ci: always use $PATH python

Consider the following output form `which python3` and `which pytest-3` in
the CI environment:

    which python3
    shell: /usr/bin/bash -e {0}
    env:
        pythonLocation: /opt/hostedtoolcache/Python/3.10.15/x64
        PKG_CONFIG_PATH: /opt/hostedtoolcache/Python/3.10.15/x64/lib/pkgconfig
        Python_ROOT_DIR: /opt/hostedtoolcache/Python/3.10.15/x64
        Python2_ROOT_DIR: /opt/hostedtoolcache/Python/3.10.15/x64
        Python3_ROOT_DIR: /opt/hostedtoolcache/Python/3.10.15/x64
        LD_LIBRARY_PATH: /opt/hostedtoolcache/Python/3.10.15/x64/lib
    /opt/hostedtoolcache/Python/3.10.15/x64/bin/python3

    which pytest-3
    shell: /usr/bin/bash -e {0}
    env:
        pythonLocation: /opt/hostedtoolcache/Python/3.10.15/x64
        PKG_CONFIG_PATH: /opt/hostedtoolcache/Python/3.10.15/x64/lib/pkgconfig
        Python_ROOT_DIR: /opt/hostedtoolcache/Python/3.10.15/x64
        Python2_ROOT_DIR: /opt/hostedtoolcache/Python/3.10.15/x64
        Python3_ROOT_DIR: /opt/hostedtoolcache/Python/3.10.15/x64
        LD_LIBRARY_PATH: /opt/hostedtoolcache/Python/3.10.15/x64/lib
    /usr/bin/pytest-3

Since we were getting e.g. cffi vi `apt-get install python3-cffi`, we were
using the *system* python's cffi for everything. That worked because we
invoked the *system* pytest-3.

Of course, that means we were not really using the github python action's
python in our CI matrix, since we were always using the system python for
everything.

Instead, let's set up a venv as a traditional project would, and that way
we can explicitly call that venv's python everywhere in the makefile, so we
always use that venv's packages and get the right python.

Then, we need only be careful to use the $PATH python, not /usr/bin/python,
to set up this venv, and we always get the right version of python to test
with.

This likely explains some weirdness that I've seen in the past, should have
investigated sooner...

Signed-off-by: Tycho Andersen <tycho@tycho.pizza>

Author: tych0

.github/workflows/ci.yaml

@@ -25,7 +25,6 @@ jobs:
               with:
                   ghc-version: '9.6'
                   cabal-version: latest
-            - run: cabal update
-            - run: sudo apt install x11-apps python3-pytest python3-cffi python3-pytest-xdist flake8
+            - run: sudo apt install x11-apps
             - run: git clone https://gitlab.freedesktop.org/xorg/proto/xcbproto.git proto && cd proto && git checkout ${{ matrix.xcbver }}
             - run: make XCBDIR=./proto/src check

.gitignore

@@ -7,6 +7,7 @@ dist-newstyle
 *__pycache__*
 *.egg-info
 xcffib
+xcffib_venv
 build/
 *egg*
 *deb

Makefile

@@ -10,6 +10,9 @@ NCPUS=$(shell grep -c processor /proc/cpuinfo)
 PARALLEL=$(shell which parallel)
 CABAL=cabal --config-file=./cabal.config
 GEN=$(CABAL) new-run --minimize-conflict-set -j$(NCPUS) exe:xcffibgen --
+VENV=xcffib_venv
+PYTHON=$(VENV)/bin/python3
+FLAKE=$(VENV)/bin/flake8
 
 # you should have xcb-proto installed to run this
 xcffib: module/*.py xcffib.cabal $(shell find . -path ./test -prune -false -o -name \*.hs)
@@ -36,7 +39,7 @@ gen: dist-newstyle
 .PHONY: clean
 clean:
 	-$(CABAL) new-clean
-	-rm -rf xcffib
+	-rm -rf xcffib xcffib_venv
 	-rm -rf module/*pyc module/__pycache__
 	-rm -rf test/*pyc test/__pycache__
 	-rm -rf build *egg* *deb .pybuild dist
@@ -51,18 +54,23 @@ newtests:
 
 # These are all split out so make -j3 check goes as fast as possible.
 .PHONY: lint
-lint:
-	flake8 --config=./test/flake8.cfg ./module
+lint: $(VENV)
+	$(FLAKE) --config=./test/flake8.cfg ./module
 
 .PHONY: htests
 htests:
 	$(CABAL) new-test -j$(NCPUS) --enable-tests
 
-check: xcffib lint htests
+$(VENV): requirements.txt
+	# the python in $PATH in CI is the python from the matrix, so it is the
+	# "right" python to start with
+	python3 -m venv $(VENV)
+	$(PYTHON) -m pip install -r requirements.txt
+
+check: xcffib htests $(VENV) lint
 	cabal check
-	flake8 -j$(NCPUS) --ignore=E128,E231,E251,E301,E302,E305,E501,F401,E402,W503,E741,E999 xcffib/*.py
-	python3 -m compileall xcffib
-	pytest-3 -v --durations=3 -n auto
+	$(PYTHON) -m compileall xcffib
+	$(PYTHON) -m pytest -v --durations=3 -n $(NCPUS)
 
 # make release ver=0.99.99
 release: xcffib

requirements.txt

@@ -1,3 +1,6 @@
 flake8
 autopep8
 cffi>=0.8.2
+pytest
+pytest-xdist
+cffi >= 1.6