diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f68ace7e..5f05e0b11 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -412,16 +412,32 @@ jobs: - name: Install / Upgrade Python requirements run: | - python -m pip install --upgrade pip wheel 'setuptools>=60.2' - python -m pip install pytest cffi + pip install --upgrade pip wheel 'setuptools>=60.2' setuptools-scm + pip install pytest cffi - name: Build and install HPy run: | make - python -m pip install . + pip install . + + - name: Build and install hpy.microbench + run: | + pip install ./microbench --no-build-isolation --no-deps - name: Run microbenchmarks run: | cd microbench - python setup.py build_ext -i - python -m pytest -v + pytest -v + + - name: Uninstall hpy.microbench + run: | + pip uninstall hpy.microbench --yes + + - name: Build and install hpy.microbench universal ABI + run: | + pip install -e ./microbench --no-build-isolation --no-deps --config-settings="--global-option=--hpy-abi=universal" + + - name: Run microbenchmarks universal ABI + run: | + cd microbench + pytest -v -m hpy diff --git a/.gitignore b/.gitignore index 287706698..dc3f00389 100644 --- a/.gitignore +++ b/.gitignore @@ -90,3 +90,7 @@ docs/_build/ # vscode .vscode + +microbench/pixi.lock +microbench/.venv_* +microbench/tmp_* diff --git a/microbench/Makefile b/microbench/Makefile new file mode 100644 index 000000000..3044a8617 --- /dev/null +++ b/microbench/Makefile @@ -0,0 +1,57 @@ +ifeq ($(PYTHON),) +PYTHON := python3 +endif + +install: + $(PYTHON) -m pip install . + +install_universal: + $(PYTHON) -m pip install . --config-settings="--global-option=--hpy-abi=universal" + +uninstall: + $(PYTHON) -m pip uninstall hpy.microbench --yes + +test: + $(PYTHON) -m pytest -v | tee tmp_results_$(shell $(PYTHON) -c "import sys; print(sys.implementation.name)").txt + +bench: test + +bench_hpy: + $(PYTHON) -m pytest -v -m hpy | tee tmp_results_$(shell $(PYTHON) -c "import sys; print(sys.implementation.name)").txt + +clean: + rm -f src/*.so src/hpy_simple.py + +cleanall: clean + rm -rf .venv_* tmp_*.txt + +create_venv_cpy: + uv python install 3.12 + $(shell uv python find 3.12) -m venv .venv_cpy --upgrade-deps + +create_venv_pypy: + uv python install pypy + $(shell uv python find pypy) -m venv .venv_pypy --upgrade-deps + +create_venv_graalpy: + uv python install graalpy + # cannot use --upgrade-deps because pip is patched for GraalPy + $(shell uv python find graalpy) -m venv .venv_graalpy + +print_cpy: + @echo =================================== CPython ==================================== + @tail tmp_results_cpython.txt -n 29 + +print_pypy: + @echo ==================================== PyPy ====================================== + @tail tmp_results_pypy.txt -n 29 + +print_graalpy: + @echo =================================== GraalPy ==================================== + @tail tmp_results_graalpy.txt -n 29 + +print_pypy_vs_cpy: + @$(PYTHON) print_other_vs_cpy.py PyPy + +print_graalpy_vs_cpy: + @$(PYTHON) print_other_vs_cpy.py GraalPy diff --git a/microbench/README.md b/microbench/README.md index c25916088..6d49489a9 100644 --- a/microbench/README.md +++ b/microbench/README.md @@ -1,29 +1,117 @@ -To run the microbenchmarks --------------------------- +# To run the microbenchmarks -1. You need to have `hpy` installed in your virtuanenv. The easiest way +## Non-Python dependencies + +The benchmarks depends on Valgrind, which can be installed with + +```sh +sudo apt update && sudo apt install -y valgrind +``` + +Alternatively, you can also use [Pixi] to get it in a new shell with + +```sh +pixi shell +``` + +Similarly, building with GraalPy requires `libffi` to build the Python package `ffi`. +`pixi shell` provides it. + +[UV] can be useful since it is used in few Makefile targets to install Python interpreters. + +## Python virtual environments + +We assume in the following that a virtual environment is activated. One can create +environments with the Makefile targets `create_venv_...` as + +```sh +cd /path/to/hpy/microbench +make create_venv_pypy +. .venv_pypy/bin/activate +``` + +## Non-editable install with build isolation + +One can build these microbenchmarks with HPy from PyPI (on CPython) or bundled with the Python implementation. + +```sh +pip install . +``` + +This builds the HPy extension with the CPython ABI for CPython and with the universal ABI for other implementations. +To build this extension with the universal ABI with CPython: + +```sh +pip install . --config-settings="--global-option=--hpy-abi=universal" +``` + +## Editable install without build isolation + +1. On CPython, you need to have `hpy` installed in your virtualenv. The easiest way to do it is: - $ cd /path/to/hpy - $ python setup.py develop + ```sh + cd /path/to/hpy + pip install -e . + ``` + +2. Install build and runtime dependencies + + ```sh + # cffi needed to build _valgrind + pip install cffi pytest + ``` + +3. Build and install the extension modules needed for the microbenchmarks + + ```sh + cd /path/to/hpy/microbench + pip install . --no-build-isolation + # or for the universal ABI (on with CPython) + rm -f src/*.so src/hpy_simple.py + pip install -e . --no-build-isolation --config-settings="--global-option=--hpy-abi=universal" + ``` + +## Run the benchmarks + +```sh +pytest -v +``` + +To run only cpy or hpy tests, use -m (to select markers): + +```sh +pytest -v -m hpy +pytest -v -m cpy +``` + +## Comparing alternative Python implementations to CPython -2. Build the extension modules needed for the microbenchmarks +One can run things like - $ cd /path/to/hpy/microbench - $ pip install cffi # needed to build _valgrind - $ python setup.py build_ext --inplace +```sh +make cleanall +pixi shell +make create_venv_cpy +make create_venv_pypy +make create_venv_graalpy -2. `py.test -v` +make install PYTHON=.venv_cpy/bin/python +make install PYTHON=.venv_pypy/bin/python +make install PYTHON=.venv_graalpy/bin/python -3. To run only cpy or hpy tests, use -m (to select markers): +make bench PYTHON=.venv_cpy/bin/python +make bench PYTHON=.venv_pypy/bin/python +# only HPy for GraalPy since the full benchmarks are a bit too long +make bench_hpy PYTHON=.venv_graalpy/bin/python - $ py.test -v -m hpy - $ py.test -v -m cpy +make print_cpy +make print_pypy +make print_graalpy -4. Step (2) build `hpy_simple` using the CPython ABI by default. If you want - to benchmark the universal mode, you need to build it explicitly: +make print_pypy_vs_cpy +make print_graalpy_vs_cpy +``` - $ cd /path/to/hpy/microbench - $ rm *.so # make sure to delete CPython-ABI versions - $ python setup.py --hpy-abi=universal build_ext --inplace - $ py.test -v +[Pixi]: https://pixi.sh +[UV]: https://docs.astral.sh/uv/ diff --git a/microbench/pixi.toml b/microbench/pixi.toml new file mode 100644 index 000000000..83b1c4f6f --- /dev/null +++ b/microbench/pixi.toml @@ -0,0 +1,13 @@ +[workspace] +authors = ["The HPy team "] +channels = ["conda-forge"] +name = "hpy.microbench" +platforms = ["linux-64"] +version = "0.1.0" + +[tasks] + +[dependencies] +gcc = ">=15.1.0,<15.2" +valgrind = ">=3.25.0,<4" +libffi = ">=3.4.6,<4" diff --git a/microbench/print_other_vs_cpy.py b/microbench/print_other_vs_cpy.py new file mode 100644 index 000000000..a303d6f4a --- /dev/null +++ b/microbench/print_other_vs_cpy.py @@ -0,0 +1,59 @@ +import sys + +from pathlib import Path + +try: + other = sys.argv[1] +except IndexError: + other = "PyPy" + +path_result_cpy = Path("tmp_results_cpython.txt") +path_result_other = Path(f"tmp_results_{other.lower()}.txt") + +assert path_result_cpy.exists() +assert path_result_other.exists() + + +def data_from_path(path): + txt = path.read_text() + _, txt = txt.split( + "================================== BENCHMARKS ==================================" + ) + lines = txt.splitlines()[3:-2] + + if "cpy" in path.name: + index_time = 1 + else: + parts = lines[0].split() + if len(parts) == 3: + index_time = 1 + else: + index_time = 3 + + names = [] + times = [] + + for line in lines: + parts = line.split() + names.append(parts[0]) + times.append(float(parts[index_time])) + + return names, times + + +names, times_cpy = data_from_path(path_result_cpy) +names, times_other = data_from_path(path_result_other) + +max_length_name = 45 +fmt_name = f"{{:{max_length_name}s}}" + +out = f" {other} HPy univ / CPy native (time ratio, smaller is better) " +num_chars = 81 +num_equals = (num_chars - len(out)) // 2 + +print("\n" + num_equals * "=" + out + num_equals * "=") + +for index, t_other in enumerate(times_other): + ratio = t_other / times_cpy[index] + name = fmt_name.format(names[index]) + print(f"{name} {ratio:.2f}") diff --git a/microbench/pyproject.toml b/microbench/pyproject.toml new file mode 100644 index 000000000..b8b50f26b --- /dev/null +++ b/microbench/pyproject.toml @@ -0,0 +1,22 @@ +[project] +name = "hpy.microbench" +authors = [ + {name = "The HPy team", email = "hpy-dev@python.org"}, +] +version = "0.1.0" +description = "HPy microbenchmarks." +readme = "README.md" +keywords = ["hpy"] +requires-python = ">=3.8" +dependencies = [ + "hpy>=0.9.0; implementation_name == 'cpython'", + "pytest", + "cffi; implementation_name != 'pypy'", + ] + +[build-system] +requires = [ + "setuptools>=64.0", + "hpy>=0.9.0; implementation_name == 'cpython'", + "cffi", +] diff --git a/microbench/setup.py b/microbench/setup.py index 16988b0ae..a25c8cf53 100644 --- a/microbench/setup.py +++ b/microbench/setup.py @@ -1,8 +1,10 @@ from setuptools import setup, Extension setup( - name="hpy.microbench", - setup_requires=['cffi', 'hpy'], + # Workaround: HPy adds files to the sources list and uses absolute paths. + # Newer setuptools complain about that if package data should be included. + # Therefore, we explicitly disable this here. + include_package_data=False, ext_modules = [ Extension('cpy_simple', ['src/cpy_simple.c'],