From 36049d86a2e21d74382d84d7a423e20e49a2ca91 Mon Sep 17 00:00:00 2001 From: Matthieu Darbois Date: Mon, 7 Aug 2023 18:01:10 +0200 Subject: [PATCH] feature: add musllinux_1_2 support (#1561) * feature: add musllinux_1_2 support * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix[test]: clean-up docker images in test_*linux*_only.py We are reaching disk space limits (14 GB) when running tests. Add a fixture to clean-up docker images after tests. This fixture is applied on tests that pull a specific image for one test only in order not to take too much time pulling that image many times. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- bin/update_docker.py | 6 ++ .../resources/pinned_docker_images.cfg | 43 +++++++------- docs/options.md | 2 +- test/conftest.py | 34 +++++++++++ test/test_container_images.py | 1 + test/test_manylinuxXXXX_only.py | 1 + test/test_musllinux_X_Y_only.py | 56 +++++++++++++++++++ test/utils.py | 15 ++--- unit_test/option_prepare_test.py | 21 +++++-- 9 files changed, 147 insertions(+), 32 deletions(-) create mode 100644 test/test_musllinux_X_Y_only.py diff --git a/bin/update_docker.py b/bin/update_docker.py index fa91cd009..9be3ae03d 100755 --- a/bin/update_docker.py +++ b/bin/update_docker.py @@ -59,6 +59,12 @@ class Image: Image("musllinux_1_1", "aarch64", "quay.io/pypa/musllinux_1_1_aarch64", None), Image("musllinux_1_1", "ppc64le", "quay.io/pypa/musllinux_1_1_ppc64le", None), Image("musllinux_1_1", "s390x", "quay.io/pypa/musllinux_1_1_s390x", None), + # musllinux_1_2 images + Image("musllinux_1_2", "x86_64", "quay.io/pypa/musllinux_1_2_x86_64", None), + Image("musllinux_1_2", "i686", "quay.io/pypa/musllinux_1_2_i686", None), + Image("musllinux_1_2", "aarch64", "quay.io/pypa/musllinux_1_2_aarch64", None), + Image("musllinux_1_2", "ppc64le", "quay.io/pypa/musllinux_1_2_ppc64le", None), + Image("musllinux_1_2", "s390x", "quay.io/pypa/musllinux_1_2_s390x", None), ] config = configparser.ConfigParser() diff --git a/cibuildwheel/resources/pinned_docker_images.cfg b/cibuildwheel/resources/pinned_docker_images.cfg index b0834e929..3f00c255f 100644 --- a/cibuildwheel/resources/pinned_docker_images.cfg +++ b/cibuildwheel/resources/pinned_docker_images.cfg @@ -1,49 +1,54 @@ [x86_64] manylinux1 = quay.io/pypa/manylinux1_x86_64:2023-06-25-d2e0575 manylinux2010 = quay.io/pypa/manylinux2010_x86_64:2022-08-05-4535177 -manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2023-07-14-55e4124 +manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2023-07-29-8793e83 manylinux_2_24 = quay.io/pypa/manylinux_2_24_x86_64:2022-12-26-0d38463 -manylinux_2_28 = quay.io/pypa/manylinux_2_28_x86_64:2023-07-14-55e4124 -musllinux_1_1 = quay.io/pypa/musllinux_1_1_x86_64:2023-07-14-55e4124 +manylinux_2_28 = quay.io/pypa/manylinux_2_28_x86_64:2023-07-29-8793e83 +musllinux_1_1 = quay.io/pypa/musllinux_1_1_x86_64:2023-07-29-8793e83 +musllinux_1_2 = quay.io/pypa/musllinux_1_2_x86_64:2023-07-29-8793e83 [i686] manylinux1 = quay.io/pypa/manylinux1_i686:2023-06-25-d2e0575 manylinux2010 = quay.io/pypa/manylinux2010_i686:2022-08-05-4535177 -manylinux2014 = quay.io/pypa/manylinux2014_i686:2023-07-14-55e4124 +manylinux2014 = quay.io/pypa/manylinux2014_i686:2023-07-29-8793e83 manylinux_2_24 = quay.io/pypa/manylinux_2_24_i686:2022-12-26-0d38463 -musllinux_1_1 = quay.io/pypa/musllinux_1_1_i686:2023-07-14-55e4124 +musllinux_1_1 = quay.io/pypa/musllinux_1_1_i686:2023-07-29-8793e83 +musllinux_1_2 = quay.io/pypa/musllinux_1_2_i686:2023-07-29-8793e83 [pypy_x86_64] manylinux2010 = quay.io/pypa/manylinux2010_x86_64:2022-08-05-4535177 -manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2023-07-14-55e4124 +manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2023-07-29-8793e83 manylinux_2_24 = quay.io/pypa/manylinux_2_24_x86_64:2022-12-26-0d38463 -manylinux_2_28 = quay.io/pypa/manylinux_2_28_x86_64:2023-07-14-55e4124 +manylinux_2_28 = quay.io/pypa/manylinux_2_28_x86_64:2023-07-29-8793e83 [pypy_i686] manylinux2010 = quay.io/pypa/manylinux2010_i686:2022-08-05-4535177 -manylinux2014 = quay.io/pypa/manylinux2014_i686:2023-07-14-55e4124 +manylinux2014 = quay.io/pypa/manylinux2014_i686:2023-07-29-8793e83 manylinux_2_24 = quay.io/pypa/manylinux_2_24_i686:2022-12-26-0d38463 [aarch64] -manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2023-07-14-55e4124 +manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2023-07-29-8793e83 manylinux_2_24 = quay.io/pypa/manylinux_2_24_aarch64:2022-12-26-0d38463 -manylinux_2_28 = quay.io/pypa/manylinux_2_28_aarch64:2023-07-14-55e4124 -musllinux_1_1 = quay.io/pypa/musllinux_1_1_aarch64:2023-07-14-55e4124 +manylinux_2_28 = quay.io/pypa/manylinux_2_28_aarch64:2023-07-29-8793e83 +musllinux_1_1 = quay.io/pypa/musllinux_1_1_aarch64:2023-07-29-8793e83 +musllinux_1_2 = quay.io/pypa/musllinux_1_2_aarch64:2023-07-29-8793e83 [ppc64le] -manylinux2014 = quay.io/pypa/manylinux2014_ppc64le:2023-07-14-55e4124 +manylinux2014 = quay.io/pypa/manylinux2014_ppc64le:2023-07-29-8793e83 manylinux_2_24 = quay.io/pypa/manylinux_2_24_ppc64le:2022-12-26-0d38463 -manylinux_2_28 = quay.io/pypa/manylinux_2_28_ppc64le:2023-07-14-55e4124 -musllinux_1_1 = quay.io/pypa/musllinux_1_1_ppc64le:2023-07-14-55e4124 +manylinux_2_28 = quay.io/pypa/manylinux_2_28_ppc64le:2023-07-29-8793e83 +musllinux_1_1 = quay.io/pypa/musllinux_1_1_ppc64le:2023-07-29-8793e83 +musllinux_1_2 = quay.io/pypa/musllinux_1_2_ppc64le:2023-07-29-8793e83 [s390x] -manylinux2014 = quay.io/pypa/manylinux2014_s390x:2023-07-14-55e4124 +manylinux2014 = quay.io/pypa/manylinux2014_s390x:2023-07-29-8793e83 manylinux_2_24 = quay.io/pypa/manylinux_2_24_s390x:2022-12-26-0d38463 -manylinux_2_28 = quay.io/pypa/manylinux_2_28_s390x:2023-07-14-55e4124 -musllinux_1_1 = quay.io/pypa/musllinux_1_1_s390x:2023-07-14-55e4124 +manylinux_2_28 = quay.io/pypa/manylinux_2_28_s390x:2023-07-29-8793e83 +musllinux_1_1 = quay.io/pypa/musllinux_1_1_s390x:2023-07-29-8793e83 +musllinux_1_2 = quay.io/pypa/musllinux_1_2_s390x:2023-07-29-8793e83 [pypy_aarch64] -manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2023-07-14-55e4124 +manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2023-07-29-8793e83 manylinux_2_24 = quay.io/pypa/manylinux_2_24_aarch64:2022-12-26-0d38463 -manylinux_2_28 = quay.io/pypa/manylinux_2_28_aarch64:2023-07-14-55e4124 +manylinux_2_28 = quay.io/pypa/manylinux_2_28_aarch64:2023-07-29-8793e83 diff --git a/docs/options.md b/docs/options.md index 2c12b9ee1..66dec0b74 100644 --- a/docs/options.md +++ b/docs/options.md @@ -981,7 +981,7 @@ Set an alternative Docker image to be used for building [manylinux / musllinux]( For `CIBW_MANYLINUX_*_IMAGE`, the value of this option can either be set to `manylinux1`, `manylinux2010`, `manylinux2014`, `manylinux_2_24` or `manylinux_2_28` to use a pinned version of the [official manylinux images](https://github.com/pypa/manylinux). Alternatively, set these options to any other valid Docker image name. For PyPy, the `manylinux1` image is not available. For architectures other than x86 (x86\_64 and i686) `manylinux2014`, `manylinux_2_24` or `manylinux_2_28` must be used, because the first version of the manylinux specification that supports additional architectures is `manylinux2014`. `manylinux_2_28` is not supported for `i686` architecture. -For `CIBW_MUSLLINUX_*_IMAGE`, the value of this option can either be set to `musllinux_1_1` to use a pinned version of the [official musllinux images](https://github.com/pypa/musllinux). Alternatively, set these options to any other valid Docker image name. +For `CIBW_MUSLLINUX_*_IMAGE`, the value of this option can either be set to `musllinux_1_1` or `musllinux_1_2` to use a pinned version of the [official musllinux images](https://github.com/pypa/musllinux). Alternatively, set these options to any other valid Docker image name. If this option is blank, it will fall though to the next available definition (environment variable -> pyproject.toml -> default). diff --git a/test/conftest.py b/test/conftest.py index 07db1c1e4..161148680 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,7 +1,15 @@ from __future__ import annotations +import json +import subprocess +from typing import Generator + import pytest +from cibuildwheel.util import detect_ci_provider + +from .utils import platform + def pytest_addoption(parser) -> None: parser.addoption( @@ -21,3 +29,29 @@ def pytest_addoption(parser) -> None: ) def build_frontend_env(request) -> dict[str, str]: return request.param # type: ignore[no-any-return] + + +@pytest.fixture() +def docker_cleanup() -> Generator[None, None, None]: + def get_images() -> set[str]: + images = subprocess.run( + ["docker", "image", "ls", "--format", "{{json .ID}}"], + text=True, + check=True, + stdout=subprocess.PIPE, + ).stdout + return {json.loads(image.strip()) for image in images.splitlines() if image.strip()} + + if detect_ci_provider() is None or platform != "linux": + try: + yield + finally: + pass + return + images_before = get_images() + try: + yield + finally: + images_after = get_images() + for image in images_after - images_before: + subprocess.run(["docker", "rmi", image], check=False) diff --git a/test/test_container_images.py b/test/test_container_images.py index 75b102e38..12d486418 100644 --- a/test/test_container_images.py +++ b/test/test_container_images.py @@ -23,6 +23,7 @@ ) +@pytest.mark.usefixtures("docker_cleanup") def test(tmp_path): if utils.platform != "linux": pytest.skip("the test is only relevant to the linux build") diff --git a/test/test_manylinuxXXXX_only.py b/test/test_manylinuxXXXX_only.py index 4d1fe87e8..06082a7bb 100644 --- a/test/test_manylinuxXXXX_only.py +++ b/test/test_manylinuxXXXX_only.py @@ -53,6 +53,7 @@ "manylinux_image", ["manylinux1", "manylinux2010", "manylinux2014", "manylinux_2_24", "manylinux_2_28"], ) +@pytest.mark.usefixtures("docker_cleanup") def test(manylinux_image, tmp_path): if utils.platform != "linux": pytest.skip("the container image test is only relevant to the linux build") diff --git a/test/test_musllinux_X_Y_only.py b/test/test_musllinux_X_Y_only.py new file mode 100644 index 000000000..d05bc9f87 --- /dev/null +++ b/test/test_musllinux_X_Y_only.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import textwrap + +import pytest + +from . import test_projects, utils + +project_with_manylinux_symbols = test_projects.new_c_project( + spam_c_top_level_add=textwrap.dedent( + r""" + #include + + #if defined(__GLIBC_PREREQ) + #error "Must not run on a glibc linux environment" + #endif + """ + ), + spam_c_function_add=textwrap.dedent( + r""" + sts = 0; + """ + ), +) + + +@pytest.mark.parametrize( + "musllinux_image", + ["musllinux_1_1", "musllinux_1_2"], +) +@pytest.mark.usefixtures("docker_cleanup") +def test(musllinux_image, tmp_path): + if utils.platform != "linux": + pytest.skip("the container image test is only relevant to the linux build") + + project_dir = tmp_path / "project" + project_with_manylinux_symbols.generate(project_dir) + + # build the wheels + add_env = { + "CIBW_BUILD": "*-musllinux*", + "CIBW_MUSLLINUX_X86_64_IMAGE": musllinux_image, + "CIBW_MUSLLINUX_I686_IMAGE": musllinux_image, + "CIBW_MUSLLINUX_AARCH64_IMAGE": musllinux_image, + "CIBW_MUSLLINUX_PPC64LE_IMAGE": musllinux_image, + "CIBW_MUSLLINUX_S390X_IMAGE": musllinux_image, + } + + actual_wheels = utils.cibuildwheel_run(project_dir, add_env=add_env) + expected_wheels = utils.expected_wheels( + "spam", + "0.1.0", + manylinux_versions=[], + musllinux_versions=[musllinux_image], + ) + assert set(actual_wheels) == set(expected_wheels) diff --git a/test/utils.py b/test/utils.py index f4578a58a..9ddc528c8 100644 --- a/test/utils.py +++ b/test/utils.py @@ -205,13 +205,14 @@ def expected_wheels( if machine_arch == "x86_64": architectures.append("i686") - platform_tags = [ - ".".join( - f"{manylinux_version}_{architecture}" - for manylinux_version in manylinux_versions - ) - for architecture in architectures - ] + if len(manylinux_versions) > 0: + platform_tags = [ + ".".join( + f"{manylinux_version}_{architecture}" + for manylinux_version in manylinux_versions + ) + for architecture in architectures + ] if len(musllinux_versions) > 0 and not python_abi_tag.startswith("pp"): platform_tags.extend( [ diff --git a/unit_test/option_prepare_test.py b/unit_test/option_prepare_test.py index f37badf20..0e965aa40 100644 --- a/unit_test/option_prepare_test.py +++ b/unit_test/option_prepare_test.py @@ -101,13 +101,15 @@ def test_build_with_override_launches(monkeypatch, tmp_path): cibw_toml.write_text( """ [tool.cibuildwheel] -manylinux-x86_64-image = "manylinux_2_24" +manylinux-x86_64-image = "manylinux_2_28" +musllinux-x86_64-image = "musllinux_1_2" -# Before Python 3.10, use manylinux2014 +# Before Python 3.10, use manylinux2014, musllinux_1_1 [[tool.cibuildwheel.overrides]] select = "cp3?-*" manylinux-x86_64-image = "manylinux2014" manylinux-i686-image = "manylinux2014" +musllinux-x86_64-image = "musllinux_1_1" [[tool.cibuildwheel.overrides]] select = "cp36-manylinux_x86_64" @@ -122,7 +124,7 @@ def test_build_with_override_launches(monkeypatch, tmp_path): build_in_container = typing.cast(mock.Mock, linux.build_in_container) - assert build_in_container.call_count == 6 + assert build_in_container.call_count == 7 kwargs = build_in_container.call_args_list[0][1] assert "quay.io/pypa/manylinux2014_x86_64" in kwargs["container"]["image"] @@ -146,7 +148,7 @@ def test_build_with_override_launches(monkeypatch, tmp_path): assert kwargs["options"].build_options("cp37-manylinux_x86_64").before_all == "" kwargs = build_in_container.call_args_list[2][1] - assert "quay.io/pypa/manylinux_2_24_x86_64" in kwargs["container"]["image"] + assert "quay.io/pypa/manylinux_2_28_x86_64" in kwargs["container"]["image"] assert kwargs["container"]["cwd"] == PurePosixPath("/project") assert not kwargs["container"]["simulate_32_bit"] identifiers = {x.identifier for x in kwargs["platform_configs"]} @@ -169,10 +171,19 @@ def test_build_with_override_launches(monkeypatch, tmp_path): identifiers = {x.identifier for x in kwargs["platform_configs"]} assert identifiers == { - f"{x}-musllinux_x86_64" for x in ALL_IDS for x in ALL_IDS if "pp" not in x + f"{x}-musllinux_x86_64" for x in ALL_IDS & {"cp36", "cp37", "cp38", "cp39"} if "pp" not in x } kwargs = build_in_container.call_args_list[5][1] + assert "quay.io/pypa/musllinux_1_2_x86_64" in kwargs["container"]["image"] + assert kwargs["container"]["cwd"] == PurePosixPath("/project") + assert not kwargs["container"]["simulate_32_bit"] + identifiers = {x.identifier for x in kwargs["platform_configs"]} + assert identifiers == { + f"{x}-musllinux_x86_64" for x in ALL_IDS - {"cp36", "cp37", "cp38", "cp39"} if "pp" not in x + } + + kwargs = build_in_container.call_args_list[6][1] assert "quay.io/pypa/musllinux_1_1_i686" in kwargs["container"]["image"] assert kwargs["container"]["cwd"] == PurePosixPath("/project") assert kwargs["container"]["simulate_32_bit"]