drop QEMU builds and use native arm runners for Linux and Mac (#329)

github made arm runners freely available, finally we can drop QEMU and cross-compiled builds

Author: hendrikmuhs

.github/workflows/python-cibuildwheel.yml

@@ -11,14 +11,13 @@ on:
 
 jobs:
   build_wheels:
-    name: cibuildwheel ${{ matrix.os }}/${{ matrix.arch }}/${{ matrix.flavor }}/${{ matrix.target }}
+    name: cibuildwheel ${{ matrix.os }}/${{ matrix.flavor }}/${{ matrix.target }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04, macos-13]
-        # separate archs, so they use individual caches
-        arch: [ 'x86_64', 'arm64' ]
+        # macos-13: x86, macos-14: Arm64
+        os: [ubuntu-22.04, ubuntu-24.04-arm, macos-13, macos-14]
         # skip pypy, https://github.com/pypa/distutils/issues/283
         flavor: ['cpython']
         # separate musl and many on linux, for mac we just skip one of those
@@ -31,31 +30,13 @@ jobs:
           #  target: musl
           #  flavor: pypy
           #- os: macos-13
-          #  arch: arm64
           #  flavor: pypy
     steps:
       - uses: actions/checkout@v4
-      - name: Set up QEMU
-        if: ${{ (runner.os == 'Linux') && (matrix.arch == 'arm64') }}
-        uses: docker/setup-qemu-action@v3
-        with:
-          platforms: all
       - name: ccache
         uses: hendrikmuhs/[email protected]
         with:
-          key: ${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.target }}-${{ matrix.flavor }}-python
-
-      - name: Sets env for x86_64
-        run: |
-          echo "CIBW_ARCHS_LINUX=auto64" >> $GITHUB_ENV
-          echo "CIBW_ARCHS_MACOS=x86_64" >> $GITHUB_ENV
-        if: matrix.arch == 'x86_64'
-
-      - name: Sets env for arm64
-        run: |
-          echo "CIBW_ARCHS_LINUX=aarch64" >> $GITHUB_ENV
-          echo "CIBW_ARCHS_MACOS=arm64" >> $GITHUB_ENV
-        if: matrix.arch == 'arm64'
+          key: ${{ matrix.os }}-${{ matrix.target }}-${{ matrix.flavor }}-python
 
       - name: Skip manylinux for musllinux target
         if: ${{ (runner.os == 'Linux') && (matrix.target == 'musl') }}
@@ -83,45 +64,29 @@ jobs:
         run: |
           brew update && \
           brew list -1 | grep python | while read formula; do brew unlink $formula; brew link --overwrite $formula; done && \
-          brew install ccache
+          brew install ccache zlib snappy boost
 
-      - name: install mac dependencies X86_64
-        if: ${{ (runner.os == 'macOS') && (matrix.arch == 'x86_64') }}
+      - name: set mac deployment target X64
+        if: runner.os == 'macOS' && runner.arch == 'X64'
         run: |
-          echo "MACOSX_DEPLOYMENT_TARGET=13.0" >> $GITHUB_ENV && \
-          brew update && \
-          brew install zlib snappy boost
+          echo "MACOSX_DEPLOYMENT_TARGET=13.0" >> $GITHUB_ENV
 
-      - name: install mac dependencies arm64
-        if: ${{ (runner.os == 'macOS') && (matrix.arch == 'arm64') }}
-        # explicitly install icu4c@76 and [email protected], not icu4c/boost to avoid unsupported combinations
-        # Note: when brew reinstall finds unsolved deps, it downloads and installs using the wrong arch(x86_64 instead of arm64)
+      - name: set mac deployment target arm64
+        if: runner.os == 'macOS' && runner.arch == 'ARM64'
         run: |
-          set -e
-          echo "MACOSX_DEPLOYMENT_TARGET=13.0" >> $GITHUB_ENV
-          echo "_CMAKE_PREFIX_PATH=${{ github.workspace }}/arm64-homebrew" >> $GITHUB_ENV
-          echo "CIBW_REPAIR_WHEEL_COMMAND_MACOS=DYLD_LIBRARY_PATH=${{ github.workspace }}/arm64-homebrew delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}" >> $GITHUB_ENV
-          mkdir arm64-homebrew && curl -L https://github.com/Homebrew/brew/tarball/master | tar xz --strip 1 -C arm64-homebrew
-          PACKAGES=( icu4c@76 xz lz4 zstd zlib snappy [email protected] )
-          for PACKAGE in "${PACKAGES[@]}"
-          do
-            arm64-homebrew/bin/brew info $PACKAGE
-            response=$(arm64-homebrew/bin/brew fetch --force --bottle-tag=arm64_ventura $PACKAGE | grep Downloaded )
-            download_path=$(echo $response | xargs -n 1 | tail -1)
-            arm64-homebrew/bin/brew reinstall -vd $download_path
-          done
-          arm64-homebrew/bin/brew config
-          ls /Users/runner/work/keyvi/keyvi/arm64-homebrew
-
-      - name: Build python wheels for ${{ matrix.os }} on ${{ matrix.arch }}
-        uses: pypa/[email protected]
+          echo "MACOSX_DEPLOYMENT_TARGET=14.0" >> $GITHUB_ENV
+
+      - name: Build python wheels for ${{ matrix.os }}
+        uses: pypa/[email protected]
         env:
-          # Skip CPython 3.6 and CPython 3.7
-          CIBW_SKIP: ${{ env.CIBW_SKIP }} cp36-* cp37-*
+          # Skip CPython 3.{6, 7, 8}
+          CIBW_SKIP: ${{ env.CIBW_SKIP }} cp36-* cp37-* cp38-*
+
+          # only build native packages
+          CIBW_ARCHS: native
 
-          # skip testing all python versions on linux arm, only test 3.12
           # skip tests on pypy, currently fails for indexer tests
-          CIBW_TEST_SKIP: "*p{38,39,310,311}-m*linux_aarch64 pp*"
+          CIBW_TEST_SKIP: "pp*"
 
           # (many)linux custom docker images
           CIBW_MANYLINUX_X86_64_IMAGE: 'keyvidev/manylinux-builder-x86_64'
@@ -149,7 +114,7 @@ jobs:
 
       - uses: actions/upload-artifact@v4
         with:
-          name: artifact-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.flavor }}-${{ matrix.target }}
+          name: artifact-${{ matrix.os }}-${{ matrix.flavor }}-${{ matrix.target }}
           path: ./wheelhouse/*.whl
 
   build_sdist:
@@ -164,7 +129,7 @@ jobs:
     - name: ccache
       uses: hendrikmuhs/[email protected]
       with:
-        key: ${{ matrix.os }}-sdist-python
+        key: ubuntu-sdist-python
 
     - name: Build SDist
       run: |

python/setup.py

@@ -413,7 +413,6 @@ class build_ext(custom_opts, build_cxx):
             "Programming Language :: C++",
             "Programming Language :: Cython",
             "Programming Language :: Python",
-            "Programming Language :: Python :: 3.8",
             "Programming Language :: Python :: 3.9",
             "Programming Language :: Python :: 3.10",
             "Programming Language :: Python :: 3.11",