[MRG] Add M1 mac builds and fix nightly build (#513)

aaronreidsmith · web-flow · commit b465ee3dfa1d · 2022-08-23T06:55:51.000-05:00
* Add cross-compile helper from scikit

* Fix lint

* Try cibuildwheel

* Add CIBW_BUILD env var

* Use CIBW_BEFORE_BUILD

* Skip 32-bit on windows

* Swap '&gt;' to '|' for yaml

* Use '&gt;' and '&amp;&amp;'

* Add CIBW_BEFORE_TEST

* Reinstall deps before testing

* Cleanup

* Use PYTHON_CROSSENV for mac builds

* Skip musl builds

* Fix format step?

* Fix README check

* Customize job names

* Remove 'Python' from job names

* Move 'make version' to CIBW_BEFORE_ALL

* Update nightly to use cibuildwheel

* Fix which versions we build

* Don't use DEPENDENCIES env var

* Skip musllinux builds

* Use batch for windows

* Try all caps?

* Single percent

* Try quotes?

* Use pip instead of python -m

* Back to 2 percents

* Just try a bash script

* Add slack notifications back

* Remove architecture from slack message

* Update whats new
diff --git a/.github/utils/build_and_test_aarch64.sh b/.github/utils/build_and_test_aarch64.sh
diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml
@@ -18,79 +18,76 @@ concurrency:
 
 jobs:
   build-and-deploy:
-    name: Build and Deploy
     strategy:
       matrix:
-        os: [windows-latest, macos-latest]
-        python-version: ['3.7', '3.8', '3.9', '3.10']
+        python3-minor-version: [7, 8, 9, 10]
+        os: [macos-latest, ubuntu-latest, windows-latest]
     defaults:
       run:
         shell: bash
+    env:
+      PYTHON_VERSION: ${{ format('3.{0}', matrix.python3-minor-version) }}
+      PYTHON_EXECUTABLE: ${{ format('cp3{0}', matrix.python3-minor-version) }}
 
     runs-on: ${{ matrix.os }}
+    name: Build and Deploy (${{ matrix.os }}, 3.${{ matrix.python3-minor-version }})
 
     steps:
       # This LOOKS like it is checking out 'master', but it is using the 'master' version of the checkout action
       # It is actually checking out the most recent version on this branch
       - name: Checkout
         uses: actions/checkout@master
 
+      # Python interpreter used by cibuildwheel, but it uses a different one internally
       - name: Setting up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow
-      - name: Checking for cached pip dependencies (macOS)
-        if: startsWith(runner.os, 'macOS')
-        uses: actions/cache@v1
+        uses: actions/setup-python@v3
         with:
-          path: ~/Library/Caches/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
+          python-version: ${{ env.PYTHON_VERSION }}
 
-      - name: Checking for cached pip dependencies (Windows)
-        if: startsWith(runner.os, 'Windows')
-        uses: actions/cache@v1
+      - name: Set up QEMU
+        if: runner.os == 'Linux'
+        uses: docker/setup-qemu-action@v1
         with:
-          path: ~\AppData\Local\pip\Cache
-          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
-
-      - name: Updating pip
-        run: python -m pip install --upgrade pip
-
-      - name: Installing requirements
-        run: |
-          pip install -r build_tools/build_requirements.txt
-          pip install -r requirements.txt
+          platforms: all
 
       # We build the source archive separately because of this: https://github.com/alkaline-ml/pmdarima/pull/136#discussion_r279781731
       # We build it first because of this: https://github.com/alkaline-ml/pmdarima/issues/448
       - name: Building source distribution
         run: make version sdist
 
-      - name: Building wheel
-        run: make version bdist_wheel
-
-      - name: Installing generated wheel
-        run: pip install --pre --no-index --find-links dist/ pmdarima
-
-      - name: Running unit tests
-        run: |
-          if [ "${{ matrix.os }}" == "macos-latest" ]; then
-            export PMD_MPL_BACKEND=TkAgg
-          fi
-          pytest --showlocals --durations=20 --pyargs pmdarima
+      - name: Install cibuildwheel
+        run: python -m pip install cibuildwheel==2.9.0 # TODO: Do we want this pinned?
 
-      - name: Checking for numpy regression
-        run: |
-          pip install --upgrade numpy
-          if [ "${{ matrix.os }}" == "macos-latest" ]; then
-            export PMD_MPL_BACKEND=TkAgg
-          fi
-          pytest --showlocals --durations=20 --pyargs pmdarima
+      - name: Building and testing wheels
+        run: python -m cibuildwheel --output-dir dist
+        env:
+          # TODO: Move Linux x86_64 builds to GHA?
+          CIBW_ARCHS_LINUX: "aarch64"
+          CIBW_ARCHS_MACOS: "x86_64 arm64"
+          CIBW_ARCHS_WINDOWS: "AMD64"
+          CIBW_BEFORE_ALL: make version
+          CIBW_BEFORE_BUILD: >
+            pip install -r build_tools/build_requirements.txt &&
+            pip install -r requirements.txt
+          # Tests are run in a separate virtual env, so we need to re-install deps
+          CIBW_BEFORE_TEST: >
+            pip install -r build_tools/build_requirements.txt &&
+            pip install -r requirements.txt
+          CIBW_BUILD: "${{ env.PYTHON_EXECUTABLE }}-*"
+          CIBW_ENVIRONMENT_MACOS: PMD_MPL_BACKEND=TkAgg PYTHON_CROSSENV=true
+          # No support for pypy or musl
+          CIBW_SKIP: "pp* *-musllinux_*"
+          # Runs tests and checks for a numpy regression by upgrading numpy and running tests again
+          CIBW_TEST_COMMAND: >
+            pytest --showlocals --durations=20 --pyargs pmdarima &&
+            pip install --upgrade numpy &&
+            pytest --showlocals --durations=20 --pyargs pmdarima
+          # Avoid testing on emulated architectures
+          CIBW_TEST_SKIP: "*-*linux_{aarch64,ppc64le,s390x} *-macosx_arm64"
 
       - name: Checking README compatibility
         run: |
+          python -m pip install "twine>=1.13.0" readme_renderer
           if python -c "from twine.commands.check import check; check(['dist/*'])" | grep  "warning"; then
               echo "README will not render properly on PyPI"
               exit 1
@@ -115,57 +112,7 @@ jobs:
       - name: Ensuring sdist can be installed
         run: pip install dist/$(ls dist | grep tar)
 
-      - name: Ensuring VERSION file existis
-        id: version_check  # Need this to refer to output below
-        run: |
-          if [ -f "${GITHUB_WORKSPACE}/pmdarima/VERSION" ]; then
-            echo "VERSION file exists"
-            echo "::set-output name=version_exists::true"
-          else
-            echo "VERSION file does not exist"
-            echo "::set-output name=version_exists::false"
-          fi
-
-      - name: Deploying to PyPI
-        # Only deploy on tags and when VERSION file created
-        if: startsWith(github.ref, 'refs/tags') && success() && steps.version_check.outputs.version_exists == 'true'
-        run: |
-          chmod +x build_tools/github/deploy.sh
-          ./build_tools/github/deploy.sh
-        env:
-          TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
-          TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
-
-  build-and-deploy-aarch64:
-    name: 'Build and Deploy (aarch64)'
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: [cp37-cp37m, cp38-cp38, cp39-cp39, cp310-cp310]
-    env:
-      python: /opt/python/${{ matrix.python-version }}/bin/python
-      image: quay.io/pypa/manylinux_2_28_aarch64
-    defaults:
-      run:
-        shell: bash
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@master
-
-      - name: Setup QEMU
-        uses: docker/setup-qemu-action@v1
-
-      - name: Build and Test
-        run: |
-          chmod +x .github/utils/build_and_test_aarch64.sh
-          docker run --rm \
-            -v ${{ github.workspace }}:/workspace \
-            --workdir=/workspace \
-            ${{ env.image }} \
-            .github/utils/build_and_test_aarch64.sh ${{ env.python }}
-
-      - name: Ensuring VERSION file existis
+      - name: Ensuring VERSION file exists
         id: version_check  # Need this to refer to output below
         run: |
           if [ -f "${GITHUB_WORKSPACE}/pmdarima/VERSION" ]; then
diff --git a/.github/workflows/nightly_cron.yml b/.github/workflows/nightly_cron.yml
@@ -24,15 +24,11 @@ jobs:
       fail-fast: false
       matrix:
         os: [windows-latest, macos-latest, ubuntu-latest]
-        python-version: ['3.9']
-        architecture: ['x86', 'x64']
-        exclude:
-          # Don't build 32-bit on Mac or Linux
-          - os: macos-latest
-            architecture: x86
-
-          - os: ubuntu-latest
-            architecture: x86
+        python-version: ['3.10']
+        python-executable: ['cp310']
+    defaults:
+      run:
+        shell: bash
 
     runs-on: ${{ matrix.os }}
 
@@ -44,19 +40,13 @@ jobs:
         uses: actions/checkout@master
 
       - name: Setting up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v3
         with:
           python-version: ${{ matrix.python-version }}
-          architecture: ${{ matrix.architecture }}
 
       - name: Set Run ID
         id: definition  # Needed to retrieve the output of this step later
         run: echo "::set-output name=run_id::$GITHUB_RUN_ID"
-        shell: bash
-
-      - name: Updating pip
-        run: python -m pip install --upgrade pip
-        shell: bash
 
       - name: Collecting naked dependencies
         id: dependencies  # Needed to retrieve the output of this step later
@@ -65,60 +55,43 @@ jobs:
           echo "::set-output name=latest_dependencies::$dependencies"
         shell: bash
 
-        # https://github.com/actions/cache/blob/master/examples.md#using-pip-to-get-cache-location
-      - name: Get pip cache dir
-        id: pip-cache-dir-path
-        run: echo "::set-output name=dir::$(pip cache dir)"
+      - name: Install cibuildwheel
+        run: python -m pip install cibuildwheel==2.9.0 # TODO: Do we want this pinned?
 
-      - name: Generating cache key file
+      - name: Generating dependency table
         id: dependency-table  # Needed to set output of job
         run: |
           pip install requests tabulate
           table=$(python .github/utils/get_dependency_releases.py $DEPENDENCIES)
-          echo $table > dependencies_key.txt
 
           # This is used in the next job (if necessary) rather than re-running the above
           echo "::set-output name=table::$table"
-        shell: bash
         env:
           DEPENDENCIES: ${{ steps.dependencies.outputs.latest_dependencies }}
 
-      - name: Check for cache hit
-        id: pip-cache
-        uses: actions/cache@v2
-        with:
-          path: ${{ steps.pip-cache-dir-path.outputs.dir }}
-          key: ${{ runner.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('**/dependencies_key.txt') }}
-
-      # We install one-by-one because some dependencies depend on others
-      - name: Installing dependencies
-        if: steps.pip-cache.outputs.cache-hit != 'true'
-        run: |
-          for dependency in $DEPENDENCIES; do
-            python -m pip install $dependency
-          done
-        shell: bash
+      - name: Building and testing wheel
+        run: python -m cibuildwheel --output-dir dist
         env:
-          DEPENDENCIES: ${{ steps.dependencies.outputs.latest_dependencies }}
-
-      - name: Building wheel
-        if: steps.pip-cache.outputs.cache-hit != 'true'
-        run: make bdist_wheel
-        shell: bash
-
-      - name: Installing generated wheel
-        if: steps.pip-cache.outputs.cache-hit != 'true'
-        run: pip install --pre --no-index --find-links dist/ pmdarima
-        shell: bash
-
-      - name: Running unit tests
-        if: steps.pip-cache.outputs.cache-hit != 'true'
-        run: |
-          if [ "${{ matrix.os }}" == "macos-latest" ]; then
-            export PMD_MPL_BACKEND=TkAgg
-          fi
-          pytest --showlocals --durations=20 --pyargs pmdarima
-        shell: bash
+          CIBW_ARCHS_LINUX: "x86_64"
+          CIBW_ARCHS_MACOS: "x86_64"
+          CIBW_ARCHS_WINDOWS: "AMD64"
+          CIBW_BEFORE_BUILD: >
+            for dependency in ${{ steps.dependencies.outputs.latest_dependencies }}; do
+              pip install $dependency
+            done
+          # Windows runs a batch script that I couldn't get to work, so we just force it to run bash
+          CIBW_BEFORE_BUILD_WINDOWS: bash -c 'for dependency in ${{ steps.dependencies.outputs.latest_dependencies }}; do pip install $dependency; done'
+          # Tests are run in a separate virtual env, so we need to re-install deps
+          CIBW_BEFORE_TEST: >
+            for dependency in ${{ steps.dependencies.outputs.latest_dependencies }}; do
+              pip install $dependency
+            done
+          CIBW_BEFORE_TEST_WINDOWS: bash -c 'for dependency in ${{ steps.dependencies.outputs.latest_dependencies }}; do pip install $dependency; done'
+          CIBW_BUILD: "${{ matrix.python-executable }}-*"
+          CIBW_ENVIRONMENT_MACOS: PMD_MPL_BACKEND=TkAGG
+          # No support for pypy or musl
+          CIBW_SKIP: "pp* *-musllinux_*"
+          CIBW_TEST_COMMAND: pytest --showlocals --durations=20 --pyargs pmdarima
 
       # https://github.com/marketplace/actions/action-slack#custom-notification
       - name: Posting to Slack
@@ -141,11 +114,6 @@ jobs:
                     value: '${{ matrix.os }}',
                     short: false
                   },
-                  {
-                    title: 'Architecture',
-                    value: '${{ matrix.architecture }}',
-                    short: false
-                  },
                   {
                     title: 'Python Version',
                     value: '${{ matrix.python-version }}',
diff --git a/doc/whats_new.rst b/doc/whats_new.rst
@@ -7,6 +7,11 @@ What's new in pmdarima
 As new releases of pmdarima are pushed out, the following list (introduced in
 v0.8.1) will document the latest features.
 
+`v2.0.1 <http://alkaline-ml.com/pmdarima/2.0.1>`_
+-------------------------------------------------
+
+* Add support for macOS with M1 chip
+
 `v2.0.0 <http://alkaline-ml.com/pmdarima/2.0.0>`_
 -------------------------------------------------
 
diff --git a/pmdarima/_build_utils/pre_build_helpers.py b/pmdarima/_build_utils/pre_build_helpers.py
@@ -50,10 +50,13 @@ def compile_test_program(code, extra_preargs=[], extra_postargs=[]):
                                       extra_preargs=extra_preargs,
                                       extra_postargs=extra_postargs)
 
-            # Run test program
-            # will raise a CalledProcessError if return code was non-zero
-            output = subprocess.check_output('./test_program')
-            output = output.decode(sys.stdout.encoding or 'utf-8').splitlines()
+            if "PYTHON_CROSSENV" not in os.environ:
+                # Run test program if not cross compiling
+                # will raise a CalledProcessError if return code was non-zero
+                output = subprocess.check_output('./test_program')
+                output = output.decode(sys.stdout.encoding or 'utf-8').splitlines() # noqa
+            else:
+                output = []
         except Exception:
             raise
         finally:
