NumPy compatibility fixes (API elements deprecated in 1.25) + Python 3.12 compatibility fixes (setuptools, etc) + upper Python version bump for CI to 3.12 #1964
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: tests+artifacts+pypi | |
| defaults: | |
| run: | |
| shell: bash | |
| on: | |
| push: | |
| branches: [ main ] | |
| pull_request: | |
| branches: [ main ] | |
| schedule: | |
| - cron: '0 13 * * 4' | |
| release: | |
| types: [published] | |
| jobs: | |
| precommit: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/[email protected] | |
| - uses: actions/[email protected] | |
| with: | |
| python-version: 3.9 | |
| - run: | | |
| pip install pre-commit | |
| pre-commit clean | |
| pre-commit autoupdate | |
| pre-commit run --all-files | |
| - if: ${{ failure() }} | |
| run: git diff | |
| nojit_and_codecov: | |
| needs: [precommit, pylint] | |
| runs-on: ubuntu-latest | |
| env: | |
| NUMBA_DISABLE_JIT: 1 | |
| steps: | |
| - uses: actions/[email protected] | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 | |
| - uses: actions/[email protected] | |
| with: | |
| python-version: 3.9 | |
| - name: Generate coverage report | |
| run: | | |
| pip install -e .[tests] | |
| pip install -e ./examples | |
| pip install pytest-cov | |
| pytest --durations=10 -We tests/unit_tests --cov-report=xml --cov=PySDM | |
| - uses: codecov/[email protected] | |
| with: | |
| fail_ci_if_error: true | |
| verbose: true | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| pdoc: | |
| strategy: | |
| matrix: | |
| platform: [ubuntu-latest, macos-13, windows-latest] | |
| runs-on: ${{ matrix.platform }} | |
| steps: | |
| - uses: actions/[email protected] | |
| with: | |
| submodules: recursive | |
| persist-credentials: false | |
| - uses: actions/[email protected] | |
| with: | |
| python-version: "3.12" | |
| - run: | | |
| pip install pdoc nbformat | |
| pip install -e . | |
| pip install -e examples | |
| python - <<EOF | |
| import glob, nbformat | |
| for notebook_path in glob.glob('examples/PySDM_examples/*/*.ipynb'): | |
| with open(notebook_path, encoding="utf8") as fin: | |
| with open(notebook_path + ".badges.md", 'w') as fout: | |
| fout.write(nbformat.read(fin, nbformat.NO_CONVERT).cells[0].source) | |
| EOF | |
| PDOC_ALLOW_EXEC=1 python -We -m pdoc -o html PySDM examples/PySDM_examples -t docs/templates --math --mermaid | |
| - if: ${{ github.ref == 'refs/heads/main' && matrix.platform == 'ubuntu-latest' }} | |
| uses: JamesIves/[email protected] | |
| with: | |
| BRANCH: pdoc | |
| FOLDER: html | |
| CLEAN: true | |
| pylint: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/[email protected] | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 | |
| - name: Set up Python 3.9 | |
| uses: actions/[email protected] | |
| with: | |
| python-version: 3.9 | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| # TODO #1160 https://github.com/pylint-dev/pylint/issues/9099 | |
| pip install "pylint<3.0.0" nbqa | |
| pip install -r tests/devops_tests/requirements.txt | |
| pip install -e ./examples[tests] | |
| pip install -e .[tests] | |
| - run: | | |
| # TODO #682 | |
| pylint --unsafe-load-any-extension=y --disable=fixme,invalid-name,missing-function-docstring,missing-class-docstring,protected-access,duplicate-code $(git ls-files '*.py' | grep -v -e ^examples -e ^tutorials) | |
| - run: | | |
| # TODO #682 | |
| pylint --max-module-lines=550 --unsafe-load-any-extension=y --disable=fixme,too-many-function-args,unsubscriptable-object,consider-using-with,protected-access,too-many-statements,too-many-public-methods,too-many-branches,duplicate-code,invalid-name,missing-function-docstring,missing-module-docstring,missing-class-docstring,too-many-locals,too-many-instance-attributes,too-few-public-methods,too-many-arguments,c-extension-no-member $(git ls-files '*.py' | grep ^examples) | |
| - run: | | |
| # TODO #682 | |
| nbqa pylint --unsafe-load-any-extension=y --disable=fixme,duplicate-code,invalid-name,trailing-whitespace,line-too-long,missing-function-docstring,wrong-import-position,missing-module-docstring,wrong-import-order,ungrouped-imports,no-member,too-many-locals,unnecessary-lambda-assignment $(git ls-files '*.ipynb') | |
| zenodo_json: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/[email protected] | |
| - uses: notiz-dev/github-action-json-property@release | |
| with: | |
| path: '.zenodo.json' | |
| prop_path: 'creators' | |
| tests: | |
| needs: [pdoc, nojit_and_codecov, zenodo_json] | |
| strategy: | |
| matrix: | |
| platform: [ubuntu-latest, macos-13, macos-14, windows-latest] | |
| python-version: ["3.9", "3.12"] | |
| test-suite: ["unit_tests/!(dynamics)", "unit_tests/dynamics/!(condensation)", "unit_tests/dynamics/condensation", "smoke_tests/no_env", "smoke_tests/box", "smoke_tests/parcel_a", "smoke_tests/parcel_b", "smoke_tests/parcel_c", "smoke_tests/kinematic_1d", "smoke_tests/kinematic_2d", "tutorials_tests"] | |
| exclude: | |
| - platform: "macos-14" | |
| python-version: "3.9" | |
| - test-suite: "devops_tests" | |
| python-version: "3.9" | |
| # TODO #1406 | |
| - platform: "macos-14" | |
| test-suite: "smoke_tests/box" | |
| - platform: "macos-14" | |
| test-suite: "smoke_tests/parcel_a" | |
| - platform: "macos-14" | |
| test-suite: "smoke_tests/kinematic_2d" | |
| - platform: "macos-14" | |
| test-suite: "unit_tests/!(dynamics)" | |
| fail-fast: false | |
| runs-on: ${{ matrix.platform }} | |
| timeout-minutes: ${{ startsWith(matrix.platform, 'windows-') && 40 || 25 }} | |
| steps: | |
| - uses: actions/[email protected] | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 | |
| - uses: actions/[email protected] | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - run: pip install -e . | |
| # sanity check if we do not depend on something from tests extras | |
| - run: python -We -c "import PySDM" | |
| - if: startsWith(matrix.test-suite, 'smoke_tests') || matrix.test-suite == 'tutorials_tests' | |
| run: pip install -e ./examples | |
| # incl. ensuring that the previous pip did not overwrite the package | |
| - run: | | |
| pip install --only-binary :all: -e .[tests] | |
| # install devops_tests for tutorials_tests | |
| - if: matrix.test-suite == 'tutorials_tests' | |
| run: pip install -r tests/devops_tests/requirements.txt | |
| # https://github.com/numba/numba/issues/6350#issuecomment-728174860 | |
| - if: startsWith(matrix.platform, 'ubuntu-') | |
| run: echo NUMBA_THREADING_LAYER=omp >> $GITHUB_ENV | |
| - name: Run pytest | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| NUMBA_OPT: 1 | |
| run: | | |
| shopt -s extglob | |
| pytest --durations=10 --timeout=900 --timeout_method=thread -p no:unraisableexception -We tests/${{ matrix.test-suite }} | |
| # - if: ${{ failure() }} | |
| # uses: mxschmitt/action-tmate@v3 | |
| # with: | |
| # limit-access-to-actor: true | |
| examples-setup: | |
| needs: [nojit_and_codecov] | |
| strategy: | |
| matrix: | |
| platform: [ubuntu-24.04, macos-13, windows-latest] | |
| python-version: ["3.9", "3.12"] | |
| runs-on: ${{ matrix.platform }} | |
| steps: | |
| - uses: actions/[email protected] | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 | |
| - uses: actions/[email protected] | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - run: | | |
| echo pip_user_site=$(python -c "import sysconfig; print(sysconfig.get_path('purelib'))") >> $GITHUB_ENV | |
| echo toml_ci_md5=$(cat pyproject.toml setup.py examples/pyproject.toml examples/setup.py tests/devops_tests/requirements.txt .github/workflows/tests+artifacts+pypi.yml \ | |
| | python -c "import hashlib;print(hashlib.md5(open(0,'rb').read()).hexdigest())") >> $GITHUB_ENV | |
| - uses: actions/cache@v4 | |
| id: cache | |
| with: | |
| path: ${{ env.pip_user_site }} | |
| key: examples_pip_cache-${{ matrix.platform }}-${{ matrix.python-version }}-${{ env.toml_ci_md5 }} | |
| lookup-only: true | |
| - if: steps.cache.outputs.cache-hit != 'true' | |
| run: | | |
| python -m pip uninstall -y jupyterlab-server # https://github.com/pypa/pip/issues/6275 | |
| python -m pip install -e . -e ./examples | |
| python -m pip install -r tests/devops_tests/requirements.txt | |
| - if: steps.cache.outputs.cache-hit != 'true' | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: ${{ env.pip_user_site }} | |
| key: examples_pip_cache-${{ matrix.platform }}-${{ matrix.python-version }}-${{ env.toml_ci_md5 }} | |
| examples: | |
| needs: [examples-setup] | |
| strategy: | |
| matrix: | |
| platform: [ubuntu-24.04, macos-13, windows-latest] | |
| python-version: ["3.9", "3.12"] | |
| test-suite: [ "chemistry_freezing_isotopes", "condensation_a", "condensation_b", "coagulation", "breakup", "multi-process_a", "multi-process_b"] | |
| fail-fast: false | |
| runs-on: ${{ matrix.platform }} | |
| timeout-minutes: ${{ !startsWith(matrix.platform, 'ubuntu-') && 65 || 50 }} | |
| steps: | |
| - uses: actions/[email protected] | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 | |
| - uses: actions/[email protected] | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - run: | | |
| echo pip_user_site=$(python -c "import sysconfig; print(sysconfig.get_path('purelib'))") >> $GITHUB_ENV | |
| echo toml_ci_md5=$(cat pyproject.toml setup.py examples/pyproject.toml examples/setup.py tests/devops_tests/requirements.txt .github/workflows/tests+artifacts+pypi.yml \ | |
| | python -c "import hashlib;print(hashlib.md5(open(0,'rb').read()).hexdigest())") >> $GITHUB_ENV | |
| - run: | | |
| rm -rf $pip_user_site | |
| - id: cache | |
| uses: actions/cache/restore@v4 | |
| with: | |
| fail-on-cache-miss: true | |
| path: ${{ env.pip_user_site }} | |
| key: examples_pip_cache-${{ matrix.platform }}-${{ matrix.python-version }}-${{ env.toml_ci_md5 }} | |
| # https://github.com/numba/numba/issues/6350#issuecomment-728174860 | |
| - if: startsWith(matrix.platform, 'ubuntu-') | |
| run: echo NUMBA_THREADING_LAYER=omp >> $GITHUB_ENV | |
| - if: startsWith(matrix.platform, 'ubuntu-') && matrix.test-suite == 'multi-process_a' | |
| run: | | |
| sudo apt-get install libegl1 libopengl0 ghostscript | |
| wget -nv "https://paraview.org/paraview-downloads/download.php?submit=Download&version=v5.13&type=binary&os=Linux&downloadFile=ParaView-5.13.1-egl-MPI-Linux-Python3.10-x86_64.tar.gz" -O paraview.tar.gz | |
| tar xzf paraview.tar.gz | |
| echo `pwd`/`dirname ParaView*/bin/pvpython` >> $GITHUB_PATH | |
| - if: startsWith(matrix.platform, 'macos-') && matrix.test-suite == 'multi-process_a' | |
| run: | | |
| brew install --cask paraview | |
| echo `dirname /Applications/ParaView-*.app/Contents/bin/pvpython | head -1` >> $GITHUB_PATH | |
| brew install ghostscript | |
| - if: startsWith(matrix.platform, 'windows-') && matrix.test-suite == 'multi-process_a' | |
| run: | | |
| curl "https://www.paraview.org/paraview-downloads/download.php?submit=Download&version=v5.13&type=binary&os=Windows&downloadFile=ParaView-5.13.1-Windows-Python3.10-msvc2017-AMD64.msi" -o paraview.msi | |
| msiexec //i paraview.msi //quiet //qn //norestart | |
| echo "C:\\Program Files\\ParaView 5.13.1\\bin\\" >> $GITHUB_PATH | |
| - if: matrix.test-suite == 'multi-process_a' | |
| run: | | |
| pvpython --version | |
| pvpython -c "from paraview import simple" | |
| - env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| python -m pytest --durations=10 -v -p no:unraisableexception -We tests/examples_tests/test_tests_completeness.py | |
| python -m pytest --durations=10 -v -p no:unraisableexception -We tests/examples_tests/test_run* --suite ${{ matrix.test-suite }} | |
| devops: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/[email protected] | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 # https://github.com/pypa/setuptools_scm/issues/480 | |
| - run: pip install -r tests/devops_tests/requirements.txt "flexparser<0.4" | |
| - env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: pytest --durations=10 -v -s -We -p no:unraisableexception -k "not test_run_notebooks" tests/devops_tests | |
| package: | |
| strategy: | |
| matrix: | |
| packages-dir: [".", "examples"] | |
| fail-fast: false | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/[email protected] | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 # https://github.com/pypa/setuptools_scm/issues/480 | |
| - uses: actions/[email protected] | |
| with: | |
| python-version: "3.12" | |
| - run: pip install twine build | |
| - run: | | |
| unset CI | |
| cd ${{ matrix.packages-dir }} | |
| python -m build 2>&1 | tee build.log | |
| exit `fgrep -i warning build.log | grep -v impl_numba/warnings.py \ | |
| | grep -v "no previously-included files matching" \ | |
| | grep -v "version of {dist_name} already set" \ | |
| | grep -v -E "UserWarning: version of PySDM(-examples)? already set" \ | |
| | wc -l` | |
| twine check --strict dist/* | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| path: ${{ matrix.packages-dir }}/dist | |
| name: dist-${{ matrix.packages-dir }} | |
| pkg_install_check: | |
| strategy: | |
| matrix: | |
| platform: [ubuntu-latest, macos-13, macos-14, windows-latest] | |
| python-version: ["3.9", "3.12"] | |
| exclude: | |
| - platform: macos-14 | |
| python-version: "3.9" | |
| runs-on: ${{ matrix.platform }} | |
| needs: [package] | |
| steps: | |
| - uses: actions/[email protected] | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| pattern: dist-* | |
| merge-multiple: true | |
| path: dist | |
| - run: python -m pip install $PIP_INSTALL_ARGS dist/*.whl | |
| - run: python -c "import PySDM; print(PySDM.__version__)" | |
| - run: python -c "import PySDM_examples; print(PySDM_examples.__version__)" | |
| publish: | |
| runs-on: ubuntu-latest | |
| needs: [tests, examples, devops, package, pkg_install_check] | |
| permissions: | |
| id-token: write | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| pattern: dist-* | |
| merge-multiple: true | |
| path: dist | |
| - if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| uses: pypa/gh-action-pypi-publish@release/v1.12 | |
| with: | |
| attestations: false | |
| repository_url: https://test.pypi.org/legacy/ | |
| packages-dir: dist | |
| - if: startsWith(github.ref, 'refs/tags') | |
| uses: pypa/gh-action-pypi-publish@release/v1.12 | |
| with: | |
| attestations: false | |
| packages-dir: dist |