tests+artifacts+pypi.yml

name: tests+artifacts+pypi

defaults:
  run:
    shell: bash

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]
  schedule:
    - cron: '0 13 * * 4'
  release:
    types: [published]

jobs:
  precommit:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4.1.6
      - uses: actions/setup-python@v5.0.0
        with:
          python-version: 3.9
      - run: |
          pip install pre-commit
          pre-commit clean
          pre-commit autoupdate
          pre-commit run --all-files
      - if: ${{ failure() }}
        run: git diff

  nojit_and_codecov:
    needs: [precommit, pylint]
    runs-on: ubuntu-latest
    env:
      NUMBA_DISABLE_JIT: 1
    steps:
    - uses: actions/checkout@v4.1.6
      with:
        submodules: recursive
        fetch-depth: 0
    - uses: actions/setup-python@v5.0.0
      with:
        python-version: 3.9
    - name: Generate coverage report
      run: |
        pip install -e .[tests]
        pip install -e ./examples
        pip install pytest-cov
        pytest --durations=10 -We tests/unit_tests --cov-report=xml --cov=PySDM
    - uses: codecov/codecov-action@v4.5.0
      with:
        fail_ci_if_error: true
        verbose: true
        token: ${{ secrets.CODECOV_TOKEN }}

  pdoc:
    strategy:
      matrix:
        platform: [ubuntu-latest, macos-13, windows-latest]
    runs-on: ${{ matrix.platform }}
    steps:
      - uses: actions/checkout@v4.1.6
        with:
          submodules: recursive
          persist-credentials: false
      - uses: actions/setup-python@v5.0.0
        with:
          python-version: 3.9
      - run: |
          pip install pdoc nbformat
          pip install -e .
          pip install -e examples
          
          python - <<EOF
          import glob, nbformat
          for notebook_path in glob.glob('examples/PySDM_examples/*/*.ipynb'):
            with open(notebook_path, encoding="utf8") as fin:
              with open(notebook_path + ".badges.md", 'w') as fout:
                fout.write(nbformat.read(fin, nbformat.NO_CONVERT).cells[0].source)
          EOF
          PDOC_ALLOW_EXEC=1 python -We -m pdoc -o html PySDM examples/PySDM_examples -t docs/templates --math --mermaid
      - if: ${{ github.ref == 'refs/heads/main' && matrix.platform == 'ubuntu-latest' }}
        uses: JamesIves/github-pages-deploy-action@4.1.1
        with:
          BRANCH: pdoc
          FOLDER: html
          CLEAN: true

  pylint:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4.1.6
      with:
        submodules: recursive
        fetch-depth: 0
    - name: Set up Python 3.9
      uses: actions/setup-python@v5.0.0
      with:
        python-version: 3.9
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        # TODO #1160 https://github.com/pylint-dev/pylint/issues/9099
        pip install "pylint<3.0.0" nbqa
        pip install -r tests/devops_tests/requirements.txt
        pip install -e ./examples[tests]
        pip install -e .[tests]
    - run: |
        # TODO #682
        pylint --unsafe-load-any-extension=y --disable=fixme,invalid-name,missing-function-docstring,missing-class-docstring,protected-access,duplicate-code $(git ls-files '*.py' | grep -v -e ^examples -e ^tutorials)
    - run: |
        # TODO #682
        pylint --max-module-lines=550 --unsafe-load-any-extension=y --disable=fixme,too-many-function-args,unsubscriptable-object,consider-using-with,protected-access,too-many-statements,too-many-public-methods,too-many-branches,duplicate-code,invalid-name,missing-function-docstring,missing-module-docstring,missing-class-docstring,too-many-locals,too-many-instance-attributes,too-few-public-methods,too-many-arguments,c-extension-no-member $(git ls-files '*.py' | grep ^examples)
    - run: |
        # TODO #682
        nbqa pylint --unsafe-load-any-extension=y --disable=fixme,duplicate-code,invalid-name,trailing-whitespace,line-too-long,missing-function-docstring,wrong-import-position,missing-module-docstring,wrong-import-order,ungrouped-imports,no-member,too-many-locals,unnecessary-lambda-assignment $(git ls-files '*.ipynb')

  zenodo_json:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4.1.6
      - uses: notiz-dev/github-action-json-property@release
        with:
          path: '.zenodo.json'
          prop_path: 'creators'

  tests:
    needs: [pdoc, nojit_and_codecov, zenodo_json]
    strategy:
      matrix:
        platform: [ubuntu-latest, macos-13, macos-14, windows-latest]
        python-version: ["3.9", "3.11"]
        test-suite: ["unit_tests/!(dynamics)", "unit_tests/dynamics/!(condensation)", "unit_tests/dynamics/condensation", "smoke_tests/no_env", "smoke_tests/box", "smoke_tests/parcel_a", "smoke_tests/parcel_b", "smoke_tests/parcel_c", "smoke_tests/kinematic_1d", "smoke_tests/kinematic_2d", "tutorials_tests"]
        exclude:
          - platform: "macos-14"
            python-version: "3.9"
          - test-suite: "devops_tests"
            python-version: "3.9"
          # TODO #1406
          - platform: "macos-14"
            test-suite: "smoke_tests/box"
          - platform: "macos-14"
            test-suite: "smoke_tests/parcel_a"
          - platform: "macos-14"
            test-suite: "smoke_tests/kinematic_2d"
          - platform: "macos-14"
            test-suite: "unit_tests/!(dynamics)"
      fail-fast: false
    runs-on: ${{ matrix.platform }}
    timeout-minutes: ${{ startsWith(matrix.platform, 'windows-') && 40 || 25 }}
    steps:
      - uses: actions/checkout@v4.1.6
        with:
          submodules: recursive
          fetch-depth: 0
      - uses: actions/setup-python@v5.0.0
        with:
          python-version: ${{ matrix.python-version }}

      - run: pip install -e .

      # sanity check if we do not depend on something from tests extras
      - run: python -We -c "import PySDM"

      - if: startsWith(matrix.test-suite, 'smoke_tests') || matrix.test-suite == 'tutorials_tests'
        run: pip install -e ./examples

      # incl. ensuring that the previous pip did not overwrite the package
      - run: |
          pip install --only-binary :all: -e .[tests]

      # install devops_tests for tutorials_tests
      - if: matrix.test-suite == 'tutorials_tests'
        run: pip install -r tests/devops_tests/requirements.txt

      # https://github.com/numba/numba/issues/6350#issuecomment-728174860
      - if: startsWith(matrix.platform, 'ubuntu-')
        run: echo NUMBA_THREADING_LAYER=omp >> $GITHUB_ENV

      - name: Run pytest
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          NUMBA_OPT: 1
        run: |
          shopt -s extglob
          pytest --durations=10 --timeout=900 --timeout_method=thread -p no:unraisableexception -We tests/${{ matrix.test-suite }}

#       - if: ${{ failure() }}
#         uses: mxschmitt/action-tmate@v3
#         with:
#           limit-access-to-actor: true

  examples-setup:
    needs: [nojit_and_codecov]
    strategy:
      matrix:
        platform: [ubuntu-24.04, macos-13, windows-latest]
        python-version: ["3.9", "3.11"]
    runs-on: ${{ matrix.platform }}
    steps:
      - uses: actions/checkout@v4.1.6
        with:
          submodules: recursive
          fetch-depth: 0
      - uses: actions/setup-python@v5.0.0
        with:
          python-version: ${{ matrix.python-version }}
      - run: |
          echo pip_user_site=$(python -c "import sysconfig; print(sysconfig.get_path('purelib'))") >> $GITHUB_ENV
          echo toml_ci_md5=$(cat pyproject.toml setup.py examples/pyproject.toml examples/setup.py tests/devops_tests/requirements.txt .github/workflows/tests+artifacts+pypi.yml \
          | python -c "import hashlib;print(hashlib.md5(open(0,'rb').read()).hexdigest())") >> $GITHUB_ENV
      - uses: actions/cache@v4
        id: cache
        with:
          path: ${{ env.pip_user_site }}
          key: examples_pip_cache-${{ matrix.platform }}-${{ matrix.python-version }}-${{ env.toml_ci_md5 }}
          lookup-only: true
      - if: steps.cache.outputs.cache-hit != 'true'
        run: |
          python -m pip uninstall -y jupyterlab-server # https://github.com/pypa/pip/issues/6275
          python -m pip install -e . -e ./examples
          python -m pip install -r tests/devops_tests/requirements.txt
      - if: steps.cache.outputs.cache-hit != 'true'
        uses: actions/cache/save@v4
        with:
          path: ${{ env.pip_user_site }}
          key: examples_pip_cache-${{ matrix.platform }}-${{ matrix.python-version }}-${{ env.toml_ci_md5 }}

  examples:
    needs: [examples-setup]
    strategy:
      matrix:
        platform: [ubuntu-24.04, macos-13, windows-latest]
        python-version: ["3.9", "3.11"]
        test-suite: [ "chemistry_freezing_isotopes", "condensation_a", "condensation_b", "coagulation", "breakup", "multi-process_a", "multi-process_b"]
      fail-fast: false
    runs-on: ${{ matrix.platform }}
    timeout-minutes: ${{ !startsWith(matrix.platform, 'ubuntu-') && 65 || 50 }}
    steps:
      - uses: actions/checkout@v4.1.6
        with:
          submodules: recursive
          fetch-depth: 0
      - uses: actions/setup-python@v5.0.0
        with:
          python-version: ${{ matrix.python-version }}
      - run: |
          echo pip_user_site=$(python -c "import sysconfig; print(sysconfig.get_path('purelib'))") >> $GITHUB_ENV
          echo toml_ci_md5=$(cat pyproject.toml setup.py examples/pyproject.toml examples/setup.py tests/devops_tests/requirements.txt .github/workflows/tests+artifacts+pypi.yml \
          | python -c "import hashlib;print(hashlib.md5(open(0,'rb').read()).hexdigest())") >> $GITHUB_ENV
      - run: |
          rm -rf $pip_user_site
      - id: cache
        uses: actions/cache/restore@v4
        with:
          fail-on-cache-miss: true
          path: ${{ env.pip_user_site }}
          key: examples_pip_cache-${{ matrix.platform }}-${{ matrix.python-version }}-${{ env.toml_ci_md5 }}

      # https://github.com/numba/numba/issues/6350#issuecomment-728174860
      - if: startsWith(matrix.platform, 'ubuntu-')
        run: echo NUMBA_THREADING_LAYER=omp >> $GITHUB_ENV

      - if: startsWith(matrix.platform, 'ubuntu-') && matrix.test-suite == 'multi-process_a'
        run: |
          sudo apt-get install libegl1 libopengl0 ghostscript
          wget -nv "https://paraview.org/paraview-downloads/download.php?submit=Download&version=v5.13&type=binary&os=Linux&downloadFile=ParaView-5.13.1-egl-MPI-Linux-Python3.10-x86_64.tar.gz" -O paraview.tar.gz
          tar xzf paraview.tar.gz
          echo `pwd`/`dirname ParaView*/bin/pvpython` >> $GITHUB_PATH
          
      - if: startsWith(matrix.platform, 'macos-') && matrix.test-suite == 'multi-process_a'
        run: |
          brew install --cask paraview
          echo `dirname /Applications/ParaView-*.app/Contents/bin/pvpython | head -1` >> $GITHUB_PATH
          brew install ghostscript

      - if: startsWith(matrix.platform, 'windows-') && matrix.test-suite == 'multi-process_a'
        run: |
          curl "https://www.paraview.org/paraview-downloads/download.php?submit=Download&version=v5.13&type=binary&os=Windows&downloadFile=ParaView-5.13.1-Windows-Python3.10-msvc2017-AMD64.msi" -o paraview.msi
          msiexec //i paraview.msi //quiet //qn //norestart 
          echo "C:\\Program Files\\ParaView 5.13.1\\bin\\" >> $GITHUB_PATH

      - if: matrix.test-suite == 'multi-process_a'
        run: |
          pvpython --version
          pvpython -c "from paraview import simple"

      - env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          python -m pytest --durations=10 -v -p no:unraisableexception -We tests/examples_tests/test_tests_completeness.py
          python -m pytest --durations=10 -v -p no:unraisableexception -We tests/examples_tests/test_run* --suite ${{ matrix.test-suite }}

  devops:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4.1.6
        with:
          submodules: recursive
          fetch-depth: 0  # https://github.com/pypa/setuptools_scm/issues/480
      - run: pip install -r tests/devops_tests/requirements.txt "flexparser<0.4"
      - env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: pytest --durations=10 -v -s -We -p no:unraisableexception -k "not test_run_notebooks" tests/devops_tests

  package:
    strategy:
      matrix:
        packages-dir: [".", "examples"]
      fail-fast: false
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4.1.6
        with:
          submodules: recursive
          fetch-depth: 0  # https://github.com/pypa/setuptools_scm/issues/480
      - uses: actions/setup-python@v5.0.0
        with:
          python-version: "3.11"

      - run: pip install twine build

      - run: |
          unset CI
          cd ${{ matrix.packages-dir }}
          python -m build 2>&1 | tee build.log
          exit `fgrep -i warning build.log | grep -v impl_numba/warnings.py | grep -v "no previously-included files matching" | wc -l`
          twine check --strict dist/*

      - uses: actions/upload-artifact@v4
        with:
          path: ${{ matrix.packages-dir }}/dist
          name: dist-${{ matrix.packages-dir }}

  pkg_install_check:
    strategy:
      matrix:
        platform: [ubuntu-latest, macos-13, macos-14, windows-latest]
        python-version: ["3.9", "3.11"]
        exclude:
          - platform: macos-14
            python-version: "3.9"
    runs-on: ${{ matrix.platform }}
    needs: [package]
    steps:
      - uses: actions/setup-python@v5.0.0
        with:
          python-version: ${{ matrix.python-version }}
      - uses: actions/download-artifact@v4
        with:
          pattern: dist-*
          merge-multiple: true
          path: dist
      - run: python -m pip install $PIP_INSTALL_ARGS dist/*.whl
      - run: python -c "import PySDM; print(PySDM.__version__)"
      - run: python -c "import PySDM_examples; print(PySDM_examples.__version__)"

  publish:
    runs-on: ubuntu-latest
    needs: [tests, examples, devops, package, pkg_install_check]
    permissions:
      id-token: write
    steps:
      - uses: actions/download-artifact@v4
        with:
          pattern: dist-*
          merge-multiple: true
          path: dist

      - if: github.event_name == 'push' && github.ref == 'refs/heads/main'
        uses: pypa/gh-action-pypi-publish@release/v1.12
        with:
          attestations: false
          repository_url: https://test.pypi.org/legacy/
          packages-dir: dist

      - if: startsWith(github.ref, 'refs/tags')
        uses: pypa/gh-action-pypi-publish@release/v1.12
        with:
          attestations: false
          packages-dir: dist