precommit.yml

name: precommit test
on:
  push:
    branches: [ main ]
  pull_request:
    branches: '**'
  workflow_dispatch:
      inputs:
        logLevel:
          default: warning
          description: "Log level"
          required: true
        tags:
          description: "Test scenario tags"

concurrency:
  group: ${{ github.ref }}
  cancel-in-progress: true

jobs:
  lint:
    name: Check linting
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.9'
      - name: Display Python version
        run: python -c "import sys; import os; print(\"\n\".join(os.environ[\"PATH\"].split(os.pathsep))); print(sys.version); print(sys.executable);"
      - name: Upgrade setuptools and pip
        run: python -m pip install -U setuptools pip
      - name: Install tox
        run: python -m pip install tox
      - name: Install protoc
        shell: bash
        run: .github/scripts/install_protoc.sh
      - name: Run fix_lint
        run: python -m tox -e fix_lint

  type_checking:
    name: Type Checking
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.9'
      - name: Display Python version
        run: python -c "import sys; import os; print(\"\n\".join(os.environ[\"PATH\"].split(os.pathsep))); print(sys.version); print(sys.executable);"
      - name: Upgrade setuptools and pip
        run: python -m pip install -U setuptools pip
      - name: Install tox
        run: python -m pip install tox
      - name: Install protoc
        shell: bash
        run: .github/scripts/install_protoc.sh
      - name: Run pyright on Selected Files
        run: python -m tox -e pyright

  build:
    needs: lint
    name: Build Wheel File
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Code
        uses: actions/checkout@v4
      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.9
      - name: Install protoc
        shell: bash
        run: .github/scripts/install_protoc.sh
      - name: Upgrade setuptools and pip
        run: python -m pip install -U setuptools pip wheel
      - name: Generate wheel
        run: python -m pip wheel -v -w dist --no-deps .
      - name: Show wheels generated
        run: ls -lh dist
      - uses: actions/upload-artifact@v4
        with:
          include-hidden-files: true
          name: wheel
          path: dist/

  test:
    name: Test py-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
    needs: build
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [macos-latest, windows-latest-64-cores, ubuntu-latest-64-cores]
        python-version: ["3.9", "3.10", "3.11"]
        cloud-provider: [aws, gcp, azure]
        exclude:
          # only run macos with aws py3.9 for doctest
          - os: macos-latest
            python-version: "3.10"
          - os: macos-latest
            python-version: "3.11"
          - os: macos-latest
            python-version: "3.9"
            cloud-provider: azure
          - os: macos-latest
            python-version: "3.9"
            cloud-provider: gcp
          # only run ubuntu with py3.9 on aws and py3.10 on azure
          - os: ubuntu-latest-64-cores
            python-version: "3.11"
          - os: ubuntu-latest-64-cores
            python-version: "3.9"
            cloud-provider: azure
          - os: ubuntu-latest-64-cores
            python-version: "3.9"
            cloud-provider: gcp
          - os: ubuntu-latest-64-cores
            python-version: "3.10"
            cloud-provider: aws
          - os: ubuntu-latest-64-cores
            python-version: "3.10"
            cloud-provider: gcp
          # only run windows with py3.10 on gcp
          - os: windows-latest-64-cores
            python-version: "3.9"
          - os: windows-latest-64-cores
            python-version: "3.10"
          - os: windows-latest-64-cores
            python-version: "3.11"
            cloud-provider: aws
          - os: windows-latest-64-cores
            python-version: "3.11"
            cloud-provider: azure
    steps:
      - name: Checkout Code
        uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Display Python version
        run: python -c "import sys; print(sys.version)"
      - name: Decrypt parameters.py
        shell: bash
        run: .github/scripts/decrypt_parameters.sh
        env:
          PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
          CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
      - name: Install protoc
        shell: bash
        run: .github/scripts/install_protoc.sh
      - name: Add protoc to Windows path
        if: runner.os == 'Windows'
        run: |
          echo "$HOME/local/bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
      - name: Download wheel(s)
        uses: actions/download-artifact@v4
        with:
          name: wheel
          path: dist
      - name: Show wheels downloaded
        run: ls -lh dist
        shell: bash
      - name: Upgrade setuptools, pip and wheel
        run: python -m pip install -U setuptools pip wheel
      - name: Install tox
        run: python -m pip install tox
      # we only run doctest on macos
      - if: ${{ matrix.os == 'macos-latest' }}
        name: Run doctests
        run: python -m tox -e "py${PYTHON_VERSION}-doctest-notudf-ci"
        env:
          PYTHON_VERSION: ${{ matrix.python-version }}
          cloud_provider: ${{ matrix.cloud-provider }}
          PYTEST_ADDOPTS: --color=yes --tb=short
          TOX_PARALLEL_NO_SPINNER: 1
          # Specify SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1 when adding >= python3.12 with no server-side support
          # For example, see https://github.com/snowflakedb/snowpark-python/pull/681
        shell: bash
      # do not run other tests for macos
      - if: ${{ matrix.os != 'macos-latest' }}
        name: Run tests (excluding doctests)
        run: python -m tox -e "py${PYTHON_VERSION/\./}-notdoctest-ci"
        env:
          PYTHON_VERSION: ${{ matrix.python-version }}
          cloud_provider: ${{ matrix.cloud-provider }}
          PYTEST_ADDOPTS: --color=yes --tb=short
          TOX_PARALLEL_NO_SPINNER: 1
        shell: bash
      - name: Combine coverages
        run: python -m tox -e coverage --skip-missing-interpreters false
        shell: bash
        env:
          SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
      - uses: actions/upload-artifact@v4
        with:
          include-hidden-files: true
          name: coverage_${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
          path: |
            .tox/.coverage
            .tox/coverage.xml

  test-local-testing:
    name: Test Local Testing Module py-${{ matrix.os }}-${{ matrix.python-version }}
    needs: build
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [macos-latest, windows-latest, ubuntu-latest]
        python-version: [ "3.8", "3.9", "3.10", "3.11" ]
        cloud-provider: [aws]
    steps:
      - name: Checkout Code
        uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Display Python version
        run: python -c "import sys; print(sys.version)"
      - name: Decrypt parameters.py
        shell: bash
        run: .github/scripts/decrypt_parameters.sh
        env:
          PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
          CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
      - name: Install protoc
        shell: bash
        run: .github/scripts/install_protoc.sh
      - name: Add protoc to Windows path
        if: runner.os == 'Windows'
        run: |
          echo "$HOME/local/bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
      - name: Download wheel(s)
        uses: actions/download-artifact@v4
        with:
          name: wheel
          path: dist
      - name: Show wheels downloaded
        run: ls -lh dist
        shell: bash
      - name: Upgrade setuptools, pip and wheel
        run: python -m pip install -U setuptools pip wheel
      - name: Install tox
        run: python -m pip install tox
      - name: Run tests
        run: python -m tox -e "py${PYTHON_VERSION/\./}-local"
        env:
          PYTHON_VERSION: ${{ matrix.python-version }}
          cloud_provider: ${{ matrix.cloud-provider }}
          PYTEST_ADDOPTS: --color=yes --tb=short
          TOX_PARALLEL_NO_SPINNER: 1
          SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
        shell: bash
      - name: Combine coverages
        run: python -m tox -e coverage --skip-missing-interpreters false
        shell: bash
        env:
          SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
      - uses: actions/upload-artifact@v4
        with:
          include-hidden-files: true
          name: coverage_${{ matrix.os }}-${{ matrix.python-version }}-local-testing
          path: |
            .tox/.coverage
            .tox/coverage.xml

  test-ast-encoding:
    name: Test AST Encoding py-${{ matrix.os }}-${{ matrix.python-version }}
    needs: build
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [macos-latest, windows-latest, ubuntu-latest]
        # TODO SNOW-1763185 use py39 by default
        # Protobuf encoding differs per python version, use 3.8 by default.
        python-version: [ "3.8"]
        cloud-provider: [aws]
        protobuf-version: ["3.20.1"]
    steps:
      - name: Checkout Code
        uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Display Python version
        run: python -c "import sys; print(sys.version)"
      - name: Decrypt parameters.py
        shell: bash
        run: .github/scripts/decrypt_parameters.sh
        env:
          PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
          CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
      - name: Install protoc
        shell: bash
        run: .github/scripts/install_protoc.sh
      - name: Download wheel(s)
        uses: actions/download-artifact@v4
        with:
          name: wheel
          path: dist
      - name: Show wheels downloaded
        run: ls -lh dist
        shell: bash
      - name: Upgrade setuptools, pip and wheel
        run: python -m pip install -U setuptools pip wheel
      - name: Install protobuf
        run: python -m pip install protobuf==${{ matrix.protobuf-version }}
      - name: Install tox
        run: python -m pip install tox
      - name: Set up Windows environment
        if: runner.os == 'Windows'
        # windows timezone management is broken, that's why we need a special fix here
        run: |
            "TZ=EST" >> $env:GITHUB_ENV
            echo "TZ=EST" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
            echo "$HOME/local/bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
      - name: Run tests
        # TODO: SNOW-1763186 use pytest marker to test multiple TZs
        run: TZ=EST python -m tox -e "py${PYTHON_VERSION/\./}-ast"
        env:
          PYTHON_VERSION: ${{ matrix.python-version }}
          cloud_provider: ${{ matrix.cloud-provider }}
          PYTEST_ADDOPTS: --color=yes --tb=short
          TOX_PARALLEL_NO_SPINNER: 1
          SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
          TZ: EST
        shell: bash

  test-snowpark-pandas:
    name: Test modin-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
    needs: build
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [macos-latest, windows-latest-64-cores, ubuntu-latest-64-cores]
        python-version: [ "3.9", "3.10", "3.11" ]
        cloud-provider: [aws, gcp, azure]
        exclude:
          # only run macos with aws py3.9 for doctest
          - os: macos-latest
            python-version: "3.10"
          - os: macos-latest
            python-version: "3.11"
          - os: macos-latest
            python-version: "3.9"
            cloud-provider: azure
          - os: macos-latest
            python-version: "3.9"
            cloud-provider: gcp
          # only run ubuntu with py3.9 on aws and py3.10 on azure
          - os: ubuntu-latest-64-cores
            python-version: "3.11"
          - os: ubuntu-latest-64-cores
            python-version: "3.9"
            cloud-provider: azure
          - os: ubuntu-latest-64-cores
            python-version: "3.9"
            cloud-provider: gcp
          - os: ubuntu-latest-64-cores
            python-version: "3.10"
            cloud-provider: aws
          - os: ubuntu-latest-64-cores
            python-version: "3.10"
            cloud-provider: gcp
          # only run windows with py3.10 on gcp
          - os: windows-latest-64-cores
            python-version: "3.9"
          - os: windows-latest-64-cores
            python-version: "3.10"
          - os: windows-latest-64-cores
            python-version: "3.11"
            cloud-provider: aws
          - os: windows-latest-64-cores
            python-version: "3.11"
            cloud-provider: azure
    steps:
      - name: Checkout Code
        uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Display Python version
        run: python -c "import sys; print(sys.version)"
      - name: Decrypt parameters.py
        shell: bash
        run: .github/scripts/decrypt_parameters.sh
        env:
          PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
          CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
      - name: Install protoc
        shell: bash
        run: .github/scripts/install_protoc.sh
      - name: Add protoc to Windows path
        if: runner.os == 'Windows'
        run: |
          echo "$HOME/local/bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
      - name: Download wheel(s)
        uses: actions/download-artifact@v4
        with:
          name: wheel
          path: dist
      - name: Show wheels downloaded
        run: ls -lh dist
        shell: bash
      - name: Upgrade setuptools, pip and wheel
        run: python -m pip install -U setuptools pip wheel
      - name: Install tox
        run: python -m pip install tox
        # only run doctest for macos on aws
      - if: ${{ matrix.os == 'macos-latest' && matrix.cloud-provider == 'aws' }}
        name: Run Snowpark pandas API doctests
        run: python -m tox -e "py${PYTHON_VERSION}-doctest-snowparkpandasdoctest-modin-ci"
        env:
          PYTHON_VERSION: ${{ matrix.python-version }}
          cloud_provider: ${{ matrix.cloud-provider }}
          PYTEST_ADDOPTS: --color=yes --tb=short
          TOX_PARALLEL_NO_SPINNER: 1
          # Specify SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1 when adding >= python3.11 with no server-side support
          # For example, see https://github.com/snowflakedb/snowpark-python/pull/681
        shell: bash
        # do not run other tests for macos on aws
      - if: ${{ !(matrix.os == 'macos-latest' && matrix.cloud-provider == 'aws') }}
        name: Run Snowpark pandas API tests (excluding doctests)
        run: python -m tox -e "py${PYTHON_VERSION/\./}-snowparkpandasnotdoctest-modin-ci"
        env:
          PYTHON_VERSION: ${{ matrix.python-version }}
          cloud_provider: ${{ matrix.cloud-provider }}
          PYTEST_ADDOPTS: --color=yes --tb=short
          TOX_PARALLEL_NO_SPINNER: 1
        shell: bash
      - name: Combine coverages
        run: python -m tox -e coverage --skip-missing-interpreters false
        shell: bash
        env:
          SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
      - uses: actions/upload-artifact@v4
        with:
          include-hidden-files: true
          name: coverage_${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}-snowpark-pandas-testing
          path: |
            .tox/.coverage
            .tox/coverage.xml

  test-modin-extra-without-pandas-extra:
    name: Test modin extra without pandas extra py-${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}.
    needs: build
    runs-on: ${{ matrix.os.image_name }}
    strategy:
      fail-fast: false
      matrix:
        os:
          - image_name: macos-latest
            download_name: macos
        python-version: ["3.9"]
        cloud-provider: [aws]
    steps:
      - name: Checkout Code
        uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Display Python version
        run: python -c "import sys; print(sys.version)"
      - name: Decrypt parameters.py
        shell: bash
        run: .github/scripts/decrypt_parameters.sh
        env:
          PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
          CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
      - name: Download wheel(s)
        uses: actions/download-artifact@v4
        with:
          name: wheel
          path: dist
      - name: Show wheels downloaded
        run: ls -lh dist
        shell: bash
      - name: Upgrade setuptools, pip and wheel
        run: python -m pip install -U setuptools pip wheel
      - name: Install protoc
        shell: bash
        run: .github/scripts/install_protoc.sh
      - name: Install tox
        run: python -m pip install tox
      - name: Run all tests
        run: python -m tox -e "modin_extra_without_pandas_extra"
        env:
          PYTHON_VERSION: ${{ matrix.python-version }}
          cloud_provider: ${{ matrix.cloud-provider }}
          PYTEST_ADDOPTS: --color=yes --tb=short
          TOX_PARALLEL_NO_SPINNER: 1
        shell: bash
      - name: Combine coverages
        run: python -m tox -e coverage --skip-missing-interpreters false
        shell: bash
      - uses: actions/upload-artifact@v4
        with:
          include-hidden-files: true
          name: coverage_modin_extra_without_pandas_extra
          path: |
            .tox/.coverage
            .tox/coverage.xml

  combine-coverage:
    if: ${{ success() || failure() }}
    name: Combine coverage
    needs:
      - test
      - test-local-testing
      - test-snowpark-pandas
      - test-modin-extra-without-pandas-extra
      - test-ast-encoding
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: actions/download-artifact@v4
        with:
          path: artifacts
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.9'
      - name: Display Python version
        run: python -c "import sys; print(sys.version)"
      - name: Install protoc
        shell: bash
        run: .github/scripts/install_protoc.sh
      - name: Upgrade setuptools and pip
        run: python -m pip install -U setuptools pip
      - name: Install tox
        run: python -m pip install tox
      - name: Collect all coverages to one dir
        run: |
          python -c '
          from pathlib import Path
          import shutil

          src_dir = Path("artifacts")
          dst_dir = Path(".") / ".tox"
          dst_dir.mkdir()
          for src_file in src_dir.glob("*/.coverage"):
              dst_file = dst_dir / ".coverage.{}".format(src_file.parent.name[9:])
              print("{} copy to {}".format(src_file, dst_file))
              shutil.copy(str(src_file), str(dst_file))'
      - name: Combine coverages
        run: python -m tox -e coverage
      - name: Publish html coverage
        uses: actions/upload-artifact@v4
        with:
          include-hidden-files: true
          name: overall_cov_html
          path: .tox/htmlcov
      - name: Publish xml coverage
        uses: actions/upload-artifact@v4
        with:
          include-hidden-files: true
          name: overall_cov_xml
          path: .tox/coverage.xml
      - uses: codecov/codecov-action@v4
        with:
          file: .tox/coverage.xml
          token: ${{ secrets.CODECOV_TOKEN }}
      - name: Show coverage diff
        run: |
          pip install diff_cover
          git log -n 5 --pretty=oneline
          git fetch -f origin main:main || true
          diff-cover .tox/coverage.xml --compare-branch=main --fail-under=95

  doc:
    needs: lint
    name: Build Doc
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Code
        uses: actions/checkout@v4
      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.9'
      - name: Upgrade setuptools and pip
        run: python -m pip install -U setuptools pip
      - name: Install tox
        run: python -m pip install tox
      - name: Install protoc
        shell: bash
        run: .github/scripts/install_protoc.sh
      - name: Build document
        run: python -m tox -e docs
      - name: Upload html files
        uses: actions/upload-artifact@v4
        with:
          include-hidden-files: true
          name: docs
          path: .tox/docs_out