Merge pull request #1528 from haberman/23-cherrypick #3425
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Python Tests | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - '[0-9]+.x' | |
| pull_request: | |
| branches: | |
| - main | |
| - '[0-9]+.x' | |
| workflow_dispatch: | |
| jobs: | |
| build_wheels: | |
| name: Build Wheels | |
| runs-on: ubuntu-large | |
| if: ${{ github.event.pull_request.head.repo.full_name == 'protocolbuffers/upb' }} | |
| env: | |
| # Bazel 5.4.0. Once we have moved to toolchains, we can update to Bazel 6.x. | |
| DOCKER_IMAGE: us-docker.pkg.dev/protobuf-build/release-containers/linux/apple@sha256:bb1d14738449916d489c1cbb062508c4dca5bd265ea3e67a2628ae40912b9b00 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up Cloud SDK | |
| uses: google-github-actions/auth@v0 | |
| with: | |
| credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }} | |
| export_environment_variables: true | |
| - name: Use gcloud CLI | |
| run: gcloud info | |
| - name: Configure Docker | |
| run: gcloud auth configure-docker -q us-docker.pkg.dev | |
| - name: Pull Docker Image | |
| run: docker pull $DOCKER_IMAGE | |
| - name: Check Bazel version | |
| run: cd ${{ github.workspace }} && docker run -v$PWD:/workspace $DOCKER_IMAGE --version | |
| - id: bazel-cache | |
| name: Set up Bazel caching | |
| uses: ./.github/actions/setup-bazel-cache | |
| - name: Build Wheels | |
| run: cd ${{ github.workspace }} && docker run -e GOOGLE_APPLICATION_CREDENTIALS=/workspace/$(basename $GOOGLE_APPLICATION_CREDENTIALS) -v$PWD:/workspace $DOCKER_IMAGE build $BAZEL_CACHE --crosstool_top=@com_google_protobuf//toolchain:clang_suite --symlink_prefix=/ -c dbg python/dist ${{ steps.bazel-cache.outputs.cache_args }} python/dist:test_wheel python/dist:source_wheel | |
| - name: Move Wheels | |
| run: mkdir wheels && find _build/out \( -name 'protobuf*.whl' -o -name 'protobuf-*.tar.gz' \) -exec mv '{}' wheels ';' | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: python-wheels | |
| path: wheels/ | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: requirements | |
| # Tests shouldn't have access to the whole upb repo, upload the one file we need | |
| path: python/requirements.txt | |
| test_wheels: | |
| name: Test Wheels | |
| needs: build_wheels | |
| strategy: | |
| fail-fast: false # Don't cancel all jobs if one fails. | |
| matrix: | |
| include: | |
| # Linux and Mac use the limited API, so all Python versions will use a single wheel. | |
| # As a result we can just test the oldest and newest supported Python versions and assume | |
| # this gives us sufficient test coverage. | |
| - { os: ubuntu-18-large, python-version: "3.7", architecture: x64, type: 'binary' } | |
| - { os: macos-11, python-version: "3.7", architecture: x64, type: 'binary' } | |
| - { os: ubuntu-20-large, python-version: "3.10", architecture: x64, type: 'binary' } | |
| - { os: macos-12, python-version: "3.10", architecture: x64, type: 'binary' } | |
| - { os: ubuntu-18-large, python-version: "3.7", architecture: x64, type: 'source' } | |
| - { os: macos-11, python-version: "3.7", architecture: x64, type: 'source' } | |
| - { os: ubuntu-20-large, python-version: "3.10", architecture: x64, type: 'source' } | |
| - { os: macos-12, python-version: "3.10", architecture: x64, type: 'source' } | |
| # Windows uses the full API up until Python 3.10, so each of these | |
| # jobs tests a distinct binary wheel. | |
| - { os: windows-2019-large, python-version: "3.7", architecture: x86, type: 'binary' } | |
| - { os: windows-2019-large, python-version: "3.8", architecture: x86, type: 'binary' } | |
| - { os: windows-2019-large, python-version: "3.9", architecture: x86, type: 'binary' } | |
| - { os: windows-2019-large, python-version: "3.10", architecture: x86, type: 'binary' } | |
| - { os: windows-2019-large, python-version: "3.7", architecture: x64, type: 'binary' } | |
| - { os: windows-2019-large, python-version: "3.8", architecture: x64, type: 'binary' } | |
| - { os: windows-2019-large, python-version: "3.9", architecture: x64, type: 'binary' } | |
| - { os: windows-2019-large, python-version: "3.10", architecture: x64, type: 'binary' } | |
| runs-on: ${{ matrix.os }} | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Download Wheels | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: python-wheels | |
| path: wheels | |
| - name: Download Requirements | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: requirements | |
| path: requirements | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| architecture: ${{ matrix.architecture }} | |
| - name: Setup Python venv | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m venv env | |
| # Windows uses 'Scripts' instead of 'bin' | |
| source env/bin/activate || source env/Scripts/activate | |
| echo "VIRTUAL ENV:" $VIRTUAL_ENV | |
| - name: Install tzdata | |
| run: pip install tzdata | |
| # Only needed on Windows, Linux ships with tzdata. | |
| if: ${{ contains(matrix.os, 'windows') }} | |
| - name: Install requirements | |
| run: pip install -r requirements/requirements.txt | |
| - name: Install Protobuf Binary Wheel | |
| run: pip install -vvv --no-index --find-links wheels protobuf | |
| if: ${{ matrix.type == 'binary' }} | |
| - name: Install Protobuf Source Wheel | |
| run: | | |
| cd wheels | |
| tar -xzvf *.tar.gz | |
| cd protobuf-*/ | |
| pip install . | |
| if: ${{ matrix.type == 'source' }} | |
| - name: Test that module is importable | |
| run: python -v -c 'from google._upb import _message; assert "google._upb._message.MessageMeta" in str(_message.MessageMeta)' | |
| if: ${{ !matrix.pure_python }} | |
| - name: Install Protobuf Test Wheel | |
| run: pip install -vvv --no-index --find-links wheels protobuftests | |
| - name: Run the unit tests | |
| run: | | |
| TESTS=$(pip show -f protobuftests | grep pb_unit_tests.*py$ | sed 's,/,.,g' | sed 's,\\,.,g' | sed -E 's,.py$,,g') | |
| for test in $TESTS; do | |
| python -m unittest -v $test | |
| done | |
| test_pure_python_wheels: | |
| name: Test Pure Python Wheels | |
| needs: build_wheels | |
| strategy: | |
| fail-fast: false # Don't cancel all jobs if one fails. | |
| matrix: | |
| python-version: ["3.7", "3.10"] | |
| runs-on: ubuntu-large | |
| steps: | |
| - name: Download Wheels | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: python-wheels | |
| path: wheels | |
| - name: Delete Binary Wheels | |
| run: find wheels -type f | grep -v none-any | xargs rm | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Setup Python venv | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m venv env | |
| source env/bin/activate | |
| echo "VIRTUAL ENV:" $VIRTUAL_ENV | |
| - name: Install numpy | |
| run: pip install numpy | |
| - name: Install Protobuf Wheels | |
| run: pip install -vvv --no-index --find-links wheels protobuf protobuftests | |
| - name: Run the unit tests | |
| run: | | |
| TESTS=$(pip show -f protobuftests | grep _test.py | sed 's,/,.,g' | sed -E 's,.py$,,g') | |
| for test in $TESTS; do | |
| python -m unittest -v $test | |
| done |