Add variable subsetting support to UMM-S #42
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This is the main build pipeline that verifies and publishes the software | |
| name: Build | |
| # Controls when the workflow will run | |
| on: | |
| # Triggers the workflow on push events | |
| push: | |
| branches: [ develop, release/**, main, feature/**, issue/**, issues/**, dependabot/** ] | |
| tags-ignore: | |
| - '*' | |
| # Do not trigger build if pyproject.toml was the only thing changed | |
| paths-ignore: | |
| - 'pyproject.toml' | |
| - 'poetry.lock' | |
| # Allows you to run this workflow manually from the Actions tab | |
| workflow_dispatch: | |
| inputs: | |
| venue: | |
| type: choice | |
| description: Venue to deploy to | |
| options: | |
| - SIT | |
| - UAT | |
| # Only allow 1 execution of this workflow to be running at any given time per-branch. | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| POETRY_VERSION: "1.8.3" | |
| PYTHON_VERSION: "3.10" | |
| REGISTRY: ghcr.io | |
| IMAGE_NAME: ${{ github.repository }} | |
| jobs: | |
| build: | |
| name: Build, Test, Verify, Publish | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash -el {0} | |
| outputs: | |
| deploy_env: ${{ steps.poetry-build.outputs.deploy_env }} | |
| version: ${{ steps.poetry-build.outputs.the_version }} | |
| pyproject_name: ${{ steps.poetry-build.outputs.pyproject_name }} | |
| python_dist: ${{ steps.poetry-build.outputs.pyproject_name }}-dist | |
| steps: | |
| - uses: getsentry/action-github-app-token@v3 | |
| name: podaac cicd token | |
| id: podaac-cicd | |
| with: | |
| app_id: ${{ secrets.CICD_APP_ID }} | |
| private_key: ${{ secrets.CICD_APP_PRIVATE_KEY }} | |
| - uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ github.repository }} | |
| token: ${{ steps.podaac-cicd.outputs.token }} | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install Poetry | |
| uses: abatilo/actions-poetry@v3 | |
| with: | |
| poetry-version: ${{ env.POETRY_VERSION }} | |
| - name: Setup a local virtual environment | |
| run: | | |
| poetry config virtualenvs.create true --local | |
| poetry config virtualenvs.in-project true --local | |
| - uses: actions/cache@v4 | |
| name: Define a cache for the virtual environment based on the dependencies lock file | |
| with: | |
| path: ./.venv | |
| key: venv-${{ hashFiles('poetry.lock') }} | |
| - name: Get pre-build version | |
| id: get-version | |
| run: | | |
| echo "current_version=$(poetry version | awk '{print $2}')" >> $GITHUB_OUTPUT | |
| echo "pyproject_name=$(poetry version | awk '{print $1}')" >> $GITHUB_ENV | |
| - name: Manual Build | |
| # If triggered by workflow dispatch, no version bump | |
| if: github.event_name == 'workflow_dispatch' | |
| id: manual | |
| run: | | |
| echo "TARGET_ENV_UPPERCASE=${{ github.event.inputs.venue }}" >> $GITHUB_ENV | |
| - name: Bump pre-alpha version | |
| # If triggered by push to a non-tracked branch | |
| if: | | |
| github.ref != 'refs/heads/develop' && | |
| github.ref != 'refs/heads/main' && | |
| !startsWith(github.ref, 'refs/heads/release/') | |
| run: | | |
| new_ver="${{ steps.get-version.outputs.current_version }}+$(git rev-parse --short ${GITHUB_SHA})" | |
| poetry version $new_ver | |
| echo "TARGET_ENV_UPPERCASE=SIT" >> $GITHUB_ENV | |
| - name: Bump alpha version | |
| # If triggered by push to the develop branch | |
| if: | | |
| github.ref == 'refs/heads/develop' && | |
| steps.manual.conclusion == 'skipped' | |
| id: alpha | |
| run: | | |
| poetry version prerelease | |
| echo "TARGET_ENV_UPPERCASE=SIT" >> $GITHUB_ENV | |
| - name: Bump rc version | |
| # If triggered by push to a release branch | |
| if: | | |
| startsWith(github.ref, 'refs/heads/release/') && | |
| steps.manual.conclusion == 'skipped' | |
| id: rc | |
| env: | |
| # True if the version already has a 'rc' pre-release identifier | |
| BUMP_RC: ${{ contains(steps.get-version.outputs.current_version, 'rc') }} | |
| run: | | |
| if [ "$BUMP_RC" = true ]; then | |
| poetry version prerelease | |
| else | |
| poetry version ${GITHUB_REF#refs/heads/release/}rc1 | |
| fi | |
| echo "TARGET_ENV_UPPERCASE=UAT" >> $GITHUB_ENV | |
| - name: Release version | |
| # If triggered by push to the main branch | |
| if: | | |
| startsWith(github.ref, 'refs/heads/main') && | |
| steps.manual.conclusion == 'skipped' | |
| id: release | |
| env: | |
| CURRENT_VERSION: ${{ steps.get-version.outputs.current_version }} | |
| # Remove rc* from end of version string | |
| # The ${string%%substring} syntax below deletes the longest match of $substring from back of $string. | |
| run: | | |
| poetry version ${CURRENT_VERSION%%rc*} | |
| echo "software_version=$(poetry version | awk '{print $2}')" >> $GITHUB_ENV | |
| echo "venue=ops" >> $GITHUB_ENV | |
| echo "TARGET_ENV_UPPERCASE=OPS" >> $GITHUB_ENV | |
| - name: Get install version | |
| # Get the version of the software being installed and save it as an ENV var | |
| run: | | |
| echo "software_version=$(poetry version | awk '{print $2}')" >> $GITHUB_ENV | |
| - name: Install software | |
| run: poetry install -E harmony | |
| - name: Lint | |
| run: | | |
| poetry run pylint net2cog | |
| poetry run flake8 net2cog | |
| - name: Test and coverage | |
| run: | | |
| poetry run pytest --junitxml=build/reports/pytest.xml --cov=net2cog/ --cov-report=xml:build/reports/coverage.xml -m "not aws and not integration" tests/ | |
| - name: SonarCloud Scan | |
| id: sonarcloud | |
| uses: sonarsource/sonarcloud-github-action@master | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} | |
| with: | |
| args: > | |
| -Dsonar.organization=${{ github.repository_owner }} | |
| -Dsonar.projectKey=${{ github.repository_owner }}_${{ github.event.repository.name }} | |
| -Dsonar.python.coverage.reportPaths=build/reports/coverage.xml | |
| -Dsonar.sources=net2cog/ | |
| -Dsonar.tests=tests/ | |
| -Dsonar.projectName=${{ github.repository }} | |
| -Dsonar.projectVersion=${{ env.software_version }} | |
| -Dsonar.python.version=3.10,3.11 | |
| - name: Run Snyk as a blocking step | |
| uses: snyk/actions/python-3.10@master | |
| env: | |
| SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} | |
| with: | |
| command: test | |
| args: > | |
| --org=${{ secrets.SNYK_ORG_ID }} | |
| --project-name=${{ github.repository }} | |
| --severity-threshold=high | |
| --fail-on=all | |
| - name: Run Snyk on Python | |
| uses: snyk/actions/python-3.10@master | |
| env: | |
| SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} | |
| with: | |
| command: monitor | |
| args: > | |
| --org=${{ secrets.SNYK_ORG_ID }} | |
| --project-name=${{ github.repository }} | |
| - name: Build Python Artifact | |
| id: poetry-build | |
| run: | | |
| poetry build | |
| echo "deploy_env=${{ env.TARGET_ENV_UPPERCASE }}" >> $GITHUB_OUTPUT | |
| echo "the_version=$(poetry version | awk '{print $2}')" >> $GITHUB_OUTPUT | |
| echo "pyproject_name=$(poetry version | awk '{print $1}')" >> $GITHUB_OUTPUT | |
| - uses: actions/upload-artifact@v4 | |
| id: python-dist | |
| with: | |
| name: ${{ steps.poetry-build.outputs.pyproject_name }}-dist | |
| path: dist/* | |
| - name: Build Docs | |
| run: | | |
| poetry run sphinx-build -b html ./docs docs/_build/ | |
| - name: Upload Docs Artifact | |
| id: docs | |
| uses: actions/upload-pages-artifact@v3 | |
| with: | |
| path: docs/_build/ | |
| - name: Commit Version Bump | |
| # If building an alpha, release candidate, or release then we commit the version bump back to the repo | |
| if: | | |
| steps.alpha.conclusion == 'success' || | |
| steps.rc.conclusion == 'success' || | |
| steps.release.conclusion == 'success' | |
| run: | | |
| git config user.name "${GITHUB_ACTOR}" | |
| git config user.email "${GITHUB_ACTOR}@users.noreply.github.com" | |
| git commit -am "/version ${{ env.software_version }}" | |
| git push | |
| - name: Push Tag | |
| if: | | |
| steps.alpha.conclusion == 'success' || | |
| steps.rc.conclusion == 'success' || | |
| steps.release.conclusion == 'success' | |
| run: | | |
| git config user.name "${GITHUB_ACTOR}" | |
| git config user.email "${GITHUB_ACTOR}@users.noreply.github.com" | |
| git tag -a "${{ env.software_version }}" -m "Version ${{ env.software_version }}" | |
| git push origin "${{ env.software_version }}" | |
| - name: Create GH release | |
| if: | | |
| steps.alpha.conclusion == 'success' || | |
| steps.rc.conclusion == 'success' || | |
| steps.release.conclusion == 'success' | |
| uses: ncipollo/release-action@v1 | |
| with: | |
| generateReleaseNotes: true | |
| name: ${{ env.software_version }} | |
| prerelease: ${{ steps.alpha.conclusion == 'success' || steps.rc.conclusion == 'success'}} | |
| tag: ${{ env.software_version }} | |
| publish-docs: | |
| name: Publish Docs | |
| needs: [ build ] | |
| runs-on: ubuntu-latest | |
| environment: | |
| name: github-pages | |
| url: ${{ steps.deployment.outputs.page_url }} | |
| permissions: | |
| pages: write # to deploy to Pages | |
| id-token: write # to verify the deployment originates from an appropriate source | |
| steps: | |
| - name: Configure Pages | |
| uses: actions/configure-pages@v4 | |
| - name: Deploy to GitHub Pages | |
| id: deployment | |
| uses: actions/deploy-pages@v4 | |
| publish-umms: | |
| name: Publish UMM-S | |
| needs: [ build ] | |
| runs-on: ubuntu-latest | |
| if: ${{ contains(fromJSON('["UAT", "OPS"]'), needs.build.outputs.deploy_env) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set environment vars | |
| id: lowercase | |
| run: | | |
| echo TARGET_ENV_LOWERCASE=${{ needs.build.outputs.deploy_env }} | tr '[:upper:]' '[:lower:]' >> "$GITHUB_OUTPUT" | |
| - name: Publish UMM-S with new version | |
| id: publish-umm-s | |
| uses: podaac/[email protected] | |
| if: | | |
| github.ref == 'refs/heads/main' || | |
| startsWith(github.ref, 'refs/heads/release') | |
| with: | |
| umm-json: 'cmr/netcdf_cmr_umm_s.json' | |
| provider: 'POCLOUD' | |
| env: ${{ steps.lowercase.outputs.TARGET_ENV_LOWERCASE }} | |
| version: ${{ needs.build.outputs.version }} | |
| timeout: 60 | |
| disable_removal: 'true' | |
| umm_type: 'umm-s' | |
| use_associations: 'true' | |
| umm_version: 1.5.2 | |
| env: | |
| LAUNCHPAD_TOKEN_SIT: ${{secrets.LAUNCHPAD_TOKEN_SIT}} | |
| LAUNCHPAD_TOKEN_UAT: ${{secrets.LAUNCHPAD_TOKEN_UAT}} | |
| LAUNCHPAD_TOKEN_OPS: ${{secrets.LAUNCHPAD_TOKEN_OPS}} | |
| continue-on-error: true | |
| - name: Wait to retry publishing UMM-S | |
| if: steps.publish-umm-s.outcome == 'failure' | |
| run: | | |
| sleep 120 | |
| - name: Publish UMM-S with new version retry | |
| id: publish-umm-s-retry | |
| uses: podaac/[email protected] | |
| if: | | |
| steps.publish-umm-s.outcome == 'failure' | |
| with: | |
| umm-json: 'cmr/netcdf_cmr_umm_s.json' | |
| provider: 'POCLOUD' | |
| env: ${{ steps.lowercase.outputs.TARGET_ENV_LOWERCASE }} | |
| version: ${{ needs.build.outputs.version }} | |
| timeout: 60 | |
| disable_removal: 'true' | |
| umm_type: 'umm-s' | |
| use_associations: 'false' | |
| umm_version: 1.5.2 | |
| env: | |
| LAUNCHPAD_TOKEN_SIT: ${{secrets.LAUNCHPAD_TOKEN_SIT}} | |
| LAUNCHPAD_TOKEN_UAT: ${{secrets.LAUNCHPAD_TOKEN_UAT}} | |
| LAUNCHPAD_TOKEN_OPS: ${{secrets.LAUNCHPAD_TOKEN_OPS}} | |
| publish-pypi: | |
| needs: [ build ] | |
| runs-on: ubuntu-latest | |
| permissions: | |
| # IMPORTANT: this permission is mandatory for trusted publishing | |
| id-token: write | |
| if: | | |
| github.ref == 'refs/heads/develop' || | |
| startsWith(github.ref, 'refs/heads/release') || | |
| github.ref == 'refs/heads/main' | |
| steps: | |
| - name: Download python dist | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ needs.build.outputs.python_dist }} | |
| path: ${{ github.workspace }}/dist | |
| - name: Publish to test.pypi.org | |
| id: pypi-test-publish | |
| if: | | |
| github.ref == 'refs/heads/develop' || | |
| startsWith(github.ref, 'refs/heads/release') | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| repository-url: https://test.pypi.org/legacy/ | |
| - name: Publish to pypi.org | |
| if: | | |
| github.ref == 'refs/heads/main' | |
| id: pypi-publish | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| publish-docker: | |
| needs: [ build, publish-pypi ] | |
| runs-on: ubuntu-latest | |
| permissions: | |
| packages: write | |
| outputs: | |
| container_image_uri: ${{ steps.set-outputs.outputs.container_image_uri }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Extract metadata (tags, labels) for Docker | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} | |
| tags: | | |
| type=pep440,pattern={{version}},value=${{ needs.build.outputs.version }} | |
| type=raw,value=${{ needs.build.outputs.deploy_env }} | |
| - name: Wait for package | |
| run: | | |
| pip install tenacity | |
| ${GITHUB_WORKSPACE}/.github/workflows/wait-for-pypi.py ${{needs.build.outputs.pyproject_name}}[harmony]==${{ needs.build.outputs.version }} | |
| - name: Build and push Docker image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| file: docker/Dockerfile | |
| build-args: | | |
| SOURCE=${{needs.build.outputs.pyproject_name}}[harmony]==${{ needs.build.outputs.version }} | |
| push: true | |
| pull: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| - name: Set output | |
| id: set-outputs | |
| run: | | |
| echo "container_image_uri=${{ fromJSON(steps.meta.outputs.json).tags[0] }}" >> $GITHUB_OUTPUT | |
| deploy: | |
| needs: [ build, publish-docker ] | |
| runs-on: ubuntu-latest | |
| environment: ${{ needs.build.outputs.deploy_env }} | |
| env: | |
| ENV: ${{ needs.build.outputs.deploy_env }} | |
| THE_VERSION: ${{ needs.build.outputs.version }} | |
| CONTAINER_IMAGE_URI: ${{ needs.publish-docker.outputs.container_image_uri }} | |
| steps: | |
| - name: Deploy Harmony | |
| env: | |
| CMR_USER: ${{ secrets.CMR_USER }} | |
| CMR_PASS: ${{ secrets.CMR_PASS }} | |
| if: | | |
| github.ref == 'refs/heads/main' || | |
| startsWith(github.ref, 'refs/heads/release') | |
| run: | |
| echo "Deploy $CONTAINER_IMAGE_URI to $ENV" |