From 41b9abe395ebbb1e1db18ad13f60f4022765d658 Mon Sep 17 00:00:00 2001 From: John SJ Anderson Date: Wed, 22 May 2024 09:42:31 -0700 Subject: [PATCH] Add local fork of pathogen-repo-ci to test refactor [#8] This modifies `pathogen-repo-ci` to allow the injection of a build target. --- .github/workflows/ci.yaml | 5 +- .github/workflows/pathogen-repo-ci.yaml | 249 ++++++++++++++++++++++++ 2 files changed, 252 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/pathogen-repo-ci.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 10b2f5e..0b12794 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -11,6 +11,7 @@ jobs: # runtime: [docker, conda] # permissions: # id-token: write - uses: nextstrain/.github/.github/workflows/pathogen-repo-ci.yaml@master + uses: ./.github/workflows/pathogen-repo-ci.yaml with: - build-args: ingest --config '[viruses="hku1"]' + build-target: ingest + build-args: --config viruses='["hku1"]' diff --git a/.github/workflows/pathogen-repo-ci.yaml b/.github/workflows/pathogen-repo-ci.yaml new file mode 100644 index 0000000..2abd90b --- /dev/null +++ b/.github/workflows/pathogen-repo-ci.yaml @@ -0,0 +1,249 @@ +# This workflow is intended to be called by workflows in our various pathogen +# build repos. See workflow-templates/pathogen-repo-ci.yaml (a "starter" +# workflow) in this repo for an example of what the caller workflow looks like. +name: CI-upstream + +on: + workflow_call: + inputs: + build-target: + description: >- + The build directory to provide as the first argument to `nextstrain build`. Defaults to ".". + type: string + default: "." + required: false + + build-args: + description: >- + Additional command-line arguments to pass to `nextstrain build` after the build directory (e.g. to Snakemake). + type: string + default: "" + required: false + + repo: + description: >- + Repository name with owner (e.g. nextstrain/zika). Defaults to the repository of the caller workflow. + type: string + default: ${{ github.repository }} + required: false + + env: + description: >- + Additional environment variables to set before the build, as a string containing YAML. This is easily produced, + for example, by pretending you're writing normal nested YAML within a literal multi-line block scalar (introduced + by "|"): #magic___^_^___line + with: + env: | + FOO: bar + I_CANT_BELIEVE: "it's not YAML" + would_you_believe: | + it's + not + yaml + + Do not use for secrets! Instead, pass them via GitHub Action's dedicated secrets mechanism. + type: string + default: "" + required: false + + runtimes: + description: >- + List of Nextstrain runtimes under which to run the build, as a string containing YAML. This is easily produced, + for example, by pretending you're writing normal nested YAML within a literal multi-line block scalar (introduced + by "|"): + + with: + runtimes: | + - docker + - conda + + Defaults to "docker" and "conda". One job per runtime will be run. + type: string + default: | + - docker + - conda + required: false + + artifact-name: + description: >- + Name to use for build results artifact uploaded at the end of the workflow. This name will be suffixed with other + information from the workflow job matrix to distinguish each artifact in a workflow run. + + If you're invoking this workflow multiple times from the same calling workflow, you should set this. Otherwise, + the default of "outputs" is probably fine. + type: string + default: outputs + required: false + + continue-on-error: + description: >- + Pass thru for . + type: boolean + default: false + required: false + +permissions: + contents: read + packages: read + +jobs: + configuration: + runs-on: ubuntu-latest + steps: + - id: inputs + env: + runtimes: ${{ inputs.runtimes }} + shell: bash + run: | + runtimes="$(yq --output-format=json --indent=0 . <<<"$runtimes")" + echo runtimes="$runtimes" | tee -a "$GITHUB_OUTPUT" + outputs: + runtimes: ${{ steps.inputs.outputs.runtimes }} + + build: + needs: configuration + strategy: + fail-fast: false + matrix: + runtime: ${{ fromJSON(needs.configuration.outputs.runtimes) }} + name: build (${{ matrix.runtime }}) + runs-on: ubuntu-latest + continue-on-error: ${{ inputs.continue-on-error }} + steps: + # Log in, if possible, to docker.io (Docker Hub), since authenticated + # requests get higher rate limits (e.g. for image pulls). Our org-level + # secret DOCKER_TOKEN_PUBLIC_READ_ONLY is available to all our public + # repos on GitHub but only available here to this reusable workflow when + # called with "secrets: inherit". On Docker Hub, the token is granted + # "public read-only" access. + # + # The secrets context is not allowed in "if:" conditions, so we must + # launder it thru env. + - if: env.token-available == 'true' + env: + token-available: ${{ secrets.DOCKER_TOKEN_PUBLIC_READ_ONLY != '' }} + name: Log in to docker.io + uses: docker/login-action@v3 + with: + registry: docker.io + username: nextstrainbot + password: ${{ secrets.DOCKER_TOKEN_PUBLIC_READ_ONLY }} + continue-on-error: true + + # Log in, if possible, to ghcr.io which we use for staging images in + # nextstrain/docker-base. The automatic GITHUB_TOKEN is restricted to + # read-only access by the "permissions:" block above. + - name: Log in to ghcr.io + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + continue-on-error: true + + # Transforms the inputs.env *string* containing YAML like this: + # + # FOO: bar + # I_CANT_BELIEVE: "it's not YAML" + # would_you_believe: | + # it's + # not + # yaml + # + # first into the equivalent JSON (with yq) and then into text (with jq) + # like this: + # + # FOO=<<__EOF__ + # bar + # __EOF__ + # I_CANT_BELIEVE<<__EOF__ + # it's not YAML + # __EOF__ + # would_you_believe<<__EOF__ + # it's + # not + # yaml + # __EOF__ + # + # which is suitable for appending to the $GITHUB_ENV file in order to set + # the environment variables for subsequent steps. + # + # See the GitHub docs for more info on this heredoc-like syntax¹, which I + # use here to avoid quoting issues in arbitrary env var values. + # + # By doing this slightly-convoluted conversion here, callers can use the + # familiar env: block syntax almost without change and avoid paying much + # in accidental complexity. We box it up here and let callers focus on + # their essential complexity. + # -trs, 23 May 2022 + # + # ¹ https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings + # + - if: inputs.env + name: Set environment variables + env: + env: ${{ inputs.env }} + run: > + # shellcheck disable=SC2154 + + echo "$env" | yq --output-format json . | jq --raw-output ' + to_entries + | map("\(.key)<<__EOF__\n\(.value)\n__EOF__") + | join("\n") + ' | tee -a "$GITHUB_ENV" + + - uses: actions/checkout@v4 + with: + repository: ${{ inputs.repo }} + + # XXX TODO: It would be better for this to call setup-nextstrain-cli + # using the same ref that this workflow was called with (e.g. if this + # workflow was invoked by the caller workflow with @foo than we invoke + # the action with @foo too), but it's not currently possible to figure + # out that ref. See discussion around this (including results of some + # investigation I did): + # + # - https://github.community/t/reusable-workflows-get-the-ref-inside-the-called-workflow/224109 + # - https://github.community/t/ref-head-in-reusable-workflows/203690/92 + # + # Once we can figure out that ref, then we can actions/checkout our + # nextstrain/.github repo at that ref as a sidecar path somewhere and + # then invoke the setup-nextstrain-cli action using a local file path + # instead of a remote owner/repo path. This separate checkout will be + # necessary since the "uses:" key can't be interpolated (${{…}}) with + # context vars. + # + # For now, update the hardcoded ref (e.g. @90af34…) below when you make + # future changes to setup-nextstrain-cli. + # + # [ Update 16 Feb 2024: We solved this for pathogen-repo-build.yaml, but + # because it required a new permission on the GitHub tokens (id-token: + # write) we decided not to update this workflow (yet?) to use the same + # approach. -trs ] + # + # -trs, 28 April 2022 + - uses: nextstrain/.github/actions/setup-nextstrain-cli@c1191de9d5e1a30e91d70b0fd1041d97ed1b2496 + with: + runtime: ${{ matrix.runtime }} + + - name: Copy example data + run: | + if [[ -d example_data ]]; then + mkdir -p data/ + cp -r -v example_data/* data/ + else + echo No example data to copy. + fi + + - run: nextstrain build ${{ inputs.build-target }} ${{ inputs.build-args }} + + - if: always() + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.artifact-name }}-${{ matrix.runtime }} + path: | + auspice/ + results/ + benchmarks/ + logs/ + .snakemake/log/