-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add local fork of pathogen-repo-ci to test refactor [#8]
This modifies `pathogen-repo-ci` to allow the injection of a build target.
- Loading branch information
Showing
2 changed files
with
272 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,249 @@ | ||
| # This workflow is intended to be called by workflows in our various pathogen | ||
| # build repos. See workflow-templates/pathogen-repo-ci.yaml (a "starter" | ||
| # workflow) in this repo for an example of what the caller workflow looks like. | ||
| name: CI-upstream | ||
|
|
||
| on: | ||
| workflow_call: | ||
| inputs: | ||
| build-target: | ||
| description: >- | ||
| The build directory to provide as the first argument to `nextstrain build`. Defaults to ".". | ||
| type: string | ||
| default: "." | ||
| required: false | ||
|
|
||
| build-args: | ||
| description: >- | ||
| Additional command-line arguments to pass to `nextstrain build` after the build directory (e.g. to Snakemake). | ||
| type: string | ||
| default: "" | ||
| required: false | ||
|
|
||
| repo: | ||
| description: >- | ||
| Repository name with owner (e.g. nextstrain/zika). Defaults to the repository of the caller workflow. | ||
| type: string | ||
| default: ${{ github.repository }} | ||
| required: false | ||
|
|
||
| env: | ||
| description: >- | ||
| Additional environment variables to set before the build, as a string containing YAML. This is easily produced, | ||
| for example, by pretending you're writing normal nested YAML within a literal multi-line block scalar (introduced | ||
| by "|"): #magic___^_^___line | ||
| with: | ||
| env: | | ||
| FOO: bar | ||
| I_CANT_BELIEVE: "it's not YAML" | ||
| would_you_believe: | | ||
| it's | ||
| not | ||
| yaml | ||
| Do not use for secrets! Instead, pass them via GitHub Action's dedicated secrets mechanism. | ||
| type: string | ||
| default: "" | ||
| required: false | ||
|
|
||
| runtimes: | ||
| description: >- | ||
| List of Nextstrain runtimes under which to run the build, as a string containing YAML. This is easily produced, | ||
| for example, by pretending you're writing normal nested YAML within a literal multi-line block scalar (introduced | ||
| by "|"): | ||
| with: | ||
| runtimes: | | ||
| - docker | ||
| - conda | ||
| Defaults to "docker" and "conda". One job per runtime will be run. | ||
| type: string | ||
| default: | | ||
| - docker | ||
| - conda | ||
| required: false | ||
|
|
||
| artifact-name: | ||
| description: >- | ||
| Name to use for build results artifact uploaded at the end of the workflow. This name will be suffixed with other | ||
| information from the workflow job matrix to distinguish each artifact in a workflow run. | ||
| If you're invoking this workflow multiple times from the same calling workflow, you should set this. Otherwise, | ||
| the default of "outputs" is probably fine. | ||
| type: string | ||
| default: outputs | ||
| required: false | ||
|
|
||
| continue-on-error: | ||
| description: >- | ||
| Pass thru for <https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idcontinue-on-error>. | ||
| type: boolean | ||
| default: false | ||
| required: false | ||
|
|
||
| permissions: | ||
| contents: read | ||
| packages: read | ||
|
|
||
| jobs: | ||
| configuration: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - id: inputs | ||
| env: | ||
| runtimes: ${{ inputs.runtimes }} | ||
| shell: bash | ||
| run: | | ||
| runtimes="$(yq --output-format=json --indent=0 . <<<"$runtimes")" | ||
| echo runtimes="$runtimes" | tee -a "$GITHUB_OUTPUT" | ||
| outputs: | ||
| runtimes: ${{ steps.inputs.outputs.runtimes }} | ||
|
|
||
| build: | ||
| needs: configuration | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| runtime: ${{ fromJSON(needs.configuration.outputs.runtimes) }} | ||
| name: build (${{ matrix.runtime }}) | ||
| runs-on: ubuntu-latest | ||
| continue-on-error: ${{ inputs.continue-on-error }} | ||
| steps: | ||
| # Log in, if possible, to docker.io (Docker Hub), since authenticated | ||
| # requests get higher rate limits (e.g. for image pulls). Our org-level | ||
| # secret DOCKER_TOKEN_PUBLIC_READ_ONLY is available to all our public | ||
| # repos on GitHub but only available here to this reusable workflow when | ||
| # called with "secrets: inherit". On Docker Hub, the token is granted | ||
| # "public read-only" access. | ||
| # | ||
| # The secrets context is not allowed in "if:" conditions, so we must | ||
| # launder it thru env. | ||
| - if: env.token-available == 'true' | ||
| env: | ||
| token-available: ${{ secrets.DOCKER_TOKEN_PUBLIC_READ_ONLY != '' }} | ||
| name: Log in to docker.io | ||
| uses: docker/login-action@v3 | ||
| with: | ||
| registry: docker.io | ||
| username: nextstrainbot | ||
| password: ${{ secrets.DOCKER_TOKEN_PUBLIC_READ_ONLY }} | ||
| continue-on-error: true | ||
|
|
||
| # Log in, if possible, to ghcr.io which we use for staging images in | ||
| # nextstrain/docker-base. The automatic GITHUB_TOKEN is restricted to | ||
| # read-only access by the "permissions:" block above. | ||
| - name: Log in to ghcr.io | ||
| uses: docker/login-action@v3 | ||
| with: | ||
| registry: ghcr.io | ||
| username: ${{ github.actor }} | ||
| password: ${{ secrets.GITHUB_TOKEN }} | ||
| continue-on-error: true | ||
|
|
||
| # Transforms the inputs.env *string* containing YAML like this: | ||
| # | ||
| # FOO: bar | ||
| # I_CANT_BELIEVE: "it's not YAML" | ||
| # would_you_believe: | | ||
| # it's | ||
| # not | ||
| # yaml | ||
| # | ||
| # first into the equivalent JSON (with yq) and then into text (with jq) | ||
| # like this: | ||
| # | ||
| # FOO=<<__EOF__ | ||
| # bar | ||
| # __EOF__ | ||
| # I_CANT_BELIEVE<<__EOF__ | ||
| # it's not YAML | ||
| # __EOF__ | ||
| # would_you_believe<<__EOF__ | ||
| # it's | ||
| # not | ||
| # yaml | ||
| # __EOF__ | ||
| # | ||
| # which is suitable for appending to the $GITHUB_ENV file in order to set | ||
| # the environment variables for subsequent steps. | ||
| # | ||
| # See the GitHub docs for more info on this heredoc-like syntax¹, which I | ||
| # use here to avoid quoting issues in arbitrary env var values. | ||
| # | ||
| # By doing this slightly-convoluted conversion here, callers can use the | ||
| # familiar env: block syntax almost without change and avoid paying much | ||
| # in accidental complexity. We box it up here and let callers focus on | ||
| # their essential complexity. | ||
| # -trs, 23 May 2022 | ||
| # | ||
| # ¹ https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings | ||
| # | ||
| - if: inputs.env | ||
| name: Set environment variables | ||
| env: | ||
| env: ${{ inputs.env }} | ||
| run: > | ||
| # shellcheck disable=SC2154 | ||
| echo "$env" | yq --output-format json . | jq --raw-output ' | ||
| to_entries | ||
| | map("\(.key)<<__EOF__\n\(.value)\n__EOF__") | ||
| | join("\n") | ||
| ' | tee -a "$GITHUB_ENV" | ||
| - uses: actions/checkout@v4 | ||
| with: | ||
| repository: ${{ inputs.repo }} | ||
|
|
||
| # XXX TODO: It would be better for this to call setup-nextstrain-cli | ||
| # using the same ref that this workflow was called with (e.g. if this | ||
| # workflow was invoked by the caller workflow with @foo than we invoke | ||
| # the action with @foo too), but it's not currently possible to figure | ||
| # out that ref. See discussion around this (including results of some | ||
| # investigation I did): | ||
| # | ||
| # - https://github.community/t/reusable-workflows-get-the-ref-inside-the-called-workflow/224109 | ||
| # - https://github.community/t/ref-head-in-reusable-workflows/203690/92 | ||
| # | ||
| # Once we can figure out that ref, then we can actions/checkout our | ||
| # nextstrain/.github repo at that ref as a sidecar path somewhere and | ||
| # then invoke the setup-nextstrain-cli action using a local file path | ||
| # instead of a remote owner/repo path. This separate checkout will be | ||
| # necessary since the "uses:" key can't be interpolated (${{…}}) with | ||
| # context vars. | ||
| # | ||
| # For now, update the hardcoded ref (e.g. @90af34…) below when you make | ||
| # future changes to setup-nextstrain-cli. | ||
| # | ||
| # [ Update 16 Feb 2024: We solved this for pathogen-repo-build.yaml, but | ||
| # because it required a new permission on the GitHub tokens (id-token: | ||
| # write) we decided not to update this workflow (yet?) to use the same | ||
| # approach. -trs ] | ||
| # | ||
| # -trs, 28 April 2022 | ||
| - uses: nextstrain/.github/actions/setup-nextstrain-cli@c1191de9d5e1a30e91d70b0fd1041d97ed1b2496 | ||
| with: | ||
| runtime: ${{ matrix.runtime }} | ||
|
|
||
| - name: Copy example data | ||
| run: | | ||
| if [[ -d example_data ]]; then | ||
| mkdir -p data/ | ||
| cp -r -v example_data/* data/ | ||
| else | ||
| echo No example data to copy. | ||
| fi | ||
| - run: nextstrain build ${{ inputs.build-target }} ${{ inputs.build-args }} | ||
|
|
||
| - if: always() | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: ${{ inputs.artifact-name }}-${{ matrix.runtime }} | ||
| path: | | ||
| ${{ inputs.build-target }}/auspice/ | ||
| ${{ inputs.build-target }}/results/ | ||
| ${{ inputs.build-target }}/benchmarks/ | ||
| ${{ inputs.build-target }}/logs/ | ||
| ${{ inputs.build-target }}/.snakemake/log/ |