Add nf-co2footprint plugin for emissions tracking #14331
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run nf-test | |
| on: | |
| push: | |
| branches: | |
| # https://docs.renovatebot.com/key-concepts/automerge/#branch-vs-pr-automerging | |
| - "renovate/**" # branches Renovate creates | |
| paths-ignore: | |
| - "**/meta.yml" | |
| pull_request: | |
| branches: [master] | |
| paths-ignore: | |
| - "**/meta.yml" | |
| merge_group: | |
| types: [checks_requested] | |
| branches: [master] | |
| workflow_dispatch: | |
| # Cancel if a newer run is started | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # renovate: datasource=github-releases depName=askimed/nf-test versioning=semver | |
| NFT_VER: "0.9.2" | |
| NXF_ANSI_LOG: false | |
| NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity | |
| NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity | |
| # renovate: datasource=github-releases depName=nextflow/nextflow versioning=semver | |
| NXF_VER: "25.04.0" | |
| jobs: | |
| nf-test-changes: | |
| name: nf-test-changes | |
| runs-on: | |
| - runs-on=${{ github.run_id }}-nf-test-changes | |
| - runner=4cpu-linux-x64 | |
| - image=ubuntu22-full-x64 | |
| outputs: | |
| # Expose detected tags as 'modules' and 'workflows' output variables | |
| paths: ${{ steps.list.outputs.components }} | |
| modules: ${{ steps.components.outputs.modules }} | |
| subworkflows: ${{ steps.components.outputs.subworkflows}} | |
| shard: ${{ steps.set-shards.outputs.shard }} | |
| total_shards: ${{ steps.set-shards.outputs.total_shards }} | |
| steps: | |
| - name: Clean Workspace # Purge the workspace in case it's running on a self-hosted runner | |
| run: | | |
| ls -la ./ | |
| rm -rf ./* || true | |
| rm -rf ./.??* || true | |
| ls -la ./ | |
| - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 | |
| with: | |
| fetch-depth: 0 | |
| - name: List nf-test files | |
| id: list | |
| uses: adamrtalbot/detect-nf-test-changes@de3c3c8e113031b4f15a3c1104b5f135e8346997 # v0.0.6 | |
| with: | |
| head: ${{ github.sha }} | |
| base: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || 'origin/master' }} | |
| n_parents: 0 | |
| exclude_tags: "gpu" | |
| - name: Separate modules and subworkflows | |
| id: components | |
| run: | | |
| echo modules=$(echo '${{ steps.list.outputs.components }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/nf-core/"; ""))') >> $GITHUB_OUTPUT | |
| echo subworkflows=$(echo '${{ steps.list.outputs.components }}' | jq '. | map(select(contains("subworkflows"))) | map(gsub("subworkflows/nf-core/"; ""))') >> $GITHUB_OUTPUT | |
| - name: get number of shards | |
| id: set-shards | |
| if: ${{ steps.list.outputs.components != '[]' }} | |
| uses: ./.github/actions/get-shards | |
| env: | |
| NFT_VER: ${{ env.NFT_VER }} | |
| with: | |
| max_shards: 15 | |
| paths: "${{ join(fromJson(steps.list.outputs.components ), ' ') }}" | |
| - name: debug | |
| run: | | |
| echo ${{ steps.list.outputs.components }} | |
| echo ${{ steps.components.outputs.modules }} | |
| echo ${{ steps.components.outputs.subworkflows }} | |
| echo ${{ steps.set-shards.outputs.shard }} | |
| echo ${{ steps.set-shards.outputs.total_shards }} | |
| nf-test: | |
| runs-on: | |
| - runs-on=${{ github.run_id }} | |
| - runner=4cpu-linux-${{ matrix.arch }} | |
| - image=ubuntu24-full-${{ matrix.arch }} | |
| name: "${{ matrix.arch }} | ${{ matrix.profile }} | ${{ matrix.shard }}" | |
| needs: [nf-test-changes] | |
| if: ${{ needs.nf-test-changes.outputs.total_shards != '0' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| shard: ${{ fromJson(needs.nf-test-changes.outputs.shard) }} | |
| profile: [conda, docker, singularity] | |
| arch: | |
| - x64 | |
| # FIXME | |
| # - arm64 | |
| env: | |
| NXF_ANSI_LOG: false | |
| TOTAL_SHARDS: ${{ needs.nf-test-changes.outputs.total_shards }} | |
| outputs: | |
| filtered_paths: ${{ steps.filter.outputs.filtered_paths }} | |
| steps: | |
| - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 | |
| with: | |
| fetch-depth: 0 | |
| - name: Filter paths | |
| id: filter | |
| run: | | |
| PATHS='${{ needs.nf-test-changes.outputs.paths }}' | |
| FILTERED=$(jq -n -c \ | |
| --arg profile "${{ matrix.profile }}" \ | |
| --argjson input_paths "$PATHS" \ | |
| ' | |
| def get_skip_list: inputs | .[$profile] // []; | |
| # Get skip list and filter | |
| get_skip_list as $skip_patterns | | |
| # Debug output | |
| ($skip_patterns | "Skip patterns: " + (map(" - " + .) | join(","))) | | |
| debug | | |
| # Do the actual filtering | |
| $input_paths | map( | |
| . as $path | | |
| select($skip_patterns | all(. as $pattern | ($path | startswith($pattern)) | not)) | |
| ) as $result | | |
| # Debug what was skipped | |
| (if ($input_paths - $result) | length > 0 then | |
| "Skipped: " + (($input_paths - $result) | map(" - " + .) | join(",")) | |
| else | |
| "No paths skipped" | |
| end) | | |
| debug | | |
| $result | |
| ' .github/skip_nf_test.json) | |
| echo "filtered_paths=${FILTERED}" >> $GITHUB_OUTPUT | |
| - name: debug | |
| run: | | |
| echo filtered_paths = ${{ steps.filter.outputs.filtered_paths }} | |
| - name: Run nf-test Action | |
| if: ${{steps.filter.outputs.filtered_paths != '[]'}} | |
| continue-on-error: ${{ runner.arch == 'ARM64' }} | |
| uses: ./.github/actions/nf-test-action | |
| env: | |
| SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }} | |
| SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }} | |
| SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }} | |
| SENTIEON_AUTH_MECH: "GitHub Actions - token" | |
| with: | |
| profile: ${{ matrix.profile }}${{ runner.arch == 'ARM64' && ',arm64,wave' || '' }} | |
| shard: ${{ matrix.shard }} | |
| total_shards: ${{ env.TOTAL_SHARDS }} | |
| paths: "${{ join(fromJson(steps.filter.outputs.filtered_paths), ' ') }}" | |
| - name: Configure AWS credentials for CO2 reports | |
| if: always() && steps.filter.outputs.filtered_paths != '[]' | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.CO2_REPORTS_AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.CO2_REPORTS_AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ secrets.CO2_REPORTS_AWS_REGION }} | |
| - name: Upload CO2 footprint reports to S3 | |
| if: always() && steps.filter.outputs.filtered_paths != '[]' | |
| shell: bash | |
| run: | | |
| # Find all CO2 footprint trace files | |
| CO2_FILES=$(find . -name "co2footprint_trace*.txt" -type f) | |
| if [ -n "$CO2_FILES" ]; then | |
| # Get current date and sanitize branch name | |
| DATE=$(date +%Y-%m-%d) | |
| BRANCH_NAME=$(echo "${{ github.head_ref || github.ref_name }}" | sed 's/[^a-zA-Z0-9._-]/_/g') | |
| # S3 destination path | |
| S3_BASE="s3://nf-core-co2-reports/modules/${DATE}/${BRANCH_NAME}/${{ matrix.profile }}/${{ matrix.shard }}" | |
| # Upload each CO2 trace file | |
| for file in $CO2_FILES; do | |
| filename=$(basename "$file") | |
| aws s3 cp "$file" "${S3_BASE}/${filename}" --no-progress | |
| echo "Uploaded: $file -> ${S3_BASE}/${filename}" | |
| done | |
| else | |
| echo "ERROR: No CO2 footprint trace files found" | |
| exit 1 | |
| fi | |
| confirm-pass-nf-test: | |
| runs-on: | |
| - runs-on=${{ github.run_id }}-confirm-pass-nf-test | |
| - runner=4cpu-linux-x64 | |
| - image=ubuntu22-full-x64 | |
| needs: [nf-test] | |
| if: always() | |
| steps: | |
| - name: One or more tests failed | |
| if: ${{ contains(needs.*.result, 'failure') }} | |
| run: exit 1 | |
| - name: One or more tests cancelled | |
| if: ${{ contains(needs.*.result, 'cancelled') }} | |
| run: exit 1 | |
| - name: If no tests run, but filtered_paths was not empty and no tests succeeded, fail | |
| if: ${{ needs.nf-test.outputs.filtered_paths != '' && !contains(needs.*.result, 'success') }} | |
| run: exit 1 | |
| - name: All tests ok | |
| if: ${{ contains(needs.*.result, 'success') }} | |
| run: exit 0 | |
| - name: debug-print | |
| if: always() | |
| run: | | |
| echo "::group::DEBUG: `needs` Contents" | |
| echo "DEBUG: toJSON(needs) = ${{ toJSON(needs) }}" | |
| echo "DEBUG: toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}" | |
| echo "DEBUG: needs.nf-test.outputs.filtered_paths = ${{ needs.nf-test.outputs.filtered_paths }}" | |
| echo "::endgroup::" | |
| - name: Clean Workspace # Purge the workspace in case it's running on a self-hosted runner | |
| if: always() | |
| run: | | |
| ls -la ./ | |
| rm -rf ./* || true | |
| rm -rf ./.??* || true | |
| ls -la ./ |