Collect New Versioned Test Results #5
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow is a separate workflow from `report.yml` for regenerating the versioned report data needed for Bowtie's UI and `bowtie trend`. | |
| # It retests all of Bowtie's supported implementations for all their multiple versions listed in `matrix-versions.json` file if one exists for them. | |
| name: Collect New Versioned Test Results | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| implementation: | |
| description: The name of an implementation. | |
| required: false | |
| type: string | |
| jobs: | |
| list-implementations: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| implementations: ${{ steps.implementations-matrix.outputs.implementations }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install Bowtie | |
| uses: ./ | |
| - name: List all Bowtie Supported Implementations having a "matrix-versions.json" file | |
| id: implementations-matrix | |
| run: | | |
| implementation=${{ inputs.implementation }} | |
| IMPLEMENTATIONS=$(bowtie filter-implementations --format json) | |
| MATRIX="[]" | |
| if [ -n "$implementation" ]; then | |
| if echo "$IMPLEMENTATIONS" | jq -e --arg impl "$implementation" 'index($impl) != null' > /dev/null; then | |
| if [ -f "implementations/$implementation/matrix-versions.json" ]; then | |
| MATRIX=$(echo $MATRIX | jq --arg impl "$implementation" '. + [$impl]') | |
| else | |
| echo "No \`matrix-versions.json\` file found for implementation ('$implementation')." | |
| exit 1 | |
| fi | |
| else | |
| echo "No such implementation ('$implementation') found. Please provide a correct implementation name." | |
| echo "To see a list of all Bowtie supported implementations, run \`bowtie filter-implementations\`." | |
| exit 1 | |
| fi | |
| else | |
| for impl in $(echo "$IMPLEMENTATIONS" | jq -r '.[]'); do | |
| if [ -f "implementations/$impl/matrix-versions.json" ]; then | |
| MATRIX=$(echo $MATRIX | jq --arg impl "$impl" '. + [$impl]') | |
| fi | |
| done | |
| fi | |
| echo "implementations=$(echo $MATRIX | jq -c .)" >> $GITHUB_OUTPUT | |
| regenerate-versioned-reports: | |
| needs: list-implementations | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| implementation: ${{ fromJson(needs.list-implementations.outputs.implementations) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install Bowtie | |
| uses: ./ | |
| # just so that we don't get rate limited (429) | |
| - name: Checkout JSON Schema Test Suite | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: json-schema-org/JSON-Schema-Test-Suite | |
| path: json-schema-test-suite | |
| - name: Make a ${{ matrix.implementation }} directory to keep all its version reports | |
| run: mkdir ${{ matrix.implementation }} | |
| - name: Generate New Versioned Reports for all Supported Dialects of ${{ matrix.implementation }} | |
| id: generate-new-versioned-report | |
| run: | | |
| impl=${{ matrix.implementation }} | |
| MATRIX_VERSIONS_FILE="implementations/$impl/matrix-versions.json" | |
| cp $MATRIX_VERSIONS_FILE $impl | |
| for version in $(jq -r '.[]' $MATRIX_VERSIONS_FILE); do | |
| SUPPORTED_DIALECTS=$(bowtie filter-dialects -i image:$impl:$version | xargs -I {} jq -r '.[] | select(.uri == "{}") | .shortName' data/dialects.json) | |
| if [ -n "$SUPPORTED_DIALECTS" ]; then | |
| mkdir "$impl/v$version" | |
| for dialect in $SUPPORTED_DIALECTS; do | |
| bowtie suite -i image:$impl:$version ./json-schema-test-suite/tests/$dialect > "$impl/v$version/$dialect.json" | |
| done | |
| fi | |
| done | |
| # This is useful to debug whether Bowtie accidentally fetched some huge | |
| # number of container images. | |
| - name: Show what images we fetched | |
| run: docker images | |
| if: always() | |
| # FIXME: We ignore for now as now this exits unsuccessfully if there | |
| # are *test* failures | |
| - name: Check that all Generated Versioned Reports of ${{ matrix.implementation }} are Valid | |
| run: | | |
| impl=${{ matrix.implementation }} | |
| MATRIX_VERSIONS=$(jq -r '.[]' "implementations/$impl/matrix-versions.json") | |
| for version in $MATRIX_VERSIONS; do | |
| SUPPORTED_DIALECTS=$(bowtie filter-dialects -i image:$impl:$version | xargs -I {} jq -r '.[] | select(.uri == "{}") | .shortName' data/dialects.json) | |
| if [ -n "$SUPPORTED_DIALECTS" ]; then | |
| for dialect in $SUPPORTED_DIALECTS; do | |
| bowtie summary --show failures "$impl/v$version/$dialect.json" --format markdown >> $GITHUB_STEP_SUMMARY | |
| done | |
| fi | |
| done | |
| continue-on-error: true | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ matrix.implementation }} | |
| path: | | |
| ${{ matrix.implementation }} | |
| combine-all-versioned-reports: | |
| runs-on: ubuntu-latest | |
| needs: regenerate-versioned-reports | |
| steps: | |
| - name: Download all uploaded artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: implementations | |
| - name: Download latest versioned test reports | |
| uses: dawidd6/action-download-artifact@v6 | |
| with: | |
| workflow: versioned-report.yml | |
| branch: main | |
| name: implementations | |
| path: latest-implementations | |
| if: ${{ inputs.implementation != null }} | |
| - name: Prepare artifact for upload | |
| run: | | |
| rm -rf "latest-implementations/${{ inputs.implementation }}" | |
| cp -r "implementations/${{ inputs.implementation }}" "latest-implementations/" | |
| if: ${{ inputs.implementation != null }} | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: implementations | |
| path: ${{ inputs.implementation != null && 'latest-implementations' || 'implementations' }} |