Collect New Versioned Test Results #4
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow is a separate workflow from `report.yml` for regenerating the versioned report data needed for Bowtie's UI and `bowtie trend`. | |
| # It retests all of Bowtie's supported implementations for all their multiple versions listed in `matrix-versions.json` file if one exists for them. | |
| name: Collect New Versioned Test Results | |
| on: | |
| workflow_dispatch: | |
| jobs: | |
| list-implementations: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| implementations: ${{ steps.implementations-matrix.outputs.implementations }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install Bowtie | |
| uses: ./ | |
| - name: List all Bowtie Supported Implementations having a "matrix-versions.json" file | |
| id: implementations-matrix | |
| run: | | |
| IMPLEMENTATIONS=$(bowtie filter-implementations --format json | jq -r '.[]') | |
| MATRIX="[]" | |
| for impl in $IMPLEMENTATIONS; do | |
| if [ -f "implementations/$impl/matrix-versions.json" ]; then | |
| MATRIX=$(echo $MATRIX | jq --arg impl "$impl" '. + [$impl]') | |
| fi | |
| done | |
| echo "implementations=$(echo $MATRIX | jq -c .)" >> $GITHUB_OUTPUT | |
| regenerate-versioned-reports: | |
| needs: list-implementations | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| implementation: ${{ fromJson(needs.list-implementations.outputs.implementations) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install Bowtie | |
| uses: ./ | |
| # just so that we don't get rate limited (429) | |
| - name: Checkout JSON Schema Test Suite | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: json-schema-org/JSON-Schema-Test-Suite | |
| path: json-schema-test-suite | |
| - name: Make a ${{ matrix.implementation }} directory to keep all its version reports | |
| run: mkdir ${{ matrix.implementation }} | |
| - name: Generate New Versioned Reports for all Supported Dialects of ${{ matrix.implementation }} | |
| id: generate-new-versioned-report | |
| run: | | |
| impl=${{ matrix.implementation }} | |
| MATRIX_VERSIONS_FILE="implementations/$impl/matrix-versions.json" | |
| cp $MATRIX_VERSIONS_FILE $impl | |
| for version in $(jq -r '.[]' $MATRIX_VERSIONS_FILE); do | |
| SUPPORTED_DIALECTS=$(bowtie filter-dialects -i image:$impl:$version | xargs -I {} jq -r '.[] | select(.uri == "{}") | .shortName' data/dialects.json) | |
| if [ -n "$SUPPORTED_DIALECTS" ]; then | |
| mkdir "$impl/v$version" | |
| for dialect in $SUPPORTED_DIALECTS; do | |
| bowtie suite -i image:$impl:$version ./json-schema-test-suite/tests/$dialect > "$impl/v$version/$dialect.json" | |
| done | |
| fi | |
| done | |
| # This is useful to debug whether Bowtie accidentally fetched some huge | |
| # number of container images. | |
| - name: Show what images we fetched | |
| run: docker images | |
| if: always() | |
| # This unfortunately can go wrong if e.g. we ever run out of memory above. | |
| # Probably we should also atomically move files into place. | |
| - name: Check that all Generated Versioned Reports of ${{ matrix.implementation }} are Valid | |
| run: | | |
| impl=${{ matrix.implementation }} | |
| MATRIX_VERSIONS=$(jq -r '.[]' "implementations/$impl/matrix-versions.json") | |
| for version in $MATRIX_VERSIONS; do | |
| SUPPORTED_DIALECTS=$(bowtie filter-dialects -i image:$impl:$version | xargs -I {} jq -r '.[] | select(.uri == "{}") | .shortName' data/dialects.json) | |
| if [ -n "$SUPPORTED_DIALECTS" ]; then | |
| for dialect in $SUPPORTED_DIALECTS; do | |
| bowtie summary --show failures "$impl/v$version/$dialect.json" --format markdown >> $GITHUB_STEP_SUMMARY | |
| done | |
| fi | |
| done | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ matrix.implementation }} | |
| path: | | |
| ${{ matrix.implementation }} | |
| combine-all-versioned-reports: | |
| runs-on: ubuntu-latest | |
| needs: regenerate-versioned-reports | |
| steps: | |
| - name: Download all uploaded artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: implementations | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: implementations | |
| path: implementations |