integration tests #2217
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: integration tests | |
| on: | |
| pull_request: | |
| schedule: | |
| - cron: "0 11 * * *" #once a day at 11 UTC | |
| concurrency: | |
| group: integration-tests-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| jobs: | |
| setup-matrix: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| outputs: | |
| pluginDirs: ${{ steps.set-plugin-dirs.outputs.pluginDirs }} | |
| canaryVersion: ${{ steps.npm-canary-version.outputs.version }} | |
| canaryDockerTag: ${{ steps.docker-canary-tag.outputs.result }} | |
| latestVersion: ${{ steps.npm-latest-version.outputs.version }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| persist-credentials: false | |
| - name: Setup plugin dir variable | |
| id: set-plugin-dirs | |
| run: echo "pluginDirs=$(find ./examples -type d -name "src" -not -path "*/node_modules*" -maxdepth 3 -exec test -e "{}/plugin.json" \; -print | sed "s/\/src$//" | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT | |
| - name: Setup NPM canary version variable | |
| id: npm-canary-version | |
| run: echo "version=$(npm view @grafana/ui dist-tags.canary)" >> $GITHUB_OUTPUT | |
| - name: Setup docker canary tag variable | |
| id: docker-canary-tag | |
| uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 | |
| env: | |
| INPUT_NPM-TAG: ${{ steps.npm-canary-version.outputs.version }} | |
| with: | |
| result-encoding: string | |
| script: | | |
| const script = require('./.github/workflows/scripts/npm-to-docker-image.js'); | |
| return await script({ core }); | |
| - name: Setup NPM latest version variable | |
| id: npm-latest-version | |
| run: echo "version=$(npm view @grafana/ui dist-tags.latest)" >> $GITHUB_OUTPUT | |
| run-integration-tests: | |
| needs: setup-matrix | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| pluginDir: ${{ fromJson(needs.setup-matrix.outputs.pluginDirs) }} | |
| fail-fast: false | |
| env: | |
| GF_AUTH_ANONYMOUS_ENABLED: "true" | |
| GF_AUTH_ANONYMOUS_ORG_ROLE: "Admin" | |
| GF_AUTH_BASIC_ENABLED: "false" | |
| GF_DEFAULT_APP_MODE: "development" | |
| GF_INSTALL_PLUGINS: "marcusolsson-static-datasource" | |
| steps: | |
| - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| persist-credentials: false | |
| - name: Set the name of the plugin-example to be tested | |
| id: example-name | |
| run: | | |
| echo "PLUGIN_NAME=$(basename $PLUGIN_NAME)" >> $GITHUB_OUTPUT | |
| env: | |
| PLUGIN_NAME: ${{ matrix.pluginDir }} | |
| - name: Setup node version | |
| uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 | |
| with: | |
| node-version: "20" | |
| registry-url: "https://registry.npmjs.org" | |
| cache: "npm" | |
| - name: Install dependencies | |
| run: | | |
| npm ci | |
| working-directory: ${{ matrix.pluginDir }} | |
| - name: Run frontend tests | |
| run: | | |
| npm run test:ci | |
| working-directory: ${{ matrix.pluginDir }} | |
| - name: Build plugin frontend | |
| run: | | |
| npm run build | |
| working-directory: ${{ matrix.pluginDir }} | |
| - name: Check for backend Mage file | |
| id: backend-check | |
| run: | | |
| echo "MAGEFILE_EXISTS=$(test -f ./Magefile.go && echo true || echo false)" >> $GITHUB_OUTPUT | |
| working-directory: ${{ matrix.pluginDir }} | |
| - uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 | |
| with: | |
| go-version: "~1.22" | |
| check-latest: true | |
| cache-dependency-path: ${{ matrix.pluginDir }}/go.sum | |
| if: steps.backend-check.outputs.MAGEFILE_EXISTS == 'true' | |
| - name: Test plugin backend | |
| uses: magefile/mage-action@6f50bbb8ea47d56e62dee92392788acbc8192d0b # v3.1.0 | |
| with: | |
| version: latest | |
| args: -v test | |
| workdir: ${{ matrix.pluginDir }} | |
| if: steps.backend-check.outputs.MAGEFILE_EXISTS == 'true' | |
| - name: Build plugin backend | |
| uses: magefile/mage-action@6f50bbb8ea47d56e62dee92392788acbc8192d0b # v3.1.0 | |
| with: | |
| version: latest | |
| args: -v build:linux | |
| workdir: ${{ matrix.pluginDir }} | |
| if: steps.backend-check.outputs.MAGEFILE_EXISTS == 'true' | |
| ## CANARY_VERSION has to use NPM as we only publish the packages on changes | |
| - name: Set environment vars for testing | |
| run: | | |
| echo "PLUGIN_ID=$(cat src/plugin.json | jq -r '.id')" >> $GITHUB_ENV | |
| echo "EXPECTED_GRAFANA_VERSION=$(npx semver@latest $(cat src/plugin.json | jq -r '.dependencies.grafanaDependency') -c)" >> $GITHUB_ENV | |
| echo "CANARY_VERSION=$CANARY_VERSION" >> $GITHUB_ENV | |
| echo "CANARY_DOCKER_TAG=$CANARY_DOCKER_TAG" >> $GITHUB_ENV | |
| echo "LATEST_STABLE_VERSION=$LATEST_STABLE_VERSION" >> $GITHUB_ENV | |
| if [ -f "${PWD}/.env" ]; then | |
| echo "ENV_FILE_OPTION=--env-file ${PWD}/.env" >> $GITHUB_ENV | |
| fi | |
| working-directory: ${{ matrix.pluginDir }} | |
| env: | |
| CANARY_VERSION: ${{ needs.setup-matrix.outputs.canaryVersion }} | |
| CANARY_DOCKER_TAG: ${{ needs.setup-matrix.outputs.canaryDockerTag }} | |
| LATEST_STABLE_VERSION: ${{ needs.setup-matrix.outputs.latestVersion }} | |
| - name: Has Integration Tests | |
| id: has-integration-tests | |
| continue-on-error: true | |
| run: | | |
| echo "DIR=$($(test -f ./playwright.config.ts) && echo "true" || echo "false")" >> $GITHUB_OUTPUT | |
| working-directory: ${{ matrix.pluginDir }} | |
| ## If we're creating examples for new features that are only found in canary versions the grafanaDependency | |
| ## will be greater than any stable release of Grafana and so will not be available on docker. | |
| - name: Should run tests against expected and latest | |
| id: should-run-expected-latest-tests | |
| continue-on-error: true | |
| run: | | |
| npm exec --package semver-compare-cli --call 'semver-compare $LATEST_STABLE_VERSION ge $EXPECTED_GRAFANA_VERSION' | |
| working-directory: ${{ matrix.pluginDir }} | |
| - name: Check if mockserver is defined | |
| id: mockserver-check | |
| run: | | |
| echo $(docker compose config --services) | |
| if docker compose config --services | grep -wq mockserver; then | |
| echo "MOCKSERVER_DEFINED=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "MOCKSERVER_DEFINED=false" >> $GITHUB_OUTPUT | |
| fi | |
| working-directory: ${{ matrix.pluginDir }} | |
| - name: Run mock server | |
| if: steps.mockserver-check.outputs.MOCKSERVER_DEFINED == 'true' | |
| run: docker compose up -d mockserver | |
| working-directory: ${{ matrix.pluginDir }} | |
| - name: Setup playwright browser | |
| if: steps.has-integration-tests.outputs.DIR == 'true' | |
| run: npx playwright install --with-deps chromium | |
| working-directory: ${{ matrix.pluginDir }} | |
| - name: Expected version - Start Grafana | |
| if: steps.has-integration-tests.outputs.DIR == 'true' && steps.should-run-expected-latest-tests.outcome == 'success' | |
| run: | | |
| docker run -d -p 3000:3000 --add-host=host.docker.internal:host-gateway --name $PLUGIN_ID -v ${PWD}/dist:/var/lib/grafana/plugins/$PLUGIN_ID -v ${PWD}/provisioning:/etc/grafana/provisioning -e GF_DEFAULT_APP_MODE -e GF_INSTALL_PLUGINS -e GF_AUTH_ANONYMOUS_ORG_ROLE -e GF_AUTH_ANONYMOUS_ENABLED -e GF_AUTH_BASIC_ENABLED $ENV_FILE_OPTION grafana/grafana:$EXPECTED_GRAFANA_VERSION | |
| working-directory: ${{ matrix.pluginDir }} | |
| - name: Expected - Wait for Grafana to start | |
| if: steps.has-integration-tests.outputs.DIR == 'true' && steps.should-run-expected-latest-tests.outcome == 'success' | |
| uses: nev7n/wait_for_response@7fef3c1a6e8939d0b09062f14fec50d3c5d15fa1 # v1.0.1 | |
| with: | |
| url: "http://localhost:3000/" | |
| responseCode: 200 | |
| timeout: 60000 | |
| interval: 500 | |
| - name: Expected - Run Playwright tests | |
| if: steps.has-integration-tests.outputs.DIR == 'true' && steps.should-run-expected-latest-tests.outcome == 'success' | |
| id: expected-version-tests | |
| continue-on-error: true | |
| run: npm run e2e --prefix $PLUGIN_DIR | |
| env: | |
| PLUGIN_DIR: ${{ matrix.pluginDir }} | |
| - name: Expected - Upload e2e test summary | |
| uses: grafana/plugin-actions/playwright-gh-pages/upload-report-artifacts@main # zizmor: ignore[unpinned-uses] | |
| if: ${{ always() && steps.expected-version-tests.outcome == 'failure' }} | |
| with: | |
| report-dir: ${{ matrix.pluginDir }}/playwright-report | |
| grafana-version: ${{ env.EXPECTED_GRAFANA_VERSION }} | |
| grafana-image: grafana-enterprise | |
| plugin-name: ${{ env.PLUGIN_ID }} | |
| test-outcome: ${{ steps.expected-version-tests.outcome }} | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Expected - Stop and remove Docker container | |
| if: steps.has-integration-tests.outputs.DIR == 'true' && steps.should-run-expected-latest-tests.outcome == 'success' | |
| run: | | |
| docker stop $PLUGIN_ID && docker rm $PLUGIN_ID | |
| working-directory: ${{ matrix.pluginDir }} | |
| # Latest Version Tests | |
| # Runs the plugin tests against the latest stable version of Grafana. | |
| # Only run if should-run-expected-latest-tests so we can build examples of new features without failing tests. | |
| - name: Latest - Start Grafana | |
| if: steps.has-integration-tests.outputs.DIR == 'true' && steps.should-run-expected-latest-tests.outcome == 'success' | |
| run: | | |
| docker run -d -p 3000:3000 --add-host=host.docker.internal:host-gateway --name $PLUGIN_ID -v ${PWD}/dist:/var/lib/grafana/plugins/$PLUGIN_ID -v ${PWD}/provisioning:/etc/grafana/provisioning -e GF_DEFAULT_APP_MODE -e GF_INSTALL_PLUGINS -e GF_AUTH_ANONYMOUS_ORG_ROLE -e GF_AUTH_ANONYMOUS_ENABLED -e GF_AUTH_BASIC_ENABLED $ENV_FILE_OPTION grafana/grafana:latest | |
| working-directory: ${{ matrix.pluginDir }} | |
| - name: Latest - Wait for Grafana to start | |
| if: steps.has-integration-tests.outputs.DIR == 'true' && steps.should-run-expected-latest-tests.outcome == 'success' | |
| uses: nev7n/wait_for_response@7fef3c1a6e8939d0b09062f14fec50d3c5d15fa1 # v1.0.1 | |
| with: | |
| url: "http://localhost:3000/" | |
| responseCode: 200 | |
| timeout: 60000 | |
| interval: 500 | |
| - name: Latest - Run Playwright tests | |
| if: steps.has-integration-tests.outputs.DIR == 'true' && steps.should-run-expected-latest-tests.outcome == 'success' | |
| id: latest-version-tests | |
| continue-on-error: true | |
| run: npm run e2e --prefix $PLUGIN_DIR | |
| env: | |
| PLUGIN_DIR: ${{ matrix.pluginDir }} | |
| - name: Latest - Upload e2e test summary | |
| uses: grafana/plugin-actions/playwright-gh-pages/upload-report-artifacts@main # zizmor: ignore[unpinned-uses] | |
| if: ${{ always() && steps.latest-version-tests.outcome == 'failure' }} | |
| with: | |
| report-dir: ${{ matrix.pluginDir }}/playwright-report | |
| grafana-version: latest | |
| grafana-image: grafana-enterprise | |
| plugin-name: ${{ env.PLUGIN_ID }} | |
| test-outcome: ${{ steps.latest-version-tests.outcome }} | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Latest - Stop and remove Docker container | |
| if: steps.has-integration-tests.outputs.DIR == 'true' && steps.should-run-expected-latest-tests.outcome == 'success' | |
| run: | | |
| docker stop $PLUGIN_ID && docker rm $PLUGIN_ID | |
| working-directory: ${{ matrix.pluginDir }} | |
| ## Canary Version Tests | |
| ## Runs the plugin tests against the latest build of Grafana main branch. | |
| - name: Canary - Upgrade @grafana packages using canary_version | |
| run: | | |
| npm install --legacy-peer-deps $(echo $(cat package.json | jq -r --arg version $CANARY_VERSION '["@grafana/eslint-config", "@grafana/tsconfig", "@grafana/scenes", "@grafana/experimental", "@grafana/plugin-e2e", "@grafana/plugin-meta-extractor", "@grafana/eslint-plugin-plugins"] as $blacklist | [(.devDependencies,.dependencies) | keys] | flatten | unique | map(select ( test("^@grafana") ) ) as $deps | $deps - $blacklist | map("\(.)@\($version)") | join(" ")') ) | |
| npm install --force --save-optional @swc/core @swc/core-linux-arm-gnueabihf @swc/core-linux-arm64-gnu @swc/core-linux-arm64-musl @swc/core-linux-x64-gnu @swc/core-linux-x64-musl | |
| working-directory: ${{ matrix.pluginDir }} | |
| - name: Canary - Run frontend tests | |
| run: | | |
| npm run test:ci | |
| working-directory: ${{ matrix.pluginDir }} | |
| - name: Canary - Build plugin with grafana dependencies | |
| run: | | |
| npm run build | |
| working-directory: ${{ matrix.pluginDir }} | |
| - name: Canary - Build plugin backend | |
| uses: magefile/mage-action@6f50bbb8ea47d56e62dee92392788acbc8192d0b # v3.1.0 | |
| with: | |
| version: latest | |
| args: -v build:linux | |
| workdir: ${{ matrix.pluginDir }} | |
| if: steps.backend-check.outputs.MAGEFILE_EXISTS == 'true' | |
| # Canary versions live at grafana/grafana-dev | |
| - name: Canary - Start Grafana dev image | |
| if: steps.has-integration-tests.outputs.DIR == 'true' | |
| run: | | |
| docker run -d -p 3000:3000 --add-host=host.docker.internal:host-gateway --name $PLUGIN_ID -v ${PWD}/dist:/var/lib/grafana/plugins/$PLUGIN_ID -v ${PWD}/provisioning:/etc/grafana/provisioning -e GF_DEFAULT_APP_MODE -e GF_INSTALL_PLUGINS -e GF_AUTH_ANONYMOUS_ORG_ROLE -e GF_AUTH_ANONYMOUS_ENABLED -e GF_AUTH_BASIC_ENABLED $ENV_FILE_OPTION grafana/grafana-dev:$CANARY_DOCKER_TAG | |
| working-directory: ${{ matrix.pluginDir }} | |
| - name: Canary - Wait for Grafana to start | |
| if: steps.has-integration-tests.outputs.DIR == 'true' | |
| uses: nev7n/wait_for_response@7fef3c1a6e8939d0b09062f14fec50d3c5d15fa1 # v1.0.1 | |
| with: | |
| url: "http://localhost:3000/" | |
| responseCode: 200 | |
| timeout: 60000 | |
| interval: 500 | |
| - name: Canary - Run Playwright tests | |
| if: steps.has-integration-tests.outputs.DIR == 'true' | |
| id: canary-version-tests | |
| continue-on-error: true | |
| run: npm run e2e --prefix $PLUGIN_DIR | |
| env: | |
| PLUGIN_DIR: ${{ matrix.pluginDir }} | |
| - name: Canary - Upload e2e test summary | |
| uses: grafana/plugin-actions/playwright-gh-pages/upload-report-artifacts@main # zizmor: ignore[unpinned-uses] | |
| if: ${{ always() && steps.canary-version-tests.outcome == 'failure' }} | |
| with: | |
| report-dir: ${{ matrix.pluginDir }}/playwright-report | |
| grafana-version: canary | |
| grafana-image: grafana-enterprise | |
| plugin-name: ${{ env.PLUGIN_ID }} | |
| test-outcome: ${{ steps.canary-version-tests.outcome }} | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Canary - Stop and remove Docker container | |
| if: steps.has-integration-tests.outputs.DIR == 'true' | |
| run: | | |
| docker stop $PLUGIN_ID && docker rm $PLUGIN_ID | |
| working-directory: ${{ matrix.pluginDir }} | |
| - name: Stop mockserver if running | |
| if: steps.mockserver-check.outputs.MOCKSERVER_DEFINED == 'true' | |
| run: docker compose down mockserver | |
| working-directory: ${{ matrix.pluginDir }} | |
| - name: Failing build due to test failures | |
| if: (steps.has-integration-tests.outputs.DIR == 'true' && (steps.canary-version-tests.outcome != 'success' || steps.latest-version-tests.outcome != 'success' || steps.expected-version-tests.outcome != 'success')) | |
| run: exit 1 | |
| publish-report: | |
| if: ${{ always() }} | |
| needs: [run-integration-tests] | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| persist-credentials: false | |
| - name: Publish report | |
| uses: grafana/plugin-actions/playwright-gh-pages/deploy-report-pages@main # zizmor: ignore[unpinned-uses] | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| notify: | |
| if: ${{ (always() && github.event_name == 'schedule') }} | |
| runs-on: ubuntu-latest | |
| needs: [run-integration-tests] | |
| permissions: | |
| id-token: write | |
| steps: | |
| - id: get-secrets | |
| uses: grafana/shared-workflows/actions/get-vault-secrets@28361cdb22223e5f1e34358c86c20908e7248760 # get-vault-secrets-v1.1.0 | |
| with: | |
| # Secrets placed in the ci/repo/grafana/grafana-plugin-examples path in Vault | |
| repo_secrets: | | |
| SLACK_WEBHOOK_URL=slack_webhook_url:slack_webhook_url | |
| - name: Send GitHub Action trigger data to Slack workflow | |
| id: slack | |
| uses: slackapi/slack-github-action@b0fa283ad8fea605de13dc3f449259339835fc52 # v2.1.0 | |
| if: contains(fromJson('["failure"]'), needs.run-integration-tests.result) | |
| with: | |
| payload-templated: true | |
| webhook-type: incoming-webhook | |
| payload: | | |
| { | |
| "blocks": [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": ":blob-sad: <${{ github.server_url }}/${{ github.repository }}|Plugin Examples> Integration Test `${{ needs.run-integration-tests.result }}`" | |
| } | |
| }, | |
| { | |
| "type": "actions", | |
| "elements": [ | |
| { | |
| "type": "button", | |
| "text": { | |
| "type": "plain_text", | |
| "text": ":octocat: Open Job", | |
| "emoji": true | |
| }, | |
| "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}", | |
| "action_id": "actionId-0" | |
| } | |
| ] | |
| }, | |
| { | |
| "type": "divider" | |
| }, | |
| { | |
| "type": "context", | |
| "elements": [ | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Commit*: <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}>" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Event*: ${{ github.event_name }}" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*Branch*: `${{ github.head_ref || github.ref_name }}`" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "*PR Number*: `${{ github.event.pull_request.number || 'Not triggered by a PR' }}`" | |
| }, | |
| { | |
| "type": "mrkdwn", | |
| "text": "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}?pr=${{ github.event.number }}|See Playwright reports>" | |
| } | |
| ] | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ env.SLACK_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK |