Run E2E against Grafana Cloud (#1400)

Author: adamyeats

.github/workflows/cron.yml

@@ -0,0 +1,86 @@
+name: cron Jobs
+
+on:
+  schedule:
+    - cron: '0 9 * * *' # Daily at 09:00 UTC
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  contents: read
+
+jobs:
+  bench-tests:
+    name: Run E2E Tests with Grafana Bench
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+
+    # ensure job never runs on forks
+    if: ${{ github.event_name != 'schedule' || github.repository_owner == 'grafana' }}
+
+    # this job needs OIDC to fetch Vault secrets
+    permissions:
+      contents: read
+      id-token: write
+
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          # Avoid leaving a token in the repo checkout; prefer explicit auth for publishing steps
+          persist-credentials: false
+
+      - name: Get secrets from Vault
+        id: get-secrets
+        uses: grafana/shared-workflows/actions/get-vault-secrets@get-vault-secrets/v1.3.0
+        with:
+          # Grafana auth (used by @grafana/plugin-e2e)
+          # + Prometheus creds for Bench metrics reporting
+          common_secrets: |
+            PLAYWRIGHT_GRAFANA_PASSWORD=data-sources/e2e:grafana-pw
+            PLAYWRIGHT_GRAFANA_USERNAME=data-sources/e2e:grafana-username
+            PROMETHEUS_PASSWORD=grafana-bench:prometheus_token
+            PROMETHEUS_URL=grafana-bench:prometheus_url
+            PROMETHEUS_USER=grafana-bench:prometheus_user
+
+          # Repo-specific backend secrets (naming varies by datasource)
+          repo_secrets: |
+            DS_INSTANCE_HOST=ds-instance:host
+            DS_INSTANCE_PASSWORD=ds-instance:password
+            DS_INSTANCE_PORT=ds-instance:port
+            DS_INSTANCE_USERNAME=ds-instance:username
+
+          # Keep secrets in step output; we’ll pass them to Bench explicitly
+          export_env: false
+
+      - name: Wait for Grafana to be reachable
+        uses: grafana/plugin-actions/wait-for-grafana@wait-for-grafana/v1.0.2
+        with:
+          # Use /login so “reachable” also implies the app is up
+          url: "https://datasourcese2e.grafana-dev.net/login"
+          timeout: 300 # 5 minutes
+          interval: 10 # 10 seconds
+
+      - name: Run Grafana Bench tests
+        run: |
+          set -euo pipefail
+          docker run \
+            -e PROMETHEUS_URL="${{ fromJSON(steps.get-secrets.outputs.secrets).PROMETHEUS_URL }}" \
+            -e PROMETHEUS_USER="${{ fromJSON(steps.get-secrets.outputs.secrets).PROMETHEUS_USER }}" \
+            -e PROMETHEUS_PASSWORD="${{ fromJSON(steps.get-secrets.outputs.secrets).PROMETHEUS_PASSWORD }}" \
+            --network=host \
+            --rm \
+            --volume "$PWD:/tests" \
+            us-docker.pkg.dev/grafanalabs-global/docker-grafana-bench-prod/grafana-bench-playwright:v0.6.11 test \
+            --grafana-admin-password "${{ fromJSON(steps.get-secrets.outputs.secrets).PLAYWRIGHT_GRAFANA_PASSWORD }}" \
+            --grafana-admin-user "${{ fromJSON(steps.get-secrets.outputs.secrets).PLAYWRIGHT_GRAFANA_USERNAME }}" \
+            --grafana-url "https://datasourcese2e.grafana-dev.net" \
+            --grafana-version "rrc-instant" \
+            --prometheus-metrics \
+            --prometheus-strict-lint \
+            --pw-execute "npm run e2e" \
+            --pw-prepare "npm ci --no-audit --fund=false; npx playwright install" \
+            --test-env 'CI=true,DS_INSTANCE_HOST=${{ fromJSON(steps.get-secrets.outputs.secrets).DS_INSTANCE_HOST }},DS_INSTANCE_PASSWORD=${{ fromJSON(steps.get-secrets.outputs.secrets).DS_INSTANCE_PASSWORD }},DS_INSTANCE_PORT=${{ fromJSON(steps.get-secrets.outputs.secrets).DS_INSTANCE_PORT }},DS_INSTANCE_USERNAME=${{ fromJSON(steps.get-secrets.outputs.secrets).DS_INSTANCE_USERNAME }},DS_PDC_NETWORK_NAME=datasources-pdc-network-aws-datasourcese2e' \
+            --test-runner playwright \
+            --test-verbose

.github/workflows/publish.yml

@@ -40,7 +40,7 @@ jobs:
       environment: ${{ matrix.environment }}
       docs-only: ${{ fromJSON(github.event.inputs.docs-only) }}
       scopes: ${{ github.event.inputs.environment == 'cloud (recommended)' && 'grafana_cloud' || github.event.inputs.environment == 'on-prem (for emergencies fix to On Prem customers)' && 'universal' }}
-      go-version: "1.24"
+      go-version: "1.25"
       golangci-lint-version: "2.1.6"
       run-playwright: false
       github-draft-release: false

.github/workflows/push.yml

@@ -12,10 +12,29 @@ jobs:
   ci:
     name: CI
     uses: grafana/plugin-ci-workflows/.github/workflows/ci.yml@ci-cd-workflows/v5.1.0
+    if: github.ref != 'refs/heads/main'
     permissions:
       contents: read
       id-token: write
     with:
+      go-version: '1.25'
+      golangci-lint-version: '2.1.6'
       plugin-version-suffix: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || '' }}
-      go-version: "1.24"
-      golangci-lint-version: "2.1.6"
+
+  publish-latest-to-catalog:
+    name: Publish `main` to Dev Catalog
+    if: github.ref == 'refs/heads/main'
+    uses: grafana/plugin-ci-workflows/.github/workflows/cd.yml@ci-cd-workflows/v5.1.0
+    permissions:
+      attestations: write
+      contents: write
+      id-token: write
+    with:
+      disable-docs-publishing: true
+      disable-github-release: true
+      environment: 'dev'
+      go-version: '1.25'
+      golangci-lint-version: '2.1.6'
+      plugin-version-suffix: ${{ github.sha }}
+      run-playwright: false
+      scopes: grafana_cloud_instance_datasourcese2e

cspell.config.json

@@ -18,6 +18,7 @@
     "CHSQL",
     "ClickHouse",
     "closedate",
+    "combobox",
     "commitish",
     "concats",
     "createddate",
@@ -78,9 +79,9 @@
     "timerange",
     "timeseries",
     "timespan",
-    "toggleTip",
     "TLSCA",
     "TLSSSL",
+    "toggleTip",
     "Toggletip",
     "totime",
     "traceid",

package-lock.json

@@ -1,3 +1,5 @@
+import { dirname } from 'path';
+
 import { defineConfig, devices } from '@playwright/test';
 import type { PluginOptions } from '@grafana/plugin-e2e';
 
@@ -42,14 +44,14 @@ export default defineConfig<PluginOptions>({
   projects: [
     {
       name: 'auth',
-      testDir: 'node_modules/@grafana/plugin-e2e/dist/auth',
+      testDir: `${dirname(require.resolve('@grafana/plugin-e2e'))}/auth`,
       testMatch: [/.*\.js/],
     },
     {
       name: 'run-tests',
       use: {
         ...devices['Desktop Chrome'],
-        storageState: 'playwright/.auth/admin.json',
+        storageState: `playwright/.auth/${process.env.GRAFANA_ADMIN_USER || 'admin'}.json`,
       },
       dependencies: ['auth'],
     },

playwright.config.ts

@@ -1,26 +1,44 @@
 import { expect, test } from '@grafana/plugin-e2e';
+import { Page } from '@playwright/test';
 
 const PLUGIN_UID = 'grafana-clickhouse-datasource';
-const CLICKHOUSE_DB_URL = Boolean(process.env.CI) ? 'clickhouse-server' : 'localhost';
+
+// Determine ClickHouse DB URL based on environment
+function resolveClickhouseUrl(env = process.env) {
+  const { CI, DS_INSTANCE_HOST } = env;
+  return CI ? (DS_INSTANCE_HOST || 'clickhouse-server') : 'localhost';
+}
+
+// Configure Private Data Source Connect if network name is provided
+async function configurePDC(page: Page, networkName: string) {
+  await page.getByRole('combobox', { name: 'Private data source connect' }).click();
+  await page.getByText(networkName).click();
+}
 
 test.describe('Config Editor', () => {
   test('invalid credentials should return an error', async ({ createDataSourceConfigPage, page }) => {
     const configPage = await createDataSourceConfigPage({ type: PLUGIN_UID });
-    await page.getByPlaceholder('Server address').fill(CLICKHOUSE_DB_URL);
+    await page.getByPlaceholder('Server address').fill(resolveClickhouseUrl());
     await expect(configPage.saveAndTest()).not.toBeOK();
+    await configPage.deleteDataSource();
   });
 
   test('valid credentials should display a success alert on the page', async ({ createDataSourceConfigPage, page }) => {
     const configPage = await createDataSourceConfigPage({ type: PLUGIN_UID });
 
-    await page.getByPlaceholder('Server address').fill(CLICKHOUSE_DB_URL);
-    await page.getByPlaceholder('9000').fill('9000');
-    await page.getByPlaceholder('default').fill('default');
+    await page.getByPlaceholder('Server address').fill(resolveClickhouseUrl());
+    await page.getByPlaceholder('9000').fill(process.env.DS_INSTANCE_PORT ?? '9000');
+    await page.getByPlaceholder('default').fill(process.env.DS_INSTANCE_USERNAME ?? 'default');
+    await page.getByPlaceholder('password').fill(process.env.DS_INSTANCE_PASSWORD ?? '');
+
+    if (process.env.DS_PDC_NETWORK_NAME) {
+      await configurePDC(page, process.env.DS_PDC_NETWORK_NAME);
+    }
 
     await configPage.saveAndTest();
     await expect(configPage).toHaveAlert('success', { hasNotText: 'Datasource updated' });
 
-    await page.pause();
+    await configPage.deleteDataSource();
   });
 
   test('mandatory fields should show error if left empty', async ({ createDataSourceConfigPage, page }) => {
@@ -35,5 +53,7 @@ test.describe('Config Editor', () => {
     await expect(page.getByText('Port is required')).toBeVisible();
 
     await expect(configPage.saveAndTest()).not.toBeOK();
+
+    await configPage.deleteDataSource();
   });
 });