visual tests

.github/workflows/test-preview.yml

@@ -0,0 +1,68 @@
+name: Playwright + Argos Tests
+
+on:
+  # Trigger on deployment event
+  deployment_status:
+  workflow_dispatch:
+  # Trigger when a label is added to a PR
+  pull_request:
+    types:
+      - labeled
+
+jobs:
+  test:
+    # Run tests only if the deployment is successful
+    if: |
+      (
+        github.event_name == 'deployment_status' &&
+        github.event.deployment_status.state == 'success'
+      ) || (
+        github.event_name == 'pull_request' &&
+        github.event.action == 'labeled' &&
+        github.event.label.name == 'visual-test'
+      )
+    runs-on: ubuntu-latest
+
+    steps:
+
+      # Step 1: Check out the repository
+      - name: Check out docs repo
+        uses: actions/checkout@v4
+        with:
+          repository: ClickHouse/clickhouse-docs
+          path: ./
+
+      # Step 2: Log url
+      - name: Log BASE_URL
+        run: echo "BASE_URL=${{ github.event.deployment_status.environment_url }}"
+        env:
+          BASE_URL: ${{ github.event.deployment_status.environment_url }}
+
+      # Step 3: Setup node
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20.18.0'
+          cache: 'yarn'
+
+      # Step 4: Install dependencies and build the site (needed for sitemap.xml)
+      - name: Install and build
+        run: |
+          export NODE_OPTIONS="--max_old_space_size=4096"
+          npm install -g yarn
+          yarn install
+
+      # Step 3: Run Playwright tests in Docker
+      - name: Run Playwright tests
+        run: |
+          docker run --rm \
+            -v ${{ github.workspace }}:/workspace \
+            -w /workspace \
+            -e GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} \
+            -e BASE_URL=${{ github.event.deployment_status.environment_url }} \
+            -e ARGOS_BRANCH="${{ github.event.deployment_status.environment == 'Production' && 'main' || github.ref_name }}" \
+            -e CI=true \
+            -e NUM_WORKERS=4 \
+            -e ARGOS_TOKEN=${{ secrets.ARGOS_TOKEN }} \
+            mcr.microsoft.com/playwright:v1.49.1-noble \
+            sh -c "git config --global --add safe.directory /workspace && npm exec -- playwright test --workers 4"

.gitignore

@@ -41,3 +41,7 @@ FormatFactorySettings.h
 Settings.cpp
 
 .vscode
+
+.vscode
+test-results/**
+tests/screenshot.spec.ts-snapshots/**

package.json

@@ -22,10 +22,10 @@
     "write-heading-ids": "docusaurus write-heading-ids"
   },
   "dependencies": {
-    "@docusaurus/core": "2.3.1",
-    "@docusaurus/plugin-client-redirects": "2.3.1",
-    "@docusaurus/preset-classic": "2.3.1",
-    "@docusaurus/theme-mermaid": "2.3.1",
+    "@docusaurus/core": "2.4.3",
+    "@docusaurus/plugin-client-redirects": "2.4.3",
+    "@docusaurus/preset-classic": "2.4.3",
+    "@docusaurus/theme-mermaid": "2.4.3",
     "@mdx-js/react": "^1.6.22",
     "@radix-ui/react-navigation-menu": "^1.1.4",
     "axios": "^1.7.9",
@@ -44,7 +44,11 @@
     "sass": "^1.82.0"
   },
   "devDependencies": {
-    "@docusaurus/module-type-aliases": "3.6.3"
+    "@argos-ci/cli": "^2.5.3",
+    "@argos-ci/playwright": "^3.9.4",
+    "@docusaurus/module-type-aliases": "3.6.3",
+    "@playwright/test": "^1.49.1",
+    "cheerio": "^1.0.0"
   },
   "browserslist": {
     "production": [
@@ -59,6 +63,6 @@
     ]
   },
   "engines": {
-    "node": ">=16.14"
+    "node": ">=20.18"
   }
 }

playwright.config.ts

@@ -0,0 +1,49 @@
+import {devices} from '@playwright/test';
+import type {PlaywrightTestConfig} from '@playwright/test';
+
+const isCI = !!process.env.CI; // Check if running in CI
+const baseURL = isCI ? process.env.BASE_URL : "http://localhost:3000";
+
+
+const config: PlaywrightTestConfig = {
+  fullyParallel: true,
+  webServer: {
+    port: 3000,
+    command: 'yarn docusaurus serve',
+  },
+  projects: [
+    {
+      name: 'chromium',
+      use: {
+        ...devices['Desktop Chrome'],
+      },
+    },
+  ],
+  reporter: [
+    // Use "dot" reporter on CI, "list" otherwise (Playwright default).
+    process.env.CI ? ["dot"] : ["list"],
+    // Add Argos reporter.
+    [
+      "@argos-ci/playwright/reporter",
+      {
+        // Upload to Argos on CI only.
+        uploadToArgos: isCI,
+
+        // Set your Argos token.
+        token: process.env.ARGOS_TOKEN,
+      },
+    ],
+  ],
+  timeout: 1800000, 
+  use: {
+    // On CI, we will set `BASE_URL` from Vercel preview URL
+    baseURL: baseURL,
+    extraHTTPHeaders: {
+        // Hide Vercel Toolbar in tests
+        "x-vercel-skip-toolbar": "0",
+    },
+  },
+
+};
+
+export default config;

tests/screenshot.css

@@ -0,0 +1,35 @@
+/*
+We need to hide some elements in Argos/Playwright screenshots
+Those elements are source of flakiness due to nondeterministic rendering
+They don't consistently render exactly the same across CI runs
+ */
+
+/******* DOCUSAURUS GLOBAL / THEME *******/
+
+/* Iframes can load lazily */
+iframe,
+/* Avatar images can be flaky due to using external sources: GitHub/Unavatar */
+.avatar__photo,
+/* Gifs load lazily and are animated */
+img[src$='.gif'],
+/* Algolia Keyboard shortcuts appear with a little delay */
+.DocSearch-Button-Keys > kbd,
+/* The live playground preview can often display dates/counters */
+[class*='playgroundPreview'] {
+  visibility: hidden;
+}
+
+/*
+Different docs last-update dates can alter layout
+"visibility: hidden" is not enough
+ */
+.theme-last-updated {
+  display: none;
+}
+
+/*
+Mermaid diagrams are rendered client-side and produce layout shifts
+ */
+.docusaurus-mermaid-container {
+  display: none;
+}

tests/screenshot.spec.ts

@@ -0,0 +1,88 @@
+import * as fs from 'fs';
+import { argosScreenshot } from '@argos-ci/playwright';
+import { test } from '@playwright/test';
+import axios from 'axios';
+import { extractSitemapPathnames, pathnameToArgosName } from './utils';
+
+// Constants
+const siteUrl = process.env.CI ? process.env.BASE_URL : 'http://localhost:3000';
+const sitemapUrl = `${siteUrl}/docs/sitemap.xml`;
+const stylesheetPath = './tests/screenshot.css';
+const stylesheet = fs.readFileSync(stylesheetPath).toString();
+const NUM_WORKERS = parseInt(process.env.NUM_WORKERS || '1', 10);
+
+// Wait for hydration, requires Docusaurus v2.4.3+
+// Docusaurus adds a <html data-has-hydrated="true"> once hydrated
+function waitForDocusaurusHydration() {
+  return document.documentElement.dataset.hasHydrated === 'true';
+}
+
+// Iterate over the number of workers
+for (let workerIndex = 0; workerIndex < NUM_WORKERS; workerIndex++) {
+  test.describe(`Docusaurus site screenshots batch ${workerIndex}`, async () => {
+    let pathnames: string[] = [];
+
+    test.beforeAll(async () => {
+      // Fetch the sitemap dynamically
+      try {
+        const response = await axios.get(sitemapUrl);
+        const sitemapContent = response.data;
+        pathnames = extractSitemapPathnames(sitemapContent).filter((pathname) =>
+          pathname.startsWith('/docs/en') // currently test en only
+        );
+      } catch (error) {
+        console.error(`Failed to fetch sitemap: ${error.message}`);
+        throw error;
+      }
+    });
+
+    test('Generate and run screenshot tests', async ({ page, browser }) => {
+      const timeout = 30000; // 30 seconds timeout for navigation
+      const workerPaths = pathnames.filter((_, index) => index % NUM_WORKERS === workerIndex);
+      console.log(`${workerPaths.length} paths to test`);
+
+      for (let i = 0; i < workerPaths.length; i++) {
+        const pathname = workerPaths[i];
+        console.log(`Processing ${pathname}`);
+        const url = siteUrl + pathname;
+
+        try {
+          // Recreate the browser context every 10 pages to reduce memory usage
+          if (i > 0 && i % 10 === 0) {
+            console.log(`Restarting browser context after processing ${i} pages.`);
+            await page.context().close();
+            const newContext = await browser.newContext();
+            page = await newContext.newPage();
+          }
+          await page.setViewportSize({ width: 1920, height: 1080 }); // Set the viewport size
+          await page.goto(url, { timeout });
+
+          // Check for meta redirect
+          const metaRedirect = await page.$('meta[http-equiv="refresh"]');
+          if (metaRedirect) {
+            console.warn(`Skipping ${pathname} due to meta redirect.`);
+            continue;
+          }
+
+          // Wait for hydration with a timeout
+          try {
+            await page.waitForFunction(waitForDocusaurusHydration, { timeout });
+          } catch (error) {
+            console.warn(`Skipping ${pathname} due to missing hydration.`);
+            continue;
+          }
+
+          // Add custom stylesheet for screenshots
+          await page.addStyleTag({ content: stylesheet });
+
+          // Take a screenshot
+
+          await argosScreenshot(page, pathnameToArgosName(pathname));
+          console.log(`Screenshot captured for ${pathname}`);
+        } catch (error) {
+          console.error(`Failed to process ${pathname}: ${error.message}`);
+        }
+      }
+    });
+  });
+}

tests/utils.ts

@@ -0,0 +1,16 @@
+import * as cheerio from "cheerio";
+import * as fs from "fs";
+
+export function extractSitemapPathnames(sitemap: string): string[] {
+  const $ = cheerio.load(sitemap, { xmlMode: true });
+  const urls: string[] = [];
+  $("loc").each(function handleLoc() {
+    urls.push($(this).text());
+  });
+  return urls.map((url) => new URL(url).pathname);
+}
+
+// Converts a pathname to a decent screenshot name
+export function pathnameToArgosName(pathname: string): string {
+  return pathname.replace(/^\/|\/$/g, "") || "index";
+}