[Scout] Test config manifests

.buildkite/scripts/steps/checks/check_scout_config.sh

@@ -5,4 +5,11 @@ set -euo pipefail
 source .buildkite/scripts/common/util.sh
 
 echo --- Check for unregistered Scout Playwright configs
-node scripts/scout discover-playwright-configs --validate 
+node scripts/scout discover-playwright-configs --validate
+
+echo --- Make sure Scout config manifests are up to date
+node scripts/scout update-test-config-manifests
+check_for_changed_files \
+  "node scripts/scout update-test-config-manifests" \
+  true \
+  "[Scout] Automated config manifest updates"

.buildkite/scripts/steps/checks/quick_checks.json

@@ -75,7 +75,8 @@
     "mayChangeFiles": true
   },
   {
-    "script": ".buildkite/scripts/steps/checks/check_scout_config.sh"
+    "script": ".buildkite/scripts/steps/checks/check_scout_config.sh",
+    "mayChangeFiles": true
   },
   {
     "script": ".buildkite/scripts/steps/checks/dependencies_diff.sh",

src/platform/packages/private/kbn-scout-info/src/paths.ts

@@ -13,18 +13,41 @@ import { REPO_ROOT } from '@kbn/repo-info';
 export const SCOUT_OUTPUT_ROOT = path.resolve(REPO_ROOT, '.scout');
 
 // Servers
+
 export const SCOUT_SERVERS_ROOT = path.resolve(SCOUT_OUTPUT_ROOT, 'servers');
 
 // Reporting
+
 export const SCOUT_REPORT_OUTPUT_ROOT = path.resolve(SCOUT_OUTPUT_ROOT, 'reports');
 export const SCOUT_TEST_CONFIG_STATS_PATH = path.resolve(
   SCOUT_OUTPUT_ROOT,
   'test_config_stats.json'
 );
 
-// Scout playwright configs
+// Scout definitions
+
 export const SCOUT_PLAYWRIGHT_CONFIGS_PATH = path.resolve(
   SCOUT_OUTPUT_ROOT,
   'test_configs',
   'scout_playwright_configs.json'
 );
+
+export const TESTABLE_COMPONENT_SCOUT_ROOT_PATH_GLOB =
+  '{src/platform,x-pack/**}/{plugins,packages}/**/test/scout';
+
+export const TESTABLE_COMPONENT_SCOUT_ROOT_PATH_REGEX = new RegExp(
+  `(?:src|x-pack)` +
+    `\/(?:(platform)|solutions\/(\\w+))` + // 1: platform, 2: solution
+    `\/(plugins|packages)` + // 3: plugin or package
+    `\/?(shared|private|)` + // 4: artifact visibility
+    `\/([\\w|-]*)` + // 5: plugin/package name
+    `\/test\/scout(?:_([^\\/]*))?` // 6: custom target config set name
+);
+export const SCOUT_CONFIG_PATH_GLOB =
+  TESTABLE_COMPONENT_SCOUT_ROOT_PATH_GLOB + '/{ui,api}/{,*.}playwright.config.ts';
+
+export const SCOUT_CONFIG_PATH_REGEX = new RegExp(
+  TESTABLE_COMPONENT_SCOUT_ROOT_PATH_REGEX.source +
+    `\/(api|ui)` + // 7: Scout test category
+    `\/(\\w*)\\.?playwright.config.ts` // 8: Scout config type
+);

src/platform/packages/private/kbn-scout-reporting/src/helpers/index.ts

@@ -10,4 +10,4 @@
 export { getKibanaModuleData, type KibanaModuleMetadata } from './read_manifest';
 export { excapeHtmlCharacters, stripFilePath, parseStdout } from './text_processing';
 export { getRunTarget, stripRunCommand } from './cli_processing';
-export { getTestIDForTitle, generateTestRunId } from './test_id_generator';
+export { computeTestID, generateTestRunId } from './test_id_generator';

src/platform/packages/private/kbn-scout-reporting/src/helpers/test_id_generator.ts

@@ -13,6 +13,8 @@ export function generateTestRunId() {
   return randomBytes(8).toString('hex');
 }
 
-export function getTestIDForTitle(title: string) {
-  return createHash('sha256').update(title).digest('hex').slice(0, 31);
+export function computeTestID(testFilePath: string, testTitle: string) {
+  return [testFilePath, testTitle]
+    .map((input) => createHash('sha256').update(input).digest('hex').slice(0, 15))
+    .join('-');
 }

src/platform/packages/private/kbn-scout-reporting/src/registry/index.ts

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+export * from './test_config';
+export * from './testable_component';

src/platform/packages/private/kbn-scout-reporting/src/registry/manifest.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { simpleGit, type SimpleGit } from 'simple-git';
+import { REPO_ROOT } from '@kbn/repo-info';
+import type { TestCase } from '@playwright/test/reporter';
+
+let git: SimpleGit;
+
+export const getGitSHA1ForPath = async (p: string) => {
+  if (git === undefined) git = simpleGit(REPO_ROOT);
+  return (await git.raw(['ls-tree', '--object-only', 'HEAD', p])).trim();
+};
+
+export interface ScoutConfigManifest {
+  path: string;
+  exists: boolean;
+  lastModified: string;
+  sha1: string;
+  tests: {
+    id: string;
+    title: string;
+    expectedStatus: string;
+    tags: string[];
+    location: TestCase['location'];
+  }[];
+}

src/platform/packages/private/kbn-scout-reporting/src/registry/test_config.ts

@@ -0,0 +1,140 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { globSync } from 'fast-glob';
+import { REPO_ROOT } from '@kbn/repo-info';
+import path from 'node:path';
+import { ToolingLog } from '@kbn/tooling-log';
+import { SCOUT_CONFIG_PATH_GLOB, SCOUT_CONFIG_PATH_REGEX } from '@kbn/scout-info';
+import { existsSync, readFileSync } from 'node:fs';
+import type { ScoutTestableComponent } from './testable_component';
+import type { ScoutConfigManifest } from './manifest';
+
+export interface ScoutTestConfig {
+  path: string;
+  category: string;
+  type: string;
+  component: ScoutTestableComponent;
+  manifest: ScoutConfigManifest;
+}
+
+export const testConfig = {
+  fromPath(configPath: string): ScoutTestConfig {
+    const match = configPath.match(SCOUT_CONFIG_PATH_REGEX);
+
+    if (match == null) {
+      throw new Error(`Scout config path ${configPath} did not match the expected regex pattern`);
+    }
+
+    const [
+      _,
+      platform,
+      solution,
+      componentType,
+      componentVisibility,
+      componentName,
+      customTargetConfigSetName,
+      testCategory,
+      testConfigType,
+    ] = match;
+
+    const scoutDirName = `scout${customTargetConfigSetName ? `_${customTargetConfigSetName}` : ''}`;
+
+    const component = {
+      name: componentName,
+      group: platform ?? solution,
+      type: componentType.slice(0, -1) as ScoutTestableComponent['type'],
+      visibility: componentVisibility as ScoutTestableComponent['visibility'],
+      root: configPath.split(`/test/${scoutDirName}`)[0],
+    };
+
+    const manifestPath = path.join(
+      component.root,
+      'test',
+      scoutDirName,
+      '.meta',
+      testCategory,
+      `${testConfigType || 'standard'}.json`
+    );
+    const manifestExists = existsSync(manifestPath);
+
+    const manifest: ScoutConfigManifest = manifestExists
+      ? {
+          path: manifestPath,
+          exists: manifestExists,
+          ...JSON.parse(readFileSync(manifestPath, 'utf8')),
+        }
+      : {
+          path: manifestPath,
+          exists: manifestExists,
+          tests: [],
+        };
+
+    return {
+      path: configPath,
+      category: testCategory,
+      type: testConfigType || 'standard',
+      component: {
+        name: componentName,
+        group: platform ?? solution,
+        type: componentType.slice(0, -1) as ScoutTestableComponent['type'],
+        visibility: componentVisibility as ScoutTestableComponent['visibility'],
+        root: configPath.split('/test/scout')[0],
+      },
+      manifest,
+    };
+  },
+};
+
+export const testConfigs = {
+  _configs: null as ScoutTestConfig[] | null,
+  log: new ToolingLog(),
+
+  findPaths(): string[] {
+    return globSync(path.join(REPO_ROOT, SCOUT_CONFIG_PATH_GLOB), { onlyFiles: true }).map(
+      (configPath) => path.relative(REPO_ROOT, configPath)
+    );
+  },
+
+  _load() {
+    this.log.info(`${this._configs == null ? 'L' : 'Rel'}oading Scout test configs`);
+
+    const loadStartTime = performance.now();
+    this._configs = this.findPaths().map((configPath) => testConfig.fromPath(configPath));
+    const loadTime = performance.now() - loadStartTime;
+
+    this.log.info(
+      `Loaded ${this._configs.length} Scout test configs in ${(loadTime / 1000).toFixed(2)}s`
+    );
+  },
+
+  reload() {
+    this._load();
+  },
+
+  get all(): ScoutTestConfig[] {
+    if (this._configs === null) this._load();
+    return this._configs!;
+  },
+
+  forComponent(name: string, type?: ScoutTestableComponent['type']): ScoutTestConfig[] {
+    const configs = this.all.filter((config) => config.component.name === name);
+    return type === undefined
+      ? configs
+      : configs.filter((config) => config.component.type === type);
+  },
+
+  forPlugin(pluginName: string) {
+    return this.forComponent(pluginName, 'plugin');
+  },
+
+  forPackage(packageName: string) {
+    return this.forComponent(packageName, 'package');
+  },
+};

src/platform/packages/private/kbn-scout-reporting/src/registry/testable_component.ts

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+export interface ScoutTestableComponent {
+  name: string;
+  group: string;
+  type: 'plugin' | 'package';
+  visibility?: 'shared' | 'private';
+  root: string;
+}
+
+export interface ScoutTestableComponentWithConfigs extends ScoutTestableComponent {
+  configs: Omit<ScoutTestConfig, 'component'>[];
+}
+
+import { type ScoutTestConfig, testConfigs } from './test_config';
+
+export const testableComponents = {
+  get all(): ScoutTestableComponent[] {
+    const seenComponents: string[] = [];
+    return testConfigs.all.reduce<ScoutTestableComponent[]>((components, config) => {
+      if (!seenComponents.includes(config.component.root)) {
+        components.push(config.component);
+        seenComponents.push(config.component.root);
+      }
+      return components;
+    }, []);
+  },
+
+  get allIncludingConfigs(): ScoutTestableComponentWithConfigs[] {
+    return this.all.map((component) => ({
+      ...component,
+      configs: [
+        ...testConfigs.forComponent(component.name, component.type).map((config) => ({
+          path: config.path,
+          category: config.category,
+          type: config.type,
+          manifest: config.manifest,
+        })),
+      ],
+    }));
+  },
+
+  ofType(componentType: ScoutTestableComponent['type']): ScoutTestableComponent[] {
+    return testableComponents.all.filter((component) => component.type === componentType);
+  },
+
+  get plugins(): ScoutTestableComponent[] {
+    return testableComponents.ofType('plugin');
+  },
+
+  get packages(): ScoutTestableComponent[] {
+    return testableComponents.ofType('package');
+  },
+};

src/platform/packages/private/kbn-scout-reporting/src/reporting/jest/reporter.ts

@@ -32,7 +32,7 @@ import type { ScoutFileInfo } from '../../..';
 import {
   datasources,
   generateTestRunId,
-  getTestIDForTitle,
+  computeTestID,
   ScoutEventsReport,
   ScoutReportEventAction,
   type ScoutTestRunInfo,
@@ -157,7 +157,7 @@ export class ScoutJestReporter extends BaseReporter {
         type: test.result.ancestorTitles.length <= 1 ? 'root' : 'suite',
       },
       test: {
-        id: getTestIDForTitle(test.result.fullName),
+        id: computeTestID(path.relative(REPO_ROOT, test.filePath), test.result.fullName),
         title: test.result.title,
         tags: [],
         file: this.getScoutFileInfoForPath(path.relative(REPO_ROOT, test.filePath)),