Merge branch 'main' into csp-ai-assistant-asset-inventory-phase1

Author: maxcold

.buildkite/package-lock.json

@@ -15,6 +15,7 @@
     "js-yaml": "^4.1.0",
     "minimatch": "^5.0.1",
     "minimist": "^1.2.8",
+    "p-limit": "^3.1.0",
     "tslib": "*"
   },
   "devDependencies": {
@@ -23,7 +24,7 @@
     "@types/jscodeshift": "^0.12.0",
     "@types/minimatch": "^3.0.5",
     "@types/minimist": "^1.2.5",
-    "@types/node": "^15.12.2",
+    "@types/node": "^22.17.1",
     "jest": "^30.0.3",
     "jscodeshift": "^17.1.2",
     "nock": "^12.0.2",

.buildkite/package.json

@@ -34,6 +34,19 @@ if (Number.isNaN(concurrency)) {
 const BASE_JOBS = 1;
 const MAX_JOBS = 500;
 
+function getScoutConfigGroupType(configPath: string): string | null {
+  // Match platform paths: x-pack/platform/... or src/platform/...
+  if (/^(x-pack|src)\/platform\//.test(configPath)) {
+    return 'platform';
+  }
+  // Match solution paths: x-pack/solutions/<solution>/plugins/...
+  const match = configPath.match(/^x-pack\/solutions\/([^/]+)\/plugins\//);
+  if (match) {
+    return match[1];
+  }
+  return null;
+}
+
 function getTestSuitesFromJson(json: string) {
   const fail = (errorMsg: string) => {
     console.error('+++ Invalid test config provided');
@@ -55,6 +68,7 @@ function getTestSuitesFromJson(json: string) {
   const testSuites: Array<
     | { type: 'group'; key: string; count: number }
     | { type: 'ftrConfig'; ftrConfig: string; count: number }
+    | { type: 'scoutConfig'; scoutConfig: string; count: number }
   > = [];
   for (const item of parsed) {
     if (typeof item !== 'object' || item === null) {
@@ -160,6 +174,33 @@ for (const testSuite of testSuites) {
     continue;
   }
 
+  if (testSuite.type === 'scoutConfig') {
+    const usesParallelWorkers = testSuite.scoutConfig.endsWith('parallel.playwright.config.ts');
+    const scoutConfigGroupType = getScoutConfigGroupType(testSuite.scoutConfig);
+
+    steps.push({
+      command: `.buildkite/scripts/steps/test/scout_configs.sh`,
+      env: {
+        SCOUT_CONFIG: testSuite.scoutConfig,
+        SCOUT_CONFIG_GROUP_TYPE: scoutConfigGroupType!,
+      },
+      key: `scout-suite-${suiteIndex++}`,
+      label: `${testSuite.scoutConfig}`,
+      parallelism: testSuite.count,
+      concurrency,
+      concurrency_group: process.env.UUID,
+      concurrency_method: 'eager',
+      agents: expandAgentQueue(usesParallelWorkers ? 'n2-8-spot' : 'n2-4-spot'),
+      depends_on: 'build',
+      timeout_in_minutes: 30,
+      cancel_on_build_failing: true,
+      retry: {
+        automatic: [{ exit_status: '-1', limit: 3 }],
+      },
+    });
+    continue;
+  }
+
   const [category, suiteName] = testSuite.key.split('/');
   switch (category) {
     case 'cypress':

.buildkite/pipelines/flaky_tests/pipeline.ts

@@ -7,9 +7,11 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { execSync } from 'child_process';
+import { execSync, spawn } from 'child_process';
 import fs from 'fs';
 import path from 'path';
+import os from 'os';
+import pLimit from 'p-limit';
 import { storybookAliases } from '../../../../src/dev/storybook/aliases';
 import { getKibanaDir } from '#pipeline-utils';
 
@@ -25,6 +27,42 @@ const STORYBOOK_BASE_URL = `${STORYBOOK_BUCKET_URL}`;
 
 const exec = (...args: string[]) => execSync(args.join(' '), { stdio: 'inherit' });
 
+const buildStorybook = (storybook: string): Promise<{ logs: string }> => {
+  return new Promise((resolve, reject) => {
+    const logsBuffer: string[] = [];
+    const handleBufferChunk = (chunk: Buffer) => {
+      logsBuffer.push(chunk.toString());
+    };
+
+    const child = spawn('yarn', ['storybook', '--site', storybook], {
+      stdio: 'pipe',
+      env: {
+        ...process.env,
+        STORYBOOK_BASE_URL,
+        NODE_OPTIONS: '--max-old-space-size=6144',
+      },
+    });
+
+    child.stdout?.on('data', handleBufferChunk);
+    child.stderr?.on('data', handleBufferChunk);
+
+    child.on('close', (code) => {
+      if (code === 0) {
+        logsBuffer.unshift(`--- ✅ ${storybook} storybook\n`);
+        resolve({ logs: logsBuffer.join('') });
+      } else {
+        logsBuffer.unshift(`--- ❌ ${storybook} storybook\n`);
+        reject(new Error(logsBuffer.join('')));
+      }
+    });
+
+    child.on('error', () => {
+      logsBuffer.unshift(`--- ❌ ${storybook} storybook\n`);
+      reject(new Error(logsBuffer.join('')));
+    });
+  });
+};
+
 const ghStatus = (state: string, description: string) =>
   exec(
     `gh api "repos/elastic/kibana/statuses/${process.env.BUILDKITE_COMMIT}"`,
@@ -35,15 +73,23 @@ const ghStatus = (state: string, description: string) =>
     `--silent`
   );
 
-const build = () => {
+const build = async () => {
   console.log('--- Building Storybooks');
 
-  for (const storybook of Object.keys(storybookAliases)) {
-    exec(
-      `STORYBOOK_BASE_URL=${STORYBOOK_BASE_URL}`,
-      `NODE_OPTIONS=--max-old-space-size=6144`,
-      `yarn storybook --site ${storybook}`
+  const limit = pLimit(os.availableParallelism());
+  const storybooks = Object.keys(storybookAliases);
+
+  try {
+    const results = await Promise.all(
+      storybooks.map((storybook) => limit(() => buildStorybook(storybook)))
     );
+
+    results.forEach(({ logs }) => {
+      console.log(logs);
+    });
+  } catch (error) {
+    console.error(error);
+    throw error;
   }
 };
 
@@ -84,8 +130,8 @@ const upload = () => {
     );
     exec(`
       ${activateScript} gs://ci-artifacts.kibana.dev
-      gsutil -h "Cache-Control:no-cache, max-age=0, no-transform" -q -m cp -r -z js,css,html,json,map,txt,svg '*' 'gs://${STORYBOOK_BUCKET}/${STORYBOOK_DIRECTORY}/'
-      gsutil -h "Cache-Control:no-cache, max-age=0, no-transform" cp -z html 'index.html' 'gs://${STORYBOOK_BUCKET}/${STORYBOOK_DIRECTORY}/latest/'
+      gcloud storage cp --cache-control="no-cache, max-age=0, no-transform" --gzip-local=js,css,html,json,map,txt,svg --recursive --no-user-output-enabled '*' 'gs://${STORYBOOK_BUCKET}/${STORYBOOK_DIRECTORY}/'
+      gcloud storage cp --cache-control="no-cache, max-age=0, no-transform" --gzip-local=html --no-user-output-enabled 'index.html' 'gs://${STORYBOOK_BUCKET}/${STORYBOOK_DIRECTORY}/latest/'
     `);
 
     if (process.env.BUILDKITE_PULL_REQUEST && process.env.BUILDKITE_PULL_REQUEST !== 'false') {
@@ -98,12 +144,14 @@ const upload = () => {
   }
 };
 
-try {
-  ghStatus('pending', 'Building Storybooks');
-  build();
-  upload();
-  ghStatus('success', 'Storybooks built');
-} catch (error) {
-  ghStatus('error', 'Building Storybooks failed');
-  throw error;
-}
+(async () => {
+  try {
+    ghStatus('pending', 'Building Storybooks');
+    await build();
+    upload();
+    ghStatus('success', 'Storybooks built');
+  } catch (error) {
+    ghStatus('error', 'Building Storybooks failed');
+    throw error;
+  }
+})();

.buildkite/scripts/steps/storybooks/build_and_upload.ts

@@ -7,44 +7,51 @@ source .buildkite/scripts/steps/functional/common.sh
 BUILDKITE_PARALLEL_JOB=${BUILDKITE_PARALLEL_JOB:-}
 SCOUT_CONFIG_GROUP_KEY=${SCOUT_CONFIG_GROUP_KEY:-}
 SCOUT_CONFIG_GROUP_TYPE=${SCOUT_CONFIG_GROUP_TYPE:-}
+SCOUT_CONFIG=${SCOUT_CONFIG:-}
 
-if [ "$SCOUT_CONFIG_GROUP_KEY" == "" ] && [ "$BUILDKITE_PARALLEL_JOB" == "" ]; then
-  echo "Missing SCOUT_CONFIG_GROUP_KEY env var"
-  exit 1
-fi
+EXTRA_ARGS=${FTR_EXTRA_ARGS:-}
+test -z "$EXTRA_ARGS" || buildkite-agent meta-data set "ftr-extra-args" "$EXTRA_ARGS"
+
+configs=""
 
 if [ "$SCOUT_CONFIG_GROUP_TYPE" == "" ]; then
   echo "Missing SCOUT_CONFIG_GROUP_TYPE env var"
   exit 1
 fi
 
-EXTRA_ARGS=${FTR_EXTRA_ARGS:-}
-test -z "$EXTRA_ARGS" || buildkite-agent meta-data set "ftr-extra-args" "$EXTRA_ARGS"
-
-export JOB="$SCOUT_CONFIG_GROUP_KEY"
-
-FAILED_CONFIGS_KEY="${BUILDKITE_STEP_ID}${SCOUT_CONFIG_GROUP_KEY}"
-
-configs=""
 group=$SCOUT_CONFIG_GROUP_TYPE
 
+if [ "$SCOUT_CONFIG" != "" ]; then
+  configs="$SCOUT_CONFIG"
+  export JOB="$SCOUT_CONFIG"
+  FAILED_CONFIGS_KEY="${BUILDKITE_STEP_ID}${SCOUT_CONFIG}"
+elif [ "$SCOUT_CONFIG_GROUP_KEY" != "" ]; then
+  export JOB="$SCOUT_CONFIG_GROUP_KEY"
+  FAILED_CONFIGS_KEY="${BUILDKITE_STEP_ID}${SCOUT_CONFIG_GROUP_KEY}"
+else
+  if [ "$BUILDKITE_PARALLEL_JOB" == "" ]; then
+    echo "Missing SCOUT_CONFIG_GROUP_KEY or SCOUT_CONFIG env var"
+    exit 1
+  fi
+fi
+
 # The first retry should only run the configs that failed in the previous attempt
 # Any subsequent retries, which would generally only happen by someone clicking the button in the UI, will run everything
-if [[ ! "$configs" && "${BUILDKITE_RETRY_COUNT:-0}" == "1" ]]; then
+if [[ -z "$configs" && "${BUILDKITE_RETRY_COUNT:-0}" == "1" ]]; then
   configs=$(buildkite-agent meta-data get "$FAILED_CONFIGS_KEY" --default '')
-  if [[ "$configs" ]]; then
+  if [[ -n "$configs" ]]; then
     echo "--- Retrying only failed configs"
     echo "$configs"
   fi
 fi
 
-if [ "$configs" == "" ] && [ "$SCOUT_CONFIG_GROUP_KEY" != "" ]; then
+if [ -z "$configs" ] && [ "$SCOUT_CONFIG_GROUP_KEY" != "" ]; then
   echo "--- downloading scout test configuration"
   download_artifact scout_playwright_configs.json .
   configs=$(jq -r '.[env.SCOUT_CONFIG_GROUP_KEY].configs[]' scout_playwright_configs.json)
 fi
 
-if [ "$configs" == "" ]; then
+if [ -z "$configs" ]; then
   echo "unable to determine configs to run"
   exit 1
 fi