[8.19] [scout] support flaky test runner (#234918) (#235494)

# Backport

This will backport the following commits from `main` to `8.19`:
- [[scout] support flaky test runner
(#234918)](#234918)

<!--- Backport version: 9.6.6 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sorenlouv/backport)

<!--BACKPORT [{"author":{"name":"Dzmitry
Lemechko","email":"[email protected]"},"sourceCommit":{"committedDate":"2025-09-18T09:18:59Z","message":"[scout]
support flaky test runner (#234918)\n\n## Summary\n\nExtending
`.buildkite/pipelines/flaky_tests/pipeline.ts` with new\n`scoutConfig`
type, that should trigger Scout tests execution from CI\nstats
flaky-test-runner","sha":"1fe19a39faf9f5d96bddc7db432732f703578549","branchLabelMapping":{"^v9.2.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","backport:all-open","test:scout","v9.2.0"],"title":"[scout]
support flaky test
runner","number":234918,"url":"https://github.com/elastic/kibana/pull/234918","mergeCommit":{"message":"[scout]
support flaky test runner (#234918)\n\n## Summary\n\nExtending
`.buildkite/pipelines/flaky_tests/pipeline.ts` with new\n`scoutConfig`
type, that should trigger Scout tests execution from CI\nstats
flaky-test-runner","sha":"1fe19a39faf9f5d96bddc7db432732f703578549"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v9.2.0","branchLabelMappingKey":"^v9.2.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/234918","number":234918,"mergeCommit":{"message":"[scout]
support flaky test runner (#234918)\n\n## Summary\n\nExtending
`.buildkite/pipelines/flaky_tests/pipeline.ts` with new\n`scoutConfig`
type, that should trigger Scout tests execution from CI\nstats
flaky-test-runner","sha":"1fe19a39faf9f5d96bddc7db432732f703578549"}}]}]
BACKPORT-->

Co-authored-by: Dzmitry Lemechko <[email protected]>

Author: kibanamachine

.buildkite/pipelines/flaky_tests/pipeline.ts

@@ -33,6 +33,19 @@ if (Number.isNaN(concurrency)) {
 const BASE_JOBS = 1;
 const MAX_JOBS = 500;
 
+function getScoutConfigGroupType(configPath: string): string | null {
+  // Match platform paths: x-pack/platform/... or src/platform/...
+  if (/^(x-pack|src)\/platform\//.test(configPath)) {
+    return 'platform';
+  }
+  // Match solution paths: x-pack/solutions/<solution>/plugins/...
+  const match = configPath.match(/^x-pack\/solutions\/([^/]+)\/plugins\//);
+  if (match) {
+    return match[1];
+  }
+  return null;
+}
+
 function getTestSuitesFromJson(json: string) {
   const fail = (errorMsg: string) => {
     console.error('+++ Invalid test config provided');
@@ -54,6 +67,7 @@ function getTestSuitesFromJson(json: string) {
   const testSuites: Array<
     | { type: 'group'; key: string; count: number }
     | { type: 'ftrConfig'; ftrConfig: string; count: number }
+    | { type: 'scoutConfig'; scoutConfig: string; count: number }
   > = [];
   for (const item of parsed) {
     if (typeof item !== 'object' || item === null) {
@@ -157,6 +171,33 @@ for (const testSuite of testSuites) {
     continue;
   }
 
+  if (testSuite.type === 'scoutConfig') {
+    const usesParallelWorkers = testSuite.scoutConfig.endsWith('parallel.playwright.config.ts');
+    const scoutConfigGroupType = getScoutConfigGroupType(testSuite.scoutConfig);
+
+    steps.push({
+      command: `.buildkite/scripts/steps/test/scout_configs.sh`,
+      env: {
+        SCOUT_CONFIG: testSuite.scoutConfig,
+        SCOUT_CONFIG_GROUP_TYPE: scoutConfigGroupType!,
+      },
+      key: `scout-suite-${suiteIndex++}`,
+      label: `${testSuite.scoutConfig}`,
+      parallelism: testSuite.count,
+      concurrency,
+      concurrency_group: process.env.UUID,
+      concurrency_method: 'eager',
+      agents: expandAgentQueue(usesParallelWorkers ? 'n2-8-spot' : 'n2-4-spot'),
+      depends_on: 'build',
+      timeout_in_minutes: 30,
+      cancel_on_build_failing: true,
+      retry: {
+        automatic: [{ exit_status: '-1', limit: 3 }],
+      },
+    });
+    continue;
+  }
+
   const [category, suiteName] = testSuite.key.split('/');
   switch (category) {
     case 'cypress':

.buildkite/scripts/steps/test/scout_configs.sh

@@ -7,44 +7,51 @@ source .buildkite/scripts/steps/functional/common.sh
 BUILDKITE_PARALLEL_JOB=${BUILDKITE_PARALLEL_JOB:-}
 SCOUT_CONFIG_GROUP_KEY=${SCOUT_CONFIG_GROUP_KEY:-}
 SCOUT_CONFIG_GROUP_TYPE=${SCOUT_CONFIG_GROUP_TYPE:-}
+SCOUT_CONFIG=${SCOUT_CONFIG:-}
 
-if [ "$SCOUT_CONFIG_GROUP_KEY" == "" ] && [ "$BUILDKITE_PARALLEL_JOB" == "" ]; then
-  echo "Missing SCOUT_CONFIG_GROUP_KEY env var"
-  exit 1
-fi
+EXTRA_ARGS=${FTR_EXTRA_ARGS:-}
+test -z "$EXTRA_ARGS" || buildkite-agent meta-data set "ftr-extra-args" "$EXTRA_ARGS"
+
+configs=""
 
 if [ "$SCOUT_CONFIG_GROUP_TYPE" == "" ]; then
   echo "Missing SCOUT_CONFIG_GROUP_TYPE env var"
   exit 1
 fi
 
-EXTRA_ARGS=${FTR_EXTRA_ARGS:-}
-test -z "$EXTRA_ARGS" || buildkite-agent meta-data set "ftr-extra-args" "$EXTRA_ARGS"
-
-export JOB="$SCOUT_CONFIG_GROUP_KEY"
-
-FAILED_CONFIGS_KEY="${BUILDKITE_STEP_ID}${SCOUT_CONFIG_GROUP_KEY}"
-
-configs=""
 group=$SCOUT_CONFIG_GROUP_TYPE
 
+if [ "$SCOUT_CONFIG" != "" ]; then
+  configs="$SCOUT_CONFIG"
+  export JOB="$SCOUT_CONFIG"
+  FAILED_CONFIGS_KEY="${BUILDKITE_STEP_ID}${SCOUT_CONFIG}"
+elif [ "$SCOUT_CONFIG_GROUP_KEY" != "" ]; then
+  export JOB="$SCOUT_CONFIG_GROUP_KEY"
+  FAILED_CONFIGS_KEY="${BUILDKITE_STEP_ID}${SCOUT_CONFIG_GROUP_KEY}"
+else
+  if [ "$BUILDKITE_PARALLEL_JOB" == "" ]; then
+    echo "Missing SCOUT_CONFIG_GROUP_KEY or SCOUT_CONFIG env var"
+    exit 1
+  fi
+fi
+
 # The first retry should only run the configs that failed in the previous attempt
 # Any subsequent retries, which would generally only happen by someone clicking the button in the UI, will run everything
-if [[ ! "$configs" && "${BUILDKITE_RETRY_COUNT:-0}" == "1" ]]; then
+if [[ -z "$configs" && "${BUILDKITE_RETRY_COUNT:-0}" == "1" ]]; then
   configs=$(buildkite-agent meta-data get "$FAILED_CONFIGS_KEY" --default '')
-  if [[ "$configs" ]]; then
+  if [[ -n "$configs" ]]; then
     echo "--- Retrying only failed configs"
     echo "$configs"
   fi
 fi
 
-if [ "$configs" == "" ] && [ "$SCOUT_CONFIG_GROUP_KEY" != "" ]; then
+if [ -z "$configs" ] && [ "$SCOUT_CONFIG_GROUP_KEY" != "" ]; then
   echo "--- downloading scout test configuration"
   download_artifact scout_playwright_configs.json .
   configs=$(jq -r '.[env.SCOUT_CONFIG_GROUP_KEY].configs[]' scout_playwright_configs.json)
 fi
 
-if [ "$configs" == "" ]; then
+if [ -z "$configs" ]; then
   echo "unable to determine configs to run"
   exit 1
 fi