[scout] support flaky test runner (#234918)

## Summary

Extending `.buildkite/pipelines/flaky_tests/pipeline.ts` with new
`scoutConfig` type, that should trigger Scout tests execution from CI
stats flaky-test-runner

Author: dmlemeshko

.buildkite/pipelines/flaky_tests/pipeline.ts

@@ -34,6 +34,19 @@ if (Number.isNaN(concurrency)) {
 const BASE_JOBS = 1;
 const MAX_JOBS = 500;
 
+function getScoutConfigGroupType(configPath: string): string | null {
+  // Match platform paths: x-pack/platform/... or src/platform/...
+  if (/^(x-pack|src)\/platform\//.test(configPath)) {
+    return 'platform';
+  }
+  // Match solution paths: x-pack/solutions/<solution>/plugins/...
+  const match = configPath.match(/^x-pack\/solutions\/([^/]+)\/plugins\//);
+  if (match) {
+    return match[1];
+  }
+  return null;
+}
+
 function getTestSuitesFromJson(json: string) {
   const fail = (errorMsg: string) => {
     console.error('+++ Invalid test config provided');
@@ -55,6 +68,7 @@ function getTestSuitesFromJson(json: string) {
   const testSuites: Array<
     | { type: 'group'; key: string; count: number }
     | { type: 'ftrConfig'; ftrConfig: string; count: number }
+    | { type: 'scoutConfig'; scoutConfig: string; count: number }
   > = [];
   for (const item of parsed) {
     if (typeof item !== 'object' || item === null) {
@@ -160,6 +174,33 @@ for (const testSuite of testSuites) {
     continue;
   }
 
+  if (testSuite.type === 'scoutConfig') {
+    const usesParallelWorkers = testSuite.scoutConfig.endsWith('parallel.playwright.config.ts');
+    const scoutConfigGroupType = getScoutConfigGroupType(testSuite.scoutConfig);
+
+    steps.push({
+      command: `.buildkite/scripts/steps/test/scout_configs.sh`,
+      env: {
+        SCOUT_CONFIG: testSuite.scoutConfig,
+        SCOUT_CONFIG_GROUP_TYPE: scoutConfigGroupType!,
+      },
+      key: `scout-suite-${suiteIndex++}`,
+      label: `${testSuite.scoutConfig}`,
+      parallelism: testSuite.count,
+      concurrency,
+      concurrency_group: process.env.UUID,
+      concurrency_method: 'eager',
+      agents: expandAgentQueue(usesParallelWorkers ? 'n2-8-spot' : 'n2-4-spot'),
+      depends_on: 'build',
+      timeout_in_minutes: 30,
+      cancel_on_build_failing: true,
+      retry: {
+        automatic: [{ exit_status: '-1', limit: 3 }],
+      },
+    });
+    continue;
+  }
+
   const [category, suiteName] = testSuite.key.split('/');
   switch (category) {
     case 'cypress':

.buildkite/scripts/steps/test/scout_configs.sh

@@ -7,44 +7,51 @@ source .buildkite/scripts/steps/functional/common.sh
 BUILDKITE_PARALLEL_JOB=${BUILDKITE_PARALLEL_JOB:-}
 SCOUT_CONFIG_GROUP_KEY=${SCOUT_CONFIG_GROUP_KEY:-}
 SCOUT_CONFIG_GROUP_TYPE=${SCOUT_CONFIG_GROUP_TYPE:-}
+SCOUT_CONFIG=${SCOUT_CONFIG:-}
 
-if [ "$SCOUT_CONFIG_GROUP_KEY" == "" ] && [ "$BUILDKITE_PARALLEL_JOB" == "" ]; then
-  echo "Missing SCOUT_CONFIG_GROUP_KEY env var"
-  exit 1
-fi
+EXTRA_ARGS=${FTR_EXTRA_ARGS:-}
+test -z "$EXTRA_ARGS" || buildkite-agent meta-data set "ftr-extra-args" "$EXTRA_ARGS"
+
+configs=""
 
 if [ "$SCOUT_CONFIG_GROUP_TYPE" == "" ]; then
   echo "Missing SCOUT_CONFIG_GROUP_TYPE env var"
   exit 1
 fi
 
-EXTRA_ARGS=${FTR_EXTRA_ARGS:-}
-test -z "$EXTRA_ARGS" || buildkite-agent meta-data set "ftr-extra-args" "$EXTRA_ARGS"
-
-export JOB="$SCOUT_CONFIG_GROUP_KEY"
-
-FAILED_CONFIGS_KEY="${BUILDKITE_STEP_ID}${SCOUT_CONFIG_GROUP_KEY}"
-
-configs=""
 group=$SCOUT_CONFIG_GROUP_TYPE
 
+if [ "$SCOUT_CONFIG" != "" ]; then
+  configs="$SCOUT_CONFIG"
+  export JOB="$SCOUT_CONFIG"
+  FAILED_CONFIGS_KEY="${BUILDKITE_STEP_ID}${SCOUT_CONFIG}"
+elif [ "$SCOUT_CONFIG_GROUP_KEY" != "" ]; then
+  export JOB="$SCOUT_CONFIG_GROUP_KEY"
+  FAILED_CONFIGS_KEY="${BUILDKITE_STEP_ID}${SCOUT_CONFIG_GROUP_KEY}"
+else
+  if [ "$BUILDKITE_PARALLEL_JOB" == "" ]; then
+    echo "Missing SCOUT_CONFIG_GROUP_KEY or SCOUT_CONFIG env var"
+    exit 1
+  fi
+fi
+
 # The first retry should only run the configs that failed in the previous attempt
 # Any subsequent retries, which would generally only happen by someone clicking the button in the UI, will run everything
-if [[ ! "$configs" && "${BUILDKITE_RETRY_COUNT:-0}" == "1" ]]; then
+if [[ -z "$configs" && "${BUILDKITE_RETRY_COUNT:-0}" == "1" ]]; then
   configs=$(buildkite-agent meta-data get "$FAILED_CONFIGS_KEY" --default '')
-  if [[ "$configs" ]]; then
+  if [[ -n "$configs" ]]; then
     echo "--- Retrying only failed configs"
     echo "$configs"
   fi
 fi
 
-if [ "$configs" == "" ] && [ "$SCOUT_CONFIG_GROUP_KEY" != "" ]; then
+if [ -z "$configs" ] && [ "$SCOUT_CONFIG_GROUP_KEY" != "" ]; then
   echo "--- downloading scout test configuration"
   download_artifact scout_playwright_configs.json .
   configs=$(jq -r '.[env.SCOUT_CONFIG_GROUP_KEY].configs[]' scout_playwright_configs.json)
 fi
 
-if [ "$configs" == "" ]; then
+if [ -z "$configs" ]; then
   echo "unable to determine configs to run"
   exit 1
 fi