[8.19] [Security Solution] [Detections] Fix flakey EQL shard test (#215757) (#233535)

# Backport

This will backport the following commits from `main` to `8.19`:
- [[Security Solution] [Detections] Fix flakey EQL shard test
(#215757)](#215757)
 
Closes #209024; details can be
found there.

<!--- Backport version: 10.0.1 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sorenlouv/backport)

<!--BACKPORT [{"author":{"name":"Devin W.
Hurley","email":"[email protected]"},"sourceCommit":{"committedDate":"2025-03-28T18:47:15Z","message":"[Security
Solution] [Detections] Fix flakey EQL shard test (#215757)\n\n##
Summary\n\nRef: https://github.com/elastic/kibana/issues/209024\n\nFlake
caused by occasionally hitting max signals on the \"good\" shard
and\nnever triggering the error from the runtime field on the \"bad\"
shard. By\nmoving the bad runtime field to the `packetbeat` index and
changing the\nrule query in the test to an `and` we can ensure the rule
queries both\ngood and bad shards.\n\n### Checklist\n\n- [x] [Flaky
Test\nRunner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1)
was\nused on any tests
changed","sha":"d869d472f0b9b55c635580c4d7d15faff8b8c215","branchLabelMapping":{"^v9.1.0$":"main","^v8.19.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["review","release_note:skip","v9.0.0","Team:Detection
Engine","backport:version","v9.1.0","v9.0.1"],"title":"[Security
Solution] [Detections] Fix flakey EQL shard
test","number":215757,"url":"https://github.com/elastic/kibana/pull/215757","mergeCommit":{"message":"[Security
Solution] [Detections] Fix flakey EQL shard test (#215757)\n\n##
Summary\n\nRef: https://github.com/elastic/kibana/issues/209024\n\nFlake
caused by occasionally hitting max signals on the \"good\" shard
and\nnever triggering the error from the runtime field on the \"bad\"
shard. By\nmoving the bad runtime field to the `packetbeat` index and
changing the\nrule query in the test to an `and` we can ensure the rule
queries both\ngood and bad shards.\n\n### Checklist\n\n- [x] [Flaky
Test\nRunner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1)
was\nused on any tests
changed","sha":"d869d472f0b9b55c635580c4d7d15faff8b8c215"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"9.0","label":"v9.0.0","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"url":"https://github.com/elastic/kibana/pull/216375","number":216375,"state":"MERGED","mergeCommit":{"sha":"a2002e0bd3692dd9dfeca80d0dbc38dd3a2af18a","message":"[9.0]
[Security Solution] [Detections] Fix flakey EQL shard test (#215757)
(#216375)\n\n# Backport\n\nThis will backport the following commits from
`main` to `9.0`:\n- [[Security Solution] [Detections] Fix flakey EQL
shard
test\n(#215757)](https://github.com/elastic/kibana/pull/215757)\n\n\n\n###
Questions ?\nPlease refer to the [Backport
tool\ndocumentation](https://github.com/sorenlouv/backport)\n\n"}},{"branch":"main","label":"v9.1.0","branchLabelMappingKey":"^v9.1.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/215757","number":215757,"mergeCommit":{"message":"[Security
Solution] [Detections] Fix flakey EQL shard test (#215757)\n\n##
Summary\n\nRef: https://github.com/elastic/kibana/issues/209024\n\nFlake
caused by occasionally hitting max signals on the \"good\" shard
and\nnever triggering the error from the runtime field on the \"bad\"
shard. By\nmoving the bad runtime field to the `packetbeat` index and
changing the\nrule query in the test to an `and` we can ensure the rule
queries both\ngood and bad shards.\n\n### Checklist\n\n- [x] [Flaky
Test\nRunner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1)
was\nused on any tests
changed","sha":"d869d472f0b9b55c635580c4d7d15faff8b8c215"}}]}]
BACKPORT-->

---------

Co-authored-by: Devin W. Hurley <[email protected]>
Co-authored-by: kibanamachine <[email protected]>

Author: 3 people

x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql.ts

@@ -54,7 +54,6 @@ import {
   deleteAllRules,
   deleteAllAlerts,
   waitForRuleFailure,
-  waitForRulePartialFailure,
   routeWithNamespace,
 } from '../../../../../../../common/utils/security_solution';
 import { FtrProviderContext } from '../../../../../../ftr_provider_context';
@@ -254,30 +253,30 @@ export default ({ getService }: FtrProviderContext) => {
 
     it('parses shard failures for EQL event query', async () => {
       await esArchiver.load(packetBeatPath);
+      await setBrokenRuntimeField({ es, index: 'packetbeat-*' });
+
+      // sometimes we would hit max signals on the good shard
+      // and never search the shard with the bad runtime field
+      // by changing the logic to be "and" broken == 1
+      // we ensure that both shards are searched
+      // which I believe was the cause of the test being flakey.
       const rule: EqlRuleCreateProps = {
         ...getEqlRuleForAlertTesting(['auditbeat-*', 'packetbeat-*']),
-        query: 'any where agent.type == "packetbeat" or broken == 1',
+        query: 'any where agent.type == "packetbeat" and broken == 1',
       };
-      await setBrokenRuntimeField({ es, index: 'auditbeat-*' });
-      const createdRule = await createRule(supertest, log, rule);
-      const createdRuleId = createdRule.id;
-      await waitForRulePartialFailure({ supertest, log, id: createdRuleId });
-      const route = routeWithNamespace(DETECTION_ENGINE_RULES_URL);
-      const response = await supertest
-        .get(route)
-        .set('kbn-xsrf', 'true')
-        .set('elastic-api-version', '2023-10-31')
-        .query({ id: createdRule.id })
-        .expect(200);
-
-      const ruleResponse = response.body;
-      expect(
-        ruleResponse.execution_summary.last_execution.message.includes(
-          'The EQL event query was only executed on the available shards. The query failed to run successfully on the following shards:'
-        )
-      ).eql(true);
-
-      await unsetBrokenRuntimeField({ es, index: 'auditbeat-*' });
+      const { logs } = await previewRule({ supertest, rule });
+      expect_(logs).toEqual(
+        expect_.arrayContaining([
+          expect_.objectContaining({
+            warnings: expect_.arrayContaining([
+              expect_.stringContaining(
+                'The EQL event query was only executed on the available shards. The query failed to run successfully on the following shards:'
+              ),
+            ]),
+          }),
+        ])
+      );
+      await unsetBrokenRuntimeField({ es, index: 'packetbeat-*' });
       await esArchiver.unload(packetBeatPath);
     });
 

x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/runtime.ts

@@ -25,6 +25,9 @@ export const setBrokenRuntimeField = async ({ es, index }: UpdateMappingsProps)
     },
     index,
   });
+  await es.indices.refresh({
+    index,
+  });
 };
 
 export const unsetBrokenRuntimeField = async ({ es, index }: UpdateMappingsProps) => {
@@ -36,4 +39,7 @@ export const unsetBrokenRuntimeField = async ({ es, index }: UpdateMappingsProps
     },
     index,
   });
+  await es.indices.refresh({
+    index,
+  });
 };