Skip to content

Commit 2927233

Browse files
[8.19] [Attack Discovery][Scheduling] Add attack discovery alerts deduplication (#12790) (#223483) (#223971)
# Backport This will backport the following commits from `main` to `8.19`: - [[Attack Discovery][Scheduling] Add attack discovery alerts deduplication (#12790) (#223483)](#223483) <!--- Backport version: 9.6.6 --> ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sorenlouv/backport) <!--BACKPORT [{"author":{"name":"Ievgen Sorokopud","email":"[email protected]"},"sourceCommit":{"committedDate":"2025-06-13T19:35:52Z","message":"[Attack Discovery][Scheduling] Add attack discovery alerts deduplication (#12790) (#223483)\n\n## Summary\n\nMain ticket ([Internal\nlink](https://github.com/elastic/security-team/issues/12790))\n\nWith these changes we add attack discovery alerts deduplication. Right\nnow identical attack are being stored within the alerts index and to\nfilter duplicates out we will use the next approach:\n\n1. Each alert will have a ID generated as a SHA256 of attack attributes:\nlist of SIEM alerts IDs and space ID\n2. After LLM generates attack discoveries we would check whether the\nattack alert with the ID that describes the attack based on the rule\nabove exists in the alerts index\n3. If such an alerts exists we would drop the generated discovery\n4. Otherwise, we would proceed with storing the attack discovery as a\nnew alert\n\n## NOTES\n\nThe feature is hidden behind the feature flag (in `kibana.dev.yml`):\n\n```\nfeature_flags.overrides:\n securitySolution.attackDiscoveryAlertsEnabled: true\n```","sha":"ed803078f99d9049ea6fcb6b7eea50ea1d3cea9d","branchLabelMapping":{"^v9.1.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","Team: SecuritySolution","Team:Security Generative AI","backport:version","v9.1.0","v8.19.0"],"title":"[Attack Discovery][Scheduling] Add attack discovery alerts deduplication (#12790)","number":223483,"url":"https://github.com/elastic/kibana/pull/223483","mergeCommit":{"message":"[Attack Discovery][Scheduling] Add attack discovery alerts deduplication (#12790) (#223483)\n\n## Summary\n\nMain ticket ([Internal\nlink](https://github.com/elastic/security-team/issues/12790))\n\nWith these changes we add attack discovery alerts deduplication. Right\nnow identical attack are being stored within the alerts index and to\nfilter duplicates out we will use the next approach:\n\n1. Each alert will have a ID generated as a SHA256 of attack attributes:\nlist of SIEM alerts IDs and space ID\n2. After LLM generates attack discoveries we would check whether the\nattack alert with the ID that describes the attack based on the rule\nabove exists in the alerts index\n3. If such an alerts exists we would drop the generated discovery\n4. Otherwise, we would proceed with storing the attack discovery as a\nnew alert\n\n## NOTES\n\nThe feature is hidden behind the feature flag (in `kibana.dev.yml`):\n\n```\nfeature_flags.overrides:\n securitySolution.attackDiscoveryAlertsEnabled: true\n```","sha":"ed803078f99d9049ea6fcb6b7eea50ea1d3cea9d"}},"sourceBranch":"main","suggestedTargetBranches":["8.19"],"targetPullRequestStates":[{"branch":"main","label":"v9.1.0","branchLabelMappingKey":"^v9.1.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/223483","number":223483,"mergeCommit":{"message":"[Attack Discovery][Scheduling] Add attack discovery alerts deduplication (#12790) (#223483)\n\n## Summary\n\nMain ticket ([Internal\nlink](https://github.com/elastic/security-team/issues/12790))\n\nWith these changes we add attack discovery alerts deduplication. Right\nnow identical attack are being stored within the alerts index and to\nfilter duplicates out we will use the next approach:\n\n1. Each alert will have a ID generated as a SHA256 of attack attributes:\nlist of SIEM alerts IDs and space ID\n2. After LLM generates attack discoveries we would check whether the\nattack alert with the ID that describes the attack based on the rule\nabove exists in the alerts index\n3. If such an alerts exists we would drop the generated discovery\n4. Otherwise, we would proceed with storing the attack discovery as a\nnew alert\n\n## NOTES\n\nThe feature is hidden behind the feature flag (in `kibana.dev.yml`):\n\n```\nfeature_flags.overrides:\n securitySolution.attackDiscoveryAlertsEnabled: true\n```","sha":"ed803078f99d9049ea6fcb6b7eea50ea1d3cea9d"}},{"branch":"8.19","label":"v8.19.0","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"}]}] BACKPORT--> Co-authored-by: Ievgen Sorokopud <[email protected]>
1 parent c9808ad commit 2927233

File tree

18 files changed

+678
-255
lines changed

18 files changed

+678
-255
lines changed

x-pack/solutions/security/plugins/elastic_assistant/server/__mocks__/data_clients.mock.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,8 @@ export const conversationsDataClientMock: {
4747
const createAttackDiscoveryDataClientMock = (): AttackDiscoveryDataClientMock => ({
4848
bulkUpdateAttackDiscoveryAlerts: jest.fn(),
4949
createAttackDiscovery: jest.fn(),
50+
getAdHocAlertsIndexPattern: jest.fn(),
51+
getScheduledAndAdHocIndexPattern: jest.fn(),
5052
createAttackDiscoveryAlerts: jest.fn(),
5153
findAllAttackDiscoveries: jest.fn(),
5254
getAlertConnectorNames: jest.fn(),

x-pack/solutions/security/plugins/elastic_assistant/server/lib/attack_discovery/evaluation/__mocks__/mock_attack_discoveries.ts

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,4 +29,33 @@ export const mockAttackDiscoveries: AttackDiscovery[] = [
2929
entitySummaryMarkdown:
3030
'Critical malware and phishing alerts detected on {{ host.name e1cb3cf0-30f3-4f99-a9c8-518b955c6f90 }} involving user {{ user.name 039c15c5-3964-43e7-a891-42fe2ceeb9ff }}.',
3131
},
32+
{
33+
alertIds: [
34+
'093abac9d2431f4e87e696aee648c44a77cc9d4e7d12ba90ad71821c399baffe',
35+
'57fbcbdc22ececf1cbc4d3978c8085b64e33f03f927997769353348d003b80ac',
36+
'5fb3aa96aa5db16a2b8f6f6dabc1b575c80325897839e4ed4fd1c4e264ce591f',
37+
'61666b168da4e77afb540a5c04b845c32d9e0113e1434f04f2e25b35980527a7',
38+
'65acae68c6f518871a70f139913a031d68275808d584731f049b24fd3690f67b',
39+
'87f2d1a10a41fd3ed6077cf85d96df744255fec1072834ca30d5a88525dc9c9d',
40+
'8b164dbdaff8b57b7b881d11ceecca8cab0364422d2bf87a009f4f897066cf23',
41+
'b1c21a113f3858b6fd0a2f3854b71f9da52bdf46fdfa2c702a3f966f8809e153',
42+
'b47c3d991f071dcae02d5de601c6ebdec565e31262ae6cd3a304f6373225c65b',
43+
'd7985a0a305644d27945ca0ffcb861c75f6144199d16e8ac9f681fcf26333308',
44+
],
45+
detailsMarkdown:
46+
'- On {{ host.name 1f217ee0-af2d-424b-b733-b0fdd726df69 }}, a suspicious process {{ process.name mimikatz.exe }} with {{ process.args "C:\\mimikatz.exe",--tr1 }} was executed by {{ user.name a2b0848d-e54f-4cb2-83db-01186ad3a33c }} from {{ user.domain 7nabs9z95j }} at {{ kibana.alert.original_time 2025-06-10T11:27:41.520Z }}, resulting in authentication failures.\n- Shortly after, the same host saw file activity on {{ file.path C:\\My Documents\\business\\January\\processName }} by process {{ process.name lsass.exe }} ({{ process.args "C:\\lsass.exe",--1ge }}) under {{ user.name 24fcfb78-d478-485f-a811-863e7f455e84 }} from {{ user.domain qevk4kbvcm }} at {{ kibana.alert.original_time 2025-06-10T11:30:47.520Z }}.\n- At {{ kibana.alert.original_time 2025-06-10T12:10:20.520Z }}, on the same host, {{ process.name powershell.exe }} and {{ file.name fake_behavior.exe }} were executed by {{ user.name 054b9510-4e58-4a0a-bc28-0286a6566e19 }} from {{ user.domain aacuihaf3x }}, with network connections from {{ source.ip 10.234.158.79 }} to {{ destination.ip 10.66.11.163 }}.\n- Simultaneously, on {{ host.name e70b7f81-e5a1-4c83-9168-568156cdc17a }}, malware {{ process.name iexlorer.exe }} was detected as {{ user.name 1e30f444-04b7-43d0-9b97-1d335d96e72c }} from {{ user.domain cwi41t2i0r }} at {{ kibana.alert.original_time 2025-06-10T12:10:12.534Z }}, suggesting propagation.\n- At {{ kibana.alert.original_time 2025-06-10T12:13:06.520Z }}, on {{ host.name 1f217ee0-af2d-424b-b733-b0fdd726df69 }}, {{ process.name explorer.exe }} ({{ process.executable C:/fake_behavior/explorer.exe }}) and {{ file.name fake_behavior.exe }} were run by {{ user.name 5b4207f4-9cf0-47b0-b875-7384b2d292d7 }} from {{ user.domain 09yzh9bg35 }}, with network connections from {{ source.ip 10.63.82.62 }} to {{ destination.ip 10.43.183.158 }}.\n- The close timing, repeated use of credential access tools, and network activity between hosts indicate a coordinated attack chain involving credential dumping, malware propagation, and data collection/exfiltration.',
47+
entitySummaryMarkdown:
48+
'Credential access and malware on {{ host.name 1f217ee0-af2d-424b-b733-b0fdd726df69 }}.',
49+
mitreAttackTactics: [
50+
'Credential Access',
51+
'Execution',
52+
'Collection',
53+
'Command and Control',
54+
'Exfiltration',
55+
],
56+
summaryMarkdown:
57+
'Credential dumping, malware, and data exfiltration on {{ host.name 1f217ee0-af2d-424b-b733-b0fdd726df69 }} and related hosts.',
58+
timestamp: '2025-06-10T12:34:53.366Z',
59+
title: 'Credential Access and Data Exfiltration',
60+
},
3261
];
Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import { estypes } from '@elastic/elasticsearch';
9+
import { elasticsearchServiceMock } from '@kbn/core/server/mocks';
10+
import { loggerMock } from '@kbn/logging-mocks';
11+
12+
import { deduplicateAttackDiscoveries } from '.';
13+
import { mockAttackDiscoveries } from '../../evaluation/__mocks__/mock_attack_discoveries';
14+
import { generateAttackDiscoveryAlertHash } from '../transforms/transform_to_alert_documents';
15+
16+
jest.mock('../transforms/transform_to_alert_documents', () => ({
17+
...jest.requireActual('../transforms/transform_to_alert_documents'),
18+
generateAttackDiscoveryAlertHash: jest.fn(),
19+
}));
20+
21+
const mockEsClient = elasticsearchServiceMock.createElasticsearchClient();
22+
const mockLogger = loggerMock.create();
23+
24+
describe('deduplicateAttackDiscoveries', () => {
25+
const uuid1 = 'test-uuid-1';
26+
const uuid2 = 'test-uuid-2';
27+
const [attack1, attack2] = mockAttackDiscoveries;
28+
const defaultProps = {
29+
attackDiscoveries: mockAttackDiscoveries,
30+
connectorId: 'test-connector-1',
31+
esClient: mockEsClient,
32+
indexPattern: '.test.alerts-*,.adhoc.alerts-*',
33+
logger: mockLogger,
34+
ownerId: 'test-owner-1',
35+
spaceId: 'test-space',
36+
};
37+
38+
beforeEach(() => {
39+
jest.clearAllMocks();
40+
mockEsClient.search.mockResponse({ hits: { hits: [] } } as unknown as estypes.SearchResponse);
41+
(generateAttackDiscoveryAlertHash as jest.Mock).mockImplementation(({ attackDiscovery }) => {
42+
if (attackDiscovery === attack1) return uuid1;
43+
if (attackDiscovery === attack2) return uuid2;
44+
return 'unknown-uuid';
45+
});
46+
});
47+
48+
it('should return empty array if no attack discoveries passed to the function', async () => {
49+
const result = await deduplicateAttackDiscoveries({ ...defaultProps, attackDiscoveries: [] });
50+
51+
expect(result).toEqual([]);
52+
});
53+
54+
it('should return all discoveries if none are duplicates', async () => {
55+
const result = await deduplicateAttackDiscoveries(defaultProps);
56+
expect(result).toEqual(mockAttackDiscoveries);
57+
});
58+
59+
it('should filter out all discoveries if all are duplicates', async () => {
60+
mockEsClient.search.mockResponse({
61+
hits: {
62+
hits: [
63+
{ _source: { 'kibana.alert.instance.id': uuid1 } },
64+
{ _source: { 'kibana.alert.instance.id': uuid2 } },
65+
],
66+
},
67+
} as unknown as estypes.SearchResponse);
68+
const result = await deduplicateAttackDiscoveries(defaultProps);
69+
expect(result).toEqual([]);
70+
});
71+
72+
it('should filter out only duplicates and keep new discoveries', async () => {
73+
mockEsClient.search.mockResponse({
74+
hits: {
75+
hits: [{ _source: { 'kibana.alert.instance.id': uuid2 } }],
76+
},
77+
} as unknown as estypes.SearchResponse);
78+
const result = await deduplicateAttackDiscoveries({
79+
...defaultProps,
80+
attackDiscoveries: [attack1, attack2],
81+
});
82+
expect(result).toEqual([attack1]);
83+
});
84+
85+
it('should handle ES hits with missing _source gracefully', async () => {
86+
mockEsClient.search.mockResponse({
87+
hits: {
88+
hits: [{ _id: 'foo' }],
89+
},
90+
} as unknown as estypes.SearchResponse);
91+
const result = await deduplicateAttackDiscoveries({
92+
...defaultProps,
93+
attackDiscoveries: [attack1],
94+
});
95+
expect(result).toEqual([attack1]);
96+
});
97+
98+
it('should log when duplicates are found', async () => {
99+
mockEsClient.search.mockResponse({
100+
hits: {
101+
hits: [{ _source: { 'kibana.alert.instance.id': uuid1 } }],
102+
},
103+
} as unknown as estypes.SearchResponse);
104+
await deduplicateAttackDiscoveries(defaultProps);
105+
expect(mockLogger.info).toHaveBeenCalledWith(
106+
'Found 1 duplicate alert(s), skipping report for those.'
107+
);
108+
});
109+
});
Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import { ElasticsearchClient, Logger } from '@kbn/core/server';
9+
import { ALERT_INSTANCE_ID } from '@kbn/rule-data-utils';
10+
import { AttackDiscoveries } from '@kbn/elastic-assistant-common';
11+
12+
import { AttackDiscoveryAlertDocument } from '../../schedules/types';
13+
import { generateAttackDiscoveryAlertHash } from '../transforms/transform_to_alert_documents';
14+
15+
interface DeduplicateAttackDiscoveriesParams {
16+
attackDiscoveries: AttackDiscoveries;
17+
connectorId: string;
18+
esClient: ElasticsearchClient;
19+
indexPattern: string;
20+
logger: Logger;
21+
ownerId: string;
22+
spaceId: string;
23+
}
24+
25+
export const deduplicateAttackDiscoveries = async ({
26+
attackDiscoveries,
27+
connectorId,
28+
esClient,
29+
indexPattern,
30+
logger,
31+
ownerId,
32+
spaceId,
33+
}: DeduplicateAttackDiscoveriesParams): Promise<AttackDiscoveries> => {
34+
if (!attackDiscoveries || attackDiscoveries.length === 0) {
35+
return attackDiscoveries;
36+
}
37+
38+
// 1. Transform all attackDiscoveries to alert documents and collect alertUuids
39+
const alertDocs = attackDiscoveries.map((attack) => {
40+
const alertHash = generateAttackDiscoveryAlertHash({
41+
attackDiscovery: attack,
42+
connectorId,
43+
ownerId,
44+
spaceId,
45+
});
46+
return { attack, alertHash };
47+
});
48+
const alertUuids = alertDocs.map((doc) => doc.alertHash);
49+
50+
// 2. Search for existing alerts in ES
51+
const searchResult = await esClient.search<AttackDiscoveryAlertDocument>({
52+
index: indexPattern,
53+
size: alertUuids.length,
54+
query: { bool: { must: [{ terms: { [ALERT_INSTANCE_ID]: alertUuids } }] } },
55+
ignore_unavailable: true,
56+
});
57+
58+
// 3. Collect found alert IDs
59+
const foundIds = new Set(
60+
searchResult.hits.hits.map((hit) => hit._source && hit._source[ALERT_INSTANCE_ID])
61+
);
62+
63+
// 4. Filter out duplicates
64+
const newDiscoveries = alertDocs
65+
.filter((doc) => !foundIds.has(doc.alertHash))
66+
.map((doc) => doc.attack);
67+
68+
const numDuplicates = attackDiscoveries.length - newDiscoveries.length;
69+
if (numDuplicates > 0) {
70+
logger.info(`Found ${numDuplicates} duplicate alert(s), skipping report for those.`);
71+
logger.debug(() => `Duplicated alerts:\n ${JSON.stringify([...foundIds].sort(), null, 2)}`);
72+
}
73+
74+
return newDiscoveries;
75+
};

x-pack/solutions/security/plugins/elastic_assistant/server/lib/attack_discovery/persistence/get_scheduled_and_ad_hoc_index_pattern/index.test.ts

Lines changed: 0 additions & 44 deletions
This file was deleted.

x-pack/solutions/security/plugins/elastic_assistant/server/lib/attack_discovery/persistence/get_scheduled_and_ad_hoc_index_pattern/index.ts

Lines changed: 0 additions & 20 deletions
This file was deleted.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import { ATTACK_DISCOVERY_ALERTS_COMMON_INDEX_PREFIX } from '@kbn/elastic-assistant-common';
9+
10+
import { getScheduledIndexPattern } from '.';
11+
12+
describe('getScheduledIndexPattern', () => {
13+
it('returns the expected scheduled index pattern for a given spaceId', () => {
14+
const spaceId = 'test-space';
15+
const result = getScheduledIndexPattern(spaceId);
16+
expect(result).toBe(`${ATTACK_DISCOVERY_ALERTS_COMMON_INDEX_PREFIX}-test-space`);
17+
});
18+
19+
it('returns the expected pattern for empty spaceId', () => {
20+
const result = getScheduledIndexPattern('');
21+
expect(result).toBe(`${ATTACK_DISCOVERY_ALERTS_COMMON_INDEX_PREFIX}-`);
22+
});
23+
});
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import { ATTACK_DISCOVERY_ALERTS_COMMON_INDEX_PREFIX } from '@kbn/elastic-assistant-common';
9+
10+
export const getScheduledIndexPattern = (spaceId: string): string => {
11+
return `${ATTACK_DISCOVERY_ALERTS_COMMON_INDEX_PREFIX}-${spaceId}`;
12+
};

x-pack/solutions/security/plugins/elastic_assistant/server/lib/attack_discovery/persistence/index.ts

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ import { getCombinedFilter } from './get_combined_filter';
3838
import { getFindAttackDiscoveryAlertsAggregation } from './get_find_attack_discovery_alerts_aggregation';
3939
import { AttackDiscoveryAlertDocument } from '../schedules/types';
4040
import { transformSearchResponseToAlerts } from './transforms/transform_search_response_to_alerts';
41-
import { getScheduledAndAdHocIndexPattern } from './get_scheduled_and_ad_hoc_index_pattern';
41+
import { getScheduledIndexPattern } from './get_scheduled_index_pattern';
4242
import { getUpdateAttackDiscoveryAlertsQuery } from '../get_update_attack_discovery_alerts_query';
4343

4444
const FIRST_PAGE = 1; // CAUTION: sever-side API uses a 1-based page index convention (for consistency with similar existing APIs)
@@ -106,6 +106,20 @@ export class AttackDiscoveryDataClient extends AIAssistantDataClient {
106106
});
107107
};
108108

109+
public getAdHocAlertsIndexPattern = () => {
110+
if (this.adhocAttackDiscoveryDataClient === undefined) {
111+
throw new Error('`adhocAttackDiscoveryDataClient` is required');
112+
}
113+
return this.adhocAttackDiscoveryDataClient.indexNameWithNamespace(this.spaceId);
114+
};
115+
116+
public getScheduledAndAdHocIndexPattern = () => {
117+
return [
118+
getScheduledIndexPattern(this.spaceId), // scheduled
119+
this.getAdHocAlertsIndexPattern(), // ad-hoc
120+
].join(',');
121+
};
122+
109123
public createAttackDiscoveryAlerts = async ({
110124
authenticatedUser,
111125
createAttackDiscoveryAlertsParams,
@@ -211,8 +225,7 @@ export class AttackDiscoveryDataClient extends AIAssistantDataClient {
211225
perPage = DEFAULT_PER_PAGE,
212226
} = findAttackDiscoveryAlertsParams;
213227

214-
const spaceId = this.spaceId;
215-
const index = getScheduledAndAdHocIndexPattern(spaceId, this.adhocAttackDiscoveryDataClient);
228+
const index = this.getScheduledAndAdHocIndexPattern();
216229

217230
const filter = combineFindAttackDiscoveryFilters({
218231
alertIds,
@@ -336,10 +349,7 @@ export class AttackDiscoveryDataClient extends AIAssistantDataClient {
336349

337350
const esClient = await this.options.elasticsearchClientPromise;
338351

339-
const indexPattern = getScheduledAndAdHocIndexPattern(
340-
this.spaceId,
341-
this.adhocAttackDiscoveryDataClient
342-
);
352+
const indexPattern = this.getScheduledAndAdHocIndexPattern();
343353

344354
if (ids.length === 0) {
345355
logger.debug(

0 commit comments

Comments
 (0)