From fd89059394a16da7f66d75257b7e077c43bca8df Mon Sep 17 00:00:00 2001 From: Davis Plumlee Date: Thu, 11 Sep 2025 11:57:45 -0400 Subject: [PATCH 01/10] adds field fixes tests runs make updates tests cleans up types fix tests updates tests updates snapshots fixes tests fixes tests fixes tests fixes tests addresses comments language updates adds logic guard for missing fields updates new tests addresses comments [CI] Auto-commit changed files from 'make api-docs' address non-test comments fixes tests --- oas_docs/output/kibana.serverless.yaml | 20 +++ oas_docs/output/kibana.yaml | 20 +++ .../rule_schema/common_attributes.gen.ts | 21 +++ .../rule_schema/common_attributes.schema.yaml | 22 +++ .../rule_schema/rule_response_schema.test.ts | 2 + .../import_rules/rule_to_import.test.ts | 2 + ...ections_api_2023_10_31.bundled.schema.yaml | 26 ++++ ...ections_api_2023_10_31.bundled.schema.yaml | 26 ++++ .../rule_details/json_diff/json_diff.test.tsx | 7 +- .../routes/__mocks__/utils.ts | 2 +- .../logic/actions/duplicate_rule.test.ts | 2 + .../logic/bulk_actions/bulk_edit_rules.ts | 25 ++-- .../common_params_camel_to_snake.test.ts | 4 + .../common_params_camel_to_snake.ts | 9 -- ...rt_prebuilt_rule_asset_to_rule_response.ts | 6 +- .../internal_rule_to_api_response.ts | 6 +- .../converters/normalize_rule_params.test.ts | 23 +++- .../converters/normalize_rule_params.ts | 113 +++++++++++----- ...detection_rules_client.import_rule.test.ts | 8 ++ .../mergers/apply_rule_defaults.ts | 25 ++-- ...d.ts => calculate_external_rule_source.ts} | 63 +++++++-- .../rule_source/calculate_rule_source.test.ts | 57 +++++++- .../rule_source/calculate_rule_source.ts | 14 +- .../create_default_external_rule_source.ts | 15 +++ .../create_default_internal_rule_source.ts | 12 ++ .../logic/export/get_export_all.test.ts | 4 +- .../export/get_export_by_object_ids.test.ts | 7 +- .../calculate_rule_source_for_import.test.ts | 33 ++++- .../calculate_rule_source_for_import.ts | 9 +- .../import_export/export_prebuilt_rules.ts | 34 ++++- .../import_multiple_prebuilt_rules.ts | 42 +++++- .../import_outdated_prebuilt_rules.ts | 126 ++++++++++++++++-- .../import_single_prebuilt_rule.ts | 48 +++++++ .../import_with_installing_package.ts | 13 +- .../import_with_missing_base_version.ts | 22 +++ .../import_with_missing_fields.ts | 8 ++ .../get_prebuilt_rule_base_version.ts | 2 + .../revert_prebuilt_rules.ts | 10 ++ .../customize_via_bulk_editing.ts | 2 +- .../detect_customization_with_base_version.ts | 2 + ...tect_customization_without_base_version.ts | 40 ++++++ .../customization/unaffected_fields.ts | 2 + 42 files changed, 791 insertions(+), 143 deletions(-) rename x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/{calculate_is_customized.ts => calculate_external_rule_source.ts} (52%) create mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/create_default_external_rule_source.ts create mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/create_default_internal_rule_source.ts diff --git a/oas_docs/output/kibana.serverless.yaml b/oas_docs/output/kibana.serverless.yaml index 888fbfdb92cfe..ef3135a589297 100644 --- a/oas_docs/output/kibana.serverless.yaml +++ b/oas_docs/output/kibana.serverless.yaml @@ -65425,10 +65425,28 @@ components: - endpoint_host_isolation_exceptions - endpoint_blocklists type: string + Security_Detections_API_ExternalRuleCustomizedFields: + description: An array of customized field names — that is, fields that the user has modified from their base value. Defaults to an empty array. + items: + type: object + properties: + field_name: + description: Name of a user-modified field in the rule object. + type: string + required: + - field_name + type: array + Security_Detections_API_ExternalRuleHasBaseVersion: + description: Determines whether an external/prebuilt rule has its original, unmodified version present when the calculation of its customization status is performed (`rule_source.is_customized` and `rule_source.customized_fields`). + type: boolean Security_Detections_API_ExternalRuleSource: description: Type of rule source for externally sourced rules, i.e. rules that have an external source, such as the Elastic Prebuilt rules repo. type: object properties: + customized_fields: + $ref: '#/components/schemas/Security_Detections_API_ExternalRuleCustomizedFields' + has_base_version: + $ref: '#/components/schemas/Security_Detections_API_ExternalRuleHasBaseVersion' is_customized: $ref: '#/components/schemas/Security_Detections_API_IsExternalRuleCustomized' type: @@ -65438,6 +65456,8 @@ components: required: - type - is_customized + - has_base_version + - customized_fields Security_Detections_API_FindRulesSortField: enum: - created_at diff --git a/oas_docs/output/kibana.yaml b/oas_docs/output/kibana.yaml index d2cefac8019a6..84fefd5f30ec9 100644 --- a/oas_docs/output/kibana.yaml +++ b/oas_docs/output/kibana.yaml @@ -78359,10 +78359,28 @@ components: - endpoint_host_isolation_exceptions - endpoint_blocklists type: string + Security_Detections_API_ExternalRuleCustomizedFields: + description: An array of customized field names — that is, fields that the user has modified from their base value. Defaults to an empty array. + items: + type: object + properties: + field_name: + description: Name of a user-modified field in the rule object. + type: string + required: + - field_name + type: array + Security_Detections_API_ExternalRuleHasBaseVersion: + description: Determines whether an external/prebuilt rule has its original, unmodified version present when the calculation of its customization status is performed (`rule_source.is_customized` and `rule_source.customized_fields`). + type: boolean Security_Detections_API_ExternalRuleSource: description: Type of rule source for externally sourced rules, i.e. rules that have an external source, such as the Elastic Prebuilt rules repo. type: object properties: + customized_fields: + $ref: '#/components/schemas/Security_Detections_API_ExternalRuleCustomizedFields' + has_base_version: + $ref: '#/components/schemas/Security_Detections_API_ExternalRuleHasBaseVersion' is_customized: $ref: '#/components/schemas/Security_Detections_API_IsExternalRuleCustomized' type: @@ -78372,6 +78390,8 @@ components: required: - type - is_customized + - has_base_version + - customized_fields Security_Detections_API_FindRulesSortField: enum: - created_at diff --git a/x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.gen.ts b/x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.gen.ts index d1e85c5b8e807..14e546ede0cb8 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.gen.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.gen.ts @@ -89,6 +89,25 @@ export const IsRuleImmutable = z.boolean(); export type IsExternalRuleCustomized = z.infer; export const IsExternalRuleCustomized = z.boolean(); +/** + * Determines whether an external/prebuilt rule has its original, unmodified version present when the calculation of its customization status is performed (`rule_source.is_customized` and `rule_source.customized_fields`). + */ +export type ExternalRuleHasBaseVersion = z.infer; +export const ExternalRuleHasBaseVersion = z.boolean(); + +/** + * An array of customized field names — that is, fields that the user has modified from their base value. Defaults to an empty array. + */ +export type ExternalRuleCustomizedFields = z.infer; +export const ExternalRuleCustomizedFields = z.array( + z.object({ + /** + * Name of a user-modified field in the rule object. + */ + field_name: z.string(), + }) +); + /** * Type of rule source for internally sourced rules, i.e. created within the Kibana apps. */ @@ -104,6 +123,8 @@ export type ExternalRuleSource = z.infer; export const ExternalRuleSource = z.object({ type: z.literal('external'), is_customized: IsExternalRuleCustomized, + has_base_version: ExternalRuleHasBaseVersion, + customized_fields: ExternalRuleCustomizedFields, }); /** diff --git a/x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml b/x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml index 3dc6e495373b3..4a1a8f258f465 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml +++ b/x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml @@ -70,6 +70,22 @@ components: type: boolean description: Determines whether an external/prebuilt rule has been customized by the user (i.e. any of its fields have been modified and diverged from the base value). + ExternalRuleHasBaseVersion: + type: boolean + description: Determines whether an external/prebuilt rule has its original, unmodified version present when the calculation of its customization status is performed (`rule_source.is_customized` and `rule_source.customized_fields`). + + ExternalRuleCustomizedFields: + type: array + description: An array of customized field names — that is, fields that the user has modified from their base value. Defaults to an empty array. + items: + type: object + properties: + field_name: + type: string + description: Name of a user-modified field in the rule object. + required: + - field_name + InternalRuleSource: description: Type of rule source for internally sourced rules, i.e. created within the Kibana apps. type: object @@ -91,9 +107,15 @@ components: - external is_customized: $ref: '#/components/schemas/IsExternalRuleCustomized' + has_base_version: + $ref: '#/components/schemas/ExternalRuleHasBaseVersion' + customized_fields: + $ref: '#/components/schemas/ExternalRuleCustomizedFields' required: - type - is_customized + - has_base_version + - customized_fields RuleSource: description: Discriminated union that determines whether the rule is internally sourced (created within the Kibana app) or has an external source, such as the Elastic Prebuilt rules repo. diff --git a/x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_response_schema.test.ts b/x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_response_schema.test.ts index 9546ab3a59b09..82d43546aa081 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_response_schema.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_response_schema.test.ts @@ -259,6 +259,8 @@ describe('rule_source', () => { payload.rule_source = { type: 'external', is_customized: true, + customized_fields: [{ field_name: 'name' }], + has_base_version: true, }; const result = RuleResponse.safeParse(payload); diff --git a/x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.test.ts b/x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.test.ts index e945683fc5019..c4eefa0d794a9 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.test.ts @@ -1056,6 +1056,8 @@ describe('RuleToImport', () => { rule_source: { type: 'external', is_customized: true, + customized_fields: [{ field_name: 'name' }], + has_base_version: true, }, }); diff --git a/x-pack/solutions/security/plugins/security_solution/docs/openapi/ess/security_solution_detections_api_2023_10_31.bundled.schema.yaml b/x-pack/solutions/security/plugins/security_solution/docs/openapi/ess/security_solution_detections_api_2023_10_31.bundled.schema.yaml index daf42d66c1a9f..dc40c61612511 100644 --- a/x-pack/solutions/security/plugins/security_solution/docs/openapi/ess/security_solution_detections_api_2023_10_31.bundled.schema.yaml +++ b/x-pack/solutions/security/plugins/security_solution/docs/openapi/ess/security_solution_detections_api_2023_10_31.bundled.schema.yaml @@ -6315,12 +6315,36 @@ components: - endpoint_host_isolation_exceptions - endpoint_blocklists type: string + ExternalRuleCustomizedFields: + description: >- + An array of customized field names — that is, fields that the user has + modified from their base value. Defaults to an empty array. + items: + type: object + properties: + field_name: + description: Name of a user-modified field in the rule object. + type: string + required: + - field_name + type: array + ExternalRuleHasBaseVersion: + description: >- + Determines whether an external/prebuilt rule has its original, + unmodified version present when the calculation of its customization + status is performed (`rule_source.is_customized` and + `rule_source.customized_fields`). + type: boolean ExternalRuleSource: description: >- Type of rule source for externally sourced rules, i.e. rules that have an external source, such as the Elastic Prebuilt rules repo. type: object properties: + customized_fields: + $ref: '#/components/schemas/ExternalRuleCustomizedFields' + has_base_version: + $ref: '#/components/schemas/ExternalRuleHasBaseVersion' is_customized: $ref: '#/components/schemas/IsExternalRuleCustomized' type: @@ -6330,6 +6354,8 @@ components: required: - type - is_customized + - has_base_version + - customized_fields FindRulesSortField: enum: - created_at diff --git a/x-pack/solutions/security/plugins/security_solution/docs/openapi/serverless/security_solution_detections_api_2023_10_31.bundled.schema.yaml b/x-pack/solutions/security/plugins/security_solution/docs/openapi/serverless/security_solution_detections_api_2023_10_31.bundled.schema.yaml index 1d7cce42adea1..3405ffd56ce02 100644 --- a/x-pack/solutions/security/plugins/security_solution/docs/openapi/serverless/security_solution_detections_api_2023_10_31.bundled.schema.yaml +++ b/x-pack/solutions/security/plugins/security_solution/docs/openapi/serverless/security_solution_detections_api_2023_10_31.bundled.schema.yaml @@ -5645,12 +5645,36 @@ components: - endpoint_host_isolation_exceptions - endpoint_blocklists type: string + ExternalRuleCustomizedFields: + description: >- + An array of customized field names — that is, fields that the user has + modified from their base value. Defaults to an empty array. + items: + type: object + properties: + field_name: + description: Name of a user-modified field in the rule object. + type: string + required: + - field_name + type: array + ExternalRuleHasBaseVersion: + description: >- + Determines whether an external/prebuilt rule has its original, + unmodified version present when the calculation of its customization + status is performed (`rule_source.is_customized` and + `rule_source.customized_fields`). + type: boolean ExternalRuleSource: description: >- Type of rule source for externally sourced rules, i.e. rules that have an external source, such as the Elastic Prebuilt rules repo. type: object properties: + customized_fields: + $ref: '#/components/schemas/ExternalRuleCustomizedFields' + has_base_version: + $ref: '#/components/schemas/ExternalRuleHasBaseVersion' is_customized: $ref: '#/components/schemas/IsExternalRuleCustomized' type: @@ -5660,6 +5684,8 @@ components: required: - type - is_customized + - has_base_version + - customized_fields FindRulesSortField: enum: - created_at diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/json_diff/json_diff.test.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/json_diff/json_diff.test.tsx index 0fbf2fc659384..99aa646f1aabb 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/json_diff/json_diff.test.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management/components/rule_details/json_diff/json_diff.test.tsx @@ -226,7 +226,12 @@ describe('Rule upgrade workflow: viewing rule changes in JSON diff view', () => created_by: 'mockUserOne', updated_at: '02/02/2024T00:00:001z', updated_by: 'mockUserThree', - rule_source: { type: 'external', is_customized: true }, + rule_source: { + type: 'external', + is_customized: true, + customized_fields: [{ field_name: 'name' }], + has_base_version: true, + }, }; renderRuleDiffComponent({ oldRule, newRule }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/utils.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/utils.ts index 687bf91655e2a..3d4a8105dc151 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/utils.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/utils.ts @@ -81,7 +81,7 @@ export const getOutputRuleAlertForRest = (): RuleResponse => ({ }, ], meta: { - someMeta: 'someField', + some_meta: 'someField', }, timeline_id: 'some-timeline-id', timeline_title: 'some-timeline-title', diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/actions/duplicate_rule.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/actions/duplicate_rule.test.ts index 2737c4f2d8085..7a103021447b9 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/actions/duplicate_rule.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/actions/duplicate_rule.test.ts @@ -158,6 +158,8 @@ describe('duplicateRule', () => { rule.params.ruleSource = { type: 'external', isCustomized: false, + customizedFields: [], + hasBaseVersion: true, }; return rule; }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/bulk_actions/bulk_edit_rules.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/bulk_actions/bulk_edit_rules.ts index 0214649fbea1d..7ea786cb79258 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/bulk_actions/bulk_edit_rules.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/bulk_actions/bulk_edit_rules.ts @@ -8,6 +8,7 @@ import type { ActionsClient } from '@kbn/actions-plugin/server'; import type { RulesClient } from '@kbn/alerting-plugin/server'; +import { convertObjectKeysToCamelCase } from '../../../../../utils/object_case_converters'; import type { BulkActionEditPayload } from '../../../../../../common/api/detection_engine/rule_management'; import type { MlAuthz } from '../../../../machine_learning/authz'; @@ -16,13 +17,14 @@ import type { RuleAlertType, RuleParams } from '../../../rule_schema'; import type { IPrebuiltRuleAssetsClient } from '../../../prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'; import { convertAlertingRuleToRuleResponse } from '../detection_rules_client/converters/convert_alerting_rule_to_rule_response'; -import { calculateIsCustomized } from '../detection_rules_client/mergers/rule_source/calculate_is_customized'; +import { calculateExternalRuleSource } from '../detection_rules_client/mergers/rule_source/calculate_external_rule_source'; import { bulkEditActionToRulesClientOperation } from './action_to_rules_client_operation'; import { ruleParamsModifier } from './rule_params_modifier'; import { splitBulkEditActions } from './split_bulk_edit_actions'; import { validateBulkEditRule } from './validations'; import type { PrebuiltRulesCustomizationStatus } from '../../../../../../common/detection_engine/prebuilt_rules/prebuilt_rule_customization_status'; import { invariant } from '../../../../../../common/utils/invariant'; +import { createDefaultInternalRuleSource } from '../detection_rules_client/mergers/rule_source/create_default_internal_rule_source'; export interface BulkEditRulesArguments { actionsClient: ActionsClient; @@ -99,25 +101,18 @@ export const bulkEditRules = async ({ params: modifiedParams, }); - let isCustomized = false; if (nextRule.immutable === true) { - isCustomized = calculateIsCustomized({ - baseRule: baseVersionsMap.get(nextRule.rule_id), + const baseRule = baseVersionsMap.get(nextRule.rule_id); + const ruleSource = calculateExternalRuleSource({ + baseRule, currentRule: convertAlertingRuleToRuleResponse(currentRule), nextRule, }); - } - const ruleSource = - nextRule.immutable === true - ? { - type: 'external' as const, - isCustomized, - } - : { - type: 'internal' as const, - }; - modifiedParams.ruleSource = ruleSource; + modifiedParams.ruleSource = convertObjectKeysToCamelCase(ruleSource); + } else { + modifiedParams.ruleSource = createDefaultInternalRuleSource(); + } return { modifiedParams, isParamsUpdateSkipped }; }, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/common_params_camel_to_snake.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/common_params_camel_to_snake.test.ts index ad4fd20243850..615c574cf99e3 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/common_params_camel_to_snake.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/common_params_camel_to_snake.test.ts @@ -15,6 +15,8 @@ describe('commonParamsCamelToSnake', () => { ruleSource: { type: 'external', isCustomized: false, + customizedFields: [], + hasBaseVersion: true, }, }); expect(transformedParams).toEqual( @@ -22,6 +24,8 @@ describe('commonParamsCamelToSnake', () => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }) ); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/common_params_camel_to_snake.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/common_params_camel_to_snake.ts index 890f8a6bad7ff..d7fc88eedd296 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/common_params_camel_to_snake.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/common_params_camel_to_snake.ts @@ -5,12 +5,10 @@ * 2.0. */ -import snakecaseKeys from 'snakecase-keys'; import { transformAlertToRuleResponseAction } from '../../../../../../../common/detection_engine/transform_actions'; import { convertObjectKeysToSnakeCase } from '../../../../../../utils/object_case_converters'; import type { BaseRuleParams } from '../../../../rule_schema'; import { migrateLegacyInvestigationFields } from '../../../utils/utils'; -import type { NormalizedRuleParams } from './normalize_rule_params'; /** * @deprecated Use convertObjectKeysToSnakeCase instead @@ -52,10 +50,3 @@ export const commonParamsCamelToSnake = (params: BaseRuleParams) => { setup: params.setup ?? '', }; }; - -export const normalizedCommonParamsCamelToSnake = (params: NormalizedRuleParams) => { - return { - ...commonParamsCamelToSnake(params), - rule_source: snakecaseKeys(params.ruleSource, { deep: true }), - }; -}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/convert_prebuilt_rule_asset_to_rule_response.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/convert_prebuilt_rule_asset_to_rule_response.ts index 1e87721557214..dc38e4cbbce3b 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/convert_prebuilt_rule_asset_to_rule_response.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/convert_prebuilt_rule_asset_to_rule_response.ts @@ -9,6 +9,7 @@ import { v4 as uuidv4 } from 'uuid'; import { RuleResponse } from '../../../../../../../common/api/detection_engine/model/rule_schema'; import type { PrebuiltRuleAsset } from '../../../../prebuilt_rules'; import { applyRuleDefaults } from '../mergers/apply_rule_defaults'; +import { createDefaultExternalRuleSource } from '../mergers/rule_source/create_default_external_rule_source'; export const convertPrebuiltRuleAssetToRuleResponse = ( prebuiltRuleAsset: PrebuiltRuleAsset @@ -22,10 +23,7 @@ export const convertPrebuiltRuleAssetToRuleResponse = ( created_at: new Date().toISOString(), created_by: '', immutable, - rule_source: { - type: 'external', - is_customized: false, - }, + rule_source: createDefaultExternalRuleSource(), revision: 1, }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/internal_rule_to_api_response.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/internal_rule_to_api_response.ts index 5c53eb2c951a1..04d85fcaee414 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/internal_rule_to_api_response.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/internal_rule_to_api_response.ts @@ -19,7 +19,7 @@ import { transformToActionFrequency, } from '../../../normalization/rule_actions'; import { typeSpecificCamelToSnake } from './type_specific_camel_to_snake'; -import { normalizedCommonParamsCamelToSnake } from './common_params_camel_to_snake'; +import { convertObjectKeysToSnakeCase } from '../../../../../../utils/object_case_converters'; import { normalizeRuleParams } from './normalize_rule_params'; export const internalRuleToAPIResponse = ( @@ -39,7 +39,7 @@ export const internalRuleToAPIResponse = ( const transformedAction = transformAlertToRuleSystemAction(action); return transformedAction; }); - const normalizedRuleParams = normalizeRuleParams(rule.params); + const normalizedRuleParams = convertObjectKeysToSnakeCase(normalizeRuleParams(rule.params)); return { // saved object properties @@ -58,7 +58,7 @@ export const internalRuleToAPIResponse = ( enabled: rule.enabled, revision: rule.revision, // Security solution shared rule params - ...normalizedCommonParamsCamelToSnake(normalizedRuleParams), + ...normalizedRuleParams, // Type specific security solution rule params ...typeSpecificCamelToSnake(rule.params), // Actions diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.test.ts index 8398bab4253f8..b684b80727dd0 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.test.ts @@ -19,7 +19,7 @@ describe('normalizeRuleSource', () => { }); }); - it('should return rule_source of type `external` and `isCustomized: false` when immutable is true and ruleSource is undefined', () => { + it('should return rule_source of type `external` and default customization field values when immutable is true and ruleSource is undefined', () => { const result = normalizeRuleSource({ immutable: true, ruleSource: undefined, @@ -27,6 +27,8 @@ describe('normalizeRuleSource', () => { expect(result).toEqual({ type: 'external', isCustomized: false, + customizedFields: [], + hasBaseVersion: true, }); }); @@ -34,11 +36,15 @@ describe('normalizeRuleSource', () => { const externalRuleSource: BaseRuleParams['ruleSource'] = { type: 'external', isCustomized: true, + customizedFields: [{ fieldName: 'name' }], + hasBaseVersion: true, }; const externalResult = normalizeRuleSource({ immutable: true, ruleSource: externalRuleSource }); expect(externalResult).toEqual({ type: externalRuleSource.type, isCustomized: externalRuleSource.isCustomized, + customizedFields: externalRuleSource.customizedFields, + hasBaseVersion: externalRuleSource.hasBaseVersion, }); const internalRuleSource: BaseRuleParams['ruleSource'] = { @@ -52,6 +58,21 @@ describe('normalizeRuleSource', () => { type: internalRuleSource.type, }); }); + + it('should return ruleSource with default customization field values when they are missing in the existing ruleSource object', () => { + // We are purposefully setting this to a value that omits fields + const externalRuleSource: BaseRuleParams['ruleSource'] = { + type: 'external', + isCustomized: false, + }; + const externalResult = normalizeRuleSource({ immutable: true, ruleSource: externalRuleSource }); + expect(externalResult).toEqual({ + type: 'external', + isCustomized: false, + customizedFields: [], + hasBaseVersion: true, + }); + }); }); describe('normalizeRuleParams', () => { diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.ts index 7917bc0a10b22..73ec606900ce5 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.ts @@ -4,44 +4,23 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import type { RequiredOptional } from '@kbn/zod-helpers'; +import { transformAlertToRuleResponseAction } from '../../../../../../../common/detection_engine/transform_actions'; +import { convertObjectKeysToCamelCase } from '../../../../../../utils/object_case_converters'; import type { BaseRuleParams, RuleSourceCamelCased } from '../../../../rule_schema'; import { migrateLegacyInvestigationFields } from '../../../utils/utils'; +import { createDefaultExternalRuleSource } from '../mergers/rule_source/create_default_external_rule_source'; + +export interface NormalizedRuleParams extends RequiredOptional { + ruleSource: Required; +} interface NormalizeRuleSourceParams { immutable: BaseRuleParams['immutable']; ruleSource: BaseRuleParams['ruleSource']; } -export interface NormalizedRuleParams extends BaseRuleParams { - ruleSource: RuleSourceCamelCased; -} - -/* - * Since there's no mechanism to migrate all rules at the same time, - * we cannot guarantee that the ruleSource params is present in all rules. - * This function will normalize the ruleSource param, creating it if does - * not exist in ES, based on the immutable param. - */ -export const normalizeRuleSource = ({ - immutable, - ruleSource, -}: NormalizeRuleSourceParams): RuleSourceCamelCased => { - if (!ruleSource) { - const normalizedRuleSource: RuleSourceCamelCased = immutable - ? { - type: 'external', - isCustomized: false, - } - : { - type: 'internal', - }; - - return normalizedRuleSource; - } - return ruleSource; -}; - -export const normalizeRuleParams = (params: BaseRuleParams): NormalizedRuleParams => { +export const normalizeRuleParams = (params: BaseRuleParams) => { const investigationFields = migrateLegacyInvestigationFields(params.investigationFields); const ruleSource = normalizeRuleSource({ immutable: params.immutable, @@ -57,5 +36,79 @@ export const normalizeRuleParams = (params: BaseRuleParams): NormalizedRuleParam // Fields to normalize investigationFields, ruleSource, + description: params.description, + risk_score: params.riskScore, + severity: params.severity, + buildingBlockType: params.buildingBlockType, + namespace: params.namespace, + note: params.note, + license: params.license, + outputIndex: params.outputIndex, + timelineId: params.timelineId, + timelineTitle: params.timelineTitle, + meta: params.meta, + ruleNameOverride: params.ruleNameOverride, + timestampOverride: params.timestampOverride, + timestampOverrideFallbackDisabled: params.timestampOverrideFallbackDisabled, + author: params.author, + falsePositives: params.falsePositives, + from: params.from, + rule_id: params.ruleId, + maxSignals: params.maxSignals, + riskScoreMapping: params.riskScoreMapping, + severityMapping: params.severityMapping, + threat: params.threat, + to: params.to, + references: params.references, + version: params.version, + exceptionsList: params.exceptionsList, + immutable: params.immutable, + related_integrations: params.relatedIntegrations ?? [], + response_actions: params.responseActions?.map(transformAlertToRuleResponseAction), }; }; + +/* + * Since there's no mechanism to migrate all rules at the same time, + * we cannot guarantee that the ruleSource params is present in all rules. + * This function will normalize the ruleSource param, creating it if does + * not exist in ES, based on the immutable param. + */ +export const normalizeRuleSource = ({ + immutable, + ruleSource, +}: NormalizeRuleSourceParams): Required => { + if (!ruleSource) { + /** + * The rule source object is not guaranteed to be present in a rule saved object. Those rules + * which were created a long time ago and haven't been updated ever since won't have it. + * However, in our domain model (`RuleResponse`) the rule source object is required - we always + * return it from the rule management API endpoints. That's why when it's missing we normalize + * it based on the legacy `immutable` field which is guaranteed to be always present. + */ + const normalizedRuleSource = immutable + ? convertObjectKeysToCamelCase(createDefaultExternalRuleSource()) + : { + type: 'internal' as const, + }; + + return normalizedRuleSource; + } else if ( + ruleSource.type === 'external' && + (ruleSource.customizedFields === undefined || ruleSource.hasBaseVersion === undefined) + /** + * If rule source exists in the rule object but does not have the new customization-related + * fields (`customizedFields` and `hasBaseVersion`), we normalize them to default values here. + * The new fields are not guaranteed to be present in the rule source object and can be missing + * in old rules which haven't been updated by teh user since a long time ago. However, in our + * domain model (`ExternalRuleSource`) they are required, so we do the normalization. + */ + ) { + return { + ...ruleSource, + customizedFields: [], + hasBaseVersion: true, + }; + } + return ruleSource as Required; +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rule.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rule.test.ts index 5fc03d13f2f50..9d29b104bac44 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rule.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client.import_rule.test.ts @@ -244,6 +244,8 @@ describe('DetectionRulesClient.importRule', () => { rule_source: { type: 'external' as const, is_customized: true, + customized_fields: [{ field_name: 'name' }], + has_base_version: true, }, }, allowMissingConnectorSecrets, @@ -257,6 +259,8 @@ describe('DetectionRulesClient.importRule', () => { ruleSource: { isCustomized: true, type: 'external', + customizedFields: [{ fieldName: 'name' }], + hasBaseVersion: true, }, }), }), @@ -368,6 +372,8 @@ describe('DetectionRulesClient.importRule', () => { rule_source: { type: 'external' as const, is_customized: true, + customized_fields: [{ field_name: 'name' }], + has_base_version: true, }, }, allowMissingConnectorSecrets, @@ -381,6 +387,8 @@ describe('DetectionRulesClient.importRule', () => { ruleSource: { isCustomized: true, type: 'external', + customizedFields: [{ fieldName: 'name' }], + hasBaseVersion: true, }, }), }), diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_defaults.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_defaults.ts index 8c91149bd5fa0..a1b913e4b62de 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_defaults.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_defaults.ts @@ -6,6 +6,10 @@ */ import { v4 as uuidv4 } from 'uuid'; +import { + convertObjectKeysToCamelCase, + convertObjectKeysToSnakeCase, +} from '../../../../../../utils/object_case_converters'; import { addEcsToRequiredFields } from '../../../../../../../common/detection_engine/rule_management/utils'; import type { RuleCreateProps, @@ -21,6 +25,7 @@ import { normalizeThresholdObject, } from '../../../../../../../common/detection_engine/utils'; import { assertUnreachable } from '../../../../../../../common/utility_types'; +import { normalizeRuleSource } from '../converters/normalize_rule_params'; export const RULE_DEFAULTS = { enabled: false, @@ -56,24 +61,16 @@ export function applyRuleDefaults( ...typeSpecificParams, rule_id: rule.rule_id ?? uuidv4(), immutable, - rule_source: rule.rule_source ?? convertImmutableToRuleSource(immutable), + rule_source: convertObjectKeysToSnakeCase( + normalizeRuleSource({ + immutable, + ruleSource: rule.rule_source ? convertObjectKeysToCamelCase(rule.rule_source) : undefined, + }) + ), required_fields: addEcsToRequiredFields(rule.required_fields), }; } -const convertImmutableToRuleSource = (immutable: boolean): RuleSource => { - if (immutable) { - return { - type: 'external', - is_customized: false, - }; - } - - return { - type: 'internal', - }; -}; - export const setTypeSpecificDefaults = (props: TypeSpecificCreateProps) => { switch (props.type) { case 'eql': { diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_is_customized.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_external_rule_source.ts similarity index 52% rename from x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_is_customized.ts rename to x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_external_rule_source.ts index 66b1e09339833..4d91c3ec3f204 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_is_customized.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_external_rule_source.ts @@ -5,27 +5,39 @@ * 2.0. */ -import { calculateRuleFieldsDiff } from '../../../../../prebuilt_rules/logic/diff/calculation/calculate_rule_fields_diff'; -import type { RuleResponse } from '../../../../../../../../common/api/detection_engine'; +import type { + ExternalRuleSource, + RuleResponse, +} from '../../../../../../../../common/api/detection_engine'; import type { PrebuiltRuleAsset } from '../../../../../prebuilt_rules'; +import { calculateRuleFieldsDiff } from '../../../../../prebuilt_rules/logic/diff/calculation/calculate_rule_fields_diff'; import { convertPrebuiltRuleAssetToRuleResponse } from '../../converters/convert_prebuilt_rule_asset_to_rule_response'; -interface CalculateIsCustomizedArgs { +interface CalculateExternalRuleSourceArgs { baseRule: PrebuiltRuleAsset | undefined; nextRule: RuleResponse; // Current rule can be undefined in case of importing a prebuilt rule that is not installed currentRule: RuleResponse | undefined; } -export function calculateIsCustomized({ +export function calculateExternalRuleSource({ baseRule, nextRule, currentRule, -}: CalculateIsCustomizedArgs) { +}: CalculateExternalRuleSourceArgs): ExternalRuleSource { if (baseRule) { // Base version is available, so we can determine the customization status // by comparing the base version with the next version - return areRulesEqual(convertPrebuiltRuleAssetToRuleResponse(baseRule), nextRule) === false; + const customizedFields = getCustomizedFields( + convertPrebuiltRuleAssetToRuleResponse(baseRule), + nextRule + ); + return { + type: 'external', + is_customized: customizedFields.length > 0, + customized_fields: customizedFields, + has_base_version: true, + }; } // Base version is not available, apply a heuristic to determine the // customization status @@ -33,7 +45,12 @@ export function calculateIsCustomized({ if (currentRule == null) { // Current rule is not installed and base rule is not available, so we can't // determine if the rule is customized. Defaulting to false. - return false; + return { + type: 'external', + is_customized: false, + customized_fields: [], + has_base_version: false, + }; } if ( @@ -43,22 +60,40 @@ export function calculateIsCustomized({ // If the rule was previously customized, there's no way to determine // whether the customization remained or was reverted. Keeping it as // customized in this case. - return true; + return { + type: 'external', + is_customized: true, + customized_fields: [], + has_base_version: false, + }; } // If the rule has not been customized before, its customization status can be - // determined by comparing the current version with the next version. - return areRulesEqual(currentRule, nextRule) === false; + // determined by comparing the current version with the next version. But as a + // base version cannot be found, we don't list the customized fields in the object + // as we cannot guarantee the correctness of these fields if the rule was + // customized again. + const customizedFields = getCustomizedFields(currentRule, nextRule); + return { + type: 'external', + is_customized: customizedFields.length > 0, + customized_fields: [], + has_base_version: false, + }; } /** - * A helper function to determine if two rules are equal + * A helper function to retrieve all customized fields between 2 rule versions * * @param ruleA * @param ruleB - * @returns true if all rule fields are equal, false otherwise + * @returns `ExternalRuleCustomizedFields` type with all fields that are different between the two given rules */ -function areRulesEqual(ruleA: RuleResponse, ruleB: RuleResponse) { +function getCustomizedFields(ruleA: RuleResponse, ruleB: RuleResponse) { const fieldsDiff = calculateRuleFieldsDiff({ ruleA, ruleB }); - return Object.values(fieldsDiff).every((field) => field.is_equal === true); + return Object.entries(fieldsDiff) + .filter(([, diff]) => !diff.is_equal) + .map(([key]) => ({ + field_name: key, + })); } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source.test.ts index c2ba2164259f9..944e701b1e8bd 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source.test.ts @@ -60,12 +60,14 @@ describe('calculateRuleSource', () => { const result = await calculateRuleSource({ prebuiltRuleAssetClient, nextRule: rule, - currentRule: rule, + currentRule: undefined, }); expect(result).toEqual( expect.objectContaining({ type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }) ); }); @@ -73,7 +75,7 @@ describe('calculateRuleSource', () => { it('returns is_customized true when the rule is prebuilt and has been customized', async () => { const rule = getSampleRule(); rule.immutable = true; - rule.name = 'Updated name'; + rule.tags = ['Updated tag']; const baseRule = getSampleRuleAsset(); prebuiltRuleAssetClient.fetchAssetsByVersion.mockResolvedValueOnce([baseRule]); @@ -81,12 +83,14 @@ describe('calculateRuleSource', () => { const result = await calculateRuleSource({ prebuiltRuleAssetClient, nextRule: rule, - currentRule: rule, + currentRule: getSampleRule(), }); expect(result).toEqual( expect.objectContaining({ type: 'external', is_customized: true, + customized_fields: [{ field_name: 'tags' }], + has_base_version: true, }) ); }); @@ -104,12 +108,14 @@ describe('calculateRuleSource', () => { const result = await calculateRuleSource({ prebuiltRuleAssetClient, nextRule: rule, - currentRule: rule, + currentRule: getSampleRule(), }); expect(result).toEqual( expect.objectContaining({ type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }) ); }); @@ -131,6 +137,8 @@ describe('calculateRuleSource', () => { expect.objectContaining({ type: 'external', is_customized: false, + customized_fields: [], + has_base_version: false, }) ); }); @@ -141,6 +149,8 @@ describe('calculateRuleSource', () => { rule.rule_source = { type: 'external', is_customized: true, + customized_fields: [], + has_base_version: false, }; // No base version @@ -155,6 +165,8 @@ describe('calculateRuleSource', () => { expect.objectContaining({ type: 'external', is_customized: true, + customized_fields: [], + has_base_version: false, }) ); }); @@ -165,6 +177,8 @@ describe('calculateRuleSource', () => { rule.rule_source = { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: false, }; // No base version @@ -179,6 +193,8 @@ describe('calculateRuleSource', () => { expect.objectContaining({ type: 'external', is_customized: false, + customized_fields: [], + has_base_version: false, }) ); }); @@ -189,6 +205,8 @@ describe('calculateRuleSource', () => { rule.rule_source = { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: false, }; const nextRule = { @@ -208,6 +226,37 @@ describe('calculateRuleSource', () => { expect.objectContaining({ type: 'external', is_customized: true, + customized_fields: [], + has_base_version: false, + }) + ); + }); + + it('returns an empty field array when rule was previously customized with a base version', async () => { + const rule = getSampleRule(); + rule.immutable = true; + rule.tags = ['Updated tag']; + rule.rule_source = { + type: 'external', + is_customized: true, + customized_fields: [{ field_name: 'tags' }], + has_base_version: true, + }; + + // No base version + prebuiltRuleAssetClient.fetchAssetsByVersion.mockResolvedValueOnce([]); + + const result = await calculateRuleSource({ + prebuiltRuleAssetClient, + nextRule: rule, + currentRule: rule, + }); + expect(result).toEqual( + expect.objectContaining({ + type: 'external', + is_customized: true, + customized_fields: [], + has_base_version: false, }) ); }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source.ts index 3b1944ccfe185..1333531897281 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/calculate_rule_source.ts @@ -11,7 +11,8 @@ import type { } from '../../../../../../../../common/api/detection_engine/model/rule_schema'; import type { PrebuiltRuleAsset } from '../../../../../prebuilt_rules'; import type { IPrebuiltRuleAssetsClient } from '../../../../../prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'; -import { calculateIsCustomized } from './calculate_is_customized'; +import { calculateExternalRuleSource } from './calculate_external_rule_source'; +import { createDefaultInternalRuleSource } from './create_default_internal_rule_source'; interface CalculateRuleSourceProps { prebuiltRuleAssetClient: IPrebuiltRuleAssetsClient; @@ -35,19 +36,12 @@ export async function calculateRuleSource({ ]); const baseRule: PrebuiltRuleAsset | undefined = prebuiltRulesResponse.at(0); - const isCustomized = calculateIsCustomized({ + return calculateExternalRuleSource({ baseRule, nextRule, currentRule, }); - - return { - type: 'external', - is_customized: isCustomized, - }; } - return { - type: 'internal', - }; + return createDefaultInternalRuleSource(); } diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/create_default_external_rule_source.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/create_default_external_rule_source.ts new file mode 100644 index 0000000000000..79bd799d2903f --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/create_default_external_rule_source.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ExternalRuleSource } from '../../../../../../../../common/api/detection_engine/model/rule_schema'; + +export const createDefaultExternalRuleSource = (): ExternalRuleSource => ({ + type: 'external', + is_customized: false, + customized_fields: [], + has_base_version: true, +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/create_default_internal_rule_source.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/create_default_internal_rule_source.ts new file mode 100644 index 0000000000000..1881496075dd2 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/create_default_internal_rule_source.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { InternalRuleSource } from '../../../../../../../../common/api/detection_engine/model/rule_schema'; + +export const createDefaultInternalRuleSource = (): InternalRuleSource => ({ + type: 'internal', +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_all.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_all.test.ts index 382df4bfa5ffc..f4f69cce8d990 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_all.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_all.test.ts @@ -121,7 +121,7 @@ describe('getExportAll', () => { setup: '', timeline_id: 'some-timeline-id', timeline_title: 'some-timeline-title', - meta: { someMeta: 'someField' }, + meta: { some_meta: 'someField' }, severity: 'high', severity_mapping: [], updated_by: 'elastic', @@ -303,7 +303,7 @@ describe('getExportAll', () => { setup: '', timeline_id: 'some-timeline-id', timeline_title: 'some-timeline-title', - meta: { someMeta: 'someField' }, + meta: { some_meta: 'someField' }, severity: 'high', severity_mapping: [], updated_by: 'elastic', diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_by_object_ids.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_by_object_ids.test.ts index f39402240d35a..4eead9459df81 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_by_object_ids.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_by_object_ids.test.ts @@ -18,6 +18,8 @@ import { getThreatMock } from '../../../../../../common/detection_engine/schemas import { internalRuleToAPIResponse } from '../detection_rules_client/converters/internal_rule_to_api_response'; import { getEqlRuleParams, getQueryRuleParams } from '../../../rule_schema/mocks'; import { getExportByObjectIds } from './get_export_by_object_ids'; +import { createDefaultExternalRuleSource } from '../detection_rules_client/mergers/rule_source/create_default_external_rule_source'; +import { convertObjectKeysToCamelCase } from '../../../../../utils/object_case_converters'; const exceptionsClient = getExceptionListClientMock(); const connectors = [ @@ -93,10 +95,7 @@ describe('getExportByObjectIds', () => { getQueryRuleParams({ ruleId: 'rule-1', immutable: true, - ruleSource: { - type: 'external', - isCustomized: false, - }, + ruleSource: convertObjectKeysToCamelCase(createDefaultExternalRuleSource()), }) ); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts index af80e0fcafbba..c6657b4c809c7 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.test.ts @@ -41,6 +41,8 @@ describe('calculateRuleSourceForImport', () => { ruleSource: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: false, }, immutable: true, }); @@ -61,6 +63,8 @@ describe('calculateRuleSourceForImport', () => { ruleSource: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: false, }, immutable: true, }); @@ -84,6 +88,8 @@ describe('calculateRuleSourceForImport', () => { ruleSource: { type: 'external', is_customized: true, + customized_fields: [], + has_base_version: false, }, immutable: true, }); @@ -92,7 +98,15 @@ describe('calculateRuleSourceForImport', () => { it('calculates as external with customizations if a matching asset/version is found', () => { const rule = getRulesSchemaMock(); rule.rule_id = 'rule_id'; - const prebuiltRuleAssetsByRuleId = { rule_id: getPrebuiltRuleMock({ rule_id: 'rule_id' }) }; + const prebuiltRuleAssetsByRuleId = { + rule_id: getPrebuiltRuleMock({ + rule_id: 'rule_id', + tags: ['updated tag'], + false_positives: ['new false positive'], + references: ['https://new.reference.co'], + index: ['new-index-pattern'], + }), + }; const result = calculateRuleSourceForImport({ importedRule: rule, @@ -105,6 +119,21 @@ describe('calculateRuleSourceForImport', () => { ruleSource: { type: 'external', is_customized: true, + customized_fields: [ + { + field_name: 'tags', + }, + { + field_name: 'false_positives', + }, + { + field_name: 'references', + }, + { + field_name: 'index', + }, + ], + has_base_version: true, }, immutable: true, }); @@ -126,6 +155,8 @@ describe('calculateRuleSourceForImport', () => { ruleSource: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, immutable: true, }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts index 7f77d749d05be..61bab9f2ac25f 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/import/calculate_rule_source_for_import.ts @@ -11,7 +11,7 @@ import type { ValidatedRuleToImport, } from '../../../../../../common/api/detection_engine'; import type { PrebuiltRuleAsset } from '../../../prebuilt_rules'; -import { calculateIsCustomized } from '../detection_rules_client/mergers/rule_source/calculate_is_customized'; +import { calculateExternalRuleSource } from '../detection_rules_client/mergers/rule_source/calculate_external_rule_source'; import { convertRuleToImportToRuleResponse } from './converters/convert_rule_to_import_to_rule_response'; /** @@ -50,17 +50,14 @@ export const calculateRuleSourceForImport = ({ // satisfy the type system. const nextRule = convertRuleToImportToRuleResponse(importedRule); - const isCustomized = calculateIsCustomized({ + const ruleSource = calculateExternalRuleSource({ baseRule, nextRule, currentRule, }); return { - ruleSource: { - type: 'external', - is_customized: isCustomized, - }, + ruleSource, immutable: true, }; }; diff --git a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/import_export/export_prebuilt_rules.ts b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/import_export/export_prebuilt_rules.ts index b1f0a2d3f9940..ebc426939647e 100644 --- a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/import_export/export_prebuilt_rules.ts +++ b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/import_export/export_prebuilt_rules.ts @@ -93,6 +93,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }), expect.objectContaining({ @@ -101,6 +103,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }), ]) @@ -163,6 +167,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: true, + customized_fields: [{ field_name: 'name' }, { field_name: 'tags' }], + has_base_version: true, }, }), expect.objectContaining({ @@ -171,6 +177,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: true, + customized_fields: [{ field_name: 'name' }, { field_name: 'tags' }], + has_base_version: true, }, }), ]) @@ -244,6 +252,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }), expect.objectContaining({ @@ -252,6 +262,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: true, + customized_fields: [{ field_name: 'name' }, { field_name: 'tags' }], + has_base_version: true, }, }), expect.objectContaining({ @@ -321,12 +333,22 @@ export default ({ getService }: FtrProviderContext): void => { expect.objectContaining({ rule_id: PREBUILT_RULE_ID_A, immutable: true, - rule_source: { type: 'external', is_customized: false }, + rule_source: { + type: 'external', + is_customized: false, + customized_fields: [], + has_base_version: true, + }, }), expect.objectContaining({ rule_id: PREBUILT_RULE_ID_B, immutable: true, - rule_source: { type: 'external', is_customized: true }, + rule_source: { + type: 'external', + is_customized: true, + customized_fields: [{ field_name: 'tags' }], + has_base_version: true, + }, }), expect.objectContaining({ rule_id: CUSTOM_RULE_ID, @@ -387,6 +409,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }), expect.objectContaining({ @@ -395,6 +419,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }), ]) @@ -433,6 +459,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }), expect.objectContaining({ @@ -441,6 +469,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }), ]) diff --git a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/import_export/import_multiple_prebuilt_rules.ts b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/import_export/import_multiple_prebuilt_rules.ts index 0e1eb5870fb81..bd1d23932755a 100644 --- a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/import_export/import_multiple_prebuilt_rules.ts +++ b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/import_export/import_multiple_prebuilt_rules.ts @@ -53,13 +53,23 @@ export default ({ getService }: FtrProviderContext): void => { const NON_CUSTOMIZED_PREBUILT_RULE_TO_IMPORT = { ...PREBUILT_RULE_ASSET_A['security-rule'], immutable: true, - rule_source: { type: 'external', is_customized: false }, + rule_source: { + type: 'external', + is_customized: false, + customized_fields: [], + has_base_version: true, + }, }; const CUSTOMIZED_PREBUILT_RULE_TO_IMPORT = { ...PREBUILT_RULE_ASSET_B['security-rule'], name: 'Customized Prebuilt Rule', immutable: true, - rule_source: { type: 'external', is_customized: true }, + rule_source: { + type: 'external', + is_customized: true, + customized_fields: [{ field_name: 'name' }], + has_base_version: true, + }, }; const CUSTOM_RULE_TO_IMPORT = getCustomQueryRuleParams({ rule_id: 'custom-rule', @@ -92,12 +102,22 @@ export default ({ getService }: FtrProviderContext): void => { expect.objectContaining({ rule_id: PREBUILT_RULE_ID_A, immutable: true, - rule_source: { type: 'external', is_customized: false }, + rule_source: { + type: 'external', + is_customized: false, + customized_fields: [], + has_base_version: true, + }, }), expect.objectContaining({ rule_id: PREBUILT_RULE_ID_B, immutable: true, - rule_source: { type: 'external', is_customized: true }, + rule_source: { + type: 'external', + is_customized: true, + customized_fields: [{ field_name: 'name' }], + has_base_version: true, + }, }), expect.objectContaining({ rule_id: 'custom-rule', @@ -182,12 +202,22 @@ export default ({ getService }: FtrProviderContext): void => { expect.objectContaining({ rule_id: PREBUILT_RULE_ID_A, immutable: true, - rule_source: { type: 'external', is_customized: false }, + rule_source: { + type: 'external', + is_customized: false, + customized_fields: [], + has_base_version: true, + }, }), expect.objectContaining({ rule_id: PREBUILT_RULE_ID_B, immutable: true, - rule_source: { type: 'external', is_customized: true }, + rule_source: { + type: 'external', + is_customized: true, + customized_fields: [{ field_name: 'name' }], + has_base_version: true, + }, }), expect.objectContaining({ rule_id: 'custom-rule', diff --git a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/import_export/import_outdated_prebuilt_rules.ts b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/import_export/import_outdated_prebuilt_rules.ts index 66c4ef239e0fa..d93f258f0a6ac 100644 --- a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/import_export/import_outdated_prebuilt_rules.ts +++ b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/import_export/import_outdated_prebuilt_rules.ts @@ -96,6 +96,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }, { @@ -105,6 +107,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }, ]; @@ -128,12 +132,22 @@ export default ({ getService }: FtrProviderContext): void => { expect.objectContaining({ rule_id: PREBUILT_RULE_ID_A, immutable: true, - rule_source: { type: 'external', is_customized: false }, + rule_source: { + type: 'external', + is_customized: false, + customized_fields: [], + has_base_version: true, + }, }), expect.objectContaining({ rule_id: PREBUILT_RULE_ID_B, immutable: true, - rule_source: { type: 'external', is_customized: true }, + rule_source: { + type: 'external', + is_customized: true, + customized_fields: [{ field_name: 'name' }], + has_base_version: true, + }, }), ]) ); @@ -230,6 +244,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }, { @@ -238,6 +254,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }, { @@ -247,6 +265,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }, { @@ -256,6 +276,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }, ]; @@ -280,22 +302,42 @@ export default ({ getService }: FtrProviderContext): void => { expect.objectContaining({ rule_id: PREBUILT_RULE_ID_A, immutable: true, - rule_source: { type: 'external', is_customized: false }, + rule_source: { + type: 'external', + is_customized: false, + customized_fields: [], + has_base_version: true, + }, }), expect.objectContaining({ rule_id: PREBUILT_RULE_ID_B, immutable: true, - rule_source: { type: 'external', is_customized: false }, + rule_source: { + type: 'external', + is_customized: false, + customized_fields: [], + has_base_version: true, + }, }), expect.objectContaining({ rule_id: PREBUILT_RULE_ID_C, immutable: true, - rule_source: { type: 'external', is_customized: true }, + rule_source: { + type: 'external', + is_customized: true, + customized_fields: [{ field_name: 'description' }], + has_base_version: true, + }, }), expect.objectContaining({ rule_id: PREBUILT_RULE_ID_D, immutable: true, - rule_source: { type: 'external', is_customized: true }, + rule_source: { + type: 'external', + is_customized: true, + customized_fields: [{ field_name: 'description' }], + has_base_version: true, + }, }), ]) ); @@ -412,6 +454,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }, { @@ -420,6 +464,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }, { @@ -429,6 +475,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }, { @@ -438,6 +486,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }, ]; @@ -462,22 +512,42 @@ export default ({ getService }: FtrProviderContext): void => { expect.objectContaining({ rule_id: PREBUILT_RULE_ID_A, immutable: true, - rule_source: { type: 'external', is_customized: false }, + rule_source: { + type: 'external', + is_customized: false, + customized_fields: [], + has_base_version: true, + }, }), expect.objectContaining({ rule_id: PREBUILT_RULE_ID_B, immutable: true, - rule_source: { type: 'external', is_customized: false }, + rule_source: { + type: 'external', + is_customized: false, + customized_fields: [], + has_base_version: true, + }, }), expect.objectContaining({ rule_id: PREBUILT_RULE_ID_C, immutable: true, - rule_source: { type: 'external', is_customized: true }, + rule_source: { + type: 'external', + is_customized: true, + customized_fields: [{ field_name: 'description' }], + has_base_version: true, + }, }), expect.objectContaining({ rule_id: PREBUILT_RULE_ID_D, immutable: true, - rule_source: { type: 'external', is_customized: true }, + rule_source: { + type: 'external', + is_customized: true, + customized_fields: [{ field_name: 'description' }], + has_base_version: true, + }, }), ]) ); @@ -594,6 +664,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }, { @@ -602,6 +674,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }, { @@ -611,6 +685,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }, { @@ -620,6 +696,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }, ]; @@ -644,22 +722,42 @@ export default ({ getService }: FtrProviderContext): void => { expect.objectContaining({ rule_id: PREBUILT_RULE_ID_A, immutable: true, - rule_source: { type: 'external', is_customized: false }, + rule_source: { + type: 'external', + is_customized: false, + customized_fields: [], + has_base_version: true, + }, }), expect.objectContaining({ rule_id: PREBUILT_RULE_ID_B, immutable: true, - rule_source: { type: 'external', is_customized: false }, + rule_source: { + type: 'external', + is_customized: false, + customized_fields: [], + has_base_version: true, + }, }), expect.objectContaining({ rule_id: PREBUILT_RULE_ID_C, immutable: true, - rule_source: { type: 'external', is_customized: true }, + rule_source: { + type: 'external', + is_customized: true, + customized_fields: [{ field_name: 'description' }], + has_base_version: true, + }, }), expect.objectContaining({ rule_id: PREBUILT_RULE_ID_D, immutable: true, - rule_source: { type: 'external', is_customized: true }, + rule_source: { + type: 'external', + is_customized: true, + customized_fields: [{ field_name: 'description' }], + has_base_version: true, + }, }), ]) ); diff --git a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/import_export/import_single_prebuilt_rule.ts b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/import_export/import_single_prebuilt_rule.ts index e1ad281d192b5..558513ad56fd3 100644 --- a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/import_export/import_single_prebuilt_rule.ts +++ b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/import_export/import_single_prebuilt_rule.ts @@ -48,6 +48,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }; @@ -67,6 +69,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }, }); @@ -91,6 +95,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }, }); @@ -120,6 +126,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }, }); @@ -135,6 +143,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: true, + customized_fields: [{ field_name: 'name' }], + has_base_version: true, }, }; @@ -154,6 +164,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: true, + customized_fields: [{ field_name: 'name' }], + has_base_version: true, }, }, }); @@ -178,6 +190,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: true, + customized_fields: [{ field_name: 'name' }], + has_base_version: true, }, }, }); @@ -207,6 +221,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: true, + customized_fields: [{ field_name: 'name' }], + has_base_version: true, }, }, }); @@ -238,6 +254,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: true, + customized_fields: [{ field_name: 'name' }], + has_base_version: true, }, }, }); @@ -353,6 +371,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }, }); @@ -367,6 +387,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }; @@ -438,6 +460,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }, }); @@ -461,6 +485,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }; @@ -478,6 +504,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }, }); @@ -499,6 +527,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: true, + customized_fields: [{ field_name: 'name' }], + has_base_version: true, }, }; @@ -516,6 +546,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: true, + customized_fields: [{ field_name: 'name' }], + has_base_version: true, }, }, }); @@ -540,6 +572,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }; @@ -557,6 +591,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }, }); @@ -580,6 +616,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }; @@ -597,6 +635,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: true, + customized_fields: [{ field_name: 'name' }], + has_base_version: true, }, }, }); @@ -628,6 +668,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }; @@ -645,6 +687,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }, }); @@ -675,6 +719,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }; @@ -692,6 +738,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: true, + customized_fields: [{ field_name: 'name' }], + has_base_version: true, }, }, }); diff --git a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/import_export/import_with_installing_package.ts b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/import_export/import_with_installing_package.ts index 639255d8b4ca3..501ee14f80b65 100644 --- a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/import_export/import_with_installing_package.ts +++ b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/import_export/import_with_installing_package.ts @@ -28,7 +28,6 @@ const NON_CUSTOMIZED_PREBUILT_RULE = PREBUILT_RULE_ASSET_A; const CUSTOMIZED_PREBUILT_RULE = { ...PREBUILT_RULE_ASSET_B, description: 'Custom description', - tags: ['custom-tag'], }; export default ({ getService }: FtrProviderContext): void => { @@ -52,6 +51,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }, { @@ -60,6 +61,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: true, + customized_fields: [{ field_name: 'description' }], + has_base_version: true, }, }, ]; @@ -87,6 +90,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }), expect.objectContaining({ @@ -95,6 +100,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: true, + customized_fields: [{ field_name: 'description' }], + has_base_version: true, }, }), ]) @@ -178,6 +185,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }), expect.objectContaining({ @@ -186,6 +195,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: true, + customized_fields: [{ field_name: 'description' }], + has_base_version: true, }, }), ]) diff --git a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/import_export/import_with_missing_base_version.ts b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/import_export/import_with_missing_base_version.ts index 29fb72106a20f..60e17ea66f757 100644 --- a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/import_export/import_with_missing_base_version.ts +++ b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/import_export/import_with_missing_base_version.ts @@ -52,6 +52,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, // Setting to the default value of rule_source, should be calculated on import }, }; @@ -87,6 +89,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, // Setting to the default value of rule_source, should be calculated on import }, }; @@ -105,6 +109,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: false, }, }, }); @@ -125,6 +131,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, // Setting to the default value of rule_source, should be calculated on import }, }; @@ -143,6 +151,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: true, + customized_fields: [], + has_base_version: false, }, }, }); @@ -170,6 +180,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: false, }, }; @@ -188,6 +200,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: true, + customized_fields: [], + has_base_version: false, }, }, }); @@ -206,6 +220,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, // Setting to the default value of rule_source, should be calculated on import }, }; @@ -224,6 +240,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: false, }, }, }); @@ -249,6 +267,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: true, + customized_fields: [], + has_base_version: true, // Setting to the default value of rule_source, should be calculated on import }, }; @@ -266,6 +286,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: true, + customized_fields: [{ field_name: 'name' }, { field_name: 'tags' }], + has_base_version: true, }, }, }); diff --git a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/import_export/import_with_missing_fields.ts b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/import_export/import_with_missing_fields.ts index d6275595f7fce..0ade4cde167c1 100644 --- a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/import_export/import_with_missing_fields.ts +++ b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/import_export/import_with_missing_fields.ts @@ -65,6 +65,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }, }); @@ -97,6 +99,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: true, + customized_fields: [{ field_name: 'name' }], + has_base_version: true, }, }, }); @@ -110,6 +114,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }; @@ -139,6 +145,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { type: 'external', is_customized: false, + customized_fields: [], + has_base_version: true, }, }; diff --git a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/revert_prebuilt_rules/get_prebuilt_rule_base_version.ts b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/revert_prebuilt_rules/get_prebuilt_rule_base_version.ts index d0504f53c09fa..cf161d394a70b 100644 --- a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/revert_prebuilt_rules/get_prebuilt_rule_base_version.ts +++ b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/revert_prebuilt_rules/get_prebuilt_rule_base_version.ts @@ -72,6 +72,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { is_customized: true, type: 'external', + customized_fields: [{ field_name: 'description' }], + has_base_version: true, }, updated_at: modifiedCurrentVersion.updated_at, }); diff --git a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/revert_prebuilt_rules/revert_prebuilt_rules.ts b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/revert_prebuilt_rules/revert_prebuilt_rules.ts index 8a90cb408b107..b0a390b26bcda 100644 --- a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/revert_prebuilt_rules/revert_prebuilt_rules.ts +++ b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/revert_prebuilt_rules/revert_prebuilt_rules.ts @@ -75,6 +75,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { is_customized: false, type: 'external', + customized_fields: [], + has_base_version: true, }, description: nonCustomizedPrebuiltRule.description, // Modified field should be set to its original asset value revision: ++customizedPrebuiltRule.revision, // We increment the revision number during reversion @@ -122,6 +124,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { is_customized: false, type: 'external', + customized_fields: [], + has_base_version: true, }, exceptions_list: [ expect.objectContaining({ @@ -163,6 +167,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { is_customized: false, type: 'external', + customized_fields: [], + has_base_version: true, }, actions: [ expect.objectContaining({ @@ -216,6 +222,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { is_customized: false, type: 'external', + customized_fields: [], + has_base_version: true, }, execution_summary: body.execution_summary, }) @@ -242,6 +250,8 @@ export default ({ getService }: FtrProviderContext): void => { rule_source: { is_customized: false, type: 'external', + customized_fields: [], + has_base_version: true, }, enabled: true, }), diff --git a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/customization_enabled/customization/customize_via_bulk_editing.ts b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/customization_enabled/customization/customize_via_bulk_editing.ts index 8b3e9c1cfc7c7..ecea5a39232ec 100644 --- a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/customization_enabled/customization/customize_via_bulk_editing.ts +++ b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/customization_enabled/customization/customize_via_bulk_editing.ts @@ -340,7 +340,7 @@ export default ({ getService }: FtrProviderContext): void => { }; describe('when base version is available', () => { - testCustomizationViaBulkEditing({ hasBaseVersion: false }); + testCustomizationViaBulkEditing({ hasBaseVersion: true }); }); describe('when base version is missing', () => { diff --git a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/customization_enabled/customization/detect_customization_with_base_version.ts b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/customization_enabled/customization/detect_customization_with_base_version.ts index dbbd6a8a40ed0..829702e8826d8 100644 --- a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/customization_enabled/customization/detect_customization_with_base_version.ts +++ b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/customization_enabled/customization/detect_customization_with_base_version.ts @@ -56,6 +56,8 @@ export default ({ getService }: FtrProviderContext): void => { expect(customizedResponse.rule_source).toMatchObject({ type: 'external', is_customized: true, + customized_fields: [{ field_name: fieldName }], + has_base_version: true, }); // Assert that patching the "fieldName" to its original value reverts the customization diff --git a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/customization_enabled/customization/detect_customization_without_base_version.ts b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/customization_enabled/customization/detect_customization_without_base_version.ts index 68d48e037b816..43885bb97a766 100644 --- a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/customization_enabled/customization/detect_customization_without_base_version.ts +++ b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/customization_enabled/customization/detect_customization_without_base_version.ts @@ -50,6 +50,8 @@ export default ({ getService }: FtrProviderContext): void => { expect(customizedResponse.rule_source).toMatchObject({ type: 'external', is_customized: true, + customized_fields: [], + has_base_version: false, }); }; @@ -304,6 +306,7 @@ export default ({ getService }: FtrProviderContext): void => { beforeEach(async () => { await createPrebuiltRuleAssetSavedObjects(es, [SAVED_QUERY_PREBUILT_RULE_ASSET]); await installPrebuiltRules(es, supertest); + await deleteAllPrebuiltRuleAssets(es, log); }); it('"saved_id" field', () => @@ -569,6 +572,43 @@ export default ({ getService }: FtrProviderContext): void => { }, })); }); + + describe('when rule is previously customized', () => { + beforeEach(async () => { + await createPrebuiltRuleAssetSavedObjects(es, [QUERY_PREBUILT_RULE_ASSET]); + await installPrebuiltRules(es, supertest); + }); + + it('should reset customized_fields to empty array', async () => { + const { body: customizedResponseWithBaseVersion } = await securitySolutionApi + .patchRule({ + body: { rule_id: PREBUILT_RULE_ID, name: 'Customized rule name' }, + }) + .expect(200); + + expect(customizedResponseWithBaseVersion.rule_source).toMatchObject({ + type: 'external', + is_customized: true, + customized_fields: [{ field_name: 'name' }], + has_base_version: true, + }); + + await deleteAllPrebuiltRuleAssets(es, log); + + const { body: customizedResponseWithoutBaseVersion } = await securitySolutionApi + .patchRule({ + body: { rule_id: PREBUILT_RULE_ID, name: 'New customized rule name' }, + }) + .expect(200); + + expect(customizedResponseWithoutBaseVersion.rule_source).toMatchObject({ + type: 'external', + is_customized: true, + customized_fields: [], + has_base_version: false, + }); + }); + }); }); }; diff --git a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/customization_enabled/customization/unaffected_fields.ts b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/customization_enabled/customization/unaffected_fields.ts index eb8e20cf11f1c..e3848a268cc22 100644 --- a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/customization_enabled/customization/unaffected_fields.ts +++ b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/customization_enabled/customization/unaffected_fields.ts @@ -60,6 +60,8 @@ export default ({ getService }: FtrProviderContext): void => { expect(body.rule_source).toMatchObject({ type: 'external', is_customized: false, + customized_fields: [], + has_base_version: hasBaseVersion, }); }; From cc348c698a53d707cff1d4dce69e18394573e3d9 Mon Sep 17 00:00:00 2001 From: Davis Plumlee Date: Mon, 29 Sep 2025 12:13:02 -0400 Subject: [PATCH 02/10] adds telemetry work --- .../common/detection_engine/constants.ts | 101 ++++++++++++++++++ .../lib/telemetry/filterlists/index.test.ts | 20 ++++ .../filterlists/prebuilt_rules_alerts.ts | 3 + .../server/lib/telemetry/helpers.test.ts | 74 +++++++++++++ .../server/lib/telemetry/helpers.ts | 14 +++ .../telemetry/tasks/prebuilt_rule_alerts.ts | 2 + .../server/lib/telemetry/types.ts | 4 + 7 files changed, 218 insertions(+) diff --git a/x-pack/solutions/security/plugins/security_solution/common/detection_engine/constants.ts b/x-pack/solutions/security/plugins/security_solution/common/detection_engine/constants.ts index a9b3fc6ff6169..b0ecbd457dede 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/detection_engine/constants.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/detection_engine/constants.ts @@ -6,6 +6,7 @@ */ import type { Severity, Type } from '@kbn/securitysolution-io-ts-alerting-types'; +import { RuleResponse } from '../api/detection_engine/model/rule_schema'; export enum RULE_PREVIEW_INVOCATION_COUNT { HOUR = 12, @@ -74,3 +75,103 @@ export const defaultRiskScoreBySeverity: Record = { high: RISK_SCORE_HIGH, critical: RISK_SCORE_CRITICAL, }; + +type AllKeys = U extends any ? keyof U : never; + +/** + * A list of all possible fields in the RuleResponse type mapped to whether or not the field is + * considered functional. We are defining "functional" to mean having a direct impact on how a + * rule executes. This means fields like `query` will be marked as functional while fields like + * `note` will be marked as non-functional. We are being conservative in our labeling of + * functional and only fields that have a 100% guaranteed impact on rule execution will be labeled + * as such. Fields like `index` that have a direct impact but don't necessarily change the alert + * rate (noise) of a rule will not be marked as functional. + * + * This categorization is intended to be used for telemetry purposes. + * + * More info here: + * x-pack/solutions/security/plugins/security_solution/docs/rfcs/detection_response/customized_rule_alert_telemetry.md + */ +export const FUNCTIONAL_FIELD_MAP: Record, boolean> = { + // Common fields + name: false, + description: false, + risk_score: false, + severity: false, + rule_name_override: false, + timestamp_override: false, + timestamp_override_fallback_disabled: false, + timeline_id: false, + timeline_title: false, + license: false, + note: false, + building_block_type: false, + investigation_fields: false, + version: false, + tags: false, + risk_score_mapping: false, + severity_mapping: false, + interval: false, + from: false, + to: false, + author: false, + false_positives: false, + references: false, + max_signals: false, + threat: false, + setup: false, + related_integrations: false, + required_fields: false, + type: true, + // Query, EQL, and ESQL rule type fields + query: true, + language: true, + index: false, + data_view_id: false, + filters: true, + event_category_override: true, + tiebreaker_field: true, + timestamp_field: true, + alert_suppression: true, + // Saved query rule type fields + saved_id: true, + // Threshold rule type fields + threshold: true, + // Threat match rule type fields + threat_query: true, + threat_mapping: true, + threat_index: false, + threat_filters: true, + threat_indicator_path: false, + threat_language: true, + // Maching learning rule type fields + anomaly_threshold: true, + machine_learning_job_id: true, + // New terms rule type fields + new_terms_fields: true, + history_window_start: true, + // Response fields - We don't use these fields for diffing purposes, setting the values to false + id: false, + rule_id: false, + rule_source: false, + outcome: false, + output_index: false, + namespace: false, + exceptions_list: false, + execution_summary: false, + actions: false, + throttle: false, + alias_purpose: false, + alias_target_id: false, + meta: false, + response_actions: false, + revision: false, + enabled: false, + items_per_search: false, + concurrent_searches: false, + immutable: false, + updated_at: false, + updated_by: false, + created_at: false, + created_by: false, +} \ No newline at end of file diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/filterlists/index.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/filterlists/index.test.ts index de977701a6a51..18528e1ce61f1 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/filterlists/index.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/filterlists/index.test.ts @@ -9,6 +9,7 @@ import { copyAllowlistedFields } from '.'; import { prebuiltRuleAllowlistFields } from './prebuilt_rules_alerts'; import type { AllowlistFields } from './types'; import { unflatten } from '../helpers'; +import { createDefaultExternalRuleSource } from '../../detection_engine/rule_management/logic/detection_rules_client/mergers/rule_source/create_default_external_rule_source'; describe('Security Telemetry filters', () => { describe('allowlistEventFields', () => { @@ -354,6 +355,25 @@ describe('Security Telemetry filters', () => { }, }); }); + + it('copies over rule source field', () => { + const event = { + not_event: 'much data, much wow', + 'kibana.alert.rule.parameters': { + rule_source: createDefaultExternalRuleSource(), + }, + }; + expect(copyAllowlistedFields(prebuiltRuleAllowlistFields, event)).toStrictEqual({ + 'kibana.alert.rule.parameters': { + rule_source: { + type: 'external', + is_customized: false, + customized_fields: [], + has_base_version: true, + }, + }, + }); + }); }); describe('when passing in unflattened args', () => { test('should preserve flattened fields in event with nested filterlist key:value', () => { diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/filterlists/prebuilt_rules_alerts.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/filterlists/prebuilt_rules_alerts.ts index a7e797227d24b..57e5787b126aa 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/filterlists/prebuilt_rules_alerts.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/filterlists/prebuilt_rules_alerts.ts @@ -49,6 +49,9 @@ export const prebuiltRuleAllowlistFields: AllowlistFields = { 'kibana.alert.rule.updated_at': true, 'kibana.alert.rule.uuid': true, 'kibana.alert.rule.version': true, + 'kibana.alert.rule.parameters': { + rule_source: true, + }, 'kibana.alert.severity': true, 'kibana.alert.status': true, 'kibana.alert.uuid': true, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.test.ts index 5b3478b03348c..5ba833028c579 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.test.ts @@ -26,6 +26,7 @@ import { setIsElasticCloudDeployment, processK8sUsernames, unflatten, + processDetectionRuleCustomizations, } from './helpers'; import type { ESClusterInfo, ESLicense, ExceptionListItem } from './types'; import type { PolicyConfig, PolicyData } from '../../../common/endpoint/types'; @@ -1073,3 +1074,76 @@ describe('unflatten', () => { expect(unflatten(input)).toEqual({ nums: [1, 2, 3] }); }); }); + +describe('processDetectionRuleCustomizations', ()=>{ + it("returns undefined if rule_source doesn't exist in alert", () => { + const customizationsField = processDetectionRuleCustomizations({}) + expect(customizationsField).toBeUndefined() + }) + + it("returns undefined if rule_source is not `external` type", () => { + const customizationsField = processDetectionRuleCustomizations({'kibana.alert.rule.parameters': { + rule_source: { + type: 'internal' + } + }}) + expect(customizationsField).toBeUndefined() + }) + + it("returns undefined if rule is not customized", () => { + const customizationsField = processDetectionRuleCustomizations({'kibana.alert.rule.parameters': { + rule_source: { + type: 'external', + is_customized: false, + customized_fields: [], + has_base_version: true, + } + }}) + expect(customizationsField).toBeUndefined() + }) + + it("returns customized fields when rule is customized with non-functional fields", () => { + const customizationsField = processDetectionRuleCustomizations({'kibana.alert.rule.parameters': { + rule_source: { + type: 'external', + is_customized: true, + customized_fields: [{field_name: 'tags'}], + has_base_version: true, + } + }}) + expect(customizationsField).toEqual({ + customized_fields: ['tags'], + num_functional_fields: 0 + }) + }) + + it("returns customized fields when rule is customized with functional fields", () => { + const customizationsField = processDetectionRuleCustomizations({'kibana.alert.rule.parameters': { + rule_source: { + type: 'external', + is_customized: true, + customized_fields: [{field_name: 'query'}], + has_base_version: true, + } + }}) + expect(customizationsField).toEqual({ + customized_fields: ['query'], + num_functional_fields: 1 + }) + }) + + it("returns customized fields when rule is customized with both functional and non-functional fields", () => { + const customizationsField = processDetectionRuleCustomizations({'kibana.alert.rule.parameters': { + rule_source: { + type: 'external', + is_customized: true, + customized_fields: [{field_name: 'query'}, {field_name: 'tags'}], + has_base_version: true, + } + }}) + expect(customizationsField).toEqual({ + customized_fields: ['query', 'tags'], + num_functional_fields: 1 + }) + }) +}) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts index a8c74fab3543d..ec2a1a82875ce 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts @@ -48,6 +48,8 @@ import { TelemetryLoggerImpl, tlog as telemetryLogger, } from './telemetry_logger'; +import { FUNCTIONAL_FIELD_MAP } from '@kbn/security-solution-plugin/common/detection_engine/constants'; +import { RuleResponse } from '@kbn/security-solution-plugin/common/api/detection_engine/model/rule_schema'; /** * Determines the when the last run was in order to execute to. @@ -389,6 +391,18 @@ export const processK8sUsernames = (clusterId: string, event: TelemetryEvent): T return event; }; +export const processDetectionRuleCustomizations = (event: TelemetryEvent) => { + const ruleSource = event['kibana.alert.rule.parameters']?.rule_source; + if (!ruleSource || ruleSource.type === 'internal' || ruleSource.is_customized === false) { + return undefined; // Don't return anything if rule isn't prebuilt or is not customized + } + const numberOfFunctionalFields = ruleSource.customized_fields.filter((field) => FUNCTIONAL_FIELD_MAP[field.field_name as keyof RuleResponse]).length + return { + customized_fields: ruleSource.customized_fields.map((fieldObj)=> fieldObj.field_name), + num_functional_fields: numberOfFunctionalFields, + } +}; + export const ranges = ( taskExecutionPeriod: TaskExecutionPeriod, defaultIntervalInHours: number = 3 diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/prebuilt_rule_alerts.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/prebuilt_rule_alerts.ts index 4a1eb0792bdb2..b9ace012fbfb6 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/prebuilt_rule_alerts.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/tasks/prebuilt_rule_alerts.ts @@ -19,6 +19,7 @@ import { getPreviousDailyTaskTimestamp, safeValue, unflatten, + processDetectionRuleCustomizations, } from '../helpers'; import { copyAllowlistedFields, filterList } from '../filterlists'; import type { AllowlistFields } from '../filterlists/types'; @@ -98,6 +99,7 @@ export function createTelemetryPrebuiltRuleAlertsTaskConfig(maxTelemetryBatch: n cluster_name: clusterInfo?.cluster_name, package_version: packageInfo?.version, task_version: taskVersion, + customizations: processDetectionRuleCustomizations(event), }) ); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/types.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/types.ts index 8784186a1c09d..7545404647cea 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/types.ts @@ -7,6 +7,7 @@ import type { Agent } from '@kbn/fleet-plugin/common'; +import type { RuleSource } from '../../../common/api/detection_engine/model/rule_schema'; import type { AlertEvent, ResolverNode, SafeResolverEvent } from '../../../common/endpoint/types'; import type { AllowlistFields } from './filterlists/types'; import type { RssGrowthCircuitBreakerConfig } from './diagnostic/circuit_breakers/rss_growth_circuit_breaker'; @@ -86,6 +87,9 @@ export interface TelemetryEvent { pod?: SearchTypes; }; }; + 'kibana.alert.rule.parameters'?: { + rule_source?: RuleSource; + }; } /** From d7a8aad2ed409a1967bf8b532bc6a8e63c9e0918 Mon Sep 17 00:00:00 2001 From: Davis Plumlee Date: Mon, 29 Sep 2025 12:34:50 -0400 Subject: [PATCH 03/10] lints --- .../server/lib/telemetry/helpers.test.ts | 154 ++++++++++-------- .../server/lib/telemetry/helpers.ts | 4 +- 2 files changed, 91 insertions(+), 67 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.test.ts index 5ba833028c579..2ec108a8afadd 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.test.ts @@ -1075,75 +1075,99 @@ describe('unflatten', () => { }); }); -describe('processDetectionRuleCustomizations', ()=>{ +describe('processDetectionRuleCustomizations', () => { it("returns undefined if rule_source doesn't exist in alert", () => { - const customizationsField = processDetectionRuleCustomizations({}) - expect(customizationsField).toBeUndefined() - }) - - it("returns undefined if rule_source is not `external` type", () => { - const customizationsField = processDetectionRuleCustomizations({'kibana.alert.rule.parameters': { - rule_source: { - type: 'internal' - } - }}) - expect(customizationsField).toBeUndefined() - }) - - it("returns undefined if rule is not customized", () => { - const customizationsField = processDetectionRuleCustomizations({'kibana.alert.rule.parameters': { - rule_source: { - type: 'external', - is_customized: false, - customized_fields: [], - has_base_version: true, - } - }}) - expect(customizationsField).toBeUndefined() - }) - - it("returns customized fields when rule is customized with non-functional fields", () => { - const customizationsField = processDetectionRuleCustomizations({'kibana.alert.rule.parameters': { - rule_source: { - type: 'external', - is_customized: true, - customized_fields: [{field_name: 'tags'}], - has_base_version: true, - } - }}) + const customizationsField = processDetectionRuleCustomizations({}); + expect(customizationsField).toBeUndefined(); + }); + + it('returns undefined if rule_source is not `external` type', () => { + const customizationsField = processDetectionRuleCustomizations({ + 'kibana.alert.rule.parameters': { + rule_source: { + type: 'internal', + }, + }, + }); + expect(customizationsField).toBeUndefined(); + }); + + it('returns undefined if rule is not customized', () => { + const customizationsField = processDetectionRuleCustomizations({ + 'kibana.alert.rule.parameters': { + rule_source: { + type: 'external', + is_customized: false, + customized_fields: [], + has_base_version: true, + }, + }, + }); + expect(customizationsField).toBeUndefined(); + }); + + it("returns undefined if rule doesn't have a base version", () => { + const customizationsField = processDetectionRuleCustomizations({ + 'kibana.alert.rule.parameters': { + rule_source: { + type: 'external', + is_customized: true, + customized_fields: [], + has_base_version: false, + }, + }, + }); + expect(customizationsField).toBeUndefined(); + }); + + it('returns customized fields when rule is customized with non-functional fields', () => { + const customizationsField = processDetectionRuleCustomizations({ + 'kibana.alert.rule.parameters': { + rule_source: { + type: 'external', + is_customized: true, + customized_fields: [{ field_name: 'tags' }], + has_base_version: true, + }, + }, + }); expect(customizationsField).toEqual({ customized_fields: ['tags'], - num_functional_fields: 0 - }) - }) - - it("returns customized fields when rule is customized with functional fields", () => { - const customizationsField = processDetectionRuleCustomizations({'kibana.alert.rule.parameters': { - rule_source: { - type: 'external', - is_customized: true, - customized_fields: [{field_name: 'query'}], - has_base_version: true, - } - }}) + num_functional_fields: 0, + }); + }); + + it('returns customized fields when rule is customized with functional fields', () => { + const customizationsField = processDetectionRuleCustomizations({ + 'kibana.alert.rule.parameters': { + rule_source: { + type: 'external', + is_customized: true, + customized_fields: [{ field_name: 'query' }], + has_base_version: true, + }, + }, + }); expect(customizationsField).toEqual({ customized_fields: ['query'], - num_functional_fields: 1 - }) - }) - - it("returns customized fields when rule is customized with both functional and non-functional fields", () => { - const customizationsField = processDetectionRuleCustomizations({'kibana.alert.rule.parameters': { - rule_source: { - type: 'external', - is_customized: true, - customized_fields: [{field_name: 'query'}, {field_name: 'tags'}], - has_base_version: true, - } - }}) + num_functional_fields: 1, + }); + }); + + it('returns customized fields when rule is customized with both functional and non-functional fields', () => { + const customizationsField = processDetectionRuleCustomizations({ + 'kibana.alert.rule.parameters': { + rule_source: { + type: 'external', + is_customized: true, + customized_fields: [{ field_name: 'query' }, { field_name: 'tags' }], + has_base_version: true, + }, + }, + }); expect(customizationsField).toEqual({ customized_fields: ['query', 'tags'], - num_functional_fields: 1 - }) - }) -}) + num_functional_fields: 1, + }); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts index ec2a1a82875ce..029a60940c325 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts @@ -393,8 +393,8 @@ export const processK8sUsernames = (clusterId: string, event: TelemetryEvent): T export const processDetectionRuleCustomizations = (event: TelemetryEvent) => { const ruleSource = event['kibana.alert.rule.parameters']?.rule_source; - if (!ruleSource || ruleSource.type === 'internal' || ruleSource.is_customized === false) { - return undefined; // Don't return anything if rule isn't prebuilt or is not customized + if (!ruleSource || ruleSource.type === 'internal' || ruleSource.is_customized === false || ruleSource.has_base_version === false) { + return undefined; // Don't return anything if rule is not customized or base version doesn't exist } const numberOfFunctionalFields = ruleSource.customized_fields.filter((field) => FUNCTIONAL_FIELD_MAP[field.field_name as keyof RuleResponse]).length return { From 017118c20550bb393db79019f027af8876ccb567 Mon Sep 17 00:00:00 2001 From: Davis Plumlee Date: Mon, 29 Sep 2025 14:40:25 -0400 Subject: [PATCH 04/10] fixes tests on merge --- .../common/detection_engine/constants.ts | 10 +++++----- .../prebuilt_rule_customization.md | 18 +++++++++++++++++- .../server/lib/telemetry/helpers.ts | 19 +++++++++++++------ ...tect_customization_without_base_version.ts | 4 ++-- 4 files changed, 37 insertions(+), 14 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/common/detection_engine/constants.ts b/x-pack/solutions/security/plugins/security_solution/common/detection_engine/constants.ts index b0ecbd457dede..20a5cea67f71e 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/detection_engine/constants.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/detection_engine/constants.ts @@ -6,7 +6,7 @@ */ import type { Severity, Type } from '@kbn/securitysolution-io-ts-alerting-types'; -import { RuleResponse } from '../api/detection_engine/model/rule_schema'; +import type { RuleResponse } from '../api/detection_engine/model/rule_schema'; export enum RULE_PREVIEW_INVOCATION_COUNT { HOUR = 12, @@ -76,7 +76,7 @@ export const defaultRiskScoreBySeverity: Record = { critical: RISK_SCORE_CRITICAL, }; -type AllKeys = U extends any ? keyof U : never; +type AllKeys = U extends unknown ? keyof U : never; /** * A list of all possible fields in the RuleResponse type mapped to whether or not the field is @@ -86,9 +86,9 @@ type AllKeys = U extends any ? keyof U : never; * functional and only fields that have a 100% guaranteed impact on rule execution will be labeled * as such. Fields like `index` that have a direct impact but don't necessarily change the alert * rate (noise) of a rule will not be marked as functional. - * + * * This categorization is intended to be used for telemetry purposes. - * + * * More info here: * x-pack/solutions/security/plugins/security_solution/docs/rfcs/detection_response/customized_rule_alert_telemetry.md */ @@ -174,4 +174,4 @@ export const FUNCTIONAL_FIELD_MAP: Record, boolean> = { updated_by: false, created_at: false, created_by: false, -} \ No newline at end of file +}; diff --git a/x-pack/solutions/security/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/prebuilt_rule_customization.md b/x-pack/solutions/security/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/prebuilt_rule_customization.md index f695bc2da5716..510e22c007640 100644 --- a/x-pack/solutions/security/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/prebuilt_rule_customization.md +++ b/x-pack/solutions/security/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/prebuilt_rule_customization.md @@ -83,6 +83,8 @@ https://marketplace.visualstudio.com/items?itemName=yzhang.markdown-all-in-one - [Common terminology](./prebuilt_rules_common_info.md#common-terminology). - **Rule source**, or **`ruleSource`**: a rule field that defines the rule's origin. Can be `internal` or `external`. Currently, custom rules have `internal` rule source and prebuilt rules have `external` rule source. - **`is_customized`**: a field within `ruleSource` that exists when rule source is set to `external`. It is a boolean value based on if the rule has been changed from its base version. +- **`customized_fields`**: a field within `ruleSource` that exists when rule source is set to `external`. It is an array of objects containing field names that have been changed from their base version counterparts. +- **`has_base_version`**: a field within `ruleSource` that exists when rule source is set to `external`. It is a boolean value based on if the rule had a matching base version during rule source calculation. - **non-semantic change**: a change to a rule field that is functionally different. We normalize certain fields so for a time-related field such as `from`, `1m` vs `60s` are treated as the same value. We also trim leading and trailing whitespace for query fields. - **rule customization**: a change to a customizable field of a prebuilt rule. Full list of customizable rule fields can be found in [Common information about prebuilt rules](./prebuilt_rules_common_info.md#customizable-rule-fields). - **insufficient license**: a license or a product tier that doesn't allow rule customization. In Serverless environments customization is only allowed on Security Essentials product tier. In non-Serverless environments customization is only allowed on Trial and Enterprise licenses. @@ -245,6 +247,8 @@ Given a prebuilt rule installed When user customizes the prebuilt rule by changing the field so it differs from the base version Then the rule's `is_customized` value should be `true` And ruleSource should be "external" +And the rule's `customized_fields` value should contain +And the rule's `has_base_version` value should be true ``` #### **Scenario: prebuilt rule's `is_customized` stays unchanged after it is saved unchanged** @@ -253,10 +257,12 @@ And ruleSource should be "external" ```Gherkin Given a prebuilt rule installed -And the prebuilt rule doesn't have a matching base version +And the prebuilt rule has a matching base version When user opens the corresponding rule editing page And saves the form unchanged Then the rule's `is_customized` value should stay unchanged (non-customized rule stays non-customized) +And the rule's `customized_fields` value should be an empty array +And the rule's `has_base_version` value should be true ``` **Examples:** @@ -272,6 +278,8 @@ Given a prebuilt rule installed And it is non-customized When a user changes the field so it differs from the base version Then the rule's `is_customized` value should remain `false` +And the rule's `customized_fields` value should be an empty array +And the rule's `has_base_version` value should be true ``` **Examples:** @@ -308,6 +316,8 @@ Given a prebuilt rule installed And it is customized When a user changes the rule fields to match the base version Then the rule's `is_customized` value should be false +And the rule's `customized_fields` value should be an empty array +And the rule's `has_base_version` value should be true ``` ### Detecting rule customizations when base version is missing @@ -324,6 +334,8 @@ And the prebuilt rule doesn't have a matching base version When user customizes the prebuilt rule by changing the field so it differs from the base version Then the rule's `is_customized` value should be `true` And ruleSource should be "external" +And the rule's `customized_fields` value should be an empty array +And the rule's `has_base_version` value should be false ``` **Examples:** @@ -340,6 +352,8 @@ And the prebuilt rule doesn't have a matching base version When user opens the corresponding rule editing page And saves the form unchanged Then the rule's `is_customized` value should stay unchanged (non-customized rule stays non-customized) +And the rule's `customized_fields` value should be an empty array +And the rule's `has_base_version` value should be false ``` **Examples:** @@ -356,6 +370,8 @@ And the prebuilt rule doesn't have a matching base version And it is non-customized When a user changes the field so it differs from the base version Then the rule's `is_customized` value should remain `false` +And the rule's `customized_fields` value should be an empty array +And the rule's `has_base_version` value should be false ``` **Examples:** diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts index 029a60940c325..bb959fda96faf 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts @@ -13,6 +13,7 @@ import { set } from '@kbn/safer-lodash-set'; import type { Logger, LogMeta } from '@kbn/core/server'; import { sha256 } from 'js-sha256'; import type { estypes } from '@elastic/elasticsearch'; +import { FUNCTIONAL_FIELD_MAP } from '../../../common/detection_engine/constants'; import { copyAllowlistedFields, filterList } from './filterlists'; import type { PolicyConfig, PolicyData, SafeEndpointEvent } from '../../../common/endpoint/types'; import type { ITelemetryReceiver } from './receiver'; @@ -48,8 +49,7 @@ import { TelemetryLoggerImpl, tlog as telemetryLogger, } from './telemetry_logger'; -import { FUNCTIONAL_FIELD_MAP } from '@kbn/security-solution-plugin/common/detection_engine/constants'; -import { RuleResponse } from '@kbn/security-solution-plugin/common/api/detection_engine/model/rule_schema'; +import type { RuleResponse } from '../../../common/api/detection_engine/model/rule_schema'; /** * Determines the when the last run was in order to execute to. @@ -393,14 +393,21 @@ export const processK8sUsernames = (clusterId: string, event: TelemetryEvent): T export const processDetectionRuleCustomizations = (event: TelemetryEvent) => { const ruleSource = event['kibana.alert.rule.parameters']?.rule_source; - if (!ruleSource || ruleSource.type === 'internal' || ruleSource.is_customized === false || ruleSource.has_base_version === false) { + if ( + !ruleSource || + ruleSource.type === 'internal' || + ruleSource.is_customized === false || + ruleSource.has_base_version === false + ) { return undefined; // Don't return anything if rule is not customized or base version doesn't exist } - const numberOfFunctionalFields = ruleSource.customized_fields.filter((field) => FUNCTIONAL_FIELD_MAP[field.field_name as keyof RuleResponse]).length + const numberOfFunctionalFields = ruleSource.customized_fields.filter( + (field) => FUNCTIONAL_FIELD_MAP[field.field_name as keyof RuleResponse] + ).length; return { - customized_fields: ruleSource.customized_fields.map((fieldObj)=> fieldObj.field_name), + customized_fields: ruleSource.customized_fields.map((fieldObj) => fieldObj.field_name), num_functional_fields: numberOfFunctionalFields, - } + }; }; export const ranges = ( diff --git a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/customization_enabled/customization/detect_customization_without_base_version.ts b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/customization_enabled/customization/detect_customization_without_base_version.ts index 8126356fdfaca..07df5aa383082 100644 --- a/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/customization_enabled/customization/detect_customization_without_base_version.ts +++ b/x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/customization_enabled/customization/detect_customization_without_base_version.ts @@ -580,7 +580,7 @@ export default ({ getService }: FtrProviderContext): void => { }); it('should reset customized_fields to empty array', async () => { - const { body: customizedResponseWithBaseVersion } = await securitySolutionApi + const { body: customizedResponseWithBaseVersion } = await detectionsApi .patchRule({ body: { rule_id: PREBUILT_RULE_ID, name: 'Customized rule name' }, }) @@ -595,7 +595,7 @@ export default ({ getService }: FtrProviderContext): void => { await deleteAllPrebuiltRuleAssets(es, log); - const { body: customizedResponseWithoutBaseVersion } = await securitySolutionApi + const { body: customizedResponseWithoutBaseVersion } = await detectionsApi .patchRule({ body: { rule_id: PREBUILT_RULE_ID, name: 'New customized rule name' }, }) From fc5b710b850ee670b012e9e991de357a931afd67 Mon Sep 17 00:00:00 2001 From: Davis Plumlee Date: Tue, 30 Sep 2025 00:22:07 -0400 Subject: [PATCH 05/10] updates test plans --- .../prebuilt_rule_customization.md | 20 +++++++++++++++---- .../prebuilt_rules/prebuilt_rule_import.md | 2 ++ .../prebuilt_rule_revert_customization.md | 3 +++ 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/prebuilt_rule_customization.md b/x-pack/solutions/security/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/prebuilt_rule_customization.md index 510e22c007640..fd37336adf43b 100644 --- a/x-pack/solutions/security/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/prebuilt_rule_customization.md +++ b/x-pack/solutions/security/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/prebuilt_rule_customization.md @@ -48,6 +48,7 @@ https://marketplace.visualstudio.com/items?itemName=yzhang.markdown-all-in-one - [**Scenario: prebuilt rule's `is_customized` is set to true after it is customized when base version is missing**](#scenario-prebuilt-rules-is_customized-is-set-to-true-after-it-is-customized-when-base-version-is-missing) - [**Scenario: prebuilt rule's `is_customized` stays unchanged after it is saved unchanged when base version is missing**](#scenario-prebuilt-rules-is_customized-stays-unchanged-after-it-is-saved-unchanged-when-base-version-is-missing) - [**Scenario: prebuilt rule's `is_customized` value is not affected by specific fields when base version is missing**](#scenario-prebuilt-rules-is_customized-value-is-not-affected-by-specific-fields-when-base-version-is-missing) + - [**Scenario: prebuilt rule's `customized_fields` resets to an empty array if rule was previously edited with base version present**](#scenario-prebuilt-rules-customized_fields-resets-to-an-empty-array-if-rule-was-previously-edited-with-base-version-present) - [Calculating the Modified badge in the UI](#calculating-the-modified-badge-in-the-ui) - [**Scenario: Modified badge should appear on the rule details page when prebuilt rule is customized**](#scenario-modified-badge-should-appear-on-the-rule-details-page-when-prebuilt-rule-is-customized) - [**Scenario: Modified badge should not appear on the rule details page when prebuilt rule isn't customized**](#scenario-modified-badge-should-not-appear-on-the-rule-details-page-when-prebuilt-rule-isnt-customized) @@ -356,10 +357,6 @@ And the rule's `customized_fields` value should be an empty array And the rule's `has_base_version` value should be false ``` -**Examples:** - -`` = all customizable rule fields - #### **Scenario: prebuilt rule's `is_customized` value is not affected by specific fields when base version is missing** **Automation**: one integration test per field. @@ -383,6 +380,21 @@ And the rule's `has_base_version` value should be false | revision | | meta | +#### **Scenario: prebuilt rule's `customized_fields` resets to an empty array if rule was previously edited with base version present** + +**Automation**: one integration test. + +```Gherkin +Given a prebuilt rule installed +And the prebuilt rule has a populated `customized_fields` value +And the prebuilt rule doesn't have a matching base version +When user opens the corresponding rule editing page +And saves the form unchanged +Then the rule's `is_customized` value should remain true +And the rule's `customized_fields` value should be an empty array +And the rule's `has_base_version` value should be false +``` + ### Calculating the Modified badge in the UI #### **Scenario: Modified badge should appear on the rule details page when prebuilt rule is customized** diff --git a/x-pack/solutions/security/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/prebuilt_rule_import.md b/x-pack/solutions/security/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/prebuilt_rule_import.md index 8a35a2d26067c..24afcf66f29fd 100644 --- a/x-pack/solutions/security/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/prebuilt_rule_import.md +++ b/x-pack/solutions/security/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/prebuilt_rule_import.md @@ -274,6 +274,7 @@ When the user imports these rules Then the rules should be created And the created rules should be correctly identified as prebuilt or custom And the created rules' is_customized field should be correctly calculated +And the created rules' customized_fields field should be correctly calculated And the created rules' parameters should match the import payload ``` @@ -292,6 +293,7 @@ When the user imports these rules Then the rules should be updated And the updated rules should be correctly identified as prebuilt or custom And the updated rules' is_customized field should be correctly calculated +And the created rules' customized_fields field should be correctly calculated And the updated rules' parameters should match the import payload ``` diff --git a/x-pack/solutions/security/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/prebuilt_rule_revert_customization.md b/x-pack/solutions/security/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/prebuilt_rule_revert_customization.md index 0b4e06a9b427b..d376bbdcec167 100644 --- a/x-pack/solutions/security/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/prebuilt_rule_revert_customization.md +++ b/x-pack/solutions/security/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/prebuilt_rule_revert_customization.md @@ -52,6 +52,7 @@ https://marketplace.visualstudio.com/items?itemName=yzhang.markdown-all-in-one - [Common terminology](./prebuilt_rules_common_info.md#common-terminology). - **Rule source**, or **`ruleSource`**: a rule field that defines the rule's origin. Can be `internal` or `external`. Currently, custom rules have `internal` rule source and prebuilt rules have `external` rule source. - **`is_customized`**: a field within `ruleSource` that exists when rule source is set to `external`. It is a boolean value based on if the rule has been changed from its base version. +- **`customized_fields`**: a field within `ruleSource` that exists when rule source is set to `external`. It is an array of objects containing field names that have been changed from their base version counterparts. - **rule customization**: a change to a customizable field of a prebuilt rule. Full list of customizable rule fields can be found in [Common information about prebuilt rules](./prebuilt_rules_common_info.md#customizable-rule-fields). ## Requirements @@ -105,6 +106,7 @@ When user reverts that rule customizations Then rule customizations should be reset And rule data should match the base version And the rule's `is_customized` value should be false +And the rule's `customized_fields` value should be an empty array ``` #### **Scenario: Showing a customizations diff view in the flyout** @@ -195,6 +197,7 @@ And that rule has an existing base version And that rule has a custom field different from the base version When user makes a request to revert the rule customizations Then the rule's `is_customized` value should be false +And the rule's `customized_fields` value should be an empty array And the field stay unchanged ``` From 14d09a9d878f4ff190629ec7d161981630b69fe6 Mon Sep 17 00:00:00 2001 From: Georgii Gorbachev Date: Tue, 30 Sep 2025 20:40:11 +0200 Subject: [PATCH 06/10] Fix rule normalization logic --- .../internal_rule_to_api_response.ts | 55 +------- .../normalize_rule_fields_common.test.ts | 26 ++++ .../normalize_rule_fields_common.ts | 120 ++++++++++++++++++ .../converters/normalize_rule_params.test.ts | 87 ------------- .../converters/normalize_rule_params.ts | 114 ----------------- .../converters/normalize_rule_source.test.ts | 104 +++++++++++++++ .../converters/normalize_rule_source.ts | 84 ++++++++++++ .../mergers/apply_rule_defaults.ts | 2 +- .../rule_schema/model/rule_schemas.ts | 36 +++--- 9 files changed, 361 insertions(+), 267 deletions(-) create mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_fields_common.test.ts create mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_fields_common.ts delete mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.test.ts delete mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.ts create mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_source.test.ts create mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_source.ts diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/internal_rule_to_api_response.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/internal_rule_to_api_response.ts index 04d85fcaee414..3f9cb73f8a640 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/internal_rule_to_api_response.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/internal_rule_to_api_response.ts @@ -8,63 +8,18 @@ import type { ResolvedSanitizedRule, SanitizedRule } from '@kbn/alerting-plugin/common'; import type { RequiredOptional } from '@kbn/zod-helpers'; import type { RuleResponse } from '../../../../../../../common/api/detection_engine/model/rule_schema'; -import { - transformAlertToRuleAction, - transformAlertToRuleSystemAction, -} from '../../../../../../../common/detection_engine/transform_actions'; -import { createRuleExecutionSummary } from '../../../../rule_monitoring'; import type { RuleParams } from '../../../../rule_schema'; -import { - transformFromAlertThrottle, - transformToActionFrequency, -} from '../../../normalization/rule_actions'; +import { normalizeCommonRuleFields } from './normalize_rule_fields_common'; import { typeSpecificCamelToSnake } from './type_specific_camel_to_snake'; -import { convertObjectKeysToSnakeCase } from '../../../../../../utils/object_case_converters'; -import { normalizeRuleParams } from './normalize_rule_params'; export const internalRuleToAPIResponse = ( rule: SanitizedRule | ResolvedSanitizedRule ): RequiredOptional => { - const executionSummary = createRuleExecutionSummary(rule); - - const isResolvedRule = (obj: unknown): obj is ResolvedSanitizedRule => { - const outcome = (obj as ResolvedSanitizedRule).outcome; - return outcome != null && outcome !== 'exactMatch'; - }; - - const alertActions = rule.actions.map(transformAlertToRuleAction); - const throttle = transformFromAlertThrottle(rule); - const actions = transformToActionFrequency(alertActions, throttle); - const systemActions = rule.systemActions?.map((action) => { - const transformedAction = transformAlertToRuleSystemAction(action); - return transformedAction; - }); - const normalizedRuleParams = convertObjectKeysToSnakeCase(normalizeRuleParams(rule.params)); + const normalizedCommonFields = normalizeCommonRuleFields(rule); + const normalizedTypeSpecificFields = typeSpecificCamelToSnake(rule.params); return { - // saved object properties - outcome: isResolvedRule(rule) ? rule.outcome : undefined, - alias_target_id: isResolvedRule(rule) ? rule.alias_target_id : undefined, - alias_purpose: isResolvedRule(rule) ? rule.alias_purpose : undefined, - // Alerting framework params - id: rule.id, - updated_at: rule.updatedAt.toISOString(), - updated_by: rule.updatedBy ?? 'elastic', - created_at: rule.createdAt.toISOString(), - created_by: rule.createdBy ?? 'elastic', - name: rule.name, - tags: rule.tags, - interval: rule.schedule.interval, - enabled: rule.enabled, - revision: rule.revision, - // Security solution shared rule params - ...normalizedRuleParams, - // Type specific security solution rule params - ...typeSpecificCamelToSnake(rule.params), - // Actions - throttle: undefined, - actions: [...actions, ...(systemActions ?? [])], - // Execution summary - execution_summary: executionSummary ?? undefined, + ...normalizedCommonFields, + ...normalizedTypeSpecificFields, }; }; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_fields_common.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_fields_common.test.ts new file mode 100644 index 0000000000000..952dbdd54cce6 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_fields_common.test.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SanitizedRule } from '@kbn/alerting-plugin/common'; +import { getQueryRuleParams } from '../../../../rule_schema/mocks'; +import { getRuleMock } from '../../../../routes/__mocks__/request_responses'; +import type { RuleParams } from '../../../../rule_schema'; +import { normalizeCommonRuleFields } from './normalize_rule_fields_common'; + +describe('normalizeCommonRuleFields', () => { + it('migrates legacy investigation fields', () => { + const mockRule: SanitizedRule = getRuleMock( + getQueryRuleParams({ + investigationFields: ['field_1', 'field_2'], + }) + ); + + const result = normalizeCommonRuleFields(mockRule); + + expect(result.investigation_fields).toMatchObject({ field_names: ['field_1', 'field_2'] }); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_fields_common.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_fields_common.ts new file mode 100644 index 0000000000000..ca3ed1cc9864d --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_fields_common.ts @@ -0,0 +1,120 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ResolvedSanitizedRule, SanitizedRule } from '@kbn/alerting-plugin/common'; +import type { RequiredOptional } from '@kbn/zod-helpers'; +import type { SharedResponseProps } from '../../../../../../../common/api/detection_engine/model'; +import { + transformAlertToRuleAction, + transformAlertToRuleResponseAction, + transformAlertToRuleSystemAction, +} from '../../../../../../../common/detection_engine/transform_actions'; +import type { RuleParams } from '../../../../rule_schema'; +import { + transformFromAlertThrottle, + transformToActionFrequency, +} from '../../../normalization/rule_actions'; +import { migrateLegacyInvestigationFields } from '../../../utils/utils'; +import { normalizeRuleSource } from './normalize_rule_source'; +import { createRuleExecutionSummary } from '../../../../rule_monitoring'; + +export function normalizeCommonRuleFields( + rule: SanitizedRule | ResolvedSanitizedRule +): RequiredOptional { + const params = rule.params; + + const normalizedRuleSource = normalizeRuleSource({ + immutable: params.immutable, + ruleSource: params.ruleSource, + }); + + const normalizedInvestigationFields = migrateLegacyInvestigationFields( + params.investigationFields + ); + + const alertActions = rule.actions.map(transformAlertToRuleAction); + const throttle = transformFromAlertThrottle(rule); + const actions = transformToActionFrequency(alertActions, throttle); + const systemActions = rule.systemActions?.map((action) => { + const transformedAction = transformAlertToRuleSystemAction(action); + return transformedAction; + }); + + const executionSummary = createRuleExecutionSummary(rule); + + return { + // Basic properties + id: rule.id, + rule_id: params.ruleId, + name: rule.name, + immutable: params.immutable, + rule_source: normalizedRuleSource, + version: params.version, + revision: rule.revision, + updated_at: rule.updatedAt.toISOString(), + updated_by: rule.updatedBy ?? 'elastic', + created_at: rule.createdAt.toISOString(), + created_by: rule.createdBy ?? 'elastic', + + // Rule schedule and execution-related data + enabled: rule.enabled, + interval: rule.schedule.interval, + from: params.from, + to: params.to, + execution_summary: executionSummary ?? undefined, + + // Additional information about the rule + description: params.description, + tags: rule.tags, + author: params.author, + license: params.license, + threat: params.threat, + timeline_id: params.timelineId, + timeline_title: params.timelineTitle, + investigation_fields: normalizedInvestigationFields, + related_integrations: params.relatedIntegrations ?? [], + required_fields: params.requiredFields ?? [], + setup: params.setup ?? '', + note: params.note, + false_positives: params.falsePositives, + references: params.references, + + // Risk score, severity, and overrides + risk_score: params.riskScore, + risk_score_mapping: params.riskScoreMapping, + severity: params.severity, + severity_mapping: params.severityMapping, + rule_name_override: params.ruleNameOverride, + timestamp_override: params.timestampOverride, + timestamp_override_fallback_disabled: params.timestampOverrideFallbackDisabled, + + // Rule's detection alerts + building_block_type: params.buildingBlockType, + output_index: params.outputIndex, + namespace: params.namespace, + max_signals: params.maxSignals, + + // Rule's exceptions + exceptions_list: params.exceptionsList, + + // Rule's notification and response actions + throttle: undefined, + actions: [...actions, ...(systemActions ?? [])], + response_actions: params.responseActions?.map(transformAlertToRuleResponseAction), + + // Technical fields + meta: params.meta, + outcome: isResolvedRule(rule) ? rule.outcome : undefined, + alias_target_id: isResolvedRule(rule) ? rule.alias_target_id : undefined, + alias_purpose: isResolvedRule(rule) ? rule.alias_purpose : undefined, + }; +} + +function isResolvedRule(obj: unknown): obj is ResolvedSanitizedRule { + const outcome = (obj as ResolvedSanitizedRule).outcome; + return outcome != null && outcome !== 'exactMatch'; +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.test.ts deleted file mode 100644 index b684b80727dd0..0000000000000 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.test.ts +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { normalizeRuleSource, normalizeRuleParams } from './normalize_rule_params'; -import type { BaseRuleParams } from '../../../../rule_schema'; - -describe('normalizeRuleSource', () => { - it('should return rule_source of type `internal` when immutable is false and ruleSource is undefined', () => { - const result = normalizeRuleSource({ - immutable: false, - ruleSource: undefined, - }); - expect(result).toEqual({ - type: 'internal', - }); - }); - - it('should return rule_source of type `external` and default customization field values when immutable is true and ruleSource is undefined', () => { - const result = normalizeRuleSource({ - immutable: true, - ruleSource: undefined, - }); - expect(result).toEqual({ - type: 'external', - isCustomized: false, - customizedFields: [], - hasBaseVersion: true, - }); - }); - - it('should return existing value when ruleSource is present', () => { - const externalRuleSource: BaseRuleParams['ruleSource'] = { - type: 'external', - isCustomized: true, - customizedFields: [{ fieldName: 'name' }], - hasBaseVersion: true, - }; - const externalResult = normalizeRuleSource({ immutable: true, ruleSource: externalRuleSource }); - expect(externalResult).toEqual({ - type: externalRuleSource.type, - isCustomized: externalRuleSource.isCustomized, - customizedFields: externalRuleSource.customizedFields, - hasBaseVersion: externalRuleSource.hasBaseVersion, - }); - - const internalRuleSource: BaseRuleParams['ruleSource'] = { - type: 'internal', - }; - const internalResult = normalizeRuleSource({ - immutable: false, - ruleSource: internalRuleSource, - }); - expect(internalResult).toEqual({ - type: internalRuleSource.type, - }); - }); - - it('should return ruleSource with default customization field values when they are missing in the existing ruleSource object', () => { - // We are purposefully setting this to a value that omits fields - const externalRuleSource: BaseRuleParams['ruleSource'] = { - type: 'external', - isCustomized: false, - }; - const externalResult = normalizeRuleSource({ immutable: true, ruleSource: externalRuleSource }); - expect(externalResult).toEqual({ - type: 'external', - isCustomized: false, - customizedFields: [], - hasBaseVersion: true, - }); - }); -}); - -describe('normalizeRuleParams', () => { - it('migrates legacy investigation fields', () => { - const params = { - investigationFields: ['field_1', 'field_2'], - } as BaseRuleParams; - const result = normalizeRuleParams(params); - - expect(result.investigationFields).toMatchObject({ field_names: ['field_1', 'field_2'] }); - }); -}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.ts deleted file mode 100644 index 73ec606900ce5..0000000000000 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_params.ts +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import type { RequiredOptional } from '@kbn/zod-helpers'; -import { transformAlertToRuleResponseAction } from '../../../../../../../common/detection_engine/transform_actions'; -import { convertObjectKeysToCamelCase } from '../../../../../../utils/object_case_converters'; -import type { BaseRuleParams, RuleSourceCamelCased } from '../../../../rule_schema'; -import { migrateLegacyInvestigationFields } from '../../../utils/utils'; -import { createDefaultExternalRuleSource } from '../mergers/rule_source/create_default_external_rule_source'; - -export interface NormalizedRuleParams extends RequiredOptional { - ruleSource: Required; -} - -interface NormalizeRuleSourceParams { - immutable: BaseRuleParams['immutable']; - ruleSource: BaseRuleParams['ruleSource']; -} - -export const normalizeRuleParams = (params: BaseRuleParams) => { - const investigationFields = migrateLegacyInvestigationFields(params.investigationFields); - const ruleSource = normalizeRuleSource({ - immutable: params.immutable, - ruleSource: params.ruleSource, - }); - - return { - ...params, - // These fields are typed as optional in the data model, but they are required in our domain - setup: params.setup ?? '', - relatedIntegrations: params.relatedIntegrations ?? [], - requiredFields: params.requiredFields ?? [], - // Fields to normalize - investigationFields, - ruleSource, - description: params.description, - risk_score: params.riskScore, - severity: params.severity, - buildingBlockType: params.buildingBlockType, - namespace: params.namespace, - note: params.note, - license: params.license, - outputIndex: params.outputIndex, - timelineId: params.timelineId, - timelineTitle: params.timelineTitle, - meta: params.meta, - ruleNameOverride: params.ruleNameOverride, - timestampOverride: params.timestampOverride, - timestampOverrideFallbackDisabled: params.timestampOverrideFallbackDisabled, - author: params.author, - falsePositives: params.falsePositives, - from: params.from, - rule_id: params.ruleId, - maxSignals: params.maxSignals, - riskScoreMapping: params.riskScoreMapping, - severityMapping: params.severityMapping, - threat: params.threat, - to: params.to, - references: params.references, - version: params.version, - exceptionsList: params.exceptionsList, - immutable: params.immutable, - related_integrations: params.relatedIntegrations ?? [], - response_actions: params.responseActions?.map(transformAlertToRuleResponseAction), - }; -}; - -/* - * Since there's no mechanism to migrate all rules at the same time, - * we cannot guarantee that the ruleSource params is present in all rules. - * This function will normalize the ruleSource param, creating it if does - * not exist in ES, based on the immutable param. - */ -export const normalizeRuleSource = ({ - immutable, - ruleSource, -}: NormalizeRuleSourceParams): Required => { - if (!ruleSource) { - /** - * The rule source object is not guaranteed to be present in a rule saved object. Those rules - * which were created a long time ago and haven't been updated ever since won't have it. - * However, in our domain model (`RuleResponse`) the rule source object is required - we always - * return it from the rule management API endpoints. That's why when it's missing we normalize - * it based on the legacy `immutable` field which is guaranteed to be always present. - */ - const normalizedRuleSource = immutable - ? convertObjectKeysToCamelCase(createDefaultExternalRuleSource()) - : { - type: 'internal' as const, - }; - - return normalizedRuleSource; - } else if ( - ruleSource.type === 'external' && - (ruleSource.customizedFields === undefined || ruleSource.hasBaseVersion === undefined) - /** - * If rule source exists in the rule object but does not have the new customization-related - * fields (`customizedFields` and `hasBaseVersion`), we normalize them to default values here. - * The new fields are not guaranteed to be present in the rule source object and can be missing - * in old rules which haven't been updated by teh user since a long time ago. However, in our - * domain model (`ExternalRuleSource`) they are required, so we do the normalization. - */ - ) { - return { - ...ruleSource, - customizedFields: [], - hasBaseVersion: true, - }; - } - return ruleSource as Required; -}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_source.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_source.test.ts new file mode 100644 index 0000000000000..05df38acac04a --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_source.test.ts @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { normalizeRuleSource } from './normalize_rule_source'; +import type { BaseRuleParams } from '../../../../rule_schema'; + +describe('normalizeRuleSource', () => { + describe('when ruleSource is missing (undefined)', () => { + describe('and immutable is false', () => { + it('returns a default rule_source of type `internal`', () => { + const result = normalizeRuleSource({ + immutable: false, + ruleSource: undefined, + }); + + expect(result).toEqual({ + type: 'internal', + }); + }); + }); + + describe('and immutable is true', () => { + it('returns a default rule_source of type `external` with an empty list of customized fields', () => { + const result = normalizeRuleSource({ + immutable: true, + ruleSource: undefined, + }); + + expect(result).toEqual({ + type: 'external', + is_customized: false, + customized_fields: [], + has_base_version: true, + }); + }); + }); + }); + + describe('when ruleSource is present', () => { + describe('and all its nested fields are present', () => { + it('normalizes existing value of internal rule source', () => { + const internalRuleSource: BaseRuleParams['ruleSource'] = { + type: 'internal', + }; + + const internalResult = normalizeRuleSource({ + immutable: false, + ruleSource: internalRuleSource, + }); + + expect(internalResult).toEqual({ + type: 'internal', + }); + }); + + it('normalizes existing value of external rule source', () => { + const externalRuleSource: BaseRuleParams['ruleSource'] = { + type: 'external', + isCustomized: true, + customizedFields: [{ fieldName: 'tags' }], + hasBaseVersion: true, + }; + + const externalResult = normalizeRuleSource({ + immutable: true, + ruleSource: externalRuleSource, + }); + + expect(externalResult).toEqual({ + type: 'external', + is_customized: true, + customized_fields: [{ field_name: 'tags' }], + has_base_version: true, + }); + }); + }); + + describe('but customization fields are missing', () => { + it('initializes the missing customization fields with default values', () => { + // We are purposefully setting this to a value that omits fields + const externalRuleSource: BaseRuleParams['ruleSource'] = { + type: 'external', + isCustomized: true, + }; + + const externalResult = normalizeRuleSource({ + immutable: true, + ruleSource: externalRuleSource, + }); + + expect(externalResult).toEqual({ + type: 'external', + is_customized: true, + customized_fields: [], + has_base_version: true, + }); + }); + }); + }); +}); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_source.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_source.ts new file mode 100644 index 0000000000000..f91a1357e3e53 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/converters/normalize_rule_source.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + ExternalRuleSource, + RuleSource, +} from '../../../../../../../common/api/detection_engine/model'; +import type { BaseRuleParams, ExternalRuleSourceCamelCased } from '../../../../rule_schema'; +import { createDefaultExternalRuleSource } from '../mergers/rule_source/create_default_external_rule_source'; +import { createDefaultInternalRuleSource } from '../mergers/rule_source/create_default_internal_rule_source'; + +interface NormalizeRuleSourceArgs { + immutable: BaseRuleParams['immutable']; + ruleSource: BaseRuleParams['ruleSource']; +} + +/* + * Since there's no mechanism to migrate all rules at the same time, + * we cannot guarantee that the ruleSource params is present in all rules. + * This function will normalize the ruleSource param, creating it if does + * not exist in ES, based on the immutable param. + */ +export const normalizeRuleSource = ({ + immutable, + ruleSource, +}: NormalizeRuleSourceArgs): RuleSource => { + if (!ruleSource) { + /** + * The rule source object is not guaranteed to be present in a rule saved object. Those rules + * which were created a long time ago and haven't been updated ever since won't have it. + * However, in our domain model (`RuleResponse`) the rule source object is required - we always + * return it from the rule management API endpoints. That's why when it's missing we normalize + * it based on the legacy `immutable` field which is guaranteed to be always present. + */ + return immutable ? createDefaultExternalRuleSource() : createDefaultInternalRuleSource(); + } + + if (ruleSource.type === 'internal') { + return { + type: ruleSource.type, + }; + } + + if (ruleSource.customizedFields == null || ruleSource.hasBaseVersion == null) { + /** + * If rule source exists in the rule object but does not have the new customization-related + * fields (`customizedFields` and `hasBaseVersion`), we normalize them to default values here. + * The new fields are not guaranteed to be present in the rule source object and can be missing + * in old rules which haven't been updated by teh user since a long time ago. However, in our + * domain model (`ExternalRuleSource`) they are required, so we do the normalization. + */ + return { + type: ruleSource.type, + is_customized: ruleSource.isCustomized, + customized_fields: [], + has_base_version: true, + }; + } + + return { + type: ruleSource.type, + is_customized: ruleSource.isCustomized, + customized_fields: normalizeCustomizedFields(ruleSource.customizedFields), + has_base_version: ruleSource.hasBaseVersion ?? true, + }; +}; + +function normalizeCustomizedFields( + customizedFields: ExternalRuleSourceCamelCased['customizedFields'] +): ExternalRuleSource['customized_fields'] { + if (customizedFields == null) { + return []; + } + + return customizedFields.map((f) => { + return { + field_name: f.fieldName, + }; + }); +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_defaults.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_defaults.ts index a1b913e4b62de..55a66b8c3161e 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_defaults.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/detection_rules_client/mergers/apply_rule_defaults.ts @@ -25,7 +25,7 @@ import { normalizeThresholdObject, } from '../../../../../../../common/detection_engine/utils'; import { assertUnreachable } from '../../../../../../../common/utility_types'; -import { normalizeRuleSource } from '../converters/normalize_rule_params'; +import { normalizeRuleSource } from '../converters/normalize_rule_source'; export const RULE_DEFAULTS = { enabled: false, diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts index ae9faa8bb006a..421e8c682ff51 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts @@ -95,27 +95,33 @@ export const InvestigationFieldsCombined = z.union([ LegacyInvestigationFields, ]); +export type ExternalRuleSourceCamelCased = z.infer; +export const ExternalRuleSourceCamelCased = z.object({ + type: z.literal('external'), + isCustomized: IsExternalRuleCustomized, + customizedFields: z + .array( + z.object({ + fieldName: z.string(), + }) + ) + .optional(), + hasBaseVersion: z.boolean().optional(), +}); + +export type InternalRuleSourceCamelCased = z.infer; +export const InternalRuleSourceCamelCased = z.object({ + type: z.literal('internal'), +}); + /** * This is the same type as RuleSource, but with the keys in camelCase. Intended * for internal use only (not for API responses). */ export type RuleSourceCamelCased = z.infer; export const RuleSourceCamelCased = z.discriminatedUnion('type', [ - z.object({ - type: z.literal('external'), - isCustomized: IsExternalRuleCustomized, - customizedFields: z - .array( - z.object({ - fieldName: z.string(), - }) - ) - .optional(), - hasBaseVersion: z.boolean().optional(), - }), - z.object({ - type: z.literal('internal'), - }), + ExternalRuleSourceCamelCased, + InternalRuleSourceCamelCased, ]); // Conversion to an interface has to be disabled for the entire file; otherwise, From 9c17115f69b4df5925ace63442482fee2a963501 Mon Sep 17 00:00:00 2001 From: Davis Plumlee Date: Tue, 30 Sep 2025 17:07:03 -0400 Subject: [PATCH 07/10] fixes tests --- .../server/lib/detection_engine/routes/__mocks__/utils.ts | 2 +- .../rule_management/logic/export/get_export_all.test.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/utils.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/utils.ts index 3d4a8105dc151..687bf91655e2a 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/utils.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/utils.ts @@ -81,7 +81,7 @@ export const getOutputRuleAlertForRest = (): RuleResponse => ({ }, ], meta: { - some_meta: 'someField', + someMeta: 'someField', }, timeline_id: 'some-timeline-id', timeline_title: 'some-timeline-title', diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_all.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_all.test.ts index f4f69cce8d990..382df4bfa5ffc 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_all.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/export/get_export_all.test.ts @@ -121,7 +121,7 @@ describe('getExportAll', () => { setup: '', timeline_id: 'some-timeline-id', timeline_title: 'some-timeline-title', - meta: { some_meta: 'someField' }, + meta: { someMeta: 'someField' }, severity: 'high', severity_mapping: [], updated_by: 'elastic', @@ -303,7 +303,7 @@ describe('getExportAll', () => { setup: '', timeline_id: 'some-timeline-id', timeline_title: 'some-timeline-title', - meta: { some_meta: 'someField' }, + meta: { someMeta: 'someField' }, severity: 'high', severity_mapping: [], updated_by: 'elastic', From cb163a4c970756fe89c4c7c74ebcfe50ad703c2b Mon Sep 17 00:00:00 2001 From: Davis Plumlee Date: Wed, 1 Oct 2025 10:34:39 -0400 Subject: [PATCH 08/10] addresses comments --- .../common/detection_engine/constants.ts | 101 ---------------- .../rule_management/constants.ts | 108 ++++++++++++++++++ .../server/lib/telemetry/helpers.test.ts | 41 +++++++ .../server/lib/telemetry/helpers.ts | 9 +- .../server/lib/telemetry/types.ts | 6 + 5 files changed, 162 insertions(+), 103 deletions(-) create mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/constants.ts diff --git a/x-pack/solutions/security/plugins/security_solution/common/detection_engine/constants.ts b/x-pack/solutions/security/plugins/security_solution/common/detection_engine/constants.ts index 20a5cea67f71e..a9b3fc6ff6169 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/detection_engine/constants.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/detection_engine/constants.ts @@ -6,7 +6,6 @@ */ import type { Severity, Type } from '@kbn/securitysolution-io-ts-alerting-types'; -import type { RuleResponse } from '../api/detection_engine/model/rule_schema'; export enum RULE_PREVIEW_INVOCATION_COUNT { HOUR = 12, @@ -75,103 +74,3 @@ export const defaultRiskScoreBySeverity: Record = { high: RISK_SCORE_HIGH, critical: RISK_SCORE_CRITICAL, }; - -type AllKeys = U extends unknown ? keyof U : never; - -/** - * A list of all possible fields in the RuleResponse type mapped to whether or not the field is - * considered functional. We are defining "functional" to mean having a direct impact on how a - * rule executes. This means fields like `query` will be marked as functional while fields like - * `note` will be marked as non-functional. We are being conservative in our labeling of - * functional and only fields that have a 100% guaranteed impact on rule execution will be labeled - * as such. Fields like `index` that have a direct impact but don't necessarily change the alert - * rate (noise) of a rule will not be marked as functional. - * - * This categorization is intended to be used for telemetry purposes. - * - * More info here: - * x-pack/solutions/security/plugins/security_solution/docs/rfcs/detection_response/customized_rule_alert_telemetry.md - */ -export const FUNCTIONAL_FIELD_MAP: Record, boolean> = { - // Common fields - name: false, - description: false, - risk_score: false, - severity: false, - rule_name_override: false, - timestamp_override: false, - timestamp_override_fallback_disabled: false, - timeline_id: false, - timeline_title: false, - license: false, - note: false, - building_block_type: false, - investigation_fields: false, - version: false, - tags: false, - risk_score_mapping: false, - severity_mapping: false, - interval: false, - from: false, - to: false, - author: false, - false_positives: false, - references: false, - max_signals: false, - threat: false, - setup: false, - related_integrations: false, - required_fields: false, - type: true, - // Query, EQL, and ESQL rule type fields - query: true, - language: true, - index: false, - data_view_id: false, - filters: true, - event_category_override: true, - tiebreaker_field: true, - timestamp_field: true, - alert_suppression: true, - // Saved query rule type fields - saved_id: true, - // Threshold rule type fields - threshold: true, - // Threat match rule type fields - threat_query: true, - threat_mapping: true, - threat_index: false, - threat_filters: true, - threat_indicator_path: false, - threat_language: true, - // Maching learning rule type fields - anomaly_threshold: true, - machine_learning_job_id: true, - // New terms rule type fields - new_terms_fields: true, - history_window_start: true, - // Response fields - We don't use these fields for diffing purposes, setting the values to false - id: false, - rule_id: false, - rule_source: false, - outcome: false, - output_index: false, - namespace: false, - exceptions_list: false, - execution_summary: false, - actions: false, - throttle: false, - alias_purpose: false, - alias_target_id: false, - meta: false, - response_actions: false, - revision: false, - enabled: false, - items_per_search: false, - concurrent_searches: false, - immutable: false, - updated_at: false, - updated_by: false, - created_at: false, - created_by: false, -}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/constants.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/constants.ts new file mode 100644 index 0000000000000..4ea2accf1c21a --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/constants.ts @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RuleResponse } from '../../../../common/api/detection_engine/model/rule_schema'; + +type AllKeys = U extends unknown ? keyof U : never; + +/** + * A list of all possible fields in the RuleResponse type mapped to whether or not the field is + * considered functional. We are defining "functional" to mean having a direct impact on how a + * rule executes. This means fields like `query` will be marked as functional while fields like + * `note` will be marked as non-functional. We are being conservative in our labeling of + * functional and only fields that have a 100% guaranteed impact on rule execution will be labeled + * as such. Fields like `index` that have a direct impact but don't necessarily change the alert + * rate (noise) of a rule will not be marked as functional. + * + * This categorization is intended to be used for telemetry purposes. + * + * More info here: + * x-pack/solutions/security/plugins/security_solution/docs/rfcs/detection_response/customized_rule_alert_telemetry.md + */ +export const FUNCTIONAL_FIELD_MAP: Record, boolean> = { + // Common fields + name: false, + description: false, + risk_score: false, + severity: false, + rule_name_override: false, + timestamp_override: false, + timestamp_override_fallback_disabled: false, + timeline_id: false, + timeline_title: false, + license: false, + note: false, + building_block_type: false, + investigation_fields: false, + version: false, + tags: false, + risk_score_mapping: false, + severity_mapping: false, + interval: false, + from: false, + to: false, + author: false, + false_positives: false, + references: false, + max_signals: false, + threat: false, + setup: false, + related_integrations: false, + required_fields: false, + type: true, + // Query, EQL, and ESQL rule type fields + query: true, + language: true, + index: false, + data_view_id: false, + filters: true, + event_category_override: true, + tiebreaker_field: true, + timestamp_field: true, + alert_suppression: true, + // Saved query rule type fields + saved_id: true, + // Threshold rule type fields + threshold: true, + // Threat match rule type fields + threat_query: true, + threat_mapping: true, + threat_index: false, + threat_filters: true, + threat_indicator_path: false, + threat_language: true, + // Maching learning rule type fields + anomaly_threshold: true, + machine_learning_job_id: true, + // New terms rule type fields + new_terms_fields: true, + history_window_start: true, + // Response fields - We don't use these fields for diffing purposes, setting the values to false + id: false, + rule_id: false, + rule_source: false, + outcome: false, + output_index: false, + namespace: false, + exceptions_list: false, + execution_summary: false, + actions: false, + throttle: false, + alias_purpose: false, + alias_target_id: false, + meta: false, + response_actions: false, + revision: false, + enabled: false, + items_per_search: false, + concurrent_searches: false, + immutable: false, + updated_at: false, + updated_by: false, + created_at: false, + created_by: false, +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.test.ts index 2ec108a8afadd..ed26c32d4bbc3 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.test.ts @@ -1106,6 +1106,47 @@ describe('processDetectionRuleCustomizations', () => { expect(customizationsField).toBeUndefined(); }); + it("returns undefined if rule doesn't have `has_base_version` field", () => { + const customizationsField = processDetectionRuleCustomizations({ + 'kibana.alert.rule.parameters': { + // @ts-expect-error + rule_source: { + type: 'external', + is_customized: true, + customized_fields: [], + }, + }, + }); + expect(customizationsField).toBeUndefined(); + }); + + it("returns undefined if rule doesn't have `customized_fields` field", () => { + const customizationsField = processDetectionRuleCustomizations({ + 'kibana.alert.rule.parameters': { + // @ts-expect-error + rule_source: { + type: 'external', + is_customized: true, + has_base_version: true, + }, + }, + }); + expect(customizationsField).toBeUndefined(); + }); + + it("returns undefined if rule doesn't have `customized_fields` or `has_base_version` fields", () => { + const customizationsField = processDetectionRuleCustomizations({ + 'kibana.alert.rule.parameters': { + // @ts-expect-error + rule_source: { + type: 'external', + is_customized: true, + }, + }, + }); + expect(customizationsField).toBeUndefined(); + }); + it("returns undefined if rule doesn't have a base version", () => { const customizationsField = processDetectionRuleCustomizations({ 'kibana.alert.rule.parameters': { diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts index bb959fda96faf..44c333c04b6a2 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.ts @@ -13,7 +13,6 @@ import { set } from '@kbn/safer-lodash-set'; import type { Logger, LogMeta } from '@kbn/core/server'; import { sha256 } from 'js-sha256'; import type { estypes } from '@elastic/elasticsearch'; -import { FUNCTIONAL_FIELD_MAP } from '../../../common/detection_engine/constants'; import { copyAllowlistedFields, filterList } from './filterlists'; import type { PolicyConfig, PolicyData, SafeEndpointEvent } from '../../../common/endpoint/types'; import type { ITelemetryReceiver } from './receiver'; @@ -33,6 +32,7 @@ import type { TimelineTelemetryEvent, ValueListResponse, AnyObject, + PrebuiltRuleCustomizations, } from './types'; import type { TaskExecutionPeriod } from './task'; import { @@ -50,6 +50,7 @@ import { tlog as telemetryLogger, } from './telemetry_logger'; import type { RuleResponse } from '../../../common/api/detection_engine/model/rule_schema'; +import { FUNCTIONAL_FIELD_MAP } from '../detection_engine/rule_management/constants'; /** * Determines the when the last run was in order to execute to. @@ -391,12 +392,16 @@ export const processK8sUsernames = (clusterId: string, event: TelemetryEvent): T return event; }; -export const processDetectionRuleCustomizations = (event: TelemetryEvent) => { +export const processDetectionRuleCustomizations = ( + event: TelemetryEvent +): PrebuiltRuleCustomizations | undefined => { const ruleSource = event['kibana.alert.rule.parameters']?.rule_source; if ( !ruleSource || ruleSource.type === 'internal' || ruleSource.is_customized === false || + ruleSource.customized_fields == null || // New fields might not appear on alert documents + ruleSource.has_base_version == null || // New fields might not appear on alert documents ruleSource.has_base_version === false ) { return undefined; // Don't return anything if rule is not customized or base version doesn't exist diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/types.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/types.ts index 7545404647cea..2d6ac18aa2f11 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/types.ts @@ -90,6 +90,7 @@ export interface TelemetryEvent { 'kibana.alert.rule.parameters'?: { rule_source?: RuleSource; }; + customizations?: PrebuiltRuleCustomizations; } /** @@ -601,3 +602,8 @@ export interface FleetAgentResponse { } export type AnyObject = Record; + +export interface PrebuiltRuleCustomizations { + customized_fields: string[]; + num_functional_fields: number; +} From b220655179a89e6edd0e26917da497ca896039b9 Mon Sep 17 00:00:00 2001 From: Davis Plumlee Date: Wed, 1 Oct 2025 15:28:09 -0400 Subject: [PATCH 09/10] addresses comments --- .../security_solution/server/lib/telemetry/helpers.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.test.ts index ed26c32d4bbc3..6000ca30412a3 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.test.ts @@ -1106,7 +1106,7 @@ describe('processDetectionRuleCustomizations', () => { expect(customizationsField).toBeUndefined(); }); - it("returns undefined if rule doesn't have `has_base_version` field", () => { + it("returns undefined if rule_source doesn't have `has_base_version` field", () => { const customizationsField = processDetectionRuleCustomizations({ 'kibana.alert.rule.parameters': { // @ts-expect-error @@ -1120,7 +1120,7 @@ describe('processDetectionRuleCustomizations', () => { expect(customizationsField).toBeUndefined(); }); - it("returns undefined if rule doesn't have `customized_fields` field", () => { + it("returns undefined if rule_source doesn't have `customized_fields` field", () => { const customizationsField = processDetectionRuleCustomizations({ 'kibana.alert.rule.parameters': { // @ts-expect-error @@ -1134,7 +1134,7 @@ describe('processDetectionRuleCustomizations', () => { expect(customizationsField).toBeUndefined(); }); - it("returns undefined if rule doesn't have `customized_fields` or `has_base_version` fields", () => { + it("returns undefined if rule_source doesn't have `customized_fields` or `has_base_version` fields", () => { const customizationsField = processDetectionRuleCustomizations({ 'kibana.alert.rule.parameters': { // @ts-expect-error From 1caa2974aaa352fef894e357070751c108a177c7 Mon Sep 17 00:00:00 2001 From: Davis Plumlee Date: Wed, 1 Oct 2025 15:31:36 -0400 Subject: [PATCH 10/10] fix message --- .../security_solution/server/lib/telemetry/helpers.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.test.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.test.ts index 6000ca30412a3..591ecd9b16add 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.test.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/telemetry/helpers.test.ts @@ -1134,7 +1134,7 @@ describe('processDetectionRuleCustomizations', () => { expect(customizationsField).toBeUndefined(); }); - it("returns undefined if rule_source doesn't have `customized_fields` or `has_base_version` fields", () => { + it("returns undefined if rule_source doesn't have both `customized_fields` and `has_base_version` fields", () => { const customizationsField = processDetectionRuleCustomizations({ 'kibana.alert.rule.parameters': { // @ts-expect-error