[Security Solution] Adds customized_fields and has_base_version fields to rule_source object schema

oas_docs/output/kibana.serverless.yaml

@@ -68986,10 +68986,28 @@ components:
         - endpoint_host_isolation_exceptions
         - endpoint_blocklists
       type: string
+    Security_Detections_API_ExternalRuleCustomizedFields:
+      description: An array of customized field names — that is, fields that the user has modified from their base value. Defaults to an empty array.
+      items:
+        type: object
+        properties:
+          field_name:
+            description: Name of a user-modified field in the rule object.
+            type: string
+        required:
+          - field_name
+      type: array
+    Security_Detections_API_ExternalRuleHasBaseVersion:
+      description: Determines whether an external/prebuilt rule has its original, unmodified version present when the calculation of its customization status is performed (`rule_source.is_customized` and `rule_source.customized_fields`).
+      type: boolean
     Security_Detections_API_ExternalRuleSource:
       description: Type of rule source for externally sourced rules, i.e. rules that have an external source, such as the Elastic Prebuilt rules repo.
       type: object
       properties:
+        customized_fields:
+          $ref: '#/components/schemas/Security_Detections_API_ExternalRuleCustomizedFields'
+        has_base_version:
+          $ref: '#/components/schemas/Security_Detections_API_ExternalRuleHasBaseVersion'
         is_customized:
           $ref: '#/components/schemas/Security_Detections_API_IsExternalRuleCustomized'
         type:
@@ -68999,6 +69017,8 @@ components:
       required:
         - type
         - is_customized
+        - has_base_version
+        - customized_fields
     Security_Detections_API_FindRulesSortField:
       enum:
         - created_at

oas_docs/output/kibana.yaml

@@ -82088,10 +82088,28 @@ components:
         - endpoint_host_isolation_exceptions
         - endpoint_blocklists
       type: string
+    Security_Detections_API_ExternalRuleCustomizedFields:
+      description: An array of customized field names — that is, fields that the user has modified from their base value. Defaults to an empty array.
+      items:
+        type: object
+        properties:
+          field_name:
+            description: Name of a user-modified field in the rule object.
+            type: string
+        required:
+          - field_name
+      type: array
+    Security_Detections_API_ExternalRuleHasBaseVersion:
+      description: Determines whether an external/prebuilt rule has its original, unmodified version present when the calculation of its customization status is performed (`rule_source.is_customized` and `rule_source.customized_fields`).
+      type: boolean
     Security_Detections_API_ExternalRuleSource:
       description: Type of rule source for externally sourced rules, i.e. rules that have an external source, such as the Elastic Prebuilt rules repo.
       type: object
       properties:
+        customized_fields:
+          $ref: '#/components/schemas/Security_Detections_API_ExternalRuleCustomizedFields'
+        has_base_version:
+          $ref: '#/components/schemas/Security_Detections_API_ExternalRuleHasBaseVersion'
         is_customized:
           $ref: '#/components/schemas/Security_Detections_API_IsExternalRuleCustomized'
         type:
@@ -82101,6 +82119,8 @@ components:
       required:
         - type
         - is_customized
+        - has_base_version
+        - customized_fields
     Security_Detections_API_FindRulesSortField:
       enum:
         - created_at

x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.gen.ts

@@ -89,6 +89,25 @@ export const IsRuleImmutable = z.boolean();
 export type IsExternalRuleCustomized = z.infer<typeof IsExternalRuleCustomized>;
 export const IsExternalRuleCustomized = z.boolean();
 
+/**
+ * Determines whether an external/prebuilt rule has its original, unmodified version present when the calculation of its customization status is performed (`rule_source.is_customized` and `rule_source.customized_fields`).
+ */
+export type ExternalRuleHasBaseVersion = z.infer<typeof ExternalRuleHasBaseVersion>;
+export const ExternalRuleHasBaseVersion = z.boolean();
+
+/**
+ * An array of customized field names — that is, fields that the user has modified from their base value. Defaults to an empty array.
+ */
+export type ExternalRuleCustomizedFields = z.infer<typeof ExternalRuleCustomizedFields>;
+export const ExternalRuleCustomizedFields = z.array(
+  z.object({
+    /**
+     * Name of a user-modified field in the rule object.
+     */
+    field_name: z.string(),
+  })
+);
+
 /**
  * Type of rule source for internally sourced rules, i.e. created within the Kibana apps.
  */
@@ -104,6 +123,8 @@ export type ExternalRuleSource = z.infer<typeof ExternalRuleSource>;
 export const ExternalRuleSource = z.object({
   type: z.literal('external'),
   is_customized: IsExternalRuleCustomized,
+  has_base_version: ExternalRuleHasBaseVersion,
+  customized_fields: ExternalRuleCustomizedFields,
 });
 
 /**

x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/model/rule_schema/common_attributes.schema.yaml

@@ -70,6 +70,22 @@ components:
       type: boolean
       description: Determines whether an external/prebuilt rule has been customized by the user (i.e. any of its fields have been modified and diverged from the base value).
 
+    ExternalRuleHasBaseVersion:
+      type: boolean
+      description: Determines whether an external/prebuilt rule has its original, unmodified version present when the calculation of its customization status is performed (`rule_source.is_customized` and `rule_source.customized_fields`).
+
+    ExternalRuleCustomizedFields:
+      type: array
+      description: An array of customized field names — that is, fields that the user has modified from their base value. Defaults to an empty array.
+      items:
+        type: object
+        properties:
+          field_name:
+            type: string
+            description: Name of a user-modified field in the rule object.
+        required:
+          - field_name
+
     InternalRuleSource:
       description: Type of rule source for internally sourced rules, i.e. created within the Kibana apps.
       type: object
@@ -91,9 +107,15 @@ components:
             - external
         is_customized:
           $ref: '#/components/schemas/IsExternalRuleCustomized'
+        has_base_version:
+          $ref: '#/components/schemas/ExternalRuleHasBaseVersion'
+        customized_fields:
+          $ref: '#/components/schemas/ExternalRuleCustomizedFields'
       required:
         - type
         - is_customized
+        - has_base_version
+        - customized_fields
 
     RuleSource:
       description: Discriminated union that determines whether the rule is internally sourced (created within the Kibana app) or has an external source, such as the Elastic Prebuilt rules repo.

x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_response_schema.test.ts

@@ -259,6 +259,8 @@ describe('rule_source', () => {
     payload.rule_source = {
       type: 'external',
       is_customized: true,
+      customized_fields: [{ field_name: 'name' }],
+      has_base_version: true,
     };
 
     const result = RuleResponse.safeParse(payload);

x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/rule_management/import_rules/rule_to_import.test.ts

@@ -1056,6 +1056,8 @@ describe('RuleToImport', () => {
         rule_source: {
           type: 'external',
           is_customized: true,
+          customized_fields: [{ field_name: 'name' }],
+          has_base_version: true,
         },
       });
 

x-pack/solutions/security/plugins/security_solution/common/detection_engine/constants.ts

@@ -6,6 +6,7 @@
  */
 
 import type { Severity, Type } from '@kbn/securitysolution-io-ts-alerting-types';
+import type { RuleResponse } from '../api/detection_engine/model/rule_schema';
 
 export enum RULE_PREVIEW_INVOCATION_COUNT {
   HOUR = 12,
@@ -74,3 +75,103 @@ export const defaultRiskScoreBySeverity: Record<Severity, number> = {
   high: RISK_SCORE_HIGH,
   critical: RISK_SCORE_CRITICAL,
 };
+
+type AllKeys<U> = U extends unknown ? keyof U : never;
+
+/**
+ * A list of all possible fields in the RuleResponse type mapped to whether or not the field is
+ * considered functional. We are defining "functional" to mean having a direct impact on how a
+ * rule executes. This means fields like `query` will be marked as functional while fields like
+ * `note` will be marked as non-functional. We are being conservative in our labeling of
+ * functional and only fields that have a 100% guaranteed impact on rule execution will be labeled
+ * as such. Fields like `index` that have a direct impact but don't necessarily change the alert
+ * rate (noise) of a rule will not be marked as functional.
+ *
+ * This categorization is intended to be used for telemetry purposes.
+ *
+ * More info here:
+ * x-pack/solutions/security/plugins/security_solution/docs/rfcs/detection_response/customized_rule_alert_telemetry.md
+ */
+export const FUNCTIONAL_FIELD_MAP: Record<AllKeys<RuleResponse>, boolean> = {
+  // Common fields
+  name: false,
+  description: false,
+  risk_score: false,
+  severity: false,
+  rule_name_override: false,
+  timestamp_override: false,
+  timestamp_override_fallback_disabled: false,
+  timeline_id: false,
+  timeline_title: false,
+  license: false,
+  note: false,
+  building_block_type: false,
+  investigation_fields: false,
+  version: false,
+  tags: false,
+  risk_score_mapping: false,
+  severity_mapping: false,
+  interval: false,
+  from: false,
+  to: false,
+  author: false,
+  false_positives: false,
+  references: false,
+  max_signals: false,
+  threat: false,
+  setup: false,
+  related_integrations: false,
+  required_fields: false,
+  type: true,
+  // Query, EQL, and ESQL rule type fields
+  query: true,
+  language: true,
+  index: false,
+  data_view_id: false,
+  filters: true,
+  event_category_override: true,
+  tiebreaker_field: true,
+  timestamp_field: true,
+  alert_suppression: true,
+  // Saved query rule type fields
+  saved_id: true,
+  // Threshold rule type fields
+  threshold: true,
+  // Threat match rule type fields
+  threat_query: true,
+  threat_mapping: true,
+  threat_index: false,
+  threat_filters: true,
+  threat_indicator_path: false,
+  threat_language: true,
+  // Maching learning rule type fields
+  anomaly_threshold: true,
+  machine_learning_job_id: true,
+  // New terms rule type fields
+  new_terms_fields: true,
+  history_window_start: true,
+  // Response fields - We don't use these fields for diffing purposes, setting the values to false
+  id: false,
+  rule_id: false,
+  rule_source: false,
+  outcome: false,
+  output_index: false,
+  namespace: false,
+  exceptions_list: false,
+  execution_summary: false,
+  actions: false,
+  throttle: false,
+  alias_purpose: false,
+  alias_target_id: false,
+  meta: false,
+  response_actions: false,
+  revision: false,
+  enabled: false,
+  items_per_search: false,
+  concurrent_searches: false,
+  immutable: false,
+  updated_at: false,
+  updated_by: false,
+  created_at: false,
+  created_by: false,
+};

x-pack/solutions/security/plugins/security_solution/docs/openapi/ess/security_solution_detections_api_2023_10_31.bundled.schema.yaml

@@ -6315,12 +6315,36 @@ components:
         - endpoint_host_isolation_exceptions
         - endpoint_blocklists
       type: string
+    ExternalRuleCustomizedFields:
+      description: >-
+        An array of customized field names — that is, fields that the user has
+        modified from their base value. Defaults to an empty array.
+      items:
+        type: object
+        properties:
+          field_name:
+            description: Name of a user-modified field in the rule object.
+            type: string
+        required:
+          - field_name
+      type: array
+    ExternalRuleHasBaseVersion:
+      description: >-
+        Determines whether an external/prebuilt rule has its original,
+        unmodified version present when the calculation of its customization
+        status is performed (`rule_source.is_customized` and
+        `rule_source.customized_fields`).
+      type: boolean
     ExternalRuleSource:
       description: >-
         Type of rule source for externally sourced rules, i.e. rules that have
         an external source, such as the Elastic Prebuilt rules repo.
       type: object
       properties:
+        customized_fields:
+          $ref: '#/components/schemas/ExternalRuleCustomizedFields'
+        has_base_version:
+          $ref: '#/components/schemas/ExternalRuleHasBaseVersion'
         is_customized:
           $ref: '#/components/schemas/IsExternalRuleCustomized'
         type:
@@ -6330,6 +6354,8 @@ components:
       required:
         - type
         - is_customized
+        - has_base_version
+        - customized_fields
     FindRulesSortField:
       enum:
         - created_at

x-pack/solutions/security/plugins/security_solution/docs/openapi/serverless/security_solution_detections_api_2023_10_31.bundled.schema.yaml

@@ -5645,12 +5645,36 @@ components:
         - endpoint_host_isolation_exceptions
         - endpoint_blocklists
       type: string
+    ExternalRuleCustomizedFields:
+      description: >-
+        An array of customized field names — that is, fields that the user has
+        modified from their base value. Defaults to an empty array.
+      items:
+        type: object
+        properties:
+          field_name:
+            description: Name of a user-modified field in the rule object.
+            type: string
+        required:
+          - field_name
+      type: array
+    ExternalRuleHasBaseVersion:
+      description: >-
+        Determines whether an external/prebuilt rule has its original,
+        unmodified version present when the calculation of its customization
+        status is performed (`rule_source.is_customized` and
+        `rule_source.customized_fields`).
+      type: boolean
     ExternalRuleSource:
       description: >-
         Type of rule source for externally sourced rules, i.e. rules that have
         an external source, such as the Elastic Prebuilt rules repo.
       type: object
       properties:
+        customized_fields:
+          $ref: '#/components/schemas/ExternalRuleCustomizedFields'
+        has_base_version:
+          $ref: '#/components/schemas/ExternalRuleHasBaseVersion'
         is_customized:
           $ref: '#/components/schemas/IsExternalRuleCustomized'
         type:
@@ -5660,6 +5684,8 @@ components:
       required:
         - type
         - is_customized
+        - has_base_version
+        - customized_fields
     FindRulesSortField:
       enum:
         - created_at