[Streams] Make feature detection prompt route

oas_docs/bundle.json

@@ -83646,6 +83646,14 @@
             "schema": {
               "type": "string"
             }
+          },
+          {
+            "in": "query",
+            "name": "sampleDocsSize",
+            "required": false,
+            "schema": {
+              "type": "number"
+            }
           }
         ],
         "requestBody": {

oas_docs/bundle.serverless.json

@@ -82725,6 +82725,14 @@
             "schema": {
               "type": "string"
             }
+          },
+          {
+            "in": "query",
+            "name": "sampleDocsSize",
+            "required": false,
+            "schema": {
+              "type": "number"
+            }
           }
         ],
         "requestBody": {

oas_docs/output/kibana.serverless.yaml

@@ -71602,6 +71602,11 @@ paths:
           required: true
           schema:
             type: string
+        - in: query
+          name: sampleDocsSize
+          required: false
+          schema:
+            type: number
       requestBody:
         content:
           application/json:

oas_docs/output/kibana.yaml

@@ -75976,6 +75976,11 @@ paths:
           required: true
           schema:
             type: string
+        - in: query
+          name: sampleDocsSize
+          required: false
+          schema:
+            type: number
       requestBody:
         content:
           application/json:

packages/kbn-check-saved-objects-cli/current_fields.json

@@ -1167,6 +1167,9 @@
     "updated",
     "updatedBy"
   ],
+  "significant-events-prompts": [
+    "type"
+  ],
   "slo": [
     "budgetingMethod",
     "description",

packages/kbn-check-saved-objects-cli/current_mappings.json

@@ -3791,6 +3791,14 @@
       }
     }
   },
+  "significant-events-prompts": {
+    "dynamic": false,
+    "properties": {
+      "type": {
+        "type": "keyword"
+      }
+    }
+  },
   "slo": {
     "dynamic": false,
     "properties": {

src/core/packages/saved-objects/server-internal/src/object_types/index.ts

@@ -11,4 +11,4 @@ export { registerCoreObjectTypes } from './registration';
 
 // set minimum number of registered saved objects to ensure no object types are removed after 8.8
 // declared in internal implementation explicitly to prevent unintended changes.
-export const SAVED_OBJECT_TYPES_COUNT = 138 as const;
+export const SAVED_OBJECT_TYPES_COUNT = 139 as const;

src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts

@@ -175,6 +175,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "siem-ui-timeline": "439d5deaa90cc74d10a13804db1b40b9c66ccadfb9c6b34b2bdfcedab8d80e41",
         "siem-ui-timeline-note": "c81b789cea05ba59973639194825838d48c571b851cd8c359c1e4aacf2c8d2c5",
         "siem-ui-timeline-pinned-event": "abd9cf88c47bd4662a898ba8f8fb736b4d3da14975f0532830b17896983bf83d",
+        "significant-events-prompts": "5a04b7dcff90a8855a6eca97460abb0a6244127f9d9b0651bfb65ef7de9c07b8",
         "slo": "682c6d9e3ba7a489def2b824da547e26f67a16343747425a1efdf618c7fdb3e7",
         "slo-settings": "eaee24c76b1c02ba4ae1bf3742c1f5eca942a1662978f3420ec1b7f951746a32",
         "space": "c87c68da91a86291dfc4d8b406849979c97790bdc12187d9c9b7b119c393bcd7",
@@ -1099,6 +1100,10 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "siem-ui-timeline-pinned-event|schemas: da39a3ee5e6b4b0d3255bfef95601890afd80709",
         "siem-ui-timeline-pinned-event|7.16.0: 91da406ec7758e5787f971ac10d62d7006d5cde5",
         "==============================================================================",
+        "significant-events-prompts|global: 0895baced3e1359416eb0d5173817d0a2bf08728",
+        "significant-events-prompts|mappings: 67889ff480917c7fef440d3a55d48b730ab74dd2",
+        "significant-events-prompts|schemas: da39a3ee5e6b4b0d3255bfef95601890afd80709",
+        "============================================================================",
         "slo|global: 3bb1282c625b0cbdaf1317157a973f0eb263b13d",
         "slo|mappings: 98c4bcb86ae664a5d21cd03fd7df79068b378caa",
         "slo|schemas: da39a3ee5e6b4b0d3255bfef95601890afd80709",
@@ -1372,6 +1377,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "siem-ui-timeline": "10.1.0",
         "siem-ui-timeline-note": "10.0.0",
         "siem-ui-timeline-pinned-event": "10.0.0",
+        "significant-events-prompts": "10.0.0",
         "slo": "10.1.0",
         "slo-settings": "10.0.0",
         "space": "10.2.0",
@@ -1520,6 +1526,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "siem-ui-timeline": "10.1.0",
         "siem-ui-timeline-note": "7.16.0",
         "siem-ui-timeline-pinned-event": "7.16.0",
+        "significant-events-prompts": "10.0.0",
         "slo": "10.1.0",
         "slo-settings": "10.0.0",
         "space": "10.2.0",

x-pack/platform/packages/shared/kbn-ai-tools/src/tools/describe_dataset/index.ts

@@ -18,13 +18,15 @@ export async function describeDataset({
   index,
   kql,
   filter,
+  sampleDocsSize,
 }: {
   esClient: ElasticsearchClient;
   start: number;
   end: number;
   index: string | string[];
   kql?: string;
   filter?: QueryDslQueryContainer | QueryDslQueryContainer[];
+  sampleDocsSize?: number;
 }) {
   const [fieldCaps, hits] = await Promise.all([
     esClient.fieldCaps({
@@ -43,6 +45,7 @@ export async function describeDataset({
       end,
       kql,
       filter,
+      size: sampleDocsSize,
     }),
   ]);
 

x-pack/platform/packages/shared/kbn-streams-ai/src/features/identify_features.ts

@@ -17,7 +17,7 @@ import {
 } from '@kbn/streams-schema';
 import type { Condition } from '@kbn/streamlang';
 import { withSpan } from '@kbn/apm-utils';
-import { IdentifySystemsPrompt } from './prompt';
+import { createIdentifySystemsPrompt } from './prompt';
 import { clusterLogs } from '../cluster_logs/cluster_logs';
 import conditionSchemaText from '../shared/condition_schema.text';
 import { generateStreamDescription } from '../description/generate_description';
@@ -33,6 +33,7 @@ export interface IdentifyFeaturesOptions {
   logger: Logger;
   signal: AbortSignal;
   analysis: DocumentAnalysis;
+  systemPromptOverride?: string;
 }
 
 /**
@@ -54,6 +55,7 @@ export async function identifySystemFeatures({
   analysis,
   dropUnmapped = false,
   maxSteps: initialMaxSteps,
+  systemPromptOverride,
 }: IdentifyFeaturesOptions & {
   dropUnmapped?: boolean;
   maxSteps?: number;
@@ -94,7 +96,9 @@ export async function identifySystemFeatures({
         initial_clustering: JSON.stringify(initialClustering),
         condition_schema: conditionSchemaText,
       },
-      prompt: IdentifySystemsPrompt,
+      prompt: createIdentifySystemsPrompt({
+        systemPromptOverride,
+      }),
       inferenceClient,
       finalToolChoice: {
         function: 'finalize_systems',

x-pack/platform/packages/shared/kbn-streams-ai/src/features/prompt.ts

@@ -7,6 +7,7 @@
 import { createPrompt } from '@kbn/inference-common';
 import { z } from '@kbn/zod';
 import { merge } from 'lodash';
+import systemPromptDefault from '../significant_events/system_prompt.text';
 import systemPromptTemplate from './system_prompt.text';
 import userPromptTemplate from './user_prompt.text';
 
@@ -59,38 +60,48 @@ export interface FinalizeSystemsResponse {
   }>;
 }
 
-export const IdentifySystemsPrompt = createPrompt({
-  name: 'identify_systems',
-  input: z.object({
-    stream: z.object({
-      name: z.string(),
-      description: z.string(),
+export function createIdentifySystemsPrompt({
+  systemPromptOverride,
+}: {
+  systemPromptOverride?: string;
+} = {}) {
+  const systemPrompt = systemPromptOverride ?? systemPromptDefault;
+
+  return createPrompt({
+    name: 'identify_systems',
+    input: z.object({
+      stream: z.object({
+        name: z.string(),
+        description: z.string(),
+      }),
+      dataset_analysis: z.string(),
+      initial_clustering: z.string(),
+      condition_schema: z.string(),
     }),
-    dataset_analysis: z.string(),
-    initial_clustering: z.string(),
-    condition_schema: z.string(),
-  }),
-})
-  .version({
-    system: {
-      mustache: {
-        template: systemPromptTemplate,
-      },
-    },
-    template: {
-      mustache: {
-        template: userPromptTemplate,
+  })
+    .version({
+      system: {
+        mustache: {
+          template: systemPrompt,
+        },
       },
-    },
-    tools: {
-      validate_systems: {
-        description: `Validate systems before finalizing`,
-        schema: systemsSchema,
+      template: {
+        mustache: {
+          template: userPromptTemplate,
+        },
       },
-      finalize_systems: {
-        description: 'Finalize system identification',
-        schema: finalSystemsSchema,
+      tools: {
+        validate_systems: {
+          description: `Validate systems before finalizing`,
+          schema: systemsSchema,
+        },
+        finalize_systems: {
+          description: 'Finalize system identification',
+          schema: finalSystemsSchema,
+        },
       },
-    },
-  })
-  .get();
+    })
+    .get();
+}
+
+export { systemPromptTemplate as featuresSystemPromptTemplate };

x-pack/platform/packages/shared/kbn-streams-ai/src/significant_events/generate_significant_events.ts

@@ -14,7 +14,7 @@ import { conditionToQueryDsl } from '@kbn/streamlang';
 import { executeAsReasoningAgent } from '@kbn/inference-prompt-utils';
 import { fromKueryExpression } from '@kbn/es-query';
 import { withSpan } from '@kbn/apm-utils';
-import { GenerateSignificantEventsPrompt } from './prompt';
+import { createGenerateSignificantEventsPrompt } from './prompt';
 import type { SignificantEventType } from './types';
 import { sumTokens } from '../helpers/sum_tokens';
 
@@ -39,6 +39,9 @@ export async function generateSignificantEvents({
   esClient,
   inferenceClient,
   signal,
+  sampleDocsSize,
+  // optional overrides for templates
+  systemPromptOverride,
   logger,
 }: {
   stream: Streams.all.Definition;
@@ -49,6 +52,8 @@ export async function generateSignificantEvents({
   inferenceClient: BoundInferenceClient;
   signal: AbortSignal;
   logger: Logger;
+  sampleDocsSize?: number;
+  systemPromptOverride?: string;
 }): Promise<{
   queries: Query[];
   tokensUsed: ChatCompletionTokenCount;
@@ -58,6 +63,7 @@ export async function generateSignificantEvents({
   logger.trace('Describing dataset for significant event generation');
   const analysis = await withSpan('describe_dataset_for_significant_event_generation', () =>
     describeDataset({
+      sampleDocsSize,
       start,
       end,
       esClient,
@@ -66,6 +72,11 @@ export async function generateSignificantEvents({
     })
   );
 
+  // create the prompt instance using provided overrides (if any)
+  const prompt = createGenerateSignificantEventsPrompt({
+    systemPromptOverride,
+  });
+
   logger.trace('Generating significant events via reasoning agent');
   const response = await withSpan('generate_significant_events', () =>
     executeAsReasoningAgent({
@@ -75,7 +86,7 @@ export async function generateSignificantEvents({
         description: feature?.description || stream.description,
       },
       maxSteps: 4,
-      prompt: GenerateSignificantEventsPrompt,
+      prompt,
       inferenceClient,
       toolCallbacks: {
         add_queries: async (toolCall) => {