[APM] Refactor accessKnownApmEventFields usage in Diagnostic & Dependencies routes (#244173)

## Summary

Closes #244142

This PR changes usages of `unflattenKnownApmEventFields` to
`accessKnownApmEventFields` for the dependencies and diagnostics routes.
The behaviour of these routes/APIs should remain the same as before.

## How to test

- Ensure the Diagnostic Tool is enabled in Advanced Settings
- Go to Discover on Observability mode, select a traces index and click
through a trace to go to APM
- Go to Service Map for a trace, enable the Diagnostic tool on a
Service, go through process to check service connections are being
recognised correctly.
- Go to Dependencies, ensure they work correctly as before.

CI must also pass with no regressions.

---------

Co-authored-by: kibanamachine <[email protected]>

Author: Bluefinger

x-pack/solutions/observability/plugins/apm/server/routes/dependencies/get_metadata_for_dependency.ts

@@ -7,7 +7,7 @@
 
 import { rangeQuery } from '@kbn/observability-plugin/server';
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
-import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
+import { accessKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
 import { asMutableArray } from '../../../common/utils/as_mutable_array';
 import { maybe } from '../../../common/utils/maybe';
 import {
@@ -58,10 +58,12 @@ export async function getMetadataForDependency({
     },
   });
 
-  const sample = unflattenKnownApmEventFields(maybe(sampleResponse.hits.hits[0])?.fields);
+  const hitFields = maybe(sampleResponse.hits.hits[0])?.fields;
+
+  const sample = hitFields && accessKnownApmEventFields(hitFields);
 
   return {
-    spanType: sample?.span?.type,
-    spanSubtype: sample?.span?.subtype,
+    spanType: sample?.[SPAN_TYPE],
+    spanSubtype: sample?.[SPAN_SUBTYPE],
   };
 }

x-pack/solutions/observability/plugins/apm/server/routes/dependencies/get_top_dependency_spans.ts

@@ -8,7 +8,7 @@
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
 import { kqlQuery, rangeQuery, termQuery, termsQuery } from '@kbn/observability-plugin/server';
 import { keyBy } from 'lodash';
-import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
+import { accessKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
 import { asMutableArray } from '../../../common/utils/as_mutable_array';
 import {
   AGENT_NAME,
@@ -111,9 +111,11 @@ export async function getTopDependencySpans({
       },
       fields: topDedsRequiredFields,
     })
-  ).hits.hits.map((hit) => unflattenKnownApmEventFields(hit.fields, topDedsRequiredFields));
+  ).hits.hits.map((hit) =>
+    accessKnownApmEventFields(hit.fields).requireFields(topDedsRequiredFields)
+  );
 
-  const traceIds = spans.map((span) => span.trace.id);
+  const traceIds = spans.map((span) => span[TRACE_ID]);
 
   const txRequiredFields = asMutableArray([
     TRACE_ID,
@@ -139,25 +141,26 @@ export async function getTopDependencySpans({
         '@timestamp': 'desc',
       },
     })
-  ).hits.hits.map((hit) => unflattenKnownApmEventFields(hit.fields, txRequiredFields));
+  ).hits.hits.map((hit) => accessKnownApmEventFields(hit.fields).requireFields(txRequiredFields));
 
-  const transactionsByTraceId = keyBy(transactions, (transaction) => transaction.trace.id);
+  const transactionsByTraceId = keyBy(transactions, (transaction) => transaction[TRACE_ID]);
 
   return spans.map((span): DependencySpan => {
-    const transaction = maybe(transactionsByTraceId[span.trace!.id]);
+    const traceId = span[TRACE_ID];
+    const transaction = maybe(transactionsByTraceId[traceId]);
 
     return {
-      '@timestamp': new Date(span['@timestamp']).getTime(),
-      spanId: span.span.id,
-      spanName: span.span.name,
-      serviceName: span.service.name,
-      agentName: span.agent.name,
-      duration: span.span.duration.us,
-      traceId: span.trace.id,
-      outcome: (span.event?.outcome || EventOutcome.unknown) as EventOutcome,
-      transactionId: transaction?.transaction.id,
-      transactionType: transaction?.transaction.type,
-      transactionName: transaction?.transaction.name,
+      [AT_TIMESTAMP]: new Date(span[AT_TIMESTAMP]).getTime(),
+      spanId: span[SPAN_ID],
+      spanName: span[SPAN_NAME],
+      serviceName: span[SERVICE_NAME],
+      agentName: span[AGENT_NAME],
+      duration: span[SPAN_DURATION],
+      traceId,
+      outcome: (span[EVENT_OUTCOME] || EventOutcome.unknown) as EventOutcome,
+      transactionId: transaction?.[TRANSACTION_ID],
+      transactionType: transaction?.[TRANSACTION_TYPE],
+      transactionName: transaction?.[TRANSACTION_NAME],
     };
   });
 }

x-pack/solutions/observability/plugins/apm/server/routes/diagnostics/bundle/get_indices.ts

@@ -5,12 +5,12 @@
  * 2.0.
  */
 
-import { compact, uniq } from 'lodash';
 import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
 import type { APMIndices } from '@kbn/apm-sources-access-plugin/server';
+import { compactMap } from '../../../utils/compact_map';
 
 export function getApmIndexPatterns(indices: string[]) {
-  return uniq(indices.flatMap((index): string[] => index.split(',')));
+  return Array.from(new Set(indices.flatMap((index): string[] => index.split(','))));
 }
 
 export async function getIndicesAndIngestPipelines({
@@ -35,12 +35,12 @@ export async function getIndicesAndIngestPipelines({
     ignore_unavailable: true,
   });
 
-  const pipelineIds = compact(
-    uniq(Object.values(indices).map((index) => index.settings?.index?.default_pipeline))
-  ).join(',');
+  const pipelineIds = Array.from(
+    new Set(compactMap(Object.values(indices), (index) => index.settings?.index?.default_pipeline))
+  );
 
   const ingestPipelines = await esClient.ingest.getPipeline({
-    id: pipelineIds,
+    id: pipelineIds.join(','),
     filter_path: ['*.processors.grok.field', '*.processors.grok.patterns'],
   });
 

x-pack/solutions/observability/plugins/apm/server/routes/diagnostics/bundle/get_non_data_stream_indices.ts

@@ -8,6 +8,7 @@
 import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
 import type { APMIndices } from '@kbn/apm-sources-access-plugin/server';
 import { getApmIndexPatterns } from './get_indices';
+import { compactMap } from '../../../utils/compact_map';
 
 export async function getNonDataStreamIndices({
   esClient,
@@ -30,9 +31,14 @@ export async function getNonDataStreamIndices({
     ignore_unavailable: true,
   });
 
-  const nonDataStreamIndices = Object.entries(nonDataStreamIndicesResponse)
-    .filter(([indexName, { data_stream: dataStream }]): boolean => !dataStream)
-    .map(([indexName]): string => indexName);
+  const nonDataStreamIndices = compactMap(
+    Object.entries(nonDataStreamIndicesResponse),
+    ([indexName, { data_stream: dataStream }]) => {
+      if (!dataStream) {
+        return indexName;
+      }
+    }
+  );
 
   return nonDataStreamIndices;
 }

x-pack/solutions/observability/plugins/apm/server/routes/diagnostics/service_map/get_exit_spans_from_samples.ts

@@ -7,7 +7,7 @@
 
 import { termQuery, rangeQuery, termsQuery } from '@kbn/observability-plugin/server';
 import { ProcessorEvent } from '@kbn/observability-plugin/common';
-import { unflattenKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
+import { accessKnownApmEventFields } from '@kbn/apm-data-access-plugin/server/utils';
 import type { ESSearchResponse, ESSearchRequest } from '@kbn/es-types';
 import {
   SERVICE_NAME,
@@ -23,6 +23,7 @@ import {
 import type { APMEventClient } from '@kbn/apm-data-access-plugin/server';
 import type { ExitSpanFields } from '../../../../common/service_map_diagnostic_types';
 import { asMutableArray } from '../../../../common/utils/as_mutable_array';
+import { compactMap } from '../../../utils/compact_map';
 
 type DestinationsBySpanId = Map<string, string | undefined>;
 
@@ -83,24 +84,27 @@ export async function getExitSpans({
     },
   });
 
-  const apmExitSpans = (
-    response?.aggregations?.matching_destination_resources?.sample_docs?.hits?.hits.map((doc) => {
-      const fields = unflattenKnownApmEventFields(doc?.fields);
-
-      if (!fields?.parent?.id || !parentSpans.get(fields?.parent?.id)) {
-        return;
-      }
-
-      return {
-        destinationService: fields?.service?.name,
-        spanId: fields?.span?.id ?? '',
-        transactionId: fields?.transaction?.id ?? '',
-        serviceNodeName: fields?.service?.node?.name ?? '',
-        traceId: fields?.trace?.id ?? '',
-        agentName: fields?.agent?.name ?? '',
-      };
-    }) ?? []
-  ).filter((span): span is ExitSpanFields => !!span);
+  const hits =
+    response?.aggregations?.matching_destination_resources?.sample_docs?.hits?.hits ?? [];
+
+  const apmExitSpans = compactMap(hits, (hit) => {
+    const fields = hit?.fields && accessKnownApmEventFields(hit.fields);
+
+    const parentId = fields?.[PARENT_ID];
+
+    if (!fields || !parentId || !parentSpans.get(parentId)) {
+      return;
+    }
+
+    return {
+      destinationService: fields[SERVICE_NAME] ?? '',
+      spanId: fields[SPAN_ID] ?? '',
+      transactionId: fields[TRANSACTION_ID] ?? '',
+      serviceNodeName: fields[SERVICE_NODE_NAME] ?? '',
+      traceId: fields[TRACE_ID] ?? '',
+      agentName: fields[AGENT_NAME] ?? '',
+    } satisfies ExitSpanFields;
+  });
 
   return {
     apmExitSpans,
@@ -178,11 +182,10 @@ export async function getSourceSpanIds({
     sourceSpanIdsRawResponse: response,
     destinationsBySpanId:
       response.aggregations?.sample_docs?.buckets?.reduce((acc, bucket) => {
-        const event = unflattenKnownApmEventFields(
-          bucket.top_span_ids.hits.hits[0].fields,
-          requiredFields
-        );
-        acc.set(event.span.id, event.span?.destination?.service?.resource);
+        const event = accessKnownApmEventFields(
+          bucket.top_span_ids.hits.hits[0].fields
+        ).requireFields(requiredFields);
+        acc.set(event[SPAN_ID], event[SPAN_DESTINATION_SERVICE_RESOURCE]);
         return acc;
       }, destinationsBySpanId) ?? destinationsBySpanId,
   };

x-pack/solutions/observability/plugins/apm/server/utils/compact_map.test.ts

@@ -29,4 +29,23 @@ describe('compactMap', () => {
 
     expect(compactMap(values, (value) => value)).toEqual([0, '', false, {}]);
   });
+
+  it('maps iterable collections into a new array', () => {
+    const values = new Set([0, 1, 1, 2, 3, 4, 4, 5]);
+
+    expect(compactMap(values, (value) => (value % 2 === 0 ? `${value}` : undefined))).toEqual([
+      '0',
+      '2',
+      '4',
+    ]);
+
+    const valuesMap = new Map([
+      ['a', 1],
+      ['b', 2],
+      ['c', 3],
+      ['c', 4],
+    ]);
+
+    expect(compactMap(valuesMap, ([key, value]) => (key === 'b' ? null : value))).toEqual([1, 4]);
+  });
 });

x-pack/solutions/observability/plugins/apm/server/utils/compact_map.ts

@@ -6,10 +6,10 @@
  */
 
 /**
- * Takes an input array and a map function, outputting a new mapped array that removes
+ * Takes an iterable input and a map function, outputs a new mapped array that removes
  * all `null` or `undefined` slots.
  */
-export function compactMap<T, U>(array: T[], mapFn: (val: T) => U | undefined | null): U[] {
+export function compactMap<T, U>(array: Iterable<T>, mapFn: (val: T) => U | undefined | null): U[] {
   const mapped: U[] = [];
 
   for (const item of array) {