[APM] Add query to check for overflow bucket in service groups (#159990)

## Summary

Closes #157479

This PR adds a check in Service Groups to display overflow bucket if
Service metrics have been overflow


![image](https://github.com/elastic/kibana/assets/7416358/714ff445-0c10-4239-b4b6-0064019178ca)

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/apm/server/lib/service_group_query_with_overflow.ts

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import { kqlQuery, termQuery } from '@kbn/observability-plugin/server';
+import { SERVICE_NAME } from '../../common/es_fields/apm';
+import { ServiceGroup } from '../../common/service_groups';
+
+export function serviceGroupWithOverflowQuery(
+  serviceGroup?: ServiceGroup | null
+): QueryDslQueryContainer[] {
+  if (serviceGroup) {
+    const serviceGroupQuery = kqlQuery(serviceGroup?.kuery);
+    const otherBucketQuery = termQuery(SERVICE_NAME, '_other');
+
+    return [
+      {
+        bool: {
+          should: [
+            {
+              bool: {
+                filter: serviceGroupQuery,
+              },
+            },
+            {
+              bool: {
+                filter: otherBucketQuery,
+              },
+            },
+          ],
+        },
+      },
+    ];
+  }
+  return [];
+}

x-pack/plugins/apm/server/routes/services/get_services/get_service_alerts.ts

@@ -24,8 +24,8 @@ import { SERVICE_NAME } from '../../../../common/es_fields/apm';
 import { ServiceGroup } from '../../../../common/service_groups';
 import { ApmAlertsClient } from '../../../lib/helpers/get_apm_alerts_client';
 import { environmentQuery } from '../../../../common/utils/environment_query';
-import { serviceGroupQuery } from '../../../lib/service_group_query';
 import { MAX_NUMBER_OF_SERVICES } from './get_services_items';
+import { serviceGroupWithOverflowQuery } from '../../../lib/service_group_query_with_overflow';
 
 interface ServiceAggResponse {
   buckets: Array<
@@ -69,7 +69,7 @@ export async function getServicesAlerts({
           ...termQuery(ALERT_STATUS, ALERT_STATUS_ACTIVE),
           ...rangeQuery(start, end),
           ...kqlQuery(kuery),
-          ...serviceGroupQuery(serviceGroup),
+          ...serviceGroupWithOverflowQuery(serviceGroup),
           ...termQuery(SERVICE_NAME, serviceName),
           ...environmentQuery(environment),
         ],

x-pack/plugins/apm/server/routes/services/get_services/get_service_transaction_stats.ts

@@ -27,8 +27,8 @@ import {
   calculateFailedTransactionRate,
   getOutcomeAggregation,
 } from '../../../lib/helpers/transaction_error_rate';
-import { serviceGroupQuery } from '../../../lib/service_group_query';
 import { maybe } from '../../../../common/utils/maybe';
+import { serviceGroupWithOverflowQuery } from '../../../lib/service_group_query_with_overflow';
 
 interface AggregationParams {
   environment: string;
@@ -102,7 +102,7 @@ export async function getServiceTransactionStats({
               ...rangeQuery(start, end),
               ...environmentQuery(environment),
               ...kqlQuery(kuery),
-              ...serviceGroupQuery(serviceGroup),
+              ...serviceGroupWithOverflowQuery(serviceGroup),
             ],
           },
         },

x-pack/plugins/apm/server/routes/services/get_services/get_services_without_transactions.ts

@@ -14,12 +14,12 @@ import {
   SERVICE_NAME,
 } from '../../../../common/es_fields/apm';
 import { environmentQuery } from '../../../../common/utils/environment_query';
-import { serviceGroupQuery } from '../../../lib/service_group_query';
 import { ServiceGroup } from '../../../../common/service_groups';
 import { RandomSampler } from '../../../lib/helpers/get_random_sampler';
 import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client';
 import { ApmDocumentType } from '../../../../common/document_type';
 import { RollupInterval } from '../../../../common/rollup';
+import { serviceGroupWithOverflowQuery } from '../../../lib/service_group_query_with_overflow';
 
 export interface ServicesWithoutTransactionsResponse {
   services: Array<{
@@ -82,7 +82,7 @@ export async function getServicesWithoutTransactions({
               ...rangeQuery(start, end),
               ...environmentQuery(environment),
               ...kqlQuery(kuery),
-              ...serviceGroupQuery(serviceGroup),
+              ...serviceGroupWithOverflowQuery(serviceGroup),
             ],
           },
         },

x-pack/test/apm_api_integration/tests/service_groups/service_group_with_overflow/es_utils.ts

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export function createServiceTransactionMetricsDocs({
+  time,
+  service,
+  agentName,
+  overflowCount,
+}: {
+  time: number;
+  service: {
+    name: string;
+    environment?: string;
+    language?: string;
+  };
+  agentName?: string;
+  overflowCount?: number;
+}) {
+  return {
+    processor: {
+      event: 'metric' as const,
+    },
+    '@timestamp': new Date(time).toISOString(),
+    ...(agentName && {
+      agent: {
+        name: agentName,
+      },
+    }),
+    event: {
+      ingested: new Date(time).toISOString(),
+    },
+    metricset: {
+      name: 'service_transaction',
+    },
+    service,
+    ...(overflowCount && {
+      service_transaction: {
+        aggregation: {
+          overflow_count: overflowCount,
+        },
+      },
+    }),
+    observer: {
+      version: '8.9.0',
+    },
+  };
+}

x-pack/test/apm_api_integration/tests/service_groups/service_group_with_overflow/generate_data.ts

@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { apm, timerange } from '@kbn/apm-synthtrace-client';
+import type { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace';
+
+export async function generateData({
+  synthtraceEsClient,
+  start,
+  end,
+}: {
+  synthtraceEsClient: ApmSynthtraceEsClient;
+  start: string;
+  end: string;
+}) {
+  const synthServices = [
+    apm
+      .service({ name: 'synth-go', environment: 'testing', agentName: 'go' })
+      .instance('instance-1'),
+    apm
+      .service({ name: 'synth-java', environment: 'testing', agentName: 'java' })
+      .instance('instance-2'),
+  ];
+
+  await synthtraceEsClient.index(
+    synthServices.map((service) =>
+      timerange(start, end)
+        .interval('5m')
+        .rate(1)
+        .generator((timestamp) =>
+          service
+            .transaction({
+              transactionName: 'GET /api/product/list',
+              transactionType: 'request',
+            })
+            .duration(2000)
+            .timestamp(timestamp)
+            .children(
+              service
+                .span({
+                  spanName: '/_search',
+                  spanType: 'db',
+                  spanSubtype: 'elasticsearch',
+                })
+                .destination('elasticsearch')
+                .duration(100)
+                .success()
+                .timestamp(timestamp)
+            )
+            .errors(service.error({ message: 'error 1', type: 'foo' }).timestamp(timestamp))
+        )
+    )
+  );
+}

x-pack/test/apm_api_integration/tests/service_groups/service_group_with_overflow/service_group_with_overflow.spec.ts

@@ -0,0 +1,104 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import expect from '@kbn/expect';
+import { ValuesType } from 'utility-types';
+import { ENVIRONMENT_ALL } from '@kbn/apm-plugin/common/environment_filter_values';
+import { ApmDocumentType } from '@kbn/apm-plugin/common/document_type';
+import { RollupInterval } from '@kbn/apm-plugin/common/rollup';
+import { FtrProviderContext } from '../../../common/ftr_provider_context';
+import { createServiceGroupApi, deleteAllServiceGroups } from '../service_groups_api_methods';
+import { createServiceTransactionMetricsDocs } from './es_utils';
+import { generateData } from './generate_data';
+
+export default function ApiTest({ getService }: FtrProviderContext) {
+  const registry = getService('registry');
+  const apmApiClient = getService('apmApiClient');
+  const es = getService('es');
+  const synthtraceEsClient = getService('synthtraceEsClient');
+
+  registry.when(
+    'Display overflow bucket in Service Groups',
+    { config: 'basic', archives: [] },
+    () => {
+      const indexName = 'metrics-apm.service_transaction.1m-default';
+      const start = '2023-06-21T06:50:15.910Z';
+      const end = '2023-06-21T06:59:15.910Z';
+      const startTime = new Date(start).getTime() + 1000;
+      const OVERFLOW_SERVICE_NAME = '_other';
+      let serviceGroupId: string;
+
+      after(async () => {
+        await deleteAllServiceGroups(apmApiClient);
+        synthtraceEsClient.clean();
+      });
+
+      before(async () => {
+        await generateData({ start, end, synthtraceEsClient });
+
+        const docs = [
+          createServiceTransactionMetricsDocs({
+            time: startTime,
+            service: {
+              name: OVERFLOW_SERVICE_NAME,
+            },
+            overflowCount: 13,
+          }),
+        ];
+
+        const bulkActions = docs.reduce(
+          (prev, doc) => {
+            return [...prev, { create: { _index: indexName } }, doc];
+          },
+          [] as Array<
+            | {
+                create: {
+                  _index: string;
+                };
+              }
+            | ValuesType<typeof docs>
+          >
+        );
+
+        await es.bulk({
+          body: bulkActions,
+          refresh: 'wait_for',
+        });
+
+        const serviceGroup = {
+          groupName: 'overflowGroup',
+          kuery: 'service.name: synth-go or service.name: synth-java',
+        };
+        const createResponse = await createServiceGroupApi({ apmApiClient, ...serviceGroup });
+        expect(createResponse.status).to.be(200);
+        serviceGroupId = createResponse.body.id;
+      });
+
+      it('get the overflow bucket even though its not added explicitly in the Service Group', async () => {
+        const response = await apmApiClient.readUser({
+          endpoint: `GET /internal/apm/services`,
+          params: {
+            query: {
+              start,
+              end,
+              environment: ENVIRONMENT_ALL.value,
+              kuery: '',
+              serviceGroup: serviceGroupId,
+              probability: 1,
+              documentType: ApmDocumentType.ServiceTransactionMetric,
+              rollupInterval: RollupInterval.OneMinute,
+            },
+          },
+        });
+
+        const overflowBucket = response.body.items.find(
+          (service) => service.serviceName === OVERFLOW_SERVICE_NAME
+        );
+        expect(overflowBucket?.serviceName).to.equal(OVERFLOW_SERVICE_NAME);
+      });
+    }
+  );
+}