[Infra] Create endpoint to verify if there is data (#189470)

closes [189247](#189247) ## Summary Create an endpoint to return whether there is data for modules passed via `modules` query param. When `modules` is not passed, the query will just check if there is any data in the index patterns configured in the Settings page ### How to test - Start a local Kibana instance - Run `node scripts/synthtrace infra_hosts_with_apm_hosts --live` - Run in the Dev tools: - `GET kbn:/api/metrics/source/hasData?modules=kubernetes&modules=system` - `GET kbn:/api/metrics/source/hasData?modules=system` - `GET kbn:/api/metrics/source/hasData?modules=kubernetes&modules=system&modules=kafka&modules=aws&modules=azure` - should return 400 - `GET kbn:/api/metrics/source/hasData`
elastic · Jul 30, 2024 · ea64b47 · ea64b47
1 parent d86e139
commit ea64b47
Show file tree

Hide file tree

Showing 9 changed files with 166 additions and 38 deletions.
diff --git a/x-pack/plugins/observability_solution/infra/common/http_api/infra/get_infra_metrics.ts b/x-pack/plugins/observability_solution/infra/common/http_api/infra/get_infra_metrics.ts
@@ -48,7 +48,6 @@ export const GetInfraMetricsRequestBodyPayloadRT = rt.intersection([
     type: rt.literal('host'),
     limit: rt.union([inRangeRt(1, 500), createLiteralValueFromUndefinedRT(20)]),
     metrics: rt.array(rt.type({ type: InfraMetricTypeRT })),
-    sourceId: rt.string,
     range: RangeRT,
   }),
 ]);

diff --git a/x-pack/plugins/observability_solution/infra/common/metrics_sources/get_has_data.ts b/x-pack/plugins/observability_solution/infra/common/metrics_sources/get_has_data.ts
@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import * as rt from 'io-ts';
+
+export const getHasDataQueryParamsRT = rt.partial({
+  // Integrations `event.module` value
+  modules: rt.union([rt.string, rt.array(rt.string)]),
+});
+
+export const getHasDataResponseRT = rt.partial({
+  hasData: rt.boolean,
+});
+
+export type GetHasDataQueryParams = rt.TypeOf<typeof getHasDataQueryParamsRT>;
+export type GetHasDataResponse = rt.TypeOf<typeof getHasDataResponseRT>;
diff --git a/...k/plugins/observability_solution/infra/public/pages/metrics/hosts/hooks/use_hosts_view.ts b/...k/plugins/observability_solution/infra/public/pages/metrics/hosts/hooks/use_hosts_view.ts
@@ -17,7 +17,6 @@ import createContainer from 'constate';
 import { BoolQuery } from '@kbn/es-query';
 import { isPending, useFetcher } from '../../../../hooks/use_fetcher';
 import { useKibanaContextForPlugin } from '../../../../hooks/use_kibana';
-import { useSourceContext } from '../../../../containers/metrics_source';
 import { useUnifiedSearchContext } from './use_unified_search';
 import {
   GetInfraMetricsRequestBodyPayload,
@@ -39,7 +38,6 @@ const HOST_TABLE_METRICS: Array<{ type: InfraAssetMetricType }> = [
 const BASE_INFRA_METRICS_PATH = '/api/metrics/infra';
 
 export const useHostsView = () => {
-  const { sourceId } = useSourceContext();
   const {
     services: { telemetry },
   } = useKibanaContextForPlugin();
@@ -50,10 +48,9 @@ export const useHostsView = () => {
       createInfraMetricsRequest({
         dateRange: parsedDateRange,
         esQuery: buildQuery(),
-        sourceId,
         limit: searchCriteria.limit,
       }),
-    [buildQuery, parsedDateRange, sourceId, searchCriteria.limit]
+    [buildQuery, parsedDateRange, searchCriteria.limit]
   );
 
   const { data, error, status } = useFetcher(
@@ -94,12 +91,10 @@ export const [HostsViewProvider, useHostsViewContext] = HostsView;
 
 const createInfraMetricsRequest = ({
   esQuery,
-  sourceId,
   dateRange,
   limit,
 }: {
   esQuery: { bool: BoolQuery };
-  sourceId: string;
   dateRange: StringDateRange;
   limit: number;
 }): GetInfraMetricsRequestBodyPayload => ({
@@ -111,5 +106,4 @@ const createInfraMetricsRequest = ({
   },
   metrics: HOST_TABLE_METRICS,
   limit,
-  sourceId,
 });
diff --git a/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.ts b/x-pack/plugins/observability_solution/infra/server/lib/helpers/get_infra_metrics_client.ts
@@ -6,8 +6,8 @@
  */
 import type { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types';
 import type { KibanaRequest } from '@kbn/core/server';
+import { MetricsDataClient } from '@kbn/metrics-data-access-plugin/server';
 import type { InfraPluginRequestHandlerContext } from '../../types';
-import { InfraSources } from '../sources';
 import { KibanaFramework } from '../adapters/framework/kibana_framework_adapter';
 
 type RequiredParams = Omit<ESSearchRequest, 'index'> & {
@@ -20,20 +20,21 @@ type RequiredParams = Omit<ESSearchRequest, 'index'> & {
 export type InfraMetricsClient = Awaited<ReturnType<typeof getInfraMetricsClient>>;
 
 export async function getInfraMetricsClient({
-  sourceId,
   framework,
-  infraSources,
+  metricsDataAccess,
   requestContext,
   request,
 }: {
-  sourceId: string;
   framework: KibanaFramework;
-  infraSources: InfraSources;
+  metricsDataAccess: MetricsDataClient;
   requestContext: InfraPluginRequestHandlerContext;
   request?: KibanaRequest;
 }) {
-  const soClient = (await requestContext.core).savedObjects.getClient();
-  const source = await infraSources.getSourceConfiguration(soClient, sourceId);
+  const coreContext = await requestContext.core;
+  const savedObjectsClient = coreContext.savedObjects.client;
+  const indices = await metricsDataAccess.getMetricIndices({
+    savedObjectsClient,
+  });
 
   return {
     search<TDocument, TParams extends RequiredParams>(
@@ -44,7 +45,7 @@ export async function getInfraMetricsClient({
         'search',
         {
           ...searchParams,
-          index: source.configuration.metricAlias,
+          index: indices,
         },
         request
       ) as Promise<any>;

diff --git a/x-pack/plugins/observability_solution/infra/server/routes/infra/index.ts b/x-pack/plugins/observability_solution/infra/server/routes/infra/index.ts
@@ -47,9 +47,8 @@ export const initInfraAssetRoutes = (libs: InfraBackendLibs) => {
         const infraMetricsClient = await getInfraMetricsClient({
           framework,
           request,
-          infraSources: libs.sources,
+          metricsDataAccess: libs.metricsClient,
           requestContext,
-          sourceId: params.sourceId,
         });
 
         const alertsClient = await getInfraAlertsClient({
@@ -102,15 +101,14 @@ export const initInfraAssetRoutes = (libs: InfraBackendLibs) => {
       const body: GetInfraAssetCountRequestBodyPayload = request.body;
       const params: GetInfraAssetCountRequestParamsPayload = request.params;
       const { assetType } = params;
-      const { query, from, to, sourceId } = body;
+      const { query, from, to } = body;
 
       try {
         const infraMetricsClient = await getInfraMetricsClient({
           framework,
           request,
-          infraSources: libs.sources,
+          metricsDataAccess: libs.metricsClient,
           requestContext,
-          sourceId,
         });
 
         const assetCount = await getHostsCount({

diff --git a/x-pack/plugins/observability_solution/infra/server/routes/metrics_sources/index.ts b/x-pack/plugins/observability_solution/infra/server/routes/metrics_sources/index.ts
@@ -8,6 +8,13 @@
 import { schema } from '@kbn/config-schema';
 import Boom from '@hapi/boom';
 import { createRouteValidationFunction } from '@kbn/io-ts-utils';
+import { termsQuery } from '@kbn/observability-plugin/server';
+import { castArray } from 'lodash';
+import { EVENT_MODULE, METRICSET_MODULE } from '../../../common/constants';
+import {
+  getHasDataQueryParamsRT,
+  getHasDataResponseRT,
+} from '../../../common/metrics_sources/get_has_data';
 import { InfraBackendLibs } from '../../lib/infra_types';
 import { hasData } from '../../lib/sources/has_data';
 import { createSearchClient } from '../../lib/create_search_client';
@@ -19,13 +26,16 @@ import {
 } from '../../../common/metrics_sources';
 import { InfraSource, InfraSourceIndexField } from '../../lib/sources';
 import { InfraPluginRequestHandlerContext } from '../../types';
+import { getInfraMetricsClient } from '../../lib/helpers/get_infra_metrics_client';
 
 const defaultStatus = {
   indexFields: [],
   metricIndicesExist: false,
   remoteClustersExist: false,
 };
 
+const MAX_MODULES = 5;
+
 export const initMetricsSourceConfigurationRoutes = (libs: InfraBackendLibs) => {
   const { framework, logger } = libs;
 
@@ -204,6 +214,75 @@ export const initMetricsSourceConfigurationRoutes = (libs: InfraBackendLibs) =>
       });
     }
   );
+
+  framework.registerRoute(
+    {
+      method: 'get',
+      path: '/api/metrics/source/hasData',
+      validate: {
+        query: createRouteValidationFunction(getHasDataQueryParamsRT),
+      },
+    },
+    async (requestContext, request, response) => {
+      try {
+        const modules = castArray(request.query.modules);
+
+        if (modules.length > MAX_MODULES) {
+          throw Boom.badRequest(
+            `'modules' size is greater than maximum of ${MAX_MODULES} allowed.`
+          );
+        }
+
+        const infraMetricsClient = await getInfraMetricsClient({
+          framework,
+          request,
+          metricsDataAccess: libs.metricsClient,
+          requestContext,
+        });
+
+        const results = await infraMetricsClient.search({
+          allow_no_indices: true,
+          ignore_unavailable: true,
+          body: {
+            track_total_hits: true,
+            terminate_after: 1,
+            size: 0,
+            ...(modules.length > 0
+              ? {
+                  query: {
+                    bool: {
+                      should: [
+                        ...termsQuery(EVENT_MODULE, ...modules),
+                        ...termsQuery(METRICSET_MODULE, ...modules),
+                      ],
+                      minimum_should_match: 1,
+                    },
+                  },
+                }
+              : {}),
+          },
+        });
+
+        return response.ok({
+          body: getHasDataResponseRT.encode({ hasData: results.hits.total.value !== 0 }),
+        });
+      } catch (err) {
+        if (Boom.isBoom(err)) {
+          return response.customError({
+            statusCode: err.output.statusCode,
+            body: { message: err.output.payload.message },
+          });
+        }
+
+        return response.customError({
+          statusCode: err.statusCode ?? 500,
+          body: {
+            message: err.message ?? 'An unexpected error occurred',
+          },
+        });
+      }
+    }
+  );
 };
 
 const isFulfilled = <Type>(

diff --git a/x-pack/test/api_integration/apis/metrics_ui/infra.ts b/x-pack/test/api_integration/apis/metrics_ui/infra.ts
@@ -54,7 +54,6 @@ export default function ({ getService }: FtrProviderContext) {
       to: new Date(DATES['8.0.0'].logs_and_metrics.max).toISOString(),
     },
     query: { bool: { must_not: [], filter: [], should: [], must: [] } },
-    sourceId: 'default',
   };
 
   const makeRequest = async ({

diff --git a/x-pack/test/api_integration/apis/metrics_ui/sources.ts b/x-pack/test/api_integration/apis/metrics_ui/sources.ts
@@ -17,25 +17,27 @@ import { FtrProviderContext } from '../../ftr_provider_context';
 export default function ({ getService }: FtrProviderContext) {
   const esArchiver = getService('esArchiver');
   const supertest = getService('supertest');
-  const SOURCE_API_URL = '/api/metrics/source/default';
+  const SOURCE_API_URL = '/api/metrics/source';
+  const SOURCE_ID = 'default';
   const kibanaServer = getService('kibanaServer');
-  const patchRequest = async (
-    body: PartialMetricsSourceConfigurationProperties
-  ): Promise<MetricsSourceConfigurationResponse | undefined> => {
-    const response = await supertest
-      .patch(SOURCE_API_URL)
-      .set('kbn-xsrf', 'xxx')
-      .send(body)
-      .expect(200);
-    return response.body;
-  };
 
   describe('sources', () => {
     before(() => esArchiver.load('x-pack/test/functional/es_archives/infra/metrics_and_logs'));
     after(() => esArchiver.unload('x-pack/test/functional/es_archives/infra/metrics_and_logs'));
     before(() => kibanaServer.savedObjects.cleanStandardList());
     after(() => kibanaServer.savedObjects.cleanStandardList());
 
+    const patchRequest = async (
+      body: PartialMetricsSourceConfigurationProperties
+    ): Promise<MetricsSourceConfigurationResponse | undefined> => {
+      const response = await supertest
+        .patch(`${SOURCE_API_URL}/${SOURCE_ID}`)
+        .set('kbn-xsrf', 'xxx')
+        .send(body)
+        .expect(200);
+      return response.body;
+    };
+
     describe('patch request', () => {
       it('applies all top-level field updates to an existing source', async () => {
         const creationResponse = await patchRequest({
@@ -103,28 +105,65 @@ export default function ({ getService }: FtrProviderContext) {
       it('validates anomalyThreshold is between range 1-100', async () => {
         // create config with bad request
         await supertest
-          .patch(SOURCE_API_URL)
+          .patch(`${SOURCE_API_URL}/${SOURCE_ID}`)
           .set('kbn-xsrf', 'xxx')
           .send({ name: 'NAME', anomalyThreshold: -20 })
           .expect(400);
         // create config with good request
         await supertest
-          .patch(SOURCE_API_URL)
+          .patch(`${SOURCE_API_URL}/${SOURCE_ID}`)
           .set('kbn-xsrf', 'xxx')
           .send({ name: 'NAME', anomalyThreshold: 20 })
           .expect(200);
 
         await supertest
-          .patch(SOURCE_API_URL)
+          .patch(`${SOURCE_API_URL}/${SOURCE_ID}`)
           .set('kbn-xsrf', 'xxx')
           .send({ anomalyThreshold: -2 })
           .expect(400);
         await supertest
-          .patch(SOURCE_API_URL)
+          .patch(`${SOURCE_API_URL}/${SOURCE_ID}`)
           .set('kbn-xsrf', 'xxx')
           .send({ anomalyThreshold: 101 })
           .expect(400);
       });
     });
+
+    describe('has data', () => {
+      const makeRequest = async (params?: {
+        modules?: string[];
+        expectedHttpStatusCode?: number;
+      }) => {
+        const { modules, expectedHttpStatusCode = 200 } = params ?? {};
+        return supertest
+          .get(`${SOURCE_API_URL}/hasData`)
+          .query(modules ? { modules } : '')
+          .set('kbn-xsrf', 'xxx')
+          .expect(expectedHttpStatusCode);
+      };
+
+      before(() => patchRequest({ name: 'default', metricAlias: 'metrics-*,metricbeat-*' }));
+
+      it('should return "hasData" true when modules is "system"', async () => {
+        const response = await makeRequest({ modules: ['system'] });
+        expect(response.body.hasData).to.be(true);
+      });
+      it('should return "hasData" false when modules is "nginx"', async () => {
+        const response = await makeRequest({ modules: ['nginx'] });
+        expect(response.body.hasData).to.be(true);
+      });
+
+      it('should return "hasData" true when modules is not passed', async () => {
+        const response = await makeRequest();
+        expect(response.body.hasData).to.be(true);
+      });
+
+      it('should fail when "modules" size is greater than 5', async () => {
+        await makeRequest({
+          modules: ['system', 'nginx', 'kubernetes', 'aws', 'kafka', 'azure'],
+          expectedHttpStatusCode: 400,
+        });
+      });
+    });
   });
 }
diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/infra/infra.ts b/x-pack/test_serverless/api_integration/test_suites/observability/infra/infra.ts
@@ -89,7 +89,6 @@ export default function ({ getService }: FtrProviderContext) {
                 from: timeRange.from,
                 to: timeRange.to,
               },
-              sourceId: 'default',
             },
             roleAuthc
           );