Move storage adapter and traced ES client into dedicated package (ela…

…stic#211578)

The storage adapter helper is a very generic package. This PR moves it
out of the observability server utils into a dedicated package to better
reflect this and to be able to use it from non-observability contexts.

The same applies to the observability es client. This PR moves it as
well and renames it to `TracedEsClient` in the same way.

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

.github/CODEOWNERS

@@ -528,7 +528,9 @@ src/platform/packages/shared/kbn-sse-utils @elastic/obs-knowledge-team
 src/platform/packages/shared/kbn-sse-utils-client @elastic/obs-knowledge-team
 src/platform/packages/shared/kbn-sse-utils-server @elastic/obs-knowledge-team
 src/platform/packages/shared/kbn-std @elastic/kibana-core
+src/platform/packages/shared/kbn-storage-adapter @elastic/observability-ui
 src/platform/packages/shared/kbn-timerange @elastic/obs-ux-logs-team
+src/platform/packages/shared/kbn-traced-es-client @elastic/observability-ui
 src/platform/packages/shared/kbn-triggers-actions-ui-types @elastic/response-ops
 src/platform/packages/shared/kbn-try-in-console @elastic/search-kibana
 src/platform/packages/shared/kbn-typed-react-router-config @elastic/obs-knowledge-team @elastic/obs-ux-infra_services-team

package.json

@@ -932,6 +932,7 @@
     "@kbn/status-plugin-a-plugin": "link:test/server_integration/plugins/status_plugin_a",
     "@kbn/status-plugin-b-plugin": "link:test/server_integration/plugins/status_plugin_b",
     "@kbn/std": "link:src/platform/packages/shared/kbn-std",
+    "@kbn/storage-adapter": "link:src/platform/packages/shared/kbn-storage-adapter",
     "@kbn/streams-app-plugin": "link:x-pack/solutions/observability/plugins/streams_app",
     "@kbn/streams-plugin": "link:x-pack/solutions/observability/plugins/streams",
     "@kbn/streams-schema": "link:x-pack/platform/packages/shared/kbn-streams-schema",
@@ -953,6 +954,7 @@
     "@kbn/timelion-grammar": "link:src/platform/packages/private/kbn-timelion-grammar",
     "@kbn/timerange": "link:src/platform/packages/shared/kbn-timerange",
     "@kbn/tinymath": "link:src/platform/packages/private/kbn-tinymath",
+    "@kbn/traced-es-client": "link:src/platform/packages/shared/kbn-traced-es-client",
     "@kbn/transform-plugin": "link:x-pack/platform/plugins/private/transform",
     "@kbn/translations-plugin": "link:x-pack/platform/plugins/private/translations",
     "@kbn/transpose-utils": "link:src/platform/packages/private/kbn-transpose-utils",

x-pack/solutions/observability/packages/utils-server/es/storage/index.ts → src/platform/packages/shared/kbn-storage-adapter/index.ts

@@ -1,9 +1,12 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
  */
+
 import type {
   BulkRequest,
   BulkResponse,
@@ -151,6 +154,6 @@ export type StorageDocumentOf<TStorageSettings extends StorageSettings> = Storag
   properties: TStorageSettings['schema']['properties'];
 }> & { _id: string };
 
-export { StorageIndexAdapter } from './index_adapter';
+export { StorageIndexAdapter } from './src/index_adapter';
 
 export { types } from './types';

src/platform/packages/shared/kbn-storage-adapter/jest.config.js

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../../../../..',
+  roots: ['<rootDir>/src/platform/packages/shared/kbn-storage-adapter'],
+};

src/platform/packages/shared/kbn-storage-adapter/jest.integration.config.js

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_integration',
+  rootDir: '../../../../..',
+  roots: ['<rootDir>/src/platform/packages/shared/kbn-storage-adapter'],
+};

src/platform/packages/shared/kbn-storage-adapter/kibana.jsonc

@@ -0,0 +1,9 @@
+{
+  "type": "shared-server",
+  "id": "@kbn/storage-adapter",
+  "owner": [
+    "@elastic/observability-ui"
+  ],
+  "group": "platform",
+  "visibility": "shared"
+}

src/platform/packages/shared/kbn-storage-adapter/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "@kbn/storage-adapter",
+  "private": true,
+  "version": "1.0.0",
+  "license": "Elastic License 2.0 OR AGPL-3.0-only OR SSPL-1.0"
+}

src/platform/packages/shared/kbn-storage-adapter/src/get_schema_version.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import stringify from 'json-stable-stringify';
+import objectHash from 'object-hash';
+import { IndexStorageSettings } from '..';
+
+export function getSchemaVersion(storage: IndexStorageSettings): string {
+  const version = objectHash(stringify(storage.schema.properties));
+  return version;
+}

x-pack/solutions/observability/packages/utils-server/es/storage/index_adapter/index.ts → src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/index.ts

@@ -1,8 +1,10 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
 import type {
@@ -35,9 +37,9 @@ import {
   StorageClientCleanResponse,
   ApplicationDocument,
   InternalIStorageClient,
-} from '..';
+} from '../..';
 import { getSchemaVersion } from '../get_schema_version';
-import { StorageMappingProperty } from '../types';
+import { StorageMappingProperty } from '../../types';
 
 function getAliasName(name: string) {
   return name;

x-pack/solutions/observability/packages/utils-server/es/storage/index_adapter/integration_tests/index.test.ts → src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/integration_tests/index.test.ts

@@ -1,8 +1,10 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
 import {
@@ -16,7 +18,7 @@ import {
   StorageClientIndexResponse,
   StorageIndexAdapter,
   type StorageSettings,
-} from '../..';
+} from '../../..';
 import type { ElasticsearchClient, Logger } from '@kbn/core/server';
 import { httpServerMock } from '@kbn/core/server/mocks';
 import * as getSchemaVersionModule from '../../get_schema_version';

src/platform/packages/shared/kbn-storage-adapter/tsconfig.json

@@ -0,0 +1,24 @@
+{
+  "extends": "../../../../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "target/types",
+    "types": [
+      "jest",
+      "node",
+      "react"
+    ]
+  },
+  "include": [
+    "**/*.ts",
+    "**/*.tsx",
+  ],
+  "exclude": [
+    "target/**/*"
+  ],
+  "kbn_references": [
+    "@kbn/core",
+    "@kbn/es-types",
+    "@kbn/es-errors",
+    "@kbn/core-test-helpers-kbn-server",
+  ]
+}

x-pack/solutions/observability/packages/utils-server/es/storage/types.ts → src/platform/packages/shared/kbn-storage-adapter/types.ts

@@ -1,8 +1,10 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
 import { MappingObjectProperty, MappingProperty } from '@elastic/elasticsearch/lib/api/types';

src/platform/packages/shared/kbn-traced-es-client/README.md

@@ -0,0 +1,4 @@
+# Traced ES Client
+
+This package provides a wrapper around the regular platform ES client which adds tracing information for better analysis.
+Each operation can be given a custom name that shows up as a trace when instrumenting using APM.

src/platform/packages/shared/kbn-traced-es-client/index.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+export * from './src/create_traced_es_client';

src/platform/packages/shared/kbn-traced-es-client/jest.config.js

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../../../../..',
+  roots: ['<rootDir>/src/platform/packages/shared/kbn-traced-es-client'],
+};

src/platform/packages/shared/kbn-traced-es-client/kibana.jsonc

@@ -0,0 +1,9 @@
+{
+  "type": "shared-server",
+  "id": "@kbn/traced-es-client",
+  "owner": [
+    "@elastic/observability-ui"
+  ],
+  "group": "platform",
+  "visibility": "shared"
+}

src/platform/packages/shared/kbn-traced-es-client/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "@kbn/traced-es-client",
+  "private": true,
+  "version": "1.0.0",
+  "license": "Elastic License 2.0 OR AGPL-3.0-only OR SSPL-1.0"
+}

x-pack/solutions/observability/packages/utils-server/es/client/create_observability_es_client.ts → src/platform/packages/shared/kbn-traced-es-client/src/create_traced_es_client.ts

@@ -1,8 +1,10 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
 import type {
@@ -19,7 +21,7 @@ import type { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types';
 import { Required, ValuesType } from 'utility-types';
 import { DedotObject } from '@kbn/utility-types';
 import { unflattenObject } from '@kbn/task-manager-plugin/server/metrics/lib';
-import { esqlResultToPlainObjects } from '../esql_result_to_plain_objects';
+import { esqlResultToPlainObjects } from './esql_result_to_plain_objects';
 
 type SearchRequest = ESSearchRequest & {
   index: string | string[];
@@ -78,7 +80,7 @@ export type EsqlQueryResponse = UnparsedEsqlResponse | ParsedEsqlResponse;
  * An Elasticsearch Client with a fully typed `search` method and built-in
  * APM instrumentation.
  */
-export interface ObservabilityElasticsearchClient {
+export interface TracedElasticsearchClient {
   search<TDocument = unknown, TSearchRequest extends SearchRequest = SearchRequest>(
     operationName: string,
     parameters: TSearchRequest
@@ -108,7 +110,7 @@ export interface ObservabilityElasticsearchClient {
   client: ElasticsearchClient;
 }
 
-export function createObservabilityEsClient({
+export function createTracedEsClient({
   client,
   logger,
   plugin,
@@ -118,7 +120,7 @@ export function createObservabilityEsClient({
   logger: Logger;
   plugin?: string;
   labels?: Record<string, string>;
-}): ObservabilityElasticsearchClient {
+}): TracedElasticsearchClient {
   // wraps the ES calls in a named APM span for better analysis
   // (otherwise it would just eg be a _search span)
   const callWithLogger = <T>(

x-pack/solutions/observability/packages/utils-server/es/esql_result_to_plain_objects.test.ts → src/platform/packages/shared/kbn-traced-es-client/src/esql_result_to_plain_objects.test.ts

@@ -1,8 +1,10 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
 import type { ESQLSearchResponse } from '@kbn/es-types';

x-pack/solutions/observability/packages/utils-server/es/esql_result_to_plain_objects.ts → src/platform/packages/shared/kbn-traced-es-client/src/esql_result_to_plain_objects.ts

@@ -1,8 +1,10 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
 import type { ESQLSearchResponse } from '@kbn/es-types';

src/platform/packages/shared/kbn-traced-es-client/tsconfig.json

@@ -0,0 +1,25 @@
+{
+  "extends": "../../../../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "target/types",
+    "types": [
+      "jest",
+      "node",
+      "react"
+    ]
+  },
+  "include": [
+    "**/*.ts",
+    "**/*.tsx",
+  ],
+  "exclude": [
+    "target/**/*"
+  ],
+  "kbn_references": [
+    "@kbn/core",
+    "@kbn/es-types",
+    "@kbn/apm-utils",
+    "@kbn/utility-types",
+    "@kbn/task-manager-plugin",
+  ]
+}

tsconfig.base.json

@@ -1894,6 +1894,8 @@
       "@kbn/std/*": ["src/platform/packages/shared/kbn-std/*"],
       "@kbn/stdio-dev-helpers": ["packages/kbn-stdio-dev-helpers"],
       "@kbn/stdio-dev-helpers/*": ["packages/kbn-stdio-dev-helpers/*"],
+      "@kbn/storage-adapter": ["src/platform/packages/shared/kbn-storage-adapter"],
+      "@kbn/storage-adapter/*": ["src/platform/packages/shared/kbn-storage-adapter/*"],
       "@kbn/storybook": ["packages/kbn-storybook"],
       "@kbn/storybook/*": ["packages/kbn-storybook/*"],
       "@kbn/streams-app-plugin": ["x-pack/solutions/observability/plugins/streams_app"],
@@ -1964,6 +1966,8 @@
       "@kbn/tinymath/*": ["src/platform/packages/private/kbn-tinymath/*"],
       "@kbn/tooling-log": ["packages/kbn-tooling-log"],
       "@kbn/tooling-log/*": ["packages/kbn-tooling-log/*"],
+      "@kbn/traced-es-client": ["src/platform/packages/shared/kbn-traced-es-client"],
+      "@kbn/traced-es-client/*": ["src/platform/packages/shared/kbn-traced-es-client/*"],
       "@kbn/transform-plugin": ["x-pack/platform/plugins/private/transform"],
       "@kbn/transform-plugin/*": ["x-pack/platform/plugins/private/transform/*"],
       "@kbn/translations-plugin": ["x-pack/platform/plugins/private/translations"],

x-pack/solutions/observability/packages/observability-ai/observability-ai-server/root_cause_analysis/run_root_cause_analysis.ts

@@ -15,7 +15,7 @@ import { findLast, pick } from 'lodash';
 import moment from 'moment';
 import { catchError, filter, from, map, mergeMap, Observable, of, switchMap } from 'rxjs';
 import { ObservabilityAIAssistantClient } from '@kbn/observability-ai-assistant-plugin/server';
-import { ObservabilityElasticsearchClient } from '@kbn/observability-utils-server/es/client/create_observability_es_client';
+import { TracedElasticsearchClient } from '@kbn/traced-es-client';
 import {
   RCA_END_PROCESS_TOOL_NAME,
   RCA_INVESTIGATE_ENTITY_TOOL_NAME,
@@ -94,7 +94,7 @@ export function runRootCauseAnalysis({
   end: number;
   alertsClient: AlertsClient;
   rulesClient: RulesClient;
-  esClient: ObservabilityElasticsearchClient;
+  esClient: TracedElasticsearchClient;
   observabilityAIAssistantClient: ObservabilityAIAssistantClient;
   indices: {
     logs: string[];