[kbn-scout] Add Synthtrace as a fixture (elastic#210505)

## Summary

Closes elastic#210340

This PR adds synthtrace clients to scout as a test fixture, so you can
use it in your test to generate data.

The clients added were `apmSynthtraceEsClient`,
`infraSynthtraceEsClient` and `otelSynthtraceEsClient`.

## How to use them in parallel tests

As `synthtrace` ingests data into our indices, and sequential runs would
be the perfect way to introduce flakiness in our tests, there is a
better way to ingest data, using a hook, at the setup phase with
`globalSetup`.
We need to create a `global_setup.ts` file and link it into our
playwright config.
Then we can use something like
```
async function globalSetup(config: FullConfig) {
  const data = {
    apm: [
      opbeans({
        from: new Date(start).getTime(),
        to: new Date(end).getTime(),
      }),
    ],
    infra: [
      generateHosts({
        from: new Date(start).toISOString(),
        to: new Date(end).toISOString(),
      }),
    ],
    otel: [
      sendotlp({
        from: new Date(start).getTime(),
        to: new Date(end).getTime(),
      }),
    ],
  };

  return ingestSynthtraceDataHook(config, data);
}
```
Each key (apm, infra, otel) accepts an array of generators.

## How to use them in sequential tests
> [!WARNING]  
> This should not be the standard behaviour, we should embrace
parallelism and use sequential testing when there is no other way.

### apmSynthtraceEsClient
```ts
 test.before(
    async ({ apmSynthtraceEsClient }) => {
      await apmSynthtraceEsClient.index(
        opbeans({
          from: new Date(start).getTime(),
          to: new Date(end).getTime(),
        })
      );
    }
  );
```
[opbeans
file](https://github.com/elastic/kibana/blob/main/x-pack/solutions/observability/plugins/apm/ftr_e2e/cypress/fixtures/synthtrace/opbeans.ts)
used in the example.

### otelSynthtraceEsClient
```ts
 test.before(
    async ({otelSynthtraceEsClient }) => {
      await otelSynthtraceEsClient.index(
        sendotlp({
          from: new Date(start).getTime(),
          to: new Date(end).getTime(),
        })
      );
    }
  );
```
[sendotlp
file](https://github.com/elastic/kibana/blob/main/x-pack/solutions/observability/plugins/apm/ftr_e2e/cypress/fixtures/synthtrace/sendotlp.ts)
which will create the data.

### infraSynthtraceEsClient
```ts
 test.before(
    async ({ infraSynthtraceEsClient }) => {
      await infraSynthtraceEsClient.index(
        generateHosts({
          from: new Date(start).toISOString(),
          to: new Date(end).toISOString(),
        })
      );
    }
  );
```
[generateHosts
file](https://github.com/elastic/kibana/blob/main/x-pack/solutions/observability/plugins/inventory/e2e/cypress/e2e/alert_count/generate_data.ts#L82)
used to generate data.

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

packages/kbn-scout/index.ts

@@ -16,6 +16,7 @@ export {
   createPlaywrightConfig,
   createLazyPageObject,
   ingestTestDataHook,
+  ingestSynthtraceDataHook,
 } from './src/playwright';
 export type {
   ScoutPlaywrightOptions,

packages/kbn-scout/src/common/services/synthtrace.ts

@@ -0,0 +1,105 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import {
+  ApmSynthtraceEsClient,
+  ApmSynthtraceKibanaClient,
+  InfraSynthtraceEsClient,
+  InfraSynthtraceKibanaClient,
+  LogLevel,
+  OtelSynthtraceEsClient,
+  createLogger,
+} from '@kbn/apm-synthtrace';
+import { ScoutLogger } from './logger';
+import { EsClient } from '../../types';
+
+let apmSynthtraceEsClientInstance: ApmSynthtraceEsClient | undefined;
+let infraSynthtraceEsClientInstance: InfraSynthtraceEsClient | undefined;
+let otelSynthtraceEsClientInstance: OtelSynthtraceEsClient | undefined;
+const logger = createLogger(LogLevel.info);
+
+export async function getApmSynthtraceEsClient(
+  esClient: EsClient,
+  target: string,
+  log: ScoutLogger
+) {
+  if (!apmSynthtraceEsClientInstance) {
+    const apmSynthtraceKibanaClient = new ApmSynthtraceKibanaClient({
+      logger,
+      target,
+    });
+
+    const version = await apmSynthtraceKibanaClient.fetchLatestApmPackageVersion();
+    await apmSynthtraceKibanaClient.installApmPackage(version);
+    apmSynthtraceEsClientInstance = new ApmSynthtraceEsClient({
+      client: esClient,
+      logger,
+      refreshAfterIndex: true,
+      version,
+    });
+
+    apmSynthtraceEsClientInstance.pipeline(
+      apmSynthtraceEsClientInstance.getDefaultPipeline({ includeSerialization: false })
+    );
+
+    log.serviceLoaded('apmSynthtraceClient');
+  }
+
+  return apmSynthtraceEsClientInstance;
+}
+
+export async function getInfraSynthtraceEsClient(
+  esClient: EsClient,
+  kbnUrl: string,
+  auth: { username: string; password: string },
+  log: ScoutLogger
+) {
+  if (!infraSynthtraceEsClientInstance) {
+    const infraSynthtraceKibanaClient = new InfraSynthtraceKibanaClient({
+      logger,
+      target: kbnUrl,
+      username: auth.username,
+      password: auth.password,
+    });
+
+    const version = await infraSynthtraceKibanaClient.fetchLatestSystemPackageVersion();
+    await infraSynthtraceKibanaClient.installSystemPackage(version);
+    infraSynthtraceEsClientInstance = new InfraSynthtraceEsClient({
+      client: esClient,
+      logger,
+      refreshAfterIndex: true,
+    });
+
+    infraSynthtraceEsClientInstance.pipeline(
+      infraSynthtraceEsClientInstance.getDefaultPipeline({ includeSerialization: false })
+    );
+
+    log.serviceLoaded('infraSynthtraceClient');
+  }
+
+  return infraSynthtraceEsClientInstance;
+}
+
+export function getOtelSynthtraceEsClient(esClient: EsClient, log: ScoutLogger) {
+  if (!otelSynthtraceEsClientInstance) {
+    otelSynthtraceEsClientInstance = new OtelSynthtraceEsClient({
+      client: esClient,
+      logger,
+      refreshAfterIndex: true,
+    });
+
+    otelSynthtraceEsClientInstance.pipeline(
+      otelSynthtraceEsClientInstance.getDefaultPipeline({ includeSerialization: false })
+    );
+
+    log.serviceLoaded('otelSynthtraceClient');
+  }
+
+  return otelSynthtraceEsClientInstance;
+}

packages/kbn-scout/src/playwright/fixtures/single_thread_fixtures.ts

@@ -14,6 +14,7 @@ import {
   coreWorkerFixtures,
   esArchiverFixture,
   uiSettingsFixture,
+  synthtraceFixture,
 } from './worker';
 import type {
   EsArchiverFixture,
@@ -23,23 +24,23 @@ import type {
   ScoutLogger,
   ScoutTestConfig,
   UiSettingsFixture,
+  SynthtraceFixture,
 } from './worker';
 import {
   scoutPageFixture,
   browserAuthFixture,
   pageObjectsFixture,
   validateTagsFixture,
-  BrowserAuthFixture,
-  ScoutPage,
-  PageObjects,
 } from './test';
-export type { PageObjects, ScoutPage } from './test';
+import type { BrowserAuthFixture, ScoutPage, PageObjects } from './test';
+export type { ScoutPage, PageObjects } from './test';
 
 export const scoutFixtures = mergeTests(
   // worker scope fixtures
   coreWorkerFixtures,
   esArchiverFixture,
   uiSettingsFixture,
+  synthtraceFixture,
   // api fixtures
   apiFixtures,
   // test scope fixtures
@@ -63,4 +64,7 @@ export interface ScoutWorkerFixtures extends ApiFixtures {
   esClient: EsClient;
   esArchiver: EsArchiverFixture;
   uiSettings: UiSettingsFixture;
+  apmSynthtraceEsClient: SynthtraceFixture['apmSynthtraceEsClient'];
+  infraSynthtraceEsClient: SynthtraceFixture['infraSynthtraceEsClient'];
+  otelSynthtraceEsClient: SynthtraceFixture['otelSynthtraceEsClient'];
 }

packages/kbn-scout/src/playwright/fixtures/worker/index.ts

@@ -28,3 +28,6 @@ export type { ScoutSpaceParallelFixture } from './scout_space';
 
 export { apiFixtures } from './apis';
 export type { ApiFixtures, ApiParallelWorkerFixtures } from './apis';
+
+export { synthtraceFixture } from './synthtrace';
+export type { SynthtraceFixture } from './synthtrace';

packages/kbn-scout/src/playwright/fixtures/worker/synthtrace.ts

@@ -0,0 +1,91 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { Readable } from 'stream';
+import type { ApmFields, Fields, InfraDocument, OtelDocument } from '@kbn/apm-synthtrace-client';
+import Url from 'url';
+import type { SynthtraceEsClient } from '@kbn/apm-synthtrace/src/lib/shared/base_client';
+import {
+  getApmSynthtraceEsClient,
+  getInfraSynthtraceEsClient,
+  getOtelSynthtraceEsClient,
+} from '../../../common/services/synthtrace';
+import { coreWorkerFixtures } from './core_fixtures';
+import type { SynthtraceEvents } from '../../global_hooks/synthtrace_ingestion';
+
+interface SynthtraceFixtureEsClient<TFields extends Fields> {
+  index: (events: SynthtraceEvents<TFields>) => Promise<void>;
+  clean: SynthtraceEsClient<TFields>['clean'];
+}
+
+export interface SynthtraceFixture {
+  apmSynthtraceEsClient: SynthtraceFixtureEsClient<ApmFields>;
+  infraSynthtraceEsClient: SynthtraceFixtureEsClient<InfraDocument>;
+  otelSynthtraceEsClient: SynthtraceFixtureEsClient<OtelDocument>;
+}
+
+const useSynthtraceClient = async <TFields extends Fields>(
+  client: SynthtraceEsClient<TFields>,
+  use: (client: SynthtraceFixtureEsClient<TFields>) => Promise<void>
+) => {
+  const index = async (events: SynthtraceEvents<TFields>) =>
+    await client.index(Readable.from(Array.from(events).flatMap((event) => event.serialize())));
+
+  const clean = async () => await client.clean();
+
+  await use({ index, clean });
+
+  // cleanup function after all tests have ran
+  await client.clean();
+};
+
+export const synthtraceFixture = coreWorkerFixtures.extend<{}, SynthtraceFixture>({
+  apmSynthtraceEsClient: [
+    async ({ esClient, config, kbnUrl, log }, use) => {
+      const { username, password } = config.auth;
+      const kibanaUrl = new URL(kbnUrl.get());
+      const kibanaUrlWithAuth = Url.format({
+        protocol: kibanaUrl.protocol,
+        hostname: kibanaUrl.hostname,
+        port: kibanaUrl.port,
+        auth: `${username}:${password}`,
+      });
+
+      const apmSynthtraceEsClient = await getApmSynthtraceEsClient(
+        esClient,
+        kibanaUrlWithAuth,
+        log
+      );
+
+      await useSynthtraceClient<ApmFields>(apmSynthtraceEsClient, use);
+    },
+    { scope: 'worker' },
+  ],
+  infraSynthtraceEsClient: [
+    async ({ esClient, config, kbnUrl, log }, use) => {
+      const infraSynthtraceEsClient = await getInfraSynthtraceEsClient(
+        esClient,
+        kbnUrl.get(),
+        config.auth,
+        log
+      );
+
+      await useSynthtraceClient<InfraDocument>(infraSynthtraceEsClient, use);
+    },
+    { scope: 'worker' },
+  ],
+  otelSynthtraceEsClient: [
+    async ({ esClient, log }, use) => {
+      const otelSynthtraceEsClient = await getOtelSynthtraceEsClient(esClient, log);
+
+      await useSynthtraceClient<OtelDocument>(otelSynthtraceEsClient, use);
+    },
+    { scope: 'worker' },
+  ],
+});

packages/kbn-scout/src/playwright/global_hooks/index.ts

@@ -8,3 +8,4 @@
  */
 
 export { ingestTestDataHook } from './data_ingestion';
+export { ingestSynthtraceDataHook } from './synthtrace_ingestion';

packages/kbn-scout/src/playwright/global_hooks/synthtrace_ingestion.ts

@@ -0,0 +1,115 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { FullConfig } from 'playwright/test';
+import Url from 'url';
+import { Readable } from 'node:stream';
+import type {
+  ApmFields,
+  Fields,
+  InfraDocument,
+  OtelDocument,
+  Serializable,
+  SynthtraceGenerator,
+} from '@kbn/apm-synthtrace-client';
+import {
+  getLogger,
+  createScoutConfig,
+  measurePerformanceAsync,
+  getEsClient,
+  ScoutLogger,
+  EsClient,
+} from '../../common';
+import { ScoutTestOptions } from '../types';
+import {
+  getApmSynthtraceEsClient,
+  getInfraSynthtraceEsClient,
+  getOtelSynthtraceEsClient,
+} from '../../common/services/synthtrace';
+
+export type SynthtraceEvents<T extends Fields> = SynthtraceGenerator<T> | Array<Serializable<T>>;
+
+interface SynthtraceIngestionData {
+  apm: Array<SynthtraceEvents<ApmFields>>;
+  infra: Array<SynthtraceEvents<InfraDocument>>;
+  otel: Array<SynthtraceEvents<OtelDocument>>;
+}
+
+const getSynthtraceClient = (
+  key: keyof SynthtraceIngestionData,
+  esClient: EsClient,
+  kbnUrl: string,
+  auth: { username: string; password: string },
+  log: ScoutLogger
+) => {
+  switch (key) {
+    case 'apm':
+      const kibanaUrl = new URL(kbnUrl);
+      const kibanaUrlWithAuth = Url.format({
+        protocol: kibanaUrl.protocol,
+        hostname: kibanaUrl.hostname,
+        port: kibanaUrl.port,
+        auth: `${auth.username}:${auth.password}`,
+      });
+      return getApmSynthtraceEsClient(esClient, kibanaUrlWithAuth, log);
+    case 'infra':
+      return getInfraSynthtraceEsClient(esClient, kbnUrl, auth, log);
+    case 'otel':
+      return getOtelSynthtraceEsClient(esClient, log);
+  }
+};
+
+export async function ingestSynthtraceDataHook(config: FullConfig, data: SynthtraceIngestionData) {
+  const log = getLogger();
+
+  const { apm, infra, otel } = data;
+  const hasApmData = apm.length > 0;
+  const hasInfraData = infra.length > 0;
+  const hasOtelData = otel.length > 0;
+  const hasAnyData = hasApmData || hasInfraData || hasOtelData;
+
+  if (!hasAnyData) {
+    log.debug('[setup] no synthtrace data to ingest');
+    return;
+  }
+
+  return measurePerformanceAsync(log, '[setup]: ingestSynthtraceDataHook', async () => {
+    // TODO: This should be configurable local vs cloud
+
+    const configName = 'local';
+    const projectUse = config.projects[0].use as ScoutTestOptions;
+    const serversConfigDir = projectUse.serversConfigDir;
+    const scoutConfig = createScoutConfig(serversConfigDir, configName, log);
+    const esClient = getEsClient(scoutConfig, log);
+    const kbnUrl = scoutConfig.hosts.kibana;
+
+    for (const key of Object.keys(data)) {
+      const typedKey = key as keyof SynthtraceIngestionData;
+      if (data[typedKey].length > 0) {
+        const client = await getSynthtraceClient(typedKey, esClient, kbnUrl, scoutConfig.auth, log);
+
+        log.debug(`[setup] ingesting ${key} synthtrace data`);
+
+        try {
+          await Promise.all(
+            data[typedKey].map((event) => {
+              return client.index(Readable.from(Array.from(event).flatMap((e) => e.serialize())));
+            })
+          );
+        } catch (e) {
+          log.debug(`[setup] error ingesting ${key} synthtrace data`, e);
+        }
+
+        log.debug(`[setup] ${key} synthtrace data ingested successfully`);
+      } else {
+        log.debug(`[setup] no synthtrace data to ingest for ${key}`);
+      }
+    }
+  });
+}

packages/kbn-scout/src/playwright/index.ts

@@ -32,4 +32,4 @@ export type {
 // use to tag tests
 export { tags } from './tags';
 
-export { ingestTestDataHook } from './global_hooks';
+export { ingestTestDataHook, ingestSynthtraceDataHook } from './global_hooks';

packages/kbn-scout/tsconfig.json

@@ -28,5 +28,7 @@
     "@kbn/test-subj-selector",
     "@kbn/scout-info",
     "@kbn/scout-reporting",
+    "@kbn/apm-synthtrace",
+    "@kbn/apm-synthtrace-client",
   ]
 }