[Obs Onboarding] Migrate API test to deployment agnostic framework (e…

…lastic#211548)

Closes [206953](elastic#206953)

This change migrates mosts of the Obs Onboarding API tests to the
deployment agnostic framework. A few tests has to be still left in the
deployment-specific setup because they require a custom roles which are
[not
supported](https://github.com/elastic/kibana/blob/main/packages/kbn-ftr-common-functional-services/services/saml_auth/serverless/auth_provider.ts#L28)
in the deployment agnostic setup for serverless-oblt.

.github/CODEOWNERS

@@ -1318,6 +1318,7 @@ packages/kbn-monaco/src/esql @elastic/kibana-esql
 
 ## Logs UI code exceptions -> @elastic/obs-ux-logs-team
 /x-pack/test/api_integration/deployment_agnostic/apis/observability/data_quality/ @elastic/obs-ux-logs-team
+/x-pack/test/api_integration/deployment_agnostic/apis/observability/onboarding/ @elastic/obs-ux-logs-team
 /x-pack/test_serverless/functional/page_objects/svl_oblt_onboarding_stream_log_file.ts @elastic/obs-ux-logs-team
 /x-pack/test_serverless/functional/page_objects/svl_oblt_onboarding_page.ts @elastic/obs-ux-logs-team
 /x-pack/solutions/observability/plugins/infra/common/http_api/log_alerts @elastic/obs-ux-logs-team

x-pack/test/api_integration/deployment_agnostic/apis/observability/onboarding/create_logs_onboarding_flow.ts

@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { OBSERVABILITY_ONBOARDING_STATE_SAVED_OBJECT_TYPE } from '@kbn/observability-onboarding-plugin/server/saved_objects/observability_onboarding_status';
+import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context';
+import { SupertestWithRoleScopeType } from '../../../services';
+
+export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
+  const kibanaServer = getService('kibanaServer');
+  const roleScopedSupertest = getService('roleScopedSupertest');
+  let viewerClient: SupertestWithRoleScopeType;
+  let adminClient: SupertestWithRoleScopeType;
+
+  describe('Creating onboarding logs flow', () => {
+    before(async () => {
+      viewerClient = await roleScopedSupertest.getSupertestWithRoleScope('viewer', {
+        withInternalHeaders: true,
+        useCookieHeader: true,
+      });
+      adminClient = await roleScopedSupertest.getSupertestWithRoleScope('admin', {
+        withInternalHeaders: true,
+        useCookieHeader: true,
+      });
+    });
+
+    it('fails with a 500 error when missing privileges', async () => {
+      const response = await viewerClient
+        .post('/internal/observability_onboarding/logs/flow')
+        .send({
+          type: 'logFiles',
+          name: 'name',
+          state: {},
+        });
+
+      expect(response.statusCode).to.be(500);
+      expect(response.body.message).to.contain('unauthorized');
+    });
+
+    it('returns a flow id and apiKey encoded', async () => {
+      const state = {
+        datasetName: 'my-dataset',
+        serviceName: 'my-service',
+        namespace: 'my-namespace',
+        logFilePaths: ['my-service-logs.log'],
+      };
+
+      const response = await adminClient.post('/internal/observability_onboarding/logs/flow').send({
+        type: 'logFiles',
+        name: 'name',
+        state,
+      });
+
+      expect(response.statusCode).to.be(200);
+      expect(response.body.apiKeyEncoded).to.not.empty();
+      expect(response.body.onboardingId).to.not.empty();
+    });
+
+    it('saves the expected state for logFiles', async () => {
+      const state = {
+        datasetName: 'my-dataset',
+        serviceName: 'my-service',
+        namespace: 'my-namespace',
+        logFilePaths: ['my-service-logs.log'],
+      };
+
+      const response = await adminClient.post('/internal/observability_onboarding/logs/flow').send({
+        type: 'logFiles',
+        name: 'name',
+        state,
+      });
+
+      const savedState = await kibanaServer.savedObjects.get({
+        type: OBSERVABILITY_ONBOARDING_STATE_SAVED_OBJECT_TYPE,
+        id: response.body.onboardingId,
+      });
+
+      expect(savedState.attributes).to.be.eql({ type: 'logFiles', state, progress: {} });
+    });
+  });
+}

x-pack/test/api_integration/deployment_agnostic/apis/observability/onboarding/get_elastic_agent_config.ts

@@ -0,0 +1,74 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { load } from 'js-yaml';
+import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context';
+import { SupertestWithRoleScopeType } from '../../../services';
+
+export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
+  const roleScopedSupertest = getService('roleScopedSupertest');
+  let adminClient: SupertestWithRoleScopeType;
+
+  describe('Generate Elastic Agent configuration', () => {
+    before(async () => {
+      adminClient = await roleScopedSupertest.getSupertestWithRoleScope('admin', {
+        withInternalHeaders: true,
+        useCookieHeader: true,
+      });
+    });
+
+    it(`should return input properties empty when onboardingId doesn't exists`, async () => {
+      const response = await adminClient
+        .get('/internal/observability_onboarding/elastic_agent/config')
+        .query({ onboardingId: 'my-onboarding-id' });
+
+      expect(response.status).to.be(200);
+
+      const ymlConfig = load(response.text);
+      expect(ymlConfig.inputs[0].data_stream.namespace).to.be('');
+      expect(ymlConfig.inputs[0].streams[0].data_stream.dataset).to.be('');
+      expect(ymlConfig.inputs[0].streams[0].paths).to.be.empty();
+    });
+
+    it('should return input properties configured when onboardingId exists', async () => {
+      const datasetName = 'api-tests';
+      const namespace = 'default';
+      const logFilepath = '/my-logs.log';
+      const serviceName = 'my-service';
+
+      const createFlowResponse = await adminClient
+        .post('/internal/observability_onboarding/logs/flow')
+        .send({
+          type: 'logFiles',
+          name: 'name',
+          state: {
+            datasetName,
+            namespace,
+            logFilePaths: [logFilepath],
+            serviceName,
+          },
+        });
+
+      const onboardingId = createFlowResponse.body.onboardingId;
+
+      const response = await adminClient
+        .get('/internal/observability_onboarding/elastic_agent/config')
+        .query({ onboardingId });
+
+      expect(response.status).to.be(200);
+
+      const ymlConfig = load(response.text);
+      expect(ymlConfig.inputs[0].data_stream.namespace).to.be(namespace);
+      expect(ymlConfig.inputs[0].streams[0].data_stream.dataset).to.be(datasetName);
+      expect(ymlConfig.inputs[0].streams[0].paths).to.be.eql([logFilepath]);
+      expect(ymlConfig.inputs[0].streams[0].processors[0].add_fields.fields.name).to.be.eql(
+        serviceName
+      );
+    });
+  });
+}

x-pack/test/api_integration/deployment_agnostic/apis/observability/onboarding/get_privileges.ts

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { type DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context';
+import { type SupertestWithRoleScopeType } from '../../../services';
+
+export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
+  const roleScopedSupertest = getService('roleScopedSupertest');
+
+  let viewerClientWithAPIKey: SupertestWithRoleScopeType;
+  let adminClientWithAPIKey: SupertestWithRoleScopeType;
+
+  describe('Api Key privileges check', () => {
+    before(async () => {
+      viewerClientWithAPIKey = await roleScopedSupertest.getSupertestWithRoleScope('viewer', {
+        withInternalHeaders: true,
+      });
+      adminClientWithAPIKey = await roleScopedSupertest.getSupertestWithRoleScope('admin', {
+        withInternalHeaders: true,
+      });
+    });
+
+    after(async () => {
+      await viewerClientWithAPIKey.destroy();
+      await adminClientWithAPIKey.destroy();
+    });
+
+    it('returns false when user has reader privileges', async () => {
+      const response = await viewerClientWithAPIKey.get(
+        `/internal/observability_onboarding/logs/setup/privileges`
+      );
+
+      expect(response.body.hasPrivileges).not.ok();
+    });
+
+    it('returns true when user has admin privileges', async () => {
+      const response = await adminClientWithAPIKey.get(
+        `/internal/observability_onboarding/logs/setup/privileges`
+      );
+
+      expect(response.body.hasPrivileges).ok();
+    });
+  });
+}

x-pack/test/api_integration/deployment_agnostic/apis/observability/onboarding/get_progress.ts

@@ -0,0 +1,175 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { type LogsSynthtraceEsClient } from '@kbn/apm-synthtrace';
+import { log, timerange } from '@kbn/apm-synthtrace-client';
+import { type DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context';
+import { type SupertestWithRoleScopeType } from '../../../services';
+
+export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
+  const roleScopedSupertest = getService('roleScopedSupertest');
+  const synthtrace = getService('synthtrace');
+
+  let synthtraceLogsEsClient: LogsSynthtraceEsClient;
+  let adminClient: SupertestWithRoleScopeType;
+
+  describe('Get progress', () => {
+    const datasetName = 'api-tests';
+    const namespace = 'default';
+    let onboardingId: string;
+
+    before(async () => {
+      synthtraceLogsEsClient = await synthtrace.createLogsSynthtraceEsClient();
+      adminClient = await roleScopedSupertest.getSupertestWithRoleScope('admin', {
+        withInternalHeaders: true,
+        useCookieHeader: true,
+      });
+
+      const createFlowResponse = await adminClient
+        .post('/internal/observability_onboarding/logs/flow')
+        .send({
+          type: 'logFiles',
+          name: 'name',
+          state: {
+            datasetName,
+            namespace,
+            logFilePaths: ['my-service.log'],
+          },
+        });
+
+      onboardingId = createFlowResponse.body.onboardingId;
+    });
+
+    it(`fails with a 404 error when onboardingId doesn't exists`, async () => {
+      const response = await adminClient.get(
+        `/internal/observability_onboarding/flow/test-onboarding-id/progress`
+      );
+
+      expect(response.status).to.be(404);
+      expect(response.body.message).to.contain('onboarding session not found');
+    });
+
+    it('should skip log verification and return log-ingest as incomplete when ea-status is not complete', async () => {
+      const response = await adminClient.get(
+        `/internal/observability_onboarding/flow/${onboardingId}/progress`
+      );
+
+      expect(response.status).to.be(200);
+
+      const logsIngestProgress = response.body.progress['logs-ingest'];
+      expect(logsIngestProgress).to.have.property('status', 'incomplete');
+    });
+
+    describe('when ea-status is complete', () => {
+      describe('should not skip logs verification', () => {
+        const agentId = 'my-agent-id';
+
+        before(async () => {
+          await adminClient
+            .post(`/internal/observability_onboarding/flow/${onboardingId}/step/ea-status`)
+            .send({
+              status: 'complete',
+              payload: {
+                agentId,
+              },
+            });
+        });
+
+        describe('when no logs have been ingested', () => {
+          it('should return log-ingest as loading', async () => {
+            const response = await adminClient.get(
+              `/internal/observability_onboarding/flow/${onboardingId}/progress`
+            );
+
+            expect(response.status).to.be(200);
+
+            const logsIngestProgress = response.body.progress['logs-ingest'];
+            expect(logsIngestProgress).to.have.property('status', 'loading');
+          });
+        });
+
+        describe('when logs have been ingested', () => {
+          describe('with a different agentId', () => {
+            describe('and onboarding type is logFiles', () => {
+              before(async () => {
+                await synthtraceLogsEsClient.index([
+                  timerange('2023-11-20T10:00:00.000Z', '2023-11-20T10:01:00.000Z')
+                    .interval('1m')
+                    .rate(1)
+                    .generator((timestamp) =>
+                      log
+                        .create()
+                        .message('This is a log message')
+                        .timestamp(timestamp)
+                        .dataset(datasetName)
+                        .namespace(namespace)
+                        .service('my-service')
+                        .defaults({
+                          'agent.id': 'another-agent-id',
+                          'log.file.path': '/my-service.log',
+                        })
+                    ),
+                ]);
+              });
+
+              it('should return log-ingest as incomplete', async () => {
+                const response = await adminClient.get(
+                  `/internal/observability_onboarding/flow/${onboardingId}/progress`
+                );
+                expect(response.status).to.be(200);
+                const logsIngestProgress = response.body.progress['logs-ingest'];
+                expect(logsIngestProgress).to.have.property('status', 'loading');
+              });
+
+              after(async () => {
+                await synthtraceLogsEsClient.clean();
+              });
+            });
+          });
+
+          describe('with the expected agentId', () => {
+            describe('and onboarding type is logFiles', () => {
+              before(async () => {
+                await synthtraceLogsEsClient.index([
+                  timerange('2023-11-20T10:00:00.000Z', '2023-11-20T10:01:00.000Z')
+                    .interval('1m')
+                    .rate(1)
+                    .generator((timestamp) =>
+                      log
+                        .create()
+                        .message('This is a log message')
+                        .timestamp(timestamp)
+                        .dataset(datasetName)
+                        .namespace(namespace)
+                        .service('my-service')
+                        .defaults({
+                          'agent.id': agentId,
+                          'log.file.path': '/my-service.log',
+                        })
+                    ),
+                ]);
+              });
+              it('should return log-ingest as complete', async () => {
+                const response = await adminClient.get(
+                  `/internal/observability_onboarding/flow/${onboardingId}/progress`
+                );
+                expect(response.status).to.be(200);
+                const logsIngestProgress = response.body.progress['logs-ingest'];
+                expect(logsIngestProgress).to.have.property('status', 'complete');
+              });
+
+              after(async () => {
+                await synthtraceLogsEsClient.clean();
+              });
+            });
+          });
+        });
+      });
+    });
+  });
+}