[Stack Monitoring] api tests for logstash (#145351)

### Summary
Part of #119658

Add api integration tests for logstash routes to validate behavior when
reading data ingested by elastic-agent.

We currently have a testing suite for legacy and another one for
metricbeat. Since metricbeat and agent documents only differ in their
metadata, for example agent will populate a `data_stream.*` property to
identify the document types while metricbeat uses `metricset.*`, the
tests assertion validating _business_ data should pass regardless of the
documents source. With this in mind the metricbeat tests were updated to
run the tests twice, one time with metricbeat data and a second time
with package data.

To generate the archives the `metrics-*` mappings were extracted with
esArchiver from an elasticsearch with the package installed, and the
documents were transformed from the metricbeat documents with [this
script](https://gist.github.com/klacabane/654497ff86053c60af6df15fa6f6f657).

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/test/api_integration/apis/monitoring/logstash/multicluster_pipelines_mb.js

@@ -13,34 +13,37 @@ export default function ({ getService }) {
   const supertest = getService('supertest');
   const { setup, tearDown } = getLifecycleMethods(getService);
 
-  describe('pipelines listing multicluster mb', () => {
-    const archive =
-      'x-pack/test/functional/es_archives/monitoring/logstash_pipelines_multicluster_mb';
-    const timeRange = {
-      min: '2019-11-11T15:13:45.266Z',
-      max: '2019-11-11T15:17:05.399Z',
-    };
-    const pagination = {
-      size: 10,
-      index: 0,
-    };
+  describe('pipelines listing multicluster - metricbeat and package', () => {
+    ['mb', 'package'].forEach((source) => {
+      describe(`pipelines listing multicluster ${source}`, () => {
+        const archive = `x-pack/test/functional/es_archives/monitoring/logstash_pipelines_multicluster_${source}`;
+        const timeRange = {
+          min: '2019-11-11T15:13:45.266Z',
+          max: '2019-11-11T15:17:05.399Z',
+        };
+        const pagination = {
+          size: 10,
+          index: 0,
+        };
 
-    before('load archive', () => {
-      return setup(archive);
-    });
+        before('load archive', () => {
+          return setup(archive);
+        });
 
-    after('unload archive', () => {
-      return tearDown();
-    });
+        after('unload archive', () => {
+          return tearDown(archive);
+        });
 
-    it('should get the pipelines', async () => {
-      const { body } = await supertest
-        .post('/api/monitoring/v1/clusters/hJS0FZ7wR9GGdYs8RNW8pw/logstash/pipelines')
-        .set('kbn-xsrf', 'xxx')
-        .send({ timeRange, pagination })
-        .expect(200);
+        it('should get the pipelines', async () => {
+          const { body } = await supertest
+            .post('/api/monitoring/v1/clusters/hJS0FZ7wR9GGdYs8RNW8pw/logstash/pipelines')
+            .set('kbn-xsrf', 'xxx')
+            .send({ timeRange, pagination })
+            .expect(200);
 
-      expect(body).to.eql(fixture);
+          expect(body).to.eql(fixture);
+        });
+      });
     });
   });
 }

x-pack/test/api_integration/apis/monitoring/logstash/node_detail_mb.js

@@ -13,44 +13,47 @@ export default function ({ getService }) {
   const supertest = getService('supertest');
   const esArchiver = getService('esArchiver');
 
-  // FAILING ES PROMOTION: https://github.com/elastic/kibana/issues/142642
-  describe.skip('node detail mb', () => {
-    const archive = 'x-pack/test/api_integration/apis/monitoring/es_archives/logstash_8';
-    const timeRange = {
-      min: '2022-06-17T13:19:00.000Z',
-      max: '2022-06-17T13:25:00.000Z',
-    };
-
-    before('load archive', () => {
-      return esArchiver.load(archive);
-    });
-
-    after('unload archive', () => {
-      return esArchiver.unload(archive);
-    });
-
-    it('should summarize the Logstash node with non-advanced chart data metrics', async () => {
-      const { body } = await supertest
-        .post(
-          '/api/monitoring/v1/clusters/__standalone_cluster__/logstash/node/f9efd237-3bbf-4a9b-9ce7-a16141b9d981'
-        )
-        .set('kbn-xsrf', 'xxx')
-        .send({ timeRange, is_advanced: false })
-        .expect(200);
-
-      expect(body).to.eql(nodeDetailFixture);
-    });
-
-    it('should summarize the Logstash node with advanced chart data metrics', async () => {
-      const { body } = await supertest
-        .post(
-          '/api/monitoring/v1/clusters/__standalone_cluster__/logstash/node/f9efd237-3bbf-4a9b-9ce7-a16141b9d981'
-        )
-        .set('kbn-xsrf', 'xxx')
-        .send({ timeRange, is_advanced: true })
-        .expect(200);
-
-      expect(body).to.eql(nodeDetailAdvancedFixture);
+  describe('node detail - metricbeat and package', () => {
+    ['logstash_8', 'logstash_package'].forEach((source) => {
+      describe(`node detail ${source}`, () => {
+        const archive = `x-pack/test/api_integration/apis/monitoring/es_archives/${source}`;
+        const timeRange = {
+          min: '2022-06-17T13:19:00.000Z',
+          max: '2022-06-17T13:25:00.000Z',
+        };
+
+        before('load archive', () => {
+          return esArchiver.load(archive);
+        });
+
+        after('unload archive', () => {
+          return esArchiver.unload(archive);
+        });
+
+        it('should summarize the Logstash node with non-advanced chart data metrics', async () => {
+          const { body } = await supertest
+            .post(
+              '/api/monitoring/v1/clusters/__standalone_cluster__/logstash/node/f9efd237-3bbf-4a9b-9ce7-a16141b9d981'
+            )
+            .set('kbn-xsrf', 'xxx')
+            .send({ timeRange, is_advanced: false })
+            .expect(200);
+
+          expect(body).to.eql(nodeDetailFixture);
+        });
+
+        it('should summarize the Logstash node with advanced chart data metrics', async () => {
+          const { body } = await supertest
+            .post(
+              '/api/monitoring/v1/clusters/__standalone_cluster__/logstash/node/f9efd237-3bbf-4a9b-9ce7-a16141b9d981'
+            )
+            .set('kbn-xsrf', 'xxx')
+            .send({ timeRange, is_advanced: true })
+            .expect(200);
+
+          expect(body).to.eql(nodeDetailAdvancedFixture);
+        });
+      });
     });
   });
 }

x-pack/test/api_integration/apis/monitoring/logstash/nodes_mb.js

@@ -13,29 +13,33 @@ export default function ({ getService }) {
   const supertest = getService('supertest');
   const { setup, tearDown } = getLifecycleMethods(getService);
 
-  describe('node listing mb', () => {
-    const archive = 'x-pack/test/functional/es_archives/monitoring/logstash_pipelines_mb';
-    const timeRange = {
-      min: '2018-01-22T09:33:13.000Z',
-      max: '2018-01-22T09:41:04.000Z',
-    };
+  describe('node listing - metricbeat and package', () => {
+    ['mb', 'package'].forEach((source) => {
+      describe(`node listing ${source}`, () => {
+        const archive = `x-pack/test/functional/es_archives/monitoring/logstash_pipelines_${source}`;
+        const timeRange = {
+          min: '2018-01-22T09:33:13.000Z',
+          max: '2018-01-22T09:41:04.000Z',
+        };
 
-    before('load archive', () => {
-      return setup(archive);
-    });
+        before('load archive', () => {
+          return setup(archive);
+        });
 
-    after('unload archive', () => {
-      return tearDown();
-    });
+        after('unload archive', () => {
+          return tearDown(archive);
+        });
 
-    it('should summarize the Logstash nodes with stats', async () => {
-      const { body } = await supertest
-        .post('/api/monitoring/v1/clusters/1rhApLfQShSh3JsNqYCkmA/logstash/nodes')
-        .set('kbn-xsrf', 'xxx')
-        .send({ timeRange })
-        .expect(200);
+        it('should summarize the Logstash nodes with stats', async () => {
+          const { body } = await supertest
+            .post('/api/monitoring/v1/clusters/1rhApLfQShSh3JsNqYCkmA/logstash/nodes')
+            .set('kbn-xsrf', 'xxx')
+            .send({ timeRange })
+            .expect(200);
 
-      expect(body).to.eql(nodesFixture);
+          expect(body).to.eql(nodesFixture);
+        });
+      });
     });
   });
 }

x-pack/test/api_integration/apis/monitoring/logstash/overview_mb.js

@@ -13,29 +13,33 @@ export default function ({ getService }) {
   const supertest = getService('supertest');
   const { setup, tearDown } = getLifecycleMethods(getService);
 
-  describe('overview mb', () => {
-    const archive = 'x-pack/test/functional/es_archives/monitoring/logstash_pipelines_mb';
-    const timeRange = {
-      min: '2018-01-22T09:33:13.000Z',
-      max: '2018-01-22T09:41:04.000Z',
-    };
+  describe('overview - metricbeat and package', () => {
+    ['mb', 'package'].forEach((source) => {
+      describe(`overview ${source}`, () => {
+        const archive = `x-pack/test/functional/es_archives/monitoring/logstash_pipelines_${source}`;
+        const timeRange = {
+          min: '2018-01-22T09:33:13.000Z',
+          max: '2018-01-22T09:41:04.000Z',
+        };
 
-    before('load archive', () => {
-      return setup(archive);
-    });
+        before('load archive', () => {
+          return setup(archive);
+        });
 
-    after('unload archive', () => {
-      return tearDown();
-    });
+        after('unload archive', () => {
+          return tearDown(archive);
+        });
 
-    it('should summarize two Logstash nodes with metrics', async () => {
-      const { body } = await supertest
-        .post('/api/monitoring/v1/clusters/1rhApLfQShSh3JsNqYCkmA/logstash')
-        .set('kbn-xsrf', 'xxx')
-        .send({ timeRange })
-        .expect(200);
+        it('should summarize two Logstash nodes with metrics', async () => {
+          const { body } = await supertest
+            .post('/api/monitoring/v1/clusters/1rhApLfQShSh3JsNqYCkmA/logstash')
+            .set('kbn-xsrf', 'xxx')
+            .send({ timeRange })
+            .expect(200);
 
-      expect(body).to.eql(overviewFixture);
+          expect(body).to.eql(overviewFixture);
+        });
+      });
     });
   });
 }

x-pack/test/api_integration/apis/monitoring/logstash/pipelines_mb.js

@@ -13,74 +13,78 @@ export default function ({ getService }) {
   const supertest = getService('supertest');
   const { setup, tearDown } = getLifecycleMethods(getService);
 
-  describe('pipelines mb', () => {
-    const archive = 'x-pack/test/functional/es_archives/monitoring/logstash/changing_pipelines_mb';
-    const timeRange = {
-      min: '2019-11-04T15:40:44.855Z',
-      max: '2019-11-04T15:50:38.667Z',
-    };
-    const pagination = {
-      size: 10,
-      index: 0,
-    };
-    const sort = {
-      field: 'id',
-      direction: 'asc',
-    };
+  describe('pipelines - metricbeat and package', () => {
+    ['mb', 'package'].forEach((source) => {
+      describe(`pipelines ${source}`, () => {
+        const archive = `x-pack/test/functional/es_archives/monitoring/logstash/changing_pipelines_${source}`;
+        const timeRange = {
+          min: '2019-11-04T15:40:44.855Z',
+          max: '2019-11-04T15:50:38.667Z',
+        };
+        const pagination = {
+          size: 10,
+          index: 0,
+        };
+        const sort = {
+          field: 'id',
+          direction: 'asc',
+        };
 
-    before('load archive', () => {
-      return setup(archive);
-    });
+        before('load archive', () => {
+          return setup(archive);
+        });
 
-    after('unload archive', () => {
-      return tearDown();
-    });
+        after('unload archive', () => {
+          return tearDown(archive);
+        });
 
-    it('should return paginated pipelines', async () => {
-      const { body } = await supertest
-        .post('/api/monitoring/v1/clusters/TUjQLdHNTh2SB9Wy0gOtWg/logstash/pipelines')
-        .set('kbn-xsrf', 'xxx')
-        .send({ timeRange, pagination, sort })
-        .expect(200);
+        it('should return paginated pipelines', async () => {
+          const { body } = await supertest
+            .post('/api/monitoring/v1/clusters/TUjQLdHNTh2SB9Wy0gOtWg/logstash/pipelines')
+            .set('kbn-xsrf', 'xxx')
+            .send({ timeRange, pagination, sort })
+            .expect(200);
 
-      expect(body).to.eql(pipelinesFixture);
-    });
+          expect(body).to.eql(pipelinesFixture);
+        });
 
-    it('should get one of each after enough pagination', async () => {
-      async function getIds(page) {
-        const { body } = await supertest
-          .post('/api/monitoring/v1/clusters/TUjQLdHNTh2SB9Wy0gOtWg/logstash/pipelines')
-          .set('kbn-xsrf', 'xxx')
-          .send({ timeRange, pagination: { ...pagination, index: page }, sort })
-          .expect(200);
+        it('should get one of each after enough pagination', async () => {
+          async function getIds(page) {
+            const { body } = await supertest
+              .post('/api/monitoring/v1/clusters/TUjQLdHNTh2SB9Wy0gOtWg/logstash/pipelines')
+              .set('kbn-xsrf', 'xxx')
+              .send({ timeRange, pagination: { ...pagination, index: page }, sort })
+              .expect(200);
 
-        return body.pipelines.map((pipeline) => pipeline.id);
-      }
+            return body.pipelines.map((pipeline) => pipeline.id);
+          }
 
-      const ids = [...(await getIds(0)), ...(await getIds(1)), ...(await getIds(2))];
-      expect(ids.length).to.be(26);
-    });
+          const ids = [...(await getIds(0)), ...(await getIds(1)), ...(await getIds(2))];
+          expect(ids.length).to.be(26);
+        });
 
-    it('should not error out if there is missing data for part of the time series', async () => {
-      const customTimeRange = {
-        ...timeRange,
-        max: '2019-11-04T15:59:38.667Z',
-      };
+        it('should not error out if there is missing data for part of the time series', async () => {
+          const customTimeRange = {
+            ...timeRange,
+            max: '2019-11-04T15:59:38.667Z',
+          };
 
-      const customSort = {
-        ...sort,
-        field: 'logstash_cluster_pipeline_throughput',
-      };
+          const customSort = {
+            ...sort,
+            field: 'logstash_cluster_pipeline_throughput',
+          };
 
-      await supertest
-        .post('/api/monitoring/v1/clusters/TUjQLdHNTh2SB9Wy0gOtWg/logstash/pipelines')
-        .set('kbn-xsrf', 'xxx')
-        .send({
-          timeRange: customTimeRange,
-          pagination: { ...pagination, index: 1 },
-          sort: customSort,
-        })
-        .expect(200);
+          await supertest
+            .post('/api/monitoring/v1/clusters/TUjQLdHNTh2SB9Wy0gOtWg/logstash/pipelines')
+            .set('kbn-xsrf', 'xxx')
+            .send({
+              timeRange: customTimeRange,
+              pagination: { ...pagination, index: 1 },
+              sort: customSort,
+            })
+            .expect(200);
+        });
+      });
     });
   });
 }