[8.10] [Fleet] fix index template from datastream name (#166941) (#16…

…7008)

# Backport

This will backport the following commits from `main` to `8.10`:
- [[Fleet] fix index template from datastream name
(#166941)](#166941)

<!--- Backport version: 8.9.7 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Julia
Bardi","email":"90178898+juliaElastic@users.noreply.github.com"},"sourceCommit":{"committedDate":"2023-09-22T08:20:05Z","message":"[Fleet]
fix index template from datastream name (#166941)\n\n##
Summary\r\n\r\nResolve
https://github.com/elastic/kibana/issues/164269\r\n\r\nSome context why
I picked this up
now:\r\nhttps://github.com//issues/162772#issuecomment-1728031080\r\n\r\nTo
verify:\r\n- Make sure 8.8+ apm package is installed\r\n- Create data
stream `PUT _data_stream/metrics-apm.app.default-default`\r\n- Reinstall
apm package from API or UI\r\n- Check kibana info logs, expect to not
see simulate template error and\r\nrollover like
below\r\n\r\n```\r\n[2023-09-21T15:54:36.559+02:00][INFO
][plugins.fleet] Mappings update for metrics-apm.app.default-default
failed due to ResponseError: illegal_argument_exception\r\n Root
causes:\r\n illegal_argument_exception: unable to simulate template
[metrics-apm.app.default] that does not
exist\r\n[2023-09-21T15:54:36.559+02:00][INFO ][plugins.fleet]
Triggering a rollover for
metrics-apm.app.default-default\r\n```\r\n\r\n\r\n### Checklist\r\n\r\n-
[x] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common
scenarios","sha":"3ff82f2c17e532fed5d5544ed9bbae4f6e7331af","branchLabelMapping":{"^v8.11.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:fix","Team:Fleet","backport:prev-minor","v8.11.0"],"number":166941,"url":"https://github.com/elastic/kibana/pull/166941","mergeCommit":{"message":"[Fleet]
fix index template from datastream name (#166941)\n\n##
Summary\r\n\r\nResolve
https://github.com/elastic/kibana/issues/164269\r\n\r\nSome context why
I picked this up
now:\r\nhttps://github.com//issues/162772#issuecomment-1728031080\r\n\r\nTo
verify:\r\n- Make sure 8.8+ apm package is installed\r\n- Create data
stream `PUT _data_stream/metrics-apm.app.default-default`\r\n- Reinstall
apm package from API or UI\r\n- Check kibana info logs, expect to not
see simulate template error and\r\nrollover like
below\r\n\r\n```\r\n[2023-09-21T15:54:36.559+02:00][INFO
][plugins.fleet] Mappings update for metrics-apm.app.default-default
failed due to ResponseError: illegal_argument_exception\r\n Root
causes:\r\n illegal_argument_exception: unable to simulate template
[metrics-apm.app.default] that does not
exist\r\n[2023-09-21T15:54:36.559+02:00][INFO ][plugins.fleet]
Triggering a rollover for
metrics-apm.app.default-default\r\n```\r\n\r\n\r\n### Checklist\r\n\r\n-
[x] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common
scenarios","sha":"3ff82f2c17e532fed5d5544ed9bbae4f6e7331af"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v8.11.0","labelRegex":"^v8.11.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/166941","number":166941,"mergeCommit":{"message":"[Fleet]
fix index template from datastream name (#166941)\n\n##
Summary\r\n\r\nResolve
https://github.com/elastic/kibana/issues/164269\r\n\r\nSome context why
I picked this up
now:\r\nhttps://github.com//issues/162772#issuecomment-1728031080\r\n\r\nTo
verify:\r\n- Make sure 8.8+ apm package is installed\r\n- Create data
stream `PUT _data_stream/metrics-apm.app.default-default`\r\n- Reinstall
apm package from API or UI\r\n- Check kibana info logs, expect to not
see simulate template error and\r\nrollover like
below\r\n\r\n```\r\n[2023-09-21T15:54:36.559+02:00][INFO
][plugins.fleet] Mappings update for metrics-apm.app.default-default
failed due to ResponseError: illegal_argument_exception\r\n Root
causes:\r\n illegal_argument_exception: unable to simulate template
[metrics-apm.app.default] that does not
exist\r\n[2023-09-21T15:54:36.559+02:00][INFO ][plugins.fleet]
Triggering a rollover for
metrics-apm.app.default-default\r\n```\r\n\r\n\r\n### Checklist\r\n\r\n-
[x] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common
scenarios","sha":"3ff82f2c17e532fed5d5544ed9bbae4f6e7331af"}}]}]
BACKPORT-->

Co-authored-by: Julia Bardi <90178898+juliaElastic@users.noreply.github.com>

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts

@@ -534,9 +534,15 @@ export function generateTemplateName(dataStream: RegistryDataStream): string {
 /**
  * Given a data stream name, return the indexTemplate name
  */
-function dataStreamNameToIndexTemplateName(dataStreamName: string): string {
-  const [type, dataset] = dataStreamName.split('-'); // ignore namespace at the end
-  return [type, dataset].join('-');
+async function getIndexTemplate(
+  esClient: ElasticsearchClient,
+  dataStreamName: string
+): Promise<string> {
+  const dataStream = await esClient.indices.getDataStream({
+    name: dataStreamName,
+    expand_wildcards: ['open', 'hidden'],
+  });
+  return dataStream.data_streams[0].template;
 }
 
 export function generateTemplateIndexPattern(dataStream: RegistryDataStream): string {
@@ -757,9 +763,9 @@ const updateExistingDataStream = async ({
   let lifecycle: any;
 
   try {
-    const simulateResult = await retryTransientEsErrors(() =>
+    const simulateResult = await retryTransientEsErrors(async () =>
       esClient.indices.simulateTemplate({
-        name: dataStreamNameToIndexTemplateName(dataStreamName),
+        name: await getIndexTemplate(esClient, dataStreamName),
       })
     );
 

x-pack/test/fleet_api_integration/apis/epm/install_hidden_datastreams.ts

@@ -23,7 +23,7 @@ export default function (providerContext: FtrProviderContext) {
     skipIfNoDockerRegistry(providerContext);
     setupFleetAndAgents(providerContext);
 
-    after(async () => {
+    afterEach(async () => {
       await deletePackage('apm', '8.8.0');
     });
 
@@ -88,5 +88,60 @@ export default function (providerContext: FtrProviderContext) {
       // datastream rolled over
       expect(Object.keys(ds).length).greaterThan(1);
     });
+
+    it('should not rollover datastreams when successfully updated mappings', async function () {
+      await supertest
+        .post(`/api/fleet/epm/packages/apm/8.8.0`)
+        .set('kbn-xsrf', 'xxxx')
+        .send({ force: true })
+        .expect(200);
+
+      await es.index({
+        index: 'metrics-apm.app.default-default',
+        document: {
+          '@timestamp': '2023-05-30T07:50:00.000Z',
+          agent: {
+            name: 'go',
+          },
+          data_stream: {
+            dataset: 'metrics-apm.app.default',
+            namespace: 'default',
+            type: 'metrics',
+          },
+          ecs: {
+            version: '8.8.0-dev',
+          },
+          event: {
+            agent_id_status: 'missing',
+            ingested: '2023-05-30T07:57:12Z',
+          },
+          observer: {
+            hostname: '047e282994fb',
+            type: 'apm-server',
+            version: '8.8.0',
+          },
+        },
+      });
+
+      let ds = await es.indices.get({
+        index: 'metrics-apm.app.default*',
+        expand_wildcards: ['open', 'hidden'],
+      });
+      const indicesBefore = Object.keys(ds).length;
+
+      await supertest
+        .post(`/api/fleet/epm/packages/apm/8.8.0`)
+        .set('kbn-xsrf', 'xxxx')
+        .send({ force: true })
+        .expect(200);
+
+      ds = await es.indices.get({
+        index: 'metrics-apm.app.default*',
+        expand_wildcards: ['open', 'hidden'],
+      });
+      const indicesAfter = Object.keys(ds).length;
+      // datastream did not roll over
+      expect(indicesAfter).equal(indicesBefore);
+    });
   });
 }