[8.x] [Index Management] Display index mode of data streams depending on `logsdb.enabled` cluster setting (#207131)

x-pack/platform/plugins/shared/index_management/common/lib/template_serialization.test.ts

@@ -62,6 +62,36 @@ describe('Template serialization', () => {
           ).toHaveProperty('indexMode', value ?? STANDARD_INDEX_MODE);
         });
       });
+
+      describe('with logs-*-* index pattern', () => {
+        test('deserializes to logsdb index mode when logsdb is enabled', () => {
+          expect(
+            deserializeTemplate(
+              {
+                ...defaultSerializedTemplate,
+                index_patterns: ['logs-*-*'],
+                name: 'my_template',
+              },
+              undefined,
+              true
+            )
+          ).toHaveProperty('indexMode', LOGSDB_INDEX_MODE);
+        });
+
+        test('deserializes to standard index mode when logsdb is disabled', () => {
+          expect(
+            deserializeTemplate(
+              {
+                ...defaultSerializedTemplate,
+                index_patterns: ['logs-*-*'],
+                name: 'my_template',
+              },
+              undefined,
+              false
+            )
+          ).toHaveProperty('indexMode', STANDARD_INDEX_MODE);
+        });
+      });
     });
 
     describe('serializeTemplate()', () => {

x-pack/platform/plugins/shared/index_management/common/lib/template_serialization.ts

@@ -63,7 +63,8 @@ export function serializeTemplate(templateDeserialized: TemplateDeserialized): T
 
 export function deserializeTemplate(
   templateEs: TemplateSerialized & { name: string },
-  cloudManagedTemplatePrefix?: string
+  cloudManagedTemplatePrefix?: string,
+  isLogsdbEnabled?: boolean
 ): TemplateDeserialized {
   const {
     name,
@@ -92,7 +93,7 @@ export function deserializeTemplate(
   const ilmPolicyName = settings?.index?.lifecycle?.name;
 
   const indexMode = (settings?.index?.mode ??
-    (indexPatterns.some((pattern) => pattern === 'logs-*-*')
+    (isLogsdbEnabled && indexPatterns.some((pattern) => pattern === 'logs-*-*')
       ? LOGSDB_INDEX_MODE
       : STANDARD_INDEX_MODE)) as IndexMode;
 
@@ -168,13 +169,15 @@ export function serializeLegacyTemplate(template: TemplateDeserialized): LegacyT
 
 export function deserializeLegacyTemplate(
   templateEs: LegacyTemplateSerialized & { name: string },
-  cloudManagedTemplatePrefix?: string
+  cloudManagedTemplatePrefix?: string,
+  isLogsdbEnabled?: boolean
 ): TemplateDeserialized {
   const { settings, aliases, mappings, ...rest } = templateEs;
 
   const deserializedTemplate = deserializeTemplate(
     { ...rest, template: { aliases, settings, mappings } },
-    cloudManagedTemplatePrefix
+    cloudManagedTemplatePrefix,
+    isLogsdbEnabled
   );
 
   return {

x-pack/platform/plugins/shared/index_management/server/lib/data_stream_serialization.ts

@@ -6,10 +6,14 @@
  */
 
 import { ByteSizeValue } from '@kbn/config-schema';
+import { LOGSDB_INDEX_MODE, STANDARD_INDEX_MODE } from '../../common/constants';
 import { IndexMode } from '../../common/types/data_streams';
 import type { DataStream, EnhancedDataStreamFromEs, Health } from '../../common';
 
-export function deserializeDataStream(dataStreamFromEs: EnhancedDataStreamFromEs): DataStream {
+export function deserializeDataStream(
+  dataStreamFromEs: EnhancedDataStreamFromEs,
+  isLogsdbEnabled: boolean
+): DataStream {
   const {
     name,
     timestamp_field: timeStampField,
@@ -75,12 +79,16 @@ export function deserializeDataStream(dataStreamFromEs: EnhancedDataStreamFromEs
       globalMaxRetention,
     },
     nextGenerationManagedBy,
-    indexMode: (indexMode ?? 'standard') as IndexMode,
+    indexMode: (indexMode ??
+      (isLogsdbEnabled && /^logs-[^-]+-[^-]+$/.test(name)
+        ? LOGSDB_INDEX_MODE
+        : STANDARD_INDEX_MODE)) as IndexMode,
   };
 }
 
 export function deserializeDataStreamList(
-  dataStreamsFromEs: EnhancedDataStreamFromEs[]
+  dataStreamsFromEs: EnhancedDataStreamFromEs[],
+  isLogsdbEnabled: boolean
 ): DataStream[] {
-  return dataStreamsFromEs.map((dataStream) => deserializeDataStream(dataStream));
+  return dataStreamsFromEs.map((dataStream) => deserializeDataStream(dataStream, isLogsdbEnabled));
 }

x-pack/platform/plugins/shared/index_management/server/routes/api/data_streams/register_get_route.ts

@@ -177,6 +177,12 @@ export function registerGetAllRoute({ router, lib: { handleEsError }, config }:
         const { index_templates: indexTemplates } =
           await client.asCurrentUser.indices.getIndexTemplate();
 
+        const { persistent, defaults } = await client.asInternalUser.cluster.getSettings({
+          include_defaults: true,
+        });
+        const isLogsdbEnabled =
+          (persistent?.cluster?.logsdb?.enabled ?? defaults?.cluster?.logsdb?.enabled) === 'true';
+
         // Only take the lifecycle of the first data stream since all data streams have the same global retention period
         const lifecycle = await getDataStreamLifecycle(client, dataStreams[0].name);
         // @ts-ignore - TS doesn't know about the `global_retention` property yet
@@ -191,7 +197,9 @@ export function registerGetAllRoute({ router, lib: { handleEsError }, config }:
           indexTemplates,
         });
 
-        return response.ok({ body: deserializeDataStreamList(enhancedDataStreams) });
+        return response.ok({
+          body: deserializeDataStreamList(enhancedDataStreams, isLogsdbEnabled),
+        });
       } catch (error) {
         return handleEsError({ error, response });
       }
@@ -262,7 +270,14 @@ export function registerGetOneRoute({ router, lib: { handleEsError }, config }:
             globalMaxRetention,
             indexTemplates,
           });
-          const body = deserializeDataStream(enhancedDataStreams[0]);
+
+          const { persistent, defaults } = await client.asInternalUser.cluster.getSettings({
+            include_defaults: true,
+          });
+          const isLogsdbEnabled =
+            (persistent?.cluster?.logsdb?.enabled ?? defaults?.cluster?.logsdb?.enabled) === 'true';
+
+          const body = deserializeDataStream(enhancedDataStreams[0], isLogsdbEnabled);
           return response.ok({ body });
         }
 

x-pack/platform/plugins/shared/index_management/server/routes/api/templates/register_get_routes.ts

@@ -37,8 +37,11 @@ export function registerGetAllRoute({ router, config, lib: { handleEsError } }:
         const { index_templates: templatesEs } =
           await client.asCurrentUser.indices.getIndexTemplate();
 
-        // @ts-expect-error TemplateSerialized.index_patterns not compatible with IndicesIndexTemplate.index_patterns
-        const templates = deserializeTemplateList(templatesEs, cloudManagedTemplatePrefix);
+        const templates = deserializeTemplateList(
+          // @ts-expect-error TemplateSerialized.index_patterns not compatible with IndicesIndexTemplate.index_patterns
+          templatesEs,
+          cloudManagedTemplatePrefix
+        );
 
         if (config.isLegacyTemplatesEnabled === false) {
           // If isLegacyTemplatesEnabled=false, we do not want to fetch legacy templates and return an empty array;
@@ -98,6 +101,12 @@ export function registerGetOneRoute({ router, config, lib: { handleEsError } }:
       try {
         const cloudManagedTemplatePrefix = await getCloudManagedTemplatePrefix(client);
 
+        const { persistent, defaults } = await client.asInternalUser.cluster.getSettings({
+          include_defaults: true,
+        });
+        const isLogsdbEnabled =
+          (persistent?.cluster?.logsdb?.enabled ?? defaults?.cluster?.logsdb?.enabled) === 'true';
+
         if (isLegacy) {
           const indexTemplateByName = await client.asCurrentUser.indices.getTemplate({
             name,
@@ -107,7 +116,8 @@ export function registerGetOneRoute({ router, config, lib: { handleEsError } }:
             return response.ok({
               body: deserializeLegacyTemplate(
                 { ...indexTemplateByName[name], name },
-                cloudManagedTemplatePrefix
+                cloudManagedTemplatePrefix,
+                isLogsdbEnabled
               ),
             });
           }
@@ -120,7 +130,8 @@ export function registerGetOneRoute({ router, config, lib: { handleEsError } }:
               body: deserializeTemplate(
                 // @ts-expect-error TemplateSerialized.index_patterns not compatible with IndicesIndexTemplate.index_patterns
                 { ...indexTemplates[0].index_template, name },
-                cloudManagedTemplatePrefix
+                cloudManagedTemplatePrefix,
+                isLogsdbEnabled
               ),
             });
           }

x-pack/test/api_integration/apis/management/index_management/data_streams.ts

@@ -14,6 +14,7 @@ import { datastreamsHelpers } from './lib/datastreams.helpers';
 
 export default function ({ getService }: FtrProviderContext) {
   const supertest = getService('supertest');
+  const es = getService('es');
 
   const {
     createDataStream,
@@ -164,20 +165,63 @@ export default function ({ getService }: FtrProviderContext) {
         });
       });
 
-      it('correctly returns index mode property', async () => {
-        const logsdbDataStreamName = 'logsdb-test-data-stream';
-        const indexMode = 'logsdb';
+      describe('index mode', () => {
+        it('correctly returns index mode property based on index settings', async () => {
+          const logsdbDataStreamName = 'logsdb-test-data-stream';
+          const indexMode = 'logsdb';
 
-        await createDataStream(logsdbDataStreamName, indexMode);
+          await createDataStream(logsdbDataStreamName, indexMode);
 
-        const { body: dataStream } = await supertest
-          .get(`${API_BASE_PATH}/data_streams/${logsdbDataStreamName}`)
-          .set('kbn-xsrf', 'xxx')
-          .expect(200);
+          const { body: dataStream } = await supertest
+            .get(`${API_BASE_PATH}/data_streams/${logsdbDataStreamName}`)
+            .set('kbn-xsrf', 'xxx')
+            .expect(200);
 
-        expect(dataStream.indexMode).to.eql(indexMode);
+          expect(dataStream.indexMode).to.eql(indexMode);
 
-        await deleteDataStream(logsdbDataStreamName);
+          await deleteDataStream(logsdbDataStreamName);
+        });
+
+        describe('index mode of logs-*-* data streams', () => {
+          const logsdbDataStreamName = 'logs-test-ds';
+
+          before(async () => {
+            await createDataStream(logsdbDataStreamName);
+          });
+
+          after(async () => {
+            await deleteDataStream(logsdbDataStreamName);
+          });
+
+          const logsdbSettings: Array<{ enabled: boolean | null; indexMode: string }> = [
+            { enabled: true, indexMode: 'logsdb' },
+            { enabled: false, indexMode: 'standard' },
+            { enabled: null, indexMode: 'standard' }, // In stateful Kibana, the cluster.logsdb.enabled setting is false by default, so standard index mode
+          ];
+
+          logsdbSettings.forEach(({ enabled, indexMode }) => {
+            it(`returns ${indexMode} index mode if logsdb.enabled setting is ${enabled}`, async () => {
+              await es.cluster.putSettings({
+                body: {
+                  persistent: {
+                    cluster: {
+                      logsdb: {
+                        enabled,
+                      },
+                    },
+                  },
+                },
+              });
+
+              const { body: dataStream } = await supertest
+                .get(`${API_BASE_PATH}/data_streams/${logsdbDataStreamName}`)
+                .set('kbn-xsrf', 'xxx')
+                .expect(200);
+
+              expect(dataStream.indexMode).to.eql(indexMode);
+            });
+          });
+        });
       });
     });
 

x-pack/test/api_integration/apis/management/index_management/templates.ts

@@ -15,6 +15,7 @@ import { getRandomString } from './lib/random';
 
 export default function ({ getService }: FtrProviderContext) {
   const log = getService('log');
+  const es = getService('es');
   const { catTemplate, getTemplatePayload, getSerializedTemplate } = templatesHelpers(getService);
   const {
     getAllTemplates,
@@ -233,6 +234,43 @@ export default function ({ getService }: FtrProviderContext) {
         expect(Object.keys(body).sort()).to.eql(expectedKeys);
         expect(Object.keys(body.template).sort()).to.eql(expectedTemplateKeys);
       });
+
+      describe('with logs-*-* index pattern', () => {
+        const logsdbTemplateName = 'test-logsdb-template';
+        before(async () => {
+          const template = getTemplatePayload(logsdbTemplateName, ['logs-*-*']);
+          await createTemplate(template).expect(200);
+        });
+
+        after(async () => {
+          await deleteTemplates([{ name: logsdbTemplateName }]);
+        });
+
+        const logsdbSettings: Array<{ enabled: boolean | null; indexMode: string }> = [
+          { enabled: true, indexMode: 'logsdb' },
+          { enabled: false, indexMode: 'standard' },
+          { enabled: null, indexMode: 'standard' }, // In stateful Kibana, the cluster.logsdb.enabled setting is false by default, so standard index mode
+        ];
+
+        logsdbSettings.forEach(({ enabled, indexMode }) => {
+          it(`returns ${indexMode} index mode if logsdb.enabled setting is ${enabled}`, async () => {
+            await es.cluster.putSettings({
+              body: {
+                persistent: {
+                  cluster: {
+                    logsdb: {
+                      enabled,
+                    },
+                  },
+                },
+              },
+            });
+
+            const { body } = await getOneTemplate(logsdbTemplateName).expect(200);
+            expect(body.indexMode).to.equal(indexMode);
+          });
+        });
+      });
     });
 
     describe('create', () => {

x-pack/test_serverless/api_integration/services/index_management/svl_templates.helpers.ts

@@ -5,7 +5,11 @@
  * 2.0.
  */
 
-import { TemplateDeserialized, TemplateSerialized } from '@kbn/index-management-plugin/common';
+import {
+  TemplateDeserialized,
+  TemplateSerialized,
+  IndexMode,
+} from '@kbn/index-management-plugin/common';
 import { INDEX_PATTERNS } from './constants';
 import { FtrProviderContext } from '../../ftr_provider_context';
 
@@ -51,12 +55,13 @@ export function SvlTemplatesHelpers({ getService }: FtrProviderContext) {
     name: string,
     indexPatterns: string[] = INDEX_PATTERNS,
     isLegacy: boolean = false,
-    isMappingsSourceFieldEnabled: boolean = true
+    isMappingsSourceFieldEnabled: boolean = true,
+    indexMode?: IndexMode
   ) => {
     const baseTemplate: TemplateDeserialized = {
       name,
       indexPatterns,
-      indexMode: 'standard',
+      indexMode,
       version: 1,
       template: { ...getTemplateMock(isMappingsSourceFieldEnabled) },
       _kbnMeta: {