[Index Management] Display index mode of data streams depending on `l…

…ogsdb.enabled` cluster setting (elastic#207131)

Closes elastic#206126

## Summary

This PR displays a LogsDB index mode for `logs-*-*` data streams and
templates with the `logs-*-*` index pattern if the
`cluster.logsdb.enabled` setting is set to true and as "Standard" if
it's set to false. If the setting is not set, it is false by default in
stateful (and so "Standard" index mode) and true by default in
serverless (and so "LogsDB" index mode).

**How to test:**
In Stateful:
1. Create an index template with a `logs-*-*` index pattern
2. Create a `logs-*-*` data stream: `PUT _data_stream/logs-test-1`
3. Go to Index Management -> Index templates and click on the created
index templates. Verify that the index mode is correct (**NOTE:** In
stateful, the `cluster.logsdb.enabled` setting is `false` by default,
while in serverless it's `true` by default. Therefore, the index mode
should be `Standard` in stateful and `LogsDB` in serverless.
4. Go to Index Management -> Data Streams and find the created data
stream. Verify that the index mode is correct (**NOTE:** In stateful,
the `cluster.logsdb.enabled` setting is `false` by default, while in
serverless it's `true` by default. Therefore, the index mode should be
`Standard` in stateful and `LogsDB` in serverless.
5. In Console, set the `cluster.logsdb.enabled` setting:
```
PUT /_cluster/settings
{
  "persistent" : {
    "cluster.logsdb.enabled" : true
  }
}
```
6. Go to the data stream/templates list and verify that the index mode
of the test data stream/template that we created is displayed as LogsDB
7. Change the cluster setting to `false` and verify that the index mode
is now displayed as Standard.

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
(cherry picked from commit b08eaa7)

x-pack/platform/plugins/shared/index_management/common/lib/template_serialization.test.ts

@@ -62,6 +62,36 @@ describe('Template serialization', () => {
           ).toHaveProperty('indexMode', value ?? STANDARD_INDEX_MODE);
         });
       });
+
+      describe('with logs-*-* index pattern', () => {
+        test('deserializes to logsdb index mode when logsdb is enabled', () => {
+          expect(
+            deserializeTemplate(
+              {
+                ...defaultSerializedTemplate,
+                index_patterns: ['logs-*-*'],
+                name: 'my_template',
+              },
+              undefined,
+              true
+            )
+          ).toHaveProperty('indexMode', LOGSDB_INDEX_MODE);
+        });
+
+        test('deserializes to standard index mode when logsdb is disabled', () => {
+          expect(
+            deserializeTemplate(
+              {
+                ...defaultSerializedTemplate,
+                index_patterns: ['logs-*-*'],
+                name: 'my_template',
+              },
+              undefined,
+              false
+            )
+          ).toHaveProperty('indexMode', STANDARD_INDEX_MODE);
+        });
+      });
     });
 
     describe('serializeTemplate()', () => {

x-pack/platform/plugins/shared/index_management/common/lib/template_serialization.ts

@@ -63,7 +63,8 @@ export function serializeTemplate(templateDeserialized: TemplateDeserialized): T
 
 export function deserializeTemplate(
   templateEs: TemplateSerialized & { name: string },
-  cloudManagedTemplatePrefix?: string
+  cloudManagedTemplatePrefix?: string,
+  isLogsdbEnabled?: boolean
 ): TemplateDeserialized {
   const {
     name,
@@ -92,7 +93,7 @@ export function deserializeTemplate(
   const ilmPolicyName = settings?.index?.lifecycle?.name;
 
   const indexMode = (settings?.index?.mode ??
-    (indexPatterns.some((pattern) => pattern === 'logs-*-*')
+    (isLogsdbEnabled && indexPatterns.some((pattern) => pattern === 'logs-*-*')
       ? LOGSDB_INDEX_MODE
       : STANDARD_INDEX_MODE)) as IndexMode;
 
@@ -168,13 +169,15 @@ export function serializeLegacyTemplate(template: TemplateDeserialized): LegacyT
 
 export function deserializeLegacyTemplate(
   templateEs: LegacyTemplateSerialized & { name: string },
-  cloudManagedTemplatePrefix?: string
+  cloudManagedTemplatePrefix?: string,
+  isLogsdbEnabled?: boolean
 ): TemplateDeserialized {
   const { settings, aliases, mappings, ...rest } = templateEs;
 
   const deserializedTemplate = deserializeTemplate(
     { ...rest, template: { aliases, settings, mappings } },
-    cloudManagedTemplatePrefix
+    cloudManagedTemplatePrefix,
+    isLogsdbEnabled
   );
 
   return {

x-pack/platform/plugins/shared/index_management/server/lib/data_stream_serialization.ts

@@ -6,10 +6,14 @@
  */
 
 import { ByteSizeValue } from '@kbn/config-schema';
+import { LOGSDB_INDEX_MODE, STANDARD_INDEX_MODE } from '../../common/constants';
 import { IndexMode } from '../../common/types/data_streams';
 import type { DataStream, EnhancedDataStreamFromEs, Health } from '../../common';
 
-export function deserializeDataStream(dataStreamFromEs: EnhancedDataStreamFromEs): DataStream {
+export function deserializeDataStream(
+  dataStreamFromEs: EnhancedDataStreamFromEs,
+  isLogsdbEnabled: boolean
+): DataStream {
   const {
     name,
     timestamp_field: timeStampField,
@@ -75,12 +79,16 @@ export function deserializeDataStream(dataStreamFromEs: EnhancedDataStreamFromEs
       globalMaxRetention,
     },
     nextGenerationManagedBy,
-    indexMode: (indexMode ?? 'standard') as IndexMode,
+    indexMode: (indexMode ??
+      (isLogsdbEnabled && /^logs-[^-]+-[^-]+$/.test(name)
+        ? LOGSDB_INDEX_MODE
+        : STANDARD_INDEX_MODE)) as IndexMode,
   };
 }
 
 export function deserializeDataStreamList(
-  dataStreamsFromEs: EnhancedDataStreamFromEs[]
+  dataStreamsFromEs: EnhancedDataStreamFromEs[],
+  isLogsdbEnabled: boolean
 ): DataStream[] {
-  return dataStreamsFromEs.map((dataStream) => deserializeDataStream(dataStream));
+  return dataStreamsFromEs.map((dataStream) => deserializeDataStream(dataStream, isLogsdbEnabled));
 }

x-pack/platform/plugins/shared/index_management/server/routes/api/data_streams/register_get_route.ts

@@ -177,6 +177,12 @@ export function registerGetAllRoute({ router, lib: { handleEsError }, config }:
         const { index_templates: indexTemplates } =
           await client.asCurrentUser.indices.getIndexTemplate();
 
+        const { persistent, defaults } = await client.asInternalUser.cluster.getSettings({
+          include_defaults: true,
+        });
+        const isLogsdbEnabled =
+          (persistent?.cluster?.logsdb?.enabled ?? defaults?.cluster?.logsdb?.enabled) === 'true';
+
         // Only take the lifecycle of the first data stream since all data streams have the same global retention period
         const lifecycle = await getDataStreamLifecycle(client, dataStreams[0].name);
         // @ts-ignore - TS doesn't know about the `global_retention` property yet
@@ -191,7 +197,9 @@ export function registerGetAllRoute({ router, lib: { handleEsError }, config }:
           indexTemplates,
         });
 
-        return response.ok({ body: deserializeDataStreamList(enhancedDataStreams) });
+        return response.ok({
+          body: deserializeDataStreamList(enhancedDataStreams, isLogsdbEnabled),
+        });
       } catch (error) {
         return handleEsError({ error, response });
       }
@@ -262,7 +270,14 @@ export function registerGetOneRoute({ router, lib: { handleEsError }, config }:
             globalMaxRetention,
             indexTemplates,
           });
-          const body = deserializeDataStream(enhancedDataStreams[0]);
+
+          const { persistent, defaults } = await client.asInternalUser.cluster.getSettings({
+            include_defaults: true,
+          });
+          const isLogsdbEnabled =
+            (persistent?.cluster?.logsdb?.enabled ?? defaults?.cluster?.logsdb?.enabled) === 'true';
+
+          const body = deserializeDataStream(enhancedDataStreams[0], isLogsdbEnabled);
           return response.ok({ body });
         }
 

x-pack/platform/plugins/shared/index_management/server/routes/api/templates/register_get_routes.ts

@@ -37,8 +37,11 @@ export function registerGetAllRoute({ router, config, lib: { handleEsError } }:
         const { index_templates: templatesEs } =
           await client.asCurrentUser.indices.getIndexTemplate();
 
-        // @ts-expect-error TemplateSerialized.index_patterns not compatible with IndicesIndexTemplate.index_patterns
-        const templates = deserializeTemplateList(templatesEs, cloudManagedTemplatePrefix);
+        const templates = deserializeTemplateList(
+          // @ts-expect-error TemplateSerialized.index_patterns not compatible with IndicesIndexTemplate.index_patterns
+          templatesEs,
+          cloudManagedTemplatePrefix
+        );
 
         if (config.isLegacyTemplatesEnabled === false) {
           // If isLegacyTemplatesEnabled=false, we do not want to fetch legacy templates and return an empty array;
@@ -98,6 +101,12 @@ export function registerGetOneRoute({ router, config, lib: { handleEsError } }:
       try {
         const cloudManagedTemplatePrefix = await getCloudManagedTemplatePrefix(client);
 
+        const { persistent, defaults } = await client.asInternalUser.cluster.getSettings({
+          include_defaults: true,
+        });
+        const isLogsdbEnabled =
+          (persistent?.cluster?.logsdb?.enabled ?? defaults?.cluster?.logsdb?.enabled) === 'true';
+
         if (isLegacy) {
           const indexTemplateByName = await client.asCurrentUser.indices.getTemplate({
             name,
@@ -107,7 +116,8 @@ export function registerGetOneRoute({ router, config, lib: { handleEsError } }:
             return response.ok({
               body: deserializeLegacyTemplate(
                 { ...indexTemplateByName[name], name },
-                cloudManagedTemplatePrefix
+                cloudManagedTemplatePrefix,
+                isLogsdbEnabled
               ),
             });
           }
@@ -120,7 +130,8 @@ export function registerGetOneRoute({ router, config, lib: { handleEsError } }:
               body: deserializeTemplate(
                 // @ts-expect-error TemplateSerialized.index_patterns not compatible with IndicesIndexTemplate.index_patterns
                 { ...indexTemplates[0].index_template, name },
-                cloudManagedTemplatePrefix
+                cloudManagedTemplatePrefix,
+                isLogsdbEnabled
               ),
             });
           }

x-pack/test/api_integration/apis/management/index_management/data_streams.ts

@@ -14,6 +14,7 @@ import { datastreamsHelpers } from './lib/datastreams.helpers';
 
 export default function ({ getService }: FtrProviderContext) {
   const supertest = getService('supertest');
+  const es = getService('es');
 
   const {
     createDataStream,
@@ -164,20 +165,63 @@ export default function ({ getService }: FtrProviderContext) {
         });
       });
 
-      it('correctly returns index mode property', async () => {
-        const logsdbDataStreamName = 'logsdb-test-data-stream';
-        const indexMode = 'logsdb';
+      describe('index mode', () => {
+        it('correctly returns index mode property based on index settings', async () => {
+          const logsdbDataStreamName = 'logsdb-test-data-stream';
+          const indexMode = 'logsdb';
 
-        await createDataStream(logsdbDataStreamName, indexMode);
+          await createDataStream(logsdbDataStreamName, indexMode);
 
-        const { body: dataStream } = await supertest
-          .get(`${API_BASE_PATH}/data_streams/${logsdbDataStreamName}`)
-          .set('kbn-xsrf', 'xxx')
-          .expect(200);
+          const { body: dataStream } = await supertest
+            .get(`${API_BASE_PATH}/data_streams/${logsdbDataStreamName}`)
+            .set('kbn-xsrf', 'xxx')
+            .expect(200);
 
-        expect(dataStream.indexMode).to.eql(indexMode);
+          expect(dataStream.indexMode).to.eql(indexMode);
 
-        await deleteDataStream(logsdbDataStreamName);
+          await deleteDataStream(logsdbDataStreamName);
+        });
+
+        describe('index mode of logs-*-* data streams', () => {
+          const logsdbDataStreamName = 'logs-test-ds';
+
+          before(async () => {
+            await createDataStream(logsdbDataStreamName);
+          });
+
+          after(async () => {
+            await deleteDataStream(logsdbDataStreamName);
+          });
+
+          const logsdbSettings: Array<{ enabled: boolean | null; indexMode: string }> = [
+            { enabled: true, indexMode: 'logsdb' },
+            { enabled: false, indexMode: 'standard' },
+            { enabled: null, indexMode: 'standard' }, // In stateful Kibana, the cluster.logsdb.enabled setting is false by default, so standard index mode
+          ];
+
+          logsdbSettings.forEach(({ enabled, indexMode }) => {
+            it(`returns ${indexMode} index mode if logsdb.enabled setting is ${enabled}`, async () => {
+              await es.cluster.putSettings({
+                body: {
+                  persistent: {
+                    cluster: {
+                      logsdb: {
+                        enabled,
+                      },
+                    },
+                  },
+                },
+              });
+
+              const { body: dataStream } = await supertest
+                .get(`${API_BASE_PATH}/data_streams/${logsdbDataStreamName}`)
+                .set('kbn-xsrf', 'xxx')
+                .expect(200);
+
+              expect(dataStream.indexMode).to.eql(indexMode);
+            });
+          });
+        });
       });
     });
 

x-pack/test/api_integration/apis/management/index_management/templates.ts

@@ -15,6 +15,7 @@ import { getRandomString } from './lib/random';
 
 export default function ({ getService }: FtrProviderContext) {
   const log = getService('log');
+  const es = getService('es');
   const { catTemplate, getTemplatePayload, getSerializedTemplate } = templatesHelpers(getService);
   const {
     getAllTemplates,
@@ -233,6 +234,43 @@ export default function ({ getService }: FtrProviderContext) {
         expect(Object.keys(body).sort()).to.eql(expectedKeys);
         expect(Object.keys(body.template).sort()).to.eql(expectedTemplateKeys);
       });
+
+      describe('with logs-*-* index pattern', () => {
+        const logsdbTemplateName = 'test-logsdb-template';
+        before(async () => {
+          const template = getTemplatePayload(logsdbTemplateName, ['logs-*-*']);
+          await createTemplate(template).expect(200);
+        });
+
+        after(async () => {
+          await deleteTemplates([{ name: logsdbTemplateName }]);
+        });
+
+        const logsdbSettings: Array<{ enabled: boolean | null; indexMode: string }> = [
+          { enabled: true, indexMode: 'logsdb' },
+          { enabled: false, indexMode: 'standard' },
+          { enabled: null, indexMode: 'standard' }, // In stateful Kibana, the cluster.logsdb.enabled setting is false by default, so standard index mode
+        ];
+
+        logsdbSettings.forEach(({ enabled, indexMode }) => {
+          it(`returns ${indexMode} index mode if logsdb.enabled setting is ${enabled}`, async () => {
+            await es.cluster.putSettings({
+              body: {
+                persistent: {
+                  cluster: {
+                    logsdb: {
+                      enabled,
+                    },
+                  },
+                },
+              },
+            });
+
+            const { body } = await getOneTemplate(logsdbTemplateName).expect(200);
+            expect(body.indexMode).to.equal(indexMode);
+          });
+        });
+      });
     });
 
     describe('create', () => {

x-pack/test_serverless/api_integration/services/index_management/svl_templates.helpers.ts

@@ -5,7 +5,11 @@
  * 2.0.
  */
 
-import { TemplateDeserialized, TemplateSerialized } from '@kbn/index-management-plugin/common';
+import {
+  TemplateDeserialized,
+  TemplateSerialized,
+  IndexMode,
+} from '@kbn/index-management-plugin/common';
 import { INDEX_PATTERNS } from './constants';
 import { FtrProviderContext } from '../../ftr_provider_context';
 
@@ -51,12 +55,13 @@ export function SvlTemplatesHelpers({ getService }: FtrProviderContext) {
     name: string,
     indexPatterns: string[] = INDEX_PATTERNS,
     isLegacy: boolean = false,
-    isMappingsSourceFieldEnabled: boolean = true
+    isMappingsSourceFieldEnabled: boolean = true,
+    indexMode?: IndexMode
   ) => {
     const baseTemplate: TemplateDeserialized = {
       name,
       indexPatterns,
-      indexMode: 'standard',
+      indexMode,
       version: 1,
       template: { ...getTemplateMock(isMappingsSourceFieldEnabled) },
       _kbnMeta: {