[Index Management] Add support for lookup index mode (elastic#207413)

Closes elastic#207350

## Summary

This PR adds the following changes:

- Adds the "Lookup" option to the index mode field in index/component
template creation flow:
<img width="1232" alt="Screenshot 2025-01-22 at 12 29 40"
src="https://github.com/user-attachments/assets/6f028dbc-65b9-40c5-a225-bafa163d8c36"
/>


- The index mode field is now always enabled, even when the data stream
toggle is disabled:
<img width="1016" alt="Screenshot 2025-01-21 at 17 16 35"
src="https://github.com/user-attachments/assets/77fdacc9-f224-4035-b662-185c614becb2"
/>


-  Adds an index mode field to the create index modal:


https://github.com/user-attachments/assets/65241543-18df-4c6a-90ad-814b93d5d410

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

x-pack/platform/plugins/shared/index_management/__jest__/client_integration/home/indices_tab.helpers.ts

@@ -45,6 +45,7 @@ export interface IndicesTestBed extends TestBed<TestSubjects> {
     clickCreateIndexButton: () => Promise<void>;
     clickCreateIndexCancelButton: () => Promise<void>;
     clickCreateIndexSaveButton: () => Promise<void>;
+    selectIndexMode: (indexModeTestSubj: string) => Promise<void>;
   };
   findDataStreamDetailPanel: () => ReactWrapper;
   findDataStreamDetailPanelTitle: () => string;
@@ -186,6 +187,21 @@ export const setup = async (
     component.update();
   };
 
+  const selectIndexMode = async (indexModeTestSubj: string) => {
+    const { find, exists, component } = testBed;
+
+    await act(async () => {
+      find('indexModeField').simulate('click');
+    });
+    component.update();
+
+    expect(exists(indexModeTestSubj)).toBe(true);
+    await act(async () => {
+      find(indexModeTestSubj).simulate('click');
+    });
+    component.update();
+  };
+
   return {
     ...testBed,
     actions: {
@@ -202,6 +218,7 @@ export const setup = async (
       clickCreateIndexButton,
       clickCreateIndexCancelButton,
       clickCreateIndexSaveButton,
+      selectIndexMode,
     },
     findDataStreamDetailPanel,
     findDataStreamDetailPanelTitle,

x-pack/platform/plugins/shared/index_management/__jest__/client_integration/home/indices_tab.test.tsx

@@ -492,11 +492,13 @@ describe('<IndexManagementHome />', () => {
       });
       component.update();
 
+      await actions.selectIndexMode('indexModeLookupOption');
+
       await actions.clickCreateIndexSaveButton();
 
       // Saves the index with expected name
       expect(httpSetup.put).toHaveBeenCalledWith(`${INTERNAL_API_BASE_PATH}/indices/create`, {
-        body: '{"indexName":"test-index-b"}',
+        body: '{"indexName":"test-index-b","indexMode":"lookup"}',
       });
       // It refresh indices after saving
       expect(httpSetup.get).toHaveBeenCalledTimes(2);

x-pack/platform/plugins/shared/index_management/__jest__/client_integration/index_template_wizard/template_clone.test.tsx

@@ -96,13 +96,14 @@ describe('<TemplateClone />', () => {
         actions.clickNextButton();
       });
 
-      const { template, priority, version, _kbnMeta, allowAutoCreate } = templateToClone;
+      const { template, indexMode, priority, version, _kbnMeta, allowAutoCreate } = templateToClone;
       expect(httpSetup.post).toHaveBeenLastCalledWith(
         `${API_BASE_PATH}/index_templates`,
         expect.objectContaining({
           body: JSON.stringify({
             name: `${templateToClone.name}-copy`,
             indexPatterns: DEFAULT_INDEX_PATTERNS,
+            indexMode,
             priority,
             version,
             allowAutoCreate,

x-pack/platform/plugins/shared/index_management/__jest__/client_integration/index_template_wizard/template_create.test.tsx

@@ -476,8 +476,8 @@ describe('<TemplateCreate />', () => {
           body: JSON.stringify({
             name: 'my_logs_template',
             indexPatterns: ['logs-*-*'],
-            allowAutoCreate: 'NO_OVERWRITE',
             indexMode: 'logsdb',
+            allowAutoCreate: 'NO_OVERWRITE',
             dataStream: {},
             _kbnMeta: {
               type: 'default',
@@ -622,8 +622,8 @@ describe('<TemplateCreate />', () => {
           body: JSON.stringify({
             name: TEMPLATE_NAME,
             indexPatterns: DEFAULT_INDEX_PATTERNS,
-            allowAutoCreate: 'TRUE',
             indexMode: 'time_series',
+            allowAutoCreate: 'TRUE',
             dataStream: {},
             _kbnMeta: {
               type: 'default',

x-pack/platform/plugins/shared/index_management/__jest__/client_integration/index_template_wizard/template_edit.test.tsx

@@ -167,9 +167,9 @@ describe('<TemplateEdit />', () => {
           body: JSON.stringify({
             name: 'test',
             indexPatterns: ['myPattern*'],
+            indexMode: 'standard',
             version: 1,
             allowAutoCreate: 'NO_OVERWRITE',
-            indexMode: 'standard',
             dataStream: {
               hidden: true,
               anyUnknownKey: 'should_be_kept',
@@ -287,6 +287,7 @@ describe('<TemplateEdit />', () => {
             body: JSON.stringify({
               name: TEMPLATE_NAME,
               indexPatterns: UPDATED_INDEX_PATTERN,
+              indexMode: 'standard',
               priority: 3,
               version: templateToEdit.version,
               allowAutoCreate: 'TRUE',
@@ -386,6 +387,7 @@ describe('<TemplateEdit />', () => {
           body: JSON.stringify({
             name: TEMPLATE_NAME,
             indexPatterns: INDEX_PATTERNS,
+            indexMode: templateToEdit.indexMode,
             version: templateToEdit.version,
             allowAutoCreate: templateToEdit.allowAutoCreate,
             _kbnMeta: templateToEdit._kbnMeta,

x-pack/platform/plugins/shared/index_management/common/constants/index_modes.ts

@@ -8,3 +8,4 @@
 export const STANDARD_INDEX_MODE = 'standard';
 export const LOGSDB_INDEX_MODE = 'logsdb';
 export const TIME_SERIES_MODE = 'time_series';
+export const LOOKUP_INDEX_MODE = 'lookup';

x-pack/platform/plugins/shared/index_management/public/application/components/template_form/steps/step_logistics.tsx

@@ -40,6 +40,7 @@ import {
   STANDARD_INDEX_MODE,
   TIME_SERIES_MODE,
   LOGSDB_INDEX_MODE,
+  LOOKUP_INDEX_MODE,
 } from '../../../../../common/constants';
 import { indexModeLabels, indexModeDescriptions } from '../../../lib/index_mode_labels';
 
@@ -146,6 +147,19 @@ function getFieldsMeta(esDocsBase: string) {
             </Fragment>
           ),
         },
+        {
+          value: LOOKUP_INDEX_MODE,
+          inputDisplay: indexModeLabels[LOOKUP_INDEX_MODE],
+          'data-test-subj': 'index_mode_logsdb',
+          dropdownDisplay: (
+            <Fragment>
+              <strong>{indexModeLabels[LOOKUP_INDEX_MODE]}</strong>
+              <EuiText size="s" color="subdued">
+                <p>{indexModeDescriptions[LOOKUP_INDEX_MODE]}</p>
+              </EuiText>
+            </Fragment>
+          ),
+        },
       ],
       testSubject: 'indexModeField',
     },
@@ -387,20 +401,18 @@ export const StepLogistics: React.FunctionComponent<Props> = React.memo(
             </FormRow>
           )}
 
-          {doCreateDataStream && (
-            <FormRow title={indexMode.title} description={indexMode.description}>
-              <UseField
-                path="indexMode"
-                componentProps={{
-                  euiFieldProps: {
-                    hasDividers: true,
-                    'data-test-subj': indexMode.testSubject,
-                    options: indexMode.options,
-                  },
-                }}
-              />
-            </FormRow>
-          )}
+          <FormRow title={indexMode.title} description={indexMode.description}>
+            <UseField
+              path="indexMode"
+              componentProps={{
+                euiFieldProps: {
+                  hasDividers: true,
+                  'data-test-subj': indexMode.testSubject,
+                  options: indexMode.options,
+                },
+              }}
+            />
+          </FormRow>
 
           {/*
             Since data stream and data retention are settings that are only allowed for non legacy,

x-pack/platform/plugins/shared/index_management/public/application/lib/index_mode_labels.ts

@@ -10,6 +10,7 @@ import {
   STANDARD_INDEX_MODE,
   LOGSDB_INDEX_MODE,
   TIME_SERIES_MODE,
+  LOOKUP_INDEX_MODE,
 } from '../../../common/constants';
 
 export const indexModeLabels = {
@@ -22,27 +23,36 @@ export const indexModeLabels = {
   [TIME_SERIES_MODE]: i18n.translate('xpack.idxMgmt.indexModeLabels.timeSeriesIndexModeLabel', {
     defaultMessage: 'Time series',
   }),
+  [LOOKUP_INDEX_MODE]: i18n.translate('xpack.idxMgmt.indexModeLabels.lookupIndexModeLabel', {
+    defaultMessage: 'Lookup',
+  }),
 };
 
 export const indexModeDescriptions = {
   [STANDARD_INDEX_MODE]: i18n.translate(
     'xpack.idxMgmt.indexModeDescriptions.standardIndexModeDescription',
     {
       defaultMessage:
-        'Standard indexing with default settings, for data other than logs or metrics',
+        'Standard indexing with default settings, for data other than logs or metrics.',
     }
   ),
   [LOGSDB_INDEX_MODE]: i18n.translate(
     'xpack.idxMgmt.indexModeDescriptions.logsdbIndexModeDescription',
     {
       defaultMessage:
-        'Optimized for storing logs data, with reduced storage and default settings that help reduce the chance of logs being rejected by Elasticsearch',
+        'Optimized for storing logs data, with reduced storage and default settings that help reduce the chance of logs being rejected by Elasticsearch.',
     }
   ),
   [TIME_SERIES_MODE]: i18n.translate(
     'xpack.idxMgmt.indexModeDescriptions.timeSeriesIndexModeDescription',
     {
-      defaultMessage: 'Optimized for metrics data with reduced storage',
+      defaultMessage: 'Optimized for metrics data with reduced storage.',
+    }
+  ),
+  [LOOKUP_INDEX_MODE]: i18n.translate(
+    'xpack.idxMgmt.indexModeDescriptions.lookupIndexModeDescription',
+    {
+      defaultMessage: 'Optimized for ES|QL join operations.',
     }
   ),
 };

x-pack/platform/plugins/shared/index_management/public/application/sections/home/index_list/create_index/create_index_modal.tsx

@@ -5,14 +5,15 @@
  * 2.0.
  */
 
-import React, { useCallback, useState } from 'react';
+import React, { Fragment, useCallback, useState } from 'react';
 import { i18n } from '@kbn/i18n';
 import { FormattedMessage } from '@kbn/i18n-react';
 import {
   EuiButton,
   EuiButtonEmpty,
   EuiCallOut,
   EuiFieldText,
+  EuiSuperSelect,
   EuiForm,
   EuiFormRow,
   EuiModal,
@@ -24,6 +25,13 @@ import {
   EuiText,
 } from '@elastic/eui';
 
+import {
+  LOGSDB_INDEX_MODE,
+  LOOKUP_INDEX_MODE,
+  STANDARD_INDEX_MODE,
+  TIME_SERIES_MODE,
+} from '../../../../../../common/constants';
+import { indexModeDescriptions, indexModeLabels } from '../../../../lib/index_mode_labels';
 import { createIndex } from '../../../../services';
 import { notificationService } from '../../../../services/notification';
 
@@ -41,14 +49,15 @@ export interface CreateIndexModalProps {
 
 export const CreateIndexModal = ({ closeModal, loadIndices }: CreateIndexModalProps) => {
   const [indexName, setIndexName] = useState<string>('');
+  const [indexMode, setIndexMode] = useState<string>(STANDARD_INDEX_MODE);
   const [indexNameError, setIndexNameError] = useState<string | undefined>();
   const [isSaving, setIsSaving] = useState<boolean>(false);
   const [createError, setCreateError] = useState<string | undefined>();
 
   const putCreateIndex = useCallback(async () => {
     setIsSaving(true);
     try {
-      const { error } = await createIndex(indexName);
+      const { error } = await createIndex(indexName, indexMode);
       setIsSaving(false);
       if (!error) {
         notificationService.showSuccessToast(
@@ -66,7 +75,7 @@ export const CreateIndexModal = ({ closeModal, loadIndices }: CreateIndexModalPr
       setIsSaving(false);
       setCreateError(e.message);
     }
-  }, [closeModal, indexName, loadIndices]);
+  }, [closeModal, indexMode, indexName, loadIndices]);
 
   const onSave = () => {
     if (isValidIndexName(indexName)) {
@@ -84,7 +93,7 @@ export const CreateIndexModal = ({ closeModal, loadIndices }: CreateIndexModalPr
   };
 
   return (
-    <EuiModal onClose={closeModal} initialFocus="[name=indexName]">
+    <EuiModal onClose={closeModal} initialFocus="[name=indexName]" css={{ width: 450 }}>
       <EuiModalHeader>
         <EuiModalHeaderTitle>
           <FormattedMessage
@@ -132,6 +141,76 @@ export const CreateIndexModal = ({ closeModal, loadIndices }: CreateIndexModalPr
               data-test-subj="createIndexNameFieldText"
             />
           </EuiFormRow>
+          <EuiFormRow
+            fullWidth
+            label={i18n.translate('xpack.idxMgmt.createIndex.modal.indexMode.label', {
+              defaultMessage: 'Index mode',
+            })}
+            isDisabled={isSaving}
+          >
+            <EuiSuperSelect
+              fullWidth
+              hasDividers
+              name="indexMode"
+              valueOfSelected={indexMode}
+              onChange={(mode) => setIndexMode(mode)}
+              data-test-subj="indexModeField"
+              options={[
+                {
+                  value: STANDARD_INDEX_MODE,
+                  inputDisplay: indexModeLabels[STANDARD_INDEX_MODE],
+                  'data-test-subj': 'indexModeStandardOption',
+                  dropdownDisplay: (
+                    <Fragment>
+                      <strong>{indexModeLabels[STANDARD_INDEX_MODE]}</strong>
+                      <EuiText size="s" color="subdued">
+                        <p>{indexModeDescriptions[STANDARD_INDEX_MODE]}</p>
+                      </EuiText>
+                    </Fragment>
+                  ),
+                },
+                {
+                  value: TIME_SERIES_MODE,
+                  inputDisplay: indexModeLabels[TIME_SERIES_MODE],
+                  'data-test-subj': 'indexModeTimeSeriesOption',
+                  dropdownDisplay: (
+                    <Fragment>
+                      <strong>{indexModeLabels[TIME_SERIES_MODE]}</strong>
+                      <EuiText size="s" color="subdued">
+                        <p>{indexModeDescriptions[TIME_SERIES_MODE]}</p>
+                      </EuiText>
+                    </Fragment>
+                  ),
+                },
+                {
+                  value: LOGSDB_INDEX_MODE,
+                  inputDisplay: indexModeLabels[LOGSDB_INDEX_MODE],
+                  'data-test-subj': 'indexModeLogsdbOption',
+                  dropdownDisplay: (
+                    <Fragment>
+                      <strong>{indexModeLabels[LOGSDB_INDEX_MODE]}</strong>
+                      <EuiText size="s" color="subdued">
+                        <p>{indexModeDescriptions[LOGSDB_INDEX_MODE]}</p>
+                      </EuiText>
+                    </Fragment>
+                  ),
+                },
+                {
+                  value: LOOKUP_INDEX_MODE,
+                  inputDisplay: indexModeLabels[LOOKUP_INDEX_MODE],
+                  'data-test-subj': 'indexModeLookupOption',
+                  dropdownDisplay: (
+                    <Fragment>
+                      <strong>{indexModeLabels[LOOKUP_INDEX_MODE]}</strong>
+                      <EuiText size="s" color="subdued">
+                        <p>{indexModeDescriptions[LOOKUP_INDEX_MODE]}</p>
+                      </EuiText>
+                    </Fragment>
+                  ),
+                },
+              ]}
+            />
+          </EuiFormRow>
         </EuiForm>
       </EuiModalBody>
       <EuiModalFooter>

x-pack/platform/plugins/shared/index_management/public/application/services/api.ts

@@ -436,12 +436,13 @@ export function useLoadIndexSettings(indexName: string) {
   });
 }
 
-export function createIndex(indexName: string) {
+export function createIndex(indexName: string, indexMode: string) {
   return sendRequest({
     path: `${INTERNAL_API_BASE_PATH}/indices/create`,
     method: 'put',
     body: JSON.stringify({
       indexName,
+      indexMode,
     }),
   });
 }