[8.6] Added Time-series indexing (TSDB) to Integrations Experimental …

…Indexing settings (#144974) (#146562) # Backport This will backport the following commits from `main` to `8.6`: - [Added Time-series indexing (TSDB) to Integrations Experimental Indexing settings (#144974)](#144974)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  Co-authored-by: Julia Bardi <90178898+juliaElastic@users.noreply.github.com>
elastic · Nov 29, 2022 · a835c33 · a835c33
1 parent bfad267
commit a835c33
Show file tree

Hide file tree

Showing 12 changed files with 414 additions and 103 deletions.
diff --git a/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts b/src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts
@@ -80,7 +80,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "endpoint:user-artifact": "f94c250a52b30d0a2d32635f8b4c5bdabd1e25c0",
         "endpoint:user-artifact-manifest": "8c14d49a385d5d1307d956aa743ec78de0b2be88",
         "enterprise_search_telemetry": "fafcc8318528d34f721c42d1270787c52565bad5",
-        "epm-packages": "cb22b422398a785e7e0565a19c6d4d5c7af6f2fd",
+        "epm-packages": "fe3716a54188b3c71327fa060dd6780a674d3994",
         "epm-packages-assets": "9fd3d6726ac77369249e9a973902c2cd615fc771",
         "event_loop_delays_daily": "d2ed39cf669577d90921c176499908b4943fb7bd",
         "exception-list": "fe8cc004fd2742177cdb9300f4a67689463faf9c",

diff --git a/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.test.ts b/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.test.ts
@@ -131,6 +131,7 @@ describe('toPackagePolicy', () => {
               data_stream: 'logs-nginx.access',
               features: {
                 synthetic_source: true,
+                tsdb: false,
               },
             },
           ],
@@ -142,6 +143,7 @@ describe('toPackagePolicy', () => {
           data_stream: 'logs-nginx.access',
           features: {
             synthetic_source: true,
+            tsdb: false,
           },
         },
       ]);

diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts
@@ -352,6 +352,7 @@ export interface RegistryElasticsearch {
   'index_template.mappings'?: estypes.MappingTypeMapping;
   'ingest_pipeline.name'?: string;
   source_mode?: 'default' | 'synthetic';
+  index_mode?: 'time_series';
 }
 
 export interface RegistryDataStreamPrivileges {
@@ -444,7 +445,7 @@ export type PackageInfo =
   | Installable<Merge<ArchivePackage, EpmPackageAdditions>>;
 
 // TODO - Expand this with other experimental indexing types
-export type ExperimentalIndexingFeature = 'synthetic_source';
+export type ExperimentalIndexingFeature = 'synthetic_source' | 'tsdb';
 
 export interface ExperimentalDataStreamFeature {
   data_stream: string;

diff --git a/x-pack/plugins/fleet/common/types/models/package_policy.ts b/x-pack/plugins/fleet/common/types/models/package_policy.ts
@@ -33,6 +33,7 @@ export interface NewPackagePolicyInputStream {
       privileges?: {
         indices?: string[];
       };
+      index_mode?: string;
     };
   };
   release?: RegistryRelease;

diff --git a/...icy/create_package_policy_page/components/steps/components/package_policy_input_panel.tsx b/...icy/create_package_policy_page/components/steps/components/package_policy_input_panel.tsx
@@ -19,6 +19,7 @@ import {
   EuiButtonEmpty,
 } from '@elastic/eui';
 
+import { getRegistryDataStreamAssetBaseName } from '../../../../../../../../../common/services';
 import type {
   NewPackagePolicy,
   NewPackagePolicyInput,
@@ -125,6 +126,41 @@ export const PackagePolicyInputPanel: React.FunctionComponent<{
       [packageInputStreams, packagePolicyInput.streams]
     );
 
+    // setting Indexing setting: TSDB to enabled by default, if the data stream's index_mode is set to time_series
+    let isUpdated = false;
+    inputStreams.forEach(({ packagePolicyInputStream }) => {
+      const dataStreamInfo = packageInfo.data_streams?.find(
+        (ds) => ds.dataset === packagePolicyInputStream?.data_stream.dataset
+      );
+
+      if (dataStreamInfo?.elasticsearch?.index_mode === 'time_series') {
+        if (!packagePolicy.package) return;
+        if (!packagePolicy.package?.experimental_data_stream_features)
+          packagePolicy.package!.experimental_data_stream_features = [];
+
+        const dsName = getRegistryDataStreamAssetBaseName(packagePolicyInputStream!.data_stream);
+        const match = packagePolicy.package!.experimental_data_stream_features.find(
+          (feat) => feat.data_stream === dsName
+        );
+        if (match) {
+          if (!match.features.tsdb) {
+            match.features.tsdb = true;
+            isUpdated = true;
+          }
+        } else {
+          packagePolicy.package!.experimental_data_stream_features.push({
+            data_stream: dsName,
+            features: { tsdb: true, synthetic_source: false },
+          });
+          isUpdated = true;
+        }
+      }
+    });
+
+    if (isUpdated) {
+      updatePackagePolicy(packagePolicy);
+    }
+
     return (
       <>
         {/* Header / input-level toggle */}

diff --git a/...cy/create_package_policy_page/components/steps/components/package_policy_input_stream.tsx b/...cy/create_package_policy_page/components/steps/components/package_policy_input_stream.tsx
@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import React, { useState, Fragment, memo, useMemo, useEffect, useRef } from 'react';
+import React, { useState, Fragment, memo, useMemo, useEffect, useRef, useCallback } from 'react';
 import ReactMarkdown from 'react-markdown';
 import styled from 'styled-components';
 import { FormattedMessage } from '@kbn/i18n-react';
@@ -18,13 +18,14 @@ import {
   EuiSpacer,
   EuiButtonEmpty,
   EuiTitle,
+  EuiToolTip,
 } from '@elastic/eui';
 import { useRouteMatch } from 'react-router-dom';
 
 import { mapPackageReleaseToIntegrationCardRelease } from '../../../../../../../../services/package_prerelease';
 
 import { getRegistryDataStreamAssetBaseName } from '../../../../../../../../../common/services';
-
+import type { ExperimentalIndexingFeature } from '../../../../../../../../../common/types/models/epm';
 import type {
   NewPackagePolicy,
   NewPackagePolicyInputStream,
@@ -120,6 +121,62 @@ export const PackagePolicyInputStreamConfig = memo<Props>(
       [advancedVars, inputStreamValidationResults?.vars]
     );
 
+    const isFeatureEnabled = useCallback(
+      (feature: ExperimentalIndexingFeature) =>
+        packagePolicy.package?.experimental_data_stream_features?.some(
+          ({ data_stream: dataStream, features }) =>
+            dataStream ===
+              getRegistryDataStreamAssetBaseName(packagePolicyInputStream.data_stream) &&
+            features[feature]
+        ) ?? false,
+      [
+        packagePolicy.package?.experimental_data_stream_features,
+        packagePolicyInputStream.data_stream,
+      ]
+    );
+
+    const newExperimentalIndexingFeature = {
+      synthetic_source: isFeatureEnabled('synthetic_source'),
+      tsdb: isFeatureEnabled('tsdb'),
+    };
+
+    const onIndexingSettingChange = (
+      features: Partial<Record<ExperimentalIndexingFeature, boolean>>
+    ) => {
+      if (!packagePolicy.package) {
+        return;
+      }
+
+      const newExperimentalDataStreamFeatures = [
+        ...(packagePolicy.package.experimental_data_stream_features ?? []),
+      ];
+
+      const dataStream = getRegistryDataStreamAssetBaseName(packagePolicyInputStream.data_stream);
+
+      const existingSettingRecord = newExperimentalDataStreamFeatures.find(
+        (x) => x.data_stream === dataStream
+      );
+
+      if (existingSettingRecord) {
+        existingSettingRecord.features = {
+          ...existingSettingRecord.features,
+          ...features,
+        };
+      } else {
+        newExperimentalDataStreamFeatures.push({
+          data_stream: dataStream,
+          features: { ...newExperimentalIndexingFeature, ...features },
+        });
+      }
+
+      updatePackagePolicy({
+        package: {
+          ...packagePolicy.package,
+          experimental_data_stream_features: newExperimentalDataStreamFeatures,
+        },
+      });
+    };
+
     return (
       <>
         <EuiFlexGrid columns={2} id={isDefaultDatstream ? 'test123' : 'asas'}>
@@ -311,59 +368,46 @@ export const PackagePolicyInputStreamConfig = memo<Props>(
                         <EuiSpacer size="s" />
                         <EuiFlexItem>
                           <EuiSwitch
-                            checked={
-                              packagePolicy.package?.experimental_data_stream_features?.some(
-                                ({ data_stream: dataStream, features }) =>
-                                  dataStream ===
-                                    getRegistryDataStreamAssetBaseName(
-                                      packagePolicyInputStream.data_stream
-                                    ) && features.synthetic_source
-                              ) ?? false
-                            }
+                            checked={isFeatureEnabled('synthetic_source')}
                             label={
                               <FormattedMessage
                                 id="xpack.fleet.createPackagePolicy.experimentalFeatures.syntheticSourceLabel"
                                 defaultMessage="Synthetic source"
                               />
                             }
                             onChange={(e) => {
-                              if (!packagePolicy.package) {
-                                return;
-                              }
-
-                              const newExperimentalDataStreamFeatures = [
-                                ...(packagePolicy.package.experimental_data_stream_features ?? []),
-                              ];
-
-                              const dataStream = getRegistryDataStreamAssetBaseName(
-                                packagePolicyInputStream.data_stream
-                              );
-
-                              const existingSettingRecord = newExperimentalDataStreamFeatures.find(
-                                (x) => x.data_stream === dataStream
-                              );
-
-                              if (existingSettingRecord) {
-                                existingSettingRecord.features.synthetic_source = e.target.checked;
-                              } else {
-                                newExperimentalDataStreamFeatures.push({
-                                  data_stream: dataStream,
-                                  features: {
-                                    synthetic_source: e.target.checked,
-                                  },
-                                });
-                              }
-
-                              updatePackagePolicy({
-                                package: {
-                                  ...packagePolicy.package,
-                                  experimental_data_stream_features:
-                                    newExperimentalDataStreamFeatures,
-                                },
+                              onIndexingSettingChange({
+                                synthetic_source: e.target.checked,
                               });
                             }}
                           />
                         </EuiFlexItem>
+                        <EuiFlexItem>
+                          <EuiToolTip
+                            content={
+                              <FormattedMessage
+                                id="xpack.fleet.createPackagePolicy.experimentalFeatures.TSDBTooltip"
+                                defaultMessage="Enabling this feature is irreversible"
+                              />
+                            }
+                          >
+                            <EuiSwitch
+                              disabled={isFeatureEnabled('tsdb')}
+                              checked={isFeatureEnabled('tsdb')}
+                              label={
+                                <FormattedMessage
+                                  id="xpack.fleet.createPackagePolicy.experimentalFeatures.TSDBLabel"
+                                  defaultMessage="Time-series indexing (TSDB)"
+                                />
+                              }
+                              onChange={(e) => {
+                                onIndexingSettingChange({
+                                  tsdb: e.target.checked,
+                                });
+                              }}
+                            />
+                          </EuiToolTip>
+                        </EuiFlexItem>
                       </EuiFlexGroup>
                     </EuiFlexItem>
                   </>

diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts
@@ -289,6 +289,7 @@ const getSavedObjectTypes = (
               type: 'nested',
               properties: {
                 synthetic_source: { type: 'boolean' },
+                tsdb: { type: 'boolean' },
               },
             },
           },

diff --git a/x-pack/plugins/fleet/server/services/epm/archive/parse.test.ts b/x-pack/plugins/fleet/server/services/epm/archive/parse.test.ts
@@ -67,6 +67,11 @@ describe('parseDataStreamElasticsearchEntry', () => {
       source_mode: 'synthetic',
     });
   });
+  it('Should add index_mode', () => {
+    expect(parseDataStreamElasticsearchEntry({ index_mode: 'time_series' })).toEqual({
+      index_mode: 'time_series',
+    });
+  });
   it('Should add index_template mappings and expand dots', () => {
     expect(
       parseDataStreamElasticsearchEntry({

diff --git a/x-pack/plugins/fleet/server/services/epm/archive/parse.ts b/x-pack/plugins/fleet/server/services/epm/archive/parse.ts
@@ -539,6 +539,10 @@ export function parseDataStreamElasticsearchEntry(
     );
   }
 
+  if (expandedElasticsearch?.index_mode) {
+    parsedElasticsearchEntry.index_mode = expandedElasticsearch.index_mode;
+  }
+
   return parsedElasticsearchEntry;
 }
-Original file line number
+Diff line change
@@ Expand Up / @@ -539,6 +539,10 @@ export function parseDataStreamElasticsearchEntry( @@
         );
       }
+      if (expandedElasticsearch?.index_mode) {
+        parsedElasticsearchEntry.index_mode = expandedElasticsearch.index_mode;
+      }
       return parsedElasticsearchEntry;
     }
@@ Expand Down @@