[Security Solution] Integration tests for data_source field diff al…

…gorithm (#189744)

## Summary

Completes #187659


Switches `data_source` fields to use the implemented diff algorithm
assigned to them in #188874


Adds integration tests in accordance to
#189669 for the `upgrade/_review`
API endpoint for the `data_source` field diff algorithm.

### Checklist

Delete any items that are not applicable to this PR.

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [x] [Flaky Test
Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was
used on any tests changed


### For maintainers

- [ ] This was checked for breaking API changes and was [labeled
appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)

x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules/model/diff/three_way_diff/three_way_diff_outcome.ts

@@ -83,9 +83,9 @@ export const determineOrderAgnosticDiffOutcome = <TValue>(
  * NOTE: uses order agnostic comparison for nested array fields (e.g. `index`)
  */
 export const determineDiffOutcomeForDataSource = (
-  baseVersion: RuleDataSource | MissingVersion,
-  currentVersion: RuleDataSource,
-  targetVersion: RuleDataSource
+  baseVersion: RuleDataSource | undefined | MissingVersion,
+  currentVersion: RuleDataSource | undefined,
+  targetVersion: RuleDataSource | undefined
 ): ThreeWayDiffOutcome => {
   const isBaseVersionMissing = baseVersion === MissingVersion;
 
@@ -151,7 +151,7 @@ export const determineIfValueCanUpdate = (diffCase: ThreeWayDiffOutcome): boolea
   );
 };
 
-const isIndexPatternDataSourceType = (
-  version: RuleDataSource
+export const isIndexPatternDataSourceType = (
+  version: RuleDataSource | undefined
 ): version is Extract<RuleDataSource, { type: DataSourceType.index_patterns }> =>
-  version.type === DataSourceType.index_patterns;
+  version !== undefined && version.type === DataSourceType.index_patterns;

x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/data_source_diff_algorithm.test.ts

@@ -21,7 +21,7 @@ import { dataSourceDiffAlgorithm } from './data_source_diff_algorithm';
 describe('dataSourceDiffAlgorithm', () => {
   describe('returns current_version as merged output if there is no update - scenario AAA', () => {
     it('if all versions are index patterns', () => {
-      const mockVersions: ThreeVersionsOf<RuleDataSource> = {
+      const mockVersions: ThreeVersionsOf<RuleDataSource | undefined> = {
         base_version: {
           type: DataSourceType.index_patterns,
           index_patterns: ['one', 'two', 'two', 'three'],
@@ -49,7 +49,7 @@ describe('dataSourceDiffAlgorithm', () => {
     });
 
     it('if all versions are data views', () => {
-      const mockVersions: ThreeVersionsOf<RuleDataSource> = {
+      const mockVersions: ThreeVersionsOf<RuleDataSource | undefined> = {
         base_version: { type: DataSourceType.data_view, data_view_id: '123' },
         current_version: { type: DataSourceType.data_view, data_view_id: '123' },
         target_version: { type: DataSourceType.data_view, data_view_id: '123' },
@@ -70,7 +70,7 @@ describe('dataSourceDiffAlgorithm', () => {
 
   describe('returns current_version as merged output if current_version is different and there is no update - scenario ABA', () => {
     it('if current version is different data type than base and target', () => {
-      const mockVersions: ThreeVersionsOf<RuleDataSource> = {
+      const mockVersions: ThreeVersionsOf<RuleDataSource | undefined> = {
         base_version: {
           type: DataSourceType.index_patterns,
           index_patterns: ['one', 'two', 'three'],
@@ -95,7 +95,7 @@ describe('dataSourceDiffAlgorithm', () => {
     });
 
     it('if all versions are same data type', () => {
-      const mockVersions: ThreeVersionsOf<RuleDataSource> = {
+      const mockVersions: ThreeVersionsOf<RuleDataSource | undefined> = {
         base_version: {
           type: DataSourceType.index_patterns,
           index_patterns: ['one', 'two', 'three'],
@@ -121,11 +121,36 @@ describe('dataSourceDiffAlgorithm', () => {
         })
       );
     });
+
+    it('if current version is undefined', () => {
+      const mockVersions: ThreeVersionsOf<RuleDataSource | undefined> = {
+        base_version: {
+          type: DataSourceType.index_patterns,
+          index_patterns: ['one', 'two', 'three'],
+        },
+        current_version: undefined,
+        target_version: {
+          type: DataSourceType.index_patterns,
+          index_patterns: ['one', 'two', 'three'],
+        },
+      };
+
+      const result = dataSourceDiffAlgorithm(mockVersions);
+
+      expect(result).toEqual(
+        expect.objectContaining({
+          merged_version: mockVersions.current_version,
+          diff_outcome: ThreeWayDiffOutcome.CustomizedValueNoUpdate,
+          merge_outcome: ThreeWayMergeOutcome.Current,
+          conflict: ThreeWayDiffConflict.NONE,
+        })
+      );
+    });
   });
 
   describe('returns target_version as merged output if current_version is the same and there is an update - scenario AAB', () => {
     it('if target version is different data type than base and current', () => {
-      const mockVersions: ThreeVersionsOf<RuleDataSource> = {
+      const mockVersions: ThreeVersionsOf<RuleDataSource | undefined> = {
         base_version: { type: DataSourceType.data_view, data_view_id: '123' },
         current_version: { type: DataSourceType.data_view, data_view_id: '123' },
         target_version: {
@@ -147,7 +172,7 @@ describe('dataSourceDiffAlgorithm', () => {
     });
 
     it('if all versions are same data type', () => {
-      const mockVersions: ThreeVersionsOf<RuleDataSource> = {
+      const mockVersions: ThreeVersionsOf<RuleDataSource | undefined> = {
         base_version: { type: DataSourceType.data_view, data_view_id: '123' },
         current_version: { type: DataSourceType.data_view, data_view_id: '123' },
         target_version: { type: DataSourceType.data_view, data_view_id: '456' },
@@ -168,7 +193,7 @@ describe('dataSourceDiffAlgorithm', () => {
 
   describe('returns current_version as merged output if current version is different but it matches the update - scenario ABB', () => {
     it('if all versions are index patterns', () => {
-      const mockVersions: ThreeVersionsOf<RuleDataSource> = {
+      const mockVersions: ThreeVersionsOf<RuleDataSource | undefined> = {
         base_version: {
           type: DataSourceType.index_patterns,
           index_patterns: ['one', 'two', 'three'],
@@ -196,7 +221,7 @@ describe('dataSourceDiffAlgorithm', () => {
     });
 
     it('if all versions are data views', () => {
-      const mockVersions: ThreeVersionsOf<RuleDataSource> = {
+      const mockVersions: ThreeVersionsOf<RuleDataSource | undefined> = {
         base_version: { type: DataSourceType.data_view, data_view_id: '123' },
         current_version: { type: DataSourceType.data_view, data_view_id: '456' },
         target_version: { type: DataSourceType.data_view, data_view_id: '456' },
@@ -217,7 +242,7 @@ describe('dataSourceDiffAlgorithm', () => {
 
   describe('returns current_version as merged output if all three versions are different - scenario ABC', () => {
     it('if all versions are index patterns', () => {
-      const mockVersions: ThreeVersionsOf<RuleDataSource> = {
+      const mockVersions: ThreeVersionsOf<RuleDataSource | undefined> = {
         base_version: {
           type: DataSourceType.index_patterns,
           index_patterns: ['one', 'two', 'three'],
@@ -250,7 +275,7 @@ describe('dataSourceDiffAlgorithm', () => {
     });
 
     it('if all versions are data views', () => {
-      const mockVersions: ThreeVersionsOf<RuleDataSource> = {
+      const mockVersions: ThreeVersionsOf<RuleDataSource | undefined> = {
         base_version: { type: DataSourceType.data_view, data_view_id: '123' },
         current_version: { type: DataSourceType.data_view, data_view_id: '456' },
         target_version: { type: DataSourceType.data_view, data_view_id: '789' },
@@ -269,7 +294,7 @@ describe('dataSourceDiffAlgorithm', () => {
     });
 
     it('if base version is a data view and others are index patterns ', () => {
-      const mockVersions: ThreeVersionsOf<RuleDataSource> = {
+      const mockVersions: ThreeVersionsOf<RuleDataSource | undefined> = {
         base_version: { type: DataSourceType.data_view, data_view_id: '123' },
         current_version: {
           type: DataSourceType.index_patterns,
@@ -299,7 +324,7 @@ describe('dataSourceDiffAlgorithm', () => {
     });
 
     it('if base version is a index patterns and other are data views', () => {
-      const mockVersions: ThreeVersionsOf<RuleDataSource> = {
+      const mockVersions: ThreeVersionsOf<RuleDataSource | undefined> = {
         base_version: {
           type: DataSourceType.index_patterns,
           index_patterns: ['one', 'three', 'four'],
@@ -326,7 +351,7 @@ describe('dataSourceDiffAlgorithm', () => {
     });
 
     it('if currrent version is a different data type', () => {
-      const mockVersions: ThreeVersionsOf<RuleDataSource> = {
+      const mockVersions: ThreeVersionsOf<RuleDataSource | undefined> = {
         base_version: { type: DataSourceType.data_view, data_view_id: '123' },
         current_version: {
           type: DataSourceType.index_patterns,
@@ -348,7 +373,7 @@ describe('dataSourceDiffAlgorithm', () => {
     });
 
     it('if target version is a different data type', () => {
-      const mockVersions: ThreeVersionsOf<RuleDataSource> = {
+      const mockVersions: ThreeVersionsOf<RuleDataSource | undefined> = {
         base_version: {
           type: DataSourceType.index_patterns,
           index_patterns: ['one', 'two', 'three'],
@@ -371,16 +396,11 @@ describe('dataSourceDiffAlgorithm', () => {
         })
       );
     });
-  });
 
-  describe('if base_version is missing', () => {
-    it('returns current_version as merged output if current_version and target_version are the same - scenario -AA', () => {
-      const mockVersions: ThreeVersionsOf<RuleDataSource> = {
-        base_version: MissingVersion,
-        current_version: {
-          type: DataSourceType.index_patterns,
-          index_patterns: ['one', 'three', 'four'],
-        },
+    it('if currrent version is undefined', () => {
+      const mockVersions: ThreeVersionsOf<RuleDataSource | undefined> = {
+        base_version: { type: DataSourceType.data_view, data_view_id: '123' },
+        current_version: undefined,
         target_version: {
           type: DataSourceType.index_patterns,
           index_patterns: ['one', 'three', 'four'],
@@ -391,20 +411,23 @@ describe('dataSourceDiffAlgorithm', () => {
 
       expect(result).toEqual(
         expect.objectContaining({
-          has_base_version: false,
-          base_version: undefined,
           merged_version: mockVersions.current_version,
-          diff_outcome: ThreeWayDiffOutcome.MissingBaseNoUpdate,
+          diff_outcome: ThreeWayDiffOutcome.CustomizedValueCanUpdate,
           merge_outcome: ThreeWayMergeOutcome.Current,
-          conflict: ThreeWayDiffConflict.NONE,
+          conflict: ThreeWayDiffConflict.NON_SOLVABLE,
         })
       );
     });
+  });
 
-    it('returns target_version as merged output if current_version and target_version are different - scenario -AB', () => {
-      const mockVersions: ThreeVersionsOf<RuleDataSource> = {
+  describe('if base_version is missing', () => {
+    it('returns current_version as merged output if current_version and target_version are the same - scenario -AA', () => {
+      const mockVersions: ThreeVersionsOf<RuleDataSource | undefined> = {
         base_version: MissingVersion,
-        current_version: { type: DataSourceType.data_view, data_view_id: '456' },
+        current_version: {
+          type: DataSourceType.index_patterns,
+          index_patterns: ['one', 'three', 'four'],
+        },
         target_version: {
           type: DataSourceType.index_patterns,
           index_patterns: ['one', 'three', 'four'],
@@ -417,12 +440,62 @@ describe('dataSourceDiffAlgorithm', () => {
         expect.objectContaining({
           has_base_version: false,
           base_version: undefined,
-          merged_version: mockVersions.target_version,
-          diff_outcome: ThreeWayDiffOutcome.MissingBaseCanUpdate,
-          merge_outcome: ThreeWayMergeOutcome.Target,
-          conflict: ThreeWayDiffConflict.SOLVABLE,
+          merged_version: mockVersions.current_version,
+          diff_outcome: ThreeWayDiffOutcome.MissingBaseNoUpdate,
+          merge_outcome: ThreeWayMergeOutcome.Current,
+          conflict: ThreeWayDiffConflict.NONE,
         })
       );
     });
+
+    describe('returns target_version as merged output if current_version and target_version are different - scenario -AB', () => {
+      it('if versions are different types', () => {
+        const mockVersions: ThreeVersionsOf<RuleDataSource | undefined> = {
+          base_version: MissingVersion,
+          current_version: { type: DataSourceType.data_view, data_view_id: '456' },
+          target_version: {
+            type: DataSourceType.index_patterns,
+            index_patterns: ['one', 'three', 'four'],
+          },
+        };
+
+        const result = dataSourceDiffAlgorithm(mockVersions);
+
+        expect(result).toEqual(
+          expect.objectContaining({
+            has_base_version: false,
+            base_version: undefined,
+            merged_version: mockVersions.target_version,
+            diff_outcome: ThreeWayDiffOutcome.MissingBaseCanUpdate,
+            merge_outcome: ThreeWayMergeOutcome.Target,
+            conflict: ThreeWayDiffConflict.SOLVABLE,
+          })
+        );
+      });
+
+      it('if current version is undefined', () => {
+        const mockVersions: ThreeVersionsOf<RuleDataSource | undefined> = {
+          base_version: MissingVersion,
+          current_version: undefined,
+          target_version: {
+            type: DataSourceType.index_patterns,
+            index_patterns: ['one', 'three', 'four'],
+          },
+        };
+
+        const result = dataSourceDiffAlgorithm(mockVersions);
+
+        expect(result).toEqual(
+          expect.objectContaining({
+            has_base_version: false,
+            base_version: undefined,
+            merged_version: mockVersions.target_version,
+            diff_outcome: ThreeWayDiffOutcome.MissingBaseCanUpdate,
+            merge_outcome: ThreeWayMergeOutcome.Target,
+            conflict: ThreeWayDiffConflict.SOLVABLE,
+          })
+        );
+      });
+    });
   });
 });

x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/diff/calculation/algorithms/data_source_diff_algorithm.ts

@@ -19,12 +19,16 @@ import {
   DataSourceType,
   ThreeWayDiffConflict,
   determineDiffOutcomeForDataSource,
+  isIndexPatternDataSourceType,
 } from '../../../../../../../../common/api/detection_engine/prebuilt_rules';
 import { getDedupedDataSourceVersion, mergeDedupedArrays } from './helpers';
 
+/**
+ * Takes a type of `RuleDataSource | undefined` because the data source can be index patterns, a data view id, or neither in some cases
+ */
 export const dataSourceDiffAlgorithm = (
-  versions: ThreeVersionsOf<RuleDataSource>
-): ThreeWayDiff<RuleDataSource> => {
+  versions: ThreeVersionsOf<RuleDataSource | undefined>
+): ThreeWayDiff<RuleDataSource | undefined> => {
   const {
     base_version: baseVersion,
     current_version: currentVersion,
@@ -60,14 +64,14 @@ export const dataSourceDiffAlgorithm = (
 
 interface MergeResult {
   mergeOutcome: ThreeWayMergeOutcome;
-  mergedVersion: RuleDataSource;
+  mergedVersion: RuleDataSource | undefined;
   conflict: ThreeWayDiffConflict;
 }
 
 interface MergeArgs {
   baseVersion: RuleDataSource | undefined;
-  currentVersion: RuleDataSource;
-  targetVersion: RuleDataSource;
+  currentVersion: RuleDataSource | undefined;
+  targetVersion: RuleDataSource | undefined;
   diffOutcome: ThreeWayDiffOutcome;
 }
 
@@ -78,8 +82,12 @@ const mergeVersions = ({
   diffOutcome,
 }: MergeArgs): MergeResult => {
   const dedupedBaseVersion = baseVersion ? getDedupedDataSourceVersion(baseVersion) : baseVersion;
-  const dedupedCurrentVersion = getDedupedDataSourceVersion(currentVersion);
-  const dedupedTargetVersion = getDedupedDataSourceVersion(targetVersion);
+  const dedupedCurrentVersion = currentVersion
+    ? getDedupedDataSourceVersion(currentVersion)
+    : currentVersion;
+  const dedupedTargetVersion = targetVersion
+    ? getDedupedDataSourceVersion(targetVersion)
+    : targetVersion;
 
   switch (diffOutcome) {
     // Scenario -AA is treated as scenario AAA:
@@ -103,8 +111,8 @@ const mergeVersions = ({
 
     case ThreeWayDiffOutcome.CustomizedValueCanUpdate: {
       if (
-        dedupedCurrentVersion.type === DataSourceType.index_patterns &&
-        dedupedTargetVersion.type === DataSourceType.index_patterns
+        isIndexPatternDataSourceType(dedupedCurrentVersion) &&
+        isIndexPatternDataSourceType(dedupedTargetVersion)
       ) {
         const baseVersionToMerge =
           dedupedBaseVersion && dedupedBaseVersion.type === DataSourceType.index_patterns