[Security Solution] adds WrapSequences method (RAC)

x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts

@@ -70,6 +70,8 @@ describe('eql_executor', () => {
         logger,
         searchAfterSize,
         bulkCreate: jest.fn(),
+        wrapHits: jest.fn(),
+        wrapSequences: jest.fn(),
       });
       expect(response.warningMessages.length).toEqual(1);
     });

x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts

@@ -21,17 +21,18 @@ import { isOutdated } from '../../migrations/helpers';
 import { getIndexVersion } from '../../routes/index/get_index_version';
 import { MIN_EQL_RULE_INDEX_VERSION } from '../../routes/index/get_signals_template';
 import { EqlRuleParams } from '../../schemas/rule_schemas';
-import { buildSignalFromEvent, buildSignalGroupFromSequence } from '../build_bulk_body';
 import { getInputIndex } from '../get_input_output_index';
-import { filterDuplicateSignals } from '../filter_duplicate_signals';
+
 import {
   AlertAttributes,
   BulkCreate,
+  WrapHits,
+  WrapSequences,
   EqlSignalSearchResponse,
   SearchAfterAndBulkCreateReturnType,
-  WrappedSignalHit,
+  SimpleHit,
 } from '../types';
-import { createSearchAfterReturnType, makeFloatString, wrapSignal } from '../utils';
+import { createSearchAfterReturnType, makeFloatString } from '../utils';
 
 export const eqlExecutor = async ({
   rule,
@@ -41,6 +42,8 @@ export const eqlExecutor = async ({
   logger,
   searchAfterSize,
   bulkCreate,
+  wrapHits,
+  wrapSequences,
 }: {
   rule: SavedObject<AlertAttributes<EqlRuleParams>>;
   exceptionItems: ExceptionListItemSchema[];
@@ -49,6 +52,8 @@ export const eqlExecutor = async ({
   logger: Logger;
   searchAfterSize: number;
   bulkCreate: BulkCreate;
+  wrapHits: WrapHits;
+  wrapSequences: WrapSequences;
 }): Promise<SearchAfterAndBulkCreateReturnType> => {
   const result = createSearchAfterReturnType();
   const ruleParams = rule.attributes.params;
@@ -101,27 +106,18 @@ export const eqlExecutor = async ({
   const eqlSignalSearchEnd = performance.now();
   const eqlSearchDuration = makeFloatString(eqlSignalSearchEnd - eqlSignalSearchStart);
   result.searchAfterTimes = [eqlSearchDuration];
-  let newSignals: WrappedSignalHit[] | undefined;
+  let newSignals: SimpleHit[] | undefined;
   if (response.hits.sequences !== undefined) {
-    newSignals = response.hits.sequences.reduce(
-      (acc: WrappedSignalHit[], sequence) =>
-        acc.concat(buildSignalGroupFromSequence(sequence, rule, ruleParams.outputIndex)),
-      []
-    );
+    newSignals = wrapSequences(response.hits.sequences);
   } else if (response.hits.events !== undefined) {
-    newSignals = filterDuplicateSignals(
-      rule.id,
-      response.hits.events.map((event) =>
-        wrapSignal(buildSignalFromEvent(event, rule, true), ruleParams.outputIndex)
-      )
-    );
+    newSignals = wrapHits(response.hits.events);
   } else {
     throw new Error(
       'eql query response should have either `sequences` or `events` but had neither'
     );
   }
 
-  if (newSignals.length > 0) {
+  if (newSignals?.length) {
     const insertResult = await bulkCreate(newSignals);
     result.bulkCreateTimes.push(insertResult.bulkCreateDuration);
     result.createdSignalsCount += insertResult.createdItemsCount;

x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_duplicate_signals.test.ts

@@ -36,11 +36,13 @@ const mockSignals = [
 ];
 
 describe('filterDuplicateSignals', () => {
-  it('filters duplicate signals', () => {
-    expect(filterDuplicateSignals(mockRuleId1, mockSignals).length).toEqual(1);
-  });
+  describe('detection engine implementation', () => {
+    it('filters duplicate signals', () => {
+      expect(filterDuplicateSignals(mockRuleId1, mockSignals, false).length).toEqual(1);
+    });
 
-  it('does not filter non-duplicate signals', () => {
-    expect(filterDuplicateSignals(mockRuleId3, mockSignals).length).toEqual(2);
+    it('does not filter non-duplicate signals', () => {
+      expect(filterDuplicateSignals(mockRuleId3, mockSignals, false).length).toEqual(2);
+    });
   });
 });

x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_duplicate_signals.ts

@@ -5,10 +5,26 @@
  * 2.0.
  */
 
-import { WrappedSignalHit } from './types';
+import { SimpleHit, WrappedSignalHit } from './types';
 
-export const filterDuplicateSignals = (ruleId: string, signals: WrappedSignalHit[]) => {
-  return signals.filter(
-    (doc) => !doc._source.signal?.ancestors.some((ancestor) => ancestor.rule === ruleId)
-  );
+const isWrappedSignalHit = (
+  signals: SimpleHit[],
+  isRuleRegistryEnabled: boolean
+): signals is WrappedSignalHit[] => {
+  return !isRuleRegistryEnabled;
+};
+
+export const filterDuplicateSignals = (
+  ruleId: string,
+  signals: SimpleHit[],
+  isRuleRegistryEnabled: boolean
+) => {
+  if (isWrappedSignalHit(signals, isRuleRegistryEnabled)) {
+    return signals.filter(
+      (doc) => !doc._source.signal?.ancestors.some((ancestor) => ancestor.rule === ruleId)
+    );
+  } else {
+    // TODO: filter duplicate signals for RAC
+    return [];
+  }
 };

x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts

@@ -66,7 +66,10 @@ describe('searchAfterAndBulkCreate', () => {
       buildRuleMessage,
       false
     );
-    wrapHits = wrapHitsFactory({ ruleSO, signalsIndex: DEFAULT_SIGNALS_INDEX });
+    wrapHits = wrapHitsFactory({
+      ruleSO,
+      signalsIndex: DEFAULT_SIGNALS_INDEX,
+    });
   });
 
   test('should return success with number of searches less than max signals', async () => {

x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts

@@ -67,6 +67,7 @@ import {
 } from '../schemas/rule_schemas';
 import { bulkCreateFactory } from './bulk_create_factory';
 import { wrapHitsFactory } from './wrap_hits_factory';
+import { wrapSequencesFactory } from './wrap_sequences_factory';
 
 export const signalRulesAlertType = ({
   logger,
@@ -233,6 +234,11 @@ export const signalRulesAlertType = ({
           signalsIndex: params.outputIndex,
         });
 
+        const wrapSequences = wrapSequencesFactory({
+          ruleSO: savedObject,
+          signalsIndex: params.outputIndex,
+        });
+
         if (isMlRule(type)) {
           const mlRuleSO = asTypeSpecificSO(savedObject, machineLearningRuleParams);
           result = await mlExecutor({
@@ -302,6 +308,8 @@ export const signalRulesAlertType = ({
             searchAfterSize,
             bulkCreate,
             logger,
+            wrapHits,
+            wrapSequences,
           });
         } else {
           throw new Error(`unknown rule type ${type}`);

x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts

@@ -25,6 +25,7 @@ import {
   BaseHit,
   RuleAlertAction,
   SearchTypes,
+  EqlSequence,
 } from '../../../../common/detection_engine/types';
 import { ListClient } from '../../../../../lists/server';
 import { Logger, SavedObject } from '../../../../../../../src/core/server';
@@ -257,9 +258,11 @@ export type SignalsEnrichment = (signals: SignalSearchResponse) => Promise<Signa
 
 export type BulkCreate = <T>(docs: Array<BaseHit<T>>) => Promise<GenericBulkCreateResponse<T>>;
 
-export type WrapHits = (
-  hits: Array<estypes.SearchHit<unknown>>
-) => Array<BaseHit<{ '@timestamp': string }>>;
+export type SimpleHit = BaseHit<{ '@timestamp': string }>;
+
+export type WrapHits = (hits: Array<estypes.SearchHit<unknown>>) => SimpleHit[];
+
+export type WrapSequences = (sequences: Array<EqlSequence<SignalSource>>) => SimpleHit[];
 
 export interface SearchAfterAndBulkCreateParams {
   tuples: Array<{

x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_hits_factory.ts

@@ -25,11 +25,15 @@ export const wrapHitsFactory = ({
   const wrappedDocs: WrappedSignalHit[] = events.flatMap((doc) => [
     {
       _index: signalsIndex,
-      // TODO: bring back doc._version
-      _id: generateId(doc._index, doc._id, '', ruleSO.attributes.params.ruleId ?? ''),
+      _id: generateId(
+        doc._index,
+        doc._id,
+        doc._version ? doc._version.toString() : '',
+        ruleSO.attributes.params.ruleId ?? ''
+      ),
       _source: buildBulkBody(ruleSO, doc as SignalSourceHit),
     },
   ]);
 
-  return filterDuplicateSignals(ruleSO.id, wrappedDocs);
+  return filterDuplicateSignals(ruleSO.id, wrappedDocs, false);
 };

x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_sequences_factory.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { SearchAfterAndBulkCreateParams, WrappedSignalHit, WrapSequences } from './types';
+import { buildSignalGroupFromSequence } from './build_bulk_body';
+
+export const wrapSequencesFactory = ({
+  ruleSO,
+  signalsIndex,
+}: {
+  ruleSO: SearchAfterAndBulkCreateParams['ruleSO'];
+  signalsIndex: string;
+}): WrapSequences => (sequences) => {
+  const wrappedDocs = sequences.reduce(
+    (acc: WrappedSignalHit[], sequence) =>
+      acc.concat(buildSignalGroupFromSequence(sequence, ruleSO, signalsIndex)),
+    []
+  );
+
+  return wrappedDocs;
+};

x-pack/plugins/security_solution/server/plugin.ts

@@ -208,8 +208,10 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
     });
 
     // TODO: Once we are past experimental phase this check can be removed along with legacy registration of rules
+    const isRuleRegistryEnabled = experimentalFeatures.ruleRegistryEnabled;
+
     let ruleDataClient: RuleDataClient | null = null;
-    if (experimentalFeatures.ruleRegistryEnabled) {
+    if (isRuleRegistryEnabled) {
       const { ruleDataService } = plugins.ruleRegistry;
       const start = () => core.getStartServices().then(([coreStart]) => coreStart);
 
@@ -293,7 +295,7 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
     const ruleTypes = [
       SIGNALS_ID,
       NOTIFICATIONS_ID,
-      ...(experimentalFeatures.ruleRegistryEnabled ? referenceRuleTypes : []),
+      ...(isRuleRegistryEnabled ? referenceRuleTypes : []),
     ];
 
     plugins.features.registerKibanaFeature({