[Security Solution] adds WrapSequences method (RAC)

x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts

@@ -78,6 +78,8 @@ describe('eql_executor', () => {
         logger,
         searchAfterSize,
         bulkCreate: jest.fn(),
+        wrapHits: jest.fn(),
+        wrapSequences: jest.fn(),
       });
       expect(response.warningMessages.length).toEqual(1);
     });

x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts

@@ -21,18 +21,19 @@ import { isOutdated } from '../../migrations/helpers';
 import { getIndexVersion } from '../../routes/index/get_index_version';
 import { MIN_EQL_RULE_INDEX_VERSION } from '../../routes/index/get_signals_template';
 import { EqlRuleParams } from '../../schemas/rule_schemas';
-import { buildSignalFromEvent, buildSignalGroupFromSequence } from '../build_bulk_body';
 import { getInputIndex } from '../get_input_output_index';
-import { filterDuplicateSignals } from '../filter_duplicate_signals';
+
 import {
   AlertAttributes,
   BulkCreate,
+  WrapHits,
+  WrapSequences,
   EqlSignalSearchResponse,
   RuleRangeTuple,
   SearchAfterAndBulkCreateReturnType,
-  WrappedSignalHit,
+  SimpleHit,
 } from '../types';
-import { createSearchAfterReturnType, makeFloatString, wrapSignal } from '../utils';
+import { createSearchAfterReturnType, makeFloatString } from '../utils';
 
 export const eqlExecutor = async ({
   rule,
@@ -43,6 +44,8 @@ export const eqlExecutor = async ({
   logger,
   searchAfterSize,
   bulkCreate,
+  wrapHits,
+  wrapSequences,
 }: {
   rule: SavedObject<AlertAttributes<EqlRuleParams>>;
   tuple: RuleRangeTuple;
@@ -52,6 +55,8 @@ export const eqlExecutor = async ({
   logger: Logger;
   searchAfterSize: number;
   bulkCreate: BulkCreate;
+  wrapHits: WrapHits;
+  wrapSequences: WrapSequences;
 }): Promise<SearchAfterAndBulkCreateReturnType> => {
   const result = createSearchAfterReturnType();
   const ruleParams = rule.attributes.params;
@@ -104,27 +109,18 @@ export const eqlExecutor = async ({
   const eqlSignalSearchEnd = performance.now();
   const eqlSearchDuration = makeFloatString(eqlSignalSearchEnd - eqlSignalSearchStart);
   result.searchAfterTimes = [eqlSearchDuration];
-  let newSignals: WrappedSignalHit[] | undefined;
+  let newSignals: SimpleHit[] | undefined;
   if (response.hits.sequences !== undefined) {
-    newSignals = response.hits.sequences.reduce(
-      (acc: WrappedSignalHit[], sequence) =>
-        acc.concat(buildSignalGroupFromSequence(sequence, rule, ruleParams.outputIndex)),
-      []
-    );
+    newSignals = wrapSequences(response.hits.sequences);
   } else if (response.hits.events !== undefined) {
-    newSignals = filterDuplicateSignals(
-      rule.id,
-      response.hits.events.map((event) =>
-        wrapSignal(buildSignalFromEvent(event, rule, true), ruleParams.outputIndex)
-      )
-    );
+    newSignals = wrapHits(response.hits.events);
   } else {
     throw new Error(
       'eql query response should have either `sequences` or `events` but had neither'
     );
   }
 
-  if (newSignals.length > 0) {
+  if (newSignals?.length) {
     const insertResult = await bulkCreate(newSignals);
     result.bulkCreateTimes.push(insertResult.bulkCreateDuration);
     result.createdSignalsCount += insertResult.createdItemsCount;

x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_duplicate_signals.test.ts

@@ -36,11 +36,13 @@ const mockSignals = [
 ];
 
 describe('filterDuplicateSignals', () => {
-  it('filters duplicate signals', () => {
-    expect(filterDuplicateSignals(mockRuleId1, mockSignals).length).toEqual(1);
-  });
+  describe('detection engine implementation', () => {
+    it('filters duplicate signals', () => {
+      expect(filterDuplicateSignals(mockRuleId1, mockSignals, false).length).toEqual(1);
+    });
 
-  it('does not filter non-duplicate signals', () => {
-    expect(filterDuplicateSignals(mockRuleId3, mockSignals).length).toEqual(2);
+    it('does not filter non-duplicate signals', () => {
+      expect(filterDuplicateSignals(mockRuleId3, mockSignals, false).length).toEqual(2);
+    });
   });
 });

x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_duplicate_signals.ts

@@ -5,10 +5,26 @@
  * 2.0.
  */
 
-import { WrappedSignalHit } from './types';
+import { SimpleHit, WrappedSignalHit } from './types';
 
-export const filterDuplicateSignals = (ruleId: string, signals: WrappedSignalHit[]) => {
-  return signals.filter(
-    (doc) => !doc._source.signal?.ancestors.some((ancestor) => ancestor.rule === ruleId)
-  );
+const isWrappedSignalHit = (
+  signals: SimpleHit[],
+  isRuleRegistryEnabled: boolean
+): signals is WrappedSignalHit[] => {
+  return !isRuleRegistryEnabled;
+};
+
+export const filterDuplicateSignals = (
+  ruleId: string,
+  signals: SimpleHit[],
+  isRuleRegistryEnabled: boolean
+) => {
+  if (isWrappedSignalHit(signals, isRuleRegistryEnabled)) {
+    return signals.filter(
+      (doc) => !doc._source.signal?.ancestors.some((ancestor) => ancestor.rule === ruleId)
+    );
+  } else {
+    // TODO: filter duplicate signals for RAC
+    return [];
+  }
 };

x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts

@@ -66,7 +66,10 @@ describe('searchAfterAndBulkCreate', () => {
       buildRuleMessage,
       false
     );
-    wrapHits = wrapHitsFactory({ ruleSO, signalsIndex: DEFAULT_SIGNALS_INDEX });
+    wrapHits = wrapHitsFactory({
+      ruleSO,
+      signalsIndex: DEFAULT_SIGNALS_INDEX,
+    });
   });
 
   test('should return success with number of searches less than max signals', async () => {

x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts

@@ -67,6 +67,7 @@ import {
 } from '../schemas/rule_schemas';
 import { bulkCreateFactory } from './bulk_create_factory';
 import { wrapHitsFactory } from './wrap_hits_factory';
+import { wrapSequencesFactory } from './wrap_sequences_factory';
 
 export const signalRulesAlertType = ({
   logger,
@@ -233,6 +234,11 @@ export const signalRulesAlertType = ({
           signalsIndex: params.outputIndex,
         });
 
+        const wrapSequences = wrapSequencesFactory({
+          ruleSO: savedObject,
+          signalsIndex: params.outputIndex,
+        });
+
         if (isMlRule(type)) {
           const mlRuleSO = asTypeSpecificSO(savedObject, machineLearningRuleParams);
           for (const tuple of tuples) {
@@ -313,6 +319,8 @@ export const signalRulesAlertType = ({
               searchAfterSize,
               bulkCreate,
               logger,
+              wrapHits,
+              wrapSequences,
             });
           }
         } else {

x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts

@@ -25,6 +25,7 @@ import {
   BaseHit,
   RuleAlertAction,
   SearchTypes,
+  EqlSequence,
 } from '../../../../common/detection_engine/types';
 import { ListClient } from '../../../../../lists/server';
 import { Logger, SavedObject } from '../../../../../../../src/core/server';
@@ -257,9 +258,11 @@ export type SignalsEnrichment = (signals: SignalSearchResponse) => Promise<Signa
 
 export type BulkCreate = <T>(docs: Array<BaseHit<T>>) => Promise<GenericBulkCreateResponse<T>>;
 
-export type WrapHits = (
-  hits: Array<estypes.SearchHit<unknown>>
-) => Array<BaseHit<{ '@timestamp': string }>>;
+export type SimpleHit = BaseHit<{ '@timestamp': string }>;
+
+export type WrapHits = (hits: Array<estypes.SearchHit<SignalSource>>) => SimpleHit[];
+
+export type WrapSequences = (sequences: Array<EqlSequence<SignalSource>>) => SimpleHit[];
 
 export interface SearchAfterAndBulkCreateParams {
   tuple: {

x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_hits_factory.ts

@@ -5,12 +5,7 @@
  * 2.0.
  */
 
-import {
-  SearchAfterAndBulkCreateParams,
-  SignalSourceHit,
-  WrapHits,
-  WrappedSignalHit,
-} from './types';
+import { SearchAfterAndBulkCreateParams, WrapHits, WrappedSignalHit } from './types';
 import { generateId } from './utils';
 import { buildBulkBody } from './build_bulk_body';
 import { filterDuplicateSignals } from './filter_duplicate_signals';
@@ -25,11 +20,15 @@ export const wrapHitsFactory = ({
   const wrappedDocs: WrappedSignalHit[] = events.flatMap((doc) => [
     {
       _index: signalsIndex,
-      // TODO: bring back doc._version
-      _id: generateId(doc._index, doc._id, '', ruleSO.attributes.params.ruleId ?? ''),
-      _source: buildBulkBody(ruleSO, doc as SignalSourceHit),
+      _id: generateId(
+        doc._index,
+        doc._id,
+        String(doc._version),
+        ruleSO.attributes.params.ruleId ?? ''
+      ),
+      _source: buildBulkBody(ruleSO, doc),
     },
   ]);
 
-  return filterDuplicateSignals(ruleSO.id, wrappedDocs);
+  return filterDuplicateSignals(ruleSO.id, wrappedDocs, false);
 };

x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_sequences_factory.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { SearchAfterAndBulkCreateParams, WrappedSignalHit, WrapSequences } from './types';
+import { buildSignalGroupFromSequence } from './build_bulk_body';
+
+export const wrapSequencesFactory = ({
+  ruleSO,
+  signalsIndex,
+}: {
+  ruleSO: SearchAfterAndBulkCreateParams['ruleSO'];
+  signalsIndex: string;
+}): WrapSequences => (sequences) =>
+  sequences.reduce(
+    (acc: WrappedSignalHit[], sequence) => [
+      ...acc,
+      ...buildSignalGroupFromSequence(sequence, ruleSO, signalsIndex),
+    ],
+    []
+  );

x-pack/plugins/security_solution/server/plugin.ts

@@ -208,8 +208,10 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
     });
 
     // TODO: Once we are past experimental phase this check can be removed along with legacy registration of rules
+    const isRuleRegistryEnabled = experimentalFeatures.ruleRegistryEnabled;
+
     let ruleDataClient: RuleDataClient | null = null;
-    if (experimentalFeatures.ruleRegistryEnabled) {
+    if (isRuleRegistryEnabled) {
       const { ruleDataService } = plugins.ruleRegistry;
       const start = () => core.getStartServices().then(([coreStart]) => coreStart);
 
@@ -293,7 +295,7 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
     const ruleTypes = [
       SIGNALS_ID,
       NOTIFICATIONS_ID,
-      ...(experimentalFeatures.ruleRegistryEnabled ? referenceRuleTypes : []),
+      ...(isRuleRegistryEnabled ? referenceRuleTypes : []),
     ];
 
     plugins.features.registerKibanaFeature({

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts

@@ -227,7 +227,7 @@ export default ({ getService }: FtrProviderContext) => {
           parents: [
             {
               rule: signalNoRule.parents[0].rule, // rule id is always changing so skip testing it
-              id: 'acf538fc082adf970012be166527c4d9fc120f0015f145e0a466a3ceb32db606',
+              id: '82421e2f4e96058baaa2ed87abbe565403b45edf36348c2b79a4f0e8cc1cd055',
               type: 'signal',
               index: '.siem-signals-default-000001',
               depth: 1,
@@ -242,7 +242,7 @@ export default ({ getService }: FtrProviderContext) => {
             },
             {
               rule: signalNoRule.ancestors[1].rule, // rule id is always changing so skip testing it
-              id: 'acf538fc082adf970012be166527c4d9fc120f0015f145e0a466a3ceb32db606',
+              id: '82421e2f4e96058baaa2ed87abbe565403b45edf36348c2b79a4f0e8cc1cd055',
               type: 'signal',
               index: '.siem-signals-default-000001',
               depth: 1,
@@ -252,7 +252,7 @@ export default ({ getService }: FtrProviderContext) => {
           depth: 2,
           parent: {
             rule: signalNoRule.parent?.rule, // parent.rule is always changing so skip testing it
-            id: 'acf538fc082adf970012be166527c4d9fc120f0015f145e0a466a3ceb32db606',
+            id: '82421e2f4e96058baaa2ed87abbe565403b45edf36348c2b79a4f0e8cc1cd055',
             type: 'signal',
             index: '.siem-signals-default-000001',
             depth: 1,
@@ -1265,7 +1265,7 @@ export default ({ getService }: FtrProviderContext) => {
           parents: [
             {
               rule: signalNoRule.parents[0].rule, // rule id is always changing so skip testing it
-              id: 'b63bcc90b9393f94899991397a3c2df2f3f5c6ebf56440434500f1e1419df7c9',
+              id: 'c4db4921f2d9152865fd6518c2a2ef3471738e49f607a21319048c69a303f83f',
               type: 'signal',
               index: '.siem-signals-default-000001',
               depth: 1,
@@ -1280,7 +1280,7 @@ export default ({ getService }: FtrProviderContext) => {
             },
             {
               rule: signalNoRule.ancestors[1].rule, // rule id is always changing so skip testing it
-              id: 'b63bcc90b9393f94899991397a3c2df2f3f5c6ebf56440434500f1e1419df7c9',
+              id: 'c4db4921f2d9152865fd6518c2a2ef3471738e49f607a21319048c69a303f83f',
               type: 'signal',
               index: '.siem-signals-default-000001',
               depth: 1,
@@ -1290,7 +1290,7 @@ export default ({ getService }: FtrProviderContext) => {
           depth: 2,
           parent: {
             rule: signalNoRule.parent?.rule, // parent.rule is always changing so skip testing it
-            id: 'b63bcc90b9393f94899991397a3c2df2f3f5c6ebf56440434500f1e1419df7c9',
+            id: 'c4db4921f2d9152865fd6518c2a2ef3471738e49f607a21319048c69a303f83f',
             type: 'signal',
             index: '.siem-signals-default-000001',
             depth: 1,
@@ -1423,7 +1423,7 @@ export default ({ getService }: FtrProviderContext) => {
           parents: [
             {
               rule: signalNoRule.parents[0].rule, // rule id is always changing so skip testing it
-              id: 'd2114ed6553816f87d6707b5bc50b88751db73b0f4930433d0890474804aa179',
+              id: '0733d5d2eaed77410a65eec95cfb2df099abc97289b78e2b0b406130e2dbdb33',
               type: 'signal',
               index: '.siem-signals-default-000001',
               depth: 1,
@@ -1438,7 +1438,7 @@ export default ({ getService }: FtrProviderContext) => {
             },
             {
               rule: signalNoRule.ancestors[1].rule, // rule id is always changing so skip testing it
-              id: 'd2114ed6553816f87d6707b5bc50b88751db73b0f4930433d0890474804aa179',
+              id: '0733d5d2eaed77410a65eec95cfb2df099abc97289b78e2b0b406130e2dbdb33',
               type: 'signal',
               index: '.siem-signals-default-000001',
               depth: 1,
@@ -1448,7 +1448,7 @@ export default ({ getService }: FtrProviderContext) => {
           depth: 2,
           parent: {
             rule: signalNoRule.parent?.rule, // parent.rule is always changing so skip testing it
-            id: 'd2114ed6553816f87d6707b5bc50b88751db73b0f4930433d0890474804aa179',
+            id: '0733d5d2eaed77410a65eec95cfb2df099abc97289b78e2b0b406130e2dbdb33',
             type: 'signal',
             index: '.siem-signals-default-000001',
             depth: 1,