[SIEM] [Detections] Fixes filtering with large value lists to use "an…

…ds" between lists (elastic#72304)

* wip - comment and sample json for exceptions

* promise.all for OR-ing exception items and quick-start script

* logging, added/updated json sample scripts, fixed  missing await on filter with lists

* WIP

* bug fix where two lists when 'anded' together were not filtering down result set

* undo changes from testing

* fix changes to example json and fixes missed conflict with master

* update log message and fix type errors

* change log statement and add unit test for when exception items without a value list are passed in to the filter function

* fix failing test

* update expect on one test and adds a new test to ensure anding of value lists when appearing in different exception items

* update test after rebasing with master

* properly ands exception item entries together with proper test cases

* fix test (log statement tests - need to come up with a better way to cover these)

* cleans up json examples

* rename test and use 'every' in lieu of 'some' when determining if the filter logic should execute

x-pack/plugins/lists/server/scripts/exception_lists/new/exception_list_item.json

@@ -8,7 +8,7 @@
   "name": "Sample Endpoint Exception List",
   "entries": [
     {
-      "field": "host.ip",
+      "field": "actingProcess.file.signer",
       "operator": "excluded",
       "type": "exists"
     },

x-pack/plugins/lists/server/scripts/exception_lists/new/exception_list_item_with_bad_ip_list.json

@@ -0,0 +1,24 @@
+{
+  "list_id": "endpoint_list",
+  "item_id": "endpoint_list_item_good_rock01",
+  "_tags": ["endpoint", "process", "malware", "os:windows"],
+  "tags": ["user added string for a tag", "malware"],
+  "type": "simple",
+  "description": "Don't signal when agent.name is rock01 and source.ip is in the goodguys.txt list",
+  "name": "Filter out good guys ip and agent.name rock01",
+  "comments": [],
+  "entries": [
+    {
+      "field": "agent.name",
+      "operator": "excluded",
+      "type": "match",
+      "value": ["rock01"]
+    },
+    {
+      "field": "source.ip",
+      "operator": "excluded",
+      "type": "list",
+      "list": { "id": "goodguys.txt", "type": "ip" }
+    }
+  ]
+}

x-pack/plugins/lists/server/scripts/lists/new/list_ip_item.json

@@ -0,0 +1,4 @@
+{
+  "id": "hand_inserted_item_id",
+  "value": "127.0.0.1"
+}

x-pack/plugins/lists/server/scripts/lists/new/list_keyword_item.json

@@ -0,0 +1,4 @@
+{
+  "list_id": "keyword_list",
+  "value": "sh"
+}

x-pack/plugins/lists/server/scripts/quick_start.sh

@@ -0,0 +1,5 @@
+./hard_reset.sh && \
+./post_list.sh lists/new/lists/keyword.json && \
+./post_list_item.sh lists/new/list_keyword_item.json && \
+./post_exception_list.sh && \
+./post_exception_list_item.sh ./exception_lists/new/exception_list_item_with_list.json

x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts

@@ -69,7 +69,8 @@ export const sampleDocNoSortIdNoVersion = (someUuid: string = sampleIdGuid): Sig
 
 export const sampleDocWithSortId = (
   someUuid: string = sampleIdGuid,
-  ip?: string
+  ip?: string,
+  destIp?: string
 ): SignalSourceHit => ({
   _index: 'myFakeSignalIndex',
   _type: 'doc',
@@ -82,6 +83,9 @@ export const sampleDocWithSortId = (
     source: {
       ip: ip ?? '127.0.0.1',
     },
+    destination: {
+      ip: destIp ?? '127.0.0.1',
+    },
   },
   sort: ['1234567891111'],
 });
@@ -307,7 +311,8 @@ export const repeatedSearchResultsWithSortId = (
   total: number,
   pageSize: number,
   guids: string[],
-  ips?: string[]
+  ips?: string[],
+  destIps?: string[]
 ) => ({
   took: 10,
   timed_out: false,
@@ -321,7 +326,11 @@ export const repeatedSearchResultsWithSortId = (
     total,
     max_score: 100,
     hits: Array.from({ length: pageSize }).map((x, index) => ({
-      ...sampleDocWithSortId(guids[index], ips ? ips[index] : '127.0.0.1'),
+      ...sampleDocWithSortId(
+        guids[index],
+        ips ? ips[index] : '127.0.0.1',
+        destIps ? destIps[index] : '127.0.0.1'
+      ),
     })),
   },
 });

x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.test.ts

@@ -44,6 +44,25 @@ describe('filterEventsAgainstList', () => {
     expect(res.hits.hits.length).toEqual(4);
   });
 
+  it('should respond with eventSearchResult if exceptionList does not contain value list exceptions', async () => {
+    const res = await filterEventsAgainstList({
+      logger: mockLogger,
+      listClient,
+      exceptionsList: [getExceptionListItemSchemaMock()],
+      eventSearchResult: repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3), [
+        '1.1.1.1',
+        '2.2.2.2',
+        '3.3.3.3',
+        '7.7.7.7',
+      ]),
+      buildRuleMessage,
+    });
+    expect(res.hits.hits.length).toEqual(4);
+    expect(((mockLogger.debug as unknown) as jest.Mock).mock.calls[0][0]).toContain(
+      'no exception items of type list found - returning original search result'
+    );
+  });
+
   describe('operator_type is included', () => {
     it('should respond with same list if no items match value list', async () => {
       const exceptionItem = getExceptionListItemSchemaMock();
@@ -106,6 +125,280 @@ describe('filterEventsAgainstList', () => {
         'ci-badguys.txt'
       );
       expect(res.hits.hits.length).toEqual(2);
+
+      // @ts-ignore
+      const ipVals = res.hits.hits.map((item) => item._source.source.ip);
+      expect(['3.3.3.3', '7.7.7.7']).toEqual(ipVals);
+    });
+
+    it('should respond with less items in the list given two exception items with entries of type list if some values match', async () => {
+      const exceptionItem = getExceptionListItemSchemaMock();
+      exceptionItem.entries = [
+        {
+          field: 'source.ip',
+          operator: 'included',
+          type: 'list',
+          list: {
+            id: 'ci-badguys.txt',
+            type: 'ip',
+          },
+        },
+      ];
+
+      const exceptionItemAgain = getExceptionListItemSchemaMock();
+      exceptionItemAgain.entries = [
+        {
+          field: 'source.ip',
+          operator: 'included',
+          type: 'list',
+          list: {
+            id: 'ci-badguys-again.txt',
+            type: 'ip',
+          },
+        },
+      ];
+
+      // this call represents an exception list with a value list containing ['2.2.2.2', '4.4.4.4']
+      (listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([
+        { ...getListItemResponseMock(), value: '2.2.2.2' },
+        { ...getListItemResponseMock(), value: '4.4.4.4' },
+      ]);
+      // this call represents an exception list with a value list containing ['6.6.6.6']
+      (listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([
+        { ...getListItemResponseMock(), value: '6.6.6.6' },
+      ]);
+
+      const res = await filterEventsAgainstList({
+        logger: mockLogger,
+        listClient,
+        exceptionsList: [exceptionItem, exceptionItemAgain],
+        eventSearchResult: repeatedSearchResultsWithSortId(9, 9, someGuids.slice(0, 9), [
+          '1.1.1.1',
+          '2.2.2.2',
+          '3.3.3.3',
+          '4.4.4.4',
+          '5.5.5.5',
+          '6.6.6.6',
+          '7.7.7.7',
+          '8.8.8.8',
+          '9.9.9.9',
+        ]),
+        buildRuleMessage,
+      });
+      expect(listClient.getListItemByValues as jest.Mock).toHaveBeenCalledTimes(2);
+      expect(res.hits.hits.length).toEqual(6);
+
+      // @ts-ignore
+      const ipVals = res.hits.hits.map((item) => item._source.source.ip);
+      expect(['1.1.1.1', '3.3.3.3', '5.5.5.5', '7.7.7.7', '8.8.8.8', '9.9.9.9']).toEqual(ipVals);
+    });
+
+    it('should respond with less items in the list given two exception items, each with one entry of type list if some values match', async () => {
+      const exceptionItem = getExceptionListItemSchemaMock();
+      exceptionItem.entries = [
+        {
+          field: 'source.ip',
+          operator: 'included',
+          type: 'list',
+          list: {
+            id: 'ci-badguys.txt',
+            type: 'ip',
+          },
+        },
+      ];
+
+      const exceptionItemAgain = getExceptionListItemSchemaMock();
+      exceptionItemAgain.entries = [
+        {
+          field: 'source.ip',
+          operator: 'included',
+          type: 'list',
+          list: {
+            id: 'ci-badguys-again.txt',
+            type: 'ip',
+          },
+        },
+      ];
+
+      // this call represents an exception list with a value list containing ['2.2.2.2', '4.4.4.4']
+      (listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([
+        { ...getListItemResponseMock(), value: '2.2.2.2' },
+      ]);
+      // this call represents an exception list with a value list containing ['6.6.6.6']
+      (listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([
+        { ...getListItemResponseMock(), value: '6.6.6.6' },
+      ]);
+
+      const res = await filterEventsAgainstList({
+        logger: mockLogger,
+        listClient,
+        exceptionsList: [exceptionItem, exceptionItemAgain],
+        eventSearchResult: repeatedSearchResultsWithSortId(9, 9, someGuids.slice(0, 9), [
+          '1.1.1.1',
+          '2.2.2.2',
+          '3.3.3.3',
+          '4.4.4.4',
+          '5.5.5.5',
+          '6.6.6.6',
+          '7.7.7.7',
+          '8.8.8.8',
+          '9.9.9.9',
+        ]),
+        buildRuleMessage,
+      });
+      expect(listClient.getListItemByValues as jest.Mock).toHaveBeenCalledTimes(2);
+      // @ts-ignore
+      const ipVals = res.hits.hits.map((item) => item._source.source.ip);
+      expect(res.hits.hits.length).toEqual(7);
+
+      expect(['1.1.1.1', '3.3.3.3', '4.4.4.4', '5.5.5.5', '7.7.7.7', '8.8.8.8', '9.9.9.9']).toEqual(
+        ipVals
+      );
+    });
+
+    it('should respond with less items in the list given one exception item with two entries of type list only if source.ip and destination.ip are in the events', async () => {
+      const exceptionItem = getExceptionListItemSchemaMock();
+      exceptionItem.entries = [
+        {
+          field: 'source.ip',
+          operator: 'included',
+          type: 'list',
+          list: {
+            id: 'ci-badguys.txt',
+            type: 'ip',
+          },
+        },
+        {
+          field: 'destination.ip',
+          operator: 'included',
+          type: 'list',
+          list: {
+            id: 'ci-badguys-again.txt',
+            type: 'ip',
+          },
+        },
+      ];
+
+      // this call represents an exception list with a value list containing ['2.2.2.2']
+      (listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([
+        { ...getListItemResponseMock(), value: '2.2.2.2' },
+      ]);
+      // this call represents an exception list with a value list containing ['4.4.4.4']
+      (listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([
+        { ...getListItemResponseMock(), value: '4.4.4.4' },
+      ]);
+
+      const res = await filterEventsAgainstList({
+        logger: mockLogger,
+        listClient,
+        exceptionsList: [exceptionItem],
+        eventSearchResult: repeatedSearchResultsWithSortId(
+          9,
+          9,
+          someGuids.slice(0, 9),
+          [
+            '1.1.1.1',
+            '2.2.2.2',
+            '3.3.3.3',
+            '4.4.4.4',
+            '5.5.5.5',
+            '6.6.6.6',
+            '2.2.2.2',
+            '8.8.8.8',
+            '9.9.9.9',
+          ],
+          [
+            '2.2.2.2',
+            '2.2.2.2',
+            '2.2.2.2',
+            '2.2.2.2',
+            '2.2.2.2',
+            '2.2.2.2',
+            '4.4.4.4',
+            '2.2.2.2',
+            '2.2.2.2',
+          ]
+        ),
+        buildRuleMessage,
+      });
+      expect(listClient.getListItemByValues as jest.Mock).toHaveBeenCalledTimes(2);
+      expect(res.hits.hits.length).toEqual(8);
+
+      // @ts-ignore
+      const ipVals = res.hits.hits.map((item) => item._source.source.ip);
+      expect([
+        '1.1.1.1',
+        '2.2.2.2',
+        '3.3.3.3',
+        '4.4.4.4',
+        '5.5.5.5',
+        '6.6.6.6',
+        '8.8.8.8',
+        '9.9.9.9',
+      ]).toEqual(ipVals);
+    });
+
+    it('should respond with the same items in the list given one exception item with two entries of type list where the entries are included and excluded', async () => {
+      const exceptionItem = getExceptionListItemSchemaMock();
+      exceptionItem.entries = [
+        {
+          field: 'source.ip',
+          operator: 'included',
+          type: 'list',
+          list: {
+            id: 'ci-badguys.txt',
+            type: 'ip',
+          },
+        },
+        {
+          field: 'source.ip',
+          operator: 'excluded',
+          type: 'list',
+          list: {
+            id: 'ci-badguys-again.txt',
+            type: 'ip',
+          },
+        },
+      ];
+
+      // this call represents an exception list with a value list containing ['2.2.2.2', '4.4.4.4']
+      (listClient.getListItemByValues as jest.Mock).mockResolvedValue([
+        { ...getListItemResponseMock(), value: '2.2.2.2' },
+      ]);
+
+      const res = await filterEventsAgainstList({
+        logger: mockLogger,
+        listClient,
+        exceptionsList: [exceptionItem],
+        eventSearchResult: repeatedSearchResultsWithSortId(9, 9, someGuids.slice(0, 9), [
+          '1.1.1.1',
+          '2.2.2.2',
+          '3.3.3.3',
+          '4.4.4.4',
+          '5.5.5.5',
+          '6.6.6.6',
+          '7.7.7.7',
+          '8.8.8.8',
+          '9.9.9.9',
+        ]),
+        buildRuleMessage,
+      });
+      expect(listClient.getListItemByValues as jest.Mock).toHaveBeenCalledTimes(2);
+      expect(res.hits.hits.length).toEqual(9);
+
+      // @ts-ignore
+      const ipVals = res.hits.hits.map((item) => item._source.source.ip);
+      expect([
+        '1.1.1.1',
+        '2.2.2.2',
+        '3.3.3.3',
+        '4.4.4.4',
+        '5.5.5.5',
+        '6.6.6.6',
+        '7.7.7.7',
+        '8.8.8.8',
+        '9.9.9.9',
+      ]).toEqual(ipVals);
     });
   });
   describe('operator type is excluded', () => {