[Security Solution][CTI] Event enrichment search strategy (elastic#10…

…1553)

* Adding boilerplate for new CTI search strategy type

This is going to be a subtype of the general SecSol search strategy;
the main functionality is going to be:

* transformation of the incoming parameters into named equivalents
* transformation of responses to include enrichment context fields
  (matched.*)

* More boilerplate, including tests

A few type errors because our functions don't actually do anything yet,
nor are our request/response types fleshed out.

* Starting to flesh out the request parsing

* Defines a basic request, along with a mock
* Defines helper function to generate should clauses from field values
* Adds placeholder tests throughout

* Fleshing out unit tests around our enrichment query

* Fleshing out response parsing of eventEnrichment strategy

* Fix types from elasticsearch

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/security_solution/common/cti/constants.ts

@@ -44,3 +44,16 @@ export const SORTED_THREAT_SUMMARY_FIELDS = [
   INDICATOR_FIRSTSEEN,
   INDICATOR_LASTSEEN,
 ];
+
+export const EVENT_ENRICHMENT_INDICATOR_FIELD_MAP = {
+  'file.hash.md5': 'threatintel.indicator.file.hash.md5',
+  'file.hash.sha1': 'threatintel.indicator.file.hash.sha1',
+  'file.hash.sha256': 'threatintel.indicator.file.hash.sha256',
+  'file.pe.imphash': 'threatintel.indicator.file.pe.imphash',
+  'file.elf.telfhash': 'threatintel.indicator.file.elf.telfhash',
+  'file.hash.ssdeep': 'threatintel.indicator.file.hash.ssdeep',
+  'source.ip': 'threatintel.indicator.ip',
+  'destination.ip': 'threatintel.indicator.ip',
+  'url.full': 'threatintel.indicator.url.full',
+  'registry.path': 'threatintel.indicator.registry.path',
+};

x-pack/plugins/security_solution/common/ecs/threat/index.ts

@@ -10,6 +10,8 @@ import { EventEcs } from '../event';
 interface ThreatMatchEcs {
   atomic?: string[];
   field?: string[];
+  id?: string[];
+  index?: string[];
   type?: string[];
 }
 

x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.mock.ts

@@ -0,0 +1,110 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { IEsSearchResponse } from 'src/plugins/data/public';
+
+import {
+  CtiEventEnrichmentRequestOptions,
+  CtiEventEnrichmentStrategyResponse,
+  CtiQueries,
+} from '.';
+
+export const buildEventEnrichmentRequestOptionsMock = (
+  overrides: Partial<CtiEventEnrichmentRequestOptions> = {}
+): CtiEventEnrichmentRequestOptions => ({
+  defaultIndex: ['filebeat-*'],
+  eventFields: {
+    'file.hash.md5': '1eee2bf3f56d8abed72da2bc523e7431',
+    'source.ip': '127.0.0.1',
+    'url.full': 'elastic.co',
+  },
+  factoryQueryType: CtiQueries.eventEnrichment,
+  filterQuery: '{"bool":{"must":[],"filter":[{"match_all":{}}],"should":[],"must_not":[]}}',
+  timerange: { interval: '', from: '2020-09-13T09:00:43.249Z', to: '2020-09-14T09:00:43.249Z' },
+  ...overrides,
+});
+
+export const buildEventEnrichmentRawResponseMock = (): IEsSearchResponse => ({
+  rawResponse: {
+    took: 17,
+    timed_out: false,
+    _shards: {
+      total: 1,
+      successful: 1,
+      skipped: 0,
+      failed: 0,
+    },
+    hits: {
+      total: {
+        value: 1,
+        relation: 'eq',
+      },
+      max_score: 6.0637846,
+      hits: [
+        {
+          _index: 'filebeat-8.0.0-2021.05.28-000001',
+          _id: '31408415b6d5601a92d29b86c2519658f210c194057588ae396d55cc20b3f03d',
+          _score: 6.0637846,
+          fields: {
+            'event.category': ['threat'],
+            'threatintel.indicator.file.type': ['html'],
+            'related.hash': [
+              '5529de7b60601aeb36f57824ed0e1ae8',
+              '15b012e6f626d0f88c2926d2bf4ca394d7b8ee07cc06d2ec05ea76bed3e8a05e',
+              '768:NXSFGJ/ooP6FawrB7Bo1MWnF/jRmhJImp:1SFXIqBo1Mwj2p',
+            ],
+            'threatintel.indicator.first_seen': ['2021-05-28T18:33:29.000Z'],
+            'threatintel.indicator.file.hash.tlsh': [
+              'FFB20B82F6617061C32784E2712F7A46B179B04FD1EA54A0F28CD8E9CFE4CAA1617F1C',
+            ],
+            'service.type': ['threatintel'],
+            'threatintel.indicator.file.hash.ssdeep': [
+              '768:NXSFGJ/ooP6FawrB7Bo1MWnF/jRmhJImp:1SFXIqBo1Mwj2p',
+            ],
+            'agent.type': ['filebeat'],
+            'event.module': ['threatintel'],
+            'threatintel.indicator.type': ['file'],
+            'agent.name': ['rylastic.local'],
+            'threatintel.indicator.file.hash.sha256': [
+              '15b012e6f626d0f88c2926d2bf4ca394d7b8ee07cc06d2ec05ea76bed3e8a05e',
+            ],
+            'event.kind': ['enrichment'],
+            'threatintel.indicator.file.hash.md5': ['5529de7b60601aeb36f57824ed0e1ae8'],
+            'fileset.name': ['abusemalware'],
+            'input.type': ['httpjson'],
+            'agent.hostname': ['rylastic.local'],
+            tags: ['threatintel-abusemalware', 'forwarded'],
+            'event.ingested': ['2021-05-28T18:33:55.086Z'],
+            '@timestamp': ['2021-05-28T18:33:52.993Z'],
+            'agent.id': ['ff93aee5-86a1-4a61-b0e6-0cdc313d01b5'],
+            'ecs.version': ['1.6.0'],
+            'event.reference': [
+              'https://urlhaus-api.abuse.ch/v1/download/15b012e6f626d0f88c2926d2bf4ca394d7b8ee07cc06d2ec05ea76bed3e8a05e/',
+            ],
+            'event.type': ['indicator'],
+            'event.created': ['2021-05-28T18:33:52.993Z'],
+            'agent.ephemeral_id': ['d6b14f65-5bf3-430d-8315-7b5613685979'],
+            'threatintel.indicator.file.size': [24738],
+            'agent.version': ['8.0.0'],
+            'event.dataset': ['threatintel.abusemalware'],
+          },
+          matched_queries: ['file.hash.md5'],
+        },
+      ],
+    },
+  },
+});
+
+export const buildEventEnrichmentResponseMock = (
+  overrides: Partial<CtiEventEnrichmentStrategyResponse> = {}
+): CtiEventEnrichmentStrategyResponse => ({
+  ...buildEventEnrichmentRawResponseMock(),
+  enrichments: [],
+  inspect: { dsl: ['{"mocked": "json"}'] },
+  totalCount: 0,
+  ...overrides,
+});

x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { IEsSearchResponse } from 'src/plugins/data/public';
+import { Inspect } from '../../common';
+import { RequestBasicOptions } from '..';
+
+export enum CtiQueries {
+  eventEnrichment = 'eventEnrichment',
+}
+
+export interface CtiEventEnrichmentRequestOptions extends RequestBasicOptions {
+  eventFields: Record<string, unknown>;
+}
+
+export type CtiEnrichment = Record<string, unknown[]>;
+
+export interface CtiEventEnrichmentStrategyResponse extends IEsSearchResponse {
+  enrichments: CtiEnrichment[];
+  inspect?: Inspect;
+  totalCount: number;
+}

x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts

@@ -66,6 +66,11 @@ import {
   MatrixHistogramStrategyResponse,
 } from './matrix_histogram';
 import { TimerangeInput, SortField, PaginationInput, PaginationInputPaginated } from '../common';
+import {
+  CtiEventEnrichmentRequestOptions,
+  CtiEventEnrichmentStrategyResponse,
+  CtiQueries,
+} from './cti';
 
 export * from './hosts';
 export * from './matrix_histogram';
@@ -76,6 +81,7 @@ export type FactoryQueryTypes =
   | HostsKpiQueries
   | NetworkQueries
   | NetworkKpiQueries
+  | CtiQueries
   | typeof MatrixHistogramQuery
   | typeof MatrixHistogramQueryEntities;
 
@@ -145,6 +151,8 @@ export type StrategyResponseType<T extends FactoryQueryTypes> = T extends HostsQ
   ? NetworkKpiUniquePrivateIpsStrategyResponse
   : T extends typeof MatrixHistogramQuery
   ? MatrixHistogramStrategyResponse
+  : T extends CtiQueries.eventEnrichment
+  ? CtiEventEnrichmentStrategyResponse
   : never;
 
 export type StrategyRequestType<T extends FactoryQueryTypes> = T extends HostsQueries.hosts
@@ -193,6 +201,8 @@ export type StrategyRequestType<T extends FactoryQueryTypes> = T extends HostsQu
   ? NetworkKpiUniquePrivateIpsRequestOptions
   : T extends typeof MatrixHistogramQuery
   ? MatrixHistogramRequestOptions
+  : T extends CtiQueries.eventEnrichment
+  ? CtiEventEnrichmentRequestOptions
   : never;
 
 export interface DocValueFieldsInput {

x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/factory.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { CtiQueries } from '../../../../../../common/search_strategy/security_solution/cti';
+import { SecuritySolutionFactory } from '../../types';
+import { buildEventEnrichmentQuery } from './query';
+import { parseEventEnrichmentResponse } from './response';
+
+export const eventEnrichment: SecuritySolutionFactory<CtiQueries.eventEnrichment> = {
+  buildDsl: buildEventEnrichmentQuery,
+  parse: parseEventEnrichmentResponse,
+};

x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/helpers.test.ts

@@ -0,0 +1,172 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { buildIndicatorEnrichments, buildIndicatorShouldClauses, getTotalCount } from './helpers';
+
+describe('buildIndicatorShouldClauses', () => {
+  it('returns an empty array given an empty fieldset', () => {
+    expect(buildIndicatorShouldClauses({})).toEqual([]);
+  });
+
+  it('returns an empty array given no relevant values', () => {
+    const eventFields = { 'url.domain': 'elastic.co' };
+    expect(buildIndicatorShouldClauses(eventFields)).toEqual([]);
+  });
+
+  it('returns a clause for each relevant value', () => {
+    const eventFields = { 'source.ip': '127.0.0.1', 'url.full': 'elastic.co' };
+    expect(buildIndicatorShouldClauses(eventFields)).toHaveLength(2);
+  });
+
+  it('excludes non-CTI fields', () => {
+    const eventFields = { 'source.ip': '127.0.0.1', 'url.domain': 'elastic.co' };
+    expect(buildIndicatorShouldClauses(eventFields)).toHaveLength(1);
+  });
+
+  it('defines a named query where the name is the event field and the value is the event field value', () => {
+    const eventFields = { 'file.hash.md5': '1eee2bf3f56d8abed72da2bc523e7431' };
+
+    expect(buildIndicatorShouldClauses(eventFields)).toContainEqual({
+      match: {
+        'threatintel.indicator.file.hash.md5': {
+          _name: 'file.hash.md5',
+          query: '1eee2bf3f56d8abed72da2bc523e7431',
+        },
+      },
+    });
+  });
+
+  it('returns valid queries for multiple valid fields', () => {
+    const eventFields = { 'source.ip': '127.0.0.1', 'url.full': 'elastic.co' };
+    expect(buildIndicatorShouldClauses(eventFields)).toEqual(
+      expect.arrayContaining([
+        { match: { 'threatintel.indicator.ip': { _name: 'source.ip', query: '127.0.0.1' } } },
+        { match: { 'threatintel.indicator.url.full': { _name: 'url.full', query: 'elastic.co' } } },
+      ])
+    );
+  });
+});
+
+describe('getTotalCount', () => {
+  it('returns 0 when total is null (not tracking)', () => {
+    expect(getTotalCount(null)).toEqual(0);
+  });
+
+  it('returns total when total is a number', () => {
+    expect(getTotalCount(5)).toEqual(5);
+  });
+
+  it('returns total.value when total is an object', () => {
+    expect(getTotalCount({ value: 20, relation: 'eq' })).toEqual(20);
+  });
+});
+
+describe('buildIndicatorEnrichments', () => {
+  it('returns nothing if hits have no matched queries', () => {
+    const hits = [{ _id: '_id', _index: '_index', matched_queries: [] }];
+    expect(buildIndicatorEnrichments(hits)).toEqual([]);
+  });
+
+  it("returns nothing if hits' matched queries are not valid", () => {
+    const hits = [{ _id: '_id', _index: '_index', matched_queries: ['invalid.field'] }];
+    expect(buildIndicatorEnrichments(hits)).toEqual([]);
+  });
+
+  it('builds a single enrichment if the hit has a matched query', () => {
+    const hits = [
+      {
+        _id: '_id',
+        _index: '_index',
+        matched_queries: ['file.hash.md5'],
+        fields: {
+          'threatintel.indicator.file.hash.md5': ['indicator_value'],
+        },
+      },
+    ];
+
+    expect(buildIndicatorEnrichments(hits)).toEqual([
+      expect.objectContaining({
+        'matched.atomic': ['indicator_value'],
+        'matched.field': ['file.hash.md5'],
+        'matched.id': ['_id'],
+        'matched.index': ['_index'],
+        'threatintel.indicator.file.hash.md5': ['indicator_value'],
+      }),
+    ]);
+  });
+
+  it('builds multiple enrichments if the hit has matched queries', () => {
+    const hits = [
+      {
+        _id: '_id',
+        _index: '_index',
+        matched_queries: ['file.hash.md5', 'source.ip'],
+        fields: {
+          'threatintel.indicator.file.hash.md5': ['indicator_value'],
+          'threatintel.indicator.ip': ['127.0.0.1'],
+        },
+      },
+    ];
+
+    expect(buildIndicatorEnrichments(hits)).toEqual([
+      expect.objectContaining({
+        'matched.atomic': ['indicator_value'],
+        'matched.field': ['file.hash.md5'],
+        'matched.id': ['_id'],
+        'matched.index': ['_index'],
+        'threatintel.indicator.file.hash.md5': ['indicator_value'],
+        'threatintel.indicator.ip': ['127.0.0.1'],
+      }),
+      expect.objectContaining({
+        'matched.atomic': ['127.0.0.1'],
+        'matched.field': ['source.ip'],
+        'matched.id': ['_id'],
+        'matched.index': ['_index'],
+        'threatintel.indicator.file.hash.md5': ['indicator_value'],
+        'threatintel.indicator.ip': ['127.0.0.1'],
+      }),
+    ]);
+  });
+
+  it('builds an enrichment for each hit', () => {
+    const hits = [
+      {
+        _id: '_id',
+        _index: '_index',
+        matched_queries: ['file.hash.md5'],
+        fields: {
+          'threatintel.indicator.file.hash.md5': ['indicator_value'],
+        },
+      },
+      {
+        _id: '_id2',
+        _index: '_index2',
+        matched_queries: ['source.ip'],
+        fields: {
+          'threatintel.indicator.ip': ['127.0.0.1'],
+        },
+      },
+    ];
+
+    expect(buildIndicatorEnrichments(hits)).toEqual([
+      expect.objectContaining({
+        'matched.atomic': ['indicator_value'],
+        'matched.field': ['file.hash.md5'],
+        'matched.id': ['_id'],
+        'matched.index': ['_index'],
+        'threatintel.indicator.file.hash.md5': ['indicator_value'],
+      }),
+      expect.objectContaining({
+        'matched.atomic': ['127.0.0.1'],
+        'matched.field': ['source.ip'],
+        'matched.id': ['_id2'],
+        'matched.index': ['_index2'],
+        'threatintel.indicator.ip': ['127.0.0.1'],
+      }),
+    ]);
+  });
+});