From 81c5fbf538eca80bab149f9dba1c97cfb1610564 Mon Sep 17 00:00:00 2001 From: Frank Hassanabad Date: Tue, 15 Feb 2022 16:05:01 -0700 Subject: [PATCH] [Security Solutions] Exposes the search_after and point in time (pit) from saved objects to exception lists (#125182) ## Summary Exposes the functionality of * search_after * point in time (pit) From saved objects to the exception lists. This _DOES NOT_ expose these to the REST API just yet. Rather this exposes it at the API level to start with and changes code that had hard limits of 10k and other limited loops. I use the batching of 1k for this at a time as I thought that would be a decent batch guess and I see other parts of the code changed to it. It's easy to change the 1k if we find we need to throttle back more as we get feedback from others. See this PR where `PIT` and `search_after` were first introduced: https://github.com/elastic/kibana/pull/89915 See these 2 issues where we should be using more paging and PIT (Point in Time) with search_after: https://github.com/elastic/kibana/issues/93770 https://github.com/elastic/kibana/issues/103944 The new methods added to the `exception_list_client.ts` client class are: * openPointInTime * closePointInTime * findExceptionListItemPointInTimeFinder * findExceptionListPointInTimeFinder * findExceptionListsItemPointInTimeFinder * findValueListExceptionListItemsPointInTimeFinder The areas of functionality that have been changed: * Exception list exports * Deletion of lists * Getting exception list items when generating signals Note that currently we use our own ways of looping over the saved objects which you can see in the codebase such as this older way below which does work but had a limitation of 10k against saved objects and did not do point in time (PIT) Older way example (deprecated): ```ts let page = 1; let ids: string[] = []; let foundExceptionListItems = await findExceptionListItem({ filter: undefined, listId, namespaceType, page, perPage: PER_PAGE, pit: undefined, savedObjectsClient, searchAfter: undefined, sortField: 'tie_breaker_id', sortOrder: 'desc', }); while (foundExceptionListItems != null && foundExceptionListItems.data.length > 0) { ids = [ ...ids, ...foundExceptionListItems.data.map((exceptionListItem) => exceptionListItem.id), ]; page += 1; foundExceptionListItems = await findExceptionListItem({ filter: undefined, listId, namespaceType, page, perPage: PER_PAGE, pit: undefined, savedObjectsClient, searchAfter: undefined, sortField: 'tie_breaker_id', sortOrder: 'desc', }); } return ids; ``` But now that is replaced with this newer way using PIT: ```ts // Stream the results from the Point In Time (PIT) finder into this array let ids: string[] = []; const executeFunctionOnStream = (response: FoundExceptionListItemSchema): void => { const responseIds = response.data.map((exceptionListItem) => exceptionListItem.id); ids = [...ids, ...responseIds]; }; await findExceptionListItemPointInTimeFinder({ executeFunctionOnStream, filter: undefined, listId, maxSize: undefined, // NOTE: This is unbounded when it is "undefined" namespaceType, perPage: 1_000, savedObjectsClient, sortField: 'tie_breaker_id', sortOrder: 'desc', }); return ids; ``` We also have areas of code that has perPage listed at 10k or a constant that represents 10k which this removes in most areas (but not all areas): ```ts const items = await client.findExceptionListsItem({ listId: listIds, namespaceType: namespaceTypes, page: 1, pit: undefined, perPage: MAX_EXCEPTION_LIST_SIZE, // <--- Really bad to send in 10k per page at a time searchAfter: undefined, filter: [], sortOrder: undefined, sortField: undefined, }); ``` That is now: ```ts // Stream the results from the Point In Time (PIT) finder into this array let items: ExceptionListItemSchema[] = []; const executeFunctionOnStream = (response: FoundExceptionListItemSchema): void => { items = [...items, ...response.data]; }; await client.findExceptionListsItemPointInTimeFinder({ executeFunctionOnStream, listId: listIds, namespaceType: namespaceTypes, perPage: 1_000, filter: [], maxSize: undefined, // NOTE: This is unbounded when it is "undefined" sortOrder: undefined, sortField: undefined, }); ``` Left over areas will be handled in separate PR's because they are in other people's code ownership areas. ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../src/common/index.ts | 3 + .../src/common/max_size/index.test.ts | 59 ++++ .../src/common/max_size/index.ts | 18 ++ .../src/common/pit/index.test.ts | 65 ++++ .../src/common/pit/index.ts | 22 ++ .../src/common/search_after/index.test.ts | 56 ++++ .../src/common/search_after/index.ts | 17 + .../found_exception_list_item_schema/index.ts | 24 +- .../found_exception_list_schema/index.ts | 20 +- .../lists/server/routes/delete_list_route.ts | 59 ++-- .../routes/find_endpoint_list_item_route.ts | 2 + .../routes/find_exception_list_item_route.ts | 2 + .../routes/find_exception_list_route.ts | 2 + .../plugins/lists/server/routes/validate.ts | 2 + .../exception_lists/close_point_in_time.ts | 31 ++ .../delete_exception_list_items_by_list.ts | 44 +-- .../exception_list_client.test.ts | 4 + .../exception_lists/exception_list_client.ts | 304 +++++++++++++++++- .../exception_list_client_types.ts | 78 ++++- .../export_exception_list_and_items.test.ts | 25 +- .../export_exception_list_and_items.ts | 23 +- .../find_exception_list.test.ts | 68 ---- .../exception_lists/find_exception_list.ts | 31 +- .../find_exception_list_item.ts | 10 +- ...xception_list_item_point_in_time_finder.ts | 89 +++++ .../find_exception_list_items.test.ts | 106 ------ .../find_exception_list_items.ts | 75 +---- ...ception_list_items_point_in_time_finder.ts | 142 ++++++++ ...ind_exception_list_point_in_time_finder.ts | 118 +++++++ .../find_value_list_exception_list_items.ts | 64 ++++ ...ception_list_items_point_in_time_finder.ts | 117 +++++++ .../exception_lists/open_point_in_time.ts | 37 +++ .../utils/get_exception_list_filter.test.ts | 66 ++++ .../utils/get_exception_list_filter.ts | 25 ++ .../get_exception_lists_item_filter.test.ts | 104 ++++++ .../utils/get_exception_lists_item_filter.ts | 36 +++ .../find_all_exception_list_item_types.ts | 2 +- .../import/find_all_exception_list_types.ts | 6 + .../services/exception_lists/utils/index.ts | 2 + .../services/items/get_list_item_by_values.ts | 3 + .../items/search_list_item_by_values.ts | 3 + .../detection_engine/signals/utils.test.ts | 60 ++-- .../lib/detection_engine/signals/utils.ts | 49 +-- 43 files changed, 1666 insertions(+), 407 deletions(-) create mode 100644 packages/kbn-securitysolution-io-ts-list-types/src/common/max_size/index.test.ts create mode 100644 packages/kbn-securitysolution-io-ts-list-types/src/common/max_size/index.ts create mode 100644 packages/kbn-securitysolution-io-ts-list-types/src/common/pit/index.test.ts create mode 100644 packages/kbn-securitysolution-io-ts-list-types/src/common/pit/index.ts create mode 100644 packages/kbn-securitysolution-io-ts-list-types/src/common/search_after/index.test.ts create mode 100644 packages/kbn-securitysolution-io-ts-list-types/src/common/search_after/index.ts create mode 100644 x-pack/plugins/lists/server/services/exception_lists/close_point_in_time.ts delete mode 100644 x-pack/plugins/lists/server/services/exception_lists/find_exception_list.test.ts create mode 100644 x-pack/plugins/lists/server/services/exception_lists/find_exception_list_item_point_in_time_finder.ts delete mode 100644 x-pack/plugins/lists/server/services/exception_lists/find_exception_list_items.test.ts create mode 100644 x-pack/plugins/lists/server/services/exception_lists/find_exception_list_items_point_in_time_finder.ts create mode 100644 x-pack/plugins/lists/server/services/exception_lists/find_exception_list_point_in_time_finder.ts create mode 100644 x-pack/plugins/lists/server/services/exception_lists/find_value_list_exception_list_items.ts create mode 100644 x-pack/plugins/lists/server/services/exception_lists/find_value_list_exception_list_items_point_in_time_finder.ts create mode 100644 x-pack/plugins/lists/server/services/exception_lists/open_point_in_time.ts create mode 100644 x-pack/plugins/lists/server/services/exception_lists/utils/get_exception_list_filter.test.ts create mode 100644 x-pack/plugins/lists/server/services/exception_lists/utils/get_exception_list_filter.ts create mode 100644 x-pack/plugins/lists/server/services/exception_lists/utils/get_exception_lists_item_filter.test.ts create mode 100644 x-pack/plugins/lists/server/services/exception_lists/utils/get_exception_lists_item_filter.ts diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/common/index.ts b/packages/kbn-securitysolution-io-ts-list-types/src/common/index.ts index 81ecd58cb397c..98c160e2c4302 100644 --- a/packages/kbn-securitysolution-io-ts-list-types/src/common/index.ts +++ b/packages/kbn-securitysolution-io-ts-list-types/src/common/index.ts @@ -35,6 +35,7 @@ export * from './list_operator'; export * from './list_type'; export * from './lists'; export * from './lists_default_array'; +export * from './max_size'; export * from './meta'; export * from './name'; export * from './non_empty_entries_array'; @@ -42,6 +43,8 @@ export * from './non_empty_nested_entries_array'; export * from './os_type'; export * from './page'; export * from './per_page'; +export * from './pit'; +export * from './search_after'; export * from './serializer'; export * from './sort_field'; export * from './sort_order'; diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/common/max_size/index.test.ts b/packages/kbn-securitysolution-io-ts-list-types/src/common/max_size/index.test.ts new file mode 100644 index 0000000000000..459195e3ec27f --- /dev/null +++ b/packages/kbn-securitysolution-io-ts-list-types/src/common/max_size/index.test.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { exactCheck } from '@kbn/securitysolution-io-ts-utils'; +import { maxSizeOrUndefined } from '.'; + +import { pipe } from 'fp-ts/lib/pipeable'; +import { left } from 'fp-ts/lib/Either'; + +import { foldLeftRight, getPaths } from '@kbn/securitysolution-io-ts-utils'; + +describe('maxSizeOrUndefined', () => { + test('it will validate a correct max value', () => { + const payload = 123; + const decoded = maxSizeOrUndefined.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it will fail to validate a 0', () => { + const payload = 0; + const decoded = maxSizeOrUndefined.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "0" supplied to "(PositiveIntegerGreaterThanZero | undefined)"', + ]); + expect(message.schema).toEqual({}); + }); + + test('it will fail to validate a -1', () => { + const payload = -1; + const decoded = maxSizeOrUndefined.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "-1" supplied to "(PositiveIntegerGreaterThanZero | undefined)"', + ]); + expect(message.schema).toEqual({}); + }); + + test('it will fail to validate a string', () => { + const payload = '123'; + const decoded = maxSizeOrUndefined.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "123" supplied to "(PositiveIntegerGreaterThanZero | undefined)"', + ]); + expect(message.schema).toEqual({}); + }); +}); diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/common/max_size/index.ts b/packages/kbn-securitysolution-io-ts-list-types/src/common/max_size/index.ts new file mode 100644 index 0000000000000..59ae5b7b7fc63 --- /dev/null +++ b/packages/kbn-securitysolution-io-ts-list-types/src/common/max_size/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +/* eslint-disable @typescript-eslint/naming-convention */ + +import { PositiveIntegerGreaterThanZero } from '@kbn/securitysolution-io-ts-types'; +import * as t from 'io-ts'; + +export const max_size = PositiveIntegerGreaterThanZero; +export type MaxSize = t.TypeOf; + +export const maxSizeOrUndefined = t.union([max_size, t.undefined]); +export type MaxSizeOrUndefined = t.TypeOf; diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/common/pit/index.test.ts b/packages/kbn-securitysolution-io-ts-list-types/src/common/pit/index.test.ts new file mode 100644 index 0000000000000..19aeb0690f13e --- /dev/null +++ b/packages/kbn-securitysolution-io-ts-list-types/src/common/pit/index.test.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { exactCheck } from '@kbn/securitysolution-io-ts-utils'; +import { pitOrUndefined } from '.'; + +import * as t from 'io-ts'; +import { pipe } from 'fp-ts/lib/pipeable'; +import { left } from 'fp-ts/lib/Either'; + +import { foldLeftRight, getPaths } from '@kbn/securitysolution-io-ts-utils'; + +describe('pitOrUndefined', () => { + test('it will validate a correct pit', () => { + const payload = { id: '123', keepAlive: '1m' }; + const decoded = pitOrUndefined.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it will validate with the value of "undefined"', () => { + const obj = t.exact( + t.type({ + pit_id: pitOrUndefined, + }) + ); + const payload: t.TypeOf = { + pit_id: undefined, + }; + const decoded = obj.decode({ + pit_id: undefined, + }); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it will validate a correct pit without having a "keepAlive"', () => { + const payload = { id: '123' }; + const decoded = pitOrUndefined.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it will fail to validate an incorrect pit', () => { + const payload = 'foo'; + const decoded = pitOrUndefined.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "foo" supplied to "({| id: string, keepAlive: (string | undefined) |} | undefined)"', + ]); + expect(message.schema).toEqual({}); + }); +}); diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/common/pit/index.ts b/packages/kbn-securitysolution-io-ts-list-types/src/common/pit/index.ts new file mode 100644 index 0000000000000..773794edaf1f6 --- /dev/null +++ b/packages/kbn-securitysolution-io-ts-list-types/src/common/pit/index.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as t from 'io-ts'; + +export const pitId = t.string; +export const pit = t.exact( + t.type({ + id: pitId, + keepAlive: t.union([t.string, t.undefined]), + }) +); +export const pitOrUndefined = t.union([pit, t.undefined]); + +export type Pit = t.TypeOf; +export type PitId = t.TypeOf; +export type PitOrUndefined = t.TypeOf; diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/common/search_after/index.test.ts b/packages/kbn-securitysolution-io-ts-list-types/src/common/search_after/index.test.ts new file mode 100644 index 0000000000000..135aa53d39783 --- /dev/null +++ b/packages/kbn-securitysolution-io-ts-list-types/src/common/search_after/index.test.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { exactCheck } from '@kbn/securitysolution-io-ts-utils'; +import { searchAfterOrUndefined } from '.'; + +import * as t from 'io-ts'; +import { pipe } from 'fp-ts/lib/pipeable'; +import { left } from 'fp-ts/lib/Either'; + +import { foldLeftRight, getPaths } from '@kbn/securitysolution-io-ts-utils'; + +describe('searchAfter', () => { + test('it will validate a correct search_after', () => { + const payload = ['test-1', 'test-2']; + const decoded = searchAfterOrUndefined.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it will validate with the value of "undefined"', () => { + const obj = t.exact( + t.type({ + search_after: searchAfterOrUndefined, + }) + ); + const payload: t.TypeOf = { + search_after: undefined, + }; + const decoded = obj.decode({ + pit_id: undefined, + }); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it will fail to validate an incorrect search_after', () => { + const payload = 'foo'; + const decoded = searchAfterOrUndefined.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "foo" supplied to "(Array | undefined)"', + ]); + expect(message.schema).toEqual({}); + }); +}); diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/common/search_after/index.ts b/packages/kbn-securitysolution-io-ts-list-types/src/common/search_after/index.ts new file mode 100644 index 0000000000000..ef39716e5bcac --- /dev/null +++ b/packages/kbn-securitysolution-io-ts-list-types/src/common/search_after/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +/* eslint-disable @typescript-eslint/naming-convention */ + +import * as t from 'io-ts'; + +export const search_after = t.array(t.string); +export type SearchAfter = t.TypeOf; + +export const searchAfterOrUndefined = t.union([search_after, t.undefined]); +export type SearchAfterOrUndefined = t.TypeOf; diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/response/found_exception_list_item_schema/index.ts b/packages/kbn-securitysolution-io-ts-list-types/src/response/found_exception_list_item_schema/index.ts index df82a70ef626c..587c39c385f91 100644 --- a/packages/kbn-securitysolution-io-ts-list-types/src/response/found_exception_list_item_schema/index.ts +++ b/packages/kbn-securitysolution-io-ts-list-types/src/response/found_exception_list_item_schema/index.ts @@ -10,16 +10,24 @@ import * as t from 'io-ts'; import { page } from '../../common/page'; import { per_page } from '../../common/per_page'; +import { pitId } from '../../common/pit'; import { total } from '../../common/total'; import { exceptionListItemSchema } from '../exception_list_item_schema'; -export const foundExceptionListItemSchema = t.exact( - t.type({ - data: t.array(exceptionListItemSchema), - page, - per_page, - total, - }) -); +export const foundExceptionListItemSchema = t.intersection([ + t.exact( + t.type({ + data: t.array(exceptionListItemSchema), + page, + per_page, + total, + }) + ), + t.exact( + t.partial({ + pit: pitId, + }) + ), +]); export type FoundExceptionListItemSchema = t.TypeOf; diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/response/found_exception_list_schema/index.ts b/packages/kbn-securitysolution-io-ts-list-types/src/response/found_exception_list_schema/index.ts index 4e430f607fb04..07b4090af9425 100644 --- a/packages/kbn-securitysolution-io-ts-list-types/src/response/found_exception_list_schema/index.ts +++ b/packages/kbn-securitysolution-io-ts-list-types/src/response/found_exception_list_schema/index.ts @@ -10,17 +10,21 @@ import * as t from 'io-ts'; import { page } from '../../common/page'; import { per_page } from '../../common/per_page'; +import { pitId } from '../../common/pit'; import { total } from '../../common/total'; import { exceptionListSchema } from '../exception_list_schema'; -export const foundExceptionListSchema = t.exact( - t.type({ - data: t.array(exceptionListSchema), - page, - per_page, - total, - }) -); +export const foundExceptionListSchema = t.intersection([ + t.exact( + t.type({ + data: t.array(exceptionListSchema), + page, + per_page, + total, + }) + ), + t.exact(t.partial({ pit: pitId })), +]); export type FoundExceptionListSchema = t.TypeOf; diff --git a/x-pack/plugins/lists/server/routes/delete_list_route.ts b/x-pack/plugins/lists/server/routes/delete_list_route.ts index 35ac490826703..6ddcce94d82e8 100644 --- a/x-pack/plugins/lists/server/routes/delete_list_route.ts +++ b/x-pack/plugins/lists/server/routes/delete_list_route.ts @@ -10,6 +10,8 @@ import { transformError } from '@kbn/securitysolution-es-utils'; import { EntriesArray, ExceptionListItemSchema, + ExceptionListSchema, + FoundExceptionListItemSchema, FoundExceptionListSchema, deleteListSchema, exceptionListItemSchema, @@ -19,7 +21,7 @@ import { getSavedObjectType } from '@kbn/securitysolution-list-utils'; import { LIST_URL } from '@kbn/securitysolution-list-constants'; import type { ListsPluginRouter } from '../types'; -import { ExceptionListClient } from '../services/exception_lists/exception_list_client'; +import type { ExceptionListClient } from '../services/exception_lists/exception_list_client'; import { escapeQuotes } from '../services/utils/escape_query'; import { buildRouteValidation, buildSiemResponse } from './utils'; @@ -47,26 +49,31 @@ export const deleteListRoute = (router: ListsPluginRouter): void => { // ignoreReferences=true maintains pre-7.11 behavior of deleting value list without performing any additional checks if (!ignoreReferences) { - const referencedExceptionListItems = await exceptionLists.findValueListExceptionListItems( - { - page: 1, - perPage: 10000, - sortField: undefined, - sortOrder: undefined, - valueListId: id, - } - ); + // Stream the results from the Point In Time (PIT) finder into this array + let referencedExceptionListItems: ExceptionListItemSchema[] = []; + const executeFunctionOnStream = (foundResponse: FoundExceptionListItemSchema): void => { + referencedExceptionListItems = [...referencedExceptionListItems, ...foundResponse.data]; + }; - if (referencedExceptionListItems?.data?.length) { + await exceptionLists.findValueListExceptionListItemsPointInTimeFinder({ + executeFunctionOnStream, + maxSize: undefined, // NOTE: This is unbounded when it is "undefined" + perPage: 1_000, // See https://github.com/elastic/kibana/issues/93770 for choice of 1k + sortField: undefined, + sortOrder: undefined, + valueListId: id, + }); + if (referencedExceptionListItems.length) { // deleteReferences=false to perform dry run and identify referenced exception lists/items if (deleteReferences) { // Delete referenced exception list items // TODO: Create deleteListItems to delete in batch deleteExceptionItemResponses = await Promise.all( - referencedExceptionListItems.data.map(async (listItem) => { + referencedExceptionListItems.map(async (listItem) => { // Ensure only the single entry is deleted as there could be a separate value list referenced that is okay to keep // TODO: Add API to delete single entry - // @ts-ignore inline way of verifying entry type is EntryList? - const remainingEntries = listItem.entries.filter((e) => e?.list?.id !== id); + const remainingEntries = listItem.entries.filter( + (e) => e.type === 'list' && e.list.id !== id + ); if (remainingEntries.length === 0) { // All entries reference value list specified in request, delete entire exception list item return deleteExceptionListItem(exceptionLists, listItem); @@ -79,14 +86,12 @@ export const deleteListRoute = (router: ListsPluginRouter): void => { } else { const referencedExceptionLists = await getReferencedExceptionLists( exceptionLists, - referencedExceptionListItems.data + referencedExceptionListItems ); const refError = `Value list '${id}' is referenced in existing exception list(s)`; - const references = referencedExceptionListItems.data.map((item) => ({ + const references = referencedExceptionListItems.map((item) => ({ exception_item: item, - exception_list: referencedExceptionLists.data.find( - (l) => l.list_id === item.list_id - ), + exception_list: referencedExceptionLists.find((l) => l.list_id === item.list_id), })); return siemResponse.error({ @@ -140,7 +145,7 @@ export const deleteListRoute = (router: ListsPluginRouter): void => { const getReferencedExceptionLists = async ( exceptionLists: ExceptionListClient, exceptionListItems: ExceptionListItemSchema[] -): Promise => { +): Promise => { const filter = exceptionListItems .map( (item) => @@ -149,14 +154,22 @@ const getReferencedExceptionLists = async ( })}.attributes.list_id: "${escapeQuotes(item.list_id)}"` ) .join(' OR '); - return exceptionLists.findExceptionList({ + + // Stream the results from the Point In Time (PIT) finder into this array + let exceptionList: ExceptionListSchema[] = []; + const executeFunctionOnStream = (response: FoundExceptionListSchema): void => { + exceptionList = [...exceptionList, ...response.data]; + }; + await exceptionLists.findExceptionListPointInTimeFinder({ + executeFunctionOnStream, filter: `(${filter})`, + maxSize: undefined, // NOTE: This is unbounded when it is "undefined" namespaceType: ['agnostic', 'single'], - page: 1, - perPage: 10000, + perPage: 1_000, // See https://github.com/elastic/kibana/issues/93770 for choice of 1k sortField: undefined, sortOrder: undefined, }); + return exceptionList; }; /** diff --git a/x-pack/plugins/lists/server/routes/find_endpoint_list_item_route.ts b/x-pack/plugins/lists/server/routes/find_endpoint_list_item_route.ts index 36b5a66c2830f..6516e88877384 100644 --- a/x-pack/plugins/lists/server/routes/find_endpoint_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/find_endpoint_list_item_route.ts @@ -48,6 +48,8 @@ export const findEndpointListItemRoute = (router: ListsPluginRouter): void => { filter, page, perPage, + pit: undefined, + searchAfter: undefined, sortField, sortOrder, }); diff --git a/x-pack/plugins/lists/server/routes/find_exception_list_item_route.ts b/x-pack/plugins/lists/server/routes/find_exception_list_item_route.ts index fe7ffaa066281..67450ca02bb20 100644 --- a/x-pack/plugins/lists/server/routes/find_exception_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/find_exception_list_item_route.ts @@ -58,6 +58,8 @@ export const findExceptionListItemRoute = (router: ListsPluginRouter): void => { namespaceType, page, perPage, + pit: undefined, + searchAfter: undefined, sortField, sortOrder, }); diff --git a/x-pack/plugins/lists/server/routes/find_exception_list_route.ts b/x-pack/plugins/lists/server/routes/find_exception_list_route.ts index 5d1b78747a89e..e49b9c39d2f04 100644 --- a/x-pack/plugins/lists/server/routes/find_exception_list_route.ts +++ b/x-pack/plugins/lists/server/routes/find_exception_list_route.ts @@ -48,6 +48,8 @@ export const findExceptionListRoute = (router: ListsPluginRouter): void => { namespaceType, page, perPage, + pit: undefined, + searchAfter: undefined, sortField, sortOrder, }); diff --git a/x-pack/plugins/lists/server/routes/validate.ts b/x-pack/plugins/lists/server/routes/validate.ts index 5a118bf2c5ae0..29b2dd3b06d28 100644 --- a/x-pack/plugins/lists/server/routes/validate.ts +++ b/x-pack/plugins/lists/server/routes/validate.ts @@ -30,6 +30,8 @@ export const validateExceptionListSize = async ( namespaceType, page: undefined, perPage: undefined, + pit: undefined, + searchAfter: undefined, sortField: undefined, sortOrder: undefined, }); diff --git a/x-pack/plugins/lists/server/services/exception_lists/close_point_in_time.ts b/x-pack/plugins/lists/server/services/exception_lists/close_point_in_time.ts new file mode 100644 index 0000000000000..0fbcd08316166 --- /dev/null +++ b/x-pack/plugins/lists/server/services/exception_lists/close_point_in_time.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + SavedObjectsClientContract, + SavedObjectsClosePointInTimeResponse, +} from 'kibana/server'; +import type { PitId } from '@kbn/securitysolution-io-ts-list-types'; + +interface ClosePointInTimeOptions { + pit: PitId; + savedObjectsClient: SavedObjectsClientContract; +} + +/** + * Closes a point in time (PIT) for either exception lists or exception list items. + * See: https://www.elastic.co/guide/en/elasticsearch/reference/current/point-in-time-api.html + * @params pit {string} The point in time to close + * @params savedObjectsClient {object} The saved objects client to delegate to + * @return {SavedObjectsOpenPointInTimeResponse} The point in time (PIT) + */ +export const closePointInTime = async ({ + pit, + savedObjectsClient, +}: ClosePointInTimeOptions): Promise => { + return savedObjectsClient.closePointInTime(pit); +}; diff --git a/x-pack/plugins/lists/server/services/exception_lists/delete_exception_list_items_by_list.ts b/x-pack/plugins/lists/server/services/exception_lists/delete_exception_list_items_by_list.ts index 873fe66b1ad3b..c23c8967fef68 100644 --- a/x-pack/plugins/lists/server/services/exception_lists/delete_exception_list_items_by_list.ts +++ b/x-pack/plugins/lists/server/services/exception_lists/delete_exception_list_items_by_list.ts @@ -5,15 +5,16 @@ * 2.0. */ -import type { ListId, NamespaceType } from '@kbn/securitysolution-io-ts-list-types'; +import type { + FoundExceptionListItemSchema, + ListId, + NamespaceType, +} from '@kbn/securitysolution-io-ts-list-types'; import { getSavedObjectType } from '@kbn/securitysolution-list-utils'; import { asyncForEach } from '@kbn/std'; +import type { SavedObjectsClientContract } from 'kibana/server'; -import { SavedObjectsClientContract } from '../../../../../../src/core/server'; - -import { findExceptionListItem } from './find_exception_list_item'; - -const PER_PAGE = 100; +import { findExceptionListItemPointInTimeFinder } from './find_exception_list_item_point_in_time_finder'; interface DeleteExceptionListItemByListOptions { listId: ListId; @@ -35,35 +36,24 @@ export const getExceptionListItemIds = async ({ savedObjectsClient, namespaceType, }: DeleteExceptionListItemByListOptions): Promise => { - let page = 1; + // Stream the results from the Point In Time (PIT) finder into this array let ids: string[] = []; - let foundExceptionListItems = await findExceptionListItem({ + const executeFunctionOnStream = (response: FoundExceptionListItemSchema): void => { + const responseIds = response.data.map((exceptionListItem) => exceptionListItem.id); + ids = [...ids, ...responseIds]; + }; + + await findExceptionListItemPointInTimeFinder({ + executeFunctionOnStream, filter: undefined, listId, + maxSize: undefined, // NOTE: This is unbounded when it is "undefined" namespaceType, - page, - perPage: PER_PAGE, + perPage: 1_000, // See https://github.com/elastic/kibana/issues/93770 for choice of 1k savedObjectsClient, sortField: 'tie_breaker_id', sortOrder: 'desc', }); - while (foundExceptionListItems != null && foundExceptionListItems.data.length > 0) { - ids = [ - ...ids, - ...foundExceptionListItems.data.map((exceptionListItem) => exceptionListItem.id), - ]; - page += 1; - foundExceptionListItems = await findExceptionListItem({ - filter: undefined, - listId, - namespaceType, - page, - perPage: PER_PAGE, - savedObjectsClient, - sortField: 'tie_breaker_id', - sortOrder: 'desc', - }); - } return ids; }; diff --git a/x-pack/plugins/lists/server/services/exception_lists/exception_list_client.test.ts b/x-pack/plugins/lists/server/services/exception_lists/exception_list_client.test.ts index 3b6f2cb6ae4f2..1a444c403d3c2 100644 --- a/x-pack/plugins/lists/server/services/exception_lists/exception_list_client.test.ts +++ b/x-pack/plugins/lists/server/services/exception_lists/exception_list_client.test.ts @@ -212,6 +212,8 @@ describe('exception_list_client', () => { namespaceType: 'agnostic', page: 1, perPage: 1, + pit: undefined, + searchAfter: undefined, sortField: 'name', sortOrder: 'asc', }); @@ -229,6 +231,8 @@ describe('exception_list_client', () => { namespaceType: ['agnostic'], page: 1, perPage: 1, + pit: undefined, + searchAfter: undefined, sortField: 'name', sortOrder: 'asc', }); diff --git a/x-pack/plugins/lists/server/services/exception_lists/exception_list_client.ts b/x-pack/plugins/lists/server/services/exception_lists/exception_list_client.ts index 93f5077f021d5..d6fc1bfec8058 100644 --- a/x-pack/plugins/lists/server/services/exception_lists/exception_list_client.ts +++ b/x-pack/plugins/lists/server/services/exception_lists/exception_list_client.ts @@ -5,7 +5,12 @@ * 2.0. */ -import { KibanaRequest, SavedObjectsClientContract } from 'kibana/server'; +import type { + KibanaRequest, + SavedObjectsClientContract, + SavedObjectsClosePointInTimeResponse, + SavedObjectsOpenPointInTimeResponse, +} from 'kibana/server'; import { ExceptionListItemSchema, ExceptionListSchema, @@ -24,7 +29,8 @@ import type { ServerExtensionCallbackContext, } from '../extension_points'; -import { +import type { + ClosePointInTimeOptions, ConstructorOptions, CreateEndpointListItemOptions, CreateExceptionListItemOptions, @@ -36,15 +42,20 @@ import { ExportExceptionListAndItemsOptions, FindEndpointListItemOptions, FindExceptionListItemOptions, + FindExceptionListItemPointInTimeFinderOptions, + FindExceptionListItemsPointInTimeFinderOptions, FindExceptionListOptions, + FindExceptionListPointInTimeFinderOptions, FindExceptionListsItemOptions, FindValueListExceptionListsItems, + FindValueListExceptionListsItemsPointInTimeFinder, GetEndpointListItemOptions, GetExceptionListItemOptions, GetExceptionListOptions, GetExceptionListSummaryOptions, ImportExceptionListAndItemsAsArrayOptions, ImportExceptionListAndItemsOptions, + OpenPointInTimeOptions, UpdateEndpointListItemOptions, UpdateExceptionListItemOptions, UpdateExceptionListOptions, @@ -64,10 +75,7 @@ import { deleteExceptionList } from './delete_exception_list'; import { deleteExceptionListItem, deleteExceptionListItemById } from './delete_exception_list_item'; import { findExceptionListItem } from './find_exception_list_item'; import { findExceptionList } from './find_exception_list'; -import { - findExceptionListsItem, - findValueListExceptionListItems, -} from './find_exception_list_items'; +import { findExceptionListsItem } from './find_exception_list_items'; import { createEndpointList } from './create_endpoint_list'; import { createEndpointTrustedAppsList } from './create_endpoint_trusted_apps_list'; import { PromiseFromStreams, importExceptions } from './import_exception_list_and_items'; @@ -80,6 +88,13 @@ import { createExceptionsStreamFromNdjson, exceptionsChecksFromArray, } from './utils/import/create_exceptions_stream_logic'; +import { openPointInTime } from './open_point_in_time'; +import { closePointInTime } from './close_point_in_time'; +import { findExceptionListPointInTimeFinder } from './find_exception_list_point_in_time_finder'; +import { findValueListExceptionListItems } from './find_value_list_exception_list_items'; +import { findExceptionListsItemPointInTimeFinder } from './find_exception_list_items_point_in_time_finder'; +import { findValueListExceptionListItemsPointInTimeFinder } from './find_value_list_exception_list_items_point_in_time_finder'; +import { findExceptionListItemPointInTimeFinder } from './find_exception_list_item_point_in_time_finder'; export class ExceptionListClient { private readonly user: string; @@ -621,7 +636,9 @@ export class ExceptionListClient { listId, filter, perPage, + pit, page, + searchAfter, sortField, sortOrder, namespaceType, @@ -637,6 +654,8 @@ export class ExceptionListClient { namespaceType, page, perPage, + pit, + searchAfter, sortField, sortOrder, }, @@ -650,7 +669,9 @@ export class ExceptionListClient { namespaceType, page, perPage, + pit, savedObjectsClient, + searchAfter, sortField, sortOrder, }); @@ -660,7 +681,9 @@ export class ExceptionListClient { listId, filter, perPage, + pit, page, + searchAfter, sortField, sortOrder, namespaceType, @@ -676,6 +699,8 @@ export class ExceptionListClient { namespaceType, page, perPage, + pit, + searchAfter, sortField, sortOrder, }, @@ -689,7 +714,9 @@ export class ExceptionListClient { namespaceType, page, perPage, + pit, savedObjectsClient, + searchAfter, sortField, sortOrder, }); @@ -697,7 +724,9 @@ export class ExceptionListClient { public findValueListExceptionListItems = async ({ perPage, + pit, page, + searchAfter, sortField, sortOrder, valueListId, @@ -706,7 +735,9 @@ export class ExceptionListClient { return findValueListExceptionListItems({ page, perPage, + pit, savedObjectsClient, + searchAfter, sortField, sortOrder, valueListId, @@ -717,6 +748,8 @@ export class ExceptionListClient { filter, perPage, page, + pit, + searchAfter, sortField, sortOrder, namespaceType, @@ -727,7 +760,9 @@ export class ExceptionListClient { namespaceType, page, perPage, + pit, savedObjectsClient, + searchAfter, sortField, sortOrder, }); @@ -745,6 +780,8 @@ export class ExceptionListClient { filter, perPage, page, + pit, + searchAfter, sortField, sortOrder, }: FindEndpointListItemOptions): Promise => { @@ -756,7 +793,9 @@ export class ExceptionListClient { namespaceType: 'agnostic', page, perPage, + pit, savedObjectsClient, + searchAfter, sortField, sortOrder, }); @@ -865,4 +904,257 @@ export class ExceptionListClient { user, }); }; + + /** + * Opens a point in time (PIT) for either exception lists or exception list items. + * See: https://www.elastic.co/guide/en/elasticsearch/reference/current/point-in-time-api.html + * @params namespaceType {string} "agnostic" or "single" depending on which namespace you are targeting + * @params options {Object} The saved object PIT options + * @return {SavedObjectsOpenPointInTimeResponse} The point in time (PIT) + */ + public openPointInTime = async ({ + namespaceType, + options, + }: OpenPointInTimeOptions): Promise => { + const { savedObjectsClient } = this; + return openPointInTime({ + namespaceType, + options, + savedObjectsClient, + }); + }; + + /** + * Closes a point in time (PIT) for either exception lists or exception list items. + * See: https://www.elastic.co/guide/en/elasticsearch/reference/current/point-in-time-api.html + * @params pit {string} The point in time to close + * @return {SavedObjectsOpenPointInTimeResponse} The point in time (PIT) + */ + public closePointInTime = async ({ + pit, + }: ClosePointInTimeOptions): Promise => { + const { savedObjectsClient } = this; + return closePointInTime({ + pit, + savedObjectsClient, + }); + }; + + /** + * Finds an exception list item within a point in time (PIT) and then calls the function + * `executeFunctionOnStream` until the maxPerPage is reached and stops. + * NOTE: This is slightly different from the saved objects version in that it takes + * an injected function, so that we avoid doing additional plumbing with generators + * to try to keep the maintenance of this machinery simpler for now. + * + * If you want to stream all results up to 10k into memory for correlation this would be: + * @example + * ```ts + * const exceptionList: ExceptionListItemSchema[] = []; + * const executeFunctionOnStream = (response: FoundExceptionListItemSchema) => { + * exceptionList = [...exceptionList, ...response.data]; + * } + * await client.findExceptionListItemPointInTimeFinder({ + * filter, + * executeFunctionOnStream, + * namespaceType, + * maxSize: 10_000, // NOTE: This is unbounded if it is "undefined" + * perPage: 1_000, // See https://github.com/elastic/kibana/issues/93770 for choice of 1k + * sortField, + * sortOrder, + * exe + * }); + * ``` + * @param filter {string} Your filter + * @param namespaceType {string} "agnostic" | "single" of your namespace + * @param perPage {number} The number of items per page. Typical value should be 1_000 here. Never go above 10_000 + * @param maxSize {number of undefined} If given a max size, this will not exceeded. Otherwise if undefined is passed down, all records will be processed. + * @param sortField {string} String of the field to sort against + * @param sortOrder "asc" | "desc" The order to sort against + */ + public findExceptionListItemPointInTimeFinder = async ({ + executeFunctionOnStream, + filter, + listId, + maxSize, + namespaceType, + perPage, + sortField, + sortOrder, + }: FindExceptionListItemPointInTimeFinderOptions): Promise => { + const { savedObjectsClient } = this; + return findExceptionListItemPointInTimeFinder({ + executeFunctionOnStream, + filter, + listId, + maxSize, + namespaceType, + perPage, + savedObjectsClient, + sortField, + sortOrder, + }); + }; + + /** + * Finds an exception list within a point in time (PIT) and then calls the function + * `executeFunctionOnStream` until the maxPerPage is reached and stops. + * NOTE: This is slightly different from the saved objects version in that it takes + * an injected function, so that we avoid doing additional plumbing with generators + * to try to keep the maintenance of this machinery simpler for now. + * + * If you want to stream all results up to 10k into memory for correlation this would be: + * @example + * ```ts + * const exceptionList: ExceptionListSchema[] = []; + * const executeFunctionOnStream = (response: FoundExceptionListSchema) => { + * exceptionList = [...exceptionList, ...response.data]; + * } + * await client.findExceptionListPointInTimeFinder({ + * filter, + * executeFunctionOnStream, + * namespaceType, + * maxSize: 10_000, // NOTE: This is unbounded if it is "undefined" + * perPage: 1_000, // See https://github.com/elastic/kibana/issues/93770 for choice of 1k + * sortField, + * sortOrder, + * exe + * }); + * ``` + * @param filter {string} Your filter + * @param namespaceType {string} "agnostic" | "single" of your namespace + * @param perPage {number} The number of items per page. Typical value should be 1_000 here. Never go above 10_000 + * @param maxSize {number of undefined} If given a max size, this will not exceeded. Otherwise if undefined is passed down, all records will be processed. + * @param sortField {string} String of the field to sort against + * @param sortOrder "asc" | "desc" The order to sort against + */ + public findExceptionListPointInTimeFinder = async ({ + executeFunctionOnStream, + filter, + maxSize, + namespaceType, + perPage, + sortField, + sortOrder, + }: FindExceptionListPointInTimeFinderOptions): Promise => { + const { savedObjectsClient } = this; + return findExceptionListPointInTimeFinder({ + executeFunctionOnStream, + filter, + maxSize, + namespaceType, + perPage, + savedObjectsClient, + sortField, + sortOrder, + }); + }; + + /** + * Finds exception list items within a point in time (PIT) and then calls the function + * `executeFunctionOnStream` until the maxPerPage is reached and stops. + * NOTE: This is slightly different from the saved objects version in that it takes + * an injected function, so that we avoid doing additional plumbing with generators + * to try to keep the maintenance of this machinery simpler for now. + * + * If you want to stream all results up to 10k into memory for correlation this would be: + * @example + * ```ts + * const exceptionList: ExceptionListItemSchema[] = []; + * const executeFunctionOnStream = (response: FoundExceptionListItemSchema) => { + * exceptionList = [...exceptionList, ...response.data]; + * } + * await client.findExceptionListsItemPointInTimeFinder({ + * filter, + * executeFunctionOnStream, + * namespaceType, + * maxSize: 10_000, // NOTE: This is unbounded if it is "undefined" + * perPage: 1_000, // See https://github.com/elastic/kibana/issues/93770 for choice of 1k + * sortField, + * sortOrder, + * exe + * }); + * ``` + * @param filter {string} Your filter + * @param namespaceType {string} "agnostic" | "single" of your namespace + * @param perPage {number} The number of items per page. Typical value should be 1_000 here. Never go above 10_000 + * @param maxSize {number of undefined} If given a max size, this will not exceeded. Otherwise if undefined is passed down, all records will be processed. + * @param sortField {string} String of the field to sort against + * @param sortOrder "asc" | "desc" The order to sort against + */ + public findExceptionListsItemPointInTimeFinder = async ({ + listId, + namespaceType, + executeFunctionOnStream, + maxSize, + filter, + perPage, + sortField, + sortOrder, + }: FindExceptionListItemsPointInTimeFinderOptions): Promise => { + const { savedObjectsClient } = this; + return findExceptionListsItemPointInTimeFinder({ + executeFunctionOnStream, + filter, + listId, + maxSize, + namespaceType, + perPage, + savedObjectsClient, + sortField, + sortOrder, + }); + }; + + /** + * Finds value lists within exception lists within a point in time (PIT) and then calls the function + * `executeFunctionOnStream` until the maxPerPage is reached and stops. + * NOTE: This is slightly different from the saved objects version in that it takes + * an injected function, so that we avoid doing additional plumbing with generators + * to try to keep the maintenance of this machinery simpler for now. + * + * If you want to stream all results up to 10k into memory for correlation this would be: + * @example + * ```ts + * const exceptionList: ExceptionListItemSchema[] = []; + * const executeFunctionOnStream = (response: FoundExceptionListItemSchema) => { + * exceptionList = [...exceptionList, ...response.data]; + * } + * await client.findValueListExceptionListItemsPointInTimeFinder({ + * valueListId, + * executeFunctionOnStream, + * namespaceType, + * maxSize: 10_000, // NOTE: This is unbounded if it is "undefined" + * perPage: 1_000, // See https://github.com/elastic/kibana/issues/93770 for choice of 1k + * sortField, + * sortOrder, + * exe + * }); + * ``` + * @param valueListId {string} Your value list id + * @param namespaceType {string} "agnostic" | "single" of your namespace + * @param perPage {number} The number of items per page. Typical value should be 1_000 here. Never go above 10_000 + * @param maxSize {number of undefined} If given a max size, this will not exceeded. Otherwise if undefined is passed down, all records will be processed. + * @param sortField {string} String of the field to sort against + * @param sortOrder "asc" | "desc" The order to sort against + */ + public findValueListExceptionListItemsPointInTimeFinder = async ({ + valueListId, + executeFunctionOnStream, + perPage, + maxSize, + sortField, + sortOrder, + }: FindValueListExceptionListsItemsPointInTimeFinder): Promise => { + const { savedObjectsClient } = this; + return findValueListExceptionListItemsPointInTimeFinder({ + executeFunctionOnStream, + maxSize, + perPage, + savedObjectsClient, + sortField, + sortOrder, + valueListId, + }); + }; } diff --git a/x-pack/plugins/lists/server/services/exception_lists/exception_list_client_types.ts b/x-pack/plugins/lists/server/services/exception_lists/exception_list_client_types.ts index 1c2762cb52c97..4c7820fc05f94 100644 --- a/x-pack/plugins/lists/server/services/exception_lists/exception_list_client_types.ts +++ b/x-pack/plugins/lists/server/services/exception_lists/exception_list_client_types.ts @@ -5,9 +5,13 @@ * 2.0. */ -import { Readable } from 'stream'; +import type { Readable } from 'stream'; -import type { KibanaRequest, SavedObjectsClientContract } from 'kibana/server'; +import type { + KibanaRequest, + SavedObjectsClientContract, + SavedObjectsOpenPointInTimeOptions, +} from 'kibana/server'; import type { CreateCommentsArray, Description, @@ -19,6 +23,8 @@ import type { ExceptionListTypeOrUndefined, ExportExceptionDetails, FilterOrUndefined, + FoundExceptionListItemSchema, + FoundExceptionListSchema, Id, IdOrUndefined, Immutable, @@ -28,6 +34,7 @@ import type { ItemIdOrUndefined, ListId, ListIdOrUndefined, + MaxSizeOrUndefined, MetaOrUndefined, Name, NameOrUndefined, @@ -36,6 +43,9 @@ import type { OsTypeArray, PageOrUndefined, PerPageOrUndefined, + PitId, + PitOrUndefined, + SearchAfterOrUndefined, SortFieldOrUndefined, SortOrderOrUndefined, Tags, @@ -43,14 +53,14 @@ import type { UpdateCommentsArray, _VersionOrUndefined, } from '@kbn/securitysolution-io-ts-list-types'; -import { +import type { EmptyStringArrayDecoded, NonEmptyStringArrayDecoded, Version, VersionOrUndefined, } from '@kbn/securitysolution-io-ts-types'; -import { ExtensionPointStorageClientInterface } from '../extension_points'; +import type { ExtensionPointStorageClientInterface } from '../extension_points'; export interface ConstructorOptions { user: string; @@ -194,6 +204,8 @@ export interface FindExceptionListItemOptions { namespaceType: NamespaceType; filter: FilterOrUndefined; perPage: PerPageOrUndefined; + pit?: PitOrUndefined; + searchAfter?: SearchAfterOrUndefined; page: PageOrUndefined; sortField: SortFieldOrUndefined; sortOrder: SortOrderOrUndefined; @@ -202,6 +214,8 @@ export interface FindExceptionListItemOptions { export interface FindEndpointListItemOptions { filter: FilterOrUndefined; perPage: PerPageOrUndefined; + pit?: PitOrUndefined; + searchAfter?: SearchAfterOrUndefined; page: PageOrUndefined; sortField: SortFieldOrUndefined; sortOrder: SortOrderOrUndefined; @@ -212,6 +226,8 @@ export interface FindExceptionListsItemOptions { namespaceType: NamespaceTypeArray; filter: EmptyStringArrayDecoded; perPage: PerPageOrUndefined; + pit?: PitOrUndefined; + searchAfter?: SearchAfterOrUndefined; page: PageOrUndefined; sortField: SortFieldOrUndefined; sortOrder: SortOrderOrUndefined; @@ -220,6 +236,8 @@ export interface FindExceptionListsItemOptions { export interface FindValueListExceptionListsItems { valueListId: Id; perPage: PerPageOrUndefined; + pit?: PitOrUndefined; + searchAfter?: SearchAfterOrUndefined; page: PageOrUndefined; sortField: SortFieldOrUndefined; sortOrder: SortOrderOrUndefined; @@ -230,6 +248,8 @@ export interface FindExceptionListOptions { filter: FilterOrUndefined; perPage: PerPageOrUndefined; page: PageOrUndefined; + pit?: PitOrUndefined; + searchAfter?: SearchAfterOrUndefined; sortField: SortFieldOrUndefined; sortOrder: SortOrderOrUndefined; } @@ -256,3 +276,53 @@ export interface ImportExceptionListAndItemsAsArrayOptions { maxExceptionsImportSize: number; overwrite: boolean; } + +export interface OpenPointInTimeOptions { + namespaceType: NamespaceType; + options: SavedObjectsOpenPointInTimeOptions | undefined; +} + +export interface ClosePointInTimeOptions { + pit: PitId; +} + +export interface FindExceptionListItemPointInTimeFinderOptions { + listId: ListId; + namespaceType: NamespaceType; + filter: FilterOrUndefined; + perPage: PerPageOrUndefined; + sortField: SortFieldOrUndefined; + sortOrder: SortOrderOrUndefined; + executeFunctionOnStream: (response: FoundExceptionListItemSchema) => void; + maxSize: MaxSizeOrUndefined; +} + +export interface FindExceptionListPointInTimeFinderOptions { + maxSize: MaxSizeOrUndefined; + namespaceType: NamespaceTypeArray; + filter: FilterOrUndefined; + perPage: PerPageOrUndefined; + sortField: SortFieldOrUndefined; + sortOrder: SortOrderOrUndefined; + executeFunctionOnStream: (response: FoundExceptionListSchema) => void; +} + +export interface FindExceptionListItemsPointInTimeFinderOptions { + listId: NonEmptyStringArrayDecoded; + namespaceType: NamespaceTypeArray; + filter: EmptyStringArrayDecoded; + perPage: PerPageOrUndefined; + sortField: SortFieldOrUndefined; + sortOrder: SortOrderOrUndefined; + executeFunctionOnStream: (response: FoundExceptionListItemSchema) => void; + maxSize: MaxSizeOrUndefined; +} + +export interface FindValueListExceptionListsItemsPointInTimeFinder { + valueListId: Id; + perPage: PerPageOrUndefined; + sortField: SortFieldOrUndefined; + sortOrder: SortOrderOrUndefined; + executeFunctionOnStream: (response: FoundExceptionListItemSchema) => void; + maxSize: MaxSizeOrUndefined; +} diff --git a/x-pack/plugins/lists/server/services/exception_lists/export_exception_list_and_items.test.ts b/x-pack/plugins/lists/server/services/exception_lists/export_exception_list_and_items.test.ts index 9f3c02fecca20..0f8e4bf7bbf45 100644 --- a/x-pack/plugins/lists/server/services/exception_lists/export_exception_list_and_items.test.ts +++ b/x-pack/plugins/lists/server/services/exception_lists/export_exception_list_and_items.test.ts @@ -5,44 +5,49 @@ * 2.0. */ -import { SavedObjectsClientContract } from 'kibana/server'; +import { savedObjectsClientMock } from 'src/core/server/mocks'; import { getExceptionListItemSchemaMock } from '../../../common/schemas/response/exception_list_item_schema.mock'; import { getExceptionListSchemaMock } from '../../../common/schemas/response/exception_list_schema.mock'; import { exportExceptionListAndItems } from './export_exception_list_and_items'; -import { findExceptionListItem } from './find_exception_list_item'; +import { findExceptionListItemPointInTimeFinder } from './find_exception_list_item_point_in_time_finder'; import { getExceptionList } from './get_exception_list'; jest.mock('./get_exception_list'); -jest.mock('./find_exception_list_item'); +jest.mock('./find_exception_list_item_point_in_time_finder'); describe('export_exception_list_and_items', () => { describe('exportExceptionListAndItems', () => { test('it should return null if no matching exception list found', async () => { (getExceptionList as jest.Mock).mockResolvedValue(null); - (findExceptionListItem as jest.Mock).mockResolvedValue({ data: [] }); + (findExceptionListItemPointInTimeFinder as jest.Mock).mockImplementationOnce( + ({ executeFunctionOnStream }) => { + executeFunctionOnStream({ data: [getExceptionListItemSchemaMock()] }); + } + ); const result = await exportExceptionListAndItems({ id: '123', listId: 'non-existent', namespaceType: 'single', - savedObjectsClient: {} as SavedObjectsClientContract, + savedObjectsClient: savedObjectsClientMock.create(), }); expect(result).toBeNull(); }); test('it should return stringified list and items', async () => { (getExceptionList as jest.Mock).mockResolvedValue(getExceptionListSchemaMock()); - (findExceptionListItem as jest.Mock).mockResolvedValue({ - data: [getExceptionListItemSchemaMock()], - }); - + (findExceptionListItemPointInTimeFinder as jest.Mock).mockImplementationOnce( + ({ executeFunctionOnStream }) => { + executeFunctionOnStream({ data: [getExceptionListItemSchemaMock()] }); + } + ); const result = await exportExceptionListAndItems({ id: '123', listId: 'non-existent', namespaceType: 'single', - savedObjectsClient: {} as SavedObjectsClientContract, + savedObjectsClient: savedObjectsClientMock.create(), }); expect(result?.exportData).toEqual( `${JSON.stringify(getExceptionListSchemaMock())}\n${JSON.stringify( diff --git a/x-pack/plugins/lists/server/services/exception_lists/export_exception_list_and_items.ts b/x-pack/plugins/lists/server/services/exception_lists/export_exception_list_and_items.ts index b071c72a9b122..b2cd7d38d8d56 100644 --- a/x-pack/plugins/lists/server/services/exception_lists/export_exception_list_and_items.ts +++ b/x-pack/plugins/lists/server/services/exception_lists/export_exception_list_and_items.ts @@ -6,15 +6,17 @@ */ import type { + ExceptionListItemSchema, ExportExceptionDetails, + FoundExceptionListItemSchema, IdOrUndefined, ListIdOrUndefined, NamespaceType, } from '@kbn/securitysolution-io-ts-list-types'; import { transformDataToNdjson } from '@kbn/securitysolution-utils'; -import { SavedObjectsClientContract } from 'kibana/server'; +import type { SavedObjectsClientContract } from 'kibana/server'; -import { findExceptionListItem } from './find_exception_list_item'; +import { findExceptionListItemPointInTimeFinder } from './find_exception_list_item_point_in_time_finder'; import { getExceptionList } from './get_exception_list'; interface ExportExceptionListAndItemsOptions { @@ -45,20 +47,23 @@ export const exportExceptionListAndItems = async ({ if (exceptionList == null) { return null; } else { - // TODO: Will need to address this when we switch over to - // using PIT, don't want it to get lost - // https://github.com/elastic/kibana/issues/103944 - const listItems = await findExceptionListItem({ + // Stream the results from the Point In Time (PIT) finder into this array + let exceptionItems: ExceptionListItemSchema[] = []; + const executeFunctionOnStream = (response: FoundExceptionListItemSchema): void => { + exceptionItems = [...exceptionItems, ...response.data]; + }; + + await findExceptionListItemPointInTimeFinder({ + executeFunctionOnStream, filter: undefined, listId: exceptionList.list_id, + maxSize: undefined, // NOTE: This is unbounded when it is "undefined" namespaceType: exceptionList.namespace_type, - page: 1, - perPage: 10000, + perPage: 1_000, // See https://github.com/elastic/kibana/issues/93770 for choice of 1k savedObjectsClient, sortField: 'exception-list.created_at', sortOrder: 'desc', }); - const exceptionItems = listItems?.data ?? []; const { exportData } = getExport([exceptionList, ...exceptionItems]); // TODO: Add logic for missing lists and items on errors diff --git a/x-pack/plugins/lists/server/services/exception_lists/find_exception_list.test.ts b/x-pack/plugins/lists/server/services/exception_lists/find_exception_list.test.ts deleted file mode 100644 index 919a1e4e90a51..0000000000000 --- a/x-pack/plugins/lists/server/services/exception_lists/find_exception_list.test.ts +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { getExceptionListFilter } from './find_exception_list'; - -describe('find_exception_list', () => { - describe('getExceptionListFilter', () => { - test('it should create a filter for agnostic lists if only searching for agnostic lists', () => { - const filter = getExceptionListFilter({ - filter: undefined, - savedObjectTypes: ['exception-list-agnostic'], - }); - expect(filter).toEqual('(exception-list-agnostic.attributes.list_type: list)'); - }); - - test('it should create a filter for agnostic lists with additional filters if only searching for agnostic lists', () => { - const filter = getExceptionListFilter({ - filter: 'exception-list-agnostic.attributes.name: "Sample Endpoint Exception List"', - savedObjectTypes: ['exception-list-agnostic'], - }); - expect(filter).toEqual( - '(exception-list-agnostic.attributes.list_type: list) AND exception-list-agnostic.attributes.name: "Sample Endpoint Exception List"' - ); - }); - - test('it should create a filter for single lists if only searching for single lists', () => { - const filter = getExceptionListFilter({ - filter: undefined, - savedObjectTypes: ['exception-list'], - }); - expect(filter).toEqual('(exception-list.attributes.list_type: list)'); - }); - - test('it should create a filter for single lists with additional filters if only searching for single lists', () => { - const filter = getExceptionListFilter({ - filter: 'exception-list.attributes.name: "Sample Endpoint Exception List"', - savedObjectTypes: ['exception-list'], - }); - expect(filter).toEqual( - '(exception-list.attributes.list_type: list) AND exception-list.attributes.name: "Sample Endpoint Exception List"' - ); - }); - - test('it should create a filter that searches for both agnostic and single lists', () => { - const filter = getExceptionListFilter({ - filter: undefined, - savedObjectTypes: ['exception-list-agnostic', 'exception-list'], - }); - expect(filter).toEqual( - '(exception-list-agnostic.attributes.list_type: list OR exception-list.attributes.list_type: list)' - ); - }); - - test('it should create a filter that searches for both agnostic and single lists with additional filters if only searching for agnostic lists', () => { - const filter = getExceptionListFilter({ - filter: 'exception-list-agnostic.attributes.name: "Sample Endpoint Exception List"', - savedObjectTypes: ['exception-list-agnostic', 'exception-list'], - }); - expect(filter).toEqual( - '(exception-list-agnostic.attributes.list_type: list OR exception-list.attributes.list_type: list) AND exception-list-agnostic.attributes.name: "Sample Endpoint Exception List"' - ); - }); - }); -}); diff --git a/x-pack/plugins/lists/server/services/exception_lists/find_exception_list.ts b/x-pack/plugins/lists/server/services/exception_lists/find_exception_list.ts index b3d5dd9ddb32b..46d292e93a2d1 100644 --- a/x-pack/plugins/lists/server/services/exception_lists/find_exception_list.ts +++ b/x-pack/plugins/lists/server/services/exception_lists/find_exception_list.ts @@ -5,21 +5,24 @@ * 2.0. */ -import { SavedObjectsClientContract } from 'kibana/server'; +import type { SavedObjectsClientContract } from 'kibana/server'; import type { FilterOrUndefined, FoundExceptionListSchema, NamespaceTypeArray, PageOrUndefined, PerPageOrUndefined, + PitOrUndefined, + SearchAfterOrUndefined, SortFieldOrUndefined, SortOrderOrUndefined, } from '@kbn/securitysolution-io-ts-list-types'; -import { SavedObjectType, getSavedObjectTypes } from '@kbn/securitysolution-list-utils'; +import { getSavedObjectTypes } from '@kbn/securitysolution-list-utils'; -import { ExceptionListSoSchema } from '../../schemas/saved_objects'; +import type { ExceptionListSoSchema } from '../../schemas/saved_objects'; import { transformSavedObjectsToFoundExceptionList } from './utils'; +import { getExceptionListFilter } from './utils/get_exception_list_filter'; interface FindExceptionListOptions { namespaceType: NamespaceTypeArray; @@ -29,6 +32,8 @@ interface FindExceptionListOptions { page: PageOrUndefined; sortField: SortFieldOrUndefined; sortOrder: SortOrderOrUndefined; + pit: PitOrUndefined; + searchAfter: SearchAfterOrUndefined; } export const findExceptionList = async ({ @@ -37,14 +42,18 @@ export const findExceptionList = async ({ filter, page, perPage, + searchAfter, sortField, sortOrder, + pit, }: FindExceptionListOptions): Promise => { const savedObjectTypes = getSavedObjectTypes({ namespaceType }); const savedObjectsFindResponse = await savedObjectsClient.find({ filter: getExceptionListFilter({ filter, savedObjectTypes }), page, perPage, + pit, + searchAfter, sortField, sortOrder, type: savedObjectTypes, @@ -52,19 +61,3 @@ export const findExceptionList = async ({ return transformSavedObjectsToFoundExceptionList({ savedObjectsFindResponse }); }; - -export const getExceptionListFilter = ({ - filter, - savedObjectTypes, -}: { - filter: FilterOrUndefined; - savedObjectTypes: SavedObjectType[]; -}): string => { - const listTypesFilter = savedObjectTypes - .map((type) => `${type}.attributes.list_type: list`) - .join(' OR '); - - if (filter != null) { - return `(${listTypesFilter}) AND ${filter}`; - } else return `(${listTypesFilter})`; -}; diff --git a/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_item.ts b/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_item.ts index 3d050652afed1..5c23475573bb6 100644 --- a/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_item.ts +++ b/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_item.ts @@ -13,6 +13,8 @@ import type { NamespaceType, PageOrUndefined, PerPageOrUndefined, + PitOrUndefined, + SearchAfterOrUndefined, SortFieldOrUndefined, SortOrderOrUndefined, } from '@kbn/securitysolution-io-ts-list-types'; @@ -24,10 +26,12 @@ interface FindExceptionListItemOptions { namespaceType: NamespaceType; savedObjectsClient: SavedObjectsClientContract; filter: FilterOrUndefined; - perPage: PerPageOrUndefined; page: PageOrUndefined; + perPage: PerPageOrUndefined; + pit: PitOrUndefined; sortField: SortFieldOrUndefined; sortOrder: SortOrderOrUndefined; + searchAfter: SearchAfterOrUndefined; } export const findExceptionListItem = async ({ @@ -37,6 +41,8 @@ export const findExceptionListItem = async ({ filter, page, perPage, + pit, + searchAfter, sortField, sortOrder, }: FindExceptionListItemOptions): Promise => { @@ -46,7 +52,9 @@ export const findExceptionListItem = async ({ namespaceType: [namespaceType], page, perPage, + pit, savedObjectsClient, + searchAfter, sortField, sortOrder, }); diff --git a/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_item_point_in_time_finder.ts b/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_item_point_in_time_finder.ts new file mode 100644 index 0000000000000..8b90315c80ada --- /dev/null +++ b/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_item_point_in_time_finder.ts @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsClientContract } from 'kibana/server'; +import type { + FilterOrUndefined, + FoundExceptionListItemSchema, + ListId, + MaxSizeOrUndefined, + NamespaceType, + PerPageOrUndefined, + SortFieldOrUndefined, + SortOrderOrUndefined, +} from '@kbn/securitysolution-io-ts-list-types'; + +import { findExceptionListsItemPointInTimeFinder } from './find_exception_list_items_point_in_time_finder'; + +interface FindExceptionListItemPointInTimeFinderOptions { + listId: ListId; + namespaceType: NamespaceType; + savedObjectsClient: SavedObjectsClientContract; + filter: FilterOrUndefined; + perPage: PerPageOrUndefined; + sortField: SortFieldOrUndefined; + sortOrder: SortOrderOrUndefined; + executeFunctionOnStream: (response: FoundExceptionListItemSchema) => void; + maxSize: MaxSizeOrUndefined; +} + +/** + * Finds an exception list item within a point in time (PIT) and then calls the function + * `executeFunctionOnStream` until the maxPerPage is reached and stops. + * NOTE: This is slightly different from the saved objects version in that it takes + * an injected function, so that we avoid doing additional plumbing with generators + * to try to keep the maintenance of this machinery simpler for now. + * + * If you want to stream all results up to 10k into memory for correlation this would be: + * @example + * ```ts + * const exceptionList: ExceptionListItemSchema[] = []; + * const executeFunctionOnStream = (response: FoundExceptionListItemSchema) => { + * exceptionList = [...exceptionList, ...response.data]; + * } + * await client.findExceptionListItemPointInTimeFinder({ + * filter, + * executeFunctionOnStream, + * namespaceType, + * maxSize: 10_000, // NOTE: This is unbounded if it is "undefined" + * perPage: 1_000, // See https://github.com/elastic/kibana/issues/93770 for choice of 1k + * sortField, + * sortOrder, + * exe + * }); + * ``` + * @param filter {string} Your filter + * @param namespaceType {string} "agnostic" | "single" of your namespace + * @param perPage {number} The number of items per page. Typical value should be 1_000 here. Never go above 10_000 + * @param maxSize {number of undefined} If given a max size, this will not exceeded. Otherwise if undefined is passed down, all records will be processed. + * @param sortField {string} String of the field to sort against + * @param savedObjectsClient {Object} The saved object client + * @param sortOrder "asc" | "desc" The order to sort against + */ +export const findExceptionListItemPointInTimeFinder = async ({ + executeFunctionOnStream, + listId, + namespaceType, + savedObjectsClient, + filter, + maxSize, + perPage, + sortField, + sortOrder, +}: FindExceptionListItemPointInTimeFinderOptions): Promise => { + return findExceptionListsItemPointInTimeFinder({ + executeFunctionOnStream, + filter: filter != null ? [filter] : [], + listId: [listId], + maxSize, + namespaceType: [namespaceType], + perPage, + savedObjectsClient, + sortField, + sortOrder, + }); +}; diff --git a/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_items.test.ts b/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_items.test.ts deleted file mode 100644 index 3a2b12c358917..0000000000000 --- a/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_items.test.ts +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { LIST_ID } from '../../../common/constants.mock'; - -import { getExceptionListsItemFilter } from './find_exception_list_items'; - -describe('find_exception_list_items', () => { - describe('getExceptionListsItemFilter', () => { - test('It should create a filter with a single listId with an empty filter', () => { - const filter = getExceptionListsItemFilter({ - filter: [], - listId: [LIST_ID], - savedObjectType: ['exception-list'], - }); - expect(filter).toEqual( - '(exception-list.attributes.list_type: item AND exception-list.attributes.list_id: "some-list-id")' - ); - }); - - test('It should create a filter escaping quotes in list ids', () => { - const filter = getExceptionListsItemFilter({ - filter: [], - listId: ['list-id-"-with-quote'], - savedObjectType: ['exception-list'], - }); - expect(filter).toEqual( - '(exception-list.attributes.list_type: item AND exception-list.attributes.list_id: "list-id-\\"-with-quote")' - ); - }); - - test('It should create a filter with a single listId with a single filter', () => { - const filter = getExceptionListsItemFilter({ - filter: ['exception-list.attributes.name: "Sample Endpoint Exception List"'], - listId: [LIST_ID], - savedObjectType: ['exception-list'], - }); - expect(filter).toEqual( - '((exception-list.attributes.list_type: item AND exception-list.attributes.list_id: "some-list-id") AND exception-list.attributes.name: "Sample Endpoint Exception List")' - ); - }); - - test('It should create a filter with 2 listIds and an empty filter', () => { - const filter = getExceptionListsItemFilter({ - filter: [], - listId: ['list-1', 'list-2'], - savedObjectType: ['exception-list', 'exception-list-agnostic'], - }); - expect(filter).toEqual( - '(exception-list.attributes.list_type: item AND exception-list.attributes.list_id: "list-1") OR (exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.list_id: "list-2")' - ); - }); - - test('It should create a filter with 2 listIds and a single filter', () => { - const filter = getExceptionListsItemFilter({ - filter: ['exception-list.attributes.name: "Sample Endpoint Exception List"'], - listId: ['list-1', 'list-2'], - savedObjectType: ['exception-list', 'exception-list-agnostic'], - }); - expect(filter).toEqual( - '((exception-list.attributes.list_type: item AND exception-list.attributes.list_id: "list-1") AND exception-list.attributes.name: "Sample Endpoint Exception List") OR (exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.list_id: "list-2")' - ); - }); - - test('It should create a filter with 3 listIds and an empty filter', () => { - const filter = getExceptionListsItemFilter({ - filter: [], - listId: ['list-1', 'list-2', 'list-3'], - savedObjectType: ['exception-list', 'exception-list-agnostic', 'exception-list-agnostic'], - }); - expect(filter).toEqual( - '(exception-list.attributes.list_type: item AND exception-list.attributes.list_id: "list-1") OR (exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.list_id: "list-2") OR (exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.list_id: "list-3")' - ); - }); - - test('It should create a filter with 3 listIds and a single filter for the first item', () => { - const filter = getExceptionListsItemFilter({ - filter: ['exception-list.attributes.name: "Sample Endpoint Exception List"'], - listId: ['list-1', 'list-2', 'list-3'], - savedObjectType: ['exception-list', 'exception-list-agnostic', 'exception-list-agnostic'], - }); - expect(filter).toEqual( - '((exception-list.attributes.list_type: item AND exception-list.attributes.list_id: "list-1") AND exception-list.attributes.name: "Sample Endpoint Exception List") OR (exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.list_id: "list-2") OR (exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.list_id: "list-3")' - ); - }); - - test('It should create a filter with 3 listIds and 3 filters for each', () => { - const filter = getExceptionListsItemFilter({ - filter: [ - 'exception-list.attributes.name: "Sample Endpoint Exception List 1"', - 'exception-list.attributes.name: "Sample Endpoint Exception List 2"', - 'exception-list.attributes.name: "Sample Endpoint Exception List 3"', - ], - listId: ['list-1', 'list-2', 'list-3'], - savedObjectType: ['exception-list', 'exception-list-agnostic', 'exception-list-agnostic'], - }); - expect(filter).toEqual( - '((exception-list.attributes.list_type: item AND exception-list.attributes.list_id: "list-1") AND exception-list.attributes.name: "Sample Endpoint Exception List 1") OR ((exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.list_id: "list-2") AND exception-list.attributes.name: "Sample Endpoint Exception List 2") OR ((exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.list_id: "list-3") AND exception-list.attributes.name: "Sample Endpoint Exception List 3")' - ); - }); - }); -}); diff --git a/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_items.ts b/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_items.ts index 99298c0304c7d..5b4601dfadc22 100644 --- a/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_items.ts +++ b/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_items.ts @@ -5,13 +5,14 @@ * 2.0. */ -import { SavedObjectsClientContract } from 'kibana/server'; +import type { SavedObjectsClientContract } from 'kibana/server'; import type { FoundExceptionListItemSchema, - Id, NamespaceTypeArray, PageOrUndefined, PerPageOrUndefined, + PitOrUndefined, + SearchAfterOrUndefined, SortFieldOrUndefined, SortOrderOrUndefined, } from '@kbn/securitysolution-io-ts-list-types'; @@ -19,18 +20,13 @@ import type { EmptyStringArrayDecoded, NonEmptyStringArrayDecoded, } from '@kbn/securitysolution-io-ts-types'; -import { - SavedObjectType, - exceptionListAgnosticSavedObjectType, - exceptionListSavedObjectType, - getSavedObjectTypes, -} from '@kbn/securitysolution-list-utils'; +import { getSavedObjectTypes } from '@kbn/securitysolution-list-utils'; -import { escapeQuotes } from '../utils/escape_query'; -import { ExceptionListSoSchema } from '../../schemas/saved_objects'; +import type { ExceptionListSoSchema } from '../../schemas/saved_objects'; import { transformSavedObjectsToFoundExceptionListItem } from './utils'; import { getExceptionList } from './get_exception_list'; +import { getExceptionListsItemFilter } from './utils/get_exception_lists_item_filter'; interface FindExceptionListItemsOptions { listId: NonEmptyStringArrayDecoded; @@ -38,9 +34,11 @@ interface FindExceptionListItemsOptions { savedObjectsClient: SavedObjectsClientContract; filter: EmptyStringArrayDecoded; perPage: PerPageOrUndefined; + pit: PitOrUndefined; page: PageOrUndefined; sortField: SortFieldOrUndefined; sortOrder: SortOrderOrUndefined; + searchAfter: SearchAfterOrUndefined; } export const findExceptionListsItem = async ({ @@ -49,7 +47,9 @@ export const findExceptionListsItem = async ({ savedObjectsClient, filter, page, + pit, perPage, + searchAfter, sortField, sortOrder, }: FindExceptionListItemsOptions): Promise => { @@ -73,6 +73,8 @@ export const findExceptionListsItem = async ({ filter: getExceptionListsItemFilter({ filter, listId, savedObjectType }), page, perPage, + pit, + searchAfter, sortField, sortOrder, type: savedObjectType, @@ -82,56 +84,3 @@ export const findExceptionListsItem = async ({ }); } }; - -export const getExceptionListsItemFilter = ({ - filter, - listId, - savedObjectType, -}: { - listId: NonEmptyStringArrayDecoded; - filter: EmptyStringArrayDecoded; - savedObjectType: SavedObjectType[]; -}): string => { - return listId.reduce((accum, singleListId, index) => { - const escapedListId = escapeQuotes(singleListId); - const listItemAppend = `(${savedObjectType[index]}.attributes.list_type: item AND ${savedObjectType[index]}.attributes.list_id: "${escapedListId}")`; - const listItemAppendWithFilter = - filter[index] != null ? `(${listItemAppend} AND ${filter[index]})` : listItemAppend; - if (accum === '') { - return listItemAppendWithFilter; - } else { - return `${accum} OR ${listItemAppendWithFilter}`; - } - }, ''); -}; - -interface FindValueListExceptionListsItems { - valueListId: Id; - savedObjectsClient: SavedObjectsClientContract; - perPage: PerPageOrUndefined; - page: PageOrUndefined; - sortField: SortFieldOrUndefined; - sortOrder: SortOrderOrUndefined; -} - -export const findValueListExceptionListItems = async ({ - valueListId, - savedObjectsClient, - page, - perPage, - sortField, - sortOrder, -}: FindValueListExceptionListsItems): Promise => { - const escapedValueListId = escapeQuotes(valueListId); - const savedObjectsFindResponse = await savedObjectsClient.find({ - filter: `(exception-list.attributes.list_type: item AND exception-list.attributes.entries.list.id:"${escapedValueListId}") OR (exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.entries.list.id:"${escapedValueListId}") `, - page, - perPage, - sortField, - sortOrder, - type: [exceptionListSavedObjectType, exceptionListAgnosticSavedObjectType], - }); - return transformSavedObjectsToFoundExceptionListItem({ - savedObjectsFindResponse, - }); -}; diff --git a/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_items_point_in_time_finder.ts b/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_items_point_in_time_finder.ts new file mode 100644 index 0000000000000..d9020ee42b6ec --- /dev/null +++ b/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_items_point_in_time_finder.ts @@ -0,0 +1,142 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsClientContract } from 'kibana/server'; +import type { + FoundExceptionListItemSchema, + MaxSizeOrUndefined, + NamespaceTypeArray, + PerPageOrUndefined, + SortFieldOrUndefined, + SortOrderOrUndefined, +} from '@kbn/securitysolution-io-ts-list-types'; +import type { + EmptyStringArrayDecoded, + NonEmptyStringArrayDecoded, +} from '@kbn/securitysolution-io-ts-types'; +import { getSavedObjectTypes } from '@kbn/securitysolution-list-utils'; + +import type { ExceptionListSoSchema } from '../../schemas/saved_objects'; + +import { transformSavedObjectsToFoundExceptionListItem } from './utils'; +import { getExceptionList } from './get_exception_list'; +import { getExceptionListsItemFilter } from './utils/get_exception_lists_item_filter'; + +interface FindExceptionListItemsPointInTimeFinderOptions { + listId: NonEmptyStringArrayDecoded; + namespaceType: NamespaceTypeArray; + savedObjectsClient: SavedObjectsClientContract; + filter: EmptyStringArrayDecoded; + perPage: PerPageOrUndefined; + sortField: SortFieldOrUndefined; + sortOrder: SortOrderOrUndefined; + executeFunctionOnStream: (response: FoundExceptionListItemSchema) => void; + maxSize: MaxSizeOrUndefined; +} + +/** + * Finds exception list items within a point in time (PIT) and then calls the function + * `executeFunctionOnStream` until the maxPerPage is reached and stops. + * NOTE: This is slightly different from the saved objects version in that it takes + * an injected function, so that we avoid doing additional plumbing with generators + * to try to keep the maintenance of this machinery simpler for now. + * + * If you want to stream all results up to 10k into memory for correlation this would be: + * @example + * ```ts + * const exceptionList: ExceptionListItemSchema[] = []; + * const executeFunctionOnStream = (response: FoundExceptionListItemSchema) => { + * exceptionList = [...exceptionList, ...response.data]; + * } + * await client.findExceptionListsItemPointInTimeFinder({ + * filter, + * executeFunctionOnStream, + * namespaceType, + * maxSize: 10_000, // NOTE: This is unbounded if it is "undefined" + * perPage: 1_000, // See https://github.com/elastic/kibana/issues/93770 for choice of 1k + * sortField, + * sortOrder, + * exe + * }); + * ``` + * @param filter {string} Your filter + * @param namespaceType {string} "agnostic" | "single" of your namespace + * @param perPage {number} The number of items per page. Typical value should be 1_000 here. Never go above 10_000 + * @param maxSize {number of undefined} If given a max size, this will not exceeded. Otherwise if undefined is passed down, all records will be processed. + * @param sortField {string} String of the field to sort against + * @param savedObjectsClient {Object} The saved object client + * @param sortOrder "asc" | "desc" The order to sort against + */ +export const findExceptionListsItemPointInTimeFinder = async ({ + listId, + namespaceType, + savedObjectsClient, + executeFunctionOnStream, + maxSize, + filter, + perPage, + sortField, + sortOrder, +}: FindExceptionListItemsPointInTimeFinderOptions): Promise => { + const savedObjectType = getSavedObjectTypes({ namespaceType }); + const exceptionLists = ( + await Promise.all( + listId.map((singleListId, index) => { + return getExceptionList({ + id: undefined, + listId: singleListId, + namespaceType: namespaceType[index], + savedObjectsClient, + }); + }) + ) + ).filter((list) => list != null); + if (exceptionLists.length !== 0) { + const finder = savedObjectsClient.createPointInTimeFinder({ + filter: getExceptionListsItemFilter({ filter, listId, savedObjectType }), + perPage, + sortField, + sortOrder, + type: savedObjectType, + }); + + let count = 0; + for await (const savedObjectsFindResponse of finder.find()) { + count += savedObjectsFindResponse.saved_objects.length; + const exceptionListItem = transformSavedObjectsToFoundExceptionListItem({ + savedObjectsFindResponse, + }); + if (maxSize != null && count > maxSize) { + const diff = count - maxSize; + exceptionListItem.data = exceptionListItem.data.slice( + -exceptionListItem.data.length, + -diff + ); + executeFunctionOnStream(exceptionListItem); + try { + finder.close(); + } catch (exception) { + // This is just a pre-caution in case the finder does a throw we don't want to blow up + // the response. We have seen this within e2e test containers but nothing happen in normal + // operational conditions which is why this try/catch is here. + } + // early return since we are at our maxSize + return; + } else { + executeFunctionOnStream(exceptionListItem); + } + } + + try { + finder.close(); + } catch (exception) { + // This is just a pre-caution in case the finder does a throw we don't want to blow up + // the response. We have seen this within e2e test containers but nothing happen in normal + // operational conditions which is why this try/catch is here. + } + } +}; diff --git a/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_point_in_time_finder.ts b/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_point_in_time_finder.ts new file mode 100644 index 0000000000000..356735e773a01 --- /dev/null +++ b/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_point_in_time_finder.ts @@ -0,0 +1,118 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsClientContract } from 'kibana/server'; +import type { + FilterOrUndefined, + FoundExceptionListSchema, + MaxSizeOrUndefined, + NamespaceTypeArray, + PerPageOrUndefined, + SortFieldOrUndefined, + SortOrderOrUndefined, +} from '@kbn/securitysolution-io-ts-list-types'; +import { getSavedObjectTypes } from '@kbn/securitysolution-list-utils'; + +import type { ExceptionListSoSchema } from '../../schemas/saved_objects'; + +import { transformSavedObjectsToFoundExceptionList } from './utils'; +import { getExceptionListFilter } from './utils/get_exception_list_filter'; + +interface FindExceptionListPointInTimeFinderOptions { + namespaceType: NamespaceTypeArray; + savedObjectsClient: SavedObjectsClientContract; + filter: FilterOrUndefined; + perPage: PerPageOrUndefined; + sortField: SortFieldOrUndefined; + sortOrder: SortOrderOrUndefined; + executeFunctionOnStream: (response: FoundExceptionListSchema) => void; + maxSize: MaxSizeOrUndefined; +} + +/** + * Finds an exception list within a point in time (PIT) and then calls the function + * `executeFunctionOnStream` until the maxPerPage is reached and stops. + * NOTE: This is slightly different from the saved objects version in that it takes + * an injected function, so that we avoid doing additional plumbing with generators + * to try to keep the maintenance of this machinery simpler for now. + * + * If you want to stream all results up to 10k into memory for correlation this would be: + * @example + * ```ts + * const exceptionList: ExceptionListSchema[] = []; + * const executeFunctionOnStream = (response: FoundExceptionListSchema) => { + * exceptionList = [...exceptionList, ...response.data]; + * } + * await client.findExceptionListPointInTimeFinder({ + * filter, + * executeFunctionOnStream, + * namespaceType, + * maxSize: 10_000, // NOTE: This is unbounded if it is "undefined" + * perPage: 1_000, // See https://github.com/elastic/kibana/issues/93770 for choice of 1k + * sortField, + * sortOrder, + * exe + * }); + * ``` + * @param filter {string} Your filter + * @param namespaceType {string} "agnostic" | "single" of your namespace + * @param perPage {number} The number of items per page. Typical value should be 1_000 here. Never go above 10_000 + * @param maxSize {number of undefined} If given a max size, this will not exceeded. Otherwise if undefined is passed down, all records will be processed. + * @param sortField {string} String of the field to sort against + * @param sortOrder "asc" | "desc" The order to sort against + * @param savedObjectsClient The saved objects client + */ +export const findExceptionListPointInTimeFinder = async ({ + namespaceType, + savedObjectsClient, + executeFunctionOnStream, + maxSize, + filter, + perPage, + sortField, + sortOrder, +}: FindExceptionListPointInTimeFinderOptions): Promise => { + const savedObjectTypes = getSavedObjectTypes({ namespaceType }); + const finder = savedObjectsClient.createPointInTimeFinder({ + filter: getExceptionListFilter({ filter, savedObjectTypes }), + perPage, + sortField, + sortOrder, + type: savedObjectTypes, + }); + + let count = 0; + for await (const savedObjectsFindResponse of finder.find()) { + count += savedObjectsFindResponse.saved_objects.length; + const exceptionList = transformSavedObjectsToFoundExceptionList({ + savedObjectsFindResponse, + }); + if (maxSize != null && count > maxSize) { + const diff = count - maxSize; + exceptionList.data = exceptionList.data.slice(-exceptionList.data.length, -diff); + executeFunctionOnStream(exceptionList); + try { + finder.close(); + } catch (exception) { + // This is just a pre-caution in case the finder does a throw we don't want to blow up + // the response. We have seen this within e2e test containers but nothing happen in normal + // operational conditions which is why this try/catch is here. + } + // early return since we are at our maxSize + return; + } + executeFunctionOnStream(exceptionList); + } + + try { + finder.close(); + } catch (exception) { + // This is just a pre-caution in case the finder does a throw we don't want to blow up + // the response. We have seen this within e2e test containers but nothing happen in normal + // operational conditions which is why this try/catch is here. + } +}; diff --git a/x-pack/plugins/lists/server/services/exception_lists/find_value_list_exception_list_items.ts b/x-pack/plugins/lists/server/services/exception_lists/find_value_list_exception_list_items.ts new file mode 100644 index 0000000000000..711c8f9c253d1 --- /dev/null +++ b/x-pack/plugins/lists/server/services/exception_lists/find_value_list_exception_list_items.ts @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsClientContract } from 'kibana/server'; +import type { + FoundExceptionListItemSchema, + Id, + PageOrUndefined, + PerPageOrUndefined, + PitOrUndefined, + SearchAfterOrUndefined, + SortFieldOrUndefined, + SortOrderOrUndefined, +} from '@kbn/securitysolution-io-ts-list-types'; +import { + exceptionListAgnosticSavedObjectType, + exceptionListSavedObjectType, +} from '@kbn/securitysolution-list-utils'; + +import { escapeQuotes } from '../utils/escape_query'; +import type { ExceptionListSoSchema } from '../../schemas/saved_objects'; + +import { transformSavedObjectsToFoundExceptionListItem } from './utils'; + +interface FindValueListExceptionListsItems { + valueListId: Id; + savedObjectsClient: SavedObjectsClientContract; + perPage: PerPageOrUndefined; + pit: PitOrUndefined; + page: PageOrUndefined; + searchAfter: SearchAfterOrUndefined; + sortField: SortFieldOrUndefined; + sortOrder: SortOrderOrUndefined; +} + +export const findValueListExceptionListItems = async ({ + valueListId, + savedObjectsClient, + page, + pit, + perPage, + searchAfter, + sortField, + sortOrder, +}: FindValueListExceptionListsItems): Promise => { + const escapedValueListId = escapeQuotes(valueListId); + const savedObjectsFindResponse = await savedObjectsClient.find({ + filter: `(exception-list.attributes.list_type: item AND exception-list.attributes.entries.list.id:"${escapedValueListId}") OR (exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.entries.list.id:"${escapedValueListId}") `, + page, + perPage, + pit, + searchAfter, + sortField, + sortOrder, + type: [exceptionListSavedObjectType, exceptionListAgnosticSavedObjectType], + }); + return transformSavedObjectsToFoundExceptionListItem({ + savedObjectsFindResponse, + }); +}; diff --git a/x-pack/plugins/lists/server/services/exception_lists/find_value_list_exception_list_items_point_in_time_finder.ts b/x-pack/plugins/lists/server/services/exception_lists/find_value_list_exception_list_items_point_in_time_finder.ts new file mode 100644 index 0000000000000..e854bd6128746 --- /dev/null +++ b/x-pack/plugins/lists/server/services/exception_lists/find_value_list_exception_list_items_point_in_time_finder.ts @@ -0,0 +1,117 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsClientContract } from 'kibana/server'; +import type { + FoundExceptionListItemSchema, + Id, + MaxSizeOrUndefined, + PerPageOrUndefined, + SortFieldOrUndefined, + SortOrderOrUndefined, +} from '@kbn/securitysolution-io-ts-list-types'; +import { + exceptionListAgnosticSavedObjectType, + exceptionListSavedObjectType, +} from '@kbn/securitysolution-list-utils'; + +import { escapeQuotes } from '../utils/escape_query'; +import type { ExceptionListSoSchema } from '../../schemas/saved_objects'; + +import { transformSavedObjectsToFoundExceptionListItem } from './utils'; + +interface FindValueListExceptionListsItemsPointInTimeFinder { + valueListId: Id; + savedObjectsClient: SavedObjectsClientContract; + perPage: PerPageOrUndefined; + sortField: SortFieldOrUndefined; + sortOrder: SortOrderOrUndefined; + executeFunctionOnStream: (response: FoundExceptionListItemSchema) => void; + maxSize: MaxSizeOrUndefined; +} + +/** + * Finds value lists within exception lists within a point in time (PIT) and then calls the function + * `executeFunctionOnStream` until the maxPerPage is reached and stops. + * NOTE: This is slightly different from the saved objects version in that it takes + * an injected function, so that we avoid doing additional plumbing with generators + * to try to keep the maintenance of this machinery simpler for now. + * + * If you want to stream all results up to 10k into memory for correlation this would be: + * @example + * ```ts + * const exceptionList: ExceptionListItemSchema[] = []; + * const executeFunctionOnStream = (response: FoundExceptionListItemSchema) => { + * exceptionList = [...exceptionList, ...response.data]; + * } + * await client.findValueListExceptionListItemsPointInTimeFinder({ + * valueListId, + * executeFunctionOnStream, + * namespaceType, + * maxSize: 10_000, // NOTE: This is unbounded if it is "undefined" + * perPage: 1_000, // See https://github.com/elastic/kibana/issues/93770 for choice of 1k + * sortField, + * sortOrder, + * exe + * }); + * ``` + * @param valueListId {string} Your value list id + * @param namespaceType {string} "agnostic" | "single" of your namespace + * @param perPage {number} The number of items per page. Typical value should be 1_000 here. Never go above 10_000 + * @param maxSize {number of undefined} If given a max size, this will not exceeded. Otherwise if undefined is passed down, all records will be processed. + * @param sortField {string} String of the field to sort against + * @param savedObjectsClient {object} The saved objects client + * @param sortOrder "asc" | "desc" The order to sort against + */ +export const findValueListExceptionListItemsPointInTimeFinder = async ({ + valueListId, + executeFunctionOnStream, + savedObjectsClient, + perPage, + maxSize, + sortField, + sortOrder, +}: FindValueListExceptionListsItemsPointInTimeFinder): Promise => { + const escapedValueListId = escapeQuotes(valueListId); + const finder = savedObjectsClient.createPointInTimeFinder({ + filter: `(exception-list.attributes.list_type: item AND exception-list.attributes.entries.list.id:"${escapedValueListId}") OR (exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.entries.list.id:"${escapedValueListId}") `, + perPage, + sortField, + sortOrder, + type: [exceptionListSavedObjectType, exceptionListAgnosticSavedObjectType], + }); + let count = 0; + for await (const savedObjectsFindResponse of finder.find()) { + count += savedObjectsFindResponse.saved_objects.length; + const exceptionList = transformSavedObjectsToFoundExceptionListItem({ + savedObjectsFindResponse, + }); + if (maxSize != null && count > maxSize) { + const diff = count - maxSize; + exceptionList.data = exceptionList.data.slice(-exceptionList.data.length, -diff); + executeFunctionOnStream(exceptionList); + try { + finder.close(); + } catch (exception) { + // This is just a pre-caution in case the finder does a throw we don't want to blow up + // the response. We have seen this within e2e test containers but nothing happen in normal + // operational conditions which is why this try/catch is here. + } + // early return since we are at our maxSize + return; + } + executeFunctionOnStream(exceptionList); + } + + try { + finder.close(); + } catch (exception) { + // This is just a pre-caution in case the finder does a throw we don't want to blow up + // the response. We have seen this within e2e test containers but nothing happen in normal + // operational conditions which is why this try/catch is here. + } +}; diff --git a/x-pack/plugins/lists/server/services/exception_lists/open_point_in_time.ts b/x-pack/plugins/lists/server/services/exception_lists/open_point_in_time.ts new file mode 100644 index 0000000000000..271676527dd2f --- /dev/null +++ b/x-pack/plugins/lists/server/services/exception_lists/open_point_in_time.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + SavedObjectsClientContract, + SavedObjectsOpenPointInTimeOptions, + SavedObjectsOpenPointInTimeResponse, +} from 'kibana/server'; +import type { NamespaceType } from '@kbn/securitysolution-io-ts-list-types'; +import { getSavedObjectTypes } from '@kbn/securitysolution-list-utils'; + +interface OpenPointInTimeOptions { + namespaceType: NamespaceType; + savedObjectsClient: SavedObjectsClientContract; + options: SavedObjectsOpenPointInTimeOptions | undefined; +} + +/** + * Opens a point in time (PIT) for either exception lists or exception list items. + * See: https://www.elastic.co/guide/en/elasticsearch/reference/current/point-in-time-api.html + * @params namespaceType {string} "agnostic" or "single" depending on which namespace you are targeting + * @params savedObjectsClient {object} The saved object client to delegate to + * @params options {Object} The saved object PIT options + * @return {SavedObjectsOpenPointInTimeResponse} The point in time (PIT) + */ +export const openPointInTime = async ({ + namespaceType, + savedObjectsClient, + options, +}: OpenPointInTimeOptions): Promise => { + const savedObjectType = getSavedObjectTypes({ namespaceType: [namespaceType] }); + return savedObjectsClient.openPointInTimeForType(savedObjectType, options); +}; diff --git a/x-pack/plugins/lists/server/services/exception_lists/utils/get_exception_list_filter.test.ts b/x-pack/plugins/lists/server/services/exception_lists/utils/get_exception_list_filter.test.ts new file mode 100644 index 0000000000000..9e0b06f8482c7 --- /dev/null +++ b/x-pack/plugins/lists/server/services/exception_lists/utils/get_exception_list_filter.test.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getExceptionListFilter } from './get_exception_list_filter'; + +describe('getExceptionListFilter', () => { + test('it should create a filter for agnostic lists if only searching for agnostic lists', () => { + const filter = getExceptionListFilter({ + filter: undefined, + savedObjectTypes: ['exception-list-agnostic'], + }); + expect(filter).toEqual('(exception-list-agnostic.attributes.list_type: list)'); + }); + + test('it should create a filter for agnostic lists with additional filters if only searching for agnostic lists', () => { + const filter = getExceptionListFilter({ + filter: 'exception-list-agnostic.attributes.name: "Sample Endpoint Exception List"', + savedObjectTypes: ['exception-list-agnostic'], + }); + expect(filter).toEqual( + '(exception-list-agnostic.attributes.list_type: list) AND exception-list-agnostic.attributes.name: "Sample Endpoint Exception List"' + ); + }); + + test('it should create a filter for single lists if only searching for single lists', () => { + const filter = getExceptionListFilter({ + filter: undefined, + savedObjectTypes: ['exception-list'], + }); + expect(filter).toEqual('(exception-list.attributes.list_type: list)'); + }); + + test('it should create a filter for single lists with additional filters if only searching for single lists', () => { + const filter = getExceptionListFilter({ + filter: 'exception-list.attributes.name: "Sample Endpoint Exception List"', + savedObjectTypes: ['exception-list'], + }); + expect(filter).toEqual( + '(exception-list.attributes.list_type: list) AND exception-list.attributes.name: "Sample Endpoint Exception List"' + ); + }); + + test('it should create a filter that searches for both agnostic and single lists', () => { + const filter = getExceptionListFilter({ + filter: undefined, + savedObjectTypes: ['exception-list-agnostic', 'exception-list'], + }); + expect(filter).toEqual( + '(exception-list-agnostic.attributes.list_type: list OR exception-list.attributes.list_type: list)' + ); + }); + + test('it should create a filter that searches for both agnostic and single lists with additional filters if searching for both single and agnostic lists', () => { + const filter = getExceptionListFilter({ + filter: 'exception-list-agnostic.attributes.name: "Sample Endpoint Exception List"', + savedObjectTypes: ['exception-list-agnostic', 'exception-list'], + }); + expect(filter).toEqual( + '(exception-list-agnostic.attributes.list_type: list OR exception-list.attributes.list_type: list) AND exception-list-agnostic.attributes.name: "Sample Endpoint Exception List"' + ); + }); +}); diff --git a/x-pack/plugins/lists/server/services/exception_lists/utils/get_exception_list_filter.ts b/x-pack/plugins/lists/server/services/exception_lists/utils/get_exception_list_filter.ts new file mode 100644 index 0000000000000..44a9be320755f --- /dev/null +++ b/x-pack/plugins/lists/server/services/exception_lists/utils/get_exception_list_filter.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { FilterOrUndefined } from '@kbn/securitysolution-io-ts-list-types'; +import type { SavedObjectType } from '@kbn/securitysolution-list-utils'; + +export const getExceptionListFilter = ({ + filter, + savedObjectTypes, +}: { + filter: FilterOrUndefined; + savedObjectTypes: SavedObjectType[]; +}): string => { + const listTypesFilter = savedObjectTypes + .map((type) => `${type}.attributes.list_type: list`) + .join(' OR '); + + if (filter != null) { + return `(${listTypesFilter}) AND ${filter}`; + } else return `(${listTypesFilter})`; +}; diff --git a/x-pack/plugins/lists/server/services/exception_lists/utils/get_exception_lists_item_filter.test.ts b/x-pack/plugins/lists/server/services/exception_lists/utils/get_exception_lists_item_filter.test.ts new file mode 100644 index 0000000000000..8e8b3499338dc --- /dev/null +++ b/x-pack/plugins/lists/server/services/exception_lists/utils/get_exception_lists_item_filter.test.ts @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { LIST_ID } from '../../../../common/constants.mock'; + +import { getExceptionListsItemFilter } from './get_exception_lists_item_filter'; + +describe('getExceptionListsItemFilter', () => { + test('It should create a filter with a single listId with an empty filter', () => { + const filter = getExceptionListsItemFilter({ + filter: [], + listId: [LIST_ID], + savedObjectType: ['exception-list'], + }); + expect(filter).toEqual( + '(exception-list.attributes.list_type: item AND exception-list.attributes.list_id: "some-list-id")' + ); + }); + + test('It should create a filter escaping quotes in list ids', () => { + const filter = getExceptionListsItemFilter({ + filter: [], + listId: ['list-id-"-with-quote'], + savedObjectType: ['exception-list'], + }); + expect(filter).toEqual( + '(exception-list.attributes.list_type: item AND exception-list.attributes.list_id: "list-id-\\"-with-quote")' + ); + }); + + test('It should create a filter with a single listId with a single filter', () => { + const filter = getExceptionListsItemFilter({ + filter: ['exception-list.attributes.name: "Sample Endpoint Exception List"'], + listId: [LIST_ID], + savedObjectType: ['exception-list'], + }); + expect(filter).toEqual( + '((exception-list.attributes.list_type: item AND exception-list.attributes.list_id: "some-list-id") AND exception-list.attributes.name: "Sample Endpoint Exception List")' + ); + }); + + test('It should create a filter with 2 listIds and an empty filter', () => { + const filter = getExceptionListsItemFilter({ + filter: [], + listId: ['list-1', 'list-2'], + savedObjectType: ['exception-list', 'exception-list-agnostic'], + }); + expect(filter).toEqual( + '(exception-list.attributes.list_type: item AND exception-list.attributes.list_id: "list-1") OR (exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.list_id: "list-2")' + ); + }); + + test('It should create a filter with 2 listIds and a single filter', () => { + const filter = getExceptionListsItemFilter({ + filter: ['exception-list.attributes.name: "Sample Endpoint Exception List"'], + listId: ['list-1', 'list-2'], + savedObjectType: ['exception-list', 'exception-list-agnostic'], + }); + expect(filter).toEqual( + '((exception-list.attributes.list_type: item AND exception-list.attributes.list_id: "list-1") AND exception-list.attributes.name: "Sample Endpoint Exception List") OR (exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.list_id: "list-2")' + ); + }); + + test('It should create a filter with 3 listIds and an empty filter', () => { + const filter = getExceptionListsItemFilter({ + filter: [], + listId: ['list-1', 'list-2', 'list-3'], + savedObjectType: ['exception-list', 'exception-list-agnostic', 'exception-list-agnostic'], + }); + expect(filter).toEqual( + '(exception-list.attributes.list_type: item AND exception-list.attributes.list_id: "list-1") OR (exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.list_id: "list-2") OR (exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.list_id: "list-3")' + ); + }); + + test('It should create a filter with 3 listIds and a single filter for the first item', () => { + const filter = getExceptionListsItemFilter({ + filter: ['exception-list.attributes.name: "Sample Endpoint Exception List"'], + listId: ['list-1', 'list-2', 'list-3'], + savedObjectType: ['exception-list', 'exception-list-agnostic', 'exception-list-agnostic'], + }); + expect(filter).toEqual( + '((exception-list.attributes.list_type: item AND exception-list.attributes.list_id: "list-1") AND exception-list.attributes.name: "Sample Endpoint Exception List") OR (exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.list_id: "list-2") OR (exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.list_id: "list-3")' + ); + }); + + test('It should create a filter with 3 listIds and 3 filters for each', () => { + const filter = getExceptionListsItemFilter({ + filter: [ + 'exception-list.attributes.name: "Sample Endpoint Exception List 1"', + 'exception-list.attributes.name: "Sample Endpoint Exception List 2"', + 'exception-list.attributes.name: "Sample Endpoint Exception List 3"', + ], + listId: ['list-1', 'list-2', 'list-3'], + savedObjectType: ['exception-list', 'exception-list-agnostic', 'exception-list-agnostic'], + }); + expect(filter).toEqual( + '((exception-list.attributes.list_type: item AND exception-list.attributes.list_id: "list-1") AND exception-list.attributes.name: "Sample Endpoint Exception List 1") OR ((exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.list_id: "list-2") AND exception-list.attributes.name: "Sample Endpoint Exception List 2") OR ((exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.list_id: "list-3") AND exception-list.attributes.name: "Sample Endpoint Exception List 3")' + ); + }); +}); diff --git a/x-pack/plugins/lists/server/services/exception_lists/utils/get_exception_lists_item_filter.ts b/x-pack/plugins/lists/server/services/exception_lists/utils/get_exception_lists_item_filter.ts new file mode 100644 index 0000000000000..935ae8839a71d --- /dev/null +++ b/x-pack/plugins/lists/server/services/exception_lists/utils/get_exception_lists_item_filter.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + EmptyStringArrayDecoded, + NonEmptyStringArrayDecoded, +} from '@kbn/securitysolution-io-ts-types'; +import type { SavedObjectType } from '@kbn/securitysolution-list-utils'; + +import { escapeQuotes } from '../../utils/escape_query'; + +export const getExceptionListsItemFilter = ({ + filter, + listId, + savedObjectType, +}: { + listId: NonEmptyStringArrayDecoded; + filter: EmptyStringArrayDecoded; + savedObjectType: SavedObjectType[]; +}): string => { + return listId.reduce((accum, singleListId, index) => { + const escapedListId = escapeQuotes(singleListId); + const listItemAppend = `(${savedObjectType[index]}.attributes.list_type: item AND ${savedObjectType[index]}.attributes.list_id: "${escapedListId}")`; + const listItemAppendWithFilter = + filter[index] != null ? `(${listItemAppend} AND ${filter[index]})` : listItemAppend; + if (accum === '') { + return listItemAppendWithFilter; + } else { + return `${accum} OR ${listItemAppendWithFilter}`; + } + }, ''); +}; diff --git a/x-pack/plugins/lists/server/services/exception_lists/utils/import/find_all_exception_list_item_types.ts b/x-pack/plugins/lists/server/services/exception_lists/utils/import/find_all_exception_list_item_types.ts index 272c64f161c9c..3884d12b4bb6f 100644 --- a/x-pack/plugins/lists/server/services/exception_lists/utils/import/find_all_exception_list_item_types.ts +++ b/x-pack/plugins/lists/server/services/exception_lists/utils/import/find_all_exception_list_item_types.ts @@ -14,7 +14,7 @@ import { getSavedObjectTypes } from '@kbn/securitysolution-list-utils'; import { SavedObjectsClientContract, SavedObjectsFindResponse } from 'kibana/server'; import { ExceptionListSoSchema } from '../../../../schemas/saved_objects'; -import { getExceptionListsItemFilter } from '../../find_exception_list_items'; +import { getExceptionListsItemFilter } from '../get_exception_lists_item_filter'; import { CHUNK_PARSED_OBJECT_SIZE } from '../../import_exception_list_and_items'; import { transformSavedObjectsToFoundExceptionListItem } from '..'; diff --git a/x-pack/plugins/lists/server/services/exception_lists/utils/import/find_all_exception_list_types.ts b/x-pack/plugins/lists/server/services/exception_lists/utils/import/find_all_exception_list_types.ts index 4b42787d8aaf9..51538faa66942 100644 --- a/x-pack/plugins/lists/server/services/exception_lists/utils/import/find_all_exception_list_types.ts +++ b/x-pack/plugins/lists/server/services/exception_lists/utils/import/find_all_exception_list_types.ts @@ -72,7 +72,9 @@ export const findAllListTypes = async ( namespaceType: ['agnostic'], page: undefined, perPage: CHUNK_PARSED_OBJECT_SIZE, + pit: undefined, savedObjectsClient, + searchAfter: undefined, sortField: undefined, sortOrder: undefined, }); @@ -82,7 +84,9 @@ export const findAllListTypes = async ( namespaceType: ['single'], page: undefined, perPage: CHUNK_PARSED_OBJECT_SIZE, + pit: undefined, savedObjectsClient, + searchAfter: undefined, sortField: undefined, sortOrder: undefined, }); @@ -92,7 +96,9 @@ export const findAllListTypes = async ( namespaceType: ['single', 'agnostic'], page: undefined, perPage: CHUNK_PARSED_OBJECT_SIZE, + pit: undefined, savedObjectsClient, + searchAfter: undefined, sortField: undefined, sortOrder: undefined, }); diff --git a/x-pack/plugins/lists/server/services/exception_lists/utils/index.ts b/x-pack/plugins/lists/server/services/exception_lists/utils/index.ts index ae1883f5767e5..b0b0151e5f537 100644 --- a/x-pack/plugins/lists/server/services/exception_lists/utils/index.ts +++ b/x-pack/plugins/lists/server/services/exception_lists/utils/index.ts @@ -247,6 +247,7 @@ export const transformSavedObjectsToFoundExceptionListItem = ({ ), page: savedObjectsFindResponse.page, per_page: savedObjectsFindResponse.per_page, + pit: savedObjectsFindResponse.pit_id, total: savedObjectsFindResponse.total, }; }; @@ -262,6 +263,7 @@ export const transformSavedObjectsToFoundExceptionList = ({ ), page: savedObjectsFindResponse.page, per_page: savedObjectsFindResponse.per_page, + pit: savedObjectsFindResponse.pit_id, total: savedObjectsFindResponse.total, }; }; diff --git a/x-pack/plugins/lists/server/services/items/get_list_item_by_values.ts b/x-pack/plugins/lists/server/services/items/get_list_item_by_values.ts index 2fb713526fce8..b0cdc77724ac7 100644 --- a/x-pack/plugins/lists/server/services/items/get_list_item_by_values.ts +++ b/x-pack/plugins/lists/server/services/items/get_list_item_by_values.ts @@ -30,6 +30,9 @@ export const getListItemByValues = async ({ type, value, }: GetListItemByValuesOptions): Promise => { + // TODO: Will need to address this when we switch over to + // using PIT, don't want it to get lost + // https://github.com/elastic/kibana/issues/103944 const response = await esClient.search({ body: { query: { diff --git a/x-pack/plugins/lists/server/services/items/search_list_item_by_values.ts b/x-pack/plugins/lists/server/services/items/search_list_item_by_values.ts index fb81594137861..2a39f863311b9 100644 --- a/x-pack/plugins/lists/server/services/items/search_list_item_by_values.ts +++ b/x-pack/plugins/lists/server/services/items/search_list_item_by_values.ts @@ -30,6 +30,9 @@ export const searchListItemByValues = async ({ type, value, }: SearchListItemByValuesOptions): Promise => { + // TODO: Will need to address this when we switch over to + // using PIT, don't want it to get lost + // https://github.com/elastic/kibana/issues/103944 const response = await esClient.search({ body: { query: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts index a1b0548a81d25..c57a446207873 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts @@ -7,7 +7,7 @@ import moment from 'moment'; import sinon from 'sinon'; -import { TransportResult } from '@elastic/elasticsearch'; +import type { TransportResult } from '@elastic/elasticsearch'; import { ALERT_REASON, ALERT_RULE_PARAMETERS, ALERT_UUID } from '@kbn/rule-data-utils'; import { alertsMock, AlertServicesMock } from '../../../../../alerting/server/mocks'; @@ -46,7 +46,7 @@ import { isRACAlert, getField, } from './utils'; -import { BulkResponseErrorAggregation, SearchAfterAndBulkCreateReturnType } from './types'; +import type { BulkResponseErrorAggregation, SearchAfterAndBulkCreateReturnType } from './types'; import { sampleBulkResponse, sampleEmptyBulkResponse, @@ -62,7 +62,7 @@ import { sampleAlertDocNoSortIdWithTimestamp, sampleAlertDocAADNoSortIdWithTimestamp, } from './__mocks__/es_results'; -import { ShardError } from '../../types'; +import type { ShardError } from '../../types'; import { ruleExecutionLogMock } from '../rule_execution_log/__mocks__'; const buildRuleMessage = buildRuleMessageFactory({ @@ -568,12 +568,11 @@ describe('utils', () => { test('it successfully returns array of exception list items', async () => { listMock.getExceptionListClient = () => ({ - findExceptionListsItem: jest.fn().mockResolvedValue({ - data: [getExceptionListItemSchemaMock()], - page: 1, - per_page: 10000, - total: 1, - }), + findExceptionListsItemPointInTimeFinder: jest + .fn() + .mockImplementationOnce(({ executeFunctionOnStream }) => { + executeFunctionOnStream({ data: [getExceptionListItemSchemaMock()] }); + }), } as unknown as ExceptionListClient); const client = listMock.getExceptionListClient(); const exceptions = await getExceptions({ @@ -581,38 +580,25 @@ describe('utils', () => { lists: getListArrayMock(), }); - expect(client.findExceptionListsItem).toHaveBeenCalledWith({ - listId: ['list_id_single', 'endpoint_list'], - namespaceType: ['single', 'agnostic'], - page: 1, - perPage: 10000, - filter: [], - sortOrder: undefined, - sortField: undefined, - }); - expect(exceptions).toEqual([getExceptionListItemSchemaMock()]); - }); - - test('it throws if "getExceptionListClient" fails', async () => { - const err = new Error('error fetching list'); - listMock.getExceptionListClient = () => - ({ - getExceptionList: jest.fn().mockRejectedValue(err), - } as unknown as ExceptionListClient); - - await expect(() => - getExceptions({ - client: listMock.getExceptionListClient(), - lists: getListArrayMock(), + expect(client.findExceptionListsItemPointInTimeFinder).toHaveBeenCalledWith( + expect.objectContaining({ + listId: ['list_id_single', 'endpoint_list'], + namespaceType: ['single', 'agnostic'], + perPage: 1_000, + filter: [], + maxSize: undefined, + sortOrder: undefined, + sortField: undefined, }) - ).rejects.toThrowError('unable to fetch exception list items'); + ); + expect(exceptions).toEqual([getExceptionListItemSchemaMock()]); }); - test('it throws if "findExceptionListsItem" fails', async () => { + test('it throws if "findExceptionListsItemPointInTimeFinder" fails anywhere', async () => { const err = new Error('error fetching list'); listMock.getExceptionListClient = () => ({ - findExceptionListsItem: jest.fn().mockRejectedValue(err), + findExceptionListsItemPointInTimeFinder: jest.fn().mockRejectedValue(err), } as unknown as ExceptionListClient); await expect(() => @@ -620,7 +606,9 @@ describe('utils', () => { client: listMock.getExceptionListClient(), lists: getListArrayMock(), }) - ).rejects.toThrowError('unable to fetch exception list items'); + ).rejects.toThrowError( + 'unable to fetch exception list items, message: "error fetching list" full error: "Error: error fetching list"' + ); }); test('it returns empty array if "findExceptionListsItem" returns null', async () => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index 31e43c694084a..cb1db88f78d31 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -11,10 +11,13 @@ import uuidv5 from 'uuid/v5'; import dateMath from '@elastic/datemath'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { TransportResult } from '@elastic/elasticsearch'; +import type { TransportResult } from '@elastic/elasticsearch'; import { ALERT_UUID, ALERT_RULE_UUID, ALERT_RULE_PARAMETERS } from '@kbn/rule-data-utils'; -import type { ListArray, ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types'; -import { MAX_EXCEPTION_LIST_SIZE } from '@kbn/securitysolution-list-constants'; +import type { + ListArray, + ExceptionListItemSchema, + FoundExceptionListItemSchema, +} from '@kbn/securitysolution-io-ts-list-types'; import { hasLargeValueList } from '@kbn/securitysolution-list-utils'; import { @@ -22,7 +25,7 @@ import { Privilege, RuleExecutionStatus, } from '../../../../common/detection_engine/schemas/common'; -import { +import type { ElasticsearchClient, Logger, SavedObjectsClientContract, @@ -33,8 +36,8 @@ import { AlertServices, parseDuration, } from '../../../../../alerting/server'; -import { ExceptionListClient, ListClient, ListPluginSetup } from '../../../../../lists/server'; -import { +import type { ExceptionListClient, ListClient, ListPluginSetup } from '../../../../../lists/server'; +import type { BulkResponseErrorAggregation, SignalHit, SearchAfterAndBulkCreateReturnType, @@ -47,9 +50,9 @@ import { SimpleHit, WrappedEventHit, } from './types'; -import { BuildRuleMessage } from './rule_messages'; -import { ShardError } from '../../types'; -import { +import type { BuildRuleMessage } from './rule_messages'; +import type { ShardError } from '../../types'; +import type { EqlRuleParams, MachineLearningRuleParams, QueryRuleParams, @@ -58,9 +61,9 @@ import { ThreatRuleParams, ThresholdRuleParams, } from '../schemas/rule_schemas'; -import { RACAlert, WrappedRACAlert } from '../rule_types/types'; -import { SearchTypes } from '../../../../common/detection_engine/types'; -import { IRuleExecutionLogForExecutors } from '../rule_execution_log'; +import type { RACAlert, WrappedRACAlert } from '../rule_types/types'; +import type { SearchTypes } from '../../../../common/detection_engine/types'; +import type { IRuleExecutionLogForExecutors } from '../rule_execution_log'; import { withSecuritySpan } from '../../../utils/with_security_span'; interface SortExceptionsReturn { @@ -269,18 +272,28 @@ export const getExceptions = async ({ try { const listIds = lists.map(({ list_id: listId }) => listId); const namespaceTypes = lists.map(({ namespace_type: namespaceType }) => namespaceType); - const items = await client.findExceptionListsItem({ + + // Stream the results from the Point In Time (PIT) finder into this array + let items: ExceptionListItemSchema[] = []; + const executeFunctionOnStream = (response: FoundExceptionListItemSchema): void => { + items = [...items, ...response.data]; + }; + + await client.findExceptionListsItemPointInTimeFinder({ + executeFunctionOnStream, listId: listIds, namespaceType: namespaceTypes, - page: 1, - perPage: MAX_EXCEPTION_LIST_SIZE, + perPage: 1_000, // See https://github.com/elastic/kibana/issues/93770 for choice of 1k filter: [], + maxSize: undefined, // NOTE: This is unbounded when it is "undefined" sortOrder: undefined, sortField: undefined, }); - return items != null ? items.data : []; - } catch { - throw new Error('unable to fetch exception list items'); + return items; + } catch (e) { + throw new Error( + `unable to fetch exception list items, message: "${e.message}" full error: "${e}"` + ); } } else { return [];