diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/common/index.ts b/packages/kbn-securitysolution-io-ts-list-types/src/common/index.ts index 81ecd58cb397c..98c160e2c4302 100644 --- a/packages/kbn-securitysolution-io-ts-list-types/src/common/index.ts +++ b/packages/kbn-securitysolution-io-ts-list-types/src/common/index.ts @@ -35,6 +35,7 @@ export * from './list_operator'; export * from './list_type'; export * from './lists'; export * from './lists_default_array'; +export * from './max_size'; export * from './meta'; export * from './name'; export * from './non_empty_entries_array'; @@ -42,6 +43,8 @@ export * from './non_empty_nested_entries_array'; export * from './os_type'; export * from './page'; export * from './per_page'; +export * from './pit'; +export * from './search_after'; export * from './serializer'; export * from './sort_field'; export * from './sort_order'; diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/common/max_size/index.test.ts b/packages/kbn-securitysolution-io-ts-list-types/src/common/max_size/index.test.ts new file mode 100644 index 0000000000000..459195e3ec27f --- /dev/null +++ b/packages/kbn-securitysolution-io-ts-list-types/src/common/max_size/index.test.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { exactCheck } from '@kbn/securitysolution-io-ts-utils'; +import { maxSizeOrUndefined } from '.'; + +import { pipe } from 'fp-ts/lib/pipeable'; +import { left } from 'fp-ts/lib/Either'; + +import { foldLeftRight, getPaths } from '@kbn/securitysolution-io-ts-utils'; + +describe('maxSizeOrUndefined', () => { + test('it will validate a correct max value', () => { + const payload = 123; + const decoded = maxSizeOrUndefined.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it will fail to validate a 0', () => { + const payload = 0; + const decoded = maxSizeOrUndefined.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "0" supplied to "(PositiveIntegerGreaterThanZero | undefined)"', + ]); + expect(message.schema).toEqual({}); + }); + + test('it will fail to validate a -1', () => { + const payload = -1; + const decoded = maxSizeOrUndefined.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "-1" supplied to "(PositiveIntegerGreaterThanZero | undefined)"', + ]); + expect(message.schema).toEqual({}); + }); + + test('it will fail to validate a string', () => { + const payload = '123'; + const decoded = maxSizeOrUndefined.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "123" supplied to "(PositiveIntegerGreaterThanZero | undefined)"', + ]); + expect(message.schema).toEqual({}); + }); +}); diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/common/max_size/index.ts b/packages/kbn-securitysolution-io-ts-list-types/src/common/max_size/index.ts new file mode 100644 index 0000000000000..59ae5b7b7fc63 --- /dev/null +++ b/packages/kbn-securitysolution-io-ts-list-types/src/common/max_size/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +/* eslint-disable @typescript-eslint/naming-convention */ + +import { PositiveIntegerGreaterThanZero } from '@kbn/securitysolution-io-ts-types'; +import * as t from 'io-ts'; + +export const max_size = PositiveIntegerGreaterThanZero; +export type MaxSize = t.TypeOf; + +export const maxSizeOrUndefined = t.union([max_size, t.undefined]); +export type MaxSizeOrUndefined = t.TypeOf; diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/common/pit/index.test.ts b/packages/kbn-securitysolution-io-ts-list-types/src/common/pit/index.test.ts new file mode 100644 index 0000000000000..19aeb0690f13e --- /dev/null +++ b/packages/kbn-securitysolution-io-ts-list-types/src/common/pit/index.test.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { exactCheck } from '@kbn/securitysolution-io-ts-utils'; +import { pitOrUndefined } from '.'; + +import * as t from 'io-ts'; +import { pipe } from 'fp-ts/lib/pipeable'; +import { left } from 'fp-ts/lib/Either'; + +import { foldLeftRight, getPaths } from '@kbn/securitysolution-io-ts-utils'; + +describe('pitOrUndefined', () => { + test('it will validate a correct pit', () => { + const payload = { id: '123', keepAlive: '1m' }; + const decoded = pitOrUndefined.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it will validate with the value of "undefined"', () => { + const obj = t.exact( + t.type({ + pit_id: pitOrUndefined, + }) + ); + const payload: t.TypeOf = { + pit_id: undefined, + }; + const decoded = obj.decode({ + pit_id: undefined, + }); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it will validate a correct pit without having a "keepAlive"', () => { + const payload = { id: '123' }; + const decoded = pitOrUndefined.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it will fail to validate an incorrect pit', () => { + const payload = 'foo'; + const decoded = pitOrUndefined.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "foo" supplied to "({| id: string, keepAlive: (string | undefined) |} | undefined)"', + ]); + expect(message.schema).toEqual({}); + }); +}); diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/common/pit/index.ts b/packages/kbn-securitysolution-io-ts-list-types/src/common/pit/index.ts new file mode 100644 index 0000000000000..773794edaf1f6 --- /dev/null +++ b/packages/kbn-securitysolution-io-ts-list-types/src/common/pit/index.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as t from 'io-ts'; + +export const pitId = t.string; +export const pit = t.exact( + t.type({ + id: pitId, + keepAlive: t.union([t.string, t.undefined]), + }) +); +export const pitOrUndefined = t.union([pit, t.undefined]); + +export type Pit = t.TypeOf; +export type PitId = t.TypeOf; +export type PitOrUndefined = t.TypeOf; diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/common/search_after/index.test.ts b/packages/kbn-securitysolution-io-ts-list-types/src/common/search_after/index.test.ts new file mode 100644 index 0000000000000..135aa53d39783 --- /dev/null +++ b/packages/kbn-securitysolution-io-ts-list-types/src/common/search_after/index.test.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { exactCheck } from '@kbn/securitysolution-io-ts-utils'; +import { searchAfterOrUndefined } from '.'; + +import * as t from 'io-ts'; +import { pipe } from 'fp-ts/lib/pipeable'; +import { left } from 'fp-ts/lib/Either'; + +import { foldLeftRight, getPaths } from '@kbn/securitysolution-io-ts-utils'; + +describe('searchAfter', () => { + test('it will validate a correct search_after', () => { + const payload = ['test-1', 'test-2']; + const decoded = searchAfterOrUndefined.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it will validate with the value of "undefined"', () => { + const obj = t.exact( + t.type({ + search_after: searchAfterOrUndefined, + }) + ); + const payload: t.TypeOf = { + search_after: undefined, + }; + const decoded = obj.decode({ + pit_id: undefined, + }); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(payload); + }); + + test('it will fail to validate an incorrect search_after', () => { + const payload = 'foo'; + const decoded = searchAfterOrUndefined.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "foo" supplied to "(Array | undefined)"', + ]); + expect(message.schema).toEqual({}); + }); +}); diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/common/search_after/index.ts b/packages/kbn-securitysolution-io-ts-list-types/src/common/search_after/index.ts new file mode 100644 index 0000000000000..ef39716e5bcac --- /dev/null +++ b/packages/kbn-securitysolution-io-ts-list-types/src/common/search_after/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +/* eslint-disable @typescript-eslint/naming-convention */ + +import * as t from 'io-ts'; + +export const search_after = t.array(t.string); +export type SearchAfter = t.TypeOf; + +export const searchAfterOrUndefined = t.union([search_after, t.undefined]); +export type SearchAfterOrUndefined = t.TypeOf; diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/response/found_exception_list_item_schema/index.ts b/packages/kbn-securitysolution-io-ts-list-types/src/response/found_exception_list_item_schema/index.ts index df82a70ef626c..587c39c385f91 100644 --- a/packages/kbn-securitysolution-io-ts-list-types/src/response/found_exception_list_item_schema/index.ts +++ b/packages/kbn-securitysolution-io-ts-list-types/src/response/found_exception_list_item_schema/index.ts @@ -10,16 +10,24 @@ import * as t from 'io-ts'; import { page } from '../../common/page'; import { per_page } from '../../common/per_page'; +import { pitId } from '../../common/pit'; import { total } from '../../common/total'; import { exceptionListItemSchema } from '../exception_list_item_schema'; -export const foundExceptionListItemSchema = t.exact( - t.type({ - data: t.array(exceptionListItemSchema), - page, - per_page, - total, - }) -); +export const foundExceptionListItemSchema = t.intersection([ + t.exact( + t.type({ + data: t.array(exceptionListItemSchema), + page, + per_page, + total, + }) + ), + t.exact( + t.partial({ + pit: pitId, + }) + ), +]); export type FoundExceptionListItemSchema = t.TypeOf; diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/response/found_exception_list_schema/index.ts b/packages/kbn-securitysolution-io-ts-list-types/src/response/found_exception_list_schema/index.ts index 4e430f607fb04..07b4090af9425 100644 --- a/packages/kbn-securitysolution-io-ts-list-types/src/response/found_exception_list_schema/index.ts +++ b/packages/kbn-securitysolution-io-ts-list-types/src/response/found_exception_list_schema/index.ts @@ -10,17 +10,21 @@ import * as t from 'io-ts'; import { page } from '../../common/page'; import { per_page } from '../../common/per_page'; +import { pitId } from '../../common/pit'; import { total } from '../../common/total'; import { exceptionListSchema } from '../exception_list_schema'; -export const foundExceptionListSchema = t.exact( - t.type({ - data: t.array(exceptionListSchema), - page, - per_page, - total, - }) -); +export const foundExceptionListSchema = t.intersection([ + t.exact( + t.type({ + data: t.array(exceptionListSchema), + page, + per_page, + total, + }) + ), + t.exact(t.partial({ pit: pitId })), +]); export type FoundExceptionListSchema = t.TypeOf; diff --git a/x-pack/plugins/lists/server/routes/delete_list_route.ts b/x-pack/plugins/lists/server/routes/delete_list_route.ts index 35ac490826703..6ddcce94d82e8 100644 --- a/x-pack/plugins/lists/server/routes/delete_list_route.ts +++ b/x-pack/plugins/lists/server/routes/delete_list_route.ts @@ -10,6 +10,8 @@ import { transformError } from '@kbn/securitysolution-es-utils'; import { EntriesArray, ExceptionListItemSchema, + ExceptionListSchema, + FoundExceptionListItemSchema, FoundExceptionListSchema, deleteListSchema, exceptionListItemSchema, @@ -19,7 +21,7 @@ import { getSavedObjectType } from '@kbn/securitysolution-list-utils'; import { LIST_URL } from '@kbn/securitysolution-list-constants'; import type { ListsPluginRouter } from '../types'; -import { ExceptionListClient } from '../services/exception_lists/exception_list_client'; +import type { ExceptionListClient } from '../services/exception_lists/exception_list_client'; import { escapeQuotes } from '../services/utils/escape_query'; import { buildRouteValidation, buildSiemResponse } from './utils'; @@ -47,26 +49,31 @@ export const deleteListRoute = (router: ListsPluginRouter): void => { // ignoreReferences=true maintains pre-7.11 behavior of deleting value list without performing any additional checks if (!ignoreReferences) { - const referencedExceptionListItems = await exceptionLists.findValueListExceptionListItems( - { - page: 1, - perPage: 10000, - sortField: undefined, - sortOrder: undefined, - valueListId: id, - } - ); + // Stream the results from the Point In Time (PIT) finder into this array + let referencedExceptionListItems: ExceptionListItemSchema[] = []; + const executeFunctionOnStream = (foundResponse: FoundExceptionListItemSchema): void => { + referencedExceptionListItems = [...referencedExceptionListItems, ...foundResponse.data]; + }; - if (referencedExceptionListItems?.data?.length) { + await exceptionLists.findValueListExceptionListItemsPointInTimeFinder({ + executeFunctionOnStream, + maxSize: undefined, // NOTE: This is unbounded when it is "undefined" + perPage: 1_000, // See https://github.com/elastic/kibana/issues/93770 for choice of 1k + sortField: undefined, + sortOrder: undefined, + valueListId: id, + }); + if (referencedExceptionListItems.length) { // deleteReferences=false to perform dry run and identify referenced exception lists/items if (deleteReferences) { // Delete referenced exception list items // TODO: Create deleteListItems to delete in batch deleteExceptionItemResponses = await Promise.all( - referencedExceptionListItems.data.map(async (listItem) => { + referencedExceptionListItems.map(async (listItem) => { // Ensure only the single entry is deleted as there could be a separate value list referenced that is okay to keep // TODO: Add API to delete single entry - // @ts-ignore inline way of verifying entry type is EntryList? - const remainingEntries = listItem.entries.filter((e) => e?.list?.id !== id); + const remainingEntries = listItem.entries.filter( + (e) => e.type === 'list' && e.list.id !== id + ); if (remainingEntries.length === 0) { // All entries reference value list specified in request, delete entire exception list item return deleteExceptionListItem(exceptionLists, listItem); @@ -79,14 +86,12 @@ export const deleteListRoute = (router: ListsPluginRouter): void => { } else { const referencedExceptionLists = await getReferencedExceptionLists( exceptionLists, - referencedExceptionListItems.data + referencedExceptionListItems ); const refError = `Value list '${id}' is referenced in existing exception list(s)`; - const references = referencedExceptionListItems.data.map((item) => ({ + const references = referencedExceptionListItems.map((item) => ({ exception_item: item, - exception_list: referencedExceptionLists.data.find( - (l) => l.list_id === item.list_id - ), + exception_list: referencedExceptionLists.find((l) => l.list_id === item.list_id), })); return siemResponse.error({ @@ -140,7 +145,7 @@ export const deleteListRoute = (router: ListsPluginRouter): void => { const getReferencedExceptionLists = async ( exceptionLists: ExceptionListClient, exceptionListItems: ExceptionListItemSchema[] -): Promise => { +): Promise => { const filter = exceptionListItems .map( (item) => @@ -149,14 +154,22 @@ const getReferencedExceptionLists = async ( })}.attributes.list_id: "${escapeQuotes(item.list_id)}"` ) .join(' OR '); - return exceptionLists.findExceptionList({ + + // Stream the results from the Point In Time (PIT) finder into this array + let exceptionList: ExceptionListSchema[] = []; + const executeFunctionOnStream = (response: FoundExceptionListSchema): void => { + exceptionList = [...exceptionList, ...response.data]; + }; + await exceptionLists.findExceptionListPointInTimeFinder({ + executeFunctionOnStream, filter: `(${filter})`, + maxSize: undefined, // NOTE: This is unbounded when it is "undefined" namespaceType: ['agnostic', 'single'], - page: 1, - perPage: 10000, + perPage: 1_000, // See https://github.com/elastic/kibana/issues/93770 for choice of 1k sortField: undefined, sortOrder: undefined, }); + return exceptionList; }; /** diff --git a/x-pack/plugins/lists/server/routes/find_endpoint_list_item_route.ts b/x-pack/plugins/lists/server/routes/find_endpoint_list_item_route.ts index 36b5a66c2830f..6516e88877384 100644 --- a/x-pack/plugins/lists/server/routes/find_endpoint_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/find_endpoint_list_item_route.ts @@ -48,6 +48,8 @@ export const findEndpointListItemRoute = (router: ListsPluginRouter): void => { filter, page, perPage, + pit: undefined, + searchAfter: undefined, sortField, sortOrder, }); diff --git a/x-pack/plugins/lists/server/routes/find_exception_list_item_route.ts b/x-pack/plugins/lists/server/routes/find_exception_list_item_route.ts index fe7ffaa066281..67450ca02bb20 100644 --- a/x-pack/plugins/lists/server/routes/find_exception_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/find_exception_list_item_route.ts @@ -58,6 +58,8 @@ export const findExceptionListItemRoute = (router: ListsPluginRouter): void => { namespaceType, page, perPage, + pit: undefined, + searchAfter: undefined, sortField, sortOrder, }); diff --git a/x-pack/plugins/lists/server/routes/find_exception_list_route.ts b/x-pack/plugins/lists/server/routes/find_exception_list_route.ts index 5d1b78747a89e..e49b9c39d2f04 100644 --- a/x-pack/plugins/lists/server/routes/find_exception_list_route.ts +++ b/x-pack/plugins/lists/server/routes/find_exception_list_route.ts @@ -48,6 +48,8 @@ export const findExceptionListRoute = (router: ListsPluginRouter): void => { namespaceType, page, perPage, + pit: undefined, + searchAfter: undefined, sortField, sortOrder, }); diff --git a/x-pack/plugins/lists/server/routes/validate.ts b/x-pack/plugins/lists/server/routes/validate.ts index 5a118bf2c5ae0..29b2dd3b06d28 100644 --- a/x-pack/plugins/lists/server/routes/validate.ts +++ b/x-pack/plugins/lists/server/routes/validate.ts @@ -30,6 +30,8 @@ export const validateExceptionListSize = async ( namespaceType, page: undefined, perPage: undefined, + pit: undefined, + searchAfter: undefined, sortField: undefined, sortOrder: undefined, }); diff --git a/x-pack/plugins/lists/server/services/exception_lists/close_point_in_time.ts b/x-pack/plugins/lists/server/services/exception_lists/close_point_in_time.ts new file mode 100644 index 0000000000000..0fbcd08316166 --- /dev/null +++ b/x-pack/plugins/lists/server/services/exception_lists/close_point_in_time.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + SavedObjectsClientContract, + SavedObjectsClosePointInTimeResponse, +} from 'kibana/server'; +import type { PitId } from '@kbn/securitysolution-io-ts-list-types'; + +interface ClosePointInTimeOptions { + pit: PitId; + savedObjectsClient: SavedObjectsClientContract; +} + +/** + * Closes a point in time (PIT) for either exception lists or exception list items. + * See: https://www.elastic.co/guide/en/elasticsearch/reference/current/point-in-time-api.html + * @params pit {string} The point in time to close + * @params savedObjectsClient {object} The saved objects client to delegate to + * @return {SavedObjectsOpenPointInTimeResponse} The point in time (PIT) + */ +export const closePointInTime = async ({ + pit, + savedObjectsClient, +}: ClosePointInTimeOptions): Promise => { + return savedObjectsClient.closePointInTime(pit); +}; diff --git a/x-pack/plugins/lists/server/services/exception_lists/delete_exception_list_items_by_list.ts b/x-pack/plugins/lists/server/services/exception_lists/delete_exception_list_items_by_list.ts index 873fe66b1ad3b..c23c8967fef68 100644 --- a/x-pack/plugins/lists/server/services/exception_lists/delete_exception_list_items_by_list.ts +++ b/x-pack/plugins/lists/server/services/exception_lists/delete_exception_list_items_by_list.ts @@ -5,15 +5,16 @@ * 2.0. */ -import type { ListId, NamespaceType } from '@kbn/securitysolution-io-ts-list-types'; +import type { + FoundExceptionListItemSchema, + ListId, + NamespaceType, +} from '@kbn/securitysolution-io-ts-list-types'; import { getSavedObjectType } from '@kbn/securitysolution-list-utils'; import { asyncForEach } from '@kbn/std'; +import type { SavedObjectsClientContract } from 'kibana/server'; -import { SavedObjectsClientContract } from '../../../../../../src/core/server'; - -import { findExceptionListItem } from './find_exception_list_item'; - -const PER_PAGE = 100; +import { findExceptionListItemPointInTimeFinder } from './find_exception_list_item_point_in_time_finder'; interface DeleteExceptionListItemByListOptions { listId: ListId; @@ -35,35 +36,24 @@ export const getExceptionListItemIds = async ({ savedObjectsClient, namespaceType, }: DeleteExceptionListItemByListOptions): Promise => { - let page = 1; + // Stream the results from the Point In Time (PIT) finder into this array let ids: string[] = []; - let foundExceptionListItems = await findExceptionListItem({ + const executeFunctionOnStream = (response: FoundExceptionListItemSchema): void => { + const responseIds = response.data.map((exceptionListItem) => exceptionListItem.id); + ids = [...ids, ...responseIds]; + }; + + await findExceptionListItemPointInTimeFinder({ + executeFunctionOnStream, filter: undefined, listId, + maxSize: undefined, // NOTE: This is unbounded when it is "undefined" namespaceType, - page, - perPage: PER_PAGE, + perPage: 1_000, // See https://github.com/elastic/kibana/issues/93770 for choice of 1k savedObjectsClient, sortField: 'tie_breaker_id', sortOrder: 'desc', }); - while (foundExceptionListItems != null && foundExceptionListItems.data.length > 0) { - ids = [ - ...ids, - ...foundExceptionListItems.data.map((exceptionListItem) => exceptionListItem.id), - ]; - page += 1; - foundExceptionListItems = await findExceptionListItem({ - filter: undefined, - listId, - namespaceType, - page, - perPage: PER_PAGE, - savedObjectsClient, - sortField: 'tie_breaker_id', - sortOrder: 'desc', - }); - } return ids; }; diff --git a/x-pack/plugins/lists/server/services/exception_lists/exception_list_client.test.ts b/x-pack/plugins/lists/server/services/exception_lists/exception_list_client.test.ts index 3b6f2cb6ae4f2..1a444c403d3c2 100644 --- a/x-pack/plugins/lists/server/services/exception_lists/exception_list_client.test.ts +++ b/x-pack/plugins/lists/server/services/exception_lists/exception_list_client.test.ts @@ -212,6 +212,8 @@ describe('exception_list_client', () => { namespaceType: 'agnostic', page: 1, perPage: 1, + pit: undefined, + searchAfter: undefined, sortField: 'name', sortOrder: 'asc', }); @@ -229,6 +231,8 @@ describe('exception_list_client', () => { namespaceType: ['agnostic'], page: 1, perPage: 1, + pit: undefined, + searchAfter: undefined, sortField: 'name', sortOrder: 'asc', }); diff --git a/x-pack/plugins/lists/server/services/exception_lists/exception_list_client.ts b/x-pack/plugins/lists/server/services/exception_lists/exception_list_client.ts index 93f5077f021d5..d6fc1bfec8058 100644 --- a/x-pack/plugins/lists/server/services/exception_lists/exception_list_client.ts +++ b/x-pack/plugins/lists/server/services/exception_lists/exception_list_client.ts @@ -5,7 +5,12 @@ * 2.0. */ -import { KibanaRequest, SavedObjectsClientContract } from 'kibana/server'; +import type { + KibanaRequest, + SavedObjectsClientContract, + SavedObjectsClosePointInTimeResponse, + SavedObjectsOpenPointInTimeResponse, +} from 'kibana/server'; import { ExceptionListItemSchema, ExceptionListSchema, @@ -24,7 +29,8 @@ import type { ServerExtensionCallbackContext, } from '../extension_points'; -import { +import type { + ClosePointInTimeOptions, ConstructorOptions, CreateEndpointListItemOptions, CreateExceptionListItemOptions, @@ -36,15 +42,20 @@ import { ExportExceptionListAndItemsOptions, FindEndpointListItemOptions, FindExceptionListItemOptions, + FindExceptionListItemPointInTimeFinderOptions, + FindExceptionListItemsPointInTimeFinderOptions, FindExceptionListOptions, + FindExceptionListPointInTimeFinderOptions, FindExceptionListsItemOptions, FindValueListExceptionListsItems, + FindValueListExceptionListsItemsPointInTimeFinder, GetEndpointListItemOptions, GetExceptionListItemOptions, GetExceptionListOptions, GetExceptionListSummaryOptions, ImportExceptionListAndItemsAsArrayOptions, ImportExceptionListAndItemsOptions, + OpenPointInTimeOptions, UpdateEndpointListItemOptions, UpdateExceptionListItemOptions, UpdateExceptionListOptions, @@ -64,10 +75,7 @@ import { deleteExceptionList } from './delete_exception_list'; import { deleteExceptionListItem, deleteExceptionListItemById } from './delete_exception_list_item'; import { findExceptionListItem } from './find_exception_list_item'; import { findExceptionList } from './find_exception_list'; -import { - findExceptionListsItem, - findValueListExceptionListItems, -} from './find_exception_list_items'; +import { findExceptionListsItem } from './find_exception_list_items'; import { createEndpointList } from './create_endpoint_list'; import { createEndpointTrustedAppsList } from './create_endpoint_trusted_apps_list'; import { PromiseFromStreams, importExceptions } from './import_exception_list_and_items'; @@ -80,6 +88,13 @@ import { createExceptionsStreamFromNdjson, exceptionsChecksFromArray, } from './utils/import/create_exceptions_stream_logic'; +import { openPointInTime } from './open_point_in_time'; +import { closePointInTime } from './close_point_in_time'; +import { findExceptionListPointInTimeFinder } from './find_exception_list_point_in_time_finder'; +import { findValueListExceptionListItems } from './find_value_list_exception_list_items'; +import { findExceptionListsItemPointInTimeFinder } from './find_exception_list_items_point_in_time_finder'; +import { findValueListExceptionListItemsPointInTimeFinder } from './find_value_list_exception_list_items_point_in_time_finder'; +import { findExceptionListItemPointInTimeFinder } from './find_exception_list_item_point_in_time_finder'; export class ExceptionListClient { private readonly user: string; @@ -621,7 +636,9 @@ export class ExceptionListClient { listId, filter, perPage, + pit, page, + searchAfter, sortField, sortOrder, namespaceType, @@ -637,6 +654,8 @@ export class ExceptionListClient { namespaceType, page, perPage, + pit, + searchAfter, sortField, sortOrder, }, @@ -650,7 +669,9 @@ export class ExceptionListClient { namespaceType, page, perPage, + pit, savedObjectsClient, + searchAfter, sortField, sortOrder, }); @@ -660,7 +681,9 @@ export class ExceptionListClient { listId, filter, perPage, + pit, page, + searchAfter, sortField, sortOrder, namespaceType, @@ -676,6 +699,8 @@ export class ExceptionListClient { namespaceType, page, perPage, + pit, + searchAfter, sortField, sortOrder, }, @@ -689,7 +714,9 @@ export class ExceptionListClient { namespaceType, page, perPage, + pit, savedObjectsClient, + searchAfter, sortField, sortOrder, }); @@ -697,7 +724,9 @@ export class ExceptionListClient { public findValueListExceptionListItems = async ({ perPage, + pit, page, + searchAfter, sortField, sortOrder, valueListId, @@ -706,7 +735,9 @@ export class ExceptionListClient { return findValueListExceptionListItems({ page, perPage, + pit, savedObjectsClient, + searchAfter, sortField, sortOrder, valueListId, @@ -717,6 +748,8 @@ export class ExceptionListClient { filter, perPage, page, + pit, + searchAfter, sortField, sortOrder, namespaceType, @@ -727,7 +760,9 @@ export class ExceptionListClient { namespaceType, page, perPage, + pit, savedObjectsClient, + searchAfter, sortField, sortOrder, }); @@ -745,6 +780,8 @@ export class ExceptionListClient { filter, perPage, page, + pit, + searchAfter, sortField, sortOrder, }: FindEndpointListItemOptions): Promise => { @@ -756,7 +793,9 @@ export class ExceptionListClient { namespaceType: 'agnostic', page, perPage, + pit, savedObjectsClient, + searchAfter, sortField, sortOrder, }); @@ -865,4 +904,257 @@ export class ExceptionListClient { user, }); }; + + /** + * Opens a point in time (PIT) for either exception lists or exception list items. + * See: https://www.elastic.co/guide/en/elasticsearch/reference/current/point-in-time-api.html + * @params namespaceType {string} "agnostic" or "single" depending on which namespace you are targeting + * @params options {Object} The saved object PIT options + * @return {SavedObjectsOpenPointInTimeResponse} The point in time (PIT) + */ + public openPointInTime = async ({ + namespaceType, + options, + }: OpenPointInTimeOptions): Promise => { + const { savedObjectsClient } = this; + return openPointInTime({ + namespaceType, + options, + savedObjectsClient, + }); + }; + + /** + * Closes a point in time (PIT) for either exception lists or exception list items. + * See: https://www.elastic.co/guide/en/elasticsearch/reference/current/point-in-time-api.html + * @params pit {string} The point in time to close + * @return {SavedObjectsOpenPointInTimeResponse} The point in time (PIT) + */ + public closePointInTime = async ({ + pit, + }: ClosePointInTimeOptions): Promise => { + const { savedObjectsClient } = this; + return closePointInTime({ + pit, + savedObjectsClient, + }); + }; + + /** + * Finds an exception list item within a point in time (PIT) and then calls the function + * `executeFunctionOnStream` until the maxPerPage is reached and stops. + * NOTE: This is slightly different from the saved objects version in that it takes + * an injected function, so that we avoid doing additional plumbing with generators + * to try to keep the maintenance of this machinery simpler for now. + * + * If you want to stream all results up to 10k into memory for correlation this would be: + * @example + * ```ts + * const exceptionList: ExceptionListItemSchema[] = []; + * const executeFunctionOnStream = (response: FoundExceptionListItemSchema) => { + * exceptionList = [...exceptionList, ...response.data]; + * } + * await client.findExceptionListItemPointInTimeFinder({ + * filter, + * executeFunctionOnStream, + * namespaceType, + * maxSize: 10_000, // NOTE: This is unbounded if it is "undefined" + * perPage: 1_000, // See https://github.com/elastic/kibana/issues/93770 for choice of 1k + * sortField, + * sortOrder, + * exe + * }); + * ``` + * @param filter {string} Your filter + * @param namespaceType {string} "agnostic" | "single" of your namespace + * @param perPage {number} The number of items per page. Typical value should be 1_000 here. Never go above 10_000 + * @param maxSize {number of undefined} If given a max size, this will not exceeded. Otherwise if undefined is passed down, all records will be processed. + * @param sortField {string} String of the field to sort against + * @param sortOrder "asc" | "desc" The order to sort against + */ + public findExceptionListItemPointInTimeFinder = async ({ + executeFunctionOnStream, + filter, + listId, + maxSize, + namespaceType, + perPage, + sortField, + sortOrder, + }: FindExceptionListItemPointInTimeFinderOptions): Promise => { + const { savedObjectsClient } = this; + return findExceptionListItemPointInTimeFinder({ + executeFunctionOnStream, + filter, + listId, + maxSize, + namespaceType, + perPage, + savedObjectsClient, + sortField, + sortOrder, + }); + }; + + /** + * Finds an exception list within a point in time (PIT) and then calls the function + * `executeFunctionOnStream` until the maxPerPage is reached and stops. + * NOTE: This is slightly different from the saved objects version in that it takes + * an injected function, so that we avoid doing additional plumbing with generators + * to try to keep the maintenance of this machinery simpler for now. + * + * If you want to stream all results up to 10k into memory for correlation this would be: + * @example + * ```ts + * const exceptionList: ExceptionListSchema[] = []; + * const executeFunctionOnStream = (response: FoundExceptionListSchema) => { + * exceptionList = [...exceptionList, ...response.data]; + * } + * await client.findExceptionListPointInTimeFinder({ + * filter, + * executeFunctionOnStream, + * namespaceType, + * maxSize: 10_000, // NOTE: This is unbounded if it is "undefined" + * perPage: 1_000, // See https://github.com/elastic/kibana/issues/93770 for choice of 1k + * sortField, + * sortOrder, + * exe + * }); + * ``` + * @param filter {string} Your filter + * @param namespaceType {string} "agnostic" | "single" of your namespace + * @param perPage {number} The number of items per page. Typical value should be 1_000 here. Never go above 10_000 + * @param maxSize {number of undefined} If given a max size, this will not exceeded. Otherwise if undefined is passed down, all records will be processed. + * @param sortField {string} String of the field to sort against + * @param sortOrder "asc" | "desc" The order to sort against + */ + public findExceptionListPointInTimeFinder = async ({ + executeFunctionOnStream, + filter, + maxSize, + namespaceType, + perPage, + sortField, + sortOrder, + }: FindExceptionListPointInTimeFinderOptions): Promise => { + const { savedObjectsClient } = this; + return findExceptionListPointInTimeFinder({ + executeFunctionOnStream, + filter, + maxSize, + namespaceType, + perPage, + savedObjectsClient, + sortField, + sortOrder, + }); + }; + + /** + * Finds exception list items within a point in time (PIT) and then calls the function + * `executeFunctionOnStream` until the maxPerPage is reached and stops. + * NOTE: This is slightly different from the saved objects version in that it takes + * an injected function, so that we avoid doing additional plumbing with generators + * to try to keep the maintenance of this machinery simpler for now. + * + * If you want to stream all results up to 10k into memory for correlation this would be: + * @example + * ```ts + * const exceptionList: ExceptionListItemSchema[] = []; + * const executeFunctionOnStream = (response: FoundExceptionListItemSchema) => { + * exceptionList = [...exceptionList, ...response.data]; + * } + * await client.findExceptionListsItemPointInTimeFinder({ + * filter, + * executeFunctionOnStream, + * namespaceType, + * maxSize: 10_000, // NOTE: This is unbounded if it is "undefined" + * perPage: 1_000, // See https://github.com/elastic/kibana/issues/93770 for choice of 1k + * sortField, + * sortOrder, + * exe + * }); + * ``` + * @param filter {string} Your filter + * @param namespaceType {string} "agnostic" | "single" of your namespace + * @param perPage {number} The number of items per page. Typical value should be 1_000 here. Never go above 10_000 + * @param maxSize {number of undefined} If given a max size, this will not exceeded. Otherwise if undefined is passed down, all records will be processed. + * @param sortField {string} String of the field to sort against + * @param sortOrder "asc" | "desc" The order to sort against + */ + public findExceptionListsItemPointInTimeFinder = async ({ + listId, + namespaceType, + executeFunctionOnStream, + maxSize, + filter, + perPage, + sortField, + sortOrder, + }: FindExceptionListItemsPointInTimeFinderOptions): Promise => { + const { savedObjectsClient } = this; + return findExceptionListsItemPointInTimeFinder({ + executeFunctionOnStream, + filter, + listId, + maxSize, + namespaceType, + perPage, + savedObjectsClient, + sortField, + sortOrder, + }); + }; + + /** + * Finds value lists within exception lists within a point in time (PIT) and then calls the function + * `executeFunctionOnStream` until the maxPerPage is reached and stops. + * NOTE: This is slightly different from the saved objects version in that it takes + * an injected function, so that we avoid doing additional plumbing with generators + * to try to keep the maintenance of this machinery simpler for now. + * + * If you want to stream all results up to 10k into memory for correlation this would be: + * @example + * ```ts + * const exceptionList: ExceptionListItemSchema[] = []; + * const executeFunctionOnStream = (response: FoundExceptionListItemSchema) => { + * exceptionList = [...exceptionList, ...response.data]; + * } + * await client.findValueListExceptionListItemsPointInTimeFinder({ + * valueListId, + * executeFunctionOnStream, + * namespaceType, + * maxSize: 10_000, // NOTE: This is unbounded if it is "undefined" + * perPage: 1_000, // See https://github.com/elastic/kibana/issues/93770 for choice of 1k + * sortField, + * sortOrder, + * exe + * }); + * ``` + * @param valueListId {string} Your value list id + * @param namespaceType {string} "agnostic" | "single" of your namespace + * @param perPage {number} The number of items per page. Typical value should be 1_000 here. Never go above 10_000 + * @param maxSize {number of undefined} If given a max size, this will not exceeded. Otherwise if undefined is passed down, all records will be processed. + * @param sortField {string} String of the field to sort against + * @param sortOrder "asc" | "desc" The order to sort against + */ + public findValueListExceptionListItemsPointInTimeFinder = async ({ + valueListId, + executeFunctionOnStream, + perPage, + maxSize, + sortField, + sortOrder, + }: FindValueListExceptionListsItemsPointInTimeFinder): Promise => { + const { savedObjectsClient } = this; + return findValueListExceptionListItemsPointInTimeFinder({ + executeFunctionOnStream, + maxSize, + perPage, + savedObjectsClient, + sortField, + sortOrder, + valueListId, + }); + }; } diff --git a/x-pack/plugins/lists/server/services/exception_lists/exception_list_client_types.ts b/x-pack/plugins/lists/server/services/exception_lists/exception_list_client_types.ts index 1c2762cb52c97..4c7820fc05f94 100644 --- a/x-pack/plugins/lists/server/services/exception_lists/exception_list_client_types.ts +++ b/x-pack/plugins/lists/server/services/exception_lists/exception_list_client_types.ts @@ -5,9 +5,13 @@ * 2.0. */ -import { Readable } from 'stream'; +import type { Readable } from 'stream'; -import type { KibanaRequest, SavedObjectsClientContract } from 'kibana/server'; +import type { + KibanaRequest, + SavedObjectsClientContract, + SavedObjectsOpenPointInTimeOptions, +} from 'kibana/server'; import type { CreateCommentsArray, Description, @@ -19,6 +23,8 @@ import type { ExceptionListTypeOrUndefined, ExportExceptionDetails, FilterOrUndefined, + FoundExceptionListItemSchema, + FoundExceptionListSchema, Id, IdOrUndefined, Immutable, @@ -28,6 +34,7 @@ import type { ItemIdOrUndefined, ListId, ListIdOrUndefined, + MaxSizeOrUndefined, MetaOrUndefined, Name, NameOrUndefined, @@ -36,6 +43,9 @@ import type { OsTypeArray, PageOrUndefined, PerPageOrUndefined, + PitId, + PitOrUndefined, + SearchAfterOrUndefined, SortFieldOrUndefined, SortOrderOrUndefined, Tags, @@ -43,14 +53,14 @@ import type { UpdateCommentsArray, _VersionOrUndefined, } from '@kbn/securitysolution-io-ts-list-types'; -import { +import type { EmptyStringArrayDecoded, NonEmptyStringArrayDecoded, Version, VersionOrUndefined, } from '@kbn/securitysolution-io-ts-types'; -import { ExtensionPointStorageClientInterface } from '../extension_points'; +import type { ExtensionPointStorageClientInterface } from '../extension_points'; export interface ConstructorOptions { user: string; @@ -194,6 +204,8 @@ export interface FindExceptionListItemOptions { namespaceType: NamespaceType; filter: FilterOrUndefined; perPage: PerPageOrUndefined; + pit?: PitOrUndefined; + searchAfter?: SearchAfterOrUndefined; page: PageOrUndefined; sortField: SortFieldOrUndefined; sortOrder: SortOrderOrUndefined; @@ -202,6 +214,8 @@ export interface FindExceptionListItemOptions { export interface FindEndpointListItemOptions { filter: FilterOrUndefined; perPage: PerPageOrUndefined; + pit?: PitOrUndefined; + searchAfter?: SearchAfterOrUndefined; page: PageOrUndefined; sortField: SortFieldOrUndefined; sortOrder: SortOrderOrUndefined; @@ -212,6 +226,8 @@ export interface FindExceptionListsItemOptions { namespaceType: NamespaceTypeArray; filter: EmptyStringArrayDecoded; perPage: PerPageOrUndefined; + pit?: PitOrUndefined; + searchAfter?: SearchAfterOrUndefined; page: PageOrUndefined; sortField: SortFieldOrUndefined; sortOrder: SortOrderOrUndefined; @@ -220,6 +236,8 @@ export interface FindExceptionListsItemOptions { export interface FindValueListExceptionListsItems { valueListId: Id; perPage: PerPageOrUndefined; + pit?: PitOrUndefined; + searchAfter?: SearchAfterOrUndefined; page: PageOrUndefined; sortField: SortFieldOrUndefined; sortOrder: SortOrderOrUndefined; @@ -230,6 +248,8 @@ export interface FindExceptionListOptions { filter: FilterOrUndefined; perPage: PerPageOrUndefined; page: PageOrUndefined; + pit?: PitOrUndefined; + searchAfter?: SearchAfterOrUndefined; sortField: SortFieldOrUndefined; sortOrder: SortOrderOrUndefined; } @@ -256,3 +276,53 @@ export interface ImportExceptionListAndItemsAsArrayOptions { maxExceptionsImportSize: number; overwrite: boolean; } + +export interface OpenPointInTimeOptions { + namespaceType: NamespaceType; + options: SavedObjectsOpenPointInTimeOptions | undefined; +} + +export interface ClosePointInTimeOptions { + pit: PitId; +} + +export interface FindExceptionListItemPointInTimeFinderOptions { + listId: ListId; + namespaceType: NamespaceType; + filter: FilterOrUndefined; + perPage: PerPageOrUndefined; + sortField: SortFieldOrUndefined; + sortOrder: SortOrderOrUndefined; + executeFunctionOnStream: (response: FoundExceptionListItemSchema) => void; + maxSize: MaxSizeOrUndefined; +} + +export interface FindExceptionListPointInTimeFinderOptions { + maxSize: MaxSizeOrUndefined; + namespaceType: NamespaceTypeArray; + filter: FilterOrUndefined; + perPage: PerPageOrUndefined; + sortField: SortFieldOrUndefined; + sortOrder: SortOrderOrUndefined; + executeFunctionOnStream: (response: FoundExceptionListSchema) => void; +} + +export interface FindExceptionListItemsPointInTimeFinderOptions { + listId: NonEmptyStringArrayDecoded; + namespaceType: NamespaceTypeArray; + filter: EmptyStringArrayDecoded; + perPage: PerPageOrUndefined; + sortField: SortFieldOrUndefined; + sortOrder: SortOrderOrUndefined; + executeFunctionOnStream: (response: FoundExceptionListItemSchema) => void; + maxSize: MaxSizeOrUndefined; +} + +export interface FindValueListExceptionListsItemsPointInTimeFinder { + valueListId: Id; + perPage: PerPageOrUndefined; + sortField: SortFieldOrUndefined; + sortOrder: SortOrderOrUndefined; + executeFunctionOnStream: (response: FoundExceptionListItemSchema) => void; + maxSize: MaxSizeOrUndefined; +} diff --git a/x-pack/plugins/lists/server/services/exception_lists/export_exception_list_and_items.test.ts b/x-pack/plugins/lists/server/services/exception_lists/export_exception_list_and_items.test.ts index 9f3c02fecca20..0f8e4bf7bbf45 100644 --- a/x-pack/plugins/lists/server/services/exception_lists/export_exception_list_and_items.test.ts +++ b/x-pack/plugins/lists/server/services/exception_lists/export_exception_list_and_items.test.ts @@ -5,44 +5,49 @@ * 2.0. */ -import { SavedObjectsClientContract } from 'kibana/server'; +import { savedObjectsClientMock } from 'src/core/server/mocks'; import { getExceptionListItemSchemaMock } from '../../../common/schemas/response/exception_list_item_schema.mock'; import { getExceptionListSchemaMock } from '../../../common/schemas/response/exception_list_schema.mock'; import { exportExceptionListAndItems } from './export_exception_list_and_items'; -import { findExceptionListItem } from './find_exception_list_item'; +import { findExceptionListItemPointInTimeFinder } from './find_exception_list_item_point_in_time_finder'; import { getExceptionList } from './get_exception_list'; jest.mock('./get_exception_list'); -jest.mock('./find_exception_list_item'); +jest.mock('./find_exception_list_item_point_in_time_finder'); describe('export_exception_list_and_items', () => { describe('exportExceptionListAndItems', () => { test('it should return null if no matching exception list found', async () => { (getExceptionList as jest.Mock).mockResolvedValue(null); - (findExceptionListItem as jest.Mock).mockResolvedValue({ data: [] }); + (findExceptionListItemPointInTimeFinder as jest.Mock).mockImplementationOnce( + ({ executeFunctionOnStream }) => { + executeFunctionOnStream({ data: [getExceptionListItemSchemaMock()] }); + } + ); const result = await exportExceptionListAndItems({ id: '123', listId: 'non-existent', namespaceType: 'single', - savedObjectsClient: {} as SavedObjectsClientContract, + savedObjectsClient: savedObjectsClientMock.create(), }); expect(result).toBeNull(); }); test('it should return stringified list and items', async () => { (getExceptionList as jest.Mock).mockResolvedValue(getExceptionListSchemaMock()); - (findExceptionListItem as jest.Mock).mockResolvedValue({ - data: [getExceptionListItemSchemaMock()], - }); - + (findExceptionListItemPointInTimeFinder as jest.Mock).mockImplementationOnce( + ({ executeFunctionOnStream }) => { + executeFunctionOnStream({ data: [getExceptionListItemSchemaMock()] }); + } + ); const result = await exportExceptionListAndItems({ id: '123', listId: 'non-existent', namespaceType: 'single', - savedObjectsClient: {} as SavedObjectsClientContract, + savedObjectsClient: savedObjectsClientMock.create(), }); expect(result?.exportData).toEqual( `${JSON.stringify(getExceptionListSchemaMock())}\n${JSON.stringify( diff --git a/x-pack/plugins/lists/server/services/exception_lists/export_exception_list_and_items.ts b/x-pack/plugins/lists/server/services/exception_lists/export_exception_list_and_items.ts index b071c72a9b122..b2cd7d38d8d56 100644 --- a/x-pack/plugins/lists/server/services/exception_lists/export_exception_list_and_items.ts +++ b/x-pack/plugins/lists/server/services/exception_lists/export_exception_list_and_items.ts @@ -6,15 +6,17 @@ */ import type { + ExceptionListItemSchema, ExportExceptionDetails, + FoundExceptionListItemSchema, IdOrUndefined, ListIdOrUndefined, NamespaceType, } from '@kbn/securitysolution-io-ts-list-types'; import { transformDataToNdjson } from '@kbn/securitysolution-utils'; -import { SavedObjectsClientContract } from 'kibana/server'; +import type { SavedObjectsClientContract } from 'kibana/server'; -import { findExceptionListItem } from './find_exception_list_item'; +import { findExceptionListItemPointInTimeFinder } from './find_exception_list_item_point_in_time_finder'; import { getExceptionList } from './get_exception_list'; interface ExportExceptionListAndItemsOptions { @@ -45,20 +47,23 @@ export const exportExceptionListAndItems = async ({ if (exceptionList == null) { return null; } else { - // TODO: Will need to address this when we switch over to - // using PIT, don't want it to get lost - // https://github.com/elastic/kibana/issues/103944 - const listItems = await findExceptionListItem({ + // Stream the results from the Point In Time (PIT) finder into this array + let exceptionItems: ExceptionListItemSchema[] = []; + const executeFunctionOnStream = (response: FoundExceptionListItemSchema): void => { + exceptionItems = [...exceptionItems, ...response.data]; + }; + + await findExceptionListItemPointInTimeFinder({ + executeFunctionOnStream, filter: undefined, listId: exceptionList.list_id, + maxSize: undefined, // NOTE: This is unbounded when it is "undefined" namespaceType: exceptionList.namespace_type, - page: 1, - perPage: 10000, + perPage: 1_000, // See https://github.com/elastic/kibana/issues/93770 for choice of 1k savedObjectsClient, sortField: 'exception-list.created_at', sortOrder: 'desc', }); - const exceptionItems = listItems?.data ?? []; const { exportData } = getExport([exceptionList, ...exceptionItems]); // TODO: Add logic for missing lists and items on errors diff --git a/x-pack/plugins/lists/server/services/exception_lists/find_exception_list.test.ts b/x-pack/plugins/lists/server/services/exception_lists/find_exception_list.test.ts deleted file mode 100644 index 919a1e4e90a51..0000000000000 --- a/x-pack/plugins/lists/server/services/exception_lists/find_exception_list.test.ts +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { getExceptionListFilter } from './find_exception_list'; - -describe('find_exception_list', () => { - describe('getExceptionListFilter', () => { - test('it should create a filter for agnostic lists if only searching for agnostic lists', () => { - const filter = getExceptionListFilter({ - filter: undefined, - savedObjectTypes: ['exception-list-agnostic'], - }); - expect(filter).toEqual('(exception-list-agnostic.attributes.list_type: list)'); - }); - - test('it should create a filter for agnostic lists with additional filters if only searching for agnostic lists', () => { - const filter = getExceptionListFilter({ - filter: 'exception-list-agnostic.attributes.name: "Sample Endpoint Exception List"', - savedObjectTypes: ['exception-list-agnostic'], - }); - expect(filter).toEqual( - '(exception-list-agnostic.attributes.list_type: list) AND exception-list-agnostic.attributes.name: "Sample Endpoint Exception List"' - ); - }); - - test('it should create a filter for single lists if only searching for single lists', () => { - const filter = getExceptionListFilter({ - filter: undefined, - savedObjectTypes: ['exception-list'], - }); - expect(filter).toEqual('(exception-list.attributes.list_type: list)'); - }); - - test('it should create a filter for single lists with additional filters if only searching for single lists', () => { - const filter = getExceptionListFilter({ - filter: 'exception-list.attributes.name: "Sample Endpoint Exception List"', - savedObjectTypes: ['exception-list'], - }); - expect(filter).toEqual( - '(exception-list.attributes.list_type: list) AND exception-list.attributes.name: "Sample Endpoint Exception List"' - ); - }); - - test('it should create a filter that searches for both agnostic and single lists', () => { - const filter = getExceptionListFilter({ - filter: undefined, - savedObjectTypes: ['exception-list-agnostic', 'exception-list'], - }); - expect(filter).toEqual( - '(exception-list-agnostic.attributes.list_type: list OR exception-list.attributes.list_type: list)' - ); - }); - - test('it should create a filter that searches for both agnostic and single lists with additional filters if only searching for agnostic lists', () => { - const filter = getExceptionListFilter({ - filter: 'exception-list-agnostic.attributes.name: "Sample Endpoint Exception List"', - savedObjectTypes: ['exception-list-agnostic', 'exception-list'], - }); - expect(filter).toEqual( - '(exception-list-agnostic.attributes.list_type: list OR exception-list.attributes.list_type: list) AND exception-list-agnostic.attributes.name: "Sample Endpoint Exception List"' - ); - }); - }); -}); diff --git a/x-pack/plugins/lists/server/services/exception_lists/find_exception_list.ts b/x-pack/plugins/lists/server/services/exception_lists/find_exception_list.ts index b3d5dd9ddb32b..46d292e93a2d1 100644 --- a/x-pack/plugins/lists/server/services/exception_lists/find_exception_list.ts +++ b/x-pack/plugins/lists/server/services/exception_lists/find_exception_list.ts @@ -5,21 +5,24 @@ * 2.0. */ -import { SavedObjectsClientContract } from 'kibana/server'; +import type { SavedObjectsClientContract } from 'kibana/server'; import type { FilterOrUndefined, FoundExceptionListSchema, NamespaceTypeArray, PageOrUndefined, PerPageOrUndefined, + PitOrUndefined, + SearchAfterOrUndefined, SortFieldOrUndefined, SortOrderOrUndefined, } from '@kbn/securitysolution-io-ts-list-types'; -import { SavedObjectType, getSavedObjectTypes } from '@kbn/securitysolution-list-utils'; +import { getSavedObjectTypes } from '@kbn/securitysolution-list-utils'; -import { ExceptionListSoSchema } from '../../schemas/saved_objects'; +import type { ExceptionListSoSchema } from '../../schemas/saved_objects'; import { transformSavedObjectsToFoundExceptionList } from './utils'; +import { getExceptionListFilter } from './utils/get_exception_list_filter'; interface FindExceptionListOptions { namespaceType: NamespaceTypeArray; @@ -29,6 +32,8 @@ interface FindExceptionListOptions { page: PageOrUndefined; sortField: SortFieldOrUndefined; sortOrder: SortOrderOrUndefined; + pit: PitOrUndefined; + searchAfter: SearchAfterOrUndefined; } export const findExceptionList = async ({ @@ -37,14 +42,18 @@ export const findExceptionList = async ({ filter, page, perPage, + searchAfter, sortField, sortOrder, + pit, }: FindExceptionListOptions): Promise => { const savedObjectTypes = getSavedObjectTypes({ namespaceType }); const savedObjectsFindResponse = await savedObjectsClient.find({ filter: getExceptionListFilter({ filter, savedObjectTypes }), page, perPage, + pit, + searchAfter, sortField, sortOrder, type: savedObjectTypes, @@ -52,19 +61,3 @@ export const findExceptionList = async ({ return transformSavedObjectsToFoundExceptionList({ savedObjectsFindResponse }); }; - -export const getExceptionListFilter = ({ - filter, - savedObjectTypes, -}: { - filter: FilterOrUndefined; - savedObjectTypes: SavedObjectType[]; -}): string => { - const listTypesFilter = savedObjectTypes - .map((type) => `${type}.attributes.list_type: list`) - .join(' OR '); - - if (filter != null) { - return `(${listTypesFilter}) AND ${filter}`; - } else return `(${listTypesFilter})`; -}; diff --git a/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_item.ts b/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_item.ts index 3d050652afed1..5c23475573bb6 100644 --- a/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_item.ts +++ b/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_item.ts @@ -13,6 +13,8 @@ import type { NamespaceType, PageOrUndefined, PerPageOrUndefined, + PitOrUndefined, + SearchAfterOrUndefined, SortFieldOrUndefined, SortOrderOrUndefined, } from '@kbn/securitysolution-io-ts-list-types'; @@ -24,10 +26,12 @@ interface FindExceptionListItemOptions { namespaceType: NamespaceType; savedObjectsClient: SavedObjectsClientContract; filter: FilterOrUndefined; - perPage: PerPageOrUndefined; page: PageOrUndefined; + perPage: PerPageOrUndefined; + pit: PitOrUndefined; sortField: SortFieldOrUndefined; sortOrder: SortOrderOrUndefined; + searchAfter: SearchAfterOrUndefined; } export const findExceptionListItem = async ({ @@ -37,6 +41,8 @@ export const findExceptionListItem = async ({ filter, page, perPage, + pit, + searchAfter, sortField, sortOrder, }: FindExceptionListItemOptions): Promise => { @@ -46,7 +52,9 @@ export const findExceptionListItem = async ({ namespaceType: [namespaceType], page, perPage, + pit, savedObjectsClient, + searchAfter, sortField, sortOrder, }); diff --git a/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_item_point_in_time_finder.ts b/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_item_point_in_time_finder.ts new file mode 100644 index 0000000000000..8b90315c80ada --- /dev/null +++ b/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_item_point_in_time_finder.ts @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsClientContract } from 'kibana/server'; +import type { + FilterOrUndefined, + FoundExceptionListItemSchema, + ListId, + MaxSizeOrUndefined, + NamespaceType, + PerPageOrUndefined, + SortFieldOrUndefined, + SortOrderOrUndefined, +} from '@kbn/securitysolution-io-ts-list-types'; + +import { findExceptionListsItemPointInTimeFinder } from './find_exception_list_items_point_in_time_finder'; + +interface FindExceptionListItemPointInTimeFinderOptions { + listId: ListId; + namespaceType: NamespaceType; + savedObjectsClient: SavedObjectsClientContract; + filter: FilterOrUndefined; + perPage: PerPageOrUndefined; + sortField: SortFieldOrUndefined; + sortOrder: SortOrderOrUndefined; + executeFunctionOnStream: (response: FoundExceptionListItemSchema) => void; + maxSize: MaxSizeOrUndefined; +} + +/** + * Finds an exception list item within a point in time (PIT) and then calls the function + * `executeFunctionOnStream` until the maxPerPage is reached and stops. + * NOTE: This is slightly different from the saved objects version in that it takes + * an injected function, so that we avoid doing additional plumbing with generators + * to try to keep the maintenance of this machinery simpler for now. + * + * If you want to stream all results up to 10k into memory for correlation this would be: + * @example + * ```ts + * const exceptionList: ExceptionListItemSchema[] = []; + * const executeFunctionOnStream = (response: FoundExceptionListItemSchema) => { + * exceptionList = [...exceptionList, ...response.data]; + * } + * await client.findExceptionListItemPointInTimeFinder({ + * filter, + * executeFunctionOnStream, + * namespaceType, + * maxSize: 10_000, // NOTE: This is unbounded if it is "undefined" + * perPage: 1_000, // See https://github.com/elastic/kibana/issues/93770 for choice of 1k + * sortField, + * sortOrder, + * exe + * }); + * ``` + * @param filter {string} Your filter + * @param namespaceType {string} "agnostic" | "single" of your namespace + * @param perPage {number} The number of items per page. Typical value should be 1_000 here. Never go above 10_000 + * @param maxSize {number of undefined} If given a max size, this will not exceeded. Otherwise if undefined is passed down, all records will be processed. + * @param sortField {string} String of the field to sort against + * @param savedObjectsClient {Object} The saved object client + * @param sortOrder "asc" | "desc" The order to sort against + */ +export const findExceptionListItemPointInTimeFinder = async ({ + executeFunctionOnStream, + listId, + namespaceType, + savedObjectsClient, + filter, + maxSize, + perPage, + sortField, + sortOrder, +}: FindExceptionListItemPointInTimeFinderOptions): Promise => { + return findExceptionListsItemPointInTimeFinder({ + executeFunctionOnStream, + filter: filter != null ? [filter] : [], + listId: [listId], + maxSize, + namespaceType: [namespaceType], + perPage, + savedObjectsClient, + sortField, + sortOrder, + }); +}; diff --git a/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_items.test.ts b/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_items.test.ts deleted file mode 100644 index 3a2b12c358917..0000000000000 --- a/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_items.test.ts +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { LIST_ID } from '../../../common/constants.mock'; - -import { getExceptionListsItemFilter } from './find_exception_list_items'; - -describe('find_exception_list_items', () => { - describe('getExceptionListsItemFilter', () => { - test('It should create a filter with a single listId with an empty filter', () => { - const filter = getExceptionListsItemFilter({ - filter: [], - listId: [LIST_ID], - savedObjectType: ['exception-list'], - }); - expect(filter).toEqual( - '(exception-list.attributes.list_type: item AND exception-list.attributes.list_id: "some-list-id")' - ); - }); - - test('It should create a filter escaping quotes in list ids', () => { - const filter = getExceptionListsItemFilter({ - filter: [], - listId: ['list-id-"-with-quote'], - savedObjectType: ['exception-list'], - }); - expect(filter).toEqual( - '(exception-list.attributes.list_type: item AND exception-list.attributes.list_id: "list-id-\\"-with-quote")' - ); - }); - - test('It should create a filter with a single listId with a single filter', () => { - const filter = getExceptionListsItemFilter({ - filter: ['exception-list.attributes.name: "Sample Endpoint Exception List"'], - listId: [LIST_ID], - savedObjectType: ['exception-list'], - }); - expect(filter).toEqual( - '((exception-list.attributes.list_type: item AND exception-list.attributes.list_id: "some-list-id") AND exception-list.attributes.name: "Sample Endpoint Exception List")' - ); - }); - - test('It should create a filter with 2 listIds and an empty filter', () => { - const filter = getExceptionListsItemFilter({ - filter: [], - listId: ['list-1', 'list-2'], - savedObjectType: ['exception-list', 'exception-list-agnostic'], - }); - expect(filter).toEqual( - '(exception-list.attributes.list_type: item AND exception-list.attributes.list_id: "list-1") OR (exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.list_id: "list-2")' - ); - }); - - test('It should create a filter with 2 listIds and a single filter', () => { - const filter = getExceptionListsItemFilter({ - filter: ['exception-list.attributes.name: "Sample Endpoint Exception List"'], - listId: ['list-1', 'list-2'], - savedObjectType: ['exception-list', 'exception-list-agnostic'], - }); - expect(filter).toEqual( - '((exception-list.attributes.list_type: item AND exception-list.attributes.list_id: "list-1") AND exception-list.attributes.name: "Sample Endpoint Exception List") OR (exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.list_id: "list-2")' - ); - }); - - test('It should create a filter with 3 listIds and an empty filter', () => { - const filter = getExceptionListsItemFilter({ - filter: [], - listId: ['list-1', 'list-2', 'list-3'], - savedObjectType: ['exception-list', 'exception-list-agnostic', 'exception-list-agnostic'], - }); - expect(filter).toEqual( - '(exception-list.attributes.list_type: item AND exception-list.attributes.list_id: "list-1") OR (exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.list_id: "list-2") OR (exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.list_id: "list-3")' - ); - }); - - test('It should create a filter with 3 listIds and a single filter for the first item', () => { - const filter = getExceptionListsItemFilter({ - filter: ['exception-list.attributes.name: "Sample Endpoint Exception List"'], - listId: ['list-1', 'list-2', 'list-3'], - savedObjectType: ['exception-list', 'exception-list-agnostic', 'exception-list-agnostic'], - }); - expect(filter).toEqual( - '((exception-list.attributes.list_type: item AND exception-list.attributes.list_id: "list-1") AND exception-list.attributes.name: "Sample Endpoint Exception List") OR (exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.list_id: "list-2") OR (exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.list_id: "list-3")' - ); - }); - - test('It should create a filter with 3 listIds and 3 filters for each', () => { - const filter = getExceptionListsItemFilter({ - filter: [ - 'exception-list.attributes.name: "Sample Endpoint Exception List 1"', - 'exception-list.attributes.name: "Sample Endpoint Exception List 2"', - 'exception-list.attributes.name: "Sample Endpoint Exception List 3"', - ], - listId: ['list-1', 'list-2', 'list-3'], - savedObjectType: ['exception-list', 'exception-list-agnostic', 'exception-list-agnostic'], - }); - expect(filter).toEqual( - '((exception-list.attributes.list_type: item AND exception-list.attributes.list_id: "list-1") AND exception-list.attributes.name: "Sample Endpoint Exception List 1") OR ((exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.list_id: "list-2") AND exception-list.attributes.name: "Sample Endpoint Exception List 2") OR ((exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.list_id: "list-3") AND exception-list.attributes.name: "Sample Endpoint Exception List 3")' - ); - }); - }); -}); diff --git a/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_items.ts b/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_items.ts index 99298c0304c7d..5b4601dfadc22 100644 --- a/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_items.ts +++ b/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_items.ts @@ -5,13 +5,14 @@ * 2.0. */ -import { SavedObjectsClientContract } from 'kibana/server'; +import type { SavedObjectsClientContract } from 'kibana/server'; import type { FoundExceptionListItemSchema, - Id, NamespaceTypeArray, PageOrUndefined, PerPageOrUndefined, + PitOrUndefined, + SearchAfterOrUndefined, SortFieldOrUndefined, SortOrderOrUndefined, } from '@kbn/securitysolution-io-ts-list-types'; @@ -19,18 +20,13 @@ import type { EmptyStringArrayDecoded, NonEmptyStringArrayDecoded, } from '@kbn/securitysolution-io-ts-types'; -import { - SavedObjectType, - exceptionListAgnosticSavedObjectType, - exceptionListSavedObjectType, - getSavedObjectTypes, -} from '@kbn/securitysolution-list-utils'; +import { getSavedObjectTypes } from '@kbn/securitysolution-list-utils'; -import { escapeQuotes } from '../utils/escape_query'; -import { ExceptionListSoSchema } from '../../schemas/saved_objects'; +import type { ExceptionListSoSchema } from '../../schemas/saved_objects'; import { transformSavedObjectsToFoundExceptionListItem } from './utils'; import { getExceptionList } from './get_exception_list'; +import { getExceptionListsItemFilter } from './utils/get_exception_lists_item_filter'; interface FindExceptionListItemsOptions { listId: NonEmptyStringArrayDecoded; @@ -38,9 +34,11 @@ interface FindExceptionListItemsOptions { savedObjectsClient: SavedObjectsClientContract; filter: EmptyStringArrayDecoded; perPage: PerPageOrUndefined; + pit: PitOrUndefined; page: PageOrUndefined; sortField: SortFieldOrUndefined; sortOrder: SortOrderOrUndefined; + searchAfter: SearchAfterOrUndefined; } export const findExceptionListsItem = async ({ @@ -49,7 +47,9 @@ export const findExceptionListsItem = async ({ savedObjectsClient, filter, page, + pit, perPage, + searchAfter, sortField, sortOrder, }: FindExceptionListItemsOptions): Promise => { @@ -73,6 +73,8 @@ export const findExceptionListsItem = async ({ filter: getExceptionListsItemFilter({ filter, listId, savedObjectType }), page, perPage, + pit, + searchAfter, sortField, sortOrder, type: savedObjectType, @@ -82,56 +84,3 @@ export const findExceptionListsItem = async ({ }); } }; - -export const getExceptionListsItemFilter = ({ - filter, - listId, - savedObjectType, -}: { - listId: NonEmptyStringArrayDecoded; - filter: EmptyStringArrayDecoded; - savedObjectType: SavedObjectType[]; -}): string => { - return listId.reduce((accum, singleListId, index) => { - const escapedListId = escapeQuotes(singleListId); - const listItemAppend = `(${savedObjectType[index]}.attributes.list_type: item AND ${savedObjectType[index]}.attributes.list_id: "${escapedListId}")`; - const listItemAppendWithFilter = - filter[index] != null ? `(${listItemAppend} AND ${filter[index]})` : listItemAppend; - if (accum === '') { - return listItemAppendWithFilter; - } else { - return `${accum} OR ${listItemAppendWithFilter}`; - } - }, ''); -}; - -interface FindValueListExceptionListsItems { - valueListId: Id; - savedObjectsClient: SavedObjectsClientContract; - perPage: PerPageOrUndefined; - page: PageOrUndefined; - sortField: SortFieldOrUndefined; - sortOrder: SortOrderOrUndefined; -} - -export const findValueListExceptionListItems = async ({ - valueListId, - savedObjectsClient, - page, - perPage, - sortField, - sortOrder, -}: FindValueListExceptionListsItems): Promise => { - const escapedValueListId = escapeQuotes(valueListId); - const savedObjectsFindResponse = await savedObjectsClient.find({ - filter: `(exception-list.attributes.list_type: item AND exception-list.attributes.entries.list.id:"${escapedValueListId}") OR (exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.entries.list.id:"${escapedValueListId}") `, - page, - perPage, - sortField, - sortOrder, - type: [exceptionListSavedObjectType, exceptionListAgnosticSavedObjectType], - }); - return transformSavedObjectsToFoundExceptionListItem({ - savedObjectsFindResponse, - }); -}; diff --git a/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_items_point_in_time_finder.ts b/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_items_point_in_time_finder.ts new file mode 100644 index 0000000000000..d9020ee42b6ec --- /dev/null +++ b/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_items_point_in_time_finder.ts @@ -0,0 +1,142 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsClientContract } from 'kibana/server'; +import type { + FoundExceptionListItemSchema, + MaxSizeOrUndefined, + NamespaceTypeArray, + PerPageOrUndefined, + SortFieldOrUndefined, + SortOrderOrUndefined, +} from '@kbn/securitysolution-io-ts-list-types'; +import type { + EmptyStringArrayDecoded, + NonEmptyStringArrayDecoded, +} from '@kbn/securitysolution-io-ts-types'; +import { getSavedObjectTypes } from '@kbn/securitysolution-list-utils'; + +import type { ExceptionListSoSchema } from '../../schemas/saved_objects'; + +import { transformSavedObjectsToFoundExceptionListItem } from './utils'; +import { getExceptionList } from './get_exception_list'; +import { getExceptionListsItemFilter } from './utils/get_exception_lists_item_filter'; + +interface FindExceptionListItemsPointInTimeFinderOptions { + listId: NonEmptyStringArrayDecoded; + namespaceType: NamespaceTypeArray; + savedObjectsClient: SavedObjectsClientContract; + filter: EmptyStringArrayDecoded; + perPage: PerPageOrUndefined; + sortField: SortFieldOrUndefined; + sortOrder: SortOrderOrUndefined; + executeFunctionOnStream: (response: FoundExceptionListItemSchema) => void; + maxSize: MaxSizeOrUndefined; +} + +/** + * Finds exception list items within a point in time (PIT) and then calls the function + * `executeFunctionOnStream` until the maxPerPage is reached and stops. + * NOTE: This is slightly different from the saved objects version in that it takes + * an injected function, so that we avoid doing additional plumbing with generators + * to try to keep the maintenance of this machinery simpler for now. + * + * If you want to stream all results up to 10k into memory for correlation this would be: + * @example + * ```ts + * const exceptionList: ExceptionListItemSchema[] = []; + * const executeFunctionOnStream = (response: FoundExceptionListItemSchema) => { + * exceptionList = [...exceptionList, ...response.data]; + * } + * await client.findExceptionListsItemPointInTimeFinder({ + * filter, + * executeFunctionOnStream, + * namespaceType, + * maxSize: 10_000, // NOTE: This is unbounded if it is "undefined" + * perPage: 1_000, // See https://github.com/elastic/kibana/issues/93770 for choice of 1k + * sortField, + * sortOrder, + * exe + * }); + * ``` + * @param filter {string} Your filter + * @param namespaceType {string} "agnostic" | "single" of your namespace + * @param perPage {number} The number of items per page. Typical value should be 1_000 here. Never go above 10_000 + * @param maxSize {number of undefined} If given a max size, this will not exceeded. Otherwise if undefined is passed down, all records will be processed. + * @param sortField {string} String of the field to sort against + * @param savedObjectsClient {Object} The saved object client + * @param sortOrder "asc" | "desc" The order to sort against + */ +export const findExceptionListsItemPointInTimeFinder = async ({ + listId, + namespaceType, + savedObjectsClient, + executeFunctionOnStream, + maxSize, + filter, + perPage, + sortField, + sortOrder, +}: FindExceptionListItemsPointInTimeFinderOptions): Promise => { + const savedObjectType = getSavedObjectTypes({ namespaceType }); + const exceptionLists = ( + await Promise.all( + listId.map((singleListId, index) => { + return getExceptionList({ + id: undefined, + listId: singleListId, + namespaceType: namespaceType[index], + savedObjectsClient, + }); + }) + ) + ).filter((list) => list != null); + if (exceptionLists.length !== 0) { + const finder = savedObjectsClient.createPointInTimeFinder({ + filter: getExceptionListsItemFilter({ filter, listId, savedObjectType }), + perPage, + sortField, + sortOrder, + type: savedObjectType, + }); + + let count = 0; + for await (const savedObjectsFindResponse of finder.find()) { + count += savedObjectsFindResponse.saved_objects.length; + const exceptionListItem = transformSavedObjectsToFoundExceptionListItem({ + savedObjectsFindResponse, + }); + if (maxSize != null && count > maxSize) { + const diff = count - maxSize; + exceptionListItem.data = exceptionListItem.data.slice( + -exceptionListItem.data.length, + -diff + ); + executeFunctionOnStream(exceptionListItem); + try { + finder.close(); + } catch (exception) { + // This is just a pre-caution in case the finder does a throw we don't want to blow up + // the response. We have seen this within e2e test containers but nothing happen in normal + // operational conditions which is why this try/catch is here. + } + // early return since we are at our maxSize + return; + } else { + executeFunctionOnStream(exceptionListItem); + } + } + + try { + finder.close(); + } catch (exception) { + // This is just a pre-caution in case the finder does a throw we don't want to blow up + // the response. We have seen this within e2e test containers but nothing happen in normal + // operational conditions which is why this try/catch is here. + } + } +}; diff --git a/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_point_in_time_finder.ts b/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_point_in_time_finder.ts new file mode 100644 index 0000000000000..356735e773a01 --- /dev/null +++ b/x-pack/plugins/lists/server/services/exception_lists/find_exception_list_point_in_time_finder.ts @@ -0,0 +1,118 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsClientContract } from 'kibana/server'; +import type { + FilterOrUndefined, + FoundExceptionListSchema, + MaxSizeOrUndefined, + NamespaceTypeArray, + PerPageOrUndefined, + SortFieldOrUndefined, + SortOrderOrUndefined, +} from '@kbn/securitysolution-io-ts-list-types'; +import { getSavedObjectTypes } from '@kbn/securitysolution-list-utils'; + +import type { ExceptionListSoSchema } from '../../schemas/saved_objects'; + +import { transformSavedObjectsToFoundExceptionList } from './utils'; +import { getExceptionListFilter } from './utils/get_exception_list_filter'; + +interface FindExceptionListPointInTimeFinderOptions { + namespaceType: NamespaceTypeArray; + savedObjectsClient: SavedObjectsClientContract; + filter: FilterOrUndefined; + perPage: PerPageOrUndefined; + sortField: SortFieldOrUndefined; + sortOrder: SortOrderOrUndefined; + executeFunctionOnStream: (response: FoundExceptionListSchema) => void; + maxSize: MaxSizeOrUndefined; +} + +/** + * Finds an exception list within a point in time (PIT) and then calls the function + * `executeFunctionOnStream` until the maxPerPage is reached and stops. + * NOTE: This is slightly different from the saved objects version in that it takes + * an injected function, so that we avoid doing additional plumbing with generators + * to try to keep the maintenance of this machinery simpler for now. + * + * If you want to stream all results up to 10k into memory for correlation this would be: + * @example + * ```ts + * const exceptionList: ExceptionListSchema[] = []; + * const executeFunctionOnStream = (response: FoundExceptionListSchema) => { + * exceptionList = [...exceptionList, ...response.data]; + * } + * await client.findExceptionListPointInTimeFinder({ + * filter, + * executeFunctionOnStream, + * namespaceType, + * maxSize: 10_000, // NOTE: This is unbounded if it is "undefined" + * perPage: 1_000, // See https://github.com/elastic/kibana/issues/93770 for choice of 1k + * sortField, + * sortOrder, + * exe + * }); + * ``` + * @param filter {string} Your filter + * @param namespaceType {string} "agnostic" | "single" of your namespace + * @param perPage {number} The number of items per page. Typical value should be 1_000 here. Never go above 10_000 + * @param maxSize {number of undefined} If given a max size, this will not exceeded. Otherwise if undefined is passed down, all records will be processed. + * @param sortField {string} String of the field to sort against + * @param sortOrder "asc" | "desc" The order to sort against + * @param savedObjectsClient The saved objects client + */ +export const findExceptionListPointInTimeFinder = async ({ + namespaceType, + savedObjectsClient, + executeFunctionOnStream, + maxSize, + filter, + perPage, + sortField, + sortOrder, +}: FindExceptionListPointInTimeFinderOptions): Promise => { + const savedObjectTypes = getSavedObjectTypes({ namespaceType }); + const finder = savedObjectsClient.createPointInTimeFinder({ + filter: getExceptionListFilter({ filter, savedObjectTypes }), + perPage, + sortField, + sortOrder, + type: savedObjectTypes, + }); + + let count = 0; + for await (const savedObjectsFindResponse of finder.find()) { + count += savedObjectsFindResponse.saved_objects.length; + const exceptionList = transformSavedObjectsToFoundExceptionList({ + savedObjectsFindResponse, + }); + if (maxSize != null && count > maxSize) { + const diff = count - maxSize; + exceptionList.data = exceptionList.data.slice(-exceptionList.data.length, -diff); + executeFunctionOnStream(exceptionList); + try { + finder.close(); + } catch (exception) { + // This is just a pre-caution in case the finder does a throw we don't want to blow up + // the response. We have seen this within e2e test containers but nothing happen in normal + // operational conditions which is why this try/catch is here. + } + // early return since we are at our maxSize + return; + } + executeFunctionOnStream(exceptionList); + } + + try { + finder.close(); + } catch (exception) { + // This is just a pre-caution in case the finder does a throw we don't want to blow up + // the response. We have seen this within e2e test containers but nothing happen in normal + // operational conditions which is why this try/catch is here. + } +}; diff --git a/x-pack/plugins/lists/server/services/exception_lists/find_value_list_exception_list_items.ts b/x-pack/plugins/lists/server/services/exception_lists/find_value_list_exception_list_items.ts new file mode 100644 index 0000000000000..711c8f9c253d1 --- /dev/null +++ b/x-pack/plugins/lists/server/services/exception_lists/find_value_list_exception_list_items.ts @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsClientContract } from 'kibana/server'; +import type { + FoundExceptionListItemSchema, + Id, + PageOrUndefined, + PerPageOrUndefined, + PitOrUndefined, + SearchAfterOrUndefined, + SortFieldOrUndefined, + SortOrderOrUndefined, +} from '@kbn/securitysolution-io-ts-list-types'; +import { + exceptionListAgnosticSavedObjectType, + exceptionListSavedObjectType, +} from '@kbn/securitysolution-list-utils'; + +import { escapeQuotes } from '../utils/escape_query'; +import type { ExceptionListSoSchema } from '../../schemas/saved_objects'; + +import { transformSavedObjectsToFoundExceptionListItem } from './utils'; + +interface FindValueListExceptionListsItems { + valueListId: Id; + savedObjectsClient: SavedObjectsClientContract; + perPage: PerPageOrUndefined; + pit: PitOrUndefined; + page: PageOrUndefined; + searchAfter: SearchAfterOrUndefined; + sortField: SortFieldOrUndefined; + sortOrder: SortOrderOrUndefined; +} + +export const findValueListExceptionListItems = async ({ + valueListId, + savedObjectsClient, + page, + pit, + perPage, + searchAfter, + sortField, + sortOrder, +}: FindValueListExceptionListsItems): Promise => { + const escapedValueListId = escapeQuotes(valueListId); + const savedObjectsFindResponse = await savedObjectsClient.find({ + filter: `(exception-list.attributes.list_type: item AND exception-list.attributes.entries.list.id:"${escapedValueListId}") OR (exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.entries.list.id:"${escapedValueListId}") `, + page, + perPage, + pit, + searchAfter, + sortField, + sortOrder, + type: [exceptionListSavedObjectType, exceptionListAgnosticSavedObjectType], + }); + return transformSavedObjectsToFoundExceptionListItem({ + savedObjectsFindResponse, + }); +}; diff --git a/x-pack/plugins/lists/server/services/exception_lists/find_value_list_exception_list_items_point_in_time_finder.ts b/x-pack/plugins/lists/server/services/exception_lists/find_value_list_exception_list_items_point_in_time_finder.ts new file mode 100644 index 0000000000000..e854bd6128746 --- /dev/null +++ b/x-pack/plugins/lists/server/services/exception_lists/find_value_list_exception_list_items_point_in_time_finder.ts @@ -0,0 +1,117 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsClientContract } from 'kibana/server'; +import type { + FoundExceptionListItemSchema, + Id, + MaxSizeOrUndefined, + PerPageOrUndefined, + SortFieldOrUndefined, + SortOrderOrUndefined, +} from '@kbn/securitysolution-io-ts-list-types'; +import { + exceptionListAgnosticSavedObjectType, + exceptionListSavedObjectType, +} from '@kbn/securitysolution-list-utils'; + +import { escapeQuotes } from '../utils/escape_query'; +import type { ExceptionListSoSchema } from '../../schemas/saved_objects'; + +import { transformSavedObjectsToFoundExceptionListItem } from './utils'; + +interface FindValueListExceptionListsItemsPointInTimeFinder { + valueListId: Id; + savedObjectsClient: SavedObjectsClientContract; + perPage: PerPageOrUndefined; + sortField: SortFieldOrUndefined; + sortOrder: SortOrderOrUndefined; + executeFunctionOnStream: (response: FoundExceptionListItemSchema) => void; + maxSize: MaxSizeOrUndefined; +} + +/** + * Finds value lists within exception lists within a point in time (PIT) and then calls the function + * `executeFunctionOnStream` until the maxPerPage is reached and stops. + * NOTE: This is slightly different from the saved objects version in that it takes + * an injected function, so that we avoid doing additional plumbing with generators + * to try to keep the maintenance of this machinery simpler for now. + * + * If you want to stream all results up to 10k into memory for correlation this would be: + * @example + * ```ts + * const exceptionList: ExceptionListItemSchema[] = []; + * const executeFunctionOnStream = (response: FoundExceptionListItemSchema) => { + * exceptionList = [...exceptionList, ...response.data]; + * } + * await client.findValueListExceptionListItemsPointInTimeFinder({ + * valueListId, + * executeFunctionOnStream, + * namespaceType, + * maxSize: 10_000, // NOTE: This is unbounded if it is "undefined" + * perPage: 1_000, // See https://github.com/elastic/kibana/issues/93770 for choice of 1k + * sortField, + * sortOrder, + * exe + * }); + * ``` + * @param valueListId {string} Your value list id + * @param namespaceType {string} "agnostic" | "single" of your namespace + * @param perPage {number} The number of items per page. Typical value should be 1_000 here. Never go above 10_000 + * @param maxSize {number of undefined} If given a max size, this will not exceeded. Otherwise if undefined is passed down, all records will be processed. + * @param sortField {string} String of the field to sort against + * @param savedObjectsClient {object} The saved objects client + * @param sortOrder "asc" | "desc" The order to sort against + */ +export const findValueListExceptionListItemsPointInTimeFinder = async ({ + valueListId, + executeFunctionOnStream, + savedObjectsClient, + perPage, + maxSize, + sortField, + sortOrder, +}: FindValueListExceptionListsItemsPointInTimeFinder): Promise => { + const escapedValueListId = escapeQuotes(valueListId); + const finder = savedObjectsClient.createPointInTimeFinder({ + filter: `(exception-list.attributes.list_type: item AND exception-list.attributes.entries.list.id:"${escapedValueListId}") OR (exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.entries.list.id:"${escapedValueListId}") `, + perPage, + sortField, + sortOrder, + type: [exceptionListSavedObjectType, exceptionListAgnosticSavedObjectType], + }); + let count = 0; + for await (const savedObjectsFindResponse of finder.find()) { + count += savedObjectsFindResponse.saved_objects.length; + const exceptionList = transformSavedObjectsToFoundExceptionListItem({ + savedObjectsFindResponse, + }); + if (maxSize != null && count > maxSize) { + const diff = count - maxSize; + exceptionList.data = exceptionList.data.slice(-exceptionList.data.length, -diff); + executeFunctionOnStream(exceptionList); + try { + finder.close(); + } catch (exception) { + // This is just a pre-caution in case the finder does a throw we don't want to blow up + // the response. We have seen this within e2e test containers but nothing happen in normal + // operational conditions which is why this try/catch is here. + } + // early return since we are at our maxSize + return; + } + executeFunctionOnStream(exceptionList); + } + + try { + finder.close(); + } catch (exception) { + // This is just a pre-caution in case the finder does a throw we don't want to blow up + // the response. We have seen this within e2e test containers but nothing happen in normal + // operational conditions which is why this try/catch is here. + } +}; diff --git a/x-pack/plugins/lists/server/services/exception_lists/open_point_in_time.ts b/x-pack/plugins/lists/server/services/exception_lists/open_point_in_time.ts new file mode 100644 index 0000000000000..271676527dd2f --- /dev/null +++ b/x-pack/plugins/lists/server/services/exception_lists/open_point_in_time.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + SavedObjectsClientContract, + SavedObjectsOpenPointInTimeOptions, + SavedObjectsOpenPointInTimeResponse, +} from 'kibana/server'; +import type { NamespaceType } from '@kbn/securitysolution-io-ts-list-types'; +import { getSavedObjectTypes } from '@kbn/securitysolution-list-utils'; + +interface OpenPointInTimeOptions { + namespaceType: NamespaceType; + savedObjectsClient: SavedObjectsClientContract; + options: SavedObjectsOpenPointInTimeOptions | undefined; +} + +/** + * Opens a point in time (PIT) for either exception lists or exception list items. + * See: https://www.elastic.co/guide/en/elasticsearch/reference/current/point-in-time-api.html + * @params namespaceType {string} "agnostic" or "single" depending on which namespace you are targeting + * @params savedObjectsClient {object} The saved object client to delegate to + * @params options {Object} The saved object PIT options + * @return {SavedObjectsOpenPointInTimeResponse} The point in time (PIT) + */ +export const openPointInTime = async ({ + namespaceType, + savedObjectsClient, + options, +}: OpenPointInTimeOptions): Promise => { + const savedObjectType = getSavedObjectTypes({ namespaceType: [namespaceType] }); + return savedObjectsClient.openPointInTimeForType(savedObjectType, options); +}; diff --git a/x-pack/plugins/lists/server/services/exception_lists/utils/get_exception_list_filter.test.ts b/x-pack/plugins/lists/server/services/exception_lists/utils/get_exception_list_filter.test.ts new file mode 100644 index 0000000000000..9e0b06f8482c7 --- /dev/null +++ b/x-pack/plugins/lists/server/services/exception_lists/utils/get_exception_list_filter.test.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getExceptionListFilter } from './get_exception_list_filter'; + +describe('getExceptionListFilter', () => { + test('it should create a filter for agnostic lists if only searching for agnostic lists', () => { + const filter = getExceptionListFilter({ + filter: undefined, + savedObjectTypes: ['exception-list-agnostic'], + }); + expect(filter).toEqual('(exception-list-agnostic.attributes.list_type: list)'); + }); + + test('it should create a filter for agnostic lists with additional filters if only searching for agnostic lists', () => { + const filter = getExceptionListFilter({ + filter: 'exception-list-agnostic.attributes.name: "Sample Endpoint Exception List"', + savedObjectTypes: ['exception-list-agnostic'], + }); + expect(filter).toEqual( + '(exception-list-agnostic.attributes.list_type: list) AND exception-list-agnostic.attributes.name: "Sample Endpoint Exception List"' + ); + }); + + test('it should create a filter for single lists if only searching for single lists', () => { + const filter = getExceptionListFilter({ + filter: undefined, + savedObjectTypes: ['exception-list'], + }); + expect(filter).toEqual('(exception-list.attributes.list_type: list)'); + }); + + test('it should create a filter for single lists with additional filters if only searching for single lists', () => { + const filter = getExceptionListFilter({ + filter: 'exception-list.attributes.name: "Sample Endpoint Exception List"', + savedObjectTypes: ['exception-list'], + }); + expect(filter).toEqual( + '(exception-list.attributes.list_type: list) AND exception-list.attributes.name: "Sample Endpoint Exception List"' + ); + }); + + test('it should create a filter that searches for both agnostic and single lists', () => { + const filter = getExceptionListFilter({ + filter: undefined, + savedObjectTypes: ['exception-list-agnostic', 'exception-list'], + }); + expect(filter).toEqual( + '(exception-list-agnostic.attributes.list_type: list OR exception-list.attributes.list_type: list)' + ); + }); + + test('it should create a filter that searches for both agnostic and single lists with additional filters if searching for both single and agnostic lists', () => { + const filter = getExceptionListFilter({ + filter: 'exception-list-agnostic.attributes.name: "Sample Endpoint Exception List"', + savedObjectTypes: ['exception-list-agnostic', 'exception-list'], + }); + expect(filter).toEqual( + '(exception-list-agnostic.attributes.list_type: list OR exception-list.attributes.list_type: list) AND exception-list-agnostic.attributes.name: "Sample Endpoint Exception List"' + ); + }); +}); diff --git a/x-pack/plugins/lists/server/services/exception_lists/utils/get_exception_list_filter.ts b/x-pack/plugins/lists/server/services/exception_lists/utils/get_exception_list_filter.ts new file mode 100644 index 0000000000000..44a9be320755f --- /dev/null +++ b/x-pack/plugins/lists/server/services/exception_lists/utils/get_exception_list_filter.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { FilterOrUndefined } from '@kbn/securitysolution-io-ts-list-types'; +import type { SavedObjectType } from '@kbn/securitysolution-list-utils'; + +export const getExceptionListFilter = ({ + filter, + savedObjectTypes, +}: { + filter: FilterOrUndefined; + savedObjectTypes: SavedObjectType[]; +}): string => { + const listTypesFilter = savedObjectTypes + .map((type) => `${type}.attributes.list_type: list`) + .join(' OR '); + + if (filter != null) { + return `(${listTypesFilter}) AND ${filter}`; + } else return `(${listTypesFilter})`; +}; diff --git a/x-pack/plugins/lists/server/services/exception_lists/utils/get_exception_lists_item_filter.test.ts b/x-pack/plugins/lists/server/services/exception_lists/utils/get_exception_lists_item_filter.test.ts new file mode 100644 index 0000000000000..8e8b3499338dc --- /dev/null +++ b/x-pack/plugins/lists/server/services/exception_lists/utils/get_exception_lists_item_filter.test.ts @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { LIST_ID } from '../../../../common/constants.mock'; + +import { getExceptionListsItemFilter } from './get_exception_lists_item_filter'; + +describe('getExceptionListsItemFilter', () => { + test('It should create a filter with a single listId with an empty filter', () => { + const filter = getExceptionListsItemFilter({ + filter: [], + listId: [LIST_ID], + savedObjectType: ['exception-list'], + }); + expect(filter).toEqual( + '(exception-list.attributes.list_type: item AND exception-list.attributes.list_id: "some-list-id")' + ); + }); + + test('It should create a filter escaping quotes in list ids', () => { + const filter = getExceptionListsItemFilter({ + filter: [], + listId: ['list-id-"-with-quote'], + savedObjectType: ['exception-list'], + }); + expect(filter).toEqual( + '(exception-list.attributes.list_type: item AND exception-list.attributes.list_id: "list-id-\\"-with-quote")' + ); + }); + + test('It should create a filter with a single listId with a single filter', () => { + const filter = getExceptionListsItemFilter({ + filter: ['exception-list.attributes.name: "Sample Endpoint Exception List"'], + listId: [LIST_ID], + savedObjectType: ['exception-list'], + }); + expect(filter).toEqual( + '((exception-list.attributes.list_type: item AND exception-list.attributes.list_id: "some-list-id") AND exception-list.attributes.name: "Sample Endpoint Exception List")' + ); + }); + + test('It should create a filter with 2 listIds and an empty filter', () => { + const filter = getExceptionListsItemFilter({ + filter: [], + listId: ['list-1', 'list-2'], + savedObjectType: ['exception-list', 'exception-list-agnostic'], + }); + expect(filter).toEqual( + '(exception-list.attributes.list_type: item AND exception-list.attributes.list_id: "list-1") OR (exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.list_id: "list-2")' + ); + }); + + test('It should create a filter with 2 listIds and a single filter', () => { + const filter = getExceptionListsItemFilter({ + filter: ['exception-list.attributes.name: "Sample Endpoint Exception List"'], + listId: ['list-1', 'list-2'], + savedObjectType: ['exception-list', 'exception-list-agnostic'], + }); + expect(filter).toEqual( + '((exception-list.attributes.list_type: item AND exception-list.attributes.list_id: "list-1") AND exception-list.attributes.name: "Sample Endpoint Exception List") OR (exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.list_id: "list-2")' + ); + }); + + test('It should create a filter with 3 listIds and an empty filter', () => { + const filter = getExceptionListsItemFilter({ + filter: [], + listId: ['list-1', 'list-2', 'list-3'], + savedObjectType: ['exception-list', 'exception-list-agnostic', 'exception-list-agnostic'], + }); + expect(filter).toEqual( + '(exception-list.attributes.list_type: item AND exception-list.attributes.list_id: "list-1") OR (exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.list_id: "list-2") OR (exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.list_id: "list-3")' + ); + }); + + test('It should create a filter with 3 listIds and a single filter for the first item', () => { + const filter = getExceptionListsItemFilter({ + filter: ['exception-list.attributes.name: "Sample Endpoint Exception List"'], + listId: ['list-1', 'list-2', 'list-3'], + savedObjectType: ['exception-list', 'exception-list-agnostic', 'exception-list-agnostic'], + }); + expect(filter).toEqual( + '((exception-list.attributes.list_type: item AND exception-list.attributes.list_id: "list-1") AND exception-list.attributes.name: "Sample Endpoint Exception List") OR (exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.list_id: "list-2") OR (exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.list_id: "list-3")' + ); + }); + + test('It should create a filter with 3 listIds and 3 filters for each', () => { + const filter = getExceptionListsItemFilter({ + filter: [ + 'exception-list.attributes.name: "Sample Endpoint Exception List 1"', + 'exception-list.attributes.name: "Sample Endpoint Exception List 2"', + 'exception-list.attributes.name: "Sample Endpoint Exception List 3"', + ], + listId: ['list-1', 'list-2', 'list-3'], + savedObjectType: ['exception-list', 'exception-list-agnostic', 'exception-list-agnostic'], + }); + expect(filter).toEqual( + '((exception-list.attributes.list_type: item AND exception-list.attributes.list_id: "list-1") AND exception-list.attributes.name: "Sample Endpoint Exception List 1") OR ((exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.list_id: "list-2") AND exception-list.attributes.name: "Sample Endpoint Exception List 2") OR ((exception-list-agnostic.attributes.list_type: item AND exception-list-agnostic.attributes.list_id: "list-3") AND exception-list.attributes.name: "Sample Endpoint Exception List 3")' + ); + }); +}); diff --git a/x-pack/plugins/lists/server/services/exception_lists/utils/get_exception_lists_item_filter.ts b/x-pack/plugins/lists/server/services/exception_lists/utils/get_exception_lists_item_filter.ts new file mode 100644 index 0000000000000..935ae8839a71d --- /dev/null +++ b/x-pack/plugins/lists/server/services/exception_lists/utils/get_exception_lists_item_filter.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + EmptyStringArrayDecoded, + NonEmptyStringArrayDecoded, +} from '@kbn/securitysolution-io-ts-types'; +import type { SavedObjectType } from '@kbn/securitysolution-list-utils'; + +import { escapeQuotes } from '../../utils/escape_query'; + +export const getExceptionListsItemFilter = ({ + filter, + listId, + savedObjectType, +}: { + listId: NonEmptyStringArrayDecoded; + filter: EmptyStringArrayDecoded; + savedObjectType: SavedObjectType[]; +}): string => { + return listId.reduce((accum, singleListId, index) => { + const escapedListId = escapeQuotes(singleListId); + const listItemAppend = `(${savedObjectType[index]}.attributes.list_type: item AND ${savedObjectType[index]}.attributes.list_id: "${escapedListId}")`; + const listItemAppendWithFilter = + filter[index] != null ? `(${listItemAppend} AND ${filter[index]})` : listItemAppend; + if (accum === '') { + return listItemAppendWithFilter; + } else { + return `${accum} OR ${listItemAppendWithFilter}`; + } + }, ''); +}; diff --git a/x-pack/plugins/lists/server/services/exception_lists/utils/import/find_all_exception_list_item_types.ts b/x-pack/plugins/lists/server/services/exception_lists/utils/import/find_all_exception_list_item_types.ts index 272c64f161c9c..3884d12b4bb6f 100644 --- a/x-pack/plugins/lists/server/services/exception_lists/utils/import/find_all_exception_list_item_types.ts +++ b/x-pack/plugins/lists/server/services/exception_lists/utils/import/find_all_exception_list_item_types.ts @@ -14,7 +14,7 @@ import { getSavedObjectTypes } from '@kbn/securitysolution-list-utils'; import { SavedObjectsClientContract, SavedObjectsFindResponse } from 'kibana/server'; import { ExceptionListSoSchema } from '../../../../schemas/saved_objects'; -import { getExceptionListsItemFilter } from '../../find_exception_list_items'; +import { getExceptionListsItemFilter } from '../get_exception_lists_item_filter'; import { CHUNK_PARSED_OBJECT_SIZE } from '../../import_exception_list_and_items'; import { transformSavedObjectsToFoundExceptionListItem } from '..'; diff --git a/x-pack/plugins/lists/server/services/exception_lists/utils/import/find_all_exception_list_types.ts b/x-pack/plugins/lists/server/services/exception_lists/utils/import/find_all_exception_list_types.ts index 4b42787d8aaf9..51538faa66942 100644 --- a/x-pack/plugins/lists/server/services/exception_lists/utils/import/find_all_exception_list_types.ts +++ b/x-pack/plugins/lists/server/services/exception_lists/utils/import/find_all_exception_list_types.ts @@ -72,7 +72,9 @@ export const findAllListTypes = async ( namespaceType: ['agnostic'], page: undefined, perPage: CHUNK_PARSED_OBJECT_SIZE, + pit: undefined, savedObjectsClient, + searchAfter: undefined, sortField: undefined, sortOrder: undefined, }); @@ -82,7 +84,9 @@ export const findAllListTypes = async ( namespaceType: ['single'], page: undefined, perPage: CHUNK_PARSED_OBJECT_SIZE, + pit: undefined, savedObjectsClient, + searchAfter: undefined, sortField: undefined, sortOrder: undefined, }); @@ -92,7 +96,9 @@ export const findAllListTypes = async ( namespaceType: ['single', 'agnostic'], page: undefined, perPage: CHUNK_PARSED_OBJECT_SIZE, + pit: undefined, savedObjectsClient, + searchAfter: undefined, sortField: undefined, sortOrder: undefined, }); diff --git a/x-pack/plugins/lists/server/services/exception_lists/utils/index.ts b/x-pack/plugins/lists/server/services/exception_lists/utils/index.ts index ae1883f5767e5..b0b0151e5f537 100644 --- a/x-pack/plugins/lists/server/services/exception_lists/utils/index.ts +++ b/x-pack/plugins/lists/server/services/exception_lists/utils/index.ts @@ -247,6 +247,7 @@ export const transformSavedObjectsToFoundExceptionListItem = ({ ), page: savedObjectsFindResponse.page, per_page: savedObjectsFindResponse.per_page, + pit: savedObjectsFindResponse.pit_id, total: savedObjectsFindResponse.total, }; }; @@ -262,6 +263,7 @@ export const transformSavedObjectsToFoundExceptionList = ({ ), page: savedObjectsFindResponse.page, per_page: savedObjectsFindResponse.per_page, + pit: savedObjectsFindResponse.pit_id, total: savedObjectsFindResponse.total, }; }; diff --git a/x-pack/plugins/lists/server/services/items/get_list_item_by_values.ts b/x-pack/plugins/lists/server/services/items/get_list_item_by_values.ts index 2fb713526fce8..b0cdc77724ac7 100644 --- a/x-pack/plugins/lists/server/services/items/get_list_item_by_values.ts +++ b/x-pack/plugins/lists/server/services/items/get_list_item_by_values.ts @@ -30,6 +30,9 @@ export const getListItemByValues = async ({ type, value, }: GetListItemByValuesOptions): Promise => { + // TODO: Will need to address this when we switch over to + // using PIT, don't want it to get lost + // https://github.com/elastic/kibana/issues/103944 const response = await esClient.search({ body: { query: { diff --git a/x-pack/plugins/lists/server/services/items/search_list_item_by_values.ts b/x-pack/plugins/lists/server/services/items/search_list_item_by_values.ts index fb81594137861..2a39f863311b9 100644 --- a/x-pack/plugins/lists/server/services/items/search_list_item_by_values.ts +++ b/x-pack/plugins/lists/server/services/items/search_list_item_by_values.ts @@ -30,6 +30,9 @@ export const searchListItemByValues = async ({ type, value, }: SearchListItemByValuesOptions): Promise => { + // TODO: Will need to address this when we switch over to + // using PIT, don't want it to get lost + // https://github.com/elastic/kibana/issues/103944 const response = await esClient.search({ body: { query: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts index a1b0548a81d25..c57a446207873 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts @@ -7,7 +7,7 @@ import moment from 'moment'; import sinon from 'sinon'; -import { TransportResult } from '@elastic/elasticsearch'; +import type { TransportResult } from '@elastic/elasticsearch'; import { ALERT_REASON, ALERT_RULE_PARAMETERS, ALERT_UUID } from '@kbn/rule-data-utils'; import { alertsMock, AlertServicesMock } from '../../../../../alerting/server/mocks'; @@ -46,7 +46,7 @@ import { isRACAlert, getField, } from './utils'; -import { BulkResponseErrorAggregation, SearchAfterAndBulkCreateReturnType } from './types'; +import type { BulkResponseErrorAggregation, SearchAfterAndBulkCreateReturnType } from './types'; import { sampleBulkResponse, sampleEmptyBulkResponse, @@ -62,7 +62,7 @@ import { sampleAlertDocNoSortIdWithTimestamp, sampleAlertDocAADNoSortIdWithTimestamp, } from './__mocks__/es_results'; -import { ShardError } from '../../types'; +import type { ShardError } from '../../types'; import { ruleExecutionLogMock } from '../rule_execution_log/__mocks__'; const buildRuleMessage = buildRuleMessageFactory({ @@ -568,12 +568,11 @@ describe('utils', () => { test('it successfully returns array of exception list items', async () => { listMock.getExceptionListClient = () => ({ - findExceptionListsItem: jest.fn().mockResolvedValue({ - data: [getExceptionListItemSchemaMock()], - page: 1, - per_page: 10000, - total: 1, - }), + findExceptionListsItemPointInTimeFinder: jest + .fn() + .mockImplementationOnce(({ executeFunctionOnStream }) => { + executeFunctionOnStream({ data: [getExceptionListItemSchemaMock()] }); + }), } as unknown as ExceptionListClient); const client = listMock.getExceptionListClient(); const exceptions = await getExceptions({ @@ -581,38 +580,25 @@ describe('utils', () => { lists: getListArrayMock(), }); - expect(client.findExceptionListsItem).toHaveBeenCalledWith({ - listId: ['list_id_single', 'endpoint_list'], - namespaceType: ['single', 'agnostic'], - page: 1, - perPage: 10000, - filter: [], - sortOrder: undefined, - sortField: undefined, - }); - expect(exceptions).toEqual([getExceptionListItemSchemaMock()]); - }); - - test('it throws if "getExceptionListClient" fails', async () => { - const err = new Error('error fetching list'); - listMock.getExceptionListClient = () => - ({ - getExceptionList: jest.fn().mockRejectedValue(err), - } as unknown as ExceptionListClient); - - await expect(() => - getExceptions({ - client: listMock.getExceptionListClient(), - lists: getListArrayMock(), + expect(client.findExceptionListsItemPointInTimeFinder).toHaveBeenCalledWith( + expect.objectContaining({ + listId: ['list_id_single', 'endpoint_list'], + namespaceType: ['single', 'agnostic'], + perPage: 1_000, + filter: [], + maxSize: undefined, + sortOrder: undefined, + sortField: undefined, }) - ).rejects.toThrowError('unable to fetch exception list items'); + ); + expect(exceptions).toEqual([getExceptionListItemSchemaMock()]); }); - test('it throws if "findExceptionListsItem" fails', async () => { + test('it throws if "findExceptionListsItemPointInTimeFinder" fails anywhere', async () => { const err = new Error('error fetching list'); listMock.getExceptionListClient = () => ({ - findExceptionListsItem: jest.fn().mockRejectedValue(err), + findExceptionListsItemPointInTimeFinder: jest.fn().mockRejectedValue(err), } as unknown as ExceptionListClient); await expect(() => @@ -620,7 +606,9 @@ describe('utils', () => { client: listMock.getExceptionListClient(), lists: getListArrayMock(), }) - ).rejects.toThrowError('unable to fetch exception list items'); + ).rejects.toThrowError( + 'unable to fetch exception list items, message: "error fetching list" full error: "Error: error fetching list"' + ); }); test('it returns empty array if "findExceptionListsItem" returns null', async () => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index 31e43c694084a..cb1db88f78d31 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -11,10 +11,13 @@ import uuidv5 from 'uuid/v5'; import dateMath from '@elastic/datemath'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { TransportResult } from '@elastic/elasticsearch'; +import type { TransportResult } from '@elastic/elasticsearch'; import { ALERT_UUID, ALERT_RULE_UUID, ALERT_RULE_PARAMETERS } from '@kbn/rule-data-utils'; -import type { ListArray, ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types'; -import { MAX_EXCEPTION_LIST_SIZE } from '@kbn/securitysolution-list-constants'; +import type { + ListArray, + ExceptionListItemSchema, + FoundExceptionListItemSchema, +} from '@kbn/securitysolution-io-ts-list-types'; import { hasLargeValueList } from '@kbn/securitysolution-list-utils'; import { @@ -22,7 +25,7 @@ import { Privilege, RuleExecutionStatus, } from '../../../../common/detection_engine/schemas/common'; -import { +import type { ElasticsearchClient, Logger, SavedObjectsClientContract, @@ -33,8 +36,8 @@ import { AlertServices, parseDuration, } from '../../../../../alerting/server'; -import { ExceptionListClient, ListClient, ListPluginSetup } from '../../../../../lists/server'; -import { +import type { ExceptionListClient, ListClient, ListPluginSetup } from '../../../../../lists/server'; +import type { BulkResponseErrorAggregation, SignalHit, SearchAfterAndBulkCreateReturnType, @@ -47,9 +50,9 @@ import { SimpleHit, WrappedEventHit, } from './types'; -import { BuildRuleMessage } from './rule_messages'; -import { ShardError } from '../../types'; -import { +import type { BuildRuleMessage } from './rule_messages'; +import type { ShardError } from '../../types'; +import type { EqlRuleParams, MachineLearningRuleParams, QueryRuleParams, @@ -58,9 +61,9 @@ import { ThreatRuleParams, ThresholdRuleParams, } from '../schemas/rule_schemas'; -import { RACAlert, WrappedRACAlert } from '../rule_types/types'; -import { SearchTypes } from '../../../../common/detection_engine/types'; -import { IRuleExecutionLogForExecutors } from '../rule_execution_log'; +import type { RACAlert, WrappedRACAlert } from '../rule_types/types'; +import type { SearchTypes } from '../../../../common/detection_engine/types'; +import type { IRuleExecutionLogForExecutors } from '../rule_execution_log'; import { withSecuritySpan } from '../../../utils/with_security_span'; interface SortExceptionsReturn { @@ -269,18 +272,28 @@ export const getExceptions = async ({ try { const listIds = lists.map(({ list_id: listId }) => listId); const namespaceTypes = lists.map(({ namespace_type: namespaceType }) => namespaceType); - const items = await client.findExceptionListsItem({ + + // Stream the results from the Point In Time (PIT) finder into this array + let items: ExceptionListItemSchema[] = []; + const executeFunctionOnStream = (response: FoundExceptionListItemSchema): void => { + items = [...items, ...response.data]; + }; + + await client.findExceptionListsItemPointInTimeFinder({ + executeFunctionOnStream, listId: listIds, namespaceType: namespaceTypes, - page: 1, - perPage: MAX_EXCEPTION_LIST_SIZE, + perPage: 1_000, // See https://github.com/elastic/kibana/issues/93770 for choice of 1k filter: [], + maxSize: undefined, // NOTE: This is unbounded when it is "undefined" sortOrder: undefined, sortField: undefined, }); - return items != null ? items.data : []; - } catch { - throw new Error('unable to fetch exception list items'); + return items; + } catch (e) { + throw new Error( + `unable to fetch exception list items, message: "${e.message}" full error: "${e}"` + ); } } else { return [];