From 63396bfdeeb2daadadaf08e05206e8c95145135f Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Wed, 27 Jan 2021 12:16:53 -0700 Subject: [PATCH 01/32] Add searchAfter param to savedObjects.find --- .../public/saved_objects/saved_objects_client.ts | 1 + src/core/server/saved_objects/routes/find.ts | 4 ++++ .../routes/integration_tests/find.test.ts | 16 ++++++++++++++++ .../saved_objects/service/lib/repository.test.js | 13 +++++++++++++ .../saved_objects/service/lib/repository.ts | 3 +++ .../service/lib/search_dsl/search_dsl.test.ts | 10 ++++++++++ .../service/lib/search_dsl/search_dsl.ts | 3 +++ src/core/server/saved_objects/types.ts | 4 ++++ 8 files changed, 54 insertions(+) diff --git a/src/core/public/saved_objects/saved_objects_client.ts b/src/core/public/saved_objects/saved_objects_client.ts index 9c0a44b2d3da0..229696f39df6d 100644 --- a/src/core/public/saved_objects/saved_objects_client.ts +++ b/src/core/public/saved_objects/saved_objects_client.ts @@ -320,6 +320,7 @@ export class SavedObjectsClient { page: 'page', perPage: 'per_page', search: 'search', + searchAfter: 'search_after', searchFields: 'search_fields', sortField: 'sort_field', type: 'type', diff --git a/src/core/server/saved_objects/routes/find.ts b/src/core/server/saved_objects/routes/find.ts index 6ba23747cf374..90a87cf418060 100644 --- a/src/core/server/saved_objects/routes/find.ts +++ b/src/core/server/saved_objects/routes/find.ts @@ -33,6 +33,9 @@ export const registerFindRoute = (router: IRouter, { coreUsageData }: RouteDepen page: schema.number({ min: 0, defaultValue: 1 }), type: schema.oneOf([schema.string(), schema.arrayOf(schema.string())]), search: schema.maybe(schema.string()), + search_after: schema.maybe( + schema.arrayOf(schema.oneOf([schema.string(), schema.number()])) + ), default_search_operator: searchOperatorSchema, search_fields: schema.maybe( schema.oneOf([schema.string(), schema.arrayOf(schema.string())]) @@ -64,6 +67,7 @@ export const registerFindRoute = (router: IRouter, { coreUsageData }: RouteDepen page: query.page, type: Array.isArray(query.type) ? query.type : [query.type], search: query.search, + searchAfter: query.search_after, defaultSearchOperator: query.default_search_operator, searchFields: typeof query.search_fields === 'string' ? [query.search_fields] : query.search_fields, diff --git a/src/core/server/saved_objects/routes/integration_tests/find.test.ts b/src/core/server/saved_objects/routes/integration_tests/find.test.ts index 3bd2484c2e30f..d9afed0aaa484 100644 --- a/src/core/server/saved_objects/routes/integration_tests/find.test.ts +++ b/src/core/server/saved_objects/routes/integration_tests/find.test.ts @@ -223,6 +223,22 @@ describe('GET /api/saved_objects/_find', () => { ); }); + it('accepts the optional query parameter search_after', async () => { + const searchAfterValues = querystring.escape(JSON.stringify([1, 'a'])); + await supertest(httpSetup.server.listener) + .get(`/api/saved_objects/_find?type=foo&search_after=${searchAfterValues}`) + .expect(200); + + expect(savedObjectsClient.find).toHaveBeenCalledTimes(1); + + const options = savedObjectsClient.find.mock.calls[0][0]; + expect(options).toEqual( + expect.objectContaining({ + searchAfter: [1, 'a'], + }) + ); + }); + it('accepts the query parameter fields as a string', async () => { await supertest(httpSetup.server.listener) .get('/api/saved_objects/_find?type=foo&fields=title') diff --git a/src/core/server/saved_objects/service/lib/repository.test.js b/src/core/server/saved_objects/service/lib/repository.test.js index aac508fb5b909..bd9a678bee81d 100644 --- a/src/core/server/saved_objects/service/lib/repository.test.js +++ b/src/core/server/saved_objects/service/lib/repository.test.js @@ -2973,6 +2973,19 @@ describe('SavedObjectsRepository', () => { }); }); + it(`accepts searchAfter`, async () => { + const relevantOpts = { + ...commonOptions, + searchAfter: [1, 'a'], + }; + + await findSuccess(relevantOpts, namespace); + expect(getSearchDslNS.getSearchDsl).toHaveBeenCalledWith(mappings, registry, { + ...relevantOpts, + searchAfter: [1, 'a'], + }); + }); + it(`accepts KQL expression filter and passes KueryNode to getSearchDsl`, async () => { const findOpts = { namespaces: [namespace], diff --git a/src/core/server/saved_objects/service/lib/repository.ts b/src/core/server/saved_objects/service/lib/repository.ts index fcd72aa4326a2..a7d892ab99fb6 100644 --- a/src/core/server/saved_objects/service/lib/repository.ts +++ b/src/core/server/saved_objects/service/lib/repository.ts @@ -708,6 +708,7 @@ export class SavedObjectsRepository { * Query field argument for more information * @property {integer} [options.page=1] * @property {integer} [options.perPage=20] + * @property {Array} [options.searchAfter] * @property {string} [options.sortField] * @property {string} [options.sortOrder] * @property {Array} [options.fields] @@ -726,6 +727,7 @@ export class SavedObjectsRepository { hasReferenceOperator, page = FIND_DEFAULT_PAGE, perPage = FIND_DEFAULT_PER_PAGE, + searchAfter, sortField, sortOrder, fields, @@ -801,6 +803,7 @@ export class SavedObjectsRepository { searchFields, rootSearchFields, type: allowedTypes, + searchAfter, sortField, sortOrder, namespaces, diff --git a/src/core/server/saved_objects/service/lib/search_dsl/search_dsl.test.ts b/src/core/server/saved_objects/service/lib/search_dsl/search_dsl.test.ts index 9e91e585f74f0..ebac0477b2866 100644 --- a/src/core/server/saved_objects/service/lib/search_dsl/search_dsl.test.ts +++ b/src/core/server/saved_objects/service/lib/search_dsl/search_dsl.test.ts @@ -101,5 +101,15 @@ describe('getSearchDsl', () => { getSortingParams.mockReturnValue({ b: 'b' }); expect(getSearchDsl(mappings, registry, { type: 'foo' })).toEqual({ a: 'a', b: 'b' }); }); + + it('returns searchAfter if provided', () => { + getQueryParams.mockReturnValue({ a: 'a' }); + getSortingParams.mockReturnValue({ b: 'b' }); + expect(getSearchDsl(mappings, registry, { type: 'foo', searchAfter: [1, 'bar'] })).toEqual({ + a: 'a', + b: 'b', + search_after: [1, 'bar'], + }); + }); }); }); diff --git a/src/core/server/saved_objects/service/lib/search_dsl/search_dsl.ts b/src/core/server/saved_objects/service/lib/search_dsl/search_dsl.ts index 4b4fa8865ee9d..7b6c702da41f7 100644 --- a/src/core/server/saved_objects/service/lib/search_dsl/search_dsl.ts +++ b/src/core/server/saved_objects/service/lib/search_dsl/search_dsl.ts @@ -21,6 +21,7 @@ interface GetSearchDslOptions { defaultSearchOperator?: SearchOperator; searchFields?: string[]; rootSearchFields?: string[]; + searchAfter?: Array; sortField?: string; sortOrder?: string; namespaces?: string[]; @@ -41,6 +42,7 @@ export function getSearchDsl( defaultSearchOperator, searchFields, rootSearchFields, + searchAfter, sortField, sortOrder, namespaces, @@ -73,5 +75,6 @@ export function getSearchDsl( kueryNode, }), ...getSortingParams(mappings, type, sortField, sortOrder), + ...(searchAfter ? { search_after: searchAfter } : {}), }; } diff --git a/src/core/server/saved_objects/types.ts b/src/core/server/saved_objects/types.ts index d122e92aba398..bdeffd734eaa2 100644 --- a/src/core/server/saved_objects/types.ts +++ b/src/core/server/saved_objects/types.ts @@ -82,6 +82,10 @@ export interface SavedObjectsFindOptions { search?: string; /** The fields to perform the parsed query against. See Elasticsearch Simple Query String `fields` argument for more information */ searchFields?: string[]; + /** + * Use the sort values from the previous page or a Point In Time (PIT) ID to retrieve the next page of results. + */ + searchAfter?: Array; /** * The fields to perform the parsed query against. Unlike the `searchFields` argument, these are expected to be root fields and will not * be modified. If used in conjunction with `searchFields`, both are concatenated together. From 021d94decb4c3cd5fe2f2d04fb61a857873f2940 Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Mon, 1 Feb 2021 08:55:26 -0700 Subject: [PATCH 02/32] Add openPointInTimeForType to SavedObjectsClient. --- .../saved_objects_client.test.ts | 73 ++++++++++++ .../saved_objects/saved_objects_client.ts | 25 ++++ .../saved_objects_service.mock.ts | 1 + .../core_usage_stats_client.mock.ts | 1 + .../core_usage_stats_client.test.ts | 76 ++++++++++++ .../core_usage_stats_client.ts | 6 + src/core/server/index.ts | 1 + src/core/server/saved_objects/index.ts | 1 + src/core/server/saved_objects/routes/index.ts | 2 + .../routes/integration_tests/pit.test.ts | 112 ++++++++++++++++++ src/core/server/saved_objects/routes/pit.ts | 56 +++++++++ .../service/lib/repository.mock.ts | 1 + .../service/lib/repository.test.js | 90 ++++++++++++++ .../saved_objects/service/lib/repository.ts | 63 ++++++++++ .../service/lib/repository_es_client.ts | 2 + .../service/saved_objects_client.mock.ts | 1 + .../service/saved_objects_client.test.js | 15 +++ .../service/saved_objects_client.ts | 12 ++ src/core/server/saved_objects/types.ts | 16 ++- 19 files changed, 553 insertions(+), 1 deletion(-) create mode 100644 src/core/server/saved_objects/routes/integration_tests/pit.test.ts create mode 100644 src/core/server/saved_objects/routes/pit.ts diff --git a/src/core/public/saved_objects/saved_objects_client.test.ts b/src/core/public/saved_objects/saved_objects_client.test.ts index 14421c871fc2b..a177ad2f10190 100644 --- a/src/core/public/saved_objects/saved_objects_client.test.ts +++ b/src/core/public/saved_objects/saved_objects_client.test.ts @@ -487,4 +487,77 @@ describe('SavedObjectsClient', () => { `); }); }); + + describe('#openPointInTimeForType', () => { + const type = 'index-pattern'; + + beforeEach(() => { + http.fetch.mockResolvedValue({ id: 'abc123' }); + }); + + test('resolves with value returned from _pit route', async () => { + const resultP = savedObjectsClient.openPointInTimeForType(type); + await expect(resultP).resolves.toHaveProperty('id'); + + const result = await resultP; + expect(http.fetch).toHaveBeenCalledTimes(1); + expect(result.id).toBe('abc123'); + }); + + test('makes HTTP call correctly without options', () => { + savedObjectsClient.openPointInTimeForType(type); + expect(http.fetch.mock.calls[0]).toMatchInlineSnapshot(` + Array [ + "/api/saved_objects/index-pattern/_pit", + Object { + "body": "{}", + "method": "POST", + "query": undefined, + }, + ] + `); + }); + + test('makes HTTP call correctly mapping options into snake case query parameters', () => { + const options = { + keepAlive: '1m', + preference: 'foo', + }; + savedObjectsClient.openPointInTimeForType(type, options); + expect(http.fetch.mock.calls[0]).toMatchInlineSnapshot(` + Array [ + "/api/saved_objects/index-pattern/_pit", + Object { + "body": "{\\"keep_alive\\":\\"1m\\",\\"preference\\":\\"foo\\"}", + "method": "POST", + "query": undefined, + }, + ] + `); + }); + + test('ignores invalid options', () => { + savedObjectsClient.openPointInTimeForType(type, { + // @ts-expect-error + invalid: true, + keepAlive: '1m', + preference: 'foo', + }); + expect(http.fetch.mock.calls[0]).toMatchInlineSnapshot(` + Array [ + "/api/saved_objects/index-pattern/_pit", + Object { + "body": "{\\"keep_alive\\":\\"1m\\",\\"preference\\":\\"foo\\"}", + "method": "POST", + "query": undefined, + }, + ] + `); + }); + + test('concatenates multiple types in the path', () => { + savedObjectsClient.openPointInTimeForType(['a', 'b', 'c']); + expect(http.fetch.mock.calls[0][0]).toBe('/api/saved_objects/a%2Cb%2Cc/_pit'); + }); + }); }); diff --git a/src/core/public/saved_objects/saved_objects_client.ts b/src/core/public/saved_objects/saved_objects_client.ts index 229696f39df6d..a1bd1b05c7d72 100644 --- a/src/core/public/saved_objects/saved_objects_client.ts +++ b/src/core/public/saved_objects/saved_objects_client.ts @@ -16,6 +16,7 @@ import { SavedObjectsClientContract as SavedObjectsApi, SavedObjectsFindOptions as SavedObjectFindOptionsServer, SavedObjectsMigrationVersion, + SavedObjectsOpenPointInTimeOptions, } from '../../server'; import { SimpleSavedObject } from './simple_saved_object'; @@ -472,6 +473,30 @@ export class SavedObjectsClient { }); } + /** + * Opens a Point In Time (PIT) against the indices for the specified Saved Object types. + * The returned `id` can then be passed to `SavedObjects.find` to search against that PIT. + * + * @param {string|Array} type + * @param {object} [options] - {@link SavedObjectsPointInTimeOptions} + * @property {string} [options.keepAlive] + * @property {string} [options.preference] + * @returns {promise} - { id: string } + */ + public openPointInTimeForType = ( + type: string | string[], + { keepAlive, preference }: SavedObjectsOpenPointInTimeOptions = {} + ): ReturnType => { + const types = Array.isArray(type) ? type.join(',') : type; + return this.savedObjectsFetch(this.getPath([types, '_pit']), { + method: 'POST', + body: JSON.stringify({ + ...(keepAlive ? { keep_alive: keepAlive } : {}), + ...(preference ? { preference } : {}), + }), + }); + }; + private createSavedObject(options: SavedObject): SimpleSavedObject { return new SimpleSavedObject(this, options); } diff --git a/src/core/public/saved_objects/saved_objects_service.mock.ts b/src/core/public/saved_objects/saved_objects_service.mock.ts index 625ea6b5dd2da..88c80109aca71 100644 --- a/src/core/public/saved_objects/saved_objects_service.mock.ts +++ b/src/core/public/saved_objects/saved_objects_service.mock.ts @@ -18,6 +18,7 @@ const createStartContractMock = () => { bulkGet: jest.fn(), find: jest.fn(), get: jest.fn(), + openPointInTimeForType: jest.fn().mockResolvedValue({ id: 'some_pit_id' }), update: jest.fn(), }, }; diff --git a/src/core/server/core_usage_data/core_usage_stats_client.mock.ts b/src/core/server/core_usage_data/core_usage_stats_client.mock.ts index 1c9c0b8fae579..05c3a5443ef07 100644 --- a/src/core/server/core_usage_data/core_usage_stats_client.mock.ts +++ b/src/core/server/core_usage_data/core_usage_stats_client.mock.ts @@ -18,6 +18,7 @@ const createUsageStatsClientMock = () => incrementSavedObjectsDelete: jest.fn().mockResolvedValue(null), incrementSavedObjectsFind: jest.fn().mockResolvedValue(null), incrementSavedObjectsGet: jest.fn().mockResolvedValue(null), + incrementSavedObjectsOpenPit: jest.fn().mockResolvedValue(null), incrementSavedObjectsResolve: jest.fn().mockResolvedValue(null), incrementSavedObjectsUpdate: jest.fn().mockResolvedValue(null), incrementSavedObjectsImport: jest.fn().mockResolvedValue(null), diff --git a/src/core/server/core_usage_data/core_usage_stats_client.test.ts b/src/core/server/core_usage_data/core_usage_stats_client.test.ts index dc4a81adf5f8e..f2ff3c1414cf6 100644 --- a/src/core/server/core_usage_data/core_usage_stats_client.test.ts +++ b/src/core/server/core_usage_data/core_usage_stats_client.test.ts @@ -22,6 +22,7 @@ import { GET_STATS_PREFIX, RESOLVE_STATS_PREFIX, UPDATE_STATS_PREFIX, + OPEN_PIT_STATS_PREFIX, IMPORT_STATS_PREFIX, RESOLVE_IMPORT_STATS_PREFIX, EXPORT_STATS_PREFIX, @@ -670,6 +671,81 @@ describe('CoreUsageStatsClient', () => { }); }); + describe('#incrementSavedObjectsOpenPit', () => { + it('does not throw an error if repository incrementCounter operation fails', async () => { + const { usageStatsClient, repositoryMock } = setup(); + repositoryMock.incrementCounter.mockRejectedValue(new Error('Oh no!')); + + const request = httpServerMock.createKibanaRequest(); + await expect( + usageStatsClient.incrementSavedObjectsOpenPit({ + request, + } as BaseIncrementOptions) + ).resolves.toBeUndefined(); + expect(repositoryMock.incrementCounter).toHaveBeenCalled(); + }); + + it('handles falsy options appropriately', async () => { + const { usageStatsClient, repositoryMock } = setup(); + + const request = httpServerMock.createKibanaRequest(); + await usageStatsClient.incrementSavedObjectsOpenPit({ + request, + } as BaseIncrementOptions); + expect(repositoryMock.incrementCounter).toHaveBeenCalledTimes(1); + expect(repositoryMock.incrementCounter).toHaveBeenCalledWith( + CORE_USAGE_STATS_TYPE, + CORE_USAGE_STATS_ID, + [ + `${OPEN_PIT_STATS_PREFIX}.total`, + `${OPEN_PIT_STATS_PREFIX}.namespace.default.total`, + `${OPEN_PIT_STATS_PREFIX}.namespace.default.kibanaRequest.no`, + ], + incrementOptions + ); + }); + + it('handles truthy options and the default namespace string appropriately', async () => { + const { usageStatsClient, repositoryMock } = setup(DEFAULT_NAMESPACE_STRING); + + const request = httpServerMock.createKibanaRequest({ headers: firstPartyRequestHeaders }); + await usageStatsClient.incrementSavedObjectsOpenPit({ + request, + } as BaseIncrementOptions); + expect(repositoryMock.incrementCounter).toHaveBeenCalledTimes(1); + expect(repositoryMock.incrementCounter).toHaveBeenCalledWith( + CORE_USAGE_STATS_TYPE, + CORE_USAGE_STATS_ID, + [ + `${OPEN_PIT_STATS_PREFIX}.total`, + `${OPEN_PIT_STATS_PREFIX}.namespace.default.total`, + `${OPEN_PIT_STATS_PREFIX}.namespace.default.kibanaRequest.yes`, + ], + incrementOptions + ); + }); + + it('handles a non-default space appropriately', async () => { + const { usageStatsClient, repositoryMock } = setup('foo'); + + const request = httpServerMock.createKibanaRequest(); + await usageStatsClient.incrementSavedObjectsOpenPit({ + request, + } as BaseIncrementOptions); + expect(repositoryMock.incrementCounter).toHaveBeenCalledTimes(1); + expect(repositoryMock.incrementCounter).toHaveBeenCalledWith( + CORE_USAGE_STATS_TYPE, + CORE_USAGE_STATS_ID, + [ + `${OPEN_PIT_STATS_PREFIX}.total`, + `${OPEN_PIT_STATS_PREFIX}.namespace.custom.total`, + `${OPEN_PIT_STATS_PREFIX}.namespace.custom.kibanaRequest.no`, + ], + incrementOptions + ); + }); + }); + describe('#incrementSavedObjectsUpdate', () => { it('does not throw an error if repository incrementCounter operation fails', async () => { const { usageStatsClient, repositoryMock } = setup(); diff --git a/src/core/server/core_usage_data/core_usage_stats_client.ts b/src/core/server/core_usage_data/core_usage_stats_client.ts index 31f044c90ee53..33c01e8c30dd7 100644 --- a/src/core/server/core_usage_data/core_usage_stats_client.ts +++ b/src/core/server/core_usage_data/core_usage_stats_client.ts @@ -42,6 +42,7 @@ export const FIND_STATS_PREFIX = 'apiCalls.savedObjectsFind'; export const GET_STATS_PREFIX = 'apiCalls.savedObjectsGet'; export const RESOLVE_STATS_PREFIX = 'apiCalls.savedObjectsResolve'; export const UPDATE_STATS_PREFIX = 'apiCalls.savedObjectsUpdate'; +export const OPEN_PIT_STATS_PREFIX = 'apiCalls.savedObjectsOpenPit'; export const IMPORT_STATS_PREFIX = 'apiCalls.savedObjectsImport'; export const RESOLVE_IMPORT_STATS_PREFIX = 'apiCalls.savedObjectsResolveImportErrors'; export const EXPORT_STATS_PREFIX = 'apiCalls.savedObjectsExport'; @@ -56,6 +57,7 @@ const ALL_COUNTER_FIELDS = [ ...getFieldsForCounter(GET_STATS_PREFIX), ...getFieldsForCounter(RESOLVE_STATS_PREFIX), ...getFieldsForCounter(UPDATE_STATS_PREFIX), + ...getFieldsForCounter(OPEN_PIT_STATS_PREFIX), // Saved Objects Management APIs ...getFieldsForCounter(IMPORT_STATS_PREFIX), `${IMPORT_STATS_PREFIX}.createNewCopiesEnabled.yes`, @@ -133,6 +135,10 @@ export class CoreUsageStatsClient { await this.updateUsageStats([], UPDATE_STATS_PREFIX, options); } + public async incrementSavedObjectsOpenPit(options: BaseIncrementOptions) { + await this.updateUsageStats([], OPEN_PIT_STATS_PREFIX, options); + } + public async incrementSavedObjectsImport(options: IncrementSavedObjectsImportOptions) { const { createNewCopies, overwrite } = options; const counterFieldNames = [ diff --git a/src/core/server/index.ts b/src/core/server/index.ts index 6f478004c204e..074b267b79ae1 100644 --- a/src/core/server/index.ts +++ b/src/core/server/index.ts @@ -277,6 +277,7 @@ export { SavedObjectsImportUnsupportedTypeError, SavedObjectMigrationContext, SavedObjectsMigrationLogger, + SavedObjectsOpenPointInTimeOptions, SavedObjectsRawDoc, SavedObjectsRawDocParseOptions, SavedObjectSanitizedDoc, diff --git a/src/core/server/saved_objects/index.ts b/src/core/server/saved_objects/index.ts index 48ae57c509daf..726a5fc5c28cb 100644 --- a/src/core/server/saved_objects/index.ts +++ b/src/core/server/saved_objects/index.ts @@ -85,6 +85,7 @@ export { export { SavedObjectsNamespaceType, + SavedObjectsOpenPointInTimeOptions, SavedObjectStatusMeta, SavedObjectsType, SavedObjectsTypeManagementDefinition, diff --git a/src/core/server/saved_objects/routes/index.ts b/src/core/server/saved_objects/routes/index.ts index 930e02de7657a..b6c3226da9688 100644 --- a/src/core/server/saved_objects/routes/index.ts +++ b/src/core/server/saved_objects/routes/index.ts @@ -21,6 +21,7 @@ import { registerBulkGetRoute } from './bulk_get'; import { registerBulkCreateRoute } from './bulk_create'; import { registerBulkUpdateRoute } from './bulk_update'; import { registerLogLegacyImportRoute } from './log_legacy_import'; +import { registerPointInTimeRoute } from './pit'; import { registerExportRoute } from './export'; import { registerImportRoute } from './import'; import { registerResolveImportErrorsRoute } from './resolve_import_errors'; @@ -51,6 +52,7 @@ export function registerRoutes({ registerBulkCreateRoute(router, { coreUsageData }); registerBulkUpdateRoute(router, { coreUsageData }); registerLogLegacyImportRoute(router, logger); + registerPointInTimeRoute(router, { coreUsageData }); registerExportRoute(router, { config, coreUsageData }); registerImportRoute(router, { config, coreUsageData }); registerResolveImportErrorsRoute(router, { config, coreUsageData }); diff --git a/src/core/server/saved_objects/routes/integration_tests/pit.test.ts b/src/core/server/saved_objects/routes/integration_tests/pit.test.ts new file mode 100644 index 0000000000000..d8cdb0cb79534 --- /dev/null +++ b/src/core/server/saved_objects/routes/integration_tests/pit.test.ts @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * and the Server Side Public License, v 1; you may not use this file except in + * compliance with, at your election, the Elastic License or the Server Side + * Public License, v 1. + */ + +import supertest from 'supertest'; + +import { UnwrapPromise } from '@kbn/utility-types'; +import { registerPointInTimeRoute } from '../pit'; +import { savedObjectsClientMock } from '../../../../../core/server/mocks'; +import { CoreUsageStatsClient } from '../../../core_usage_data'; +import { coreUsageStatsClientMock } from '../../../core_usage_data/core_usage_stats_client.mock'; +import { coreUsageDataServiceMock } from '../../../core_usage_data/core_usage_data_service.mock'; +import { setupServer } from '../test_utils'; + +type SetupServerReturn = UnwrapPromise>; + +describe('POST /api/saved_objects/{type}/_pit', () => { + let server: SetupServerReturn['server']; + let httpSetup: SetupServerReturn['httpSetup']; + let handlerContext: SetupServerReturn['handlerContext']; + let savedObjectsClient: ReturnType; + let coreUsageStatsClient: jest.Mocked; + + const clientResponse = { + id: 'abc123', + }; + + beforeEach(async () => { + ({ server, httpSetup, handlerContext } = await setupServer()); + savedObjectsClient = handlerContext.savedObjects.client; + + savedObjectsClient.openPointInTimeForType.mockResolvedValue(clientResponse); + + const router = httpSetup.createRouter('/api/saved_objects/'); + coreUsageStatsClient = coreUsageStatsClientMock.create(); + coreUsageStatsClient.incrementSavedObjectsOpenPit.mockRejectedValue(new Error('Oh no!')); // intentionally throw this error, which is swallowed, so we can assert that the operation does not fail + const coreUsageData = coreUsageDataServiceMock.createSetupContract(coreUsageStatsClient); + registerPointInTimeRoute(router, { coreUsageData }); + + await server.start(); + }); + + afterEach(async () => { + await server.stop(); + }); + + it('works without optional body settings', async () => { + const result = await supertest(httpSetup.server.listener) + .post('/api/saved_objects/index-pattern/_pit') + .expect(200); + + expect(result.body).toEqual(clientResponse); + }); + + it('records usage stats', async () => { + await supertest(httpSetup.server.listener) + .post('/api/saved_objects/index-pattern/_pit') + .expect(200); + + expect(coreUsageStatsClient.incrementSavedObjectsOpenPit).toHaveBeenCalledWith({ + request: expect.anything(), + }); + }); + + it('calls savedObjectClient.openPointInTimeForType with the type', async () => { + await supertest(httpSetup.server.listener) + .post('/api/saved_objects/index-pattern/_pit') + .expect(200); + + expect(savedObjectsClient.openPointInTimeForType).toHaveBeenCalledTimes(1); + + const options = savedObjectsClient.openPointInTimeForType.mock.calls[0][0]; + expect(options).toEqual(['index-pattern']); + }); + + it('handles multiple comma-separated types in the path', async () => { + await supertest(httpSetup.server.listener).post('/api/saved_objects/a,b,c/_pit').expect(200); + + expect(savedObjectsClient.openPointInTimeForType).toHaveBeenCalledTimes(1); + + const options = savedObjectsClient.openPointInTimeForType.mock.calls[0][0]; + expect(options).toEqual(['a', 'b', 'c']); + }); + + it('accepts keep_alive in the body', async () => { + await supertest(httpSetup.server.listener) + .post('/api/saved_objects/index-pattern/_pit') + .send({ keepAlive: '1m' }) + .expect(200); + + expect(savedObjectsClient.openPointInTimeForType).toHaveBeenCalledTimes(1); + + const options = savedObjectsClient.openPointInTimeForType.mock.calls[0][1]; + expect(options).toEqual({ keepAlive: '1m' }); + }); + + it('accepts preference in the body', async () => { + await supertest(httpSetup.server.listener) + .post('/api/saved_objects/index-pattern/_pit') + .send({ preference: 'foo' }) + .expect(200); + + expect(savedObjectsClient.openPointInTimeForType).toHaveBeenCalledTimes(1); + + const options = savedObjectsClient.openPointInTimeForType.mock.calls[0][1]; + expect(options).toEqual({ preference: 'foo' }); + }); +}); diff --git a/src/core/server/saved_objects/routes/pit.ts b/src/core/server/saved_objects/routes/pit.ts new file mode 100644 index 0000000000000..2ae37be556ab9 --- /dev/null +++ b/src/core/server/saved_objects/routes/pit.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * and the Server Side Public License, v 1; you may not use this file except in + * compliance with, at your election, the Elastic License or the Server Side + * Public License, v 1. + */ + +import { schema } from '@kbn/config-schema'; +import { IRouter } from '../../http'; +import { CoreUsageDataSetup } from '../../core_usage_data'; + +interface RouteDependencies { + coreUsageData: CoreUsageDataSetup; +} + +const TYPE_DELIMITER = ','; + +export const registerPointInTimeRoute = (router: IRouter, { coreUsageData }: RouteDependencies) => { + router.post( + { + path: '/{types}/_pit', + validate: { + params: schema.object({ + types: schema.string(), + }), + body: schema.nullable( + schema.object( + { + keepAlive: schema.maybe(schema.string()), + preference: schema.maybe(schema.string()), + }, + { + defaultValue: {}, + } + ) + ), + }, + }, + router.handleLegacyErrors(async (context, req, res) => { + const { types } = req.params; + + const usageStatsClient = coreUsageData.getClient(); + usageStatsClient.incrementSavedObjectsOpenPit({ request: req }).catch(() => {}); + + const result = await context.core.savedObjects.client.openPointInTimeForType( + types.split(TYPE_DELIMITER), + { + ...(req.body?.keepAlive ? { keepAlive: req.body.keepAlive } : {}), + ...(req.body?.preference ? { preference: req.body.preference } : {}), + } + ); + return res.ok({ body: result }); + }) + ); +}; diff --git a/src/core/server/saved_objects/service/lib/repository.mock.ts b/src/core/server/saved_objects/service/lib/repository.mock.ts index c853e208f27aa..c574999b4b95f 100644 --- a/src/core/server/saved_objects/service/lib/repository.mock.ts +++ b/src/core/server/saved_objects/service/lib/repository.mock.ts @@ -17,6 +17,7 @@ const create = (): jest.Mocked => ({ bulkGet: jest.fn(), find: jest.fn(), get: jest.fn(), + openPointInTimeForType: jest.fn().mockResolvedValue({ id: 'some_pit_id' }), resolve: jest.fn(), update: jest.fn(), addToNamespaces: jest.fn(), diff --git a/src/core/server/saved_objects/service/lib/repository.test.js b/src/core/server/saved_objects/service/lib/repository.test.js index bd9a678bee81d..4ceb045ad174f 100644 --- a/src/core/server/saved_objects/service/lib/repository.test.js +++ b/src/core/server/saved_objects/service/lib/repository.test.js @@ -4406,4 +4406,94 @@ describe('SavedObjectsRepository', () => { }); }); }); + + describe('#openPointInTimeForType', () => { + const type = 'index-pattern'; + + const generateResults = (id) => ({ id: id || null }); + const successResponse = async (type, options) => { + client.openPointInTime.mockResolvedValueOnce( + elasticsearchClientMock.createSuccessTransportRequestPromise(generateResults()) + ); + const result = await savedObjectsRepository.openPointInTimeForType(type, options); + expect(client.openPointInTime).toHaveBeenCalledTimes(1); + return result; + }; + + describe('client calls', () => { + it(`should use the ES search action`, async () => { + await successResponse(type); + expect(client.openPointInTime).toHaveBeenCalledTimes(1); + }); + + it(`accepts preference`, async () => { + await successResponse(type, { preference: 'pref' }); + expect(client.openPointInTime).toHaveBeenCalledWith( + expect.objectContaining({ + preference: 'pref', + }), + expect.anything() + ); + }); + + it(`accepts keepAlive`, async () => { + await successResponse(type, { keepAlive: '1m' }); + expect(client.openPointInTime).toHaveBeenCalledWith( + expect.objectContaining({ + keep_alive: '1m', + }), + expect.anything() + ); + }); + + it(`defaults keepAlive to 5m`, async () => { + await successResponse(type); + expect(client.openPointInTime).toHaveBeenCalledWith( + expect.objectContaining({ + keep_alive: '5m', + }), + expect.anything() + ); + }); + }); + + describe('errors', () => { + const expectNotFoundError = async (types) => { + await expect(savedObjectsRepository.openPointInTimeForType(types)).rejects.toThrowError( + createGenericNotFoundError() + ); + }; + + it(`throws when ES is unable to find the index`, async () => { + client.openPointInTime.mockResolvedValueOnce( + elasticsearchClientMock.createSuccessTransportRequestPromise({}, { statusCode: 404 }) + ); + await expectNotFoundError(type); + expect(client.openPointInTime).toHaveBeenCalledTimes(1); + }); + + it(`should return generic not found error when attempting to find only invalid or hidden types`, async () => { + const test = async (types) => { + await expectNotFoundError(types); + expect(client.openPointInTime).not.toHaveBeenCalled(); + }; + + await test('unknownType'); + await test(HIDDEN_TYPE); + await test(['unknownType', HIDDEN_TYPE]); + }); + }); + + describe('returns', () => { + it(`returns id in the expected format`, async () => { + const id = 'abc123'; + const results = generateResults(id); + client.openPointInTime.mockResolvedValueOnce( + elasticsearchClientMock.createSuccessTransportRequestPromise(results) + ); + const response = await savedObjectsRepository.openPointInTimeForType(type); + expect(response).toEqual({ id }); + }); + }); + }); }); diff --git a/src/core/server/saved_objects/service/lib/repository.ts b/src/core/server/saved_objects/service/lib/repository.ts index a7d892ab99fb6..86fce9370e2a8 100644 --- a/src/core/server/saved_objects/service/lib/repository.ts +++ b/src/core/server/saved_objects/service/lib/repository.ts @@ -54,6 +54,7 @@ import { SavedObjectsBaseOptions, SavedObjectsFindOptions, SavedObjectsMigrationVersion, + SavedObjectsOpenPointInTimeOptions, MutatingOperationRefreshSetting, } from '../../types'; import { LegacyUrlAlias, LEGACY_URL_ALIAS_TYPE } from '../../object_types'; @@ -1767,6 +1768,68 @@ export class SavedObjectsRepository { }; } + /** + * Opens a Point In Time (PIT) against the indices for the specified Saved Object types. + * The returned `id` can then be passed to `SavedObjects.find` to search against that PIT. + * + * @example + * ```ts + * const repository = coreStart.savedObjects.createInternalRepository(); + * + * const { id } = await repository.openPointInTimeForType( + * type: 'index-pattern', + * { keepAlive: '1m' }, + * ); + * + * const response = await repository.find({ + * type: 'index-pattern', + * search: 'foo*', + * sortField: 'name', + * sortOrder: 'desc', + * pit: { + * id: 'abc123', + * keepAlive: '1m', + * }, + * searchAfter: [1234, 'abcd'], + * }); + * ``` + * + * @param {string|Array} type + * @param {object} [options] - {@link SavedObjectsPointInTimeOptions} + * @property {string} [options.keepAlive] + * @property {string} [options.preference] + * @returns {promise} - { id: string } + */ + async openPointInTimeForType( + type: string | string[], + { keepAlive, preference }: SavedObjectsOpenPointInTimeOptions = {} + ): Promise<{ id: string }> { + const defaultKeepAlive = '5m'; + + const types = Array.isArray(type) ? type : [type]; + const allowedTypes = types.filter((t) => this._allowedTypes.includes(t)); + if (allowedTypes.length === 0) { + throw SavedObjectsErrorHelpers.createGenericNotFoundError(); + } + + const esOptions = { + index: this.getIndicesForTypes(allowedTypes), + keep_alive: keepAlive || defaultKeepAlive, + ...(preference ? { preference } : {}), + }; + + const { body, statusCode } = await this.client.openPointInTime<{ id: string }>(esOptions, { + ignore: [404], + }); + if (statusCode === 404) { + throw SavedObjectsErrorHelpers.createGenericNotFoundError(); + } + + return { + id: body.id, + }; + } + /** * Returns index specified by the given type or the default index * diff --git a/src/core/server/saved_objects/service/lib/repository_es_client.ts b/src/core/server/saved_objects/service/lib/repository_es_client.ts index dae72819726ad..6a601b1ed0c83 100644 --- a/src/core/server/saved_objects/service/lib/repository_es_client.ts +++ b/src/core/server/saved_objects/service/lib/repository_es_client.ts @@ -14,11 +14,13 @@ import { decorateEsError } from './decorate_es_error'; const methods = [ 'bulk', + 'closePointInTime', 'create', 'delete', 'get', 'index', 'mget', + 'openPointInTime', 'search', 'update', 'updateByQuery', diff --git a/src/core/server/saved_objects/service/saved_objects_client.mock.ts b/src/core/server/saved_objects/service/saved_objects_client.mock.ts index 72f5561aa7027..79fb068319838 100644 --- a/src/core/server/saved_objects/service/saved_objects_client.mock.ts +++ b/src/core/server/saved_objects/service/saved_objects_client.mock.ts @@ -20,6 +20,7 @@ const create = () => bulkGet: jest.fn(), find: jest.fn(), get: jest.fn(), + openPointInTimeForType: jest.fn().mockResolvedValue({ id: 'some_pit_id' }), resolve: jest.fn(), update: jest.fn(), addToNamespaces: jest.fn(), diff --git a/src/core/server/saved_objects/service/saved_objects_client.test.js b/src/core/server/saved_objects/service/saved_objects_client.test.js index 45b0cf70b0dc6..b17582b4b286b 100644 --- a/src/core/server/saved_objects/service/saved_objects_client.test.js +++ b/src/core/server/saved_objects/service/saved_objects_client.test.js @@ -115,6 +115,21 @@ test(`#get`, async () => { expect(result).toBe(returnValue); }); +test(`#openPointInTimeForType`, async () => { + const returnValue = Symbol(); + const mockRepository = { + openPointInTimeForType: jest.fn().mockResolvedValue(returnValue), + }; + const client = new SavedObjectsClient(mockRepository); + + const type = Symbol(); + const options = Symbol(); + const result = await client.openPointInTimeForType(type, options); + + expect(mockRepository.openPointInTimeForType).toHaveBeenCalledWith(type, options); + expect(result).toBe(returnValue); +}); + test(`#resolve`, async () => { const returnValue = Symbol(); const mockRepository = { diff --git a/src/core/server/saved_objects/service/saved_objects_client.ts b/src/core/server/saved_objects/service/saved_objects_client.ts index b90540fbfa971..d5e447c0de209 100644 --- a/src/core/server/saved_objects/service/saved_objects_client.ts +++ b/src/core/server/saved_objects/service/saved_objects_client.ts @@ -15,6 +15,7 @@ import { SavedObjectsBaseOptions, MutatingOperationRefreshSetting, SavedObjectsFindOptions, + SavedObjectsOpenPointInTimeOptions, } from '../types'; import { SavedObjectsErrorHelpers } from './lib/errors'; @@ -504,4 +505,15 @@ export class SavedObjectsClient { ) { return await this._repository.removeReferencesTo(type, id, options); } + + /** + * Opens a Point In Time (PIT) against the indices for the specified Saved Object types. + * The returned `id` can then be passed to `SavedObjects.find` to search against that PIT. + */ + async openPointInTimeForType( + type: string | string[], + options: SavedObjectsOpenPointInTimeOptions + ) { + return await this._repository.openPointInTimeForType(type, options); + } } diff --git a/src/core/server/saved_objects/types.ts b/src/core/server/saved_objects/types.ts index bdeffd734eaa2..c23509884cccc 100644 --- a/src/core/server/saved_objects/types.ts +++ b/src/core/server/saved_objects/types.ts @@ -83,7 +83,7 @@ export interface SavedObjectsFindOptions { /** The fields to perform the parsed query against. See Elasticsearch Simple Query String `fields` argument for more information */ searchFields?: string[]; /** - * Use the sort values from the previous page or a Point In Time (PIT) ID to retrieve the next page of results. + * Use the sort values from the previous page to retrieve the next page of results. */ searchAfter?: Array; /** @@ -120,6 +120,20 @@ export interface SavedObjectsFindOptions { preference?: string; } +/** + * @public + */ +export interface SavedObjectsOpenPointInTimeOptions { + /** + * Optionally specify how long ES should keep the PIT alive until the next request. Defaults to `5m`. + */ + keepAlive?: string; + /** + * An optional ES preference value to be used for the query. + */ + preference?: string; +} + /** * * @public From b91ac5b0d0187c362ce62a07dac3ef7ad2cf068f Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Mon, 1 Feb 2021 15:02:11 -0700 Subject: [PATCH 03/32] Add pit option to savedObjects.find. --- .../saved_objects_client.test.ts | 7 ++++ .../saved_objects/saved_objects_client.ts | 10 ++++-- src/core/server/elasticsearch/client/types.ts | 1 + src/core/server/saved_objects/index.ts | 1 - src/core/server/saved_objects/routes/find.ts | 9 ++++++ .../routes/integration_tests/find.test.ts | 19 +++++++++++ .../service/lib/repository.test.js | 13 ++++++++ .../saved_objects/service/lib/repository.ts | 15 +++++++-- .../service/lib/search_dsl/pit_params.test.ts | 32 +++++++++++++++++++ .../service/lib/search_dsl/pit_params.ts | 25 +++++++++++++++ .../service/lib/search_dsl/search_dsl.ts | 4 +++ .../service/saved_objects_client.ts | 16 +++++++++- src/core/server/saved_objects/types.ts | 14 ++------ 13 files changed, 147 insertions(+), 19 deletions(-) create mode 100644 src/core/server/saved_objects/service/lib/search_dsl/pit_params.test.ts create mode 100644 src/core/server/saved_objects/service/lib/search_dsl/pit_params.ts diff --git a/src/core/public/saved_objects/saved_objects_client.test.ts b/src/core/public/saved_objects/saved_objects_client.test.ts index a177ad2f10190..66828587779d5 100644 --- a/src/core/public/saved_objects/saved_objects_client.test.ts +++ b/src/core/public/saved_objects/saved_objects_client.test.ts @@ -428,7 +428,9 @@ describe('SavedObjectsClient', () => { hasReference: { id: '1', type: 'reference' }, page: 10, perPage: 100, + pit: { id: 'abc', keepAlive: '1m' }, search: 'what is the meaning of life?|life', + searchAfter: [123, 'abc'], searchFields: ['title^5', 'body'], sortField: 'sort_field', type: 'index-pattern', @@ -450,7 +452,12 @@ describe('SavedObjectsClient', () => { "has_reference": "{\\"id\\":\\"1\\",\\"type\\":\\"reference\\"}", "page": 10, "per_page": 100, + "pit": "{\\"id\\":\\"abc\\",\\"keep_alive\\":\\"1m\\"}", "search": "what is the meaning of life?|life", + "search_after": Array [ + 123, + "abc", + ], "search_fields": Array [ "title^5", "body", diff --git a/src/core/public/saved_objects/saved_objects_client.ts b/src/core/public/saved_objects/saved_objects_client.ts index a1bd1b05c7d72..e8580543169b9 100644 --- a/src/core/public/saved_objects/saved_objects_client.ts +++ b/src/core/public/saved_objects/saved_objects_client.ts @@ -106,6 +106,7 @@ export interface SavedObjectsFindResponsePublic extends SavedObject total: number; perPage: number; page: number; + pit_id?: string; } interface BatchQueueEntry { @@ -320,6 +321,7 @@ export class SavedObjectsClient { hasReferenceOperator: 'has_reference_operator', page: 'page', perPage: 'per_page', + pit: 'pit', search: 'search', searchAfter: 'search_after', searchFields: 'search_fields', @@ -336,11 +338,14 @@ export class SavedObjectsClient { any >; - // `has_references` is a structured object. we need to stringify it before sending it, as `fetch` - // is not doing it implicitly. + // `has_references` and `pit` are structured objects. We need to stringify before sending, + // as `fetch` is not doing it implicitly. if (query.has_reference) { query.has_reference = JSON.stringify(query.has_reference); } + if (query.pit) { + query.pit = JSON.stringify(renameKeys({ id: 'id', keepAlive: 'keep_alive' }, query.pit)); + } const request: ReturnType = this.savedObjectsFetch(path, { method: 'GET', @@ -356,6 +361,7 @@ export class SavedObjectsClient { total: 'total', per_page: 'perPage', page: 'page', + pit_id: 'pit_id', }, { ...resp, diff --git a/src/core/server/elasticsearch/client/types.ts b/src/core/server/elasticsearch/client/types.ts index 2e99398efdfba..11521becb46af 100644 --- a/src/core/server/elasticsearch/client/types.ts +++ b/src/core/server/elasticsearch/client/types.ts @@ -100,6 +100,7 @@ export interface SearchResponse { }>; }; aggregations?: any; + pit_id?: string; } /** diff --git a/src/core/server/saved_objects/index.ts b/src/core/server/saved_objects/index.ts index 726a5fc5c28cb..48ae57c509daf 100644 --- a/src/core/server/saved_objects/index.ts +++ b/src/core/server/saved_objects/index.ts @@ -85,7 +85,6 @@ export { export { SavedObjectsNamespaceType, - SavedObjectsOpenPointInTimeOptions, SavedObjectStatusMeta, SavedObjectsType, SavedObjectsTypeManagementDefinition, diff --git a/src/core/server/saved_objects/routes/find.ts b/src/core/server/saved_objects/routes/find.ts index 90a87cf418060..92550e4d4f3ad 100644 --- a/src/core/server/saved_objects/routes/find.ts +++ b/src/core/server/saved_objects/routes/find.ts @@ -16,6 +16,10 @@ interface RouteDependencies { } export const registerFindRoute = (router: IRouter, { coreUsageData }: RouteDependencies) => { + const pitSchema = schema.object({ + id: schema.string(), + keep_alive: schema.maybe(schema.string()), + }); const referenceSchema = schema.object({ type: schema.string(), id: schema.string(), @@ -36,6 +40,7 @@ export const registerFindRoute = (router: IRouter, { coreUsageData }: RouteDepen search_after: schema.maybe( schema.arrayOf(schema.oneOf([schema.string(), schema.number()])) ), + pit: schema.maybe(pitSchema), default_search_operator: searchOperatorSchema, search_fields: schema.maybe( schema.oneOf([schema.string(), schema.arrayOf(schema.string())]) @@ -68,6 +73,10 @@ export const registerFindRoute = (router: IRouter, { coreUsageData }: RouteDepen type: Array.isArray(query.type) ? query.type : [query.type], search: query.search, searchAfter: query.search_after, + pit: query.pit && { + id: query.pit.id, + keepAlive: query.pit.keep_alive, + }, defaultSearchOperator: query.default_search_operator, searchFields: typeof query.search_fields === 'string' ? [query.search_fields] : query.search_fields, diff --git a/src/core/server/saved_objects/routes/integration_tests/find.test.ts b/src/core/server/saved_objects/routes/integration_tests/find.test.ts index d9afed0aaa484..0e22c153af3e0 100644 --- a/src/core/server/saved_objects/routes/integration_tests/find.test.ts +++ b/src/core/server/saved_objects/routes/integration_tests/find.test.ts @@ -239,6 +239,25 @@ describe('GET /api/saved_objects/_find', () => { ); }); + it('accepts the optional query parameter pit', async () => { + const pitValues = querystring.escape(JSON.stringify({ id: 'abc', keep_alive: '1m' })); + await supertest(httpSetup.server.listener) + .get(`/api/saved_objects/_find?type=foo&pit=${pitValues}`) + .expect(200); + + expect(savedObjectsClient.find).toHaveBeenCalledTimes(1); + + const options = savedObjectsClient.find.mock.calls[0][0]; + expect(options).toEqual( + expect.objectContaining({ + pit: { + id: 'abc', + keepAlive: '1m', + }, + }) + ); + }); + it('accepts the query parameter fields as a string', async () => { await supertest(httpSetup.server.listener) .get('/api/saved_objects/_find?type=foo&fields=title') diff --git a/src/core/server/saved_objects/service/lib/repository.test.js b/src/core/server/saved_objects/service/lib/repository.test.js index 4ceb045ad174f..781637138c7ff 100644 --- a/src/core/server/saved_objects/service/lib/repository.test.js +++ b/src/core/server/saved_objects/service/lib/repository.test.js @@ -2986,6 +2986,19 @@ describe('SavedObjectsRepository', () => { }); }); + it(`accepts pit`, async () => { + const relevantOpts = { + ...commonOptions, + pit: { id: 'abc123', keepAlive: '1m' }, + }; + + await findSuccess(relevantOpts, namespace); + expect(getSearchDslNS.getSearchDsl).toHaveBeenCalledWith(mappings, registry, { + ...relevantOpts, + pit: { id: 'abc123', keepAlive: '1m' }, + }); + }); + it(`accepts KQL expression filter and passes KueryNode to getSearchDsl`, async () => { const findOpts = { namespaces: [namespace], diff --git a/src/core/server/saved_objects/service/lib/repository.ts b/src/core/server/saved_objects/service/lib/repository.ts index 86fce9370e2a8..b5215e8887714 100644 --- a/src/core/server/saved_objects/service/lib/repository.ts +++ b/src/core/server/saved_objects/service/lib/repository.ts @@ -36,6 +36,7 @@ import { SavedObjectsCreateOptions, SavedObjectsFindResponse, SavedObjectsFindResult, + SavedObjectsOpenPointInTimeOptions, SavedObjectsUpdateOptions, SavedObjectsUpdateResponse, SavedObjectsBulkUpdateObject, @@ -54,7 +55,6 @@ import { SavedObjectsBaseOptions, SavedObjectsFindOptions, SavedObjectsMigrationVersion, - SavedObjectsOpenPointInTimeOptions, MutatingOperationRefreshSetting, } from '../../types'; import { LegacyUrlAlias, LEGACY_URL_ALIAS_TYPE } from '../../object_types'; @@ -728,6 +728,7 @@ export class SavedObjectsRepository { hasReferenceOperator, page = FIND_DEFAULT_PAGE, perPage = FIND_DEFAULT_PER_PAGE, + pit, searchAfter, sortField, sortOrder, @@ -790,18 +791,25 @@ export class SavedObjectsRepository { } const esOptions = { - index: this.getIndicesForTypes(allowedTypes), + // If `pit` is provided, we drop the `index` and `preference` as those are already + // associated with the PIT in ES, and will otherwise return a 400. + ...(pit + ? {} + : { + index: this.getIndicesForTypes(allowedTypes), + preference, + }), size: perPage, from: perPage * (page - 1), _source: includedFields(type, fields), rest_total_hits_as_int: true, - preference, body: { seq_no_primary_term: true, ...getSearchDsl(this._mappings, this._registry, { search, defaultSearchOperator, searchFields, + pit, rootSearchFields, type: allowedTypes, searchAfter, @@ -840,6 +848,7 @@ export class SavedObjectsRepository { score: (hit as any)._score, }) ), + ...(body.pit_id ? { pit_id: body.pit_id } : {}), } as SavedObjectsFindResponse; } diff --git a/src/core/server/saved_objects/service/lib/search_dsl/pit_params.test.ts b/src/core/server/saved_objects/service/lib/search_dsl/pit_params.test.ts new file mode 100644 index 0000000000000..e626900415adf --- /dev/null +++ b/src/core/server/saved_objects/service/lib/search_dsl/pit_params.test.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * and the Server Side Public License, v 1; you may not use this file except in + * compliance with, at your election, the Elastic License or the Server Side + * Public License, v 1. + */ + +import { getPitParams } from './pit_params'; + +describe('searchDsl/getPitParams', () => { + it('returns only an ID by default', () => { + expect(getPitParams({ id: 'abc123' })).toEqual({ + pit: { + id: 'abc123', + }, + }); + }); + + it('includes keepAlive if provided and rewrites to snake case', () => { + expect(getPitParams({ id: 'abc123', keepAlive: '1m' })).toEqual({ + pit: { + id: 'abc123', + keep_alive: '1m', + }, + }); + }); + + it('returns empty object if pit is undefined', () => { + expect(getPitParams()).toEqual({}); + }); +}); diff --git a/src/core/server/saved_objects/service/lib/search_dsl/pit_params.ts b/src/core/server/saved_objects/service/lib/search_dsl/pit_params.ts new file mode 100644 index 0000000000000..353217240452d --- /dev/null +++ b/src/core/server/saved_objects/service/lib/search_dsl/pit_params.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * and the Server Side Public License, v 1; you may not use this file except in + * compliance with, at your election, the Elastic License or the Server Side + * Public License, v 1. + */ + +interface SavedObjectsPitParams { + id: string; + keepAlive?: string; +} + +export function getPitParams(pit?: SavedObjectsPitParams) { + if (!pit) { + return {}; + } + + return { + pit: { + id: pit.id, + ...(pit.keepAlive ? { keep_alive: pit.keepAlive } : {}), + }, + }; +} diff --git a/src/core/server/saved_objects/service/lib/search_dsl/search_dsl.ts b/src/core/server/saved_objects/service/lib/search_dsl/search_dsl.ts index 7b6c702da41f7..253d3738728c9 100644 --- a/src/core/server/saved_objects/service/lib/search_dsl/search_dsl.ts +++ b/src/core/server/saved_objects/service/lib/search_dsl/search_dsl.ts @@ -10,6 +10,7 @@ import Boom from '@hapi/boom'; import { IndexMapping } from '../../../mappings'; import { getQueryParams, HasReferenceQueryParams, SearchOperator } from './query_params'; +import { getPitParams } from './pit_params'; import { getSortingParams } from './sorting_params'; import { ISavedObjectTypeRegistry } from '../../../saved_objects_type_registry'; @@ -25,6 +26,7 @@ interface GetSearchDslOptions { sortField?: string; sortOrder?: string; namespaces?: string[]; + pit?: { id: string; keepAlive?: string }; typeToNamespacesMap?: Map; hasReference?: HasReferenceQueryParams | HasReferenceQueryParams[]; hasReferenceOperator?: SearchOperator; @@ -46,6 +48,7 @@ export function getSearchDsl( sortField, sortOrder, namespaces, + pit, typeToNamespacesMap, hasReference, hasReferenceOperator, @@ -75,6 +78,7 @@ export function getSearchDsl( kueryNode, }), ...getSortingParams(mappings, type, sortField, sortOrder), + ...getPitParams(pit), ...(searchAfter ? { search_after: searchAfter } : {}), }; } diff --git a/src/core/server/saved_objects/service/saved_objects_client.ts b/src/core/server/saved_objects/service/saved_objects_client.ts index d5e447c0de209..83e297187e258 100644 --- a/src/core/server/saved_objects/service/saved_objects_client.ts +++ b/src/core/server/saved_objects/service/saved_objects_client.ts @@ -15,7 +15,6 @@ import { SavedObjectsBaseOptions, MutatingOperationRefreshSetting, SavedObjectsFindOptions, - SavedObjectsOpenPointInTimeOptions, } from '../types'; import { SavedObjectsErrorHelpers } from './lib/errors'; @@ -145,6 +144,7 @@ export interface SavedObjectsFindResponse { total: number; per_page: number; page: number; + pit_id?: string; } /** @@ -312,6 +312,20 @@ export interface SavedObjectsResolveResponse { outcome: 'exactMatch' | 'aliasMatch' | 'conflict'; } +/** + * @public + */ +export interface SavedObjectsOpenPointInTimeOptions { + /** + * Optionally specify how long ES should keep the PIT alive until the next request. Defaults to `5m`. + */ + keepAlive?: string; + /** + * An optional ES preference value to be used for the query. + */ + preference?: string; +} + /** * * @public diff --git a/src/core/server/saved_objects/types.ts b/src/core/server/saved_objects/types.ts index c23509884cccc..56a828d0b1214 100644 --- a/src/core/server/saved_objects/types.ts +++ b/src/core/server/saved_objects/types.ts @@ -118,20 +118,10 @@ export interface SavedObjectsFindOptions { typeToNamespacesMap?: Map; /** An optional ES preference value to be used for the query **/ preference?: string; -} - -/** - * @public - */ -export interface SavedObjectsOpenPointInTimeOptions { - /** - * Optionally specify how long ES should keep the PIT alive until the next request. Defaults to `5m`. - */ - keepAlive?: string; /** - * An optional ES preference value to be used for the query. + * Search against a specific Point In Time (PIT) that you've opened with `savedObjects.openPointInTimeForType`. */ - preference?: string; + pit?: { id: string; keepAlive?: string }; } /** From 67333ff26ea541852c1492b1362586bf809c6d5c Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Mon, 1 Feb 2021 15:03:07 -0700 Subject: [PATCH 04/32] Add openPointInTimeForType to SO client wrappers and audit logs. --- .../encrypted_saved_objects_client_wrapper.ts | 8 +++++ .../security/server/audit/audit_events.ts | 3 ++ .../secure_saved_objects_client_wrapper.ts | 29 +++++++++++++++++++ .../spaces_saved_objects_client.ts | 18 ++++++++++++ 4 files changed, 58 insertions(+) diff --git a/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.ts b/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.ts index 73414e8559192..a14a292622f19 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.ts @@ -18,6 +18,7 @@ import { SavedObjectsCreateOptions, SavedObjectsFindOptions, SavedObjectsFindResponse, + SavedObjectsOpenPointInTimeOptions, SavedObjectsUpdateOptions, SavedObjectsUpdateResponse, SavedObjectsAddToNamespacesOptions, @@ -249,6 +250,13 @@ export class EncryptedSavedObjectsClientWrapper implements SavedObjectsClientCon return await this.options.baseClient.removeReferencesTo(type, id, options); } + public async openPointInTimeForType( + type: string | string[], + options: SavedObjectsOpenPointInTimeOptions = {} + ) { + return await this.options.baseClient.openPointInTimeForType(type, options); + } + /** * Strips encrypted attributes from any non-bulk Saved Objects API response. If type isn't * registered, response is returned as is. diff --git a/x-pack/plugins/security/server/audit/audit_events.ts b/x-pack/plugins/security/server/audit/audit_events.ts index 25bcfd683b0dc..1205f03f7d4b5 100644 --- a/x-pack/plugins/security/server/audit/audit_events.ts +++ b/x-pack/plugins/security/server/audit/audit_events.ts @@ -190,6 +190,7 @@ export enum SavedObjectAction { ADD_TO_SPACES = 'saved_object_add_to_spaces', DELETE_FROM_SPACES = 'saved_object_delete_from_spaces', REMOVE_REFERENCES = 'saved_object_remove_references', + OPEN_POINT_IN_TIME = 'saved_object_open_point_in_time', } type VerbsTuple = [string, string, string]; @@ -203,6 +204,7 @@ const savedObjectAuditVerbs: Record = { saved_object_find: ['access', 'accessing', 'accessed'], saved_object_add_to_spaces: ['update', 'updating', 'updated'], saved_object_delete_from_spaces: ['update', 'updating', 'updated'], + saved_object_open_point_in_time: ['open', 'opening', 'opened'], saved_object_remove_references: [ 'remove references to', 'removing references to', @@ -219,6 +221,7 @@ const savedObjectAuditTypes: Record = { saved_object_find: EventType.ACCESS, saved_object_add_to_spaces: EventType.CHANGE, saved_object_delete_from_spaces: EventType.CHANGE, + saved_object_open_point_in_time: EventType.CREATION, saved_object_remove_references: EventType.CHANGE, }; diff --git a/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts b/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts index 4a886e5addb46..e0f8dd68eba7c 100644 --- a/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts +++ b/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts @@ -17,6 +17,7 @@ import { SavedObjectsCreateOptions, SavedObjectsDeleteFromNamespacesOptions, SavedObjectsFindOptions, + SavedObjectsOpenPointInTimeOptions, SavedObjectsRemoveReferencesToOptions, SavedObjectsUpdateOptions, SavedObjectsUtils, @@ -562,6 +563,34 @@ export class SecureSavedObjectsClientWrapper implements SavedObjectsClientContra return await this.baseClient.removeReferencesTo(type, id, options); } + public async openPointInTimeForType( + type: string | string[], + options: SavedObjectsOpenPointInTimeOptions + ) { + try { + const args = { type, options }; + await this.ensureAuthorized(type, 'open_point_in_time', undefined, { args }); + } catch (error) { + this.auditLogger.log( + savedObjectEvent({ + action: SavedObjectAction.OPEN_POINT_IN_TIME, + error, + }) + ); + throw error; + } + + const pit = await this.baseClient.openPointInTimeForType(type, options); + + this.auditLogger.log( + savedObjectEvent({ + action: SavedObjectAction.OPEN_POINT_IN_TIME, + }) + ); + + return pit; + } + private async checkPrivileges( actions: string | string[], namespaceOrNamespaces?: string | Array diff --git a/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts b/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts index 9316d86b19bdd..3053d836f6661 100644 --- a/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts +++ b/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts @@ -15,6 +15,7 @@ import { SavedObjectsClientContract, SavedObjectsCreateOptions, SavedObjectsFindOptions, + SavedObjectsOpenPointInTimeOptions, SavedObjectsUpdateOptions, SavedObjectsAddToNamespacesOptions, SavedObjectsDeleteFromNamespacesOptions, @@ -378,4 +379,21 @@ export class SpacesSavedObjectsClient implements SavedObjectsClientContract { namespace: spaceIdToNamespace(this.spaceId), }); } + + /** + * Opens a Point In Time (PIT) against the indices for the specified Saved Object types. + * The returned `id` can then be passed to `SavedObjects.find` to search against that PIT. + * + * @param {string|Array} type + * @param {object} [options] - {@link SavedObjectsPointInTimeOptions} + * @property {string} [options.keepAlive] + * @property {string} [options.preference] + * @returns {promise} - { id: string } + */ + async openPointInTimeForType( + type: string | string[], + options: SavedObjectsOpenPointInTimeOptions = {} + ) { + return await this.client.openPointInTimeForType(type, options); + } } From 6e583ace6d3f984ec86348bb0ec1fc9865e32d23 Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Tue, 2 Feb 2021 14:24:36 -0700 Subject: [PATCH 05/32] Automatically add _shard_doc tiebreaker when PIT is provided. --- .../service/lib/search_dsl/pit_params.test.ts | 4 --- .../service/lib/search_dsl/pit_params.ts | 6 +---- .../service/lib/search_dsl/search_dsl.ts | 4 +-- .../lib/search_dsl/sorting_params.test.ts | 26 +++++++++++++++++++ .../service/lib/search_dsl/sorting_params.ts | 11 +++++++- 5 files changed, 39 insertions(+), 12 deletions(-) diff --git a/src/core/server/saved_objects/service/lib/search_dsl/pit_params.test.ts b/src/core/server/saved_objects/service/lib/search_dsl/pit_params.test.ts index e626900415adf..9192b0a231027 100644 --- a/src/core/server/saved_objects/service/lib/search_dsl/pit_params.test.ts +++ b/src/core/server/saved_objects/service/lib/search_dsl/pit_params.test.ts @@ -25,8 +25,4 @@ describe('searchDsl/getPitParams', () => { }, }); }); - - it('returns empty object if pit is undefined', () => { - expect(getPitParams()).toEqual({}); - }); }); diff --git a/src/core/server/saved_objects/service/lib/search_dsl/pit_params.ts b/src/core/server/saved_objects/service/lib/search_dsl/pit_params.ts index 353217240452d..8c1defd04e4a7 100644 --- a/src/core/server/saved_objects/service/lib/search_dsl/pit_params.ts +++ b/src/core/server/saved_objects/service/lib/search_dsl/pit_params.ts @@ -11,11 +11,7 @@ interface SavedObjectsPitParams { keepAlive?: string; } -export function getPitParams(pit?: SavedObjectsPitParams) { - if (!pit) { - return {}; - } - +export function getPitParams(pit: SavedObjectsPitParams) { return { pit: { id: pit.id, diff --git a/src/core/server/saved_objects/service/lib/search_dsl/search_dsl.ts b/src/core/server/saved_objects/service/lib/search_dsl/search_dsl.ts index 253d3738728c9..7d51483b6b20d 100644 --- a/src/core/server/saved_objects/service/lib/search_dsl/search_dsl.ts +++ b/src/core/server/saved_objects/service/lib/search_dsl/search_dsl.ts @@ -77,8 +77,8 @@ export function getSearchDsl( hasReferenceOperator, kueryNode, }), - ...getSortingParams(mappings, type, sortField, sortOrder), - ...getPitParams(pit), + ...getSortingParams(mappings, type, sortField, sortOrder, pit), + ...(pit ? getPitParams(pit) : {}), ...(searchAfter ? { search_after: searchAfter } : {}), }; } diff --git a/src/core/server/saved_objects/service/lib/search_dsl/sorting_params.test.ts b/src/core/server/saved_objects/service/lib/search_dsl/sorting_params.test.ts index 1376f0d50a9da..73c7065705fc5 100644 --- a/src/core/server/saved_objects/service/lib/search_dsl/sorting_params.test.ts +++ b/src/core/server/saved_objects/service/lib/search_dsl/sorting_params.test.ts @@ -79,6 +79,11 @@ describe('searchDsl/getSortParams', () => { ], }); }); + it('appends tiebreaker when PIT is provided', () => { + expect(getSortingParams(MAPPINGS, 'saved', 'title', undefined, { id: 'abc' }).sort).toEqual( + expect.arrayContaining([{ _shard_doc: 'asc' }]) + ); + }); }); describe('sortField is simple root property with multiple types', () => { it('returns correct params', () => { @@ -93,6 +98,11 @@ describe('searchDsl/getSortParams', () => { ], }); }); + it('appends tiebreaker when PIT is provided', () => { + expect( + getSortingParams(MAPPINGS, ['saved', 'pending'], 'type', undefined, { id: 'abc' }).sort + ).toEqual(expect.arrayContaining([{ _shard_doc: 'asc' }])); + }); }); describe('sortField is simple non-root property with multiple types', () => { it('returns correct params', () => { @@ -114,6 +124,11 @@ describe('searchDsl/getSortParams', () => { ], }); }); + it('appends tiebreaker when PIT is provided', () => { + expect( + getSortingParams(MAPPINGS, 'saved', 'title.raw', undefined, { id: 'abc' }).sort + ).toEqual(expect.arrayContaining([{ _shard_doc: 'asc' }])); + }); }); describe('sortField is multi-field with single type as array', () => { it('returns correct params', () => { @@ -128,6 +143,11 @@ describe('searchDsl/getSortParams', () => { ], }); }); + it('appends tiebreaker when PIT is provided', () => { + expect( + getSortingParams(MAPPINGS, ['saved'], 'title.raw', undefined, { id: 'abc' }).sort + ).toEqual(expect.arrayContaining([{ _shard_doc: 'asc' }])); + }); }); describe('sortField is root multi-field with multiple types', () => { it('returns correct params', () => { @@ -142,6 +162,12 @@ describe('searchDsl/getSortParams', () => { ], }); }); + it('appends tiebreaker when PIT is provided', () => { + expect( + getSortingParams(MAPPINGS, ['saved', 'pending'], 'type.raw', undefined, { id: 'abc' }) + .sort + ).toEqual(expect.arrayContaining([{ _shard_doc: 'asc' }])); + }); }); describe('sortField is not-root multi-field with multiple types', () => { it('returns correct params', () => { diff --git a/src/core/server/saved_objects/service/lib/search_dsl/sorting_params.ts b/src/core/server/saved_objects/service/lib/search_dsl/sorting_params.ts index e3bfba6a80f59..45a9473a5df0e 100644 --- a/src/core/server/saved_objects/service/lib/search_dsl/sorting_params.ts +++ b/src/core/server/saved_objects/service/lib/search_dsl/sorting_params.ts @@ -9,13 +9,19 @@ import Boom from '@hapi/boom'; import { getProperty, IndexMapping } from '../../../mappings'; +// TODO: The plan is for ES to automatically add this tiebreaker when +// using PIT. We should remove this logic one that is resolved. +// https://github.com/elastic/elasticsearch/issues/56828 +const ES_PROVIDED_TIEBREAKER = { _shard_doc: 'asc' }; + const TOP_LEVEL_FIELDS = ['_id', '_score']; export function getSortingParams( mappings: IndexMapping, type: string | string[], sortField?: string, - sortOrder?: string + sortOrder?: string, + pit?: { id: string; keepAlive?: string } ) { if (!sortField) { return {}; @@ -31,6 +37,7 @@ export function getSortingParams( order: sortOrder, }, }, + ...(pit ? [ES_PROVIDED_TIEBREAKER] : []), ], }; } @@ -51,6 +58,7 @@ export function getSortingParams( unmapped_type: rootField.type, }, }, + ...(pit ? [ES_PROVIDED_TIEBREAKER] : []), ], }; } @@ -75,6 +83,7 @@ export function getSortingParams( unmapped_type: field.type, }, }, + ...(pit ? [ES_PROVIDED_TIEBREAKER] : []), ], }; } From a4646a01eca25cfba9551a882213af5ae32b9669 Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Tue, 2 Feb 2021 14:55:25 -0700 Subject: [PATCH 06/32] Clean up types. --- .../saved_objects/service/lib/repository.ts | 4 +++- .../service/lib/search_dsl/search_dsl.ts | 2 +- .../service/saved_objects_client.ts | 16 +++++++++++++++- src/core/server/saved_objects/types.ts | 2 +- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/core/server/saved_objects/service/lib/repository.ts b/src/core/server/saved_objects/service/lib/repository.ts index b5215e8887714..a1f77945057c6 100644 --- a/src/core/server/saved_objects/service/lib/repository.ts +++ b/src/core/server/saved_objects/service/lib/repository.ts @@ -37,6 +37,7 @@ import { SavedObjectsFindResponse, SavedObjectsFindResult, SavedObjectsOpenPointInTimeOptions, + SavedObjectsOpenPointInTimeResponse, SavedObjectsUpdateOptions, SavedObjectsUpdateResponse, SavedObjectsBulkUpdateObject, @@ -846,6 +847,7 @@ export class SavedObjectsRepository { (hit: SavedObjectsRawDoc): SavedObjectsFindResult => ({ ...this._rawToSavedObject(hit), score: (hit as any)._score, + ...((hit as any).sort ? { sort: (hit as any).sort } : {}), }) ), ...(body.pit_id ? { pit_id: body.pit_id } : {}), @@ -1812,7 +1814,7 @@ export class SavedObjectsRepository { async openPointInTimeForType( type: string | string[], { keepAlive, preference }: SavedObjectsOpenPointInTimeOptions = {} - ): Promise<{ id: string }> { + ): Promise { const defaultKeepAlive = '5m'; const types = Array.isArray(type) ? type : [type]; diff --git a/src/core/server/saved_objects/service/lib/search_dsl/search_dsl.ts b/src/core/server/saved_objects/service/lib/search_dsl/search_dsl.ts index 7d51483b6b20d..ca47a0c2bd6e3 100644 --- a/src/core/server/saved_objects/service/lib/search_dsl/search_dsl.ts +++ b/src/core/server/saved_objects/service/lib/search_dsl/search_dsl.ts @@ -22,7 +22,7 @@ interface GetSearchDslOptions { defaultSearchOperator?: SearchOperator; searchFields?: string[]; rootSearchFields?: string[]; - searchAfter?: Array; + searchAfter?: unknown[]; sortField?: string; sortOrder?: string; namespaces?: string[]; diff --git a/src/core/server/saved_objects/service/saved_objects_client.ts b/src/core/server/saved_objects/service/saved_objects_client.ts index 83e297187e258..8445cd9889e02 100644 --- a/src/core/server/saved_objects/service/saved_objects_client.ts +++ b/src/core/server/saved_objects/service/saved_objects_client.ts @@ -129,6 +129,10 @@ export interface SavedObjectsFindResult extends SavedObject { * The Elasticsearch `_score` of this result. */ score: number; + /** + * The Elasticsearch `sort` value of this result. + */ + sort?: unknown[]; } /** @@ -326,6 +330,16 @@ export interface SavedObjectsOpenPointInTimeOptions { preference?: string; } +/** + * @public + */ +export interface SavedObjectsOpenPointInTimeResponse { + /** + * PIT ID returned from ES. + */ + id: string; +} + /** * * @public @@ -526,7 +540,7 @@ export class SavedObjectsClient { */ async openPointInTimeForType( type: string | string[], - options: SavedObjectsOpenPointInTimeOptions + options: SavedObjectsOpenPointInTimeOptions = {} ) { return await this._repository.openPointInTimeForType(type, options); } diff --git a/src/core/server/saved_objects/types.ts b/src/core/server/saved_objects/types.ts index 56a828d0b1214..739bf5f78bb51 100644 --- a/src/core/server/saved_objects/types.ts +++ b/src/core/server/saved_objects/types.ts @@ -85,7 +85,7 @@ export interface SavedObjectsFindOptions { /** * Use the sort values from the previous page to retrieve the next page of results. */ - searchAfter?: Array; + searchAfter?: unknown[]; /** * The fields to perform the parsed query against. Unlike the `searchFields` argument, these are expected to be root fields and will not * be modified. If used in conjunction with `searchFields`, both are concatenated together. From fee2339cebbc796c97b280471b802697ef8dc5a7 Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Wed, 3 Feb 2021 21:05:52 -0700 Subject: [PATCH 07/32] Update export code (still need to update tests). --- .../export/saved_objects_exporter.ts | 84 +++++++++++++++++-- .../saved_objects/saved_objects_service.ts | 1 + .../saved_objects/service/lib/repository.ts | 3 +- 3 files changed, 81 insertions(+), 7 deletions(-) diff --git a/src/core/server/saved_objects/export/saved_objects_exporter.ts b/src/core/server/saved_objects/export/saved_objects_exporter.ts index 295a3d7a119d4..f2c4d44b39e9e 100644 --- a/src/core/server/saved_objects/export/saved_objects_exporter.ts +++ b/src/core/server/saved_objects/export/saved_objects_exporter.ts @@ -8,7 +8,9 @@ import { createListStream } from '@kbn/utils'; import { PublicMethodsOf } from '@kbn/utility-types'; -import { SavedObject, SavedObjectsClientContract } from '../types'; +import { Logger } from '../../logging'; +import { SavedObject, SavedObjectsClientContract, SavedObjectsFindOptions } from '../types'; +import { SavedObjectsFindResult, SavedObjectsFindResponse } from '../service'; import { ISavedObjectTypeRegistry } from '../saved_objects_type_registry'; import { fetchNestedDependencies } from './fetch_nested_dependencies'; import { sortObjects } from './sort_objects'; @@ -35,16 +37,20 @@ export class SavedObjectsExporter { readonly #savedObjectsClient: SavedObjectsClientContract; readonly #exportTransforms: Record; readonly #exportSizeLimit: number; + readonly #log: Logger; constructor({ savedObjectsClient, typeRegistry, exportSizeLimit, + logger, }: { savedObjectsClient: SavedObjectsClientContract; typeRegistry: ISavedObjectTypeRegistry; exportSizeLimit: number; + logger: Logger; }) { + this.#log = logger; this.#savedObjectsClient = savedObjectsClient; this.#exportSizeLimit = exportSizeLimit; this.#exportTransforms = typeRegistry.getAllTypes().reduce((transforms, type) => { @@ -66,6 +72,7 @@ export class SavedObjectsExporter { * @throws SavedObjectsExportError */ public async exportByTypes(options: SavedObjectsExportByTypeOptions) { + this.#log.debug(`Initiating export for types: [${options.types.join(',')}]`); const objects = await this.fetchByTypes(options); return this.processObjects(objects, byIdAscComparator, { request: options.request, @@ -83,6 +90,7 @@ export class SavedObjectsExporter { * @throws SavedObjectsExportError */ public async exportByObjects(options: SavedObjectsExportByObjectOptions) { + this.#log.debug(`Initiating export of [${options.objects.length}] objects.`); if (options.objects.length > this.#exportSizeLimit) { throw SavedObjectsExportError.exportSizeExceeded(this.#exportSizeLimit); } @@ -106,6 +114,7 @@ export class SavedObjectsExporter { namespace, }: SavedObjectExportBaseOptions ) { + this.#log.debug(`Processing [${savedObjects.length}] saved objects.`); let exportedObjects: Array>; let missingReferences: SavedObjectsExportResultDetails['missingReferences'] = []; @@ -117,6 +126,7 @@ export class SavedObjectsExporter { }); if (includeReferencesDeep) { + this.#log.debug(`Fetching saved objects references.`); const fetchResult = await fetchNestedDependencies( savedObjects, this.#savedObjectsClient, @@ -138,6 +148,7 @@ export class SavedObjectsExporter { missingRefCount: missingReferences.length, missingReferences, }; + this.#log.debug(`Exporting [${redactedObjects.length}] saved objects.`); return createListStream([...redactedObjects, ...(excludeExportDetails ? [] : [exportDetails])]); } @@ -150,27 +161,88 @@ export class SavedObjectsExporter { return bulkGetResult.saved_objects; } + /** + * Generator which wraps calls to `SavedObjects.find` and iterates over + * multiple pages of results using `_pit` and `search_after`. This will + * continue paging until a set of results is received that's smaller than + * the designated `perPage`. + */ + private async *findWithPointInTime(findOptions: SavedObjectsFindOptions) { + const getLastHitSortValue = (res: SavedObjectsFindResponse) => + res.saved_objects.length && res.saved_objects[res.saved_objects.length - 1].sort; + + const findWithPit = async ({ id, searchAfter }: { id: string; searchAfter?: unknown[] }) => { + return await this.#savedObjectsClient.find({ + sortField: 'updated_at', // sort field is required to use search_after + sortOrder: 'desc', + pit: { + id, + keepAlive: '1m', // bump keep_alive by 1m on every new request + }, + ...(searchAfter ? { searchAfter } : {}), + ...findOptions, + }); + }; + + // Open PIT and request our first page of hits + let { id: pitId } = await this.#savedObjectsClient.openPointInTimeForType(findOptions.type); + let results = await findWithPit({ id: pitId }); + + let lastHitCount = results.saved_objects.length; + let lastHitSortValue = getLastHitSortValue(results); + pitId = results.pit_id!; + + this.#log.debug(`Collected [${lastHitCount}] saved objects for export.`); + + yield results; + + // We've reached the end when there are fewer hits than our perPage size + while (lastHitSortValue && lastHitCount === findOptions.perPage) { + results = await findWithPit({ + id: pitId, + searchAfter: lastHitSortValue, + }); + + lastHitCount = results.saved_objects.length; + lastHitSortValue = getLastHitSortValue(results); + pitId = results.pit_id!; + + this.#log.debug(`Collected [${lastHitCount}] more saved objects for export.`); + + yield results; + } + } + private async fetchByTypes({ types, namespace, hasReference, search, }: SavedObjectsExportByTypeOptions) { - const findResponse = await this.#savedObjectsClient.find({ + const options: SavedObjectsFindOptions = { type: types, hasReference, hasReferenceOperator: hasReference ? 'OR' : undefined, search, perPage: this.#exportSizeLimit, namespaces: namespace ? [namespace] : undefined, - }); - if (findResponse.total > this.#exportSizeLimit) { - throw SavedObjectsExportError.exportSizeExceeded(this.#exportSizeLimit); + }; + const finder = this.findWithPointInTime(options); + + let hits: SavedObjectsFindResult[] = []; + for await (const result of finder) { + hits = hits.concat(result.saved_objects); + if (hits.length > this.#exportSizeLimit) { + throw SavedObjectsExportError.exportSizeExceeded(this.#exportSizeLimit); + } } + // TODO: Need to close our PIT after we have collected all hits + // await this.#savedObjectsClient.closePointInTime(id); + // sorts server-side by _id, since it's only available in fielddata return ( - findResponse.saved_objects + hits // exclude the find-specific `score` property from the exported objects .map(({ score, ...obj }) => obj) .sort(byIdAscComparator) diff --git a/src/core/server/saved_objects/saved_objects_service.ts b/src/core/server/saved_objects/saved_objects_service.ts index 4ad0a34acc2ef..6d416ac2cdd7f 100644 --- a/src/core/server/saved_objects/saved_objects_service.ts +++ b/src/core/server/saved_objects/saved_objects_service.ts @@ -459,6 +459,7 @@ export class SavedObjectsService savedObjectsClient, typeRegistry: this.typeRegistry, exportSizeLimit: this.config!.maxImportExportSize, + logger: this.logger, }), createImporter: (savedObjectsClient) => new SavedObjectsImporter({ diff --git a/src/core/server/saved_objects/service/lib/repository.ts b/src/core/server/saved_objects/service/lib/repository.ts index a1f77945057c6..651b6c65d1bfd 100644 --- a/src/core/server/saved_objects/service/lib/repository.ts +++ b/src/core/server/saved_objects/service/lib/repository.ts @@ -800,8 +800,9 @@ export class SavedObjectsRepository { index: this.getIndicesForTypes(allowedTypes), preference, }), + // If `searchAfter` is provided, we drop `from` as it will not be used for pagination. + ...(searchAfter ? {} : { from: perPage * (page - 1) }), size: perPage, - from: perPage * (page - 1), _source: includedFields(type, fields), rest_total_hits_as_int: true, body: { From f9ce6d9fa726c64620c4a99d74c331df0ccc10ca Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Wed, 3 Feb 2021 21:08:33 -0700 Subject: [PATCH 08/32] Update license headers. --- .../saved_objects/routes/integration_tests/pit.test.ts | 6 +++--- src/core/server/saved_objects/routes/pit.ts | 6 +++--- .../saved_objects/service/lib/search_dsl/pit_params.test.ts | 6 +++--- .../saved_objects/service/lib/search_dsl/pit_params.ts | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/core/server/saved_objects/routes/integration_tests/pit.test.ts b/src/core/server/saved_objects/routes/integration_tests/pit.test.ts index d8cdb0cb79534..f3f40282af5c7 100644 --- a/src/core/server/saved_objects/routes/integration_tests/pit.test.ts +++ b/src/core/server/saved_objects/routes/integration_tests/pit.test.ts @@ -1,9 +1,9 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * and the Server Side Public License, v 1; you may not use this file except in - * compliance with, at your election, the Elastic License or the Server Side - * Public License, v 1. + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ import supertest from 'supertest'; diff --git a/src/core/server/saved_objects/routes/pit.ts b/src/core/server/saved_objects/routes/pit.ts index 2ae37be556ab9..4be8d4bbf08d6 100644 --- a/src/core/server/saved_objects/routes/pit.ts +++ b/src/core/server/saved_objects/routes/pit.ts @@ -1,9 +1,9 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * and the Server Side Public License, v 1; you may not use this file except in - * compliance with, at your election, the Elastic License or the Server Side - * Public License, v 1. + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ import { schema } from '@kbn/config-schema'; diff --git a/src/core/server/saved_objects/service/lib/search_dsl/pit_params.test.ts b/src/core/server/saved_objects/service/lib/search_dsl/pit_params.test.ts index 9192b0a231027..6d6051e33f1cf 100644 --- a/src/core/server/saved_objects/service/lib/search_dsl/pit_params.test.ts +++ b/src/core/server/saved_objects/service/lib/search_dsl/pit_params.test.ts @@ -1,9 +1,9 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * and the Server Side Public License, v 1; you may not use this file except in - * compliance with, at your election, the Elastic License or the Server Side - * Public License, v 1. + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ import { getPitParams } from './pit_params'; diff --git a/src/core/server/saved_objects/service/lib/search_dsl/pit_params.ts b/src/core/server/saved_objects/service/lib/search_dsl/pit_params.ts index 8c1defd04e4a7..1f06df4f3329d 100644 --- a/src/core/server/saved_objects/service/lib/search_dsl/pit_params.ts +++ b/src/core/server/saved_objects/service/lib/search_dsl/pit_params.ts @@ -1,9 +1,9 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * and the Server Side Public License, v 1; you may not use this file except in - * compliance with, at your election, the Elastic License or the Server Side - * Public License, v 1. + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ interface SavedObjectsPitParams { From d5eb72a474b8e4267fcaef51e4da42424d117404 Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Thu, 4 Feb 2021 16:29:47 -0700 Subject: [PATCH 09/32] Remove PIT route and client-side PIT APIs. --- .../saved_objects_client.test.ts | 73 ------------ .../saved_objects/saved_objects_client.ts | 25 ---- .../saved_objects_service.mock.ts | 1 - .../core_usage_stats_client.mock.ts | 1 - .../core_usage_stats_client.test.ts | 76 ------------ .../core_usage_stats_client.ts | 6 - src/core/server/saved_objects/routes/index.ts | 2 - .../routes/integration_tests/pit.test.ts | 112 ------------------ src/core/server/saved_objects/routes/pit.ts | 56 --------- 9 files changed, 352 deletions(-) delete mode 100644 src/core/server/saved_objects/routes/integration_tests/pit.test.ts delete mode 100644 src/core/server/saved_objects/routes/pit.ts diff --git a/src/core/public/saved_objects/saved_objects_client.test.ts b/src/core/public/saved_objects/saved_objects_client.test.ts index 66828587779d5..2ff02c1224870 100644 --- a/src/core/public/saved_objects/saved_objects_client.test.ts +++ b/src/core/public/saved_objects/saved_objects_client.test.ts @@ -494,77 +494,4 @@ describe('SavedObjectsClient', () => { `); }); }); - - describe('#openPointInTimeForType', () => { - const type = 'index-pattern'; - - beforeEach(() => { - http.fetch.mockResolvedValue({ id: 'abc123' }); - }); - - test('resolves with value returned from _pit route', async () => { - const resultP = savedObjectsClient.openPointInTimeForType(type); - await expect(resultP).resolves.toHaveProperty('id'); - - const result = await resultP; - expect(http.fetch).toHaveBeenCalledTimes(1); - expect(result.id).toBe('abc123'); - }); - - test('makes HTTP call correctly without options', () => { - savedObjectsClient.openPointInTimeForType(type); - expect(http.fetch.mock.calls[0]).toMatchInlineSnapshot(` - Array [ - "/api/saved_objects/index-pattern/_pit", - Object { - "body": "{}", - "method": "POST", - "query": undefined, - }, - ] - `); - }); - - test('makes HTTP call correctly mapping options into snake case query parameters', () => { - const options = { - keepAlive: '1m', - preference: 'foo', - }; - savedObjectsClient.openPointInTimeForType(type, options); - expect(http.fetch.mock.calls[0]).toMatchInlineSnapshot(` - Array [ - "/api/saved_objects/index-pattern/_pit", - Object { - "body": "{\\"keep_alive\\":\\"1m\\",\\"preference\\":\\"foo\\"}", - "method": "POST", - "query": undefined, - }, - ] - `); - }); - - test('ignores invalid options', () => { - savedObjectsClient.openPointInTimeForType(type, { - // @ts-expect-error - invalid: true, - keepAlive: '1m', - preference: 'foo', - }); - expect(http.fetch.mock.calls[0]).toMatchInlineSnapshot(` - Array [ - "/api/saved_objects/index-pattern/_pit", - Object { - "body": "{\\"keep_alive\\":\\"1m\\",\\"preference\\":\\"foo\\"}", - "method": "POST", - "query": undefined, - }, - ] - `); - }); - - test('concatenates multiple types in the path', () => { - savedObjectsClient.openPointInTimeForType(['a', 'b', 'c']); - expect(http.fetch.mock.calls[0][0]).toBe('/api/saved_objects/a%2Cb%2Cc/_pit'); - }); - }); }); diff --git a/src/core/public/saved_objects/saved_objects_client.ts b/src/core/public/saved_objects/saved_objects_client.ts index e8580543169b9..e735adbdac048 100644 --- a/src/core/public/saved_objects/saved_objects_client.ts +++ b/src/core/public/saved_objects/saved_objects_client.ts @@ -16,7 +16,6 @@ import { SavedObjectsClientContract as SavedObjectsApi, SavedObjectsFindOptions as SavedObjectFindOptionsServer, SavedObjectsMigrationVersion, - SavedObjectsOpenPointInTimeOptions, } from '../../server'; import { SimpleSavedObject } from './simple_saved_object'; @@ -479,30 +478,6 @@ export class SavedObjectsClient { }); } - /** - * Opens a Point In Time (PIT) against the indices for the specified Saved Object types. - * The returned `id` can then be passed to `SavedObjects.find` to search against that PIT. - * - * @param {string|Array} type - * @param {object} [options] - {@link SavedObjectsPointInTimeOptions} - * @property {string} [options.keepAlive] - * @property {string} [options.preference] - * @returns {promise} - { id: string } - */ - public openPointInTimeForType = ( - type: string | string[], - { keepAlive, preference }: SavedObjectsOpenPointInTimeOptions = {} - ): ReturnType => { - const types = Array.isArray(type) ? type.join(',') : type; - return this.savedObjectsFetch(this.getPath([types, '_pit']), { - method: 'POST', - body: JSON.stringify({ - ...(keepAlive ? { keep_alive: keepAlive } : {}), - ...(preference ? { preference } : {}), - }), - }); - }; - private createSavedObject(options: SavedObject): SimpleSavedObject { return new SimpleSavedObject(this, options); } diff --git a/src/core/public/saved_objects/saved_objects_service.mock.ts b/src/core/public/saved_objects/saved_objects_service.mock.ts index 88c80109aca71..625ea6b5dd2da 100644 --- a/src/core/public/saved_objects/saved_objects_service.mock.ts +++ b/src/core/public/saved_objects/saved_objects_service.mock.ts @@ -18,7 +18,6 @@ const createStartContractMock = () => { bulkGet: jest.fn(), find: jest.fn(), get: jest.fn(), - openPointInTimeForType: jest.fn().mockResolvedValue({ id: 'some_pit_id' }), update: jest.fn(), }, }; diff --git a/src/core/server/core_usage_data/core_usage_stats_client.mock.ts b/src/core/server/core_usage_data/core_usage_stats_client.mock.ts index 05c3a5443ef07..1c9c0b8fae579 100644 --- a/src/core/server/core_usage_data/core_usage_stats_client.mock.ts +++ b/src/core/server/core_usage_data/core_usage_stats_client.mock.ts @@ -18,7 +18,6 @@ const createUsageStatsClientMock = () => incrementSavedObjectsDelete: jest.fn().mockResolvedValue(null), incrementSavedObjectsFind: jest.fn().mockResolvedValue(null), incrementSavedObjectsGet: jest.fn().mockResolvedValue(null), - incrementSavedObjectsOpenPit: jest.fn().mockResolvedValue(null), incrementSavedObjectsResolve: jest.fn().mockResolvedValue(null), incrementSavedObjectsUpdate: jest.fn().mockResolvedValue(null), incrementSavedObjectsImport: jest.fn().mockResolvedValue(null), diff --git a/src/core/server/core_usage_data/core_usage_stats_client.test.ts b/src/core/server/core_usage_data/core_usage_stats_client.test.ts index f2ff3c1414cf6..dc4a81adf5f8e 100644 --- a/src/core/server/core_usage_data/core_usage_stats_client.test.ts +++ b/src/core/server/core_usage_data/core_usage_stats_client.test.ts @@ -22,7 +22,6 @@ import { GET_STATS_PREFIX, RESOLVE_STATS_PREFIX, UPDATE_STATS_PREFIX, - OPEN_PIT_STATS_PREFIX, IMPORT_STATS_PREFIX, RESOLVE_IMPORT_STATS_PREFIX, EXPORT_STATS_PREFIX, @@ -671,81 +670,6 @@ describe('CoreUsageStatsClient', () => { }); }); - describe('#incrementSavedObjectsOpenPit', () => { - it('does not throw an error if repository incrementCounter operation fails', async () => { - const { usageStatsClient, repositoryMock } = setup(); - repositoryMock.incrementCounter.mockRejectedValue(new Error('Oh no!')); - - const request = httpServerMock.createKibanaRequest(); - await expect( - usageStatsClient.incrementSavedObjectsOpenPit({ - request, - } as BaseIncrementOptions) - ).resolves.toBeUndefined(); - expect(repositoryMock.incrementCounter).toHaveBeenCalled(); - }); - - it('handles falsy options appropriately', async () => { - const { usageStatsClient, repositoryMock } = setup(); - - const request = httpServerMock.createKibanaRequest(); - await usageStatsClient.incrementSavedObjectsOpenPit({ - request, - } as BaseIncrementOptions); - expect(repositoryMock.incrementCounter).toHaveBeenCalledTimes(1); - expect(repositoryMock.incrementCounter).toHaveBeenCalledWith( - CORE_USAGE_STATS_TYPE, - CORE_USAGE_STATS_ID, - [ - `${OPEN_PIT_STATS_PREFIX}.total`, - `${OPEN_PIT_STATS_PREFIX}.namespace.default.total`, - `${OPEN_PIT_STATS_PREFIX}.namespace.default.kibanaRequest.no`, - ], - incrementOptions - ); - }); - - it('handles truthy options and the default namespace string appropriately', async () => { - const { usageStatsClient, repositoryMock } = setup(DEFAULT_NAMESPACE_STRING); - - const request = httpServerMock.createKibanaRequest({ headers: firstPartyRequestHeaders }); - await usageStatsClient.incrementSavedObjectsOpenPit({ - request, - } as BaseIncrementOptions); - expect(repositoryMock.incrementCounter).toHaveBeenCalledTimes(1); - expect(repositoryMock.incrementCounter).toHaveBeenCalledWith( - CORE_USAGE_STATS_TYPE, - CORE_USAGE_STATS_ID, - [ - `${OPEN_PIT_STATS_PREFIX}.total`, - `${OPEN_PIT_STATS_PREFIX}.namespace.default.total`, - `${OPEN_PIT_STATS_PREFIX}.namespace.default.kibanaRequest.yes`, - ], - incrementOptions - ); - }); - - it('handles a non-default space appropriately', async () => { - const { usageStatsClient, repositoryMock } = setup('foo'); - - const request = httpServerMock.createKibanaRequest(); - await usageStatsClient.incrementSavedObjectsOpenPit({ - request, - } as BaseIncrementOptions); - expect(repositoryMock.incrementCounter).toHaveBeenCalledTimes(1); - expect(repositoryMock.incrementCounter).toHaveBeenCalledWith( - CORE_USAGE_STATS_TYPE, - CORE_USAGE_STATS_ID, - [ - `${OPEN_PIT_STATS_PREFIX}.total`, - `${OPEN_PIT_STATS_PREFIX}.namespace.custom.total`, - `${OPEN_PIT_STATS_PREFIX}.namespace.custom.kibanaRequest.no`, - ], - incrementOptions - ); - }); - }); - describe('#incrementSavedObjectsUpdate', () => { it('does not throw an error if repository incrementCounter operation fails', async () => { const { usageStatsClient, repositoryMock } = setup(); diff --git a/src/core/server/core_usage_data/core_usage_stats_client.ts b/src/core/server/core_usage_data/core_usage_stats_client.ts index 33c01e8c30dd7..31f044c90ee53 100644 --- a/src/core/server/core_usage_data/core_usage_stats_client.ts +++ b/src/core/server/core_usage_data/core_usage_stats_client.ts @@ -42,7 +42,6 @@ export const FIND_STATS_PREFIX = 'apiCalls.savedObjectsFind'; export const GET_STATS_PREFIX = 'apiCalls.savedObjectsGet'; export const RESOLVE_STATS_PREFIX = 'apiCalls.savedObjectsResolve'; export const UPDATE_STATS_PREFIX = 'apiCalls.savedObjectsUpdate'; -export const OPEN_PIT_STATS_PREFIX = 'apiCalls.savedObjectsOpenPit'; export const IMPORT_STATS_PREFIX = 'apiCalls.savedObjectsImport'; export const RESOLVE_IMPORT_STATS_PREFIX = 'apiCalls.savedObjectsResolveImportErrors'; export const EXPORT_STATS_PREFIX = 'apiCalls.savedObjectsExport'; @@ -57,7 +56,6 @@ const ALL_COUNTER_FIELDS = [ ...getFieldsForCounter(GET_STATS_PREFIX), ...getFieldsForCounter(RESOLVE_STATS_PREFIX), ...getFieldsForCounter(UPDATE_STATS_PREFIX), - ...getFieldsForCounter(OPEN_PIT_STATS_PREFIX), // Saved Objects Management APIs ...getFieldsForCounter(IMPORT_STATS_PREFIX), `${IMPORT_STATS_PREFIX}.createNewCopiesEnabled.yes`, @@ -135,10 +133,6 @@ export class CoreUsageStatsClient { await this.updateUsageStats([], UPDATE_STATS_PREFIX, options); } - public async incrementSavedObjectsOpenPit(options: BaseIncrementOptions) { - await this.updateUsageStats([], OPEN_PIT_STATS_PREFIX, options); - } - public async incrementSavedObjectsImport(options: IncrementSavedObjectsImportOptions) { const { createNewCopies, overwrite } = options; const counterFieldNames = [ diff --git a/src/core/server/saved_objects/routes/index.ts b/src/core/server/saved_objects/routes/index.ts index b6c3226da9688..930e02de7657a 100644 --- a/src/core/server/saved_objects/routes/index.ts +++ b/src/core/server/saved_objects/routes/index.ts @@ -21,7 +21,6 @@ import { registerBulkGetRoute } from './bulk_get'; import { registerBulkCreateRoute } from './bulk_create'; import { registerBulkUpdateRoute } from './bulk_update'; import { registerLogLegacyImportRoute } from './log_legacy_import'; -import { registerPointInTimeRoute } from './pit'; import { registerExportRoute } from './export'; import { registerImportRoute } from './import'; import { registerResolveImportErrorsRoute } from './resolve_import_errors'; @@ -52,7 +51,6 @@ export function registerRoutes({ registerBulkCreateRoute(router, { coreUsageData }); registerBulkUpdateRoute(router, { coreUsageData }); registerLogLegacyImportRoute(router, logger); - registerPointInTimeRoute(router, { coreUsageData }); registerExportRoute(router, { config, coreUsageData }); registerImportRoute(router, { config, coreUsageData }); registerResolveImportErrorsRoute(router, { config, coreUsageData }); diff --git a/src/core/server/saved_objects/routes/integration_tests/pit.test.ts b/src/core/server/saved_objects/routes/integration_tests/pit.test.ts deleted file mode 100644 index f3f40282af5c7..0000000000000 --- a/src/core/server/saved_objects/routes/integration_tests/pit.test.ts +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import supertest from 'supertest'; - -import { UnwrapPromise } from '@kbn/utility-types'; -import { registerPointInTimeRoute } from '../pit'; -import { savedObjectsClientMock } from '../../../../../core/server/mocks'; -import { CoreUsageStatsClient } from '../../../core_usage_data'; -import { coreUsageStatsClientMock } from '../../../core_usage_data/core_usage_stats_client.mock'; -import { coreUsageDataServiceMock } from '../../../core_usage_data/core_usage_data_service.mock'; -import { setupServer } from '../test_utils'; - -type SetupServerReturn = UnwrapPromise>; - -describe('POST /api/saved_objects/{type}/_pit', () => { - let server: SetupServerReturn['server']; - let httpSetup: SetupServerReturn['httpSetup']; - let handlerContext: SetupServerReturn['handlerContext']; - let savedObjectsClient: ReturnType; - let coreUsageStatsClient: jest.Mocked; - - const clientResponse = { - id: 'abc123', - }; - - beforeEach(async () => { - ({ server, httpSetup, handlerContext } = await setupServer()); - savedObjectsClient = handlerContext.savedObjects.client; - - savedObjectsClient.openPointInTimeForType.mockResolvedValue(clientResponse); - - const router = httpSetup.createRouter('/api/saved_objects/'); - coreUsageStatsClient = coreUsageStatsClientMock.create(); - coreUsageStatsClient.incrementSavedObjectsOpenPit.mockRejectedValue(new Error('Oh no!')); // intentionally throw this error, which is swallowed, so we can assert that the operation does not fail - const coreUsageData = coreUsageDataServiceMock.createSetupContract(coreUsageStatsClient); - registerPointInTimeRoute(router, { coreUsageData }); - - await server.start(); - }); - - afterEach(async () => { - await server.stop(); - }); - - it('works without optional body settings', async () => { - const result = await supertest(httpSetup.server.listener) - .post('/api/saved_objects/index-pattern/_pit') - .expect(200); - - expect(result.body).toEqual(clientResponse); - }); - - it('records usage stats', async () => { - await supertest(httpSetup.server.listener) - .post('/api/saved_objects/index-pattern/_pit') - .expect(200); - - expect(coreUsageStatsClient.incrementSavedObjectsOpenPit).toHaveBeenCalledWith({ - request: expect.anything(), - }); - }); - - it('calls savedObjectClient.openPointInTimeForType with the type', async () => { - await supertest(httpSetup.server.listener) - .post('/api/saved_objects/index-pattern/_pit') - .expect(200); - - expect(savedObjectsClient.openPointInTimeForType).toHaveBeenCalledTimes(1); - - const options = savedObjectsClient.openPointInTimeForType.mock.calls[0][0]; - expect(options).toEqual(['index-pattern']); - }); - - it('handles multiple comma-separated types in the path', async () => { - await supertest(httpSetup.server.listener).post('/api/saved_objects/a,b,c/_pit').expect(200); - - expect(savedObjectsClient.openPointInTimeForType).toHaveBeenCalledTimes(1); - - const options = savedObjectsClient.openPointInTimeForType.mock.calls[0][0]; - expect(options).toEqual(['a', 'b', 'c']); - }); - - it('accepts keep_alive in the body', async () => { - await supertest(httpSetup.server.listener) - .post('/api/saved_objects/index-pattern/_pit') - .send({ keepAlive: '1m' }) - .expect(200); - - expect(savedObjectsClient.openPointInTimeForType).toHaveBeenCalledTimes(1); - - const options = savedObjectsClient.openPointInTimeForType.mock.calls[0][1]; - expect(options).toEqual({ keepAlive: '1m' }); - }); - - it('accepts preference in the body', async () => { - await supertest(httpSetup.server.listener) - .post('/api/saved_objects/index-pattern/_pit') - .send({ preference: 'foo' }) - .expect(200); - - expect(savedObjectsClient.openPointInTimeForType).toHaveBeenCalledTimes(1); - - const options = savedObjectsClient.openPointInTimeForType.mock.calls[0][1]; - expect(options).toEqual({ preference: 'foo' }); - }); -}); diff --git a/src/core/server/saved_objects/routes/pit.ts b/src/core/server/saved_objects/routes/pit.ts deleted file mode 100644 index 4be8d4bbf08d6..0000000000000 --- a/src/core/server/saved_objects/routes/pit.ts +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { schema } from '@kbn/config-schema'; -import { IRouter } from '../../http'; -import { CoreUsageDataSetup } from '../../core_usage_data'; - -interface RouteDependencies { - coreUsageData: CoreUsageDataSetup; -} - -const TYPE_DELIMITER = ','; - -export const registerPointInTimeRoute = (router: IRouter, { coreUsageData }: RouteDependencies) => { - router.post( - { - path: '/{types}/_pit', - validate: { - params: schema.object({ - types: schema.string(), - }), - body: schema.nullable( - schema.object( - { - keepAlive: schema.maybe(schema.string()), - preference: schema.maybe(schema.string()), - }, - { - defaultValue: {}, - } - ) - ), - }, - }, - router.handleLegacyErrors(async (context, req, res) => { - const { types } = req.params; - - const usageStatsClient = coreUsageData.getClient(); - usageStatsClient.incrementSavedObjectsOpenPit({ request: req }).catch(() => {}); - - const result = await context.core.savedObjects.client.openPointInTimeForType( - types.split(TYPE_DELIMITER), - { - ...(req.body?.keepAlive ? { keepAlive: req.body.keepAlive } : {}), - ...(req.body?.preference ? { preference: req.body.preference } : {}), - } - ); - return res.ok({ body: result }); - }) - ); -}; From fcfc507e34d03ee4252c8d0729c6ca78a16ad33f Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Fri, 5 Feb 2021 14:12:46 -0700 Subject: [PATCH 10/32] Add closePointInTime to SO client. --- .../export/saved_objects_exporter.ts | 7 ++- .../service/lib/repository.mock.ts | 1 + .../service/lib/repository.test.js | 44 +++++++++++++++- .../saved_objects/service/lib/repository.ts | 50 ++++++++++++++++++- .../service/saved_objects_client.mock.ts | 1 + .../service/saved_objects_client.test.js | 14 ++++++ .../service/saved_objects_client.ts | 24 +++++++++ 7 files changed, 137 insertions(+), 4 deletions(-) diff --git a/src/core/server/saved_objects/export/saved_objects_exporter.ts b/src/core/server/saved_objects/export/saved_objects_exporter.ts index f2c4d44b39e9e..2faced60f72d0 100644 --- a/src/core/server/saved_objects/export/saved_objects_exporter.ts +++ b/src/core/server/saved_objects/export/saved_objects_exporter.ts @@ -229,16 +229,19 @@ export class SavedObjectsExporter { }; const finder = this.findWithPointInTime(options); + let lastPit: string | undefined; let hits: SavedObjectsFindResult[] = []; for await (const result of finder) { + lastPit = result.pit_id; hits = hits.concat(result.saved_objects); if (hits.length > this.#exportSizeLimit) { throw SavedObjectsExportError.exportSizeExceeded(this.#exportSizeLimit); } } - // TODO: Need to close our PIT after we have collected all hits - // await this.#savedObjectsClient.closePointInTime(id); + if (lastPit) { + await this.#savedObjectsClient.closePointInTime(lastPit); + } // sorts server-side by _id, since it's only available in fielddata return ( diff --git a/src/core/server/saved_objects/service/lib/repository.mock.ts b/src/core/server/saved_objects/service/lib/repository.mock.ts index c574999b4b95f..a3610b1e437e2 100644 --- a/src/core/server/saved_objects/service/lib/repository.mock.ts +++ b/src/core/server/saved_objects/service/lib/repository.mock.ts @@ -17,6 +17,7 @@ const create = (): jest.Mocked => ({ bulkGet: jest.fn(), find: jest.fn(), get: jest.fn(), + closePointInTime: jest.fn(), openPointInTimeForType: jest.fn().mockResolvedValue({ id: 'some_pit_id' }), resolve: jest.fn(), update: jest.fn(), diff --git a/src/core/server/saved_objects/service/lib/repository.test.js b/src/core/server/saved_objects/service/lib/repository.test.js index 781637138c7ff..e1135ade6690a 100644 --- a/src/core/server/saved_objects/service/lib/repository.test.js +++ b/src/core/server/saved_objects/service/lib/repository.test.js @@ -4434,7 +4434,7 @@ describe('SavedObjectsRepository', () => { }; describe('client calls', () => { - it(`should use the ES search action`, async () => { + it(`should use the ES PIT API`, async () => { await successResponse(type); expect(client.openPointInTime).toHaveBeenCalledTimes(1); }); @@ -4509,4 +4509,46 @@ describe('SavedObjectsRepository', () => { }); }); }); + + describe('#closePointInTime', () => { + const generateResults = () => ({ succeeded: true, num_freed: 3 }); + const successResponse = async (id) => { + client.closePointInTime.mockResolvedValueOnce( + elasticsearchClientMock.createSuccessTransportRequestPromise(generateResults()) + ); + const result = await savedObjectsRepository.closePointInTime(id); + expect(client.closePointInTime).toHaveBeenCalledTimes(1); + return result; + }; + + describe('client calls', () => { + it(`should use the ES PIT API`, async () => { + await successResponse('abc123'); + expect(client.closePointInTime).toHaveBeenCalledTimes(1); + }); + + it(`accepts id`, async () => { + await successResponse('abc123'); + expect(client.closePointInTime).toHaveBeenCalledWith( + expect.objectContaining({ + body: expect.objectContaining({ + id: 'abc123', + }), + }), + expect.anything() + ); + }); + }); + + describe('returns', () => { + it(`returns response body from ES`, async () => { + const results = generateResults('abc123'); + client.closePointInTime.mockResolvedValueOnce( + elasticsearchClientMock.createSuccessTransportRequestPromise(results) + ); + const response = await savedObjectsRepository.closePointInTime('abc123'); + expect(response).toEqual(results); + }); + }); + }); }); diff --git a/src/core/server/saved_objects/service/lib/repository.ts b/src/core/server/saved_objects/service/lib/repository.ts index 651b6c65d1bfd..db8faa654f5ab 100644 --- a/src/core/server/saved_objects/service/lib/repository.ts +++ b/src/core/server/saved_objects/service/lib/repository.ts @@ -36,6 +36,7 @@ import { SavedObjectsCreateOptions, SavedObjectsFindResponse, SavedObjectsFindResult, + SavedObjectsClosePointInTimeResponse, SavedObjectsOpenPointInTimeOptions, SavedObjectsOpenPointInTimeResponse, SavedObjectsUpdateOptions, @@ -1830,7 +1831,10 @@ export class SavedObjectsRepository { ...(preference ? { preference } : {}), }; - const { body, statusCode } = await this.client.openPointInTime<{ id: string }>(esOptions, { + const { + body, + statusCode, + } = await this.client.openPointInTime(esOptions, { ignore: [404], }); if (statusCode === 404) { @@ -1842,6 +1846,50 @@ export class SavedObjectsRepository { }; } + /** + * Closes a Point In Time (PIT) by ID. This simply proxies the request to ES + * via the Elasticsearch client, and is included in the Saved Objects Client + * as a convenience for consumers who are using `openPointInTimeForType`. + * + * @remarks + * While the `keepAlive` that is provided will cause a PIT to automatically close, + * it is highly recommended to explicitly close a PIT when you are done with it + * in order to avoid consuming unneeded resources in Elasticsearch. + * + * @example + * ```ts + * const repository = coreStart.savedObjects.createInternalRepository(); + * + * const { id } = await repository.openPointInTimeForType( + * type: 'index-pattern', + * { keepAlive: '1m' }, + * ); + * + * const response = await repository.find({ + * type: 'index-pattern', + * search: 'foo*', + * sortField: 'name', + * sortOrder: 'desc', + * pit: { + * id: 'abc123', + * keepAlive: '1m', + * }, + * searchAfter: [1234, 'abcd'], + * }); + * + * await repository.closePointInTime(response.pit_id); + * ``` + * + * @param {string} id + * @returns {promise} - { id: string } + */ + async closePointInTime(id: string): Promise { + const { body } = await this.client.closePointInTime({ + body: { id }, + }); + return body; + } + /** * Returns index specified by the given type or the default index * diff --git a/src/core/server/saved_objects/service/saved_objects_client.mock.ts b/src/core/server/saved_objects/service/saved_objects_client.mock.ts index 79fb068319838..ecca652cace37 100644 --- a/src/core/server/saved_objects/service/saved_objects_client.mock.ts +++ b/src/core/server/saved_objects/service/saved_objects_client.mock.ts @@ -20,6 +20,7 @@ const create = () => bulkGet: jest.fn(), find: jest.fn(), get: jest.fn(), + closePointInTime: jest.fn(), openPointInTimeForType: jest.fn().mockResolvedValue({ id: 'some_pit_id' }), resolve: jest.fn(), update: jest.fn(), diff --git a/src/core/server/saved_objects/service/saved_objects_client.test.js b/src/core/server/saved_objects/service/saved_objects_client.test.js index b17582b4b286b..7f803a6e96373 100644 --- a/src/core/server/saved_objects/service/saved_objects_client.test.js +++ b/src/core/server/saved_objects/service/saved_objects_client.test.js @@ -130,6 +130,20 @@ test(`#openPointInTimeForType`, async () => { expect(result).toBe(returnValue); }); +test(`#closePointInTime`, async () => { + const returnValue = Symbol(); + const mockRepository = { + closePointInTime: jest.fn().mockResolvedValue(returnValue), + }; + const client = new SavedObjectsClient(mockRepository); + + const id = Symbol(); + const result = await client.closePointInTime(id); + + expect(mockRepository.closePointInTime).toHaveBeenCalledWith(id); + expect(result).toBe(returnValue); +}); + test(`#resolve`, async () => { const returnValue = Symbol(); const mockRepository = { diff --git a/src/core/server/saved_objects/service/saved_objects_client.ts b/src/core/server/saved_objects/service/saved_objects_client.ts index 8445cd9889e02..5ccd69c2506fc 100644 --- a/src/core/server/saved_objects/service/saved_objects_client.ts +++ b/src/core/server/saved_objects/service/saved_objects_client.ts @@ -340,6 +340,21 @@ export interface SavedObjectsOpenPointInTimeResponse { id: string; } +/** + * @public + */ +export interface SavedObjectsClosePointInTimeResponse { + /** + * If true, all search contexts associated with the PIT id are + * successfully closed. + */ + succeeded: boolean; + /** + * The number of search contexts that have been successfully closed. + */ + num_freed: number; +} + /** * * @public @@ -544,4 +559,13 @@ export class SavedObjectsClient { ) { return await this._repository.openPointInTimeForType(type, options); } + + /** + * Closes a Point In Time (PIT) by ID. This simply proxies the request to ES + * via the Elasticsearch client, and is included in the Saved Objects Client + * as a convenience for consumers who are using `openPointInTimeForType`. + */ + async closePointInTime(id: string) { + return await this._repository.closePointInTime(id); + } } From b0c11ee5aef84a690abb468dfdfe29ea881013f1 Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Sat, 6 Feb 2021 13:43:15 -0700 Subject: [PATCH 11/32] Update SO exporter snapshots. --- .../export/saved_objects_exporter.test.ts | 299 +++++++++++------- .../export/saved_objects_exporter.ts | 49 ++- 2 files changed, 208 insertions(+), 140 deletions(-) diff --git a/src/core/server/saved_objects/export/saved_objects_exporter.test.ts b/src/core/server/saved_objects/export/saved_objects_exporter.test.ts index c16623f785b08..cac3a81f10107 100644 --- a/src/core/server/saved_objects/export/saved_objects_exporter.test.ts +++ b/src/core/server/saved_objects/export/saved_objects_exporter.test.ts @@ -11,6 +11,7 @@ import { SavedObjectsExporter } from './saved_objects_exporter'; import { savedObjectsClientMock } from '../service/saved_objects_client.mock'; import { SavedObjectTypeRegistry } from '../saved_objects_type_registry'; import { httpServerMock } from '../../http/http_server.mocks'; +import { loggerMock, MockedLogger } from '../../logging/logger.mock'; import { Readable } from 'stream'; import { createPromiseFromStreams, createConcatStream } from '@kbn/utils'; @@ -22,14 +23,21 @@ const exportSizeLimit = 500; const request = httpServerMock.createKibanaRequest(); describe('getSortedObjectsForExport()', () => { + let logger: MockedLogger; let savedObjectsClient: ReturnType; let typeRegistry: SavedObjectTypeRegistry; let exporter: SavedObjectsExporter; beforeEach(() => { + logger = loggerMock.create(); typeRegistry = new SavedObjectTypeRegistry(); savedObjectsClient = savedObjectsClientMock.create(); - exporter = new SavedObjectsExporter({ savedObjectsClient, exportSizeLimit, typeRegistry }); + exporter = new SavedObjectsExporter({ + exportSizeLimit, + logger, + savedObjectsClient, + typeRegistry, + }); }); describe('#exportByTypes', () => { @@ -96,30 +104,36 @@ describe('getSortedObjectsForExport()', () => { ] `); expect(savedObjectsClient.find).toMatchInlineSnapshot(` - [MockFunction] { - "calls": Array [ - Array [ - Object { - "hasReference": undefined, - "hasReferenceOperator": undefined, - "namespaces": undefined, - "perPage": 500, - "search": undefined, - "type": Array [ - "index-pattern", - "search", - ], - }, - ], - ], - "results": Array [ - Object { - "type": "return", - "value": Promise {}, - }, + [MockFunction] { + "calls": Array [ + Array [ + Object { + "hasReference": undefined, + "hasReferenceOperator": undefined, + "namespaces": undefined, + "perPage": 500, + "pit": Object { + "id": "some_pit_id", + "keepAlive": "1m", + }, + "search": undefined, + "sortField": "updated_at", + "sortOrder": "desc", + "type": Array [ + "index-pattern", + "search", ], - } - `); + }, + ], + ], + "results": Array [ + Object { + "type": "return", + "value": Promise {}, + }, + ], + } + `); }); test('applies the export transforms', async () => { @@ -138,7 +152,12 @@ describe('getSortedObjectsForExport()', () => { }, }, }); - exporter = new SavedObjectsExporter({ savedObjectsClient, exportSizeLimit, typeRegistry }); + exporter = new SavedObjectsExporter({ + exportSizeLimit, + logger, + savedObjectsClient, + typeRegistry, + }); savedObjectsClient.find.mockResolvedValueOnce({ total: 1, @@ -233,30 +252,36 @@ describe('getSortedObjectsForExport()', () => { ] `); expect(savedObjectsClient.find).toMatchInlineSnapshot(` - [MockFunction] { - "calls": Array [ - Array [ - Object { - "hasReference": undefined, - "hasReferenceOperator": undefined, - "namespaces": undefined, - "perPage": 500, - "search": undefined, - "type": Array [ - "index-pattern", - "search", - ], - }, - ], - ], - "results": Array [ - Object { - "type": "return", - "value": Promise {}, - }, + [MockFunction] { + "calls": Array [ + Array [ + Object { + "hasReference": undefined, + "hasReferenceOperator": undefined, + "namespaces": undefined, + "perPage": 500, + "pit": Object { + "id": "some_pit_id", + "keepAlive": "1m", + }, + "search": undefined, + "sortField": "updated_at", + "sortOrder": "desc", + "type": Array [ + "index-pattern", + "search", ], - } - `); + }, + ], + ], + "results": Array [ + Object { + "type": "return", + "value": Promise {}, + }, + ], + } + `); }); test('exclude export details if option is specified', async () => { @@ -383,30 +408,36 @@ describe('getSortedObjectsForExport()', () => { ] `); expect(savedObjectsClient.find).toMatchInlineSnapshot(` - [MockFunction] { - "calls": Array [ - Array [ - Object { - "hasReference": undefined, - "hasReferenceOperator": undefined, - "namespaces": undefined, - "perPage": 500, - "search": "foo", - "type": Array [ - "index-pattern", - "search", - ], - }, - ], - ], - "results": Array [ - Object { - "type": "return", - "value": Promise {}, - }, + [MockFunction] { + "calls": Array [ + Array [ + Object { + "hasReference": undefined, + "hasReferenceOperator": undefined, + "namespaces": undefined, + "perPage": 500, + "pit": Object { + "id": "some_pit_id", + "keepAlive": "1m", + }, + "search": "foo", + "sortField": "updated_at", + "sortOrder": "desc", + "type": Array [ + "index-pattern", + "search", ], - } - `); + }, + ], + ], + "results": Array [ + Object { + "type": "return", + "value": Promise {}, + }, + ], + } + `); }); test('exports selected types with references when present', async () => { @@ -465,35 +496,41 @@ describe('getSortedObjectsForExport()', () => { ] `); expect(savedObjectsClient.find).toMatchInlineSnapshot(` - [MockFunction] { - "calls": Array [ - Array [ - Object { - "hasReference": Array [ - Object { - "id": "1", - "type": "index-pattern", - }, - ], - "hasReferenceOperator": "OR", - "namespaces": undefined, - "perPage": 500, - "search": undefined, - "type": Array [ - "index-pattern", - "search", - ], - }, - ], - ], - "results": Array [ + [MockFunction] { + "calls": Array [ + Array [ + Object { + "hasReference": Array [ Object { - "type": "return", - "value": Promise {}, + "id": "1", + "type": "index-pattern", }, ], - } - `); + "hasReferenceOperator": "OR", + "namespaces": undefined, + "perPage": 500, + "pit": Object { + "id": "some_pit_id", + "keepAlive": "1m", + }, + "search": undefined, + "sortField": "updated_at", + "sortOrder": "desc", + "type": Array [ + "index-pattern", + "search", + ], + }, + ], + ], + "results": Array [ + Object { + "type": "return", + "value": Promise {}, + }, + ], + } + `); }); test('exports from the provided namespace when present', async () => { @@ -560,36 +597,47 @@ describe('getSortedObjectsForExport()', () => { ] `); expect(savedObjectsClient.find).toMatchInlineSnapshot(` - [MockFunction] { - "calls": Array [ - Array [ - Object { - "hasReference": undefined, - "hasReferenceOperator": undefined, - "namespaces": Array [ - "foo", - ], - "perPage": 500, - "search": undefined, - "type": Array [ - "index-pattern", - "search", - ], - }, - ], + [MockFunction] { + "calls": Array [ + Array [ + Object { + "hasReference": undefined, + "hasReferenceOperator": undefined, + "namespaces": Array [ + "foo", ], - "results": Array [ - Object { - "type": "return", - "value": Promise {}, - }, + "perPage": 500, + "pit": Object { + "id": "some_pit_id", + "keepAlive": "1m", + }, + "search": undefined, + "sortField": "updated_at", + "sortOrder": "desc", + "type": Array [ + "index-pattern", + "search", ], - } - `); + }, + ], + ], + "results": Array [ + Object { + "type": "return", + "value": Promise {}, + }, + ], + } + `); }); test('export selected types throws error when exceeding exportSizeLimit', async () => { - exporter = new SavedObjectsExporter({ savedObjectsClient, exportSizeLimit: 1, typeRegistry }); + exporter = new SavedObjectsExporter({ + exportSizeLimit: 1, + logger, + savedObjectsClient, + typeRegistry, + }); savedObjectsClient.find.mockResolvedValueOnce({ total: 2, @@ -836,7 +884,12 @@ describe('getSortedObjectsForExport()', () => { }); test('export selected objects throws error when exceeding exportSizeLimit', async () => { - exporter = new SavedObjectsExporter({ savedObjectsClient, exportSizeLimit: 1, typeRegistry }); + exporter = new SavedObjectsExporter({ + exportSizeLimit: 1, + logger, + savedObjectsClient, + typeRegistry, + }); const exportOpts = { request, diff --git a/src/core/server/saved_objects/export/saved_objects_exporter.ts b/src/core/server/saved_objects/export/saved_objects_exporter.ts index 2faced60f72d0..9f797ca00a135 100644 --- a/src/core/server/saved_objects/export/saved_objects_exporter.ts +++ b/src/core/server/saved_objects/export/saved_objects_exporter.ts @@ -164,8 +164,22 @@ export class SavedObjectsExporter { /** * Generator which wraps calls to `SavedObjects.find` and iterates over * multiple pages of results using `_pit` and `search_after`. This will - * continue paging until a set of results is received that's smaller than - * the designated `perPage`. + * open a new PIT, continue paging until a set of results is received + * that's smaller than the designated `perPage`, and then close the PIT. + * + * @example + * ```ts + * const finder = findWithPointInTime({ + * type: 'index-pattern', + * search: 'foo*', + * perPage: 100, + * }); + * + * const responses: SavedObjectFindResponse[] = []; + * for await (const response of finder) { + * responses.push(...response); + * } + * ``` */ private async *findWithPointInTime(findOptions: SavedObjectsFindOptions) { const getLastHitSortValue = (res: SavedObjectsFindResponse) => @@ -173,11 +187,13 @@ export class SavedObjectsExporter { const findWithPit = async ({ id, searchAfter }: { id: string; searchAfter?: unknown[] }) => { return await this.#savedObjectsClient.find({ - sortField: 'updated_at', // sort field is required to use search_after + // Sort fields are required to use searchAfter, so we set some defaults here + sortField: 'updated_at', sortOrder: 'desc', pit: { - id, keepAlive: '1m', // bump keep_alive by 1m on every new request + ...findOptions.pit, + id, }, ...(searchAfter ? { searchAfter } : {}), ...findOptions, @@ -188,26 +204,31 @@ export class SavedObjectsExporter { let { id: pitId } = await this.#savedObjectsClient.openPointInTimeForType(findOptions.type); let results = await findWithPit({ id: pitId }); - let lastHitCount = results.saved_objects.length; + let lastResultsCount = results.saved_objects.length; let lastHitSortValue = getLastHitSortValue(results); pitId = results.pit_id!; - this.#log.debug(`Collected [${lastHitCount}] saved objects for export.`); + this.#log.debug(`Collected [${lastResultsCount}] saved objects for export.`); yield results; // We've reached the end when there are fewer hits than our perPage size - while (lastHitSortValue && lastHitCount === findOptions.perPage) { + while (lastHitSortValue && lastResultsCount === findOptions.perPage) { results = await findWithPit({ id: pitId, searchAfter: lastHitSortValue, }); - lastHitCount = results.saved_objects.length; + lastResultsCount = results.saved_objects.length; lastHitSortValue = getLastHitSortValue(results); pitId = results.pit_id!; - this.#log.debug(`Collected [${lastHitCount}] more saved objects for export.`); + this.#log.debug(`Collected [${lastResultsCount}] more saved objects for export.`); + + // Close PIT if this was our last page + if (lastResultsCount < findOptions.perPage) { + await this.#savedObjectsClient.closePointInTime(pitId); + } yield results; } @@ -229,20 +250,14 @@ export class SavedObjectsExporter { }; const finder = this.findWithPointInTime(options); - let lastPit: string | undefined; - let hits: SavedObjectsFindResult[] = []; + const hits: SavedObjectsFindResult[] = []; for await (const result of finder) { - lastPit = result.pit_id; - hits = hits.concat(result.saved_objects); + hits.push(...result.saved_objects); if (hits.length > this.#exportSizeLimit) { throw SavedObjectsExportError.exportSizeExceeded(this.#exportSizeLimit); } } - if (lastPit) { - await this.#savedObjectsClient.closePointInTime(lastPit); - } - // sorts server-side by _id, since it's only available in fielddata return ( hits From d7ecaea99c59a5506bd061783ea3f215161aea4c Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Sat, 6 Feb 2021 20:26:54 -0700 Subject: [PATCH 12/32] Clean up exporter; add tests. --- .../export/saved_objects_exporter.test.ts | 205 +++++++++++++++++- .../export/saved_objects_exporter.ts | 11 +- 2 files changed, 208 insertions(+), 8 deletions(-) diff --git a/src/core/server/saved_objects/export/saved_objects_exporter.test.ts b/src/core/server/saved_objects/export/saved_objects_exporter.test.ts index cac3a81f10107..f96b634b73297 100644 --- a/src/core/server/saved_objects/export/saved_objects_exporter.test.ts +++ b/src/core/server/saved_objects/export/saved_objects_exporter.test.ts @@ -19,7 +19,7 @@ async function readStreamToCompletion(stream: Readable): Promise { @@ -111,7 +111,7 @@ describe('getSortedObjectsForExport()', () => { "hasReference": undefined, "hasReferenceOperator": undefined, "namespaces": undefined, - "perPage": 500, + "perPage": 10000, "pit": Object { "id": "some_pit_id", "keepAlive": "1m", @@ -136,6 +136,199 @@ describe('getSortedObjectsForExport()', () => { `); }); + describe('pages through results with PIT', () => { + function generateHits( + hitCount: number, + { + attributes = {}, + sort = [], + type = 'index-pattern', + }: { + attributes?: Record; + sort?: unknown[]; + type?: string; + } = {} + ) { + const hits = []; + for (let i = 1; i <= hitCount; i++) { + hits.push({ + id: `${i}`, + type, + attributes, + sort, + score: 1, + references: [], + }); + } + return hits; + } + + describe('<10k hits', () => { + const mockHits = generateHits(20); + + test('requests a single page', async () => { + savedObjectsClient.find.mockResolvedValueOnce({ + total: 20, + saved_objects: mockHits, + per_page: 10000, + page: 0, + }); + + const exportStream = await exporter.exportByTypes({ + request, + types: ['index-pattern'], + }); + + const response = await readStreamToCompletion(exportStream); + + expect(savedObjectsClient.find).toHaveBeenCalledTimes(1); + expect(response[response.length - 1]).toMatchInlineSnapshot(` + Object { + "exportedCount": 20, + "missingRefCount": 0, + "missingReferences": Array [], + } + `); + }); + + test('opens and closes PIT', async () => { + savedObjectsClient.find.mockResolvedValueOnce({ + total: 20, + saved_objects: mockHits, + per_page: 10000, + page: 0, + }); + + const exportStream = await exporter.exportByTypes({ + request, + types: ['index-pattern'], + }); + + await readStreamToCompletion(exportStream); + + expect(savedObjectsClient.openPointInTimeForType).toHaveBeenCalledTimes(1); + expect(savedObjectsClient.closePointInTime).toHaveBeenCalledTimes(1); + }); + + test('passes correct PIT ID to `find`', async () => { + savedObjectsClient.openPointInTimeForType.mockResolvedValueOnce({ + id: 'abc123', + }); + savedObjectsClient.find.mockResolvedValueOnce({ + total: 20, + saved_objects: mockHits, + per_page: 10000, + page: 0, + }); + + const exportStream = await exporter.exportByTypes({ + request, + types: ['index-pattern'], + }); + + await readStreamToCompletion(exportStream); + + expect(savedObjectsClient.find).toHaveBeenCalledWith( + expect.objectContaining({ + pit: expect.objectContaining({ id: 'abc123', keepAlive: '1m' }), + sortField: 'updated_at', + sortOrder: 'desc', + type: ['index-pattern'], + }) + ); + }); + }); + + describe('>10k hits', () => { + const firstMockHits = generateHits(10000, { sort: ['a', 'b'] }); + const secondMockHits = generateHits(5000); + + test('requests multiple pages', async () => { + savedObjectsClient.find.mockResolvedValueOnce({ + total: 15000, + saved_objects: firstMockHits, + per_page: 10000, + page: 0, + }); + savedObjectsClient.find.mockResolvedValueOnce({ + total: 15000, + saved_objects: secondMockHits, + per_page: 5000, + page: 1, + }); + + const exportStream = await exporter.exportByTypes({ + request, + types: ['index-pattern'], + }); + + const response = await readStreamToCompletion(exportStream); + + expect(savedObjectsClient.find).toHaveBeenCalledTimes(2); + expect(response[response.length - 1]).toMatchInlineSnapshot(` + Object { + "exportedCount": 15000, + "missingRefCount": 0, + "missingReferences": Array [], + } + `); + }); + + test('opens and closes PIT', async () => { + savedObjectsClient.find.mockResolvedValueOnce({ + total: 15000, + saved_objects: firstMockHits, + per_page: 10000, + page: 0, + }); + savedObjectsClient.find.mockResolvedValueOnce({ + total: 15000, + saved_objects: secondMockHits, + per_page: 5000, + page: 1, + }); + + const exportStream = await exporter.exportByTypes({ + request, + types: ['index-pattern'], + }); + + await readStreamToCompletion(exportStream); + + expect(savedObjectsClient.openPointInTimeForType).toHaveBeenCalledTimes(1); + expect(savedObjectsClient.closePointInTime).toHaveBeenCalledTimes(1); + }); + + test('passes sort values to searchAfter', async () => { + savedObjectsClient.find.mockResolvedValueOnce({ + total: 15000, + saved_objects: firstMockHits, + per_page: 10000, + page: 0, + }); + savedObjectsClient.find.mockResolvedValueOnce({ + total: 15000, + saved_objects: secondMockHits, + per_page: 5000, + page: 1, + }); + + const exportStream = await exporter.exportByTypes({ + request, + types: ['index-pattern'], + }); + + await readStreamToCompletion(exportStream); + + expect(savedObjectsClient.find.mock.calls[1][0]).toEqual( + expect.objectContaining({ + searchAfter: ['a', 'b'], + }) + ); + }); + }); + }); + test('applies the export transforms', async () => { typeRegistry.registerType({ name: 'foo', @@ -259,7 +452,7 @@ describe('getSortedObjectsForExport()', () => { "hasReference": undefined, "hasReferenceOperator": undefined, "namespaces": undefined, - "perPage": 500, + "perPage": 10000, "pit": Object { "id": "some_pit_id", "keepAlive": "1m", @@ -415,7 +608,7 @@ describe('getSortedObjectsForExport()', () => { "hasReference": undefined, "hasReferenceOperator": undefined, "namespaces": undefined, - "perPage": 500, + "perPage": 10000, "pit": Object { "id": "some_pit_id", "keepAlive": "1m", @@ -508,7 +701,7 @@ describe('getSortedObjectsForExport()', () => { ], "hasReferenceOperator": "OR", "namespaces": undefined, - "perPage": 500, + "perPage": 10000, "pit": Object { "id": "some_pit_id", "keepAlive": "1m", @@ -606,7 +799,7 @@ describe('getSortedObjectsForExport()', () => { "namespaces": Array [ "foo", ], - "perPage": 500, + "perPage": 10000, "pit": Object { "id": "some_pit_id", "keepAlive": "1m", diff --git a/src/core/server/saved_objects/export/saved_objects_exporter.ts b/src/core/server/saved_objects/export/saved_objects_exporter.ts index 9f797ca00a135..11ecf19a2cc48 100644 --- a/src/core/server/saved_objects/export/saved_objects_exporter.ts +++ b/src/core/server/saved_objects/export/saved_objects_exporter.ts @@ -210,6 +210,11 @@ export class SavedObjectsExporter { this.#log.debug(`Collected [${lastResultsCount}] saved objects for export.`); + // Close PIT if this was our last page + if (lastResultsCount < findOptions.perPage!) { + await this.#savedObjectsClient.closePointInTime(pitId); + } + yield results; // We've reached the end when there are fewer hits than our perPage size @@ -225,7 +230,6 @@ export class SavedObjectsExporter { this.#log.debug(`Collected [${lastResultsCount}] more saved objects for export.`); - // Close PIT if this was our last page if (lastResultsCount < findOptions.perPage) { await this.#savedObjectsClient.closePointInTime(pitId); } @@ -245,8 +249,11 @@ export class SavedObjectsExporter { hasReference, hasReferenceOperator: hasReference ? 'OR' : undefined, search, - perPage: this.#exportSizeLimit, namespaces: namespace ? [namespace] : undefined, + // We aren't using `exportSizeLimit` here because a user may opt to set it + // higher than the 10k ES default for `index.max_result_window`, in which + // case we will use PIT to "scroll" through pages of hits, 10k at a time. + perPage: 10000, }; const finder = this.findWithPointInTime(options); From ba0342323a554359c2de8fca976281d942ce70f9 Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Sat, 6 Feb 2021 21:54:59 -0700 Subject: [PATCH 13/32] Update spaces and security. --- .../service/saved_objects_client.ts | 2 +- .../encrypted_saved_objects_client_wrapper.ts | 4 ++ .../security/server/audit/audit_events.ts | 13 ++++++- .../feature_privilege_builder/saved_object.ts | 2 +- ...ecure_saved_objects_client_wrapper.test.ts | 39 +++++++++++++++++++ .../secure_saved_objects_client_wrapper.ts | 14 ++++++- .../spaces_saved_objects_client.test.ts | 26 +++++++++++++ .../spaces_saved_objects_client.ts | 15 ++++++- 8 files changed, 110 insertions(+), 5 deletions(-) diff --git a/src/core/server/saved_objects/service/saved_objects_client.ts b/src/core/server/saved_objects/service/saved_objects_client.ts index 5ccd69c2506fc..0b76661b57672 100644 --- a/src/core/server/saved_objects/service/saved_objects_client.ts +++ b/src/core/server/saved_objects/service/saved_objects_client.ts @@ -319,7 +319,7 @@ export interface SavedObjectsResolveResponse { /** * @public */ -export interface SavedObjectsOpenPointInTimeOptions { +export interface SavedObjectsOpenPointInTimeOptions extends SavedObjectsBaseOptions { /** * Optionally specify how long ES should keep the PIT alive until the next request. Defaults to `5m`. */ diff --git a/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.ts b/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.ts index a14a292622f19..48bfadb0cde49 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.ts @@ -257,6 +257,10 @@ export class EncryptedSavedObjectsClientWrapper implements SavedObjectsClientCon return await this.options.baseClient.openPointInTimeForType(type, options); } + public async closePointInTime(id: string) { + return await this.options.baseClient.closePointInTime(id); + } + /** * Strips encrypted attributes from any non-bulk Saved Objects API response. If type isn't * registered, response is returned as is. diff --git a/x-pack/plugins/security/server/audit/audit_events.ts b/x-pack/plugins/security/server/audit/audit_events.ts index 1205f03f7d4b5..f353362e33513 100644 --- a/x-pack/plugins/security/server/audit/audit_events.ts +++ b/x-pack/plugins/security/server/audit/audit_events.ts @@ -191,6 +191,7 @@ export enum SavedObjectAction { DELETE_FROM_SPACES = 'saved_object_delete_from_spaces', REMOVE_REFERENCES = 'saved_object_remove_references', OPEN_POINT_IN_TIME = 'saved_object_open_point_in_time', + CLOSE_POINT_IN_TIME = 'saved_object_close_point_in_time', } type VerbsTuple = [string, string, string]; @@ -204,7 +205,16 @@ const savedObjectAuditVerbs: Record = { saved_object_find: ['access', 'accessing', 'accessed'], saved_object_add_to_spaces: ['update', 'updating', 'updated'], saved_object_delete_from_spaces: ['update', 'updating', 'updated'], - saved_object_open_point_in_time: ['open', 'opening', 'opened'], + saved_object_open_point_in_time: [ + 'open point-in-time', + 'opening point-in-time', + 'opened point-in-time', + ], + saved_object_close_point_in_time: [ + 'close point-in-time', + 'closing point-in-time', + 'closed point-in-time', + ], saved_object_remove_references: [ 'remove references to', 'removing references to', @@ -222,6 +232,7 @@ const savedObjectAuditTypes: Record = { saved_object_add_to_spaces: EventType.CHANGE, saved_object_delete_from_spaces: EventType.CHANGE, saved_object_open_point_in_time: EventType.CREATION, + saved_object_close_point_in_time: EventType.DELETION, saved_object_remove_references: EventType.CHANGE, }; diff --git a/x-pack/plugins/security/server/authorization/privileges/feature_privilege_builder/saved_object.ts b/x-pack/plugins/security/server/authorization/privileges/feature_privilege_builder/saved_object.ts index 571d588037f36..08c623d04dd78 100644 --- a/x-pack/plugins/security/server/authorization/privileges/feature_privilege_builder/saved_object.ts +++ b/x-pack/plugins/security/server/authorization/privileges/feature_privilege_builder/saved_object.ts @@ -9,7 +9,7 @@ import { flatten, uniq } from 'lodash'; import { FeatureKibanaPrivileges } from '../../../../../features/server'; import { BaseFeaturePrivilegeBuilder } from './feature_privilege_builder'; -const readOperations: string[] = ['bulk_get', 'get', 'find']; +const readOperations: string[] = ['bulk_get', 'get', 'find', 'open_point_in_time']; const writeOperations: string[] = [ 'create', 'bulk_create', diff --git a/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.test.ts b/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.test.ts index aeddba051a186..f3012cba0c236 100644 --- a/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.test.ts +++ b/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.test.ts @@ -987,6 +987,45 @@ describe('#get', () => { }); }); +describe('#openPointInTimeForType', () => { + const type = 'foo'; + const namespace = 'some-ns'; + + test(`throws decorated GeneralError when hasPrivileges rejects promise`, async () => { + await expectGeneralError(client.openPointInTimeForType, { type }); + }); + + test(`throws decorated ForbiddenError when unauthorized`, async () => { + const options = { namespace }; + await expectForbiddenError(client.openPointInTimeForType, { type, options }); + }); + + test(`returns result of baseClient.openPointInTimeForType when authorized`, async () => { + const apiCallReturnValue = Symbol(); + clientOpts.baseClient.openPointInTimeForType.mockReturnValue(apiCallReturnValue as any); + + const options = { namespace }; + const result = await expectSuccess(client.openPointInTimeForType, { type, options }); + expect(result).toBe(apiCallReturnValue); + }); + + test(`adds audit event when successful`, async () => { + const apiCallReturnValue = Symbol(); + clientOpts.baseClient.openPointInTimeForType.mockReturnValue(apiCallReturnValue as any); + const options = { namespace }; + await expectSuccess(client.openPointInTimeForType, { type, options }); + expect(clientOpts.auditLogger.log).toHaveBeenCalledTimes(1); + expectAuditEvent('saved_object_open_point_in_time', EventOutcome.SUCCESS); + }); + + test(`adds audit event when not successful`, async () => { + clientOpts.checkSavedObjectsPrivilegesAsCurrentUser.mockRejectedValue(new Error()); + await expect(() => client.openPointInTimeForType(type, { namespace })).rejects.toThrow(); + expect(clientOpts.auditLogger.log).toHaveBeenCalledTimes(1); + expectAuditEvent('saved_object_open_point_in_time', EventOutcome.FAILURE); + }); +}); + describe('#resolve', () => { const type = 'foo'; const id = `${type}-id`; diff --git a/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts b/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts index e0f8dd68eba7c..0721e1e00596f 100644 --- a/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts +++ b/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts @@ -569,7 +569,7 @@ export class SecureSavedObjectsClientWrapper implements SavedObjectsClientContra ) { try { const args = { type, options }; - await this.ensureAuthorized(type, 'open_point_in_time', undefined, { args }); + await this.ensureAuthorized(type, 'open_point_in_time', options?.namespace, { args }); } catch (error) { this.auditLogger.log( savedObjectEvent({ @@ -591,6 +591,18 @@ export class SecureSavedObjectsClientWrapper implements SavedObjectsClientContra return pit; } + public async closePointInTime(id: string) { + const response = await this.baseClient.closePointInTime(id); + + this.auditLogger.log( + savedObjectEvent({ + action: SavedObjectAction.CLOSE_POINT_IN_TIME, + }) + ); + + return response; + } + private async checkPrivileges( actions: string | string[], namespaceOrNamespaces?: string | Array diff --git a/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.test.ts b/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.test.ts index 8a749b5009334..632100de691d8 100644 --- a/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.test.ts +++ b/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.test.ts @@ -589,5 +589,31 @@ const ERROR_NAMESPACE_SPECIFIED = 'Spaces currently determines the namespaces'; }); }); }); + + describe('#openPointInTimeForType', () => { + test(`throws error if options.namespace is specified`, async () => { + const { client } = createSpacesSavedObjectsClient(); + + await expect(client.openPointInTimeForType('foo', { namespace: 'bar' })).rejects.toThrow( + ERROR_NAMESPACE_SPECIFIED + ); + }); + + test(`supplements options with the current namespace`, async () => { + const { client, baseClient } = createSpacesSavedObjectsClient(); + const expectedReturnValue = { id: 'abc123' }; + baseClient.openPointInTimeForType.mockReturnValue(Promise.resolve(expectedReturnValue)); + + const options = Object.freeze({ foo: 'bar' }); + // @ts-expect-error + const actualReturnValue = await client.openPointInTimeForType('foo', options); + + expect(actualReturnValue).toBe(expectedReturnValue); + expect(baseClient.openPointInTimeForType).toHaveBeenCalledWith('foo', { + foo: 'bar', + namespace: currentSpace.expectedNamespace, + }); + }); + }); }); }); diff --git a/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts b/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts index 3053d836f6661..3441d14877b32 100644 --- a/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts +++ b/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts @@ -394,6 +394,19 @@ export class SpacesSavedObjectsClient implements SavedObjectsClientContract { type: string | string[], options: SavedObjectsOpenPointInTimeOptions = {} ) { - return await this.client.openPointInTimeForType(type, options); + throwErrorIfNamespaceSpecified(options); + return await this.client.openPointInTimeForType(type, { + ...options, + namespace: spaceIdToNamespace(this.spaceId), + }); + } + + /** + * Closes a Point In Time (PIT) by ID. This simply proxies the request to ES + * via the Elasticsearch client, and is included in the Saved Objects Client + * as a convenience for consumers who are using `openPointInTimeForType`. + */ + async closePointInTime(id: string) { + return await this.client.closePointInTime(id); } } From eec7e2bc548261957e2f34ae401abf2e20bafad3 Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Sun, 7 Feb 2021 21:25:22 -0700 Subject: [PATCH 14/32] Fix some tests and clean up. --- src/core/server/index.ts | 3 +++ .../saved_objects/service/lib/repository.ts | 9 +++++-- .../service/lib/search_dsl/search_dsl.test.ts | 25 ++++++++++++++++++- .../service/saved_objects_client.ts | 9 +++++-- .../encrypted_saved_objects_client_wrapper.ts | 5 ++-- .../feature_privilege_builder/saved_object.ts | 8 +++++- .../secure_saved_objects_client_wrapper.ts | 5 ++-- .../spaces_saved_objects_client.ts | 5 ++-- 8 files changed, 57 insertions(+), 12 deletions(-) diff --git a/src/core/server/index.ts b/src/core/server/index.ts index 074b267b79ae1..e30d9019e6236 100644 --- a/src/core/server/index.ts +++ b/src/core/server/index.ts @@ -260,6 +260,8 @@ export { SavedObjectsClientWrapperOptions, SavedObjectsClientFactory, SavedObjectsClientFactoryProvider, + SavedObjectsClosePointInTimeOptions, + SavedObjectsClosePointInTimeResponse, SavedObjectsCreateOptions, SavedObjectsErrorHelpers, SavedObjectsExportResultDetails, @@ -278,6 +280,7 @@ export { SavedObjectMigrationContext, SavedObjectsMigrationLogger, SavedObjectsOpenPointInTimeOptions, + SavedObjectsOpenPointInTimeResponse, SavedObjectsRawDoc, SavedObjectsRawDocParseOptions, SavedObjectSanitizedDoc, diff --git a/src/core/server/saved_objects/service/lib/repository.ts b/src/core/server/saved_objects/service/lib/repository.ts index db8faa654f5ab..12e8be3887d5e 100644 --- a/src/core/server/saved_objects/service/lib/repository.ts +++ b/src/core/server/saved_objects/service/lib/repository.ts @@ -36,6 +36,7 @@ import { SavedObjectsCreateOptions, SavedObjectsFindResponse, SavedObjectsFindResult, + SavedObjectsClosePointInTimeOptions, SavedObjectsClosePointInTimeResponse, SavedObjectsOpenPointInTimeOptions, SavedObjectsOpenPointInTimeResponse, @@ -1881,9 +1882,13 @@ export class SavedObjectsRepository { * ``` * * @param {string} id - * @returns {promise} - { id: string } + * @param {object} [options] - {@link SavedObjectsClosePointInTimeOptions} + * @returns {promise} - {@link SavedObjectsClosePointInTimeResponse} */ - async closePointInTime(id: string): Promise { + async closePointInTime( + id: string, + options?: SavedObjectsClosePointInTimeOptions + ): Promise { const { body } = await this.client.closePointInTime({ body: { id }, }); diff --git a/src/core/server/saved_objects/service/lib/search_dsl/search_dsl.test.ts b/src/core/server/saved_objects/service/lib/search_dsl/search_dsl.test.ts index ebac0477b2866..fc26c837d5e52 100644 --- a/src/core/server/saved_objects/service/lib/search_dsl/search_dsl.test.ts +++ b/src/core/server/saved_objects/service/lib/search_dsl/search_dsl.test.ts @@ -6,14 +6,17 @@ * Side Public License, v 1. */ +jest.mock('./pit_params'); jest.mock('./query_params'); jest.mock('./sorting_params'); import { typeRegistryMock } from '../../../saved_objects_type_registry.mock'; +import * as pitParamsNS from './pit_params'; import * as queryParamsNS from './query_params'; import { getSearchDsl } from './search_dsl'; import * as sortParamsNS from './sorting_params'; +const getPitParams = pitParamsNS.getPitParams as jest.Mock; const getQueryParams = queryParamsNS.getQueryParams as jest.Mock; const getSortingParams = sortParamsNS.getSortingParams as jest.Mock; @@ -84,6 +87,7 @@ describe('getSearchDsl', () => { type: 'foo', sortField: 'bar', sortOrder: 'baz', + pit: { id: 'abc123' }, }; getSearchDsl(mappings, registry, opts); @@ -92,7 +96,8 @@ describe('getSearchDsl', () => { mappings, opts.type, opts.sortField, - opts.sortOrder + opts.sortOrder, + opts.pit ); }); @@ -111,5 +116,23 @@ describe('getSearchDsl', () => { search_after: [1, 'bar'], }); }); + + it('returns pit if provided', () => { + getQueryParams.mockReturnValue({ a: 'a' }); + getSortingParams.mockReturnValue({ b: 'b' }); + getPitParams.mockReturnValue({ pit: { id: 'abc123' } }); + expect( + getSearchDsl(mappings, registry, { + type: 'foo', + searchAfter: [1, 'bar'], + pit: { id: 'abc123' }, + }) + ).toEqual({ + a: 'a', + b: 'b', + pit: { id: 'abc123' }, + search_after: [1, 'bar'], + }); + }); }); }); diff --git a/src/core/server/saved_objects/service/saved_objects_client.ts b/src/core/server/saved_objects/service/saved_objects_client.ts index 0b76661b57672..4cd5316d78f73 100644 --- a/src/core/server/saved_objects/service/saved_objects_client.ts +++ b/src/core/server/saved_objects/service/saved_objects_client.ts @@ -340,6 +340,11 @@ export interface SavedObjectsOpenPointInTimeResponse { id: string; } +/** + * @public + */ +export type SavedObjectsClosePointInTimeOptions = SavedObjectsBaseOptions; + /** * @public */ @@ -565,7 +570,7 @@ export class SavedObjectsClient { * via the Elasticsearch client, and is included in the Saved Objects Client * as a convenience for consumers who are using `openPointInTimeForType`. */ - async closePointInTime(id: string) { - return await this._repository.closePointInTime(id); + async closePointInTime(id: string, options?: SavedObjectsClosePointInTimeOptions) { + return await this._repository.closePointInTime(id, options); } } diff --git a/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.ts b/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.ts index 48bfadb0cde49..a602f3606e0a9 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.ts @@ -15,6 +15,7 @@ import { SavedObjectsBulkUpdateResponse, SavedObjectsCheckConflictsObject, SavedObjectsClientContract, + SavedObjectsClosePointInTimeOptions, SavedObjectsCreateOptions, SavedObjectsFindOptions, SavedObjectsFindResponse, @@ -257,8 +258,8 @@ export class EncryptedSavedObjectsClientWrapper implements SavedObjectsClientCon return await this.options.baseClient.openPointInTimeForType(type, options); } - public async closePointInTime(id: string) { - return await this.options.baseClient.closePointInTime(id); + public async closePointInTime(id: string, options?: SavedObjectsClosePointInTimeOptions) { + return await this.options.baseClient.closePointInTime(id, options); } /** diff --git a/x-pack/plugins/security/server/authorization/privileges/feature_privilege_builder/saved_object.ts b/x-pack/plugins/security/server/authorization/privileges/feature_privilege_builder/saved_object.ts index 08c623d04dd78..3a0d9f4a5a100 100644 --- a/x-pack/plugins/security/server/authorization/privileges/feature_privilege_builder/saved_object.ts +++ b/x-pack/plugins/security/server/authorization/privileges/feature_privilege_builder/saved_object.ts @@ -9,7 +9,13 @@ import { flatten, uniq } from 'lodash'; import { FeatureKibanaPrivileges } from '../../../../../features/server'; import { BaseFeaturePrivilegeBuilder } from './feature_privilege_builder'; -const readOperations: string[] = ['bulk_get', 'get', 'find', 'open_point_in_time']; +const readOperations: string[] = [ + 'bulk_get', + 'get', + 'find', + 'open_point_in_time', + 'close_point_in_time', +]; const writeOperations: string[] = [ 'create', 'bulk_create', diff --git a/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts b/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts index 0721e1e00596f..c1fbf2ff60519 100644 --- a/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts +++ b/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts @@ -18,6 +18,7 @@ import { SavedObjectsDeleteFromNamespacesOptions, SavedObjectsFindOptions, SavedObjectsOpenPointInTimeOptions, + SavedObjectsClosePointInTimeOptions, SavedObjectsRemoveReferencesToOptions, SavedObjectsUpdateOptions, SavedObjectsUtils, @@ -591,8 +592,8 @@ export class SecureSavedObjectsClientWrapper implements SavedObjectsClientContra return pit; } - public async closePointInTime(id: string) { - const response = await this.baseClient.closePointInTime(id); + public async closePointInTime(id: string, options?: SavedObjectsClosePointInTimeOptions) { + const response = await this.baseClient.closePointInTime(id, options); this.auditLogger.log( savedObjectEvent({ diff --git a/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts b/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts index 3441d14877b32..95e67bcd1b51b 100644 --- a/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts +++ b/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts @@ -15,6 +15,7 @@ import { SavedObjectsClientContract, SavedObjectsCreateOptions, SavedObjectsFindOptions, + SavedObjectsClosePointInTimeOptions, SavedObjectsOpenPointInTimeOptions, SavedObjectsUpdateOptions, SavedObjectsAddToNamespacesOptions, @@ -406,7 +407,7 @@ export class SpacesSavedObjectsClient implements SavedObjectsClientContract { * via the Elasticsearch client, and is included in the Saved Objects Client * as a convenience for consumers who are using `openPointInTimeForType`. */ - async closePointInTime(id: string) { - return await this.client.closePointInTime(id); + async closePointInTime(id: string, options?: SavedObjectsClosePointInTimeOptions) { + return await this.client.closePointInTime(id, options); } } From 85f949a6ac5f3458cd164ae9f9ff958f19ce9f71 Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Sun, 7 Feb 2021 21:28:37 -0700 Subject: [PATCH 15/32] Update generated docs. --- ...core-public.savedobjectsfindoptions.pit.md | 16 +++++ ...lic.savedobjectsfindoptions.searchafter.md | 13 +++++ ...c.savedobjectsfindresponsepublic.pit_id.md | 11 ++++ .../core/server/kibana-plugin-core-server.md | 4 ++ ...ver.savedobjectsclient.closepointintime.md | 25 ++++++++ ...a-plugin-core-server.savedobjectsclient.md | 2 + ...vedobjectsclient.openpointintimefortype.md | 25 ++++++++ ...ver.savedobjectsclosepointintimeoptions.md | 12 ++++ ...er.savedobjectsclosepointintimeresponse.md | 20 +++++++ ...jectsclosepointintimeresponse.num_freed.md | 13 +++++ ...jectsclosepointintimeresponse.succeeded.md | 13 +++++ ...rver.savedobjectsexporter._constructor_.md | 5 +- ...plugin-core-server.savedobjectsexporter.md | 2 +- ...gin-core-server.savedobjectsfindoptions.md | 2 + ...core-server.savedobjectsfindoptions.pit.md | 16 +++++ ...ver.savedobjectsfindoptions.searchafter.md | 13 +++++ ...in-core-server.savedobjectsfindresponse.md | 1 + ...-server.savedobjectsfindresponse.pit_id.md | 11 ++++ ...ugin-core-server.savedobjectsfindresult.md | 1 + ...core-server.savedobjectsfindresult.sort.md | 13 +++++ ...objectsopenpointintimeoptions.keepalive.md | 13 +++++ ...rver.savedobjectsopenpointintimeoptions.md | 20 +++++++ ...bjectsopenpointintimeoptions.preference.md | 13 +++++ ....savedobjectsopenpointintimeresponse.id.md | 13 +++++ ...ver.savedobjectsopenpointintimeresponse.md | 19 ++++++ ...savedobjectsrepository.closepointintime.md | 58 +++++++++++++++++++ ...ugin-core-server.savedobjectsrepository.md | 2 + ...bjectsrepository.openpointintimefortype.md | 52 +++++++++++++++++ ...ibana-plugin-core-server.searchresponse.md | 1 + ...lugin-core-server.searchresponse.pit_id.md | 11 ++++ src/core/public/public.api.md | 7 +++ src/core/server/server.api.md | 38 +++++++++++- src/plugins/data/server/server.api.md | 2 +- 33 files changed, 462 insertions(+), 5 deletions(-) create mode 100644 docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.pit.md create mode 100644 docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.searchafter.md create mode 100644 docs/development/core/public/kibana-plugin-core-public.savedobjectsfindresponsepublic.pit_id.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.closepointintime.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.openpointintimefortype.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.savedobjectsclosepointintimeoptions.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.savedobjectsclosepointintimeresponse.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.savedobjectsclosepointintimeresponse.num_freed.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.savedobjectsclosepointintimeresponse.succeeded.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.pit.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.searchafter.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresponse.pit_id.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresult.sort.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.savedobjectsopenpointintimeoptions.keepalive.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.savedobjectsopenpointintimeoptions.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.savedobjectsopenpointintimeoptions.preference.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.savedobjectsopenpointintimeresponse.id.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.savedobjectsopenpointintimeresponse.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.closepointintime.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.openpointintimefortype.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.searchresponse.pit_id.md diff --git a/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.pit.md b/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.pit.md new file mode 100644 index 0000000000000..df8b8de7d96d6 --- /dev/null +++ b/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.pit.md @@ -0,0 +1,16 @@ + + +[Home](./index.md) > [kibana-plugin-core-public](./kibana-plugin-core-public.md) > [SavedObjectsFindOptions](./kibana-plugin-core-public.savedobjectsfindoptions.md) > [pit](./kibana-plugin-core-public.savedobjectsfindoptions.pit.md) + +## SavedObjectsFindOptions.pit property + +Search against a specific Point In Time (PIT) that you've opened with `savedObjects.openPointInTimeForType`. + +Signature: + +```typescript +pit?: { + id: string; + keepAlive?: string; + }; +``` diff --git a/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.searchafter.md b/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.searchafter.md new file mode 100644 index 0000000000000..99ca2c34e77be --- /dev/null +++ b/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.searchafter.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-core-public](./kibana-plugin-core-public.md) > [SavedObjectsFindOptions](./kibana-plugin-core-public.savedobjectsfindoptions.md) > [searchAfter](./kibana-plugin-core-public.savedobjectsfindoptions.searchafter.md) + +## SavedObjectsFindOptions.searchAfter property + +Use the sort values from the previous page to retrieve the next page of results. + +Signature: + +```typescript +searchAfter?: unknown[]; +``` diff --git a/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindresponsepublic.pit_id.md b/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindresponsepublic.pit_id.md new file mode 100644 index 0000000000000..f1228390bdbf0 --- /dev/null +++ b/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindresponsepublic.pit_id.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-core-public](./kibana-plugin-core-public.md) > [SavedObjectsFindResponsePublic](./kibana-plugin-core-public.savedobjectsfindresponsepublic.md) > [pit\_id](./kibana-plugin-core-public.savedobjectsfindresponsepublic.pit_id.md) + +## SavedObjectsFindResponsePublic.pit\_id property + +Signature: + +```typescript +pit_id?: string; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.md b/docs/development/core/server/kibana-plugin-core-server.md index 5fe5eda7a8172..a1cfaf5b4725c 100644 --- a/docs/development/core/server/kibana-plugin-core-server.md +++ b/docs/development/core/server/kibana-plugin-core-server.md @@ -155,6 +155,7 @@ The plugin integrates with the core system via lifecycle events: `setup` | [SavedObjectsCheckConflictsResponse](./kibana-plugin-core-server.savedobjectscheckconflictsresponse.md) | | | [SavedObjectsClientProviderOptions](./kibana-plugin-core-server.savedobjectsclientprovideroptions.md) | Options to control the creation of the Saved Objects Client. | | [SavedObjectsClientWrapperOptions](./kibana-plugin-core-server.savedobjectsclientwrapperoptions.md) | Options passed to each SavedObjectsClientWrapperFactory to aid in creating the wrapper instance. | +| [SavedObjectsClosePointInTimeResponse](./kibana-plugin-core-server.savedobjectsclosepointintimeresponse.md) | | | [SavedObjectsComplexFieldMapping](./kibana-plugin-core-server.savedobjectscomplexfieldmapping.md) | See [SavedObjectsFieldMapping](./kibana-plugin-core-server.savedobjectsfieldmapping.md) for documentation. | | [SavedObjectsCoreFieldMapping](./kibana-plugin-core-server.savedobjectscorefieldmapping.md) | See [SavedObjectsFieldMapping](./kibana-plugin-core-server.savedobjectsfieldmapping.md) for documentation. | | [SavedObjectsCreateOptions](./kibana-plugin-core-server.savedobjectscreateoptions.md) | | @@ -188,6 +189,8 @@ The plugin integrates with the core system via lifecycle events: `setup` | [SavedObjectsMappingProperties](./kibana-plugin-core-server.savedobjectsmappingproperties.md) | Describe the fields of a [saved object type](./kibana-plugin-core-server.savedobjectstypemappingdefinition.md). | | [SavedObjectsMigrationLogger](./kibana-plugin-core-server.savedobjectsmigrationlogger.md) | | | [SavedObjectsMigrationVersion](./kibana-plugin-core-server.savedobjectsmigrationversion.md) | Information about the migrations that have been applied to this SavedObject. When Kibana starts up, KibanaMigrator detects outdated documents and migrates them based on this value. For each migration that has been applied, the plugin's name is used as a key and the latest migration version as the value. | +| [SavedObjectsOpenPointInTimeOptions](./kibana-plugin-core-server.savedobjectsopenpointintimeoptions.md) | | +| [SavedObjectsOpenPointInTimeResponse](./kibana-plugin-core-server.savedobjectsopenpointintimeresponse.md) | | | [SavedObjectsRawDoc](./kibana-plugin-core-server.savedobjectsrawdoc.md) | A raw document as represented directly in the saved object index. | | [SavedObjectsRawDocParseOptions](./kibana-plugin-core-server.savedobjectsrawdocparseoptions.md) | Options that can be specified when using the saved objects serializer to parse a raw document. | | [SavedObjectsRemoveReferencesToOptions](./kibana-plugin-core-server.savedobjectsremovereferencestooptions.md) | | @@ -301,6 +304,7 @@ The plugin integrates with the core system via lifecycle events: `setup` | [SavedObjectsClientFactory](./kibana-plugin-core-server.savedobjectsclientfactory.md) | Describes the factory used to create instances of the Saved Objects Client. | | [SavedObjectsClientFactoryProvider](./kibana-plugin-core-server.savedobjectsclientfactoryprovider.md) | Provider to invoke to retrieve a [SavedObjectsClientFactory](./kibana-plugin-core-server.savedobjectsclientfactory.md). | | [SavedObjectsClientWrapperFactory](./kibana-plugin-core-server.savedobjectsclientwrapperfactory.md) | Describes the factory used to create instances of Saved Objects Client Wrappers. | +| [SavedObjectsClosePointInTimeOptions](./kibana-plugin-core-server.savedobjectsclosepointintimeoptions.md) | | | [SavedObjectsExportTransform](./kibana-plugin-core-server.savedobjectsexporttransform.md) | Transformation function used to mutate the exported objects of the associated type.A type's export transform function will be executed once per user-initiated export, for all objects of that type. | | [SavedObjectsFieldMapping](./kibana-plugin-core-server.savedobjectsfieldmapping.md) | Describe a [saved object type mapping](./kibana-plugin-core-server.savedobjectstypemappingdefinition.md) field.Please refer to [elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html) For the mapping documentation | | [SavedObjectsImportHook](./kibana-plugin-core-server.savedobjectsimporthook.md) | A hook associated with a specific saved object type, that will be invoked during the import process. The hook will have access to the objects of the registered type.Currently, the only supported feature for import hooks is to return warnings to be displayed in the UI when the import succeeds. The only interactions the hook can have with the import process is via the hook's response. Mutating the objects inside the hook's code will have no effect. | diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.closepointintime.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.closepointintime.md new file mode 100644 index 0000000000000..f6de3b9ba24cb --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.closepointintime.md @@ -0,0 +1,25 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsClient](./kibana-plugin-core-server.savedobjectsclient.md) > [closePointInTime](./kibana-plugin-core-server.savedobjectsclient.closepointintime.md) + +## SavedObjectsClient.closePointInTime() method + +Closes a Point In Time (PIT) by ID. This simply proxies the request to ES via the Elasticsearch client, and is included in the Saved Objects Client as a convenience for consumers who are using `openPointInTimeForType`. + +Signature: + +```typescript +closePointInTime(id: string, options?: SavedObjectsClosePointInTimeOptions): Promise; +``` + +## Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| id | string | | +| options | SavedObjectsClosePointInTimeOptions | | + +Returns: + +`Promise` + diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.md index da1f4d029ea2b..787a0e1b653d1 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.md @@ -30,11 +30,13 @@ The constructor for this class is marked as internal. Third-party code should no | [bulkGet(objects, options)](./kibana-plugin-core-server.savedobjectsclient.bulkget.md) | | Returns an array of objects by id | | [bulkUpdate(objects, options)](./kibana-plugin-core-server.savedobjectsclient.bulkupdate.md) | | Bulk Updates multiple SavedObject at once | | [checkConflicts(objects, options)](./kibana-plugin-core-server.savedobjectsclient.checkconflicts.md) | | Check what conflicts will result when creating a given array of saved objects. This includes "unresolvable conflicts", which are multi-namespace objects that exist in a different namespace; such conflicts cannot be resolved/overwritten. | +| [closePointInTime(id, options)](./kibana-plugin-core-server.savedobjectsclient.closepointintime.md) | | Closes a Point In Time (PIT) by ID. This simply proxies the request to ES via the Elasticsearch client, and is included in the Saved Objects Client as a convenience for consumers who are using openPointInTimeForType. | | [create(type, attributes, options)](./kibana-plugin-core-server.savedobjectsclient.create.md) | | Persists a SavedObject | | [delete(type, id, options)](./kibana-plugin-core-server.savedobjectsclient.delete.md) | | Deletes a SavedObject | | [deleteFromNamespaces(type, id, namespaces, options)](./kibana-plugin-core-server.savedobjectsclient.deletefromnamespaces.md) | | Removes namespaces from a SavedObject | | [find(options)](./kibana-plugin-core-server.savedobjectsclient.find.md) | | Find all SavedObjects matching the search query | | [get(type, id, options)](./kibana-plugin-core-server.savedobjectsclient.get.md) | | Retrieves a single object | +| [openPointInTimeForType(type, options)](./kibana-plugin-core-server.savedobjectsclient.openpointintimefortype.md) | | Opens a Point In Time (PIT) against the indices for the specified Saved Object types. The returned id can then be passed to SavedObjects.find to search against that PIT. | | [removeReferencesTo(type, id, options)](./kibana-plugin-core-server.savedobjectsclient.removereferencesto.md) | | Updates all objects containing a reference to the given {type, id} tuple to remove the said reference. | | [resolve(type, id, options)](./kibana-plugin-core-server.savedobjectsclient.resolve.md) | | Resolves a single object, using any legacy URL alias if it exists | | [update(type, id, attributes, options)](./kibana-plugin-core-server.savedobjectsclient.update.md) | | Updates an SavedObject | diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.openpointintimefortype.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.openpointintimefortype.md new file mode 100644 index 0000000000000..c7dc50f9cd066 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.openpointintimefortype.md @@ -0,0 +1,25 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsClient](./kibana-plugin-core-server.savedobjectsclient.md) > [openPointInTimeForType](./kibana-plugin-core-server.savedobjectsclient.openpointintimefortype.md) + +## SavedObjectsClient.openPointInTimeForType() method + +Opens a Point In Time (PIT) against the indices for the specified Saved Object types. The returned `id` can then be passed to `SavedObjects.find` to search against that PIT. + +Signature: + +```typescript +openPointInTimeForType(type: string | string[], options?: SavedObjectsOpenPointInTimeOptions): Promise; +``` + +## Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| type | string | string[] | | +| options | SavedObjectsOpenPointInTimeOptions | | + +Returns: + +`Promise` + diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsclosepointintimeoptions.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsclosepointintimeoptions.md new file mode 100644 index 0000000000000..27432a8805b06 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsclosepointintimeoptions.md @@ -0,0 +1,12 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsClosePointInTimeOptions](./kibana-plugin-core-server.savedobjectsclosepointintimeoptions.md) + +## SavedObjectsClosePointInTimeOptions type + + +Signature: + +```typescript +export declare type SavedObjectsClosePointInTimeOptions = SavedObjectsBaseOptions; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsclosepointintimeresponse.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsclosepointintimeresponse.md new file mode 100644 index 0000000000000..43ecd1298d5d9 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsclosepointintimeresponse.md @@ -0,0 +1,20 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsClosePointInTimeResponse](./kibana-plugin-core-server.savedobjectsclosepointintimeresponse.md) + +## SavedObjectsClosePointInTimeResponse interface + + +Signature: + +```typescript +export interface SavedObjectsClosePointInTimeResponse +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [num\_freed](./kibana-plugin-core-server.savedobjectsclosepointintimeresponse.num_freed.md) | number | The number of search contexts that have been successfully closed. | +| [succeeded](./kibana-plugin-core-server.savedobjectsclosepointintimeresponse.succeeded.md) | boolean | If true, all search contexts associated with the PIT id are successfully closed. | + diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsclosepointintimeresponse.num_freed.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsclosepointintimeresponse.num_freed.md new file mode 100644 index 0000000000000..b64932fcee8f6 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsclosepointintimeresponse.num_freed.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsClosePointInTimeResponse](./kibana-plugin-core-server.savedobjectsclosepointintimeresponse.md) > [num\_freed](./kibana-plugin-core-server.savedobjectsclosepointintimeresponse.num_freed.md) + +## SavedObjectsClosePointInTimeResponse.num\_freed property + +The number of search contexts that have been successfully closed. + +Signature: + +```typescript +num_freed: number; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsclosepointintimeresponse.succeeded.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsclosepointintimeresponse.succeeded.md new file mode 100644 index 0000000000000..225a549a4cf59 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsclosepointintimeresponse.succeeded.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsClosePointInTimeResponse](./kibana-plugin-core-server.savedobjectsclosepointintimeresponse.md) > [succeeded](./kibana-plugin-core-server.savedobjectsclosepointintimeresponse.succeeded.md) + +## SavedObjectsClosePointInTimeResponse.succeeded property + +If true, all search contexts associated with the PIT id are successfully closed. + +Signature: + +```typescript +succeeded: boolean; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsexporter._constructor_.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsexporter._constructor_.md index 5e959bbee7beb..3f3d708c590ee 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsexporter._constructor_.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsexporter._constructor_.md @@ -9,10 +9,11 @@ Constructs a new instance of the `SavedObjectsExporter` class Signature: ```typescript -constructor({ savedObjectsClient, typeRegistry, exportSizeLimit, }: { +constructor({ savedObjectsClient, typeRegistry, exportSizeLimit, logger, }: { savedObjectsClient: SavedObjectsClientContract; typeRegistry: ISavedObjectTypeRegistry; exportSizeLimit: number; + logger: Logger; }); ``` @@ -20,5 +21,5 @@ constructor({ savedObjectsClient, typeRegistry, exportSizeLimit, }: { | Parameter | Type | Description | | --- | --- | --- | -| { savedObjectsClient, typeRegistry, exportSizeLimit, } | {
savedObjectsClient: SavedObjectsClientContract;
typeRegistry: ISavedObjectTypeRegistry;
exportSizeLimit: number;
} | | +| { savedObjectsClient, typeRegistry, exportSizeLimit, logger, } | {
savedObjectsClient: SavedObjectsClientContract;
typeRegistry: ISavedObjectTypeRegistry;
exportSizeLimit: number;
logger: Logger;
} | | diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsexporter.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsexporter.md index 727108b824c84..ce23e91633b07 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsexporter.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsexporter.md @@ -15,7 +15,7 @@ export declare class SavedObjectsExporter | Constructor | Modifiers | Description | | --- | --- | --- | -| [(constructor)({ savedObjectsClient, typeRegistry, exportSizeLimit, })](./kibana-plugin-core-server.savedobjectsexporter._constructor_.md) | | Constructs a new instance of the SavedObjectsExporter class | +| [(constructor)({ savedObjectsClient, typeRegistry, exportSizeLimit, logger, })](./kibana-plugin-core-server.savedobjectsexporter._constructor_.md) | | Constructs a new instance of the SavedObjectsExporter class | ## Properties diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.md index d393d579dbdd2..cf97d45066e70 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.md @@ -23,9 +23,11 @@ export interface SavedObjectsFindOptions | [namespaces](./kibana-plugin-core-server.savedobjectsfindoptions.namespaces.md) | string[] | | | [page](./kibana-plugin-core-server.savedobjectsfindoptions.page.md) | number | | | [perPage](./kibana-plugin-core-server.savedobjectsfindoptions.perpage.md) | number | | +| [pit](./kibana-plugin-core-server.savedobjectsfindoptions.pit.md) | {
id: string;
keepAlive?: string;
} | Search against a specific Point In Time (PIT) that you've opened with savedObjects.openPointInTimeForType. | | [preference](./kibana-plugin-core-server.savedobjectsfindoptions.preference.md) | string | An optional ES preference value to be used for the query \* | | [rootSearchFields](./kibana-plugin-core-server.savedobjectsfindoptions.rootsearchfields.md) | string[] | The fields to perform the parsed query against. Unlike the searchFields argument, these are expected to be root fields and will not be modified. If used in conjunction with searchFields, both are concatenated together. | | [search](./kibana-plugin-core-server.savedobjectsfindoptions.search.md) | string | Search documents using the Elasticsearch Simple Query String syntax. See Elasticsearch Simple Query String query argument for more information | +| [searchAfter](./kibana-plugin-core-server.savedobjectsfindoptions.searchafter.md) | unknown[] | Use the sort values from the previous page to retrieve the next page of results. | | [searchFields](./kibana-plugin-core-server.savedobjectsfindoptions.searchfields.md) | string[] | The fields to perform the parsed query against. See Elasticsearch Simple Query String fields argument for more information | | [sortField](./kibana-plugin-core-server.savedobjectsfindoptions.sortfield.md) | string | | | [sortOrder](./kibana-plugin-core-server.savedobjectsfindoptions.sortorder.md) | string | | diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.pit.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.pit.md new file mode 100644 index 0000000000000..2c6b280f01c5b --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.pit.md @@ -0,0 +1,16 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsFindOptions](./kibana-plugin-core-server.savedobjectsfindoptions.md) > [pit](./kibana-plugin-core-server.savedobjectsfindoptions.pit.md) + +## SavedObjectsFindOptions.pit property + +Search against a specific Point In Time (PIT) that you've opened with `savedObjects.openPointInTimeForType`. + +Signature: + +```typescript +pit?: { + id: string; + keepAlive?: string; + }; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.searchafter.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.searchafter.md new file mode 100644 index 0000000000000..6364370948976 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.searchafter.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsFindOptions](./kibana-plugin-core-server.savedobjectsfindoptions.md) > [searchAfter](./kibana-plugin-core-server.savedobjectsfindoptions.searchafter.md) + +## SavedObjectsFindOptions.searchAfter property + +Use the sort values from the previous page to retrieve the next page of results. + +Signature: + +```typescript +searchAfter?: unknown[]; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresponse.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresponse.md index 4ed069d1598fe..fd56e8ce40e24 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresponse.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresponse.md @@ -20,6 +20,7 @@ export interface SavedObjectsFindResponse | --- | --- | --- | | [page](./kibana-plugin-core-server.savedobjectsfindresponse.page.md) | number | | | [per\_page](./kibana-plugin-core-server.savedobjectsfindresponse.per_page.md) | number | | +| [pit\_id](./kibana-plugin-core-server.savedobjectsfindresponse.pit_id.md) | string | | | [saved\_objects](./kibana-plugin-core-server.savedobjectsfindresponse.saved_objects.md) | Array<SavedObjectsFindResult<T>> | | | [total](./kibana-plugin-core-server.savedobjectsfindresponse.total.md) | number | | diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresponse.pit_id.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresponse.pit_id.md new file mode 100644 index 0000000000000..dc4f9b509d606 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresponse.pit_id.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsFindResponse](./kibana-plugin-core-server.savedobjectsfindresponse.md) > [pit\_id](./kibana-plugin-core-server.savedobjectsfindresponse.pit_id.md) + +## SavedObjectsFindResponse.pit\_id property + +Signature: + +```typescript +pit_id?: string; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresult.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresult.md index e455074a7d11b..0f8e9c59236bb 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresult.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresult.md @@ -16,4 +16,5 @@ export interface SavedObjectsFindResult extends SavedObject | Property | Type | Description | | --- | --- | --- | | [score](./kibana-plugin-core-server.savedobjectsfindresult.score.md) | number | The Elasticsearch _score of this result. | +| [sort](./kibana-plugin-core-server.savedobjectsfindresult.sort.md) | unknown[] | The Elasticsearch sort value of this result. | diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresult.sort.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresult.sort.md new file mode 100644 index 0000000000000..2f85403e56a38 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresult.sort.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsFindResult](./kibana-plugin-core-server.savedobjectsfindresult.md) > [sort](./kibana-plugin-core-server.savedobjectsfindresult.sort.md) + +## SavedObjectsFindResult.sort property + +The Elasticsearch `sort` value of this result. + +Signature: + +```typescript +sort?: unknown[]; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsopenpointintimeoptions.keepalive.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsopenpointintimeoptions.keepalive.md new file mode 100644 index 0000000000000..57752318cb96a --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsopenpointintimeoptions.keepalive.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsOpenPointInTimeOptions](./kibana-plugin-core-server.savedobjectsopenpointintimeoptions.md) > [keepAlive](./kibana-plugin-core-server.savedobjectsopenpointintimeoptions.keepalive.md) + +## SavedObjectsOpenPointInTimeOptions.keepAlive property + +Optionally specify how long ES should keep the PIT alive until the next request. Defaults to `5m`. + +Signature: + +```typescript +keepAlive?: string; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsopenpointintimeoptions.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsopenpointintimeoptions.md new file mode 100644 index 0000000000000..46516be2329e9 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsopenpointintimeoptions.md @@ -0,0 +1,20 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsOpenPointInTimeOptions](./kibana-plugin-core-server.savedobjectsopenpointintimeoptions.md) + +## SavedObjectsOpenPointInTimeOptions interface + + +Signature: + +```typescript +export interface SavedObjectsOpenPointInTimeOptions extends SavedObjectsBaseOptions +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [keepAlive](./kibana-plugin-core-server.savedobjectsopenpointintimeoptions.keepalive.md) | string | Optionally specify how long ES should keep the PIT alive until the next request. Defaults to 5m. | +| [preference](./kibana-plugin-core-server.savedobjectsopenpointintimeoptions.preference.md) | string | An optional ES preference value to be used for the query. | + diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsopenpointintimeoptions.preference.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsopenpointintimeoptions.preference.md new file mode 100644 index 0000000000000..7a9f3a49e8663 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsopenpointintimeoptions.preference.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsOpenPointInTimeOptions](./kibana-plugin-core-server.savedobjectsopenpointintimeoptions.md) > [preference](./kibana-plugin-core-server.savedobjectsopenpointintimeoptions.preference.md) + +## SavedObjectsOpenPointInTimeOptions.preference property + +An optional ES preference value to be used for the query. + +Signature: + +```typescript +preference?: string; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsopenpointintimeresponse.id.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsopenpointintimeresponse.id.md new file mode 100644 index 0000000000000..66387e5b3b89f --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsopenpointintimeresponse.id.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsOpenPointInTimeResponse](./kibana-plugin-core-server.savedobjectsopenpointintimeresponse.md) > [id](./kibana-plugin-core-server.savedobjectsopenpointintimeresponse.id.md) + +## SavedObjectsOpenPointInTimeResponse.id property + +PIT ID returned from ES. + +Signature: + +```typescript +id: string; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsopenpointintimeresponse.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsopenpointintimeresponse.md new file mode 100644 index 0000000000000..c4be2692763a5 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsopenpointintimeresponse.md @@ -0,0 +1,19 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsOpenPointInTimeResponse](./kibana-plugin-core-server.savedobjectsopenpointintimeresponse.md) + +## SavedObjectsOpenPointInTimeResponse interface + + +Signature: + +```typescript +export interface SavedObjectsOpenPointInTimeResponse +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [id](./kibana-plugin-core-server.savedobjectsopenpointintimeresponse.id.md) | string | PIT ID returned from ES. | + diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.closepointintime.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.closepointintime.md new file mode 100644 index 0000000000000..27d732ac8f3e6 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.closepointintime.md @@ -0,0 +1,58 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsRepository](./kibana-plugin-core-server.savedobjectsrepository.md) > [closePointInTime](./kibana-plugin-core-server.savedobjectsrepository.closepointintime.md) + +## SavedObjectsRepository.closePointInTime() method + +Closes a Point In Time (PIT) by ID. This simply proxies the request to ES via the Elasticsearch client, and is included in the Saved Objects Client as a convenience for consumers who are using `openPointInTimeForType`. + +Signature: + +```typescript +closePointInTime(id: string, options?: SavedObjectsClosePointInTimeOptions): Promise; +``` + +## Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| id | string | | +| options | SavedObjectsClosePointInTimeOptions | | + +Returns: + +`Promise` + +{promise} - [SavedObjectsClosePointInTimeResponse](./kibana-plugin-core-server.savedobjectsclosepointintimeresponse.md) + +## Remarks + +While the `keepAlive` that is provided will cause a PIT to automatically close, it is highly recommended to explicitly close a PIT when you are done with it in order to avoid consuming unneeded resources in Elasticsearch. + +## Example + + +```ts +const repository = coreStart.savedObjects.createInternalRepository(); + +const { id } = await repository.openPointInTimeForType( + type: 'index-pattern', + { keepAlive: '1m' }, +); + +const response = await repository.find({ + type: 'index-pattern', + search: 'foo*', + sortField: 'name', + sortOrder: 'desc', + pit: { + id: 'abc123', + keepAlive: '1m', + }, + searchAfter: [1234, 'abcd'], +}); + +await repository.closePointInTime(response.pit_id); + +``` + diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.md index 4d13fea12572c..632d9c279cb88 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.md @@ -20,6 +20,7 @@ export declare class SavedObjectsRepository | [bulkGet(objects, options)](./kibana-plugin-core-server.savedobjectsrepository.bulkget.md) | | Returns an array of objects by id | | [bulkUpdate(objects, options)](./kibana-plugin-core-server.savedobjectsrepository.bulkupdate.md) | | Updates multiple objects in bulk | | [checkConflicts(objects, options)](./kibana-plugin-core-server.savedobjectsrepository.checkconflicts.md) | | Check what conflicts will result when creating a given array of saved objects. This includes "unresolvable conflicts", which are multi-namespace objects that exist in a different namespace; such conflicts cannot be resolved/overwritten. | +| [closePointInTime(id, options)](./kibana-plugin-core-server.savedobjectsrepository.closepointintime.md) | | Closes a Point In Time (PIT) by ID. This simply proxies the request to ES via the Elasticsearch client, and is included in the Saved Objects Client as a convenience for consumers who are using openPointInTimeForType. | | [create(type, attributes, options)](./kibana-plugin-core-server.savedobjectsrepository.create.md) | | Persists an object | | [delete(type, id, options)](./kibana-plugin-core-server.savedobjectsrepository.delete.md) | | Deletes an object | | [deleteByNamespace(namespace, options)](./kibana-plugin-core-server.savedobjectsrepository.deletebynamespace.md) | | Deletes all objects from the provided namespace. | @@ -27,6 +28,7 @@ export declare class SavedObjectsRepository | [find(options)](./kibana-plugin-core-server.savedobjectsrepository.find.md) | | | | [get(type, id, options)](./kibana-plugin-core-server.savedobjectsrepository.get.md) | | Gets a single object | | [incrementCounter(type, id, counterFields, options)](./kibana-plugin-core-server.savedobjectsrepository.incrementcounter.md) | | Increments all the specified counter fields (by one by default). Creates the document if one doesn't exist for the given id. | +| [openPointInTimeForType(type, { keepAlive, preference })](./kibana-plugin-core-server.savedobjectsrepository.openpointintimefortype.md) | | Opens a Point In Time (PIT) against the indices for the specified Saved Object types. The returned id can then be passed to SavedObjects.find to search against that PIT. | | [removeReferencesTo(type, id, options)](./kibana-plugin-core-server.savedobjectsrepository.removereferencesto.md) | | Updates all objects containing a reference to the given {type, id} tuple to remove the said reference. | | [resolve(type, id, options)](./kibana-plugin-core-server.savedobjectsrepository.resolve.md) | | Resolves a single object, using any legacy URL alias if it exists | | [update(type, id, attributes, options)](./kibana-plugin-core-server.savedobjectsrepository.update.md) | | Updates an object | diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.openpointintimefortype.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.openpointintimefortype.md new file mode 100644 index 0000000000000..1845dbc64e43d --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.openpointintimefortype.md @@ -0,0 +1,52 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsRepository](./kibana-plugin-core-server.savedobjectsrepository.md) > [openPointInTimeForType](./kibana-plugin-core-server.savedobjectsrepository.openpointintimefortype.md) + +## SavedObjectsRepository.openPointInTimeForType() method + +Opens a Point In Time (PIT) against the indices for the specified Saved Object types. The returned `id` can then be passed to `SavedObjects.find` to search against that PIT. + +Signature: + +```typescript +openPointInTimeForType(type: string | string[], { keepAlive, preference }?: SavedObjectsOpenPointInTimeOptions): Promise; +``` + +## Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| type | string | string[] | | +| { keepAlive, preference } | SavedObjectsOpenPointInTimeOptions | | + +Returns: + +`Promise` + +{promise} - { id: string } + +## Example + + +```ts +const repository = coreStart.savedObjects.createInternalRepository(); + +const { id } = await repository.openPointInTimeForType( + type: 'index-pattern', + { keepAlive: '1m' }, +); + +const response = await repository.find({ + type: 'index-pattern', + search: 'foo*', + sortField: 'name', + sortOrder: 'desc', + pit: { + id: 'abc123', + keepAlive: '1m', + }, + searchAfter: [1234, 'abcd'], +}); + +``` + diff --git a/docs/development/core/server/kibana-plugin-core-server.searchresponse.md b/docs/development/core/server/kibana-plugin-core-server.searchresponse.md index b53cbf0d87f24..e7ec6f96ada65 100644 --- a/docs/development/core/server/kibana-plugin-core-server.searchresponse.md +++ b/docs/development/core/server/kibana-plugin-core-server.searchresponse.md @@ -19,6 +19,7 @@ export interface SearchResponse | [\_shards](./kibana-plugin-core-server.searchresponse._shards.md) | ShardsResponse | | | [aggregations](./kibana-plugin-core-server.searchresponse.aggregations.md) | any | | | [hits](./kibana-plugin-core-server.searchresponse.hits.md) | {
total: number;
max_score: number;
hits: Array<{
_index: string;
_type: string;
_id: string;
_score: number;
_source: T;
_version?: number;
_explanation?: Explanation;
fields?: any;
highlight?: any;
inner_hits?: any;
matched_queries?: string[];
sort?: string[];
}>;
} | | +| [pit\_id](./kibana-plugin-core-server.searchresponse.pit_id.md) | string | | | [timed\_out](./kibana-plugin-core-server.searchresponse.timed_out.md) | boolean | | | [took](./kibana-plugin-core-server.searchresponse.took.md) | number | | diff --git a/docs/development/core/server/kibana-plugin-core-server.searchresponse.pit_id.md b/docs/development/core/server/kibana-plugin-core-server.searchresponse.pit_id.md new file mode 100644 index 0000000000000..f214bc0538045 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.searchresponse.pit_id.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SearchResponse](./kibana-plugin-core-server.searchresponse.md) > [pit\_id](./kibana-plugin-core-server.searchresponse.pit_id.md) + +## SearchResponse.pit\_id property + +Signature: + +```typescript +pit_id?: string; +``` diff --git a/src/core/public/public.api.md b/src/core/public/public.api.md index 99579ada8ec58..04004bb549c6e 100644 --- a/src/core/public/public.api.md +++ b/src/core/public/public.api.md @@ -1197,9 +1197,14 @@ export interface SavedObjectsFindOptions { page?: number; // (undocumented) perPage?: number; + pit?: { + id: string; + keepAlive?: string; + }; preference?: string; rootSearchFields?: string[]; search?: string; + searchAfter?: unknown[]; searchFields?: string[]; // (undocumented) sortField?: string; @@ -1225,6 +1230,8 @@ export interface SavedObjectsFindResponsePublic extends SavedObject // (undocumented) perPage: number; // (undocumented) + pit_id?: string; + // (undocumented) total: number; } diff --git a/src/core/server/server.api.md b/src/core/server/server.api.md index 09207608908a4..3739fc9f4d6f4 100644 --- a/src/core/server/server.api.md +++ b/src/core/server/server.api.md @@ -2222,6 +2222,7 @@ export class SavedObjectsClient { bulkGet(objects?: SavedObjectsBulkGetObject[], options?: SavedObjectsBaseOptions): Promise>; bulkUpdate(objects: Array>, options?: SavedObjectsBulkUpdateOptions): Promise>; checkConflicts(objects?: SavedObjectsCheckConflictsObject[], options?: SavedObjectsBaseOptions): Promise; + closePointInTime(id: string, options?: SavedObjectsClosePointInTimeOptions): Promise; create(type: string, attributes: T, options?: SavedObjectsCreateOptions): Promise>; delete(type: string, id: string, options?: SavedObjectsDeleteOptions): Promise<{}>; deleteFromNamespaces(type: string, id: string, namespaces: string[], options?: SavedObjectsDeleteFromNamespacesOptions): Promise; @@ -2231,6 +2232,7 @@ export class SavedObjectsClient { errors: typeof SavedObjectsErrorHelpers; find(options: SavedObjectsFindOptions): Promise>; get(type: string, id: string, options?: SavedObjectsBaseOptions): Promise>; + openPointInTimeForType(type: string | string[], options?: SavedObjectsOpenPointInTimeOptions): Promise; removeReferencesTo(type: string, id: string, options?: SavedObjectsRemoveReferencesToOptions): Promise; resolve(type: string, id: string, options?: SavedObjectsBaseOptions): Promise>; update(type: string, id: string, attributes: Partial, options?: SavedObjectsUpdateOptions): Promise>; @@ -2269,6 +2271,15 @@ export interface SavedObjectsClientWrapperOptions { typeRegistry: ISavedObjectTypeRegistry; } +// @public (undocumented) +export type SavedObjectsClosePointInTimeOptions = SavedObjectsBaseOptions; + +// @public (undocumented) +export interface SavedObjectsClosePointInTimeResponse { + num_freed: number; + succeeded: boolean; +} + // @public export interface SavedObjectsComplexFieldMapping { // (undocumented) @@ -2419,10 +2430,11 @@ export interface SavedObjectsExportByTypeOptions extends SavedObjectExportBaseOp export class SavedObjectsExporter { // (undocumented) #private; - constructor({ savedObjectsClient, typeRegistry, exportSizeLimit, }: { + constructor({ savedObjectsClient, typeRegistry, exportSizeLimit, logger, }: { savedObjectsClient: SavedObjectsClientContract; typeRegistry: ISavedObjectTypeRegistry; exportSizeLimit: number; + logger: Logger; }); exportByObjects(options: SavedObjectsExportByObjectOptions): Promise; exportByTypes(options: SavedObjectsExportByTypeOptions): Promise; @@ -2480,9 +2492,14 @@ export interface SavedObjectsFindOptions { page?: number; // (undocumented) perPage?: number; + pit?: { + id: string; + keepAlive?: string; + }; preference?: string; rootSearchFields?: string[]; search?: string; + searchAfter?: unknown[]; searchFields?: string[]; // (undocumented) sortField?: string; @@ -2508,6 +2525,8 @@ export interface SavedObjectsFindResponse { // (undocumented) per_page: number; // (undocumented) + pit_id?: string; + // (undocumented) saved_objects: Array>; // (undocumented) total: number; @@ -2516,6 +2535,7 @@ export interface SavedObjectsFindResponse { // @public (undocumented) export interface SavedObjectsFindResult extends SavedObject { score: number; + sort?: unknown[]; } // @public @@ -2742,6 +2762,17 @@ export interface SavedObjectsMigrationVersion { // @public export type SavedObjectsNamespaceType = 'single' | 'multiple' | 'agnostic'; +// @public (undocumented) +export interface SavedObjectsOpenPointInTimeOptions extends SavedObjectsBaseOptions { + keepAlive?: string; + preference?: string; +} + +// @public (undocumented) +export interface SavedObjectsOpenPointInTimeResponse { + id: string; +} + // @public export interface SavedObjectsRawDoc { // (undocumented) @@ -2778,6 +2809,7 @@ export class SavedObjectsRepository { bulkGet(objects?: SavedObjectsBulkGetObject[], options?: SavedObjectsBaseOptions): Promise>; bulkUpdate(objects: Array>, options?: SavedObjectsBulkUpdateOptions): Promise>; checkConflicts(objects?: SavedObjectsCheckConflictsObject[], options?: SavedObjectsBaseOptions): Promise; + closePointInTime(id: string, options?: SavedObjectsClosePointInTimeOptions): Promise; create(type: string, attributes: T, options?: SavedObjectsCreateOptions): Promise>; // Warning: (ae-forgotten-export) The symbol "IKibanaMigrator" needs to be exported by the entry point index.d.ts // @@ -2790,6 +2822,8 @@ export class SavedObjectsRepository { find(options: SavedObjectsFindOptions): Promise>; get(type: string, id: string, options?: SavedObjectsBaseOptions): Promise>; incrementCounter(type: string, id: string, counterFields: Array, options?: SavedObjectsIncrementCounterOptions): Promise>; + // Warning: (ae-unresolved-link) The @link reference could not be resolved: The package "kibana" does not have an export "SavedObjectsPointInTimeOptions" + openPointInTimeForType(type: string | string[], { keepAlive, preference }?: SavedObjectsOpenPointInTimeOptions): Promise; removeReferencesTo(type: string, id: string, options?: SavedObjectsRemoveReferencesToOptions): Promise; resolve(type: string, id: string, options?: SavedObjectsBaseOptions): Promise>; update(type: string, id: string, attributes: Partial, options?: SavedObjectsUpdateOptions): Promise>; @@ -2958,6 +2992,8 @@ export interface SearchResponse { }>; }; // (undocumented) + pit_id?: string; + // (undocumented) _scroll_id?: string; // (undocumented) _shards: ShardsResponse; diff --git a/src/plugins/data/server/server.api.md b/src/plugins/data/server/server.api.md index 3b1440f211bfe..60f57e9e6ca7d 100644 --- a/src/plugins/data/server/server.api.md +++ b/src/plugins/data/server/server.api.md @@ -1137,7 +1137,7 @@ export class Plugin implements Plugin_2 Promise; }; indexPatterns: { - indexPatternsServiceFactory: (savedObjectsClient: Pick, elasticsearchClient: import("../../../core/server").ElasticsearchClient) => Promise; + indexPatternsServiceFactory: (savedObjectsClient: Pick, elasticsearchClient: import("../../../core/server").ElasticsearchClient) => Promise; }; search: ISearchStart>; }; From fcad6db6221aaeb116c195ada12b88b99e5a75f5 Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Mon, 8 Feb 2021 10:37:09 -0700 Subject: [PATCH 16/32] Fix SO client and security jest tests. --- .../service/saved_objects_client.test.js | 5 +- .../privileges/privileges.test.ts | 212 ++++++++++++++++++ 2 files changed, 215 insertions(+), 2 deletions(-) diff --git a/src/core/server/saved_objects/service/saved_objects_client.test.js b/src/core/server/saved_objects/service/saved_objects_client.test.js index 7f803a6e96373..7cbddaf195dc9 100644 --- a/src/core/server/saved_objects/service/saved_objects_client.test.js +++ b/src/core/server/saved_objects/service/saved_objects_client.test.js @@ -138,9 +138,10 @@ test(`#closePointInTime`, async () => { const client = new SavedObjectsClient(mockRepository); const id = Symbol(); - const result = await client.closePointInTime(id); + const options = Symbol(); + const result = await client.closePointInTime(id, options); - expect(mockRepository.closePointInTime).toHaveBeenCalledWith(id); + expect(mockRepository.closePointInTime).toHaveBeenCalledWith(id, options); expect(result).toBe(returnValue); }); diff --git a/x-pack/plugins/security/server/authorization/privileges/privileges.test.ts b/x-pack/plugins/security/server/authorization/privileges/privileges.test.ts index 6c2a57e57dcd8..da2639aba1c6b 100644 --- a/x-pack/plugins/security/server/authorization/privileges/privileges.test.ts +++ b/x-pack/plugins/security/server/authorization/privileges/privileges.test.ts @@ -101,6 +101,8 @@ describe('features', () => { actions.savedObject.get('all-savedObject-all-1', 'bulk_get'), actions.savedObject.get('all-savedObject-all-1', 'get'), actions.savedObject.get('all-savedObject-all-1', 'find'), + actions.savedObject.get('all-savedObject-all-1', 'open_point_in_time'), + actions.savedObject.get('all-savedObject-all-1', 'close_point_in_time'), actions.savedObject.get('all-savedObject-all-1', 'create'), actions.savedObject.get('all-savedObject-all-1', 'bulk_create'), actions.savedObject.get('all-savedObject-all-1', 'update'), @@ -110,6 +112,8 @@ describe('features', () => { actions.savedObject.get('all-savedObject-all-2', 'bulk_get'), actions.savedObject.get('all-savedObject-all-2', 'get'), actions.savedObject.get('all-savedObject-all-2', 'find'), + actions.savedObject.get('all-savedObject-all-2', 'open_point_in_time'), + actions.savedObject.get('all-savedObject-all-2', 'close_point_in_time'), actions.savedObject.get('all-savedObject-all-2', 'create'), actions.savedObject.get('all-savedObject-all-2', 'bulk_create'), actions.savedObject.get('all-savedObject-all-2', 'update'), @@ -119,9 +123,13 @@ describe('features', () => { actions.savedObject.get('all-savedObject-read-1', 'bulk_get'), actions.savedObject.get('all-savedObject-read-1', 'get'), actions.savedObject.get('all-savedObject-read-1', 'find'), + actions.savedObject.get('all-savedObject-read-1', 'open_point_in_time'), + actions.savedObject.get('all-savedObject-read-1', 'close_point_in_time'), actions.savedObject.get('all-savedObject-read-2', 'bulk_get'), actions.savedObject.get('all-savedObject-read-2', 'get'), actions.savedObject.get('all-savedObject-read-2', 'find'), + actions.savedObject.get('all-savedObject-read-2', 'open_point_in_time'), + actions.savedObject.get('all-savedObject-read-2', 'close_point_in_time'), actions.ui.get('foo', 'all-ui-1'), actions.ui.get('foo', 'all-ui-2'), ]; @@ -132,6 +140,8 @@ describe('features', () => { actions.savedObject.get('read-savedObject-all-1', 'bulk_get'), actions.savedObject.get('read-savedObject-all-1', 'get'), actions.savedObject.get('read-savedObject-all-1', 'find'), + actions.savedObject.get('read-savedObject-all-1', 'open_point_in_time'), + actions.savedObject.get('read-savedObject-all-1', 'close_point_in_time'), actions.savedObject.get('read-savedObject-all-1', 'create'), actions.savedObject.get('read-savedObject-all-1', 'bulk_create'), actions.savedObject.get('read-savedObject-all-1', 'update'), @@ -141,6 +151,8 @@ describe('features', () => { actions.savedObject.get('read-savedObject-all-2', 'bulk_get'), actions.savedObject.get('read-savedObject-all-2', 'get'), actions.savedObject.get('read-savedObject-all-2', 'find'), + actions.savedObject.get('read-savedObject-all-2', 'open_point_in_time'), + actions.savedObject.get('read-savedObject-all-2', 'close_point_in_time'), actions.savedObject.get('read-savedObject-all-2', 'create'), actions.savedObject.get('read-savedObject-all-2', 'bulk_create'), actions.savedObject.get('read-savedObject-all-2', 'update'), @@ -150,9 +162,13 @@ describe('features', () => { actions.savedObject.get('read-savedObject-read-1', 'bulk_get'), actions.savedObject.get('read-savedObject-read-1', 'get'), actions.savedObject.get('read-savedObject-read-1', 'find'), + actions.savedObject.get('read-savedObject-read-1', 'open_point_in_time'), + actions.savedObject.get('read-savedObject-read-1', 'close_point_in_time'), actions.savedObject.get('read-savedObject-read-2', 'bulk_get'), actions.savedObject.get('read-savedObject-read-2', 'get'), actions.savedObject.get('read-savedObject-read-2', 'find'), + actions.savedObject.get('read-savedObject-read-2', 'open_point_in_time'), + actions.savedObject.get('read-savedObject-read-2', 'close_point_in_time'), actions.ui.get('foo', 'read-ui-1'), actions.ui.get('foo', 'read-ui-2'), ]; @@ -274,6 +290,8 @@ describe('features', () => { actions.savedObject.get('all-savedObject-all-1', 'bulk_get'), actions.savedObject.get('all-savedObject-all-1', 'get'), actions.savedObject.get('all-savedObject-all-1', 'find'), + actions.savedObject.get('all-savedObject-all-1', 'open_point_in_time'), + actions.savedObject.get('all-savedObject-all-1', 'close_point_in_time'), actions.savedObject.get('all-savedObject-all-1', 'create'), actions.savedObject.get('all-savedObject-all-1', 'bulk_create'), actions.savedObject.get('all-savedObject-all-1', 'update'), @@ -283,6 +301,8 @@ describe('features', () => { actions.savedObject.get('all-savedObject-all-2', 'bulk_get'), actions.savedObject.get('all-savedObject-all-2', 'get'), actions.savedObject.get('all-savedObject-all-2', 'find'), + actions.savedObject.get('all-savedObject-all-2', 'open_point_in_time'), + actions.savedObject.get('all-savedObject-all-2', 'close_point_in_time'), actions.savedObject.get('all-savedObject-all-2', 'create'), actions.savedObject.get('all-savedObject-all-2', 'bulk_create'), actions.savedObject.get('all-savedObject-all-2', 'update'), @@ -292,9 +312,13 @@ describe('features', () => { actions.savedObject.get('all-savedObject-read-1', 'bulk_get'), actions.savedObject.get('all-savedObject-read-1', 'get'), actions.savedObject.get('all-savedObject-read-1', 'find'), + actions.savedObject.get('all-savedObject-read-1', 'open_point_in_time'), + actions.savedObject.get('all-savedObject-read-1', 'close_point_in_time'), actions.savedObject.get('all-savedObject-read-2', 'bulk_get'), actions.savedObject.get('all-savedObject-read-2', 'get'), actions.savedObject.get('all-savedObject-read-2', 'find'), + actions.savedObject.get('all-savedObject-read-2', 'open_point_in_time'), + actions.savedObject.get('all-savedObject-read-2', 'close_point_in_time'), actions.ui.get('foo', 'all-ui-1'), actions.ui.get('foo', 'all-ui-2'), actions.ui.get('catalogue', 'read-catalogue-1'), @@ -304,6 +328,8 @@ describe('features', () => { actions.savedObject.get('read-savedObject-all-1', 'bulk_get'), actions.savedObject.get('read-savedObject-all-1', 'get'), actions.savedObject.get('read-savedObject-all-1', 'find'), + actions.savedObject.get('read-savedObject-all-1', 'open_point_in_time'), + actions.savedObject.get('read-savedObject-all-1', 'close_point_in_time'), actions.savedObject.get('read-savedObject-all-1', 'create'), actions.savedObject.get('read-savedObject-all-1', 'bulk_create'), actions.savedObject.get('read-savedObject-all-1', 'update'), @@ -313,6 +339,8 @@ describe('features', () => { actions.savedObject.get('read-savedObject-all-2', 'bulk_get'), actions.savedObject.get('read-savedObject-all-2', 'get'), actions.savedObject.get('read-savedObject-all-2', 'find'), + actions.savedObject.get('read-savedObject-all-2', 'open_point_in_time'), + actions.savedObject.get('read-savedObject-all-2', 'close_point_in_time'), actions.savedObject.get('read-savedObject-all-2', 'create'), actions.savedObject.get('read-savedObject-all-2', 'bulk_create'), actions.savedObject.get('read-savedObject-all-2', 'update'), @@ -322,9 +350,13 @@ describe('features', () => { actions.savedObject.get('read-savedObject-read-1', 'bulk_get'), actions.savedObject.get('read-savedObject-read-1', 'get'), actions.savedObject.get('read-savedObject-read-1', 'find'), + actions.savedObject.get('read-savedObject-read-1', 'open_point_in_time'), + actions.savedObject.get('read-savedObject-read-1', 'close_point_in_time'), actions.savedObject.get('read-savedObject-read-2', 'bulk_get'), actions.savedObject.get('read-savedObject-read-2', 'get'), actions.savedObject.get('read-savedObject-read-2', 'find'), + actions.savedObject.get('read-savedObject-read-2', 'open_point_in_time'), + actions.savedObject.get('read-savedObject-read-2', 'close_point_in_time'), actions.ui.get('foo', 'read-ui-1'), actions.ui.get('foo', 'read-ui-2'), ]); @@ -388,6 +420,8 @@ describe('features', () => { actions.savedObject.get('read-savedObject-all-1', 'bulk_get'), actions.savedObject.get('read-savedObject-all-1', 'get'), actions.savedObject.get('read-savedObject-all-1', 'find'), + actions.savedObject.get('read-savedObject-all-1', 'open_point_in_time'), + actions.savedObject.get('read-savedObject-all-1', 'close_point_in_time'), actions.savedObject.get('read-savedObject-all-1', 'create'), actions.savedObject.get('read-savedObject-all-1', 'bulk_create'), actions.savedObject.get('read-savedObject-all-1', 'update'), @@ -397,6 +431,8 @@ describe('features', () => { actions.savedObject.get('read-savedObject-all-2', 'bulk_get'), actions.savedObject.get('read-savedObject-all-2', 'get'), actions.savedObject.get('read-savedObject-all-2', 'find'), + actions.savedObject.get('read-savedObject-all-2', 'open_point_in_time'), + actions.savedObject.get('read-savedObject-all-2', 'close_point_in_time'), actions.savedObject.get('read-savedObject-all-2', 'create'), actions.savedObject.get('read-savedObject-all-2', 'bulk_create'), actions.savedObject.get('read-savedObject-all-2', 'update'), @@ -406,9 +442,13 @@ describe('features', () => { actions.savedObject.get('read-savedObject-read-1', 'bulk_get'), actions.savedObject.get('read-savedObject-read-1', 'get'), actions.savedObject.get('read-savedObject-read-1', 'find'), + actions.savedObject.get('read-savedObject-read-1', 'open_point_in_time'), + actions.savedObject.get('read-savedObject-read-1', 'close_point_in_time'), actions.savedObject.get('read-savedObject-read-2', 'bulk_get'), actions.savedObject.get('read-savedObject-read-2', 'get'), actions.savedObject.get('read-savedObject-read-2', 'find'), + actions.savedObject.get('read-savedObject-read-2', 'open_point_in_time'), + actions.savedObject.get('read-savedObject-read-2', 'close_point_in_time'), actions.ui.get('foo', 'read-ui-1'), actions.ui.get('foo', 'read-ui-2'), ]); @@ -691,6 +731,8 @@ describe('reserved', () => { actions.savedObject.get('savedObject-all-1', 'bulk_get'), actions.savedObject.get('savedObject-all-1', 'get'), actions.savedObject.get('savedObject-all-1', 'find'), + actions.savedObject.get('savedObject-all-1', 'open_point_in_time'), + actions.savedObject.get('savedObject-all-1', 'close_point_in_time'), actions.savedObject.get('savedObject-all-1', 'create'), actions.savedObject.get('savedObject-all-1', 'bulk_create'), actions.savedObject.get('savedObject-all-1', 'update'), @@ -700,6 +742,8 @@ describe('reserved', () => { actions.savedObject.get('savedObject-all-2', 'bulk_get'), actions.savedObject.get('savedObject-all-2', 'get'), actions.savedObject.get('savedObject-all-2', 'find'), + actions.savedObject.get('savedObject-all-2', 'open_point_in_time'), + actions.savedObject.get('savedObject-all-2', 'close_point_in_time'), actions.savedObject.get('savedObject-all-2', 'create'), actions.savedObject.get('savedObject-all-2', 'bulk_create'), actions.savedObject.get('savedObject-all-2', 'update'), @@ -709,9 +753,13 @@ describe('reserved', () => { actions.savedObject.get('savedObject-read-1', 'bulk_get'), actions.savedObject.get('savedObject-read-1', 'get'), actions.savedObject.get('savedObject-read-1', 'find'), + actions.savedObject.get('savedObject-read-1', 'open_point_in_time'), + actions.savedObject.get('savedObject-read-1', 'close_point_in_time'), actions.savedObject.get('savedObject-read-2', 'bulk_get'), actions.savedObject.get('savedObject-read-2', 'get'), actions.savedObject.get('savedObject-read-2', 'find'), + actions.savedObject.get('savedObject-read-2', 'open_point_in_time'), + actions.savedObject.get('savedObject-read-2', 'close_point_in_time'), actions.ui.get('foo', 'ui-1'), actions.ui.get('foo', 'ui-2'), ]); @@ -823,6 +871,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-sub-feature-type', 'get'), actions.savedObject.get('all-sub-feature-type', 'find'), + actions.savedObject.get('all-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-sub-feature-type', 'create'), actions.savedObject.get('all-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-sub-feature-type', 'update'), @@ -832,6 +882,8 @@ describe('subFeatures', () => { actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), actions.savedObject.get('read-sub-feature-type', 'find'), + actions.savedObject.get('read-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-sub-feature-type', 'close_point_in_time'), actions.ui.get('foo', 'sub-feature-ui'), ]); @@ -952,6 +1004,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-sub-feature-type', 'get'), actions.savedObject.get('all-sub-feature-type', 'find'), + actions.savedObject.get('all-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-sub-feature-type', 'create'), actions.savedObject.get('all-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-sub-feature-type', 'update'), @@ -961,6 +1015,8 @@ describe('subFeatures', () => { actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), actions.savedObject.get('read-sub-feature-type', 'find'), + actions.savedObject.get('read-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-sub-feature-type', 'close_point_in_time'), actions.ui.get('foo', 'sub-feature-ui'), ]); @@ -970,6 +1026,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-sub-feature-type', 'get'), actions.savedObject.get('all-sub-feature-type', 'find'), + actions.savedObject.get('all-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-sub-feature-type', 'create'), actions.savedObject.get('all-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-sub-feature-type', 'update'), @@ -979,6 +1037,8 @@ describe('subFeatures', () => { actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), actions.savedObject.get('read-sub-feature-type', 'find'), + actions.savedObject.get('read-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-sub-feature-type', 'close_point_in_time'), actions.ui.get('foo', 'foo'), actions.ui.get('foo', 'sub-feature-ui'), ]); @@ -995,6 +1055,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-sub-feature-type', 'get'), actions.savedObject.get('all-sub-feature-type', 'find'), + actions.savedObject.get('all-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-sub-feature-type', 'create'), actions.savedObject.get('all-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-sub-feature-type', 'update'), @@ -1004,6 +1066,8 @@ describe('subFeatures', () => { actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), actions.savedObject.get('read-sub-feature-type', 'find'), + actions.savedObject.get('read-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-sub-feature-type', 'close_point_in_time'), actions.ui.get('foo', 'foo'), actions.ui.get('foo', 'sub-feature-ui'), ]); @@ -1026,6 +1090,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-sub-feature-type', 'get'), actions.savedObject.get('all-sub-feature-type', 'find'), + actions.savedObject.get('all-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-sub-feature-type', 'create'), actions.savedObject.get('all-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-sub-feature-type', 'update'), @@ -1035,6 +1101,8 @@ describe('subFeatures', () => { actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), actions.savedObject.get('read-sub-feature-type', 'find'), + actions.savedObject.get('read-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-sub-feature-type', 'close_point_in_time'), actions.ui.get('foo', 'foo'), actions.ui.get('foo', 'sub-feature-ui'), ]); @@ -1044,6 +1112,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-sub-feature-type', 'get'), actions.savedObject.get('all-sub-feature-type', 'find'), + actions.savedObject.get('all-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-sub-feature-type', 'create'), actions.savedObject.get('all-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-sub-feature-type', 'update'), @@ -1053,6 +1123,8 @@ describe('subFeatures', () => { actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), actions.savedObject.get('read-sub-feature-type', 'find'), + actions.savedObject.get('read-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-sub-feature-type', 'close_point_in_time'), actions.ui.get('foo', 'foo'), actions.ui.get('foo', 'sub-feature-ui'), ]); @@ -1063,6 +1135,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-sub-feature-type', 'get'), actions.savedObject.get('all-sub-feature-type', 'find'), + actions.savedObject.get('all-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-sub-feature-type', 'create'), actions.savedObject.get('all-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-sub-feature-type', 'update'), @@ -1072,6 +1146,8 @@ describe('subFeatures', () => { actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), actions.savedObject.get('read-sub-feature-type', 'find'), + actions.savedObject.get('read-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-sub-feature-type', 'close_point_in_time'), actions.ui.get('foo', 'foo'), actions.ui.get('foo', 'sub-feature-ui'), ]); @@ -1081,6 +1157,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-sub-feature-type', 'get'), actions.savedObject.get('all-sub-feature-type', 'find'), + actions.savedObject.get('all-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-sub-feature-type', 'create'), actions.savedObject.get('all-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-sub-feature-type', 'update'), @@ -1090,6 +1168,8 @@ describe('subFeatures', () => { actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), actions.savedObject.get('read-sub-feature-type', 'find'), + actions.savedObject.get('read-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-sub-feature-type', 'close_point_in_time'), actions.ui.get('foo', 'foo'), actions.ui.get('foo', 'sub-feature-ui'), ]); @@ -1160,6 +1240,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-sub-feature-type', 'get'), actions.savedObject.get('all-sub-feature-type', 'find'), + actions.savedObject.get('all-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-sub-feature-type', 'create'), actions.savedObject.get('all-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-sub-feature-type', 'update'), @@ -1169,6 +1251,8 @@ describe('subFeatures', () => { actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), actions.savedObject.get('read-sub-feature-type', 'find'), + actions.savedObject.get('read-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-sub-feature-type', 'close_point_in_time'), actions.ui.get('foo', 'sub-feature-ui'), ]); @@ -1178,6 +1262,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-sub-feature-type', 'get'), actions.savedObject.get('all-sub-feature-type', 'find'), + actions.savedObject.get('all-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-sub-feature-type', 'create'), actions.savedObject.get('all-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-sub-feature-type', 'update'), @@ -1187,6 +1273,8 @@ describe('subFeatures', () => { actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), actions.savedObject.get('read-sub-feature-type', 'find'), + actions.savedObject.get('read-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-sub-feature-type', 'close_point_in_time'), actions.ui.get('foo', 'foo'), actions.ui.get('foo', 'sub-feature-ui'), ]); @@ -1203,6 +1291,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-sub-feature-type', 'get'), actions.savedObject.get('all-sub-feature-type', 'find'), + actions.savedObject.get('all-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-sub-feature-type', 'create'), actions.savedObject.get('all-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-sub-feature-type', 'update'), @@ -1212,6 +1302,8 @@ describe('subFeatures', () => { actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), actions.savedObject.get('read-sub-feature-type', 'find'), + actions.savedObject.get('read-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-sub-feature-type', 'close_point_in_time'), actions.ui.get('foo', 'foo'), actions.ui.get('foo', 'sub-feature-ui'), ]); @@ -1304,6 +1396,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-sub-feature-type', 'get'), actions.savedObject.get('all-sub-feature-type', 'find'), + actions.savedObject.get('all-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-sub-feature-type', 'create'), actions.savedObject.get('all-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-sub-feature-type', 'update'), @@ -1313,6 +1407,8 @@ describe('subFeatures', () => { actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), actions.savedObject.get('read-sub-feature-type', 'find'), + actions.savedObject.get('read-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-sub-feature-type', 'close_point_in_time'), actions.ui.get('foo', 'sub-feature-ui'), ]); @@ -1322,6 +1418,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-sub-feature-type', 'get'), actions.savedObject.get('all-sub-feature-type', 'find'), + actions.savedObject.get('all-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-sub-feature-type', 'create'), actions.savedObject.get('all-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-sub-feature-type', 'update'), @@ -1331,6 +1429,8 @@ describe('subFeatures', () => { actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), actions.savedObject.get('read-sub-feature-type', 'find'), + actions.savedObject.get('read-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-sub-feature-type', 'close_point_in_time'), actions.ui.get('foo', 'foo'), actions.ui.get('foo', 'sub-feature-ui'), ]); @@ -1365,6 +1465,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-sub-feature-type', 'get'), actions.savedObject.get('all-sub-feature-type', 'find'), + actions.savedObject.get('all-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-sub-feature-type', 'create'), actions.savedObject.get('all-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-sub-feature-type', 'update'), @@ -1374,6 +1476,8 @@ describe('subFeatures', () => { actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), actions.savedObject.get('read-sub-feature-type', 'find'), + actions.savedObject.get('read-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-sub-feature-type', 'close_point_in_time'), actions.ui.get('foo', 'foo'), actions.ui.get('foo', 'sub-feature-ui'), ]); @@ -1389,6 +1493,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-sub-feature-type', 'get'), actions.savedObject.get('all-sub-feature-type', 'find'), + actions.savedObject.get('all-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-sub-feature-type', 'create'), actions.savedObject.get('all-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-sub-feature-type', 'update'), @@ -1398,6 +1504,8 @@ describe('subFeatures', () => { actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), actions.savedObject.get('read-sub-feature-type', 'find'), + actions.savedObject.get('read-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-sub-feature-type', 'close_point_in_time'), actions.ui.get('foo', 'foo'), actions.ui.get('foo', 'sub-feature-ui'), ]); @@ -1473,6 +1581,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-sub-feature-type', 'get'), actions.savedObject.get('all-sub-feature-type', 'find'), + actions.savedObject.get('all-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-sub-feature-type', 'create'), actions.savedObject.get('all-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-sub-feature-type', 'update'), @@ -1482,6 +1592,8 @@ describe('subFeatures', () => { actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), actions.savedObject.get('read-sub-feature-type', 'find'), + actions.savedObject.get('read-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-sub-feature-type', 'close_point_in_time'), actions.ui.get('foo', 'sub-feature-ui'), ]); @@ -1491,6 +1603,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-sub-feature-type', 'get'), actions.savedObject.get('all-sub-feature-type', 'find'), + actions.savedObject.get('all-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-sub-feature-type', 'create'), actions.savedObject.get('all-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-sub-feature-type', 'update'), @@ -1500,6 +1614,8 @@ describe('subFeatures', () => { actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), actions.savedObject.get('read-sub-feature-type', 'find'), + actions.savedObject.get('read-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-sub-feature-type', 'close_point_in_time'), actions.ui.get('foo', 'foo'), actions.ui.get('foo', 'sub-feature-ui'), ]); @@ -1606,6 +1722,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-sub-feature-type', 'get'), actions.savedObject.get('all-sub-feature-type', 'find'), + actions.savedObject.get('all-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-sub-feature-type', 'create'), actions.savedObject.get('all-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-sub-feature-type', 'update'), @@ -1615,6 +1733,8 @@ describe('subFeatures', () => { actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), actions.savedObject.get('read-sub-feature-type', 'find'), + actions.savedObject.get('read-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-sub-feature-type', 'close_point_in_time'), actions.ui.get('foo', 'foo'), actions.ui.get('foo', 'sub-feature-ui'), ]); @@ -1627,6 +1747,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-sub-feature-type', 'get'), actions.savedObject.get('all-sub-feature-type', 'find'), + actions.savedObject.get('all-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-sub-feature-type', 'create'), actions.savedObject.get('all-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-sub-feature-type', 'update'), @@ -1636,6 +1758,8 @@ describe('subFeatures', () => { actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), actions.savedObject.get('read-sub-feature-type', 'find'), + actions.savedObject.get('read-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-sub-feature-type', 'close_point_in_time'), actions.ui.get('foo', 'foo'), actions.ui.get('foo', 'sub-feature-ui'), ]); @@ -1654,6 +1778,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-sub-feature-type', 'get'), actions.savedObject.get('all-sub-feature-type', 'find'), + actions.savedObject.get('all-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-sub-feature-type', 'create'), actions.savedObject.get('all-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-sub-feature-type', 'update'), @@ -1663,6 +1789,8 @@ describe('subFeatures', () => { actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), actions.savedObject.get('read-sub-feature-type', 'find'), + actions.savedObject.get('read-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-sub-feature-type', 'close_point_in_time'), actions.ui.get('foo', 'foo'), actions.ui.get('foo', 'sub-feature-ui'), ]); @@ -1672,6 +1800,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-sub-feature-type', 'get'), actions.savedObject.get('all-sub-feature-type', 'find'), + actions.savedObject.get('all-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-sub-feature-type', 'create'), actions.savedObject.get('all-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-sub-feature-type', 'update'), @@ -1681,6 +1811,8 @@ describe('subFeatures', () => { actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), actions.savedObject.get('read-sub-feature-type', 'find'), + actions.savedObject.get('read-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-sub-feature-type', 'close_point_in_time'), actions.ui.get('foo', 'foo'), actions.ui.get('foo', 'sub-feature-ui'), ]); @@ -1691,6 +1823,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-sub-feature-type', 'get'), actions.savedObject.get('all-sub-feature-type', 'find'), + actions.savedObject.get('all-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-sub-feature-type', 'create'), actions.savedObject.get('all-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-sub-feature-type', 'update'), @@ -1700,6 +1834,8 @@ describe('subFeatures', () => { actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), actions.savedObject.get('read-sub-feature-type', 'find'), + actions.savedObject.get('read-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-sub-feature-type', 'close_point_in_time'), actions.ui.get('foo', 'foo'), actions.ui.get('foo', 'sub-feature-ui'), ]); @@ -1709,6 +1845,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-sub-feature-type', 'get'), actions.savedObject.get('all-sub-feature-type', 'find'), + actions.savedObject.get('all-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-sub-feature-type', 'create'), actions.savedObject.get('all-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-sub-feature-type', 'update'), @@ -1718,6 +1856,8 @@ describe('subFeatures', () => { actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), actions.savedObject.get('read-sub-feature-type', 'find'), + actions.savedObject.get('read-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-sub-feature-type', 'close_point_in_time'), actions.ui.get('foo', 'foo'), actions.ui.get('foo', 'sub-feature-ui'), ]); @@ -1808,6 +1948,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-sub-feature-type', 'get'), actions.savedObject.get('all-sub-feature-type', 'find'), + actions.savedObject.get('all-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-sub-feature-type', 'create'), actions.savedObject.get('all-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-sub-feature-type', 'update'), @@ -1817,6 +1959,8 @@ describe('subFeatures', () => { actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), actions.savedObject.get('read-sub-feature-type', 'find'), + actions.savedObject.get('read-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-sub-feature-type', 'close_point_in_time'), actions.ui.get('foo', 'foo'), actions.ui.get('foo', 'sub-feature-ui'), ]); @@ -1833,6 +1977,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-sub-feature-type', 'get'), actions.savedObject.get('all-sub-feature-type', 'find'), + actions.savedObject.get('all-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-sub-feature-type', 'create'), actions.savedObject.get('all-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-sub-feature-type', 'update'), @@ -1842,6 +1988,8 @@ describe('subFeatures', () => { actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), actions.savedObject.get('read-sub-feature-type', 'find'), + actions.savedObject.get('read-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-sub-feature-type', 'close_point_in_time'), actions.ui.get('foo', 'foo'), actions.ui.get('foo', 'sub-feature-ui'), ]); @@ -1864,6 +2012,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-sub-feature-type', 'get'), actions.savedObject.get('all-sub-feature-type', 'find'), + actions.savedObject.get('all-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-sub-feature-type', 'create'), actions.savedObject.get('all-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-sub-feature-type', 'update'), @@ -1873,6 +2023,8 @@ describe('subFeatures', () => { actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), actions.savedObject.get('read-sub-feature-type', 'find'), + actions.savedObject.get('read-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-sub-feature-type', 'close_point_in_time'), actions.ui.get('foo', 'foo'), actions.ui.get('foo', 'sub-feature-ui'), ]); @@ -1882,6 +2034,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-sub-feature-type', 'get'), actions.savedObject.get('all-sub-feature-type', 'find'), + actions.savedObject.get('all-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-sub-feature-type', 'create'), actions.savedObject.get('all-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-sub-feature-type', 'update'), @@ -1891,6 +2045,8 @@ describe('subFeatures', () => { actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), actions.savedObject.get('read-sub-feature-type', 'find'), + actions.savedObject.get('read-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-sub-feature-type', 'close_point_in_time'), actions.ui.get('foo', 'foo'), actions.ui.get('foo', 'sub-feature-ui'), ]); @@ -1901,6 +2057,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-sub-feature-type', 'get'), actions.savedObject.get('all-sub-feature-type', 'find'), + actions.savedObject.get('all-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-sub-feature-type', 'create'), actions.savedObject.get('all-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-sub-feature-type', 'update'), @@ -1910,6 +2068,8 @@ describe('subFeatures', () => { actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), actions.savedObject.get('read-sub-feature-type', 'find'), + actions.savedObject.get('read-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-sub-feature-type', 'close_point_in_time'), actions.ui.get('foo', 'foo'), actions.ui.get('foo', 'sub-feature-ui'), ]); @@ -1919,6 +2079,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-sub-feature-type', 'get'), actions.savedObject.get('all-sub-feature-type', 'find'), + actions.savedObject.get('all-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-sub-feature-type', 'create'), actions.savedObject.get('all-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-sub-feature-type', 'update'), @@ -1928,6 +2090,8 @@ describe('subFeatures', () => { actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), actions.savedObject.get('read-sub-feature-type', 'find'), + actions.savedObject.get('read-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-sub-feature-type', 'close_point_in_time'), actions.ui.get('foo', 'foo'), actions.ui.get('foo', 'sub-feature-ui'), ]); @@ -2018,6 +2182,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-sub-feature-type', 'get'), actions.savedObject.get('all-sub-feature-type', 'find'), + actions.savedObject.get('all-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-sub-feature-type', 'create'), actions.savedObject.get('all-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-sub-feature-type', 'update'), @@ -2027,6 +2193,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-licensed-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-licensed-sub-feature-type', 'get'), actions.savedObject.get('all-licensed-sub-feature-type', 'find'), + actions.savedObject.get('all-licensed-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-licensed-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-licensed-sub-feature-type', 'create'), actions.savedObject.get('all-licensed-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-licensed-sub-feature-type', 'update'), @@ -2036,9 +2204,13 @@ describe('subFeatures', () => { actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), actions.savedObject.get('read-sub-feature-type', 'find'), + actions.savedObject.get('read-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('read-licensed-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-licensed-sub-feature-type', 'get'), actions.savedObject.get('read-licensed-sub-feature-type', 'find'), + actions.savedObject.get('read-licensed-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-licensed-sub-feature-type', 'close_point_in_time'), actions.ui.get('foo', 'foo'), actions.ui.get('foo', 'sub-feature-ui'), actions.ui.get('foo', 'licensed-sub-feature-ui'), @@ -2056,6 +2228,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-sub-feature-type', 'get'), actions.savedObject.get('all-sub-feature-type', 'find'), + actions.savedObject.get('all-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-sub-feature-type', 'create'), actions.savedObject.get('all-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-sub-feature-type', 'update'), @@ -2065,6 +2239,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-licensed-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-licensed-sub-feature-type', 'get'), actions.savedObject.get('all-licensed-sub-feature-type', 'find'), + actions.savedObject.get('all-licensed-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-licensed-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-licensed-sub-feature-type', 'create'), actions.savedObject.get('all-licensed-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-licensed-sub-feature-type', 'update'), @@ -2074,9 +2250,13 @@ describe('subFeatures', () => { actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), actions.savedObject.get('read-sub-feature-type', 'find'), + actions.savedObject.get('read-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('read-licensed-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-licensed-sub-feature-type', 'get'), actions.savedObject.get('read-licensed-sub-feature-type', 'find'), + actions.savedObject.get('read-licensed-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-licensed-sub-feature-type', 'close_point_in_time'), actions.ui.get('foo', 'foo'), actions.ui.get('foo', 'sub-feature-ui'), actions.ui.get('foo', 'licensed-sub-feature-ui'), @@ -2100,6 +2280,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-sub-feature-type', 'get'), actions.savedObject.get('all-sub-feature-type', 'find'), + actions.savedObject.get('all-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-sub-feature-type', 'create'), actions.savedObject.get('all-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-sub-feature-type', 'update'), @@ -2109,6 +2291,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-licensed-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-licensed-sub-feature-type', 'get'), actions.savedObject.get('all-licensed-sub-feature-type', 'find'), + actions.savedObject.get('all-licensed-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-licensed-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-licensed-sub-feature-type', 'create'), actions.savedObject.get('all-licensed-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-licensed-sub-feature-type', 'update'), @@ -2118,9 +2302,13 @@ describe('subFeatures', () => { actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), actions.savedObject.get('read-sub-feature-type', 'find'), + actions.savedObject.get('read-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('read-licensed-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-licensed-sub-feature-type', 'get'), actions.savedObject.get('read-licensed-sub-feature-type', 'find'), + actions.savedObject.get('read-licensed-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-licensed-sub-feature-type', 'close_point_in_time'), actions.ui.get('foo', 'foo'), actions.ui.get('foo', 'sub-feature-ui'), actions.ui.get('foo', 'licensed-sub-feature-ui'), @@ -2131,6 +2319,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-sub-feature-type', 'get'), actions.savedObject.get('all-sub-feature-type', 'find'), + actions.savedObject.get('all-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-sub-feature-type', 'create'), actions.savedObject.get('all-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-sub-feature-type', 'update'), @@ -2140,6 +2330,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-licensed-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-licensed-sub-feature-type', 'get'), actions.savedObject.get('all-licensed-sub-feature-type', 'find'), + actions.savedObject.get('all-licensed-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-licensed-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-licensed-sub-feature-type', 'create'), actions.savedObject.get('all-licensed-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-licensed-sub-feature-type', 'update'), @@ -2149,9 +2341,13 @@ describe('subFeatures', () => { actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), actions.savedObject.get('read-sub-feature-type', 'find'), + actions.savedObject.get('read-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('read-licensed-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-licensed-sub-feature-type', 'get'), actions.savedObject.get('read-licensed-sub-feature-type', 'find'), + actions.savedObject.get('read-licensed-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-licensed-sub-feature-type', 'close_point_in_time'), actions.ui.get('foo', 'foo'), actions.ui.get('foo', 'sub-feature-ui'), actions.ui.get('foo', 'licensed-sub-feature-ui'), @@ -2163,6 +2359,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-sub-feature-type', 'get'), actions.savedObject.get('all-sub-feature-type', 'find'), + actions.savedObject.get('all-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-sub-feature-type', 'create'), actions.savedObject.get('all-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-sub-feature-type', 'update'), @@ -2172,6 +2370,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-licensed-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-licensed-sub-feature-type', 'get'), actions.savedObject.get('all-licensed-sub-feature-type', 'find'), + actions.savedObject.get('all-licensed-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-licensed-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-licensed-sub-feature-type', 'create'), actions.savedObject.get('all-licensed-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-licensed-sub-feature-type', 'update'), @@ -2181,9 +2381,13 @@ describe('subFeatures', () => { actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), actions.savedObject.get('read-sub-feature-type', 'find'), + actions.savedObject.get('read-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('read-licensed-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-licensed-sub-feature-type', 'get'), actions.savedObject.get('read-licensed-sub-feature-type', 'find'), + actions.savedObject.get('read-licensed-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-licensed-sub-feature-type', 'close_point_in_time'), actions.ui.get('foo', 'foo'), actions.ui.get('foo', 'sub-feature-ui'), actions.ui.get('foo', 'licensed-sub-feature-ui'), @@ -2194,6 +2398,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-sub-feature-type', 'get'), actions.savedObject.get('all-sub-feature-type', 'find'), + actions.savedObject.get('all-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-sub-feature-type', 'create'), actions.savedObject.get('all-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-sub-feature-type', 'update'), @@ -2203,6 +2409,8 @@ describe('subFeatures', () => { actions.savedObject.get('all-licensed-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-licensed-sub-feature-type', 'get'), actions.savedObject.get('all-licensed-sub-feature-type', 'find'), + actions.savedObject.get('all-licensed-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('all-licensed-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('all-licensed-sub-feature-type', 'create'), actions.savedObject.get('all-licensed-sub-feature-type', 'bulk_create'), actions.savedObject.get('all-licensed-sub-feature-type', 'update'), @@ -2212,9 +2420,13 @@ describe('subFeatures', () => { actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), actions.savedObject.get('read-sub-feature-type', 'find'), + actions.savedObject.get('read-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-sub-feature-type', 'close_point_in_time'), actions.savedObject.get('read-licensed-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-licensed-sub-feature-type', 'get'), actions.savedObject.get('read-licensed-sub-feature-type', 'find'), + actions.savedObject.get('read-licensed-sub-feature-type', 'open_point_in_time'), + actions.savedObject.get('read-licensed-sub-feature-type', 'close_point_in_time'), actions.ui.get('foo', 'foo'), actions.ui.get('foo', 'sub-feature-ui'), actions.ui.get('foo', 'licensed-sub-feature-ui'), From 07ece370048630762dde9fe463dc618015232875 Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Mon, 8 Feb 2021 17:20:41 -0700 Subject: [PATCH 17/32] Clean up exporter & adjust functional tests. --- .../export/saved_objects_exporter.ts | 59 +++-- .../apis/saved_objects/export.ts | 248 ++++++++++-------- test/api_integration/config.js | 1 + 3 files changed, 181 insertions(+), 127 deletions(-) diff --git a/src/core/server/saved_objects/export/saved_objects_exporter.ts b/src/core/server/saved_objects/export/saved_objects_exporter.ts index 11ecf19a2cc48..ada9c80b6e735 100644 --- a/src/core/server/saved_objects/export/saved_objects_exporter.ts +++ b/src/core/server/saved_objects/export/saved_objects_exporter.ts @@ -72,7 +72,7 @@ export class SavedObjectsExporter { * @throws SavedObjectsExportError */ public async exportByTypes(options: SavedObjectsExportByTypeOptions) { - this.#log.debug(`Initiating export for types: [${options.types.join(',')}]`); + this.#log.debug(`Initiating export for types: [${options.types}]`); const objects = await this.fetchByTypes(options); return this.processObjects(objects, byIdAscComparator, { request: options.request, @@ -185,33 +185,51 @@ export class SavedObjectsExporter { const getLastHitSortValue = (res: SavedObjectsFindResponse) => res.saved_objects.length && res.saved_objects[res.saved_objects.length - 1].sort; - const findWithPit = async ({ id, searchAfter }: { id: string; searchAfter?: unknown[] }) => { - return await this.#savedObjectsClient.find({ - // Sort fields are required to use searchAfter, so we set some defaults here - sortField: 'updated_at', - sortOrder: 'desc', - pit: { - keepAlive: '1m', // bump keep_alive by 1m on every new request - ...findOptions.pit, - id, - }, - ...(searchAfter ? { searchAfter } : {}), - ...findOptions, - }); + const findWithPit = async ({ id, searchAfter }: { id?: string; searchAfter?: unknown[] }) => { + try { + return await this.#savedObjectsClient.find({ + // Sort fields are required to use searchAfter, so we set some defaults here + sortField: 'updated_at', + sortOrder: 'desc', + // Bump keep_alive by 1m on every new request + ...(id ? { pit: { keepAlive: '1m', ...findOptions.pit, id } } : {}), + ...(searchAfter ? { searchAfter } : {}), + ...findOptions, + }); + } catch (e) { + if (id) { + // Clean up PIT on any errors. + this.#log.debug(`Closing PIT for types [${findOptions.type}]`); + await this.#savedObjectsClient.closePointInTime(id); + } + throw e; + } }; // Open PIT and request our first page of hits - let { id: pitId } = await this.#savedObjectsClient.openPointInTimeForType(findOptions.type); - let results = await findWithPit({ id: pitId }); + let pitId: string | undefined; + try { + const { id } = await this.#savedObjectsClient.openPointInTimeForType(findOptions.type); + pitId = id; + } catch (e) { + // Since `find` swallows 404s, it is expected that exporter will do the same, + // so we only rethrow non-404 errors here. + if (e.output.statusCode !== 404) { + throw e; + } + this.#log.debug(`Unable to open PIT for types [${findOptions.type}]: 404 ${e}`); + } + let results = await findWithPit({ id: pitId }); let lastResultsCount = results.saved_objects.length; let lastHitSortValue = getLastHitSortValue(results); - pitId = results.pit_id!; + pitId = results.pit_id; this.#log.debug(`Collected [${lastResultsCount}] saved objects for export.`); // Close PIT if this was our last page - if (lastResultsCount < findOptions.perPage!) { + if (pitId && lastResultsCount < findOptions.perPage!) { + this.#log.debug(`Closing PIT for types [${findOptions.type}]`); await this.#savedObjectsClient.closePointInTime(pitId); } @@ -226,11 +244,12 @@ export class SavedObjectsExporter { lastResultsCount = results.saved_objects.length; lastHitSortValue = getLastHitSortValue(results); - pitId = results.pit_id!; + pitId = results.pit_id; this.#log.debug(`Collected [${lastResultsCount}] more saved objects for export.`); - if (lastResultsCount < findOptions.perPage) { + if (pitId && lastResultsCount < findOptions.perPage) { + this.#log.debug(`Closing PIT for types [${findOptions.type}]`); await this.#savedObjectsClient.closePointInTime(pitId); } diff --git a/test/api_integration/apis/saved_objects/export.ts b/test/api_integration/apis/saved_objects/export.ts index 5206d51054745..8af2dbdea31dc 100644 --- a/test/api_integration/apis/saved_objects/export.ts +++ b/test/api_integration/apis/saved_objects/export.ts @@ -295,43 +295,43 @@ export default function ({ getService }: FtrProviderContext) { ); expect(resp.header['content-type']).to.eql('application/ndjson'); const objects = ndjsonToObject(resp.text); - expect(objects).to.eql([ - { - attributes: { - description: '', - hits: 0, - kibanaSavedObjectMeta: { - searchSourceJSON: - objects[0].attributes.kibanaSavedObjectMeta.searchSourceJSON, - }, - optionsJSON: objects[0].attributes.optionsJSON, - panelsJSON: objects[0].attributes.panelsJSON, - refreshInterval: { - display: 'Off', - pause: false, - value: 0, - }, - timeFrom: 'Wed Sep 16 2015 22:52:17 GMT-0700', - timeRestore: true, - timeTo: 'Fri Sep 18 2015 12:24:38 GMT-0700', - title: 'Requests', - version: 1, + + // Sort values aren't deterministic so we need to exclude them + const { sort, ...obj } = objects[0]; + expect(obj).to.eql({ + attributes: { + description: '', + hits: 0, + kibanaSavedObjectMeta: { + searchSourceJSON: objects[0].attributes.kibanaSavedObjectMeta.searchSourceJSON, }, - id: 'be3733a0-9efe-11e7-acb3-3dab96693fab', - migrationVersion: objects[0].migrationVersion, - coreMigrationVersion: KIBANA_VERSION, - references: [ - { - id: 'dd7caf20-9efd-11e7-acb3-3dab96693fab', - name: 'panel_0', - type: 'visualization', - }, - ], - type: 'dashboard', - updated_at: '2017-09-21T18:57:40.826Z', - version: objects[0].version, + optionsJSON: objects[0].attributes.optionsJSON, + panelsJSON: objects[0].attributes.panelsJSON, + refreshInterval: { + display: 'Off', + pause: false, + value: 0, + }, + timeFrom: 'Wed Sep 16 2015 22:52:17 GMT-0700', + timeRestore: true, + timeTo: 'Fri Sep 18 2015 12:24:38 GMT-0700', + title: 'Requests', + version: 1, }, - ]); + id: 'be3733a0-9efe-11e7-acb3-3dab96693fab', + migrationVersion: objects[0].migrationVersion, + coreMigrationVersion: KIBANA_VERSION, + references: [ + { + id: 'dd7caf20-9efd-11e7-acb3-3dab96693fab', + name: 'panel_0', + type: 'visualization', + }, + ], + type: 'dashboard', + updated_at: '2017-09-21T18:57:40.826Z', + version: objects[0].version, + }); expect(objects[0].migrationVersion).to.be.ok(); expect(() => JSON.parse(objects[0].attributes.kibanaSavedObjectMeta.searchSourceJSON) @@ -355,43 +355,43 @@ export default function ({ getService }: FtrProviderContext) { ); expect(resp.header['content-type']).to.eql('application/ndjson'); const objects = ndjsonToObject(resp.text); - expect(objects).to.eql([ - { - attributes: { - description: '', - hits: 0, - kibanaSavedObjectMeta: { - searchSourceJSON: - objects[0].attributes.kibanaSavedObjectMeta.searchSourceJSON, - }, - optionsJSON: objects[0].attributes.optionsJSON, - panelsJSON: objects[0].attributes.panelsJSON, - refreshInterval: { - display: 'Off', - pause: false, - value: 0, - }, - timeFrom: 'Wed Sep 16 2015 22:52:17 GMT-0700', - timeRestore: true, - timeTo: 'Fri Sep 18 2015 12:24:38 GMT-0700', - title: 'Requests', - version: 1, + + // Sort values aren't deterministic so we need to exclude them + const { sort, ...obj } = objects[0]; + expect(obj).to.eql({ + attributes: { + description: '', + hits: 0, + kibanaSavedObjectMeta: { + searchSourceJSON: objects[0].attributes.kibanaSavedObjectMeta.searchSourceJSON, }, - id: 'be3733a0-9efe-11e7-acb3-3dab96693fab', - migrationVersion: objects[0].migrationVersion, - coreMigrationVersion: KIBANA_VERSION, - references: [ - { - id: 'dd7caf20-9efd-11e7-acb3-3dab96693fab', - name: 'panel_0', - type: 'visualization', - }, - ], - type: 'dashboard', - updated_at: '2017-09-21T18:57:40.826Z', - version: objects[0].version, + optionsJSON: objects[0].attributes.optionsJSON, + panelsJSON: objects[0].attributes.panelsJSON, + refreshInterval: { + display: 'Off', + pause: false, + value: 0, + }, + timeFrom: 'Wed Sep 16 2015 22:52:17 GMT-0700', + timeRestore: true, + timeTo: 'Fri Sep 18 2015 12:24:38 GMT-0700', + title: 'Requests', + version: 1, }, - ]); + id: 'be3733a0-9efe-11e7-acb3-3dab96693fab', + migrationVersion: objects[0].migrationVersion, + coreMigrationVersion: KIBANA_VERSION, + references: [ + { + id: 'dd7caf20-9efd-11e7-acb3-3dab96693fab', + name: 'panel_0', + type: 'visualization', + }, + ], + type: 'dashboard', + updated_at: '2017-09-21T18:57:40.826Z', + version: objects[0].version, + }); expect(objects[0].migrationVersion).to.be.ok(); expect(() => JSON.parse(objects[0].attributes.kibanaSavedObjectMeta.searchSourceJSON) @@ -420,43 +420,43 @@ export default function ({ getService }: FtrProviderContext) { ); expect(resp.header['content-type']).to.eql('application/ndjson'); const objects = ndjsonToObject(resp.text); - expect(objects).to.eql([ - { - attributes: { - description: '', - hits: 0, - kibanaSavedObjectMeta: { - searchSourceJSON: - objects[0].attributes.kibanaSavedObjectMeta.searchSourceJSON, - }, - optionsJSON: objects[0].attributes.optionsJSON, - panelsJSON: objects[0].attributes.panelsJSON, - refreshInterval: { - display: 'Off', - pause: false, - value: 0, - }, - timeFrom: 'Wed Sep 16 2015 22:52:17 GMT-0700', - timeRestore: true, - timeTo: 'Fri Sep 18 2015 12:24:38 GMT-0700', - title: 'Requests', - version: 1, + + // Sort values aren't deterministic so we need to exclude them + const { sort, ...obj } = objects[0]; + expect(obj).to.eql({ + attributes: { + description: '', + hits: 0, + kibanaSavedObjectMeta: { + searchSourceJSON: objects[0].attributes.kibanaSavedObjectMeta.searchSourceJSON, }, - id: 'be3733a0-9efe-11e7-acb3-3dab96693fab', - migrationVersion: objects[0].migrationVersion, - coreMigrationVersion: KIBANA_VERSION, - references: [ - { - id: 'dd7caf20-9efd-11e7-acb3-3dab96693fab', - name: 'panel_0', - type: 'visualization', - }, - ], - type: 'dashboard', - updated_at: '2017-09-21T18:57:40.826Z', - version: objects[0].version, + optionsJSON: objects[0].attributes.optionsJSON, + panelsJSON: objects[0].attributes.panelsJSON, + refreshInterval: { + display: 'Off', + pause: false, + value: 0, + }, + timeFrom: 'Wed Sep 16 2015 22:52:17 GMT-0700', + timeRestore: true, + timeTo: 'Fri Sep 18 2015 12:24:38 GMT-0700', + title: 'Requests', + version: 1, }, - ]); + id: 'be3733a0-9efe-11e7-acb3-3dab96693fab', + migrationVersion: objects[0].migrationVersion, + coreMigrationVersion: KIBANA_VERSION, + references: [ + { + id: 'dd7caf20-9efd-11e7-acb3-3dab96693fab', + name: 'panel_0', + type: 'visualization', + }, + ], + type: 'dashboard', + updated_at: '2017-09-21T18:57:40.826Z', + version: objects[0].version, + }); expect(objects[0].migrationVersion).to.be.ok(); expect(() => JSON.parse(objects[0].attributes.kibanaSavedObjectMeta.searchSourceJSON) @@ -511,7 +511,37 @@ export default function ({ getService }: FtrProviderContext) { await esArchiver.unload('saved_objects/10k'); }); - it('should return 400 when exporting more than 10,000', async () => { + it('should allow exporting more than 10,000 objects if permitted by maxImportExportSize', async () => { + await supertest + .post('/api/saved_objects/_export') + .send({ + type: ['dashboard', 'visualization', 'search', 'index-pattern'], + excludeExportDetails: true, + }) + .expect(200) + .then((resp) => { + expect(resp.header['content-disposition']).to.eql( + 'attachment; filename="export.ndjson"' + ); + expect(resp.header['content-type']).to.eql('application/ndjson'); + const objects = ndjsonToObject(resp.text); + expect(objects.length).to.eql(10001); + }); + }); + + it('should return 400 when exporting more than allowed by maxImportExportSize', async () => { + let anotherCustomVisId: string; + await supertest + .post('/api/saved_objects/visualization') + .send({ + attributes: { + title: 'My other favorite vis', + }, + }) + .expect(200) + .then((resp) => { + anotherCustomVisId = resp.body.id; + }); await supertest .post('/api/saved_objects/_export') .send({ @@ -523,9 +553,13 @@ export default function ({ getService }: FtrProviderContext) { expect(resp.body).to.eql({ statusCode: 400, error: 'Bad Request', - message: `Can't export more than 10000 objects`, + message: `Can't export more than 10001 objects`, }); }); + await supertest + // @ts-expect-error TS complains about using `anotherCustomVisId` before it is assigned + .delete(`/api/saved_objects/visualization/${anotherCustomVisId}`) + .expect(200); }); }); }); diff --git a/test/api_integration/config.js b/test/api_integration/config.js index bd8f10606a45a..1c19dd24fa96b 100644 --- a/test/api_integration/config.js +++ b/test/api_integration/config.js @@ -30,6 +30,7 @@ export default async function ({ readConfigFile }) { '--elasticsearch.healthCheck.delay=3600000', '--server.xsrf.disableProtection=true', '--server.compression.referrerWhitelist=["some-host.com"]', + `--savedObjects.maxImportExportSize=10001`, ], }, }; From 95000742a444954cc6e13744573a752b6fcb5129 Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Mon, 8 Feb 2021 23:51:58 -0700 Subject: [PATCH 18/32] Relocate findWithPointInTime and add/cleanup tests. --- .../export/find_with_point_in_time.test.ts | 238 ++++++++++++++++++ .../export/find_with_point_in_time.ts | 189 ++++++++++++++ .../export/saved_objects_exporter.test.ts | 15 +- .../export/saved_objects_exporter.ts | 107 +------- .../migrations/core/index_migrator.test.ts | 2 +- .../routes/integration_tests/find.test.ts | 4 +- .../service/lib/repository.test.js | 8 +- .../saved_objects/service/lib/repository.ts | 8 +- .../service/lib/search_dsl/pit_params.test.ts | 4 +- 9 files changed, 457 insertions(+), 118 deletions(-) create mode 100644 src/core/server/saved_objects/export/find_with_point_in_time.test.ts create mode 100644 src/core/server/saved_objects/export/find_with_point_in_time.ts diff --git a/src/core/server/saved_objects/export/find_with_point_in_time.test.ts b/src/core/server/saved_objects/export/find_with_point_in_time.test.ts new file mode 100644 index 0000000000000..a652acc8f5291 --- /dev/null +++ b/src/core/server/saved_objects/export/find_with_point_in_time.test.ts @@ -0,0 +1,238 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { savedObjectsClientMock } from '../service/saved_objects_client.mock'; +import { loggerMock, MockedLogger } from '../../logging/logger.mock'; +import { SavedObjectsFindOptions } from '../types'; +import { SavedObjectsFindResult } from '../service'; + +import type { FindWithPointInTime } from './find_with_point_in_time'; +import { findWithPointInTime } from './find_with_point_in_time'; + +const mockHits = [ + { + id: '2', + type: 'search', + attributes: {}, + score: 1, + references: [ + { + name: 'name', + type: 'visualization', + id: '1', + }, + ], + sort: [], + }, + { + id: '1', + type: 'visualization', + attributes: {}, + score: 1, + references: [], + sort: [], + }, +]; + +describe('findWithPointInTime()', () => { + let logger: MockedLogger; + let savedObjectsClient: ReturnType; + let finder: FindWithPointInTime; + + beforeEach(() => { + logger = loggerMock.create(); + savedObjectsClient = savedObjectsClientMock.create(); + finder = findWithPointInTime({ savedObjectsClient, logger }); + }); + + describe('#find', () => { + test('works with a single page of results', async () => { + savedObjectsClient.openPointInTimeForType.mockResolvedValueOnce({ + id: 'abc123', + }); + savedObjectsClient.find.mockResolvedValueOnce({ + total: 2, + saved_objects: mockHits, + pit_id: 'abc123', + per_page: 2, + page: 0, + }); + + const options: SavedObjectsFindOptions = { + type: ['visualization'], + search: 'foo*', + }; + + const hits: SavedObjectsFindResult[] = []; + for await (const result of finder.find(options)) { + hits.push(...result.saved_objects); + } + + expect(hits.length).toBe(2); + expect(savedObjectsClient.openPointInTimeForType).toHaveBeenCalledTimes(1); + expect(savedObjectsClient.closePointInTime).toHaveBeenCalledTimes(1); + expect(savedObjectsClient.find).toHaveBeenCalledTimes(1); + expect(savedObjectsClient.find).toHaveBeenCalledWith( + expect.objectContaining({ + pit: expect.objectContaining({ id: 'abc123', keepAlive: '2m' }), + sortField: 'updated_at', + sortOrder: 'desc', + type: ['visualization'], + }) + ); + }); + + test('works with multiple pages of results', async () => { + savedObjectsClient.openPointInTimeForType.mockResolvedValueOnce({ + id: 'abc123', + }); + savedObjectsClient.find.mockResolvedValueOnce({ + total: 2, + saved_objects: [mockHits[0]], + pit_id: 'abc123', + per_page: 1, + page: 0, + }); + savedObjectsClient.find.mockResolvedValueOnce({ + total: 2, + saved_objects: [mockHits[1]], + pit_id: 'abc123', + per_page: 1, + page: 0, + }); + savedObjectsClient.find.mockResolvedValueOnce({ + total: 2, + saved_objects: [], + per_page: 1, + pit_id: 'abc123', + page: 0, + }); + + const options: SavedObjectsFindOptions = { + type: ['visualization'], + search: 'foo*', + perPage: 1, + }; + + const hits: SavedObjectsFindResult[] = []; + for await (const result of finder.find(options)) { + hits.push(...result.saved_objects); + } + + expect(hits.length).toBe(2); + expect(savedObjectsClient.openPointInTimeForType).toHaveBeenCalledTimes(1); + expect(savedObjectsClient.closePointInTime).toHaveBeenCalledTimes(1); + // called 3 times since we need a 3rd request to check if we + // are done paginating through results. + expect(savedObjectsClient.find).toHaveBeenCalledTimes(3); + expect(savedObjectsClient.find).toHaveBeenCalledWith( + expect.objectContaining({ + pit: expect.objectContaining({ id: 'abc123', keepAlive: '2m' }), + sortField: 'updated_at', + sortOrder: 'desc', + type: ['visualization'], + }) + ); + }); + }); + + describe('#close', () => { + test('calls closePointInTime with correct ID', async () => { + savedObjectsClient.openPointInTimeForType.mockResolvedValueOnce({ + id: 'test', + }); + savedObjectsClient.find.mockResolvedValueOnce({ + total: 1, + saved_objects: [mockHits[0]], + pit_id: 'test', + per_page: 1, + page: 0, + }); + + const options: SavedObjectsFindOptions = { + type: ['visualization'], + search: 'foo*', + perPage: 2, + }; + + const hits: SavedObjectsFindResult[] = []; + for await (const result of finder.find(options)) { + hits.push(...result.saved_objects); + await finder.close(); + } + + expect(savedObjectsClient.closePointInTime).toHaveBeenCalledWith('test'); + }); + + test('causes generator to stop', async () => { + savedObjectsClient.openPointInTimeForType.mockResolvedValueOnce({ + id: 'test', + }); + savedObjectsClient.find.mockResolvedValueOnce({ + total: 2, + saved_objects: [mockHits[0]], + pit_id: 'test', + per_page: 1, + page: 0, + }); + savedObjectsClient.find.mockResolvedValueOnce({ + total: 2, + saved_objects: [mockHits[1]], + pit_id: 'test', + per_page: 1, + page: 0, + }); + savedObjectsClient.find.mockResolvedValueOnce({ + total: 2, + saved_objects: [], + per_page: 1, + pit_id: 'test', + page: 0, + }); + + const options: SavedObjectsFindOptions = { + type: ['visualization'], + search: 'foo*', + perPage: 1, + }; + + const hits: SavedObjectsFindResult[] = []; + for await (const result of finder.find(options)) { + hits.push(...result.saved_objects); + await finder.close(); + } + + expect(savedObjectsClient.closePointInTime).toHaveBeenCalledTimes(1); + expect(hits.length).toBe(1); + }); + + test('is called if `find` throws an error', async () => { + savedObjectsClient.openPointInTimeForType.mockResolvedValueOnce({ + id: 'test', + }); + savedObjectsClient.find.mockRejectedValueOnce(new Error('oops')); + + const options: SavedObjectsFindOptions = { + type: ['visualization'], + search: 'foo*', + perPage: 2, + }; + + const hits: SavedObjectsFindResult[] = []; + try { + for await (const result of finder.find(options)) { + hits.push(...result.saved_objects); + } + } catch (e) { + // intentionally empty + } + + expect(savedObjectsClient.closePointInTime).toHaveBeenCalledWith('test'); + }); + }); +}); diff --git a/src/core/server/saved_objects/export/find_with_point_in_time.ts b/src/core/server/saved_objects/export/find_with_point_in_time.ts new file mode 100644 index 0000000000000..5ce271aa76859 --- /dev/null +++ b/src/core/server/saved_objects/export/find_with_point_in_time.ts @@ -0,0 +1,189 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { Logger } from '../../logging'; +import { SavedObjectsClientContract, SavedObjectsFindOptions } from '../types'; +import { SavedObjectsFindResponse } from '../service'; + +/** + * Returns a generator to help page through large sets of saved objects. + * + * The generator wraps calls to `SavedObjects.find` and iterates over + * multiple pages of results using `_pit` and `search_after`. This will + * open a new Point In Time (PIT), and continue paging until a set of + * results is received that's smaller than the designated `perPage`. + * + * Once you have retrieved all of the results you need, it is recommended + * to call `close()` to clean up the PIT and prevent Elasticsearch from + * consuming resources unnecessarily. This will automatically be done for + * you if you reach the last page of results. + * + * @example + * ```ts + * const finder = findWithPointInTime({ + * logger, + * savedObjectsClient, + * }); + * + * const options: SavedObjectsFindOptions = { + * type: 'visualization', + * search: 'foo*', + * perPage: 100, + * }; + * + * const responses: SavedObjectFindResponse[] = []; + * for await (const response of finder.find(options)) { + * responses.push(...response); + * if (doneSearching) { + * await finder.close(); + * } + * } + * ``` + */ +export function findWithPointInTime({ + logger, + savedObjectsClient, +}: { + logger: Logger; + savedObjectsClient: SavedObjectsClientContract; +}) { + return new FindWithPointInTime({ logger, savedObjectsClient }); +} + +/** + * @internal + */ +export class FindWithPointInTime { + readonly #log: Logger; + readonly #savedObjectsClient: SavedObjectsClientContract; + #open?: boolean; + #perPage?: number; + #pitId?: string; + #type?: string | string[]; + + constructor({ + savedObjectsClient, + logger, + }: { + savedObjectsClient: SavedObjectsClientContract; + logger: Logger; + }) { + this.#log = logger; + this.#savedObjectsClient = savedObjectsClient; + } + + async *find(findOptions: SavedObjectsFindOptions) { + this.#open = true; + this.#type = findOptions.type; + // Default to 1000 items per page as a tradeoff between + // speed and memory consumption. + this.#perPage = findOptions.perPage ?? 1000; + + // Open PIT and request our first page of hits + await this.open(); + + let results = await this.findNext({ findOptions, id: this.#pitId }); + this.#pitId = results.pit_id; + let lastResultsCount = results.saved_objects.length; + let lastHitSortValue = this.getLastHitSortValue(results); + + this.#log.debug(`Collected [${lastResultsCount}] saved objects for export.`); + + // Close PIT if this was our last page + if (this.#pitId && lastResultsCount < this.#perPage!) { + await this.close(); + } + + yield results; + + // We've reached the end when there are fewer hits than our perPage size + while (this.#open && lastHitSortValue && lastResultsCount === this.#perPage) { + results = await this.findNext({ + findOptions, + id: this.#pitId, + searchAfter: lastHitSortValue, + }); + + lastResultsCount = results.saved_objects.length; + lastHitSortValue = this.getLastHitSortValue(results); + this.#pitId = results.pit_id; + + this.#log.debug(`Collected [${lastResultsCount}] more saved objects for export.`); + + if (this.#pitId && lastResultsCount < this.#perPage) { + await this.close(); + } + + yield results; + } + + return; + } + + async close() { + try { + if (this.#pitId) { + this.#log.debug(`Closing PIT for types [${this.#type}]`); + await this.#savedObjectsClient.closePointInTime(this.#pitId); + this.#pitId = undefined; + } + this.#type = undefined; + this.#open = false; + } catch (e) { + this.#log.error(`Failed to close PIT for types [${this.#type}]`); + throw e; + } + } + + private async open() { + try { + const { id } = await this.#savedObjectsClient.openPointInTimeForType(this.#type!); + this.#pitId = id; + } catch (e) { + // Since `find` swallows 404s, it is expected that exporter will do the same, + // so we only rethrow non-404 errors here. + if (e.output.statusCode !== 404) { + throw e; + } + this.#log.debug(`Unable to open PIT for types [${this.#type}]: 404 ${e}`); + } + } + + private async findNext({ + findOptions, + id, + searchAfter, + }: { + findOptions: SavedObjectsFindOptions; + id?: string; + searchAfter?: unknown[]; + }) { + try { + return await this.#savedObjectsClient.find({ + // Sort fields are required to use searchAfter, so we set some defaults here + sortField: 'updated_at', + sortOrder: 'desc', + // Bump keep_alive by 2m on every new request to allow for the ES client + // to make multiple retries in the event of a network failure. + ...(id ? { pit: { keepAlive: '2m', ...findOptions.pit, id } } : {}), + ...(searchAfter ? { searchAfter } : {}), + ...findOptions, + }); + } catch (e) { + if (id) { + // Clean up PIT on any errors. + await this.close(); + } + throw e; + } + } + + private getLastHitSortValue(res: SavedObjectsFindResponse) { + return res.saved_objects.length && res.saved_objects[res.saved_objects.length - 1].sort; + } +} diff --git a/src/core/server/saved_objects/export/saved_objects_exporter.test.ts b/src/core/server/saved_objects/export/saved_objects_exporter.test.ts index f96b634b73297..b6ef1dabe93c6 100644 --- a/src/core/server/saved_objects/export/saved_objects_exporter.test.ts +++ b/src/core/server/saved_objects/export/saved_objects_exporter.test.ts @@ -114,7 +114,7 @@ describe('getSortedObjectsForExport()', () => { "perPage": 10000, "pit": Object { "id": "some_pit_id", - "keepAlive": "1m", + "keepAlive": "2m", }, "search": undefined, "sortField": "updated_at", @@ -197,6 +197,7 @@ describe('getSortedObjectsForExport()', () => { saved_objects: mockHits, per_page: 10000, page: 0, + pit_id: 'abc123', }); const exportStream = await exporter.exportByTypes({ @@ -230,7 +231,7 @@ describe('getSortedObjectsForExport()', () => { expect(savedObjectsClient.find).toHaveBeenCalledWith( expect.objectContaining({ - pit: expect.objectContaining({ id: 'abc123', keepAlive: '1m' }), + pit: expect.objectContaining({ id: 'abc123', keepAlive: '2m' }), sortField: 'updated_at', sortOrder: 'desc', type: ['index-pattern'], @@ -280,12 +281,14 @@ describe('getSortedObjectsForExport()', () => { saved_objects: firstMockHits, per_page: 10000, page: 0, + pit_id: 'abc123', }); savedObjectsClient.find.mockResolvedValueOnce({ total: 15000, saved_objects: secondMockHits, per_page: 5000, page: 1, + pit_id: 'abc123', }); const exportStream = await exporter.exportByTypes({ @@ -455,7 +458,7 @@ describe('getSortedObjectsForExport()', () => { "perPage": 10000, "pit": Object { "id": "some_pit_id", - "keepAlive": "1m", + "keepAlive": "2m", }, "search": undefined, "sortField": "updated_at", @@ -611,7 +614,7 @@ describe('getSortedObjectsForExport()', () => { "perPage": 10000, "pit": Object { "id": "some_pit_id", - "keepAlive": "1m", + "keepAlive": "2m", }, "search": "foo", "sortField": "updated_at", @@ -704,7 +707,7 @@ describe('getSortedObjectsForExport()', () => { "perPage": 10000, "pit": Object { "id": "some_pit_id", - "keepAlive": "1m", + "keepAlive": "2m", }, "search": undefined, "sortField": "updated_at", @@ -802,7 +805,7 @@ describe('getSortedObjectsForExport()', () => { "perPage": 10000, "pit": Object { "id": "some_pit_id", - "keepAlive": "1m", + "keepAlive": "2m", }, "search": undefined, "sortField": "updated_at", diff --git a/src/core/server/saved_objects/export/saved_objects_exporter.ts b/src/core/server/saved_objects/export/saved_objects_exporter.ts index ada9c80b6e735..96eec644f085e 100644 --- a/src/core/server/saved_objects/export/saved_objects_exporter.ts +++ b/src/core/server/saved_objects/export/saved_objects_exporter.ts @@ -10,7 +10,7 @@ import { createListStream } from '@kbn/utils'; import { PublicMethodsOf } from '@kbn/utility-types'; import { Logger } from '../../logging'; import { SavedObject, SavedObjectsClientContract, SavedObjectsFindOptions } from '../types'; -import { SavedObjectsFindResult, SavedObjectsFindResponse } from '../service'; +import { SavedObjectsFindResult } from '../service'; import { ISavedObjectTypeRegistry } from '../saved_objects_type_registry'; import { fetchNestedDependencies } from './fetch_nested_dependencies'; import { sortObjects } from './sort_objects'; @@ -23,6 +23,7 @@ import { } from './types'; import { SavedObjectsExportError } from './errors'; import { applyExportTransforms } from './apply_export_transforms'; +import { findWithPointInTime } from './find_with_point_in_time'; import { byIdAscComparator, getPreservedOrderComparator, SavedObjectComparator } from './utils'; /** @@ -161,108 +162,17 @@ export class SavedObjectsExporter { return bulkGetResult.saved_objects; } - /** - * Generator which wraps calls to `SavedObjects.find` and iterates over - * multiple pages of results using `_pit` and `search_after`. This will - * open a new PIT, continue paging until a set of results is received - * that's smaller than the designated `perPage`, and then close the PIT. - * - * @example - * ```ts - * const finder = findWithPointInTime({ - * type: 'index-pattern', - * search: 'foo*', - * perPage: 100, - * }); - * - * const responses: SavedObjectFindResponse[] = []; - * for await (const response of finder) { - * responses.push(...response); - * } - * ``` - */ - private async *findWithPointInTime(findOptions: SavedObjectsFindOptions) { - const getLastHitSortValue = (res: SavedObjectsFindResponse) => - res.saved_objects.length && res.saved_objects[res.saved_objects.length - 1].sort; - - const findWithPit = async ({ id, searchAfter }: { id?: string; searchAfter?: unknown[] }) => { - try { - return await this.#savedObjectsClient.find({ - // Sort fields are required to use searchAfter, so we set some defaults here - sortField: 'updated_at', - sortOrder: 'desc', - // Bump keep_alive by 1m on every new request - ...(id ? { pit: { keepAlive: '1m', ...findOptions.pit, id } } : {}), - ...(searchAfter ? { searchAfter } : {}), - ...findOptions, - }); - } catch (e) { - if (id) { - // Clean up PIT on any errors. - this.#log.debug(`Closing PIT for types [${findOptions.type}]`); - await this.#savedObjectsClient.closePointInTime(id); - } - throw e; - } - }; - - // Open PIT and request our first page of hits - let pitId: string | undefined; - try { - const { id } = await this.#savedObjectsClient.openPointInTimeForType(findOptions.type); - pitId = id; - } catch (e) { - // Since `find` swallows 404s, it is expected that exporter will do the same, - // so we only rethrow non-404 errors here. - if (e.output.statusCode !== 404) { - throw e; - } - this.#log.debug(`Unable to open PIT for types [${findOptions.type}]: 404 ${e}`); - } - - let results = await findWithPit({ id: pitId }); - let lastResultsCount = results.saved_objects.length; - let lastHitSortValue = getLastHitSortValue(results); - pitId = results.pit_id; - - this.#log.debug(`Collected [${lastResultsCount}] saved objects for export.`); - - // Close PIT if this was our last page - if (pitId && lastResultsCount < findOptions.perPage!) { - this.#log.debug(`Closing PIT for types [${findOptions.type}]`); - await this.#savedObjectsClient.closePointInTime(pitId); - } - - yield results; - - // We've reached the end when there are fewer hits than our perPage size - while (lastHitSortValue && lastResultsCount === findOptions.perPage) { - results = await findWithPit({ - id: pitId, - searchAfter: lastHitSortValue, - }); - - lastResultsCount = results.saved_objects.length; - lastHitSortValue = getLastHitSortValue(results); - pitId = results.pit_id; - - this.#log.debug(`Collected [${lastResultsCount}] more saved objects for export.`); - - if (pitId && lastResultsCount < findOptions.perPage) { - this.#log.debug(`Closing PIT for types [${findOptions.type}]`); - await this.#savedObjectsClient.closePointInTime(pitId); - } - - yield results; - } - } - private async fetchByTypes({ types, namespace, hasReference, search, }: SavedObjectsExportByTypeOptions) { + const finder = findWithPointInTime({ + logger: this.#log, + savedObjectsClient: this.#savedObjectsClient, + }); + const options: SavedObjectsFindOptions = { type: types, hasReference, @@ -274,10 +184,9 @@ export class SavedObjectsExporter { // case we will use PIT to "scroll" through pages of hits, 10k at a time. perPage: 10000, }; - const finder = this.findWithPointInTime(options); const hits: SavedObjectsFindResult[] = []; - for await (const result of finder) { + for await (const result of finder.find(options)) { hits.push(...result.saved_objects); if (hits.length > this.#exportSizeLimit) { throw SavedObjectsExportError.exportSizeExceeded(this.#exportSizeLimit); diff --git a/src/core/server/saved_objects/migrations/core/index_migrator.test.ts b/src/core/server/saved_objects/migrations/core/index_migrator.test.ts index 0d1939231ce6c..0413c1a44e160 100644 --- a/src/core/server/saved_objects/migrations/core/index_migrator.test.ts +++ b/src/core/server/saved_objects/migrations/core/index_migrator.test.ts @@ -28,7 +28,7 @@ describe('IndexMigrator', () => { log: loggingSystemMock.create().get(), mappingProperties: {}, pollInterval: 1, - scrollDuration: '1m', + scrollDuration: '2m', documentMigrator: { migrationVersion: {}, migrate: _.identity, diff --git a/src/core/server/saved_objects/routes/integration_tests/find.test.ts b/src/core/server/saved_objects/routes/integration_tests/find.test.ts index 0e22c153af3e0..58a86edc3414c 100644 --- a/src/core/server/saved_objects/routes/integration_tests/find.test.ts +++ b/src/core/server/saved_objects/routes/integration_tests/find.test.ts @@ -240,7 +240,7 @@ describe('GET /api/saved_objects/_find', () => { }); it('accepts the optional query parameter pit', async () => { - const pitValues = querystring.escape(JSON.stringify({ id: 'abc', keep_alive: '1m' })); + const pitValues = querystring.escape(JSON.stringify({ id: 'abc', keep_alive: '2m' })); await supertest(httpSetup.server.listener) .get(`/api/saved_objects/_find?type=foo&pit=${pitValues}`) .expect(200); @@ -252,7 +252,7 @@ describe('GET /api/saved_objects/_find', () => { expect.objectContaining({ pit: { id: 'abc', - keepAlive: '1m', + keepAlive: '2m', }, }) ); diff --git a/src/core/server/saved_objects/service/lib/repository.test.js b/src/core/server/saved_objects/service/lib/repository.test.js index e1135ade6690a..d46a37d8ffa76 100644 --- a/src/core/server/saved_objects/service/lib/repository.test.js +++ b/src/core/server/saved_objects/service/lib/repository.test.js @@ -2989,13 +2989,13 @@ describe('SavedObjectsRepository', () => { it(`accepts pit`, async () => { const relevantOpts = { ...commonOptions, - pit: { id: 'abc123', keepAlive: '1m' }, + pit: { id: 'abc123', keepAlive: '2m' }, }; await findSuccess(relevantOpts, namespace); expect(getSearchDslNS.getSearchDsl).toHaveBeenCalledWith(mappings, registry, { ...relevantOpts, - pit: { id: 'abc123', keepAlive: '1m' }, + pit: { id: 'abc123', keepAlive: '2m' }, }); }); @@ -4450,10 +4450,10 @@ describe('SavedObjectsRepository', () => { }); it(`accepts keepAlive`, async () => { - await successResponse(type, { keepAlive: '1m' }); + await successResponse(type, { keepAlive: '2m' }); expect(client.openPointInTime).toHaveBeenCalledWith( expect.objectContaining({ - keep_alive: '1m', + keep_alive: '2m', }), expect.anything() ); diff --git a/src/core/server/saved_objects/service/lib/repository.ts b/src/core/server/saved_objects/service/lib/repository.ts index 12e8be3887d5e..c7373b6aa824a 100644 --- a/src/core/server/saved_objects/service/lib/repository.ts +++ b/src/core/server/saved_objects/service/lib/repository.ts @@ -1792,7 +1792,7 @@ export class SavedObjectsRepository { * * const { id } = await repository.openPointInTimeForType( * type: 'index-pattern', - * { keepAlive: '1m' }, + * { keepAlive: '2m' }, * ); * * const response = await repository.find({ @@ -1802,7 +1802,7 @@ export class SavedObjectsRepository { * sortOrder: 'desc', * pit: { * id: 'abc123', - * keepAlive: '1m', + * keepAlive: '2m', * }, * searchAfter: [1234, 'abcd'], * }); @@ -1863,7 +1863,7 @@ export class SavedObjectsRepository { * * const { id } = await repository.openPointInTimeForType( * type: 'index-pattern', - * { keepAlive: '1m' }, + * { keepAlive: '2m' }, * ); * * const response = await repository.find({ @@ -1873,7 +1873,7 @@ export class SavedObjectsRepository { * sortOrder: 'desc', * pit: { * id: 'abc123', - * keepAlive: '1m', + * keepAlive: '2m', * }, * searchAfter: [1234, 'abcd'], * }); diff --git a/src/core/server/saved_objects/service/lib/search_dsl/pit_params.test.ts b/src/core/server/saved_objects/service/lib/search_dsl/pit_params.test.ts index 6d6051e33f1cf..5a99168792e83 100644 --- a/src/core/server/saved_objects/service/lib/search_dsl/pit_params.test.ts +++ b/src/core/server/saved_objects/service/lib/search_dsl/pit_params.test.ts @@ -18,10 +18,10 @@ describe('searchDsl/getPitParams', () => { }); it('includes keepAlive if provided and rewrites to snake case', () => { - expect(getPitParams({ id: 'abc123', keepAlive: '1m' })).toEqual({ + expect(getPitParams({ id: 'abc123', keepAlive: '2m' })).toEqual({ pit: { id: 'abc123', - keep_alive: '1m', + keep_alive: '2m', }, }); }); From dfcdee3b33130c2ccd2f23d61593bc5d04ebe65d Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Tue, 9 Feb 2021 11:44:39 -0700 Subject: [PATCH 19/32] Set perPage default to 1000. --- .../export/find_with_point_in_time.ts | 12 +++- .../export/saved_objects_exporter.test.ts | 58 +++++++++---------- .../export/saved_objects_exporter.ts | 4 -- .../apis/saved_objects/import.ts | 4 +- 4 files changed, 40 insertions(+), 38 deletions(-) diff --git a/src/core/server/saved_objects/export/find_with_point_in_time.ts b/src/core/server/saved_objects/export/find_with_point_in_time.ts index 5ce271aa76859..ef6b01673fb21 100644 --- a/src/core/server/saved_objects/export/find_with_point_in_time.ts +++ b/src/core/server/saved_objects/export/find_with_point_in_time.ts @@ -77,12 +77,18 @@ export class FindWithPointInTime { this.#savedObjectsClient = savedObjectsClient; } - async *find(findOptions: SavedObjectsFindOptions) { + async *find(options: SavedObjectsFindOptions) { this.#open = true; - this.#type = findOptions.type; + this.#type = options.type; // Default to 1000 items per page as a tradeoff between // speed and memory consumption. - this.#perPage = findOptions.perPage ?? 1000; + this.#perPage = options.perPage ?? 1000; + + const findOptions: SavedObjectsFindOptions = { + ...options, + perPage: this.#perPage, + type: this.#type, + }; // Open PIT and request our first page of hits await this.open(); diff --git a/src/core/server/saved_objects/export/saved_objects_exporter.test.ts b/src/core/server/saved_objects/export/saved_objects_exporter.test.ts index b6ef1dabe93c6..7c6cc409e1de7 100644 --- a/src/core/server/saved_objects/export/saved_objects_exporter.test.ts +++ b/src/core/server/saved_objects/export/saved_objects_exporter.test.ts @@ -19,7 +19,7 @@ async function readStreamToCompletion(stream: Readable): Promise { @@ -66,7 +66,7 @@ describe('getSortedObjectsForExport()', () => { references: [], }, ], - per_page: 1, + per_page: 1000, page: 0, }); const exportStream = await exporter.exportByTypes({ @@ -111,7 +111,7 @@ describe('getSortedObjectsForExport()', () => { "hasReference": undefined, "hasReferenceOperator": undefined, "namespaces": undefined, - "perPage": 10000, + "perPage": 1000, "pit": Object { "id": "some_pit_id", "keepAlive": "2m", @@ -163,14 +163,14 @@ describe('getSortedObjectsForExport()', () => { return hits; } - describe('<10k hits', () => { + describe('<1k hits', () => { const mockHits = generateHits(20); test('requests a single page', async () => { savedObjectsClient.find.mockResolvedValueOnce({ total: 20, saved_objects: mockHits, - per_page: 10000, + per_page: 1000, page: 0, }); @@ -195,7 +195,7 @@ describe('getSortedObjectsForExport()', () => { savedObjectsClient.find.mockResolvedValueOnce({ total: 20, saved_objects: mockHits, - per_page: 10000, + per_page: 1000, page: 0, pit_id: 'abc123', }); @@ -218,7 +218,7 @@ describe('getSortedObjectsForExport()', () => { savedObjectsClient.find.mockResolvedValueOnce({ total: 20, saved_objects: mockHits, - per_page: 10000, + per_page: 1000, page: 0, }); @@ -240,21 +240,21 @@ describe('getSortedObjectsForExport()', () => { }); }); - describe('>10k hits', () => { - const firstMockHits = generateHits(10000, { sort: ['a', 'b'] }); - const secondMockHits = generateHits(5000); + describe('>1k hits', () => { + const firstMockHits = generateHits(1000, { sort: ['a', 'b'] }); + const secondMockHits = generateHits(500); test('requests multiple pages', async () => { savedObjectsClient.find.mockResolvedValueOnce({ - total: 15000, + total: 1500, saved_objects: firstMockHits, - per_page: 10000, + per_page: 1000, page: 0, }); savedObjectsClient.find.mockResolvedValueOnce({ - total: 15000, + total: 1500, saved_objects: secondMockHits, - per_page: 5000, + per_page: 500, page: 1, }); @@ -268,7 +268,7 @@ describe('getSortedObjectsForExport()', () => { expect(savedObjectsClient.find).toHaveBeenCalledTimes(2); expect(response[response.length - 1]).toMatchInlineSnapshot(` Object { - "exportedCount": 15000, + "exportedCount": 1500, "missingRefCount": 0, "missingReferences": Array [], } @@ -277,16 +277,16 @@ describe('getSortedObjectsForExport()', () => { test('opens and closes PIT', async () => { savedObjectsClient.find.mockResolvedValueOnce({ - total: 15000, + total: 1500, saved_objects: firstMockHits, - per_page: 10000, + per_page: 1000, page: 0, pit_id: 'abc123', }); savedObjectsClient.find.mockResolvedValueOnce({ - total: 15000, + total: 1500, saved_objects: secondMockHits, - per_page: 5000, + per_page: 500, page: 1, pit_id: 'abc123', }); @@ -304,15 +304,15 @@ describe('getSortedObjectsForExport()', () => { test('passes sort values to searchAfter', async () => { savedObjectsClient.find.mockResolvedValueOnce({ - total: 15000, + total: 1500, saved_objects: firstMockHits, - per_page: 10000, + per_page: 1000, page: 0, }); savedObjectsClient.find.mockResolvedValueOnce({ - total: 15000, + total: 1500, saved_objects: secondMockHits, - per_page: 5000, + per_page: 500, page: 1, }); @@ -455,7 +455,7 @@ describe('getSortedObjectsForExport()', () => { "hasReference": undefined, "hasReferenceOperator": undefined, "namespaces": undefined, - "perPage": 10000, + "perPage": 1000, "pit": Object { "id": "some_pit_id", "keepAlive": "2m", @@ -611,7 +611,7 @@ describe('getSortedObjectsForExport()', () => { "hasReference": undefined, "hasReferenceOperator": undefined, "namespaces": undefined, - "perPage": 10000, + "perPage": 1000, "pit": Object { "id": "some_pit_id", "keepAlive": "2m", @@ -704,7 +704,7 @@ describe('getSortedObjectsForExport()', () => { ], "hasReferenceOperator": "OR", "namespaces": undefined, - "perPage": 10000, + "perPage": 1000, "pit": Object { "id": "some_pit_id", "keepAlive": "2m", @@ -754,7 +754,7 @@ describe('getSortedObjectsForExport()', () => { references: [], }, ], - per_page: 1, + per_page: 1000, page: 0, }); const exportStream = await exporter.exportByTypes({ @@ -802,7 +802,7 @@ describe('getSortedObjectsForExport()', () => { "namespaces": Array [ "foo", ], - "perPage": 10000, + "perPage": 1000, "pit": Object { "id": "some_pit_id", "keepAlive": "2m", @@ -873,7 +873,7 @@ describe('getSortedObjectsForExport()', () => { test('sorts objects within type', async () => { savedObjectsClient.find.mockResolvedValueOnce({ total: 3, - per_page: 10000, + per_page: 1000, page: 1, saved_objects: [ { diff --git a/src/core/server/saved_objects/export/saved_objects_exporter.ts b/src/core/server/saved_objects/export/saved_objects_exporter.ts index 96eec644f085e..1819e8ea4d50d 100644 --- a/src/core/server/saved_objects/export/saved_objects_exporter.ts +++ b/src/core/server/saved_objects/export/saved_objects_exporter.ts @@ -179,10 +179,6 @@ export class SavedObjectsExporter { hasReferenceOperator: hasReference ? 'OR' : undefined, search, namespaces: namespace ? [namespace] : undefined, - // We aren't using `exportSizeLimit` here because a user may opt to set it - // higher than the 10k ES default for `index.max_result_window`, in which - // case we will use PIT to "scroll" through pages of hits, 10k at a time. - perPage: 10000, }; const hits: SavedObjectsFindResult[] = []; diff --git a/test/api_integration/apis/saved_objects/import.ts b/test/api_integration/apis/saved_objects/import.ts index b0aa9b0eef8fc..d463b9498a52a 100644 --- a/test/api_integration/apis/saved_objects/import.ts +++ b/test/api_integration/apis/saved_objects/import.ts @@ -166,7 +166,7 @@ export default function ({ getService }: FtrProviderContext) { it('should return 400 when trying to import more than 10,000 objects', async () => { const fileChunks = []; - for (let i = 0; i < 10001; i++) { + for (let i = 0; i <= 10001; i++) { fileChunks.push(`{"type":"visualization","id":"${i}","attributes":{},"references":[]}`); } await supertest @@ -177,7 +177,7 @@ export default function ({ getService }: FtrProviderContext) { expect(resp.body).to.eql({ statusCode: 400, error: 'Bad Request', - message: "Can't import more than 10000 objects", + message: "Can't import more than 10001 objects", }); }); }); From 004a062052b192dc6427887796f84adcaff386d6 Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Tue, 9 Feb 2021 12:23:23 -0700 Subject: [PATCH 20/32] Remove some duplication in findWithPointInTime. --- .../export/find_with_point_in_time.ts | 40 ++++++++----------- 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/src/core/server/saved_objects/export/find_with_point_in_time.ts b/src/core/server/saved_objects/export/find_with_point_in_time.ts index ef6b01673fb21..6668e2184e571 100644 --- a/src/core/server/saved_objects/export/find_with_point_in_time.ts +++ b/src/core/server/saved_objects/export/find_with_point_in_time.ts @@ -93,40 +93,29 @@ export class FindWithPointInTime { // Open PIT and request our first page of hits await this.open(); - let results = await this.findNext({ findOptions, id: this.#pitId }); - this.#pitId = results.pit_id; - let lastResultsCount = results.saved_objects.length; - let lastHitSortValue = this.getLastHitSortValue(results); - - this.#log.debug(`Collected [${lastResultsCount}] saved objects for export.`); - - // Close PIT if this was our last page - if (this.#pitId && lastResultsCount < this.#perPage!) { - await this.close(); - } - - yield results; - - // We've reached the end when there are fewer hits than our perPage size - while (this.#open && lastHitSortValue && lastResultsCount === this.#perPage) { - results = await this.findNext({ + let lastResultsCount: number; + let lastHitSortValue: unknown[] | undefined; + do { + const results = await this.findNext({ findOptions, id: this.#pitId, - searchAfter: lastHitSortValue, + ...(lastHitSortValue ? { searchAfter: lastHitSortValue } : {}), }); - + this.#pitId = results.pit_id; lastResultsCount = results.saved_objects.length; lastHitSortValue = this.getLastHitSortValue(results); - this.#pitId = results.pit_id; - this.#log.debug(`Collected [${lastResultsCount}] more saved objects for export.`); + this.#log.debug(`Collected [${lastResultsCount}] saved objects for export.`); + // Close PIT if this was our last page if (this.#pitId && lastResultsCount < this.#perPage) { await this.close(); } yield results; - } + // We've reached the end when there are fewer hits than our perPage size, + // or when `close()` has been called. + } while (this.#open && lastHitSortValue && lastResultsCount >= this.#perPage); return; } @@ -189,7 +178,10 @@ export class FindWithPointInTime { } } - private getLastHitSortValue(res: SavedObjectsFindResponse) { - return res.saved_objects.length && res.saved_objects[res.saved_objects.length - 1].sort; + private getLastHitSortValue(res: SavedObjectsFindResponse): unknown[] | undefined { + if (res.saved_objects.length < 1) { + return undefined; + } + return res.saved_objects[res.saved_objects.length - 1].sort; } } From 255779e4ab2c53764be2ea02c51f827fe858f793 Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Tue, 9 Feb 2021 14:05:47 -0700 Subject: [PATCH 21/32] Address feedback on security/spaces clients. --- .../saved_objects/service/lib/repository.ts | 2 +- ...ecure_saved_objects_client_wrapper.test.ts | 31 +++++++++++++++---- .../secure_saved_objects_client_wrapper.ts | 15 ++++----- .../spaces_saved_objects_client.test.ts | 26 ++++++++++++++++ .../spaces_saved_objects_client.ts | 12 +++++-- 5 files changed, 70 insertions(+), 16 deletions(-) diff --git a/src/core/server/saved_objects/service/lib/repository.ts b/src/core/server/saved_objects/service/lib/repository.ts index c7373b6aa824a..371cdbeee1dcb 100644 --- a/src/core/server/saved_objects/service/lib/repository.ts +++ b/src/core/server/saved_objects/service/lib/repository.ts @@ -1809,7 +1809,7 @@ export class SavedObjectsRepository { * ``` * * @param {string|Array} type - * @param {object} [options] - {@link SavedObjectsPointInTimeOptions} + * @param {object} [options] - {@link SavedObjectsOpenPointInTimeOptions} * @property {string} [options.keepAlive] * @property {string} [options.preference] * @returns {promise} - { id: string } diff --git a/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.test.ts b/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.test.ts index f3012cba0c236..9e4f9fafda970 100644 --- a/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.test.ts +++ b/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.test.ts @@ -995,11 +995,6 @@ describe('#openPointInTimeForType', () => { await expectGeneralError(client.openPointInTimeForType, { type }); }); - test(`throws decorated ForbiddenError when unauthorized`, async () => { - const options = { namespace }; - await expectForbiddenError(client.openPointInTimeForType, { type, options }); - }); - test(`returns result of baseClient.openPointInTimeForType when authorized`, async () => { const apiCallReturnValue = Symbol(); clientOpts.baseClient.openPointInTimeForType.mockReturnValue(apiCallReturnValue as any); @@ -1015,7 +1010,7 @@ describe('#openPointInTimeForType', () => { const options = { namespace }; await expectSuccess(client.openPointInTimeForType, { type, options }); expect(clientOpts.auditLogger.log).toHaveBeenCalledTimes(1); - expectAuditEvent('saved_object_open_point_in_time', EventOutcome.SUCCESS); + expectAuditEvent('saved_object_open_point_in_time', EventOutcome.UNKNOWN); }); test(`adds audit event when not successful`, async () => { @@ -1026,6 +1021,30 @@ describe('#openPointInTimeForType', () => { }); }); +describe('#closePointInTime', () => { + const id = 'abc123'; + const namespace = 'some-ns'; + + test(`returns result of baseClient.closePointInTime`, async () => { + const apiCallReturnValue = Symbol(); + clientOpts.baseClient.closePointInTime.mockReturnValue(apiCallReturnValue as any); + + const options = { namespace }; + const result = await client.closePointInTime(id, options); + expect(result).toBe(apiCallReturnValue); + }); + + test(`adds audit event`, async () => { + const apiCallReturnValue = Symbol(); + clientOpts.baseClient.closePointInTime.mockReturnValue(apiCallReturnValue as any); + + const options = { namespace }; + await client.closePointInTime(id, options); + expect(clientOpts.auditLogger.log).toHaveBeenCalledTimes(1); + expectAuditEvent('saved_object_close_point_in_time', EventOutcome.UNKNOWN); + }); +}); + describe('#resolve', () => { const type = 'foo'; const id = `${type}-id`; diff --git a/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts b/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts index c1fbf2ff60519..a0d148a14f9b7 100644 --- a/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts +++ b/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts @@ -570,7 +570,10 @@ export class SecureSavedObjectsClientWrapper implements SavedObjectsClientContra ) { try { const args = { type, options }; - await this.ensureAuthorized(type, 'open_point_in_time', options?.namespace, { args }); + await this.ensureAuthorized(type, 'open_point_in_time', options?.namespace, { + args, + requireFullAuthorization: false, + }); } catch (error) { this.auditLogger.log( savedObjectEvent({ @@ -581,27 +584,25 @@ export class SecureSavedObjectsClientWrapper implements SavedObjectsClientContra throw error; } - const pit = await this.baseClient.openPointInTimeForType(type, options); - this.auditLogger.log( savedObjectEvent({ action: SavedObjectAction.OPEN_POINT_IN_TIME, + outcome: EventOutcome.UNKNOWN, }) ); - return pit; + return await this.baseClient.openPointInTimeForType(type, options); } public async closePointInTime(id: string, options?: SavedObjectsClosePointInTimeOptions) { - const response = await this.baseClient.closePointInTime(id, options); - this.auditLogger.log( savedObjectEvent({ action: SavedObjectAction.CLOSE_POINT_IN_TIME, + outcome: EventOutcome.UNKNOWN, }) ); - return response; + return await this.baseClient.closePointInTime(id, options); } private async checkPrivileges( diff --git a/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.test.ts b/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.test.ts index 632100de691d8..f5917e78135ec 100644 --- a/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.test.ts +++ b/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.test.ts @@ -615,5 +615,31 @@ const ERROR_NAMESPACE_SPECIFIED = 'Spaces currently determines the namespaces'; }); }); }); + + describe('#closePointInTime', () => { + test(`throws error if options.namespace is specified`, async () => { + const { client } = createSpacesSavedObjectsClient(); + + await expect(client.closePointInTime('foo', { namespace: 'bar' })).rejects.toThrow( + ERROR_NAMESPACE_SPECIFIED + ); + }); + + test(`supplements options with the current namespace`, async () => { + const { client, baseClient } = createSpacesSavedObjectsClient(); + const expectedReturnValue = { succeeded: true, num_freed: 1 }; + baseClient.closePointInTime.mockReturnValue(Promise.resolve(expectedReturnValue)); + + const options = Object.freeze({ foo: 'bar' }); + // @ts-expect-error + const actualReturnValue = await client.closePointInTime('foo', options); + + expect(actualReturnValue).toBe(expectedReturnValue); + expect(baseClient.closePointInTime).toHaveBeenCalledWith('foo', { + foo: 'bar', + namespace: currentSpace.expectedNamespace, + }); + }); + }); }); }); diff --git a/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts b/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts index 95e67bcd1b51b..af795a44ddf79 100644 --- a/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts +++ b/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts @@ -386,7 +386,7 @@ export class SpacesSavedObjectsClient implements SavedObjectsClientContract { * The returned `id` can then be passed to `SavedObjects.find` to search against that PIT. * * @param {string|Array} type - * @param {object} [options] - {@link SavedObjectsPointInTimeOptions} + * @param {object} [options] - {@link SavedObjectsOpenPointInTimeOptions} * @property {string} [options.keepAlive] * @property {string} [options.preference] * @returns {promise} - { id: string } @@ -406,8 +406,16 @@ export class SpacesSavedObjectsClient implements SavedObjectsClientContract { * Closes a Point In Time (PIT) by ID. This simply proxies the request to ES * via the Elasticsearch client, and is included in the Saved Objects Client * as a convenience for consumers who are using `openPointInTimeForType`. + * + * @param {string} id - ID returned from `openPointInTimeForType` + * @param {object} [options] - {@link SavedObjectsClosePointInTimeOptions} + * @returns {promise} - { succeeded: boolean; num_freed: number } */ async closePointInTime(id: string, options?: SavedObjectsClosePointInTimeOptions) { - return await this.client.closePointInTime(id, options); + throwErrorIfNamespaceSpecified(options); + return await this.client.closePointInTime(id, { + ...options, + namespace: spaceIdToNamespace(this.spaceId), + }); } } From 6f0f875d9ce27c2692c523ce14cfbe3a06d9f628 Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Tue, 9 Feb 2021 16:39:19 -0700 Subject: [PATCH 22/32] Update generated docs. --- ...gin-core-server.savedobjectsrepository.closepointintime.md | 4 ++-- ...re-server.savedobjectsrepository.openpointintimefortype.md | 4 ++-- src/core/server/server.api.md | 1 - 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.closepointintime.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.closepointintime.md index 27d732ac8f3e6..8f9dca35fa362 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.closepointintime.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.closepointintime.md @@ -37,7 +37,7 @@ const repository = coreStart.savedObjects.createInternalRepository(); const { id } = await repository.openPointInTimeForType( type: 'index-pattern', - { keepAlive: '1m' }, + { keepAlive: '2m' }, ); const response = await repository.find({ @@ -47,7 +47,7 @@ const response = await repository.find({ sortOrder: 'desc', pit: { id: 'abc123', - keepAlive: '1m', + keepAlive: '2m', }, searchAfter: [1234, 'abcd'], }); diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.openpointintimefortype.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.openpointintimefortype.md index 1845dbc64e43d..11e356f845b31 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.openpointintimefortype.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.openpointintimefortype.md @@ -33,7 +33,7 @@ const repository = coreStart.savedObjects.createInternalRepository(); const { id } = await repository.openPointInTimeForType( type: 'index-pattern', - { keepAlive: '1m' }, + { keepAlive: '2m' }, ); const response = await repository.find({ @@ -43,7 +43,7 @@ const response = await repository.find({ sortOrder: 'desc', pit: { id: 'abc123', - keepAlive: '1m', + keepAlive: '2m', }, searchAfter: [1234, 'abcd'], }); diff --git a/src/core/server/server.api.md b/src/core/server/server.api.md index 3739fc9f4d6f4..8c55f1fe873f6 100644 --- a/src/core/server/server.api.md +++ b/src/core/server/server.api.md @@ -2822,7 +2822,6 @@ export class SavedObjectsRepository { find(options: SavedObjectsFindOptions): Promise>; get(type: string, id: string, options?: SavedObjectsBaseOptions): Promise>; incrementCounter(type: string, id: string, counterFields: Array, options?: SavedObjectsIncrementCounterOptions): Promise>; - // Warning: (ae-unresolved-link) The @link reference could not be resolved: The package "kibana" does not have an export "SavedObjectsPointInTimeOptions" openPointInTimeForType(type: string | string[], { keepAlive, preference }?: SavedObjectsOpenPointInTimeOptions): Promise; removeReferencesTo(type: string, id: string, options?: SavedObjectsRemoveReferencesToOptions): Promise; resolve(type: string, id: string, options?: SavedObjectsBaseOptions): Promise>; From 6f30ae5498f327b1a76b877806381c72926d7bf3 Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Tue, 9 Feb 2021 19:54:11 -0700 Subject: [PATCH 23/32] Fix remaining test failures. --- .../apis/saved_objects/resolve_import_errors.ts | 6 +++--- .../server/saved_objects/spaces_saved_objects_client.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/api_integration/apis/saved_objects/resolve_import_errors.ts b/test/api_integration/apis/saved_objects/resolve_import_errors.ts index b203a2c7b7071..b93f3a52d73d9 100644 --- a/test/api_integration/apis/saved_objects/resolve_import_errors.ts +++ b/test/api_integration/apis/saved_objects/resolve_import_errors.ts @@ -167,9 +167,9 @@ export default function ({ getService }: FtrProviderContext) { }); }); - it('should return 400 when resolving conflicts with a file containing more than 10,000 objects', async () => { + it('should return 400 when resolving conflicts with a file containing more than 10,001 objects', async () => { const fileChunks = []; - for (let i = 0; i < 10001; i++) { + for (let i = 0; i <= 10001; i++) { fileChunks.push(`{"type":"visualization","id":"${i}","attributes":{},"references":[]}`); } await supertest @@ -181,7 +181,7 @@ export default function ({ getService }: FtrProviderContext) { expect(resp.body).to.eql({ statusCode: 400, error: 'Bad Request', - message: "Can't import more than 10000 objects", + message: "Can't import more than 10001 objects", }); }); }); diff --git a/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts b/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts index af795a44ddf79..433f95d2b5cf6 100644 --- a/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts +++ b/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts @@ -411,7 +411,7 @@ export class SpacesSavedObjectsClient implements SavedObjectsClientContract { * @param {object} [options] - {@link SavedObjectsClosePointInTimeOptions} * @returns {promise} - { succeeded: boolean; num_freed: number } */ - async closePointInTime(id: string, options?: SavedObjectsClosePointInTimeOptions) { + async closePointInTime(id: string, options: SavedObjectsClosePointInTimeOptions = {}) { throwErrorIfNamespaceSpecified(options); return await this.client.closePointInTime(id, { ...options, From cd8be897a91656f2bd012ee81bc1ded0f9990ebe Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Tue, 9 Feb 2021 20:24:35 -0700 Subject: [PATCH 24/32] Throw if both preference and pit are provided to SO.find --- .../service/lib/repository.test.js | 7 +++++++ .../saved_objects/service/lib/repository.ts | 20 +++++++++---------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/core/server/saved_objects/service/lib/repository.test.js b/src/core/server/saved_objects/service/lib/repository.test.js index d46a37d8ffa76..e77143d13612f 100644 --- a/src/core/server/saved_objects/service/lib/repository.test.js +++ b/src/core/server/saved_objects/service/lib/repository.test.js @@ -2813,6 +2813,13 @@ describe('SavedObjectsRepository', () => { expect(client.search).not.toHaveBeenCalled(); }); + it(`throws when a preference is provided with pit`, async () => { + await expect( + savedObjectsRepository.find({ type: 'foo', pit: { id: 'abc123' }, preference: 'hi' }) + ).rejects.toThrowError('options.preference must be excluded when options.pit is used'); + expect(client.search).not.toHaveBeenCalled(); + }); + it(`throws when KQL filter syntax is invalid`, async () => { const findOpts = { namespaces: [namespace], diff --git a/src/core/server/saved_objects/service/lib/repository.ts b/src/core/server/saved_objects/service/lib/repository.ts index 371cdbeee1dcb..a48677b313147 100644 --- a/src/core/server/saved_objects/service/lib/repository.ts +++ b/src/core/server/saved_objects/service/lib/repository.ts @@ -712,12 +712,13 @@ export class SavedObjectsRepository { * Query field argument for more information * @property {integer} [options.page=1] * @property {integer} [options.perPage=20] - * @property {Array} [options.searchAfter] + * @property {Array} [options.searchAfter] * @property {string} [options.sortField] * @property {string} [options.sortOrder] * @property {Array} [options.fields] * @property {string} [options.namespace] * @property {object} [options.hasReference] - { type, id } + * @property {string} [options.pit] * @property {string} [options.preference] * @returns {promise} - { saved_objects: [{ id, type, version, attributes }], total, per_page, page } */ @@ -759,6 +760,10 @@ export class SavedObjectsRepository { throw SavedObjectsErrorHelpers.createBadRequestError( 'options.namespaces must be an empty array when options.typeToNamespacesMap is used' ); + } else if (preference?.length && pit) { + throw SavedObjectsErrorHelpers.createBadRequestError( + 'options.preference must be excluded when options.pit is used' + ); } const types = type @@ -794,19 +799,14 @@ export class SavedObjectsRepository { } const esOptions = { - // If `pit` is provided, we drop the `index` and `preference` as those are already - // associated with the PIT in ES, and will otherwise return a 400. - ...(pit - ? {} - : { - index: this.getIndicesForTypes(allowedTypes), - preference, - }), + // If `pit` is provided, we drop the `index`, otherwise ES returns 400. + ...(pit ? {} : { index: this.getIndicesForTypes(allowedTypes) }), // If `searchAfter` is provided, we drop `from` as it will not be used for pagination. ...(searchAfter ? {} : { from: perPage * (page - 1) }), - size: perPage, _source: includedFields(type, fields), + preference, rest_total_hits_as_int: true, + size: perPage, body: { seq_no_primary_term: true, ...getSearchDsl(this._mappings, this._registry, { From b5c6614fb025379813f6ab6ba262fa854ac3c834 Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Tue, 9 Feb 2021 20:45:47 -0700 Subject: [PATCH 25/32] Don't merge user-provided PIT inside findWithPointInTime. --- src/core/server/saved_objects/export/find_with_point_in_time.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/server/saved_objects/export/find_with_point_in_time.ts b/src/core/server/saved_objects/export/find_with_point_in_time.ts index 6668e2184e571..684f692e1d72b 100644 --- a/src/core/server/saved_objects/export/find_with_point_in_time.ts +++ b/src/core/server/saved_objects/export/find_with_point_in_time.ts @@ -165,7 +165,7 @@ export class FindWithPointInTime { sortOrder: 'desc', // Bump keep_alive by 2m on every new request to allow for the ES client // to make multiple retries in the event of a network failure. - ...(id ? { pit: { keepAlive: '2m', ...findOptions.pit, id } } : {}), + ...(id ? { pit: { id, keepAlive: '2m' } } : {}), ...(searchAfter ? { searchAfter } : {}), ...findOptions, }); From 710c886aa8541ff4e0d2b327e6e668213f6dd0da Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Tue, 9 Feb 2021 20:51:00 -0700 Subject: [PATCH 26/32] Revert unintentional change to migrations test. --- .../server/saved_objects/migrations/core/index_migrator.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/server/saved_objects/migrations/core/index_migrator.test.ts b/src/core/server/saved_objects/migrations/core/index_migrator.test.ts index 0413c1a44e160..0d1939231ce6c 100644 --- a/src/core/server/saved_objects/migrations/core/index_migrator.test.ts +++ b/src/core/server/saved_objects/migrations/core/index_migrator.test.ts @@ -28,7 +28,7 @@ describe('IndexMigrator', () => { log: loggingSystemMock.create().get(), mappingProperties: {}, pollInterval: 1, - scrollDuration: '2m', + scrollDuration: '1m', documentMigrator: { migrationVersion: {}, migrate: _.identity, From 4255b1eba0e78c1c5b172169a56e63ea4abf4e45 Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Tue, 9 Feb 2021 20:57:47 -0700 Subject: [PATCH 27/32] Fix typo in comments. --- .../saved_objects/service/lib/search_dsl/sorting_params.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/server/saved_objects/service/lib/search_dsl/sorting_params.ts b/src/core/server/saved_objects/service/lib/search_dsl/sorting_params.ts index 45a9473a5df0e..162fd95e3d68d 100644 --- a/src/core/server/saved_objects/service/lib/search_dsl/sorting_params.ts +++ b/src/core/server/saved_objects/service/lib/search_dsl/sorting_params.ts @@ -10,7 +10,7 @@ import Boom from '@hapi/boom'; import { getProperty, IndexMapping } from '../../../mappings'; // TODO: The plan is for ES to automatically add this tiebreaker when -// using PIT. We should remove this logic one that is resolved. +// using PIT. We should remove this logic once that is resolved. // https://github.com/elastic/elasticsearch/issues/56828 const ES_PROVIDED_TIEBREAKER = { _shard_doc: 'asc' }; From 740a8cde6a6590c8c0f9c28182f9fdb2aa23e779 Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Wed, 10 Feb 2021 10:40:38 -0700 Subject: [PATCH 28/32] Remove searchAfter and pit from SO.find on the client-side. --- ...kibana-plugin-core-public.doclinksstart.md | 2 +- ...gin-core-public.savedobjectsfindoptions.md | 2 ++ ...c.savedobjectsfindresponsepublic.pit_id.md | 11 ------ src/core/public/public.api.md | 2 -- .../saved_objects_client.test.ts | 7 ---- .../saved_objects/saved_objects_client.ts | 22 ++++-------- src/core/server/saved_objects/routes/find.ts | 13 ------- .../routes/integration_tests/find.test.ts | 35 ------------------- 8 files changed, 10 insertions(+), 84 deletions(-) delete mode 100644 docs/development/core/public/kibana-plugin-core-public.savedobjectsfindresponsepublic.pit_id.md diff --git a/docs/development/core/public/kibana-plugin-core-public.doclinksstart.md b/docs/development/core/public/kibana-plugin-core-public.doclinksstart.md index f4bce8b51ebb1..f1c046c51b728 100644 --- a/docs/development/core/public/kibana-plugin-core-public.doclinksstart.md +++ b/docs/development/core/public/kibana-plugin-core-public.doclinksstart.md @@ -17,5 +17,5 @@ export interface DocLinksStart | --- | --- | --- | | [DOC\_LINK\_VERSION](./kibana-plugin-core-public.doclinksstart.doc_link_version.md) | string | | | [ELASTIC\_WEBSITE\_URL](./kibana-plugin-core-public.doclinksstart.elastic_website_url.md) | string | | -| [links](./kibana-plugin-core-public.doclinksstart.links.md) | {
readonly dashboard: {
readonly guide: string;
readonly drilldowns: string;
readonly drilldownsTriggerPicker: string;
readonly urlDrilldownTemplateSyntax: string;
readonly urlDrilldownVariables: string;
};
readonly discover: Record<string, string>;
readonly filebeat: {
readonly base: string;
readonly installation: string;
readonly configuration: string;
readonly elasticsearchOutput: string;
readonly startup: string;
readonly exportedFields: string;
};
readonly auditbeat: {
readonly base: string;
};
readonly metricbeat: {
readonly base: string;
};
readonly enterpriseSearch: {
readonly base: string;
readonly appSearchBase: string;
readonly workplaceSearchBase: string;
};
readonly heartbeat: {
readonly base: string;
};
readonly logstash: {
readonly base: string;
};
readonly functionbeat: {
readonly base: string;
};
readonly winlogbeat: {
readonly base: string;
};
readonly aggs: {
readonly date_histogram: string;
readonly date_range: string;
readonly date_format_pattern: string;
readonly filter: string;
readonly filters: string;
readonly geohash_grid: string;
readonly histogram: string;
readonly ip_range: string;
readonly range: string;
readonly significant_terms: string;
readonly terms: string;
readonly avg: string;
readonly avg_bucket: string;
readonly max_bucket: string;
readonly min_bucket: string;
readonly sum_bucket: string;
readonly cardinality: string;
readonly count: string;
readonly cumulative_sum: string;
readonly derivative: string;
readonly geo_bounds: string;
readonly geo_centroid: string;
readonly max: string;
readonly median: string;
readonly min: string;
readonly moving_avg: string;
readonly percentile_ranks: string;
readonly serial_diff: string;
readonly std_dev: string;
readonly sum: string;
readonly top_hits: string;
};
readonly runtimeFields: string;
readonly scriptedFields: {
readonly scriptFields: string;
readonly scriptAggs: string;
readonly painless: string;
readonly painlessApi: string;
readonly painlessSyntax: string;
readonly luceneExpressions: string;
};
readonly indexPatterns: {
readonly loadingData: string;
readonly introduction: string;
};
readonly addData: string;
readonly kibana: string;
readonly elasticsearch: Record<string, string>;
readonly siem: {
readonly guide: string;
readonly gettingStarted: string;
};
readonly query: {
readonly eql: string;
readonly luceneQuerySyntax: string;
readonly queryDsl: string;
readonly kueryQuerySyntax: string;
};
readonly date: {
readonly dateMath: string;
};
readonly management: Record<string, string>;
readonly ml: Record<string, string>;
readonly transforms: Record<string, string>;
readonly visualize: Record<string, string>;
readonly apis: Readonly<{
createIndex: string;
createSnapshotLifecyclePolicy: string;
createRoleMapping: string;
createRoleMappingTemplates: string;
createApiKey: string;
createPipeline: string;
createTransformRequest: string;
executeWatchActionModes: string;
openIndex: string;
putComponentTemplate: string;
painlessExecute: string;
putComponentTemplateMetadata: string;
putWatch: string;
updateTransform: string;
}>;
readonly observability: Record<string, string>;
readonly alerting: Record<string, string>;
readonly maps: Record<string, string>;
readonly monitoring: Record<string, string>;
readonly security: Readonly<{
apiKeyServiceSettings: string;
clusterPrivileges: string;
elasticsearchSettings: string;
elasticsearchEnableSecurity: string;
indicesPrivileges: string;
kibanaTLS: string;
kibanaPrivileges: string;
mappingRoles: string;
mappingRolesFieldRules: string;
runAsPrivilege: string;
}>;
readonly watcher: Record<string, string>;
readonly ccs: Record<string, string>;
} | | +| [links](./kibana-plugin-core-public.doclinksstart.links.md) | {
readonly dashboard: {
readonly guide: string;
readonly drilldowns: string;
readonly drilldownsTriggerPicker: string;
readonly urlDrilldownTemplateSyntax: string;
readonly urlDrilldownVariables: string;
};
readonly discover: Record<string, string>;
readonly filebeat: {
readonly base: string;
readonly installation: string;
readonly configuration: string;
readonly elasticsearchOutput: string;
readonly elasticsearchModule: string;
readonly startup: string;
readonly exportedFields: string;
};
readonly auditbeat: {
readonly base: string;
};
readonly metricbeat: {
readonly base: string;
readonly configure: string;
readonly httpEndpoint: string;
readonly install: string;
readonly start: string;
};
readonly enterpriseSearch: {
readonly base: string;
readonly appSearchBase: string;
readonly workplaceSearchBase: string;
};
readonly heartbeat: {
readonly base: string;
};
readonly logstash: {
readonly base: string;
};
readonly functionbeat: {
readonly base: string;
};
readonly winlogbeat: {
readonly base: string;
};
readonly aggs: {
readonly date_histogram: string;
readonly date_range: string;
readonly date_format_pattern: string;
readonly filter: string;
readonly filters: string;
readonly geohash_grid: string;
readonly histogram: string;
readonly ip_range: string;
readonly range: string;
readonly significant_terms: string;
readonly terms: string;
readonly avg: string;
readonly avg_bucket: string;
readonly max_bucket: string;
readonly min_bucket: string;
readonly sum_bucket: string;
readonly cardinality: string;
readonly count: string;
readonly cumulative_sum: string;
readonly derivative: string;
readonly geo_bounds: string;
readonly geo_centroid: string;
readonly max: string;
readonly median: string;
readonly min: string;
readonly moving_avg: string;
readonly percentile_ranks: string;
readonly serial_diff: string;
readonly std_dev: string;
readonly sum: string;
readonly top_hits: string;
};
readonly runtimeFields: string;
readonly scriptedFields: {
readonly scriptFields: string;
readonly scriptAggs: string;
readonly painless: string;
readonly painlessApi: string;
readonly painlessSyntax: string;
readonly luceneExpressions: string;
};
readonly indexPatterns: {
readonly loadingData: string;
readonly introduction: string;
};
readonly addData: string;
readonly kibana: string;
readonly elasticsearch: Record<string, string>;
readonly siem: {
readonly guide: string;
readonly gettingStarted: string;
};
readonly query: {
readonly eql: string;
readonly luceneQuerySyntax: string;
readonly queryDsl: string;
readonly kueryQuerySyntax: string;
};
readonly date: {
readonly dateMath: string;
};
readonly management: Record<string, string>;
readonly ml: Record<string, string>;
readonly transforms: Record<string, string>;
readonly visualize: Record<string, string>;
readonly apis: Readonly<{
createIndex: string;
createSnapshotLifecyclePolicy: string;
createRoleMapping: string;
createRoleMappingTemplates: string;
createApiKey: string;
createPipeline: string;
createTransformRequest: string;
executeWatchActionModes: string;
indexExists: string;
openIndex: string;
putComponentTemplate: string;
painlessExecute: string;
putComponentTemplateMetadata: string;
putWatch: string;
updateTransform: string;
}>;
readonly observability: Record<string, string>;
readonly alerting: Record<string, string>;
readonly maps: Record<string, string>;
readonly monitoring: Record<string, string>;
readonly security: Readonly<{
apiKeyServiceSettings: string;
clusterPrivileges: string;
elasticsearchSettings: string;
elasticsearchEnableSecurity: string;
indicesPrivileges: string;
kibanaTLS: string;
kibanaPrivileges: string;
mappingRoles: string;
mappingRolesFieldRules: string;
runAsPrivilege: string;
}>;
readonly watcher: Record<string, string>;
readonly ccs: Record<string, string>;
} | | diff --git a/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.md b/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.md index 8bd87c2f6ea35..d084bd28941ac 100644 --- a/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.md +++ b/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.md @@ -23,9 +23,11 @@ export interface SavedObjectsFindOptions | [namespaces](./kibana-plugin-core-public.savedobjectsfindoptions.namespaces.md) | string[] | | | [page](./kibana-plugin-core-public.savedobjectsfindoptions.page.md) | number | | | [perPage](./kibana-plugin-core-public.savedobjectsfindoptions.perpage.md) | number | | +| [pit](./kibana-plugin-core-public.savedobjectsfindoptions.pit.md) | {
id: string;
keepAlive?: string;
} | Search against a specific Point In Time (PIT) that you've opened with savedObjects.openPointInTimeForType. | | [preference](./kibana-plugin-core-public.savedobjectsfindoptions.preference.md) | string | An optional ES preference value to be used for the query \* | | [rootSearchFields](./kibana-plugin-core-public.savedobjectsfindoptions.rootsearchfields.md) | string[] | The fields to perform the parsed query against. Unlike the searchFields argument, these are expected to be root fields and will not be modified. If used in conjunction with searchFields, both are concatenated together. | | [search](./kibana-plugin-core-public.savedobjectsfindoptions.search.md) | string | Search documents using the Elasticsearch Simple Query String syntax. See Elasticsearch Simple Query String query argument for more information | +| [searchAfter](./kibana-plugin-core-public.savedobjectsfindoptions.searchafter.md) | unknown[] | Use the sort values from the previous page to retrieve the next page of results. | | [searchFields](./kibana-plugin-core-public.savedobjectsfindoptions.searchfields.md) | string[] | The fields to perform the parsed query against. See Elasticsearch Simple Query String fields argument for more information | | [sortField](./kibana-plugin-core-public.savedobjectsfindoptions.sortfield.md) | string | | | [sortOrder](./kibana-plugin-core-public.savedobjectsfindoptions.sortorder.md) | string | | diff --git a/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindresponsepublic.pit_id.md b/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindresponsepublic.pit_id.md deleted file mode 100644 index f1228390bdbf0..0000000000000 --- a/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindresponsepublic.pit_id.md +++ /dev/null @@ -1,11 +0,0 @@ - - -[Home](./index.md) > [kibana-plugin-core-public](./kibana-plugin-core-public.md) > [SavedObjectsFindResponsePublic](./kibana-plugin-core-public.savedobjectsfindresponsepublic.md) > [pit\_id](./kibana-plugin-core-public.savedobjectsfindresponsepublic.pit_id.md) - -## SavedObjectsFindResponsePublic.pit\_id property - -Signature: - -```typescript -pit_id?: string; -``` diff --git a/src/core/public/public.api.md b/src/core/public/public.api.md index 04004bb549c6e..4ff3f97313081 100644 --- a/src/core/public/public.api.md +++ b/src/core/public/public.api.md @@ -1230,8 +1230,6 @@ export interface SavedObjectsFindResponsePublic extends SavedObject // (undocumented) perPage: number; // (undocumented) - pit_id?: string; - // (undocumented) total: number; } diff --git a/src/core/public/saved_objects/saved_objects_client.test.ts b/src/core/public/saved_objects/saved_objects_client.test.ts index 2ff02c1224870..14421c871fc2b 100644 --- a/src/core/public/saved_objects/saved_objects_client.test.ts +++ b/src/core/public/saved_objects/saved_objects_client.test.ts @@ -428,9 +428,7 @@ describe('SavedObjectsClient', () => { hasReference: { id: '1', type: 'reference' }, page: 10, perPage: 100, - pit: { id: 'abc', keepAlive: '1m' }, search: 'what is the meaning of life?|life', - searchAfter: [123, 'abc'], searchFields: ['title^5', 'body'], sortField: 'sort_field', type: 'index-pattern', @@ -452,12 +450,7 @@ describe('SavedObjectsClient', () => { "has_reference": "{\\"id\\":\\"1\\",\\"type\\":\\"reference\\"}", "page": 10, "per_page": 100, - "pit": "{\\"id\\":\\"abc\\",\\"keep_alive\\":\\"1m\\"}", "search": "what is the meaning of life?|life", - "search_after": Array [ - 123, - "abc", - ], "search_fields": Array [ "title^5", "body", diff --git a/src/core/public/saved_objects/saved_objects_client.ts b/src/core/public/saved_objects/saved_objects_client.ts index e735adbdac048..44466025de7e3 100644 --- a/src/core/public/saved_objects/saved_objects_client.ts +++ b/src/core/public/saved_objects/saved_objects_client.ts @@ -21,12 +21,14 @@ import { import { SimpleSavedObject } from './simple_saved_object'; import { HttpFetchOptions, HttpSetup } from '../http'; +type PromiseType> = T extends Promise ? U : never; + type SavedObjectsFindOptions = Omit< SavedObjectFindOptionsServer, - 'sortOrder' | 'rootSearchFields' | 'typeToNamespacesMap' + 'pit' | 'rootSearchFields' | 'searchAfter' | 'sortOrder' | 'typeToNamespacesMap' >; -type PromiseType> = T extends Promise ? U : never; +type SavedObjectsFindResponse = Omit>, 'pit_id'>; /** @public */ export interface SavedObjectsCreateOptions { @@ -105,7 +107,6 @@ export interface SavedObjectsFindResponsePublic extends SavedObject total: number; perPage: number; page: number; - pit_id?: string; } interface BatchQueueEntry { @@ -320,9 +321,7 @@ export class SavedObjectsClient { hasReferenceOperator: 'has_reference_operator', page: 'page', perPage: 'per_page', - pit: 'pit', search: 'search', - searchAfter: 'search_after', searchFields: 'search_fields', sortField: 'sort_field', type: 'type', @@ -337,30 +336,23 @@ export class SavedObjectsClient { any >; - // `has_references` and `pit` are structured objects. We need to stringify before sending, - // as `fetch` is not doing it implicitly. + // `has_references` is a structured object. we need to stringify it before sending it, as `fetch` + // is not doing it implicitly. if (query.has_reference) { query.has_reference = JSON.stringify(query.has_reference); } - if (query.pit) { - query.pit = JSON.stringify(renameKeys({ id: 'id', keepAlive: 'keep_alive' }, query.pit)); - } const request: ReturnType = this.savedObjectsFetch(path, { method: 'GET', query, }); return request.then((resp) => { - return renameKeys< - PromiseType>, - SavedObjectsFindResponsePublic - >( + return renameKeys( { saved_objects: 'savedObjects', total: 'total', per_page: 'perPage', page: 'page', - pit_id: 'pit_id', }, { ...resp, diff --git a/src/core/server/saved_objects/routes/find.ts b/src/core/server/saved_objects/routes/find.ts index 92550e4d4f3ad..6ba23747cf374 100644 --- a/src/core/server/saved_objects/routes/find.ts +++ b/src/core/server/saved_objects/routes/find.ts @@ -16,10 +16,6 @@ interface RouteDependencies { } export const registerFindRoute = (router: IRouter, { coreUsageData }: RouteDependencies) => { - const pitSchema = schema.object({ - id: schema.string(), - keep_alive: schema.maybe(schema.string()), - }); const referenceSchema = schema.object({ type: schema.string(), id: schema.string(), @@ -37,10 +33,6 @@ export const registerFindRoute = (router: IRouter, { coreUsageData }: RouteDepen page: schema.number({ min: 0, defaultValue: 1 }), type: schema.oneOf([schema.string(), schema.arrayOf(schema.string())]), search: schema.maybe(schema.string()), - search_after: schema.maybe( - schema.arrayOf(schema.oneOf([schema.string(), schema.number()])) - ), - pit: schema.maybe(pitSchema), default_search_operator: searchOperatorSchema, search_fields: schema.maybe( schema.oneOf([schema.string(), schema.arrayOf(schema.string())]) @@ -72,11 +64,6 @@ export const registerFindRoute = (router: IRouter, { coreUsageData }: RouteDepen page: query.page, type: Array.isArray(query.type) ? query.type : [query.type], search: query.search, - searchAfter: query.search_after, - pit: query.pit && { - id: query.pit.id, - keepAlive: query.pit.keep_alive, - }, defaultSearchOperator: query.default_search_operator, searchFields: typeof query.search_fields === 'string' ? [query.search_fields] : query.search_fields, diff --git a/src/core/server/saved_objects/routes/integration_tests/find.test.ts b/src/core/server/saved_objects/routes/integration_tests/find.test.ts index 58a86edc3414c..3bd2484c2e30f 100644 --- a/src/core/server/saved_objects/routes/integration_tests/find.test.ts +++ b/src/core/server/saved_objects/routes/integration_tests/find.test.ts @@ -223,41 +223,6 @@ describe('GET /api/saved_objects/_find', () => { ); }); - it('accepts the optional query parameter search_after', async () => { - const searchAfterValues = querystring.escape(JSON.stringify([1, 'a'])); - await supertest(httpSetup.server.listener) - .get(`/api/saved_objects/_find?type=foo&search_after=${searchAfterValues}`) - .expect(200); - - expect(savedObjectsClient.find).toHaveBeenCalledTimes(1); - - const options = savedObjectsClient.find.mock.calls[0][0]; - expect(options).toEqual( - expect.objectContaining({ - searchAfter: [1, 'a'], - }) - ); - }); - - it('accepts the optional query parameter pit', async () => { - const pitValues = querystring.escape(JSON.stringify({ id: 'abc', keep_alive: '2m' })); - await supertest(httpSetup.server.listener) - .get(`/api/saved_objects/_find?type=foo&pit=${pitValues}`) - .expect(200); - - expect(savedObjectsClient.find).toHaveBeenCalledTimes(1); - - const options = savedObjectsClient.find.mock.calls[0][0]; - expect(options).toEqual( - expect.objectContaining({ - pit: { - id: 'abc', - keepAlive: '2m', - }, - }) - ); - }); - it('accepts the query parameter fields as a string', async () => { await supertest(httpSetup.server.listener) .get('/api/saved_objects/_find?type=foo&fields=title') From 669ab9e9fc7144a8195771846acfda6828dc7f60 Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Wed, 10 Feb 2021 11:22:58 -0700 Subject: [PATCH 29/32] Address client/repository/types feedback. --- ...gin-core-public.savedobjectsfindoptions.md | 2 +- ...core-public.savedobjectsfindoptions.pit.md | 7 ++-- .../core/server/kibana-plugin-core-server.md | 1 + ...ver.savedobjectsclient.closepointintime.md | 2 +- ...a-plugin-core-server.savedobjectsclient.md | 4 +-- ...vedobjectsclient.openpointintimefortype.md | 2 +- ...gin-core-server.savedobjectsfindoptions.md | 2 +- ...core-server.savedobjectsfindoptions.pit.md | 7 ++-- ...core-server.savedobjectsfindresult.sort.md | 28 +++++++++++++++ ...in-core-server.savedobjectspitparams.id.md | 11 ++++++ ...-server.savedobjectspitparams.keepalive.md | 11 ++++++ ...lugin-core-server.savedobjectspitparams.md | 20 +++++++++++ ...bjectsrepository.openpointintimefortype.md | 25 +++++++------ ...-plugin-core-server.searchresponse.hits.md | 2 +- ...ibana-plugin-core-server.searchresponse.md | 2 +- src/core/public/public.api.md | 7 ++-- src/core/server/elasticsearch/client/types.ts | 2 +- src/core/server/index.ts | 1 + .../saved_objects/saved_objects_service.ts | 2 +- .../saved_objects/service/lib/repository.ts | 35 ++++++++++--------- .../service/lib/search_dsl/pit_params.ts | 5 +-- .../service/lib/search_dsl/search_dsl.ts | 3 +- .../service/lib/search_dsl/sorting_params.ts | 3 +- .../service/saved_objects_client.ts | 34 +++++++++++++++--- src/core/server/saved_objects/types.ts | 12 +++++-- src/core/server/server.api.md | 15 +++++--- src/core/server/types.ts | 1 + 27 files changed, 179 insertions(+), 67 deletions(-) create mode 100644 docs/development/core/server/kibana-plugin-core-server.savedobjectspitparams.id.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.savedobjectspitparams.keepalive.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.savedobjectspitparams.md diff --git a/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.md b/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.md index d084bd28941ac..69cfb818561e5 100644 --- a/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.md +++ b/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.md @@ -23,7 +23,7 @@ export interface SavedObjectsFindOptions | [namespaces](./kibana-plugin-core-public.savedobjectsfindoptions.namespaces.md) | string[] | | | [page](./kibana-plugin-core-public.savedobjectsfindoptions.page.md) | number | | | [perPage](./kibana-plugin-core-public.savedobjectsfindoptions.perpage.md) | number | | -| [pit](./kibana-plugin-core-public.savedobjectsfindoptions.pit.md) | {
id: string;
keepAlive?: string;
} | Search against a specific Point In Time (PIT) that you've opened with savedObjects.openPointInTimeForType. | +| [pit](./kibana-plugin-core-public.savedobjectsfindoptions.pit.md) | SavedObjectsPitParams | Search against a specific Point In Time (PIT) that you've opened with . | | [preference](./kibana-plugin-core-public.savedobjectsfindoptions.preference.md) | string | An optional ES preference value to be used for the query \* | | [rootSearchFields](./kibana-plugin-core-public.savedobjectsfindoptions.rootsearchfields.md) | string[] | The fields to perform the parsed query against. Unlike the searchFields argument, these are expected to be root fields and will not be modified. If used in conjunction with searchFields, both are concatenated together. | | [search](./kibana-plugin-core-public.savedobjectsfindoptions.search.md) | string | Search documents using the Elasticsearch Simple Query String syntax. See Elasticsearch Simple Query String query argument for more information | diff --git a/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.pit.md b/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.pit.md index df8b8de7d96d6..2284a4d8d210d 100644 --- a/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.pit.md +++ b/docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.pit.md @@ -4,13 +4,10 @@ ## SavedObjectsFindOptions.pit property -Search against a specific Point In Time (PIT) that you've opened with `savedObjects.openPointInTimeForType`. +Search against a specific Point In Time (PIT) that you've opened with . Signature: ```typescript -pit?: { - id: string; - keepAlive?: string; - }; +pit?: SavedObjectsPitParams; ``` diff --git a/docs/development/core/server/kibana-plugin-core-server.md b/docs/development/core/server/kibana-plugin-core-server.md index a1cfaf5b4725c..1791335d58fef 100644 --- a/docs/development/core/server/kibana-plugin-core-server.md +++ b/docs/development/core/server/kibana-plugin-core-server.md @@ -191,6 +191,7 @@ The plugin integrates with the core system via lifecycle events: `setup` | [SavedObjectsMigrationVersion](./kibana-plugin-core-server.savedobjectsmigrationversion.md) | Information about the migrations that have been applied to this SavedObject. When Kibana starts up, KibanaMigrator detects outdated documents and migrates them based on this value. For each migration that has been applied, the plugin's name is used as a key and the latest migration version as the value. | | [SavedObjectsOpenPointInTimeOptions](./kibana-plugin-core-server.savedobjectsopenpointintimeoptions.md) | | | [SavedObjectsOpenPointInTimeResponse](./kibana-plugin-core-server.savedobjectsopenpointintimeresponse.md) | | +| [SavedObjectsPitParams](./kibana-plugin-core-server.savedobjectspitparams.md) | | | [SavedObjectsRawDoc](./kibana-plugin-core-server.savedobjectsrawdoc.md) | A raw document as represented directly in the saved object index. | | [SavedObjectsRawDocParseOptions](./kibana-plugin-core-server.savedobjectsrawdocparseoptions.md) | Options that can be specified when using the saved objects serializer to parse a raw document. | | [SavedObjectsRemoveReferencesToOptions](./kibana-plugin-core-server.savedobjectsremovereferencestooptions.md) | | diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.closepointintime.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.closepointintime.md index f6de3b9ba24cb..dc765260a08ca 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.closepointintime.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.closepointintime.md @@ -4,7 +4,7 @@ ## SavedObjectsClient.closePointInTime() method -Closes a Point In Time (PIT) by ID. This simply proxies the request to ES via the Elasticsearch client, and is included in the Saved Objects Client as a convenience for consumers who are using `openPointInTimeForType`. +Closes a Point In Time (PIT) by ID. This simply proxies the request to ES via the Elasticsearch client, and is included in the Saved Objects Client as a convenience for consumers who are using [SavedObjectsClient.openPointInTimeForType()](./kibana-plugin-core-server.savedobjectsclient.openpointintimefortype.md). Signature: diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.md index 787a0e1b653d1..887f7f7d93a87 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.md @@ -30,13 +30,13 @@ The constructor for this class is marked as internal. Third-party code should no | [bulkGet(objects, options)](./kibana-plugin-core-server.savedobjectsclient.bulkget.md) | | Returns an array of objects by id | | [bulkUpdate(objects, options)](./kibana-plugin-core-server.savedobjectsclient.bulkupdate.md) | | Bulk Updates multiple SavedObject at once | | [checkConflicts(objects, options)](./kibana-plugin-core-server.savedobjectsclient.checkconflicts.md) | | Check what conflicts will result when creating a given array of saved objects. This includes "unresolvable conflicts", which are multi-namespace objects that exist in a different namespace; such conflicts cannot be resolved/overwritten. | -| [closePointInTime(id, options)](./kibana-plugin-core-server.savedobjectsclient.closepointintime.md) | | Closes a Point In Time (PIT) by ID. This simply proxies the request to ES via the Elasticsearch client, and is included in the Saved Objects Client as a convenience for consumers who are using openPointInTimeForType. | +| [closePointInTime(id, options)](./kibana-plugin-core-server.savedobjectsclient.closepointintime.md) | | Closes a Point In Time (PIT) by ID. This simply proxies the request to ES via the Elasticsearch client, and is included in the Saved Objects Client as a convenience for consumers who are using [SavedObjectsClient.openPointInTimeForType()](./kibana-plugin-core-server.savedobjectsclient.openpointintimefortype.md). | | [create(type, attributes, options)](./kibana-plugin-core-server.savedobjectsclient.create.md) | | Persists a SavedObject | | [delete(type, id, options)](./kibana-plugin-core-server.savedobjectsclient.delete.md) | | Deletes a SavedObject | | [deleteFromNamespaces(type, id, namespaces, options)](./kibana-plugin-core-server.savedobjectsclient.deletefromnamespaces.md) | | Removes namespaces from a SavedObject | | [find(options)](./kibana-plugin-core-server.savedobjectsclient.find.md) | | Find all SavedObjects matching the search query | | [get(type, id, options)](./kibana-plugin-core-server.savedobjectsclient.get.md) | | Retrieves a single object | -| [openPointInTimeForType(type, options)](./kibana-plugin-core-server.savedobjectsclient.openpointintimefortype.md) | | Opens a Point In Time (PIT) against the indices for the specified Saved Object types. The returned id can then be passed to SavedObjects.find to search against that PIT. | +| [openPointInTimeForType(type, options)](./kibana-plugin-core-server.savedobjectsclient.openpointintimefortype.md) | | Opens a Point In Time (PIT) against the indices for the specified Saved Object types. The returned id can then be passed to [SavedObjectsClient.find()](./kibana-plugin-core-server.savedobjectsclient.find.md) to search against that PIT. | | [removeReferencesTo(type, id, options)](./kibana-plugin-core-server.savedobjectsclient.removereferencesto.md) | | Updates all objects containing a reference to the given {type, id} tuple to remove the said reference. | | [resolve(type, id, options)](./kibana-plugin-core-server.savedobjectsclient.resolve.md) | | Resolves a single object, using any legacy URL alias if it exists | | [update(type, id, attributes, options)](./kibana-plugin-core-server.savedobjectsclient.update.md) | | Updates an SavedObject | diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.openpointintimefortype.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.openpointintimefortype.md index c7dc50f9cd066..56c1d6d1ddc33 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.openpointintimefortype.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsclient.openpointintimefortype.md @@ -4,7 +4,7 @@ ## SavedObjectsClient.openPointInTimeForType() method -Opens a Point In Time (PIT) against the indices for the specified Saved Object types. The returned `id` can then be passed to `SavedObjects.find` to search against that PIT. +Opens a Point In Time (PIT) against the indices for the specified Saved Object types. The returned `id` can then be passed to [SavedObjectsClient.find()](./kibana-plugin-core-server.savedobjectsclient.find.md) to search against that PIT. Signature: diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.md index cf97d45066e70..6f7c05ea469bc 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.md @@ -23,7 +23,7 @@ export interface SavedObjectsFindOptions | [namespaces](./kibana-plugin-core-server.savedobjectsfindoptions.namespaces.md) | string[] | | | [page](./kibana-plugin-core-server.savedobjectsfindoptions.page.md) | number | | | [perPage](./kibana-plugin-core-server.savedobjectsfindoptions.perpage.md) | number | | -| [pit](./kibana-plugin-core-server.savedobjectsfindoptions.pit.md) | {
id: string;
keepAlive?: string;
} | Search against a specific Point In Time (PIT) that you've opened with savedObjects.openPointInTimeForType. | +| [pit](./kibana-plugin-core-server.savedobjectsfindoptions.pit.md) | SavedObjectsPitParams | Search against a specific Point In Time (PIT) that you've opened with [SavedObjectsClient.openPointInTimeForType()](./kibana-plugin-core-server.savedobjectsclient.openpointintimefortype.md). | | [preference](./kibana-plugin-core-server.savedobjectsfindoptions.preference.md) | string | An optional ES preference value to be used for the query \* | | [rootSearchFields](./kibana-plugin-core-server.savedobjectsfindoptions.rootsearchfields.md) | string[] | The fields to perform the parsed query against. Unlike the searchFields argument, these are expected to be root fields and will not be modified. If used in conjunction with searchFields, both are concatenated together. | | [search](./kibana-plugin-core-server.savedobjectsfindoptions.search.md) | string | Search documents using the Elasticsearch Simple Query String syntax. See Elasticsearch Simple Query String query argument for more information | diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.pit.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.pit.md index 2c6b280f01c5b..fac333227088c 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.pit.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.pit.md @@ -4,13 +4,10 @@ ## SavedObjectsFindOptions.pit property -Search against a specific Point In Time (PIT) that you've opened with `savedObjects.openPointInTimeForType`. +Search against a specific Point In Time (PIT) that you've opened with [SavedObjectsClient.openPointInTimeForType()](./kibana-plugin-core-server.savedobjectsclient.openpointintimefortype.md). Signature: ```typescript -pit?: { - id: string; - keepAlive?: string; - }; +pit?: SavedObjectsPitParams; ``` diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresult.sort.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresult.sort.md index 2f85403e56a38..3cc02c404c8d7 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresult.sort.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresult.sort.md @@ -11,3 +11,31 @@ The Elasticsearch `sort` value of this result. ```typescript sort?: unknown[]; ``` + +## Remarks + +This can be passed directly to the `searchAfter` param in the [SavedObjectsFindOptions](./kibana-plugin-core-server.savedobjectsfindoptions.md) in order to page through large numbers of hits. It is recommended you use this alongside a Point In Time (PIT) that was opened with [SavedObjectsClient.openPointInTimeForType()](./kibana-plugin-core-server.savedobjectsclient.openpointintimefortype.md). + +## Example + + +```ts +const { id } = await savedObjectsClient.openPointInTimeForType('visualization'); +const page1 = await savedObjectsClient.find({ + type: 'visualization', + sortField: 'updated_at', + sortOrder: 'asc', + pit, +}); +const lastHit = page1.saved_objects[page1.saved_objects.length - 1]; +const page2 = await savedObjectsClient.find({ + type: 'visualization', + sortField: 'updated_at', + sortOrder: 'asc', + pit: { id: page1.pit_id }, + searchAfter: lastHit.sort, +}); +await savedObjectsClient.closePointInTime(page2.pit_id); + +``` + diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectspitparams.id.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectspitparams.id.md new file mode 100644 index 0000000000000..cb4d4a65727d7 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectspitparams.id.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsPitParams](./kibana-plugin-core-server.savedobjectspitparams.md) > [id](./kibana-plugin-core-server.savedobjectspitparams.id.md) + +## SavedObjectsPitParams.id property + +Signature: + +```typescript +id: string; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectspitparams.keepalive.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectspitparams.keepalive.md new file mode 100644 index 0000000000000..1011a908f210a --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectspitparams.keepalive.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsPitParams](./kibana-plugin-core-server.savedobjectspitparams.md) > [keepAlive](./kibana-plugin-core-server.savedobjectspitparams.keepalive.md) + +## SavedObjectsPitParams.keepAlive property + +Signature: + +```typescript +keepAlive?: string; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectspitparams.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectspitparams.md new file mode 100644 index 0000000000000..7bffca7cda281 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectspitparams.md @@ -0,0 +1,20 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsPitParams](./kibana-plugin-core-server.savedobjectspitparams.md) + +## SavedObjectsPitParams interface + + +Signature: + +```typescript +export interface SavedObjectsPitParams +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [id](./kibana-plugin-core-server.savedobjectspitparams.id.md) | string | | +| [keepAlive](./kibana-plugin-core-server.savedobjectspitparams.keepalive.md) | string | | + diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.openpointintimefortype.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.openpointintimefortype.md index 11e356f845b31..63956ebee68f7 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.openpointintimefortype.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsrepository.openpointintimefortype.md @@ -35,18 +35,23 @@ const { id } = await repository.openPointInTimeForType( type: 'index-pattern', { keepAlive: '2m' }, ); +const page1 = await savedObjectsClient.find({ + type: 'visualization', + sortField: 'updated_at', + sortOrder: 'asc', + pit, +}); -const response = await repository.find({ - type: 'index-pattern', - search: 'foo*', - sortField: 'name', - sortOrder: 'desc', - pit: { - id: 'abc123', - keepAlive: '2m', - }, - searchAfter: [1234, 'abcd'], +const lastHit = page1.saved_objects[page1.saved_objects.length - 1]; +const page2 = await savedObjectsClient.find({ + type: 'visualization', + sortField: 'updated_at', + sortOrder: 'asc', + pit: { id: page1.pit_id }, + searchAfter: lastHit.sort, }); +await savedObjectsClient.closePointInTime(page2.pit_id); + ``` diff --git a/docs/development/core/server/kibana-plugin-core-server.searchresponse.hits.md b/docs/development/core/server/kibana-plugin-core-server.searchresponse.hits.md index 1629e77425525..599c4e3ad6319 100644 --- a/docs/development/core/server/kibana-plugin-core-server.searchresponse.hits.md +++ b/docs/development/core/server/kibana-plugin-core-server.searchresponse.hits.md @@ -22,7 +22,7 @@ hits: { highlight?: any; inner_hits?: any; matched_queries?: string[]; - sort?: string[]; + sort?: unknown[]; }>; }; ``` diff --git a/docs/development/core/server/kibana-plugin-core-server.searchresponse.md b/docs/development/core/server/kibana-plugin-core-server.searchresponse.md index e7ec6f96ada65..cbaab4632014d 100644 --- a/docs/development/core/server/kibana-plugin-core-server.searchresponse.md +++ b/docs/development/core/server/kibana-plugin-core-server.searchresponse.md @@ -18,7 +18,7 @@ export interface SearchResponse | [\_scroll\_id](./kibana-plugin-core-server.searchresponse._scroll_id.md) | string | | | [\_shards](./kibana-plugin-core-server.searchresponse._shards.md) | ShardsResponse | | | [aggregations](./kibana-plugin-core-server.searchresponse.aggregations.md) | any | | -| [hits](./kibana-plugin-core-server.searchresponse.hits.md) | {
total: number;
max_score: number;
hits: Array<{
_index: string;
_type: string;
_id: string;
_score: number;
_source: T;
_version?: number;
_explanation?: Explanation;
fields?: any;
highlight?: any;
inner_hits?: any;
matched_queries?: string[];
sort?: string[];
}>;
} | | +| [hits](./kibana-plugin-core-server.searchresponse.hits.md) | {
total: number;
max_score: number;
hits: Array<{
_index: string;
_type: string;
_id: string;
_score: number;
_source: T;
_version?: number;
_explanation?: Explanation;
fields?: any;
highlight?: any;
inner_hits?: any;
matched_queries?: string[];
sort?: unknown[];
}>;
} | | | [pit\_id](./kibana-plugin-core-server.searchresponse.pit_id.md) | string | | | [timed\_out](./kibana-plugin-core-server.searchresponse.timed_out.md) | boolean | | | [took](./kibana-plugin-core-server.searchresponse.took.md) | number | | diff --git a/src/core/public/public.api.md b/src/core/public/public.api.md index 4ff3f97313081..21e970d203dda 100644 --- a/src/core/public/public.api.md +++ b/src/core/public/public.api.md @@ -1197,10 +1197,9 @@ export interface SavedObjectsFindOptions { page?: number; // (undocumented) perPage?: number; - pit?: { - id: string; - keepAlive?: string; - }; + // Warning: (ae-forgotten-export) The symbol "SavedObjectsPitParams" needs to be exported by the entry point index.d.ts + // Warning: (ae-unresolved-link) The @link reference could not be resolved: No member was found with name "openPointInTimeForType" + pit?: SavedObjectsPitParams; preference?: string; rootSearchFields?: string[]; search?: string; diff --git a/src/core/server/elasticsearch/client/types.ts b/src/core/server/elasticsearch/client/types.ts index 11521becb46af..f5a6fa1f0b1fd 100644 --- a/src/core/server/elasticsearch/client/types.ts +++ b/src/core/server/elasticsearch/client/types.ts @@ -96,7 +96,7 @@ export interface SearchResponse { highlight?: any; inner_hits?: any; matched_queries?: string[]; - sort?: string[]; + sort?: unknown[]; }>; }; aggregations?: any; diff --git a/src/core/server/index.ts b/src/core/server/index.ts index e30d9019e6236..dac2d210eb395 100644 --- a/src/core/server/index.ts +++ b/src/core/server/index.ts @@ -377,6 +377,7 @@ export { SavedObjectsClientContract, SavedObjectsFindOptions, SavedObjectsFindOptionsReference, + SavedObjectsPitParams, SavedObjectsMigrationVersion, } from './types'; diff --git a/src/core/server/saved_objects/saved_objects_service.ts b/src/core/server/saved_objects/saved_objects_service.ts index 6d416ac2cdd7f..fce7f12384456 100644 --- a/src/core/server/saved_objects/saved_objects_service.ts +++ b/src/core/server/saved_objects/saved_objects_service.ts @@ -459,7 +459,7 @@ export class SavedObjectsService savedObjectsClient, typeRegistry: this.typeRegistry, exportSizeLimit: this.config!.maxImportExportSize, - logger: this.logger, + logger: this.logger.get('exporter'), }), createImporter: (savedObjectsClient) => new SavedObjectsImporter({ diff --git a/src/core/server/saved_objects/service/lib/repository.ts b/src/core/server/saved_objects/service/lib/repository.ts index a48677b313147..3bb274936c32b 100644 --- a/src/core/server/saved_objects/service/lib/repository.ts +++ b/src/core/server/saved_objects/service/lib/repository.ts @@ -850,10 +850,10 @@ export class SavedObjectsRepository { (hit: SavedObjectsRawDoc): SavedObjectsFindResult => ({ ...this._rawToSavedObject(hit), score: (hit as any)._score, - ...((hit as any).sort ? { sort: (hit as any).sort } : {}), + ...((hit as any).sort && { sort: (hit as any).sort }), }) ), - ...(body.pit_id ? { pit_id: body.pit_id } : {}), + ...(body.pit_id && { pit_id: body.pit_id }), } as SavedObjectsFindResponse; } @@ -1794,18 +1794,23 @@ export class SavedObjectsRepository { * type: 'index-pattern', * { keepAlive: '2m' }, * ); + * const page1 = await savedObjectsClient.find({ + * type: 'visualization', + * sortField: 'updated_at', + * sortOrder: 'asc', + * pit, + * }); * - * const response = await repository.find({ - * type: 'index-pattern', - * search: 'foo*', - * sortField: 'name', - * sortOrder: 'desc', - * pit: { - * id: 'abc123', - * keepAlive: '2m', - * }, - * searchAfter: [1234, 'abcd'], + * const lastHit = page1.saved_objects[page1.saved_objects.length - 1]; + * const page2 = await savedObjectsClient.find({ + * type: 'visualization', + * sortField: 'updated_at', + * sortOrder: 'asc', + * pit: { id: page1.pit_id }, + * searchAfter: lastHit.sort, * }); + * + * await savedObjectsClient.closePointInTime(page2.pit_id); * ``` * * @param {string|Array} type @@ -1816,10 +1821,8 @@ export class SavedObjectsRepository { */ async openPointInTimeForType( type: string | string[], - { keepAlive, preference }: SavedObjectsOpenPointInTimeOptions = {} + { keepAlive = '5m', preference }: SavedObjectsOpenPointInTimeOptions = {} ): Promise { - const defaultKeepAlive = '5m'; - const types = Array.isArray(type) ? type : [type]; const allowedTypes = types.filter((t) => this._allowedTypes.includes(t)); if (allowedTypes.length === 0) { @@ -1828,7 +1831,7 @@ export class SavedObjectsRepository { const esOptions = { index: this.getIndicesForTypes(allowedTypes), - keep_alive: keepAlive || defaultKeepAlive, + keep_alive: keepAlive, ...(preference ? { preference } : {}), }; diff --git a/src/core/server/saved_objects/service/lib/search_dsl/pit_params.ts b/src/core/server/saved_objects/service/lib/search_dsl/pit_params.ts index 1f06df4f3329d..1a8dcb5cca2e9 100644 --- a/src/core/server/saved_objects/service/lib/search_dsl/pit_params.ts +++ b/src/core/server/saved_objects/service/lib/search_dsl/pit_params.ts @@ -6,10 +6,7 @@ * Side Public License, v 1. */ -interface SavedObjectsPitParams { - id: string; - keepAlive?: string; -} +import { SavedObjectsPitParams } from '../../../types'; export function getPitParams(pit: SavedObjectsPitParams) { return { diff --git a/src/core/server/saved_objects/service/lib/search_dsl/search_dsl.ts b/src/core/server/saved_objects/service/lib/search_dsl/search_dsl.ts index ca47a0c2bd6e3..cae5e43897bcf 100644 --- a/src/core/server/saved_objects/service/lib/search_dsl/search_dsl.ts +++ b/src/core/server/saved_objects/service/lib/search_dsl/search_dsl.ts @@ -9,6 +9,7 @@ import Boom from '@hapi/boom'; import { IndexMapping } from '../../../mappings'; +import { SavedObjectsPitParams } from '../../../types'; import { getQueryParams, HasReferenceQueryParams, SearchOperator } from './query_params'; import { getPitParams } from './pit_params'; import { getSortingParams } from './sorting_params'; @@ -26,7 +27,7 @@ interface GetSearchDslOptions { sortField?: string; sortOrder?: string; namespaces?: string[]; - pit?: { id: string; keepAlive?: string }; + pit?: SavedObjectsPitParams; typeToNamespacesMap?: Map; hasReference?: HasReferenceQueryParams | HasReferenceQueryParams[]; hasReferenceOperator?: SearchOperator; diff --git a/src/core/server/saved_objects/service/lib/search_dsl/sorting_params.ts b/src/core/server/saved_objects/service/lib/search_dsl/sorting_params.ts index 162fd95e3d68d..abef9bfa0a300 100644 --- a/src/core/server/saved_objects/service/lib/search_dsl/sorting_params.ts +++ b/src/core/server/saved_objects/service/lib/search_dsl/sorting_params.ts @@ -8,6 +8,7 @@ import Boom from '@hapi/boom'; import { getProperty, IndexMapping } from '../../../mappings'; +import { SavedObjectsPitParams } from '../../../types'; // TODO: The plan is for ES to automatically add this tiebreaker when // using PIT. We should remove this logic once that is resolved. @@ -21,7 +22,7 @@ export function getSortingParams( type: string | string[], sortField?: string, sortOrder?: string, - pit?: { id: string; keepAlive?: string } + pit?: SavedObjectsPitParams ) { if (!sortField) { return {}; diff --git a/src/core/server/saved_objects/service/saved_objects_client.ts b/src/core/server/saved_objects/service/saved_objects_client.ts index 4cd5316d78f73..7b7ef36e1cb0c 100644 --- a/src/core/server/saved_objects/service/saved_objects_client.ts +++ b/src/core/server/saved_objects/service/saved_objects_client.ts @@ -131,6 +131,31 @@ export interface SavedObjectsFindResult extends SavedObject { score: number; /** * The Elasticsearch `sort` value of this result. + * + * @remarks + * This can be passed directly to the `searchAfter` param in the {@link SavedObjectsFindOptions} + * in order to page through large numbers of hits. It is recommended you use this alongside + * a Point In Time (PIT) that was opened with {@link SavedObjectsClient.openPointInTimeForType}. + * + * @example + * ```ts + * const { id } = await savedObjectsClient.openPointInTimeForType('visualization'); + * const page1 = await savedObjectsClient.find({ + * type: 'visualization', + * sortField: 'updated_at', + * sortOrder: 'asc', + * pit, + * }); + * const lastHit = page1.saved_objects[page1.saved_objects.length - 1]; + * const page2 = await savedObjectsClient.find({ + * type: 'visualization', + * sortField: 'updated_at', + * sortOrder: 'asc', + * pit: { id: page1.pit_id }, + * searchAfter: lastHit.sort, + * }); + * await savedObjectsClient.closePointInTime(page2.pit_id); + * ``` */ sort?: unknown[]; } @@ -556,7 +581,8 @@ export class SavedObjectsClient { /** * Opens a Point In Time (PIT) against the indices for the specified Saved Object types. - * The returned `id` can then be passed to `SavedObjects.find` to search against that PIT. + * The returned `id` can then be passed to {@link SavedObjectsClient.find} to search + * against that PIT. */ async openPointInTimeForType( type: string | string[], @@ -566,9 +592,9 @@ export class SavedObjectsClient { } /** - * Closes a Point In Time (PIT) by ID. This simply proxies the request to ES - * via the Elasticsearch client, and is included in the Saved Objects Client - * as a convenience for consumers who are using `openPointInTimeForType`. + * Closes a Point In Time (PIT) by ID. This simply proxies the request to ES via the + * Elasticsearch client, and is included in the Saved Objects Client as a convenience + * for consumers who are using {@link SavedObjectsClient.openPointInTimeForType}. */ async closePointInTime(id: string, options?: SavedObjectsClosePointInTimeOptions) { return await this._repository.closePointInTime(id, options); diff --git a/src/core/server/saved_objects/types.ts b/src/core/server/saved_objects/types.ts index 739bf5f78bb51..66110d096213f 100644 --- a/src/core/server/saved_objects/types.ts +++ b/src/core/server/saved_objects/types.ts @@ -62,6 +62,14 @@ export interface SavedObjectsFindOptionsReference { id: string; } +/** + * @public + */ +export interface SavedObjectsPitParams { + id: string; + keepAlive?: string; +} + /** * * @public @@ -119,9 +127,9 @@ export interface SavedObjectsFindOptions { /** An optional ES preference value to be used for the query **/ preference?: string; /** - * Search against a specific Point In Time (PIT) that you've opened with `savedObjects.openPointInTimeForType`. + * Search against a specific Point In Time (PIT) that you've opened with {@link SavedObjectsClient.openPointInTimeForType}. */ - pit?: { id: string; keepAlive?: string }; + pit?: SavedObjectsPitParams; } /** diff --git a/src/core/server/server.api.md b/src/core/server/server.api.md index 8c55f1fe873f6..ca94a75d89f89 100644 --- a/src/core/server/server.api.md +++ b/src/core/server/server.api.md @@ -2492,10 +2492,7 @@ export interface SavedObjectsFindOptions { page?: number; // (undocumented) perPage?: number; - pit?: { - id: string; - keepAlive?: string; - }; + pit?: SavedObjectsPitParams; preference?: string; rootSearchFields?: string[]; search?: string; @@ -2773,6 +2770,14 @@ export interface SavedObjectsOpenPointInTimeResponse { id: string; } +// @public (undocumented) +export interface SavedObjectsPitParams { + // (undocumented) + id: string; + // (undocumented) + keepAlive?: string; +} + // @public export interface SavedObjectsRawDoc { // (undocumented) @@ -2987,7 +2992,7 @@ export interface SearchResponse { highlight?: any; inner_hits?: any; matched_queries?: string[]; - sort?: string[]; + sort?: unknown[]; }>; }; // (undocumented) diff --git a/src/core/server/types.ts b/src/core/server/types.ts index 1839ee68190aa..2ae51d4452a4e 100644 --- a/src/core/server/types.ts +++ b/src/core/server/types.ts @@ -31,6 +31,7 @@ export type { SavedObjectStatusMeta, SavedObjectsFindOptionsReference, SavedObjectsFindOptions, + SavedObjectsPitParams, SavedObjectsBaseOptions, MutatingOperationRefreshSetting, SavedObjectsClientContract, From 199e29f5d492526adac0619210dc052689358d59 Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Wed, 10 Feb 2021 12:55:34 -0700 Subject: [PATCH 30/32] Address security feedback. --- docs/user/security/audit-logging.asciidoc | 8 +++ ...ypted_saved_objects_client_wrapper.test.ts | 62 +++++++++++++++++++ ...ecure_saved_objects_client_wrapper.test.ts | 11 ++++ .../secure_saved_objects_client_wrapper.ts | 15 +++++ 4 files changed, 96 insertions(+) diff --git a/docs/user/security/audit-logging.asciidoc b/docs/user/security/audit-logging.asciidoc index 12a87b1422c5c..b9fc0c9c4ac46 100644 --- a/docs/user/security/audit-logging.asciidoc +++ b/docs/user/security/audit-logging.asciidoc @@ -85,6 +85,10 @@ Refer to the corresponding {es} logs for potential write errors. | `unknown` | User is creating a saved object. | `failure` | User is not authorized to create a saved object. +.2+| `saved_object_open_point_in_time` +| `unknown` | User is creating a Point In Time to use when querying saved objects. +| `failure` | User is not authorized to create a Point In Time for the provided saved object types. + .2+| `connector_create` | `unknown` | User is creating a connector. | `failure` | User is not authorized to create a connector. @@ -171,6 +175,10 @@ Refer to the corresponding {es} logs for potential write errors. | `unknown` | User is deleting a saved object. | `failure` | User is not authorized to delete a saved object. +.2+| `saved_object_close_point_in_time` +| `unknown` | User is deleting a Point In Time that was used to query saved objects. +| `failure` | User is not authorized to delete a Point In Time. + .2+| `connector_delete` | `unknown` | User is deleting a connector. | `failure` | User is not authorized to delete a connector. diff --git a/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.test.ts b/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.test.ts index 3405f196960cd..474a283b5e3cb 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.test.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.test.ts @@ -1757,3 +1757,65 @@ describe('#removeReferencesTo', () => { expect(mockBaseClient.removeReferencesTo).toHaveBeenCalledTimes(1); }); }); + +describe('#openPointInTimeForType', () => { + it('redirects request to underlying base client', async () => { + const options = { keepAlive: '1m' }; + + await wrapper.openPointInTimeForType('some-type', options); + + expect(mockBaseClient.openPointInTimeForType).toHaveBeenCalledTimes(1); + expect(mockBaseClient.openPointInTimeForType).toHaveBeenCalledWith('some-type', options); + }); + + it('returns response from underlying client', async () => { + const returnValue = { + id: 'abc123', + }; + mockBaseClient.openPointInTimeForType.mockResolvedValue(returnValue); + + const result = await wrapper.openPointInTimeForType('known-type'); + + expect(result).toBe(returnValue); + }); + + it('fails if base client fails', async () => { + const failureReason = new Error('Something bad happened...'); + mockBaseClient.openPointInTimeForType.mockRejectedValue(failureReason); + + await expect(wrapper.openPointInTimeForType('known-type')).rejects.toThrowError(failureReason); + + expect(mockBaseClient.openPointInTimeForType).toHaveBeenCalledTimes(1); + }); +}); + +describe('#closePointInTime', () => { + it('redirects request to underlying base client', async () => { + const id = 'abc123'; + await wrapper.closePointInTime(id); + + expect(mockBaseClient.closePointInTime).toHaveBeenCalledTimes(1); + expect(mockBaseClient.closePointInTime).toHaveBeenCalledWith(id, undefined); + }); + + it('returns response from underlying client', async () => { + const returnValue = { + succeeded: true, + num_freed: 1, + }; + mockBaseClient.closePointInTime.mockResolvedValue(returnValue); + + const result = await wrapper.closePointInTime('abc123'); + + expect(result).toBe(returnValue); + }); + + it('fails if base client fails', async () => { + const failureReason = new Error('Something bad happened...'); + mockBaseClient.closePointInTime.mockRejectedValue(failureReason); + + await expect(wrapper.closePointInTime('abc123')).rejects.toThrowError(failureReason); + + expect(mockBaseClient.closePointInTime).toHaveBeenCalledTimes(1); + }); +}); diff --git a/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.test.ts b/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.test.ts index 9e4f9fafda970..1293d3f2c84a3 100644 --- a/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.test.ts +++ b/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.test.ts @@ -905,6 +905,17 @@ describe('#find', () => { ); }); + test(`throws BadRequestError when searching across namespaces when pit is provided`, async () => { + const options = { + type: [type1, type2], + pit: { id: 'abc123' }, + namespaces: ['some-ns', 'another-ns'], + }; + await expect(client.find(options)).rejects.toThrowErrorMatchingInlineSnapshot( + `"_find across namespaces is not permitted when using the \`pit\` option."` + ); + }); + test(`checks privileges for user, actions, and namespaces`, async () => { const options = { type: [type1, type2], namespaces }; await expectPrivilegeCheck(client.find, { options }, namespaces); diff --git a/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts b/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts index a0d148a14f9b7..73bee302363ab 100644 --- a/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts +++ b/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts @@ -225,6 +225,11 @@ export class SecureSavedObjectsClientWrapper implements SavedObjectsClientContra `_find across namespaces is not permitted when the Spaces plugin is disabled.` ); } + if (options.pit && Array.isArray(options.namespaces) && options.namespaces.length > 1) { + throw this.errors.createBadRequestError( + '_find across namespaces is not permitted when using the `pit` option.' + ); + } const args = { options }; const { status, typeMap } = await this.ensureAuthorized( @@ -572,6 +577,8 @@ export class SecureSavedObjectsClientWrapper implements SavedObjectsClientContra const args = { type, options }; await this.ensureAuthorized(type, 'open_point_in_time', options?.namespace, { args, + // Partial authorization is acceptable in this case because this method is only designed + // to be used with `find`, which already allows for partial authorization. requireFullAuthorization: false, }); } catch (error) { @@ -595,6 +602,14 @@ export class SecureSavedObjectsClientWrapper implements SavedObjectsClientContra } public async closePointInTime(id: string, options?: SavedObjectsClosePointInTimeOptions) { + // We are intentionally omitting a call to `ensureAuthorized` here, because `closePointInTime` + // doesn't take in `types`, which are required to perform authorization. As there is no way + // to know what index/indices a PIT was created against, we have no practical means of + // authorizing users. We've decided we are okay with this because: + // (a) Elasticsearch only requires `read` privileges on an index in order to open/close + // a PIT against it, and; + // (b) By the time a user is accessing this service, they are already authenticated + // to Kibana, which is our closest equivalent to Elasticsearch's `read`. this.auditLogger.log( savedObjectEvent({ action: SavedObjectAction.CLOSE_POINT_IN_TIME, From a883a94cf17d065ff872e62263c943e643bf05a6 Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Wed, 10 Feb 2021 16:29:32 -0700 Subject: [PATCH 31/32] Address feedback on createPointInTimeFinder. --- ...e.test.ts => point_in_time_finder.test.ts} | 113 +++++++++++++++--- ...int_in_time.ts => point_in_time_finder.ts} | 75 ++++++------ .../export/saved_objects_exporter.test.ts | 11 ++ .../export/saved_objects_exporter.ts | 18 +-- 4 files changed, 159 insertions(+), 58 deletions(-) rename src/core/server/saved_objects/export/{find_with_point_in_time.test.ts => point_in_time_finder.test.ts} (63%) rename src/core/server/saved_objects/export/{find_with_point_in_time.ts => point_in_time_finder.ts} (76%) diff --git a/src/core/server/saved_objects/export/find_with_point_in_time.test.ts b/src/core/server/saved_objects/export/point_in_time_finder.test.ts similarity index 63% rename from src/core/server/saved_objects/export/find_with_point_in_time.test.ts rename to src/core/server/saved_objects/export/point_in_time_finder.test.ts index a652acc8f5291..cd79c7a4b81e5 100644 --- a/src/core/server/saved_objects/export/find_with_point_in_time.test.ts +++ b/src/core/server/saved_objects/export/point_in_time_finder.test.ts @@ -11,8 +11,7 @@ import { loggerMock, MockedLogger } from '../../logging/logger.mock'; import { SavedObjectsFindOptions } from '../types'; import { SavedObjectsFindResult } from '../service'; -import type { FindWithPointInTime } from './find_with_point_in_time'; -import { findWithPointInTime } from './find_with_point_in_time'; +import { createPointInTimeFinder } from './point_in_time_finder'; const mockHits = [ { @@ -39,18 +38,55 @@ const mockHits = [ }, ]; -describe('findWithPointInTime()', () => { +describe('createPointInTimeFinder()', () => { let logger: MockedLogger; let savedObjectsClient: ReturnType; - let finder: FindWithPointInTime; beforeEach(() => { logger = loggerMock.create(); savedObjectsClient = savedObjectsClientMock.create(); - finder = findWithPointInTime({ savedObjectsClient, logger }); }); describe('#find', () => { + test('throws if a PIT is already open', async () => { + savedObjectsClient.openPointInTimeForType.mockResolvedValueOnce({ + id: 'abc123', + }); + savedObjectsClient.find.mockResolvedValueOnce({ + total: 2, + saved_objects: mockHits, + pit_id: 'abc123', + per_page: 1, + page: 0, + }); + savedObjectsClient.find.mockResolvedValueOnce({ + total: 2, + saved_objects: mockHits, + pit_id: 'abc123', + per_page: 1, + page: 1, + }); + + const findOptions: SavedObjectsFindOptions = { + type: ['visualization'], + search: 'foo*', + perPage: 1, + }; + + const finder = createPointInTimeFinder({ findOptions, logger, savedObjectsClient }); + await finder.find().next(); + + expect(savedObjectsClient.find).toHaveBeenCalledTimes(1); + savedObjectsClient.find.mockClear(); + + expect(async () => { + await finder.find().next(); + }).rejects.toThrowErrorMatchingInlineSnapshot( + `"Point In Time has already been opened for this finder instance. Please call \`close()\` before calling \`find()\` again."` + ); + expect(savedObjectsClient.find).toHaveBeenCalledTimes(0); + }); + test('works with a single page of results', async () => { savedObjectsClient.openPointInTimeForType.mockResolvedValueOnce({ id: 'abc123', @@ -63,13 +99,14 @@ describe('findWithPointInTime()', () => { page: 0, }); - const options: SavedObjectsFindOptions = { + const findOptions: SavedObjectsFindOptions = { type: ['visualization'], search: 'foo*', }; + const finder = createPointInTimeFinder({ findOptions, logger, savedObjectsClient }); const hits: SavedObjectsFindResult[] = []; - for await (const result of finder.find(options)) { + for await (const result of finder.find()) { hits.push(...result.saved_objects); } @@ -113,14 +150,15 @@ describe('findWithPointInTime()', () => { page: 0, }); - const options: SavedObjectsFindOptions = { + const findOptions: SavedObjectsFindOptions = { type: ['visualization'], search: 'foo*', perPage: 1, }; + const finder = createPointInTimeFinder({ findOptions, logger, savedObjectsClient }); const hits: SavedObjectsFindResult[] = []; - for await (const result of finder.find(options)) { + for await (const result of finder.find()) { hits.push(...result.saved_objects); } @@ -154,14 +192,15 @@ describe('findWithPointInTime()', () => { page: 0, }); - const options: SavedObjectsFindOptions = { + const findOptions: SavedObjectsFindOptions = { type: ['visualization'], search: 'foo*', perPage: 2, }; + const finder = createPointInTimeFinder({ findOptions, logger, savedObjectsClient }); const hits: SavedObjectsFindResult[] = []; - for await (const result of finder.find(options)) { + for await (const result of finder.find()) { hits.push(...result.saved_objects); await finder.close(); } @@ -195,14 +234,15 @@ describe('findWithPointInTime()', () => { page: 0, }); - const options: SavedObjectsFindOptions = { + const findOptions: SavedObjectsFindOptions = { type: ['visualization'], search: 'foo*', perPage: 1, }; + const finder = createPointInTimeFinder({ findOptions, logger, savedObjectsClient }); const hits: SavedObjectsFindResult[] = []; - for await (const result of finder.find(options)) { + for await (const result of finder.find()) { hits.push(...result.saved_objects); await finder.close(); } @@ -217,15 +257,16 @@ describe('findWithPointInTime()', () => { }); savedObjectsClient.find.mockRejectedValueOnce(new Error('oops')); - const options: SavedObjectsFindOptions = { + const findOptions: SavedObjectsFindOptions = { type: ['visualization'], search: 'foo*', perPage: 2, }; + const finder = createPointInTimeFinder({ findOptions, logger, savedObjectsClient }); const hits: SavedObjectsFindResult[] = []; try { - for await (const result of finder.find(options)) { + for await (const result of finder.find()) { hits.push(...result.saved_objects); } } catch (e) { @@ -234,5 +275,47 @@ describe('findWithPointInTime()', () => { expect(savedObjectsClient.closePointInTime).toHaveBeenCalledWith('test'); }); + + test('finder can be reused after closing', async () => { + savedObjectsClient.openPointInTimeForType.mockResolvedValueOnce({ + id: 'abc123', + }); + savedObjectsClient.find.mockResolvedValueOnce({ + total: 2, + saved_objects: mockHits, + pit_id: 'abc123', + per_page: 1, + page: 0, + }); + savedObjectsClient.find.mockResolvedValueOnce({ + total: 2, + saved_objects: mockHits, + pit_id: 'abc123', + per_page: 1, + page: 1, + }); + + const findOptions: SavedObjectsFindOptions = { + type: ['visualization'], + search: 'foo*', + perPage: 1, + }; + + const finder = createPointInTimeFinder({ findOptions, logger, savedObjectsClient }); + + const findA = finder.find(); + await findA.next(); + await finder.close(); + + const findB = finder.find(); + await findB.next(); + await finder.close(); + + expect((await findA.next()).done).toBe(true); + expect((await findB.next()).done).toBe(true); + expect(savedObjectsClient.openPointInTimeForType).toHaveBeenCalledTimes(2); + expect(savedObjectsClient.find).toHaveBeenCalledTimes(2); + expect(savedObjectsClient.closePointInTime).toHaveBeenCalledTimes(2); + }); }); }); diff --git a/src/core/server/saved_objects/export/find_with_point_in_time.ts b/src/core/server/saved_objects/export/point_in_time_finder.ts similarity index 76% rename from src/core/server/saved_objects/export/find_with_point_in_time.ts rename to src/core/server/saved_objects/export/point_in_time_finder.ts index 684f692e1d72b..1edd518c437f4 100644 --- a/src/core/server/saved_objects/export/find_with_point_in_time.ts +++ b/src/core/server/saved_objects/export/point_in_time_finder.ts @@ -25,19 +25,20 @@ import { SavedObjectsFindResponse } from '../service'; * * @example * ```ts - * const finder = findWithPointInTime({ - * logger, - * savedObjectsClient, - * }); - * - * const options: SavedObjectsFindOptions = { + * const findOptions: SavedObjectsFindOptions = { * type: 'visualization', * search: 'foo*', * perPage: 100, * }; * + * const finder = createPointInTimeFinder({ + * logger, + * savedObjectsClient, + * findOptions, + * }); + * * const responses: SavedObjectFindResponse[] = []; - * for await (const response of finder.find(options)) { + * for await (const response of finder.find()) { * responses.push(...response); * if (doneSearching) { * await finder.close(); @@ -45,50 +46,54 @@ import { SavedObjectsFindResponse } from '../service'; * } * ``` */ -export function findWithPointInTime({ +export function createPointInTimeFinder({ + findOptions, logger, savedObjectsClient, }: { + findOptions: SavedObjectsFindOptions; logger: Logger; savedObjectsClient: SavedObjectsClientContract; }) { - return new FindWithPointInTime({ logger, savedObjectsClient }); + return new PointInTimeFinder({ findOptions, logger, savedObjectsClient }); } /** * @internal */ -export class FindWithPointInTime { +export class PointInTimeFinder { readonly #log: Logger; readonly #savedObjectsClient: SavedObjectsClientContract; - #open?: boolean; - #perPage?: number; + readonly #findOptions: SavedObjectsFindOptions; + #open: boolean = false; #pitId?: string; - #type?: string | string[]; constructor({ - savedObjectsClient, + findOptions, logger, + savedObjectsClient, }: { - savedObjectsClient: SavedObjectsClientContract; + findOptions: SavedObjectsFindOptions; logger: Logger; + savedObjectsClient: SavedObjectsClientContract; }) { this.#log = logger; this.#savedObjectsClient = savedObjectsClient; + this.#findOptions = { + // Default to 1000 items per page as a tradeoff between + // speed and memory consumption. + perPage: 1000, + ...findOptions, + }; } - async *find(options: SavedObjectsFindOptions) { - this.#open = true; - this.#type = options.type; - // Default to 1000 items per page as a tradeoff between - // speed and memory consumption. - this.#perPage = options.perPage ?? 1000; - - const findOptions: SavedObjectsFindOptions = { - ...options, - perPage: this.#perPage, - type: this.#type, - }; + async *find() { + if (this.#open) { + throw new Error( + 'Point In Time has already been opened for this finder instance. ' + + 'Please call `close()` before calling `find()` again.' + ); + } // Open PIT and request our first page of hits await this.open(); @@ -97,7 +102,7 @@ export class FindWithPointInTime { let lastHitSortValue: unknown[] | undefined; do { const results = await this.findNext({ - findOptions, + findOptions: this.#findOptions, id: this.#pitId, ...(lastHitSortValue ? { searchAfter: lastHitSortValue } : {}), }); @@ -108,14 +113,14 @@ export class FindWithPointInTime { this.#log.debug(`Collected [${lastResultsCount}] saved objects for export.`); // Close PIT if this was our last page - if (this.#pitId && lastResultsCount < this.#perPage) { + if (this.#pitId && lastResultsCount < this.#findOptions.perPage!) { await this.close(); } yield results; // We've reached the end when there are fewer hits than our perPage size, // or when `close()` has been called. - } while (this.#open && lastHitSortValue && lastResultsCount >= this.#perPage); + } while (this.#open && lastHitSortValue && lastResultsCount >= this.#findOptions.perPage!); return; } @@ -123,29 +128,29 @@ export class FindWithPointInTime { async close() { try { if (this.#pitId) { - this.#log.debug(`Closing PIT for types [${this.#type}]`); + this.#log.debug(`Closing PIT for types [${this.#findOptions.type}]`); await this.#savedObjectsClient.closePointInTime(this.#pitId); this.#pitId = undefined; } - this.#type = undefined; this.#open = false; } catch (e) { - this.#log.error(`Failed to close PIT for types [${this.#type}]`); + this.#log.error(`Failed to close PIT for types [${this.#findOptions.type}]`); throw e; } } private async open() { try { - const { id } = await this.#savedObjectsClient.openPointInTimeForType(this.#type!); + const { id } = await this.#savedObjectsClient.openPointInTimeForType(this.#findOptions.type); this.#pitId = id; + this.#open = true; } catch (e) { // Since `find` swallows 404s, it is expected that exporter will do the same, // so we only rethrow non-404 errors here. if (e.output.statusCode !== 404) { throw e; } - this.#log.debug(`Unable to open PIT for types [${this.#type}]: 404 ${e}`); + this.#log.debug(`Unable to open PIT for types [${this.#findOptions.type}]: 404 ${e}`); } } diff --git a/src/core/server/saved_objects/export/saved_objects_exporter.test.ts b/src/core/server/saved_objects/export/saved_objects_exporter.test.ts index 7c6cc409e1de7..cf60ada5ba90a 100644 --- a/src/core/server/saved_objects/export/saved_objects_exporter.test.ts +++ b/src/core/server/saved_objects/export/saved_objects_exporter.test.ts @@ -835,6 +835,15 @@ describe('getSortedObjectsForExport()', () => { typeRegistry, }); + savedObjectsClient.openPointInTimeForType.mockResolvedValueOnce({ + id: 'abc123', + }); + + savedObjectsClient.closePointInTime.mockResolvedValueOnce({ + succeeded: true, + num_freed: 1, + }); + savedObjectsClient.find.mockResolvedValueOnce({ total: 2, saved_objects: [ @@ -861,6 +870,7 @@ describe('getSortedObjectsForExport()', () => { ], per_page: 1, page: 0, + pit_id: 'abc123', }); await expect( exporter.exportByTypes({ @@ -868,6 +878,7 @@ describe('getSortedObjectsForExport()', () => { types: ['index-pattern', 'search'], }) ).rejects.toThrowErrorMatchingInlineSnapshot(`"Can't export more than 1 objects"`); + expect(savedObjectsClient.closePointInTime).toHaveBeenCalledTimes(1); }); test('sorts objects within type', async () => { diff --git a/src/core/server/saved_objects/export/saved_objects_exporter.ts b/src/core/server/saved_objects/export/saved_objects_exporter.ts index 1819e8ea4d50d..c1c0ea73f0bd3 100644 --- a/src/core/server/saved_objects/export/saved_objects_exporter.ts +++ b/src/core/server/saved_objects/export/saved_objects_exporter.ts @@ -23,7 +23,7 @@ import { } from './types'; import { SavedObjectsExportError } from './errors'; import { applyExportTransforms } from './apply_export_transforms'; -import { findWithPointInTime } from './find_with_point_in_time'; +import { createPointInTimeFinder } from './point_in_time_finder'; import { byIdAscComparator, getPreservedOrderComparator, SavedObjectComparator } from './utils'; /** @@ -168,12 +168,7 @@ export class SavedObjectsExporter { hasReference, search, }: SavedObjectsExportByTypeOptions) { - const finder = findWithPointInTime({ - logger: this.#log, - savedObjectsClient: this.#savedObjectsClient, - }); - - const options: SavedObjectsFindOptions = { + const findOptions: SavedObjectsFindOptions = { type: types, hasReference, hasReferenceOperator: hasReference ? 'OR' : undefined, @@ -181,10 +176,17 @@ export class SavedObjectsExporter { namespaces: namespace ? [namespace] : undefined, }; + const finder = createPointInTimeFinder({ + findOptions, + logger: this.#log, + savedObjectsClient: this.#savedObjectsClient, + }); + const hits: SavedObjectsFindResult[] = []; - for await (const result of finder.find(options)) { + for await (const result of finder.find()) { hits.push(...result.saved_objects); if (hits.length > this.#exportSizeLimit) { + await finder.close(); throw SavedObjectsExportError.exportSizeExceeded(this.#exportSizeLimit); } } From 917845669bee60fe05833153f1466e9ca10514de Mon Sep 17 00:00:00 2001 From: Luke Elmers Date: Thu, 11 Feb 2021 10:02:38 -0700 Subject: [PATCH 32/32] Final cleanup. --- .../saved_objects/export/point_in_time_finder.ts | 2 +- .../server/saved_objects/service/lib/repository.ts | 12 ++++-------- .../saved_objects/service/saved_objects_client.ts | 2 +- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/core/server/saved_objects/export/point_in_time_finder.ts b/src/core/server/saved_objects/export/point_in_time_finder.ts index 1edd518c437f4..dc0bac6b6bfd9 100644 --- a/src/core/server/saved_objects/export/point_in_time_finder.ts +++ b/src/core/server/saved_objects/export/point_in_time_finder.ts @@ -120,7 +120,7 @@ export class PointInTimeFinder { yield results; // We've reached the end when there are fewer hits than our perPage size, // or when `close()` has been called. - } while (this.#open && lastHitSortValue && lastResultsCount >= this.#findOptions.perPage!); + } while (this.#open && lastResultsCount >= this.#findOptions.perPage!); return; } diff --git a/src/core/server/saved_objects/service/lib/repository.ts b/src/core/server/saved_objects/service/lib/repository.ts index 3bb274936c32b..b8a72377b0d76 100644 --- a/src/core/server/saved_objects/service/lib/repository.ts +++ b/src/core/server/saved_objects/service/lib/repository.ts @@ -1788,19 +1788,16 @@ export class SavedObjectsRepository { * * @example * ```ts - * const repository = coreStart.savedObjects.createInternalRepository(); - * - * const { id } = await repository.openPointInTimeForType( - * type: 'index-pattern', - * { keepAlive: '2m' }, + * const { id } = await savedObjectsClient.openPointInTimeForType( + * type: 'visualization', + * { keepAlive: '5m' }, * ); * const page1 = await savedObjectsClient.find({ * type: 'visualization', * sortField: 'updated_at', * sortOrder: 'asc', - * pit, + * pit: { id, keepAlive: '2m' }, * }); - * * const lastHit = page1.saved_objects[page1.saved_objects.length - 1]; * const page2 = await savedObjectsClient.find({ * type: 'visualization', @@ -1809,7 +1806,6 @@ export class SavedObjectsRepository { * pit: { id: page1.pit_id }, * searchAfter: lastHit.sort, * }); - * * await savedObjectsClient.closePointInTime(page2.pit_id); * ``` * diff --git a/src/core/server/saved_objects/service/saved_objects_client.ts b/src/core/server/saved_objects/service/saved_objects_client.ts index 7b7ef36e1cb0c..b93f3022e4236 100644 --- a/src/core/server/saved_objects/service/saved_objects_client.ts +++ b/src/core/server/saved_objects/service/saved_objects_client.ts @@ -144,7 +144,7 @@ export interface SavedObjectsFindResult extends SavedObject { * type: 'visualization', * sortField: 'updated_at', * sortOrder: 'asc', - * pit, + * pit: { id }, * }); * const lastHit = page1.saved_objects[page1.saved_objects.length - 1]; * const page2 = await savedObjectsClient.find({