From 03c4ef443443561ca06bb712310a618e02cad0ca Mon Sep 17 00:00:00 2001 From: ymao1 Date: Wed, 28 Jul 2021 13:04:20 -0400 Subject: [PATCH] Gracefully handle decryption errors during ESO migrations (#105968) * Updating unit tests * Fixing types * Updating readme and adding warning message * Updating README * PR fixes * collapsing args to create migration fn * Adding functional tests * Adding comment to functional test * Adding stripOrDecryptAttributesSync * Using stripOrDecryptAttributesSync * Fixing unit tests * PR fixes * PR fixes * Moving validation of apikey existence in alerting task runner * Cleanup * Reverting changes to alerting task runner * PR fixes Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../server/saved_objects/migrations.test.ts | 227 +- .../server/saved_objects/migrations.ts | 26 +- .../server/saved_objects/migrations.test.ts | 1541 ++++++----- .../server/saved_objects/migrations.ts | 36 +- .../plugins/encrypted_saved_objects/README.md | 31 +- .../server/create_migration.test.ts | 447 ++- .../server/create_migration.ts | 74 +- .../encrypted_saved_objects_service.mocks.ts | 1 + .../encrypted_saved_objects_service.test.ts | 204 ++ .../crypto/encrypted_saved_objects_service.ts | 106 +- .../server/crypto/index.ts | 2 +- .../encrypted_saved_objects/server/index.ts | 1 + .../saved_objects/migrations/to_v7_10_0.ts | 12 +- .../api_consumer_plugin/server/index.ts | 26 +- .../data.json | 86 + .../mappings.json | 2440 +++++++++++++++++ .../encrypted_saved_objects_decryption.ts | 65 + .../tests/index.ts | 1 + 18 files changed, 4362 insertions(+), 964 deletions(-) create mode 100644 x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/encrypted_saved_objects_different_key/data.json create mode 100644 x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/encrypted_saved_objects_different_key/mappings.json create mode 100644 x-pack/test/encrypted_saved_objects_api_integration/tests/encrypted_saved_objects_decryption.ts diff --git a/x-pack/plugins/actions/server/saved_objects/migrations.test.ts b/x-pack/plugins/actions/server/saved_objects/migrations.test.ts index 4c30925e61894..beaea76756113 100644 --- a/x-pack/plugins/actions/server/saved_objects/migrations.test.ts +++ b/x-pack/plugins/actions/server/saved_objects/migrations.test.ts @@ -15,121 +15,170 @@ import { migrationMocks } from 'src/core/server/mocks'; const context = migrationMocks.createContext(); const encryptedSavedObjectsSetup = encryptedSavedObjectsMock.createSetup(); -describe('7.10.0', () => { +describe('successful migrations', () => { beforeEach(() => { jest.resetAllMocks(); - encryptedSavedObjectsSetup.createMigration.mockImplementation( - (shouldMigrateWhenPredicate, migration) => migration - ); + encryptedSavedObjectsSetup.createMigration.mockImplementation(({ migration }) => migration); }); - test('add hasAuth config property for .email actions', () => { - const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; - const action = getMockDataForEmail({}); - const migratedAction = migration710(action, context); - expect(migratedAction.attributes.config).toEqual({ - hasAuth: true, - }); - expect(migratedAction).toEqual({ - ...action, - attributes: { - ...action.attributes, - config: { - hasAuth: true, + describe('7.10.0', () => { + test('add hasAuth config property for .email actions', () => { + const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; + const action = getMockDataForEmail({}); + const migratedAction = migration710(action, context); + expect(migratedAction.attributes.config).toEqual({ + hasAuth: true, + }); + expect(migratedAction).toEqual({ + ...action, + attributes: { + ...action.attributes, + config: { + hasAuth: true, + }, }, - }, + }); }); - }); - test('rename cases configuration object', () => { - const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; - const action = getCasesMockData({}); - const migratedAction = migration710(action, context); - expect(migratedAction.attributes.config).toEqual({ - incidentConfiguration: { mapping: [] }, - }); - expect(migratedAction).toEqual({ - ...action, - attributes: { - ...action.attributes, - config: { - incidentConfiguration: { mapping: [] }, + test('rename cases configuration object', () => { + const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; + const action = getCasesMockData({}); + const migratedAction = migration710(action, context); + expect(migratedAction.attributes.config).toEqual({ + incidentConfiguration: { mapping: [] }, + }); + expect(migratedAction).toEqual({ + ...action, + attributes: { + ...action.attributes, + config: { + incidentConfiguration: { mapping: [] }, + }, }, - }, + }); }); }); -}); - -describe('7.11.0', () => { - beforeEach(() => { - jest.resetAllMocks(); - encryptedSavedObjectsSetup.createMigration.mockImplementation( - (shouldMigrateWhenPredicate, migration) => migration - ); - }); - test('add hasAuth = true for .webhook actions with user and password', () => { - const migration711 = getMigrations(encryptedSavedObjectsSetup)['7.11.0']; - const action = getMockDataForWebhook({}, true); - expect(migration711(action, context)).toMatchObject({ - ...action, - attributes: { - ...action.attributes, - config: { - hasAuth: true, + describe('7.11.0', () => { + test('add hasAuth = true for .webhook actions with user and password', () => { + const migration711 = getMigrations(encryptedSavedObjectsSetup)['7.11.0']; + const action = getMockDataForWebhook({}, true); + expect(migration711(action, context)).toMatchObject({ + ...action, + attributes: { + ...action.attributes, + config: { + hasAuth: true, + }, }, - }, + }); }); - }); - test('add hasAuth = false for .webhook actions without user and password', () => { - const migration711 = getMigrations(encryptedSavedObjectsSetup)['7.11.0']; - const action = getMockDataForWebhook({}, false); - expect(migration711(action, context)).toMatchObject({ - ...action, - attributes: { - ...action.attributes, - config: { - hasAuth: false, + test('add hasAuth = false for .webhook actions without user and password', () => { + const migration711 = getMigrations(encryptedSavedObjectsSetup)['7.11.0']; + const action = getMockDataForWebhook({}, false); + expect(migration711(action, context)).toMatchObject({ + ...action, + attributes: { + ...action.attributes, + config: { + hasAuth: false, + }, }, - }, + }); }); - }); - test('remove cases mapping object', () => { - const migration711 = getMigrations(encryptedSavedObjectsSetup)['7.11.0']; - const action = getMockData({ - config: { incidentConfiguration: { mapping: [] }, isCaseOwned: true, another: 'value' }, + test('remove cases mapping object', () => { + const migration711 = getMigrations(encryptedSavedObjectsSetup)['7.11.0']; + const action = getMockData({ + config: { incidentConfiguration: { mapping: [] }, isCaseOwned: true, another: 'value' }, + }); + expect(migration711(action, context)).toEqual({ + ...action, + attributes: { + ...action.attributes, + config: { + another: 'value', + }, + }, + }); }); - expect(migration711(action, context)).toEqual({ - ...action, - attributes: { - ...action.attributes, - config: { - another: 'value', + }); + + describe('7.14.0', () => { + test('add isMissingSecrets property for actions', () => { + const migration714 = getMigrations(encryptedSavedObjectsSetup)['7.14.0']; + const action = getMockData({ isMissingSecrets: undefined }); + const migratedAction = migration714(action, context); + expect(migratedAction).toEqual({ + ...action, + attributes: { + ...action.attributes, + isMissingSecrets: false, }, - }, + }); }); }); }); -describe('7.14.0', () => { +describe('handles errors during migrations', () => { beforeEach(() => { jest.resetAllMocks(); - encryptedSavedObjectsSetup.createMigration.mockImplementation( - (shouldMigrateWhenPredicate, migration) => migration - ); + encryptedSavedObjectsSetup.createMigration.mockImplementation(() => () => { + throw new Error(`Can't migrate!`); + }); + }); + + describe('7.10.0 throws if migration fails', () => { + test('should show the proper exception', () => { + const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; + const action = getMockDataForEmail({}); + expect(() => { + migration710(action, context); + }).toThrowError(`Can't migrate!`); + expect(context.log.error).toHaveBeenCalledWith( + `encryptedSavedObject 7.10.0 migration failed for action ${action.id} with error: Can't migrate!`, + { + migrations: { + actionDocument: action, + }, + } + ); + }); + }); + + describe('7.11.0 throws if migration fails', () => { + test('should show the proper exception', () => { + const migration711 = getMigrations(encryptedSavedObjectsSetup)['7.11.0']; + const action = getMockDataForEmail({}); + expect(() => { + migration711(action, context); + }).toThrowError(`Can't migrate!`); + expect(context.log.error).toHaveBeenCalledWith( + `encryptedSavedObject 7.11.0 migration failed for action ${action.id} with error: Can't migrate!`, + { + migrations: { + actionDocument: action, + }, + } + ); + }); }); - test('add isMissingSecrets property for actions', () => { - const migration714 = getMigrations(encryptedSavedObjectsSetup)['7.14.0']; - const action = getMockData({ isMissingSecrets: undefined }); - const migratedAction = migration714(action, context); - expect(migratedAction).toEqual({ - ...action, - attributes: { - ...action.attributes, - isMissingSecrets: false, - }, + describe('7.14.0 throws if migration fails', () => { + test('should show the proper exception', () => { + const migration714 = getMigrations(encryptedSavedObjectsSetup)['7.14.0']; + const action = getMockDataForEmail({}); + expect(() => { + migration714(action, context); + }).toThrowError(`Can't migrate!`); + expect(context.log.error).toHaveBeenCalledWith( + `encryptedSavedObject 7.14.0 migration failed for action ${action.id} with error: Can't migrate!`, + { + migrations: { + actionDocument: action, + }, + } + ); }); }); }); diff --git a/x-pack/plugins/actions/server/saved_objects/migrations.ts b/x-pack/plugins/actions/server/saved_objects/migrations.ts index 17932b6b90f97..de15de7b15e23 100644 --- a/x-pack/plugins/actions/server/saved_objects/migrations.ts +++ b/x-pack/plugins/actions/server/saved_objects/migrations.ts @@ -14,6 +14,7 @@ import { } from '../../../../../src/core/server'; import { RawAction } from '../types'; import { EncryptedSavedObjectsPluginSetup } from '../../../encrypted_saved_objects/server'; +import type { IsMigrationNeededPredicate } from '../../../encrypted_saved_objects/server'; interface ActionsLogMeta extends LogMeta { migrations: { actionDocument: SavedObjectUnsanitizedDoc }; @@ -23,17 +24,31 @@ type ActionMigration = ( doc: SavedObjectUnsanitizedDoc ) => SavedObjectUnsanitizedDoc; +function createEsoMigration( + encryptedSavedObjects: EncryptedSavedObjectsPluginSetup, + isMigrationNeededPredicate: IsMigrationNeededPredicate, + migrationFunc: ActionMigration +) { + return encryptedSavedObjects.createMigration({ + isMigrationNeededPredicate, + migration: migrationFunc, + shouldMigrateIfDecryptionFails: true, // shouldMigrateIfDecryptionFails flag that applies the migration to undecrypted document if decryption fails + }); +} + export function getMigrations( encryptedSavedObjects: EncryptedSavedObjectsPluginSetup ): SavedObjectMigrationMap { - const migrationActionsTen = encryptedSavedObjects.createMigration( + const migrationActionsTen = createEsoMigration( + encryptedSavedObjects, (doc): doc is SavedObjectUnsanitizedDoc => doc.attributes.config?.hasOwnProperty('casesConfiguration') || doc.attributes.actionTypeId === '.email', pipeMigrations(renameCasesConfigurationObject, addHasAuthConfigurationObject) ); - const migrationActionsEleven = encryptedSavedObjects.createMigration( + const migrationActionsEleven = createEsoMigration( + encryptedSavedObjects, (doc): doc is SavedObjectUnsanitizedDoc => doc.attributes.config?.hasOwnProperty('isCaseOwned') || doc.attributes.config?.hasOwnProperty('incidentConfiguration') || @@ -41,7 +56,8 @@ export function getMigrations( pipeMigrations(removeCasesFieldMappings, addHasAuthConfigurationObject) ); - const migrationActionsFourteen = encryptedSavedObjects.createMigration( + const migrationActionsFourteen = createEsoMigration( + encryptedSavedObjects, (doc): doc is SavedObjectUnsanitizedDoc => true, pipeMigrations(addisMissingSecretsField) ); @@ -69,8 +85,8 @@ function executeMigrationWithErrorHandling( }, } ); + throw ex; } - return doc; }; } @@ -120,7 +136,7 @@ const addHasAuthConfigurationObject = ( if (doc.attributes.actionTypeId !== '.email' && doc.attributes.actionTypeId !== '.webhook') { return doc; } - const hasAuth = !!doc.attributes.secrets.user || !!doc.attributes.secrets.password; + const hasAuth = !!doc.attributes.secrets?.user || !!doc.attributes.secrets?.password; return { ...doc, attributes: { diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations.test.ts b/x-pack/plugins/alerting/server/saved_objects/migrations.test.ts index 4888116e43602..9403c0c28c153 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations.test.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations.test.ts @@ -15,99 +15,80 @@ import { migrationMocks } from 'src/core/server/mocks'; const migrationContext = migrationMocks.createContext(); const encryptedSavedObjectsSetup = encryptedSavedObjectsMock.createSetup(); -describe('7.10.0', () => { +describe('successful migrations', () => { beforeEach(() => { jest.resetAllMocks(); - encryptedSavedObjectsSetup.createMigration.mockImplementation( - (shouldMigrateWhenPredicate, migration) => migration - ); + encryptedSavedObjectsSetup.createMigration.mockImplementation(({ migration }) => migration); }); - - test('marks alerts as legacy', () => { - const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; - const alert = getMockData({}); - expect(migration710(alert, migrationContext)).toMatchObject({ - ...alert, - attributes: { - ...alert.attributes, - meta: { - versionApiKeyLastmodified: 'pre-7.10.0', - }, - }, - }); - }); - - test('migrates the consumer for metrics', () => { - const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; - const alert = getMockData({ - consumer: 'metrics', - }); - expect(migration710(alert, migrationContext)).toMatchObject({ - ...alert, - attributes: { - ...alert.attributes, - consumer: 'infrastructure', - meta: { - versionApiKeyLastmodified: 'pre-7.10.0', + describe('7.10.0', () => { + test('marks alerts as legacy', () => { + const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; + const alert = getMockData({}); + expect(migration710(alert, migrationContext)).toMatchObject({ + ...alert, + attributes: { + ...alert.attributes, + meta: { + versionApiKeyLastmodified: 'pre-7.10.0', + }, }, - }, + }); }); - }); - test('migrates the consumer for siem', () => { - const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; - const alert = getMockData({ - consumer: 'securitySolution', - }); - expect(migration710(alert, migrationContext)).toMatchObject({ - ...alert, - attributes: { - ...alert.attributes, - consumer: 'siem', - meta: { - versionApiKeyLastmodified: 'pre-7.10.0', + test('migrates the consumer for metrics', () => { + const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; + const alert = getMockData({ + consumer: 'metrics', + }); + expect(migration710(alert, migrationContext)).toMatchObject({ + ...alert, + attributes: { + ...alert.attributes, + consumer: 'infrastructure', + meta: { + versionApiKeyLastmodified: 'pre-7.10.0', + }, }, - }, + }); }); - }); - test('migrates the consumer for alerting', () => { - const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; - const alert = getMockData({ - consumer: 'alerting', - }); - expect(migration710(alert, migrationContext)).toMatchObject({ - ...alert, - attributes: { - ...alert.attributes, - consumer: 'alerts', - meta: { - versionApiKeyLastmodified: 'pre-7.10.0', + test('migrates the consumer for siem', () => { + const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; + const alert = getMockData({ + consumer: 'securitySolution', + }); + expect(migration710(alert, migrationContext)).toMatchObject({ + ...alert, + attributes: { + ...alert.attributes, + consumer: 'siem', + meta: { + versionApiKeyLastmodified: 'pre-7.10.0', + }, }, - }, + }); }); - }); - test('migrates PagerDuty actions to set a default dedupkey of the AlertId', () => { - const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; - const alert = getMockData({ - actions: [ - { - actionTypeId: '.pagerduty', - group: 'default', - params: { - summary: 'fired {{alertInstanceId}}', - eventAction: 'resolve', - component: '', + test('migrates the consumer for alerting', () => { + const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; + const alert = getMockData({ + consumer: 'alerting', + }); + expect(migration710(alert, migrationContext)).toMatchObject({ + ...alert, + attributes: { + ...alert.attributes, + consumer: 'alerts', + meta: { + versionApiKeyLastmodified: 'pre-7.10.0', }, - id: 'b62ea790-5366-4abc-a7df-33db1db78410', }, - ], + }); }); - expect(migration710(alert, migrationContext)).toMatchObject({ - ...alert, - attributes: { - ...alert.attributes, + + test('migrates PagerDuty actions to set a default dedupkey of the AlertId', () => { + const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; + const alert = getMockData({ actions: [ { actionTypeId: '.pagerduty', @@ -115,37 +96,36 @@ describe('7.10.0', () => { params: { summary: 'fired {{alertInstanceId}}', eventAction: 'resolve', - dedupKey: '{{alertId}}', component: '', }, id: 'b62ea790-5366-4abc-a7df-33db1db78410', }, ], - }, - }); - }); - - test('skips PagerDuty actions with a specified dedupkey', () => { - const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; - const alert = getMockData({ - actions: [ - { - actionTypeId: '.pagerduty', - group: 'default', - params: { - summary: 'fired {{alertInstanceId}}', - eventAction: 'trigger', - dedupKey: '{{alertInstanceId}}', - component: '', - }, - id: 'b62ea790-5366-4abc-a7df-33db1db78410', + }); + expect(migration710(alert, migrationContext)).toMatchObject({ + ...alert, + attributes: { + ...alert.attributes, + actions: [ + { + actionTypeId: '.pagerduty', + group: 'default', + params: { + summary: 'fired {{alertInstanceId}}', + eventAction: 'resolve', + dedupKey: '{{alertId}}', + component: '', + }, + id: 'b62ea790-5366-4abc-a7df-33db1db78410', + }, + ], }, - ], + }); }); - expect(migration710(alert, migrationContext)).toMatchObject({ - ...alert, - attributes: { - ...alert.attributes, + + test('skips PagerDuty actions with a specified dedupkey', () => { + const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; + const alert = getMockData({ actions: [ { actionTypeId: '.pagerduty', @@ -159,33 +139,31 @@ describe('7.10.0', () => { id: 'b62ea790-5366-4abc-a7df-33db1db78410', }, ], - }, - }); - }); - - test('skips PagerDuty actions with an eventAction of "trigger"', () => { - const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; - const alert = getMockData({ - actions: [ - { - actionTypeId: '.pagerduty', - group: 'default', - params: { - summary: 'fired {{alertInstanceId}}', - eventAction: 'trigger', - component: '', - }, - id: 'b62ea790-5366-4abc-a7df-33db1db78410', + }); + expect(migration710(alert, migrationContext)).toMatchObject({ + ...alert, + attributes: { + ...alert.attributes, + actions: [ + { + actionTypeId: '.pagerduty', + group: 'default', + params: { + summary: 'fired {{alertInstanceId}}', + eventAction: 'trigger', + dedupKey: '{{alertInstanceId}}', + component: '', + }, + id: 'b62ea790-5366-4abc-a7df-33db1db78410', + }, + ], }, - ], + }); }); - expect(migration710(alert, migrationContext)).toMatchObject({ - ...alert, - attributes: { - ...alert.attributes, - meta: { - versionApiKeyLastmodified: 'pre-7.10.0', - }, + + test('skips PagerDuty actions with an eventAction of "trigger"', () => { + const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; + const alert = getMockData({ actions: [ { actionTypeId: '.pagerduty', @@ -198,214 +176,115 @@ describe('7.10.0', () => { id: 'b62ea790-5366-4abc-a7df-33db1db78410', }, ], - }, - }); - }); - - test('creates execution status', () => { - const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; - const alert = getMockData(); - const dateStart = Date.now(); - const migratedAlert = migration710(alert, migrationContext); - const dateStop = Date.now(); - const dateExecutionStatus = Date.parse( - migratedAlert.attributes.executionStatus.lastExecutionDate - ); - - expect(dateStart).toBeLessThanOrEqual(dateExecutionStatus); - expect(dateStop).toBeGreaterThanOrEqual(dateExecutionStatus); - - expect(migratedAlert).toMatchObject({ - ...alert, - attributes: { - ...alert.attributes, - executionStatus: { - lastExecutionDate: migratedAlert.attributes.executionStatus.lastExecutionDate, - status: 'pending', - error: null, + }); + expect(migration710(alert, migrationContext)).toMatchObject({ + ...alert, + attributes: { + ...alert.attributes, + meta: { + versionApiKeyLastmodified: 'pre-7.10.0', + }, + actions: [ + { + actionTypeId: '.pagerduty', + group: 'default', + params: { + summary: 'fired {{alertInstanceId}}', + eventAction: 'trigger', + component: '', + }, + id: 'b62ea790-5366-4abc-a7df-33db1db78410', + }, + ], }, - }, + }); }); - }); -}); -describe('7.10.0 migrates with failure', () => { - beforeEach(() => { - jest.resetAllMocks(); - encryptedSavedObjectsSetup.createMigration.mockImplementationOnce(() => () => { - throw new Error(`Can't migrate!`); - }); - }); + test('creates execution status', () => { + const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; + const alert = getMockData(); + const dateStart = Date.now(); + const migratedAlert = migration710(alert, migrationContext); + const dateStop = Date.now(); + const dateExecutionStatus = Date.parse( + migratedAlert.attributes.executionStatus.lastExecutionDate + ); - test('should show the proper exception', () => { - const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; - const alert = getMockData({ - consumer: 'alerting', - }); - const res = migration710(alert, migrationContext); - expect(res).toMatchObject({ - ...alert, - attributes: { - ...alert.attributes, - }, - }); - expect(migrationContext.log.error).toHaveBeenCalledWith( - `encryptedSavedObject 7.10.0 migration failed for alert ${alert.id} with error: Can't migrate!`, - { - migrations: { - alertDocument: { - ...alert, - attributes: { - ...alert.attributes, - }, + expect(dateStart).toBeLessThanOrEqual(dateExecutionStatus); + expect(dateStop).toBeGreaterThanOrEqual(dateExecutionStatus); + + expect(migratedAlert).toMatchObject({ + ...alert, + attributes: { + ...alert.attributes, + executionStatus: { + lastExecutionDate: migratedAlert.attributes.executionStatus.lastExecutionDate, + status: 'pending', + error: null, }, }, - } - ); - }); -}); - -describe('7.11.0', () => { - beforeEach(() => { - jest.resetAllMocks(); - encryptedSavedObjectsSetup.createMigration.mockImplementation( - (shouldMigrateWhenPredicate, migration) => migration - ); - }); - - test('add updatedAt field to alert - set to SavedObject updated_at attribute', () => { - const migration711 = getMigrations(encryptedSavedObjectsSetup)['7.11.0']; - const alert = getMockData({}, true); - expect(migration711(alert, migrationContext)).toEqual({ - ...alert, - attributes: { - ...alert.attributes, - updatedAt: alert.updated_at, - notifyWhen: 'onActiveAlert', - }, + }); }); }); - test('add updatedAt field to alert - set to createdAt when SavedObject updated_at is not defined', () => { - const migration711 = getMigrations(encryptedSavedObjectsSetup)['7.11.0']; - const alert = getMockData({}); - expect(migration711(alert, migrationContext)).toEqual({ - ...alert, - attributes: { - ...alert.attributes, - updatedAt: alert.attributes.createdAt, - notifyWhen: 'onActiveAlert', - }, + describe('7.11.0', () => { + test('add updatedAt field to alert - set to SavedObject updated_at attribute', () => { + const migration711 = getMigrations(encryptedSavedObjectsSetup)['7.11.0']; + const alert = getMockData({}, true); + expect(migration711(alert, migrationContext)).toEqual({ + ...alert, + attributes: { + ...alert.attributes, + updatedAt: alert.updated_at, + notifyWhen: 'onActiveAlert', + }, + }); }); - }); - test('add notifyWhen=onActiveAlert when throttle is null', () => { - const migration711 = getMigrations(encryptedSavedObjectsSetup)['7.11.0']; - const alert = getMockData({}); - expect(migration711(alert, migrationContext)).toEqual({ - ...alert, - attributes: { - ...alert.attributes, - updatedAt: alert.attributes.createdAt, - notifyWhen: 'onActiveAlert', - }, + test('add updatedAt field to alert - set to createdAt when SavedObject updated_at is not defined', () => { + const migration711 = getMigrations(encryptedSavedObjectsSetup)['7.11.0']; + const alert = getMockData({}); + expect(migration711(alert, migrationContext)).toEqual({ + ...alert, + attributes: { + ...alert.attributes, + updatedAt: alert.attributes.createdAt, + notifyWhen: 'onActiveAlert', + }, + }); }); - }); - test('add notifyWhen=onActiveAlert when throttle is set', () => { - const migration711 = getMigrations(encryptedSavedObjectsSetup)['7.11.0']; - const alert = getMockData({ throttle: '5m' }); - expect(migration711(alert, migrationContext)).toEqual({ - ...alert, - attributes: { - ...alert.attributes, - updatedAt: alert.attributes.createdAt, - notifyWhen: 'onThrottleInterval', - }, + test('add notifyWhen=onActiveAlert when throttle is null', () => { + const migration711 = getMigrations(encryptedSavedObjectsSetup)['7.11.0']; + const alert = getMockData({}); + expect(migration711(alert, migrationContext)).toEqual({ + ...alert, + attributes: { + ...alert.attributes, + updatedAt: alert.attributes.createdAt, + notifyWhen: 'onActiveAlert', + }, + }); }); - }); -}); - -describe('7.11.2', () => { - beforeEach(() => { - jest.resetAllMocks(); - encryptedSavedObjectsSetup.createMigration.mockImplementation( - (shouldMigrateWhenPredicate, migration) => migration - ); - }); - test('transforms connectors that support incident correctly', () => { - const migration7112 = getMigrations(encryptedSavedObjectsSetup)['7.11.2']; - const alert = getMockData({ - actions: [ - { - actionTypeId: '.jira', - group: 'threshold met', - params: { - subAction: 'pushToService', - subActionParams: { - title: 'Jira summary', - issueType: '10001', - comments: [ - { - commentId: '1', - comment: 'jira comment', - }, - ], - description: 'Jira description', - savedObjectId: '{{alertId}}', - priority: 'Highest', - parent: 'CASES-78', - labels: ['test'], - }, - }, - id: 'b1abe42d-ae1a-4a6a-b5ec-482ce0492c14', - }, - { - actionTypeId: '.resilient', - group: 'threshold met', - params: { - subAction: 'pushToService', - subActionParams: { - savedObjectId: '{{alertId}}', - incidentTypes: ['17', '21'], - severityCode: '5', - title: 'IBM name', - description: 'IBM description', - comments: [ - { - commentId: 'alert-comment', - comment: 'IBM comment', - }, - ], - }, - }, - id: '75d63268-9a83-460f-9026-0028f4f7dac4', - }, - { - actionTypeId: '.servicenow', - group: 'threshold met', - params: { - subAction: 'pushToService', - subActionParams: { - severity: '2', - impact: '2', - urgency: '2', - savedObjectId: '{{alertId}}', - title: 'SN short desc', - description: 'SN desc', - comment: 'sn comment', - }, - }, - id: '1266562a-4e1f-4305-99ca-1b44c469b26e', + test('add notifyWhen=onActiveAlert when throttle is set', () => { + const migration711 = getMigrations(encryptedSavedObjectsSetup)['7.11.0']; + const alert = getMockData({ throttle: '5m' }); + expect(migration711(alert, migrationContext)).toEqual({ + ...alert, + attributes: { + ...alert.attributes, + updatedAt: alert.attributes.createdAt, + notifyWhen: 'onThrottleInterval', }, - ], + }); }); + }); - expect(migration7112(alert, migrationContext)).toEqual({ - ...alert, - attributes: { - ...alert.attributes, + describe('7.11.2', () => { + test('transforms connectors that support incident correctly', () => { + const migration7112 = getMigrations(encryptedSavedObjectsSetup)['7.11.2']; + const alert = getMockData({ actions: [ { actionTypeId: '.jira', @@ -413,20 +292,19 @@ describe('7.11.2', () => { params: { subAction: 'pushToService', subActionParams: { - incident: { - summary: 'Jira summary', - description: 'Jira description', - issueType: '10001', - priority: 'Highest', - parent: 'CASES-78', - labels: ['test'], - }, + title: 'Jira summary', + issueType: '10001', comments: [ { commentId: '1', comment: 'jira comment', }, ], + description: 'Jira description', + savedObjectId: '{{alertId}}', + priority: 'Highest', + parent: 'CASES-78', + labels: ['test'], }, }, id: 'b1abe42d-ae1a-4a6a-b5ec-482ce0492c14', @@ -437,12 +315,11 @@ describe('7.11.2', () => { params: { subAction: 'pushToService', subActionParams: { - incident: { - name: 'IBM name', - description: 'IBM description', - incidentTypes: ['17', '21'], - severityCode: '5', - }, + savedObjectId: '{{alertId}}', + incidentTypes: ['17', '21'], + severityCode: '5', + title: 'IBM name', + description: 'IBM description', comments: [ { commentId: 'alert-comment', @@ -459,81 +336,205 @@ describe('7.11.2', () => { params: { subAction: 'pushToService', subActionParams: { - incident: { - short_description: 'SN short desc', - description: 'SN desc', - severity: '2', - impact: '2', - urgency: '2', - }, - comments: [{ commentId: '1', comment: 'sn comment' }], + severity: '2', + impact: '2', + urgency: '2', + savedObjectId: '{{alertId}}', + title: 'SN short desc', + description: 'SN desc', + comment: 'sn comment', }, }, id: '1266562a-4e1f-4305-99ca-1b44c469b26e', }, ], - }, - }); - }); + }); - test('it transforms only subAction=pushToService', () => { - const migration7112 = getMigrations(encryptedSavedObjectsSetup)['7.11.2']; - const alert = getMockData({ - actions: [ - { - actionTypeId: '.jira', - group: 'threshold met', - params: { - subAction: 'issues', - subActionParams: { issues: 'Task' }, - }, - id: '1266562a-4e1f-4305-99ca-1b44c469b26e', + expect(migration7112(alert, migrationContext)).toEqual({ + ...alert, + attributes: { + ...alert.attributes, + actions: [ + { + actionTypeId: '.jira', + group: 'threshold met', + params: { + subAction: 'pushToService', + subActionParams: { + incident: { + summary: 'Jira summary', + description: 'Jira description', + issueType: '10001', + priority: 'Highest', + parent: 'CASES-78', + labels: ['test'], + }, + comments: [ + { + commentId: '1', + comment: 'jira comment', + }, + ], + }, + }, + id: 'b1abe42d-ae1a-4a6a-b5ec-482ce0492c14', + }, + { + actionTypeId: '.resilient', + group: 'threshold met', + params: { + subAction: 'pushToService', + subActionParams: { + incident: { + name: 'IBM name', + description: 'IBM description', + incidentTypes: ['17', '21'], + severityCode: '5', + }, + comments: [ + { + commentId: 'alert-comment', + comment: 'IBM comment', + }, + ], + }, + }, + id: '75d63268-9a83-460f-9026-0028f4f7dac4', + }, + { + actionTypeId: '.servicenow', + group: 'threshold met', + params: { + subAction: 'pushToService', + subActionParams: { + incident: { + short_description: 'SN short desc', + description: 'SN desc', + severity: '2', + impact: '2', + urgency: '2', + }, + comments: [{ commentId: '1', comment: 'sn comment' }], + }, + }, + id: '1266562a-4e1f-4305-99ca-1b44c469b26e', + }, + ], }, - ], + }); }); - expect(migration7112(alert, migrationContext)).toEqual(alert); - }); + test('it transforms only subAction=pushToService', () => { + const migration7112 = getMigrations(encryptedSavedObjectsSetup)['7.11.2']; + const alert = getMockData({ + actions: [ + { + actionTypeId: '.jira', + group: 'threshold met', + params: { + subAction: 'issues', + subActionParams: { issues: 'Task' }, + }, + id: '1266562a-4e1f-4305-99ca-1b44c469b26e', + }, + ], + }); - test('it does not transforms other connectors', () => { - const migration7112 = getMigrations(encryptedSavedObjectsSetup)['7.11.2']; - const alert = getMockData({ - actions: [ - { - actionTypeId: '.server-log', - group: 'threshold met', - params: { - level: 'info', - message: 'log message', + expect(migration7112(alert, migrationContext)).toEqual(alert); + }); + + test('it does not transforms other connectors', () => { + const migration7112 = getMigrations(encryptedSavedObjectsSetup)['7.11.2']; + const alert = getMockData({ + actions: [ + { + actionTypeId: '.server-log', + group: 'threshold met', + params: { + level: 'info', + message: 'log message', + }, + id: '99257478-e591-4560-b264-441bdd4fe1d9', }, - id: '99257478-e591-4560-b264-441bdd4fe1d9', - }, - { - actionTypeId: '.servicenow', - group: 'threshold met', - params: { - subAction: 'pushToService', - subActionParams: { - severity: '2', - impact: '2', - urgency: '2', - savedObjectId: '{{alertId}}', - title: 'SN short desc', - description: 'SN desc', - comment: 'sn comment', + { + actionTypeId: '.servicenow', + group: 'threshold met', + params: { + subAction: 'pushToService', + subActionParams: { + severity: '2', + impact: '2', + urgency: '2', + savedObjectId: '{{alertId}}', + title: 'SN short desc', + description: 'SN desc', + comment: 'sn comment', + }, }, + id: '1266562a-4e1f-4305-99ca-1b44c469b26e', }, - id: '1266562a-4e1f-4305-99ca-1b44c469b26e', + ], + }); + + expect(migration7112(alert, migrationContext)).toEqual({ + ...alert, + attributes: { + ...alert.attributes, + actions: [ + alert.attributes.actions![0], + { + actionTypeId: '.servicenow', + group: 'threshold met', + params: { + subAction: 'pushToService', + subActionParams: { + incident: { + short_description: 'SN short desc', + description: 'SN desc', + severity: '2', + impact: '2', + urgency: '2', + }, + comments: [{ commentId: '1', comment: 'sn comment' }], + }, + }, + id: '1266562a-4e1f-4305-99ca-1b44c469b26e', + }, + ], }, - ], + }); + }); + + test.each(['.jira', '.servicenow', '.resilient'])( + 'isAnyActionSupportIncidents should return true when %s is in actions', + (actionTypeId) => { + const doc = { + attributes: { actions: [{ actionTypeId }, { actionTypeId: '.server-log' }] }, + } as SavedObjectUnsanitizedDoc; + expect(isAnyActionSupportIncidents(doc)).toBe(true); + } + ); + + test('isAnyActionSupportIncidents should return false when there is no connector that supports incidents', () => { + const doc = { + attributes: { actions: [{ actionTypeId: '.server-log' }] }, + } as SavedObjectUnsanitizedDoc; + expect(isAnyActionSupportIncidents(doc)).toBe(false); }); - expect(migration7112(alert, migrationContext)).toEqual({ - ...alert, - attributes: { - ...alert.attributes, + test('it does not transforms alerts when the right structure connectors is already applied', () => { + const migration7112 = getMigrations(encryptedSavedObjectsSetup)['7.11.2']; + const alert = getMockData({ actions: [ - alert.attributes.actions![0], + { + actionTypeId: '.server-log', + group: 'threshold met', + params: { + level: 'info', + message: 'log message', + }, + id: '99257478-e591-4560-b264-441bdd4fe1d9', + }, { actionTypeId: '.servicenow', group: 'threshold met', @@ -553,218 +554,109 @@ describe('7.11.2', () => { id: '1266562a-4e1f-4305-99ca-1b44c469b26e', }, ], - }, - }); - }); - - test.each(['.jira', '.servicenow', '.resilient'])( - 'isAnyActionSupportIncidents should return true when %s is in actions', - (actionTypeId) => { - const doc = { - attributes: { actions: [{ actionTypeId }, { actionTypeId: '.server-log' }] }, - } as SavedObjectUnsanitizedDoc; - expect(isAnyActionSupportIncidents(doc)).toBe(true); - } - ); + }); - test('isAnyActionSupportIncidents should return false when there is no connector that supports incidents', () => { - const doc = { - attributes: { actions: [{ actionTypeId: '.server-log' }] }, - } as SavedObjectUnsanitizedDoc; - expect(isAnyActionSupportIncidents(doc)).toBe(false); - }); + expect(migration7112(alert, migrationContext)).toEqual(alert); + }); - test('it does not transforms alerts when the right structure connectors is already applied', () => { - const migration7112 = getMigrations(encryptedSavedObjectsSetup)['7.11.2']; - const alert = getMockData({ - actions: [ - { - actionTypeId: '.server-log', - group: 'threshold met', - params: { - level: 'info', - message: 'log message', + test('if incident attribute is an empty object, copy back the related attributes from subActionParams back to incident', () => { + const migration7112 = getMigrations(encryptedSavedObjectsSetup)['7.11.2']; + const alert = getMockData({ + actions: [ + { + actionTypeId: '.server-log', + group: 'threshold met', + params: { + level: 'info', + message: 'log message', + }, + id: '99257478-e591-4560-b264-441bdd4fe1d9', }, - id: '99257478-e591-4560-b264-441bdd4fe1d9', - }, - { - actionTypeId: '.servicenow', - group: 'threshold met', - params: { - subAction: 'pushToService', - subActionParams: { - incident: { + { + actionTypeId: '.servicenow', + group: 'threshold met', + params: { + subAction: 'pushToService', + subActionParams: { short_description: 'SN short desc', description: 'SN desc', severity: '2', impact: '2', urgency: '2', + incident: {}, + comments: [{ commentId: '1', comment: 'sn comment' }], }, - comments: [{ commentId: '1', comment: 'sn comment' }], }, + id: '1266562a-4e1f-4305-99ca-1b44c469b26e', }, - id: '1266562a-4e1f-4305-99ca-1b44c469b26e', - }, - ], - }); - - expect(migration7112(alert, migrationContext)).toEqual(alert); - }); + ], + }); - test('if incident attribute is an empty object, copy back the related attributes from subActionParams back to incident', () => { - const migration7112 = getMigrations(encryptedSavedObjectsSetup)['7.11.2']; - const alert = getMockData({ - actions: [ - { - actionTypeId: '.server-log', - group: 'threshold met', - params: { - level: 'info', - message: 'log message', - }, - id: '99257478-e591-4560-b264-441bdd4fe1d9', - }, - { - actionTypeId: '.servicenow', - group: 'threshold met', - params: { - subAction: 'pushToService', - subActionParams: { - short_description: 'SN short desc', - description: 'SN desc', - severity: '2', - impact: '2', - urgency: '2', - incident: {}, - comments: [{ commentId: '1', comment: 'sn comment' }], + expect(migration7112(alert, migrationContext)).toEqual({ + ...alert, + attributes: { + ...alert.attributes, + actions: [ + alert.attributes.actions![0], + { + actionTypeId: '.servicenow', + group: 'threshold met', + params: { + subAction: 'pushToService', + subActionParams: { + incident: { + short_description: 'SN short desc', + description: 'SN desc', + severity: '2', + impact: '2', + urgency: '2', + }, + comments: [{ commentId: '1', comment: 'sn comment' }], + }, + }, + id: '1266562a-4e1f-4305-99ca-1b44c469b26e', }, - }, - id: '1266562a-4e1f-4305-99ca-1b44c469b26e', + ], }, - ], + }); }); - expect(migration7112(alert, migrationContext)).toEqual({ - ...alert, - attributes: { - ...alert.attributes, + test('custom action does not get migrated/loss', () => { + const migration7112 = getMigrations(encryptedSavedObjectsSetup)['7.11.2']; + const alert = getMockData({ actions: [ - alert.attributes.actions![0], { - actionTypeId: '.servicenow', + actionTypeId: '.mike', group: 'threshold met', params: { subAction: 'pushToService', subActionParams: { - incident: { - short_description: 'SN short desc', - description: 'SN desc', - severity: '2', - impact: '2', - urgency: '2', - }, + short_description: 'SN short desc', + description: 'SN desc', + severity: '2', + impact: '2', + urgency: '2', + incident: {}, comments: [{ commentId: '1', comment: 'sn comment' }], }, }, id: '1266562a-4e1f-4305-99ca-1b44c469b26e', }, ], - }, - }); - }); + }); - test('custom action does not get migrated/loss', () => { - const migration7112 = getMigrations(encryptedSavedObjectsSetup)['7.11.2']; - const alert = getMockData({ - actions: [ - { - actionTypeId: '.mike', - group: 'threshold met', - params: { - subAction: 'pushToService', - subActionParams: { - short_description: 'SN short desc', - description: 'SN desc', - severity: '2', - impact: '2', - urgency: '2', - incident: {}, - comments: [{ commentId: '1', comment: 'sn comment' }], - }, - }, - id: '1266562a-4e1f-4305-99ca-1b44c469b26e', - }, - ], + expect(migration7112(alert, migrationContext)).toEqual(alert); }); - - expect(migration7112(alert, migrationContext)).toEqual(alert); - }); -}); - -describe('7.13.0', () => { - beforeEach(() => { - jest.resetAllMocks(); - encryptedSavedObjectsSetup.createMigration.mockImplementation( - (shouldMigrateWhenPredicate, migration) => migration - ); }); - test('security solution alerts get migrated and remove null values', () => { - const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0']; - const alert = getMockData({ - alertTypeId: 'siem.signals', - params: { - author: ['Elastic'], - buildingBlockType: null, - description: - "This rule detects a known command and control pattern in network events. The FIN7 threat group is known to use this command and control technique, while maintaining persistence in their target's network.", - ruleId: '4a4e23cf-78a2-449c-bac3-701924c269d3', - index: ['packetbeat-*'], - falsePositives: [ - "This rule could identify benign domains that are formatted similarly to FIN7's command and control algorithm. Alerts should be investigated by an analyst to assess the validity of the individual observations.", - ], - from: 'now-6m', - immutable: true, - query: - 'event.category:(network OR network_traffic) AND type:(tls OR http) AND network.transport:tcp AND destination.domain:/[a-zA-Z]{4,5}.(pw|us|club|info|site|top)/ AND NOT destination.domain:zoom.us', - language: 'lucene', - license: 'Elastic License', - outputIndex: '.siem-signals-rylandherrick_2-default', - savedId: null, - timelineId: null, - timelineTitle: null, - meta: null, - filters: null, - maxSignals: 100, - riskScore: 73, - riskScoreMapping: [], - ruleNameOverride: null, - severity: 'high', - severityMapping: null, - threat: null, - threatFilters: null, - timestampOverride: null, - to: 'now', - type: 'query', - references: [ - 'https://www.fireeye.com/blog/threat-research/2018/08/fin7-pursuing-an-enigmatic-and-evasive-global-criminal-operation.html', - ], - note: - 'In the event this rule identifies benign domains in your environment, the `destination.domain` field in the rule can be modified to include those domains. Example: `...AND NOT destination.domain:(zoom.us OR benign.domain1 OR benign.domain2)`.', - version: 1, - exceptionsList: null, - threshold: { - field: null, - value: 5, - }, - }, - }); - expect(migration713(alert, migrationContext)).toEqual({ - ...alert, - attributes: { - ...alert.attributes, + describe('7.13.0', () => { + test('security solution alerts get migrated and remove null values', () => { + const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0']; + const alert = getMockData({ + alertTypeId: 'siem.signals', params: { author: ['Elastic'], + buildingBlockType: null, description: "This rule detects a known command and control pattern in network events. The FIN7 threat group is known to use this command and control technique, while maintaining persistence in their target's network.", ruleId: '4a4e23cf-78a2-449c-bac3-701924c269d3', @@ -779,12 +671,20 @@ describe('7.13.0', () => { language: 'lucene', license: 'Elastic License', outputIndex: '.siem-signals-rylandherrick_2-default', + savedId: null, + timelineId: null, + timelineTitle: null, + meta: null, + filters: null, maxSignals: 100, riskScore: 73, riskScoreMapping: [], + ruleNameOverride: null, severity: 'high', - severityMapping: [], - threat: [], + severityMapping: null, + threat: null, + threatFilters: null, + timestampOverride: null, to: 'now', type: 'query', references: [ @@ -793,169 +693,193 @@ describe('7.13.0', () => { note: 'In the event this rule identifies benign domains in your environment, the `destination.domain` field in the rule can be modified to include those domains. Example: `...AND NOT destination.domain:(zoom.us OR benign.domain1 OR benign.domain2)`.', version: 1, - exceptionsList: [], + exceptionsList: null, threshold: { - field: [], + field: null, value: 5, - cardinality: [], }, }, - }, - }); - }); + }); - test('non-null values in security solution alerts are not modified', () => { - const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0']; - const alert = getMockData({ - alertTypeId: 'siem.signals', - params: { - author: ['Elastic'], - buildingBlockType: 'default', - description: - "This rule detects a known command and control pattern in network events. The FIN7 threat group is known to use this command and control technique, while maintaining persistence in their target's network.", - ruleId: '4a4e23cf-78a2-449c-bac3-701924c269d3', - index: ['packetbeat-*'], - falsePositives: [ - "This rule could identify benign domains that are formatted similarly to FIN7's command and control algorithm. Alerts should be investigated by an analyst to assess the validity of the individual observations.", - ], - from: 'now-6m', - immutable: true, - query: - 'event.category:(network OR network_traffic) AND type:(tls OR http) AND network.transport:tcp AND destination.domain:/[a-zA-Z]{4,5}.(pw|us|club|info|site|top)/ AND NOT destination.domain:zoom.us', - language: 'lucene', - license: 'Elastic License', - outputIndex: '.siem-signals-rylandherrick_2-default', - savedId: 'saved-id', - timelineId: 'timeline-id', - timelineTitle: 'timeline-title', - meta: { - field: 'value', - }, - filters: ['filters'], - maxSignals: 100, - riskScore: 73, - riskScoreMapping: ['risk-score-mapping'], - ruleNameOverride: 'field.name', - severity: 'high', - severityMapping: ['severity-mapping'], - threat: [ - { - framework: 'MITRE ATT&CK', - tactic: { - id: 'TA0011', - name: 'Command and Control', - reference: 'https://attack.mitre.org/tactics/TA0011/', - }, - technique: [ - { - id: 'T1483', - name: 'Domain Generation Algorithms', - reference: 'https://attack.mitre.org/techniques/T1483/', - }, + expect(migration713(alert, migrationContext)).toEqual({ + ...alert, + attributes: { + ...alert.attributes, + params: { + author: ['Elastic'], + description: + "This rule detects a known command and control pattern in network events. The FIN7 threat group is known to use this command and control technique, while maintaining persistence in their target's network.", + ruleId: '4a4e23cf-78a2-449c-bac3-701924c269d3', + index: ['packetbeat-*'], + falsePositives: [ + "This rule could identify benign domains that are formatted similarly to FIN7's command and control algorithm. Alerts should be investigated by an analyst to assess the validity of the individual observations.", + ], + from: 'now-6m', + immutable: true, + query: + 'event.category:(network OR network_traffic) AND type:(tls OR http) AND network.transport:tcp AND destination.domain:/[a-zA-Z]{4,5}.(pw|us|club|info|site|top)/ AND NOT destination.domain:zoom.us', + language: 'lucene', + license: 'Elastic License', + outputIndex: '.siem-signals-rylandherrick_2-default', + maxSignals: 100, + riskScore: 73, + riskScoreMapping: [], + severity: 'high', + severityMapping: [], + threat: [], + to: 'now', + type: 'query', + references: [ + 'https://www.fireeye.com/blog/threat-research/2018/08/fin7-pursuing-an-enigmatic-and-evasive-global-criminal-operation.html', ], + note: + 'In the event this rule identifies benign domains in your environment, the `destination.domain` field in the rule can be modified to include those domains. Example: `...AND NOT destination.domain:(zoom.us OR benign.domain1 OR benign.domain2)`.', + version: 1, + exceptionsList: [], + threshold: { + field: [], + value: 5, + cardinality: [], + }, }, - ], - threatFilters: ['threat-filter'], - timestampOverride: 'event.ingested', - to: 'now', - type: 'query', - references: [ - 'https://www.fireeye.com/blog/threat-research/2018/08/fin7-pursuing-an-enigmatic-and-evasive-global-criminal-operation.html', - ], - note: - 'In the event this rule identifies benign domains in your environment, the `destination.domain` field in the rule can be modified to include those domains. Example: `...AND NOT destination.domain:(zoom.us OR benign.domain1 OR benign.domain2)`.', - version: 1, - exceptionsList: ['exceptions-list'], - }, + }, + }); }); - expect(migration713(alert, migrationContext)).toEqual(alert); - }); - - test('security solution threshold alert with string in threshold.field is migrated to array', () => { - const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0']; - const alert = getMockData({ - alertTypeId: 'siem.signals', - params: { - threshold: { - field: 'host.id', - value: 5, + test('non-null values in security solution alerts are not modified', () => { + const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0']; + const alert = getMockData({ + alertTypeId: 'siem.signals', + params: { + author: ['Elastic'], + buildingBlockType: 'default', + description: + "This rule detects a known command and control pattern in network events. The FIN7 threat group is known to use this command and control technique, while maintaining persistence in their target's network.", + ruleId: '4a4e23cf-78a2-449c-bac3-701924c269d3', + index: ['packetbeat-*'], + falsePositives: [ + "This rule could identify benign domains that are formatted similarly to FIN7's command and control algorithm. Alerts should be investigated by an analyst to assess the validity of the individual observations.", + ], + from: 'now-6m', + immutable: true, + query: + 'event.category:(network OR network_traffic) AND type:(tls OR http) AND network.transport:tcp AND destination.domain:/[a-zA-Z]{4,5}.(pw|us|club|info|site|top)/ AND NOT destination.domain:zoom.us', + language: 'lucene', + license: 'Elastic License', + outputIndex: '.siem-signals-rylandherrick_2-default', + savedId: 'saved-id', + timelineId: 'timeline-id', + timelineTitle: 'timeline-title', + meta: { + field: 'value', + }, + filters: ['filters'], + maxSignals: 100, + riskScore: 73, + riskScoreMapping: ['risk-score-mapping'], + ruleNameOverride: 'field.name', + severity: 'high', + severityMapping: ['severity-mapping'], + threat: [ + { + framework: 'MITRE ATT&CK', + tactic: { + id: 'TA0011', + name: 'Command and Control', + reference: 'https://attack.mitre.org/tactics/TA0011/', + }, + technique: [ + { + id: 'T1483', + name: 'Domain Generation Algorithms', + reference: 'https://attack.mitre.org/techniques/T1483/', + }, + ], + }, + ], + threatFilters: ['threat-filter'], + timestampOverride: 'event.ingested', + to: 'now', + type: 'query', + references: [ + 'https://www.fireeye.com/blog/threat-research/2018/08/fin7-pursuing-an-enigmatic-and-evasive-global-criminal-operation.html', + ], + note: + 'In the event this rule identifies benign domains in your environment, the `destination.domain` field in the rule can be modified to include those domains. Example: `...AND NOT destination.domain:(zoom.us OR benign.domain1 OR benign.domain2)`.', + version: 1, + exceptionsList: ['exceptions-list'], }, - }, + }); + + expect(migration713(alert, migrationContext)).toEqual(alert); }); - expect(migration713(alert, migrationContext)).toEqual({ - ...alert, - attributes: { - ...alert.attributes, + test('security solution threshold alert with string in threshold.field is migrated to array', () => { + const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0']; + const alert = getMockData({ + alertTypeId: 'siem.signals', params: { threshold: { - field: ['host.id'], + field: 'host.id', value: 5, - cardinality: [], }, - exceptionsList: [], - riskScoreMapping: [], - severityMapping: [], - threat: [], }, - }, - }); - }); + }); - test('security solution threshold alert with empty string in threshold.field is migrated to empty array', () => { - const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0']; - const alert = getMockData({ - alertTypeId: 'siem.signals', - params: { - threshold: { - field: '', - value: 5, + expect(migration713(alert, migrationContext)).toEqual({ + ...alert, + attributes: { + ...alert.attributes, + params: { + threshold: { + field: ['host.id'], + value: 5, + cardinality: [], + }, + exceptionsList: [], + riskScoreMapping: [], + severityMapping: [], + threat: [], + }, }, - }, + }); }); - expect(migration713(alert, migrationContext)).toEqual({ - ...alert, - attributes: { - ...alert.attributes, + test('security solution threshold alert with empty string in threshold.field is migrated to empty array', () => { + const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0']; + const alert = getMockData({ + alertTypeId: 'siem.signals', params: { threshold: { - field: [], + field: '', value: 5, - cardinality: [], }, - exceptionsList: [], - riskScoreMapping: [], - severityMapping: [], - threat: [], }, - }, - }); - }); + }); - test('security solution threshold alert with array in threshold.field and cardinality is left alone', () => { - const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0']; - const alert = getMockData({ - alertTypeId: 'siem.signals', - params: { - threshold: { - field: ['host.id'], - value: 5, - cardinality: [ - { - field: 'source.ip', - value: 10, + expect(migration713(alert, migrationContext)).toEqual({ + ...alert, + attributes: { + ...alert.attributes, + params: { + threshold: { + field: [], + value: 5, + cardinality: [], }, - ], + exceptionsList: [], + riskScoreMapping: [], + severityMapping: [], + threat: [], + }, }, - }, + }); }); - expect(migration713(alert, migrationContext)).toEqual({ - ...alert, - attributes: { - ...alert.attributes, + test('security solution threshold alert with array in threshold.field and cardinality is left alone', () => { + const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0']; + const alert = getMockData({ + alertTypeId: 'siem.signals', params: { threshold: { field: ['host.id'], @@ -967,64 +891,191 @@ describe('7.13.0', () => { }, ], }, - exceptionsList: [], - riskScoreMapping: [], - severityMapping: [], - threat: [], }, - }, - }); - }); + }); - test('security solution ML alert with string in machineLearningJobId is converted to an array', () => { - const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0']; - const alert = getMockData({ - alertTypeId: 'siem.signals', - params: { - anomalyThreshold: 20, - machineLearningJobId: 'my_job_id', - }, + expect(migration713(alert, migrationContext)).toEqual({ + ...alert, + attributes: { + ...alert.attributes, + params: { + threshold: { + field: ['host.id'], + value: 5, + cardinality: [ + { + field: 'source.ip', + value: 10, + }, + ], + }, + exceptionsList: [], + riskScoreMapping: [], + severityMapping: [], + threat: [], + }, + }, + }); }); - expect(migration713(alert, migrationContext)).toEqual({ - ...alert, - attributes: { - ...alert.attributes, + test('security solution ML alert with string in machineLearningJobId is converted to an array', () => { + const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0']; + const alert = getMockData({ + alertTypeId: 'siem.signals', params: { anomalyThreshold: 20, - machineLearningJobId: ['my_job_id'], - exceptionsList: [], - riskScoreMapping: [], - severityMapping: [], - threat: [], + machineLearningJobId: 'my_job_id', }, - }, - }); - }); + }); - test('security solution ML alert with an array in machineLearningJobId is preserved', () => { - const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0']; - const alert = getMockData({ - alertTypeId: 'siem.signals', - params: { - anomalyThreshold: 20, - machineLearningJobId: ['my_job_id', 'my_other_job_id'], - }, + expect(migration713(alert, migrationContext)).toEqual({ + ...alert, + attributes: { + ...alert.attributes, + params: { + anomalyThreshold: 20, + machineLearningJobId: ['my_job_id'], + exceptionsList: [], + riskScoreMapping: [], + severityMapping: [], + threat: [], + }, + }, + }); }); - expect(migration713(alert, migrationContext)).toEqual({ - ...alert, - attributes: { - ...alert.attributes, + test('security solution ML alert with an array in machineLearningJobId is preserved', () => { + const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0']; + const alert = getMockData({ + alertTypeId: 'siem.signals', params: { anomalyThreshold: 20, machineLearningJobId: ['my_job_id', 'my_other_job_id'], - exceptionsList: [], - riskScoreMapping: [], - severityMapping: [], - threat: [], }, - }, + }); + + expect(migration713(alert, migrationContext)).toEqual({ + ...alert, + attributes: { + ...alert.attributes, + params: { + anomalyThreshold: 20, + machineLearningJobId: ['my_job_id', 'my_other_job_id'], + exceptionsList: [], + riskScoreMapping: [], + severityMapping: [], + threat: [], + }, + }, + }); + }); + }); +}); + +describe('handles errors during migrations', () => { + beforeEach(() => { + jest.resetAllMocks(); + encryptedSavedObjectsSetup.createMigration.mockImplementation(() => () => { + throw new Error(`Can't migrate!`); + }); + }); + describe('7.10.0 throws if migration fails', () => { + test('should show the proper exception', () => { + const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; + const alert = getMockData({ + consumer: 'alerting', + }); + expect(() => { + migration710(alert, migrationContext); + }).toThrowError(`Can't migrate!`); + expect(migrationContext.log.error).toHaveBeenCalledWith( + `encryptedSavedObject 7.10.0 migration failed for alert ${alert.id} with error: Can't migrate!`, + { + migrations: { + alertDocument: { + ...alert, + attributes: { + ...alert.attributes, + }, + }, + }, + } + ); + }); + }); + + describe('7.11.0 throws if migration fails', () => { + test('should show the proper exception', () => { + const migration711 = getMigrations(encryptedSavedObjectsSetup)['7.11.0']; + const alert = getMockData({ + consumer: 'alerting', + }); + expect(() => { + migration711(alert, migrationContext); + }).toThrowError(`Can't migrate!`); + expect(migrationContext.log.error).toHaveBeenCalledWith( + `encryptedSavedObject 7.11.0 migration failed for alert ${alert.id} with error: Can't migrate!`, + { + migrations: { + alertDocument: { + ...alert, + attributes: { + ...alert.attributes, + }, + }, + }, + } + ); + }); + }); + + describe('7.11.2 throws if migration fails', () => { + test('should show the proper exception', () => { + const migration7112 = getMigrations(encryptedSavedObjectsSetup)['7.11.2']; + const alert = getMockData({ + consumer: 'alerting', + }); + expect(() => { + migration7112(alert, migrationContext); + }).toThrowError(`Can't migrate!`); + expect(migrationContext.log.error).toHaveBeenCalledWith( + `encryptedSavedObject 7.11.2 migration failed for alert ${alert.id} with error: Can't migrate!`, + { + migrations: { + alertDocument: { + ...alert, + attributes: { + ...alert.attributes, + }, + }, + }, + } + ); + }); + }); + + describe('7.13.0 throws if migration fails', () => { + test('should show the proper exception', () => { + const migration7130 = getMigrations(encryptedSavedObjectsSetup)['7.13.0']; + const alert = getMockData({ + consumer: 'alerting', + }); + expect(() => { + migration7130(alert, migrationContext); + }).toThrowError(`Can't migrate!`); + expect(migrationContext.log.error).toHaveBeenCalledWith( + `encryptedSavedObject 7.13.0 migration failed for alert ${alert.id} with error: Can't migrate!`, + { + migrations: { + alertDocument: { + ...alert, + attributes: { + ...alert.attributes, + }, + }, + }, + } + ); }); }); }); diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations.ts b/x-pack/plugins/alerting/server/saved_objects/migrations.ts index 8969e3ad0fdef..9f6adeb27083a 100644 --- a/x-pack/plugins/alerting/server/saved_objects/migrations.ts +++ b/x-pack/plugins/alerting/server/saved_objects/migrations.ts @@ -16,6 +16,7 @@ import { } from '../../../../../src/core/server'; import { RawAlert, RawAlertAction } from '../types'; import { EncryptedSavedObjectsPluginSetup } from '../../../encrypted_saved_objects/server'; +import type { IsMigrationNeededPredicate } from '../../../encrypted_saved_objects/server'; const SIEM_APP_ID = 'securitySolution'; const SIEM_SERVER_APP_ID = 'siem'; @@ -29,6 +30,18 @@ type AlertMigration = ( doc: SavedObjectUnsanitizedDoc ) => SavedObjectUnsanitizedDoc; +function createEsoMigration( + encryptedSavedObjects: EncryptedSavedObjectsPluginSetup, + isMigrationNeededPredicate: IsMigrationNeededPredicate, + migrationFunc: AlertMigration +) { + return encryptedSavedObjects.createMigration({ + isMigrationNeededPredicate, + migration: migrationFunc, + shouldMigrateIfDecryptionFails: true, // shouldMigrateIfDecryptionFails flag that applies the migration to undecrypted document if decryption fails + }); +} + const SUPPORT_INCIDENTS_ACTION_TYPES = ['.servicenow', '.jira', '.resilient']; export const isAnyActionSupportIncidents = (doc: SavedObjectUnsanitizedDoc): boolean => @@ -42,11 +55,10 @@ export const isSecuritySolutionRule = (doc: SavedObjectUnsanitizedDoc) export function getMigrations( encryptedSavedObjects: EncryptedSavedObjectsPluginSetup ): SavedObjectMigrationMap { - const migrationWhenRBACWasIntroduced = encryptedSavedObjects.createMigration( - function shouldBeMigrated(doc): doc is SavedObjectUnsanitizedDoc { - // migrate all documents in 7.10 in order to add the "meta" RBAC field - return true; - }, + const migrationWhenRBACWasIntroduced = createEsoMigration( + encryptedSavedObjects, + // migrate all documents in 7.10 in order to add the "meta" RBAC field + (doc): doc is SavedObjectUnsanitizedDoc => true, pipeMigrations( markAsLegacyAndChangeConsumer, setAlertIdAsDefaultDedupkeyOnPagerDutyActions, @@ -54,21 +66,21 @@ export function getMigrations( ) ); - const migrationAlertUpdatedAtAndNotifyWhen = encryptedSavedObjects.createMigration< - RawAlert, - RawAlert - >( + const migrationAlertUpdatedAtAndNotifyWhen = createEsoMigration( + encryptedSavedObjects, // migrate all documents in 7.11 in order to add the "updatedAt" and "notifyWhen" fields (doc): doc is SavedObjectUnsanitizedDoc => true, pipeMigrations(setAlertUpdatedAtDate, setNotifyWhen) ); - const migrationActions7112 = encryptedSavedObjects.createMigration( + const migrationActions7112 = createEsoMigration( + encryptedSavedObjects, (doc): doc is SavedObjectUnsanitizedDoc => isAnyActionSupportIncidents(doc), pipeMigrations(restructureConnectorsThatSupportIncident) ); - const migrationSecurityRules713 = encryptedSavedObjects.createMigration( + const migrationSecurityRules713 = createEsoMigration( + encryptedSavedObjects, (doc): doc is SavedObjectUnsanitizedDoc => isSecuritySolutionRule(doc), pipeMigrations(removeNullsFromSecurityRules) ); @@ -97,8 +109,8 @@ function executeMigrationWithErrorHandling( }, } ); + throw ex; } - return doc; }; } diff --git a/x-pack/plugins/encrypted_saved_objects/README.md b/x-pack/plugins/encrypted_saved_objects/README.md index 99ebf771126d5..97e1ea5b657b3 100644 --- a/x-pack/plugins/encrypted_saved_objects/README.md +++ b/x-pack/plugins/encrypted_saved_objects/README.md @@ -111,6 +111,7 @@ The `createMigration` function takes four arguments: |---|---|---| |isMigrationNeededPredicate|A predicate which is called for each document, prior to being decrypted, which confirms whether a document requires migration or not. This predicate is important as the decryption step is costly and we would rather not decrypt and re-encrypt a document if we can avoid it.|function| |migration|A migration function which will migrate each decrypted document from the old shape to the new one.|function| +|shouldMigrateIfDecryptionFails|Optional. A boolean flag which indicates whether to proceed with migration if a document fails to decrypt. If this is not set or if it is set to `false`, decryption errors will be thrown. If set to `true`, a warning will be logged and the migration function will be applied to the original encrypted document. Set this to `true` if you don't want decryption failures to block Kibana upgrades. |boolean| |inputType|Optional. An `EncryptedSavedObjectTypeRegistration` which describes the ESOType of the input (the document prior to migration). If this type isn't provided, we'll assume the input doc follows the registered type. |object| |migratedType| Optional. An `EncryptedSavedObjectTypeRegistration` which describes the ESOType of the output (the document after migration). If this type isn't provided, we'll assume the migrated doc follows the registered type.|object| @@ -123,11 +124,11 @@ encryptedSavedObjects.registerType({ attributesToExcludeFromAAD: new Set(['mutedInstanceIds', 'updatedBy']), }); -const migration790 = encryptedSavedObjects.createMigration( - function shouldBeMigrated(doc): doc is SavedObjectUnsanitizedDoc { +const migration790 = encryptedSavedObjects.createMigration({ + isMigrationNeededPredicate: function shouldBeMigrated(doc): doc is SavedObjectUnsanitizedDoc { return doc.consumer === 'alerting' || doc.consumer === undefined; }, - (doc: SavedObjectUnsanitizedDoc): SavedObjectUnsanitizedDoc => { + migration: (doc: SavedObjectUnsanitizedDoc): SavedObjectUnsanitizedDoc => { const { attributes: { consumer }, } = doc; @@ -139,7 +140,7 @@ const migration790 = encryptedSavedObjects.createMigration( }, }; } -); +}); ``` In the above example you can see thwe following: @@ -174,11 +175,11 @@ encryptedSavedObjects.registerType({ attributesToExcludeFromAAD: new Set(['mutedInstanceIds', 'updatedBy']), }); -const migration790 = encryptedSavedObjects.createMigration( - function shouldBeMigrated(doc): doc is SavedObjectUnsanitizedDoc { +const migration790 = encryptedSavedObjects.createMigration({ + isMigrationNeededPredicate: function shouldBeMigrated(doc): doc is SavedObjectUnsanitizedDoc { return doc.consumer === 'alerting' || doc.consumer === undefined; }, - (doc: SavedObjectUnsanitizedDoc): SavedObjectUnsanitizedDoc => { + migration: (doc: SavedObjectUnsanitizedDoc): SavedObjectUnsanitizedDoc => { const { attributes: { legacyEncryptedField, ...attributes }, } = doc; @@ -189,12 +190,12 @@ const migration790 = encryptedSavedObjects.createMigration( }, }; }, - { + inputType: { type: 'alert', attributesToEncrypt: new Set(['apiKey', 'legacyEncryptedField']), attributesToExcludeFromAAD: new Set(['mutedInstanceIds', 'updatedBy']), } -); +}); ``` As you can see in this example we provide a legacy type which describes the _input_ which needs to be decrypted. @@ -209,26 +210,26 @@ encryptedSavedObjects.registerType({ attributesToExcludeFromAAD: new Set(['mutedInstanceIds', 'updatedBy']), }); -const migration780 = encryptedSavedObjects.createMigration( - function shouldBeMigrated(doc): doc is SavedObjectUnsanitizedDoc { +const migration780 = encryptedSavedObjects.createMigration({ + isMigrationNeededPredicate: function shouldBeMigrated(doc): doc is SavedObjectUnsanitizedDoc { // ... }, - (doc: SavedObjectUnsanitizedDoc): SavedObjectUnsanitizedDoc => { + migration: (doc: SavedObjectUnsanitizedDoc): SavedObjectUnsanitizedDoc => { // ... }, // legacy input type - { + inputType: { type: 'alert', attributesToEncrypt: new Set(['apiKey', 'legacyEncryptedField']), attributesToExcludeFromAAD: new Set(['mutedInstanceIds', 'updatedBy']), }, // legacy migration type - { + migratedType: { type: 'alert', attributesToEncrypt: new Set(['apiKey', 'legacyEncryptedField']), attributesToExcludeFromAAD: new Set(['mutedInstanceIds', 'updatedBy', 'legacyEncryptedField']), } -); +}); ``` ## Testing diff --git a/x-pack/plugins/encrypted_saved_objects/server/create_migration.test.ts b/x-pack/plugins/encrypted_saved_objects/server/create_migration.test.ts index 548340fbb6463..32aac4a37e4c2 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/create_migration.test.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/create_migration.test.ts @@ -9,6 +9,7 @@ import type { SavedObjectUnsanitizedDoc } from 'src/core/server'; import { migrationMocks } from 'src/core/server/mocks'; import { getCreateMigration } from './create_migration'; +import { EncryptionError, EncryptionErrorOperation } from './crypto'; import { encryptedSavedObjectsServiceMock } from './crypto/index.mock'; afterEach(() => { @@ -39,20 +40,20 @@ describe('createMigration()', () => { encryptedSavedObjectsServiceMock.create() ); expect(() => - migrationCreator( - function (doc): doc is SavedObjectUnsanitizedDoc { + migrationCreator({ + isMigrationNeededPredicate(doc): doc is SavedObjectUnsanitizedDoc { return true; }, - (doc) => doc, - { + migration: (doc) => doc, + inputType: { type: 'known-type-1', attributesToEncrypt: new Set(), }, - { + migratedType: { type: 'known-type-2', attributesToEncrypt: new Set(), - } - ) + }, + }) ).toThrowErrorMatchingInlineSnapshot( `"An Invalid Encrypted Saved Objects migration is trying to migrate across types (\\"known-type-1\\" => \\"known-type-2\\"), which isn't permitted"` ); @@ -68,12 +69,12 @@ describe('createMigration()', () => { encryptionSavedObjectService, instantiateServiceWithLegacyType ); - const noopMigration = migrationCreator( - function (doc): doc is SavedObjectUnsanitizedDoc { + const noopMigration = migrationCreator({ + isMigrationNeededPredicate(doc): doc is SavedObjectUnsanitizedDoc { return true; }, - (doc) => doc - ); + migration: (doc) => doc, + }); const attributes = { firstAttr: 'first_attr', @@ -111,6 +112,402 @@ describe('createMigration()', () => { attributes ); }); + + it('throws error on decryption failure if shouldMigrateIfDecryptionFails is false', () => { + const instantiateServiceWithLegacyType = jest.fn(() => + encryptedSavedObjectsServiceMock.create() + ); + const migrationFunc = jest.fn((doc) => doc); + + const migrationCreator = getCreateMigration( + encryptionSavedObjectService, + instantiateServiceWithLegacyType + ); + const noopMigration = migrationCreator({ + isMigrationNeededPredicate(doc): doc is SavedObjectUnsanitizedDoc { + return true; + }, + migration: migrationFunc, + }); + + const attributes = { + firstAttr: 'first_attr', + }; + + encryptionSavedObjectService.decryptAttributesSync.mockImplementationOnce(() => { + throw new Error('decryption failed!'); + }); + + expect(() => { + noopMigration( + { + id: '123', + type: 'known-type-1', + namespace: 'namespace', + attributes, + }, + migrationContext + ); + }).toThrowError(`decryption failed!`); + + expect(encryptionSavedObjectService.decryptAttributesSync).toHaveBeenCalledWith( + { + id: '123', + type: 'known-type-1', + namespace: 'namespace', + }, + attributes, + { convertToMultiNamespaceType: false } + ); + + expect(migrationFunc).not.toHaveBeenCalled(); + expect(encryptionSavedObjectService.encryptAttributesSync).not.toHaveBeenCalled(); + }); + + it('throws error on decryption failure if shouldMigrateIfDecryptionFails is true but error is not encryption error', () => { + const instantiateServiceWithLegacyType = jest.fn(() => + encryptedSavedObjectsServiceMock.create() + ); + const migrationFunc = jest.fn((doc) => doc); + + const migrationCreator = getCreateMigration( + encryptionSavedObjectService, + instantiateServiceWithLegacyType + ); + const noopMigration = migrationCreator({ + isMigrationNeededPredicate(doc): doc is SavedObjectUnsanitizedDoc { + return true; + }, + migration: migrationFunc, + shouldMigrateIfDecryptionFails: true, + }); + + const attributes = { + firstAttr: 'first_attr', + }; + + encryptionSavedObjectService.decryptAttributesSync.mockImplementationOnce(() => { + throw new Error('decryption failed!'); + }); + + expect(() => { + noopMigration( + { + id: '123', + type: 'known-type-1', + namespace: 'namespace', + attributes, + }, + migrationContext + ); + }).toThrowError(`decryption failed!`); + + expect(encryptionSavedObjectService.decryptAttributesSync).toHaveBeenCalledWith( + { + id: '123', + type: 'known-type-1', + namespace: 'namespace', + }, + attributes, + { convertToMultiNamespaceType: false } + ); + + expect(encryptionSavedObjectService.stripOrDecryptAttributesSync).not.toHaveBeenCalled(); + expect(migrationFunc).not.toHaveBeenCalled(); + expect(encryptionSavedObjectService.encryptAttributesSync).not.toHaveBeenCalled(); + }); + + it('runs migration function on decryption failure if shouldMigrateIfDecryptionFails is true and error is encryption error', () => { + const instantiateServiceWithLegacyType = jest.fn(() => + encryptedSavedObjectsServiceMock.create() + ); + const migrationFunc = jest.fn((doc) => doc); + + const migrationCreator = getCreateMigration( + encryptionSavedObjectService, + instantiateServiceWithLegacyType + ); + const noopMigration = migrationCreator({ + isMigrationNeededPredicate(doc): doc is SavedObjectUnsanitizedDoc { + return true; + }, + migration: migrationFunc, + shouldMigrateIfDecryptionFails: true, + }); + + const attributes = { + firstAttr: 'first_attr', + attrToStrip: 'secret', + }; + const strippedAttributes = { + firstAttr: 'first_attr', + }; + + encryptionSavedObjectService.decryptAttributesSync.mockImplementationOnce(() => { + throw new EncryptionError( + `Unable to decrypt attribute "'attribute'"`, + 'attribute', + EncryptionErrorOperation.Decryption, + new Error('decryption failed') + ); + }); + + encryptionSavedObjectService.stripOrDecryptAttributesSync.mockReturnValueOnce({ + attributes: strippedAttributes, + }); + + noopMigration( + { + id: '123', + type: 'known-type-1', + namespace: 'namespace', + attributes, + }, + migrationContext + ); + + expect(encryptionSavedObjectService.stripOrDecryptAttributesSync).toHaveBeenCalledWith( + { + id: '123', + type: 'known-type-1', + namespace: 'namespace', + }, + attributes, + { convertToMultiNamespaceType: false } + ); + + expect(migrationFunc).toHaveBeenCalled(); + expect(encryptionSavedObjectService.encryptAttributesSync).toHaveBeenCalledWith( + { + id: '123', + type: 'known-type-1', + namespace: 'namespace', + }, + strippedAttributes + ); + }); + + it('throws error on migration failure', () => { + const instantiateServiceWithLegacyType = jest.fn(() => + encryptedSavedObjectsServiceMock.create() + ); + const migrationFunc = jest.fn(() => { + throw new Error('migration failed!'); + }); + + const migrationCreator = getCreateMigration( + encryptionSavedObjectService, + instantiateServiceWithLegacyType + ); + const noopMigration = migrationCreator({ + isMigrationNeededPredicate(doc): doc is SavedObjectUnsanitizedDoc { + return true; + }, + migration: migrationFunc, + }); + + const attributes = { + firstAttr: 'first_attr', + }; + + encryptionSavedObjectService.decryptAttributesSync.mockReturnValueOnce(attributes); + + expect(() => { + noopMigration( + { + id: '123', + type: 'known-type-1', + namespace: 'namespace', + attributes, + }, + migrationContext + ); + }).toThrowError(`migration failed!`); + + expect(encryptionSavedObjectService.decryptAttributesSync).toHaveBeenCalledWith( + { + id: '123', + type: 'known-type-1', + namespace: 'namespace', + }, + attributes, + { convertToMultiNamespaceType: false } + ); + + expect(migrationFunc).toHaveBeenCalled(); + expect(encryptionSavedObjectService.encryptAttributesSync).not.toHaveBeenCalled(); + }); + + it('throws error on migration failure even if shouldMigrateIfDecryptionFails is true', () => { + const instantiateServiceWithLegacyType = jest.fn(() => + encryptedSavedObjectsServiceMock.create() + ); + const migrationFunc = jest.fn(() => { + throw new Error('migration failed!'); + }); + + const migrationCreator = getCreateMigration( + encryptionSavedObjectService, + instantiateServiceWithLegacyType + ); + const noopMigration = migrationCreator({ + isMigrationNeededPredicate(doc): doc is SavedObjectUnsanitizedDoc { + return true; + }, + migration: migrationFunc, + shouldMigrateIfDecryptionFails: true, + }); + + const attributes = { + firstAttr: 'first_attr', + }; + + encryptionSavedObjectService.decryptAttributesSync.mockReturnValueOnce(attributes); + + expect(() => { + noopMigration( + { + id: '123', + type: 'known-type-1', + namespace: 'namespace', + attributes, + }, + migrationContext + ); + }).toThrowError(`migration failed!`); + + expect(encryptionSavedObjectService.decryptAttributesSync).toHaveBeenCalledWith( + { + id: '123', + type: 'known-type-1', + namespace: 'namespace', + }, + attributes, + { convertToMultiNamespaceType: false } + ); + + expect(migrationFunc).toHaveBeenCalled(); + expect(encryptionSavedObjectService.encryptAttributesSync).not.toHaveBeenCalled(); + }); + + it('throws error on encryption failure', () => { + const instantiateServiceWithLegacyType = jest.fn(() => + encryptedSavedObjectsServiceMock.create() + ); + const migrationFunc = jest.fn((doc) => doc); + + const migrationCreator = getCreateMigration( + encryptionSavedObjectService, + instantiateServiceWithLegacyType + ); + const noopMigration = migrationCreator({ + isMigrationNeededPredicate(doc): doc is SavedObjectUnsanitizedDoc { + return true; + }, + migration: migrationFunc, + }); + + const attributes = { + firstAttr: 'first_attr', + }; + + encryptionSavedObjectService.decryptAttributesSync.mockReturnValueOnce(attributes); + encryptionSavedObjectService.encryptAttributesSync.mockImplementationOnce(() => { + throw new Error('encryption failed!'); + }); + + expect(() => { + noopMigration( + { + id: '123', + type: 'known-type-1', + namespace: 'namespace', + attributes, + }, + migrationContext + ); + }).toThrowError(`encryption failed!`); + + expect(encryptionSavedObjectService.decryptAttributesSync).toHaveBeenCalledWith( + { + id: '123', + type: 'known-type-1', + namespace: 'namespace', + }, + attributes, + { convertToMultiNamespaceType: false } + ); + + expect(migrationFunc).toHaveBeenCalled(); + expect(encryptionSavedObjectService.encryptAttributesSync).toHaveBeenCalledWith( + { + id: '123', + type: 'known-type-1', + namespace: 'namespace', + }, + attributes + ); + }); + + it('throws error on encryption failure even if shouldMigrateIfDecryptionFails is true', () => { + const instantiateServiceWithLegacyType = jest.fn(() => + encryptedSavedObjectsServiceMock.create() + ); + const migrationFunc = jest.fn((doc) => doc); + + const migrationCreator = getCreateMigration( + encryptionSavedObjectService, + instantiateServiceWithLegacyType + ); + const noopMigration = migrationCreator({ + isMigrationNeededPredicate(doc): doc is SavedObjectUnsanitizedDoc { + return true; + }, + migration: migrationFunc, + shouldMigrateIfDecryptionFails: true, + }); + + const attributes = { + firstAttr: 'first_attr', + }; + + encryptionSavedObjectService.decryptAttributesSync.mockReturnValueOnce(attributes); + encryptionSavedObjectService.encryptAttributesSync.mockImplementationOnce(() => { + throw new Error('encryption failed!'); + }); + + expect(() => { + noopMigration( + { + id: '123', + type: 'known-type-1', + namespace: 'namespace', + attributes, + }, + migrationContext + ); + }).toThrowError(`encryption failed!`); + + expect(encryptionSavedObjectService.decryptAttributesSync).toHaveBeenCalledWith( + { + id: '123', + type: 'known-type-1', + namespace: 'namespace', + }, + attributes, + { convertToMultiNamespaceType: false } + ); + + expect(migrationFunc).toHaveBeenCalled(); + expect(encryptionSavedObjectService.encryptAttributesSync).toHaveBeenCalledWith( + { + id: '123', + type: 'known-type-1', + namespace: 'namespace', + }, + attributes + ); + }); }); describe('migration of a single legacy type', () => { @@ -122,13 +519,13 @@ describe('createMigration()', () => { encryptionSavedObjectService, instantiateServiceWithLegacyType ); - const noopMigration = migrationCreator( - function (doc): doc is SavedObjectUnsanitizedDoc { + const noopMigration = migrationCreator({ + isMigrationNeededPredicate(doc): doc is SavedObjectUnsanitizedDoc { return true; }, - (doc) => doc, - inputType - ); + migration: (doc) => doc, + inputType, + }); const attributes = { firstAttr: 'first_attr', @@ -183,12 +580,12 @@ describe('createMigration()', () => { encryptionSavedObjectService, instantiateServiceWithLegacyType ); - const noopMigration = migrationCreator( - function (doc): doc is SavedObjectUnsanitizedDoc { + const noopMigration = migrationCreator({ + isMigrationNeededPredicate(doc): doc is SavedObjectUnsanitizedDoc { return true; }, - (doc) => doc - ); + migration: (doc) => doc, + }); const attributes = { firstAttr: 'first_attr', @@ -257,15 +654,15 @@ describe('createMigration()', () => { encryptionSavedObjectService, instantiateServiceWithLegacyType ); - return migrationCreator( - function (doc): doc is SavedObjectUnsanitizedDoc { + return migrationCreator({ + isMigrationNeededPredicate(doc): doc is SavedObjectUnsanitizedDoc { // migrate doc that have the second field return ( typeof (doc as SavedObjectUnsanitizedDoc).attributes.nonEncryptedAttr === 'string' ); }, - ({ attributes: { firstAttr, nonEncryptedAttr }, ...doc }) => ({ + migration: ({ attributes: { firstAttr, nonEncryptedAttr }, ...doc }) => ({ attributes: { // modify an encrypted field firstAttr: `~~${firstAttr}~~`, @@ -275,8 +672,8 @@ describe('createMigration()', () => { ...doc, }), inputType, - migrationType - ); + migratedType: migrationType, + }); } it('doesnt decrypt saved objects that dont need to be migrated', async () => { diff --git a/x-pack/plugins/encrypted_saved_objects/server/create_migration.ts b/x-pack/plugins/encrypted_saved_objects/server/create_migration.ts index beace2b17fe08..b9e6dcf710924 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/create_migration.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/create_migration.ts @@ -11,6 +11,7 @@ import type { SavedObjectUnsanitizedDoc, } from 'src/core/server'; +import { EncryptionError } from './crypto'; import type { EncryptedSavedObjectsService, EncryptedSavedObjectTypeRegistration } from './crypto'; import { normalizeNamespace } from './saved_objects'; @@ -19,20 +20,28 @@ type SavedObjectOptionalMigrationFn = ( context: SavedObjectMigrationContext ) => SavedObjectUnsanitizedDoc; -type IsMigrationNeededPredicate = ( +export type IsMigrationNeededPredicate = ( encryptedDoc: | SavedObjectUnsanitizedDoc | SavedObjectUnsanitizedDoc ) => encryptedDoc is SavedObjectUnsanitizedDoc; +export interface CreateEncryptedSavedObjectsMigrationFnOpts< + InputAttributes = unknown, + MigratedAttributes = InputAttributes +> { + isMigrationNeededPredicate: IsMigrationNeededPredicate; + migration: SavedObjectMigrationFn; + shouldMigrateIfDecryptionFails?: boolean; + inputType?: EncryptedSavedObjectTypeRegistration; + migratedType?: EncryptedSavedObjectTypeRegistration; +} + export type CreateEncryptedSavedObjectsMigrationFn = < InputAttributes = unknown, MigratedAttributes = InputAttributes >( - isMigrationNeededPredicate: IsMigrationNeededPredicate, - migration: SavedObjectMigrationFn, - inputType?: EncryptedSavedObjectTypeRegistration, - migratedType?: EncryptedSavedObjectTypeRegistration + opts: CreateEncryptedSavedObjectsMigrationFnOpts ) => SavedObjectOptionalMigrationFn; export const getCreateMigration = ( @@ -40,12 +49,15 @@ export const getCreateMigration = ( instantiateServiceWithLegacyType: ( typeRegistration: EncryptedSavedObjectTypeRegistration ) => EncryptedSavedObjectsService -): CreateEncryptedSavedObjectsMigrationFn => ( - isMigrationNeededPredicate, - migration, - inputType, - migratedType -) => { +): CreateEncryptedSavedObjectsMigrationFn => (opts) => { + const { + isMigrationNeededPredicate, + migration, + shouldMigrateIfDecryptionFails, + inputType, + migratedType, + } = opts; + if (inputType && migratedType && inputType.type !== migratedType.type) { throw new Error( `An Invalid Encrypted Saved Objects migration is trying to migrate across types ("${inputType.type}" => "${migratedType.type}"), which isn't permitted` @@ -80,20 +92,32 @@ export const getCreateMigration = ( const encryptDescriptor = { id, type, namespace: encryptedDoc.namespace }; // decrypt the attributes using the input type definition - // then migrate the document - // then encrypt the attributes using the migration type definition - return mapAttributes( - migration( - mapAttributes(encryptedDoc, (inputAttributes) => - inputService.decryptAttributesSync(decryptDescriptor, inputAttributes, { - convertToMultiNamespaceType, - }) - ), - context - ), - (migratedAttributes) => - migratedService.encryptAttributesSync(encryptDescriptor, migratedAttributes) - ); + // if an error occurs during decryption, use the shouldMigrateIfDecryptionFails flag + // to determine whether to throw the error or continue the migration + // if we are continuing the migration, strip encrypted attributes from the document using stripOrDecryptAttributesSync + const documentToMigrate = mapAttributes(encryptedDoc, (inputAttributes) => { + try { + return inputService.decryptAttributesSync(decryptDescriptor, inputAttributes, { + convertToMultiNamespaceType, + }); + } catch (err) { + if (!shouldMigrateIfDecryptionFails || !(err instanceof EncryptionError)) { + throw err; + } + + context.log.warn( + `Decryption failed for encrypted Saved Object "${encryptedDoc.id}" of type "${encryptedDoc.type}" with error: ${err.message}. Encrypted attributes have been stripped from the original document and migration will be applied but this may cause errors later on.` + ); + return inputService.stripOrDecryptAttributesSync(decryptDescriptor, inputAttributes, { + convertToMultiNamespaceType, + }).attributes; + } + }); + + // migrate and encrypt the document + return mapAttributes(migration(documentToMigrate, context), (migratedAttributes) => { + return migratedService.encryptAttributesSync(encryptDescriptor, migratedAttributes); + }); }; }; diff --git a/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.mocks.ts b/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.mocks.ts index d7ff27ced38c6..2e8e37e61deed 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.mocks.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.mocks.ts @@ -19,6 +19,7 @@ function createEncryptedSavedObjectsServiceMock() { decryptAttributes: jest.fn(), encryptAttributesSync: jest.fn(), decryptAttributesSync: jest.fn(), + stripOrDecryptAttributesSync: jest.fn(), } as unknown) as jest.Mocked; } diff --git a/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.test.ts b/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.test.ts index 5ac6467e8d78b..0625199eeed63 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.test.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.test.ts @@ -295,6 +295,210 @@ describe('#stripOrDecryptAttributes', () => { }); }); +describe('#stripOrDecryptAttributesSync', () => { + it('does not strip attributes from unknown types', () => { + const attributes = { attrOne: 'one', attrTwo: 'two', attrThree: 'three' }; + + expect( + service.stripOrDecryptAttributesSync({ id: 'unknown-id', type: 'unknown-type' }, attributes) + ).toEqual({ attributes: { attrOne: 'one', attrTwo: 'two', attrThree: 'three' } }); + }); + + it('does not strip any attributes if none of them are supposed to be encrypted', () => { + const attributes = { attrOne: 'one', attrTwo: 'two', attrThree: 'three' }; + + service.registerType({ type: 'known-type-1', attributesToEncrypt: new Set(['attrFour']) }); + + expect( + service.stripOrDecryptAttributesSync({ id: 'known-id', type: 'known-type-1' }, attributes) + ).toEqual({ attributes: { attrOne: 'one', attrTwo: 'two', attrThree: 'three' } }); + }); + + it('strips only attributes that are supposed to be encrypted', () => { + const attributes = { attrOne: 'one', attrTwo: 'two', attrThree: 'three' }; + + service.registerType({ + type: 'known-type-1', + attributesToEncrypt: new Set(['attrOne', 'attrThree']), + }); + + expect( + service.stripOrDecryptAttributesSync({ id: 'known-id', type: 'known-type-1' }, attributes) + ).toEqual({ attributes: { attrTwo: 'two' } }); + }); + + describe('with `dangerouslyExposeValue`', () => { + it('decrypts and exposes values with `dangerouslyExposeValue` set to `true`', () => { + service.registerType({ + type: 'known-type-1', + attributesToEncrypt: new Set([ + 'attrOne', + { key: 'attrThree', dangerouslyExposeValue: true }, + ]), + }); + + const attributes = { attrOne: 'one', attrTwo: 'two', attrThree: 'three' }; + const encryptedAttributes = service.encryptAttributesSync( + { type: 'known-type-1', id: 'object-id' }, + attributes + ); + + const mockUser = mockAuthenticatedUser(); + expect( + service.stripOrDecryptAttributesSync( + { type: 'known-type-1', id: 'object-id' }, + encryptedAttributes, + undefined, + { user: mockUser } + ) + ).toEqual({ attributes: { attrTwo: 'two', attrThree: 'three' } }); + + expect(mockAuditLogger.decryptAttributesSuccess).toHaveBeenCalledTimes(1); + expect(mockAuditLogger.decryptAttributesSuccess).toHaveBeenCalledWith( + ['attrThree'], + { type: 'known-type-1', id: 'object-id' }, + mockUser + ); + }); + + it('exposes values with `dangerouslyExposeValue` set to `true` using original attributes if provided', () => { + service.registerType({ + type: 'known-type-1', + attributesToEncrypt: new Set([ + 'attrOne', + { key: 'attrThree', dangerouslyExposeValue: true }, + ]), + }); + + const attributes = { attrOne: 'one', attrTwo: 'two', attrThree: 'three' }; + const encryptedAttributes = { + attrOne: 'fake-enc-one', + attrTwo: 'two', + attrThree: 'fake-enc-three', + }; + + expect( + service.stripOrDecryptAttributesSync( + { type: 'known-type-1', id: 'object-id' }, + encryptedAttributes, + attributes + ) + ).toEqual({ attributes: { attrTwo: 'two', attrThree: 'three' } }); + + expect(mockAuditLogger.decryptAttributesSuccess).not.toHaveBeenCalled(); + expect(mockAuditLogger.decryptAttributeFailure).not.toHaveBeenCalled(); + }); + + it('strips attributes with `dangerouslyExposeValue` set to `true` if failed to decrypt', () => { + service.registerType({ + type: 'known-type-1', + attributesToEncrypt: new Set([ + 'attrOne', + { key: 'attrThree', dangerouslyExposeValue: true }, + ]), + }); + + const attributes = { + attrZero: 'zero', + attrOne: 'one', + attrTwo: 'two', + attrThree: 'three', + attrFour: 'four', + }; + const encryptedAttributes = service.encryptAttributesSync( + { type: 'known-type-1', id: 'object-id' }, + attributes + ); + + encryptedAttributes.attrThree = 'some-undecryptable-value'; + + const mockUser = mockAuthenticatedUser(); + const { attributes: decryptedAttributes, error } = service.stripOrDecryptAttributesSync( + { type: 'known-type-1', id: 'object-id' }, + encryptedAttributes, + undefined, + { user: mockUser } + ); + + expect(decryptedAttributes).toEqual({ attrZero: 'zero', attrTwo: 'two', attrFour: 'four' }); + expect(error).toMatchInlineSnapshot(`[Error: Unable to decrypt attribute "attrThree"]`); + + expect(mockAuditLogger.decryptAttributesSuccess).not.toHaveBeenCalled(); + expect(mockAuditLogger.decryptAttributeFailure).toHaveBeenCalledWith( + 'attrThree', + { type: 'known-type-1', id: 'object-id' }, + mockUser + ); + }); + }); + + describe('without encryption key', () => { + beforeEach(() => { + service = new EncryptedSavedObjectsService({ + logger: loggingSystemMock.create().get(), + audit: mockAuditLogger, + }); + }); + + it('does not fail if none of attributes are supposed to be encrypted', () => { + const attributes = { attrOne: 'one', attrTwo: 'two', attrThree: 'three' }; + + service.registerType({ type: 'known-type-1', attributesToEncrypt: new Set(['attrFour']) }); + + expect( + service.stripOrDecryptAttributesSync({ id: 'known-id', type: 'known-type-1' }, attributes) + ).toEqual({ attributes: { attrOne: 'one', attrTwo: 'two', attrThree: 'three' } }); + }); + + it('does not fail if there are attributes are supposed to be encrypted, but should be stripped', () => { + const attributes = { attrOne: 'one', attrTwo: 'two', attrThree: 'three' }; + + service.registerType({ + type: 'known-type-1', + attributesToEncrypt: new Set(['attrOne', 'attrThree']), + }); + + expect( + service.stripOrDecryptAttributesSync({ id: 'known-id', type: 'known-type-1' }, attributes) + ).toEqual({ attributes: { attrTwo: 'two' } }); + }); + + it('fails if needs to decrypt any attribute', () => { + service.registerType({ + type: 'known-type-1', + attributesToEncrypt: new Set([ + 'attrOne', + { key: 'attrThree', dangerouslyExposeValue: true }, + ]), + }); + + const mockUser = mockAuthenticatedUser(); + const { attributes, error } = service.stripOrDecryptAttributesSync( + { type: 'known-type-1', id: 'object-id' }, + { attrOne: 'one', attrTwo: 'two', attrThree: 'three' }, + undefined, + { user: mockUser } + ); + + expect(attributes).toEqual({ attrTwo: 'two' }); + + const encryptionError = error as EncryptionError; + expect(encryptionError.attributeName).toBe('attrThree'); + expect(encryptionError.message).toBe('Unable to decrypt attribute "attrThree"'); + expect(encryptionError.cause).toEqual( + new Error('Decryption is disabled because of missing decryption keys.') + ); + + expect(mockAuditLogger.decryptAttributeFailure).toHaveBeenCalledTimes(1); + expect(mockAuditLogger.decryptAttributeFailure).toHaveBeenCalledWith( + 'attrThree', + { type: 'known-type-1', id: 'object-id' }, + mockUser + ); + }); + }); +}); + describe('#encryptAttributes', () => { beforeEach(() => { mockNodeCrypto.encrypt.mockImplementation( diff --git a/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.ts b/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.ts index 652a2c8b6870e..cc1a8414924c3 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.ts @@ -168,25 +168,78 @@ export class EncryptedSavedObjectsService { */ public async stripOrDecryptAttributes>( descriptor: SavedObjectDescriptor, - attributes: T, + attributesToStripOrDecrypt: T, + originalAttributes?: T, + params?: DecryptParameters + ) { + const { attributes, attributesToDecrypt } = this.prepareAttributesForStripOrDecrypt( + descriptor, + attributesToStripOrDecrypt, + originalAttributes + ); + try { + const decryptedAttributes = attributesToDecrypt + ? await this.decryptAttributes(descriptor, attributesToDecrypt, params) + : {}; + return { attributes: { ...attributes, ...decryptedAttributes } }; + } catch (error) { + return { attributes, error }; + } + } + + /** + * Takes saved object attributes for the specified type and, depending on the type definition, + * either decrypts or strips encrypted attributes (e.g. in case AAD or encryption key has changed + * and decryption is no longer possible). + * @param descriptor Saved object descriptor (ID, type and optional namespace) + * @param attributesToStripOrDecrypt Object that includes a dictionary of __ALL__ saved object attributes stored + * in Elasticsearch. + * @param [originalAttributes] An optional dictionary of __ALL__ saved object original attributes + * that were used to create that saved object (i.e. values are NOT encrypted). + * @param [params] Parameters that control the way encrypted attributes are handled. + */ + public stripOrDecryptAttributesSync>( + descriptor: SavedObjectDescriptor, + attributesToStripOrDecrypt: T, originalAttributes?: T, params?: DecryptParameters + ) { + const { attributes, attributesToDecrypt } = this.prepareAttributesForStripOrDecrypt( + descriptor, + attributesToStripOrDecrypt, + originalAttributes + ); + try { + const decryptedAttributes = attributesToDecrypt + ? this.decryptAttributesSync(descriptor, attributesToDecrypt, params) + : {}; + return { attributes: { ...attributes, ...decryptedAttributes } }; + } catch (error) { + return { attributes, error }; + } + } + + /** + * Takes saved object attributes for the specified type and, depending on the type definition, + * either strips encrypted attributes, replaces with original decrypted value if available, or + * prepares them for decryption. + * @private + */ + private prepareAttributesForStripOrDecrypt>( + descriptor: SavedObjectDescriptor, + attributes: T, + originalAttributes?: T ) { const typeDefinition = this.typeDefinitions.get(descriptor.type); if (typeDefinition === undefined) { - return { attributes }; + return { attributes, attributesToDecrypt: null }; } - let decryptedAttributes: T | null = null; - let decryptionError: Error | undefined; + let attributesToDecrypt: T | undefined; const clonedAttributes: Record = {}; for (const [attributeName, attributeValue] of Object.entries(attributes)) { - // We should strip encrypted attribute if definition explicitly mandates that or decryption - // failed. - if ( - typeDefinition.shouldBeStripped(attributeName) || - (!!decryptionError && typeDefinition.shouldBeEncrypted(attributeName)) - ) { + // We should strip encrypted attribute if definition explicitly mandates that. + if (typeDefinition.shouldBeStripped(attributeName)) { continue; } @@ -197,30 +250,21 @@ export class EncryptedSavedObjectsService { // If attribute should be decrypted, but we have original attributes used to create object // we should get raw unencrypted value from there to avoid performance penalty. clonedAttributes[attributeName] = originalAttributes[attributeName]; - } else { - // Otherwise just try to decrypt attribute. We decrypt all attributes at once, cache it and - // reuse for any other attributes. - if (decryptedAttributes === null) { - try { - decryptedAttributes = await this.decryptAttributes( - descriptor, - // Decrypt only attributes that are supposed to be exposed. - Object.fromEntries( - Object.entries(attributes).filter(([key]) => !typeDefinition.shouldBeStripped(key)) - ) as T, - params - ); - } catch (err) { - decryptionError = err; - continue; - } - } - - clonedAttributes[attributeName] = decryptedAttributes[attributeName]; + } else if (!attributesToDecrypt) { + // Decrypt only attributes that are supposed to be exposed. + attributesToDecrypt = Object.fromEntries( + Object.entries(attributes).filter(([key]) => !typeDefinition.shouldBeStripped(key)) + ) as T; } } - return { attributes: clonedAttributes as T, error: decryptionError }; + return { + attributes: clonedAttributes as T, + attributesToDecrypt: + attributesToDecrypt && Object.keys(attributesToDecrypt).length > 0 + ? attributesToDecrypt + : null, + }; } private *attributesToEncryptIterator>( diff --git a/x-pack/plugins/encrypted_saved_objects/server/crypto/index.ts b/x-pack/plugins/encrypted_saved_objects/server/crypto/index.ts index 3d838d7cba69a..31086f56c3b86 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/crypto/index.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/crypto/index.ts @@ -11,6 +11,6 @@ export { descriptorToArray, SavedObjectDescriptor, } from './encrypted_saved_objects_service'; -export { EncryptionError } from './encryption_error'; +export { EncryptionError, EncryptionErrorOperation } from './encryption_error'; export { EncryptedSavedObjectAttributesDefinition } from './encrypted_saved_object_type_definition'; export { EncryptionKeyRotationService } from './encryption_key_rotation_service'; diff --git a/x-pack/plugins/encrypted_saved_objects/server/index.ts b/x-pack/plugins/encrypted_saved_objects/server/index.ts index 95337b8c92913..2706da22d108b 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/index.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/index.ts @@ -13,6 +13,7 @@ import { EncryptedSavedObjectsPlugin } from './plugin'; export { EncryptedSavedObjectTypeRegistration, EncryptionError } from './crypto'; export { EncryptedSavedObjectsPluginSetup, EncryptedSavedObjectsPluginStart } from './plugin'; export { EncryptedSavedObjectsClient } from './saved_objects'; +export type { IsMigrationNeededPredicate } from './create_migration'; export const config = { schema: ConfigSchema }; export const plugin = (initializerContext: PluginInitializerContext) => diff --git a/x-pack/plugins/fleet/server/saved_objects/migrations/to_v7_10_0.ts b/x-pack/plugins/fleet/server/saved_objects/migrations/to_v7_10_0.ts index 50780f168c459..39e65efcf2ab1 100644 --- a/x-pack/plugins/fleet/server/saved_objects/migrations/to_v7_10_0.ts +++ b/x-pack/plugins/fleet/server/saved_objects/migrations/to_v7_10_0.ts @@ -89,12 +89,14 @@ export const migrateSettingsToV7100: SavedObjectMigrationFn< export const migrateAgentActionToV7100 = ( encryptedSavedObjects: EncryptedSavedObjectsPluginSetup ): SavedObjectMigrationFn => { - return encryptedSavedObjects.createMigration( - (agentActionDoc): agentActionDoc is SavedObjectUnsanitizedDoc => { + return encryptedSavedObjects.createMigration({ + isMigrationNeededPredicate: ( + agentActionDoc + ): agentActionDoc is SavedObjectUnsanitizedDoc => { // @ts-expect-error return agentActionDoc.attributes.type === 'CONFIG_CHANGE'; }, - (agentActionDoc) => { + migration: (agentActionDoc) => { let agentActionData; try { agentActionData = agentActionDoc.attributes.data @@ -122,8 +124,8 @@ export const migrateAgentActionToV7100 = ( } else { return agentActionDoc; } - } - ); + }, + }); }; export const migrateInstallationToV7100: SavedObjectMigrationFn< diff --git a/x-pack/test/encrypted_saved_objects_api_integration/fixtures/api_consumer_plugin/server/index.ts b/x-pack/test/encrypted_saved_objects_api_integration/fixtures/api_consumer_plugin/server/index.ts index 96a0a3b2fa427..10846442d1c84 100644 --- a/x-pack/test/encrypted_saved_objects_api_integration/fixtures/api_consumer_plugin/server/index.ts +++ b/x-pack/test/encrypted_saved_objects_api_integration/fixtures/api_consumer_plugin/server/index.ts @@ -150,11 +150,13 @@ function defineTypeWithMigration(core: CoreSetup, deps: PluginsSet }, migrations: { // in this version we migrated a non encrypted field and type didnt change - '7.8.0': deps.encryptedSavedObjects.createMigration( - function shouldBeMigrated(doc): doc is SavedObjectUnsanitizedDoc { + '7.8.0': deps.encryptedSavedObjects.createMigration({ + isMigrationNeededPredicate: function shouldBeMigrated( + doc + ): doc is SavedObjectUnsanitizedDoc { return true; }, - ( + migration: ( doc: SavedObjectUnsanitizedDoc ): SavedObjectUnsanitizedDoc => { const { @@ -169,15 +171,17 @@ function defineTypeWithMigration(core: CoreSetup, deps: PluginsSet }; }, // type hasn't changed as the field we're updating is not an encrypted one - typePriorTo790, - typePriorTo790 - ), + inputType: typePriorTo790, + migratedType: typePriorTo790, + }), // in this version we encrypted an existing non encrypted field - '7.9.0': deps.encryptedSavedObjects.createMigration( - function shouldBeMigrated(doc): doc is SavedObjectUnsanitizedDoc { + '7.9.0': deps.encryptedSavedObjects.createMigration({ + isMigrationNeededPredicate: function shouldBeMigrated( + doc + ): doc is SavedObjectUnsanitizedDoc { return true; }, - ( + migration: ( doc: SavedObjectUnsanitizedDoc ): SavedObjectUnsanitizedDoc => { const { @@ -193,8 +197,8 @@ function defineTypeWithMigration(core: CoreSetup, deps: PluginsSet }, }; }, - typePriorTo790 - ), + inputType: typePriorTo790, + }), }, }); } diff --git a/x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/encrypted_saved_objects_different_key/data.json b/x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/encrypted_saved_objects_different_key/data.json new file mode 100644 index 0000000000000..6c3071fd311ab --- /dev/null +++ b/x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/encrypted_saved_objects_different_key/data.json @@ -0,0 +1,86 @@ +{ + "type": "doc", + "value": { + "id": "alert:a0d18560-e985-11eb-b1e3-5b27f0de1e72", + "index": ".kibana_1", + "source": { + "alert" : { + "params" : { + "aggType" : "count", + "termSize" : 5, + "thresholdComparator" : ">", + "timeWindowSize" : 5, + "timeWindowUnit" : "m", + "groupBy" : "all", + "threshold" : [ + 1000 + ], + "index" : [ + ".kibana-event-log-8.0.0" + ], + "timeField" : "@timestamp" + }, + "consumer" : "alerts", + "schedule" : { + "interval" : "1m" + }, + "tags" : [ ], + "name" : "test rule", + "actions" : [ ], + "enabled" : true, + "throttle" : null, + "alertTypeId" : ".index-threshold", + "apiKeyOwner" : "elastic", + "apiKey" : "XpJzeSrg3p/zB+Xz8CcrylW7q5/NVlgC1de0xbbm5E0FbhcT9DskOoUH8jmmyCMOEn7SiPKm62LXbuuLknUmd0EKBDCecK0Mf8NMcqZbTusgWLmQDu5DDx+xAXcI3wsI2KD/wqLhE+RAbiBVuNtFcfm+gBwAKNikei7qtcGL5TzfmC1Cqhn2RjrnTfzCQ0csZwHrKpbBlMhUDA==", + "createdBy" : "elastic", + "updatedBy" : "elastic", + "createdAt" : "2021-07-20T18:09:31.067Z", + "muteAll" : false, + "mutedInstanceIds" : [ ], + "executionStatus" : { + "status" : "ok", + "lastExecutionDate" : "2021-07-20T18:09:34.970Z", + "error" : null + }, + "meta" : { + "versionApiKeyLastmodified" : "8.0.0" + }, + "scheduledTaskId" : "a1b4b970-e985-11eb-b1e3-5b27f0de1e72" + }, + "type" : "alert", + "references" : [ ], + "migrationVersion" : { + "alert" : "7.10.0" + }, + "updated_at" : "2021-07-20T18:09:35.093Z" + } + } +} + +{ + "type": "doc", + "value": { + "id": "action:b9127990-e985-11eb-b1e3-5b27f0de1e72", + "index": ".kibana_1", + "source": { + "action" : { + "actionTypeId" : ".email", + "name" : "email connector", + "config" : { + "from" : "admin@company.com", + "host" : "mail.company.com", + "port" : 465, + "service" : null, + "secure" : null + }, + "secrets" : "NGVraK7+8QRoKXp4++g8mtwf8WxqFQn+RXn35Pa1gZSq/M8E2/yCycUbTEfzmhRm35It4dn9C0AQeFmlL/3nzY8fZgen4HGRVmn+FGSIoFbVY65Rkiy7v3neigO9NcZlFZ7UQAHn+mubdYrVRjFGwEzN9YbG9zK5zsCVUmVZ8w==" + }, + "type" : "action", + "references" : [ ], + "migrationVersion" : { + "alert" : "7.13.0" + }, + "updated_at" : "2021-07-20T18:10:11.024Z" + } + } +} diff --git a/x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/encrypted_saved_objects_different_key/mappings.json b/x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/encrypted_saved_objects_different_key/mappings.json new file mode 100644 index 0000000000000..6de44ddece61d --- /dev/null +++ b/x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/encrypted_saved_objects_different_key/mappings.json @@ -0,0 +1,2440 @@ +{ + "type": "index", + "value": { + "aliases": { + ".kibana": { + } + }, + "index": ".kibana_1", + "mappings": { + "_meta": { + "migrationMappingPropertyHashes": { + "action": "6e96ac5e648f57523879661ea72525b7", + "action_task_params": "a9d49f184ee89641044be0ca2950fa3a", + "alert": "7b44fba6773e37c806ce290ea9b7024e", + "apm-indices": "9bb9b2bf1fa636ed8619cbab5ce6a1dd", + "apm-telemetry": "3525d7c22c42bc80f5e6e9cb3f2b26a2", + "application_usage_totals": "c897e4310c5f24b07caaff3db53ae2c1", + "application_usage_transactional": "965839e75f809fefe04f92dc4d99722a", + "canvas-element": "7390014e1091044523666d97247392fc", + "canvas-workpad": "b0a1706d356228dbdcb4a17e6b9eb231", + "cases": "32aa96a6d3855ddda53010ae2048ac22", + "cases-comments": "c2061fb929f585df57425102fa928b4b", + "cases-configure": "42711cbb311976c0687853f4c1354572", + "cases-user-actions": "32277330ec6b721abe3b846cfd939a71", + "config": "ae24d22d5986d04124cc6568f771066f", + "dashboard": "d00f614b29a80360e1190193fd333bab", + "file-upload-telemetry": "0ed4d3e1983d1217a30982630897092e", + "graph-workspace": "cd7ba1330e6682e9cc00b78850874be1", + "index-pattern": "66eccb05066c5a89924f48a9e9736499", + "infrastructure-ui-source": "ddc0ecb18383f6b26101a2fadb2dab0c", + "inventory-view": "88fc7e12fd1b45b6f0787323ce4f18d2", + "kql-telemetry": "d12a98a6f19a2d273696597547e064ee", + "lens": "d33c68a69ff1e78c9888dedd2164ac22", + "lens-ui-telemetry": "509bfa5978586998e05f9e303c07a327", + "maps-telemetry": "bfd39d88aadadb4be597ea984d433dbe", + "metrics-explorer-view": "428e319af3e822c80a84cf87123ca35c", + "migrationVersion": "4a1746014a75ade3a714e1db5763276f", + "ml-telemetry": "257fd1d4b4fdbb9cb4b8a3b27da201e9", + "namespace": "2f4316de49999235636386fe51dc06c1", + "namespaces": "2f4316de49999235636386fe51dc06c1", + "query": "11aaeb7f5f7fa5bb43f25e18ce26e7d9", + "references": "7997cf5a56cc02bdc9c93361bde732b0", + "sample-data-telemetry": "7d3cfeb915303c9641c59681967ffeb4", + "search": "181661168bbadd1eff5902361e2a0d5c", + "space": "c5ca8acafa0beaa4d08d014a97b6bc6b", + "telemetry": "36a616f7026dfa617d6655df850fe16d", + "timelion-sheet": "9a2a2748877c7a7b582fef201ab1d4cf", + "todo": "082a2cc96a590268344d5cd74c159ac4", + "tsvb-validation-telemetry": "3a37ef6c8700ae6fc97d5c7da00e9215", + "type": "2f4316de49999235636386fe51dc06c1", + "ui-metric": "0d409297dc5ebe1e3a1da691c6ee32e3", + "updated_at": "00da57df13e94e9d98437d13ace4bfe0", + "upgrade-assistant-reindex-operation": "296a89039fc4260292be36b1b005d8f2", + "upgrade-assistant-telemetry": "56702cec857e0a9dacfb696655b4ff7b", + "uptime-dynamic-settings": "fcdb453a30092f022f2642db29523d80", + "url": "b675c3be8d76ecf029294d51dc7ec65d", + "visualization": "52d7a13ad68a150c4525b292d23e12cc" + } + }, + "dynamic": "strict", + "properties": { + "action": { + "properties": { + "actionTypeId": { + "type": "keyword" + }, + "config": { + "enabled": false, + "type": "object" + }, + "name": { + "fields": { + "keyword": { + "type": "keyword" + } + }, + "type": "text" + }, + "secrets": { + "type": "binary" + } + } + }, + "action_task_params": { + "properties": { + "actionId": { + "type": "keyword" + }, + "apiKey": { + "type": "binary" + }, + "params": { + "enabled": false, + "type": "object" + } + } + }, + "alert": { + "properties": { + "actions": { + "properties": { + "actionRef": { + "type": "keyword" + }, + "actionTypeId": { + "type": "keyword" + }, + "group": { + "type": "keyword" + }, + "params": { + "enabled": false, + "type": "object" + } + }, + "type": "nested" + }, + "alertTypeId": { + "type": "keyword" + }, + "apiKey": { + "type": "binary" + }, + "apiKeyOwner": { + "type": "keyword" + }, + "consumer": { + "type": "keyword" + }, + "createdAt": { + "type": "date" + }, + "createdBy": { + "type": "keyword" + }, + "enabled": { + "type": "boolean" + }, + "executionStatus": { + "properties": { + "status": { + "type": "keyword" + }, + "lastExecutionDate": { + "type": "date" + }, + "error": { + "properties": { + "reason": { + "type": "keyword" + }, + "message": { + "type": "keyword" + } + } + } + } + }, + "meta": { + "properties": { + "versionApiKeyLastmodified": { + "type": "keyword" + } + } + }, + "muteAll": { + "type": "boolean" + }, + "mutedInstanceIds": { + "type": "keyword" + }, + "name": { + "fields": { + "keyword": { + "type": "keyword" + } + }, + "type": "text" + }, + "params": { + "enabled": false, + "type": "object" + }, + "schedule": { + "properties": { + "interval": { + "type": "keyword" + } + } + }, + "scheduledTaskId": { + "type": "keyword" + }, + "tags": { + "type": "keyword" + }, + "throttle": { + "type": "keyword" + }, + "updatedBy": { + "type": "keyword" + } + } + }, + "saved-object-with-migration": { + "properties": { + "encryptedAttribute": { + "type": "binary" + }, + "nonEncryptedAttribute": { + "type": "keyword" + }, + "additionalEncryptedAttribute": { + "type": "binary" + } + } + }, + "apm-indices": { + "properties": { + "apm_oss": { + "properties": { + "errorIndices": { + "type": "keyword" + }, + "metricsIndices": { + "type": "keyword" + }, + "onboardingIndices": { + "type": "keyword" + }, + "sourcemapIndices": { + "type": "keyword" + }, + "spanIndices": { + "type": "keyword" + }, + "transactionIndices": { + "type": "keyword" + } + } + } + } + }, + "apm-telemetry": { + "properties": { + "agents": { + "properties": { + "dotnet": { + "properties": { + "agent": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "language": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "runtime": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + }, + "go": { + "properties": { + "agent": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "language": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "runtime": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + }, + "java": { + "properties": { + "agent": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "language": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "runtime": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + }, + "js-base": { + "properties": { + "agent": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "language": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "runtime": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + }, + "nodejs": { + "properties": { + "agent": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "language": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "runtime": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + }, + "python": { + "properties": { + "agent": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "language": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "runtime": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + }, + "ruby": { + "properties": { + "agent": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "language": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "runtime": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + }, + "rum-js": { + "properties": { + "agent": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "service": { + "properties": { + "framework": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "language": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "runtime": { + "properties": { + "composite": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + } + } + }, + "cardinality": { + "properties": { + "transaction": { + "properties": { + "name": { + "properties": { + "all_agents": { + "properties": { + "1d": { + "type": "long" + } + } + }, + "rum": { + "properties": { + "1d": { + "type": "long" + } + } + } + } + } + } + }, + "user_agent": { + "properties": { + "original": { + "properties": { + "all_agents": { + "properties": { + "1d": { + "type": "long" + } + } + }, + "rum": { + "properties": { + "1d": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "counts": { + "properties": { + "agent_configuration": { + "properties": { + "all": { + "type": "long" + } + } + }, + "error": { + "properties": { + "1d": { + "type": "long" + }, + "all": { + "type": "long" + } + } + }, + "max_error_groups_per_service": { + "properties": { + "1d": { + "type": "long" + } + } + }, + "max_transaction_groups_per_service": { + "properties": { + "1d": { + "type": "long" + } + } + }, + "metric": { + "properties": { + "1d": { + "type": "long" + }, + "all": { + "type": "long" + } + } + }, + "onboarding": { + "properties": { + "1d": { + "type": "long" + }, + "all": { + "type": "long" + } + } + }, + "services": { + "properties": { + "1d": { + "type": "long" + } + } + }, + "sourcemap": { + "properties": { + "1d": { + "type": "long" + }, + "all": { + "type": "long" + } + } + }, + "span": { + "properties": { + "1d": { + "type": "long" + }, + "all": { + "type": "long" + } + } + }, + "traces": { + "properties": { + "1d": { + "type": "long" + } + } + }, + "transaction": { + "properties": { + "1d": { + "type": "long" + }, + "all": { + "type": "long" + } + } + } + } + }, + "has_any_services": { + "type": "boolean" + }, + "indices": { + "properties": { + "all": { + "properties": { + "total": { + "properties": { + "docs": { + "properties": { + "count": { + "type": "long" + } + } + }, + "store": { + "properties": { + "size_in_bytes": { + "type": "long" + } + } + } + } + } + } + }, + "shards": { + "properties": { + "total": { + "type": "long" + } + } + } + } + }, + "integrations": { + "properties": { + "ml": { + "properties": { + "all_jobs_count": { + "type": "long" + } + } + } + } + }, + "retainment": { + "properties": { + "error": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "metric": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "onboarding": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "span": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "transaction": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "services_per_agent": { + "properties": { + "dotnet": { + "null_value": 0, + "type": "long" + }, + "go": { + "null_value": 0, + "type": "long" + }, + "java": { + "null_value": 0, + "type": "long" + }, + "js-base": { + "null_value": 0, + "type": "long" + }, + "nodejs": { + "null_value": 0, + "type": "long" + }, + "python": { + "null_value": 0, + "type": "long" + }, + "ruby": { + "null_value": 0, + "type": "long" + }, + "rum-js": { + "null_value": 0, + "type": "long" + } + } + }, + "tasks": { + "properties": { + "agent_configuration": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "agents": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "cardinality": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "groupings": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "indices_stats": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "integrations": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "processor_events": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "services": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "versions": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + } + } + }, + "version": { + "properties": { + "apm_server": { + "properties": { + "major": { + "type": "long" + }, + "minor": { + "type": "long" + }, + "patch": { + "type": "long" + } + } + } + } + } + } + }, + "application_usage_totals": { + "properties": { + "appId": { + "type": "keyword" + }, + "minutesOnScreen": { + "type": "float" + }, + "numberOfClicks": { + "type": "long" + } + } + }, + "application_usage_transactional": { + "properties": { + "appId": { + "type": "keyword" + }, + "minutesOnScreen": { + "type": "float" + }, + "numberOfClicks": { + "type": "long" + }, + "timestamp": { + "type": "date" + } + } + }, + "canvas-element": { + "dynamic": "false", + "properties": { + "@created": { + "type": "date" + }, + "@timestamp": { + "type": "date" + }, + "content": { + "type": "text" + }, + "help": { + "type": "text" + }, + "image": { + "type": "text" + }, + "name": { + "fields": { + "keyword": { + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "canvas-workpad": { + "dynamic": "false", + "properties": { + "@created": { + "type": "date" + }, + "@timestamp": { + "type": "date" + }, + "name": { + "fields": { + "keyword": { + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "cases": { + "properties": { + "closed_at": { + "type": "date" + }, + "closed_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + }, + "connector_id": { + "type": "keyword" + }, + "created_at": { + "type": "date" + }, + "created_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + }, + "description": { + "type": "text" + }, + "external_service": { + "properties": { + "connector_id": { + "type": "keyword" + }, + "connector_name": { + "type": "keyword" + }, + "external_id": { + "type": "keyword" + }, + "external_title": { + "type": "text" + }, + "external_url": { + "type": "text" + }, + "pushed_at": { + "type": "date" + }, + "pushed_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + } + } + }, + "status": { + "type": "keyword" + }, + "tags": { + "type": "keyword" + }, + "title": { + "type": "keyword" + }, + "updated_at": { + "type": "date" + }, + "updated_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + } + } + }, + "cases-comments": { + "properties": { + "comment": { + "type": "text" + }, + "created_at": { + "type": "date" + }, + "created_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + }, + "pushed_at": { + "type": "date" + }, + "pushed_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + }, + "updated_at": { + "type": "date" + }, + "updated_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + } + } + }, + "cases-configure": { + "properties": { + "closure_type": { + "type": "keyword" + }, + "connector_id": { + "type": "keyword" + }, + "connector_name": { + "type": "keyword" + }, + "created_at": { + "type": "date" + }, + "created_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + }, + "updated_at": { + "type": "date" + }, + "updated_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + } + } + }, + "cases-user-actions": { + "properties": { + "action": { + "type": "keyword" + }, + "action_at": { + "type": "date" + }, + "action_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + }, + "action_field": { + "type": "keyword" + }, + "new_value": { + "type": "text" + }, + "old_value": { + "type": "text" + } + } + }, + "config": { + "dynamic": "true", + "properties": { + "buildNum": { + "type": "keyword" + } + } + }, + "dashboard": { + "properties": { + "description": { + "type": "text" + }, + "hits": { + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "optionsJSON": { + "type": "text" + }, + "panelsJSON": { + "type": "text" + }, + "refreshInterval": { + "properties": { + "display": { + "type": "keyword" + }, + "pause": { + "type": "boolean" + }, + "section": { + "type": "integer" + }, + "value": { + "type": "integer" + } + } + }, + "timeFrom": { + "type": "keyword" + }, + "timeRestore": { + "type": "boolean" + }, + "timeTo": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "file-upload-telemetry": { + "properties": { + "filesUploadedTotalCount": { + "type": "long" + } + } + }, + "graph-workspace": { + "properties": { + "description": { + "type": "text" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "numLinks": { + "type": "integer" + }, + "numVertices": { + "type": "integer" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + }, + "wsState": { + "type": "text" + } + } + }, + "index-pattern": { + "properties": { + "fieldFormatMap": { + "type": "text" + }, + "fields": { + "type": "text" + }, + "intervalName": { + "type": "keyword" + }, + "notExpandable": { + "type": "boolean" + }, + "sourceFilters": { + "type": "text" + }, + "timeFieldName": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "type": { + "type": "keyword" + }, + "typeMeta": { + "type": "keyword" + } + } + }, + "infrastructure-ui-source": { + "properties": { + "description": { + "type": "text" + }, + "fields": { + "properties": { + "container": { + "type": "keyword" + }, + "host": { + "type": "keyword" + }, + "pod": { + "type": "keyword" + }, + "tiebreaker": { + "type": "keyword" + }, + "timestamp": { + "type": "keyword" + } + } + }, + "logAlias": { + "type": "keyword" + }, + "logColumns": { + "properties": { + "fieldColumn": { + "properties": { + "field": { + "type": "keyword" + }, + "id": { + "type": "keyword" + } + } + }, + "messageColumn": { + "properties": { + "id": { + "type": "keyword" + } + } + }, + "timestampColumn": { + "properties": { + "id": { + "type": "keyword" + } + } + } + }, + "type": "nested" + }, + "metricAlias": { + "type": "keyword" + }, + "name": { + "type": "text" + } + } + }, + "inventory-view": { + "properties": { + "accountId": { + "type": "keyword" + }, + "autoBounds": { + "type": "boolean" + }, + "autoReload": { + "type": "boolean" + }, + "boundsOverride": { + "properties": { + "max": { + "type": "integer" + }, + "min": { + "type": "integer" + } + } + }, + "customMetrics": { + "properties": { + "aggregation": { + "type": "keyword" + }, + "field": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "label": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + }, + "type": "nested" + }, + "customOptions": { + "properties": { + "field": { + "type": "keyword" + }, + "text": { + "type": "keyword" + } + }, + "type": "nested" + }, + "filterQuery": { + "properties": { + "expression": { + "type": "keyword" + }, + "kind": { + "type": "keyword" + } + } + }, + "groupBy": { + "properties": { + "field": { + "type": "keyword" + }, + "label": { + "type": "keyword" + } + }, + "type": "nested" + }, + "legend": { + "properties": { + "palette": { + "type": "keyword" + }, + "reverseColors": { + "type": "boolean" + }, + "steps": { + "type": "long" + } + } + }, + "metric": { + "properties": { + "aggregation": { + "type": "keyword" + }, + "field": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "label": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + } + }, + "name": { + "type": "keyword" + }, + "nodeType": { + "type": "keyword" + }, + "region": { + "type": "keyword" + }, + "sort": { + "properties": { + "by": { + "type": "keyword" + }, + "direction": { + "type": "keyword" + } + } + }, + "time": { + "type": "long" + }, + "view": { + "type": "keyword" + } + } + }, + "kql-telemetry": { + "properties": { + "optInCount": { + "type": "long" + }, + "optOutCount": { + "type": "long" + } + } + }, + "lens": { + "properties": { + "description": { + "type": "text" + }, + "expression": { + "index": false, + "type": "keyword" + }, + "state": { + "type": "flattened" + }, + "title": { + "type": "text" + }, + "visualizationType": { + "type": "keyword" + } + } + }, + "lens-ui-telemetry": { + "properties": { + "count": { + "type": "integer" + }, + "date": { + "type": "date" + }, + "name": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + } + }, + "map": { + "properties": { + "bounds": { + "dynamic": false, + "properties": {} + }, + "description": { + "type": "text" + }, + "layerListJSON": { + "type": "text" + }, + "mapStateJSON": { + "type": "text" + }, + "title": { + "type": "text" + }, + "uiStateJSON": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "maps-telemetry": { + "properties": { + "attributesPerMap": { + "properties": { + "dataSourcesCount": { + "properties": { + "avg": { + "type": "long" + }, + "max": { + "type": "long" + }, + "min": { + "type": "long" + } + } + }, + "emsVectorLayersCount": { + "dynamic": "true", + "type": "object" + }, + "layerTypesCount": { + "dynamic": "true", + "type": "object" + }, + "layersCount": { + "properties": { + "avg": { + "type": "long" + }, + "max": { + "type": "long" + }, + "min": { + "type": "long" + } + } + } + } + }, + "indexPatternsWithGeoFieldCount": { + "type": "long" + }, + "indexPatternsWithGeoPointFieldCount": { + "type": "long" + }, + "indexPatternsWithGeoShapeFieldCount": { + "type": "long" + }, + "mapsTotalCount": { + "type": "long" + }, + "settings": { + "properties": { + "showMapVisualizationTypes": { + "type": "boolean" + } + } + }, + "timeCaptured": { + "type": "date" + } + } + }, + "metrics-explorer-view": { + "properties": { + "chartOptions": { + "properties": { + "stack": { + "type": "boolean" + }, + "type": { + "type": "keyword" + }, + "yAxisMode": { + "type": "keyword" + } + } + }, + "currentTimerange": { + "properties": { + "from": { + "type": "keyword" + }, + "interval": { + "type": "keyword" + }, + "to": { + "type": "keyword" + } + } + }, + "name": { + "type": "keyword" + }, + "options": { + "properties": { + "aggregation": { + "type": "keyword" + }, + "filterQuery": { + "type": "keyword" + }, + "forceInterval": { + "type": "boolean" + }, + "groupBy": { + "type": "keyword" + }, + "limit": { + "type": "integer" + }, + "metrics": { + "properties": { + "aggregation": { + "type": "keyword" + }, + "color": { + "type": "keyword" + }, + "field": { + "type": "keyword" + }, + "label": { + "type": "keyword" + } + }, + "type": "nested" + } + } + } + } + }, + "migrationVersion": { + "dynamic": "true", + "properties": { + "alert": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "saved-object-with-migration": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "config": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "space": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "ml-telemetry": { + "properties": { + "file_data_visualizer": { + "properties": { + "index_creation_count": { + "type": "long" + } + } + } + } + }, + "namespace": { + "type": "keyword" + }, + "namespaces": { + "type": "keyword" + }, + "query": { + "properties": { + "description": { + "type": "text" + }, + "filters": { + "enabled": false, + "type": "object" + }, + "query": { + "properties": { + "language": { + "type": "keyword" + }, + "query": { + "index": false, + "type": "keyword" + } + } + }, + "timefilter": { + "enabled": false, + "type": "object" + }, + "title": { + "type": "text" + } + } + }, + "references": { + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + }, + "type": "nested" + }, + "sample-data-telemetry": { + "properties": { + "installCount": { + "type": "long" + }, + "unInstallCount": { + "type": "long" + } + } + }, + "search": { + "properties": { + "columns": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "hits": { + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "sort": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "space": { + "properties": { + "_reserved": { + "type": "boolean" + }, + "color": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "disabledFeatures": { + "type": "keyword" + }, + "imageUrl": { + "index": false, + "type": "text" + }, + "initials": { + "type": "keyword" + }, + "name": { + "fields": { + "keyword": { + "ignore_above": 2048, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "telemetry": { + "properties": { + "allowChangingOptInStatus": { + "type": "boolean" + }, + "enabled": { + "type": "boolean" + }, + "lastReported": { + "type": "date" + }, + "lastVersionChecked": { + "type": "keyword" + }, + "reportFailureCount": { + "type": "integer" + }, + "reportFailureVersion": { + "type": "keyword" + }, + "sendUsageFrom": { + "type": "keyword" + }, + "userHasSeenNotice": { + "type": "boolean" + } + } + }, + "timelion-sheet": { + "properties": { + "description": { + "type": "text" + }, + "hits": { + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "timelion_chart_height": { + "type": "integer" + }, + "timelion_columns": { + "type": "integer" + }, + "timelion_interval": { + "type": "keyword" + }, + "timelion_other_interval": { + "type": "keyword" + }, + "timelion_rows": { + "type": "integer" + }, + "timelion_sheet": { + "type": "text" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "todo": { + "properties": { + "icon": { + "type": "keyword" + }, + "task": { + "type": "text" + }, + "title": { + "type": "keyword" + } + } + }, + "tsvb-validation-telemetry": { + "properties": { + "failedRequests": { + "type": "long" + } + } + }, + "type": { + "type": "keyword" + }, + "ui-metric": { + "properties": { + "count": { + "type": "integer" + } + } + }, + "updated_at": { + "type": "date" + }, + "upgrade-assistant-reindex-operation": { + "properties": { + "errorMessage": { + "type": "keyword" + }, + "indexName": { + "type": "keyword" + }, + "lastCompletedStep": { + "type": "integer" + }, + "locked": { + "type": "date" + }, + "newIndexName": { + "type": "keyword" + }, + "reindexOptions": { + "properties": { + "openAndClose": { + "type": "boolean" + }, + "queueSettings": { + "properties": { + "queuedAt": { + "type": "long" + }, + "startedAt": { + "type": "long" + } + } + } + } + }, + "reindexTaskId": { + "type": "keyword" + }, + "reindexTaskPercComplete": { + "type": "float" + }, + "runningReindexCount": { + "type": "integer" + }, + "status": { + "type": "integer" + } + } + }, + "upgrade-assistant-telemetry": { + "properties": { + "features": { + "properties": { + "deprecation_logging": { + "properties": { + "enabled": { + "null_value": true, + "type": "boolean" + } + } + } + } + }, + "ui_open": { + "properties": { + "cluster": { + "null_value": 0, + "type": "long" + }, + "indices": { + "null_value": 0, + "type": "long" + }, + "overview": { + "null_value": 0, + "type": "long" + } + } + }, + "ui_reindex": { + "properties": { + "close": { + "null_value": 0, + "type": "long" + }, + "open": { + "null_value": 0, + "type": "long" + }, + "start": { + "null_value": 0, + "type": "long" + }, + "stop": { + "null_value": 0, + "type": "long" + } + } + } + } + }, + "uptime-dynamic-settings": { + "properties": { + "certAgeThreshold": { + "type": "long" + }, + "certExpirationThreshold": { + "type": "long" + }, + "heartbeatIndices": { + "type": "keyword" + } + } + }, + "url": { + "properties": { + "accessCount": { + "type": "long" + }, + "accessDate": { + "type": "date" + }, + "createDate": { + "type": "date" + }, + "url": { + "fields": { + "keyword": { + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "visualization": { + "properties": { + "description": { + "type": "text" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "savedSearchRefName": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "uiStateJSON": { + "type": "text" + }, + "version": { + "type": "integer" + }, + "visState": { + "type": "text" + } + } + } + } + }, + "settings": { + "index": { + "auto_expand_replicas": "0-1", + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +} diff --git a/x-pack/test/encrypted_saved_objects_api_integration/tests/encrypted_saved_objects_decryption.ts b/x-pack/test/encrypted_saved_objects_api_integration/tests/encrypted_saved_objects_decryption.ts new file mode 100644 index 0000000000000..5045f912fc26f --- /dev/null +++ b/x-pack/test/encrypted_saved_objects_api_integration/tests/encrypted_saved_objects_decryption.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../ftr_provider_context'; + +export default function ({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + + describe('encrypted saved objects decryption', () => { + // This test uses esArchiver to load alert and action saved objects that have been created with a different encryption key + // than what is used in the test. The SOs are from an older Kibana version to ensure that migrations will be applied, + + // When the test runs, you will see in the console logs both the decryption error and a warning that the migration will run anyway. + // The test asserts that the alert and action SOs have the new fields expected post-migration + + describe('migrations', () => { + before(async () => { + await esArchiver.load( + 'x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/encrypted_saved_objects_different_key' + ); + }); + + after(async () => { + await esArchiver.unload( + 'x-pack/test/encrypted_saved_objects_api_integration/fixtures/es_archiver/encrypted_saved_objects_different_key' + ); + }); + + it('migrates alert and actions saved objects even if decryption fails', async () => { + const { body: migratedRule } = await supertest + .get(`/api/alerting/rule/a0d18560-e985-11eb-b1e3-5b27f0de1e72`) + .expect(200); + + await supertest + .get( + `/api/hidden_saved_objects/get-decrypted-as-internal-user/alert/a0d18560-e985-11eb-b1e3-5b27f0de1e72` + ) + .expect(200); + + expect(migratedRule.apiKey).to.be(undefined); + expect(migratedRule.notify_when).to.eql('onActiveAlert'); + expect(migratedRule.updated_at).to.eql('2021-07-20T18:09:35.093Z'); + + const { body: migratedConnector } = await supertest + .get(`/api/actions/connector/b9127990-e985-11eb-b1e3-5b27f0de1e72`) + .expect(200); + + await supertest + .get( + `/api/hidden_saved_objects/get-decrypted-as-internal-user/action/b9127990-e985-11eb-b1e3-5b27f0de1e72` + ) + .expect(200); + + expect(migratedRule.secrets).to.be(undefined); + expect(migratedConnector.is_missing_secrets).to.eql(false); + }); + }); + }); +} diff --git a/x-pack/test/encrypted_saved_objects_api_integration/tests/index.ts b/x-pack/test/encrypted_saved_objects_api_integration/tests/index.ts index 535342ae7416a..de87d627ac486 100644 --- a/x-pack/test/encrypted_saved_objects_api_integration/tests/index.ts +++ b/x-pack/test/encrypted_saved_objects_api_integration/tests/index.ts @@ -11,5 +11,6 @@ export default function ({ loadTestFile }: FtrProviderContext) { describe('encryptedSavedObjects', function encryptedSavedObjectsSuite() { this.tags('ciGroup13'); loadTestFile(require.resolve('./encrypted_saved_objects_api')); + loadTestFile(require.resolve('./encrypted_saved_objects_decryption')); }); }