From 3f504b95d4bbdf1af8a9a1947e46fa89eb00f84a Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Thu, 4 Nov 2021 11:55:54 -0400 Subject: [PATCH 01/18] Call setup on fleet start, remove API calls --- .../fleet/public/applications/fleet/app.tsx | 16 --------------- x-pack/plugins/fleet/server/plugin.ts | 20 ++++++++++++++----- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/x-pack/plugins/fleet/public/applications/fleet/app.tsx b/x-pack/plugins/fleet/public/applications/fleet/app.tsx index 682c889d80b97..1626c40aab4c6 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/app.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/app.tsx @@ -32,7 +32,6 @@ import { FleetStatusProvider, KibanaVersionContext, sendGetPermissionsCheck, - sendSetup, useBreadcrumbs, useStartServices, UIExtensionsContext, @@ -141,21 +140,6 @@ export const WithPermissionsAndSetup: React.FC = memo(({ children }) => { const permissionsResponse = await sendGetPermissionsCheck(); setIsPermissionsLoading(false); if (permissionsResponse.data?.success) { - try { - const setupResponse = await sendSetup(); - if (setupResponse.error) { - setInitializationError(setupResponse.error); - } - if (setupResponse.data?.nonFatalErrors?.length) { - notifications.toasts.addError(setupResponse.data.nonFatalErrors[0], { - title: i18n.translate('xpack.fleet.setup.uiPreconfigurationErrorTitle', { - defaultMessage: 'Configuration error', - }), - }); - } - } catch (err) { - setInitializationError(err); - } setIsInitialized(true); } else { setPermissionsError(permissionsResponse.data?.error || 'REQUEST_ERROR'); diff --git a/x-pack/plugins/fleet/server/plugin.ts b/x-pack/plugins/fleet/server/plugin.ts index 7cc1b8b1cfcc9..6b8013b8fcf98 100644 --- a/x-pack/plugins/fleet/server/plugin.ts +++ b/x-pack/plugins/fleet/server/plugin.ts @@ -6,6 +6,7 @@ */ import type { Observable } from 'rxjs'; +import { first } from 'rxjs/operators'; import type { CoreSetup, CoreStart, @@ -20,7 +21,7 @@ import type { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import type { TelemetryPluginSetup, TelemetryPluginStart } from 'src/plugins/telemetry/server'; -import { DEFAULT_APP_CATEGORIES } from '../../../../src/core/server'; +import { DEFAULT_APP_CATEGORIES, SavedObjectsClient } from '../../../../src/core/server'; import type { PluginStart as DataPluginStart } from '../../../../src/plugins/data/server'; import type { LicensingPluginSetup, ILicense } from '../../licensing/server'; import type { @@ -86,6 +87,7 @@ import { startFleetServerSetup } from './services/fleet_server'; import { FleetArtifactsClient } from './services/artifacts'; import type { FleetRouter } from './types/request_context'; import { TelemetryEventsSender } from './telemetry/sender'; +import { setupFleet } from './services/setup'; export interface FleetSetupDeps { licensing: LicensingPluginSetup; @@ -335,14 +337,22 @@ export class FleetPlugin }); licenseService.start(this.licensing$); - const fleetServerSetup = startFleetServerSetup(); - this.telemetryEventsSender.start(plugins.telemetry, core); return { fleetSetupCompleted: () => - new Promise((resolve) => { - Promise.all([fleetServerSetup]).finally(() => resolve()); + new Promise(async (resolve, reject) => { + try { + await startFleetServerSetup(); + await setupFleet( + new SavedObjectsClient(core.savedObjects.createInternalRepository()), + core.elasticsearch.client.asInternalUser + ); + + resolve(); + } catch (error) { + reject(error); + } }), esIndexPatternService: new ESIndexPatternSavedObjectService(), packageService: { From ff41a0052f83af9e2130fd74a7eb5cb78252c054 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Thu, 4 Nov 2021 15:23:53 -0400 Subject: [PATCH 02/18] Fix unused import --- x-pack/plugins/fleet/server/plugin.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/fleet/server/plugin.ts b/x-pack/plugins/fleet/server/plugin.ts index 6b8013b8fcf98..15cdb22826d43 100644 --- a/x-pack/plugins/fleet/server/plugin.ts +++ b/x-pack/plugins/fleet/server/plugin.ts @@ -6,7 +6,6 @@ */ import type { Observable } from 'rxjs'; -import { first } from 'rxjs/operators'; import type { CoreSetup, CoreStart, From 7f5330c913d56ec473ddffa5c82980deb610ce98 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Thu, 4 Nov 2021 15:57:35 -0400 Subject: [PATCH 03/18] Revert removal of setup API call --- .../fleet/public/applications/fleet/app.tsx | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/x-pack/plugins/fleet/public/applications/fleet/app.tsx b/x-pack/plugins/fleet/public/applications/fleet/app.tsx index 1626c40aab4c6..682c889d80b97 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/app.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/app.tsx @@ -32,6 +32,7 @@ import { FleetStatusProvider, KibanaVersionContext, sendGetPermissionsCheck, + sendSetup, useBreadcrumbs, useStartServices, UIExtensionsContext, @@ -140,6 +141,21 @@ export const WithPermissionsAndSetup: React.FC = memo(({ children }) => { const permissionsResponse = await sendGetPermissionsCheck(); setIsPermissionsLoading(false); if (permissionsResponse.data?.success) { + try { + const setupResponse = await sendSetup(); + if (setupResponse.error) { + setInitializationError(setupResponse.error); + } + if (setupResponse.data?.nonFatalErrors?.length) { + notifications.toasts.addError(setupResponse.data.nonFatalErrors[0], { + title: i18n.translate('xpack.fleet.setup.uiPreconfigurationErrorTitle', { + defaultMessage: 'Configuration error', + }), + }); + } + } catch (err) { + setInitializationError(err); + } setIsInitialized(true); } else { setPermissionsError(permissionsResponse.data?.error || 'REQUEST_ERROR'); From 252cf54df2300cd3ac1168900f45ff40961aafe4 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Mon, 8 Nov 2021 12:05:25 -0500 Subject: [PATCH 04/18] Restructor fleetSetupCompleted promise --- x-pack/plugins/fleet/server/plugin.ts | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/x-pack/plugins/fleet/server/plugin.ts b/x-pack/plugins/fleet/server/plugin.ts index 15cdb22826d43..f02ba9eee940a 100644 --- a/x-pack/plugins/fleet/server/plugin.ts +++ b/x-pack/plugins/fleet/server/plugin.ts @@ -339,20 +339,13 @@ export class FleetPlugin this.telemetryEventsSender.start(plugins.telemetry, core); return { - fleetSetupCompleted: () => - new Promise(async (resolve, reject) => { - try { - await startFleetServerSetup(); - await setupFleet( - new SavedObjectsClient(core.savedObjects.createInternalRepository()), - core.elasticsearch.client.asInternalUser - ); - - resolve(); - } catch (error) { - reject(error); - } - }), + fleetSetupCompleted: async () => { + await startFleetServerSetup(); + await setupFleet( + new SavedObjectsClient(core.savedObjects.createInternalRepository()), + core.elasticsearch.client.asInternalUser + ); + }, esIndexPatternService: new ESIndexPatternSavedObjectService(), packageService: { getInstallation, From b5d900b25a7f9b12db58a348f743b6630a84ffd2 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Tue, 9 Nov 2021 08:07:41 -0500 Subject: [PATCH 05/18] Add logging + handle setup failures --- x-pack/plugins/fleet/server/plugin.ts | 23 +++++++++++++++---- .../fleet/server/services/preconfiguration.ts | 7 +++++- x-pack/plugins/fleet/server/services/setup.ts | 14 ++++++++++- 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/fleet/server/plugin.ts b/x-pack/plugins/fleet/server/plugin.ts index f02ba9eee940a..f46af8012bea7 100644 --- a/x-pack/plugins/fleet/server/plugin.ts +++ b/x-pack/plugins/fleet/server/plugin.ts @@ -338,13 +338,26 @@ export class FleetPlugin this.telemetryEventsSender.start(plugins.telemetry, core); - return { - fleetSetupCompleted: async () => { - await startFleetServerSetup(); - await setupFleet( + const logger = appContextService.getLogger(); + + const fleetSetupPromise = startFleetServerSetup() + .then(() => + setupFleet( new SavedObjectsClient(core.savedObjects.createInternalRepository()), core.elasticsearch.client.asInternalUser - ); + ) + ) + .then(() => { + logger.info('Fleet setup completed'); + }) + .catch((error) => { + logger.warn('Fleet setup failed'); + logger.warn(error); + }); + + return { + fleetSetupCompleted: async () => { + return fleetSetupPromise; }, esIndexPatternService: new ESIndexPatternSavedObjectService(), packageService: { diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.ts b/x-pack/plugins/fleet/server/services/preconfiguration.ts index e5fea73815ea7..b592d3cdd170b 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.ts @@ -72,6 +72,8 @@ export async function ensurePreconfiguredOutputs( esClient: ElasticsearchClient, outputs: PreconfiguredOutput[] ) { + const logger = appContextService.getLogger(); + if (outputs.length === 0) { return; } @@ -114,8 +116,11 @@ export async function ensurePreconfiguredOutputs( if (isCreate) { await outputService.create(soClient, data, { id, overwrite: true }); + logger.info(`Created output ${id}`); } else if (isUpdateWithNewData) { await outputService.update(soClient, id, data); + logger.info(`Updated output ${id}`); + // Bump revision of all policies using that output if (outputData.is_default) { await agentPolicyService.bumpAllAgentPolicies(soClient, esClient); @@ -342,7 +347,7 @@ export async function ensurePreconfiguredPackagesAndPolicies( await soClient .delete(AGENT_POLICY_SAVED_OBJECT_TYPE, policy!.id) // swallow error - .catch((deleteErr) => appContextService.getLogger().error(deleteErr)); + .catch((deleteErr) => logger.error(deleteErr)); throw err; } diff --git a/x-pack/plugins/fleet/server/services/setup.ts b/x-pack/plugins/fleet/server/services/setup.ts index 37d79c1bb691d..d7cfce8821861 100644 --- a/x-pack/plugins/fleet/server/services/setup.ts +++ b/x-pack/plugins/fleet/server/services/setup.ts @@ -52,6 +52,9 @@ async function createSetupSideEffects( soClient: SavedObjectsClientContract, esClient: ElasticsearchClient ): Promise { + const logger = appContextService.getLogger(); + logger.info('Beginning Fleet setup'); + const { agentPolicies: policiesOrUndefined, packages: packagesOrUndefined, @@ -61,6 +64,7 @@ async function createSetupSideEffects( const policies = policiesOrUndefined ?? []; let packages = packagesOrUndefined ?? []; + logger.info('Setting up Fleet outputs'); await Promise.all([ ensurePreconfiguredOutputs(soClient, esClient, outputsOrUndefined ?? []), settingsService.settingsSetup(soClient), @@ -70,6 +74,7 @@ async function createSetupSideEffects( await awaitIfFleetServerSetupPending(); if (appContextService.getConfig()?.agentIdVerificationEnabled) { + logger.info('Setting up Fleet Elasticsearch assets'); await ensureFleetGlobalEsAssets(soClient, esClient); } @@ -93,6 +98,8 @@ async function createSetupSideEffects( ...autoUpdateablePackages.filter((pkg) => !preconfiguredPackageNames.has(pkg.name)), ]; + logger.info('Setting up initial Fleet packages'); + const { nonFatalErrors } = await ensurePreconfiguredPackagesAndPolicies( soClient, esClient, @@ -101,9 +108,13 @@ async function createSetupSideEffects( defaultOutput ); + logger.info('Cleaning up Fleet outputs'); await cleanPreconfiguredOutputs(soClient, outputsOrUndefined ?? []); + logger.info('Setting up Fleet enrollment keys'); await ensureDefaultEnrollmentAPIKeysExists(soClient, esClient); + + logger.info('Setting up Fleet Server agent policies'); await ensureFleetServerAgentPoliciesExists(soClient, esClient); return { @@ -121,6 +132,7 @@ export async function ensureFleetGlobalEsAssets( ) { const logger = appContextService.getLogger(); // Ensure Global Fleet ES assets are installed + logger.info('Creating Fleet component template and ingest pipeline'); const globalAssetsRes = await Promise.all([ ensureDefaultComponentTemplate(esClient), ensureFleetFinalPipelineIsInstalled(esClient), @@ -143,7 +155,7 @@ export async function ensureFleetGlobalEsAssets( savedObjectsClient: soClient, pkgkey: pkgToPkgKey({ name: installation.name, version: installation.version }), esClient, - // Force install the pacakge will update the index template and the datastream write indices + // Force install the package will update the index template and the datastream write indices force: true, }).catch((err) => { logger.error( From 1fc6f016c0f8d2b14b146da4de08cc68a22c1387 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Tue, 9 Nov 2021 11:04:42 -0500 Subject: [PATCH 06/18] Restructure logging to mix of debug/info --- x-pack/plugins/fleet/server/plugin.ts | 10 ++++++---- .../fleet/server/services/preconfiguration.ts | 6 +++--- x-pack/plugins/fleet/server/services/setup.ts | 15 +++++++-------- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/x-pack/plugins/fleet/server/plugin.ts b/x-pack/plugins/fleet/server/plugin.ts index f46af8012bea7..79385d211d896 100644 --- a/x-pack/plugins/fleet/server/plugin.ts +++ b/x-pack/plugins/fleet/server/plugin.ts @@ -341,12 +341,14 @@ export class FleetPlugin const logger = appContextService.getLogger(); const fleetSetupPromise = startFleetServerSetup() - .then(() => - setupFleet( + .then(() => { + logger.info('Beginning fleet setup'); + + return setupFleet( new SavedObjectsClient(core.savedObjects.createInternalRepository()), core.elasticsearch.client.asInternalUser - ) - ) + ); + }) .then(() => { logger.info('Fleet setup completed'); }) diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.ts b/x-pack/plugins/fleet/server/services/preconfiguration.ts index b592d3cdd170b..91ebfe306fa6f 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.ts @@ -116,10 +116,10 @@ export async function ensurePreconfiguredOutputs( if (isCreate) { await outputService.create(soClient, data, { id, overwrite: true }); - logger.info(`Created output ${id}`); + logger.debug(`Created output ${id}`); } else if (isUpdateWithNewData) { await outputService.update(soClient, id, data); - logger.info(`Updated output ${id}`); + logger.debug(`Updated output ${id}`); // Bump revision of all policies using that output if (outputData.is_default) { @@ -143,7 +143,7 @@ export async function cleanPreconfiguredOutputs( for (const output of existingPreconfiguredOutput) { if (!outputs.find(({ id }) => output.id === id)) { - logger.info(`Deleting preconfigured output ${output.id}`); + logger.debug(`Deleting preconfigured output ${output.id}`); await outputService.delete(soClient, output.id); } } diff --git a/x-pack/plugins/fleet/server/services/setup.ts b/x-pack/plugins/fleet/server/services/setup.ts index d7cfce8821861..4acf855dc5ddf 100644 --- a/x-pack/plugins/fleet/server/services/setup.ts +++ b/x-pack/plugins/fleet/server/services/setup.ts @@ -53,7 +53,6 @@ async function createSetupSideEffects( esClient: ElasticsearchClient ): Promise { const logger = appContextService.getLogger(); - logger.info('Beginning Fleet setup'); const { agentPolicies: policiesOrUndefined, @@ -64,7 +63,7 @@ async function createSetupSideEffects( const policies = policiesOrUndefined ?? []; let packages = packagesOrUndefined ?? []; - logger.info('Setting up Fleet outputs'); + logger.debug('Setting up Fleet outputs'); await Promise.all([ ensurePreconfiguredOutputs(soClient, esClient, outputsOrUndefined ?? []), settingsService.settingsSetup(soClient), @@ -74,7 +73,7 @@ async function createSetupSideEffects( await awaitIfFleetServerSetupPending(); if (appContextService.getConfig()?.agentIdVerificationEnabled) { - logger.info('Setting up Fleet Elasticsearch assets'); + logger.debug('Setting up Fleet Elasticsearch assets'); await ensureFleetGlobalEsAssets(soClient, esClient); } @@ -98,7 +97,7 @@ async function createSetupSideEffects( ...autoUpdateablePackages.filter((pkg) => !preconfiguredPackageNames.has(pkg.name)), ]; - logger.info('Setting up initial Fleet packages'); + logger.debug('Setting up initial Fleet packages'); const { nonFatalErrors } = await ensurePreconfiguredPackagesAndPolicies( soClient, @@ -108,13 +107,13 @@ async function createSetupSideEffects( defaultOutput ); - logger.info('Cleaning up Fleet outputs'); + logger.debug('Cleaning up Fleet outputs'); await cleanPreconfiguredOutputs(soClient, outputsOrUndefined ?? []); - logger.info('Setting up Fleet enrollment keys'); + logger.debug('Setting up Fleet enrollment keys'); await ensureDefaultEnrollmentAPIKeysExists(soClient, esClient); - logger.info('Setting up Fleet Server agent policies'); + logger.debug('Setting up Fleet Server agent policies'); await ensureFleetServerAgentPoliciesExists(soClient, esClient); return { @@ -132,7 +131,7 @@ export async function ensureFleetGlobalEsAssets( ) { const logger = appContextService.getLogger(); // Ensure Global Fleet ES assets are installed - logger.info('Creating Fleet component template and ingest pipeline'); + logger.debug('Creating Fleet component template and ingest pipeline'); const globalAssetsRes = await Promise.all([ ensureDefaultComponentTemplate(esClient), ensureFleetFinalPipelineIsInstalled(esClient), From 23c4ac108ecad9473a45d15b15f3c7970773c6d3 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Tue, 9 Nov 2021 15:57:11 -0500 Subject: [PATCH 07/18] Maybe fix failing tests --- test/accessibility/apps/dashboard.ts | 1 + test/examples/embeddables/adding_children.ts | 1 + test/functional/apps/management/_mgmt_import_saved_objects.js | 1 + 3 files changed, 3 insertions(+) diff --git a/test/accessibility/apps/dashboard.ts b/test/accessibility/apps/dashboard.ts index 54eb5e7df4178..9a7b0ee77f4a0 100644 --- a/test/accessibility/apps/dashboard.ts +++ b/test/accessibility/apps/dashboard.ts @@ -45,6 +45,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); it('add a visualization', async () => { + await listingTable.searchForItemWithName('[Flights]'); await testSubjects.click('savedObjectTitle[Flights]-Delay-Buckets'); await a11y.testAppSnapshot(); }); diff --git a/test/examples/embeddables/adding_children.ts b/test/examples/embeddables/adding_children.ts index ee06622a33f51..094c028bdb46d 100644 --- a/test/examples/embeddables/adding_children.ts +++ b/test/examples/embeddables/adding_children.ts @@ -25,6 +25,7 @@ export default function ({ getService }: PluginFunctionalProviderContext) { await testSubjects.waitForDeleted('savedObjectFinderLoadingIndicator'); await testSubjects.click('savedObjectTitleGarbage'); await testSubjects.moveMouseTo('euiFlyoutCloseButton'); + await testSubjects.setValue('savedObjectFinderSearchInput', 'todo'); await flyout.ensureClosed('dashboardAddPanel'); const tasks = await testSubjects.getVisibleTextAll('todoEmbeddableTask'); expect(tasks).to.eql(['Goes out on Wednesdays!', 'Take the garbage out']); diff --git a/test/functional/apps/management/_mgmt_import_saved_objects.js b/test/functional/apps/management/_mgmt_import_saved_objects.js index cf30b6f4ccf0d..95b0bbb7ed03b 100644 --- a/test/functional/apps/management/_mgmt_import_saved_objects.js +++ b/test/functional/apps/management/_mgmt_import_saved_objects.js @@ -39,6 +39,7 @@ export default function ({ getService, getPageObjects }) { await PageObjects.savedObjects.clickConfirmChanges(); await PageObjects.savedObjects.clickImportDone(); await PageObjects.savedObjects.waitTableIsLoaded(); + await PageObjects.savedObjects.searchForObject('mysaved'); //instead of asserting on count- am asserting on the titles- which is more accurate than count. const objects = await PageObjects.savedObjects.getRowTitles(); From 8fe61e281bcbdcc5fb26bcac2fcc9917f37414cf Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Wed, 10 Nov 2021 09:13:05 -0500 Subject: [PATCH 08/18] Try fixing tests again --- test/accessibility/apps/dashboard.ts | 2 +- test/examples/embeddables/adding_children.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test/accessibility/apps/dashboard.ts b/test/accessibility/apps/dashboard.ts index 9a7b0ee77f4a0..44cb0aa7ce117 100644 --- a/test/accessibility/apps/dashboard.ts +++ b/test/accessibility/apps/dashboard.ts @@ -45,7 +45,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); it('add a visualization', async () => { - await listingTable.searchForItemWithName('[Flights]'); + await testSubjects.setValue('savedObjectFinderSearchInput', '[Flights]'); await testSubjects.click('savedObjectTitle[Flights]-Delay-Buckets'); await a11y.testAppSnapshot(); }); diff --git a/test/examples/embeddables/adding_children.ts b/test/examples/embeddables/adding_children.ts index 094c028bdb46d..dcc7467689a18 100644 --- a/test/examples/embeddables/adding_children.ts +++ b/test/examples/embeddables/adding_children.ts @@ -23,9 +23,10 @@ export default function ({ getService }: PluginFunctionalProviderContext) { await testSubjects.click('embeddablePanelToggleMenuIcon'); await testSubjects.click('embeddablePanelAction-ACTION_ADD_PANEL'); await testSubjects.waitForDeleted('savedObjectFinderLoadingIndicator'); + await testSubjects.click('savedObjectFinderFilterButton'); + await testSubjects.click('savedObjectFinderFilter-todo'); await testSubjects.click('savedObjectTitleGarbage'); await testSubjects.moveMouseTo('euiFlyoutCloseButton'); - await testSubjects.setValue('savedObjectFinderSearchInput', 'todo'); await flyout.ensureClosed('dashboardAddPanel'); const tasks = await testSubjects.getVisibleTextAll('todoEmbeddableTask'); expect(tasks).to.eql(['Goes out on Wednesdays!', 'Take the garbage out']); From e135fdd9ded4b56dc3490b4692fea64b6e45c54a Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Wed, 10 Nov 2021 09:46:45 -0500 Subject: [PATCH 09/18] Fix another dashboard test --- test/accessibility/apps/dashboard.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/test/accessibility/apps/dashboard.ts b/test/accessibility/apps/dashboard.ts index 44cb0aa7ce117..847f7b9eff3e9 100644 --- a/test/accessibility/apps/dashboard.ts +++ b/test/accessibility/apps/dashboard.ts @@ -86,6 +86,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); it('Add one more saved object to cancel it', async () => { + await testSubjects.setValue('savedObjectFinderSearchInput', '[Flights]'); await testSubjects.click('savedObjectTitle[Flights]-Destination-Weather'); await a11y.testAppSnapshot(); }); From 4c2e113c855dd10d2e94da4592fb382cf6641c35 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Wed, 10 Nov 2021 11:55:57 -0500 Subject: [PATCH 10/18] Re-add output logs after merge --- x-pack/plugins/fleet/server/services/preconfiguration.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.ts b/x-pack/plugins/fleet/server/services/preconfiguration.ts index 63b07476d4c25..b16eae266d28c 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.ts @@ -108,8 +108,10 @@ export async function ensurePreconfiguredOutputs( existingOutput && isPreconfiguredOutputDifferentFromCurrent(existingOutput, data); if (isCreate) { + logger.debug(`Creating output ${output.id}`); await outputService.create(soClient, data, { id, fromPreconfiguration: true }); } else if (isUpdateWithNewData) { + logger.debug(`Updating output ${output.id}`); await outputService.update(soClient, id, data, { fromPreconfiguration: true }); // Bump revision of all policies using that output if (outputData.is_default || outputData.is_default_monitoring) { From 3c019ba38b2b6be564754b270c5750fb997863c2 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Thu, 11 Nov 2021 10:24:00 -0500 Subject: [PATCH 11/18] Log non-fatal errors during Fleet setup on boot --- x-pack/plugins/fleet/server/plugin.ts | 11 +++++++-- .../fleet/server/routes/setup/handlers.ts | 20 ++-------------- x-pack/plugins/fleet/server/services/setup.ts | 24 +++++++++++++++++++ 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/x-pack/plugins/fleet/server/plugin.ts b/x-pack/plugins/fleet/server/plugin.ts index 4959b75733fa9..88ee26f0c3e6c 100644 --- a/x-pack/plugins/fleet/server/plugin.ts +++ b/x-pack/plugins/fleet/server/plugin.ts @@ -83,7 +83,7 @@ import { RouterWrappers } from './routes/security'; import { FleetArtifactsClient } from './services/artifacts'; import type { FleetRouter } from './types/request_context'; import { TelemetryEventsSender } from './telemetry/sender'; -import { setupFleet } from './services/setup'; +import { formatNonFatalErrors, setupFleet } from './services/setup'; export interface FleetSetupDeps { licensing: LicensingPluginSetup; @@ -340,11 +340,18 @@ export class FleetPlugin try { logger.info('Beginning fleet setup'); - await setupFleet( + const { nonFatalErrors } = await setupFleet( new SavedObjectsClient(core.savedObjects.createInternalRepository()), core.elasticsearch.client.asInternalUser ); + if (nonFatalErrors.length > 0) { + logger.info('Encountered non fatal errors during Fleet setup'); + formatNonFatalErrors(nonFatalErrors).forEach((error) => + logger.info(JSON.stringify(error)) + ); + } + logger.info('Fleet setup completed'); } catch (error) { logger.warn('Fleet setup failed'); diff --git a/x-pack/plugins/fleet/server/routes/setup/handlers.ts b/x-pack/plugins/fleet/server/routes/setup/handlers.ts index fad5d93c3f5d5..60094c532b913 100644 --- a/x-pack/plugins/fleet/server/routes/setup/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/setup/handlers.ts @@ -7,7 +7,7 @@ import { appContextService } from '../../services'; import type { GetFleetStatusResponse, PostFleetSetupResponse } from '../../../common'; -import { setupFleet } from '../../services/setup'; +import { formatNonFatalErrors, setupFleet } from '../../services/setup'; import { hasFleetServers } from '../../services/fleet_server'; import { defaultIngestErrorHandler } from '../../errors'; import type { FleetRequestHandler } from '../../types'; @@ -50,24 +50,8 @@ export const fleetSetupHandler: FleetRequestHandler = async (context, request, r const setupStatus = await setupFleet(soClient, esClient); const body: PostFleetSetupResponse = { ...setupStatus, - nonFatalErrors: setupStatus.nonFatalErrors.flatMap((e) => { - // JSONify the error object so it can be displayed properly in the UI - if ('error' in e) { - return { - name: e.error.name, - message: e.error.message, - }; - } else { - return e.errors.map((upgradePackagePolicyError: any) => { - return { - name: upgradePackagePolicyError.key, - message: upgradePackagePolicyError.message, - }; - }); - } - }), + nonFatalErrors: formatNonFatalErrors(setupStatus.nonFatalErrors), }; - return response.ok({ body }); } catch (error) { return defaultIngestErrorHandler({ error, response }); diff --git a/x-pack/plugins/fleet/server/services/setup.ts b/x-pack/plugins/fleet/server/services/setup.ts index 5465a35081305..f3d1381c5265b 100644 --- a/x-pack/plugins/fleet/server/services/setup.ts +++ b/x-pack/plugins/fleet/server/services/setup.ts @@ -198,3 +198,27 @@ export async function ensureDefaultEnrollmentAPIKeysExists( }) ); } + +/** + * Maps the `nonFatalErrors` object returned by the setup process to a more readable + * and predictable format suitable for logging output or UI presentation. + */ +export function formatNonFatalErrors( + nonFatalErrors: SetupStatus['nonFatalErrors'] +): Array<{ name: string; message: string }> { + return nonFatalErrors.flatMap((e) => { + if ('error' in e) { + return { + name: e.error.name, + message: e.error.message, + }; + } else { + return e.errors.map((upgradePackagePolicyError: any) => { + return { + name: upgradePackagePolicyError.key, + message: upgradePackagePolicyError.message, + }; + }); + } + }); +} From 8e2e1beb9f4409e35044c13cd0c6af52188e0ef3 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Thu, 11 Nov 2021 11:40:16 -0500 Subject: [PATCH 12/18] Don't rely on fleetSetupCompleted to be called --- x-pack/plugins/fleet/server/plugin.ts | 46 +++++++++++++++------------ 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/x-pack/plugins/fleet/server/plugin.ts b/x-pack/plugins/fleet/server/plugin.ts index 88ee26f0c3e6c..9786b07ccec6e 100644 --- a/x-pack/plugins/fleet/server/plugin.ts +++ b/x-pack/plugins/fleet/server/plugin.ts @@ -335,29 +335,33 @@ export class FleetPlugin const logger = appContextService.getLogger(); - return { - fleetSetupCompleted: async () => { - try { - logger.info('Beginning fleet setup'); - - const { nonFatalErrors } = await setupFleet( - new SavedObjectsClient(core.savedObjects.createInternalRepository()), - core.elasticsearch.client.asInternalUser + const fleetSetupPromise = async () => { + try { + logger.info('Beginning fleet setup'); + + const { nonFatalErrors } = await setupFleet( + new SavedObjectsClient(core.savedObjects.createInternalRepository()), + core.elasticsearch.client.asInternalUser + ); + + if (nonFatalErrors.length > 0) { + logger.info('Encountered non fatal errors during Fleet setup'); + formatNonFatalErrors(nonFatalErrors).forEach((error) => + logger.info(JSON.stringify(error)) ); - - if (nonFatalErrors.length > 0) { - logger.info('Encountered non fatal errors during Fleet setup'); - formatNonFatalErrors(nonFatalErrors).forEach((error) => - logger.info(JSON.stringify(error)) - ); - } - - logger.info('Fleet setup completed'); - } catch (error) { - logger.warn('Fleet setup failed'); - logger.warn(error); } - }, + + logger.info('Fleet setup completed'); + } catch (error) { + logger.warn('Fleet setup failed'); + logger.warn(error); + } + }; + + fleetSetupPromise(); + + return { + fleetSetupCompleted: fleetSetupPromise, esIndexPatternService: new ESIndexPatternSavedObjectService(), packageService: { getInstallation, From ede4a18509470a21257e18c3fa5c5bd392efb6cd Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Thu, 11 Nov 2021 12:08:06 -0500 Subject: [PATCH 13/18] Fix failing test --- x-pack/plugins/fleet/server/routes/setup/handlers.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/fleet/server/routes/setup/handlers.test.ts b/x-pack/plugins/fleet/server/routes/setup/handlers.test.ts index b39c6e7686110..ffdec9509b05c 100644 --- a/x-pack/plugins/fleet/server/routes/setup/handlers.test.ts +++ b/x-pack/plugins/fleet/server/routes/setup/handlers.test.ts @@ -18,6 +18,7 @@ import { fleetSetupHandler } from './handlers'; jest.mock('../../services/setup', () => { return { + ...jest.requireActual('../../services/setup'), setupFleet: jest.fn(), }; }); From 95475d4c020701b9ce6346287299271849eada3b Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Thu, 11 Nov 2021 14:13:11 -0500 Subject: [PATCH 14/18] Track fleet setup status to avoid double calls --- x-pack/plugins/fleet/server/plugin.ts | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/fleet/server/plugin.ts b/x-pack/plugins/fleet/server/plugin.ts index 9786b07ccec6e..e81b2c6f6872a 100644 --- a/x-pack/plugins/fleet/server/plugin.ts +++ b/x-pack/plugins/fleet/server/plugin.ts @@ -179,6 +179,7 @@ export class FleetPlugin private securitySetup?: SecurityPluginSetup; private encryptedSavedObjectsSetup?: EncryptedSavedObjectsPluginSetup; private readonly telemetryEventsSender: TelemetryEventsSender; + private fleetSetupStatus: 'initial' | 'pending' | 'complete'; constructor(private readonly initializerContext: PluginInitializerContext) { this.config$ = this.initializerContext.config.create(); @@ -188,6 +189,7 @@ export class FleetPlugin this.logger = this.initializerContext.logger.get(); this.configInitialValue = this.initializerContext.config.get(); this.telemetryEventsSender = new TelemetryEventsSender(this.logger.get('telemetry_events')); + this.fleetSetupStatus = 'initial'; } public setup(core: CoreSetup, deps: FleetSetupDeps) { @@ -335,9 +337,14 @@ export class FleetPlugin const logger = appContextService.getLogger(); - const fleetSetupPromise = async () => { + const fleetSetupPromise = new Promise(async (resolve, reject) => { try { + if (this.fleetSetupStatus === 'pending' || this.fleetSetupStatus === 'complete') { + resolve(); + } + logger.info('Beginning fleet setup'); + this.fleetSetupStatus = 'pending'; const { nonFatalErrors } = await setupFleet( new SavedObjectsClient(core.savedObjects.createInternalRepository()), @@ -352,16 +359,21 @@ export class FleetPlugin } logger.info('Fleet setup completed'); + this.fleetSetupStatus = 'complete'; + + resolve(); } catch (error) { logger.warn('Fleet setup failed'); logger.warn(error); - } - }; - fleetSetupPromise(); + this.fleetSetupStatus = 'initial'; + + reject(error); + } + }); return { - fleetSetupCompleted: fleetSetupPromise, + fleetSetupCompleted: () => fleetSetupPromise, esIndexPatternService: new ESIndexPatternSavedObjectService(), packageService: { getInstallation, From c937d62210b1bd459accb4706ab2936e0176ac03 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Thu, 11 Nov 2021 14:53:31 -0500 Subject: [PATCH 15/18] Use IIFE in place of Promise ctor --- x-pack/plugins/fleet/server/plugin.ts | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/fleet/server/plugin.ts b/x-pack/plugins/fleet/server/plugin.ts index e81b2c6f6872a..0e28c7cec3278 100644 --- a/x-pack/plugins/fleet/server/plugin.ts +++ b/x-pack/plugins/fleet/server/plugin.ts @@ -337,10 +337,10 @@ export class FleetPlugin const logger = appContextService.getLogger(); - const fleetSetupPromise = new Promise(async (resolve, reject) => { + const fleetSetupPromise = (async () => { try { if (this.fleetSetupStatus === 'pending' || this.fleetSetupStatus === 'complete') { - resolve(); + return; } logger.info('Beginning fleet setup'); @@ -360,17 +360,13 @@ export class FleetPlugin logger.info('Fleet setup completed'); this.fleetSetupStatus = 'complete'; - - resolve(); } catch (error) { logger.warn('Fleet setup failed'); logger.warn(error); this.fleetSetupStatus = 'initial'; - - reject(error); } - }); + })(); return { fleetSetupCompleted: () => fleetSetupPromise, From 76e4829b9ab1507c844268d6b4fbff4fc876a840 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Thu, 11 Nov 2021 15:14:40 -0500 Subject: [PATCH 16/18] Remove unnecessary fleetSetupStatus value --- x-pack/plugins/fleet/server/plugin.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/x-pack/plugins/fleet/server/plugin.ts b/x-pack/plugins/fleet/server/plugin.ts index 0e28c7cec3278..73bb7d5f0d0f9 100644 --- a/x-pack/plugins/fleet/server/plugin.ts +++ b/x-pack/plugins/fleet/server/plugin.ts @@ -179,7 +179,6 @@ export class FleetPlugin private securitySetup?: SecurityPluginSetup; private encryptedSavedObjectsSetup?: EncryptedSavedObjectsPluginSetup; private readonly telemetryEventsSender: TelemetryEventsSender; - private fleetSetupStatus: 'initial' | 'pending' | 'complete'; constructor(private readonly initializerContext: PluginInitializerContext) { this.config$ = this.initializerContext.config.create(); @@ -189,7 +188,6 @@ export class FleetPlugin this.logger = this.initializerContext.logger.get(); this.configInitialValue = this.initializerContext.config.get(); this.telemetryEventsSender = new TelemetryEventsSender(this.logger.get('telemetry_events')); - this.fleetSetupStatus = 'initial'; } public setup(core: CoreSetup, deps: FleetSetupDeps) { @@ -339,12 +337,7 @@ export class FleetPlugin const fleetSetupPromise = (async () => { try { - if (this.fleetSetupStatus === 'pending' || this.fleetSetupStatus === 'complete') { - return; - } - logger.info('Beginning fleet setup'); - this.fleetSetupStatus = 'pending'; const { nonFatalErrors } = await setupFleet( new SavedObjectsClient(core.savedObjects.createInternalRepository()), @@ -359,12 +352,9 @@ export class FleetPlugin } logger.info('Fleet setup completed'); - this.fleetSetupStatus = 'complete'; } catch (error) { logger.warn('Fleet setup failed'); logger.warn(error); - - this.fleetSetupStatus = 'initial'; } })(); From 76138ec1c31c57ef08373e3f5361dc25b24a5dd7 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Mon, 15 Nov 2021 07:50:39 -0500 Subject: [PATCH 17/18] Move non-error logs into setupFleet method --- x-pack/plugins/fleet/server/plugin.ts | 13 +------------ x-pack/plugins/fleet/server/services/setup.ts | 8 ++++++++ 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/fleet/server/plugin.ts b/x-pack/plugins/fleet/server/plugin.ts index 73bb7d5f0d0f9..b3602b54604b9 100644 --- a/x-pack/plugins/fleet/server/plugin.ts +++ b/x-pack/plugins/fleet/server/plugin.ts @@ -337,21 +337,10 @@ export class FleetPlugin const fleetSetupPromise = (async () => { try { - logger.info('Beginning fleet setup'); - - const { nonFatalErrors } = await setupFleet( + await setupFleet( new SavedObjectsClient(core.savedObjects.createInternalRepository()), core.elasticsearch.client.asInternalUser ); - - if (nonFatalErrors.length > 0) { - logger.info('Encountered non fatal errors during Fleet setup'); - formatNonFatalErrors(nonFatalErrors).forEach((error) => - logger.info(JSON.stringify(error)) - ); - } - - logger.info('Fleet setup completed'); } catch (error) { logger.warn('Fleet setup failed'); logger.warn(error); diff --git a/x-pack/plugins/fleet/server/services/setup.ts b/x-pack/plugins/fleet/server/services/setup.ts index f3d1381c5265b..1c84073552e57 100644 --- a/x-pack/plugins/fleet/server/services/setup.ts +++ b/x-pack/plugins/fleet/server/services/setup.ts @@ -52,6 +52,7 @@ async function createSetupSideEffects( esClient: ElasticsearchClient ): Promise { const logger = appContextService.getLogger(); + logger.info('Beginning fleet setup'); const { agentPolicies: policiesOrUndefined, @@ -114,6 +115,13 @@ async function createSetupSideEffects( logger.debug('Setting up Fleet Server agent policies'); await ensureFleetServerAgentPoliciesExists(soClient, esClient); + if (nonFatalErrors.length > 0) { + logger.info('Encountered non fatal errors during Fleet setup'); + formatNonFatalErrors(nonFatalErrors).forEach((error) => logger.info(JSON.stringify(error))); + } + + logger.info('Fleet setup completed'); + return { isInitialized: true, nonFatalErrors, From 59190ba96e383089a02006cbcbfa6fccaa2e45ce Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Mon, 15 Nov 2021 07:51:42 -0500 Subject: [PATCH 18/18] Remove unused formatNonFatalErrors import --- x-pack/plugins/fleet/server/plugin.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/server/plugin.ts b/x-pack/plugins/fleet/server/plugin.ts index b3602b54604b9..f7593e32c25c9 100644 --- a/x-pack/plugins/fleet/server/plugin.ts +++ b/x-pack/plugins/fleet/server/plugin.ts @@ -83,7 +83,7 @@ import { RouterWrappers } from './routes/security'; import { FleetArtifactsClient } from './services/artifacts'; import type { FleetRouter } from './types/request_context'; import { TelemetryEventsSender } from './telemetry/sender'; -import { formatNonFatalErrors, setupFleet } from './services/setup'; +import { setupFleet } from './services/setup'; export interface FleetSetupDeps { licensing: LicensingPluginSetup;