[Fleet] Improve performance of Fleet setup (#102219) (#102559)

* Remove endpoint from the default packages * Change the default spinner for the initial load * Export fleet endpoint package as a constant * Use constants for special packages * Simplify type signature of `isRequiredPackage` * Remove unused types * Simplify required and default package definitions * Treat REQUIRED_PACKAGES as independent from DEFAULT_PACKAGES We want to keep the assumption that the lists contain the same packages only in `epm/constants.ts` * Install all default packages, not only the required ones * Document the purpose of each package list * Handle auto-update for non-default packages * Make `endpoint` non-removable * Make endpoint package be installed by default in tests * Rename requiredPackages to unremovablePackages * Fix type check * Add Endpoint to be installed by default on Fleet tests too Co-authored-by: Jen Huang <its.jenetic@gmail.com> Co-authored-by: Alejandro Fernández Gómez <alejandro.fernandez@elastic.co> Co-authored-by: Jen Huang <its.jenetic@gmail.com>
elastic · Jun 17, 2021 · cb18f07 · cb18f07
1 parent 6af72cd
commit cb18f07
Show file tree

Hide file tree

Showing 18 changed files with 118 additions and 50 deletions.
diff --git a/x-pack/plugins/fleet/common/constants/epm.ts b/x-pack/plugins/fleet/common/constants/epm.ts
@@ -9,17 +9,35 @@ export const PACKAGES_SAVED_OBJECT_TYPE = 'epm-packages';
 export const ASSETS_SAVED_OBJECT_TYPE = 'epm-packages-assets';
 export const MAX_TIME_COMPLETE_INSTALL = 60000;
 
+export const FLEET_SYSTEM_PACKAGE = 'system';
+export const FLEET_ELASTIC_AGENT_PACKAGE = 'elastic_agent';
 export const FLEET_SERVER_PACKAGE = 'fleet_server';
+export const FLEET_ENDPOINT_PACKAGE = 'endpoint';
 
-export const requiredPackages = {
-  System: 'system',
-  Endpoint: 'endpoint',
-  ElasticAgent: 'elastic_agent',
-  FleetServer: FLEET_SERVER_PACKAGE,
-} as const;
+/*
+ Package rules:
+|               | unremovablePackages | defaultPackages | autoUpdatePackages |
+|---------------|:---------------------:|:---------------:|:------------------:|
+| Removable     |         ❌             |        ✔️        |          ✔️         |
+| Auto-installs |         ❌             |        ✔️        |          ❌         |
+| Auto-updates  |         ❌             |        ✔️        |          ✔️         |
+
+`endpoint` is a special package. It needs to autoupdate, it needs to _not_ be
+removable, but it doesn't install by default. Following the table, it needs to
+be in `unremovablePackages` and in `autoUpdatePackages`, but not in
+`defaultPackages`.
+*/
+
+export const unremovablePackages = [
+  FLEET_SYSTEM_PACKAGE,
+  FLEET_ELASTIC_AGENT_PACKAGE,
+  FLEET_SERVER_PACKAGE,
+  FLEET_ENDPOINT_PACKAGE,
+];
+
+export const defaultPackages = unremovablePackages.filter((p) => p !== FLEET_ENDPOINT_PACKAGE);
 
-// these are currently identical. we can separate if they later diverge
-export const defaultPackages = requiredPackages;
+export const autoUpdatePackages = [FLEET_ENDPOINT_PACKAGE];
 
 export const agentAssetTypes = {
   Input: 'input',

diff --git a/x-pack/plugins/fleet/common/constants/preconfiguration.ts b/x-pack/plugins/fleet/common/constants/preconfiguration.ts
@@ -7,7 +7,12 @@
 
 import type { PreconfiguredAgentPolicy } from '../types';
 
-import { defaultPackages } from './epm';
+import {
+  defaultPackages,
+  FLEET_SYSTEM_PACKAGE,
+  FLEET_SERVER_PACKAGE,
+  autoUpdatePackages,
+} from './epm';
 
 export const PRECONFIGURATION_DELETION_RECORD_SAVED_OBJECT_TYPE =
   'fleet-preconfiguration-deletion-record';
@@ -27,9 +32,9 @@ export const DEFAULT_AGENT_POLICY: PreconfiguredAgentPolicyWithDefaultInputs = {
   description: 'Default agent policy created by Kibana',
   package_policies: [
     {
-      name: `${defaultPackages.System}-1`,
+      name: `${FLEET_SYSTEM_PACKAGE}-1`,
       package: {
-        name: defaultPackages.System,
+        name: FLEET_SYSTEM_PACKAGE,
       },
     },
   ],
@@ -44,9 +49,9 @@ export const DEFAULT_FLEET_SERVER_AGENT_POLICY: PreconfiguredAgentPolicyWithDefa
   description: 'Default Fleet Server agent policy created by Kibana',
   package_policies: [
     {
-      name: `${defaultPackages.FleetServer}-1`,
+      name: `${FLEET_SERVER_PACKAGE}-1`,
       package: {
-        name: defaultPackages.FleetServer,
+        name: FLEET_SERVER_PACKAGE,
       },
     },
   ],
@@ -56,13 +61,15 @@ export const DEFAULT_FLEET_SERVER_AGENT_POLICY: PreconfiguredAgentPolicyWithDefa
   monitoring_enabled: ['logs', 'metrics'] as Array<'logs' | 'metrics'>,
 };
 
-export const DEFAULT_PACKAGES = Object.values(defaultPackages).map((name) => ({
+export const DEFAULT_PACKAGES = defaultPackages.map((name) => ({
   name,
   version: PRECONFIGURATION_LATEST_KEYWORD,
 }));
 
-// these are currently identical. we can separate if they later diverge
-export const REQUIRED_PACKAGES = DEFAULT_PACKAGES;
+export const AUTO_UPDATE_PACKAGES = autoUpdatePackages.map((name) => ({
+  name,
+  version: PRECONFIGURATION_LATEST_KEYWORD,
+}));
 
 export interface PreconfigurationError {
   package?: { name: string; version: string };

diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts
@@ -13,9 +13,7 @@ import type {
   ASSETS_SAVED_OBJECT_TYPE,
   agentAssetTypes,
   dataTypes,
-  defaultPackages,
   installationStatuses,
-  requiredPackages,
 } from '../../constants';
 import type { ValueOf } from '../../types';
 
@@ -408,10 +406,6 @@ export type PackageAssetReference = Pick<SavedObjectReference, 'id'> & {
   type: typeof ASSETS_SAVED_OBJECT_TYPE;
 };
 
-export type RequiredPackage = typeof requiredPackages;
-
-export type DefaultPackages = typeof defaultPackages;
-
 export interface IndexTemplateMappings {
   properties: any;
 }

diff --git a/x-pack/plugins/fleet/public/applications/fleet/app.tsx b/x-pack/plugins/fleet/public/applications/fleet/app.tsx
@@ -35,7 +35,7 @@ import {
   useStartServices,
   UIExtensionsContext,
 } from './hooks';
-import { Error, Loading, SettingFlyout } from './components';
+import { Error, Loading, SettingFlyout, FleetSetupLoading } from './components';
 import type { UIExtensionsStorage } from './types';
 
 import { FLEET_ROUTING_PATHS } from './constants';
@@ -180,7 +180,7 @@ export const WithPermissionsAndSetup: React.FC = memo(({ children }) => {
             error={initializationError}
           />
         ) : (
-          <Loading />
+          <FleetSetupLoading />
         )}
       </ErrorLayout>
     );

diff --git a/x-pack/plugins/fleet/public/components/fleet_setup_loading.tsx b/x-pack/plugins/fleet/public/components/fleet_setup_loading.tsx
@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+import { FormattedMessage } from '@kbn/i18n/react';
+import { EuiEmptyPrompt, EuiLoadingSpinner } from '@elastic/eui';
+
+export const FleetSetupLoading: React.FunctionComponent = () => (
+  <EuiEmptyPrompt
+    title={
+      <h2>
+        <FormattedMessage id="xpack.fleet.setup.titleLabel" defaultMessage="Loading Fleet..." />
+      </h2>
+    }
+    titleSize="m"
+    body={<EuiLoadingSpinner size="xl" />}
+  />
+);
diff --git a/x-pack/plugins/fleet/public/components/index.ts b/x-pack/plugins/fleet/public/components/index.ts
@@ -6,6 +6,7 @@
  */
 export { Loading } from './loading';
 export { Error } from './error';
+export { FleetSetupLoading } from './fleet_setup_loading';
 export { PackageIcon } from './package_icon';
 export { ContextMenuActions } from './context_menu_actions';
 export { LinkedAgentCount } from './linked_agent_count';

diff --git a/x-pack/plugins/fleet/server/constants/index.ts b/x-pack/plugins/fleet/server/constants/index.ts
@@ -49,7 +49,6 @@ export {
   DEFAULT_FLEET_SERVER_AGENT_POLICY,
   DEFAULT_OUTPUT,
   DEFAULT_PACKAGES,
-  REQUIRED_PACKAGES,
   // Fleet Server index
   FLEET_SERVER_SERVERS_INDEX,
   ENROLLMENT_API_KEYS_INDEX,

diff --git a/x-pack/plugins/fleet/server/routes/agent_policy/handlers.ts b/x-pack/plugins/fleet/server/routes/agent_policy/handlers.ts
@@ -23,7 +23,7 @@ import type {
   GetFullAgentPolicyRequestSchema,
 } from '../../types';
 import type { AgentPolicy, NewPackagePolicy } from '../../types';
-import { defaultPackages } from '../../../common';
+import { FLEET_SYSTEM_PACKAGE } from '../../../common';
 import type {
   GetAgentPoliciesResponse,
   GetAgentPoliciesResponseItem,
@@ -120,7 +120,7 @@ export const createAgentPolicyHandler: RequestHandler<
       // successfully
       withSysMonitoring
         ? packagePolicyService
-            .buildPackagePolicyFromPackage(soClient, defaultPackages.System)
+            .buildPackagePolicyFromPackage(soClient, FLEET_SYSTEM_PACKAGE)
             .catch(() => undefined)
         : undefined,
     ]);

diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.ts
@@ -28,7 +28,7 @@ import { getEsPackage } from '../archive/storage';
 import { getArchivePackage } from '../archive';
 import { normalizeKuery } from '../../saved_object';
 
-import { createInstallableFrom, isRequiredPackage } from './index';
+import { createInstallableFrom, isUnremovablePackage } from './index';
 
 export { getFile, SearchParams } from '../registry';
 
@@ -125,7 +125,7 @@ export async function getPackageInfo(options: {
     latestVersion: latestPackage.version,
     title: packageInfo.title || nameAsTitle(packageInfo.name),
     assets: Registry.groupPathsByService(paths || []),
-    removable: !isRequiredPackage(pkgName),
+    removable: !isUnremovablePackage(pkgName),
     notice: Registry.getNoticePath(paths || []),
   };
   const updated = { ...packageInfo, ...additions };

diff --git a/x-pack/plugins/fleet/server/services/epm/packages/index.ts b/x-pack/plugins/fleet/server/services/epm/packages/index.ts
@@ -7,8 +7,7 @@
 
 import type { SavedObject } from 'src/core/server';
 
-import { requiredPackages, installationStatuses } from '../../../../common';
-import type { RequiredPackage, ValueOf } from '../../../../common';
+import { unremovablePackages, installationStatuses } from '../../../../common';
 import { KibanaAssetType } from '../../../types';
 import type { AssetType, Installable, Installation } from '../../../types';
 
@@ -33,8 +32,8 @@ export {
 } from './install';
 export { removeInstallation } from './remove';
 
-export function isRequiredPackage(value: string): value is ValueOf<RequiredPackage> {
-  return Object.values(requiredPackages).some((required) => value === required);
+export function isUnremovablePackage(value: string): boolean {
+  return unremovablePackages.includes(value);
 }
 
 export class PackageNotInstalledError extends Error {

diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts
@@ -34,11 +34,19 @@ import { toAssetReference } from '../kibana/assets/install';
 import type { ArchiveAsset } from '../kibana/assets/install';
 import { installIndexPatterns } from '../kibana/index_pattern/install';
 
-import { isRequiredPackage, getInstallation, getInstallationObject } from './index';
+import { isUnremovablePackage, getInstallation, getInstallationObject } from './index';
 import { removeInstallation } from './remove';
 import { getPackageSavedObjects } from './get';
 import { _installPackage } from './_install_package';
 
+export async function isPackageInstalled(options: {
+  savedObjectsClient: SavedObjectsClientContract;
+  pkgName: string;
+}): Promise<boolean> {
+  const installedPackage = await getInstallation(options);
+  return installedPackage !== undefined;
+}
+
 export async function isPackageVersionOrLaterInstalled(options: {
   savedObjectsClient: SavedObjectsClientContract;
   pkgName: string;
@@ -426,7 +434,7 @@ export async function createInstallation(options: {
 }) {
   const { savedObjectsClient, packageInfo, installSource } = options;
   const { internal = false, name: pkgName, version: pkgVersion } = packageInfo;
-  const removable = !isRequiredPackage(pkgName);
+  const removable = !isUnremovablePackage(pkgName);
   const toSaveESIndexPatterns = generateESIndexPatterns(packageInfo.data_streams);
 
   const created = await savedObjectsClient.create<Installation>(

diff --git a/x-pack/plugins/fleet/server/services/setup.ts b/x-pack/plugins/fleet/server/services/setup.ts
@@ -5,10 +5,13 @@
  * 2.0.
  */
 
+import { compact } from 'lodash';
+
 import type { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/server';
 
+import { AUTO_UPDATE_PACKAGES } from '../../common';
 import type { DefaultPackagesInstallationError, PreconfigurationError } from '../../common';
-import { SO_SEARCH_LIMIT, REQUIRED_PACKAGES } from '../constants';
+import { SO_SEARCH_LIMIT, DEFAULT_PACKAGES } from '../constants';
 
 import { appContextService } from './app_context';
 import { agentPolicyService } from './agent_policy';
@@ -21,6 +24,7 @@ import { awaitIfPending } from './setup_utils';
 import { ensureAgentActionPolicyChangeExists } from './agents';
 import { awaitIfFleetServerSetupPending } from './fleet_server';
 import { ensureFleetFinalPipelineIsInstalled } from './epm/elasticsearch/ingest_pipeline/install';
+import { isPackageInstalled } from './epm/packages/install';
 
 export interface SetupStatus {
   isInitialized: boolean;
@@ -53,11 +57,25 @@ async function createSetupSideEffects(
   const policies = policiesOrUndefined ?? [];
 
   let packages = packagesOrUndefined ?? [];
+
   // Ensure that required packages are always installed even if they're left out of the config
   const preconfiguredPackageNames = new Set(packages.map((pkg) => pkg.name));
+
+  const autoUpdateablePackages = compact(
+    await Promise.all(
+      AUTO_UPDATE_PACKAGES.map((pkg) =>
+        isPackageInstalled({
+          savedObjectsClient: soClient,
+          pkgName: pkg.name,
+        }).then((installed) => (installed ? pkg : undefined))
+      )
+    )
+  );
+
   packages = [
     ...packages,
-    ...REQUIRED_PACKAGES.filter((pkg) => !preconfiguredPackageNames.has(pkg.name)),
+    ...DEFAULT_PACKAGES.filter((pkg) => !preconfiguredPackageNames.has(pkg.name)),
+    ...autoUpdateablePackages.filter((pkg) => !preconfiguredPackageNames.has(pkg.name)),
   ];
 
   const { nonFatalErrors } = await ensurePreconfiguredPackagesAndPolicies(

diff --git a/x-pack/plugins/fleet/server/types/index.tsx b/x-pack/plugins/fleet/server/types/index.tsx
@@ -63,7 +63,6 @@ export {
   IndexTemplate,
   RegistrySearchResults,
   RegistrySearchResult,
-  DefaultPackages,
   TemplateRef,
   IndexTemplateMappings,
   Settings,

diff --git a/x-pack/plugins/security_solution/server/usage/endpoints/endpoint.mocks.ts b/x-pack/plugins/security_solution/server/usage/endpoints/endpoint.mocks.ts
@@ -7,11 +7,8 @@
 
 import { SavedObjectsFindResponse } from 'src/core/server';
 
-import { Agent } from '../../../../fleet/common';
-import {
-  FLEET_ENDPOINT_PACKAGE_CONSTANT,
-  AGENT_EVENT_SAVED_OBJECT_TYPE,
-} from './fleet_saved_objects';
+import { Agent, FLEET_ENDPOINT_PACKAGE } from '../../../../fleet/common';
+import { AGENT_EVENT_SAVED_OBJECT_TYPE } from './fleet_saved_objects';
 
 const testAgentId = 'testAgentId';
 const testAgentPolicyId = 'testAgentPolicyId';
@@ -65,7 +62,7 @@ export const mockFleetObjectsResponse = (
           full: MockOSFullName,
         },
       },
-      packages: [FLEET_ENDPOINT_PACKAGE_CONSTANT, 'system'],
+      packages: [FLEET_ENDPOINT_PACKAGE, 'system'],
       last_checkin: lastCheckIn,
     },
     {
@@ -93,7 +90,7 @@ export const mockFleetObjectsResponse = (
           full: MockOSFullName,
         },
       },
-      packages: [FLEET_ENDPOINT_PACKAGE_CONSTANT, 'system'],
+      packages: [FLEET_ENDPOINT_PACKAGE, 'system'],
       last_checkin: lastCheckIn,
     },
   ],

diff --git a/x-pack/plugins/security_solution/server/usage/endpoints/fleet_saved_objects.ts b/x-pack/plugins/security_solution/server/usage/endpoints/fleet_saved_objects.ts
@@ -11,9 +11,7 @@ import {
   SavedObjectsFindResponse,
 } from 'src/core/server';
 import { AgentService } from '../../../../fleet/server';
-import { defaultPackages as FleetDefaultPackages } from '../../../../fleet/common';
-
-export const FLEET_ENDPOINT_PACKAGE_CONSTANT = FleetDefaultPackages.Endpoint;
+import { FLEET_ENDPOINT_PACKAGE } from '../../../../fleet/common';
 
 export const AGENT_EVENT_SAVED_OBJECT_TYPE = 'donotexistsanymore-since-7.13';
 
@@ -22,7 +20,7 @@ export const getEndpointIntegratedFleetMetadata = async (
   esClient: ElasticsearchClient
 ) => {
   return agentService?.listAgents(esClient, {
-    kuery: `(packages : ${FLEET_ENDPOINT_PACKAGE_CONSTANT})`,
+    kuery: `(packages : ${FLEET_ENDPOINT_PACKAGE})`,
     perPage: 10000,
     showInactive: false,
     sortField: 'enrolled_at',

diff --git a/x-pack/test/fleet_api_integration/config.ts b/x-pack/test/fleet_api_integration/config.ts
@@ -66,6 +66,9 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
       ...xPackAPITestsConfig.get('kbnTestServer'),
       serverArgs: [
         ...xPackAPITestsConfig.get('kbnTestServer.serverArgs'),
+        // always install Endpoint package by default when Fleet sets up
+        `--xpack.fleet.packages.0.name=endpoint`,
+        `--xpack.fleet.packages.0.version=latest`,
         ...(registryPort ? [`--xpack.fleet.registryUrl=http://localhost:${registryPort}`] : []),
       ],
     },

diff --git a/x-pack/test/security_solution_endpoint/config.ts b/x-pack/test/security_solution_endpoint/config.ts
@@ -39,9 +39,11 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
       ...xpackFunctionalConfig.get('kbnTestServer'),
       serverArgs: [
         ...xpackFunctionalConfig.get('kbnTestServer.serverArgs'),
-        '--xpack.fleet.enabled=true',
         // if you return an empty string here the kibana server will not start properly but an empty array works
         ...getRegistryUrlAsArray(),
+        // always install Endpoint package by default when Fleet sets up
+        `--xpack.fleet.packages.0.name=endpoint`,
+        `--xpack.fleet.packages.0.version=latest`,
       ],
     },
     layout: {