diff --git a/.gitignore b/.gitignore index 818d3a472d52c..7e45158458238 100644 --- a/.gitignore +++ b/.gitignore @@ -95,4 +95,3 @@ fleet-server-* elastic-agent.yml fleet-server.yml -/x-pack/plugins/fleet/server/bundled_packages diff --git a/src/dev/build/tasks/bundle_fleet_packages.ts b/src/dev/build/tasks/bundle_fleet_packages.ts index 7d0dc6a25a47e..b2faed818b55b 100644 --- a/src/dev/build/tasks/bundle_fleet_packages.ts +++ b/src/dev/build/tasks/bundle_fleet_packages.ts @@ -11,7 +11,7 @@ import JSON5 from 'json5'; import { readCliArgs } from '../args'; import { Task, read, downloadToDisk } from '../lib'; -const BUNDLED_PACKAGES_DIR = 'x-pack/plugins/fleet/server/bundled_packages'; +const BUNDLED_PACKAGES_DIR = 'x-pack/plugins/fleet/target/bundled_packages'; interface FleetPackage { name: string; diff --git a/src/dev/precommit_hook/casing_check_config.js b/src/dev/precommit_hook/casing_check_config.js index 860886811da54..932fdaf6c2e28 100644 --- a/src/dev/precommit_hook/casing_check_config.js +++ b/src/dev/precommit_hook/casing_check_config.js @@ -61,9 +61,6 @@ export const IGNORE_FILE_GLOBS = [ 'x-pack/plugins/maps/server/fonts/**/*', - // Bundled package names typically use a format like ${pkgName}-${pkgVersion}, so don't lint them - 'x-pack/plugins/fleet/server/bundled_packages/**/*', - // Bazel default files '**/WORKSPACE.bazel', '**/BUILD.bazel', diff --git a/x-pack/plugins/fleet/common/openapi/bundled.json b/x-pack/plugins/fleet/common/openapi/bundled.json index 46cd3e998ea7f..9c79397e25e10 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.json +++ b/x-pack/plugins/fleet/common/openapi/bundled.json @@ -621,6 +621,19 @@ "type" ] } + }, + "_meta": { + "type": "object", + "properties": { + "install_source": { + "type": "string", + "enum": [ + "registry", + "upload", + "bundled" + ] + } + } } }, "required": [ diff --git a/x-pack/plugins/fleet/common/openapi/bundled.yaml b/x-pack/plugins/fleet/common/openapi/bundled.yaml index ae8fdb3b87d4d..1ec0df0e51641 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.yaml +++ b/x-pack/plugins/fleet/common/openapi/bundled.yaml @@ -382,6 +382,15 @@ paths: required: - id - type + _meta: + type: object + properties: + install_source: + type: string + enum: + - registry + - upload + - bundled required: - items operationId: install-package diff --git a/x-pack/plugins/fleet/common/openapi/paths/epm@packages@{pkg_name}@{pkg_version}.yaml b/x-pack/plugins/fleet/common/openapi/paths/epm@packages@{pkg_name}@{pkg_version}.yaml index ef0964b66e045..6ef61788acd62 100644 --- a/x-pack/plugins/fleet/common/openapi/paths/epm@packages@{pkg_name}@{pkg_version}.yaml +++ b/x-pack/plugins/fleet/common/openapi/paths/epm@packages@{pkg_name}@{pkg_version}.yaml @@ -64,6 +64,15 @@ post: required: - id - type + _meta: + type: object + properties: + install_source: + type: string + enum: + - registry + - upload + - bundled required: - items operationId: install-package diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index 64ea5665241e1..1c7e09a51c5da 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -43,7 +43,7 @@ export interface DefaultPackagesInstallationError { } export type InstallType = 'reinstall' | 'reupdate' | 'rollback' | 'update' | 'install' | 'unknown'; -export type InstallSource = 'registry' | 'upload'; +export type InstallSource = 'registry' | 'upload' | 'bundled'; export type EpmPackageInstallStatus = | 'installed' diff --git a/x-pack/plugins/fleet/common/types/rest_spec/epm.ts b/x-pack/plugins/fleet/common/types/rest_spec/epm.ts index 6a72792e780ef..f1ccaae05487b 100644 --- a/x-pack/plugins/fleet/common/types/rest_spec/epm.ts +++ b/x-pack/plugins/fleet/common/types/rest_spec/epm.ts @@ -12,6 +12,7 @@ import type { PackageInfo, PackageUsageStats, InstallType, + InstallSource, } from '../models/epm'; export interface GetCategoriesRequest { @@ -108,6 +109,9 @@ export interface InstallPackageRequest { export interface InstallPackageResponse { items: AssetReference[]; + _meta: { + install_source: InstallSource; + }; // deprecated in 8.0 response?: AssetReference[]; } @@ -123,6 +127,7 @@ export interface InstallResult { status?: 'installed' | 'already_installed'; error?: Error; installType: InstallType; + installSource: InstallSource; } export interface BulkInstallPackageInfo { diff --git a/x-pack/plugins/fleet/server/routes/epm/handlers.ts b/x-pack/plugins/fleet/server/routes/epm/handlers.ts index 9bfcffa04bf35..7ba2d3f194eeb 100644 --- a/x-pack/plugins/fleet/server/routes/epm/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/epm/handlers.ts @@ -267,9 +267,13 @@ export const installPackageFromRegistryHandler: FleetRequestHandler< force: request.body?.force, ignoreConstraints: request.body?.ignore_constraints, }); + if (!res.error) { const body: InstallPackageResponse = { items: res.assets || [], + _meta: { + install_source: res.installSource, + }, }; return response.ok({ body }); } else { @@ -342,6 +346,9 @@ export const installPackageByUploadHandler: FleetRequestHandler< const body: InstallPackageResponse = { items: res.assets || [], response: res.assets || [], + _meta: { + install_source: res.installSource, + }, }; return response.ok({ body }); } else { diff --git a/x-pack/plugins/fleet/server/routes/epm/index.ts b/x-pack/plugins/fleet/server/routes/epm/index.ts index 95aadf1b8555a..544ab8b288cb4 100644 --- a/x-pack/plugins/fleet/server/routes/epm/index.ts +++ b/x-pack/plugins/fleet/server/routes/epm/index.ts @@ -242,7 +242,7 @@ export const registerRoutes = (router: FleetAuthzRouter) => { response ); if (resp.payload?.items) { - return response.ok({ body: { response: resp.payload.items } }); + return response.ok({ body: { ...resp.payload, response: resp.payload.items } }); } return resp; } diff --git a/x-pack/plugins/fleet/server/services/epm/packages/bundled_packages.ts b/x-pack/plugins/fleet/server/services/epm/packages/bundled_packages.ts index 8ccd2006ad846..77ece9e1d7787 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/bundled_packages.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/bundled_packages.ts @@ -12,7 +12,7 @@ import type { BundledPackage } from '../../../types'; import { appContextService } from '../../app_context'; import { splitPkgKey } from '../registry'; -const BUNDLED_PACKAGE_DIRECTORY = path.join(__dirname, '../../../bundled_packages'); +const BUNDLED_PACKAGE_DIRECTORY = path.join(__dirname, '../../../../target/bundled_packages'); export async function getBundledPackages(): Promise { try { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts index 1a1f1aa617f54..c803b0ff18a44 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts @@ -83,6 +83,7 @@ describe('install', () => { .mockImplementation(() => Promise.resolve({ packageInfo: { license: 'basic' } } as any)); mockGetBundledPackages.mockReset(); + (install._installPackage as jest.Mock).mockClear(); }); describe('registry', () => { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts index 107b906a969c8..23883f90d4248 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.ts @@ -276,6 +276,7 @@ async function installPackageFromRegistry({ ], status: 'already_installed', installType, + installSource: 'registry', }; } } @@ -307,7 +308,7 @@ async function installPackageFromRegistry({ ...telemetryEvent, errorMessage: err.message, }); - return { error: err, installType }; + return { error: err, installType, installSource: 'registry' }; } const savedObjectsImporter = appContextService @@ -338,7 +339,7 @@ async function installPackageFromRegistry({ ...telemetryEvent, status: 'success', }); - return { assets, status: 'installed', installType }; + return { assets, status: 'installed', installType, installSource: 'registry' }; }) .catch(async (err: Error) => { logger.warn(`Failure to install package [${pkgName}]: [${err.toString()}]`); @@ -355,7 +356,7 @@ async function installPackageFromRegistry({ ...telemetryEvent, errorMessage: err.message, }); - return { error: err, installType }; + return { error: err, installType, installSource: 'registry' }; }); } catch (e) { sendEvent({ @@ -365,6 +366,7 @@ async function installPackageFromRegistry({ return { error: e, installType, + installSource: 'registry', }; } } @@ -454,7 +456,7 @@ async function installPackageByUpload({ ...telemetryEvent, errorMessage: e.message, }); - return { error: e, installType }; + return { error: e, installType, installSource: 'upload' }; } } @@ -463,9 +465,10 @@ export type InstallPackageParams = { } & ( | ({ installSource: Extract } & InstallRegistryPackageParams) | ({ installSource: Extract } & InstallUploadedArchiveParams) + | ({ installSource: Extract } & InstallUploadedArchiveParams) ); -export async function installPackage(args: InstallPackageParams) { +export async function installPackage(args: InstallPackageParams): Promise { if (!('installSource' in args)) { throw new Error('installSource is required'); } @@ -487,7 +490,7 @@ export async function installPackage(args: InstallPackageParams) { `found bundled package for requested install of ${pkgkey} - installing from bundled package archive` ); - const response = installPackageByUpload({ + const response = await installPackageByUpload({ savedObjectsClient, esClient, archiveBuffer: matchingBundledPackage.buffer, @@ -495,11 +498,11 @@ export async function installPackage(args: InstallPackageParams) { spaceId, }); - return response; + return { ...response, installSource: 'bundled' }; } logger.debug(`kicking off install of ${pkgkey} from registry`); - const response = installPackageFromRegistry({ + const response = await installPackageFromRegistry({ savedObjectsClient, pkgkey, esClient, @@ -510,7 +513,7 @@ export async function installPackage(args: InstallPackageParams) { return response; } else if (args.installSource === 'upload') { const { archiveBuffer, contentType, spaceId } = args; - const response = installPackageByUpload({ + const response = await installPackageByUpload({ savedObjectsClient, esClient, archiveBuffer, @@ -519,7 +522,6 @@ export async function installPackage(args: InstallPackageParams) { }); return response; } - // @ts-expect-error s/b impossibe b/c `never` by this point, but just in case throw new Error(`Unknown installSource: ${args.installSource}`); } diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts index 518b79b9e8547..2a6e235580811 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts @@ -143,6 +143,7 @@ jest.mock('./epm/packages/install', () => ({ return { error: new Error(installError), installType: 'install', + installSource: 'registry', }; } @@ -157,6 +158,7 @@ jest.mock('./epm/packages/install', () => ({ return { status: 'installed', installType: 'install', + installSource: 'registry', }; } else if (args.installSource === 'upload') { const { archiveBuffer } = args; @@ -168,7 +170,7 @@ jest.mock('./epm/packages/install', () => ({ const packageInstallation = { name: pkgName, version: '1.0.0', title: pkgName }; mockInstalledPackages.set(pkgName, packageInstallation); - return { status: 'installed', installType: 'install' }; + return { status: 'installed', installType: 'install', installSource: 'upload' }; } }, ensurePackagesCompletedInstall() { diff --git a/x-pack/test/fleet_api_integration/apis/epm/index.js b/x-pack/test/fleet_api_integration/apis/epm/index.js index 70c8748932f6e..4247be10e0792 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/index.js +++ b/x-pack/test/fleet_api_integration/apis/epm/index.js @@ -14,6 +14,7 @@ export default function loadTests({ loadTestFile }) { loadTestFile(require.resolve('./file')); loadTestFile(require.resolve('./template')); loadTestFile(require.resolve('./ilm')); + // loadTestFile(require.resolve('./install_bundled')); loadTestFile(require.resolve('./install_by_upload')); loadTestFile(require.resolve('./install_endpoint')); loadTestFile(require.resolve('./install_overrides')); diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_bundled.ts b/x-pack/test/fleet_api_integration/apis/epm/install_bundled.ts new file mode 100644 index 0000000000000..c70495ea7809d --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/epm/install_bundled.ts @@ -0,0 +1,126 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import fs from 'fs/promises'; +import path from 'path'; + +import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; +import { skipIfNoDockerRegistry } from '../../helpers'; +import { setupFleetAndAgents } from '../agents/services'; + +export default function (providerContext: FtrProviderContext) { + const { getService } = providerContext; + const supertest = getService('supertest'); + const log = getService('log'); + + const BUNDLED_PACKAGE_FIXTURES_DIR = path.join( + path.dirname(__filename), + '../fixtures/bundled_packages' + ); + const BUNDLED_PACKAGES_DIR = path.join( + path.dirname(__filename), + '../../../../plugins/fleet/target/bundled_packages' + ); + + const bundlePackage = async (name: string) => { + try { + await fs.access(BUNDLED_PACKAGES_DIR); + } catch (error) { + await fs.mkdir(BUNDLED_PACKAGES_DIR); + } + + await fs.copyFile( + path.join(BUNDLED_PACKAGE_FIXTURES_DIR, `${name}.zip`), + path.join(BUNDLED_PACKAGES_DIR, `${name}.zip`) + ); + }; + + const removeBundledPackages = async () => { + try { + const files = await fs.readdir(BUNDLED_PACKAGES_DIR); + + for (const file of files) { + await fs.unlink(path.join(BUNDLED_PACKAGES_DIR, file)); + } + } catch (error) { + log.error('Error removing bundled packages'); + log.error(error); + } + }; + + describe('installing bundled packages', async () => { + skipIfNoDockerRegistry(providerContext); + setupFleetAndAgents(providerContext); + + afterEach(async () => { + await removeBundledPackages(); + }); + + describe('without registry', () => { + it('installs from bundled source via api', async () => { + await bundlePackage('elastic_agent-1.2.0'); + + const response = await supertest + .post(`/api/fleet/epm/packages/elastic_agent/1.2.0`) + .set('kbn-xsrf', 'xxxx') + .type('application/json') + .send({ force: true }) + .expect(200); + + expect(response.body._meta.install_source).to.be('bundled'); + }); + + it('allows for upgrading from newer bundled source when outdated package was installed from bundled source', async () => { + await bundlePackage('elastic_agent-1.0.0'); + await bundlePackage('elastic_agent-1.2.0'); + + const installResponse = await supertest + .post(`/api/fleet/epm/packages/elastic_agent/1.0.0`) + .set('kbn-xsrf', 'xxxx') + .type('application/json') + .send({ force: true }) + .expect(200); + + expect(installResponse.body._meta.install_source).to.be('bundled'); + + const updateResponse = await supertest + .post(`/api/fleet/epm/packages/elastic_agent/1.2.0`) + .set('kbn-xsrf', 'xxxx') + .type('application/json') + .send({ force: true }) + .expect(200); + + expect(updateResponse.body._meta.install_source).to.be('bundled'); + }); + }); + + describe('with registry', () => { + it('allows for updating from registry when outdated package is installed from bundled source', async () => { + await bundlePackage('elastic_agent-1.2.0'); + + const bundledInstallResponse = await supertest + .post(`/api/fleet/epm/packages/elastic_agent/1.2.0`) + .set('kbn-xsrf', 'xxxx') + .type('application/json') + .send({ force: true }) + .expect(200); + + expect(bundledInstallResponse.body._meta.install_source).to.be('bundled'); + + const registryUpdateResponse = await supertest + .post(`/api/fleet/epm/packages/elastic_agent/1.3.0`) + .set('kbn-xsrf', 'xxxx') + .type('application/json') + .send({ force: true }) + .expect(200); + + expect(registryUpdateResponse.body._meta.install_source).to.be('registry'); + }); + }); + }); +} diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/bundled_packages/elastic_agent-1.0.0.zip b/x-pack/test/fleet_api_integration/apis/fixtures/bundled_packages/elastic_agent-1.0.0.zip new file mode 100644 index 0000000000000..77961d6b14c53 Binary files /dev/null and b/x-pack/test/fleet_api_integration/apis/fixtures/bundled_packages/elastic_agent-1.0.0.zip differ diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/bundled_packages/elastic_agent-1.2.0.zip b/x-pack/test/fleet_api_integration/apis/fixtures/bundled_packages/elastic_agent-1.2.0.zip new file mode 100644 index 0000000000000..a41061b7212ae Binary files /dev/null and b/x-pack/test/fleet_api_integration/apis/fixtures/bundled_packages/elastic_agent-1.2.0.zip differ