[Fleet][EPM] Unified install and archive (#83384) (#83944)

## Summary

 * Further reduce differences between installing uploaded vs registry package
 * Improve cache/store names, TS types, etc. Including key by name + version + source
 * Add a cache/store for PackageInfo (e.g. results metadata from registry's /package/version/ response)
 * Remove ensureCachedArchiveInfo

x-pack/plugins/fleet/server/services/epm/archive/cache.ts

@@ -3,21 +3,57 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-import { pkgToPkgKey } from '../registry/index';
+import { ArchiveEntry } from './index';
+import { InstallSource, ArchivePackage, RegistryPackage } from '../../../../common';
 
-const cache: Map<string, Buffer> = new Map();
-export const cacheGet = (key: string) => cache.get(key);
-export const cacheSet = (key: string, value: Buffer) => cache.set(key, value);
-export const cacheHas = (key: string) => cache.has(key);
-export const cacheClear = () => cache.clear();
-export const cacheDelete = (key: string) => cache.delete(key);
+const archiveEntryCache: Map<ArchiveEntry['path'], ArchiveEntry['buffer']> = new Map();
+export const getArchiveEntry = (key: string) => archiveEntryCache.get(key);
+export const setArchiveEntry = (key: string, value: Buffer) => archiveEntryCache.set(key, value);
+export const hasArchiveEntry = (key: string) => archiveEntryCache.has(key);
+export const clearArchiveEntries = () => archiveEntryCache.clear();
+export const deleteArchiveEntry = (key: string) => archiveEntryCache.delete(key);
 
-const archiveFilelistCache: Map<string, string[]> = new Map();
-export const getArchiveFilelist = (name: string, version: string) =>
-  archiveFilelistCache.get(pkgToPkgKey({ name, version }));
+export interface SharedKey {
+  name: string;
+  version: string;
+  installSource: InstallSource;
+}
+type SharedKeyString = string;
 
-export const setArchiveFilelist = (name: string, version: string, paths: string[]) =>
-  archiveFilelistCache.set(pkgToPkgKey({ name, version }), paths);
+type ArchiveFilelist = string[];
+const archiveFilelistCache: Map<SharedKeyString, ArchiveFilelist> = new Map();
+export const getArchiveFilelist = (keyArgs: SharedKey) =>
+  archiveFilelistCache.get(sharedKey(keyArgs));
 
-export const deleteArchiveFilelist = (name: string, version: string) =>
-  archiveFilelistCache.delete(pkgToPkgKey({ name, version }));
+export const setArchiveFilelist = (keyArgs: SharedKey, paths: string[]) =>
+  archiveFilelistCache.set(sharedKey(keyArgs), paths);
+
+export const deleteArchiveFilelist = (keyArgs: SharedKey) =>
+  archiveFilelistCache.delete(sharedKey(keyArgs));
+
+const packageInfoCache: Map<SharedKeyString, ArchivePackage | RegistryPackage> = new Map();
+const sharedKey = ({ name, version, installSource }: SharedKey) =>
+  `${name}-${version}-${installSource}`;
+
+export const getPackageInfo = (args: SharedKey) => {
+  const packageInfo = packageInfoCache.get(sharedKey(args));
+  if (args.installSource === 'registry') {
+    return packageInfo as RegistryPackage;
+  } else if (args.installSource === 'upload') {
+    return packageInfo as ArchivePackage;
+  } else {
+    throw new Error(`Unknown installSource: ${args.installSource}`);
+  }
+};
+
+export const setPackageInfo = ({
+  name,
+  version,
+  installSource,
+  packageInfo,
+}: SharedKey & { packageInfo: ArchivePackage | RegistryPackage }) => {
+  const key = sharedKey({ name, version, installSource });
+  return packageInfoCache.set(key, packageInfo);
+};
+
+export const deletePackageInfo = (args: SharedKey) => packageInfoCache.delete(sharedKey(args));

x-pack/plugins/fleet/server/services/epm/archive/index.ts

@@ -4,68 +4,57 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { ArchivePackage, AssetParts } from '../../../../common/types';
+import { AssetParts, InstallSource } from '../../../../common/types';
 import { PackageInvalidArchiveError, PackageUnsupportedMediaTypeError } from '../../../errors';
 import {
-  cacheGet,
-  cacheSet,
-  cacheDelete,
+  SharedKey,
+  getArchiveEntry,
+  setArchiveEntry,
+  deleteArchiveEntry,
   getArchiveFilelist,
   setArchiveFilelist,
   deleteArchiveFilelist,
+  deletePackageInfo,
 } from './cache';
 import { getBufferExtractor } from './extract';
-import { parseAndVerifyArchiveEntries } from './validation';
 
 export * from './cache';
-export { untarBuffer, unzipBuffer, getBufferExtractor } from './extract';
+export { getBufferExtractor, untarBuffer, unzipBuffer } from './extract';
+export { parseAndVerifyArchiveBuffer as parseAndVerifyArchiveEntries } from './validation';
 
 export interface ArchiveEntry {
   path: string;
   buffer?: Buffer;
 }
 
-export async function getArchivePackage({
-  archiveBuffer,
+export async function unpackBufferToCache({
+  name,
+  version,
   contentType,
+  archiveBuffer,
+  installSource,
 }: {
-  archiveBuffer: Buffer;
+  name: string;
+  version: string;
   contentType: string;
-}): Promise<{ paths: string[]; archivePackageInfo: ArchivePackage }> {
-  const entries = await unpackArchiveEntries(archiveBuffer, contentType);
-  const { archivePackageInfo } = await parseAndVerifyArchiveEntries(entries);
-  const paths = addEntriesToMemoryStore(entries);
-
-  setArchiveFilelist(archivePackageInfo.name, archivePackageInfo.version, paths);
-
-  return {
-    paths,
-    archivePackageInfo,
-  };
-}
-
-export async function unpackArchiveToCache(
-  archiveBuffer: Buffer,
-  contentType: string
-): Promise<string[]> {
-  const entries = await unpackArchiveEntries(archiveBuffer, contentType);
-  return addEntriesToMemoryStore(entries);
-}
-
-function addEntriesToMemoryStore(entries: ArchiveEntry[]) {
+  archiveBuffer: Buffer;
+  installSource: InstallSource;
+}): Promise<string[]> {
+  const entries = await unpackBufferEntries(archiveBuffer, contentType);
   const paths: string[] = [];
   entries.forEach((entry) => {
     const { path, buffer } = entry;
     if (buffer) {
-      cacheSet(path, buffer);
+      setArchiveEntry(path, buffer);
       paths.push(path);
     }
   });
+  setArchiveFilelist({ name, version, installSource }, paths);
 
   return paths;
 }
 
-export async function unpackArchiveEntries(
+export async function unpackBufferEntries(
   archiveBuffer: Buffer,
   contentType: string
 ): Promise<ArchiveEntry[]> {
@@ -96,16 +85,18 @@ export async function unpackArchiveEntries(
   return entries;
 }
 
-export const deletePackageCache = (name: string, version: string) => {
+export const deletePackageCache = ({ name, version, installSource }: SharedKey) => {
   // get cached archive filelist
-  const paths = getArchiveFilelist(name, version);
+  const paths = getArchiveFilelist({ name, version, installSource });
 
   // delete cached archive filelist
-  deleteArchiveFilelist(name, version);
+  deleteArchiveFilelist({ name, version, installSource });
 
   // delete cached archive files
-  // this has been populated in unpackArchiveToCache()
-  paths?.forEach((path) => cacheDelete(path));
+  // this has been populated in unpackBufferToCache()
+  paths?.forEach(deleteArchiveEntry);
+
+  deletePackageInfo({ name, version, installSource });
 };
 
 export function getPathParts(path: string): AssetParts {
@@ -139,7 +130,7 @@ export function getPathParts(path: string): AssetParts {
 }
 
 export function getAsset(key: string) {
-  const buffer = cacheGet(key);
+  const buffer = getArchiveEntry(key);
   if (buffer === undefined) throw new Error(`Cannot find asset ${key}`);
 
   return buffer;

x-pack/plugins/fleet/server/services/epm/archive/validation.ts

@@ -15,7 +15,7 @@ import {
   RegistryVarsEntry,
 } from '../../../../common/types';
 import { PackageInvalidArchiveError } from '../../../errors';
-import { ArchiveEntry } from './index';
+import { unpackBufferEntries } from './index';
 import { pkgToPkgKey } from '../registry';
 
 const MANIFESTS: Record<string, Buffer> = {};
@@ -24,22 +24,24 @@ const MANIFEST_NAME = 'manifest.yml';
 // TODO: everything below performs verification of manifest.yml files, and hence duplicates functionality already implemented in the
 // package registry. At some point this should probably be replaced (or enhanced) with verification based on
 // https://github.com/elastic/package-spec/
-export async function parseAndVerifyArchiveEntries(
-  entries: ArchiveEntry[]
-): Promise<{ paths: string[]; archivePackageInfo: ArchivePackage }> {
+export async function parseAndVerifyArchiveBuffer(
+  archiveBuffer: Buffer,
+  contentType: string
+): Promise<{ paths: string[]; packageInfo: ArchivePackage }> {
+  const entries = await unpackBufferEntries(archiveBuffer, contentType);
   const paths: string[] = [];
   entries.forEach(({ path, buffer }) => {
     paths.push(path);
     if (path.endsWith(MANIFEST_NAME) && buffer) MANIFESTS[path] = buffer;
   });
 
   return {
-    archivePackageInfo: parseAndVerifyArchive(paths),
+    packageInfo: parseAndVerifyArchive(paths),
     paths,
   };
 }
 
-export function parseAndVerifyArchive(paths: string[]): ArchivePackage {
+function parseAndVerifyArchive(paths: string[]): ArchivePackage {
   // The top-level directory must match pkgName-pkgVersion, and no other top-level files or directories may be present
   const toplevelDir = paths[0].split('/')[0];
   paths.forEach((path) => {

x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/install.ts

@@ -85,14 +85,6 @@ export async function installIndexPatterns(
     savedObjectsClient,
     installationStatuses.Installed
   );
-  // TODO: move to install package
-  // cache all installed packages if they don't exist
-  const packagePromises = installedPackages.map((pkg) =>
-    // TODO: this hard-codes 'registry' as installSource, so uploaded packages are ignored
-    // and their fields will be removed from the generated index patterns after this runs.
-    Registry.ensureCachedArchiveInfo(pkg.pkgName, pkg.pkgVersion, 'registry')
-  );
-  await Promise.all(packagePromises);
 
   const packageVersionsToFetch = [...installedPackages];
   if (pkgName && pkgVersion) {

x-pack/plugins/fleet/server/services/epm/packages/assets.ts

@@ -5,7 +5,6 @@
  */
 
 import { InstallablePackage } from '../../../types';
-import * as Registry from '../registry';
 import { ArchiveEntry, getArchiveFilelist, getAsset } from '../archive';
 
 // paths from RegistryPackage are routes to the assets on EPR
@@ -21,7 +20,8 @@ export function getAssets(
   datasetName?: string
 ): string[] {
   const assets: string[] = [];
-  const paths = getArchiveFilelist(packageInfo.name, packageInfo.version);
+  const { name, version } = packageInfo;
+  const paths = getArchiveFilelist({ name, version, installSource: 'registry' });
   // TODO: might be better to throw a PackageCacheError here
   if (!paths || paths.length === 0) return assets;
 
@@ -47,15 +47,13 @@ export function getAssets(
   return assets;
 }
 
+// ASK: Does getAssetsData need an installSource now?
+// if so, should it be an Installation vs InstallablePackage or add another argument?
 export async function getAssetsData(
   packageInfo: InstallablePackage,
   filter = (path: string): boolean => true,
   datasetName?: string
 ): Promise<ArchiveEntry[]> {
-  // TODO: Needs to be called to fill the cache but should not be required
-
-  await Registry.ensureCachedArchiveInfo(packageInfo.name, packageInfo.version, 'registry');
-
   // Gather all asset data
   const assets = getAssets(packageInfo, filter, datasetName);
   const entries: ArchiveEntry[] = assets.map((path) => {

x-pack/plugins/fleet/server/services/epm/packages/get.ts

@@ -109,11 +109,7 @@ export async function getPackageInfo(options: {
   pkgVersion: string;
 }): Promise<PackageInfo> {
   const { savedObjectsClient, pkgName, pkgVersion } = options;
-  const [
-    savedObject,
-    latestPackage,
-    { paths: assets, registryPackageInfo: item },
-  ] = await Promise.all([
+  const [savedObject, latestPackage, { paths: assets, packageInfo: item }] = await Promise.all([
     getInstallationObject({ savedObjectsClient, pkgName }),
     Registry.fetchFindLatestPackage(pkgName),
     Registry.getRegistryPackage(pkgName, pkgVersion),

x-pack/plugins/fleet/server/services/epm/packages/install.ts

@@ -28,6 +28,7 @@ import {
   KibanaAssetType,
 } from '../../../types';
 import * as Registry from '../registry';
+import { setPackageInfo, parseAndVerifyArchiveEntries, unpackBufferToCache } from '../archive';
 import {
   getInstallation,
   getInstallationObject,
@@ -43,7 +44,6 @@ import {
 } from '../../../errors';
 import { getPackageSavedObjects } from './get';
 import { appContextService } from '../../app_context';
-import { getArchivePackage } from '../archive';
 import { _installPackage } from './_install_package';
 
 export async function installLatestPackage(options: {
@@ -245,29 +245,26 @@ async function installPackageFromRegistry({
 }: InstallRegistryPackageParams): Promise<AssetReference[]> {
   // TODO: change epm API to /packageName/version so we don't need to do this
   const { pkgName, pkgVersion } = Registry.splitPkgKey(pkgkey);
-  // TODO: calls to getInstallationObject, Registry.fetchInfo, and Registry.fetchFindLatestPackge
-  // and be replaced by getPackageInfo after adjusting for it to not group/use archive assets
-  const latestPackage = await Registry.fetchFindLatestPackage(pkgName);
   // get the currently installed package
   const installedPkg = await getInstallationObject({ savedObjectsClient, pkgName });
-
   const installType = getInstallType({ pkgVersion, installedPkg });
-
   // let the user install if using the force flag or needing to reinstall or install a previous version due to failed update
   const installOutOfDateVersionOk =
     installType === 'reinstall' || installType === 'reupdate' || installType === 'rollback';
+
+  const latestPackage = await Registry.fetchFindLatestPackage(pkgName);
   if (semverLt(pkgVersion, latestPackage.version) && !force && !installOutOfDateVersionOk) {
     throw new PackageOutdatedError(`${pkgkey} is out-of-date and cannot be installed or updated`);
   }
 
-  const { paths, registryPackageInfo } = await Registry.getRegistryPackage(pkgName, pkgVersion);
+  const { paths, packageInfo } = await Registry.getRegistryPackage(pkgName, pkgVersion);
 
   return _installPackage({
     savedObjectsClient,
     callCluster,
     installedPkg,
     paths,
-    packageInfo: registryPackageInfo,
+    packageInfo,
     installType,
     installSource: 'registry',
   });
@@ -290,27 +287,44 @@ async function installPackageByUpload({
   archiveBuffer,
   contentType,
 }: InstallUploadedArchiveParams): Promise<AssetReference[]> {
-  const { paths, archivePackageInfo } = await getArchivePackage({ archiveBuffer, contentType });
+  const { packageInfo } = await parseAndVerifyArchiveEntries(archiveBuffer, contentType);
 
   const installedPkg = await getInstallationObject({
     savedObjectsClient,
-    pkgName: archivePackageInfo.name,
+    pkgName: packageInfo.name,
   });
-  const installType = getInstallType({ pkgVersion: archivePackageInfo.version, installedPkg });
+
+  const installType = getInstallType({ pkgVersion: packageInfo.version, installedPkg });
   if (installType !== 'install') {
     throw new PackageOperationNotSupportedError(
-      `Package upload only supports fresh installations. Package ${archivePackageInfo.name} is already installed, please uninstall first.`
+      `Package upload only supports fresh installations. Package ${packageInfo.name} is already installed, please uninstall first.`
     );
   }
 
+  const installSource = 'upload';
+  const paths = await unpackBufferToCache({
+    name: packageInfo.name,
+    version: packageInfo.version,
+    installSource,
+    archiveBuffer,
+    contentType,
+  });
+
+  setPackageInfo({
+    name: packageInfo.name,
+    version: packageInfo.version,
+    installSource,
+    packageInfo,
+  });
+
   return _installPackage({
     savedObjectsClient,
     callCluster,
     installedPkg,
     paths,
-    packageInfo: archivePackageInfo,
+    packageInfo,
     installType,
-    installSource: 'upload',
+    installSource,
   });
 }