[7.x] [esArchiver] drop support for --dir, use repo-relative paths in…

…stead (#101345) (#101676)

* [esArchiver] drop support for --dir, use repo-relative paths instead (#101345)

Co-authored-by: spalger <spalger@users.noreply.github.com>
# Conflicts:
#	test/api_integration/apis/suggestions/suggestions.js
#	test/functional/apps/discover/_large_string.ts
#	test/functional/apps/visualize/index.ts
#	x-pack/test/functional/apps/infra/feature_controls/logs_security.ts
#	x-pack/test/functional/apps/saved_objects_management/import_saved_objects_between_versions_6.x_7.x.ts
#	x-pack/test/functional/apps/upgrade_assistant/upgrade_assistant.ts

* convert references to `saved_objects/basic` archive

* adapt other `saved_objects/*` archives

* update management/saved_obejcts/relationships archives

* replace old monitoring setup() usage

* remove reference to `empty_kibana` archive

Co-authored-by: spalger <spalger@users.noreply.github.com>

docs/developer/contributing/development-functional-tests.asciidoc

@@ -229,9 +229,9 @@ export default function ({ getService, getPageObject }) {
     before(async () => {
       await Promise.all([
         // start with an empty .kibana index
-        esArchiver.load('empty_kibana'),
+        esArchiver.load('test/functional/fixtures/es_archiver/empty_kibana'),
         // load some basic log data only if the index doesn't exist
-        esArchiver.loadIfNeeded('makelogs')
+        esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/makelogs')
       ]);
       // go to the page described by `apps.visualize` in the config
       await PageObjects.common.navigateTo('visualize');
@@ -243,7 +243,7 @@ export default function ({ getService, getPageObject }) {
       // we unload the empty_kibana archive but not the makelogs
       // archive because we don't make any changes to it, and subsequent
       // suites could use it if they call `.loadIfNeeded()`.
-      await esArchiver.unload('empty_kibana');
+      await esArchiver.unload('test/functional/fixtures/es_archiver/empty_kibana');
     });
 
     // This series of tests illustrate how tests generally verify
@@ -370,9 +370,9 @@ await testSubjects.click(‘containerButton’);
 * Source: link:{kib-repo}tree/{branch}/test/common/services/es_archiver.ts[test/common/services/es_archiver.ts]
 * Load/unload archives created with the `esArchiver`
 * Popular methods:
-** `esArchiver.load(name)`
-** `esArchiver.loadIfNeeded(name)`
-** `esArchiver.unload(name)`
+** `esArchiver.load(path)`
+** `esArchiver.loadIfNeeded(path)`
+** `esArchiver.unload(path)`
 
 Full list of services that are used in functional tests can be found here: link:{kib-repo}tree/{branch}/test/functional/services[test/functional/services]
 

docs/developer/plugin/external-plugin-functional-tests.asciidoc

@@ -58,11 +58,6 @@ export default async function ({ readConfigFile }) {
       }
     },
 
-    // choose where esArchiver should load archives from
-    esArchiver: {
-      directory: resolve(__dirname, './es_archives'),
-    },
-
     // choose where screenshots should be saved
     screenshots: {
       directory: resolve(__dirname, './tmp/screenshots'),

packages/kbn-es-archiver/src/actions/edit.ts

@@ -6,9 +6,9 @@
  * Side Public License, v 1.
  */
 
-import { resolve, relative } from 'path';
+import { relative } from 'path';
 import Fs from 'fs';
-import { createGunzip, createGzip, Z_BEST_COMPRESSION } from 'zlib';
+import { createGunzip, createGzip, constants } from 'zlib';
 import { promisify } from 'util';
 import globby from 'globby';
 import { ToolingLog } from '@kbn/dev-utils';
@@ -17,24 +17,22 @@ import { createPromiseFromStreams } from '@kbn/utils';
 const unlinkAsync = promisify(Fs.unlink);
 
 export async function editAction({
-  prefix,
-  dataDir,
+  path,
   log,
   handler,
 }: {
-  prefix: string;
-  dataDir: string;
+  path: string;
   log: ToolingLog;
   handler: () => Promise<any>;
 }) {
   const archives = (
     await globby('**/*.gz', {
-      cwd: prefix ? resolve(dataDir, prefix) : dataDir,
+      cwd: path,
       absolute: true,
     })
-  ).map((path) => ({
-    path,
-    rawPath: path.slice(0, -3),
+  ).map((found) => ({
+    path: found,
+    rawPath: found.slice(0, -3),
   }));
 
   await Promise.all(
@@ -61,7 +59,7 @@ export async function editAction({
     archives.map(async (archive) => {
       await createPromiseFromStreams([
         Fs.createReadStream(archive.rawPath),
-        createGzip({ level: Z_BEST_COMPRESSION }),
+        createGzip({ level: constants.Z_BEST_COMPRESSION }),
         Fs.createWriteStream(archive.path),
       ]);
 

packages/kbn-es-archiver/src/actions/load.ts

@@ -6,10 +6,10 @@
  * Side Public License, v 1.
  */
 
-import { resolve } from 'path';
+import { resolve, relative } from 'path';
 import { createReadStream } from 'fs';
 import { Readable } from 'stream';
-import { ToolingLog } from '@kbn/dev-utils';
+import { ToolingLog, REPO_ROOT } from '@kbn/dev-utils';
 import { KbnClient } from '@kbn/test';
 import type { KibanaClient } from '@elastic/elasticsearch/api/kibana';
 import { createPromiseFromStreams, concatStreamProviders } from '@kbn/utils';
@@ -37,23 +37,21 @@ const pipeline = (...streams: Readable[]) =>
   );
 
 export async function loadAction({
-  name,
+  inputDir,
   skipExisting,
   useCreate,
   client,
-  dataDir,
   log,
   kbnClient,
 }: {
-  name: string;
+  inputDir: string;
   skipExisting: boolean;
   useCreate: boolean;
   client: KibanaClient;
-  dataDir: string;
   log: ToolingLog;
   kbnClient: KbnClient;
 }) {
-  const inputDir = resolve(dataDir, name);
+  const name = relative(REPO_ROOT, inputDir);
   const stats = createStats(name, log);
   const files = prioritizeMappings(await readDirectory(inputDir));
   const kibanaPluginIds = await kbnClient.plugins.getEnabledIds();

packages/kbn-es-archiver/src/actions/rebuild_all.ts

@@ -6,11 +6,11 @@
  * Side Public License, v 1.
  */
 
-import { resolve, dirname, relative } from 'path';
+import { resolve, relative } from 'path';
 import { stat, Stats, rename, createReadStream, createWriteStream } from 'fs';
 import { Readable, Writable } from 'stream';
 import { fromNode } from 'bluebird';
-import { ToolingLog } from '@kbn/dev-utils';
+import { ToolingLog, REPO_ROOT } from '@kbn/dev-utils';
 import { createPromiseFromStreams } from '@kbn/utils';
 import {
   prioritizeMappings,
@@ -25,15 +25,7 @@ async function isDirectory(path: string): Promise<boolean> {
   return stats.isDirectory();
 }
 
-export async function rebuildAllAction({
-  dataDir,
-  log,
-  rootDir = dataDir,
-}: {
-  dataDir: string;
-  log: ToolingLog;
-  rootDir?: string;
-}) {
+export async function rebuildAllAction({ dataDir, log }: { dataDir: string; log: ToolingLog }) {
   const childNames = prioritizeMappings(await readDirectory(dataDir));
   for (const childName of childNames) {
     const childPath = resolve(dataDir, childName);
@@ -42,13 +34,12 @@ export async function rebuildAllAction({
       await rebuildAllAction({
         dataDir: childPath,
         log,
-        rootDir,
       });
       continue;
     }
 
-    const archiveName = dirname(relative(rootDir, childPath));
-    log.info(`${archiveName} Rebuilding ${childName}`);
+    const archiveName = relative(REPO_ROOT, childPath);
+    log.info('[%s] Rebuilding %j', archiveName, childName);
     const gzip = isGzip(childPath);
     const tempFile = childPath + (gzip ? '.rebuilding.gz' : '.rebuilding');
 
@@ -60,6 +51,6 @@ export async function rebuildAllAction({
     ] as [Readable, ...Writable[]]);
 
     await fromNode((cb) => rename(tempFile, childPath, cb));
-    log.info(`${archiveName} Rebuilt ${childName}`);
+    log.info('[%s] Rebuilt %j', archiveName, childName);
   }
 }

packages/kbn-es-archiver/src/actions/save.ts

@@ -6,11 +6,11 @@
  * Side Public License, v 1.
  */
 
-import { resolve } from 'path';
+import { resolve, relative } from 'path';
 import { createWriteStream, mkdirSync } from 'fs';
 import { Readable, Writable } from 'stream';
 import type { KibanaClient } from '@elastic/elasticsearch/api/kibana';
-import { ToolingLog } from '@kbn/dev-utils';
+import { ToolingLog, REPO_ROOT } from '@kbn/dev-utils';
 import { createListStream, createPromiseFromStreams } from '@kbn/utils';
 
 import {
@@ -22,23 +22,21 @@ import {
 } from '../lib';
 
 export async function saveAction({
-  name,
+  outputDir,
   indices,
   client,
-  dataDir,
   log,
   raw,
   query,
 }: {
-  name: string;
+  outputDir: string;
   indices: string | string[];
   client: KibanaClient;
-  dataDir: string;
   log: ToolingLog;
   raw: boolean;
   query?: Record<string, any>;
 }) {
-  const outputDir = resolve(dataDir, name);
+  const name = relative(REPO_ROOT, outputDir);
   const stats = createStats(name, log);
 
   log.info('[%s] Creating archive of %j', name, indices);

packages/kbn-es-archiver/src/actions/unload.ts

@@ -6,11 +6,11 @@
  * Side Public License, v 1.
  */
 
-import { resolve } from 'path';
+import { resolve, relative } from 'path';
 import { createReadStream } from 'fs';
 import { Readable, Writable } from 'stream';
 import type { KibanaClient } from '@elastic/elasticsearch/api/kibana';
-import { ToolingLog } from '@kbn/dev-utils';
+import { ToolingLog, REPO_ROOT } from '@kbn/dev-utils';
 import { KbnClient } from '@kbn/test';
 import { createPromiseFromStreams } from '@kbn/utils';
 
@@ -25,19 +25,17 @@ import {
 } from '../lib';
 
 export async function unloadAction({
-  name,
+  inputDir,
   client,
-  dataDir,
   log,
   kbnClient,
 }: {
-  name: string;
+  inputDir: string;
   client: KibanaClient;
-  dataDir: string;
   log: ToolingLog;
   kbnClient: KbnClient;
 }) {
-  const inputDir = resolve(dataDir, name);
+  const name = relative(REPO_ROOT, inputDir);
   const stats = createStats(name, log);
   const kibanaPluginIds = await kbnClient.plugins.getEnabledIds();