[7.x] [esArchiver] drop support for --dir, use repo-relative paths in…

…stead (#101345) (#101676) * [esArchiver] drop support for --dir, use repo-relative paths instead (#101345) Co-authored-by: spalger <spalger@users.noreply.github.com> # Conflicts: # test/api_integration/apis/suggestions/suggestions.js # test/functional/apps/discover/_large_string.ts # test/functional/apps/visualize/index.ts # x-pack/test/functional/apps/infra/feature_controls/logs_security.ts # x-pack/test/functional/apps/saved_objects_management/import_saved_objects_between_versions_6.x_7.x.ts # x-pack/test/functional/apps/upgrade_assistant/upgrade_assistant.ts * convert references to `saved_objects/basic` archive * adapt other `saved_objects/*` archives * update management/saved_obejcts/relationships archives * replace old monitoring setup() usage * remove reference to `empty_kibana` archive Co-authored-by: spalger <spalger@users.noreply.github.com>
elastic · Jun 9, 2021 · bcdeccb · bcdeccb
1 parent 81d67b3
commit bcdeccb
Show file tree

Hide file tree

Showing 729 changed files with 2,608 additions and 1,888 deletions.
diff --git a/docs/developer/contributing/development-functional-tests.asciidoc b/docs/developer/contributing/development-functional-tests.asciidoc
@@ -229,9 +229,9 @@ export default function ({ getService, getPageObject }) {
  before(async () => {
  await Promise.all([
  // start with an empty .kibana index
- esArchiver.load('empty_kibana'),
+ esArchiver.load('test/functional/fixtures/es_archiver/empty_kibana'),
  // load some basic log data only if the index doesn't exist
- esArchiver.loadIfNeeded('makelogs')
+ esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/makelogs')
  ]);
  // go to the page described by `apps.visualize` in the config
  await PageObjects.common.navigateTo('visualize');
@@ -243,7 +243,7 @@ export default function ({ getService, getPageObject }) {
  // we unload the empty_kibana archive but not the makelogs
  // archive because we don't make any changes to it, and subsequent
  // suites could use it if they call `.loadIfNeeded()`.
- await esArchiver.unload('empty_kibana');
+ await esArchiver.unload('test/functional/fixtures/es_archiver/empty_kibana');
  });
 
  // This series of tests illustrate how tests generally verify
@@ -370,9 +370,9 @@ await testSubjects.click(‘containerButton’);
 * Source: link:{kib-repo}tree/{branch}/test/common/services/es_archiver.ts[test/common/services/es_archiver.ts]
 * Load/unload archives created with the `esArchiver`
 * Popular methods:
-** `esArchiver.load(name)`
-** `esArchiver.loadIfNeeded(name)`
-** `esArchiver.unload(name)`
+** `esArchiver.load(path)`
+** `esArchiver.loadIfNeeded(path)`
+** `esArchiver.unload(path)`
 
 Full list of services that are used in functional tests can be found here: link:{kib-repo}tree/{branch}/test/functional/services[test/functional/services]
 

diff --git a/docs/developer/plugin/external-plugin-functional-tests.asciidoc b/docs/developer/plugin/external-plugin-functional-tests.asciidoc
@@ -58,11 +58,6 @@ export default async function ({ readConfigFile }) {
  }
  },
 
- // choose where esArchiver should load archives from
- esArchiver: {
- directory: resolve(__dirname, './es_archives'),
- },
-
  // choose where screenshots should be saved
  screenshots: {
  directory: resolve(__dirname, './tmp/screenshots'),

diff --git a/packages/kbn-es-archiver/src/actions/edit.ts b/packages/kbn-es-archiver/src/actions/edit.ts
@@ -6,9 +6,9 @@
  * Side Public License, v 1.
  */
 
-import { resolve, relative } from 'path';
+import { relative } from 'path';
 import Fs from 'fs';
-import { createGunzip, createGzip, Z_BEST_COMPRESSION } from 'zlib';
+import { createGunzip, createGzip, constants } from 'zlib';
 import { promisify } from 'util';
 import globby from 'globby';
 import { ToolingLog } from '@kbn/dev-utils';
@@ -17,24 +17,22 @@ import { createPromiseFromStreams } from '@kbn/utils';
 const unlinkAsync = promisify(Fs.unlink);
 
 export async function editAction({
- prefix,
- dataDir,
+ path,
  log,
  handler,
 }: {
- prefix: string;
- dataDir: string;
+ path: string;
  log: ToolingLog;
  handler: () => Promise<any>;
 }) {
  const archives = (
  await globby('**/*.gz', {
- cwd: prefix ? resolve(dataDir, prefix) : dataDir,
+ cwd: path,
  absolute: true,
  })
- ).map((path) => ({
- path,
- rawPath: path.slice(0, -3),
+ ).map((found) => ({
+ path: found,
+ rawPath: found.slice(0, -3),
  }));
 
  await Promise.all(
@@ -61,7 +59,7 @@ export async function editAction({
  archives.map(async (archive) => {
  await createPromiseFromStreams([
  Fs.createReadStream(archive.rawPath),
- createGzip({ level: Z_BEST_COMPRESSION }),
+ createGzip({ level: constants.Z_BEST_COMPRESSION }),
  Fs.createWriteStream(archive.path),
  ]);
 

diff --git a/packages/kbn-es-archiver/src/actions/load.ts b/packages/kbn-es-archiver/src/actions/load.ts
@@ -6,10 +6,10 @@
  * Side Public License, v 1.
  */
 
-import { resolve } from 'path';
+import { resolve, relative } from 'path';
 import { createReadStream } from 'fs';
 import { Readable } from 'stream';
-import { ToolingLog } from '@kbn/dev-utils';
+import { ToolingLog, REPO_ROOT } from '@kbn/dev-utils';
 import { KbnClient } from '@kbn/test';
 import type { KibanaClient } from '@elastic/elasticsearch/api/kibana';
 import { createPromiseFromStreams, concatStreamProviders } from '@kbn/utils';
@@ -37,23 +37,21 @@ const pipeline = (...streams: Readable[]) =>
  );
 
 export async function loadAction({
- name,
+ inputDir,
  skipExisting,
  useCreate,
  client,
- dataDir,
  log,
  kbnClient,
 }: {
- name: string;
+ inputDir: string;
  skipExisting: boolean;
  useCreate: boolean;
  client: KibanaClient;
- dataDir: string;
  log: ToolingLog;
  kbnClient: KbnClient;
 }) {
- const inputDir = resolve(dataDir, name);
+ const name = relative(REPO_ROOT, inputDir);
  const stats = createStats(name, log);
  const files = prioritizeMappings(await readDirectory(inputDir));
  const kibanaPluginIds = await kbnClient.plugins.getEnabledIds();

diff --git a/packages/kbn-es-archiver/src/actions/rebuild_all.ts b/packages/kbn-es-archiver/src/actions/rebuild_all.ts
@@ -6,11 +6,11 @@
  * Side Public License, v 1.
  */
 
-import { resolve, dirname, relative } from 'path';
+import { resolve, relative } from 'path';
 import { stat, Stats, rename, createReadStream, createWriteStream } from 'fs';
 import { Readable, Writable } from 'stream';
 import { fromNode } from 'bluebird';
-import { ToolingLog } from '@kbn/dev-utils';
+import { ToolingLog, REPO_ROOT } from '@kbn/dev-utils';
 import { createPromiseFromStreams } from '@kbn/utils';
 import {
  prioritizeMappings,
@@ -25,15 +25,7 @@ async function isDirectory(path: string): Promise<boolean> {
  return stats.isDirectory();
 }
 
-export async function rebuildAllAction({
- dataDir,
- log,
- rootDir = dataDir,
-}: {
- dataDir: string;
- log: ToolingLog;
- rootDir?: string;
-}) {
+export async function rebuildAllAction({ dataDir, log }: { dataDir: string; log: ToolingLog }) {
  const childNames = prioritizeMappings(await readDirectory(dataDir));
  for (const childName of childNames) {
  const childPath = resolve(dataDir, childName);
@@ -42,13 +34,12 @@ export async function rebuildAllAction({
  await rebuildAllAction({
  dataDir: childPath,
  log,
- rootDir,
  });
  continue;
  }
 
- const archiveName = dirname(relative(rootDir, childPath));
- log.info(`${archiveName} Rebuilding ${childName}`);
+ const archiveName = relative(REPO_ROOT, childPath);
+ log.info('[%s] Rebuilding %j', archiveName, childName);
  const gzip = isGzip(childPath);
  const tempFile = childPath + (gzip ? '.rebuilding.gz' : '.rebuilding');
 
@@ -60,6 +51,6 @@ export async function rebuildAllAction({
  ] as [Readable, ...Writable[]]);
 
  await fromNode((cb) => rename(tempFile, childPath, cb));
- log.info(`${archiveName} Rebuilt ${childName}`);
+ log.info('[%s] Rebuilt %j', archiveName, childName);
  }
 }
diff --git a/packages/kbn-es-archiver/src/actions/save.ts b/packages/kbn-es-archiver/src/actions/save.ts
@@ -6,11 +6,11 @@
  * Side Public License, v 1.
  */
 
-import { resolve } from 'path';
+import { resolve, relative } from 'path';
 import { createWriteStream, mkdirSync } from 'fs';
 import { Readable, Writable } from 'stream';
 import type { KibanaClient } from '@elastic/elasticsearch/api/kibana';
-import { ToolingLog } from '@kbn/dev-utils';
+import { ToolingLog, REPO_ROOT } from '@kbn/dev-utils';
 import { createListStream, createPromiseFromStreams } from '@kbn/utils';
 
 import {
@@ -22,23 +22,21 @@ import {
 } from '../lib';
 
 export async function saveAction({
- name,
+ outputDir,
  indices,
  client,
- dataDir,
  log,
  raw,
  query,
 }: {
- name: string;
+ outputDir: string;
  indices: string | string[];
  client: KibanaClient;
- dataDir: string;
  log: ToolingLog;
  raw: boolean;
  query?: Record<string, any>;
 }) {
- const outputDir = resolve(dataDir, name);
+ const name = relative(REPO_ROOT, outputDir);
  const stats = createStats(name, log);
 
  log.info('[%s] Creating archive of %j', name, indices);

diff --git a/packages/kbn-es-archiver/src/actions/unload.ts b/packages/kbn-es-archiver/src/actions/unload.ts
@@ -6,11 +6,11 @@
  * Side Public License, v 1.
  */
 
-import { resolve } from 'path';
+import { resolve, relative } from 'path';
 import { createReadStream } from 'fs';
 import { Readable, Writable } from 'stream';
 import type { KibanaClient } from '@elastic/elasticsearch/api/kibana';
-import { ToolingLog } from '@kbn/dev-utils';
+import { ToolingLog, REPO_ROOT } from '@kbn/dev-utils';
 import { KbnClient } from '@kbn/test';
 import { createPromiseFromStreams } from '@kbn/utils';
 
@@ -25,19 +25,17 @@ import {
 } from '../lib';
 
 export async function unloadAction({
- name,
+ inputDir,
  client,
- dataDir,
  log,
  kbnClient,
 }: {
- name: string;
+ inputDir: string;
  client: KibanaClient;
- dataDir: string;
  log: ToolingLog;
  kbnClient: KbnClient;
 }) {
- const inputDir = resolve(dataDir, name);
+ const name = relative(REPO_ROOT, inputDir);
  const stats = createStats(name, log);
  const kibanaPluginIds = await kbnClient.plugins.getEnabledIds();