Skip to content

Commit

Permalink
[7.16] Improve es-archiver saving/loading (#118255) (#118576)
Browse files Browse the repository at this point in the history
* Improve es-archiver saving/loading (#118255)

# Conflicts:
#	packages/kbn-es-archiver/src/actions/load.ts
#	packages/kbn-es-archiver/src/lib/indices/create_index_stream.ts
#	packages/kbn-es-archiver/src/lib/indices/generate_index_records_stream.ts
#	packages/kbn-es-archiver/src/lib/indices/kibana_index.ts

* Fix failing unit test

Accidentally deleted some lines when fixing merge conflicts, had to add
them back again.
  • Loading branch information
jportner authored Nov 15, 2021
1 parent bea4f87 commit aa1d1f9
Show file tree
Hide file tree
Showing 12 changed files with 377 additions and 93 deletions.
4 changes: 3 additions & 1 deletion packages/kbn-es-archiver/src/actions/load.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,13 +40,15 @@ export async function loadAction({
inputDir,
skipExisting,
useCreate,
docsOnly,
client,
log,
kbnClient,
}: {
inputDir: string;
skipExisting: boolean;
useCreate: boolean;
docsOnly?: boolean;
client: KibanaClient;
log: ToolingLog;
kbnClient: KbnClient;
Expand Down Expand Up @@ -76,7 +78,7 @@ export async function loadAction({

await createPromiseFromStreams([
recordStream,
createCreateIndexStream({ client, stats, skipExisting, log }),
createCreateIndexStream({ client, stats, skipExisting, docsOnly, log }),
createIndexDocRecordsStream(client, stats, progress, useCreate),
]);

Expand Down
6 changes: 4 additions & 2 deletions packages/kbn-es-archiver/src/actions/save.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,15 @@ export async function saveAction({
client,
log,
raw,
keepIndexNames,
query,
}: {
outputDir: string;
indices: string | string[];
client: KibanaClient;
log: ToolingLog;
raw: boolean;
keepIndexNames?: boolean;
query?: Record<string, any>;
}) {
const name = relative(REPO_ROOT, outputDir);
Expand All @@ -50,15 +52,15 @@ export async function saveAction({
// export and save the matching indices to mappings.json
createPromiseFromStreams([
createListStream(indices),
createGenerateIndexRecordsStream(client, stats),
createGenerateIndexRecordsStream({ client, stats, keepIndexNames }),
...createFormatArchiveStreams(),
createWriteStream(resolve(outputDir, 'mappings.json')),
] as [Readable, ...Writable[]]),

// export all documents from matching indexes into data.json.gz
createPromiseFromStreams([
createListStream(indices),
createGenerateDocRecordsStream({ client, stats, progress, query }),
createGenerateDocRecordsStream({ client, stats, progress, keepIndexNames, query }),
...createFormatArchiveStreams({ gzip: !raw }),
createWriteStream(resolve(outputDir, `data.json${raw ? '' : '.gz'}`)),
] as [Readable, ...Writable[]]),
Expand Down
24 changes: 18 additions & 6 deletions packages/kbn-es-archiver/src/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -142,11 +142,12 @@ export function runCli() {
$ node scripts/es_archiver save test/functional/es_archives/my_test_data logstash-*
`,
flags: {
boolean: ['raw'],
boolean: ['raw', 'keep-index-names'],
string: ['query'],
help: `
--raw don't gzip the archives
--query query object to limit the documents being archived, needs to be properly escaped JSON
--raw don't gzip the archives
--keep-index-names don't change the names of Kibana indices to .kibana_1
--query query object to limit the documents being archived, needs to be properly escaped JSON
`,
},
async run({ flags, esArchiver, statsMeta }) {
Expand All @@ -167,6 +168,11 @@ export function runCli() {
throw createFlagError('--raw does not take a value');
}

const keepIndexNames = flags['keep-index-names'];
if (typeof keepIndexNames !== 'boolean') {
throw createFlagError('--keep-index-names does not take a value');
}

const query = flags.query;
let parsedQuery;
if (typeof query === 'string' && query.length > 0) {
Expand All @@ -177,7 +183,7 @@ export function runCli() {
}
}

await esArchiver.save(path, indices, { raw, query: parsedQuery });
await esArchiver.save(path, indices, { raw, keepIndexNames, query: parsedQuery });
},
})
.command({
Expand All @@ -195,9 +201,10 @@ export function runCli() {
$ node scripts/es_archiver load my_test_data --config ../config.js
`,
flags: {
boolean: ['use-create'],
boolean: ['use-create', 'docs-only'],
help: `
--use-create use create instead of index for loading documents
--docs-only load only documents, not indices
`,
},
async run({ flags, esArchiver, statsMeta }) {
Expand All @@ -216,7 +223,12 @@ export function runCli() {
throw createFlagError('--use-create does not take a value');
}

await esArchiver.load(path, { useCreate });
const docsOnly = flags['docs-only'];
if (typeof docsOnly !== 'boolean') {
throw createFlagError('--docs-only does not take a value');
}

await esArchiver.load(path, { useCreate, docsOnly });
},
})
.command({
Expand Down
13 changes: 11 additions & 2 deletions packages/kbn-es-archiver/src/es_archiver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,16 +50,22 @@ export class EsArchiver {
* @param {String|Array<String>} indices - the indices to archive
* @param {Object} options
* @property {Boolean} options.raw - should the archive be raw (unzipped) or not
* @property {Boolean} options.keepIndexNames - should the Kibana index name be kept as-is or renamed
*/
async save(
path: string,
indices: string | string[],
{ raw = false, query }: { raw?: boolean; query?: Record<string, any> } = {}
{
raw = false,
keepIndexNames = false,
query,
}: { raw?: boolean; keepIndexNames?: boolean; query?: Record<string, any> } = {}
) {
return await saveAction({
outputDir: Path.resolve(this.baseDir, path),
indices,
raw,
keepIndexNames,
client: this.client,
log: this.log,
query,
Expand All @@ -74,18 +80,21 @@ export class EsArchiver {
* @property {Boolean} options.skipExisting - should existing indices
* be ignored or overwritten
* @property {Boolean} options.useCreate - use a create operation instead of index for documents
* @property {Boolean} options.docsOnly - load only documents, not indices
*/
async load(
path: string,
{
skipExisting = false,
useCreate = false,
}: { skipExisting?: boolean; useCreate?: boolean } = {}
docsOnly = false,
}: { skipExisting?: boolean; useCreate?: boolean; docsOnly?: boolean } = {}
) {
return await loadAction({
inputDir: this.findArchive(path),
skipExisting: !!skipExisting,
useCreate: !!useCreate,
docsOnly,
client: this.client,
log: this.log,
kbnClient: this.kbnClient,
Expand Down
Loading

0 comments on commit aa1d1f9

Please sign in to comment.