From 41ba84b46cf97b486661b786cecf87abef86cf09 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Fri, 24 Apr 2020 16:23:44 -0400 Subject: [PATCH 1/7] [Ingest] Support yaml variables in datasource --- .../server/services/datasource.ts | 18 ++--- .../server/services/epm/agent/agent.test.ts | 76 +++++++++++++------ .../server/services/epm/agent/agent.ts | 58 ++++++++++++-- 3 files changed, 111 insertions(+), 41 deletions(-) diff --git a/x-pack/plugins/ingest_manager/server/services/datasource.ts b/x-pack/plugins/ingest_manager/server/services/datasource.ts index f27252aaa9a84..7ed87e507514b 100644 --- a/x-pack/plugins/ingest_manager/server/services/datasource.ts +++ b/x-pack/plugins/ingest_manager/server/services/datasource.ts @@ -239,19 +239,11 @@ async function _assignPackageStreamToStream( throw new Error(`Stream template not found for dataset ${dataset}`); } - // Populate template variables from input config and stream config - const data: { [k: string]: string | string[] } = {}; - if (input.config) { - for (const key of Object.keys(input.config)) { - data[key] = input.config[key].value; - } - } - if (stream.config) { - for (const key of Object.keys(stream.config)) { - data[key] = stream.config[key].value; - } - } - const yaml = safeLoad(createStream(data, pkgStream.buffer.toString())); + const yaml = createStream( + // Populate template variables from input config and stream config + Object.assign({}, input.config, stream.config), + pkgStream.buffer.toString() + ); stream.pkg_stream = yaml; return { ...stream }; } diff --git a/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.test.ts b/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.test.ts index 21de625532f03..6fb72c6e42b98 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.test.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.test.ts @@ -6,29 +6,59 @@ import { createStream } from './agent'; -test('Test creating a stream from template', () => { - const streamTemplate = ` -input: log -paths: -{{#each paths}} - - {{this}} -{{/each}} -exclude_files: [".gz$"] -processors: - - add_locale: ~ - `; - const vars = { - paths: ['/usr/local/var/log/nginx/access.log'], - }; +describe('createStream', () => { + it('should work', () => { + const streamTemplate = ` + input: log + paths: + {{#each paths}} + - {{this}} + {{/each}} + exclude_files: [".gz$"] + processors: + - add_locale: ~ + `; + const vars = { + paths: { value: ['/usr/local/var/log/nginx/access.log'] }, + }; - const output = createStream(vars, streamTemplate); + const output = createStream(vars, streamTemplate); + expect(output).toEqual({ + input: 'log', + paths: ['/usr/local/var/log/nginx/access.log'], + exclude_files: ['.gz$'], + processors: [{ add_locale: null }], + }); + }); - expect(output).toBe(` -input: log -paths: - - /usr/local/var/log/nginx/access.log -exclude_files: [".gz$"] -processors: - - add_locale: ~ - `); + it('should support yaml values', () => { + const streamTemplate = ` + input: redis/metrics + metricsets: ["key"] + {{#if key.patterns}} + key.patterns: {{key.patterns}} + {{/if}} + `; + const vars = { + 'key.patterns': { + type: 'yaml', + value: ` + - limit: 20 + pattern: '*' + `, + }, + }; + + const output = createStream(vars, streamTemplate); + expect(output).toEqual({ + input: 'redis/metrics', + metricsets: ['key'], + 'key.patterns': [ + { + limit: 20, + pattern: '*', + }, + ], + }); + }); }); diff --git a/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.ts b/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.ts index 5d9a6d409aa1a..3b0f0655260ee 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.ts @@ -5,12 +5,60 @@ */ import Handlebars from 'handlebars'; +import { safeLoad } from 'js-yaml'; +import { DatasourceConfigRecord } from '../../../../common'; -interface StreamVars { - [k: string]: string | string[]; +function replaceVariablesInYaml(yamlVariables: { [k: string]: any }, yaml: any) { + const yamlKeys = Object.keys(yamlVariables); + if (yamlKeys.length === 0) { + return yaml; + } + + Object.entries(yaml).forEach(([key, value]: [string, any]) => { + if (typeof value === 'object') { + yaml[key] = replaceVariablesInYaml(yamlVariables, value); + } + + if (typeof value === 'string' && yamlKeys.indexOf(value) >= 0) { + yaml[key] = yamlVariables[value]; + } + }); + + return yaml; } -export function createStream(vars: StreamVars, streamTemplate: string) { - const template = Handlebars.compile(streamTemplate); - return template(vars); +export function createStream(variables: DatasourceConfigRecord, streamTemplate: string) { + const yamlValues: { [k: string]: any } = {}; + const vars = Object.entries(variables).reduce((acc, [key, recordEntry]) => { + // support variables with . like key.patterns + const keyParts = key.split('.'); + const lastKeyPart = keyParts.pop(); + + if (!lastKeyPart) { + throw new Error('Invalid key'); + } + + let varPart = acc; + for (const keyPart of keyParts) { + if (!varPart[keyPart]) { + varPart[keyPart] = {}; + } + varPart = varPart[keyPart]; + } + + if (recordEntry.type && recordEntry.type === 'yaml') { + const yamlKeyPlaceholder = `{{${key}}}`; + varPart[lastKeyPart] = `"${yamlKeyPlaceholder}"`; + yamlValues[yamlKeyPlaceholder] = recordEntry.value ? safeLoad(recordEntry.value) : null; + } else { + varPart[lastKeyPart] = recordEntry.value; + } + return acc; + }, {} as { [k: string]: any }); + + const template = Handlebars.compile(streamTemplate, { noEscape: true }); + const stream = template(vars); + const yamlFromStream = safeLoad(stream, {}); + + return replaceVariablesInYaml(yamlValues, yamlFromStream); } From 792bf79f39f98689ec1a4f877c3182490503a334 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Mon, 27 Apr 2020 11:52:31 -0400 Subject: [PATCH 2/7] Fix types --- x-pack/plugins/ingest_manager/server/services/datasource.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/ingest_manager/server/services/datasource.ts b/x-pack/plugins/ingest_manager/server/services/datasource.ts index 7ed87e507514b..31a086e1b18f8 100644 --- a/x-pack/plugins/ingest_manager/server/services/datasource.ts +++ b/x-pack/plugins/ingest_manager/server/services/datasource.ts @@ -4,7 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ import { SavedObjectsClientContract } from 'src/core/server'; -import { safeLoad } from 'js-yaml'; import { AuthenticatedUser } from '../../../security/server'; import { DeleteDatasourcesResponse, From d3933c05fc6500f7e82f630073f01193cf49524a Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Tue, 28 Apr 2020 14:16:24 -0400 Subject: [PATCH 3/7] Make createStream function more readable --- .../server/services/epm/agent/agent.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.ts b/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.ts index 3b0f0655260ee..847d7489cba94 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.ts @@ -9,8 +9,7 @@ import { safeLoad } from 'js-yaml'; import { DatasourceConfigRecord } from '../../../../common'; function replaceVariablesInYaml(yamlVariables: { [k: string]: any }, yaml: any) { - const yamlKeys = Object.keys(yamlVariables); - if (yamlKeys.length === 0) { + if (Object.keys(yamlVariables).length === 0) { return yaml; } @@ -19,7 +18,7 @@ function replaceVariablesInYaml(yamlVariables: { [k: string]: any }, yaml: any) yaml[key] = replaceVariablesInYaml(yamlVariables, value); } - if (typeof value === 'string' && yamlKeys.indexOf(value) >= 0) { + if (typeof value === 'string' && value in yamlVariables) { yaml[key] = yamlVariables[value]; } }); @@ -27,7 +26,7 @@ function replaceVariablesInYaml(yamlVariables: { [k: string]: any }, yaml: any) return yaml; } -export function createStream(variables: DatasourceConfigRecord, streamTemplate: string) { +function buildTemplateVariables(variables: DatasourceConfigRecord) { const yamlValues: { [k: string]: any } = {}; const vars = Object.entries(variables).reduce((acc, [key, recordEntry]) => { // support variables with . like key.patterns @@ -56,8 +55,15 @@ export function createStream(variables: DatasourceConfigRecord, streamTemplate: return acc; }, {} as { [k: string]: any }); + return { vars, yamlValues }; +} + +export function createStream(variables: DatasourceConfigRecord, streamTemplate: string) { + const { vars, yamlValues } = buildTemplateVariables(variables); + const template = Handlebars.compile(streamTemplate, { noEscape: true }); const stream = template(vars); + const yamlFromStream = safeLoad(stream, {}); return replaceVariablesInYaml(yamlValues, yamlFromStream); From 2268f1dbf1c1eb9a2ef1acbc956b3419c5bbb2a4 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Tue, 28 Apr 2020 15:21:01 -0400 Subject: [PATCH 4/7] More tests and use variable ##key## --- .../server/services/epm/agent/agent.test.ts | 2 ++ .../server/services/epm/agent/agent.ts | 13 ++++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.test.ts b/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.test.ts index 6fb72c6e42b98..db2e4fe474640 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.test.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.test.ts @@ -35,6 +35,7 @@ describe('createStream', () => { const streamTemplate = ` input: redis/metrics metricsets: ["key"] + test: null {{#if key.patterns}} key.patterns: {{key.patterns}} {{/if}} @@ -53,6 +54,7 @@ describe('createStream', () => { expect(output).toEqual({ input: 'redis/metrics', metricsets: ['key'], + test: null, 'key.patterns': [ { limit: 20, diff --git a/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.ts b/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.ts index 847d7489cba94..4b591ecf7b70e 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.ts @@ -8,8 +8,12 @@ import Handlebars from 'handlebars'; import { safeLoad } from 'js-yaml'; import { DatasourceConfigRecord } from '../../../../common'; +function isValidKey(key: string) { + return key !== '__proto__' && key !== 'constructor' && key !== 'prototype'; +} + function replaceVariablesInYaml(yamlVariables: { [k: string]: any }, yaml: any) { - if (Object.keys(yamlVariables).length === 0) { + if (Object.keys(yamlVariables).length === 0 || !yaml) { return yaml; } @@ -33,12 +37,15 @@ function buildTemplateVariables(variables: DatasourceConfigRecord) { const keyParts = key.split('.'); const lastKeyPart = keyParts.pop(); - if (!lastKeyPart) { + if (!lastKeyPart || !isValidKey(lastKeyPart)) { throw new Error('Invalid key'); } let varPart = acc; for (const keyPart of keyParts) { + if (!isValidKey(keyPart)) { + throw new Error('Invalid key'); + } if (!varPart[keyPart]) { varPart[keyPart] = {}; } @@ -46,7 +53,7 @@ function buildTemplateVariables(variables: DatasourceConfigRecord) { } if (recordEntry.type && recordEntry.type === 'yaml') { - const yamlKeyPlaceholder = `{{${key}}}`; + const yamlKeyPlaceholder = `##${key}##`; varPart[lastKeyPart] = `"${yamlKeyPlaceholder}"`; yamlValues[yamlKeyPlaceholder] = recordEntry.value ? safeLoad(recordEntry.value) : null; } else { From 8d33767efe3d6e1375dcacb8dde64117ec5a02b7 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Tue, 28 Apr 2020 15:58:49 -0400 Subject: [PATCH 5/7] Update x-pack/plugins/ingest_manager/server/services/epm/agent/agent.ts Co-Authored-By: John Schulz --- .../plugins/ingest_manager/server/services/epm/agent/agent.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.ts b/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.ts index 4b591ecf7b70e..cc2c10810a7d6 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.ts @@ -68,7 +68,7 @@ function buildTemplateVariables(variables: DatasourceConfigRecord) { export function createStream(variables: DatasourceConfigRecord, streamTemplate: string) { const { vars, yamlValues } = buildTemplateVariables(variables); - const template = Handlebars.compile(streamTemplate, { noEscape: true }); + const template = Handlebars.compile(streamTemplate); const stream = template(vars); const yamlFromStream = safeLoad(stream, {}); From e10a43bc8c23606cfc4d977c04841564c4ad83b6 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Tue, 28 Apr 2020 16:17:23 -0400 Subject: [PATCH 6/7] Update x-pack/plugins/ingest_manager/server/services/epm/agent/agent.ts Co-Authored-By: John Schulz --- .../plugins/ingest_manager/server/services/epm/agent/agent.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.ts b/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.ts index cc2c10810a7d6..5e72d20afac0a 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.ts @@ -54,7 +54,7 @@ function buildTemplateVariables(variables: DatasourceConfigRecord) { if (recordEntry.type && recordEntry.type === 'yaml') { const yamlKeyPlaceholder = `##${key}##`; - varPart[lastKeyPart] = `"${yamlKeyPlaceholder}"`; + varPart[lastKeyPart] = yamlKeyPlaceholder; yamlValues[yamlKeyPlaceholder] = recordEntry.value ? safeLoad(recordEntry.value) : null; } else { varPart[lastKeyPart] = recordEntry.value; From 87a185cd8bbcaa901f0b32db3d27cf3781c28265 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Tue, 28 Apr 2020 16:29:07 -0400 Subject: [PATCH 7/7] Do not escape html characters in yaml template --- .../ingest_manager/server/services/epm/agent/agent.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.ts b/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.ts index 5e72d20afac0a..8254c0d8aaa37 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/agent/agent.ts @@ -21,7 +21,6 @@ function replaceVariablesInYaml(yamlVariables: { [k: string]: any }, yaml: any) if (typeof value === 'object') { yaml[key] = replaceVariablesInYaml(yamlVariables, value); } - if (typeof value === 'string' && value in yamlVariables) { yaml[key] = yamlVariables[value]; } @@ -54,7 +53,7 @@ function buildTemplateVariables(variables: DatasourceConfigRecord) { if (recordEntry.type && recordEntry.type === 'yaml') { const yamlKeyPlaceholder = `##${key}##`; - varPart[lastKeyPart] = yamlKeyPlaceholder; + varPart[lastKeyPart] = `"${yamlKeyPlaceholder}"`; yamlValues[yamlKeyPlaceholder] = recordEntry.value ? safeLoad(recordEntry.value) : null; } else { varPart[lastKeyPart] = recordEntry.value; @@ -68,9 +67,8 @@ function buildTemplateVariables(variables: DatasourceConfigRecord) { export function createStream(variables: DatasourceConfigRecord, streamTemplate: string) { const { vars, yamlValues } = buildTemplateVariables(variables); - const template = Handlebars.compile(streamTemplate); + const template = Handlebars.compile(streamTemplate, { noEscape: true }); const stream = template(vars); - const yamlFromStream = safeLoad(stream, {}); return replaceVariablesInYaml(yamlValues, yamlFromStream);