From cef21139f0dc16d999e53be8f9bdb74714d29246 Mon Sep 17 00:00:00 2001 From: neptunian Date: Thu, 29 Oct 2020 16:26:11 -0400 Subject: [PATCH 1/2] check if pipeline refs for version already exists --- .../elasticsearch/ingest_pipeline/install.ts | 16 + .../apis/epm/install_remove_assets.ts | 414 ++++++++++-------- 2 files changed, 253 insertions(+), 177 deletions(-) diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/ingest_pipeline/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/ingest_pipeline/install.ts index 43c0179c0aa8a7..fae6a672e4578e 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/ingest_pipeline/install.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/ingest_pipeline/install.ts @@ -14,6 +14,8 @@ import { import * as Registry from '../../registry'; import { CallESAsCurrentUser } from '../../../../types'; import { saveInstalledEsRefs } from '../../packages/install'; +import { getInstallationObject } from '../../packages'; +import { deletePipelineRefs } from './remove'; interface RewriteSubstitution { source: string; @@ -50,6 +52,20 @@ export const installPipelines = async ( acc.push(...pipelineObjectRefs); return acc; }, []); + + // check that we don't duplicate the pipeline refs if the user is reinstalling + const installedPkg = await getInstallationObject({ + savedObjectsClient, + pkgName: installablePackage.name, + }); + if (!installedPkg) throw new Error("integration wasn't found while installing pipelines"); + // remove the current pipeline refs, if any exist, associated with this version before saving new ones so no duplicates occur + await deletePipelineRefs( + savedObjectsClient, + installedPkg.attributes.installed_es, + installedPkg.attributes.name, + installedPkg.attributes.version + ); await saveInstalledEsRefs(savedObjectsClient, installablePackage.name, pipelineRefs); const pipelines = dataStreams.reduce>>((acc, dataStream) => { if (dataStream.ingest_pipeline) { diff --git a/x-pack/test/ingest_manager_api_integration/apis/epm/install_remove_assets.ts b/x-pack/test/ingest_manager_api_integration/apis/epm/install_remove_assets.ts index cc6a384dcaafe6..72ea9cb4e7ef35 100644 --- a/x-pack/test/ingest_manager_api_integration/apis/epm/install_remove_assets.ts +++ b/x-pack/test/ingest_manager_api_integration/apis/epm/install_remove_assets.ts @@ -5,6 +5,8 @@ */ import expect from '@kbn/expect'; +import { sortBy } from 'lodash'; +import { AssetReference } from '../../../../plugins/ingest_manager/common'; import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; import { skipIfNoDockerRegistry } from '../../helpers'; @@ -12,6 +14,8 @@ export default function (providerContext: FtrProviderContext) { const { getService } = providerContext; const kibanaServer = getService('kibanaServer'); const supertest = getService('supertest'); + const dockerServers = getService('dockerServers'); + const server = dockerServers.get('registry'); const es = getService('es'); const pkgName = 'all_assets'; const pkgVersion = '0.1.0'; @@ -33,194 +37,27 @@ export default function (providerContext: FtrProviderContext) { describe('installs all assets when installing a package for the first time', async () => { skipIfNoDockerRegistry(providerContext); before(async () => { + if (!server.enabled) return; await installPackage(pkgKey); }); after(async () => { + if (!server.enabled) return; await uninstallPackage(pkgKey); }); - it('should have installed the ILM policy', async function () { - const resPolicy = await es.transport.request({ - method: 'GET', - path: `/_ilm/policy/all_assets`, - }); - expect(resPolicy.statusCode).equal(200); - }); - it('should have installed the index templates', async function () { - const resLogsTemplate = await es.transport.request({ - method: 'GET', - path: `/_index_template/${logsTemplateName}`, - }); - expect(resLogsTemplate.statusCode).equal(200); - - const resMetricsTemplate = await es.transport.request({ - method: 'GET', - path: `/_index_template/${metricsTemplateName}`, - }); - expect(resMetricsTemplate.statusCode).equal(200); - }); - it('should have installed the pipelines', async function () { - const res = await es.transport.request({ - method: 'GET', - path: `/_ingest/pipeline/${logsTemplateName}-${pkgVersion}`, - }); - expect(res.statusCode).equal(200); - const resPipeline1 = await es.transport.request({ - method: 'GET', - path: `/_ingest/pipeline/${logsTemplateName}-${pkgVersion}-pipeline1`, - }); - expect(resPipeline1.statusCode).equal(200); - const resPipeline2 = await es.transport.request({ - method: 'GET', - path: `/_ingest/pipeline/${logsTemplateName}-${pkgVersion}-pipeline2`, - }); - expect(resPipeline2.statusCode).equal(200); - }); - it('should have installed the template components', async function () { - const res = await es.transport.request({ - method: 'GET', - path: `/_component_template/${logsTemplateName}-mappings`, - }); - expect(res.statusCode).equal(200); - const resSettings = await es.transport.request({ - method: 'GET', - path: `/_component_template/${logsTemplateName}-settings`, - }); - expect(resSettings.statusCode).equal(200); - }); - it('should have installed the transform components', async function () { - const res = await es.transport.request({ - method: 'GET', - path: `/_transform/${pkgName}.test-default-${pkgVersion}`, - }); - expect(res.statusCode).equal(200); - }); - it('should have created the index for the transform', async function () { - // the index is defined in the transform file - const res = await es.transport.request({ - method: 'GET', - path: `/logs-all_assets.test_log_current_default`, - }); - expect(res.statusCode).equal(200); - }); - it('should have installed the kibana assets', async function () { - const resIndexPatternLogs = await kibanaServer.savedObjects.get({ - type: 'index-pattern', - id: 'logs-*', - }); - expect(resIndexPatternLogs.id).equal('logs-*'); - const resIndexPatternMetrics = await kibanaServer.savedObjects.get({ - type: 'index-pattern', - id: 'metrics-*', - }); - expect(resIndexPatternMetrics.id).equal('metrics-*'); - const resDashboard = await kibanaServer.savedObjects.get({ - type: 'dashboard', - id: 'sample_dashboard', - }); - expect(resDashboard.id).equal('sample_dashboard'); - const resDashboard2 = await kibanaServer.savedObjects.get({ - type: 'dashboard', - id: 'sample_dashboard2', - }); - expect(resDashboard2.id).equal('sample_dashboard2'); - const resVis = await kibanaServer.savedObjects.get({ - type: 'visualization', - id: 'sample_visualization', - }); - expect(resVis.id).equal('sample_visualization'); - const resSearch = await kibanaServer.savedObjects.get({ - type: 'search', - id: 'sample_search', - }); - expect(resSearch.id).equal('sample_search'); - }); - it('should create an index pattern with the package fields', async () => { - const resIndexPatternLogs = await kibanaServer.savedObjects.get({ - type: 'index-pattern', - id: 'logs-*', - }); - const fields = JSON.parse(resIndexPatternLogs.attributes.fields); - const exists = fields.find((field: { name: string }) => field.name === 'logs_test_name'); - expect(exists).not.to.be(undefined); - const resIndexPatternMetrics = await kibanaServer.savedObjects.get({ - type: 'index-pattern', - id: 'metrics-*', - }); - const fieldsMetrics = JSON.parse(resIndexPatternMetrics.attributes.fields); - const metricsExists = fieldsMetrics.find( - (field: { name: string }) => field.name === 'metrics_test_name' - ); - expect(metricsExists).not.to.be(undefined); - }); - it('should have created the correct saved object', async function () { - const res = await kibanaServer.savedObjects.get({ - type: 'epm-packages', - id: 'all_assets', - }); - expect(res.attributes).eql({ - installed_kibana: [ - { - id: 'sample_dashboard', - type: 'dashboard', - }, - { - id: 'sample_dashboard2', - type: 'dashboard', - }, - { - id: 'sample_search', - type: 'search', - }, - { - id: 'sample_visualization', - type: 'visualization', - }, - ], - installed_es: [ - { - id: 'logs-all_assets.test_logs-0.1.0', - type: 'ingest_pipeline', - }, - { - id: 'logs-all_assets.test_logs-0.1.0-pipeline1', - type: 'ingest_pipeline', - }, - { - id: 'logs-all_assets.test_logs-0.1.0-pipeline2', - type: 'ingest_pipeline', - }, - { - id: 'logs-all_assets.test_logs', - type: 'index_template', - }, - { - id: 'metrics-all_assets.test_metrics', - type: 'index_template', - }, - { - id: 'all_assets.test-default-0.1.0', - type: 'transform', - }, - ], - es_index_patterns: { - test_logs: 'logs-all_assets.test_logs-*', - test_metrics: 'metrics-all_assets.test_metrics-*', - }, - name: 'all_assets', - version: '0.1.0', - internal: false, - removable: true, - install_version: '0.1.0', - install_status: 'installed', - install_started_at: res.attributes.install_started_at, - install_source: 'registry', - }); + expectAssetsInstalled({ + logsTemplateName, + metricsTemplateName, + pkgVersion, + pkgName, + es, + kibanaServer, }); }); describe('uninstalls all assets when uninstalling a package', async () => { skipIfNoDockerRegistry(providerContext); before(async () => { + if (!server.enabled) return; // these tests ensure that uninstall works properly so make sure that the package gets installed and uninstalled // and then we'll test that not artifacts are left behind. await installPackage(pkgKey); @@ -403,5 +240,228 @@ export default function (providerContext: FtrProviderContext) { expect(res.response.data.statusCode).equal(404); }); }); + + describe('reinstalls all assets', async () => { + skipIfNoDockerRegistry(providerContext); + before(async () => { + if (!server.enabled) return; + await installPackage(pkgKey); + // reinstall + await installPackage(pkgKey); + }); + after(async () => { + if (!server.enabled) return; + await uninstallPackage(pkgKey); + }); + expectAssetsInstalled({ + logsTemplateName, + metricsTemplateName, + pkgVersion, + pkgName, + es, + kibanaServer, + }); + }); }); } + +const expectAssetsInstalled = ({ + logsTemplateName, + metricsTemplateName, + pkgVersion, + pkgName, + es, + kibanaServer, +}: { + logsTemplateName: string; + metricsTemplateName: string; + pkgVersion: string; + pkgName: string; + es: any; + kibanaServer: any; +}) => { + it('should have installed the ILM policy', async function () { + const resPolicy = await es.transport.request({ + method: 'GET', + path: `/_ilm/policy/all_assets`, + }); + expect(resPolicy.statusCode).equal(200); + }); + it('should have installed the index templates', async function () { + const resLogsTemplate = await es.transport.request({ + method: 'GET', + path: `/_index_template/${logsTemplateName}`, + }); + expect(resLogsTemplate.statusCode).equal(200); + + const resMetricsTemplate = await es.transport.request({ + method: 'GET', + path: `/_index_template/${metricsTemplateName}`, + }); + expect(resMetricsTemplate.statusCode).equal(200); + }); + it('should have installed the pipelines', async function () { + const res = await es.transport.request({ + method: 'GET', + path: `/_ingest/pipeline/${logsTemplateName}-${pkgVersion}`, + }); + expect(res.statusCode).equal(200); + const resPipeline1 = await es.transport.request({ + method: 'GET', + path: `/_ingest/pipeline/${logsTemplateName}-${pkgVersion}-pipeline1`, + }); + expect(resPipeline1.statusCode).equal(200); + const resPipeline2 = await es.transport.request({ + method: 'GET', + path: `/_ingest/pipeline/${logsTemplateName}-${pkgVersion}-pipeline2`, + }); + expect(resPipeline2.statusCode).equal(200); + }); + it('should have installed the template components', async function () { + const res = await es.transport.request({ + method: 'GET', + path: `/_component_template/${logsTemplateName}-mappings`, + }); + expect(res.statusCode).equal(200); + const resSettings = await es.transport.request({ + method: 'GET', + path: `/_component_template/${logsTemplateName}-settings`, + }); + expect(resSettings.statusCode).equal(200); + }); + it('should have installed the transform components', async function () { + const res = await es.transport.request({ + method: 'GET', + path: `/_transform/${pkgName}.test-default-${pkgVersion}`, + }); + expect(res.statusCode).equal(200); + }); + it('should have created the index for the transform', async function () { + // the index is defined in the transform file + const res = await es.transport.request({ + method: 'GET', + path: `/logs-all_assets.test_log_current_default`, + }); + expect(res.statusCode).equal(200); + }); + it('should have installed the kibana assets', async function () { + const resIndexPatternLogs = await kibanaServer.savedObjects.get({ + type: 'index-pattern', + id: 'logs-*', + }); + expect(resIndexPatternLogs.id).equal('logs-*'); + const resIndexPatternMetrics = await kibanaServer.savedObjects.get({ + type: 'index-pattern', + id: 'metrics-*', + }); + expect(resIndexPatternMetrics.id).equal('metrics-*'); + const resDashboard = await kibanaServer.savedObjects.get({ + type: 'dashboard', + id: 'sample_dashboard', + }); + expect(resDashboard.id).equal('sample_dashboard'); + const resDashboard2 = await kibanaServer.savedObjects.get({ + type: 'dashboard', + id: 'sample_dashboard2', + }); + expect(resDashboard2.id).equal('sample_dashboard2'); + const resVis = await kibanaServer.savedObjects.get({ + type: 'visualization', + id: 'sample_visualization', + }); + expect(resVis.id).equal('sample_visualization'); + const resSearch = await kibanaServer.savedObjects.get({ + type: 'search', + id: 'sample_search', + }); + expect(resSearch.id).equal('sample_search'); + }); + it('should create an index pattern with the package fields', async () => { + const resIndexPatternLogs = await kibanaServer.savedObjects.get({ + type: 'index-pattern', + id: 'logs-*', + }); + const fields = JSON.parse(resIndexPatternLogs.attributes.fields); + const exists = fields.find((field: { name: string }) => field.name === 'logs_test_name'); + expect(exists).not.to.be(undefined); + const resIndexPatternMetrics = await kibanaServer.savedObjects.get({ + type: 'index-pattern', + id: 'metrics-*', + }); + const fieldsMetrics = JSON.parse(resIndexPatternMetrics.attributes.fields); + const metricsExists = fieldsMetrics.find( + (field: { name: string }) => field.name === 'metrics_test_name' + ); + expect(metricsExists).not.to.be(undefined); + }); + it('should have created the correct saved object', async function () { + const res = await kibanaServer.savedObjects.get({ + type: 'epm-packages', + id: 'all_assets', + }); + // during a reinstall the items can change + const sortedRes = { + ...res.attributes, + installed_kibana: sortBy(res.attributes.installed_kibana, (o: AssetReference) => o.type), + installed_es: sortBy(res.attributes.installed_es, (o: AssetReference) => o.type), + }; + expect(sortedRes).eql({ + installed_kibana: [ + { + id: 'sample_dashboard', + type: 'dashboard', + }, + { + id: 'sample_dashboard2', + type: 'dashboard', + }, + { + id: 'sample_search', + type: 'search', + }, + { + id: 'sample_visualization', + type: 'visualization', + }, + ], + installed_es: [ + { + id: 'logs-all_assets.test_logs', + type: 'index_template', + }, + { + id: 'metrics-all_assets.test_metrics', + type: 'index_template', + }, + { + id: 'logs-all_assets.test_logs-0.1.0', + type: 'ingest_pipeline', + }, + { + id: 'logs-all_assets.test_logs-0.1.0-pipeline1', + type: 'ingest_pipeline', + }, + { + id: 'logs-all_assets.test_logs-0.1.0-pipeline2', + type: 'ingest_pipeline', + }, + { + id: 'all_assets.test-default-0.1.0', + type: 'transform', + }, + ], + es_index_patterns: { + test_logs: 'logs-all_assets.test_logs-*', + test_metrics: 'metrics-all_assets.test_metrics-*', + }, + name: 'all_assets', + version: '0.1.0', + internal: false, + removable: true, + install_version: '0.1.0', + install_status: 'installed', + install_started_at: res.attributes.install_started_at, + install_source: 'registry', + }); + }); +}; From 97a445e28fe5974de107951d9f06e417ba771c9e Mon Sep 17 00:00:00 2001 From: neptunian Date: Thu, 29 Oct 2020 16:52:26 -0400 Subject: [PATCH 2/2] use the right version --- .../services/epm/elasticsearch/ingest_pipeline/install.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/ingest_pipeline/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/ingest_pipeline/install.ts index fae6a672e4578e..58abdeb0d443d5 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/ingest_pipeline/install.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/ingest_pipeline/install.ts @@ -33,6 +33,7 @@ export const installPipelines = async ( // it can be created pointing to the new template, without removing the old one and effecting data // so do not remove the currently installed pipelines here const dataStreams = installablePackage.data_streams; + const { name: pkgName, version: pkgVersion } = installablePackage; if (!dataStreams?.length) return []; const pipelinePaths = paths.filter((path) => isPipeline(path)); // get and save pipeline refs before installing pipelines @@ -56,15 +57,15 @@ export const installPipelines = async ( // check that we don't duplicate the pipeline refs if the user is reinstalling const installedPkg = await getInstallationObject({ savedObjectsClient, - pkgName: installablePackage.name, + pkgName, }); if (!installedPkg) throw new Error("integration wasn't found while installing pipelines"); // remove the current pipeline refs, if any exist, associated with this version before saving new ones so no duplicates occur await deletePipelineRefs( savedObjectsClient, installedPkg.attributes.installed_es, - installedPkg.attributes.name, - installedPkg.attributes.version + pkgName, + pkgVersion ); await saveInstalledEsRefs(savedObjectsClient, installablePackage.name, pipelineRefs); const pipelines = dataStreams.reduce>>((acc, dataStream) => {