From cb86e22014d6b1be0c388a27ee5ce6cd3ef91ad4 Mon Sep 17 00:00:00 2001 From: Mike Cote Date: Wed, 19 Feb 2020 09:23:28 -0500 Subject: [PATCH 1/5] Initial work --- .../server/es/cluster_client_adapter.mock.ts | 3 ++ .../server/es/cluster_client_adapter.test.ts | 16 ++++++++++ .../server/es/cluster_client_adapter.ts | 24 +++++++++++++-- .../plugins/event_log/server/es/documents.ts | 5 ++-- x-pack/plugins/event_log/server/es/init.ts | 30 ++++++++++++++++++- 5 files changed, 72 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.mock.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.mock.ts index 87e8fb0f521a9..943fb2d78f22f 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.mock.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.mock.ts @@ -15,6 +15,9 @@ const createClusterClientMock = () => { createIndexTemplate: jest.fn(), doesAliasExist: jest.fn(), createIndex: jest.fn(), + getIndexTemplate: jest.fn(), + updateIndexTemplate: jest.fn(), + rolloverIndex: jest.fn(), }; return mock; }; diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts index ecefd4bfa271e..f9266bae97df1 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts @@ -140,6 +140,22 @@ describe('createIndexTemplate', () => { }); }); +describe('getIndexTemplate', () => { + test('should call cluster with given name', async () => { + await clusterClientAdapter.getIndexTemplate('foo'); + expect(clusterClient.callAsInternalUser).toHaveBeenCalledWith('indices.getTemplate', { + name: 'foo', + }); + }); + + test('should throw error if call cluster trows', async () => { + clusterClient.callAsInternalUser.mockRejectedValueOnce(new Error('Fail')); + await expect( + clusterClientAdapter.getIndexTemplate('foo') + ).rejects.toThrowErrorMatchingInlineSnapshot(`"Fail"`); + }); +}); + describe('doesAliasExist', () => { test('should call cluster with proper arguments', async () => { await clusterClientAdapter.doesAliasExist('foo'); diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts index c74eeacc9bb19..7e68fd891ea44 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts @@ -84,6 +84,19 @@ export class ClusterClientAdapter { } } + public async updateIndexTemplate(name: string, template: any): Promise { + const updateTemplateParams = { + name, + body: template, + }; + await this.callEs('indices.putTemplate', updateTemplateParams); + } + + public async getIndexTemplate(name: string): Promise { + const response = await this.callEs('indices.getTemplate', { name }); + return response[name]; + } + public async doesAliasExist(name: string): Promise { let result; try { @@ -94,9 +107,12 @@ export class ClusterClientAdapter { return result as boolean; } - public async createIndex(name: string): Promise { + public async createIndex(name: string, body: any = {}): Promise { try { - await this.callEs('indices.create', { index: name }); + await this.callEs('indices.create', { + index: name, + body, + }); } catch (err) { if (err.body?.error?.type !== 'resource_already_exists_exception') { throw new Error(`error creating initial index: ${err.message}`); @@ -104,6 +120,10 @@ export class ClusterClientAdapter { } } + public async rolloverIndex(body: any): Promise { + await this.callEs('indices.rollover', body); + } + private async callEs(operation: string, body?: any): Promise { try { this.debug(`callEs(${operation}) calls:`, body); diff --git a/x-pack/plugins/event_log/server/es/documents.ts b/x-pack/plugins/event_log/server/es/documents.ts index 09dd7383c4c5e..66acb7ab3ea8a 100644 --- a/x-pack/plugins/event_log/server/es/documents.ts +++ b/x-pack/plugins/event_log/server/es/documents.ts @@ -6,14 +6,12 @@ import { EsNames } from './names'; import mappings from '../../generated/mappings.json'; +import { getCurrentVersionAsInteger } from '../lib/get_current_version_as_integer'; // returns the body of an index template used in an ES indices.putTemplate call export function getIndexTemplate(esNames: EsNames) { const indexTemplateBody: any = { index_patterns: [esNames.indexPattern], - aliases: { - [esNames.alias]: {}, - }, settings: { number_of_shards: 1, number_of_replicas: 1, @@ -21,6 +19,7 @@ export function getIndexTemplate(esNames: EsNames) { 'index.lifecycle.rollover_alias': esNames.alias, }, mappings, + version: getCurrentVersionAsInteger(), }; return indexTemplateBody; diff --git a/x-pack/plugins/event_log/server/es/init.ts b/x-pack/plugins/event_log/server/es/init.ts index 7094277f7aa9f..77b68ac95a8b2 100644 --- a/x-pack/plugins/event_log/server/es/init.ts +++ b/x-pack/plugins/event_log/server/es/init.ts @@ -27,6 +27,7 @@ async function initializeEsResources(esContext: EsContext) { await steps.createIlmPolicyIfNotExists(); await steps.createIndexTemplateIfNotExists(); await steps.createInitialIndexIfNotExists(); + await steps.migrateIndexTemplate(); } class EsInitializationSteps { @@ -62,7 +63,34 @@ class EsInitializationSteps { async createInitialIndexIfNotExists(): Promise { const exists = await this.esContext.esAdapter.doesAliasExist(this.esContext.esNames.alias); if (!exists) { - await this.esContext.esAdapter.createIndex(this.esContext.esNames.initialIndex); + await this.esContext.esAdapter.createIndex(this.esContext.esNames.initialIndex, { + aliases: { + [this.esContext.esNames.alias]: { + is_write_index: true, + }, + }, + }); + } + } + + async migrateIndexTemplate() { + const existingTemplate = await this.esContext.esAdapter.getIndexTemplate( + this.esContext.esNames.indexTemplate + ); + const updatedTemplateBody = getIndexTemplate(this.esContext.esNames); + if (updatedTemplateBody.version > existingTemplate.version) { + await this.esContext.esAdapter.updateIndexTemplate( + this.esContext.esNames.indexTemplate, + updatedTemplateBody + ); + await this.esContext.esAdapter.rolloverIndex({ + alias: this.esContext.esNames.alias, + body: { + conditions: { + max_age: '0d', + }, + }, + }); } } } From c2b83ca2c64a1d272510538723ff7d9beb71a0a1 Mon Sep 17 00:00:00 2001 From: Mike Cote Date: Wed, 19 Feb 2020 10:08:59 -0500 Subject: [PATCH 2/5] Add missing file --- .../lib/get_current_version_as_integer.ts | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 x-pack/plugins/event_log/server/lib/get_current_version_as_integer.ts diff --git a/x-pack/plugins/event_log/server/lib/get_current_version_as_integer.ts b/x-pack/plugins/event_log/server/lib/get_current_version_as_integer.ts new file mode 100644 index 0000000000000..8a203a39fd8e9 --- /dev/null +++ b/x-pack/plugins/event_log/server/lib/get_current_version_as_integer.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { padLeft } from 'lodash'; +import xPackage from '../../../../package.json'; + +export function getCurrentVersionAsInteger(): number { + // break up the string parts + const splitted = xPackage.version.split('.'); + + // pad each part with leading 0 to make 2 characters + const padded = splitted.map((v: string) => { + const vMatches = v.match(/\d+/); + if (vMatches) { + return padLeft(vMatches[0], 2, '0'); + } + return '00'; + }); + return parseInt(padded.join(''), 10); +} From dbc3111763a9771533f989d860128dd4d0e7ff30 Mon Sep 17 00:00:00 2001 From: Mike Cote Date: Wed, 19 Feb 2020 15:19:12 -0500 Subject: [PATCH 3/5] Add tests where missing --- .../server/es/cluster_client_adapter.test.ts | 38 +++++++++++++++++ .../event_log/server/es/documents.test.ts | 2 +- .../plugins/event_log/server/es/init.test.ts | 37 +++++++++++++++++ x-pack/plugins/event_log/server/es/init.ts | 7 +++- .../get_current_version_as_integer.test.ts | 41 +++++++++++++++++++ 5 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugins/event_log/server/lib/get_current_version_as_integer.test.ts diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts index f9266bae97df1..aaa844f60da8c 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts @@ -140,7 +140,28 @@ describe('createIndexTemplate', () => { }); }); +describe('updateIndexTemplate', () => { + test('should call cluster with given template', async () => { + await clusterClientAdapter.updateIndexTemplate('foo', { args: true }); + expect(clusterClient.callAsInternalUser).toHaveBeenCalledWith('indices.putTemplate', { + name: 'foo', + body: { args: true }, + }); + }); + + test('should throw error when call cluster throws', async () => { + clusterClient.callAsInternalUser.mockRejectedValue(new Error('Fail')); + await expect( + clusterClientAdapter.updateIndexTemplate('foo', { args: true }) + ).rejects.toThrowErrorMatchingInlineSnapshot(`"Fail"`); + }); +}); + describe('getIndexTemplate', () => { + beforeEach(() => { + clusterClient.callAsInternalUser.mockResolvedValue({}); + }); + test('should call cluster with given name', async () => { await clusterClientAdapter.getIndexTemplate('foo'); expect(clusterClient.callAsInternalUser).toHaveBeenCalledWith('indices.getTemplate', { @@ -189,6 +210,7 @@ describe('createIndex', () => { await clusterClientAdapter.createIndex('foo'); expect(clusterClient.callAsInternalUser).toHaveBeenCalledWith('indices.create', { index: 'foo', + body: {}, }); }); @@ -210,3 +232,19 @@ describe('createIndex', () => { await clusterClientAdapter.createIndex('foo'); }); }); + +describe('rolloverIndex', () => { + test('should call cluster with given body', async () => { + await clusterClientAdapter.rolloverIndex({ args: true }); + expect(clusterClient.callAsInternalUser).toHaveBeenCalledWith('indices.rollover', { + args: true, + }); + }); + + test('should throw error when call cluster throws', async () => { + clusterClient.callAsInternalUser.mockRejectedValue(new Error('Fail')); + await expect( + clusterClientAdapter.rolloverIndex({ args: true }) + ).rejects.toThrowErrorMatchingInlineSnapshot(`"Fail"`); + }); +}); diff --git a/x-pack/plugins/event_log/server/es/documents.test.ts b/x-pack/plugins/event_log/server/es/documents.test.ts index 7edca4b3943a6..7a29cca3f6553 100644 --- a/x-pack/plugins/event_log/server/es/documents.test.ts +++ b/x-pack/plugins/event_log/server/es/documents.test.ts @@ -23,11 +23,11 @@ describe('getIndexTemplate()', () => { test('returns the correct details of the index template', () => { const indexTemplate = getIndexTemplate(esNames); expect(indexTemplate.index_patterns).toEqual([esNames.indexPattern]); - expect(indexTemplate.aliases[esNames.alias]).toEqual({}); expect(indexTemplate.settings.number_of_shards).toBeGreaterThanOrEqual(0); expect(indexTemplate.settings.number_of_replicas).toBeGreaterThanOrEqual(0); expect(indexTemplate.settings['index.lifecycle.name']).toBe(esNames.ilmPolicy); expect(indexTemplate.settings['index.lifecycle.rollover_alias']).toBe(esNames.alias); expect(indexTemplate.mappings).toMatchObject({}); + expect(indexTemplate.version).toBeGreaterThanOrEqual(0); }); }); diff --git a/x-pack/plugins/event_log/server/es/init.test.ts b/x-pack/plugins/event_log/server/es/init.test.ts index ad237e522c0a5..803b3b416a650 100644 --- a/x-pack/plugins/event_log/server/es/init.test.ts +++ b/x-pack/plugins/event_log/server/es/init.test.ts @@ -7,6 +7,10 @@ import { contextMock } from './context.mock'; import { initializeEs } from './init'; +jest.mock('../lib/get_current_version_as_integer', () => ({ + getCurrentVersionAsInteger: () => 1, +})); + describe('initializeEs', () => { let esContext = contextMock.create(); @@ -61,4 +65,37 @@ describe('initializeEs', () => { expect(esContext.esAdapter.doesAliasExist).toHaveBeenCalled(); expect(esContext.esAdapter.createIndex).not.toHaveBeenCalled(); }); + + test('should migrate index template when kibana version is more recent', async () => { + esContext.esAdapter.getIndexTemplate.mockResolvedValue({ + version: 0, + }); + + await initializeEs(esContext); + expect(esContext.esAdapter.getIndexTemplate).toHaveBeenCalled(); + expect(esContext.esAdapter.updateIndexTemplate).toHaveBeenCalled(); + expect(esContext.esAdapter.rolloverIndex).toHaveBeenCalled(); + }); + + test(`shouldn't migrate index template when kibana version is the same`, async () => { + esContext.esAdapter.getIndexTemplate.mockResolvedValue({ + version: 1, + }); + + await initializeEs(esContext); + expect(esContext.esAdapter.getIndexTemplate).toHaveBeenCalled(); + expect(esContext.esAdapter.updateIndexTemplate).not.toHaveBeenCalled(); + expect(esContext.esAdapter.rolloverIndex).not.toHaveBeenCalled(); + }); + + test('should log error if template version in Elasticsearch is more recent', async () => { + esContext.esAdapter.getIndexTemplate.mockResolvedValue({ + version: 2, + }); + + await initializeEs(esContext); + expect(esContext.logger.error).toHaveBeenLastCalledWith( + 'error initializing elasticsearch resources: Index template belongs to a more recent version of Kibana (2)' + ); + }); }); diff --git a/x-pack/plugins/event_log/server/es/init.ts b/x-pack/plugins/event_log/server/es/init.ts index 77b68ac95a8b2..f506a73e87ff4 100644 --- a/x-pack/plugins/event_log/server/es/init.ts +++ b/x-pack/plugins/event_log/server/es/init.ts @@ -74,10 +74,11 @@ class EsInitializationSteps { } async migrateIndexTemplate() { + const updatedTemplateBody = getIndexTemplate(this.esContext.esNames); const existingTemplate = await this.esContext.esAdapter.getIndexTemplate( this.esContext.esNames.indexTemplate ); - const updatedTemplateBody = getIndexTemplate(this.esContext.esNames); + if (updatedTemplateBody.version > existingTemplate.version) { await this.esContext.esAdapter.updateIndexTemplate( this.esContext.esNames.indexTemplate, @@ -91,6 +92,10 @@ class EsInitializationSteps { }, }, }); + } else if (updatedTemplateBody.version < existingTemplate.version) { + throw new Error( + `Index template belongs to a more recent version of Kibana (${existingTemplate.version})` + ); } } } diff --git a/x-pack/plugins/event_log/server/lib/get_current_version_as_integer.test.ts b/x-pack/plugins/event_log/server/lib/get_current_version_as_integer.test.ts new file mode 100644 index 0000000000000..c3ad19ec7df2b --- /dev/null +++ b/x-pack/plugins/event_log/server/lib/get_current_version_as_integer.test.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +describe('getCurrentVersionAsInteger', () => { + beforeEach(() => jest.resetModules()); + + test('should parse 8.0.0', () => { + jest.mock('../../../../package.json', () => ({ + version: '8.0.0', + })); + const { getCurrentVersionAsInteger } = jest.requireActual('./get_current_version_as_integer'); + expect(getCurrentVersionAsInteger()).toEqual(80000); + }); + + test('should parse 8.1.0', () => { + jest.mock('../../../../package.json', () => ({ + version: '8.1.0', + })); + const { getCurrentVersionAsInteger } = jest.requireActual('./get_current_version_as_integer'); + expect(getCurrentVersionAsInteger()).toEqual(80100); + }); + + test('should parse 8.1.2', () => { + jest.mock('../../../../package.json', () => ({ + version: '8.1.2', + })); + const { getCurrentVersionAsInteger } = jest.requireActual('./get_current_version_as_integer'); + expect(getCurrentVersionAsInteger()).toEqual(80102); + }); + + test('should parse v8.1.2', () => { + jest.mock('../../../../package.json', () => ({ + version: 'v8.1.2', + })); + const { getCurrentVersionAsInteger } = jest.requireActual('./get_current_version_as_integer'); + expect(getCurrentVersionAsInteger()).toEqual(80102); + }); +}); From 0d5bde1005906087ee8494620fcd896316bd298c Mon Sep 17 00:00:00 2001 From: Mike Cote Date: Fri, 21 Feb 2020 10:42:30 -0500 Subject: [PATCH 4/5] Add kibana version to esNames --- .../server/es/cluster_client_adapter.mock.ts | 3 -- .../server/es/cluster_client_adapter.test.ts | 53 ------------------- .../server/es/cluster_client_adapter.ts | 17 ------ .../event_log/server/es/documents.test.ts | 1 - .../plugins/event_log/server/es/documents.ts | 2 - .../plugins/event_log/server/es/init.test.ts | 37 ------------- x-pack/plugins/event_log/server/es/init.ts | 27 ---------- .../plugins/event_log/server/es/names.mock.ts | 8 +-- .../plugins/event_log/server/es/names.test.ts | 13 +++-- x-pack/plugins/event_log/server/es/names.ts | 14 +++-- .../get_current_version_as_integer.test.ts | 41 -------------- .../lib/get_current_version_as_integer.ts | 23 -------- 12 files changed, 22 insertions(+), 217 deletions(-) delete mode 100644 x-pack/plugins/event_log/server/lib/get_current_version_as_integer.test.ts delete mode 100644 x-pack/plugins/event_log/server/lib/get_current_version_as_integer.ts diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.mock.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.mock.ts index 943fb2d78f22f..87e8fb0f521a9 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.mock.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.mock.ts @@ -15,9 +15,6 @@ const createClusterClientMock = () => { createIndexTemplate: jest.fn(), doesAliasExist: jest.fn(), createIndex: jest.fn(), - getIndexTemplate: jest.fn(), - updateIndexTemplate: jest.fn(), - rolloverIndex: jest.fn(), }; return mock; }; diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts index aaa844f60da8c..b61196439ee4f 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.test.ts @@ -140,43 +140,6 @@ describe('createIndexTemplate', () => { }); }); -describe('updateIndexTemplate', () => { - test('should call cluster with given template', async () => { - await clusterClientAdapter.updateIndexTemplate('foo', { args: true }); - expect(clusterClient.callAsInternalUser).toHaveBeenCalledWith('indices.putTemplate', { - name: 'foo', - body: { args: true }, - }); - }); - - test('should throw error when call cluster throws', async () => { - clusterClient.callAsInternalUser.mockRejectedValue(new Error('Fail')); - await expect( - clusterClientAdapter.updateIndexTemplate('foo', { args: true }) - ).rejects.toThrowErrorMatchingInlineSnapshot(`"Fail"`); - }); -}); - -describe('getIndexTemplate', () => { - beforeEach(() => { - clusterClient.callAsInternalUser.mockResolvedValue({}); - }); - - test('should call cluster with given name', async () => { - await clusterClientAdapter.getIndexTemplate('foo'); - expect(clusterClient.callAsInternalUser).toHaveBeenCalledWith('indices.getTemplate', { - name: 'foo', - }); - }); - - test('should throw error if call cluster trows', async () => { - clusterClient.callAsInternalUser.mockRejectedValueOnce(new Error('Fail')); - await expect( - clusterClientAdapter.getIndexTemplate('foo') - ).rejects.toThrowErrorMatchingInlineSnapshot(`"Fail"`); - }); -}); - describe('doesAliasExist', () => { test('should call cluster with proper arguments', async () => { await clusterClientAdapter.doesAliasExist('foo'); @@ -232,19 +195,3 @@ describe('createIndex', () => { await clusterClientAdapter.createIndex('foo'); }); }); - -describe('rolloverIndex', () => { - test('should call cluster with given body', async () => { - await clusterClientAdapter.rolloverIndex({ args: true }); - expect(clusterClient.callAsInternalUser).toHaveBeenCalledWith('indices.rollover', { - args: true, - }); - }); - - test('should throw error when call cluster throws', async () => { - clusterClient.callAsInternalUser.mockRejectedValue(new Error('Fail')); - await expect( - clusterClientAdapter.rolloverIndex({ args: true }) - ).rejects.toThrowErrorMatchingInlineSnapshot(`"Fail"`); - }); -}); diff --git a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts index 7e68fd891ea44..d585fd4f539b5 100644 --- a/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts +++ b/x-pack/plugins/event_log/server/es/cluster_client_adapter.ts @@ -84,19 +84,6 @@ export class ClusterClientAdapter { } } - public async updateIndexTemplate(name: string, template: any): Promise { - const updateTemplateParams = { - name, - body: template, - }; - await this.callEs('indices.putTemplate', updateTemplateParams); - } - - public async getIndexTemplate(name: string): Promise { - const response = await this.callEs('indices.getTemplate', { name }); - return response[name]; - } - public async doesAliasExist(name: string): Promise { let result; try { @@ -120,10 +107,6 @@ export class ClusterClientAdapter { } } - public async rolloverIndex(body: any): Promise { - await this.callEs('indices.rollover', body); - } - private async callEs(operation: string, body?: any): Promise { try { this.debug(`callEs(${operation}) calls:`, body); diff --git a/x-pack/plugins/event_log/server/es/documents.test.ts b/x-pack/plugins/event_log/server/es/documents.test.ts index 7a29cca3f6553..1a71500b66624 100644 --- a/x-pack/plugins/event_log/server/es/documents.test.ts +++ b/x-pack/plugins/event_log/server/es/documents.test.ts @@ -28,6 +28,5 @@ describe('getIndexTemplate()', () => { expect(indexTemplate.settings['index.lifecycle.name']).toBe(esNames.ilmPolicy); expect(indexTemplate.settings['index.lifecycle.rollover_alias']).toBe(esNames.alias); expect(indexTemplate.mappings).toMatchObject({}); - expect(indexTemplate.version).toBeGreaterThanOrEqual(0); }); }); diff --git a/x-pack/plugins/event_log/server/es/documents.ts b/x-pack/plugins/event_log/server/es/documents.ts index 66acb7ab3ea8a..78ec9f653f5e6 100644 --- a/x-pack/plugins/event_log/server/es/documents.ts +++ b/x-pack/plugins/event_log/server/es/documents.ts @@ -6,7 +6,6 @@ import { EsNames } from './names'; import mappings from '../../generated/mappings.json'; -import { getCurrentVersionAsInteger } from '../lib/get_current_version_as_integer'; // returns the body of an index template used in an ES indices.putTemplate call export function getIndexTemplate(esNames: EsNames) { @@ -19,7 +18,6 @@ export function getIndexTemplate(esNames: EsNames) { 'index.lifecycle.rollover_alias': esNames.alias, }, mappings, - version: getCurrentVersionAsInteger(), }; return indexTemplateBody; diff --git a/x-pack/plugins/event_log/server/es/init.test.ts b/x-pack/plugins/event_log/server/es/init.test.ts index 803b3b416a650..ad237e522c0a5 100644 --- a/x-pack/plugins/event_log/server/es/init.test.ts +++ b/x-pack/plugins/event_log/server/es/init.test.ts @@ -7,10 +7,6 @@ import { contextMock } from './context.mock'; import { initializeEs } from './init'; -jest.mock('../lib/get_current_version_as_integer', () => ({ - getCurrentVersionAsInteger: () => 1, -})); - describe('initializeEs', () => { let esContext = contextMock.create(); @@ -65,37 +61,4 @@ describe('initializeEs', () => { expect(esContext.esAdapter.doesAliasExist).toHaveBeenCalled(); expect(esContext.esAdapter.createIndex).not.toHaveBeenCalled(); }); - - test('should migrate index template when kibana version is more recent', async () => { - esContext.esAdapter.getIndexTemplate.mockResolvedValue({ - version: 0, - }); - - await initializeEs(esContext); - expect(esContext.esAdapter.getIndexTemplate).toHaveBeenCalled(); - expect(esContext.esAdapter.updateIndexTemplate).toHaveBeenCalled(); - expect(esContext.esAdapter.rolloverIndex).toHaveBeenCalled(); - }); - - test(`shouldn't migrate index template when kibana version is the same`, async () => { - esContext.esAdapter.getIndexTemplate.mockResolvedValue({ - version: 1, - }); - - await initializeEs(esContext); - expect(esContext.esAdapter.getIndexTemplate).toHaveBeenCalled(); - expect(esContext.esAdapter.updateIndexTemplate).not.toHaveBeenCalled(); - expect(esContext.esAdapter.rolloverIndex).not.toHaveBeenCalled(); - }); - - test('should log error if template version in Elasticsearch is more recent', async () => { - esContext.esAdapter.getIndexTemplate.mockResolvedValue({ - version: 2, - }); - - await initializeEs(esContext); - expect(esContext.logger.error).toHaveBeenLastCalledWith( - 'error initializing elasticsearch resources: Index template belongs to a more recent version of Kibana (2)' - ); - }); }); diff --git a/x-pack/plugins/event_log/server/es/init.ts b/x-pack/plugins/event_log/server/es/init.ts index f506a73e87ff4..c67d6541ce002 100644 --- a/x-pack/plugins/event_log/server/es/init.ts +++ b/x-pack/plugins/event_log/server/es/init.ts @@ -27,7 +27,6 @@ async function initializeEsResources(esContext: EsContext) { await steps.createIlmPolicyIfNotExists(); await steps.createIndexTemplateIfNotExists(); await steps.createInitialIndexIfNotExists(); - await steps.migrateIndexTemplate(); } class EsInitializationSteps { @@ -72,30 +71,4 @@ class EsInitializationSteps { }); } } - - async migrateIndexTemplate() { - const updatedTemplateBody = getIndexTemplate(this.esContext.esNames); - const existingTemplate = await this.esContext.esAdapter.getIndexTemplate( - this.esContext.esNames.indexTemplate - ); - - if (updatedTemplateBody.version > existingTemplate.version) { - await this.esContext.esAdapter.updateIndexTemplate( - this.esContext.esNames.indexTemplate, - updatedTemplateBody - ); - await this.esContext.esAdapter.rolloverIndex({ - alias: this.esContext.esNames.alias, - body: { - conditions: { - max_age: '0d', - }, - }, - }); - } else if (updatedTemplateBody.version < existingTemplate.version) { - throw new Error( - `Index template belongs to a more recent version of Kibana (${existingTemplate.version})` - ); - } - } } diff --git a/x-pack/plugins/event_log/server/es/names.mock.ts b/x-pack/plugins/event_log/server/es/names.mock.ts index 7b013a0d263da..74d8dfafef34c 100644 --- a/x-pack/plugins/event_log/server/es/names.mock.ts +++ b/x-pack/plugins/event_log/server/es/names.mock.ts @@ -9,11 +9,11 @@ import { EsNames } from './names'; const createNamesMock = () => { const mock: jest.Mocked = { base: '.kibana', - alias: '.kibana-event-log', - ilmPolicy: '.kibana-event-log-policy', + alias: '.kibana-event-log-8.0.0', + ilmPolicy: '.kibana-event-log-8.0.0-policy', indexPattern: '.kibana-event-log-*', - initialIndex: '.kibana-event-log-000001', - indexTemplate: '.kibana-event-log-template', + initialIndex: '.kibana-event-log-8.0.0-000001', + indexTemplate: '.kibana-event-log-8.0.0-template', }; return mock; }; diff --git a/x-pack/plugins/event_log/server/es/names.test.ts b/x-pack/plugins/event_log/server/es/names.test.ts index d88c4212df91c..3f811d9fc4cb0 100644 --- a/x-pack/plugins/event_log/server/es/names.test.ts +++ b/x-pack/plugins/event_log/server/es/names.test.ts @@ -6,15 +6,20 @@ import { getEsNames } from './names'; +jest.mock('../lib/../../../../package.json', () => ({ + version: '1.2.3', +})); + describe('getEsNames()', () => { test('works as expected', () => { const base = 'XYZ'; + const version = '1.2.3'; const esNames = getEsNames(base); expect(esNames.base).toEqual(base); - expect(esNames.alias).toEqual(`${base}-event-log`); - expect(esNames.ilmPolicy).toEqual(`${base}-event-log-policy`); + expect(esNames.alias).toEqual(`${base}-event-log-${version}`); + expect(esNames.ilmPolicy).toEqual(`${base}-event-log-${version}-policy`); expect(esNames.indexPattern).toEqual(`${base}-event-log-*`); - expect(esNames.initialIndex).toEqual(`${base}-event-log-000001`); - expect(esNames.indexTemplate).toEqual(`${base}-event-log-template`); + expect(esNames.initialIndex).toEqual(`${base}-event-log-${version}-000001`); + expect(esNames.indexTemplate).toEqual(`${base}-event-log-${version}-template`); }); }); diff --git a/x-pack/plugins/event_log/server/es/names.ts b/x-pack/plugins/event_log/server/es/names.ts index be737d23625f1..958da8e6016a8 100644 --- a/x-pack/plugins/event_log/server/es/names.ts +++ b/x-pack/plugins/event_log/server/es/names.ts @@ -4,7 +4,10 @@ * you may not use this file except in compliance with the Elastic License. */ -const EVENT_LOG_NAME_SUFFIX = '-event-log'; +import xPackage from '../../../../package.json'; + +const EVENT_LOG_NAME_SUFFIX = `-event-log`; +const EVENT_LOG_VERSION_SUFFIX = `-${xPackage.version}`; export interface EsNames { base: string; @@ -17,12 +20,13 @@ export interface EsNames { export function getEsNames(baseName: string): EsNames { const eventLogName = `${baseName}${EVENT_LOG_NAME_SUFFIX}`; + const eventLogNameWithVersion = `${eventLogName}${EVENT_LOG_VERSION_SUFFIX}`; return { base: baseName, - alias: eventLogName, - ilmPolicy: `${eventLogName}-policy`, + alias: eventLogNameWithVersion, + ilmPolicy: `${eventLogNameWithVersion}-policy`, indexPattern: `${eventLogName}-*`, - initialIndex: `${eventLogName}-000001`, - indexTemplate: `${eventLogName}-template`, + initialIndex: `${eventLogNameWithVersion}-000001`, + indexTemplate: `${eventLogNameWithVersion}-template`, }; } diff --git a/x-pack/plugins/event_log/server/lib/get_current_version_as_integer.test.ts b/x-pack/plugins/event_log/server/lib/get_current_version_as_integer.test.ts deleted file mode 100644 index c3ad19ec7df2b..0000000000000 --- a/x-pack/plugins/event_log/server/lib/get_current_version_as_integer.test.ts +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -describe('getCurrentVersionAsInteger', () => { - beforeEach(() => jest.resetModules()); - - test('should parse 8.0.0', () => { - jest.mock('../../../../package.json', () => ({ - version: '8.0.0', - })); - const { getCurrentVersionAsInteger } = jest.requireActual('./get_current_version_as_integer'); - expect(getCurrentVersionAsInteger()).toEqual(80000); - }); - - test('should parse 8.1.0', () => { - jest.mock('../../../../package.json', () => ({ - version: '8.1.0', - })); - const { getCurrentVersionAsInteger } = jest.requireActual('./get_current_version_as_integer'); - expect(getCurrentVersionAsInteger()).toEqual(80100); - }); - - test('should parse 8.1.2', () => { - jest.mock('../../../../package.json', () => ({ - version: '8.1.2', - })); - const { getCurrentVersionAsInteger } = jest.requireActual('./get_current_version_as_integer'); - expect(getCurrentVersionAsInteger()).toEqual(80102); - }); - - test('should parse v8.1.2', () => { - jest.mock('../../../../package.json', () => ({ - version: 'v8.1.2', - })); - const { getCurrentVersionAsInteger } = jest.requireActual('./get_current_version_as_integer'); - expect(getCurrentVersionAsInteger()).toEqual(80102); - }); -}); diff --git a/x-pack/plugins/event_log/server/lib/get_current_version_as_integer.ts b/x-pack/plugins/event_log/server/lib/get_current_version_as_integer.ts deleted file mode 100644 index 8a203a39fd8e9..0000000000000 --- a/x-pack/plugins/event_log/server/lib/get_current_version_as_integer.ts +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { padLeft } from 'lodash'; -import xPackage from '../../../../package.json'; - -export function getCurrentVersionAsInteger(): number { - // break up the string parts - const splitted = xPackage.version.split('.'); - - // pad each part with leading 0 to make 2 characters - const padded = splitted.map((v: string) => { - const vMatches = v.match(/\d+/); - if (vMatches) { - return padLeft(vMatches[0], 2, '0'); - } - return '00'; - }); - return parseInt(padded.join(''), 10); -} From c7209d0273b1d23083d006d35e9e98a2d4a5dde0 Mon Sep 17 00:00:00 2001 From: Mike Cote Date: Mon, 24 Feb 2020 09:47:54 -0500 Subject: [PATCH 5/5] Share ILM policy across versions --- x-pack/plugins/event_log/server/es/documents.test.ts | 2 +- x-pack/plugins/event_log/server/es/documents.ts | 2 +- x-pack/plugins/event_log/server/es/names.mock.ts | 3 ++- x-pack/plugins/event_log/server/es/names.test.ts | 3 ++- x-pack/plugins/event_log/server/es/names.ts | 4 +++- 5 files changed, 9 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/event_log/server/es/documents.test.ts b/x-pack/plugins/event_log/server/es/documents.test.ts index 1a71500b66624..c08d0ac978bc9 100644 --- a/x-pack/plugins/event_log/server/es/documents.test.ts +++ b/x-pack/plugins/event_log/server/es/documents.test.ts @@ -22,7 +22,7 @@ describe('getIndexTemplate()', () => { test('returns the correct details of the index template', () => { const indexTemplate = getIndexTemplate(esNames); - expect(indexTemplate.index_patterns).toEqual([esNames.indexPattern]); + expect(indexTemplate.index_patterns).toEqual([esNames.indexPatternWithVersion]); expect(indexTemplate.settings.number_of_shards).toBeGreaterThanOrEqual(0); expect(indexTemplate.settings.number_of_replicas).toBeGreaterThanOrEqual(0); expect(indexTemplate.settings['index.lifecycle.name']).toBe(esNames.ilmPolicy); diff --git a/x-pack/plugins/event_log/server/es/documents.ts b/x-pack/plugins/event_log/server/es/documents.ts index 78ec9f653f5e6..982454e671008 100644 --- a/x-pack/plugins/event_log/server/es/documents.ts +++ b/x-pack/plugins/event_log/server/es/documents.ts @@ -10,7 +10,7 @@ import mappings from '../../generated/mappings.json'; // returns the body of an index template used in an ES indices.putTemplate call export function getIndexTemplate(esNames: EsNames) { const indexTemplateBody: any = { - index_patterns: [esNames.indexPattern], + index_patterns: [esNames.indexPatternWithVersion], settings: { number_of_shards: 1, number_of_replicas: 1, diff --git a/x-pack/plugins/event_log/server/es/names.mock.ts b/x-pack/plugins/event_log/server/es/names.mock.ts index 74d8dfafef34c..268421235b4b2 100644 --- a/x-pack/plugins/event_log/server/es/names.mock.ts +++ b/x-pack/plugins/event_log/server/es/names.mock.ts @@ -10,8 +10,9 @@ const createNamesMock = () => { const mock: jest.Mocked = { base: '.kibana', alias: '.kibana-event-log-8.0.0', - ilmPolicy: '.kibana-event-log-8.0.0-policy', + ilmPolicy: '.kibana-event-log-policy', indexPattern: '.kibana-event-log-*', + indexPatternWithVersion: '.kibana-event-log-8.0.0-*', initialIndex: '.kibana-event-log-8.0.0-000001', indexTemplate: '.kibana-event-log-8.0.0-template', }; diff --git a/x-pack/plugins/event_log/server/es/names.test.ts b/x-pack/plugins/event_log/server/es/names.test.ts index 3f811d9fc4cb0..baefd756bb1ed 100644 --- a/x-pack/plugins/event_log/server/es/names.test.ts +++ b/x-pack/plugins/event_log/server/es/names.test.ts @@ -17,8 +17,9 @@ describe('getEsNames()', () => { const esNames = getEsNames(base); expect(esNames.base).toEqual(base); expect(esNames.alias).toEqual(`${base}-event-log-${version}`); - expect(esNames.ilmPolicy).toEqual(`${base}-event-log-${version}-policy`); + expect(esNames.ilmPolicy).toEqual(`${base}-event-log-policy`); expect(esNames.indexPattern).toEqual(`${base}-event-log-*`); + expect(esNames.indexPatternWithVersion).toEqual(`${base}-event-log-${version}-*`); expect(esNames.initialIndex).toEqual(`${base}-event-log-${version}-000001`); expect(esNames.indexTemplate).toEqual(`${base}-event-log-${version}-template`); }); diff --git a/x-pack/plugins/event_log/server/es/names.ts b/x-pack/plugins/event_log/server/es/names.ts index 958da8e6016a8..d55d02a16fc9a 100644 --- a/x-pack/plugins/event_log/server/es/names.ts +++ b/x-pack/plugins/event_log/server/es/names.ts @@ -14,6 +14,7 @@ export interface EsNames { alias: string; ilmPolicy: string; indexPattern: string; + indexPatternWithVersion: string; initialIndex: string; indexTemplate: string; } @@ -24,8 +25,9 @@ export function getEsNames(baseName: string): EsNames { return { base: baseName, alias: eventLogNameWithVersion, - ilmPolicy: `${eventLogNameWithVersion}-policy`, + ilmPolicy: `${eventLogName}-policy`, indexPattern: `${eventLogName}-*`, + indexPatternWithVersion: `${eventLogNameWithVersion}-*`, initialIndex: `${eventLogNameWithVersion}-000001`, indexTemplate: `${eventLogNameWithVersion}-template`, };