From 551bd14c861a00a41c195b37ed6f07d9e5fb3bb6 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner <56361221+jonathan-buttner@users.noreply.github.com> Date: Mon, 10 Aug 2020 13:42:48 -0400 Subject: [PATCH] [Ingest Manager] Allow prerelease in package version (#74452) (#74670) * Allow prerelease in version * Adding integration test for prerelease version of a package * Tests for invalid package key * Removing inter-test dependency --- src/dev/precommit_hook/casing_check_config.js | 2 + .../server/routes/epm/handlers.ts | 5 ++- .../server/services/epm/packages/install.ts | 2 +- .../server/services/epm/packages/remove.ts | 3 +- .../services/epm/registry/index.test.ts | 34 ++++++++++++++- .../server/services/epm/registry/index.ts | 21 ++++++++++ .../apis/epm/get.ts | 33 +++++++++++++++ .../apis/epm/index.js | 2 + .../apis/epm/install_overrides.ts | 6 +-- .../apis/epm/install_prerelease.ts | 41 +++++++++++++++++++ .../dataset/test/fields/fields.yml | 16 ++++++++ .../0.1.0-dev.0+abc/dataset/test/manifest.yml | 3 ++ .../prerelease/0.1.0-dev.0+abc/docs/README.md | 3 ++ .../img/logo_prerelease_64_color.svg | 7 ++++ .../prerelease/0.1.0-dev.0+abc/manifest.yml | 20 +++++++++ 15 files changed, 190 insertions(+), 8 deletions(-) create mode 100644 x-pack/test/ingest_manager_api_integration/apis/epm/get.ts create mode 100644 x-pack/test/ingest_manager_api_integration/apis/epm/install_prerelease.ts create mode 100644 x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/prerelease/0.1.0-dev.0+abc/dataset/test/fields/fields.yml create mode 100644 x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/prerelease/0.1.0-dev.0+abc/dataset/test/manifest.yml create mode 100644 x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/prerelease/0.1.0-dev.0+abc/docs/README.md create mode 100644 x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/prerelease/0.1.0-dev.0+abc/img/logo_prerelease_64_color.svg create mode 100644 x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/prerelease/0.1.0-dev.0+abc/manifest.yml diff --git a/src/dev/precommit_hook/casing_check_config.js b/src/dev/precommit_hook/casing_check_config.js index 87f57b60621f4..9eb29261a4ea9 100644 --- a/src/dev/precommit_hook/casing_check_config.js +++ b/src/dev/precommit_hook/casing_check_config.js @@ -70,6 +70,8 @@ export const IGNORE_FILE_GLOBS = [ 'x-pack/plugins/apm/e2e/**/*', 'x-pack/plugins/maps/server/fonts/**/*', + // packages for the ingest manager's api integration tests could be valid semver which has dashes + 'x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/**/*', ]; /** diff --git a/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts b/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts index f8f39f6294260..bbd91b97a86c9 100644 --- a/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts +++ b/x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts @@ -33,6 +33,7 @@ import { getInstallationObject, } from '../../services/epm/packages'; import { IngestManagerError, getHTTPResponseCode } from '../../errors'; +import { splitPkgKey } from '../../services/epm/registry'; export const getCategoriesHandler: RequestHandler< undefined, @@ -131,7 +132,7 @@ export const getInfoHandler: RequestHandler { // TODO: change epm API to /packageName/version so we don't need to do this - const [pkgName, pkgVersion] = pkgkey.split('-'); + const { pkgName, pkgVersion } = Registry.splitPkgKey(pkgkey); // TODO: calls to getInstallationObject, Registry.fetchInfo, and Registry.fetchFindLatestPackge // and be replaced by getPackageInfo after adjusting for it to not group/use archive assets const latestPackage = await Registry.fetchFindLatestPackage(pkgName); diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/remove.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/remove.ts index 1acf2131dcb01..1e50c67d63c42 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/remove.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/remove.ts @@ -13,6 +13,7 @@ import { getInstallation, savedObjectTypes } from './index'; import { deletePipeline } from '../elasticsearch/ingest_pipeline/'; import { installIndexPatterns } from '../kibana/index_pattern/install'; import { packageConfigService, appContextService } from '../..'; +import { splitPkgKey } from '../registry'; export async function removeInstallation(options: { savedObjectsClient: SavedObjectsClientContract; @@ -21,7 +22,7 @@ export async function removeInstallation(options: { }): Promise { const { savedObjectsClient, pkgkey, callCluster } = options; // TODO: the epm api should change to /name/version so we don't need to do this - const [pkgName] = pkgkey.split('-'); + const { pkgName } = splitPkgKey(pkgkey); const installation = await getInstallation({ savedObjectsClient, pkgName }); if (!installation) throw Boom.badRequest(`${pkgName} is not installed`); if (installation.removable === false) diff --git a/x-pack/plugins/ingest_manager/server/services/epm/registry/index.test.ts b/x-pack/plugins/ingest_manager/server/services/epm/registry/index.test.ts index eae84275a49b9..085dc990fa376 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/registry/index.test.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/registry/index.test.ts @@ -5,7 +5,7 @@ */ import { AssetParts } from '../../../types'; -import { pathParts } from './index'; +import { pathParts, splitPkgKey } from './index'; const testPaths = [ { @@ -48,3 +48,35 @@ test('testPathParts', () => { expect(pathParts(value.path)).toStrictEqual(value.assetParts as AssetParts); } }); + +describe('splitPkgKey tests', () => { + it('throws an error if the delimiter is not found', () => { + expect(() => { + splitPkgKey('awesome_package'); + }).toThrow(); + }); + + it('throws an error if there is nothing before the delimiter', () => { + expect(() => { + splitPkgKey('-0.0.1-dev1'); + }).toThrow(); + }); + + it('throws an error if the version is not a semver', () => { + expect(() => { + splitPkgKey('awesome-laskdfj'); + }).toThrow(); + }); + + it('returns the name and version if the delimiter is found once', () => { + const { pkgName, pkgVersion } = splitPkgKey('awesome-0.1.0'); + expect(pkgName).toBe('awesome'); + expect(pkgVersion).toBe('0.1.0'); + }); + + it('returns the name and version if the delimiter is found multiple times', () => { + const { pkgName, pkgVersion } = splitPkgKey('endpoint-0.13.0-alpha.1+abcd'); + expect(pkgName).toBe('endpoint'); + expect(pkgVersion).toBe('0.13.0-alpha.1+abcd'); + }); +}); diff --git a/x-pack/plugins/ingest_manager/server/services/epm/registry/index.ts b/x-pack/plugins/ingest_manager/server/services/epm/registry/index.ts index c701762e50b50..b635378960468 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/registry/index.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/registry/index.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import semver from 'semver'; import { Response } from 'node-fetch'; import { URL } from 'url'; import { @@ -35,6 +36,26 @@ export interface CategoriesParams { experimental?: boolean; } +/** + * Extract the package name and package version from a string. + * + * @param pkgkey a string containing the package name delimited by the package version + */ +export function splitPkgKey(pkgkey: string): { pkgName: string; pkgVersion: string } { + // this will return an empty string if `indexOf` returns -1 + const pkgName = pkgkey.substr(0, pkgkey.indexOf('-')); + if (pkgName === '') { + throw new Error('Package key parsing failed: package name was empty'); + } + + // this will return the entire string if `indexOf` return -1 + const pkgVersion = pkgkey.substr(pkgkey.indexOf('-') + 1); + if (!semver.valid(pkgVersion)) { + throw new Error('Package key parsing failed: package version was not a valid semver'); + } + return { pkgName, pkgVersion }; +} + export const pkgToPkgKey = ({ name, version }: { name: string; version: string }) => `${name}-${version}`; diff --git a/x-pack/test/ingest_manager_api_integration/apis/epm/get.ts b/x-pack/test/ingest_manager_api_integration/apis/epm/get.ts new file mode 100644 index 0000000000000..382bd6beb2e2f --- /dev/null +++ b/x-pack/test/ingest_manager_api_integration/apis/epm/get.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; +import { warnAndSkipTest } from '../../helpers'; + +export default function ({ getService }: FtrProviderContext) { + const log = getService('log'); + const supertest = getService('supertest'); + const dockerServers = getService('dockerServers'); + const server = dockerServers.get('registry'); + + describe('EPM - get', () => { + it('returns a 500 for a package key without a proper name', async function () { + if (server.enabled) { + await supertest.get('/api/ingest_manager/epm/packages/-0.1.0').expect(500); + } else { + warnAndSkipTest(this, log); + } + }); + + it('returns a 500 for a package key without a proper semver version', async function () { + if (server.enabled) { + await supertest.get('/api/ingest_manager/epm/packages/endpoint-0.1.0.1.2.3').expect(500); + } else { + warnAndSkipTest(this, log); + } + }); + }); +} diff --git a/x-pack/test/ingest_manager_api_integration/apis/epm/index.js b/x-pack/test/ingest_manager_api_integration/apis/epm/index.js index 1582f72dd1cd8..8e1ae67b02741 100644 --- a/x-pack/test/ingest_manager_api_integration/apis/epm/index.js +++ b/x-pack/test/ingest_manager_api_integration/apis/epm/index.js @@ -7,10 +7,12 @@ export default function loadTests({ loadTestFile }) { describe('EPM Endpoints', () => { loadTestFile(require.resolve('./list')); + loadTestFile(require.resolve('./get')); loadTestFile(require.resolve('./file')); //loadTestFile(require.resolve('./template')); loadTestFile(require.resolve('./ilm')); loadTestFile(require.resolve('./install_overrides')); + loadTestFile(require.resolve('./install_prerelease')); loadTestFile(require.resolve('./install_remove_assets')); loadTestFile(require.resolve('./install_update')); }); diff --git a/x-pack/test/ingest_manager_api_integration/apis/epm/install_overrides.ts b/x-pack/test/ingest_manager_api_integration/apis/epm/install_overrides.ts index f73ba56c172c4..c75c51f6a5000 100644 --- a/x-pack/test/ingest_manager_api_integration/apis/epm/install_overrides.ts +++ b/x-pack/test/ingest_manager_api_integration/apis/epm/install_overrides.ts @@ -14,13 +14,13 @@ export default function ({ getService }: FtrProviderContext) { const dockerServers = getService('dockerServers'); const log = getService('log'); + const mappingsPackage = 'overrides-0.1.0'; + const server = dockerServers.get('registry'); + const deletePackage = async (pkgkey: string) => { await supertest.delete(`/api/ingest_manager/epm/packages/${pkgkey}`).set('kbn-xsrf', 'xxxx'); }; - const mappingsPackage = 'overrides-0.1.0'; - const server = dockerServers.get('registry'); - describe('installs packages that include settings and mappings overrides', async () => { after(async () => { if (server.enabled) { diff --git a/x-pack/test/ingest_manager_api_integration/apis/epm/install_prerelease.ts b/x-pack/test/ingest_manager_api_integration/apis/epm/install_prerelease.ts new file mode 100644 index 0000000000000..a641a105c66e0 --- /dev/null +++ b/x-pack/test/ingest_manager_api_integration/apis/epm/install_prerelease.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; +import { warnAndSkipTest } from '../../helpers'; + +export default function ({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const dockerServers = getService('dockerServers'); + const log = getService('log'); + + const testPackage = 'prerelease-0.1.0-dev.0+abc'; + const server = dockerServers.get('registry'); + + const deletePackage = async (pkgkey: string) => { + await supertest.delete(`/api/ingest_manager/epm/packages/${pkgkey}`).set('kbn-xsrf', 'xxxx'); + }; + + describe('installs package that has a prerelease version', async () => { + after(async () => { + if (server.enabled) { + // remove the package just in case it being installed will affect other tests + await deletePackage(testPackage); + } + }); + + it('should install the package correctly', async function () { + if (server.enabled) { + await supertest + .post(`/api/ingest_manager/epm/packages/${testPackage}`) + .set('kbn-xsrf', 'xxxx') + .expect(200); + } else { + warnAndSkipTest(this, log); + } + }); + }); +} diff --git a/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/prerelease/0.1.0-dev.0+abc/dataset/test/fields/fields.yml b/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/prerelease/0.1.0-dev.0+abc/dataset/test/fields/fields.yml new file mode 100644 index 0000000000000..6e003ed0ad147 --- /dev/null +++ b/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/prerelease/0.1.0-dev.0+abc/dataset/test/fields/fields.yml @@ -0,0 +1,16 @@ +- name: data_stream.type + type: constant_keyword + description: > + Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: > + Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: > + Data stream namespace. +- name: '@timestamp' + type: date + description: > + Event timestamp. diff --git a/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/prerelease/0.1.0-dev.0+abc/dataset/test/manifest.yml b/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/prerelease/0.1.0-dev.0+abc/dataset/test/manifest.yml new file mode 100644 index 0000000000000..17c33c745ce74 --- /dev/null +++ b/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/prerelease/0.1.0-dev.0+abc/dataset/test/manifest.yml @@ -0,0 +1,3 @@ +title: Test Dataset + +type: logs diff --git a/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/prerelease/0.1.0-dev.0+abc/docs/README.md b/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/prerelease/0.1.0-dev.0+abc/docs/README.md new file mode 100644 index 0000000000000..0002afd9cdfc0 --- /dev/null +++ b/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/prerelease/0.1.0-dev.0+abc/docs/README.md @@ -0,0 +1,3 @@ +# Test package + +For testing a prerelease package diff --git a/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/prerelease/0.1.0-dev.0+abc/img/logo_prerelease_64_color.svg b/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/prerelease/0.1.0-dev.0+abc/img/logo_prerelease_64_color.svg new file mode 100644 index 0000000000000..b03007a76ffcc --- /dev/null +++ b/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/prerelease/0.1.0-dev.0+abc/img/logo_prerelease_64_color.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/prerelease/0.1.0-dev.0+abc/manifest.yml b/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/prerelease/0.1.0-dev.0+abc/manifest.yml new file mode 100644 index 0000000000000..a0adb184cfc59 --- /dev/null +++ b/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/prerelease/0.1.0-dev.0+abc/manifest.yml @@ -0,0 +1,20 @@ +format_version: 1.0.0 +name: prerelease +title: Prerelease package +description: This is a test package for testing that parsing a prerelease version works +version: 0.1.0-dev.0+abc +categories: ['security'] +release: beta +type: integration +license: basic + +requirement: + elasticsearch: + versions: '>7.7.0' + kibana: + versions: '>7.7.0' + +icons: + - src: '/img/logo_prerelease_64_color.svg' + size: '16x16' + type: 'image/svg+xml'