From d2829daaaca461a80f6391638e08afbd00148a13 Mon Sep 17 00:00:00 2001 From: Spencer Date: Tue, 4 Aug 2020 15:34:01 -0700 Subject: [PATCH] [src/dev/build] build Kibana Platform bundles from source (#73591) Co-authored-by: spalger --- .../serializers/absolute_path_serializer.ts | 7 +++-- .../kbn-optimizer/src/common/bundle_cache.ts | 14 +++++++++ .../src/optimizer/get_plugin_bundles.test.ts | 31 ++++++++++--------- .../src/optimizer/get_plugin_bundles.ts | 12 +++++-- .../src/optimizer/optimizer_config.test.ts | 31 +++++++++++++++---- .../src/optimizer/optimizer_config.ts | 20 ++++++++++-- .../tasks/build_kibana_platform_plugins.ts | 18 +++++------ src/dev/build/tasks/copy_source_task.ts | 2 +- vars/kibanaPipeline.groovy | 1 + 9 files changed, 99 insertions(+), 37 deletions(-) diff --git a/packages/kbn-dev-utils/src/serializers/absolute_path_serializer.ts b/packages/kbn-dev-utils/src/serializers/absolute_path_serializer.ts index 884614c8b9551..4008cf852c3a8 100644 --- a/packages/kbn-dev-utils/src/serializers/absolute_path_serializer.ts +++ b/packages/kbn-dev-utils/src/serializers/absolute_path_serializer.ts @@ -19,9 +19,12 @@ import { REPO_ROOT } from '../repo_root'; -export function createAbsolutePathSerializer(rootPath: string = REPO_ROOT) { +export function createAbsolutePathSerializer( + rootPath: string = REPO_ROOT, + replacement = '' +) { return { test: (value: any) => typeof value === 'string' && value.startsWith(rootPath), - serialize: (value: string) => value.replace(rootPath, '').replace(/\\/g, '/'), + serialize: (value: string) => value.replace(rootPath, replacement).replace(/\\/g, '/'), }; } diff --git a/packages/kbn-optimizer/src/common/bundle_cache.ts b/packages/kbn-optimizer/src/common/bundle_cache.ts index 7607e270b5b4f..578108fce51fa 100644 --- a/packages/kbn-optimizer/src/common/bundle_cache.ts +++ b/packages/kbn-optimizer/src/common/bundle_cache.ts @@ -104,4 +104,18 @@ export class BundleCache { public getOptimizerCacheKey() { return this.get().optimizerCacheKey; } + + public clear() { + this.state = undefined; + + if (this.path) { + try { + Fs.unlinkSync(this.path); + } catch (error) { + if (error.code !== 'ENOENT') { + throw error; + } + } + } + } } diff --git a/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.test.ts b/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.test.ts index a823f66cf767b..702ad16144e7b 100644 --- a/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.test.ts +++ b/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.test.ts @@ -21,7 +21,9 @@ import { createAbsolutePathSerializer } from '@kbn/dev-utils'; import { getPluginBundles } from './get_plugin_bundles'; -expect.addSnapshotSerializer(createAbsolutePathSerializer('/repo')); +expect.addSnapshotSerializer(createAbsolutePathSerializer('/repo', '')); +expect.addSnapshotSerializer(createAbsolutePathSerializer('/output', '')); +expect.addSnapshotSerializer(createAbsolutePathSerializer('/outside/of/repo', '')); it('returns a bundle for core and each plugin', () => { expect( @@ -56,46 +58,47 @@ it('returns a bundle for core and each plugin', () => { manifestPath: '/repo/x-pack/plugins/box/kibana.json', }, ], - '/repo' + '/repo', + '/output' ).map((b) => b.toSpec()) ).toMatchInlineSnapshot(` Array [ Object { "banner": undefined, - "contextDir": /plugins/foo, + "contextDir": /plugins/foo, "id": "foo", - "manifestPath": /plugins/foo/kibana.json, - "outputDir": /plugins/foo/target/public, + "manifestPath": /plugins/foo/kibana.json, + "outputDir": /plugins/foo/target/public, "publicDirNames": Array [ "public", ], - "sourceRoot": , + "sourceRoot": , "type": "plugin", }, Object { "banner": undefined, - "contextDir": "/outside/of/repo/plugins/baz", + "contextDir": /plugins/baz, "id": "baz", - "manifestPath": "/outside/of/repo/plugins/baz/kibana.json", - "outputDir": "/outside/of/repo/plugins/baz/target/public", + "manifestPath": /plugins/baz/kibana.json, + "outputDir": /plugins/baz/target/public, "publicDirNames": Array [ "public", ], - "sourceRoot": , + "sourceRoot": , "type": "plugin", }, Object { "banner": "/*! Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one or more contributor license agreements. * Licensed under the Elastic License; you may not use this file except in compliance with the Elastic License. */ ", - "contextDir": /x-pack/plugins/box, + "contextDir": /x-pack/plugins/box, "id": "box", - "manifestPath": /x-pack/plugins/box/kibana.json, - "outputDir": /x-pack/plugins/box/target/public, + "manifestPath": /x-pack/plugins/box/kibana.json, + "outputDir": /x-pack/plugins/box/target/public, "publicDirNames": Array [ "public", ], - "sourceRoot": , + "sourceRoot": , "type": "plugin", }, ] diff --git a/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts b/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts index 9350b9464242a..d2d19dcd87cca 100644 --- a/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts +++ b/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts @@ -23,7 +23,11 @@ import { Bundle } from '../common'; import { KibanaPlatformPlugin } from './kibana_platform_plugins'; -export function getPluginBundles(plugins: KibanaPlatformPlugin[], repoRoot: string) { +export function getPluginBundles( + plugins: KibanaPlatformPlugin[], + repoRoot: string, + outputRoot: string +) { const xpackDirSlash = Path.resolve(repoRoot, 'x-pack') + Path.sep; return plugins @@ -36,7 +40,11 @@ export function getPluginBundles(plugins: KibanaPlatformPlugin[], repoRoot: stri publicDirNames: ['public', ...p.extraPublicDirs], sourceRoot: repoRoot, contextDir: p.directory, - outputDir: Path.resolve(p.directory, 'target/public'), + outputDir: Path.resolve( + outputRoot, + Path.relative(repoRoot, p.directory), + 'target/public' + ), manifestPath: p.manifestPath, banner: p.directory.startsWith(xpackDirSlash) ? `/*! Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one or more contributor license agreements.\n` + diff --git a/packages/kbn-optimizer/src/optimizer/optimizer_config.test.ts b/packages/kbn-optimizer/src/optimizer/optimizer_config.test.ts index f97646e2bbbd3..afc2dc8952c87 100644 --- a/packages/kbn-optimizer/src/optimizer/optimizer_config.test.ts +++ b/packages/kbn-optimizer/src/optimizer/optimizer_config.test.ts @@ -23,16 +23,20 @@ jest.mock('./get_plugin_bundles.ts'); jest.mock('../common/theme_tags.ts'); jest.mock('./filter_by_id.ts'); -import Path from 'path'; -import Os from 'os'; +jest.mock('os', () => { + const realOs = jest.requireActual('os'); + jest.spyOn(realOs, 'cpus').mockImplementation(() => { + return ['foo'] as any; + }); + return realOs; +}); +import Path from 'path'; import { REPO_ROOT, createAbsolutePathSerializer } from '@kbn/dev-utils'; -import { OptimizerConfig } from './optimizer_config'; +import { OptimizerConfig, ParsedOptions } from './optimizer_config'; import { parseThemeTags } from '../common'; -jest.spyOn(Os, 'cpus').mockReturnValue(['foo'] as any); - expect.addSnapshotSerializer(createAbsolutePathSerializer()); beforeEach(() => { @@ -118,6 +122,7 @@ describe('OptimizerConfig::parseOptions()', () => { "includeCoreBundle": false, "inspectWorkers": false, "maxWorkerCount": 2, + "outputRoot": , "pluginPaths": Array [], "pluginScanDirs": Array [ /src/plugins, @@ -145,6 +150,7 @@ describe('OptimizerConfig::parseOptions()', () => { "includeCoreBundle": false, "inspectWorkers": false, "maxWorkerCount": 2, + "outputRoot": , "pluginPaths": Array [], "pluginScanDirs": Array [ /src/plugins, @@ -172,6 +178,7 @@ describe('OptimizerConfig::parseOptions()', () => { "includeCoreBundle": false, "inspectWorkers": false, "maxWorkerCount": 2, + "outputRoot": , "pluginPaths": Array [], "pluginScanDirs": Array [ /src/plugins, @@ -201,6 +208,7 @@ describe('OptimizerConfig::parseOptions()', () => { "includeCoreBundle": false, "inspectWorkers": false, "maxWorkerCount": 2, + "outputRoot": , "pluginPaths": Array [], "pluginScanDirs": Array [ /src/plugins, @@ -227,6 +235,7 @@ describe('OptimizerConfig::parseOptions()', () => { "includeCoreBundle": false, "inspectWorkers": false, "maxWorkerCount": 2, + "outputRoot": , "pluginPaths": Array [], "pluginScanDirs": Array [ /x/y/z, @@ -253,6 +262,7 @@ describe('OptimizerConfig::parseOptions()', () => { "includeCoreBundle": false, "inspectWorkers": false, "maxWorkerCount": 100, + "outputRoot": , "pluginPaths": Array [], "pluginScanDirs": Array [], "profileWebpack": false, @@ -276,6 +286,7 @@ describe('OptimizerConfig::parseOptions()', () => { "includeCoreBundle": false, "inspectWorkers": false, "maxWorkerCount": 100, + "outputRoot": , "pluginPaths": Array [], "pluginScanDirs": Array [], "profileWebpack": false, @@ -299,6 +310,7 @@ describe('OptimizerConfig::parseOptions()', () => { "includeCoreBundle": false, "inspectWorkers": false, "maxWorkerCount": 100, + "outputRoot": , "pluginPaths": Array [], "pluginScanDirs": Array [], "profileWebpack": false, @@ -323,6 +335,7 @@ describe('OptimizerConfig::parseOptions()', () => { "includeCoreBundle": false, "inspectWorkers": false, "maxWorkerCount": 100, + "outputRoot": , "pluginPaths": Array [], "pluginScanDirs": Array [], "profileWebpack": false, @@ -347,6 +360,7 @@ describe('OptimizerConfig::parseOptions()', () => { "includeCoreBundle": false, "inspectWorkers": false, "maxWorkerCount": 100, + "outputRoot": , "pluginPaths": Array [], "pluginScanDirs": Array [], "profileWebpack": false, @@ -384,18 +398,22 @@ describe('OptimizerConfig::create()', () => { getPluginBundles.mockReturnValue([Symbol('bundle1'), Symbol('bundle2')]); filterById.mockReturnValue(Symbol('filtered bundles')); - jest.spyOn(OptimizerConfig, 'parseOptions').mockImplementation((): any => ({ + jest.spyOn(OptimizerConfig, 'parseOptions').mockImplementation((): { + [key in keyof ParsedOptions]: any; + } => ({ cache: Symbol('parsed cache'), dist: Symbol('parsed dist'), maxWorkerCount: Symbol('parsed max worker count'), pluginPaths: Symbol('parsed plugin paths'), pluginScanDirs: Symbol('parsed plugin scan dirs'), repoRoot: Symbol('parsed repo root'), + outputRoot: Symbol('parsed output root'), watch: Symbol('parsed watch'), themeTags: Symbol('theme tags'), inspectWorkers: Symbol('parsed inspect workers'), profileWebpack: Symbol('parsed profile webpack'), filters: [], + includeCoreBundle: false, })); }); @@ -474,6 +492,7 @@ describe('OptimizerConfig::create()', () => { Array [ Symbol(new platform plugins), Symbol(parsed repo root), + Symbol(parsed output root), ], ], "instances": Array [ diff --git a/packages/kbn-optimizer/src/optimizer/optimizer_config.ts b/packages/kbn-optimizer/src/optimizer/optimizer_config.ts index 0e588ab36238b..45598ff8831b0 100644 --- a/packages/kbn-optimizer/src/optimizer/optimizer_config.ts +++ b/packages/kbn-optimizer/src/optimizer/optimizer_config.ts @@ -55,6 +55,13 @@ function omit(obj: T, keys: K[]): Omit { interface Options { /** absolute path to root of the repo/build */ repoRoot: string; + /** + * absolute path to the root directory where output should be written to. This + * defaults to the repoRoot but can be customized to write output somewhere else. + * + * This is how we write output to the build directory in the Kibana build tasks. + */ + outputRoot?: string; /** enable to run the optimizer in watch mode */ watch?: boolean; /** the maximum number of workers that will be created */ @@ -107,8 +114,9 @@ interface Options { themes?: ThemeTag | '*' | ThemeTag[]; } -interface ParsedOptions { +export interface ParsedOptions { repoRoot: string; + outputRoot: string; watch: boolean; maxWorkerCount: number; profileWebpack: boolean; @@ -139,6 +147,11 @@ export class OptimizerConfig { throw new TypeError('repoRoot must be an absolute path'); } + const outputRoot = options.outputRoot ?? repoRoot; + if (!Path.isAbsolute(outputRoot)) { + throw new TypeError('outputRoot must be an absolute path'); + } + /** * BEWARE: this needs to stay roughly synchronized with * `src/core/server/config/env.ts` which determines which paths @@ -182,6 +195,7 @@ export class OptimizerConfig { watch, dist, repoRoot, + outputRoot, maxWorkerCount, profileWebpack, cache, @@ -206,11 +220,11 @@ export class OptimizerConfig { publicDirNames: ['public', 'public/utils'], sourceRoot: options.repoRoot, contextDir: Path.resolve(options.repoRoot, 'src/core'), - outputDir: Path.resolve(options.repoRoot, 'src/core/target/public'), + outputDir: Path.resolve(options.outputRoot, 'src/core/target/public'), }), ] : []), - ...getPluginBundles(plugins, options.repoRoot), + ...getPluginBundles(plugins, options.repoRoot, options.outputRoot), ]; return new OptimizerConfig( diff --git a/src/dev/build/tasks/build_kibana_platform_plugins.ts b/src/dev/build/tasks/build_kibana_platform_plugins.ts index beb5ad40229df..48625078e9bd1 100644 --- a/src/dev/build/tasks/build_kibana_platform_plugins.ts +++ b/src/dev/build/tasks/build_kibana_platform_plugins.ts @@ -17,7 +17,7 @@ * under the License. */ -import { CiStatsReporter } from '@kbn/dev-utils'; +import { CiStatsReporter, REPO_ROOT } from '@kbn/dev-utils'; import { runOptimizer, OptimizerConfig, @@ -29,9 +29,10 @@ import { Task } from '../lib'; export const BuildKibanaPlatformPlugins: Task = { description: 'Building distributable versions of Kibana platform plugins', - async run(config, log, build) { - const optimizerConfig = OptimizerConfig.create({ - repoRoot: build.resolvePath(), + async run(_, log, build) { + const config = OptimizerConfig.create({ + repoRoot: REPO_ROOT, + outputRoot: build.resolvePath(), cache: false, oss: build.isOss(), examples: false, @@ -42,11 +43,10 @@ export const BuildKibanaPlatformPlugins: Task = { const reporter = CiStatsReporter.fromEnv(log); - await runOptimizer(optimizerConfig) - .pipe( - reportOptimizerStats(reporter, optimizerConfig, log), - logOptimizerState(log, optimizerConfig) - ) + await runOptimizer(config) + .pipe(reportOptimizerStats(reporter, config, log), logOptimizerState(log, config)) .toPromise(); + + await Promise.all(config.bundles.map((b) => b.cache.clear())); }, }; diff --git a/src/dev/build/tasks/copy_source_task.ts b/src/dev/build/tasks/copy_source_task.ts index c8489673b83af..79279997671e5 100644 --- a/src/dev/build/tasks/copy_source_task.ts +++ b/src/dev/build/tasks/copy_source_task.ts @@ -30,7 +30,7 @@ export const CopySource: Task = { 'src/**', '!src/**/*.{test,test.mocks,mock}.{js,ts,tsx}', '!src/**/mocks.ts', // special file who imports .mock files - '!src/**/{__tests__,__snapshots__,__mocks__}/**', + '!src/**/{target,__tests__,__snapshots__,__mocks__}/**', '!src/test_utils/**', '!src/fixtures/**', '!src/legacy/core_plugins/console/public/tests/**', diff --git a/vars/kibanaPipeline.groovy b/vars/kibanaPipeline.groovy index 0f1e11a1fb70a..2ff7dca085d31 100644 --- a/vars/kibanaPipeline.groovy +++ b/vars/kibanaPipeline.groovy @@ -108,6 +108,7 @@ def uploadGcsArtifact(uploadPrefix, pattern) { def withGcsArtifactUpload(workerName, closure) { def uploadPrefix = "kibana-ci-artifacts/jobs/${env.JOB_NAME}/${BUILD_NUMBER}/${workerName}" def ARTIFACT_PATTERNS = [ + '**/target/public/.kbn-optimizer-cache', 'target/kibana-*', 'target/kibana-security-solution/**/*.png', 'target/junit/**/*',