[7.x] [kbn/optimizer] implement "requiredBundles" property of KP plug…

…ins (#70911) (#71355)

Co-authored-by: Josh Dover <me@joshdover.com>
Co-authored-by: spalger <spalger@users.noreply.github.com>
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
# Conflicts:
#	packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap
#	src/plugins/embeddable/kibana.json
#	src/plugins/telemetry/kibana.json
#	src/plugins/ui_actions/kibana.json
#	x-pack/plugins/apm/kibana.json
#	x-pack/plugins/upgrade_assistant/kibana.json

docs/development/core/server/kibana-plugin-core-server.discoveredplugin.md

@@ -19,5 +19,6 @@ export interface DiscoveredPlugin
 |  [configPath](./kibana-plugin-core-server.discoveredplugin.configpath.md) | <code>ConfigPath</code> | Root configuration path used by the plugin, defaults to "id" in snake\_case format. |
 |  [id](./kibana-plugin-core-server.discoveredplugin.id.md) | <code>PluginName</code> | Identifier of the plugin. |
 |  [optionalPlugins](./kibana-plugin-core-server.discoveredplugin.optionalplugins.md) | <code>readonly PluginName[]</code> | An optional list of the other plugins that if installed and enabled \*\*may be\*\* leveraged by this plugin for some additional functionality but otherwise are not required for this plugin to work properly. |
+|  [requiredBundles](./kibana-plugin-core-server.discoveredplugin.requiredbundles.md) | <code>readonly PluginName[]</code> | List of plugin ids that this plugin's UI code imports modules from that are not in <code>requiredPlugins</code>. |
 |  [requiredPlugins](./kibana-plugin-core-server.discoveredplugin.requiredplugins.md) | <code>readonly PluginName[]</code> | An optional list of the other plugins that \*\*must be\*\* installed and enabled for this plugin to function properly. |
 

docs/development/core/server/kibana-plugin-core-server.discoveredplugin.requiredbundles.md

@@ -0,0 +1,18 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [DiscoveredPlugin](./kibana-plugin-core-server.discoveredplugin.md) &gt; [requiredBundles](./kibana-plugin-core-server.discoveredplugin.requiredbundles.md)
+
+## DiscoveredPlugin.requiredBundles property
+
+List of plugin ids that this plugin's UI code imports modules from that are not in `requiredPlugins`<!-- -->.
+
+<b>Signature:</b>
+
+```typescript
+readonly requiredBundles: readonly PluginName[];
+```
+
+## Remarks
+
+The plugins listed here will be loaded in the browser, even if the plugin is disabled. Required by `@kbn/optimizer` to support cross-plugin imports. "core" and plugins already listed in `requiredPlugins` do not need to be duplicated here.
+

docs/development/core/server/kibana-plugin-core-server.pluginmanifest.md

@@ -25,6 +25,7 @@ Should never be used in code outside of Core but is exported for documentation p
 |  [id](./kibana-plugin-core-server.pluginmanifest.id.md) | <code>PluginName</code> | Identifier of the plugin. Must be a string in camelCase. Part of a plugin public contract. Other plugins leverage it to access plugin API, navigate to the plugin, etc. |
 |  [kibanaVersion](./kibana-plugin-core-server.pluginmanifest.kibanaversion.md) | <code>string</code> | The version of Kibana the plugin is compatible with, defaults to "version". |
 |  [optionalPlugins](./kibana-plugin-core-server.pluginmanifest.optionalplugins.md) | <code>readonly PluginName[]</code> | An optional list of the other plugins that if installed and enabled \*\*may be\*\* leveraged by this plugin for some additional functionality but otherwise are not required for this plugin to work properly. |
+|  [requiredBundles](./kibana-plugin-core-server.pluginmanifest.requiredbundles.md) | <code>readonly string[]</code> | List of plugin ids that this plugin's UI code imports modules from that are not in <code>requiredPlugins</code>. |
 |  [requiredPlugins](./kibana-plugin-core-server.pluginmanifest.requiredplugins.md) | <code>readonly PluginName[]</code> | An optional list of the other plugins that \*\*must be\*\* installed and enabled for this plugin to function properly. |
 |  [server](./kibana-plugin-core-server.pluginmanifest.server.md) | <code>boolean</code> | Specifies whether plugin includes some server-side specific functionality. |
 |  [ui](./kibana-plugin-core-server.pluginmanifest.ui.md) | <code>boolean</code> | Specifies whether plugin includes some client/browser specific functionality that should be included into client bundle via <code>public/ui_plugin.js</code> file. |

docs/development/core/server/kibana-plugin-core-server.pluginmanifest.requiredbundles.md

@@ -0,0 +1,18 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [PluginManifest](./kibana-plugin-core-server.pluginmanifest.md) &gt; [requiredBundles](./kibana-plugin-core-server.pluginmanifest.requiredbundles.md)
+
+## PluginManifest.requiredBundles property
+
+List of plugin ids that this plugin's UI code imports modules from that are not in `requiredPlugins`<!-- -->.
+
+<b>Signature:</b>
+
+```typescript
+readonly requiredBundles: readonly string[];
+```
+
+## Remarks
+
+The plugins listed here will be loaded in the browser, even if the plugin is disabled. Required by `@kbn/optimizer` to support cross-plugin imports. "core" and plugins already listed in `requiredPlugins` do not need to be duplicated here.
+

examples/bfetch_explorer/kibana.json

@@ -5,5 +5,6 @@
   "server": true,
   "ui": true,
   "requiredPlugins": ["bfetch", "developerExamples"],
-  "optionalPlugins": []
+  "optionalPlugins": [],
+  "requiredBundles": ["kibanaReact"]
 }

examples/dashboard_embeddable_examples/kibana.json

@@ -5,5 +5,6 @@
   "server": false,
   "ui": true,
   "requiredPlugins": ["embeddable", "embeddableExamples", "dashboard", "developerExamples"],
-  "optionalPlugins": []
+  "optionalPlugins": [],
+  "requiredBundles": ["esUiShared"]
 }

examples/embeddable_examples/kibana.json

@@ -6,5 +6,6 @@
   "ui": true,
   "requiredPlugins": ["embeddable", "uiActions"],
   "optionalPlugins": [],
-  "extraPublicDirs": ["public/todo", "public/hello_world", "public/todo/todo_ref_embeddable"]
+  "extraPublicDirs": ["public/todo", "public/hello_world", "public/todo/todo_ref_embeddable"],
+  "requiredBundles": ["kibanaReact"]
 }

examples/state_containers_examples/kibana.json

@@ -5,5 +5,6 @@
   "server": true,
   "ui": true,
   "requiredPlugins": ["navigation", "data", "developerExamples"],
-  "optionalPlugins": []
+  "optionalPlugins": [],
+  "requiredBundles": ["kibanaUtils", "kibanaReact"]
 }

examples/ui_action_examples/kibana.json

@@ -5,5 +5,6 @@
   "server": false,
   "ui": true,
   "requiredPlugins": ["uiActions"],
-  "optionalPlugins": []
+  "optionalPlugins": [],
+  "requiredBundles": ["kibanaReact"]
 }

examples/ui_actions_explorer/kibana.json

@@ -5,5 +5,6 @@
   "server": false,
   "ui": true,
   "requiredPlugins": ["uiActions", "uiActionsExamples", "developerExamples"],
-  "optionalPlugins": []
+  "optionalPlugins": [],
+  "requiredBundles": ["kibanaReact"]
 }

packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/kibana.json

@@ -1,4 +1,5 @@
 {
   "id": "bar",
-  "ui": true
+  "ui": true,
+  "requiredBundles": ["foo"]
 }

packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/public/legacy/_other_styles.scss

@@ -0,0 +1,3 @@
+p {
+  background-color: rebeccapurple;
+}

packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/public/legacy/styles.scss

@@ -1,3 +1,5 @@
+@import "./other_styles.scss";
+
 body {
   width: $globalStyleConstant;
   background-image: url("ui/icon.svg");

packages/kbn-optimizer/src/cli.ts

@@ -87,6 +87,11 @@ run(
       throw createFlagError('expected --report-stats to have no value');
     }
 
+    const filter = typeof flags.filter === 'string' ? [flags.filter] : flags.filter;
+    if (!Array.isArray(filter) || !filter.every((f) => typeof f === 'string')) {
+      throw createFlagError('expected --filter to be one or more strings');
+    }
+
     const config = OptimizerConfig.create({
       repoRoot: REPO_ROOT,
       watch,
@@ -99,6 +104,7 @@ run(
       extraPluginScanDirs,
       inspectWorkers,
       includeCoreBundle,
+      filter,
     });
 
     let update$ = runOptimizer(config);
@@ -128,12 +134,13 @@ run(
         'inspect-workers',
         'report-stats',
       ],
-      string: ['workers', 'scan-dir'],
+      string: ['workers', 'scan-dir', 'filter'],
       default: {
         core: true,
         examples: true,
         cache: true,
         'inspect-workers': true,
+        filter: [],
       },
       help: `
         --watch            run the optimizer in watch mode
@@ -142,6 +149,7 @@ run(
         --profile          profile the webpack builds and write stats.json files to build outputs
         --no-core          disable generating the core bundle
         --no-cache         disable the cache
+        --filter           comma-separated list of bundle id filters, results from multiple flags are merged, * and ! are supported
         --no-examples      don't build the example plugins
         --dist             create bundles that are suitable for inclusion in the Kibana distributable
         --scan-dir         add a directory to the list of directories scanned for plugins (specify as many times as necessary)

packages/kbn-optimizer/src/common/bundle.test.ts

@@ -50,6 +50,7 @@ it('creates cache keys', () => {
       "spec": Object {
         "contextDir": "/foo/bar",
         "id": "bar",
+        "manifestPath": undefined,
         "outputDir": "/foo/bar/target",
         "publicDirNames": Array [
           "public",
@@ -85,6 +86,7 @@ it('parses bundles from JSON specs', () => {
         },
         "contextDir": "/foo/bar",
         "id": "bar",
+        "manifestPath": undefined,
         "outputDir": "/foo/bar/target",
         "publicDirNames": Array [
           "public",

packages/kbn-optimizer/src/common/bundle.ts

@@ -18,13 +18,19 @@
  */
 
 import Path from 'path';
+import Fs from 'fs';
 
 import { BundleCache } from './bundle_cache';
 import { UnknownVals } from './ts_helpers';
 import { includes, ascending, entriesToObject } from './array_helpers';
 
 const VALID_BUNDLE_TYPES = ['plugin' as const, 'entry' as const];
 
+const DEFAULT_IMPLICIT_BUNDLE_DEPS = ['core'];
+
+const isStringArray = (input: any): input is string[] =>
+  Array.isArray(input) && input.every((x) => typeof x === 'string');
+
 export interface BundleSpec {
   readonly type: typeof VALID_BUNDLE_TYPES[0];
   /** Unique id for this bundle */
@@ -37,6 +43,8 @@ export interface BundleSpec {
   readonly sourceRoot: string;
   /** Absolute path to the directory where output should be written */
   readonly outputDir: string;
+  /** Absolute path to a kibana.json manifest file, if omitted we assume there are not dependenices */
+  readonly manifestPath?: string;
 }
 
 export class Bundle {
@@ -56,6 +64,12 @@ export class Bundle {
   public readonly sourceRoot: BundleSpec['sourceRoot'];
   /** Absolute path to the output directory for this bundle */
   public readonly outputDir: BundleSpec['outputDir'];
+  /**
+   * Absolute path to a manifest file with "requiredBundles" which will be
+   * used to allow bundleRefs from this bundle to the exports of another bundle.
+   * Every bundle mentioned in the `requiredBundles` must be built together.
+   */
+  public readonly manifestPath: BundleSpec['manifestPath'];
 
   public readonly cache: BundleCache;
 
@@ -66,6 +80,7 @@ export class Bundle {
     this.contextDir = spec.contextDir;
     this.sourceRoot = spec.sourceRoot;
     this.outputDir = spec.outputDir;
+    this.manifestPath = spec.manifestPath;
 
     this.cache = new BundleCache(Path.resolve(this.outputDir, '.kbn-optimizer-cache'));
   }
@@ -96,8 +111,54 @@ export class Bundle {
       contextDir: this.contextDir,
       sourceRoot: this.sourceRoot,
       outputDir: this.outputDir,
+      manifestPath: this.manifestPath,
     };
   }
+
+  readBundleDeps(): { implicit: string[]; explicit: string[] } {
+    if (!this.manifestPath) {
+      return {
+        implicit: [...DEFAULT_IMPLICIT_BUNDLE_DEPS],
+        explicit: [],
+      };
+    }
+
+    let json: string;
+    try {
+      json = Fs.readFileSync(this.manifestPath, 'utf8');
+    } catch (error) {
+      if (error.code !== 'ENOENT') {
+        throw error;
+      }
+
+      json = '{}';
+    }
+
+    let parsedManifest: { requiredPlugins?: string[]; requiredBundles?: string[] };
+    try {
+      parsedManifest = JSON.parse(json);
+    } catch (error) {
+      throw new Error(
+        `unable to parse manifest at [${this.manifestPath}], error: [${error.message}]`
+      );
+    }
+
+    if (typeof parsedManifest === 'object' && parsedManifest) {
+      const explicit = parsedManifest.requiredBundles || [];
+      const implicit = [...DEFAULT_IMPLICIT_BUNDLE_DEPS, ...(parsedManifest.requiredPlugins || [])];
+
+      if (isStringArray(explicit) && isStringArray(implicit)) {
+        return {
+          explicit,
+          implicit,
+        };
+      }
+    }
+
+    throw new Error(
+      `Expected "requiredBundles" and "requiredPlugins" in manifest file [${this.manifestPath}] to be arrays of strings`
+    );
+  }
 }
 
 /**
@@ -152,13 +213,21 @@ export function parseBundles(json: string) {
           throw new Error('`bundles[]` must have an absolute path `outputDir` property');
         }
 
+        const { manifestPath } = spec;
+        if (manifestPath !== undefined) {
+          if (!(typeof manifestPath === 'string' && Path.isAbsolute(manifestPath))) {
+            throw new Error('`bundles[]` must have an absolute path `manifestPath` property');
+          }
+        }
+
         return new Bundle({
           type,
           id,
           publicDirNames,
           contextDir,
           sourceRoot,
           outputDir,
+          manifestPath,
         });
       }
     );

packages/kbn-optimizer/src/common/bundle_cache.ts

@@ -24,6 +24,7 @@ export interface State {
   optimizerCacheKey?: unknown;
   cacheKey?: unknown;
   moduleCount?: number;
+  workUnits?: number;
   files?: string[];
   bundleRefExportIds?: string[];
 }
@@ -96,6 +97,10 @@ export class BundleCache {
     return this.get().cacheKey;
   }
 
+  public getWorkUnits() {
+    return this.get().workUnits;
+  }
+
   public getOptimizerCacheKey() {
     return this.get().optimizerCacheKey;
   }

packages/kbn-optimizer/src/common/bundle_refs.ts

@@ -114,6 +114,10 @@ export class BundleRefs {
 
   constructor(private readonly refs: BundleRef[]) {}
 
+  public forBundleIds(bundleIds: string[]) {
+    return this.refs.filter((r) => bundleIds.includes(r.bundleId));
+  }
+
   public filterByExportIds(exportIds: string[]) {
     return this.refs.filter((r) => exportIds.includes(r.exportId));
   }

packages/kbn-optimizer/src/integration_tests/basic_optimization.test.ts

@@ -139,6 +139,7 @@ it('builds expected bundles, saves bundle counts to metadata', async () => {
   expect(foo.cache.getModuleCount()).toBe(6);
   expect(foo.cache.getReferencedFiles()).toMatchInlineSnapshot(`
     Array [
+      <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/kibana.json,
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/async_import.ts,
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/ext.ts,
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/index.ts,
@@ -160,12 +161,17 @@ it('builds expected bundles, saves bundle counts to metadata', async () => {
     Array [
       <absolute path>/node_modules/css-loader/package.json,
       <absolute path>/node_modules/style-loader/package.json,
+      <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/kibana.json,
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/public/index.scss,
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/public/index.ts,
+      <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/public/legacy/_other_styles.scss,
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/public/legacy/styles.scss,
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/public/lib.ts,
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/src/legacy/ui/public/icon.svg,
+      <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/src/legacy/ui/public/styles/_globals_v7dark.scss,
+      <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/src/legacy/ui/public/styles/_globals_v7light.scss,
       <absolute path>/packages/kbn-optimizer/target/worker/entry_point_creator.js,
+      <absolute path>/packages/kbn-optimizer/target/worker/postcss.config.js,
       <absolute path>/packages/kbn-ui-shared-deps/public_path_module_creator.js,
     ]
   `);

packages/kbn-optimizer/src/log_optimizer_state.ts

@@ -54,12 +54,18 @@ export function logOptimizerState(log: ToolingLog, config: OptimizerConfig) {
 
         if (event?.type === 'worker started') {
           let moduleCount = 0;
+          let workUnits = 0;
           for (const bundle of event.bundles) {
             moduleCount += bundle.cache.getModuleCount() ?? NaN;
+            workUnits += bundle.cache.getWorkUnits() ?? NaN;
           }
-          const mcString = isFinite(moduleCount) ? String(moduleCount) : '?';
-          const bcString = String(event.bundles.length);
-          log.info(`starting worker [${bcString} bundles, ${mcString} modules]`);
+
+          log.info(
+            `starting worker [${event.bundles.length} ${
+              event.bundles.length === 1 ? 'bundle' : 'bundles'
+            }]`
+          );
+          log.debug(`modules [${moduleCount}] work units [${workUnits}]`);
         }
 
         if (state.phase === 'reallocating') {