[kbn/optimizer] implement "requiredBundles" property of KP plugins

docs/development/core/server/kibana-plugin-core-server.discoveredplugin.md

@@ -19,5 +19,6 @@ export interface DiscoveredPlugin
 |  [configPath](./kibana-plugin-core-server.discoveredplugin.configpath.md) | <code>ConfigPath</code> | Root configuration path used by the plugin, defaults to "id" in snake\_case format. |
 |  [id](./kibana-plugin-core-server.discoveredplugin.id.md) | <code>PluginName</code> | Identifier of the plugin. |
 |  [optionalPlugins](./kibana-plugin-core-server.discoveredplugin.optionalplugins.md) | <code>readonly PluginName[]</code> | An optional list of the other plugins that if installed and enabled \*\*may be\*\* leveraged by this plugin for some additional functionality but otherwise are not required for this plugin to work properly. |
+|  [requiredBundles](./kibana-plugin-core-server.discoveredplugin.requiredbundles.md) | <code>readonly PluginName[]</code> | An optional list of the other plugins which are imported by the UI bundles of this plugin. If the plugins listed here are disabled their bundles will still be loaded into the browser. Required by the @<!-- -->kbn/optimizer to allow cross plugin imports. "core" and plugins listed in "requiredPlugins" do not need to be listed here. |
 |  [requiredPlugins](./kibana-plugin-core-server.discoveredplugin.requiredplugins.md) | <code>readonly PluginName[]</code> | An optional list of the other plugins that \*\*must be\*\* installed and enabled for this plugin to function properly. |
 

docs/development/core/server/kibana-plugin-core-server.discoveredplugin.requiredbundles.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [DiscoveredPlugin](./kibana-plugin-core-server.discoveredplugin.md) &gt; [requiredBundles](./kibana-plugin-core-server.discoveredplugin.requiredbundles.md)
+
+## DiscoveredPlugin.requiredBundles property
+
+An optional list of the other plugins which are imported by the UI bundles of this plugin. If the plugins listed here are disabled their bundles will still be loaded into the browser. Required by the @<!-- -->kbn/optimizer to allow cross plugin imports. "core" and plugins listed in "requiredPlugins" do not need to be listed here.
+
+<b>Signature:</b>
+
+```typescript
+readonly requiredBundles: readonly PluginName[];
+```

docs/development/core/server/kibana-plugin-core-server.pluginmanifest.md

@@ -25,6 +25,7 @@ Should never be used in code outside of Core but is exported for documentation p
 |  [id](./kibana-plugin-core-server.pluginmanifest.id.md) | <code>PluginName</code> | Identifier of the plugin. Must be a string in camelCase. Part of a plugin public contract. Other plugins leverage it to access plugin API, navigate to the plugin, etc. |
 |  [kibanaVersion](./kibana-plugin-core-server.pluginmanifest.kibanaversion.md) | <code>string</code> | The version of Kibana the plugin is compatible with, defaults to "version". |
 |  [optionalPlugins](./kibana-plugin-core-server.pluginmanifest.optionalplugins.md) | <code>readonly PluginName[]</code> | An optional list of the other plugins that if installed and enabled \*\*may be\*\* leveraged by this plugin for some additional functionality but otherwise are not required for this plugin to work properly. |
+|  [requiredBundles](./kibana-plugin-core-server.pluginmanifest.requiredbundles.md) | <code>readonly string[]</code> | An optional list of the other plugins which are imported by the UI bundles of this plugin. If the plugins listed here are disabled their bundles will still be loaded into the browser. Required by the @<!-- -->kbn/optimizer to allow cross plugin imports. "core" and plugins listed in "requiredPlugins" do not need to be listed here. |
 |  [requiredPlugins](./kibana-plugin-core-server.pluginmanifest.requiredplugins.md) | <code>readonly PluginName[]</code> | An optional list of the other plugins that \*\*must be\*\* installed and enabled for this plugin to function properly. |
 |  [server](./kibana-plugin-core-server.pluginmanifest.server.md) | <code>boolean</code> | Specifies whether plugin includes some server-side specific functionality. |
 |  [ui](./kibana-plugin-core-server.pluginmanifest.ui.md) | <code>boolean</code> | Specifies whether plugin includes some client/browser specific functionality that should be included into client bundle via <code>public/ui_plugin.js</code> file. |

docs/development/core/server/kibana-plugin-core-server.pluginmanifest.requiredbundles.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [PluginManifest](./kibana-plugin-core-server.pluginmanifest.md) &gt; [requiredBundles](./kibana-plugin-core-server.pluginmanifest.requiredbundles.md)
+
+## PluginManifest.requiredBundles property
+
+An optional list of the other plugins which are imported by the UI bundles of this plugin. If the plugins listed here are disabled their bundles will still be loaded into the browser. Required by the @<!-- -->kbn/optimizer to allow cross plugin imports. "core" and plugins listed in "requiredPlugins" do not need to be listed here.
+
+<b>Signature:</b>
+
+```typescript
+readonly requiredBundles: readonly string[];
+```

examples/bfetch_explorer/kibana.json

@@ -5,5 +5,6 @@
   "server": true,
   "ui": true,
   "requiredPlugins": ["bfetch", "developerExamples"],
-  "optionalPlugins": []
+  "optionalPlugins": [],
+  "requiredBundles": ["kibanaReact"]
 }

examples/dashboard_embeddable_examples/kibana.json

@@ -5,5 +5,6 @@
   "server": false,
   "ui": true,
   "requiredPlugins": ["embeddable", "embeddableExamples", "dashboard", "developerExamples"],
-  "optionalPlugins": []
+  "optionalPlugins": [],
+  "requiredBundles": ["esUiShared"]
 }

examples/embeddable_examples/kibana.json

@@ -6,5 +6,6 @@
   "ui": true,
   "requiredPlugins": ["embeddable"],
   "optionalPlugins": [],
-  "extraPublicDirs": ["public/todo", "public/hello_world", "public/todo/todo_ref_embeddable"]
+  "extraPublicDirs": ["public/todo", "public/hello_world", "public/todo/todo_ref_embeddable"],
+  "requiredBundles": ["kibanaReact"]
 }

examples/state_containers_examples/kibana.json

@@ -5,5 +5,6 @@
   "server": true,
   "ui": true,
   "requiredPlugins": ["navigation", "data", "developerExamples"],
-  "optionalPlugins": []
+  "optionalPlugins": [],
+  "requiredBundles": ["kibanaUtils", "kibanaReact"]
 }

examples/ui_action_examples/kibana.json

@@ -5,5 +5,6 @@
   "server": false,
   "ui": true,
   "requiredPlugins": ["uiActions"],
-  "optionalPlugins": []
+  "optionalPlugins": [],
+  "requiredBundles": ["kibanaReact"]
 }

examples/ui_actions_explorer/kibana.json

@@ -5,5 +5,6 @@
   "server": false,
   "ui": true,
   "requiredPlugins": ["uiActions", "uiActionsExamples", "developerExamples"],
-  "optionalPlugins": []
+  "optionalPlugins": [],
+  "requiredBundles": ["kibanaReact"]
 }

packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/kibana.json

@@ -1,4 +1,5 @@
 {
   "id": "bar",
-  "ui": true
+  "ui": true,
+  "requiredBundles": ["foo"]
 }

packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/public/legacy/_other_styles.scss

@@ -0,0 +1,3 @@
+p {
+  background-color: rebeccapurple;
+}

packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/public/legacy/styles.scss

@@ -1,3 +1,5 @@
+@import "./other_styles.scss";
+
 body {
   width: $globalStyleConstant;
   background-image: url("ui/icon.svg");

packages/kbn-optimizer/src/cli.ts

@@ -87,6 +87,11 @@ run(
       throw createFlagError('expected --report-stats to have no value');
     }
 
+    const filter = typeof flags.filter === 'string' ? [flags.filter] : flags.filter;
+    if (!Array.isArray(filter) || !filter.every((f) => typeof f === 'string')) {
+      throw createFlagError('expected --filter to be one or more strings');
+    }
+
     const config = OptimizerConfig.create({
       repoRoot: REPO_ROOT,
       watch,
@@ -99,6 +104,7 @@ run(
       extraPluginScanDirs,
       inspectWorkers,
       includeCoreBundle,
+      filter,
     });
 
     let update$ = runOptimizer(config);
@@ -128,12 +134,13 @@ run(
         'inspect-workers',
         'report-stats',
       ],
-      string: ['workers', 'scan-dir'],
+      string: ['workers', 'scan-dir', 'filter'],
       default: {
         core: true,
         examples: true,
         cache: true,
         'inspect-workers': true,
+        filter: [],
       },
       help: `
         --watch            run the optimizer in watch mode
@@ -142,6 +149,7 @@ run(
         --profile          profile the webpack builds and write stats.json files to build outputs
         --no-core          disable generating the core bundle
         --no-cache         disable the cache
+        --filter           filter the list of bundles, results from multiple flags are merged, * and ! are supported
         --no-examples      don't build the example plugins
         --dist             create bundles that are suitable for inclusion in the Kibana distributable
         --scan-dir         add a directory to the list of directories scanned for plugins (specify as many times as necessary)

packages/kbn-optimizer/src/common/bundle.test.ts

@@ -50,6 +50,7 @@ it('creates cache keys', () => {
       "spec": Object {
         "contextDir": "/foo/bar",
         "id": "bar",
+        "manifestPath": undefined,
         "outputDir": "/foo/bar/target",
         "publicDirNames": Array [
           "public",
@@ -85,6 +86,7 @@ it('parses bundles from JSON specs', () => {
         },
         "contextDir": "/foo/bar",
         "id": "bar",
+        "manifestPath": undefined,
         "outputDir": "/foo/bar/target",
         "publicDirNames": Array [
           "public",

packages/kbn-optimizer/src/common/bundle.ts

@@ -18,13 +18,19 @@
  */
 
 import Path from 'path';
+import Fs from 'fs';
 
 import { BundleCache } from './bundle_cache';
 import { UnknownVals } from './ts_helpers';
 import { includes, ascending, entriesToObject } from './array_helpers';
 
 const VALID_BUNDLE_TYPES = ['plugin' as const, 'entry' as const];
 
+const DEFAULT_IMPLICIT_BUNDLE_DEPS = ['core'];
+
+const isStringArray = (input: any): input is string[] =>
+  Array.isArray(input) && input.every((x) => typeof x === 'string');
+
 export interface BundleSpec {
   readonly type: typeof VALID_BUNDLE_TYPES[0];
   /** Unique id for this bundle */
@@ -37,6 +43,8 @@ export interface BundleSpec {
   readonly sourceRoot: string;
   /** Absolute path to the directory where output should be written */
   readonly outputDir: string;
+  /** Absolute path to a kibana.json manifest file, if omitted we assume there are not dependenices */
+  readonly manifestPath?: string;
 }
 
 export class Bundle {
@@ -56,6 +64,12 @@ export class Bundle {
   public readonly sourceRoot: BundleSpec['sourceRoot'];
   /** Absolute path to the output directory for this bundle */
   public readonly outputDir: BundleSpec['outputDir'];
+  /**
+   * Absolute path to a manifest file with "requiredBundles" which will be
+   * used to allow bundleRefs from this bundle to the exports of another bundle.
+   * Every bundle mentioned in the `requiredBundles` must be built together.
+   */
+  public readonly manifestPath: BundleSpec['manifestPath'];
 
   public readonly cache: BundleCache;
 
@@ -66,6 +80,7 @@ export class Bundle {
     this.contextDir = spec.contextDir;
     this.sourceRoot = spec.sourceRoot;
     this.outputDir = spec.outputDir;
+    this.manifestPath = spec.manifestPath;
 
     this.cache = new BundleCache(Path.resolve(this.outputDir, '.kbn-optimizer-cache'));
   }
@@ -96,8 +111,54 @@ export class Bundle {
       contextDir: this.contextDir,
       sourceRoot: this.sourceRoot,
       outputDir: this.outputDir,
+      manifestPath: this.manifestPath,
     };
   }
+
+  readBundleDeps(): { implicit: string[]; explicit: string[] } {
+    if (!this.manifestPath) {
+      return {
+        implicit: [...DEFAULT_IMPLICIT_BUNDLE_DEPS],
+        explicit: [],
+      };
+    }
+
+    let json: string;
+    try {
+      json = Fs.readFileSync(this.manifestPath, 'utf8');
+    } catch (error) {
+      if (error.code !== 'ENOENT') {
+        throw error;
+      }
+
+      json = '{}';
+    }
+
+    let parsedManifest: { requiredPlugins?: string[]; requiredBundles?: string[] };
+    try {
+      parsedManifest = JSON.parse(json);
+    } catch (error) {
+      throw new Error(
+        `unable to parse manifest at [${this.manifestPath}], error: [${error.message}]`
+      );
+    }
+
+    if (typeof parsedManifest === 'object' && parsedManifest) {
+      const explicit = parsedManifest.requiredBundles || [];
+      const implicit = [...DEFAULT_IMPLICIT_BUNDLE_DEPS, ...(parsedManifest.requiredPlugins || [])];
+
+      if (isStringArray(explicit) && isStringArray(implicit)) {
+        return {
+          explicit,
+          implicit,
+        };
+      }
+    }
+
+    throw new Error(
+      `Expected "requiredBundles" and "requiredPlugins" in manifest file [${this.manifestPath}] to be arrays of strings`
+    );
+  }
 }
 
 /**
@@ -152,13 +213,21 @@ export function parseBundles(json: string) {
           throw new Error('`bundles[]` must have an absolute path `outputDir` property');
         }
 
+        const { manifestPath } = spec;
+        if (manifestPath !== undefined) {
+          if (!(typeof manifestPath === 'string' && Path.isAbsolute(manifestPath))) {
+            throw new Error('`bundles[]` must have an absolute path `manifestPath` property');
+          }
+        }
+
         return new Bundle({
           type,
           id,
           publicDirNames,
           contextDir,
           sourceRoot,
           outputDir,
+          manifestPath,
         });
       }
     );

packages/kbn-optimizer/src/common/bundle_cache.ts

@@ -24,6 +24,7 @@ export interface State {
   optimizerCacheKey?: unknown;
   cacheKey?: unknown;
   moduleCount?: number;
+  workUnits?: number;
   files?: string[];
   bundleRefExportIds?: string[];
 }
@@ -96,6 +97,10 @@ export class BundleCache {
     return this.get().cacheKey;
   }
 
+  public getWorkUnits() {
+    return this.get().workUnits;
+  }
+
   public getOptimizerCacheKey() {
     return this.get().optimizerCacheKey;
   }

packages/kbn-optimizer/src/common/bundle_refs.ts

@@ -114,6 +114,10 @@ export class BundleRefs {
 
   constructor(private readonly refs: BundleRef[]) {}
 
+  public forBundleIds(bundleIds: string[]) {
+    return this.refs.filter((r) => bundleIds.includes(r.bundleId));
+  }
+
   public filterByExportIds(exportIds: string[]) {
     return this.refs.filter((r) => exportIds.includes(r.exportId));
   }

packages/kbn-optimizer/src/integration_tests/basic_optimization.test.ts

@@ -139,6 +139,7 @@ it('builds expected bundles, saves bundle counts to metadata', async () => {
   expect(foo.cache.getModuleCount()).toBe(6);
   expect(foo.cache.getReferencedFiles()).toMatchInlineSnapshot(`
     Array [
+      <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/kibana.json,
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/async_import.ts,
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/ext.ts,
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/index.ts,
@@ -160,12 +161,17 @@ it('builds expected bundles, saves bundle counts to metadata', async () => {
     Array [
       <absolute path>/node_modules/css-loader/package.json,
       <absolute path>/node_modules/style-loader/package.json,
+      <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/kibana.json,
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/public/index.scss,
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/public/index.ts,
+      <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/public/legacy/_other_styles.scss,
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/public/legacy/styles.scss,
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/public/lib.ts,
       <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/src/legacy/ui/public/icon.svg,
+      <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/src/legacy/ui/public/styles/_globals_v7dark.scss,
+      <absolute path>/packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/src/legacy/ui/public/styles/_globals_v7light.scss,
       <absolute path>/packages/kbn-optimizer/target/worker/entry_point_creator.js,
+      <absolute path>/packages/kbn-optimizer/target/worker/postcss.config.js,
       <absolute path>/packages/kbn-ui-shared-deps/public_path_module_creator.js,
     ]
   `);

packages/kbn-optimizer/src/log_optimizer_state.ts

@@ -54,12 +54,18 @@ export function logOptimizerState(log: ToolingLog, config: OptimizerConfig) {
 
         if (event?.type === 'worker started') {
           let moduleCount = 0;
+          let workUnits = 0;
           for (const bundle of event.bundles) {
             moduleCount += bundle.cache.getModuleCount() ?? NaN;
+            workUnits += bundle.cache.getWorkUnits() ?? NaN;
           }
-          const mcString = isFinite(moduleCount) ? String(moduleCount) : '?';
-          const bcString = String(event.bundles.length);
-          log.info(`starting worker [${bcString} bundles, ${mcString} modules]`);
+
+          log.info(
+            `starting worker [${event.bundles.length} ${
+              event.bundles.length === 1 ? 'bundle' : 'bundles'
+            }]`
+          );
+          log.debug(`modules [${moduleCount}] work units [${workUnits}]`);
         }
 
         if (state.phase === 'reallocating') {