No longer package X-Pack as a node module

package.json

@@ -247,7 +247,6 @@
     "webpack-merge": "4.1.4",
     "whatwg-fetch": "^3.0.0",
     "wreck": "^14.0.2",
-    "x-pack": "8.0.0",
     "yauzl": "2.7.0"
   },
   "devDependencies": {

packages/kbn-pm/src/commands/bootstrap.test.ts

@@ -27,12 +27,13 @@ import { linkProjectExecutables } from '../utils/link_project_executables';
 import { IPackageJson } from '../utils/package_json';
 import { Project } from '../utils/project';
 import { buildProjectGraph } from '../utils/projects';
-import { installInDir, runScriptInPackageStreaming } from '../utils/scripts';
+import { installInDir, runScriptInPackageStreaming, yarnWorkspacesInfo } from '../utils/scripts';
 import { BootstrapCommand } from './bootstrap';
 
 const mockInstallInDir = installInDir as jest.Mock;
 const mockRunScriptInPackageStreaming = runScriptInPackageStreaming as jest.Mock;
 const mockLinkProjectExecutables = linkProjectExecutables as jest.Mock;
+const mockYarnWorkspacesInfo = yarnWorkspacesInfo as jest.Mock;
 
 const createProject = (packageJson: IPackageJson, path = '.') => {
   const project = new Project(
@@ -57,6 +58,10 @@ const noop = () => {
   // noop
 };
 
+beforeEach(() => {
+  mockYarnWorkspacesInfo.mockResolvedValue({});
+});
+
 afterEach(() => {
   jest.resetAllMocks();
   jest.restoreAllMocks();

packages/kbn-pm/src/config.ts

@@ -27,7 +27,7 @@ export interface IProjectPathOptions {
 /**
  * Returns all the paths where plugins are located
  */
-export function getProjectPaths(rootPath: string, options: IProjectPathOptions) {
+export function getProjectPaths(rootPath: string, options: IProjectPathOptions = {}) {
   const skipKibanaPlugins = Boolean(options['skip-kibana-plugins']);
   const ossOnly = Boolean(options.oss);
 

packages/kbn-pm/src/index.ts

@@ -19,4 +19,7 @@
 
 export { run } from './cli';
 export { buildProductionProjects, prepareExternalProjectDependencies } from './production';
+export { getProjects } from './utils/projects';
+export { Project } from './utils/project';
 export { copyWorkspacePackages } from './utils/workspaces';
+export { getProjectPaths } from './config';

packages/kbn-pm/src/production/build_production_projects.ts

@@ -35,12 +35,14 @@ import {
 
 export async function buildProductionProjects({
   kibanaRoot,
-  buildRoots,
+  buildRoot,
+  onlyOSS,
 }: {
   kibanaRoot: string;
-  buildRoots: string[];
+  buildRoot: string;
+  onlyOSS?: boolean;
 }) {
-  const projects = await getProductionProjects(kibanaRoot);
+  const projects = await getProductionProjects(kibanaRoot, onlyOSS);
   const projectGraph = buildProjectGraph(projects);
   const batchedProjects = topologicallyBatchProjects(projects, projectGraph);
 
@@ -51,9 +53,7 @@ export async function buildProductionProjects({
     for (const project of batch) {
       await deleteTarget(project);
       await buildProject(project);
-      for (const buildRoot of buildRoots) {
-        await copyToBuild(project, kibanaRoot, buildRoot);
-      }
+      await copyToBuild(project, kibanaRoot, buildRoot);
     }
   }
 }
@@ -62,19 +62,33 @@ export async function buildProductionProjects({
  * Returns the subset of projects that should be built into the production
  * bundle. As we copy these into Kibana's `node_modules` during the build step,
  * and let Kibana's build process be responsible for installing dependencies,
- * we only include Kibana's transitive _production_ dependencies.
+ * we only include Kibana's transitive _production_ dependencies. If onlyOSS
+ * is supplied, we omit projects with build.oss in their package.json set to false.
  */
-async function getProductionProjects(rootPath: string) {
+async function getProductionProjects(rootPath: string, onlyOSS?: boolean) {
   const projectPaths = getProjectPaths(rootPath, {});
   const projects = await getProjects(rootPath, projectPaths);
+  const projectsSubset = [projects.get('kibana')!];
+
+  if (projects.has('x-pack')) {
+    projectsSubset.push(projects.get('x-pack')!);
+  }
 
-  const productionProjects = includeTransitiveProjects([projects.get('kibana')!], projects, {
+  const productionProjects = includeTransitiveProjects(projectsSubset, projects, {
     onlyProductionDependencies: true,
   });
 
   // We remove Kibana, as we're already building Kibana
   productionProjects.delete('kibana');
 
+  if (onlyOSS) {
+    productionProjects.forEach(project => {
+      if (project.getBuildConfig().oss === false) {
+        productionProjects.delete(project.json.name);
+      }
+    });
+  }
+
   return productionProjects;
 }
 

packages/kbn-pm/src/production/integration_tests/__fixtures__/packages/foo/package.json

@@ -3,6 +3,11 @@
   "version": "1.0.0",
   "private": true,
   "main": "./target/index.js",
+  "kibana": {
+    "build": {
+      "oss": false
+    }
+  },
   "dependencies": {
     "@elastic/bar": "link:../bar"
   },

packages/kbn-pm/src/production/integration_tests/build_production_projects.test.ts

@@ -27,31 +27,38 @@ import { getProjects } from '../../utils/projects';
 import { buildProductionProjects } from '../build_production_projects';
 
 describe('kbn-pm production', () => {
-  test(
-    'builds and copies projects for production',
-    async () => {
-      const tmpDir = tempy.directory();
-      const buildRoot = tempy.directory();
-      const fixturesPath = resolve(__dirname, '__fixtures__');
+  let tmpDir: string;
+  let buildRoot: string;
 
-      // Copy all the test fixtures into a tmp dir, as we will be mutating them
-      await copy(['**/*'], tmpDir, {
-        cwd: fixturesPath,
-        dot: true,
-        nodir: true,
-        parents: true,
-      });
+  const timeout = 1 * 60 * 1000;
 
-      const projects = await getProjects(tmpDir, ['.', './packages/*']);
+  beforeEach(async () => {
+    tmpDir = tempy.directory();
+    buildRoot = tempy.directory();
+    const fixturesPath = resolve(__dirname, '__fixtures__');
 
-      for (const project of projects.values()) {
-        // This will both install dependencies and generate `yarn.lock` files
-        await project.installDependencies({
-          extraArgs: ['--silent', '--no-progress'],
-        });
-      }
+    // Copy all the test fixtures into a tmp dir, as we will be mutating them
+    await copy(['**/*'], tmpDir, {
+      cwd: fixturesPath,
+      dot: true,
+      nodir: true,
+      parents: true,
+    });
+
+    const projects = await getProjects(tmpDir, ['.', './packages/*']);
 
-      await buildProductionProjects({ kibanaRoot: tmpDir, buildRoots: [buildRoot] });
+    for (const project of projects.values()) {
+      // This will both install dependencies and generate `yarn.lock` files
+      await project.installDependencies({
+        extraArgs: ['--silent', '--no-progress'],
+      });
+    }
+  }, timeout);
+
+  test(
+    'builds and copies projects for production',
+    async () => {
+      await buildProductionProjects({ kibanaRoot: tmpDir, buildRoot });
 
       const files = await globby(['**/*', '!**/node_modules/**'], {
         cwd: buildRoot,
@@ -65,6 +72,20 @@ describe('kbn-pm production', () => {
         }
       }
     },
-    2 * 60 * 1000
+    timeout
+  );
+
+  test(
+    'builds and copies only OSS projects for production',
+    async () => {
+      await buildProductionProjects({ kibanaRoot: tmpDir, buildRoot, onlyOSS: true });
+
+      const files = await globby(['**/*', '!**/node_modules/**'], {
+        cwd: buildRoot,
+      });
+
+      expect(files.sort()).toMatchSnapshot();
+    },
+    timeout
   );
 });

packages/kbn-pm/src/utils/child_process.ts

@@ -33,8 +33,8 @@ function generateColors() {
 
 export function spawn(command: string, args: string[], opts: execa.Options) {
   return execa(command, args, {
-    ...opts,
     stdio: 'inherit',
+    ...opts,
   });
 }
 
@@ -47,8 +47,8 @@ export function spawnStreaming(
   { prefix }: { prefix: string }
 ) {
   const spawned = execa(command, args, {
-    ...opts,
     stdio: ['ignore', 'pipe', 'pipe'],
+    ...opts,
   });
 
   const color = nextColor();

packages/kbn-pm/src/utils/fs.ts

@@ -24,7 +24,7 @@ import { ncp } from 'ncp';
 import { dirname, relative } from 'path';
 import { promisify } from 'util';
 
-const stat = promisify(fs.stat);
+const lstat = promisify(fs.lstat);
 const readFile = promisify(fs.readFile);
 const symlink = promisify(fs.symlink);
 const chmod = promisify(fs.chmod);
@@ -37,7 +37,7 @@ export { chmod, readFile, mkdirp };
 
 async function statTest(path: string, block: (stats: fs.Stats) => boolean) {
   try {
-    return block(await stat(path));
+    return block(await lstat(path));
   } catch (e) {
     if (e.code === 'ENOENT') {
       return false;
@@ -46,6 +46,14 @@ async function statTest(path: string, block: (stats: fs.Stats) => boolean) {
   }
 }
 
+/**
+ * Test if a path points to a symlink.
+ * @param path
+ */
+export async function isSymlink(path: string) {
+  return await statTest(path, stats => stats.isSymbolicLink());
+}
+
 /**
  * Test if a path points to a directory.
  * @param path

packages/kbn-pm/src/utils/project.ts

@@ -18,6 +18,7 @@
  */
 
 import chalk from 'chalk';
+import fs from 'fs';
 import { relative, resolve as resolvePath } from 'path';
 import { inspect } from 'util';
 
@@ -30,11 +31,17 @@ import {
   isLinkDependency,
   readPackageJson,
 } from './package_json';
-import { installInDir, runScriptInPackage, runScriptInPackageStreaming } from './scripts';
+import {
+  installInDir,
+  runScriptInPackage,
+  runScriptInPackageStreaming,
+  yarnWorkspacesInfo,
+} from './scripts';
 
 interface BuildConfig {
   skip?: boolean;
   intermediateBuildDirectory?: string;
+  oss?: boolean;
 }
 
 interface CleanConfig {
@@ -190,7 +197,41 @@ export class Project {
 
   public async installDependencies({ extraArgs }: { extraArgs: string[] }) {
     log.write(chalk.bold(`\n\nInstalling dependencies in [${chalk.green(this.name)}]:\n`));
-    return installInDir(this.path, extraArgs);
+    await installInDir(this.path, extraArgs);
+    await this.removeExtraneousNodeModules();
+  }
+
+  /**
+   * Yarn workspaces symlinks workspace projects to the root node_modules, even
+   * when there is no depenency on the project. This results in unnecicary, and
+   * often duplicated code in the build archives.
+   */
+  public async removeExtraneousNodeModules() {
+    // this is only relevant for the root workspace
+    if (!this.isWorkspaceRoot) {
+      return;
+    }
+
+    const workspacesInfo = await yarnWorkspacesInfo(this.path);
+    const unusedWorkspaces = new Set(Object.keys(workspacesInfo));
+
+    // check for any cross-project dependency
+    for (const name of Object.keys(workspacesInfo)) {
+      const workspace = workspacesInfo[name];
+      workspace.workspaceDependencies.forEach(w => unusedWorkspaces.delete(w));
+    }
+
+    unusedWorkspaces.forEach(name => {
+      const { dependencies, devDependencies } = this.json;
+      const nodeModulesPath = resolvePath(this.nodeModulesLocation, name);
+      const isDependency = dependencies && dependencies.hasOwnProperty(name);
+      const isDevDependency = devDependencies && devDependencies.hasOwnProperty(name);
+
+      if (!isDependency && !isDevDependency && fs.existsSync(nodeModulesPath)) {
+        log.write(`No dependency on ${name}, removing link in node_modules`);
+        fs.unlinkSync(nodeModulesPath);
+      }
+    });
   }
 }