[7.x] chore(NA): introduce new yarn kbn reset command to support baze…

…l workflow (#89597) (#90244)

* chore(NA): introduce new yarn kbn reset command to support bazel workflow (#89597)

* chore(NA): introduce new yarn kbn destroy command to support bazel workflow

* chore(NA): update wording for new kbn pm changes

* chore(NA): update .bazelrc.common

Co-authored-by: Tyler Smalley <tylersmalley@me.com>

* chore(NA): update .bazelrc

Co-authored-by: Tyler Smalley <tylersmalley@me.com>

* chore(NA): update packages/kbn-pm/src/commands/clean.ts

Co-authored-by: Tyler Smalley <tylersmalley@me.com>

* chore(NA): update packages/kbn-pm/src/commands/destroy.ts

Co-authored-by: Tyler Smalley <tylersmalley@me.com>

* chore(NA): update packages/kbn-pm/src/commands/clean.ts

Co-authored-by: Tyler Smalley <tylersmalley@me.com>

* chore(NA): update packages/kbn-pm/src/commands/destroy.ts

Co-authored-by: Tyler Smalley <tylersmalley@me.com>

* chore(NA): rename destroy command into reset

* chore(NA): update packages/kbn-pm/src/commands/clean.ts

Co-authored-by: Tyler Smalley <tylersmalley@me.com>

* chore(NA): restore old behaviour on kbn clean

* chore(NA): update reset command to delete bazel caches on disk

* chore(NA): update packages/kbn-pm/src/commands/clean.ts

Co-authored-by: Tyler Smalley <tylersmalley@me.com>

* chore(NA): update prefix to match bazel defined one for cache settings

* chore(NA): update kbn pm dist file

* chore(NA): update kbn pm dist

* chore(NA): update gitignore

* chore(NA): add new ignore files after changed bazel aggregated folder

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Tyler Smalley <tylersmalley@me.com>
# Conflicts:
#	packages/kbn-pm/dist/index.js

* chore(NA): update changes on kbm pm

* chore(NA): fix jest test

.bazelignore

@@ -8,8 +8,7 @@
 .idea
 .teamcity
 .yarn-local-mirror
-bazel-cache
-bazel-dist
+/bazel
 build
 node_modules
 target

.bazelrc

@@ -1,4 +1,4 @@
-# Inspired on from https://raw.githubusercontent.com/bazelbuild/rules_nodejs/master/.bazelrc
+# Inspired by https://raw.githubusercontent.com/bazelbuild/rules_nodejs/master/.bazelrc
 # Import shared settings first so we can override below
 import %workspace%/.bazelrc.common
 

.bazelrc.common

@@ -5,18 +5,22 @@
 #
 # The full list of Bazel options: https://docs.bazel.build/versions/master/command-line-reference.html
 
-# Cache action outputs on disk so they persist across output_base and bazel shutdown (eg. changing branches)
-build --disk_cache=bazel-cache/disk-cache
+# Local Cache Settings
+## Avoid cache results from being corrupt when changing source during build
+common --experimental_guard_against_concurrent_changes
 
-# Bazel repo cache settings
-build --repository_cache=bazel-cache/repository-cache
+## Cache action outputs on disk so they persist across output_base and bazel shutdown (eg. changing branches)
+build --disk_cache=~/.bazel-cache/disk-cache
+
+## Bazel repo cache settings
+build --repository_cache=~/.bazel-cache/repository-cache
 
 # Bazel will create symlinks from the workspace directory to output artifacts.
-# Build results will be placed in a directory called "bazel-dist/bin"
+# Build results will be placed in a directory called "bazel/bin"
 # This will still create a bazel-out symlink in
 # the project directory, which must be excluded from the
 # editor's search path.
-build --symlink_prefix=bazel-dist/
+build --symlink_prefix=bazel/
 # To disable the symlinks altogether (including bazel-out) we can use
 # build --symlink_prefix=/
 # however this makes it harder to find outputs.
@@ -37,9 +41,7 @@ common --color=yes
 build --show_task_finish
 build --noshow_progress
 build --noshow_loading_progress
-
-## enforced default values
-build --show_result=1
+build --show_result=0
 
 # Specifies desired output mode for running tests.
 # Valid values are
@@ -78,7 +80,8 @@ test:debug --test_output=streamed --test_strategy=exclusive --test_timeout=9999
 # The node process will break before user code starts and wait for the debugger to connect.
 run:debug --define=VERBOSE_LOGS=1 -- --node_options=--inspect-brk
 # The following option will change the build output of certain rules such as terser and may not be desirable in all cases
-build:debug --compilation_mode=dbg
+# It will also output both the repo cache and action cache to a folder inside the repo
+build:debug --compilation_mode=dbg --show_result=1 --disk_cache=bazel/disk-cache --repository_cache=bazel/repository-cache
 
 # Turn off legacy external runfiles
 # This prevents accidentally depending on this feature, which Bazel will remove.

.eslintignore

@@ -46,4 +46,4 @@ snapshots.js
 /packages/kbn-monaco/src/painless/antlr
 
 # Bazel
-/bazel-*
+/bazel

.gitignore

@@ -81,5 +81,5 @@ report.asciidoc
 .yarn-local-mirror
 
 # Bazel
-/bazel-*
+/bazel
 /.bazelrc.user

packages/kbn-dev-utils/stdio/package.json

@@ -0,0 +1,3 @@
+{
+  "main": "../target/stdio"
+}

packages/kbn-pm/src/commands/clean.ts

@@ -6,19 +6,27 @@
  * Public License, v 1.
  */
 
+import dedent from 'dedent';
 import del from 'del';
 import ora from 'ora';
 import { join, relative } from 'path';
 
+import { runBazel } from '../utils/bazel';
 import { isDirectory } from '../utils/fs';
 import { log } from '../utils/log';
 import { ICommand } from './';
 
 export const CleanCommand: ICommand = {
-  description: 'Remove the node_modules and target directories from all projects.',
+  description: 'Deletes output directories, node_modules and resets internal caches.',
   name: 'clean',
 
   async run(projects) {
+    log.warning(dedent`
+      This command is only necessary for the rare circumstance where you need to recover a consistent
+      state when problems arise. If you need to run this command often, please let us know by
+      filling out this form: https://ela.st/yarn-kbn-clean
+    `);
+
     const toDelete = [];
     for (const project of projects.values()) {
       if (await isDirectory(project.nodeModulesLocation)) {
@@ -44,6 +52,10 @@ export const CleanCommand: ICommand = {
       }
     }
 
+    // Runs Bazel soft clean
+    await runBazel(['clean']);
+    log.success('Soft cleaned bazel');
+
     if (toDelete.length === 0) {
       log.success('Nothing to delete');
     } else {

packages/kbn-pm/src/commands/index.ts

@@ -24,13 +24,15 @@ export interface ICommand {
 
 import { BootstrapCommand } from './bootstrap';
 import { CleanCommand } from './clean';
+import { ResetCommand } from './reset';
 import { RunCommand } from './run';
 import { WatchCommand } from './watch';
 import { Kibana } from '../utils/kibana';
 
 export const commands: { [key: string]: ICommand } = {
   bootstrap: BootstrapCommand,
   clean: CleanCommand,
+  reset: ResetCommand,
   run: RunCommand,
   watch: WatchCommand,
 };

packages/kbn-pm/src/commands/reset.ts

@@ -0,0 +1,94 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * and the Server Side Public License, v 1; you may not use this file except in
+ * compliance with, at your election, the Elastic License or the Server Side
+ * Public License, v 1.
+ */
+
+import dedent from 'dedent';
+import del from 'del';
+import ora from 'ora';
+import { join, relative } from 'path';
+
+import { getBazelDiskCacheFolder, getBazelRepositoryCacheFolder, runBazel } from '../utils/bazel';
+import { isDirectory } from '../utils/fs';
+import { log } from '../utils/log';
+import { ICommand } from './';
+
+export const ResetCommand: ICommand = {
+  description:
+    'Deletes node_modules and output directories, resets internal and disk caches, and stops Bazel server',
+  name: 'reset',
+
+  async run(projects) {
+    log.warning(dedent`
+      In most cases, 'yarn kbn clean' is all that should be needed to recover a consistent state when
+      problems arise. If you need to use this command, please let us know, as it should not be necessary.
+    `);
+
+    const toDelete = [];
+    for (const project of projects.values()) {
+      if (await isDirectory(project.nodeModulesLocation)) {
+        toDelete.push({
+          cwd: project.path,
+          pattern: relative(project.path, project.nodeModulesLocation),
+        });
+      }
+
+      if (await isDirectory(project.targetLocation)) {
+        toDelete.push({
+          cwd: project.path,
+          pattern: relative(project.path, project.targetLocation),
+        });
+      }
+
+      const { extraPatterns } = project.getCleanConfig();
+      if (extraPatterns) {
+        toDelete.push({
+          cwd: project.path,
+          pattern: extraPatterns,
+        });
+      }
+    }
+
+    // Runs Bazel hard clean
+    await runBazel(['clean', '--expunge']);
+    log.success('Hard cleaned bazel');
+
+    // Deletes Bazel Cache Folders
+    await del([await getBazelDiskCacheFolder(), await getBazelRepositoryCacheFolder()], {
+      force: true,
+    });
+    log.success('Removed disk caches');
+
+    if (toDelete.length === 0) {
+      return;
+    }
+
+    /**
+     * In order to avoid patterns like `/build` in packages from accidentally
+     * impacting files outside the package we use `process.chdir()` to change
+     * the cwd to the package and execute `del()` without the `force` option
+     * so it will check that each file being deleted is within the package.
+     *
+     * `del()` does support a `cwd` option, but it's only for resolving the
+     * patterns and does not impact the cwd check.
+     */
+    const originalCwd = process.cwd();
+    try {
+      for (const { pattern, cwd } of toDelete) {
+        process.chdir(cwd);
+        const promise = del(pattern);
+
+        if (log.wouldLogLevel('info')) {
+          ora.promise(promise, relative(originalCwd, join(cwd, String(pattern))));
+        }
+
+        await promise;
+      }
+    } finally {
+      process.chdir(originalCwd);
+    }
+  },
+};

packages/kbn-pm/src/utils/bazel/get_cache_folders.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * and the Server Side Public License, v 1; you may not use this file except in
+ * compliance with, at your election, the Elastic License or the Server Side
+ * Public License, v 1.
+ */
+
+import { dirname, resolve } from 'path';
+import { spawn } from '../child_process';
+
+async function rawRunBazelInfoRepoCache() {
+  const { stdout: bazelRepositoryCachePath } = await spawn('bazel', ['info', 'repository_cache'], {
+    stdio: 'pipe',
+  });
+  return bazelRepositoryCachePath;
+}
+
+export async function getBazelDiskCacheFolder() {
+  return resolve(dirname(await rawRunBazelInfoRepoCache()), 'disk-cache');
+}
+
+export async function getBazelRepositoryCacheFolder() {
+  return await rawRunBazelInfoRepoCache();
+}

packages/kbn-pm/src/utils/bazel/index.ts

@@ -6,4 +6,6 @@
  * Public License, v 1.
  */
 
+export * from './get_cache_folders';
 export * from './install_tools';
+export * from './run';

packages/kbn-pm/src/utils/bazel/run.ts

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * and the Server Side Public License, v 1; you may not use this file except in
+ * compliance with, at your election, the Elastic License or the Server Side
+ * Public License, v 1.
+ */
+
+import chalk from 'chalk';
+import execa from 'execa';
+import * as Rx from 'rxjs';
+import { tap } from 'rxjs/operators';
+import { observeLines } from '@kbn/dev-utils/stdio';
+import { spawn } from '../child_process';
+import { log } from '../log';
+
+export async function runBazel(bazelArgs: string[], runOpts: execa.Options = {}) {
+  // Force logs to pipe in order to control the output of them
+  const bazelOpts: execa.Options = {
+    ...runOpts,
+    stdio: 'pipe',
+  };
+
+  const bazelProc = spawn('bazel', bazelArgs, bazelOpts);
+
+  const bazelLogs$ = new Rx.Subject<string>();
+
+  // Bazel outputs machine readable output into stdout and human readable output goes to stderr.
+  // Therefore we need to get both. In order to get errors we need to parse the actual text line
+  const bazelLogSubscription = Rx.merge(
+    observeLines(bazelProc.stdout!).pipe(
+      tap((line) => log.info(`${chalk.cyan('[bazel]')} ${line}`))
+    ),
+    observeLines(bazelProc.stderr!).pipe(
+      tap((line) => log.info(`${chalk.cyan('[bazel]')} ${line}`))
+    )
+  ).subscribe(bazelLogs$);
+
+  // Wait for process and logs to finish, unsubscribing in the end
+  await bazelProc;
+  await bazelLogs$.toPromise();
+  await bazelLogSubscription.unsubscribe();
+}