Separate no-op versus one-shot executions (#251)

Previously, we had a single `ScriptExecution` class which handled all scripts. However, some scripts are significantly simpler than others. Namely, we now distinguish between "no-op" and "one-shot" scripts.

A "no-op" script is one that has no command defined. It just has some files and/or dependencies, and is mostly just a pass-through for fingerprints. There is no sense in writing fingerprint files, doing caching, acquiring locks, etc. for these scripts, so we no longer do that.

A "one-shot" script is a script with an actual command. It's called "one-shot" to distinguish it from an upcoming 3rd kind of script called a [service](#33).

**Note to reviewer**: The `OneShotExecution` class is almost identical to what used to be called `ScriptExecution`, but with some methods moved into a new `BaseExecution` class. The `NoOpExecution` class is where the meaningful changes are, because a lot of logic that used to run around reading/writing to the `.wireit` directory no longer does.

CHANGELOG.md

@@ -22,6 +22,11 @@ Versioning](https://semver.org/spec/v2.0.0.html).
 - The internal `.wireit/*/state` file was renamed to `.wireit/*/fingerprint`.
   Should have no effect.
 
+- If a script does not define a `"command"`, then fingerprints, lock files, and
+  cache entries are no longer written to the `.wireit` directory. This change
+  should have no user-facing effect apart from a very minor performance
+  improvement.
+
 ## [0.4.3] - 2022-05-15
 
 ### Changed

src/execution/base.ts

@@ -0,0 +1,214 @@
+/**
+ * @license
+ * Copyright 2022 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import {createReadStream} from 'fs';
+import {resolve} from 'path';
+import {createHash} from 'crypto';
+import {glob} from '../util/glob.js';
+import {shuffle} from '../util/shuffle.js';
+import {getScriptDataDir} from '../util/script-data-dir.js';
+
+import type {Result} from '../error.js';
+import type {Executor} from '../executor.js';
+import {
+  Fingerprint,
+  ScriptConfig,
+  ScriptReference,
+  ScriptReferenceString,
+  scriptReferenceToString,
+  Sha256HexDigest,
+} from '../script.js';
+import type {Logger} from '../logging/logger.js';
+import type {Failure, StartCancelled} from '../event.js';
+
+export type ExecutionResult = Result<Fingerprint, Failure[]>;
+
+/**
+ * What to do when a script failure occurs:
+ *
+ * - `no-new`: Allow running scripts to finish, but don't start new ones.
+ * - `continue`: Allow running scripts to finish, and start new ones unless a
+ *   dependency failed.
+ * - `kill`: Immediately kill running scripts, and don't start new ones.
+ */
+export type FailureMode = 'no-new' | 'continue' | 'kill';
+
+/**
+ * A single execution of a specific script.
+ */
+export abstract class BaseExecution<T extends ScriptConfig> {
+  protected readonly script: T;
+  protected readonly executor: Executor;
+  protected readonly logger: Logger;
+
+  protected constructor(script: T, executor: Executor, logger: Logger) {
+    this.script = script;
+    this.executor = executor;
+    this.logger = logger;
+  }
+
+  /**
+   * Whether we should return early instead of starting this script.
+   *
+   * We should check this as the first thing we do, and then after any
+   * significant amount of time might have elapsed.
+   */
+  protected get shouldNotStart(): boolean {
+    return this.executor.shouldStopStartingNewScripts;
+  }
+
+  /**
+   * Convenience to generate a cancellation failure event for this script.
+   */
+  protected get startCancelledEvent(): StartCancelled {
+    return {
+      script: this.script,
+      type: 'failure',
+      reason: 'start-cancelled',
+    };
+  }
+
+  /**
+   * Get the directory name where Wireit data can be saved for this script.
+   */
+  protected get dataDir(): string {
+    return getScriptDataDir(this.script);
+  }
+
+  /**
+   * Execute all of this script's dependencies.
+   */
+  protected async executeDependencies(): Promise<
+    Result<Array<[ScriptReference, Fingerprint]>, Failure[]>
+  > {
+    // Randomize the order we execute dependencies to make it less likely for a
+    // user to inadvertently depend on any specific order, which could indicate
+    // a missing edge in the dependency graph.
+    shuffle(this.script.dependencies);
+    // Note we use Promise.allSettled instead of Promise.all so that we can
+    // collect all errors, instead of just the first one.
+    const dependencyResults = await Promise.allSettled(
+      this.script.dependencies.map((dependency) => {
+        return this.executor.execute(dependency.config);
+      })
+    );
+    const errors = new Set<Failure>();
+    const results: Array<[ScriptReference, Fingerprint]> = [];
+    for (let i = 0; i < dependencyResults.length; i++) {
+      const result = dependencyResults[i];
+      if (result.status === 'rejected') {
+        const error: unknown = result.reason;
+        errors.add({
+          type: 'failure',
+          reason: 'unknown-error-thrown',
+          script: this.script.dependencies[i].config,
+          error: error,
+        });
+      } else {
+        if (!result.value.ok) {
+          for (const error of result.value.error) {
+            errors.add(error);
+          }
+        } else {
+          results.push([
+            this.script.dependencies[i].config,
+            result.value.value,
+          ]);
+        }
+      }
+    }
+    if (errors.size > 0) {
+      return {ok: false, error: [...errors]};
+    }
+    return {ok: true, value: results};
+  }
+
+  /**
+   * Generate the fingerprint data object for this script based on its current
+   * configuration, input files, and the fingerprints of its dependencies.
+   */
+  protected async computeFingerprint(
+    dependencyFingerprints: Array<[ScriptReference, Fingerprint]>
+  ): Promise<Fingerprint> {
+    let allDependenciesAreCacheable = true;
+    const filteredDependencyStates: Array<
+      [ScriptReferenceString, Fingerprint]
+    > = [];
+    for (const [dep, depState] of dependencyFingerprints) {
+      if (!depState.cacheable) {
+        allDependenciesAreCacheable = false;
+      }
+      filteredDependencyStates.push([scriptReferenceToString(dep), depState]);
+    }
+
+    let fileHashes: Array<[string, Sha256HexDigest]>;
+    if (this.script.files?.values.length) {
+      const files = await glob(this.script.files.values, {
+        cwd: this.script.packageDir,
+        absolute: false,
+        followSymlinks: true,
+        // TODO(aomarks) This means that empty directories are not reflected in
+        // the fingerprint, however an empty directory could modify the behavior
+        // of a script. We should probably include empty directories; we'll just
+        // need special handling when we compute the fingerprint, because there
+        // is no hash we can compute.
+        includeDirectories: false,
+        // We must expand directories here, because we need the complete
+        // explicit list of files to hash.
+        expandDirectories: true,
+        throwIfOutsideCwd: false,
+      });
+      // TODO(aomarks) Instead of reading and hashing every input file on every
+      // build, use inode/mtime/ctime/size metadata (which is much faster to
+      // read) as a heuristic to detect files that have likely changed, and
+      // otherwise re-use cached hashes that we store in e.g.
+      // ".wireit/<script>/hashes".
+      fileHashes = await Promise.all(
+        files.map(async (file): Promise<[string, Sha256HexDigest]> => {
+          const absolutePath = resolve(this.script.packageDir, file.path);
+          const hash = createHash('sha256');
+          for await (const chunk of createReadStream(absolutePath)) {
+            hash.update(chunk as Buffer);
+          }
+          return [file.path, hash.digest('hex') as Sha256HexDigest];
+        })
+      );
+    } else {
+      fileHashes = [];
+    }
+
+    const cacheable =
+      // If command is undefined, then we simply propagate the fingerprints of
+      // our dependencies, and don't have any effect ourselves on cacheability.
+      this.script.command === undefined ||
+      // Otherwise, If files are undefined, then it's not safe to be cached,
+      // because we don't know what the inputs are, so we can't know if the
+      // output of this script could change.
+      (this.script.files !== undefined &&
+        // Similarly, if any of our dependencies are uncacheable, then we're
+        // uncacheable too, because that dependency could also have an effect on
+        // our output.
+        allDependenciesAreCacheable);
+
+    return {
+      cacheable,
+      platform: process.platform,
+      arch: process.arch,
+      nodeVersion: process.version,
+      command: this.script.command?.value,
+      clean: this.script.clean,
+      files: Object.fromEntries(
+        fileHashes.sort(([aFile], [bFile]) => aFile.localeCompare(bFile))
+      ),
+      output: this.script.output?.values ?? [],
+      dependencies: Object.fromEntries(
+        filteredDependencyStates.sort(([aRef], [bRef]) =>
+          aRef.localeCompare(bRef)
+        )
+      ),
+    };
+  }
+}

src/execution/no-op.ts

@@ -0,0 +1,45 @@
+/**
+ * @license
+ * Copyright 2022 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import {BaseExecution} from './base.js';
+
+import type {ExecutionResult} from './base.js';
+import type {Executor} from '../executor.js';
+import type {NoOpScriptConfig} from '../script.js';
+import type {Logger} from '../logging/logger.js';
+
+/**
+ * Execution for a {@link NoOpScriptConfig}.
+ */
+export class NoOpExecution extends BaseExecution<NoOpScriptConfig> {
+  static execute(
+    script: NoOpScriptConfig,
+    executor: Executor,
+    logger: Logger
+  ): Promise<ExecutionResult> {
+    return new NoOpExecution(script, executor, logger).#execute();
+  }
+
+  async #execute(): Promise<ExecutionResult> {
+    if (this.shouldNotStart) {
+      return {ok: false, error: [this.startCancelledEvent]};
+    }
+
+    const dependencyFingerprints = await this.executeDependencies();
+    if (!dependencyFingerprints.ok) {
+      return dependencyFingerprints;
+    }
+    const fingerprint = await this.computeFingerprint(
+      dependencyFingerprints.value
+    );
+    this.logger.log({
+      script: this.script,
+      type: 'success',
+      reason: 'no-command',
+    });
+    return {ok: true, value: fingerprint};
+  }
+}