feat(worker): new worker package

Author: alexeagle

WORKSPACE

@@ -370,4 +370,5 @@ local_repository(
     "vendored_node_and_yarn",
     "web_testing",
     "webapp",
+    "worker",
 ]]

commitlint.config.js

@@ -18,6 +18,7 @@ module.exports = {
         'stylus',
         'rollup',
         'typescript',
+        'worker',
       ]
     ]
   }

examples/BUILD.bazel

@@ -228,3 +228,24 @@ bazel_integration_test(
     ],
     workspace_files = "@examples_vendored_node//:all_files",
 )
+
+bazel_integration_test(
+    name = "examples_worker",
+    # There are no tests in this example
+    bazel_commands = [
+        # By default this will build with worker enabled
+        "build //:do_work",
+        # Build again without the worker
+        "build --define=cache_bust=true --strategy=DoWork=standalone //:do_work",
+    ],
+    bazelrc_imports = {
+        "//:common.bazelrc": "import %workspace%/../../common.bazelrc",
+    },
+    check_npm_packages = NPM_PACKAGES,
+    npm_packages = {"//packages/worker:npm_package": "@bazel/worker"},
+    repositories = {
+        "//:release": "build_bazel_rules_nodejs",
+    },
+    tags = ["examples"],
+    workspace_files = "@examples_worker//:all_files",
+)

examples/worker/.bazelrc

@@ -0,0 +1 @@
+import %workspace%/../../common.bazelrc

examples/worker/BUILD.bazel

@@ -0,0 +1,35 @@
+load("@build_bazel_rules_nodejs//:defs.bzl", "nodejs_binary")
+load(":uses_workers.bzl", "work")
+
+# This is our program that we want to run as a worker
+# Imagine that it takes a long time to start, or benefits from caching work
+nodejs_binary(
+    name = "tool",
+    # For the integration test, allow a second bazel build
+    # to explicitly be a cache miss, letting us test both
+    # worker and standalone modes.
+    configuration_env_vars = ["cache_bust"],
+    data = ["@npm//@bazel/worker"],
+    entry_point = ":tool.js",
+)
+
+# How a user would call our rule that uses workers.
+work(
+    name = "do_work",
+    src = "foo.js",
+)
+
+# For running this example as a bazel_integration_test
+# See //examples:BUILD.bazel
+filegroup(
+    name = "all_files",
+    srcs = glob(
+        include = ["**/*"],
+        exclude = [
+            "bazel-out/**/*",
+            "dist/**/*",
+            "node_modules/**/*",
+        ],
+    ),
+    visibility = ["//visibility:public"],
+)

examples/worker/README.md

@@ -0,0 +1,25 @@
+# Worker example
+
+This shows how to keep a tool running a persistent worker. This is like a daemon process that Bazel will start and manage as needed to perform actions.
+
+Bazel's protocol for workers is:
+
+- start a pool of processes
+- when an action needs to be run, it encodes the request as a protocol buffer and writes it to the worker's stdin
+- the `@bazel/worker` package provides a utility to speak this protocol, and dispatches to a function you provide that performs the work of the tool. See /packages/worker/README.md for a description of that utility.
+- the tool returns a response written as another protocol buffer to stdout (note this means you cannot log to stdout)
+
+## Files in the example
+
+`foo.js` is some arbitrary input to the rule. You can run `ibazel build :do_work` and then make edits to this JS input to observe how every change triggers the action to run, and it's quite fast because the worker process stays running.
+
+The `tool.js` file shows how to use the `@bazel/worker` package to implement the worker protocol.
+Note that the main method first checks whether the tool is being run under the worker mode, or should just do the work once and exit.
+
+`uses_workers.bzl` shows how the tool is wrapped in a Bazel rule. When the action is declared, we mark it with attribute `execution_requirements = {"supports-workers": "1"}` which informs Bazel that it speaks the worker protocol. Bazel will decide whether to actually keep the process running as a persistent worker.
+
+By also providing `mnemonic` attribute to the action, users will be able to control the scheduling if desired.
+Note the `--strategy=DoWork=standalone` flag passed to Bazel in the integration test in the /examples directory. This tells Bazel not to use workers. Similarly the user could set some other strategy like `--strategy=DoWork=worker` to explicitly opt-in.
+
+`BUILD.bazel` defines the binary for the tool, then shows how it would be used by calling `work()`. Note that the usage site just calls the rule without knowing whether it uses workers for performance.
+

examples/worker/WORKSPACE

@@ -0,0 +1,34 @@
+# Copyright 2019 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+workspace(
+    name = "examples_worker",
+    managed_directories = {"@npm": ["node_modules"]},
+)
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+http_archive(
+    name = "build_bazel_rules_nodejs",
+    sha256 = "6625259f9f77ef90d795d20df1d0385d9b3ce63b6619325f702b6358abb4ab33",
+    urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/0.35.0/rules_nodejs-0.35.0.tar.gz"],
+)
+
+load("@build_bazel_rules_nodejs//:defs.bzl", "yarn_install")
+
+yarn_install(
+    name = "npm",
+    package_json = "//:package.json",
+    yarn_lock = "//:yarn.lock",
+)

examples/worker/foo.js

@@ -0,0 +1,3 @@
+// This is an arbitrary file used as an input to the worker action
+// Any time this file is changed, the action will need to re-run
+export const num = 0;

examples/worker/package.json

@@ -0,0 +1,6 @@
+{
+    "private": true,
+    "devDependencies": {
+        "@bazel/worker": "latest"
+    }
+}

examples/worker/tool.js

@@ -0,0 +1,52 @@
+/**
+ * @fileoverview this program does a trivial job of writing a dummy string to an output
+ */
+const worker = require('@bazel/worker');
+
+function runOneBuild(args, inputs) {
+  // IMPORTANT don't log with console.out - stdout is reserved for the worker protocol.
+  // This is true for any code running in the program, even if it comes from a third-party library.
+  worker.log('Performing a build with args', args);
+  if (inputs) {
+    // The inputs help you manage a cache within the worker process
+    // They are available only when run as a worker, not in standalone mode
+    worker.log('We were run as a worker so we also got a manifest of all the inputs', inputs);
+  }
+
+  // Parse our arguments as usual. The worker library handles getting these out of the protocol
+  // buffer.
+  const [output] = args;
+  require('fs').writeFileSync(output, 'Dummy output', {encoding: 'utf-8'});
+
+  // Return true if the tool succeeded, false otherwise.
+  return true;
+}
+
+if (require.main === module) {
+  // One reason to run a program under a worker is that it takes a long time to start
+  // Imagine that several seconds are spent here
+
+  // Bazel will pass a special argument to the program when it's running us as a worker
+  if (worker.runAsWorker(process.argv)) {
+    worker.log('Running as a Bazel worker');
+
+    worker.runWorkerLoop(runOneBuild);
+  } else {
+    // Running standalone so stdout is available as usual
+    console.log('Running as a standalone process');
+
+    // Help our users get on the fast path
+    console.error(
+        'Started a new process to perform this action. Your build might be misconfigured, try --strategy=DoWork=worker');
+
+    // The first argument to the program is prefixed with '@'
+    // because Bazel does that for param files. Strip it first.
+    const paramFile = process.argv[2].replace(/^@/, '');
+    const args = require('fs').readFileSync(paramFile, 'utf-8').trim().split('\n');
+
+    // Bazel is just running the program as a single action, don't act like a worker
+    if (!runOneBuild(args)) {
+      process.exitCode = 1;
+    }
+  }
+}