From edb36752eef5f17551643c1609f156a331d23c34 Mon Sep 17 00:00:00 2001
From: aleksdmladenovic <aleksandr.mladenovic@gmail.com>
Date: Fri, 17 May 2024 15:30:54 +0000
Subject: [PATCH] Add remote execution example for NativeLink

---
 .../remote_execution/nativelink/.buckconfig   | 13 +++++
 .../remote_execution/nativelink/.buckroot     |  0
 .../remote_execution/nativelink/.gitignore    |  1 +
 .../remote_execution/nativelink/README.md     | 58 +++++++++++++++++++
 .../nativelink/platforms/BUCK                 |  3 +
 .../nativelink/platforms/defs.bzl             | 33 +++++++++++
 .../nativelink/prelude/prelude.bzl            |  0
 .../remote_execution/nativelink/tests/BUCK    |  3 +
 .../nativelink/tests/defs.bzl                 | 50 ++++++++++++++++
 9 files changed, 161 insertions(+)
 create mode 100644 examples/remote_execution/nativelink/.buckconfig
 create mode 100644 examples/remote_execution/nativelink/.buckroot
 create mode 100644 examples/remote_execution/nativelink/.gitignore
 create mode 100644 examples/remote_execution/nativelink/README.md
 create mode 100644 examples/remote_execution/nativelink/platforms/BUCK
 create mode 100644 examples/remote_execution/nativelink/platforms/defs.bzl
 create mode 100644 examples/remote_execution/nativelink/prelude/prelude.bzl
 create mode 100644 examples/remote_execution/nativelink/tests/BUCK
 create mode 100644 examples/remote_execution/nativelink/tests/defs.bzl

diff --git a/examples/remote_execution/nativelink/.buckconfig b/examples/remote_execution/nativelink/.buckconfig
new file mode 100644
index 0000000000000..894aba54aa216
--- /dev/null
+++ b/examples/remote_execution/nativelink/.buckconfig
@@ -0,0 +1,13 @@
+[cells]
+root = .
+prelude = prelude
+
+[buck2_re_client]
+action_cache_address = grpc://localhost:50051
+engine_address = grpc://localhost:50051
+cas_address = grpc://localhost:50051
+tls = false
+instance_name = main
+
+[build]
+  execution_platforms = root//platforms:platforms
diff --git a/examples/remote_execution/nativelink/.buckroot b/examples/remote_execution/nativelink/.buckroot
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/examples/remote_execution/nativelink/.gitignore b/examples/remote_execution/nativelink/.gitignore
new file mode 100644
index 0000000000000..d60c5d299b29c
--- /dev/null
+++ b/examples/remote_execution/nativelink/.gitignore
@@ -0,0 +1 @@
+buck-out
diff --git a/examples/remote_execution/nativelink/README.md b/examples/remote_execution/nativelink/README.md
new file mode 100644
index 0000000000000..28019f7c07de5
--- /dev/null
+++ b/examples/remote_execution/nativelink/README.md
@@ -0,0 +1,58 @@
+# Remote execution integration with NativeLink
+
+This project provides a small example of what a project that utilizes
+[NativeLink](https://github.com/Tracemachina/nativelink).
+
+In this document, we will go over the key configs used in this setup. If you
+already have a `NativeLink` deployment you can use that instead.
+
+## Deploy a local NativeLink
+
+### 📦 Installing with Cargo
+
+1. First install Rust, but skip to step 2 if you have it already.
+```bash
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
+```
+2. Install NativeLink with Cargo.
+```bash
+cargo install --git https://github.com/TraceMachina/nativelink --tag v0.4.0
+```
+
+### ⚙️ Configure and 🦾 Start NativeLink
+
+The `nativelink` executable reads a JSON file as it's only parameter,
+`--config`. See [`nativelink-config`](https://github.com/TraceMachina/nativelink/tree/main/nativelink-config/examples/basic_cas.json)
+for more details and examples.
+
+To grab the example in your current working directory, run:
+
+```bash
+curl -O https://raw.githubusercontent.com/TraceMachina/nativelink/main/nativelink-config/examples/basic_cas.json
+
+### you can modify the example above to replace the filesystem store with the memory store if you favor speed over data durability.
+nativelink basic_cas.json
+```
+
+More information is available in the
+[repo](https://github.com/Tracemachina/nativelink).
+
+## Relevant configs in .buckconfig
+
+Configure the `NativeLink` endpoint as follows:
+
+```ini
+[buck2_re_client]
+action_cache_address = grpc://localhost:50051
+engine_address = grpc://localhost:50051
+cas_address = grpc://localhost:50051
+tls = false
+instance_name = main
+```
+
+TLS is not used in this example.
+
+## Relevant configs in `ExecutionPlatformInfo`
+
+`NativeLink` takes in a Docker image and `OSFamily` in its RE properties to
+select a worker. This is configured in `root//platforms:platforms`.
diff --git a/examples/remote_execution/nativelink/platforms/BUCK b/examples/remote_execution/nativelink/platforms/BUCK
new file mode 100644
index 0000000000000..63f852afecbda
--- /dev/null
+++ b/examples/remote_execution/nativelink/platforms/BUCK
@@ -0,0 +1,3 @@
+load(":defs.bzl", "platforms")
+
+platforms(name = "platforms")
diff --git a/examples/remote_execution/nativelink/platforms/defs.bzl b/examples/remote_execution/nativelink/platforms/defs.bzl
new file mode 100644
index 0000000000000..754201cbbeea8
--- /dev/null
+++ b/examples/remote_execution/nativelink/platforms/defs.bzl
@@ -0,0 +1,33 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under both the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree and the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree.
+
+def _platforms(ctx):
+    configuration = ConfigurationInfo(
+        constraints = {},
+        values = {},
+    )
+
+    platform = ExecutionPlatformInfo(
+        label = ctx.label.raw_target(),
+        configuration = configuration,
+        executor_config = CommandExecutorConfig(
+            local_enabled = True,
+            remote_enabled = True,
+            use_limited_hybrid = True,
+            # Set those up based on what workers you've registered with NativeLink.
+            remote_execution_properties = {
+                "OSFamily": "linux",
+                "container-image": "docker://ghcr.io/catthehacker/ubuntu:act-22.04@sha256:5f9c35c25db1d51a8ddaae5c0ba8d3c163c5e9a4a6cc97acd409ac7eae239448",
+            },
+            remote_execution_use_case = "buck2-default",
+            remote_output_paths = "output_paths",
+        ),
+    )
+
+    return [DefaultInfo(), ExecutionPlatformRegistrationInfo(platforms = [platform])]
+
+platforms = rule(attrs = {}, impl = _platforms)
diff --git a/examples/remote_execution/nativelink/prelude/prelude.bzl b/examples/remote_execution/nativelink/prelude/prelude.bzl
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/examples/remote_execution/nativelink/tests/BUCK b/examples/remote_execution/nativelink/tests/BUCK
new file mode 100644
index 0000000000000..8200f35ccd441
--- /dev/null
+++ b/examples/remote_execution/nativelink/tests/BUCK
@@ -0,0 +1,3 @@
+load(":defs.bzl", "tests")
+
+tests(name = "tests")
diff --git a/examples/remote_execution/nativelink/tests/defs.bzl b/examples/remote_execution/nativelink/tests/defs.bzl
new file mode 100644
index 0000000000000..3533c234c001c
--- /dev/null
+++ b/examples/remote_execution/nativelink/tests/defs.bzl
@@ -0,0 +1,50 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under both the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree and the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree.
+
+def _tests(ctx):
+    # Create locally
+    stage0 = ctx.actions.declare_output("stage0")
+    ctx.actions.run(
+        ["sh", "-c", 'head -c 10 /dev/urandom > "$1"', "--", stage0.as_output()],
+        category = "stage0",
+        local_only = True,
+    )
+
+    # Use on RE
+    stage1 = ctx.actions.declare_output("stage1")
+    ctx.actions.run(["sh", "-c", 'cat "$1" "$1" > "$2"', "--", stage0, stage1.as_output()], category = "stage1")
+
+    # Reuse on RE
+    stage2 = ctx.actions.declare_output("stage2")
+    ctx.actions.run(["sh", "-c", 'cat "$1" "$1" > "$2"', "--", stage1, stage2.as_output()], category = "stage2")
+
+    # Reuse locally
+    stage3 = ctx.actions.declare_output("stage3")
+    ctx.actions.run(
+        ["sh", "-c", 'cat "$1" "$1" > "$2"', "--", stage2, stage3.as_output()],
+        category = "stage3",
+        local_only = True,
+    )
+
+    # Verify
+    stage4 = ctx.actions.declare_output("stage4")
+    ctx.actions.run(
+        [
+            "sh",
+            "-c",
+            'cat "$1" "$1" "$1" "$1" "$1" "$1" "$1" "$1" > "$3" && diff "$2" "$3"',
+            "--",
+            stage0,
+            stage3,
+            stage4.as_output(),
+        ],
+        category = "stage4",
+    )
+
+    return [DefaultInfo(stage4)]
+
+tests = rule(attrs = {}, impl = _tests)