Scaffold for buildkite CI. Fixes #4801

.buildkite/agent.nix

@@ -0,0 +1,19 @@
+# Minimal nix shell config for running local agents for testing purposes
+#
+# Usage:
+#
+# nix-shell agent.nix
+# buildkite-agent start --tags size=small --token <TOKEN> --build-path ~/buildkite
+#
+# Replace `tags` with anything relevant and token with the buildkite token for O(1) Labs
+
+{ pkgs ? import <nixpkgs> {} }:
+  pkgs.mkShell {
+    buildInputs = [ pkgs.buildkite-agent pkgs.dhall pkgs.dhall-json ];
+    shellHook = ''
+      mkdir -p ~/buildkite
+      export GIT_SSL_CAINFO=/etc/ssl/certs/ca-certificates.crt
+      export SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
+    '';
+}
+

.buildkite/pipeline.dhall

@@ -0,0 +1,2 @@
+./src/Monorepo.dhall
+

.buildkite/scripts/generate-diff.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Base against origin/develop by default, but use pull-request base otherwise
+BASE=${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-origin/develop}
+
+# Finds the greatest common commit that is shared between these two branches
+# Or nothing if there isn't one
+COMMIT=$(diff -u <(git rev-list --first-parent HEAD) \
+        <(git rev-list --first-parent $BASE) | \
+        sed -ne 's/^ //p' | head -1)
+
+if [[ $COMMIT != "" ]]; then
+  # Get the files that have changed since that shared commit
+  git diff $COMMIT --name-only
+else
+  # TODO: Dump commits as artifacts when build succeeds so we can diff against
+  # that on develop instead of always running all the tests
+  git ls-files
+fi
+

.buildkite/src/External/Prelude.dhall

@@ -0,0 +1 @@
+https://prelude.dhall-lang.org/v15.0.0/package.dhall sha256:6b90326dc39ab738d7ed87b970ba675c496bed0194071b332840a87261649dcd

.buildkite/src/Jobs.dhall

@@ -0,0 +1,6 @@
+-- Add any new jobs here
+[
+  ./jobs/Sample/Spec.dhall,
+  ./jobs/Sample2/Spec.dhall
+]
+

.buildkite/src/Lib/Command.dhall

@@ -0,0 +1,64 @@
+-- Commands are the individual command steps that CI runs
+
+let Prelude = ../External/Prelude.dhall
+
+let Map = ../Lib/Map.dhall
+
+let Shared =
+    { Type =
+        { command : List Text
+        , label : Text
+        , key : Text
+        }
+    , default = {=}
+    }
+
+-- Everything here is taken directly from the buildkite Command documentation
+-- https://buildkite.com/docs/pipelines/command-step#command-step-attributes
+-- except "target" replaces "agents"
+--
+-- Target is our explicit union of large or small instances. As we build more
+-- complicated targeting rules we can replace this abstraction with something
+-- more powerful.
+let Config =
+    { Type = Shared.Type  //\\
+        { target : <Large | Small>
+        , depends_on : List Text
+        }
+    , default = Shared.default /\ {
+        depends_on = [] : List Text
+      }
+    }
+in
+-- The result type wraps our containers in optionals so that they are omitted
+-- from the rendered yaml if they are empty.
+let Result =
+  { Type = Shared.Type //\\
+    { agents : Optional (Map.Type Text)
+    , depends_on : Optional (List Text)
+    }
+  , default = Shared.default /\ {
+      depends_on = None (List Text)
+    }
+  }
+in
+
+let targetToAgent = \(target : <Large | Small>) ->
+  merge { Large = [ { mapKey = "size", mapValue = "large" } ],
+          Small = [ { mapKey = "size", mapValue = "small" } ]
+        }
+        target
+
+let build : Config.Type -> Result.Type = \(c : Config.Type) ->
+  {
+    command = c.command,
+    label = c.label,
+    key = c.key,
+    depends_on = if Prelude.List.null Text c.depends_on then None (List Text) else Some c.depends_on,
+    agents =
+      let agents = targetToAgent c.target in
+      if Prelude.List.null (Map.Entry.Type Text) agents then None (Map.Type Text) else Some agents
+  }
+
+in {Config = Config, build = build} /\ Result
+

.buildkite/src/Lib/JobSpec.dhall

@@ -0,0 +1,8 @@
+-- Defines info used for selecting a job to run
+{
+  Type = {
+    name: Text,
+    dirtyWhen: Text
+  },
+  default = {=}
+}

.buildkite/src/Lib/Map.dhall

@@ -0,0 +1,4 @@
+-- Maps are lists of special "entry" records. This is made explicit here
+let Entry = { Type =  \(value: Type) -> { mapKey : Text, mapValue : value } } in
+{ Entry = Entry, Type = \(value : Type) -> List (Entry.Type value) }
+

.buildkite/src/Lib/Pipeline.dhall

@@ -0,0 +1,37 @@
+-- A Pipeline is a series of build steps
+--
+-- Pipelines are rendered by separate invocations to dhall-to-yaml when our
+-- monorepo triage step determines it be necessary.
+
+let Prelude = ../External/Prelude.dhall
+let List/map = Prelude.List.map
+
+let Command = ./Command.dhall
+let JobSpec = ./JobSpec.dhall
+
+let Result = {
+  Type = {
+    -- TODO: Union type with block steps
+    steps: List Command.Type
+  },
+  default = {=}
+}
+
+-- We build a pipeline out of a spec and the commands in a step
+let Config = {
+  Type = {
+    spec: JobSpec.Type,
+    -- TODO: Union type with block steps
+    steps: List Command.Config.Type
+  },
+  default = {=}
+}
+
+let build : Config.Type -> Result.Type = \(c : Config.Type) ->
+  let name = c.spec.name
+  let buildCommand = \(c : Command.Config.Type) ->
+    Command.build c // { key = "$${name}-${c.key}" }
+  in
+  { steps = List/map Command.Config.Type Command.Type buildCommand c.steps }
+
+in {Config = Config, build = build} /\ Result

.buildkite/src/Monorepo.dhall

@@ -0,0 +1,36 @@
+let Prelude = ./External/Prelude.dhall
+
+let Command = ./Lib/Command.dhall
+let JobSpec = ./Lib/JobSpec.dhall
+let Pipeline = ./Lib/Pipeline.dhall
+
+let jobs : List JobSpec.Type = ./Jobs.dhall
+
+-- precompute the diffed files as this command takes a few seconds to run
+let prepareCommand = "./.buildkite/scripts/generate-diff.sh > computed_diff.txt"
+
+-- Run a job if we touched a dirty path
+let makeCommand = \(job : JobSpec.Type) -> ''
+  if cat computed_diff.txt | grep -q ${job.dirtyWhen}; then
+      dhall-to-yaml --quoted <<< './.buildkite/src/jobs/${job.name}/Pipeline.dhall' | buildkite-agent pipeline upload
+  fi
+''
+
+let commands = Prelude.List.map JobSpec.Type Text makeCommand jobs
+
+in Pipeline.build Pipeline.Config::{
+  spec = JobSpec::{
+    name = "monorepo-triage",
+    -- TODO: Clean up this code so we don't need an unused dirtyWhen here
+    dirtyWhen = ""
+  },
+  steps = [
+    Command.Config::{
+      command = [ prepareCommand ] # commands,
+      label = "Monorepo triage",
+      key = "cmds",
+      target = <Large | Small>.Small
+    }
+  ]
+}
+

.buildkite/src/jobs/Sample/Pipeline.dhall

@@ -0,0 +1,12 @@
+let Pipeline = ../../Lib/Pipeline.dhall
+let Command = ../../Lib/Command.dhall
+
+in
+
+Pipeline.build
+  Pipeline.Config::{
+    spec = ./Spec.dhall,
+    steps = [
+      Command.Config::{ command = [ "echo \"hello\"" ], label = "Test Echo", key = "hello", target = <Large | Small>.Small }
+    ]
+  }

.buildkite/src/jobs/Sample/Spec.dhall

@@ -0,0 +1,7 @@
+let JobSpec = ../../Lib/JobSpec.dhall
+
+in
+JobSpec::{
+  dirtyWhen = "transition",
+  name = "Sample"
+}

.buildkite/src/jobs/Sample2/Pipeline.dhall

@@ -0,0 +1,12 @@
+let Pipeline = ../../Lib/Pipeline.dhall
+let Command = ../../Lib/Command.dhall
+
+in
+
+Pipeline.build
+  Pipeline.Config::{
+    spec = ./Spec.dhall,
+    steps = [
+      Command.Config::{ command = [ "echo \"hello2\"" ], label = "Test Echo2", key = "hello2", target = <Large | Small>.Small }
+    ]
+  }

.buildkite/src/jobs/Sample2/Spec.dhall

@@ -0,0 +1,7 @@
+let JobSpec = ../../Lib/JobSpec.dhall
+
+in
+JobSpec::{
+  dirtyWhen = "^src/lib",
+  name = "Sample2"
+}

CODEOWNERS

@@ -1,4 +1,5 @@
 /CODEOWNERS @nholland94 @bkase @imeckler
+/.buildkite @CodaProtocol/infra-eng-reviewers
 /frontend/ @CodaProtocol/product-eng-reviewers
 /dockerfiles/ @CodaProtocol/infra-eng-reviewers
 /scripts/ @CodaProtocol/infra-eng-reviewers @CodaProtocol/protocol-eng-reviewers