feat(ci): Setup target-determinator

.github/workflows/workflow.yml

@@ -4,10 +4,9 @@ on:
   push:
     branches:
       - main
-      - release-*
-    tags:
-      - '*.*.*'
   pull_request:
+    branches:
+      - main
 
 jobs:
   check:
@@ -20,7 +19,14 @@ jobs:
     name: check - (${{ matrix.os }})
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v3
+      - name: 'PR commits + 1'
+        run: echo "PR_FETCH_DEPTH=$(( ${{ github.event.pull_request.commits }} + 1 ))" >> "${GITHUB_ENV}"
+
+      - name: 'Checkout PR branch and PR commits'
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.ref }}
+          fetch-depth: ${{ env.PR_FETCH_DEPTH }}
 
       # https://github.com/tweag/rules_haskell/issues/704
       - name: "[Haskell] Install required packages"
@@ -38,4 +44,7 @@ jobs:
         run: ./tools/ci/install_bazelisk.sh v1.16.0 ${{ matrix.os }}
 
       - name: Test
-        run: ./bin/bazel test --remote_header=x-buildbuddy-api-key=${{ secrets.BUILDBUDDY_API_KEY }} //...
+        env:
+          BASE_REF: ${{ github.event.pull_request.base.sha }}
+          BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }}
+        run: ./tools/ci/pr.sh

Pipfile

@@ -14,4 +14,5 @@ jinja2 = "*"
 autoflake = "*"
 isort = "*"
 pip-tools = "*"
+PyGithub = "*"
 yapf = "*"

WORKSPACE

@@ -461,3 +461,9 @@ container_deps()
 load("@io_bazel_rules_docker//go:image.bzl", _go_image_repos = "repositories")
 
 _go_image_repos()
+
+load("//tools/target-determinator:td.bzl", "driver_deps", "target_determinator_deps")
+
+driver_deps()
+
+target_determinator_deps()

go.mod

@@ -4,6 +4,8 @@ go 1.19
 
 require (
 	github.com/antlr/antlr4/runtime/Go/antlr v1.4.10
+	github.com/bazel-contrib/target-determinator v0.16.0
+	github.com/bazelbuild/bazel-gazelle v0.29.0
 	github.com/bazelbuild/rules_go v0.38.1
 	github.com/google/cel-go v0.13.0
 	github.com/google/go-cmp v0.5.9
@@ -27,6 +29,7 @@ require (
 
 require (
 	github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect
+	github.com/aristanetworks/goarista v0.0.0-20220211174905-526022c8b178 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cloudflare/circl v1.1.0 // indirect
@@ -41,7 +44,12 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect
 	github.com/stoewer/go-strcase v1.2.0 // indirect
+	github.com/tidwall/gjson v1.14.0 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
+	github.com/wI2L/jsondiff v0.2.0 // indirect
 	golang.org/x/crypto v0.7.0 // indirect
+	golang.org/x/exp v0.0.0-20220328175248-053ad81199eb // indirect
 	golang.org/x/net v0.8.0 // indirect
 	golang.org/x/oauth2 v0.6.0 // indirect
 	golang.org/x/sys v0.6.0 // indirect

go.sum

@@ -37,6 +37,12 @@ github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 h1:wPbRQzjjwF
 github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g=
 github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves=
 github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY=
+github.com/aristanetworks/goarista v0.0.0-20220211174905-526022c8b178 h1:U7Y+d65r8YW6PRIu+YaTqQGkmTx7PMI+oDVic5FTP8c=
+github.com/aristanetworks/goarista v0.0.0-20220211174905-526022c8b178/go.mod h1:9zxrD1FatJPUgxIsMwWVrALau7/v1sI1OJETI63r670=
+github.com/bazel-contrib/target-determinator v0.16.0 h1:2acmnbb+AfOYoSs/P4zSLmOiKH0p+FbLXITAi1Fxr4w=
+github.com/bazel-contrib/target-determinator v0.16.0/go.mod h1:j51obIjvMqH8lSf+q2W+C9jk61LW2/C5iYj6nobjewI=
+github.com/bazelbuild/bazel-gazelle v0.29.0 h1:uFYs0rkVtEwZ6PlVQAXkBrIvWtIljMG8YR67LGiopeY=
+github.com/bazelbuild/bazel-gazelle v0.29.0/go.mod h1:L6O3ptp0eoujyGx2KyJrv86w93S0hxgh9HB3Cbw0cdI=
 github.com/bazelbuild/rules_go v0.38.1 h1:YGNsLhWe18Ielebav7cClP3GMwBxBE+xEArLHtmXDx8=
 github.com/bazelbuild/rules_go v0.38.1/go.mod h1:TMHmtfpvyfsxaqfL9WnahCsXMWDMICTw7XeK9yVb+YU=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -164,6 +170,7 @@ github.com/mndrix/tap-go v0.0.0-20171203230836-629fa407e90b/go.mod h1:pzzDgJWZ34
 github.com/mvdan/sh v2.6.4+incompatible h1:D4oEWW0J8cL7zeQkrXw76IAYXF0mJfDaBwjgzmKb6zs=
 github.com/mvdan/sh v2.6.4+incompatible/go.mod h1:kipHzrJQZEDCMTNRVRAlMMFjqHEYrthfIlFkJSrmDZE=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/otiai10/copy v1.7.1-0.20211223015809-9aae5f77261f h1:P7Ab27T4In6ExIHmjOe88b1BHpuHlr4Vr75hX2QKAXw=
 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -200,6 +207,14 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w=
+github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/wI2L/jsondiff v0.2.0 h1:dE00WemBa1uCjrzQUUTE/17I6m5qAaN0EMFOg2Ynr/k=
+github.com/wI2L/jsondiff v0.2.0/go.mod h1:axTcwtBkY4TsKuV+RgoMhHyHKKFRI6nnjRLi8LLYQnA=
 github.com/whilp/git-urls v1.0.0 h1:95f6UMWN5FKW71ECsXRUd3FVYiXdrE7aX4NZKcPmIjU=
 github.com/whilp/git-urls v1.0.0/go.mod h1:J16SAmobsqc3Qcy98brfl5f5+e0clUvg1krgwk/qCfE=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -230,6 +245,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/exp v0.0.0-20220328175248-053ad81199eb h1:pC9Okm6BVmxEw76PUu0XUbOTQ92JX11hfvqTjAV3qxM=
+golang.org/x/exp v0.0.0-20220328175248-053ad81199eb/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=

src/go/ci/driver/BUILD.bazel

@@ -0,0 +1,20 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+    name = "driver_lib",
+    srcs = ["driver.go"],
+    importpath = "github.com/satreix/everest/src/go/ci/driver",
+    visibility = ["//visibility:private"],
+    deps = [
+        "@bazel_gazelle//label:go_default_library",
+        "@com_github_bazel_contrib_target_determinator//cli",
+        "@com_github_bazel_contrib_target_determinator//pkg",
+        "@com_github_bazel_contrib_target_determinator//third_party/protobuf/bazel/analysis",
+    ],
+)
+
+go_binary(
+    name = "driver",
+    embed = [":driver_lib"],
+    visibility = ["//visibility:public"],
+)

src/go/ci/driver/driver.go

@@ -0,0 +1,163 @@
+// driver is a binary for driving the CI process based on the affected targets.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/bazel-contrib/target-determinator/cli"
+	"github.com/bazel-contrib/target-determinator/pkg"
+	"github.com/bazel-contrib/target-determinator/third_party/protobuf/bazel/analysis"
+	"github.com/bazelbuild/bazel-gazelle/label"
+)
+
+type driverFlags struct {
+	commonFlags    *cli.CommonFlags
+	revisionBefore string
+}
+
+type config struct {
+	Context        *pkg.Context
+	RevisionBefore pkg.LabelledGitRev
+	Targets        pkg.TargetsList
+}
+
+func main() {
+	log.SetFlags(0)
+
+	flags, err := parseFlags()
+	if err != nil {
+		fmt.Fprintf(flag.CommandLine.Output(), "Failed to parse flags: %v\n", err)
+		fmt.Fprintf(flag.CommandLine.Output(), "Usage of %s:\n", os.Args[0])
+		fmt.Fprintf(flag.CommandLine.Output(), "  %s <before-revision>\n", filepath.Base(os.Args[0]))
+		fmt.Fprintf(flag.CommandLine.Output(), "Where <before-revision> may be any commit-like strings - full commit hashes, short commit hashes, tags, branches, etc.\n")
+		fmt.Fprintf(flag.CommandLine.Output(), "Optional flags:\n")
+		flag.PrintDefaults()
+		os.Exit(1)
+	}
+
+	config, err := resolveConfig(*flags)
+	if err != nil {
+		log.Fatalf("Error during preprocessing: %v", err)
+	}
+
+	var targets []label.Label
+	targetsSet := make(map[label.Label]struct{})
+	commandVerb := "build"
+
+	log.Println("Discovering affected targets")
+	callback := func(l label.Label, differences []pkg.Difference, configuredTarget *analysis.ConfiguredTarget) {
+		if isTaggedManual(configuredTarget) {
+			return
+		}
+		if _, seen := targetsSet[l]; seen {
+			return
+		}
+		targets = append(targets, l)
+		targetsSet[l] = struct{}{}
+		// This is not an ideal heuristic, ideally cquery would expose to us whether a target is a test target.
+		if strings.HasSuffix(configuredTarget.GetTarget().GetRule().GetRuleClass(), "_test") {
+			commandVerb = "test"
+		}
+	}
+
+	if err := pkg.WalkAffectedTargets(config.Context,
+		config.RevisionBefore,
+		config.Targets,
+		false,
+		callback); err != nil {
+		log.Fatal(err)
+	}
+
+	if len(targets) == 0 {
+		log.Println("No targets were affected, not running Bazel")
+		os.Exit(0)
+	}
+
+	log.Printf("Discovered %d affected targets", len(targets))
+
+	targetPatternFile, err := os.CreateTemp("", "")
+	if err != nil {
+		log.Fatalf("Failed to create temporary file for target patterns: %v", err)
+	}
+	for _, target := range targets {
+		if _, err := targetPatternFile.WriteString(target.String()); err != nil {
+			log.Fatalf("Failed to write target pattern to target pattern file: %v", err)
+		}
+		if _, err := targetPatternFile.WriteString("\n"); err != nil {
+			log.Fatalf("Failed to write target pattern to target pattern file: %v", err)
+		}
+	}
+	if err := targetPatternFile.Sync(); err != nil {
+		log.Fatalf("Failed to sync target pattern file: %v", err)
+	}
+	if err := targetPatternFile.Close(); err != nil {
+		log.Fatalf("Failed to close target pattern file: %v", err)
+	}
+
+	buildBuddyAPIToken := os.Getenv("BUILDBUDDY_API_KEY")
+
+	args := []string{
+		commandVerb,
+		"--target_pattern_file", targetPatternFile.Name(),
+	}
+
+	if buildBuddyAPIToken != "" {
+		args = append(args, "--remote_header=x-buildbuddy-api-key="+buildBuddyAPIToken)
+	}
+
+	log.Printf("Running %s on %d targets", commandVerb, len(targets))
+	result, err := config.Context.BazelCmd.Execute(
+		pkg.BazelCmdConfig{Dir: config.Context.WorkspacePath, Stdout: os.Stdout, Stderr: os.Stderr},
+		[]string{},
+		commandVerb,
+		args...,
+	)
+	if err != nil || result != 0 {
+		log.Fatal(err)
+	}
+}
+
+func isTaggedManual(target *analysis.ConfiguredTarget) bool {
+	for _, attr := range target.GetTarget().GetRule().GetAttribute() {
+		if attr.GetName() == "tags" {
+			for _, tag := range attr.GetStringListValue() {
+				if tag == "manual" {
+					return true
+				}
+			}
+		}
+	}
+	return false
+}
+
+func parseFlags() (*driverFlags, error) {
+	var flags driverFlags
+	flags.commonFlags = cli.RegisterCommonFlags()
+	flag.Parse()
+
+	var err error
+	flags.revisionBefore, err = cli.ValidateCommonFlags()
+	if err != nil {
+		return nil, err
+	}
+
+	return &flags, nil
+}
+
+func resolveConfig(flags driverFlags) (*config, error) {
+	commonArgs, err := cli.ResolveCommonConfig(flags.commonFlags, flags.revisionBefore)
+	if err != nil {
+		return nil, err
+	}
+
+	return &config{
+		Context:        commonArgs.Context,
+		RevisionBefore: commonArgs.RevisionBefore,
+		Targets:        commonArgs.Targets,
+	}, nil
+}

src/python/gh-release-to-bzl/BUILD.bazel

@@ -0,0 +1,15 @@
+load("@rules_python//python:defs.bzl", "py_binary")
+load("//third_party/python:requirements.bzl", "requirement")
+
+py_binary(
+    name = "gh-release-to-bzl",
+    srcs = ["__main__.py"],
+    main = "__main__.py",
+    visibility = ["//visibility:public"],
+    deps = [
+        requirement("jinja2"),
+        requirement("markupsafe"),
+        requirement("PyGithub"),
+        requirement("requests"),
+    ],
+)