feat(bolt): run tests in containers

.gitignore

@@ -30,9 +30,11 @@ Bolt.local.toml
 
 # Generated code
 gen/build_script.sh
+gen/test_build_script.sh
 gen/svc/
 gen/tf/
 gen/docker/
+gen/tests/
 gen/k8s/
 
 # Rust

infra/misc/svc_scripts/install_ca.sh

@@ -8,4 +8,3 @@ exec >> "/var/log/install-ca.txt" 2>&1
 #
 # Overriding LD_LIBRARY_PATH prevents apt from using the OpenSSL installation from /nix/store (if mounted).
 LD_LIBRARY_PATH=/lib:/usr/lib:/usr/local/lib update-ca-certificates
-

lib/bolt/core/src/dep/cargo/cli.rs

@@ -4,17 +4,21 @@ use anyhow::{ensure, Context, Result};
 use indoc::{formatdoc, indoc};
 use regex::Regex;
 use serde_json::json;
-use tokio::{fs, io::AsyncReadExt, process::Command, task::block_in_place};
+use tokio::{fs, process::Command, task::block_in_place};
+use uuid::Uuid;
 
-use crate::{config, context::ProjectContext};
+use crate::context::ProjectContext;
 
 const DOCKERIGNORE: &str = indoc!(
 	r#"
 	*
 	# Rivet
 	!Bolt.toml
+	!infra/misc/svc_scripts
+	!oss/infra/misc/svc_scripts
 	!gen/docker
 	!gen/build_script.sh
+	!gen/test_build_script.sh
 	sdks/runtime
 	oss/sdks/runtime
 	!lib
@@ -184,18 +188,18 @@ pub async fn build<'a, T: AsRef<str>>(ctx: &ProjectContext, opts: BuildOpts<'a,
 					COPY . .
 					COPY {build_script_path} build_script.sh
 
+					# Build and copy all binaries from target directory into an empty image (it is not
+					# included in the output because of cache mount)
 					RUN chmod +x ./build_script.sh
 					RUN \
 						--mount=type=cache,target=/usr/local/cargo/git \
 						--mount=type=cache,target=/usr/local/cargo/registry \
 						--mount=type=cache,target=/usr/rivet/target \
 						--mount=type=cache,target=/usr/rivet/oss/target \
 						sh -c ./build_script.sh && mkdir /usr/bin/rivet && find target/{optimization} -maxdepth 1 -type f ! -name "*.*" -exec mv {{}} /usr/bin/rivet/ \;
-
-					# Copy all binaries from target directory into an empty image (it is not included in
-					# the output because of cache mount)
 
-					# Create an empty image and copy binaries to it (this is to minimize the size of the image)
+					# Create an empty image and copy binaries + test outputs to it (this is to minimize the
+					# size of the image)
 					FROM scratch
 					COPY --from=rust /usr/bin/rivet/ /
 					"#,
@@ -300,8 +304,12 @@ pub async fn build<'a, T: AsRef<str>>(ctx: &ProjectContext, opts: BuildOpts<'a,
 	Ok(())
 }
 
+pub const TEST_IMAGE_NAME: &str = "test";
+
 pub struct BuildTestOpts<'a, T: AsRef<str>> {
 	pub build_calls: Vec<BuildTestCall<'a, T>>,
+	/// Builds for release mode.
+	pub release: bool,
 	/// How many threads to run in parallel when building.
 	pub jobs: Option<usize>,
 	pub test_filters: &'a [String],
@@ -324,44 +332,220 @@ pub async fn build_tests<'a, T: AsRef<str>>(
 	ctx: &ProjectContext,
 	opts: BuildTestOpts<'a, T>,
 ) -> Result<Vec<TestBinary>> {
+	let jobs_flag = if let Some(jobs) = opts.jobs {
+		format!("--jobs {jobs}")
+	} else {
+		String::new()
+	};
+
+	let build_calls =
+		opts.build_calls
+			.iter()
+			.map(|build_call| {
+				let path = build_call.path.display();
+				let package_flags = build_call
+					.packages
+					.iter()
+					.map(|x| format!("--package {}", x.as_ref()))
+					.collect::<Vec<_>>()
+					.join(" ");
+				// Generate a name from the build call path
+				let name = build_call.path.iter()
+					.map(|s| s.to_string_lossy())
+					.collect::<Vec<_>>()
+					.join("-");
+
+				format!("(cd {path} && cargo test --no-run --message-format=json-render-diagnostics {jobs_flag} {package_flags}) > gen/tests/{name}.out")
+			})
+			.collect::<Vec<_>>()
+			.join("\n");
+
+	let build_script_path = ctx.gen_path().join("test_build_script.sh");
+	let build_script_path_relative = build_script_path
+		.strip_prefix(ctx.path())
+		.context("failed to strip prefix")?;
+
+	// TODO: Not sure why the .cargo/config.toml isn't working with nested projects, have to hardcode
+	// the target dir
+	// Generate build script
+	let build_script = formatdoc!(
+		r#"
+		#!/bin/bash
+		set -euf
+
+		[ -z "${{CARGO_TARGET_DIR+x}}" ] && export CARGO_TARGET_DIR=$(readlink -f ./target)
+		# Used for Tokio Console. See https://github.com/tokio-rs/console#using-it
+		export RUSTFLAGS="--cfg tokio_unstable"
+
+		{build_calls}
+		"#,
+	);
+
+	// Write build script to file
+	fs::write(&build_script_path, build_script).await?;
+
 	let mut test_binaries = vec![];
-	for build_call in opts.build_calls {
-		let abs_path = ctx.path().join(build_call.path);
-
-		// Build command
-		let mut cmd = Command::new("cargo");
-		cmd.args(&[
-			"test",
-			"--no-run",
-			"--message-format=json-render-diagnostics",
-		])
-		.stdout(std::process::Stdio::piped())
-		.current_dir(abs_path)
-		// Used for Tokio Console. See https://github.com/tokio-rs/console#using-it
-		.env("RUSTFLAGS", "--cfg tokio_unstable");
-		if let Some(jobs) = opts.jobs {
-			cmd.args(&["--jobs", &jobs.to_string()]);
-		}
-		for test in build_call.packages {
-			cmd.args(&["--package", test.as_ref()]);
-		}
-		if std::env::var("CARGO_TARGET_DIR").is_err() {
-			cmd.env("CARGO_TARGET_DIR", ctx.cargo_target_dir());
-		}
-		let mut child = cmd.spawn()?;
 
-		// Capture stdout
-		let mut stdout = child.stdout.take().context("missing stdout")?;
-		let mut stdout_str = String::new();
-		stdout.read_to_string(&mut stdout_str).await?;
+	// Execute build command
+	let temp_container_name = if ctx.build_svcs_locally() {
+		// Create directory for test outputs
+		fs::create_dir_all(ctx.gen_path().join("tests")).await?;
+
+		// Make build script executable
+		let mut cmd = Command::new("chmod");
+		cmd.current_dir(ctx.path());
+		cmd.arg("+x");
+		cmd.arg(build_script_path.display().to_string());
+		let status = cmd.status().await?;
+		ensure!(status.success());
+
+		// Execute
+		let mut cmd = Command::new(build_script_path.display().to_string());
+		cmd.current_dir(ctx.path());
+		let status = cmd.status().await?;
+		ensure!(status.success());
+
+		None
+	} else {
+		let optimization = if opts.release { "release" } else { "debug" };
+		// Get repo to push to
+		let (push_repo, _) = ctx.docker_repos().await;
+		let source_hash = ctx.source_hash();
+
+		// Create directory for docker files
+		let gen_path = ctx.gen_path().join("docker");
+		fs::create_dir_all(&gen_path).await?;
+
+		let image_tag = format!("{push_repo}{TEST_IMAGE_NAME}:{source_hash}");
+		let dockerfile_path = gen_path.join(format!("Dockerfile.test_build"));
+
+		// Resolve the symlink for the svc_scripts dir since Docker does not resolve
+		// symlinks in COPY commands
+		let infra_path = ctx.path().join("infra");
+		let infra_path_resolved = tokio::fs::read_link(&infra_path)
+			.await
+			.map_or_else(|_| infra_path.clone(), |path| ctx.path().join(path));
+		let svc_scripts_path = infra_path_resolved.join("misc").join("svc_scripts");
+		let svc_scripts_path_relative = svc_scripts_path
+			.strip_prefix(ctx.path())
+			.context("failed to strip prefix")?;
+
+		// See above `build` fn for more info
+		fs::write(
+			&dockerfile_path,
+			formatdoc!(
+				r#"
+				# syntax=docker/dockerfile:1.2
+
+				FROM rust:1.77.2-slim AS build
+
+				RUN apt-get update && apt-get install -y protobuf-compiler pkg-config libssl-dev g++ git
+
+				RUN apt-get install --yes libpq-dev wget
+				RUN wget https://github.com/mozilla/sccache/releases/download/v0.2.15/sccache-v0.2.15-x86_64-unknown-linux-musl.tar.gz \
+					&& tar xzf sccache-v0.2.15-x86_64-unknown-linux-musl.tar.gz \
+					&& mv sccache-v0.2.15-x86_64-unknown-linux-musl/sccache /usr/local/bin/sccache \
+					&& chmod +x /usr/local/bin/sccache
+
+				WORKDIR /usr/rivet
+
+				COPY . .
+				COPY {build_script_path} build_script.sh
+
+				# TODO: Only copy test binaries (currently copies all binaries in target/opt/deps)
+				# Build and copy all binaries from target directory into an empty image (it is not
+				# included in the output because of cache mount)
+				RUN chmod +x ./build_script.sh
+				RUN mkdir -p gen/tests
+				RUN \
+					--mount=type=cache,target=/usr/local/cargo/git \
+					--mount=type=cache,target=/usr/local/cargo/registry \
+					--mount=type=cache,target=/usr/rivet/target \
+					--mount=type=cache,target=/usr/rivet/oss/target \
+					sh -c ./build_script.sh && mkdir /usr/bin/rivet && find target/{optimization}/deps -maxdepth 1 -type f ! -name "*.*" -exec mv {{}} /usr/bin/rivet/ \;
+
+				FROM debian:12.1-slim AS run
+
+				# Update ca-certificates. Install curl for health checks.
+				RUN DEBIAN_FRONTEND=noninteractive apt-get update -y && apt-get install -y --no-install-recommends ca-certificates openssl curl
+
+				# Copy supporting scripts
+				COPY {svc_scripts_path}/health_check.sh {svc_scripts_path}/install_ca.sh /usr/bin/
+				RUN chmod +x /usr/bin/health_check.sh /usr/bin/install_ca.sh
+
+				# Copy generated test outputs
+				COPY --from=build /usr/rivet/gen/tests/ /usr/rivet/gen/tests/
+
+				# Copy final binaries
+				COPY --from=build /usr/bin/rivet/ /usr/bin/rivet/
+				"#,
+				build_script_path = build_script_path_relative.display(),
+				svc_scripts_path = svc_scripts_path_relative.display(),
+			),
+		)
+		.await?;
+
+		let dockerignore_path = gen_path.join("Dockerfile.test_build.dockerignore");
+		fs::write(&dockerignore_path, DOCKERIGNORE).await?;
+
+		// Build image
+		let mut cmd = Command::new("docker");
+		cmd.env("DOCKER_BUILDKIT", "1");
+		cmd.current_dir(ctx.path());
+		cmd.arg("build");
+		cmd.arg("-f").arg(dockerfile_path);
+		// Prints plain console output for debugging
+		// cmd.arg("--progress=plain");
+		cmd.arg("-t").arg(&image_tag);
+		cmd.arg(".");
+
+		let status = cmd.status().await?;
+		ensure!(status.success(), "failed to run build command");
+
+		// TODO: Find better way to copy files from image?
+		let temp_container_name = Uuid::new_v4().to_string();
+		let output_path = ctx.gen_path().display().to_string();
+
+		// Create the temporary container
+		let mut cmd = Command::new("docker");
+		cmd.current_dir(ctx.path());
+		cmd.arg("run");
+		cmd.arg("--rm");
+		cmd.arg("-d");
+		cmd.arg("--name").arg(&temp_container_name);
+		cmd.arg(&image_tag);
+		cmd.arg("sleep").arg("120");
+
+		let output = cmd.output().await?;
+		ensure!(output.status.success(), "failed to run temp container");
+
+		// Copy the files from the temporary container
+		let mut cmd = Command::new("docker");
+		cmd.current_dir(ctx.path());
+		cmd.arg("cp")
+			.arg(format!("{}:/usr/rivet/gen/tests", temp_container_name))
+			.arg(output_path);
+
+		let status = cmd.status().await?;
+		ensure!(status.success(), "failed to copy files from container");
+
+		Some(temp_container_name)
+	};
+
+	for build_call in opts.build_calls {
+		let name = build_call
+			.path
+			.iter()
+			.map(|s| s.to_string_lossy())
+			.collect::<Vec<_>>()
+			.join("-");
+		let output_path = ctx.gen_path().join("tests").join(format!("{name}.out"));
 
-		// Wait for finish
-		let status = child.wait().await?;
-		ensure!(status.success(), "build test failed");
+		let stdout = fs::read_to_string(output_path).await?;
 
 		// Parse artifacts
 		let test_count_re = Regex::new(r"(?m)^(.*): test$").unwrap();
-		for line in stdout_str.lines() {
+		for line in stdout.lines() {
 			let v = serde_json::from_str::<serde_json::Value>(line).context("invalid json")?;
 			if v["reason"] == "compiler-artifact" && v["target"]["kind"] == json!(["test"]) {
 				if let Some(executable) = v["filenames"][0].as_str() {
@@ -382,21 +566,54 @@ pub async fn build_tests<'a, T: AsRef<str>>(
 						.context("missing target name")?;
 
 					// Parse the test count from the binary
-					let test_list_args = [
-						&["--list".to_string(), "--format".into(), "terse".into()],
-						opts.test_filters,
-					]
-					.concat();
-					let test_list_stdout =
-						block_in_place(|| duct::cmd(executable, &test_list_args).read())?;
+					let (exec_path, test_list_stdout) =
+						if let Some(temp_container_name) = &temp_container_name {
+							let exec_name = Path::new(executable);
+							let exec_path = Path::new("/usr/bin/rivet")
+								.join(exec_name.file_name().expect("no file name"));
+
+							let test_list_args = [
+								&[
+									"exec".to_string(),
+									temp_container_name.clone(),
+									exec_path.display().to_string(),
+									"--list".into(),
+									"--format".into(),
+									"terse".into(),
+								],
+								opts.test_filters,
+							]
+							.concat();
+
+							(
+								exec_path,
+								block_in_place(|| duct::cmd("docker", &test_list_args).read())?,
+							)
+						} else {
+							// Make path relative to project
+							let relative_path = Path::new(executable)
+								.strip_prefix(ctx.cargo_target_dir())
+								.context(format!("path not in project: {executable}"))?;
+
+							let test_list_args = [
+								&["--list".to_string(), "--format".into(), "terse".into()],
+								opts.test_filters,
+							]
+							.concat();
+
+							(
+								relative_path.to_path_buf(),
+								block_in_place(|| duct::cmd(executable, &test_list_args).read())?,
+							)
+						};
 
 					// Run a test container for every test in the binary
 					for cap in test_count_re.captures_iter(&test_list_stdout) {
 						let test_name = &cap[1];
 						test_binaries.push(TestBinary {
 							package: package.to_string(),
 							target: target.to_string(),
-							path: PathBuf::from(executable),
+							path: exec_path.clone(),
 							test_name: test_name.to_string(),
 						})
 					}