feat(pegboard): add gc service (#1160)

<!-- Please make sure there is an issue that this PR is correlated to. -->
Fixes RVTEE-621
## Changes

<!-- If there are frontend changes, please include screenshots. -->

infra/tf/infra_artifacts/pegboard_manager.tf

@@ -27,7 +27,7 @@ resource "null_resource" "pegboard_manager_build" {
 		# Variables
 		IMAGE_NAME="pegboard-manager:${local.pegboard_manager_src_hash}"
 		CONTAINER_NAME="temp-pegboard-manager-${local.pegboard_manager_src_hash}"
-		BINARY_PATH_IN_CONTAINER="/app/lib/pegboard/target/x86_64-unknown-linux-musl/release/pegboard-manager"
+		BINARY_PATH_IN_CONTAINER="/pegboard-manager"
 		DST_BINARY_PATH="${local.pegboard_manager_dst_binary_path}"
 
 		# Build the Docker image

lib/pegboard/manager/Dockerfile

@@ -1,4 +1,4 @@
-FROM clux/muslrust:1.80.0-stable
+FROM clux/muslrust:1.80.0-stable AS rust
 
 RUN ln -s /bin/g++ /bin/musl-g++
 
@@ -8,4 +8,15 @@ COPY svc/ svc/
 COPY errors/ errors/
 COPY Bolt.toml Bolt.toml
 COPY proto/ proto/
-RUN cd lib/pegboard/manager && RUSTFLAGS="--cfg tokio_unstable" cargo build --release
+RUN \
+	--mount=type=cache,target=/root/.cargo/git \
+	--mount=type=cache,target=/root/.cargo/registry \
+	--mount=type=cache,target=/app/lib/pegboard/target \
+	cd lib/pegboard/manager && RUSTFLAGS="--cfg tokio_unstable" cargo build --release && mkdir /usr/bin/rivet && mv /app/lib/pegboard/target/x86_64-unknown-linux-musl/release/pegboard-manager /usr/bin/rivet/
+
+# Create an empty image and copy binaries into it to minimize the size of the image
+FROM scratch
+COPY --from=rust /usr/bin/rivet/ /
+
+# Allows `docker create` to work even though this fails
+CMD [""]

lib/pegboard/manager/src/container/mod.rs

@@ -395,27 +395,34 @@ impl Container {
 		}
 
 		// Update stop_ts
-		if matches!(signal, Signal::SIGTERM) || pid.is_none() {
-			utils::query(|| async {
-				sqlx::query(indoc!(
+		if matches!(signal, Signal::SIGTERM | Signal::SIGKILL) || pid.is_none() {
+			let stop_ts_set = utils::query(|| async {
+				sqlx::query_as::<_, (bool,)>(indoc!(
 					"
 					UPDATE containers
 					SET stop_ts = ?2
-					container_id = ?1
+					WHERE
+						container_id = ?1 AND
+						stop_ts IS NULL
+					RETURNING 1
 					",
 				))
 				.bind(self.container_id)
 				.bind(utils::now())
-				.execute(&mut *ctx.sql().await?)
+				.fetch_optional(&mut *ctx.sql().await?)
 				.await
 			})
-			.await?;
-
-			ctx.event(protocol::Event::ContainerStateUpdate {
-				container_id: self.container_id,
-				state: protocol::ContainerState::Stopped,
-			})
-			.await?;
+			.await?
+			.is_some();
+
+			// Emit event if not stopped before
+			if stop_ts_set {
+				ctx.event(protocol::Event::ContainerStateUpdate {
+					container_id: self.container_id,
+					state: protocol::ContainerState::Stopped,
+				})
+				.await?;
+			}
 		}
 
 		Ok(())

lib/util/core/src/serde.rs

@@ -343,3 +343,10 @@ where
 		})
 	}
 }
+
+impl<T> sqlx::postgres::PgHasArrayType for Raw<T> {
+    fn array_type_info() -> sqlx::postgres::PgTypeInfo {
+		// JSONB array
+        sqlx::postgres::PgTypeInfo::with_name("_jsonb")
+	}
+}

svc/Cargo.toml

@@ -179,6 +179,7 @@ members = [
 	"pkg/nomad/standalone/monitor",
 	"pkg/nsfw/ops/image-score",
 	"pkg/pegboard",
+	"pkg/pegboard/standalone/gc",
 	"pkg/pegboard/standalone/ws",
 	"pkg/perf/ops/log-get",
 	"pkg/profanity/ops/check",

svc/pkg/ds/db/servers/migrations/20240918220629_game_config.up.sql

@@ -2,5 +2,5 @@ CREATE TABLE game_config (
 	game_id UUID PRIMARY KEY,
 	host_networking_enabled BOOLEAN NOT NULL DEFAULT FALSE,
 	root_user_enabled BOOLEAN NOT NULL DEFAULT FALSE,
-	runtime INT NOT NULL, -- ds::types::GameRuntime
+	runtime INT NOT NULL -- ds::types::GameRuntime
 );

svc/pkg/ds/src/lib.rs

@@ -16,7 +16,6 @@ pub fn registry() -> WorkflowResult<Registry> {
 	registry.register_workflow::<server::nomad::alloc_plan::Workflow>()?;
 	registry.register_workflow::<server::nomad::alloc_update::Workflow>()?;
 	registry.register_workflow::<server::nomad::eval_update::Workflow>()?;
-	registry.register_workflow::<server::nomad::eval_update::Workflow>()?;
 	registry.register_workflow::<server::pegboard::Workflow>()?;
 	registry.register_workflow::<server::pegboard::destroy::Workflow>()?;
 

svc/pkg/pegboard/Cargo.toml

@@ -7,6 +7,7 @@ license = "Apache-2.0"
 
 [dependencies]
 chirp-workflow = { path = "../../../lib/chirp-workflow/core" }
+nix = { version = "0.27", default-features = false, features = ["user", "signal"] }
 serde = { version = "1.0.198", features = ["derive"] }
 strum = { version = "0.24", features = ["derive"] }
 thiserror = "1.0"

svc/pkg/pegboard/db/pegboard/migrations/20240913005543_init.up.sql

@@ -5,10 +5,12 @@ CREATE TABLE clients (
 	last_event_idx INT NOT NULL DEFAULT 0,
 	last_command_idx INT NOT NULL DEFAULT 0,
 
+	-- Total resources
 	cpu INT NOT NULL DEFAULT 0,
 	memory INT NOT NULL DEFAULT 0,
 
-	drain_ts INT
+	drain_ts INT,
+	delete_ts INT
 );
 
 CREATE TABLE client_events (