fix: get mm tests working again with provisioning

lib/bolt/core/src/dep/cargo/cli.rs

@@ -300,6 +300,9 @@ pub async fn build_tests<'a, T: AsRef<str>>(
 		for test in build_call.packages {
 			cmd.args(&["--package", test.as_ref()]);
 		}
+		if std::env::var("CARGO_TARGET_DIR").is_err() {
+			cmd.env("CARGO_TARGET_DIR", ctx.cargo_target_dir());
+		}
 		let mut child = cmd.spawn()?;
 
 		// Capture stdout

svc/api/internal-monolith/src/route/mod.rs

@@ -20,7 +20,7 @@ define_router! {
 	mounts: [
 		{
 			path: api_traefik_provider::route::Router,
-			prefix: "route",
+			prefix: "traefik-provider",
 		},
 		{
 			path: api_provision::route::Router,

svc/pkg/cluster/ops/server-list/src/lib.rs

@@ -24,8 +24,8 @@ pub async fn handle(
 			[ctx, Server]
 			"
 			SELECT d.cluster_id, s.server_id
-			FROM servers AS s
-			JOIN datacenters AS d
+			FROM db_cluster.servers AS s
+			JOIN db_cluster.datacenters AS d
 			ON s.datacenter_id = d.datacenter_id
 			WHERE
 				d.cluster_id = ANY($1) AND
@@ -39,8 +39,8 @@ pub async fn handle(
 			[ctx, Server]
 			"
 			SELECT d.cluster_id, s.server_id
-			FROM servers AS s
-			JOIN datacenters AS d
+			FROM db_cluster.servers AS s
+			JOIN db_cluster.datacenters AS d
 			ON s.datacenter_id = d.datacenter_id
 			WHERE
 				d.cluster_id = ANY($1) AND

svc/pkg/faker/ops/game/src/lib.rs

@@ -50,12 +50,11 @@ async fn handle(
 		let version_id = unwrap_ref!(version_create_res.version_id).as_uuid();
 		version_ids.push(version_id.into());
 
-		let namespace_name_ids = vec!["prod".to_owned(), "staging".to_owned()];
-		for name_id in &namespace_name_ids {
+		for name_id in ["prod", "staging"] {
 			let ns_create_res = op!([ctx] faker_game_namespace {
 				game_id: game_create_res.game_id,
 				version_id: version_create_res.version_id,
-				override_name_id: name_id.clone(),
+				override_name_id: name_id.to_owned(),
 				..Default::default()
 			})
 			.await?;

svc/pkg/faker/types/game.proto

@@ -8,7 +8,6 @@ message Request {
 	// Will not create the default namespaces and versions.
 	bool skip_namespaces_and_versions = 1;
 
-
 	// The default team ID to set as the game's dev team.
 	optional rivet.common.Uuid dev_team_id = 2;
 }

svc/pkg/load-test/standalone/mm-sustain/Cargo.toml

@@ -13,11 +13,13 @@ rivet-runtime = { path = "../../../../../lib/runtime" }
 tokio = { version = "1.29", features = ["full"] }
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "json", "ansi"] }
+rand = "0.8"
 rivet-api = { path = "../../../../../sdks/full/rust" }
 reqwest = "0.11"
 
 faker-game = { path = "../../../faker/ops/game" }
 faker-team = { path = "../../../faker/ops/team" }
+faker-game-namespace = { path = "../../../faker/ops/game-namespace" }
 faker-game-version = { path = "../../../faker/ops/game-version" }
 faker-region = { path = "../../../faker/ops/region" }
 faker-build = { path = "../../../faker/ops/build" }

svc/pkg/load-test/standalone/mm-sustain/src/lib.rs

@@ -1,10 +1,12 @@
-use proto::backend::{self, pkg::*};
+use std::collections::HashMap;
 
+use proto::backend::{self, pkg::*};
+use rand::prelude::*;
 use rivet_operation::prelude::*;
-
-use std::collections::HashMap;
 use tokio::time::Instant;
 
+const PARALLEL_WORKERS: usize = 1;
+
 #[tracing::instrument(skip_all)]
 pub async fn run_from_env(_ts: i64) -> GlobalResult<()> {
 	let pools = rivet_pools::from_env("load-test-mm-sustain").await?;
@@ -29,77 +31,30 @@ pub async fn run_from_env(_ts: i64) -> GlobalResult<()> {
 
 	// Game
 	let game_res = op!([ctx] faker_game {
+		skip_namespaces_and_versions: true,
 		..Default::default()
 	})
 	.await?;
-	let namespace_id = unwrap!(game_res.namespace_ids.first()).clone().as_uuid();
 
 	let build_res = op!([ctx] faker_build {
 		game_id: game_res.game_id,
 		image: backend::faker::Image::MmLobbyEcho as i32,
 	})
 	.await?;
+	let build_id = build_res.build_id.unwrap().as_uuid();
 
 	let game_version_res = op!([ctx] faker_game_version {
 		game_id: game_res.game_id,
 		override_lobby_groups: Some(faker::game_version::request::OverrideLobbyGroups {
-			lobby_groups: vec![backend::matchmaker::LobbyGroup {
-				name_id: "test-1".into(),
-
-				regions: vec![backend::matchmaker::lobby_group::Region {
-					region_id: Some(region_id.into()),
-					tier_name_id: util_mm::test::TIER_NAME_ID.to_owned(),
-					idle_lobbies: Some(backend::matchmaker::lobby_group::IdleLobbies {
-						min_idle_lobbies: 0,
-						// Don't auto-destroy lobbies from tests
-						max_idle_lobbies: 32,
-					}),
-				}],
-				max_players_normal: 8,
-				max_players_direct: 10,
-				max_players_party: 12,
-				listable: true,
-				taggable: false,
-				allow_dynamic_max_players: false,
-
-				runtime: Some(backend::matchmaker::lobby_runtime::Docker {
-					build_id: build_res.build_id,
-					args: Vec::new(),
-					env_vars: vec![
-						backend::matchmaker::lobby_runtime::EnvVar {
-							key: "HELLO".into(),
-							value: "world".into(),
-						},
-					],
-					network_mode: backend::matchmaker::lobby_runtime::NetworkMode::Bridge as i32,
-					ports: vec![
-						backend::matchmaker::lobby_runtime::Port {
-							label: "test-http".into(),
-							target_port: Some(8001),
-							port_range: None,
-							proxy_protocol: backend::matchmaker::lobby_runtime::ProxyProtocol::Http as i32,
-							proxy_kind: backend::matchmaker::lobby_runtime::ProxyKind::GameGuard as i32,
-						},
-						backend::matchmaker::lobby_runtime::Port {
-							label: "test-tcp".into(),
-							target_port: Some(8002),
-							port_range: None,
-							proxy_protocol: backend::matchmaker::lobby_runtime::ProxyProtocol::Tcp as i32,
-							proxy_kind: backend::matchmaker::lobby_runtime::ProxyKind::GameGuard as i32,
-						},
-						backend::matchmaker::lobby_runtime::Port {
-							label: "test-udp".into(),
-							target_port: Some(8003),
-							port_range: None,
-							proxy_protocol: backend::matchmaker::lobby_runtime::ProxyProtocol::Udp as i32,
-							proxy_kind: backend::matchmaker::lobby_runtime::ProxyKind::GameGuard as i32,
-						},
-					],
-
-				}.into()),
-
-				actions: None,
-			}]
+			lobby_groups: vec![
+				// lobby_group_config(build_id, region_id, "basic-4d1"),
+				// lobby_group_config(build_id, region_id, "basic-2d1"),
+				// lobby_group_config(build_id, region_id, "basic-1d1"),
+				lobby_group_config(build_id, region_id, "basic-1d2"),
+				lobby_group_config(build_id, region_id, "basic-1d4"),
+				lobby_group_config(build_id, region_id, "basic-1d8"),
+				lobby_group_config(build_id, region_id, "basic-1d16"),
+			],
 		}),
 		..Default::default()
 	})
@@ -113,36 +68,121 @@ pub async fn run_from_env(_ts: i64) -> GlobalResult<()> {
 		.config_meta
 		.as_ref())
 	.lobby_groups;
-	let lobby_group_id = unwrap!(lobby_groups[0].lobby_group_id.as_ref()).as_uuid();
+	let lobby_group_ids = lobby_groups
+		.iter()
+		.map(|x| x.lobby_group_id.unwrap().as_uuid())
+		.collect::<Vec<_>>();
 
-	op!([ctx] game_namespace_version_set {
-		namespace_id: Some(namespace_id.into()),
+	let ns_create_res = op!([ctx] faker_game_namespace {
+		game_id: game_res.game_id,
 		version_id: game_version_res.version_id,
+		override_name_id: "prod".to_owned(),
+		..Default::default()
 	})
-	.await?;
+	.await
+	.unwrap();
+	let namespace_id = ns_create_res.namespace_id.unwrap().as_uuid();
+
+	let mut handles = Vec::new();
 
-	let parallel_workers = 4;
-	for i in 0..parallel_workers {
-		tokio::spawn(run_lobby_worker(
+	for i in 0..PARALLEL_WORKERS {
+		handles.push(tokio::spawn(run_lobby_worker(
 			ctx.clone(),
 			i,
 			namespace_id,
 			region_id,
-			lobby_group_id,
-		));
+			lobby_group_ids.clone(),
+		)));
+	}
+
+	for handle in handles {
+		handle.await?;
 	}
 
 	Ok(())
 }
 
+fn lobby_group_config(
+	build_id: Uuid,
+	region_id: Uuid,
+	tier: &str,
+) -> backend::matchmaker::LobbyGroup {
+	backend::matchmaker::LobbyGroup {
+		name_id: util::faker::ident(),
+
+		regions: vec![backend::matchmaker::lobby_group::Region {
+			region_id: Some(region_id.into()),
+			tier_name_id: tier.to_owned(),
+			idle_lobbies: Some(backend::matchmaker::lobby_group::IdleLobbies {
+				min_idle_lobbies: 0,
+				// Don't auto-destroy lobbies from tests
+				max_idle_lobbies: 32,
+			}),
+		}],
+		max_players_normal: 8,
+		max_players_direct: 10,
+		max_players_party: 12,
+		listable: true,
+		taggable: false,
+		allow_dynamic_max_players: false,
+
+		runtime: Some(
+			backend::matchmaker::lobby_runtime::Docker {
+				build_id: Some(build_id.into()),
+				args: Vec::new(),
+				env_vars: vec![backend::matchmaker::lobby_runtime::EnvVar {
+					key: "HELLO".into(),
+					value: "world".into(),
+				}],
+				network_mode: backend::matchmaker::lobby_runtime::NetworkMode::Bridge as i32,
+				ports: vec![
+					backend::matchmaker::lobby_runtime::Port {
+						label: "test-http".into(),
+						target_port: Some(8001),
+						port_range: None,
+						proxy_protocol: backend::matchmaker::lobby_runtime::ProxyProtocol::Http
+							as i32,
+						proxy_kind: backend::matchmaker::lobby_runtime::ProxyKind::GameGuard as i32,
+					},
+					backend::matchmaker::lobby_runtime::Port {
+						label: "test-tcp".into(),
+						target_port: Some(8002),
+						port_range: None,
+						proxy_protocol: backend::matchmaker::lobby_runtime::ProxyProtocol::Tcp
+							as i32,
+						proxy_kind: backend::matchmaker::lobby_runtime::ProxyKind::GameGuard as i32,
+					},
+					backend::matchmaker::lobby_runtime::Port {
+						label: "test-udp".into(),
+						target_port: Some(8003),
+						port_range: None,
+						proxy_protocol: backend::matchmaker::lobby_runtime::ProxyProtocol::Udp
+							as i32,
+						proxy_kind: backend::matchmaker::lobby_runtime::ProxyKind::GameGuard as i32,
+					},
+				],
+			}
+			.into(),
+		),
+
+		actions: None,
+	}
+}
+
 async fn run_lobby_worker(
 	ctx: OperationContext<()>,
 	_worker_idx: usize,
 	namespace_id: Uuid,
 	region_id: Uuid,
-	lobby_group_id: Uuid,
+	lobby_group_ids: Vec<Uuid>,
 ) {
 	loop {
+		// Choose random lobby group id
+		let lobby_group_id = {
+			let mut rng = thread_rng();
+			*lobby_group_ids.choose(&mut rng).unwrap()
+		};
+
 		let start = Instant::now();
 		let lobby_id = Uuid::new_v4();
 		match run_lobby_lifecycle(&ctx, lobby_id, namespace_id, region_id, lobby_group_id).await {
@@ -173,22 +213,26 @@ async fn run_lobby_lifecycle(
 ) -> GlobalResult<()> {
 	// Create lobby
 	tracing::info!(?lobby_id, "creating lobby");
-	msg!([ctx] @notrace mm::msg::lobby_create(lobby_id) -> mm::msg::lobby_ready_complete(lobby_id) {
-			lobby_id: Some(lobby_id.into()),
-			namespace_id: Some(namespace_id.into()),
-			lobby_group_id: Some(lobby_group_id.into()),
-			region_id: Some(region_id.into()),
-			create_ray_id: None,
-			preemptively_created: false,
-
-			creator_user_id: None,
-			is_custom: false,
-			publicity: None,
-			lobby_config_json: None,
-			tags: HashMap::new(),
-			dynamic_max_players: None,
-		})
-		.await?;
+	let res = msg!([ctx] @notrace mm::msg::lobby_create(lobby_id) -> Result<mm::msg::lobby_ready_complete, mm::msg::lobby_create_fail> {
+		lobby_id: Some(lobby_id.into()),
+		namespace_id: Some(namespace_id.into()),
+		lobby_group_id: Some(lobby_group_id.into()),
+		region_id: Some(region_id.into()),
+		create_ray_id: None,
+		preemptively_created: false,
+
+		creator_user_id: None,
+		is_custom: false,
+		publicity: None,
+		lobby_config_json: None,
+		tags: HashMap::new(),
+		dynamic_max_players: None,
+	})
+	.await?;
+
+	if let Err(err) = res {
+		bail!(format!("{err:?}"));
+	}
 
 	// Test HTTP connectivity
 	let (hostname, _) = get_lobby_addr(ctx, lobby_id, "test-http").await?;
@@ -253,7 +297,10 @@ async fn get_lobby_addr(
 	lobby_id: Uuid,
 	port: &str,
 ) -> GlobalResult<(String, u16)> {
-	let lobby_res = op!([ctx] mm_lobby_get { lobby_ids: vec![lobby_id.into()] }).await?;
+	let lobby_res = op!([ctx] mm_lobby_get {
+		lobby_ids: vec![lobby_id.into()],
+	})
+	.await?;
 	let lobby = unwrap!(lobby_res.lobbies.first());
 	let run_id = unwrap!(lobby.run_id);
 

svc/pkg/mm/worker/Cargo.toml

@@ -60,6 +60,7 @@ chirp-worker = { path = "../../../../lib/chirp/worker" }
 
 faker-build = { path = "../../faker/ops/build" }
 faker-game = { path = "../../faker/ops/game" }
+faker-game-namespace = { path = "../../faker/ops/game-namespace" }
 faker-game-version = { path = "../../faker/ops/game-version" }
 faker-mm-lobby = { path = "../../faker/ops/mm-lobby" }
 faker-region = { path = "../../faker/ops/region" }