From d4c4a9a92fb9e8205d1ae8ef02b58db67a7a4339 Mon Sep 17 00:00:00 2001 From: MasterPtato Date: Fri, 12 Apr 2024 19:59:43 +0000 Subject: [PATCH] chore: dont destroy anything --- .../migrations/20231201000927_init.up.sql | 39 +++++++++++++++---- .../types/msg/server-install-complete.proto | 11 +++--- .../cluster/types/msg/server-install.proto | 15 +++---- .../worker/src/workers/server_dns_delete.rs | 12 ++++-- .../worker/src/workers/server_install/mod.rs | 4 +- .../worker/src/workers/server_provision.rs | 5 ++- .../cluster/worker/tests/server_dns_create.rs | 4 +- .../cluster/worker/tests/server_dns_delete.rs | 4 +- svc/pkg/cluster/worker/tests/server_drain.rs | 4 +- svc/pkg/linode/ops/server-destroy/src/lib.rs | 16 +++++--- .../linode/ops/server-provision/src/lib.rs | 8 +++- svc/pkg/linode/standalone/gc/src/lib.rs | 17 +++++--- .../src/workers/prebake_install_complete.rs | 6 ++- .../worker/src/workers/prebake_provision.rs | 26 +++++++++---- .../worker/tests/prebake_install_complete.rs | 6 ++- .../linode/worker/tests/prebake_provision.rs | 3 +- 16 files changed, 126 insertions(+), 54 deletions(-) diff --git a/svc/pkg/cluster/db/cluster/migrations/20231201000927_init.up.sql b/svc/pkg/cluster/db/cluster/migrations/20231201000927_init.up.sql index 38c636090e..d1f9bd1ca0 100644 --- a/svc/pkg/cluster/db/cluster/migrations/20231201000927_init.up.sql +++ b/svc/pkg/cluster/db/cluster/migrations/20231201000927_init.up.sql @@ -69,20 +69,34 @@ WHERE cloud_destroy_ts IS NULL; -- Stores data for destroying linode resources CREATE TABLE servers_linode ( - server_id UUID PRIMARY KEY REFERENCES servers (server_id), + server_id UUID NOT NULL REFERENCES servers (server_id), ssh_key_id INT NOT NULL, linode_id INT, - firewall_id INT + firewall_id INT, + + destroy_ts INT ); +-- Effectively a conditional primary key +CREATE UNIQUE INDEX idx_servers_linode_pkey +ON servers_linode (server_id) +WHERE destroy_ts IS NULL; + -- Stores data for destroying cloudflare resources CREATE TABLE servers_cloudflare ( - server_id UUID PRIMARY KEY REFERENCES servers (server_id), + server_id UUID NOT NULL REFERENCES servers (server_id), dns_record_id TEXT NOT NULL, -- Secondary DNS route which doesn't have a wildcard. Used for discord activities. - secondary_dns_record_id TEXT + secondary_dns_record_id TEXT, + + destroy_ts INT ); +-- Effectively a conditional primary key +CREATE UNIQUE INDEX idx_servers_cloudflare_pkey +ON servers_cloudflare (server_id) +WHERE destroy_ts IS NULL; + CREATE TABLE server_images ( provider INT, install_hash TEXT, @@ -108,11 +122,22 @@ CREATE TABLE server_images_linode ( public_ip INET, image_id TEXT, - PRIMARY KEY (install_hash, datacenter_id, pool_type), - INDEX (public_ip), - INDEX (image_id) + destroy_ts INT ); +-- Effectively a conditional primary key +CREATE UNIQUE INDEX idx_server_images_linode_pkey +ON server_images_linode (install_hash, datacenter_id, pool_type) +WHERE destroy_ts IS NULL; + +CREATE INDEX idx_server_images_linode_public_ip +ON server_images_linode (public_ip) +WHERE destroy_ts IS NULL; + +CREATE INDEX idx_server_images_linode_image_id +ON server_images_linode (image_id) +WHERE destroy_ts IS NULL; + -- Dictates which cluster a game's lobbies will be created in CREATE TABLE games ( game_id UUID PRIMARY KEY, diff --git a/svc/pkg/cluster/types/msg/server-install-complete.proto b/svc/pkg/cluster/types/msg/server-install-complete.proto index 9860389c99..3c515e2ff5 100644 --- a/svc/pkg/cluster/types/msg/server-install-complete.proto +++ b/svc/pkg/cluster/types/msg/server-install-complete.proto @@ -7,12 +7,13 @@ import "proto/backend/cluster.proto"; /// name = "msg-cluster-server-install-complete" /// parameters = [ -/// { name = "public_ip" }, +/// { name = "request_id" }, /// ] message Message { - string public_ip = 1; - rivet.common.Uuid datacenter_id = 2; + rivet.common.Uuid request_id = 1; + string public_ip = 2; + rivet.common.Uuid datacenter_id = 3; // If set in server install message - optional rivet.common.Uuid server_id = 3; - rivet.backend.cluster.Provider provider = 4; + optional rivet.common.Uuid server_id = 4; + rivet.backend.cluster.Provider provider = 5; } diff --git a/svc/pkg/cluster/types/msg/server-install.proto b/svc/pkg/cluster/types/msg/server-install.proto index b27d2b3975..d0364d844c 100644 --- a/svc/pkg/cluster/types/msg/server-install.proto +++ b/svc/pkg/cluster/types/msg/server-install.proto @@ -7,18 +7,19 @@ import "proto/backend/cluster.proto"; /// name = "msg-cluster-server-install" /// parameters = [ -/// { name = "public_ip" }, +/// { name = "request_id" }, /// ] message Message { - string public_ip = 1; - rivet.backend.cluster.PoolType pool_type = 2; + rivet.common.Uuid request_id = 1; + string public_ip = 2; + rivet.backend.cluster.PoolType pool_type = 3; - rivet.common.Uuid datacenter_id = 6; + rivet.common.Uuid datacenter_id = 4; // Unset when installing prebake servers since they don't have an id. Used to check if // the server is currently being deleted to prevent installation - optional rivet.common.Uuid server_id = 3; + optional rivet.common.Uuid server_id = 5; // Simply passed to the install complete message - rivet.backend.cluster.Provider provider = 4; - bool initialize_immediately = 5; + rivet.backend.cluster.Provider provider = 6; + bool initialize_immediately = 7; } diff --git a/svc/pkg/cluster/worker/src/workers/server_dns_delete.rs b/svc/pkg/cluster/worker/src/workers/server_dns_delete.rs index a251a22b87..19946787c1 100644 --- a/svc/pkg/cluster/worker/src/workers/server_dns_delete.rs +++ b/svc/pkg/cluster/worker/src/workers/server_dns_delete.rs @@ -16,7 +16,9 @@ async fn worker( " SELECT dns_record_id, secondary_dns_record_id FROM db_cluster.servers_cloudflare - WHERE server_id = $1 + WHERE + server_id = $1 AND + destroy_ts IS NULL ", &server_id, util::timestamp::now(), @@ -61,10 +63,14 @@ async fn worker( sql_execute!( [ctx, &crdb] " - DELETE FROM db_cluster.servers_cloudflare - WHERE server_id = $1 + UPDATE db_cluster.servers_cloudflare + SET destroy_ts = $2 + WHERE + server_id = $1 AND + destroy_ts IS NULL ", server_id, + util::timestamp::now(), ) .await?; diff --git a/svc/pkg/cluster/worker/src/workers/server_install/mod.rs b/svc/pkg/cluster/worker/src/workers/server_install/mod.rs index de1380ff4d..830c3e0be3 100644 --- a/svc/pkg/cluster/worker/src/workers/server_install/mod.rs +++ b/svc/pkg/cluster/worker/src/workers/server_install/mod.rs @@ -156,7 +156,9 @@ async fn worker(ctx: &OperationContext) - } } - msg!([ctx] cluster::msg::server_install_complete(&ctx.public_ip) { + let request_id = unwrap_ref!(ctx.request_id).as_uuid(); + msg!([ctx] cluster::msg::server_install_complete(request_id) { + request_id: ctx.request_id, public_ip: ctx.public_ip.clone(), datacenter_id: ctx.datacenter_id, server_id: ctx.server_id, diff --git a/svc/pkg/cluster/worker/src/workers/server_provision.rs b/svc/pkg/cluster/worker/src/workers/server_provision.rs index 5ca9819e81..b058679168 100644 --- a/svc/pkg/cluster/worker/src/workers/server_provision.rs +++ b/svc/pkg/cluster/worker/src/workers/server_provision.rs @@ -137,7 +137,10 @@ async fn worker( if let Some(provision_res) = provision_res { // Install components if !provision_res.already_installed { - msg!([ctx] cluster::msg::server_install(&provision_res.public_ip) { + let request_id = Uuid::new_v4(); + + msg!([ctx] cluster::msg::server_install(request_id) { + request_id: Some(request_id.into()), public_ip: provision_res.public_ip, datacenter_id: ctx.datacenter_id, server_id: ctx.server_id, diff --git a/svc/pkg/cluster/worker/tests/server_dns_create.rs b/svc/pkg/cluster/worker/tests/server_dns_create.rs index c332b0188a..fcdfcd2010 100644 --- a/svc/pkg/cluster/worker/tests/server_dns_create.rs +++ b/svc/pkg/cluster/worker/tests/server_dns_create.rs @@ -53,7 +53,9 @@ async fn server_dns_create(ctx: TestCtx) { SELECT EXISTS ( SELECT 1 FROM db_cluster.servers_cloudflare - WHERE server_id = $1 + WHERE + server_id = $1 AND + destroy_ts IS NULL ) ", server_id, diff --git a/svc/pkg/cluster/worker/tests/server_dns_delete.rs b/svc/pkg/cluster/worker/tests/server_dns_delete.rs index adbe61e80a..6da2338819 100644 --- a/svc/pkg/cluster/worker/tests/server_dns_delete.rs +++ b/svc/pkg/cluster/worker/tests/server_dns_delete.rs @@ -52,7 +52,9 @@ async fn server_dns_delete(ctx: TestCtx) { SELECT EXISTS ( SELECT 1 FROM db_cluster.servers_cloudflare - WHERE server_id = $1 + WHERE + server_id = $1 AND + destroy_ts IS NULL ) ", server_id, diff --git a/svc/pkg/cluster/worker/tests/server_drain.rs b/svc/pkg/cluster/worker/tests/server_drain.rs index 76ae34f305..89785a5f7a 100644 --- a/svc/pkg/cluster/worker/tests/server_drain.rs +++ b/svc/pkg/cluster/worker/tests/server_drain.rs @@ -95,7 +95,9 @@ async fn gg_server_drain(ctx: TestCtx) { SELECT EXISTS ( SELECT 1 FROM db_cluster.servers_cloudflare - WHERE server_id = $1 + WHERE + server_id = $1 AND + destroy_ts IS NULL ) ", server_id, diff --git a/svc/pkg/linode/ops/server-destroy/src/lib.rs b/svc/pkg/linode/ops/server-destroy/src/lib.rs index ee87008867..e98d87a679 100644 --- a/svc/pkg/linode/ops/server-destroy/src/lib.rs +++ b/svc/pkg/linode/ops/server-destroy/src/lib.rs @@ -13,7 +13,6 @@ struct LinodeData { pub async fn handle( ctx: OperationContext, ) -> GlobalResult { - let crdb = ctx.crdb().await?; let server_id = unwrap_ref!(ctx.server_id).as_uuid(); let datacenter_id = unwrap!(ctx.datacenter_id); @@ -24,11 +23,13 @@ pub async fn handle( let datacenter = unwrap!(datacenter_res.datacenters.first()); let data = sql_fetch_optional!( - [ctx, LinodeData, &crdb] + [ctx, LinodeData] " SELECT ssh_key_id, linode_id, firewall_id FROM db_cluster.servers_linode - WHERE server_id = $1 + WHERE + server_id = $1 AND + destroy_ts IS NULL ", server_id, ) @@ -54,10 +55,13 @@ pub async fn handle( // Remove record sql_execute!( - [ctx, &crdb] + [ctx] " - DELETE FROM db_cluster.servers_linode - WHERE server_id = $1 + UPDATE db_cluster.servers_linode + SET destroy_ts = $2 + WHERE + server_id = $1 AND + destroy_ts IS NULL ", server_id, ) diff --git a/svc/pkg/linode/ops/server-provision/src/lib.rs b/svc/pkg/linode/ops/server-provision/src/lib.rs index 5bea4b3100..8574f0c290 100644 --- a/svc/pkg/linode/ops/server-provision/src/lib.rs +++ b/svc/pkg/linode/ops/server-provision/src/lib.rs @@ -96,7 +96,9 @@ pub async fn handle( " UPDATE db_cluster.servers_linode SET linode_id = $2 - WHERE server_id = $1 + WHERE + server_id = $1 AND + destroy_ts IS NULL ", server_id, linode_id as i64, @@ -130,7 +132,9 @@ pub async fn handle( " UPDATE db_cluster.servers_linode SET firewall_id = $2 - WHERE server_id = $1 + WHERE + server_id = $1 AND + destroy_ts IS NULL ", server_id, firewall_res.id as i64, diff --git a/svc/pkg/linode/standalone/gc/src/lib.rs b/svc/pkg/linode/standalone/gc/src/lib.rs index eb49bf1a81..863d9f3612 100644 --- a/svc/pkg/linode/standalone/gc/src/lib.rs +++ b/svc/pkg/linode/standalone/gc/src/lib.rs @@ -126,8 +126,10 @@ async fn get_prebake_servers( install_hash, datacenter_id, pool_type, ssh_key_id, linode_id, firewall_id FROM db_cluster.server_images_linode - WHERE image_id = ANY($1) - FOR UPDATE + WHERE + image_id = ANY($1) AND + destroy_ts IS NULL + FOR UPDATE ", image_ids, ) @@ -164,6 +166,7 @@ async fn get_prebake_servers( s.install_hash = (q->>0)::TEXT AND s.datacenter_id = (q->>1)::UUID AND s.pool_type = (q->>2)::INT + WHERE destroy_ts IS NULL ) AS m WHERE i.provider = $2 AND @@ -180,15 +183,17 @@ async fn get_prebake_servers( sql_execute!( [ctx, @tx tx] " - DELETE FROM db_cluster.server_images_linode AS s - USING jsonb_array_elements($1::JSONB) AS q + UPDATE db_cluster.server_images_linode AS s + SET destroy_ts = $2 + FROM jsonb_array_elements($1::JSONB) AS q WHERE s.install_hash = (q->>0)::TEXT AND s.datacenter_id = (q->>1)::UUID AND - s.pool_type = (q->>2)::INT + s.pool_type = (q->>2)::INT AND + destroy_ts IS NULL ", &primary_keys, - backend::cluster::Provider::Linode as i64, + util::timestamp::now(), ) .await?; diff --git a/svc/pkg/linode/worker/src/workers/prebake_install_complete.rs b/svc/pkg/linode/worker/src/workers/prebake_install_complete.rs index 17b72682a5..38b0872ff6 100644 --- a/svc/pkg/linode/worker/src/workers/prebake_install_complete.rs +++ b/svc/pkg/linode/worker/src/workers/prebake_install_complete.rs @@ -28,7 +28,8 @@ async fn worker( install_hash, datacenter_id, pool_type, linode_id, disk_id FROM db_cluster.server_images_linode WHERE - public_ip = $1 + public_ip = $1 AND + destroy_ts IS NULL ", public_ip, ) @@ -62,7 +63,8 @@ async fn worker( WHERE install_hash = $1 AND datacenter_id = $2 AND - pool_type = $3 + pool_type = $3 AND + destroy_ts IS NULL ", prebake_server.install_hash, prebake_server.datacenter_id, diff --git a/svc/pkg/linode/worker/src/workers/prebake_provision.rs b/svc/pkg/linode/worker/src/workers/prebake_provision.rs index a6ac9e17ad..fb834448aa 100644 --- a/svc/pkg/linode/worker/src/workers/prebake_provision.rs +++ b/svc/pkg/linode/worker/src/workers/prebake_provision.rs @@ -54,8 +54,11 @@ async fn worker( match provision(ctx, &crdb, &client, datacenter_id, &prebake_server).await { Ok(public_ip) => { + let request_id = Uuid::new_v4(); + // Continue to install - msg!([ctx] cluster::msg::server_install(&public_ip) { + msg!([ctx] cluster::msg::server_install(request_id) { + request_id: Some(request_id.into()), public_ip: public_ip, pool_type: ctx.pool_type, server_id: None, @@ -123,7 +126,8 @@ async fn provision( WHERE install_hash = $1 AND datacenter_id = $2 AND - pool_type = $3 + pool_type = $3 AND + destroy_ts IS NULL ", util_cluster::INSTALL_SCRIPT_HASH, datacenter_id, @@ -156,7 +160,8 @@ async fn provision( WHERE install_hash = $1 AND datacenter_id = $2 AND - pool_type = $3 + pool_type = $3 AND + destroy_ts IS NULL ", util_cluster::INSTALL_SCRIPT_HASH, datacenter_id, @@ -180,7 +185,8 @@ async fn provision( WHERE install_hash = $1 AND datacenter_id = $2 AND - pool_type = $3 + pool_type = $3 AND + destroy_ts IS NULL ", util_cluster::INSTALL_SCRIPT_HASH, datacenter_id, @@ -214,7 +220,8 @@ async fn destroy( WHERE install_hash = $1 AND datacenter_id = $2 AND - pool_type = $3 + pool_type = $3 AND + destroy_ts IS NULL ", util_cluster::INSTALL_SCRIPT_HASH, datacenter_id, @@ -239,17 +246,20 @@ async fn destroy( // Remove record sql_execute!( - [ctx] + [ctx, &crdb] " - DELETE FROM db_cluster.server_images_linode + UPDATE db_cluster.server_images_linode + SET destroy_ts = $4 WHERE install_hash = $1 AND datacenter_id = $2 AND - pool_type = $3 + pool_type = $3 AND + destroy_ts IS NULL ", util_cluster::INSTALL_SCRIPT_HASH, datacenter_id, ctx.pool_type as i64, + util::timestamp::now(), ) .await?; diff --git a/svc/pkg/linode/worker/tests/prebake_install_complete.rs b/svc/pkg/linode/worker/tests/prebake_install_complete.rs index 55b92c02ba..f0a57c6e1d 100644 --- a/svc/pkg/linode/worker/tests/prebake_install_complete.rs +++ b/svc/pkg/linode/worker/tests/prebake_install_complete.rs @@ -53,7 +53,8 @@ async fn prebake_install_complete(ctx: TestCtx) { install_hash = $2 AND datacenter_id = $3 AND pool_type = $4 AND - public_ip IS NOT NULL + public_ip IS NOT NULL AND + destroy_ts IS NULL ", backend::cluster::Provider::Linode as i64, util_cluster::INSTALL_SCRIPT_HASH, @@ -89,7 +90,8 @@ async fn prebake_install_complete(ctx: TestCtx) { install_hash = $2 AND datacenter_id = $3 AND pool_type = $4 AND - image_id IS NOT NULL + image_id IS NOT NULL AND + destroy_ts IS NULL ) ", backend::cluster::Provider::Linode as i64, diff --git a/svc/pkg/linode/worker/tests/prebake_provision.rs b/svc/pkg/linode/worker/tests/prebake_provision.rs index f990758fce..0bde395d23 100644 --- a/svc/pkg/linode/worker/tests/prebake_provision.rs +++ b/svc/pkg/linode/worker/tests/prebake_provision.rs @@ -54,7 +54,8 @@ async fn prebake_provision(ctx: TestCtx) { install_hash = $2 AND datacenter_id = $3 AND pool_type = $4 AND - public_ip IS NOT NULL + public_ip IS NOT NULL AND + destroy_ts IS NULL ) ", backend::cluster::Provider::Linode as i64,