feat: dynamic TLS generation (#635)

<!-- Please make sure there is an issue that this PR is correlated to. -->

## Changes

<!-- If there are frontend changes, please include screenshots. -->

fern/definition/provision/datacenters/__package__.yml

@@ -0,0 +1,19 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+
+service:
+  auth: true
+  base-path: /datacenters
+  endpoints:
+    getTls:
+      path: /{datacenter_id}/tls
+      method: GET
+      path-parameters:
+        datacenter_id:
+          type: uuid
+      response: GetTlsResponse
+
+types:
+  GetTlsResponse:
+    properties:
+      job_cert_pem: string
+      job_private_key_pem: string

fern/definition/provision/servers/__package__.yml

@@ -4,16 +4,16 @@ service:
   auth: true
   base-path: /servers
   endpoints:
-    getServerInfo:
-      path: /{ip}/info
+    getInfo:
+      path: /{ip}
       method: GET
       path-parameters:
         ip:
           type: string
-      response: GetServerInfoResponse
+      response: GetInfoResponse
 
 types:
-  GetServerInfoResponse:
+  GetInfoResponse:
     properties:
       name: string
       server_id: uuid

infra/tf/tls/acme.tf

@@ -1,6 +1,8 @@
 # MARK: Private key
 resource "tls_private_key" "acme_account_key" {
-	algorithm = "RSA"
+	# Must be EC key to work with acme_lib
+	algorithm = "ECDSA"
+	ecdsa_curve = "P256"
 }
 
 # MARK: Registration
@@ -15,12 +17,6 @@ resource "acme_certificate" "rivet_gg" {
 	common_name = var.domain_main
 	subject_alternative_names = flatten([
 		"*.${var.domain_main}",
-
-		// Add dedicated subdomains for each datacenter
-		[
-			for name_id, datacenter in var.datacenters:
-			"*.${datacenter.datacenter_id}.${var.domain_main}"
-		],
 	])
 
 	recursive_nameservers = ["1.1.1.1:53", "1.0.0.1:53"]
@@ -61,38 +57,3 @@ resource "acme_certificate" "rivet_game" {
 		}
 	}
 }
-
-resource "acme_certificate" "rivet_job" {
-	account_key_pem = acme_registration.main.account_key_pem
-	common_name = var.domain_job
-	subject_alternative_names = flatten([
-		# Add dedicated subdomains for each region
-		flatten([
-			for name_id, datacenter in var.datacenters:
-			[
-				"*.lobby.${datacenter.datacenter_id}.${var.domain_job}",
-				"*.${datacenter.datacenter_id}.${var.domain_job}",
-			]
-		]),
-
-		// Add wildcard
-		"*.${var.domain_job}",
-	])
-
-	recursive_nameservers = ["1.1.1.1:53", "1.0.0.1:53"]
-
-	# LetsEncrypt issues for 90 days, issue a new cert at 75 days
-	min_days_remaining = 75
-
-	# This certificate may not have been deployed yet
-	revoke_certificate_on_destroy = false
-
-	dns_challenge {
-		provider = "cloudflare"
-
-		config = {
-			CF_DNS_API_TOKEN = module.secrets.values["cloudflare/terraform/auth_token"]
-		}
-	}
-}
-

infra/tf/tls/outputs.tf

@@ -1,25 +1,4 @@
 locals {
-	tls_cert_letsencrypt_rivet_gg = {
-		# Build full chain by concatenating the certificate with issuer.
-		#
-		# See
-		# https://registry.terraform.io/providers/vancluever/acme/latest/docs/resources/certificate#certificate_pem
-		cert_pem = "${acme_certificate.rivet_gg.certificate_pem}${acme_certificate.rivet_gg.issuer_pem}"
-		key_pem = acme_certificate.rivet_gg.private_key_pem
-	}
-
-	tls_cert_letsencrypt_rivet_game = {
-		# See above
-		cert_pem = "${acme_certificate.rivet_game.certificate_pem}${acme_certificate.rivet_game.issuer_pem}"
-		key_pem = acme_certificate.rivet_game.private_key_pem
-	}
-
-	tls_cert_letsencrypt_rivet_job = {
-		# See above
-		cert_pem = "${acme_certificate.rivet_job.certificate_pem}${acme_certificate.rivet_job.issuer_pem}"
-		key_pem = acme_certificate.rivet_job.private_key_pem
-	}
-
 	tls_cert_locally_signed_tunnel_server = {
 		cert_pem = tls_locally_signed_cert.locally_signed_tunnel_server.cert_pem
 		key_pem = tls_private_key.locally_signed_tunnel_server.private_key_pem
@@ -37,21 +16,6 @@ locals {
 }
 
 # MARK: Write secrets
-output "tls_cert_letsencrypt_rivet_gg" {
-	value = local.tls_cert_letsencrypt_rivet_gg
-	sensitive = true
-}
-
-output "tls_cert_letsencrypt_rivet_game" {
-	value = local.tls_cert_letsencrypt_rivet_game
-	sensitive = true
-}
-
-output "tls_cert_letsencrypt_rivet_job" {
-	value = local.tls_cert_letsencrypt_rivet_job
-	sensitive = true
-}
-
 output "tls_cert_locally_signed_tunnel_server" {
 	value = local.tls_cert_locally_signed_tunnel_server
 	sensitive = true
@@ -71,3 +35,7 @@ output "root_ca_cert_pem" {
 	value = tls_self_signed_cert.root_ca.cert_pem
 }
 
+output "acme_account_private_key_pem" {
+	value = tls_private_key.acme_account_key.private_key_pem
+	sensitive = true
+}

infra/tf/tls/vars.tf

@@ -15,13 +15,6 @@ variable "domain_job" {
 	type = string
 }
 
-# MARK: Datacenters
-variable "datacenters" {
-	type = map(object({
-		datacenter_id = string
-	}))
-}
-
 # MARK: K8s
 variable "kubeconfig_path" {
 	type = string

lib/bolt/config/src/ns.rs

@@ -603,17 +603,22 @@ pub struct Provisioning {
 	/// How many empty job servers to have at all times. Used in the simple provisioning algorithm.
 	#[serde(default = "default_job_server_provision_margin")]
 	pub job_server_provision_margin: u32,
+	#[serde(default)]
+	pub acme_directory: ProvisioningAcmeDirectory,
 }
 
-#[derive(Serialize, Deserialize, Clone, Debug, Default, strum_macros::Display)]
-pub enum ProvisioningBuildDeliveryMethod {
-	#[serde(rename = "traffic_server")]
-	#[strum(serialize = "traffic_server")]
-	#[default]
-	TrafficServer,
-	#[serde(rename = "s3_direct")]
-	#[strum(serialize = "s3_direct")]
-	S3Direct,
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub enum ProvisioningAcmeDirectory {
+	#[serde(rename = "lets_encrypt")]
+	LetsEncrypt,
+	#[serde(rename = "lets_encrypt_staging")]
+	LetsEncryptStaging,
+}
+
+impl Default for ProvisioningAcmeDirectory {
+	fn default() -> Self {
+		ProvisioningAcmeDirectory::LetsEncryptStaging
+	}
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug)]
@@ -646,6 +651,17 @@ pub enum ProvisioningProvider {
 	Linode,
 }
 
+#[derive(Serialize, Deserialize, Clone, Debug, Default, strum_macros::Display)]
+pub enum ProvisioningBuildDeliveryMethod {
+	#[serde(rename = "traffic_server")]
+	#[strum(serialize = "traffic_server")]
+	#[default]
+	TrafficServer,
+	#[serde(rename = "s3_direct")]
+	#[strum(serialize = "s3_direct")]
+	S3Direct,
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug)]
 #[serde(deny_unknown_fields)]
 pub struct ProvisioningDatacenterPool {

lib/bolt/core/src/context/service.rs

@@ -1039,6 +1039,11 @@ impl ServiceContextData {
 					provisioning.job_server_provision_margin.to_string(),
 				));
 			}
+
+			env.push((
+				format!("TLS_ACME_DIRECTORY"),
+				serde_json::to_string(&provisioning.acme_directory)?,
+			));
 		}
 
 		// Sort env by keys so it's always in the same order
@@ -1251,18 +1256,14 @@ impl ServiceContextData {
 				"TLS_CERT_LOCALLY_SIGNED_JOB_KEY_PEM".into(),
 				tls.tls_cert_locally_signed_job.key_pem.clone(),
 			));
-			env.push((
-				"TLS_CERT_LETSENCRYPT_RIVET_JOB_CERT_PEM".into(),
-				tls.tls_cert_letsencrypt_rivet_job.cert_pem.clone(),
-			));
-			env.push((
-				"TLS_CERT_LETSENCRYPT_RIVET_JOB_KEY_PEM".into(),
-				tls.tls_cert_letsencrypt_rivet_job.key_pem.clone(),
-			));
 			env.push((
 				"TLS_ROOT_CA_CERT_PEM".into(),
 				(*tls.root_ca_cert_pem).clone(),
 			));
+			env.push((
+				"TLS_ACME_ACCOUNT_PRIVATE_KEY_PEM".into(),
+				(*tls.acme_account_private_key_pem).clone(),
+			));
 			env.push((
 				"K8S_TRAEFIK_TUNNEL_EXTERNAL_IP".into(),
 				(*k8s_infra.traefik_tunnel_external_ip).clone(),

lib/bolt/core/src/dep/terraform/gen.rs

@@ -279,31 +279,6 @@ async fn vars(ctx: &ProjectContext) {
 		vars.insert("services".into(), json!(services));
 	}
 
-	// Datacenters
-	if let Some(cluster) = config
-		.rivet
-		.provisioning
-		.as_ref()
-		.and_then(|p| p.cluster.as_ref())
-	{
-		let datacenters = cluster
-			.datacenters
-			.iter()
-			.map(|(name_id, dc)| {
-				(
-					name_id,
-					json!({
-						"datacenter_id": dc.datacenter_id,
-					}),
-				)
-			})
-			.collect::<HashMap<_, _>>();
-
-		vars.insert("datacenters".into(), json!(datacenters));
-	} else {
-		vars.insert("datacenters".into(), json!({}));
-	};
-
 	// Docker
 	vars.insert(
 		"authenticate_all_docker_hub_pulls".into(),

lib/bolt/core/src/dep/terraform/output.rs

@@ -29,11 +29,11 @@ pub struct K8sInfra {
 
 #[derive(Debug, Clone, Deserialize)]
 pub struct Tls {
-	pub tls_cert_letsencrypt_rivet_job: TerraformOutputValue<Cert>,
 	pub tls_cert_locally_signed_tunnel_server: TerraformOutputValue<Cert>,
 	pub tls_cert_locally_signed_job: TerraformOutputValue<Cert>,
 	pub tls_cert_locally_signed_gg: TerraformOutputValue<Cert>,
 	pub root_ca_cert_pem: TerraformOutputValue<String>,
+	pub acme_account_private_key_pem: TerraformOutputValue<String>,
 }
 
 #[derive(Debug, Clone, Deserialize)]

proto/backend/cluster.proto

@@ -22,6 +22,7 @@ message Datacenter {
     rivet.common.Uuid cluster_id = 2;
     string name_id = 3;
     string display_name = 4;
+    int64 create_ts = 11;
 
     Provider provider = 5;
     string provider_datacenter_id = 6;
@@ -67,3 +68,9 @@ message Server {
 
     // TODO: Add the rest of the sql columns
 }
+
+enum TlsState {
+    CREATING = 0;
+    ACTIVE = 1;
+    RENEWING = 2;
+}