fix: get pegboard -> fdb connection over vlan

docs-internal/infrastructure/fdb/CLUSTER.md

@@ -0,0 +1,6 @@
+# Cluster layout
+
+Ideally, 3 servers per DC that coordinate with each other.
+
+Each process should have around 4GB of memory, so servers with more than 4GB can run multiple processes
+(unconfirmed).

packages/services/build/standalone/default-create/src/lib.rs

@@ -97,7 +97,7 @@ async fn upload_build(
 			backend::upload::PrepareFile {
 				path: unwrap!(unwrap!(build.key.file_name(), "should have file name").to_str()).to_string(),
 				content_length: len,
-				multipart: false,
+				multipart: len > util::file_size::mebibytes(100),
 				..Default::default()
 			},
 		],

packages/services/cluster/src/workflows/server/install/install_scripts/components/fdb.rs

@@ -1,8 +1,12 @@
+pub const FDB_VERSION: &str = "7.1.60";
+
 pub fn install(initialize_immediately: bool) -> String {
-	let mut script = include_str!("../files/fdb_install.sh").replace(
-		"__PROMETHEUS_PROXY_SCRIPT__",
-		include_str!("../files/fdp_prometheus_proxy.py"),
-	);
+	let mut script = include_str!("../files/fdb_install.sh")
+		.replace("__FDB_VERSION__", FDB_VERSION)
+		.replace(
+			"__PROMETHEUS_PROXY_SCRIPT__",
+			include_str!("../files/fdp_prometheus_proxy.py"),
+		);
 
 	if initialize_immediately {
 		// Run script immediately

packages/services/cluster/src/workflows/server/install/install_scripts/components/pegboard.rs

@@ -1,6 +1,6 @@
 use chirp_workflow::prelude::*;
 
-use super::TUNNEL_API_EDGE_PORT;
+use super::{fdb::FDB_VERSION, TUNNEL_API_EDGE_PORT};
 
 pub const TUNNEL_PEGBOARD_WS_PORT: u16 = 5030;
 
@@ -23,7 +23,8 @@ pub async fn install(
 		.replace(
 			"__ISOLATE_V8_RUNNER_BINARY_URL__",
 			provision_config.isolate_runner_binary_url.as_ref(),
-		))
+		)
+		.replace("__FDB_VERSION__", FDB_VERSION))
 }
 
 pub fn configure(

packages/services/cluster/src/workflows/server/install/install_scripts/files/fdb_configure.sh

@@ -1,8 +1,52 @@
 # Append config
-# cat << 'EOF' >> /etc/foundationdb/foundationdb.conf
-# [fdbserver]
-# EOF
+cat << 'EOF' > /etc/foundationdb/foundationdb.conf
+## foundationdb.conf
+##
+## Configuration file for FoundationDB server processes
+## Full documentation is available at
+## https://apple.github.io/foundationdb/configuration.html#the-configuration-file
 
-# TODO: add -t flag for TLS (https://apple.github.io/foundationdb/tls.html#enable-tls)
-# Make fdb accessible on VLAN
-python3 /usr/lib/foundationdb/make_public.py -a ___VLAN_IP___
+[fdbmonitor]
+user = foundationdb
+group = foundationdb
+
+[general]
+restart-delay = 60
+## by default, restart-backoff = restart-delay-reset-interval = restart-delay
+# initial-restart-delay = 0
+# restart-backoff = 60
+# restart-delay-reset-interval = 60
+cluster-file = /etc/foundationdb/fdb.cluster
+# delete-envvars =
+# kill-on-configuration-change = true
+ 
+## Default parameters for individual fdbserver processes
+[fdbserver]
+command = /usr/sbin/fdbserver
+public-address = ___VLAN_IP___:$ID
+listen-address = 0.0.0.0:$ID
+datadir = /var/lib/foundationdb/data/$ID
+logdir = /var/log/foundationdb
+machine-id = ___SERVER_ID___
+datacenter-id = ___DATACENTER_ID___
+# logsize = 10MiB
+# maxlogssize = 100MiB
+# class = 
+# memory = 8GiB
+# storage-memory = 1GiB
+# cache-memory = 2GiB
+# metrics-cluster =
+# metrics-prefix =
+# TODO: TLS
+
+# Individual servers
+[fdbserver.4500]
+[fdbserver.4501]
+ 
+[backup_agent]
+command = /usr/lib/foundationdb/backup_agent/backup_agent
+logdir = /var/log/foundationdb
+
+[backup_agent.1]
+
+EOF

packages/services/cluster/src/workflows/server/install/install_scripts/files/fdb_install.sh

@@ -1,27 +1,35 @@
 sysctl --system
 
+# For clarity
+FDB_VERSION="__FDB_VERSION__"
+
 mkdir -p /etc/foundationdb
 
-curl -Lf -o /tmp/foundationdb-clients_OFF-1_amd64.deb "https://github.com/apple/foundationdb/releases/download/7.3.43/foundationdb-clients_7.3.43-1_amd64.deb"
-dpkg -i /tmp/foundationdb-clients_OFF-1_amd64.deb
+# Custom cluster description and ID (https://apple.github.io/foundationdb/administration.html#cluster-file-format)
+cat << 'EOF' > /etc/foundationdb/fdb.cluster
+fdb:fdb@127.0.0.1:4500
+EOF
+
+curl -Lf -o "/tmp/foundationdb-clients_${FDB_VERSION}-1_amd64.deb" "https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/foundationdb-clients_${FDB_VERSION}-1_amd64.deb"
+dpkg -i "/tmp/foundationdb-clients_${FDB_VERSION}-1_amd64.deb"
 
 # Verify installation
 fdbcli --version
 
-curl -Lf -o /tmp/foundationdb-server_OFF-1_amd64.deb "https://github.com/apple/foundationdb/releases/download/7.3.43/foundationdb-server_7.3.43-1_amd64.deb"
-dpkg -i /tmp/foundationdb-server_OFF-1_amd64.deb
+curl -Lf -o "/tmp/foundationdb-server_${FDB_VERSION}-1_amd64.deb" "https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/foundationdb-server_${FDB_VERSION}-1_amd64.deb"
+dpkg -i "/tmp/foundationdb-server_${FDB_VERSION}-1_amd64.deb"
 
 # Verify installation
 fdbserver --version
 
+# Required for 7.3
+# fdbcli --exec "configure perpetual_storage_wiggle=1 storage_migration_type=gradual"
+
 # https://apple.github.io/foundationdb/administration.html#administration-running-foundationdb
 # Configure redundancy and storage engine
-fdbcli --exec "configure perpetual_storage_wiggle=1 storage_migration_type=gradual"
-fdbcli --exec "configure single ssd"
-service foundationdb stop
-
+fdbcli --exec "configure new single ssd"
 
-pip install wheel foundationdb prometheus_client
+pip install wheel "foundationdb==${FDB_VERSION}" prometheus_client
 
 cat << 'EOF' > /usr/local/bin/fdb_prometheus_proxy.py
 __PROMETHEUS_PROXY_SCRIPT__
@@ -46,5 +54,5 @@ EOF
 systemctl daemon-reload
 systemctl enable fdb_prometheus_proxy
 
-# NOTE: we dont have a systemd service for fdbserver because it uses `service`:
+# NOTE: systemd service for foundationdb automatically created from dpkg:
 # https://apple.github.io/foundationdb/administration.html#administration-running-foundationdb

packages/services/cluster/src/workflows/server/install/install_scripts/files/fdp_prometheus_proxy.py

@@ -998,7 +998,7 @@ def main():
         status_file = args.status_file
     else:
         assert args.fdb_cluster_file
-        fdb.api_version(730)
+        fdb.api_version(710)
         try:
             db = fdb.open(cluster_file=cluster_file_path)
         except Exception:

packages/services/cluster/src/workflows/server/install/install_scripts/files/pegboard_configure.sh

@@ -28,7 +28,7 @@ cat << 'EOF' > /etc/rivet-client/config.json
 		},
 		"reserved_resources": {
 			"cpu": 0,
-			"memory": "__RESERVED_MEMORY__"
+			"memory": __RESERVED_MEMORY__
 		},
 		"vector": {
 			"address": "127.0.0.1:5021"

packages/services/cluster/src/workflows/server/install/install_scripts/files/pegboard_install.sh

@@ -21,3 +21,9 @@ if [ "__FLAVOR__" = "isolate" ]; then
 	curl -Lf -o /usr/local/bin/rivet-isolate-v8-runner "__ISOLATE_V8_RUNNER_BINARY_URL__"
 	chmod +x /usr/local/bin/rivet-isolate-v8-runner
 fi
+
+# For clarity
+FDB_VERSION="__FDB_VERSION__"
+
+# Shared object for fdb client
+curl -Lf -o /lib/libfdb_c.so "https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/libfdb_c.x86_64.so"

packages/services/cluster/src/workflows/server/install/install_scripts/mod.rs

@@ -168,7 +168,7 @@ pub async fn gen_initialize(
 		PoolType::Fdb => {
 			script.push(components::fdb::configure());
 			prometheus_targets.insert(
-				"pegboard".into(),
+				"foundationdb".into(),
 				components::vector::PrometheusTarget {
 					endpoint: "http://127.0.0.1:9161".into(),
 					scrape_interval: 15,