diff --git a/docs-internal/infrastructure/fdb/CLUSTER.md b/docs-internal/infrastructure/fdb/CLUSTER.md new file mode 100644 index 0000000000..2cc151b4b8 --- /dev/null +++ b/docs-internal/infrastructure/fdb/CLUSTER.md @@ -0,0 +1,6 @@ +# Cluster layout + +Ideally, 3 servers per DC that coordinate with each other. + +Each process should have around 4GB of memory, so servers with more than 4GB can run multiple processes +(unconfirmed). diff --git a/docs-old/infrastructure/pegboard/FDB.md b/docs-internal/infrastructure/pegboard/FDB.md similarity index 100% rename from docs-old/infrastructure/pegboard/FDB.md rename to docs-internal/infrastructure/pegboard/FDB.md diff --git a/packages/services/build/standalone/default-create/src/lib.rs b/packages/services/build/standalone/default-create/src/lib.rs index 457bd1874c..b55fa8c96c 100644 --- a/packages/services/build/standalone/default-create/src/lib.rs +++ b/packages/services/build/standalone/default-create/src/lib.rs @@ -97,7 +97,7 @@ async fn upload_build( backend::upload::PrepareFile { path: unwrap!(unwrap!(build.key.file_name(), "should have file name").to_str()).to_string(), content_length: len, - multipart: false, + multipart: len > util::file_size::mebibytes(100), ..Default::default() }, ], diff --git a/packages/services/cluster/src/workflows/server/install/install_scripts/components/fdb.rs b/packages/services/cluster/src/workflows/server/install/install_scripts/components/fdb.rs index 2764a415a6..1a84017b0b 100644 --- a/packages/services/cluster/src/workflows/server/install/install_scripts/components/fdb.rs +++ b/packages/services/cluster/src/workflows/server/install/install_scripts/components/fdb.rs @@ -1,8 +1,12 @@ +pub const FDB_VERSION: &str = "7.1.60"; + pub fn install(initialize_immediately: bool) -> String { - let mut script = include_str!("../files/fdb_install.sh").replace( - "__PROMETHEUS_PROXY_SCRIPT__", - include_str!("../files/fdp_prometheus_proxy.py"), - ); + let mut script = include_str!("../files/fdb_install.sh") + .replace("__FDB_VERSION__", FDB_VERSION) + .replace( + "__PROMETHEUS_PROXY_SCRIPT__", + include_str!("../files/fdp_prometheus_proxy.py"), + ); if initialize_immediately { // Run script immediately diff --git a/packages/services/cluster/src/workflows/server/install/install_scripts/components/pegboard.rs b/packages/services/cluster/src/workflows/server/install/install_scripts/components/pegboard.rs index d2825bc134..d5827395f1 100644 --- a/packages/services/cluster/src/workflows/server/install/install_scripts/components/pegboard.rs +++ b/packages/services/cluster/src/workflows/server/install/install_scripts/components/pegboard.rs @@ -1,6 +1,6 @@ use chirp_workflow::prelude::*; -use super::TUNNEL_API_EDGE_PORT; +use super::{fdb::FDB_VERSION, TUNNEL_API_EDGE_PORT}; pub const TUNNEL_PEGBOARD_WS_PORT: u16 = 5030; @@ -23,7 +23,8 @@ pub async fn install( .replace( "__ISOLATE_V8_RUNNER_BINARY_URL__", provision_config.isolate_runner_binary_url.as_ref(), - )) + ) + .replace("__FDB_VERSION__", FDB_VERSION)) } pub fn configure( diff --git a/packages/services/cluster/src/workflows/server/install/install_scripts/files/fdb_configure.sh b/packages/services/cluster/src/workflows/server/install/install_scripts/files/fdb_configure.sh index 3772b0d406..b979455b49 100644 --- a/packages/services/cluster/src/workflows/server/install/install_scripts/files/fdb_configure.sh +++ b/packages/services/cluster/src/workflows/server/install/install_scripts/files/fdb_configure.sh @@ -1,8 +1,52 @@ # Append config -# cat << 'EOF' >> /etc/foundationdb/foundationdb.conf -# [fdbserver] -# EOF +cat << 'EOF' > /etc/foundationdb/foundationdb.conf +## foundationdb.conf +## +## Configuration file for FoundationDB server processes +## Full documentation is available at +## https://apple.github.io/foundationdb/configuration.html#the-configuration-file -# TODO: add -t flag for TLS (https://apple.github.io/foundationdb/tls.html#enable-tls) -# Make fdb accessible on VLAN -python3 /usr/lib/foundationdb/make_public.py -a ___VLAN_IP___ +[fdbmonitor] +user = foundationdb +group = foundationdb + +[general] +restart-delay = 60 +## by default, restart-backoff = restart-delay-reset-interval = restart-delay +# initial-restart-delay = 0 +# restart-backoff = 60 +# restart-delay-reset-interval = 60 +cluster-file = /etc/foundationdb/fdb.cluster +# delete-envvars = +# kill-on-configuration-change = true + +## Default parameters for individual fdbserver processes +[fdbserver] +command = /usr/sbin/fdbserver +public-address = ___VLAN_IP___:$ID +listen-address = 0.0.0.0:$ID +datadir = /var/lib/foundationdb/data/$ID +logdir = /var/log/foundationdb +machine-id = ___SERVER_ID___ +datacenter-id = ___DATACENTER_ID___ +# logsize = 10MiB +# maxlogssize = 100MiB +# class = +# memory = 8GiB +# storage-memory = 1GiB +# cache-memory = 2GiB +# metrics-cluster = +# metrics-prefix = +# TODO: TLS + +# Individual servers +[fdbserver.4500] +[fdbserver.4501] + +[backup_agent] +command = /usr/lib/foundationdb/backup_agent/backup_agent +logdir = /var/log/foundationdb + +[backup_agent.1] + +EOF diff --git a/packages/services/cluster/src/workflows/server/install/install_scripts/files/fdb_install.sh b/packages/services/cluster/src/workflows/server/install/install_scripts/files/fdb_install.sh index a850ae7a55..d7e166fc00 100644 --- a/packages/services/cluster/src/workflows/server/install/install_scripts/files/fdb_install.sh +++ b/packages/services/cluster/src/workflows/server/install/install_scripts/files/fdb_install.sh @@ -1,27 +1,35 @@ sysctl --system +# For clarity +FDB_VERSION="__FDB_VERSION__" + mkdir -p /etc/foundationdb -curl -Lf -o /tmp/foundationdb-clients_OFF-1_amd64.deb "https://github.com/apple/foundationdb/releases/download/7.3.43/foundationdb-clients_7.3.43-1_amd64.deb" -dpkg -i /tmp/foundationdb-clients_OFF-1_amd64.deb +# Custom cluster description and ID (https://apple.github.io/foundationdb/administration.html#cluster-file-format) +cat << 'EOF' > /etc/foundationdb/fdb.cluster +fdb:fdb@127.0.0.1:4500 +EOF + +curl -Lf -o "/tmp/foundationdb-clients_${FDB_VERSION}-1_amd64.deb" "https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/foundationdb-clients_${FDB_VERSION}-1_amd64.deb" +dpkg -i "/tmp/foundationdb-clients_${FDB_VERSION}-1_amd64.deb" # Verify installation fdbcli --version -curl -Lf -o /tmp/foundationdb-server_OFF-1_amd64.deb "https://github.com/apple/foundationdb/releases/download/7.3.43/foundationdb-server_7.3.43-1_amd64.deb" -dpkg -i /tmp/foundationdb-server_OFF-1_amd64.deb +curl -Lf -o "/tmp/foundationdb-server_${FDB_VERSION}-1_amd64.deb" "https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/foundationdb-server_${FDB_VERSION}-1_amd64.deb" +dpkg -i "/tmp/foundationdb-server_${FDB_VERSION}-1_amd64.deb" # Verify installation fdbserver --version +# Required for 7.3 +# fdbcli --exec "configure perpetual_storage_wiggle=1 storage_migration_type=gradual" + # https://apple.github.io/foundationdb/administration.html#administration-running-foundationdb # Configure redundancy and storage engine -fdbcli --exec "configure perpetual_storage_wiggle=1 storage_migration_type=gradual" -fdbcli --exec "configure single ssd" -service foundationdb stop - +fdbcli --exec "configure new single ssd" -pip install wheel foundationdb prometheus_client +pip install wheel "foundationdb==${FDB_VERSION}" prometheus_client cat << 'EOF' > /usr/local/bin/fdb_prometheus_proxy.py __PROMETHEUS_PROXY_SCRIPT__ @@ -46,5 +54,5 @@ EOF systemctl daemon-reload systemctl enable fdb_prometheus_proxy -# NOTE: we dont have a systemd service for fdbserver because it uses `service`: +# NOTE: systemd service for foundationdb automatically created from dpkg: # https://apple.github.io/foundationdb/administration.html#administration-running-foundationdb diff --git a/packages/services/cluster/src/workflows/server/install/install_scripts/files/fdp_prometheus_proxy.py b/packages/services/cluster/src/workflows/server/install/install_scripts/files/fdp_prometheus_proxy.py index 8c5ad3a05c..6ebe67591d 100644 --- a/packages/services/cluster/src/workflows/server/install/install_scripts/files/fdp_prometheus_proxy.py +++ b/packages/services/cluster/src/workflows/server/install/install_scripts/files/fdp_prometheus_proxy.py @@ -998,7 +998,7 @@ def main(): status_file = args.status_file else: assert args.fdb_cluster_file - fdb.api_version(730) + fdb.api_version(710) try: db = fdb.open(cluster_file=cluster_file_path) except Exception: diff --git a/packages/services/cluster/src/workflows/server/install/install_scripts/files/pegboard_configure.sh b/packages/services/cluster/src/workflows/server/install/install_scripts/files/pegboard_configure.sh index 29327441e4..869f43da21 100644 --- a/packages/services/cluster/src/workflows/server/install/install_scripts/files/pegboard_configure.sh +++ b/packages/services/cluster/src/workflows/server/install/install_scripts/files/pegboard_configure.sh @@ -28,7 +28,7 @@ cat << 'EOF' > /etc/rivet-client/config.json }, "reserved_resources": { "cpu": 0, - "memory": "__RESERVED_MEMORY__" + "memory": __RESERVED_MEMORY__ }, "vector": { "address": "127.0.0.1:5021" diff --git a/packages/services/cluster/src/workflows/server/install/install_scripts/files/pegboard_install.sh b/packages/services/cluster/src/workflows/server/install/install_scripts/files/pegboard_install.sh index c4dce85252..657f9e3462 100644 --- a/packages/services/cluster/src/workflows/server/install/install_scripts/files/pegboard_install.sh +++ b/packages/services/cluster/src/workflows/server/install/install_scripts/files/pegboard_install.sh @@ -21,3 +21,9 @@ if [ "__FLAVOR__" = "isolate" ]; then curl -Lf -o /usr/local/bin/rivet-isolate-v8-runner "__ISOLATE_V8_RUNNER_BINARY_URL__" chmod +x /usr/local/bin/rivet-isolate-v8-runner fi + +# For clarity +FDB_VERSION="__FDB_VERSION__" + +# Shared object for fdb client +curl -Lf -o /lib/libfdb_c.so "https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/libfdb_c.x86_64.so" diff --git a/packages/services/cluster/src/workflows/server/install/install_scripts/mod.rs b/packages/services/cluster/src/workflows/server/install/install_scripts/mod.rs index 8491fbd38f..cc2a750984 100644 --- a/packages/services/cluster/src/workflows/server/install/install_scripts/mod.rs +++ b/packages/services/cluster/src/workflows/server/install/install_scripts/mod.rs @@ -168,7 +168,7 @@ pub async fn gen_initialize( PoolType::Fdb => { script.push(components::fdb::configure()); prometheus_targets.insert( - "pegboard".into(), + "foundationdb".into(), components::vector::PrometheusTarget { endpoint: "http://127.0.0.1:9161".into(), scrape_interval: 15,