Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: improve scale sorting #1717

Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ pub struct ClusterPools {
#[derive(Debug, Serialize, Deserialize, Clone, JsonSchema)]
#[serde(rename_all = "snake_case", deny_unknown_fields)]
pub struct ClusterPoolJob {
pub provision_margin: u32,
pub autoscale_margin: u32,
// All other properties are read from Pegboard since they're identical
}

Expand All @@ -53,7 +53,7 @@ pub struct ClusterPoolJob {
#[derive(Debug, Serialize, Deserialize, Clone, Default, JsonSchema)]
#[serde(rename_all = "snake_case", deny_unknown_fields)]
pub struct ClusterPoolPegboard {
pub provision_margin: u32,
pub autoscale_margin: u32,

pub vlan_addr_range_min: Option<Ipv4Addr>,
pub vlan_addr_range_max: Option<Ipv4Addr>,
Expand Down Expand Up @@ -125,6 +125,8 @@ impl ClusterPoolPegboard {
#[derive(Debug, Serialize, Deserialize, Clone, JsonSchema)]
#[serde(rename_all = "snake_case", deny_unknown_fields)]
pub struct ClusterPoolGg {
pub autoscale_margin: u32,

#[schemars(with = "Option<String>")]
pub vlan_ip_net: Option<Ipv4Net>,
pub firewall_rules: Option<Vec<FirewallRule>>,
Expand Down Expand Up @@ -205,6 +207,8 @@ impl ClusterPoolGg {
#[derive(Debug, Serialize, Deserialize, Clone, JsonSchema)]
#[serde(rename_all = "snake_case", deny_unknown_fields)]
pub struct ClusterPoolAts {
pub autoscale_margin: u32,

#[schemars(with = "Option<String>")]
pub vlan_ip_net: Option<Ipv4Net>,
pub firewall_rules: Option<Vec<FirewallRule>>,
Expand Down
1 change: 1 addition & 0 deletions packages/common/util/core/src/math.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
/// Divide integers of any type, rounding up. Panics on dividing by 0.
#[deprecated(note="use x.div_ceil(y) instead")]
#[macro_export]
macro_rules! div_up {
($a:expr, $b:expr) => {
Expand Down
37 changes: 30 additions & 7 deletions packages/services/cluster/src/workflows/datacenter/scale.rs
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,12 @@ struct PoolCtx {
desired_count: usize,
}

#[derive(Clone)]
struct ServerSortTopology {
cpu: u32,
memory: u32,
}

#[derive(Debug, Serialize, Deserialize)]
pub struct Input {
pub datacenter_id: Uuid,
Expand Down Expand Up @@ -189,20 +195,29 @@ async fn calculate_diff(

let dc = unwrap!(datacenter_res.datacenters.first());

// Build hashmap from topos for sorting
let topology = unwrap!(topology_res.datacenters.first());
let memory_by_server = topology
let topo_by_server = topology
.servers
.iter()
.map(|server| Ok((server.server_id, server.usage.memory)))
.map(|server| {
Ok((
server.server_id,
ServerSortTopology {
cpu: server.usage.cpu,
memory: server.usage.memory,
},
))
})
.collect::<GlobalResult<HashMap<_, _>>>()?;

// Run everything in a locking transaction
let actions = rivet_pools::utils::crdb::tx(&ctx.crdb().await?, |tx| {
let ctx = ctx.clone();
let dc = dc.clone();
let memory_by_server = memory_by_server.clone();
let topo_by_server = topo_by_server.clone();

inner(ctx, tx, dc, memory_by_server).boxed()
inner(ctx, tx, dc, topo_by_server).boxed()
})
.await?;

Expand All @@ -213,7 +228,7 @@ async fn inner(
ctx: ActivityCtx,
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
dc: Datacenter,
memory_by_server: HashMap<Uuid, u32>,
topo_by_server: HashMap<Uuid, ServerSortTopology>,
) -> GlobalResult<Vec<Action>> {
let servers = sql_fetch_all!(
[ctx, ServerRow, @tx tx]
Expand Down Expand Up @@ -244,8 +259,16 @@ async fn inner(
.map(TryInto::try_into)
.collect::<GlobalResult<Vec<Server>>>()?;

// Sort servers by memory usage
servers.sort_by_key(|server| memory_by_server.get(&server.server_id));
// Sort servers
servers.sort_by_key(|server| {
let topo = topo_by_server.get(&server.server_id);
match server.pool_type {
// Sort job and pb servers by memory
PoolType::Job | PoolType::Pegboard => (topo.map(|t| t.memory), None),
// Sort the rest by cpu
_ => (None, topo.map(|t| t.cpu)),
}
});

let mut actions = Vec::new();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ use indoc::formatdoc;
use super::{
ok_server::OK_SERVER_PORT,
pegboard::TUNNEL_PEGBOARD_WS_PORT,
vector::{TUNNEL_VECTOR_PORT, TUNNEL_VECTOR_TCP_JSON_PORT},
rivet::TUNNEL_API_EDGE_PORT,
vector::{TUNNEL_VECTOR_PORT, TUNNEL_VECTOR_TCP_JSON_PORT},
};

pub const TUNNEL_SERVICES: &[TunnelService] = &[
Expand Down
Loading