Skip to content

Add tunable parameters for Nexus config #1086

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
May 18, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions nexus/examples/config.toml
Original file line number Diff line number Diff line change
Expand Up @@ -51,3 +51,10 @@ mode = "stderr-terminal"
# Configuration for interacting with the timeseries database
[timeseries_db]
address = "[::1]:8123"

# Tunable configuration parameters, for testing or experimentation
[tunables]

# The maximum allowed prefix (thus smallest size) for a VPC Subnet's
# IPv4 subnetwork. This size allows for ~60 hosts.
max_vpc_ipv4_subnet_prefix = 26
9 changes: 9 additions & 0 deletions nexus/src/app/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,9 @@ pub struct Nexus {
/// Contents of the trusted root role for the TUF repository.
updates_config: Option<config::UpdatesConfig>,

/// The tunable parameters from a configuration file
tunables: config::Tunables,

/// Operational context used for Instance allocation
opctx_alloc: OpContext,

Expand Down Expand Up @@ -151,6 +154,7 @@ impl Nexus {
populate_status,
timeseries_client,
updates_config: config.updates.clone(),
tunables: config.tunables.clone(),
opctx_alloc: OpContext::for_background(
log.new(o!("component" => "InstanceAllocator")),
Arc::clone(&authz),
Expand Down Expand Up @@ -191,6 +195,11 @@ impl Nexus {
nexus
}

/// Return the tunable configuration parameters, e.g. for use in tests.
pub fn tunables(&self) -> &config::Tunables {
&self.tunables
}

pub async fn wait_for_populate(&self) -> Result<(), anyhow::Error> {
let mut my_rx = self.populate_status.clone();
loop {
Expand Down
5 changes: 3 additions & 2 deletions nexus/src/app/vpc_subnet.rs
Original file line number Diff line number Diff line change
Expand Up @@ -49,15 +49,16 @@ impl super::Nexus {
));
}
if params.ipv4_block.prefix() < defaults::MIN_VPC_IPV4_SUBNET_PREFIX
|| params.ipv4_block.prefix() > defaults::MAX_VPC_IPV4_SUBNET_PREFIX
|| params.ipv4_block.prefix()
> self.tunables.max_vpc_ipv4_subnet_prefix
{
return Err(external::Error::invalid_request(&format!(
concat!(
"VPC Subnet IPv4 address ranges must have prefix ",
"length between {} and {}, inclusive"
),
defaults::MIN_VPC_IPV4_SUBNET_PREFIX,
defaults::MAX_VPC_IPV4_SUBNET_PREFIX
self.tunables.max_vpc_ipv4_subnet_prefix,
)));
}

Expand Down
149 changes: 149 additions & 0 deletions nexus/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,77 @@ pub struct TimeseriesDbConfig {
pub address: SocketAddr,
}

// A deserializable type that does no validation on the tunable parameters.
#[derive(Clone, Debug, Deserialize, PartialEq)]
struct UnvalidatedTunables {
max_vpc_ipv4_subnet_prefix: u8,
}

/// Tunable configuration parameters, intended for use in test environments or
/// other situations in which experimentation / tuning is valuable.
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
#[serde(try_from = "UnvalidatedTunables")]
pub struct Tunables {
/// The maximum prefix size supported for VPC Subnet IPv4 subnetworks.
///
/// Note that this is the maximum _prefix_ size, which sets the minimum size
/// of the subnet.
pub max_vpc_ipv4_subnet_prefix: u8,
}

// Convert from the unvalidated tunables, verifying each parameter as needed.
impl TryFrom<UnvalidatedTunables> for Tunables {
type Error = InvalidTunable;

fn try_from(unvalidated: UnvalidatedTunables) -> Result<Self, Self::Error> {
Tunables::validate_ipv4_prefix(unvalidated.max_vpc_ipv4_subnet_prefix)?;
Ok(Tunables {
max_vpc_ipv4_subnet_prefix: unvalidated.max_vpc_ipv4_subnet_prefix,
})
}
}

impl Tunables {
fn validate_ipv4_prefix(prefix: u8) -> Result<(), InvalidTunable> {
let absolute_max: u8 = 32_u8.checked_sub(
// Always need space for the reserved Oxide addresses, including the
// broadcast address at the end of the subnet.
((crate::defaults::NUM_INITIAL_RESERVED_IP_ADDRESSES + 1) as f32)
.log2() // Subnet size to bit prefix.
.ceil() // Round up to a whole number of bits.
as u8
).expect("Invalid absolute maximum IPv4 subnet prefix");
if prefix >= crate::defaults::MIN_VPC_IPV4_SUBNET_PREFIX
&& prefix <= absolute_max
{
Ok(())
} else {
Err(InvalidTunable {
tunable: String::from("max_vpc_ipv4_subnet_prefix"),
message: format!(
"IPv4 subnet prefix must be in the range [0, {}], found: {}",
absolute_max,
prefix,
),
})
}
}
}

/// The maximum prefix size by default.
///
/// There are 6 Oxide reserved IP addresses, 5 at the beginning for DNS and the
/// like, and the broadcast address at the end of the subnet. This size provides
/// room for 2 ** 6 - 6 = 58 IP addresses, which seems like a reasonable size
/// for the smallest subnet that's still useful in many contexts.
pub const MAX_VPC_IPV4_SUBNET_PREFIX: u8 = 26;

impl Default for Tunables {
fn default() -> Self {
Tunables { max_vpc_ipv4_subnet_prefix: MAX_VPC_IPV4_SUBNET_PREFIX }
}
}

/// Configuration for a nexus server
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
pub struct Config {
Expand All @@ -74,8 +145,25 @@ pub struct Config {
/// unconfigured.
#[serde(default)]
pub updates: Option<UpdatesConfig>,
/// Tunable configuration for testing and experimentation
#[serde(default)]
pub tunables: Tunables,
}

#[derive(Debug)]
pub struct InvalidTunable {
tunable: String,
message: String,
}

impl std::fmt::Display for InvalidTunable {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "invalid \"{}\": \"{}\"", self.tunable, self.message)
}
}

impl std::error::Error for InvalidTunable {}

#[derive(Debug)]
pub struct LoadError {
path: PathBuf,
Expand All @@ -85,6 +173,7 @@ pub struct LoadError {
pub enum LoadErrorKind {
Io(std::io::Error),
Parse(toml::de::Error),
InvalidTunable(InvalidTunable),
}

impl From<(PathBuf, std::io::Error)> for LoadError {
Expand All @@ -110,6 +199,14 @@ impl fmt::Display for LoadError {
LoadErrorKind::Parse(e) => {
write!(f, "parse \"{}\": {}", self.path.display(), e)
}
LoadErrorKind::InvalidTunable(inner) => {
write!(
f,
"invalid tunable \"{}\": {}",
self.path.display(),
inner,
)
}
}
}
}
Expand Down Expand Up @@ -175,6 +272,7 @@ impl Config {

#[cfg(test)]
mod test {
use super::Tunables;
use super::{
AuthnConfig, Config, ConsoleConfig, LoadError, LoadErrorKind,
SchemeName, TimeseriesDbConfig, UpdatesConfig,
Expand Down Expand Up @@ -301,6 +399,8 @@ mod test {
[updates]
trusted_root = "/path/to/root.json"
default_base_url = "http://example.invalid/"
[tunables]
max_vpc_ipv4_subnet_prefix = 27
"##,
)
.unwrap();
Expand Down Expand Up @@ -345,6 +445,7 @@ mod test {
trusted_root: PathBuf::from("/path/to/root.json"),
default_base_url: "http://example.invalid/".into(),
}),
tunables: Tunables { max_vpc_ipv4_subnet_prefix: 27 },
}
);

Expand Down Expand Up @@ -427,4 +528,52 @@ mod test {
);
}
}

#[test]
fn test_invalid_ipv4_prefix_tunable() {
let error = read_config(
"invalid_ipv4_prefix_tunable",
r##"
id = "28b90dc4-c22a-65ba-f49a-f051fe01208f"
[console]
static_dir = "tests/static"
cache_control_max_age_minutes = 10
session_idle_timeout_minutes = 60
session_absolute_timeout_minutes = 480
[authn]
schemes_external = []
[dropshot_external]
bind_address = "10.1.2.3:4567"
request_body_max_bytes = 1024
[dropshot_internal]
bind_address = "10.1.2.3:4568"
request_body_max_bytes = 1024
[database]
url = "postgresql://127.0.0.1?sslmode=disable"
[log]
mode = "file"
level = "debug"
path = "/nonexistent/path"
if_exists = "fail"
[timeseries_db]
address = "[::1]:8123"
[updates]
trusted_root = "/path/to/root.json"
default_base_url = "http://example.invalid/"
[tunables]
max_vpc_ipv4_subnet_prefix = 100
"##,
)
.expect_err("Expected failure");
if let LoadErrorKind::Parse(error) = &error.kind {
assert!(error.to_string().starts_with(
r#"invalid "max_vpc_ipv4_subnet_prefix": "IPv4 subnet prefix must"#,
));
} else {
panic!(
"Got an unexpected error, expected Parse but got {:?}",
error
);
}
}
}
5 changes: 0 additions & 5 deletions nexus/src/defaults.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,6 @@ use std::net::Ipv6Addr;
/// NOTE: This is the minimum _prefix_, which sets the maximum subnet size.
pub const MIN_VPC_IPV4_SUBNET_PREFIX: u8 = 8;

/// Maximum prefix size supported in IPv4 VPC Subnets.
///
/// NOTE: This is the maximum _prefix_, which sets the minimum subnet size.
pub const MAX_VPC_IPV4_SUBNET_PREFIX: u8 = 26;

/// The number of reserved addresses at the beginning of a subnet range.
pub const NUM_INITIAL_RESERVED_IP_ADDRESSES: usize = 5;

Expand Down
5 changes: 5 additions & 0 deletions nexus/tests/config.test.toml
Original file line number Diff line number Diff line change
Expand Up @@ -54,3 +54,8 @@ path = "UNUSED"
# is listening.
[timeseries_db]
address = "[::1]:0"

# Tunable parameters used during tests
[tunables]
# Allow small subnets, so we can test IP address exhaustion easily / quickly
max_vpc_ipv4_subnet_prefix = 29
13 changes: 9 additions & 4 deletions nexus/tests/integration_tests/subnet_allocation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ use dropshot::test_util::ClientTestContext;
use dropshot::HttpErrorResponseBody;
use http::method::Method;
use http::StatusCode;
use ipnetwork::Ipv4Network;
use nexus_test_utils::http_testing::AuthnMode;
use nexus_test_utils::http_testing::NexusRequest;
use nexus_test_utils::http_testing::RequestBuilder;
Expand All @@ -23,6 +24,7 @@ use omicron_common::api::external::{
};
use omicron_nexus::defaults::NUM_INITIAL_RESERVED_IP_ADDRESSES;
use omicron_nexus::external_api::params;
use std::net::Ipv4Addr;

async fn create_instance_expect_failure(
client: &ClientTestContext,
Expand Down Expand Up @@ -87,12 +89,16 @@ async fn test_subnet_allocation(cptestctx: &ControlPlaneTestContext) {

// Create a new, small VPC Subnet, so we don't need to issue many requests
// to test address exhaustion.
let subnet_size =
cptestctx.server.apictx.nexus.tunables().max_vpc_ipv4_subnet_prefix;
let url_subnets = format!(
"/organizations/{}/projects/{}/vpcs/default/subnets",
organization_name, project_name
);
let subnet_name = "small";
let subnet = "192.168.42.0/26".parse().unwrap();
let network_address = Ipv4Addr::new(192, 168, 42, 0);
let subnet = Ipv4Network::new(network_address, subnet_size)
.expect("Invalid IPv4 network");
let subnet_create = params::VpcSubnetCreate {
identity: IdentityMetadataCreateParams {
name: subnet_name.parse().unwrap(),
Expand Down Expand Up @@ -122,9 +128,8 @@ async fn test_subnet_allocation(cptestctx: &ControlPlaneTestContext) {
},
]);

// Create enough instances to fill the subnet. There should be 58 instances
// total created here. The subnet is a /26, and there are 6 reserved
// addresses. So 2 ** (32 - 26) - 6 = 2 ** 6 - 6 = 64 - 6 = 58.
// Create enough instances to fill the subnet. There are subnet.size() total
// addresses, 6 of which are reserved.
let n_final_reserved_addresses = 1;
let n_reserved_addresses =
NUM_INITIAL_RESERVED_IP_ADDRESSES + n_final_reserved_addresses;
Expand Down