diff --git a/mgmtd/assets/beegfs-mgmtd.toml b/mgmtd/assets/beegfs-mgmtd.toml index c876e21..55acc4f 100644 --- a/mgmtd/assets/beegfs-mgmtd.toml +++ b/mgmtd/assets/beegfs-mgmtd.toml @@ -62,10 +62,9 @@ # # interfaces = ["*"] -# Prefers an interfaces ipv6 addresses over ipv4. -# By default, ipv4 addresses are preferred. If the interface filter is given, the interface -# order has higher priority than the address family, which is sorted per interface. -# interfaces_prefer_ipv6 = false + +# Force disable IPv6. +# ipv6_disable = false, # Maximum number of outgoing connections per node. # connection-limit = 12 diff --git a/mgmtd/src/config.rs b/mgmtd/src/config.rs index 3f1726c..19fe43a 100644 --- a/mgmtd/src/config.rs +++ b/mgmtd/src/config.rs @@ -232,6 +232,11 @@ generate_structs! { #[arg(value_parser = nic::NicFilter::parse)] interfaces: Vec = vec![], + /// Force disable IPv6. + #[arg(long)] + #[arg(num_args = 0..=1, default_missing_value = "true")] + ipv6_disable: bool = false, + /// Maximum number of outgoing BeeMsg connections per node. [default: 12] #[arg(long)] #[arg(value_name = "LIMIT")] diff --git a/mgmtd/src/grpc.rs b/mgmtd/src/grpc.rs index 06ceb64..01827fe 100644 --- a/mgmtd/src/grpc.rs +++ b/mgmtd/src/grpc.rs @@ -16,7 +16,7 @@ use sqlite::{TransactionExt, check_affected_rows}; use sqlite_check::sql; use std::fmt::Debug; use std::future::Future; -use std::net::SocketAddr; +use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr}; use std::pin::Pin; use tonic::transport::{Identity, Server, ServerTlsConfig}; use tonic::{Code, Request, Response, Status}; @@ -210,7 +210,14 @@ pub(crate) fn serve(ctx: Context, mut shutdown: RunStateHandle) -> Result<()> { }, ); - let serve_addr = shared::nic::select_bind_addr(ctx.info.user_config.grpc_port); + let serve_addr = SocketAddr::new( + if ctx.info.use_ipv6 { + Ipv6Addr::UNSPECIFIED.into() + } else { + Ipv4Addr::UNSPECIFIED.into() + }, + ctx.info.user_config.grpc_port, + ); log::info!("Serving gRPC requests on {serve_addr}"); diff --git a/mgmtd/src/lib.rs b/mgmtd/src/lib.rs index 8f9f69d..cb3eba8 100644 --- a/mgmtd/src/lib.rs +++ b/mgmtd/src/lib.rs @@ -27,6 +27,7 @@ use sqlite::TransactionExt; use sqlite_check::sql; use std::collections::HashSet; use std::future::Future; +use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr}; use std::sync::Arc; use tokio::net::UdpSocket; use tokio::sync::mpsc; @@ -39,6 +40,7 @@ pub struct StaticInfo { pub user_config: Config, pub auth_secret: Option, pub network_addrs: Vec, + pub use_ipv6: bool, } /// Starts the management service. @@ -59,7 +61,14 @@ pub async fn start(info: StaticInfo, license: LicenseVerifier) -> Result Result, auth_secret: Option, + use_ipv6: bool, } impl Pool { @@ -34,11 +35,13 @@ impl Pool { udp_socket: Arc, connection_limit: usize, auth_secret: Option, + use_ipv6: bool, ) -> Self { Self { store: Store::new(connection_limit), auth_secret, udp_socket, + use_ipv6, } } @@ -118,6 +121,10 @@ impl Pool { log::debug!("Connecting new stream to {node_uid:?}"); for addr in addrs.iter() { + if addr.is_ipv6() && !self.use_ipv6 { + continue; + } + match Stream::connect_tcp(addr).await { Ok(stream) => { let mut stream = StoredStream::from_stream(stream, permit); @@ -210,6 +217,10 @@ impl Pool { let mut errs = vec![]; for addr in addrs.iter() { + if addr.is_ipv6() && !self.use_ipv6 { + continue; + } + if let Err(err) = buf.send_to_socket(&self.udp_socket, addr).await { log::debug!( "Sending datagram to node with uid {node_uid} using {addr} failed: {err}" diff --git a/shared/src/nic.rs b/shared/src/nic.rs index 6ac8a15..9e53d68 100644 --- a/shared/src/nic.rs +++ b/shared/src/nic.rs @@ -2,7 +2,7 @@ use crate::types::NicType; use anyhow::{Result, anyhow}; use serde::Deserializer; use serde::de::{Unexpected, Visitor}; -use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; +use std::net::{IpAddr, Ipv6Addr}; use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; use std::str::FromStr; @@ -187,11 +187,15 @@ impl Ord for Nic { /// /// Only interfaces matching one of the given names in `filter` will be returned, unless the list /// is empty. -pub fn query_nics(filter: &[NicFilter]) -> Result> { +pub fn query_nics(filter: &[NicFilter], use_ipv6: bool) -> Result> { let mut filtered_nics = vec![]; for interface in pnet_datalink::interfaces() { for ip in interface.ips { + if !use_ipv6 && ip.is_ipv6() { + continue; + } + if let Some(priority) = nic_priority(filter, &interface.name, &ip.ip()) { filtered_nics.push(Nic { name: interface.name.clone(), @@ -208,17 +212,22 @@ pub fn query_nics(filter: &[NicFilter]) -> Result> { Ok(filtered_nics) } -/// Selects address to bind to for listening: Checks if IPv6 sockets are available on this host +/// Checks if IPv6 sockets are available on this host /// according to our rules: IPv6 must be enabled during boot and at runtime, and IPv6 sockets must -/// be dual stack. Then it returns `::` (IPv6), otherwise `0.0.0.0` (IPv4). -pub fn select_bind_addr(port: u16) -> SocketAddr { +/// be dual stack. +pub fn check_ipv6(port: u16, use_ipv6: bool) -> bool { + if !use_ipv6 { + log::info!("IPv6 is disabled by the configuration, falling back to IPv4 sockets"); + return false; + } + // SAFETY: Any data used in the libc calls is local only unsafe { // Check if IPv6 socket can be created let sock = libc::socket(libc::AF_INET6, libc::SOCK_STREAM, 0); if sock < 0 { log::info!("IPv6 is unavailable on this host, falling back to IPv4 sockets"); - return SocketAddr::new(Ipv4Addr::UNSPECIFIED.into(), port); + return false; } // Make sure the socket is closed on drop let sock = OwnedFd::from_raw_fd(sock); @@ -244,7 +253,7 @@ pub fn select_bind_addr(port: u16) -> SocketAddr { if res < 0 && std::io::Error::last_os_error().raw_os_error() == Some(libc::EADDRNOTAVAIL) { log::info!("IPv6 is disabled on this host, falling back to IPv4 sockets"); - return SocketAddr::new(Ipv4Addr::UNSPECIFIED.into(), port); + return false; } // Check if dual stack sockets are enabled by querying the socket option @@ -263,11 +272,11 @@ pub fn select_bind_addr(port: u16) -> SocketAddr { log::info!( "IPv6 dual stack sockets are unavailable on this host, falling back to IPv4 sockets" ); - return SocketAddr::new(Ipv4Addr::UNSPECIFIED.into(), port); + return false; } } - SocketAddr::new(Ipv6Addr::UNSPECIFIED.into(), port) + true } #[cfg(test)]