From 2f78672ba3f98e0f8999668b0b8f1f7cd0b76d46 Mon Sep 17 00:00:00 2001 From: Carl Date: Wed, 8 Apr 2020 01:41:25 -0700 Subject: [PATCH] ifb fixes --- net-shaper/src/main.rs | 236 +++++++++++++++++++++-------------------- 1 file changed, 123 insertions(+), 113 deletions(-) diff --git a/net-shaper/src/main.rs b/net-shaper/src/main.rs index 858e7909ad9973..d20fb11c6f8b29 100644 --- a/net-shaper/src/main.rs +++ b/net-shaper/src/main.rs @@ -221,50 +221,98 @@ fn flush_iptables_rule() { ); } -fn insert_tc_root(interface: &str, num_bands: &str) -> bool { - // tc qdisc add dev root handle 1: prio - // tc qdisc add dev root handle 1: prio bands +fn setup_ifb(interface: &str) -> bool { + // modprobe ifb numifbs=1 + run( + "modprobe", + &[ + "ifb", "numifbs=1", + ], + "Failed to load ifb module", + "modprobe ifb numifbs=1", + false + ) && + // ip link set dev ifb0 up + run( + "ip", + &[ + "link", "set", "dev", "ifb0", "up" + ], + "Failed to bring ifb0 online", + "ip link set dev ifb0 up", + false + ) && + // tc qdisc add dev handle ffff: ingress run( "tc", &[ - "qdisc", "add", "dev", interface, "root", "handle", "1:", "prio", "bands", num_bands, + "qdisc", "add", "dev", interface, "handle", "ffff:", "ingress" ], - "Failed to add root qdisc", - "tc add root qdisc", + "Failed to setup ingress qdisc", + "tc qdisc add dev handle ffff: ingress", + false + ) + && + // tc filter add dev parent ffff: protocol ip u32 match u32 0 0 action mirred egress redirect dev ifb0 + run( + "tc", + &[ + "filter", "add", "dev", interface, "parent", "ffff:", "protocol", "ip", "u32", "match", "u32", "0", "0", "action", "mirred", "egress", "redirect", "dev", "ifb0" + ], + "Failed to redirect ingress traffc", + "tc filter add dev parent ffff: protocol ip u32 match u32 0 0 action mirred egress redirect dev ifb0", + false + ) +} + +fn delete_ifb(interface: &str) -> bool { + run( + "tc", + &[ + "qdisc", "delete", "dev", interface, "handle", "ffff:", "ingress", + ], + "Failed to setup ingress qdisc", + "tc qdisc delete dev handle ffff: ingress", + false, + ) && run( + "modprobe", + &["ifb", "--remove"], + "Failed to delete ifb module", + "modprobe ifb --remove", false, ) } -fn delete_tc_root(interface: &str) { - // tc qdisc delete dev root handle 1: prio +fn insert_tc_ifb_root(num_bands: &str) -> bool { + // tc qdisc add dev ifb0 root handle 1: prio bands run( "tc", &[ - "qdisc", "delete", "dev", interface, "root", "handle", "1:", "prio", + "qdisc", "add", "dev", "ifb0", "root", "handle", "1:", "prio", "bands", num_bands, ], - "Failed to delete root qdisc", - "tc qdisc delete root", - true, - ); + "Failed to add root ifb qdisc", + "tc qdisc add dev ifb0 root handle 1: prio bands ", + false, + ) } -fn insert_tc_netem(interface: &str, class: &str, handle: &str, filter: &str) -> bool { +fn insert_tc_ifb_netem(class: &str, handle: &str, filter: &str) -> bool { let mut filters: Vec<&str> = filter.split(' ').collect(); let mut args = vec![ - "qdisc", "add", "dev", interface, "parent", class, "handle", handle, "netem", + "qdisc", "add", "dev", "ifb0", "parent", class, "handle", handle, "netem", ]; args.append(&mut filters); - // tc qdisc add dev parent 1: handle : netem + // tc qdisc add dev ifb0 parent handle netem run("tc", &args, "Failed to add tc child", "tc add child", false) } -fn delete_tc_netem(interface: &str, class: &str, handle: &str, filter: &str) { +fn delete_tc_ifb_netem(interface: &str, class: &str, handle: &str, filter: &str) { let mut filters: Vec<&str> = filter.split(' ').collect(); let mut args = vec![ "qdisc", "delete", "dev", interface, "parent", class, "handle", handle, "netem", ]; args.append(&mut filters); - // tc qdisc delete dev parent 1: handle : netem + // tc qdisc delete dev parent handle : netem run( "tc", &args, @@ -274,22 +322,22 @@ fn delete_tc_netem(interface: &str, class: &str, handle: &str, filter: &str) { ); } -fn insert_tos_filter(interface: &str, class: &str, tos: &str) -> bool { - // tc filter add dev protocol ip parent 1: prio 1 u32 match ip tos 0xff flowid 1: +fn insert_tos_ifb_filter(class: &str, tos: &str) -> bool { + // tc filter add dev ifb0 protocol ip parent 1: prio 1 u32 match ip tos 0xff flowid run( "tc", &[ - "filter", "add", "dev", interface, "protocol", "ip", "parent", "1:", "prio", "1", + "filter", "add", "dev", "ifb0", "protocol", "ip", "parent", "1:", "prio", "1", "u32", "match", "ip", "tos", tos, "0xff", "flowid", class, ], "Failed to add tos filter", - "tc add filter", + "tc filter add dev ifb0 protocol ip parent 1: prio 1 u32 match ip tos 0xff flowid ", false, ) } fn delete_tos_filter(interface: &str, class: &str, tos: &str) { - // tc filter delete dev protocol ip parent 1: prio 10 u32 match ip tos 0xff flowid 1: + // tc filter delete dev protocol ip parent 1: prio 10 u32 match ip tos 0xff flowid run( "tc", &[ @@ -302,34 +350,20 @@ fn delete_tos_filter(interface: &str, class: &str, tos: &str) { ); } -fn insert_default_filter(interface: &str, class: &str) -> bool { - // tc filter add dev protocol ip parent 1: prio 2 u32 match ip src 0/0 flowid 1: +fn insert_default_ifb_filter(class: &str) -> bool { + // tc filter add dev ifb0 parent 1: protocol all prio 2 u32 match u32 0 0 flowid 1: run( "tc", &[ - "filter", "add", "dev", interface, "protocol", "ip", "parent", "1:", "prio", "2", - "u32", "match", "ip", "tos", "0", "0xff", "flowid", class, + "filter", "add", "dev", "ifb0", "parent", "1:", "protocol", "all", "prio", "2", "u32", + "match", "u32", "0", "0", "flowid", class, ], - "Failed to add default filter", - "tc add default filter", + "Failed to add catch-all filter", + "tc filter add dev ifb0 parent 1: protocol all prio 2 u32 match u32 0 0 flowid 1:", false, ) } -fn delete_default_filter(interface: &str, class: &str) { - // tc filter delete dev protocol ip parent 1: prio 2 flowid 1: - run( - "tc", - &[ - "filter", "delete", "dev", interface, "protocol", "ip", "parent", "1:", "prio", "2", - "flowid", class, - ], - "Failed to delete default filter", - "tc delete default filter", - true, - ); -} - fn delete_all_filters(interface: &str) { // tc filter delete dev run( @@ -372,94 +406,84 @@ fn shape_network(matches: &ArgMatches) { let config = fs::read_to_string(&config_path).expect("Unable to read config file"); let topology: NetworkTopology = serde_json::from_str(&config).expect("Failed to parse log as JSON"); + let interface = value_t_or_exit!(matches, "iface", String); + let network_size = value_t_or_exit!(matches, "size", u64); + let my_index = value_t_or_exit!(matches, "position", u64); + if !shape_network_steps(&topology, &interface, network_size, my_index) { + delete_ifb(interface.as_str()); + flush_iptables_rule(); + } +} +fn shape_network_steps( + topology: &NetworkTopology, + interface: &str, + network_size: u64, + my_index: u64, +) -> bool { + // Integrity checks if !topology.verify() { panic!("Failed to verify the configuration file"); } - - let network_size = value_t_or_exit!(matches, "size", u64); - let my_index = value_t_or_exit!(matches, "position", u64); - let interface = value_t_or_exit!(matches, "iface", String); - assert!(my_index < network_size); + // Figure out partition we belong in let my_partition = identify_my_partition(&topology.partitions, my_index + 1, network_size); + // Clear any lingering state println!( "my_index: {}, network_size: {}, partitions: {:?}", my_index, network_size, topology.partitions ); println!("My partition is {}", my_partition); - flush_iptables_rule(); + force_cleanup_network(interface); + // Mark egress packets with our partition id if !insert_iptables_rule(partition_id_to_tos(my_partition)) { - return; + return false; } - delete_tc_root(interface.as_str()); let num_bands = topology.partitions.len() + 1; let default_filter_class = format!("1:{}", num_bands); if !topology.interconnects.is_empty() { let num_bands_str = num_bands.to_string(); - if !insert_tc_root(interface.as_str(), num_bands_str.as_str()) - || !insert_default_filter(interface.as_str(), default_filter_class.as_str()) + // Redirect ingress traffic to the virtual interface ifb0 so we can + // apply egress rules + if !setup_ifb(interface) + // Setup root qdisc on ifb0 + || !insert_tc_ifb_root(num_bands_str.as_str()) + // Catch all so regular traffic/traffic within the same partition + // is not filtered out + || !insert_default_ifb_filter(default_filter_class.as_str()) { - delete_tc_root(interface.as_str()); - flush_iptables_rule(); - return; + return false; } } - println!("Processing interconnects"); - topology.interconnects.iter().for_each(|i| { - println!("my partition 2: {}", my_partition); + println!("Setting up interconnects"); + for i in &topology.interconnects { println!("interconnects: {:#?}", i); if i.b as usize == my_partition { let tos = partition_id_to_tos(i.a as usize); if tos == 0 { println!("Incorrect value of TOS/Partition in config {}", i.a); - delete_default_filter(interface.as_str(), default_filter_class.as_str()); - delete_tc_root(interface.as_str()); - return; + return false; } let tos_string = tos.to_string(); // First valid class is 1:1 - let x = { - if i.a == 1 { - 2 - } else { - 1 - } - }; - let class = format!("1:{}", x); - println!("here"); - if !insert_tc_netem( - interface.as_str(), - class.as_str(), - tos_string.as_str(), - i.config.as_str(), - ) { - println!("failed"); - delete_default_filter(interface.as_str(), default_filter_class.as_str()); - delete_tc_root(interface.as_str()); - return; + let class = format!("1:{}", i.a + 1); + if !insert_tc_ifb_netem(class.as_str(), tos_string.as_str(), i.config.as_str()) { + return false; } - if !insert_tos_filter(interface.as_str(), class.as_str(), tos_string.as_str()) { - println!("failed 2"); - delete_tc_netem( - interface.as_str(), - class.as_str(), - tos_string.as_str(), - i.config.as_str(), - ); - delete_default_filter(interface.as_str(), default_filter_class.as_str()); - delete_tc_root(interface.as_str()); - return; + if !insert_tos_ifb_filter(class.as_str(), tos_string.as_str()) { + return false; } } - }) + } + + true } fn cleanup_network(matches: &ArgMatches) { @@ -492,7 +516,7 @@ fn cleanup_network(matches: &ArgMatches) { let class = format!("1:{}", i.a + 1); let tos_string = i.a.to_string(); delete_tos_filter(interface.as_str(), class.as_str(), tos_string.as_str()); - delete_tc_netem( + delete_tc_ifb_netem( interface.as_str(), class.as_str(), handle.as_str(), @@ -500,17 +524,12 @@ fn cleanup_network(matches: &ArgMatches) { ); } }); - let num_bands = topology.partitions.len() + 1; - let default_filter_class = format!("1:{}", num_bands); - delete_default_filter(interface.as_str(), default_filter_class.as_str()); - delete_tc_root(interface.as_str()); - flush_iptables_rule(); + force_cleanup_network(&interface); } -fn force_cleanup_network(matches: &ArgMatches) { - let interface = value_t_or_exit!(matches, "iface", String); - delete_all_filters(interface.as_str()); - delete_tc_root(interface.as_str()); +fn force_cleanup_network(interface: &str) { + delete_all_filters("ifb0"); + delete_ifb(interface); flush_iptables_rule(); } @@ -622,15 +641,6 @@ fn main() { .subcommand( SubCommand::with_name("force_cleanup") .about("Remove the network filters") - .arg( - Arg::with_name("iface") - .short("i") - .long("iface") - .value_name("network interface name") - .takes_value(true) - .required(true) - .help("Name of network interface"), - ), ) .subcommand( SubCommand::with_name("configure") @@ -675,7 +685,7 @@ fn main() { match matches.subcommand() { ("shape", Some(args_matches)) => shape_network(args_matches), ("cleanup", Some(args_matches)) => cleanup_network(args_matches), - ("force_cleanup", Some(args_matches)) => force_cleanup_network(args_matches), + ("force_cleanup", Some(args_matches)) => force_cleanup_network(&value_t_or_exit!(args_matches, "iface", String)), ("configure", Some(args_matches)) => configure(args_matches), _ => {} };