diff --git a/cmd/kg/main.go b/cmd/kg/main.go index d465cfec..fab6adcc 100644 --- a/cmd/kg/main.go +++ b/cmd/kg/main.go @@ -65,6 +65,7 @@ var ( availableGranularities = strings.Join([]string{ string(mesh.LogicalGranularity), string(mesh.FullGranularity), + string(mesh.CrossGranularity), }, ", ") availableLogLevels = strings.Join([]string{ logLevelAll, @@ -157,6 +158,7 @@ func Main() error { switch gr { case mesh.LogicalGranularity: case mesh.FullGranularity: + case mesh.CrossGranularity: default: return fmt.Errorf("mesh granularity %v unknown; possible values are: %s", *granularity, availableGranularities) } diff --git a/docs/topology.md b/docs/topology.md index f5633882..e4e82ead 100644 --- a/docs/topology.md +++ b/docs/topology.md @@ -46,6 +46,12 @@ kgctl graph | circo -Tsvg > cluster.svg +# Cross Mesh + +In this topology all nodes within the same location are not encrypted. Traffic to any other node outside of current location is encrypted +with direct node-to-node encryption. To use this mesh specify `--mesh-granularity=cross`. + + ## Mixed The `kilo.squat.ai/location` annotation can be used to create cluster mixing some fully meshed nodes and some nodes grouped by logical location. diff --git a/pkg/mesh/backend.go b/pkg/mesh/backend.go index cb693c11..2fb73049 100644 --- a/pkg/mesh/backend.go +++ b/pkg/mesh/backend.go @@ -47,6 +47,9 @@ const ( // FullGranularity indicates that the network should create // a mesh between every node. FullGranularity Granularity = "full" + // CrossGranularity indicates that network is encrypted only + // between nodes in different locations. + CrossGranularity Granularity = "cross" ) // Node represents a node in the network. diff --git a/pkg/mesh/routes.go b/pkg/mesh/routes.go index 6764fc3e..5c173c6e 100644 --- a/pkg/mesh/routes.go +++ b/pkg/mesh/routes.go @@ -125,7 +125,7 @@ func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface } for _, segment := range t.segments { // Add routes for the current segment if local is true. - if segment.location == t.location { + if (segment.location == t.location) || (t.nodeLocation != "" && segment.nodeLocation == t.nodeLocation) { if local { for i := range segment.cidrs { // Don't add routes for the local node. diff --git a/pkg/mesh/topology.go b/pkg/mesh/topology.go index 74efa07c..0518d0bf 100644 --- a/pkg/mesh/topology.go +++ b/pkg/mesh/topology.go @@ -27,8 +27,11 @@ type Topology struct { // key is the private key of the node creating the topology. key []byte port uint32 - // Location is the logical location of the local host. + // Location is the logical location of the local host of host name. location string + // nodeLocation is the location annotation of the node. This is set only in cross location topology. + nodeLocation string + segments []*segment peers []*Peer @@ -56,7 +59,8 @@ type segment struct { key []byte // Location is the logical location of this segment. location string - + // nodeLocation is the node location annotation. This is set only for cross location topology. + nodeLocation string // cidrs is a slice of subnets of all peers in the segment. cidrs []*net.IPNet // hostnames is a slice of the hostnames of the peers in the segment. @@ -70,35 +74,58 @@ type segment struct { wireGuardIP net.IP } +// topoKey is used to group nodes into locations. +type topoKey struct { + location string + nodeLocation string +} + // NewTopology creates a new Topology struct from a given set of nodes and peers. func NewTopology(nodes map[string]*Node, peers map[string]*Peer, granularity Granularity, hostname string, port uint32, key []byte, subnet *net.IPNet, persistentKeepalive int) (*Topology, error) { - topoMap := make(map[string][]*Node) + topoMap := make(map[topoKey][]*Node) + var localLocation, localNodeLocation string + switch granularity { + case LogicalGranularity: + localLocation = nodes[hostname].Location + case CrossGranularity: + localLocation = hostname + localNodeLocation = nodes[hostname].Location + case FullGranularity: + localLocation = hostname + } + for _, node := range nodes { - var location string + var location, nodeLocation string switch granularity { case LogicalGranularity: location = node.Location + case CrossGranularity: + location = node.Name + nodeLocation = node.Location case FullGranularity: location = node.Name } - topoMap[location] = append(topoMap[location], node) - } - var localLocation string - switch granularity { - case LogicalGranularity: - localLocation = nodes[hostname].Location - case FullGranularity: - localLocation = hostname + key := topoKey{location: location, nodeLocation: nodeLocation} + topoMap[key] = append(topoMap[key], node) } - t := Topology{key: key, port: port, hostname: hostname, location: localLocation, persistentKeepalive: persistentKeepalive, privateIP: nodes[hostname].InternalIP, subnet: nodes[hostname].Subnet} + t := Topology{ + key: key, + port: port, + hostname: hostname, + location: localLocation, + nodeLocation: localNodeLocation, + persistentKeepalive: persistentKeepalive, + privateIP: nodes[hostname].InternalIP, + subnet: nodes[hostname].Subnet, + } for location := range topoMap { // Sort the location so the result is stable. sort.Slice(topoMap[location], func(i, j int) bool { return topoMap[location][i].Name < topoMap[location][j].Name }) leader := findLeader(topoMap[location]) - if location == localLocation && topoMap[location][leader].Name == hostname { + if location.nodeLocation != "" || (location.location == localLocation && topoMap[location][leader].Name == hostname) { t.leader = true } var allowedIPs []*net.IPNet @@ -116,14 +143,15 @@ func NewTopology(nodes map[string]*Node, peers map[string]*Peer, granularity Gra privateIPs = append(privateIPs, node.InternalIP.IP) } t.segments = append(t.segments, &segment{ - allowedIPs: allowedIPs, - endpoint: topoMap[location][leader].Endpoint, - key: topoMap[location][leader].Key, - location: location, - cidrs: cidrs, - hostnames: hostnames, - leader: leader, - privateIPs: privateIPs, + allowedIPs: allowedIPs, + endpoint: topoMap[location][leader].Endpoint, + key: topoMap[location][leader].Key, + location: location.location, + nodeLocation: location.nodeLocation, + cidrs: cidrs, + hostnames: hostnames, + leader: leader, + privateIPs: privateIPs, }) } // Sort the Topology segments so the result is stable. @@ -167,7 +195,7 @@ func (t *Topology) Conf() *wireguard.Conf { }, } for _, s := range t.segments { - if s.location == t.location { + if (s.location == t.location) || (t.nodeLocation != "" && t.nodeLocation == s.nodeLocation) { continue } peer := &wireguard.Peer{ diff --git a/pkg/mesh/topology_test.go b/pkg/mesh/topology_test.go index d13c645d..b1a39b08 100644 --- a/pkg/mesh/topology_test.go +++ b/pkg/mesh/topology_test.go @@ -349,6 +349,156 @@ func TestNewTopology(t *testing.T) { peers: []*Peer{peers["a"], peers["b"]}, }, }, + { + name: "cross from a", + granularity: CrossGranularity, + hostname: nodes["a"].Name, + result: &Topology{ + hostname: nodes["a"].Name, + leader: true, + location: nodes["a"].Name, + nodeLocation: nodes["a"].Location, + subnet: nodes["a"].Subnet, + privateIP: nodes["a"].InternalIP, + wireGuardCIDR: &net.IPNet{IP: w1, Mask: net.CIDRMask(16, 32)}, + segments: []*segment{ + { + allowedIPs: []*net.IPNet{nodes["a"].Subnet, nodes["a"].InternalIP, {IP: w1, Mask: net.CIDRMask(32, 32)}}, + endpoint: nodes["a"].Endpoint, + key: nodes["a"].Key, + location: nodes["a"].Name, + nodeLocation: nodes["a"].Location, + cidrs: []*net.IPNet{nodes["a"].Subnet}, + hostnames: []string{"a"}, + privateIPs: []net.IP{nodes["a"].InternalIP.IP}, + wireGuardIP: w1, + }, + { + allowedIPs: []*net.IPNet{nodes["b"].Subnet, nodes["b"].InternalIP, {IP: w2, Mask: net.CIDRMask(32, 32)}}, + endpoint: nodes["b"].Endpoint, + key: nodes["b"].Key, + location: nodes["b"].Name, + nodeLocation: nodes["b"].Location, + cidrs: []*net.IPNet{nodes["b"].Subnet}, + hostnames: []string{"b"}, + privateIPs: []net.IP{nodes["b"].InternalIP.IP}, + wireGuardIP: w2, + }, + { + allowedIPs: []*net.IPNet{nodes["c"].Subnet, nodes["c"].InternalIP, {IP: w3, Mask: net.CIDRMask(32, 32)}}, + endpoint: nodes["c"].Endpoint, + key: nodes["c"].Key, + location: nodes["c"].Name, + nodeLocation: nodes["c"].Location, + cidrs: []*net.IPNet{nodes["c"].Subnet}, + hostnames: []string{"c"}, + privateIPs: []net.IP{nodes["c"].InternalIP.IP}, + wireGuardIP: w3, + }, + }, + peers: []*Peer{peers["a"], peers["b"]}, + }, + }, + { + name: "cross from b", + granularity: CrossGranularity, + hostname: nodes["b"].Name, + result: &Topology{ + hostname: nodes["b"].Name, + leader: true, + location: nodes["b"].Name, + nodeLocation: nodes["b"].Location, + subnet: nodes["b"].Subnet, + privateIP: nodes["b"].InternalIP, + wireGuardCIDR: &net.IPNet{IP: w2, Mask: net.CIDRMask(16, 32)}, + segments: []*segment{ + { + allowedIPs: []*net.IPNet{nodes["a"].Subnet, nodes["a"].InternalIP, {IP: w1, Mask: net.CIDRMask(32, 32)}}, + endpoint: nodes["a"].Endpoint, + key: nodes["a"].Key, + location: nodes["a"].Name, + nodeLocation: nodes["a"].Location, + cidrs: []*net.IPNet{nodes["a"].Subnet}, + hostnames: []string{"a"}, + privateIPs: []net.IP{nodes["a"].InternalIP.IP}, + wireGuardIP: w1, + }, + { + allowedIPs: []*net.IPNet{nodes["b"].Subnet, nodes["b"].InternalIP, {IP: w2, Mask: net.CIDRMask(32, 32)}}, + endpoint: nodes["b"].Endpoint, + key: nodes["b"].Key, + location: nodes["b"].Name, + nodeLocation: nodes["b"].Location, + cidrs: []*net.IPNet{nodes["b"].Subnet}, + hostnames: []string{"b"}, + privateIPs: []net.IP{nodes["b"].InternalIP.IP}, + wireGuardIP: w2, + }, + { + allowedIPs: []*net.IPNet{nodes["c"].Subnet, nodes["c"].InternalIP, {IP: w3, Mask: net.CIDRMask(32, 32)}}, + endpoint: nodes["c"].Endpoint, + key: nodes["c"].Key, + location: nodes["c"].Name, + nodeLocation: nodes["c"].Location, + cidrs: []*net.IPNet{nodes["c"].Subnet}, + hostnames: []string{"c"}, + privateIPs: []net.IP{nodes["c"].InternalIP.IP}, + wireGuardIP: w3, + }, + }, + peers: []*Peer{peers["a"], peers["b"]}, + }, + }, + { + name: "cross from c", + granularity: CrossGranularity, + hostname: nodes["c"].Name, + result: &Topology{ + hostname: nodes["c"].Name, + leader: true, + location: nodes["c"].Name, + nodeLocation: nodes["c"].Location, + subnet: nodes["c"].Subnet, + privateIP: nodes["c"].InternalIP, + wireGuardCIDR: &net.IPNet{IP: w3, Mask: net.CIDRMask(16, 32)}, + segments: []*segment{ + { + allowedIPs: []*net.IPNet{nodes["a"].Subnet, nodes["a"].InternalIP, {IP: w1, Mask: net.CIDRMask(32, 32)}}, + endpoint: nodes["a"].Endpoint, + key: nodes["a"].Key, + location: nodes["a"].Name, + nodeLocation: nodes["a"].Location, + cidrs: []*net.IPNet{nodes["a"].Subnet}, + hostnames: []string{"a"}, + privateIPs: []net.IP{nodes["a"].InternalIP.IP}, + wireGuardIP: w1, + }, + { + allowedIPs: []*net.IPNet{nodes["b"].Subnet, nodes["b"].InternalIP, {IP: w2, Mask: net.CIDRMask(32, 32)}}, + endpoint: nodes["b"].Endpoint, + key: nodes["b"].Key, + location: nodes["b"].Name, + nodeLocation: nodes["b"].Location, + cidrs: []*net.IPNet{nodes["b"].Subnet}, + hostnames: []string{"b"}, + privateIPs: []net.IP{nodes["b"].InternalIP.IP}, + wireGuardIP: w2, + }, + { + allowedIPs: []*net.IPNet{nodes["c"].Subnet, nodes["c"].InternalIP, {IP: w3, Mask: net.CIDRMask(32, 32)}}, + endpoint: nodes["c"].Endpoint, + key: nodes["c"].Key, + location: nodes["c"].Name, + nodeLocation: nodes["c"].Location, + cidrs: []*net.IPNet{nodes["c"].Subnet}, + hostnames: []string{"c"}, + privateIPs: []net.IP{nodes["c"].InternalIP.IP}, + wireGuardIP: w3, + }, + }, + peers: []*Peer{peers["a"], peers["b"]}, + }, + }, } { tc.result.key = key tc.result.port = port @@ -531,6 +681,85 @@ PersistentKeepalive = 25 AllowedIPs = 10.5.0.3/24 `, }, + { + name: "cross from a", + topology: mustTopo(t, nodes, peers, CrossGranularity, nodes["a"].Name, port, key, DefaultKiloSubnet, nodes["a"].PersistentKeepalive), + result: `[Interface] + PrivateKey = private + ListenPort = 51820 + + [Peer] + AllowedIPs = 10.2.2.0/24, 192.168.0.1/32, 10.4.0.2/32 + Endpoint = 10.1.0.2:51820 + PersistentKeepalive = 25 + PublicKey = key2 + + [Peer] + AllowedIPs = 10.2.3.0/24, 192.168.0.2/32, 10.4.0.3/32 + Endpoint = 10.1.0.3:51820 + PersistentKeepalive = 25 + PublicKey = key3 + + [Peer] + AllowedIPs = 10.5.0.1/24, 10.5.0.2/24 + PersistentKeepalive = 25 + PublicKey = key4 + + [Peer] + PublicKey = key5 + Endpoint = 192.168.0.1:51820 + PersistentKeepalive = 25 + AllowedIPs = 10.5.0.3/24 + `, + }, + { + name: "cross from b", + topology: mustTopo(t, nodes, peers, CrossGranularity, nodes["b"].Name, port, key, DefaultKiloSubnet, nodes["b"].PersistentKeepalive), + result: `[Interface] + PrivateKey = private + ListenPort = 51820 + + [Peer] + PublicKey = key1 + Endpoint = 10.1.0.1:51820 + AllowedIPs = 10.2.1.0/24, 192.168.0.1/32, 10.4.0.1/32 + + [Peer] + AllowedIPs = 10.5.0.1/24, 10.5.0.2/24 + PersistentKeepalive = 0 + PublicKey = key4 + + [Peer] + AllowedIPs = 10.5.0.3/24 + Endpoint = [192.168.0.1]:51820 + PersistentKeepalive = 0 + PublicKey = key5 + `, + }, + { + name: "cross from c", + topology: mustTopo(t, nodes, peers, CrossGranularity, nodes["b"].Name, port, key, DefaultKiloSubnet, nodes["b"].PersistentKeepalive), + result: `[Interface] + PrivateKey = private + ListenPort = 51820 + + [Peer] + PublicKey = key1 + Endpoint = 10.1.0.1:51820 + AllowedIPs = 10.2.1.0/24, 192.168.0.1/32, 10.4.0.1/32 + + [Peer] + AllowedIPs = 10.5.0.1/24, 10.5.0.2/24 + PersistentKeepalive = 0 + PublicKey = key4 + + [Peer] + AllowedIPs = 10.5.0.3/24 + Endpoint = [192.168.0.1]:51820 + PersistentKeepalive = 0 + PublicKey = key5 + `, + }, } { conf := tc.topology.Conf() if !conf.Equal(wireguard.Parse([]byte(tc.result))) {