diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index c891ff0a6d9c..2f4e3073ac3a 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -418,7 +418,6 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N nodeConfig.AgentConfig.ClusterDomain = controlConfig.ClusterDomain nodeConfig.AgentConfig.ResolvConf = locateOrGenerateResolvConf(envInfo) nodeConfig.AgentConfig.ClientCA = clientCAFile - nodeConfig.AgentConfig.ListenAddress = "0.0.0.0" nodeConfig.AgentConfig.KubeConfigKubelet = kubeconfigKubelet nodeConfig.AgentConfig.KubeConfigKubeProxy = kubeconfigKubeproxy nodeConfig.AgentConfig.KubeConfigK3sController = kubeconfigK3sController @@ -460,8 +459,13 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N nodeConfig.AgentConfig.NodeIPs = nodeIPs nodeIP, err := util.GetFirst4(nodeIPs) + nodeConfig.AgentConfig.ListenAddress = "0.0.0.0" if err != nil { - return nil, errors.Wrap(err, "cannot configure IPv4 node-ip") + nodeIP, err = util.GetFirst6(nodeIPs) + if err != nil { + return nil, errors.Wrap(err, "cannot configure IPv4/IPv6 node-ip") + } + nodeConfig.AgentConfig.ListenAddress = "::" } nodeConfig.AgentConfig.NodeIP = nodeIP.String() @@ -476,10 +480,14 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N } // if configured, set NodeExternalIP to the first IPv4 address, for legacy clients + // unless only IPv6 address given if len(nodeConfig.AgentConfig.NodeExternalIPs) > 0 { nodeExternalIP, err := util.GetFirst4(nodeConfig.AgentConfig.NodeExternalIPs) if err != nil { - return nil, errors.Wrap(err, "cannot configure IPv4 node-external-ip") + nodeExternalIP, err = util.GetFirst6(nodeConfig.AgentConfig.NodeExternalIPs) + if err != nil { + return nil, errors.Wrap(err, "cannot configure IPv4/IPv6 node-external-ip") + } } nodeConfig.AgentConfig.NodeExternalIP = nodeExternalIP.String() } diff --git a/pkg/agent/run.go b/pkg/agent/run.go index cec764237f13..9501d4bf3987 100644 --- a/pkg/agent/run.go +++ b/pkg/agent/run.go @@ -58,8 +58,14 @@ func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error { if err != nil { return errors.Wrap(err, "failed to validate node-ip") } + serviceIPv6 := false + for _, cidr := range nodeConfig.AgentConfig.ServiceCIDRs { + if utilsnet.IsIPv6CIDR(cidr) { + serviceIPv6 = true + } + } - enableIPv6 := dualCluster || dualService || dualNode + enableIPv6 := dualCluster || dualService || dualNode || serviceIPv6 conntrackConfig, err := getConntrackConfig(nodeConfig) if err != nil { return errors.Wrap(err, "failed to validate kube-proxy conntrack configuration") diff --git a/pkg/cli/server/server.go b/pkg/cli/server/server.go index 77a683c8015e..cca576b4bd37 100644 --- a/pkg/cli/server/server.go +++ b/pkg/cli/server/server.go @@ -224,16 +224,25 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont if serverConfig.ControlConfig.PrivateIP == "" && len(cmds.AgentConfig.NodeIP) != 0 { // ignoring the error here is fine since etcd will fall back to the interface's IPv4 address serverConfig.ControlConfig.PrivateIP, _ = util.GetFirst4String(cmds.AgentConfig.NodeIP) + if serverConfig.ControlConfig.PrivateIP == "" { + serverConfig.ControlConfig.PrivateIP, _ = util.GetFirst6String(cmds.AgentConfig.NodeIP) + } } // if not set, try setting advertise-ip from agent node-external-ip if serverConfig.ControlConfig.AdvertiseIP == "" && len(cmds.AgentConfig.NodeExternalIP) != 0 { serverConfig.ControlConfig.AdvertiseIP, _ = util.GetFirst4String(cmds.AgentConfig.NodeExternalIP) + if serverConfig.ControlConfig.AdvertiseIP == "" { + serverConfig.ControlConfig.AdvertiseIP, _ = util.GetFirst6String(cmds.AgentConfig.NodeExternalIP) + } } // if not set, try setting advertise-up from agent node-ip if serverConfig.ControlConfig.AdvertiseIP == "" && len(cmds.AgentConfig.NodeIP) != 0 { serverConfig.ControlConfig.AdvertiseIP, _ = util.GetFirst4String(cmds.AgentConfig.NodeIP) + if serverConfig.ControlConfig.AdvertiseIP == "" { + serverConfig.ControlConfig.AdvertiseIP, _ = util.GetFirst6String(cmds.AgentConfig.NodeIP) + } } // if we ended up with any advertise-ips, ensure they're added to the SAN list; @@ -251,14 +260,22 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont return err } serverConfig.ControlConfig.ServerNodeName = nodeName - serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, "127.0.0.1", "localhost", nodeName) + serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, "127.0.0.1", "::1", "localhost", nodeName) for _, ip := range nodeIPs { serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, ip.String()) } // configure ClusterIPRanges if len(cmds.ServerConfig.ClusterCIDR) == 0 { - cmds.ServerConfig.ClusterCIDR.Set("10.42.0.0/16") + clusterCIDR := "10.42.0.0/16" + + // default to IPv6 ClusterIPRange on IPv6 nodes + _, ipv4Err := util.GetFirst4(nodeIPs) + _, ipv6Err := util.GetFirst6(nodeIPs) + if ipv6Err == nil && ipv4Err != nil { + clusterCIDR = "fd:42::/56" + } + cmds.ServerConfig.ClusterCIDR.Set(clusterCIDR) } for _, cidr := range cmds.ServerConfig.ClusterCIDR { for _, v := range strings.Split(cidr, ",") { @@ -271,15 +288,27 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont } // set ClusterIPRange to the first IPv4 block, for legacy clients + // unless only IPv6 range given clusterIPRange, err := util.GetFirst4Net(serverConfig.ControlConfig.ClusterIPRanges) if err != nil { - return errors.Wrap(err, "cannot configure IPv4 cluster-cidr") + clusterIPRange, err = util.GetFirst6Net(serverConfig.ControlConfig.ClusterIPRanges) + if err != nil { + return errors.Wrap(err, "cannot configure IPv4/IPv6 cluster-cidr") + } } serverConfig.ControlConfig.ClusterIPRange = clusterIPRange // configure ServiceIPRanges if len(cmds.ServerConfig.ServiceCIDR) == 0 { - cmds.ServerConfig.ServiceCIDR.Set("10.43.0.0/16") + serviceCIDR := "10.43.0.0/16" + + // default to IPv6 ServiceIPRange on IPv6 nodes + _, ipv4Err := util.GetFirst4(nodeIPs) + _, ipv6Err := util.GetFirst6(nodeIPs) + if ipv6Err == nil && ipv4Err != nil { + serviceCIDR = "fd:43::/112" + } + cmds.ServerConfig.ServiceCIDR.Set(serviceCIDR) } for _, cidr := range cmds.ServerConfig.ServiceCIDR { for _, v := range strings.Split(cidr, ",") { @@ -292,9 +321,13 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont } // set ServiceIPRange to the first IPv4 block, for legacy clients + // unless only IPv6 range given serviceIPRange, err := util.GetFirst4Net(serverConfig.ControlConfig.ServiceIPRanges) if err != nil { - return errors.Wrap(err, "cannot configure IPv4 service-cidr") + serviceIPRange, err = util.GetFirst6Net(serverConfig.ControlConfig.ServiceIPRanges) + if err != nil { + return errors.Wrap(err, "cannot configure IPv4/IPv6 service-cidr") + } } serverConfig.ControlConfig.ServiceIPRange = serviceIPRange @@ -312,7 +345,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont // If cluster-dns CLI arg is not set, we set ClusterDNS address to be the first IPv4 ServiceCIDR network + 10, // i.e. when you set service-cidr to 192.168.0.0/16 and don't provide cluster-dns, it will be set to 192.168.0.10 - // If there are no IPv4 ServiceCIDRs, an error will be raised. + // If there are no IPv4 ServiceCIDRs, an IPv6 ServiceCIDRs will be used. if len(cmds.ServerConfig.ClusterDNS) == 0 { clusterDNS, err := utilsnet.GetIndexedIP(serverConfig.ControlConfig.ServiceIPRange, 10) if err != nil { @@ -331,9 +364,13 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont } } // Set ClusterDNS to the first IPv4 address, for legacy clients + // unless only IPv6 range given clusterDNS, err := util.GetFirst4(serverConfig.ControlConfig.ClusterDNSs) if err != nil { - return errors.Wrap(err, "cannot configure IPv4 cluster-dns address") + clusterDNS, err = util.GetFirst6(serverConfig.ControlConfig.ClusterDNSs) + if err != nil { + return errors.Wrap(err, "cannot configure IPv4/IPv6 cluster-dns address") + } } serverConfig.ControlConfig.ClusterDNS = clusterDNS } @@ -445,6 +482,11 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont ip := serverConfig.ControlConfig.BindAddress if ip == "" { ip = "127.0.0.1" + _, ipv4Err := util.GetFirst6(nodeIPs) + _, ipv6Err := util.GetFirst6(nodeIPs) + if ipv6Err == nil && ipv4Err != nil { + ip = "[::1]" + } } url := fmt.Sprintf("https://%s:%d", ip, serverConfig.ControlConfig.SupervisorPort) @@ -505,6 +547,25 @@ func validateNetworkConfiguration(serverConfig server.Config) error { return errors.New("dual-stack cluster-dns is not supported") } + serviceIPv4 := false + serviceIPv6 := false + for _, cidr := range serverConfig.ControlConfig.ServiceIPRanges { + if utilsnet.IsIPv4CIDR(cidr) { + serviceIPv4 = true + } + if utilsnet.IsIPv6CIDR(cidr) { + serviceIPv6 = true + } + } + if serviceIPv6 && !serviceIPv4 { + if serverConfig.ControlConfig.DisableNPC == false { + return errors.New("network policy enforcement is not compatible with IPv6 only operation; server must be restarted with --disable-network-policy") + } + if serverConfig.ControlConfig.FlannelBackend != config.FlannelBackendNone { + return errors.New("Flannel is not compatible with IPv6 only operation; server must be restarted with --flannel-backend=none") + } + } + return nil } diff --git a/pkg/daemons/agent/agent_linux.go b/pkg/daemons/agent/agent_linux.go index 2ad12bb03fc3..6bc1cab82e5a 100644 --- a/pkg/daemons/agent/agent_linux.go +++ b/pkg/daemons/agent/agent_linux.go @@ -39,9 +39,15 @@ func checkRuntimeEndpoint(cfg *config.Agent, argsMap map[string]string) { } func kubeProxyArgs(cfg *config.Agent) map[string]string { + bindAddress := "127.0.0.1" + _, ipv4Err := util.GetFirst4String([]string{cfg.NodeIP}) + _, ipv6Err := util.GetFirst6String([]string{cfg.NodeIP}) + if ipv6Err == nil && ipv4Err != nil { + bindAddress = "::1" + } argsMap := map[string]string{ "proxy-mode": "iptables", - "healthz-bind-address": "127.0.0.1", + "healthz-bind-address": bindAddress, "kubeconfig": cfg.KubeConfigKubeProxy, "cluster-cidr": util.JoinIPNets(cfg.ClusterCIDRs), "conntrack-max-per-core": "0", @@ -55,8 +61,14 @@ func kubeProxyArgs(cfg *config.Agent) map[string]string { } func kubeletArgs(cfg *config.Agent) map[string]string { + bindAddress := "127.0.0.1" + _, ipv4Err := util.GetFirst4String([]string{cfg.NodeIP}) + _, ipv6Err := util.GetFirst6String([]string{cfg.NodeIP}) + if ipv6Err == nil && ipv4Err != nil { + bindAddress = "::1" + } argsMap := map[string]string{ - "healthz-bind-address": "127.0.0.1", + "healthz-bind-address": bindAddress, "read-only-port": "0", "cluster-domain": cfg.ClusterDomain, "kubeconfig": cfg.KubeConfigKubelet, diff --git a/pkg/daemons/agent/agent_windows.go b/pkg/daemons/agent/agent_windows.go index 39cd086142f1..a4e1dc5d4ac5 100644 --- a/pkg/daemons/agent/agent_windows.go +++ b/pkg/daemons/agent/agent_windows.go @@ -29,9 +29,15 @@ func checkRuntimeEndpoint(cfg *config.Agent, argsMap map[string]string) { } func kubeProxyArgs(cfg *config.Agent) map[string]string { + bindAddress := "127.0.0.1" + _, ipv4Err := util.GetFirst4(cfg.NodeIP) + _, ipv6Err := util.GetFirst6(cfg.NodeIP) + if ipv6Err == nil && ipv4Err != nil { + bindAddress = "::1" + } argsMap := map[string]string{ "proxy-mode": "kernelspace", - "healthz-bind-address": "127.0.0.1", + "healthz-bind-address": bindAddress, "kubeconfig": cfg.KubeConfigKubeProxy, "cluster-cidr": util.JoinIPNets(cfg.ClusterCIDRs), } @@ -47,8 +53,14 @@ func kubeProxyArgs(cfg *config.Agent) map[string]string { } func kubeletArgs(cfg *config.Agent) map[string]string { + bindAddress := "127.0.0.1" + _, ipv4Err := util.GetFirst4(cfg.NodeIP) + _, ipv6Err := util.GetFirst6(cfg.NodeIP) + if ipv6Err == nil && ipv4Err != nil { + bindAddress = "::1" + } argsMap := map[string]string{ - "healthz-bind-address": "127.0.0.1", + "healthz-bind-address": bindAddress, "read-only-port": "0", "cluster-domain": cfg.ClusterDomain, "kubeconfig": cfg.KubeConfigKubelet, diff --git a/pkg/util/net.go b/pkg/util/net.go index 94a8742a0ed7..2bd2f78d4677 100644 --- a/pkg/util/net.go +++ b/pkg/util/net.go @@ -80,6 +80,46 @@ func JoinIP4Nets(elems []*net.IPNet) string { return strings.Join(strs, ",") } +// GetFirst6 returns the first IPv6 address from the list of IP addresses. +// If no IPv6 addresses are found, an error is raised. +func GetFirst6(elems []net.IP) (net.IP, error) { + for _, elem := range elems { + if elem == nil || elem.To16() == nil { + continue + } + return elem, nil + } + return nil, errors.New("no IPv6 address found") +} + +// GetFirst6Net returns the first IPv4 network from the list of IP networks. +// If no IPv6 addresses are found, an error is raised. +func GetFirst6Net(elems []*net.IPNet) (*net.IPNet, error) { + for _, elem := range elems { + if elem == nil || elem.IP.To16() == nil { + continue + } + return elem, nil + } + return nil, errors.New("no IPv6 CIDRs found") +} + +// GetFirst6String returns the first IPv6 address from a list of IP address strings. +// If no IPv6 addresses are found, an error is raised. +func GetFirst6String(elems []string) (string, error) { + ips := []net.IP{} + for _, elem := range elems { + for _, v := range strings.Split(elem, ",") { + ips = append(ips, net.ParseIP(v)) + } + } + ip, err := GetFirst6(ips) + if err != nil { + return "", err + } + return ip.String(), nil +} + // JoinIP6Nets stringifies and joins a list of IPv6 networks with commas. func JoinIP6Nets(elems []*net.IPNet) string { var strs []string