Merge pull request #673 from soider/ipv6-support

Introduce configuration changes for the ipv6 support

.github/workflows/go.yml

@@ -300,6 +300,37 @@ jobs:
           path: |
             /tmp/*.log
 
+  smoketest-dualstack:
+    name: Smoke test for IPv6 dualstack
+    needs: build
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Get PR Reference and Set Cache Name
+        run: |
+          PR_NUMBER=$(echo ${GITHUB_REF} | cut -d / -f 3 )
+          echo "cachePrefix=k0s-${PR_NUMBER}-${{ github.sha }}" >> $GITHUB_ENV
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Cache compiled binary for smoke testing
+        uses: actions/cache@v2
+        id: restore-compiled-binary
+        with:
+          path: |
+            k0s
+          key: build-${{env.cachePrefix}}
+
+      - name: Run smoke test .
+        run: make -C inttest check-dualstack
+
+      - name: Collect test logs
+        if: failure()
+        uses: actions/upload-artifact@v2
+        with:
+          path: |
+            /tmp/*.log
+
   lint:
     name: Lint
     runs-on: ubuntu-latest

Makefile

@@ -81,7 +81,7 @@ k0s.exe k0s: $(GO_SRCS)
 lint: pkg/assets/zz_generated_offsets_$(TARGET_OS).go
 	$(golint) run ./...
 
-smoketests := check-addons check-basic check-byocri check-hacontrolplane check-kine check-network check-singlenode check-install
+smoketests := check-addons check-basic check-byocri check-hacontrolplane check-kine check-network check-singlenode check-install check-dualstack
 .PHONY: $(smoketests)
 $(smoketests): k0s
 	$(MAKE) -C inttest $@

docs/dual-stack.md

@@ -0,0 +1,35 @@
+# Dual-stack networking
+
+To enable dual-stack networking use the following k0s.yaml as an example.
+This settings will set up bundled calico cni, enable feature gates for the Kubernetes components and set up kubernetes-controller-manager.
+```
+spec:
+  network:
+    podCIDR: "10.244.0.0/16"
+    serviceCIDR: "10.96.0.0/12"
+    calico:
+      mode: "bird"
+    dualStack:
+      enabled: true
+      IPv6podCIDR: "fd00::/108"
+      IPv6serviceCIDR: "fd01::/108"
+```
+## CNI settings
+
+### Calico settings
+
+Calico doesn't support tunneling for the IPv6, so "vxlan" and "ipip" backend wouldn't work. 
+If you need to have cross-pod connectivity, you need to use "bird" as a backend mode. 
+In any other mode the pods would be able to reach only pods on the same node.
+
+### External CNI
+The `k0s.yaml` dualStack section will enable all the neccessary feature gates for the Kubernetes components but in case of using external CNI it must be set up with IPv6 support.
+
+## Additional materials
+https://kubernetes.io/docs/concepts/services-networking/dual-stack/
+
+https://kubernetes.io/docs/tasks/network/validate-dual-stack/ 
+
+https://www.projectcalico.org/dual-stack-operation-with-calico-on-kubernetes/
+
+https://docs.projectcalico.org/networking/ipv6

inttest/Makefile

@@ -30,7 +30,7 @@ check-network-vm: bin/sonobuoy
 		go test -count=1 -v -timeout 20m github.com/k0sproject/k0s/inttest/sonobuoy/ -run ^TestVMNetworkSuite$
 
 TIMEOUT ?= 4m
-smoketests := check-addons check-basic check-byocri check-hacontrolplane check-kine check-singlenode check-install
+smoketests := check-addons check-basic check-byocri check-hacontrolplane check-kine check-singlenode check-install check-dualstack
 
 check-byocri: TIMEOUT=5m
 

inttest/dualstack/dualstack_test.go

@@ -0,0 +1,124 @@
+/*
+Copyright 2020 Mirantis, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dualstack
+
+// Package implements basic smoke test for dualstack setup.
+// Since we run tests under containers environment in the GHA we can't
+// actually check proper network connectivity.
+// Until wi migrate toward VM based test suites
+// this test only checks that nodes in the cluster
+// have proper values for spec.PodCIDRs
+
+import (
+	"context"
+	"fmt"
+	"github.com/stretchr/testify/suite"
+	v1meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/k0sproject/k0s/inttest/common"
+	k8s "k8s.io/client-go/kubernetes"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"testing"
+)
+
+type DualstackSuite struct {
+	common.FootlooseSuite
+
+	client *k8s.Clientset
+}
+
+func (ds *DualstackSuite) TestDualStackNodesHavePodCIDRs() {
+	nl, err := ds.client.CoreV1().Nodes().List(context.Background(), v1meta.ListOptions{})
+	ds.Require().NoError(err)
+	for _, n := range nl.Items {
+		ds.Require().Len(n.Spec.PodCIDRs, 2, "Each node must have ipv4 and ipv6 pod cidr")
+	}
+
+}
+
+func (ds *DualstackSuite) getKubeConfig(node string) *restclient.Config {
+	machine, err := ds.MachineForName(node)
+	ds.Require().NoError(err)
+	ssh, err := ds.SSH(node)
+	ds.Require().NoError(err)
+	kubeConf, err := ssh.ExecWithOutput("cat /var/lib/k0s/pki/admin.conf")
+	ds.Require().NoError(err)
+	cfg, err := clientcmd.RESTConfigFromKubeConfig([]byte(kubeConf))
+	ds.Require().NoError(err)
+	hostPort, err := machine.HostPort(6443)
+	ds.Require().NoError(err)
+	cfg.Host = fmt.Sprintf("localhost:%d", hostPort)
+	return cfg
+}
+
+func (ds *DualstackSuite) SetupSuite() {
+	ds.FootlooseSuite.SetupSuite()
+	ds.prepareConfigWithDualStackEnabled()
+	ds.Require().NoError(ds.InitMainController("/tmp/k0s.yaml", ""))
+	ds.Require().NoError(ds.RunWorkers("/var/lib/k0s"))
+	client, err := k8s.NewForConfig(ds.getKubeConfig("controller0"))
+	ds.Require().NoError(err)
+	err = ds.WaitForNodeReady("worker0", client)
+	ds.Require().NoError(err)
+
+	err = ds.WaitForNodeReady("worker1", client)
+	ds.Require().NoError(err)
+
+	ds.client = client
+
+}
+
+func TestDualStack(t *testing.T) {
+
+	s := DualstackSuite{
+		common.FootlooseSuite{
+			ControllerCount: 1,
+			WorkerCount:     2,
+		},
+		nil,
+	}
+
+	suite.Run(t, &s)
+
+}
+
+func (ds *DualstackSuite) prepareConfigWithDualStackEnabled() {
+	ds.putFile("/tmp/k0s.yaml", k0sConfigWithAddon)
+}
+
+func (ds *DualstackSuite) putFile(path string, content string) {
+	controllerNode := fmt.Sprintf("controller%d", 0)
+	ssh, err := ds.SSH(controllerNode)
+	ds.Require().NoError(err)
+	defer ssh.Disconnect()
+	_, err = ssh.ExecWithOutput(fmt.Sprintf("echo '%s' >%s", content, path))
+
+	ds.Require().NoError(err)
+
+}
+
+const k0sConfigWithAddon = `
+spec:
+  network:
+    calico:
+      mode: "bird"
+    dualStack:
+      enabled: true
+      IPv6podCIDR: "fd00::/108"
+      IPv6serviceCIDR: "fd01::/108"
+`

mkdocs.yml

@@ -25,6 +25,7 @@ nav:
       - Using Cloud Providers: cloud-providers.md
       - Create a cluster with Ansible: examples/ansible-playbook.md
       - Running k0s as a service: install.md
+      - IPv4\IPv6 dual-stack networking: dual-stack.md
   - Integrations:
       - Running k0s with Traefik: examples/traefik-ingress.md
       - Running k0s with Ambassador: examples/ambassador-ingress.md

pkg/apis/v1beta1/dualstack.go

@@ -0,0 +1,27 @@
+package v1beta1
+
+// DualStack defines network configuration for ipv4\ipv6 mixed cluster setup
+type DualStack struct {
+	Enabled         bool   `yaml:"enabled,omitempty"`
+	IPv6PodCIDR     string `yaml:"IPv6podCIDR,omitempty"`
+	IPv6ServiceCIDR string `yaml:"IPv6serviceCIDR,omitempty"`
+}
+
+// EnableDualStackFeatureGate adds ipv6 feature gate to the given args colllection
+func (ds DualStack) EnableDualStackFeatureGate(args map[string]string) {
+	if !ds.Enabled {
+		return
+	}
+	fg, found := args["feature-gates"]
+	if !found {
+		args["feature-gates"] = "IPv6DualStack=true"
+	} else {
+		fg = fg + ",IPv6DualStack=true"
+		args["feature-gates"] = fg
+	}
+}
+
+// DefaultDualStack builds default values
+func DefaultDualStack() DualStack {
+	return DualStack{}
+}

pkg/apis/v1beta1/dualstack_test.go

@@ -0,0 +1,36 @@
+package v1beta1
+
+import (
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func TestAddDualStackArguments(t *testing.T) {
+	ds := DualStack{Enabled: true}
+	t.Run("If no extrargs given, just add DualStack", func(t *testing.T) {
+		args := map[string]string{}
+		ds.EnableDualStackFeatureGate(args)
+		require.Equal(t, "IPv6DualStack=true", args["feature-gates"])
+	})
+	t.Run("keep already existed extra-args", func(t *testing.T) {
+		args := map[string]string{
+			"some-argument": "value",
+		}
+		ds.EnableDualStackFeatureGate(args)
+		require.Equal(t, "IPv6DualStack=true", args["feature-gates"])
+		require.Equal(t, "value", args["some-argument"])
+	})
+	t.Run("keep already existed extra-args feature gates", func(t *testing.T) {
+		args := map[string]string{
+			"feature-gates": "Magic=true",
+		}
+		ds.EnableDualStackFeatureGate(args)
+		require.Equal(t, "Magic=true,IPv6DualStack=true", args["feature-gates"])
+	})
+	t.Run("do nothing if dual-stack disabled", func(t *testing.T) {
+		ds := DualStack{}
+		args := map[string]string{}
+		ds.EnableDualStackFeatureGate(args)
+		require.Empty(t, args)
+	})
+}

pkg/apis/v1beta1/network.go

@@ -24,10 +24,11 @@ import (
 
 // Network defines the network related config options
 type Network struct {
-	PodCIDR     string  `yaml:"podCIDR"`
-	ServiceCIDR string  `yaml:"serviceCIDR"`
-	Provider    string  `yaml:"provider"`
-	Calico      *Calico `yaml:"calico"`
+	PodCIDR     string    `yaml:"podCIDR"`
+	ServiceCIDR string    `yaml:"serviceCIDR"`
+	Provider    string    `yaml:"provider"`
+	Calico      *Calico   `yaml:"calico"`
+	DualStack   DualStack `yaml:"dualStack,omitempty"`
 }
 
 // DefaultNetwork creates the Network config struct with sane default values
@@ -37,6 +38,7 @@ func DefaultNetwork() *Network {
 		ServiceCIDR: "10.96.0.0/12",
 		Provider:    "calico",
 		Calico:      DefaultCalico(),
+		DualStack:   DefaultDualStack(),
 	}
 }
 
@@ -46,6 +48,9 @@ func (n *Network) Validate() []error {
 	if n.Provider != "calico" && n.Provider != "custom" {
 		errors = append(errors, fmt.Errorf("unsupported network provider: %s", n.Provider))
 	}
+	if n.Provider == "calico" && n.DualStack.Enabled && n.Calico.Mode != "bird" {
+		errors = append(errors, fmt.Errorf("network dual stack is supported only for calico mode `bird`"))
+	}
 	return errors
 }
 
@@ -100,3 +105,19 @@ func (n *Network) UnmarshalYAML(unmarshal func(interface{}) error) error {
 
 	return nil
 }
+
+// BuildServiceCIDR returns actual argument value for service cidr
+func (n *Network) BuildServiceCIDR() string {
+	if n.DualStack.Enabled {
+		return n.ServiceCIDR + "," + n.DualStack.IPv6ServiceCIDR
+	}
+	return n.ServiceCIDR
+}
+
+// BuildPodCIDR returns actual argument value for pod cidr
+func (n *Network) BuildPodCIDR() string {
+	if n.DualStack.Enabled {
+		return n.PodCIDR + "," + n.DualStack.IPv6PodCIDR
+	}
+	return n.PodCIDR
+}

pkg/component/server/apiserver.go

@@ -96,7 +96,7 @@ func (a *APIServer) Run() error {
 		"requestheader-allowed-names":      "front-proxy-client",
 		"requestheader-client-ca-file":     path.Join(a.K0sVars.CertRootDir, "front-proxy-ca.crt"),
 		"service-account-key-file":         path.Join(a.K0sVars.CertRootDir, "sa.pub"),
-		"service-cluster-ip-range":         a.ClusterConfig.Spec.Network.ServiceCIDR,
+		"service-cluster-ip-range":         a.ClusterConfig.Spec.Network.BuildServiceCIDR(),
 		"tls-cert-file":                    path.Join(a.K0sVars.CertRootDir, "server.crt"),
 		"tls-private-key-file":             path.Join(a.K0sVars.CertRootDir, "server.key"),
 		"egress-selector-config-file":      path.Join(a.K0sVars.DataDir, "konnectivity.conf"),
@@ -114,6 +114,7 @@ func (a *APIServer) Run() error {
 		}
 		args[name] = value
 	}
+	a.ClusterConfig.Spec.Network.DualStack.EnableDualStackFeatureGate(args)
 
 	for name, value := range apiDefaultArgs {
 		if args[name] == "" {