Update k8s dependencies to v0.20.13

hetznercloud · Nov 22, 2021 · 5e0db56 · 5e0db56
1 parent 1b33f52
commit 5e0db56
Show file tree

Hide file tree

Showing 14 changed files with 1,098 additions and 1,097 deletions.
diff --git a/.github/workflows/test_e2e.yml b/.github/workflows/test_e2e.yml
@@ -5,7 +5,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        k8s: [ 1.19.10, 1.20.6, 1.21.0 ]
+        k8s: [ 1.20.12, 1.21.6, 1.22.3 ]
       fail-fast: false
     name: k8s ${{ matrix.k8s }}
     steps:
@@ -27,7 +27,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        k3s: [ v1.19.10+k3s1, v1.20.6+k3s1 ]
+        k3s: [ v1.20.12+k3s1, v1.21.6+k3s1, v1.22.3+k3s1 ]
       fail-fast: false
     name: k3s ${{ matrix.k3s }}
     steps:

diff --git a/.gitignore b/.gitignore
@@ -4,3 +4,4 @@ dist/
 deploy/gen/
 .coverage.out
 .envrc
+hcloud-cloud-controller-manager
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -9,7 +9,11 @@ stages:
   - test
   - build
   - build:image
-  - e2e
+  - e2e-test
+  - e2e-test-k8s
+  - e2e-test-k8s-networks
+  - e2e-test-k3s
+  - e2e-test-k3s-networks
   - release
   - release:image
 
@@ -73,7 +77,7 @@ build:image:
   stage: build:image
 
 .e2e:k8s: &testk8se2e
-  stage: e2e
+  stage: e2e-test
   image: docker:git
   variables:
     K8S_VERSION: k8s-1.19.10
@@ -85,37 +89,95 @@ build:image:
   tags:
     - hc-bladerunner-build
 
-k8s-1.19:
+k8s-1.20:
   <<: *testk8se2e
+  stage: e2e-test-k8s
   variables:
-    K8S_VERSION: k8s-1.19.10
+    K8S_VERSION: k8s-1.20.12
 
-k8s-1.19 networks:
+k8s-1.20-networks:
   <<: *testk8se2e
+  stage: e2e-test-k8s-networks
   variables:
-    K8S_VERSION: k8s-1.19.10
+    K8S_VERSION: k8s-1.20.12
     USE_NETWORKS: "yes"
 
-k8s-1.20:
+k8s-1.21:
   <<: *testk8se2e
+  stage: e2e-test-k8s
   variables:
-    K8S_VERSION: k8s-1.20.6
+    K8S_VERSION: k8s-1.21.6
 
-k8s-1.20 networks:
+k8s-1.21-networks:
   <<: *testk8se2e
+  stage: e2e-test-k8s-networks
   variables:
-    K8S_VERSION: k8s-1.20.6
+    K8S_VERSION: k8s-1.21.6
     USE_NETWORKS: "yes"
 
-k8s-1.21:
+k8s-1.22:
   <<: *testk8se2e
+  stage: e2e-test-k8s
   variables:
-    K8S_VERSION: k8s-1.21.0
+    K8S_VERSION: k8s-1.22.3
 
-k8s-1.21 networks:
+k8s-1.22-networks:
   <<: *testk8se2e
+  stage: e2e-test-k8s-networks
+  variables:
+    K8S_VERSION: k8s-1.22.3
+    USE_NETWORKS: "yes"
+
+.e2e:k3s: &testk3se2e
+  stage: e2e-test
+  image: docker:git
+  variables:
+    K8S_VERSION: k3s-1.20.12+k3s1
+    CCM_IMAGE_NAME: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
+  before_script:
+    - apk add --no-cache git make musl-dev go openssh-client
+  script:
+    - go test $(go list ./... | grep e2etests) -v -timeout 60m
+  tags:
+    - hc-bladerunner-build
+
+k3s-1.20:
+  <<: *testk3se2e
+  stage: e2e-test-k3s
+  variables:
+    K8S_VERSION: k3s-1.20.12+k3s1
+
+k3s-1.20-networks:
+  <<: *testk3se2e
+  stage: e2e-test-k3s-networks
+  variables:
+    K8S_VERSION: k3s-1.20.12+k3s1
+    USE_NETWORKS: "yes"
+
+k3s-1.21:
+  <<: *testk3se2e
+  stage: e2e-test-k3s
+  variables:
+    K8S_VERSION: k3s-1.21.6+k3s1
+
+k3s-1.21-networks:
+  <<: *testk3se2e
+  stage: e2e-test-k3s-networks
+  variables:
+    K8S_VERSION: k3s-1.21.6+k3s1
+    USE_NETWORKS: "yes"
+
+k3s-1.22:
+  <<: *testk3se2e
+  stage: e2e-test-k3s
+  variables:
+    K8S_VERSION: k3s-1.22.3+k3s1
+
+k3s-1.22-networks:
+  <<: *testk3se2e
+  stage: e2e-test-k3s-networks
   variables:
-    K8S_VERSION: k8s-1.21.0
+    K8S_VERSION: k3s-1.22.3+k3s1
     USE_NETWORKS: "yes"
 
 release:image:

diff --git a/.goreleaser.yml b/.goreleaser.yml
@@ -32,7 +32,7 @@ dockers:
       - "hetznercloud/hcloud-cloud-controller-manager:latest"
       - "hetznercloud/hcloud-cloud-controller-manager:{{ if not .IsSnapshot }}v{{ end }}{{ .Version }}"
     dockerfile: Dockerfile
-    use_buildx: true
+    use: buildx
     build_flag_templates:
       - "--platform=linux/amd64"
 release:

diff --git a/README.md b/README.md
@@ -156,7 +156,7 @@ You also need to add the network name/ID to the secret: `kubectl -n kube-system
 
 ## Kube-proxy mode IPVS and HCloud LoadBalancer
 
-If `kube-proxy` is run in IPVS mode, the `Service` manifest needs to have the annotation `load-balancer.hetzner.cloud/hostname` where the FQDN resolves to the HCloud LoadBalancer IP. 
+If `kube-proxy` is run in IPVS mode, the `Service` manifest needs to have the annotation `load-balancer.hetzner.cloud/hostname` where the FQDN resolves to the HCloud LoadBalancer IP.
 
 See https://github.com/hetznercloud/hcloud-cloud-controller-manager/issues/212
 
@@ -173,20 +173,19 @@ release.
 
 ### With Networks support
 
-| Kubernetes | k3s           | cloud controller Manager   | Deployment File                                                                                                 |
-| ---------- | -------------:| --------------------------:|----------------------------------------------------------------------------------------------------------------:|
-| 1.21       | -             | 1.12.1, master                     | https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/latest/download/ccm-networks.yaml      |
-| 1.20       | v1.20.0+k3s2  | 1.12.1, master                     | https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/latest/download/ccm-networks.yaml      |
-| 1.19       | v1.19.5+k3s2  | 1.8.1, master              | https://raw.githubusercontent.com/hetznercloud/hcloud-cloud-controller-manager/v1.8.1/deploy/ccm-networks.yaml  |
-
+| Kubernetes | k3s               | Cloud Controller Manager | Deployment File                                                                                            |
+| ---------- | -----------------:| ------------------------:|-----------------------------------------------------------------------------------------------------------:|
+| 1.22       | v1.22.3+k3s1      | master                   | https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/latest/download/ccm-networks.yaml |
+| 1.21       | v1.21.6+k3s1      | 1.12.0, master           | https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/latest/download/ccm-networks.yaml |
+| 1.20       | v1.20.12+k3s2     | 1.12.0, master           | https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/latest/download/ccm-networks.yaml |
 
 ### Without Networks support
 
-| Kubernetes | k3s           | cloud controller Manager   | Deployment File                                                                                                 |
-| ---------- | -------------:| --------------------------:|----------------------------------------------------------------------------------------------------------------:|
-| 1.21       | -             | 1.12.0, master                     | https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/latest/download/ccm.yaml               |
-| 1.20       | v1.20.0+k3s2  | 1.12.0, master                     | https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/latest/download/ccm.yaml               |
-| 1.19       | v1.19.5+k3s2  | 1.8.1, master              | https://raw.githubusercontent.com/hetznercloud/hcloud-cloud-controller-manager/v1.8.1/deploy/ccm.yaml           |
+| Kubernetes | k3s               | Cloud Controller Manager | Deployment File                                                                                   |
+| ---------- | -----------------:| ------------------------:|--------------------------------------------------------------------------------------------------:|
+| 1.22       | v1.22.3+k3s1      | master                   | https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/latest/download/ccm.yaml |
+| 1.21       | v1.21.6+k3s1      | 1.12.0, master           | https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/latest/download/ccm.yaml |
+| 1.20       | v1.20.12+k3s1     | 1.12.0, master           | https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/latest/download/ccm.yaml |
 
 ## E2E Tests
 

diff --git a/e2etests/setup.go b/e2etests/setup.go
@@ -41,6 +41,7 @@ type hcloudK8sSetup struct {
 	KeepOnFailure   bool
 	ClusterNode     *hcloud.Server
 	ExtServer       *hcloud.Server
+	UseNetworks     bool
 	privKey         string
 	sshKey          *hcloud.SSHKey
 	network         *hcloud.Network
@@ -55,6 +56,7 @@ type cloudInitTmpl struct {
 	HcloudNetwork   string
 	IsClusterServer bool
 	JoinCMD         string
+	UseFlannel      bool
 }
 
 // PrepareTestEnv setups a test environment for the Cloud Controller Manager
@@ -106,7 +108,7 @@ func (s *hcloudK8sSetup) PrepareTestEnv(ctx context.Context, additionalSSHKeys [
 	if err != nil {
 		return "", fmt.Errorf("%s: Load image %s", op, err)
 	}
-	kubeconfigPath, err := s.PrepareK8s(useNetworks)
+	kubeconfigPath, err := s.PrepareK8s()
 	if err != nil {
 		return "", fmt.Errorf("%s: %s", op, err)
 	}
@@ -252,22 +254,23 @@ func (s *hcloudK8sSetup) createServer(ctx context.Context, name, typ string, add
 
 // PrepareK8s patches an existing kubernetes cluster with a CNI and the correct
 // Cloud Controller Manager version from this test run
-func (s *hcloudK8sSetup) PrepareK8s(withNetworks bool) (string, error) {
+func (s *hcloudK8sSetup) PrepareK8s() (string, error) {
 	const op = "hcloudK8sSetup/PrepareK8s"
 
-	if withNetworks {
+	if s.UseNetworks {
 		err := s.deployCilium()
 		if err != nil {
 			return "", fmt.Errorf("%s: %s", op, err)
 		}
-	} else {
+	}
+	if s.K8sDistribution != K8sDistributionK3s && !s.UseNetworks {
 		err := s.deployFlannel()
 		if err != nil {
 			return "", fmt.Errorf("%s: %s", op, err)
 		}
 	}
 
-	err := s.prepareCCMDeploymentFile(withNetworks)
+	err := s.prepareCCMDeploymentFile(s.UseNetworks)
 	if err != nil {
 		return "", fmt.Errorf("%s: %s", op, err)
 	}
@@ -278,19 +281,6 @@ func (s *hcloudK8sSetup) PrepareK8s(withNetworks bool) (string, error) {
 		return "", fmt.Errorf("%s Deploy ccm: %s", op, err)
 	}
 
-	/*	fmt.Printf("[%s] %s: Ensure Server is not labeled as master\n", s.ClusterNode.Name, op)
-		err = RunCommandOnServer(s.privKey, s.ClusterNode, "KUBECONFIG=/root/.kube/config kubectl label nodes --all node-role.kubernetes.io/master-")
-		if err != nil {
-			return "", fmt.Errorf("%s Ensure Server is not labeled as master: %s", op, err)
-		}
-
-		if s.K8sDistribution == K8sDistributionK8s {
-			fmt.Printf("[%s] %s: Ensure Server is not tainted as master\n", s.ClusterNode.Name, op)
-			err = RunCommandOnServer(s.privKey, s.ClusterNode, "KUBECONFIG=/root/.kube/config kubectl taint nodes --all node-role.kubernetes.io/master-")
-			if err != nil {
-				return "", fmt.Errorf("%s Ensure Server is not tainted as master: %s", op, err)
-			}
-		}*/
 	fmt.Printf("[%s] %s: Download kubeconfig\n", s.ClusterNode.Name, op)
 
 	err = scp("ssh_key", fmt.Sprintf("root@%s:/root/.kube/config", s.ClusterNode.PublicNet.IPv4.IP.String()), "kubeconfig")
@@ -401,7 +391,7 @@ func (s *hcloudK8sSetup) deployCilium() error {
 	if err != nil {
 		return fmt.Errorf("%s: read cilium deployment file %s: %v", op, "templates/cilium.yml", err)
 	}
-	err = RunCommandOnServer(s.privKey, s.ClusterNode, fmt.Sprintf("echo '%s' >> cilium.yml", deploymentFile))
+	err = RunCommandOnServer(s.privKey, s.ClusterNode, fmt.Sprintf("cat <<EOF > cilium.yml\n%s\nEOF", deploymentFile))
 	if err != nil {
 		return fmt.Errorf("%s: Transfer cilium deployment: %s", op, err)
 	}
@@ -514,6 +504,14 @@ func (s *hcloudK8sSetup) TearDown(testFailed bool) error {
 func (s *hcloudK8sSetup) getCloudInitConfig(isClusterServer bool) (string, error) {
 	const op = "hcloudK8sSetup/getCloudInitConfig"
 
+	data := cloudInitTmpl{
+		K8sVersion:      s.K8sVersion,
+		HcloudToken:     s.HcloudToken,
+		HcloudNetwork:   s.network.Name,
+		IsClusterServer: isClusterServer,
+		JoinCMD:         s.clusterJoinCMD,
+		UseFlannel:      s.K8sDistribution == K8sDistributionK3s && !s.UseNetworks,
+	}
 	str, err := ioutil.ReadFile(fmt.Sprintf("templates/cloudinit_%s.txt.tpl", s.K8sDistribution))
 	if err != nil {
 		return "", fmt.Errorf("%s: read template file %s: %v", "templates/cloudinit.txt.tpl", op, err)
@@ -523,7 +521,7 @@ func (s *hcloudK8sSetup) getCloudInitConfig(isClusterServer bool) (string, error
 		return "", fmt.Errorf("%s: parsing template file %s: %v", "templates/cloudinit.txt.tpl", op, err)
 	}
 	var buf bytes.Buffer
-	if err := tmpl.Execute(&buf, cloudInitTmpl{K8sVersion: s.K8sVersion, HcloudToken: s.HcloudToken, HcloudNetwork: s.network.Name, IsClusterServer: isClusterServer, JoinCMD: s.clusterJoinCMD}); err != nil {
+	if err := tmpl.Execute(&buf, data); err != nil {
 		return "", fmt.Errorf("%s: execute template: %v", op, err)
 	}
 	return buf.String(), nil
@@ -611,6 +609,7 @@ func RunCommandOnServer(privKey string, server *hcloud.Server, command string) e
 	return WithSSHSession(privKey, server.PublicNet.IPv4.IP.String(), func(session *ssh.Session) error {
 		if ok := os.Getenv("TEST_DEBUG_MODE"); ok != "" {
 			session.Stdout = os.Stdout
+			session.Stderr = os.Stderr
 		}
 		return session.Run(command)
 	})