diff --git a/e2etests/.gitignore b/e2etests/.gitignore index 4456063e9..108479e84 100644 --- a/e2etests/.gitignore +++ b/e2etests/.gitignore @@ -1,3 +1,4 @@ ci-hcloud-ccm.tar ssh_key kubeconfig +join.txt diff --git a/e2etests/setup.go b/e2etests/setup.go index 8982c8929..d36d61108 100644 --- a/e2etests/setup.go +++ b/e2etests/setup.go @@ -29,6 +29,8 @@ const ( K8sDistributionK3s K8sDistribution = "k3s" ) +var instanceType = "cpx21" + type hcloudK8sSetup struct { Hcloud *hcloud.Client HcloudToken string @@ -42,87 +44,176 @@ type hcloudK8sSetup struct { privKey string sshKey *hcloud.SSHKey network *hcloud.Network + clusterJoinCMD string + WorkerNodes []*hcloud.Server testLabels map[string]string } type cloudInitTmpl struct { - K8sVersion string - HcloudToken string - HcloudNetwork string + K8sVersion string + HcloudToken string + HcloudNetwork string + IsClusterServer bool + JoinCMD string } // PrepareTestEnv setups a test environment for the Cloud Controller Manager // This includes the creation of a Network, SSH Key and Server. // The server will be created with a Cloud Init UserData // The template can be found under e2etests/templates/cloudinit_.ixt.tpl -func (s *hcloudK8sSetup) PrepareTestEnv(ctx context.Context, additionalSSHKeys []*hcloud.SSHKey) error { +func (s *hcloudK8sSetup) PrepareTestEnv(ctx context.Context, additionalSSHKeys []*hcloud.SSHKey, useNetworks bool) (string, error) { const op = "hcloudK8sSetup/PrepareTestEnv" s.testLabels = map[string]string{"K8sDistribution": string(s.K8sDistribution), "K8sVersion": strings.ReplaceAll(s.K8sVersion, "+", ""), "test": s.TestIdentifier} err := s.getSSHKey(ctx) if err != nil { - return fmt.Errorf("%s getSSHKey: %s", op, err) + return "", fmt.Errorf("%s getSSHKey: %s", op, err) } err = s.getNetwork(ctx) if err != nil { - return fmt.Errorf("%s getNetwork: %s", op, err) + return "", fmt.Errorf("%s getNetwork: %s", op, err) } - - srv, err := s.createServer(ctx, "cluster-node", "cpx21", additionalSSHKeys) + userData, err := s.getCloudInitConfig(true) + if err != nil { + fmt.Printf("[cluster-node] %s getCloudInitConfig: %s", op, err) + return "", err + } + srv, err := s.createServer(ctx, "cluster-node", instanceType, additionalSSHKeys, userData) if err != nil { - return fmt.Errorf("%s: create cluster node: %v", op, err) + return "", fmt.Errorf("%s: create cluster node: %v", op, err) } s.ClusterNode = srv + s.waitUntilSSHable(srv) + err = s.waitForCloudInit(srv) + if err != nil { + return "", err + } - srv, err = s.createServer(ctx, "ext-server", "cx11", additionalSSHKeys) + joinCmd, err := s.getJoinCmd() if err != nil { - return fmt.Errorf("%s: create ext server: %v", op, err) + return "", err } - s.ExtServer = srv + s.clusterJoinCMD = joinCmd - fmt.Printf("%s Waiting for server to be sshable:", op) - for { - conn, err := net.Dial("tcp", fmt.Sprintf("%s:22", s.ClusterNode.PublicNet.IPv4.IP.String())) - if err != nil { - fmt.Print(".") - time.Sleep(1 * time.Second) - continue - } - _ = conn.Close() - fmt.Print("Connection successful\n") - break + err = s.transferDockerImage(s.ClusterNode) + if err != nil { + return "", fmt.Errorf("%s: %s", op, err) } - err = s.waitForCloudInit() + + fmt.Printf("[%s] %s: Load Image:\n", s.ClusterNode.Name, op) + err = RunCommandOnServer(s.privKey, s.ClusterNode, "ctr -n=k8s.io image import ci-hcloud-ccm.tar") if err != nil { - return err + return "", fmt.Errorf("%s: Load image %s", op, err) } + kubeconfigPath, err := s.PrepareK8s(useNetworks) + if err != nil { + return "", fmt.Errorf("%s: %s", op, err) + } + + var workers = 1 // Change this value if you want to have more workers for the test + var wg sync.WaitGroup + for worker := 1; worker <= workers; worker++ { + wg.Add(1) + go s.createClusterWorker(ctx, additionalSSHKeys, &wg, worker) + } + wg.Wait() - err = s.transferCCMDockerImage() + srv, err = s.createServer(ctx, "ext-server", instanceType, additionalSSHKeys, "") if err != nil { - return fmt.Errorf("%s: %s", op, err) + return "", fmt.Errorf("%s: create ext server: %v", op, err) } + s.ExtServer = srv + s.waitUntilSSHable(srv) + + return kubeconfigPath, nil +} + +func (s *hcloudK8sSetup) createClusterWorker(ctx context.Context, additionalSSHKeys []*hcloud.SSHKey, wg *sync.WaitGroup, worker int) { + const op = "hcloudK8sSetup/createClusterWorker" + defer wg.Done() - fmt.Printf("%s Load Image:\n", op) - transferCmd := "docker load --input ci-hcloud-ccm.tar" - if s.K8sDistribution == K8sDistributionK3s { - transferCmd = "ctr -n=k8s.io image import ci-hcloud-ccm.tar" + workerName := fmt.Sprintf("cluster-worker-%d", worker) + fmt.Printf("[%s] %s Create worker node:\n", workerName, op) + + userData, err := s.getCloudInitConfig(false) + if err != nil { + fmt.Printf("[%s] %s getCloudInitConfig: %s", workerName, op, err) + return } - err = RunCommandOnServer(s.privKey, s.ClusterNode, transferCmd) + srv, err := s.createServer(ctx, workerName, instanceType, additionalSSHKeys, userData) if err != nil { - return fmt.Errorf("%s: Load image %s", op, err) + fmt.Printf("[%s] %s createServer: %s", workerName, op, err) + return } + s.WorkerNodes = append(s.WorkerNodes, srv) - return nil + s.waitUntilSSHable(srv) + + err = s.waitForCloudInit(srv) + if err != nil { + fmt.Printf("[%s] %s: wait for cloud init on worker: %v", srv.Name, op, err) + return + } + + err = s.transferDockerImage(srv) + if err != nil { + fmt.Printf("[%s] %s: transfer image on worker: %v", srv.Name, op, err) + return + } + + fmt.Printf("[%s] %s Load Image\n", srv.Name, op) + err = RunCommandOnServer(s.privKey, srv, "ctr -n=k8s.io image import ci-hcloud-ccm.tar") + if err != nil { + fmt.Printf("[%s] %s: load image on worker: %v", srv.Name, op, err) + return + } } -func (s *hcloudK8sSetup) createServer(ctx context.Context, name, typ string, additionalSSHKeys []*hcloud.SSHKey) (*hcloud.Server, error) { - const op = "e2etest/createServer" +// waitForCloudInit waits on cloud init on the server. +// when cloud init is ready we can assume that the server +// and the plain k8s installation is ready +func (s *hcloudK8sSetup) getJoinCmd() (string, error) { + const op = "hcloudK8sSetup/getJoinCmd" + fmt.Printf("[%s] %s: Download join cmd\n", s.ClusterNode.Name, op) + if s.K8sDistribution == K8sDistributionK8s { + err := scp("ssh_key", fmt.Sprintf("root@%s:/root/join.txt", s.ClusterNode.PublicNet.IPv4.IP.String()), "join.txt") + if err != nil { + return "", fmt.Errorf("[%s] %s download join cmd: %s", s.ClusterNode.Name, op, err) + } + cmd, err := ioutil.ReadFile("join.txt") + if err != nil { + return "", fmt.Errorf("[%s] %s reading join cmd file: %s", s.ClusterNode.Name, op, err) + } - userData, err := s.getCloudInitConfig() + return string(cmd), nil + } + err := scp("ssh_key", fmt.Sprintf("root@%s:/var/lib/rancher/k3s/server/node-token", s.ClusterNode.PublicNet.IPv4.IP.String()), "join.txt") if err != nil { - return nil, fmt.Errorf("%s getCloudInitConfig: %s", op, err) + return "", fmt.Errorf("[%s] %s download join cmd: %s", s.ClusterNode.Name, op, err) + } + token, err := ioutil.ReadFile("join.txt") + return fmt.Sprintf("K3S_URL=https://%s:6443 K3S_TOKEN=%s", s.ClusterNode.PublicNet.IPv4.IP.String(), token), nil +} + +func (s *hcloudK8sSetup) waitUntilSSHable(server *hcloud.Server) { + const op = "hcloudK8sSetup/PrepareTestEnv" + fmt.Printf("[%s] %s: Waiting for server to be sshable:\n", server.Name, op) + for { + conn, err := net.Dial("tcp", fmt.Sprintf("%s:22", server.PublicNet.IPv4.IP.String())) + if err != nil { + time.Sleep(1 * time.Second) + continue + } + _ = conn.Close() + fmt.Printf("[%s] %s: SSH Connection successful\n", server.Name, op) + break } +} + +func (s *hcloudK8sSetup) createServer(ctx context.Context, name, typ string, additionalSSHKeys []*hcloud.SSHKey, userData string) (*hcloud.Server, error) { + const op = "e2etest/createServer" + sshKeys := []*hcloud.SSHKey{s.sshKey} for _, additionalSSHKey := range additionalSSHKeys { sshKeys = append(sshKeys, additionalSSHKey) @@ -181,33 +272,33 @@ func (s *hcloudK8sSetup) PrepareK8s(withNetworks bool) (string, error) { return "", fmt.Errorf("%s: %s", op, err) } - fmt.Printf("%s: Apply ccm deployment\n", op) + fmt.Printf("[%s] %s: Apply ccm deployment\n", s.ClusterNode.Name, op) err = RunCommandOnServer(s.privKey, s.ClusterNode, "KUBECONFIG=/root/.kube/config kubectl apply -f ccm.yml") if err != nil { return "", fmt.Errorf("%s Deploy ccm: %s", op, err) } - fmt.Printf("%s: Ensure Server is not labeled as master\n", op) - err = RunCommandOnServer(s.privKey, s.ClusterNode, "KUBECONFIG=/root/.kube/config kubectl label nodes --all node-role.kubernetes.io/master-") - if err != nil { - return "", fmt.Errorf("%s Ensure Server is not labeled as master: %s", op, err) - } - - if s.K8sDistribution == K8sDistributionK8s { - fmt.Printf("%s: Ensure Server is not tainted as master\n", op) - err = RunCommandOnServer(s.privKey, s.ClusterNode, "KUBECONFIG=/root/.kube/config kubectl taint nodes --all node-role.kubernetes.io/master-") + /* fmt.Printf("[%s] %s: Ensure Server is not labeled as master\n", s.ClusterNode.Name, op) + err = RunCommandOnServer(s.privKey, s.ClusterNode, "KUBECONFIG=/root/.kube/config kubectl label nodes --all node-role.kubernetes.io/master-") if err != nil { - return "", fmt.Errorf("%s Ensure Server is not tainted as master: %s", op, err) + return "", fmt.Errorf("%s Ensure Server is not labeled as master: %s", op, err) } - } - fmt.Printf("%s: Download kubeconfig\n", op) + + if s.K8sDistribution == K8sDistributionK8s { + fmt.Printf("[%s] %s: Ensure Server is not tainted as master\n", s.ClusterNode.Name, op) + err = RunCommandOnServer(s.privKey, s.ClusterNode, "KUBECONFIG=/root/.kube/config kubectl taint nodes --all node-role.kubernetes.io/master-") + if err != nil { + return "", fmt.Errorf("%s Ensure Server is not tainted as master: %s", op, err) + } + }*/ + fmt.Printf("[%s] %s: Download kubeconfig\n", s.ClusterNode.Name, op) err = scp("ssh_key", fmt.Sprintf("root@%s:/root/.kube/config", s.ClusterNode.PublicNet.IPv4.IP.String()), "kubeconfig") if err != nil { return "", fmt.Errorf("%s download kubeconfig: %s", op, err) } - fmt.Printf("%s: Ensure correct server is set\n", op) + fmt.Printf("[%s] %s: Ensure correct server is set\n", s.ClusterNode.Name, op) kubeconfigBefore, err := ioutil.ReadFile("kubeconfig") if err != nil { return "", fmt.Errorf("%s reading kubeconfig: %s", op, err) @@ -324,11 +415,11 @@ func (s *hcloudK8sSetup) deployCilium() error { return nil } -// transferCCMDockerImage transfers the local build docker image tar via SCP -func (s *hcloudK8sSetup) transferCCMDockerImage() error { - const op = "hcloudK8sSetup/transferCCMDockerImage" - fmt.Printf("%s: Transfer docker image\n", op) - err := WithSSHSession(s.privKey, s.ClusterNode.PublicNet.IPv4.IP.String(), func(session *ssh.Session) error { +// transferDockerImage transfers the local build docker image tar via SCP +func (s *hcloudK8sSetup) transferDockerImage(server *hcloud.Server) error { + const op = "hcloudK8sSetup/transferDockerImage" + fmt.Printf("[%s] %s: Transfer docker image\n", server.Name, op) + err := WithSSHSession(s.privKey, server.PublicNet.IPv4.IP.String(), func(session *ssh.Session) error { file, err := os.Open("ci-hcloud-ccm.tar") if err != nil { return fmt.Errorf("%s read ci-hcloud-ccm.tar: %s", op, err) @@ -363,10 +454,10 @@ func (s *hcloudK8sSetup) transferCCMDockerImage() error { // waitForCloudInit waits on cloud init on the server. // when cloud init is ready we can assume that the server // and the plain k8s installation is ready -func (s *hcloudK8sSetup) waitForCloudInit() error { +func (s *hcloudK8sSetup) waitForCloudInit(server *hcloud.Server) error { const op = "hcloudK8sSetup/PrepareTestEnv" - fmt.Printf("%s: Wait for cloud-init\n", op) - err := RunCommandOnServer(s.privKey, s.ClusterNode, fmt.Sprintf("cloud-init status --wait > /dev/null")) + fmt.Printf("[%s] %s: Wait for cloud-init\n", server.Name, op) + err := RunCommandOnServer(s.privKey, server, fmt.Sprintf("cloud-init status --wait > /dev/null")) if err != nil { return fmt.Errorf("%s: Wait for cloud-init: %s", op, err) } @@ -393,6 +484,13 @@ func (s *hcloudK8sSetup) TearDown(testFailed bool) error { } s.ClusterNode = nil + for _, wn := range s.WorkerNodes { + _, err := s.Hcloud.Server.Delete(ctx, wn) + if err != nil { + return fmt.Errorf("[%s] %s Hcloud.Server.Delete: %s", wn.Name, op, err) + } + } + _, err = s.Hcloud.Server.Delete(ctx, s.ExtServer) if err != nil { return fmt.Errorf("%s Hcloud.Server.Delete: %s", op, err) @@ -413,7 +511,7 @@ func (s *hcloudK8sSetup) TearDown(testFailed bool) error { } // getCloudInitConfig returns the generated cloud init configuration -func (s *hcloudK8sSetup) getCloudInitConfig() (string, error) { +func (s *hcloudK8sSetup) getCloudInitConfig(isClusterServer bool) (string, error) { const op = "hcloudK8sSetup/getCloudInitConfig" str, err := ioutil.ReadFile(fmt.Sprintf("templates/cloudinit_%s.txt.tpl", s.K8sDistribution)) @@ -425,7 +523,7 @@ func (s *hcloudK8sSetup) getCloudInitConfig() (string, error) { return "", fmt.Errorf("%s: parsing template file %s: %v", "templates/cloudinit.txt.tpl", op, err) } var buf bytes.Buffer - if err := tmpl.Execute(&buf, cloudInitTmpl{K8sVersion: s.K8sVersion, HcloudToken: s.HcloudToken, HcloudNetwork: s.network.Name}); err != nil { + if err := tmpl.Execute(&buf, cloudInitTmpl{K8sVersion: s.K8sVersion, HcloudToken: s.HcloudToken, HcloudNetwork: s.network.Name, IsClusterServer: isClusterServer, JoinCMD: s.clusterJoinCMD}); err != nil { return "", fmt.Errorf("%s: execute template: %v", op, err) } return buf.String(), nil diff --git a/e2etests/templates/cloudinit_k3s.txt.tpl b/e2etests/templates/cloudinit_k3s.txt.tpl index 5a8e5498c..8f257713e 100644 --- a/e2etests/templates/cloudinit_k3s.txt.tpl +++ b/e2etests/templates/cloudinit_k3s.txt.tpl @@ -15,11 +15,16 @@ runcmd: - sysctl --system - apt install -y apt-transport-https curl - export INSTALL_K3S_VERSION={{.K8sVersion}} +# Download and install latest hcloud cli release for easier debugging on host +- curl -s https://api.github.com/repos/hetznercloud/cli/releases/latest | grep browser_download_url | grep linux-amd64 | cut -d '"' -f 4 | wget -qi - +- tar xvzf hcloud-linux-amd64.tar.gz && cp hcloud /usr/bin/hcloud && chmod +x /usr/bin/hcloud +{{if .IsClusterServer}} - curl -sfL https://get.k3s.io | sh -s - --disable servicelb --disable traefik --disable-cloud-controller --kubelet-arg="cloud-provider=external" --no-flannel - mkdir -p /root/.kube - cp -i /etc/rancher/k3s/k3s.yaml /root/.kube/config - until KUBECONFIG=/root/.kube/config kubectl get node; do sleep 2;done - KUBECONFIG=/root/.kube/config kubectl -n kube-system create secret generic hcloud --from-literal=token={{.HcloudToken}} --from-literal=network={{.HcloudNetwork}} -# Download and install latest hcloud cli release for easier debugging on host -- curl -s https://api.github.com/repos/hetznercloud/cli/releases/latest | grep browser_download_url | grep linux-amd64 | cut -d '"' -f 4 | wget -qi - -- tar xvzf hcloud-linux-amd64.tar.gz && cp hcloud /usr/bin/hcloud && chmod +x /usr/bin/hcloud +{{else}} +- curl -sfL https://get.k3s.io | {{.JoinCMD}} sh -s - --kubelet-arg="cloud-provider=external" --no-flannel +- sleep 10 # to get the joining work +{{end}} diff --git a/e2etests/templates/cloudinit_k8s.txt.tpl b/e2etests/templates/cloudinit_k8s.txt.tpl index 92de239e0..7382c63e6 100644 --- a/e2etests/templates/cloudinit_k8s.txt.tpl +++ b/e2etests/templates/cloudinit_k8s.txt.tpl @@ -1,8 +1,13 @@ #cloud-config write_files: +- content: | + overlay + br_netfilter + path: /etc/modules-load.d/containerd.conf - content: | net.bridge.bridge-nf-call-ip6tables = 1 net.bridge.bridge-nf-call-iptables = 1 + net.ipv4.ip_forward = 1 path: /etc/sysctl.d/k8s.conf - content: | apiVersion: kubeadm.k8s.io/v1beta2 @@ -20,23 +25,37 @@ write_files: alias ksy="kubectl -n kube-system" alias kgp="kubectl get pods" alias kgs="kubectl get services" + alias cilog="cat /var/log/cloud-init-output.log" export HCLOUD_TOKEN={{.HcloudToken}} path: /root/.bashrc runcmd: - export HOME=/root +- modprobe overlay +- modprobe br_netfilter - sysctl --system - apt install -y apt-transport-https curl - curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - - echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list +- curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg +- echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null - apt update -- apt install -y kubectl={{.K8sVersion}}-00 kubeadm={{.K8sVersion}}-00 kubelet={{.K8sVersion}}-00 docker.io +- apt install -y kubectl={{.K8sVersion}}-00 kubeadm={{.K8sVersion}}-00 kubelet={{.K8sVersion}}-00 containerd.io - systemctl daemon-reload +- mkdir -p /etc/containerd +- containerd config default | tee /etc/containerd/config.toml +- systemctl restart containerd - systemctl restart kubelet +# Download and install latest hcloud cli release for easier debugging on host +- curl -s https://api.github.com/repos/hetznercloud/cli/releases/latest | grep browser_download_url | grep linux-amd64 | cut -d '"' -f 4 | wget -qi - +- tar xvzf hcloud-linux-amd64.tar.gz && cp hcloud /usr/bin/hcloud && chmod +x /usr/bin/hcloud +{{if .IsClusterServer}} - kubeadm init --config /tmp/kubeadm-config.yaml - mkdir -p /root/.kube - cp -i /etc/kubernetes/admin.conf /root/.kube/config - until KUBECONFIG=/root/.kube/config kubectl get node; do sleep 2;done - KUBECONFIG=/root/.kube/config kubectl -n kube-system create secret generic hcloud --from-literal=token={{.HcloudToken}} --from-literal=network={{.HcloudNetwork}} -# Download and install latest hcloud cli release for easier debugging on host -- curl -s https://api.github.com/repos/hetznercloud/cli/releases/latest | grep browser_download_url | grep linux-amd64 | cut -d '"' -f 4 | wget -qi - -- tar xvzf hcloud-linux-amd64.tar.gz && cp hcloud /usr/bin/hcloud && chmod +x /usr/bin/hcloud +- kubeadm token create --print-join-command >> /root/join.txt +{{else}} +- {{.JoinCMD}} +- sleep 10 # to get the joining work +{{end}} diff --git a/e2etests/testing.go b/e2etests/testing.go index c50a24be5..fda7aa7c3 100644 --- a/e2etests/testing.go +++ b/e2etests/testing.go @@ -106,7 +106,7 @@ func (tc *TestCluster) initialize() error { } } - fmt.Printf("Test against %s\n", k8sVersion) + fmt.Printf("%s: Test against %s\n", op, k8sVersion) imageName := os.Getenv("CCM_IMAGE_NAME") buildImage := false @@ -115,7 +115,7 @@ func (tc *TestCluster) initialize() error { buildImage = true } if buildImage { - fmt.Println("Building ccm image") + fmt.Printf("%s: Building ccm image\n", op) err := runCmd( "go", @@ -131,7 +131,7 @@ func (tc *TestCluster) initialize() error { } } - fmt.Println("Saving ccm image to disk") + fmt.Printf("%s: Saving ccm image to disk\n", op) if err := runCmd("docker", []string{"save", "--output", "ci-hcloud-ccm.tar", imageName}, nil); err != nil { return fmt.Errorf("%s: %v", op, err) } @@ -145,14 +145,9 @@ func (tc *TestCluster) initialize() error { HcloudToken: token, KeepOnFailure: tc.KeepOnFailure, } - fmt.Println("Setting up test env") + fmt.Printf("%s: Setting up test env\n", op) - err := tc.setup.PrepareTestEnv(context.Background(), additionalSSHKeys) - if err != nil { - return fmt.Errorf("%s: %s", op, err) - } - - kubeconfigPath, err := tc.setup.PrepareK8s(tc.useNetworks) + kubeconfigPath, err := tc.setup.PrepareTestEnv(context.Background(), additionalSSHKeys, tc.useNetworks) if err != nil { return fmt.Errorf("%s: %s", op, err) } @@ -215,8 +210,8 @@ func (tc *TestCluster) ensureNodesReady() error { const op = "e2etests/ensureNodesReady" err := wait.Poll(1*time.Second, 5*time.Minute, func() (bool, error) { - var available bool - + var totalNodes = len(tc.setup.WorkerNodes) + 1 // Number Worker Nodes + 1 Cluster Node + var readyNodes int nodes, err := tc.k8sClient.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) if err != nil { return false, err @@ -224,11 +219,13 @@ func (tc *TestCluster) ensureNodesReady() error { for _, node := range nodes.Items { for _, cond := range node.Status.Conditions { if cond.Type == corev1.NodeReady && cond.Status == corev1.ConditionTrue { - available = true + readyNodes++ } } } - return available, nil + pendingNodes := totalNodes - readyNodes + fmt.Printf("Waiting for %d/%d nodes\n", pendingNodes, totalNodes) + return pendingNodes == 0, err }) if err != nil {