From 536fc85002372f12d63f668d8beadd7068357cda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=94=A6=E5=8D=97=E8=B7=AF=E4=B9=8B=E8=8A=B1?= Date: Sun, 15 Sep 2024 19:24:39 +0200 Subject: [PATCH 1/2] feat:add calico api server to calico cni plugin --- .../calico_version/update_calico_version.go | 22 ++ pkg/minikube/bootstrapper/images/images.go | 6 +- pkg/minikube/cni/calico-apiserver.yaml | 297 ++++++++++++++++++ pkg/minikube/cni/calico.go | 70 ++++- 4 files changed, 392 insertions(+), 3 deletions(-) create mode 100644 pkg/minikube/cni/calico-apiserver.yaml diff --git a/hack/update/calico_version/update_calico_version.go b/hack/update/calico_version/update_calico_version.go index b20cfaa2c55a..c620c554a277 100644 --- a/hack/update/calico_version/update_calico_version.go +++ b/hack/update/calico_version/update_calico_version.go @@ -58,6 +58,8 @@ func main() { } func updateYAML(version string) { + // for calico we are going to install both calico and calico api server via mainifest + // first we update the calico itself res, err := http.Get(fmt.Sprintf("https://raw.githubusercontent.com/projectcalico/calico/%s/manifests/calico.yaml", version)) if err != nil { klog.Fatalf("failed to get calico.yaml: %v", err) @@ -79,4 +81,24 @@ func updateYAML(version string) { if err := os.WriteFile("../../../pkg/minikube/cni/calico.yaml", yaml, 0644); err != nil { klog.Fatalf("failed to write to YAML file: %v", err) } + // the second step is the calico api server + // doc: https://docs.tigera.io/calico/latest/operations/install-apiserver + resAPIServer, err := http.Get(fmt.Sprintf("https://raw.githubusercontent.com/projectcalico/calico/%s/manifests/apiserver.yaml", version)) + if err != nil { + klog.Fatalf("failed to get calico.yaml: %v", err) + } + defer resAPIServer.Body.Close() + yamlAPIServer, err := io.ReadAll(resAPIServer.Body) + if err != nil { + klog.Fatalf("failed to read body: %v", err) + } + replacementsAPIServer := map[string]string{ + `calico\/apiserver:.*`: "{{ .APIServerImageName }}", + } + for re, repl := range replacementsAPIServer { + yamlAPIServer = regexp.MustCompile(re).ReplaceAll(yamlAPIServer, []byte(repl)) + } + if err := os.WriteFile("../../../pkg/minikube/cni/calico-apiserver.yaml", yamlAPIServer, 0644); err != nil { + klog.Fatalf("failed to write to YAML file: %v", err) + } } diff --git a/pkg/minikube/bootstrapper/images/images.go b/pkg/minikube/bootstrapper/images/images.go index 2b5bdb7e7f72..4f6d683ec9e8 100644 --- a/pkg/minikube/bootstrapper/images/images.go +++ b/pkg/minikube/bootstrapper/images/images.go @@ -183,7 +183,7 @@ func KindNet(repo string) string { } // all calico images are from https://github.com/projectcalico/calico/blob/master/manifests/calico.yaml -const calicoVersion = "v3.28.1" +const calicoVersion = "v3.28.2" const calicoRepo = "docker.io/calico" // CalicoDaemonSet returns the image used for calicoDaemonSet @@ -202,6 +202,10 @@ func CalicoBin(repo string) string { return calicoCommon(repo, "cni") } +// CalicoAPIServer returns image used for calico apiserver image +func CalicoAPIServer(repo string) string { + return calicoCommon(repo, "apiserver") +} func calicoCommon(repo string, name string) string { if repo == "" { repo = calicoRepo diff --git a/pkg/minikube/cni/calico-apiserver.yaml b/pkg/minikube/cni/calico-apiserver.yaml new file mode 100644 index 000000000000..a5310cadc770 --- /dev/null +++ b/pkg/minikube/cni/calico-apiserver.yaml @@ -0,0 +1,297 @@ +# This is a tech-preview manifest which installs the Calico API server. Note that this manifest is liable to change +# or be removed in future releases without further warning. +# +# Namespace and namespace-scoped resources. +apiVersion: v1 +kind: Namespace +metadata: + labels: + name: calico-apiserver + name: calico-apiserver +spec: + +--- + +# Policy to ensure the API server isn't cut off. Can be modified, but ensure +# that the main API server is always able to reach the Calico API server. +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: allow-apiserver + namespace: calico-apiserver +spec: + podSelector: + matchLabels: + apiserver: "true" + ingress: + - ports: + - protocol: TCP + port: 5443 + +--- + +apiVersion: v1 +kind: Service +metadata: + name: calico-api + namespace: calico-apiserver +spec: + ports: + - name: apiserver + port: 443 + protocol: TCP + targetPort: 5443 + selector: + apiserver: "true" + type: ClusterIP + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + apiserver: "true" + k8s-app: calico-apiserver + name: calico-apiserver + namespace: calico-apiserver +spec: + replicas: 1 + selector: + matchLabels: + apiserver: "true" + strategy: + type: Recreate + template: + metadata: + labels: + apiserver: "true" + k8s-app: calico-apiserver + name: calico-apiserver + namespace: calico-apiserver + spec: + containers: + - args: + - --secure-port=5443 + - -v=5 + env: + - name: DATASTORE_TYPE + value: kubernetes + image: {{ .APIServerImageName }} + name: calico-apiserver + readinessProbe: + httpGet: + path: /readyz + port: 5443 + scheme: HTTPS + timeoutSeconds: 5 + periodSeconds: 60 + securityContext: + privileged: false + runAsUser: 0 + volumeMounts: + - mountPath: /code/apiserver.local.config/certificates + name: calico-apiserver-certs + dnsPolicy: ClusterFirst + nodeSelector: + kubernetes.io/os: linux + restartPolicy: Always + serviceAccount: calico-apiserver + serviceAccountName: calico-apiserver + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + volumes: + - name: calico-apiserver-certs + secret: + secretName: calico-apiserver-certs + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-apiserver + namespace: calico-apiserver + +--- + +# Cluster-scoped resources below here. +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + name: v3.projectcalico.org +spec: + group: projectcalico.org + groupPriorityMinimum: 1500 + service: + name: calico-api + namespace: calico-apiserver + port: 443 + version: v3 + versionPriority: 200 + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: calico-crds +rules: +- apiGroups: + - extensions + - networking.k8s.io + - "" + resources: + - networkpolicies + - nodes + - namespaces + - pods + - serviceaccounts + verbs: + - get + - list + - watch +- apiGroups: + - crd.projectcalico.org + resources: + - globalnetworkpolicies + - networkpolicies + - clusterinformations + - hostendpoints + - globalnetworksets + - networksets + - bgpconfigurations + - bgppeers + - bgpfilters + - felixconfigurations + - kubecontrollersconfigurations + - ippools + - ipreservations + - ipamblocks + - blockaffinities + - caliconodestatuses + - ipamconfigs + verbs: + - get + - list + - watch + - create + - update + - delete +- apiGroups: + - policy + resourceNames: + - calico-apiserver + resources: + - podsecuritypolicies + verbs: + - use + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: calico-extension-apiserver-auth-access +rules: +- apiGroups: + - "" + resourceNames: + - extension-apiserver-authentication + resources: + - configmaps + verbs: + - list + - watch + - get +- apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterroles + - clusterrolebindings + - roles + - rolebindings + verbs: + - get + - list + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: calico-webhook-reader +rules: +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + - validatingwebhookconfigurations + verbs: + - get + - list + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: calico-apiserver-access-crds +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-crds +subjects: +- kind: ServiceAccount + name: calico-apiserver + namespace: calico-apiserver + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: calico-apiserver-delegate-auth +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: calico-apiserver + namespace: calico-apiserver + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: calico-apiserver-webhook-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-webhook-reader +subjects: +- kind: ServiceAccount + name: calico-apiserver + namespace: calico-apiserver + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: calico-extension-apiserver-auth-access +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-extension-apiserver-auth-access +subjects: +- kind: ServiceAccount + name: calico-apiserver + namespace: calico-apiserver diff --git a/pkg/minikube/cni/calico.go b/pkg/minikube/cni/calico.go index 6f7b55c4b7c1..d62118cef35f 100644 --- a/pkg/minikube/cni/calico.go +++ b/pkg/minikube/cni/calico.go @@ -19,6 +19,8 @@ package cni import ( "bytes" "fmt" + "os/exec" + "path" // goembed needs this _ "embed" @@ -26,9 +28,13 @@ import ( "github.com/blang/semver/v4" "github.com/pkg/errors" + + "k8s.io/klog/v2" + "k8s.io/minikube/pkg/kapi" "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/bootstrapper/images" "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/vmpath" "k8s.io/minikube/pkg/util" ) @@ -37,8 +43,12 @@ import ( //go:embed calico.yaml var calicoYaml string +//go:embed calico-apiserver.yaml +var calicoAPIServerYaml string + // calicoTmpl is from https://docs.projectcalico.org/manifests/calico.yaml var calicoTmpl = template.Must(template.New("calico").Parse(calicoYaml)) +var calicoAPIServerTmpl = template.Must(template.New("calico-apiserver").Parse(calicoAPIServerYaml)) // Calico is the Calico CNI manager type Calico struct { @@ -80,13 +90,66 @@ func (c Calico) manifest() (assets.CopyableFile, error) { return manifestAsset(b.Bytes()), nil } +func (c Calico) apiServerManifest() (assets.CopyableFile, error) { + b := bytes.Buffer{} + if err := calicoAPIServerTmpl.Execute(&b, map[string]string{ + "APIServerImageName": images.CalicoAPIServer(c.cc.KubernetesConfig.ImageRepository), + }); err != nil { + return nil, err + } + return assets.NewMemoryAssetTarget(b.Bytes(), apiServerMainifestPath(), "0644"), nil +} + // Apply enables the CNI func (c Calico) Apply(r Runner) error { - m, err := c.manifest() + manifest, err := c.manifest() if err != nil { return errors.Wrap(err, "manifest") } - return applyManifest(c.cc, r, m) + if err := applyManifest(c.cc, r, manifest); err != nil { + return err + } + // then we apply the mainifest + kubectl := kapi.KubectlBinaryPath(c.cc.KubernetesConfig.KubernetesVersion) + + apiServerManifest, err := c.apiServerManifest() + if err != nil { + return errors.Wrap(err, "apiserver manifest") + } + if err := r.Copy(apiServerManifest); err != nil { + return errors.Wrapf(err, "copy") + } + cmd := exec.Command("sudo", kubectl, "apply", fmt.Sprintf("--kubeconfig=%s", path.Join(vmpath.GuestPersistentDir, "kubeconfig")), "-f", apiServerMainifestPath()) + if rr, err := r.RunCmd(cmd); err != nil { + return errors.Wrapf(err, "cmd: %s output: %s", rr.Command(), rr.Output()) + } + + // according to https://docs.tigera.io/calico/3.26/operations/install-apiserver (Manifest Install) + // there is something more we need to do + // from now on, we won't return any error if any failure happens + // because executing the previous manifest.yaml is sufficient for calico cni plugin to work + // apiserver.yaml is just for managing the calico, it affect cni plugin's duty if the api server is down + cmd = exec.Command("openssl", "req", "-x509", "-nodes", "-newkey", "rsa:4096", "-keyout", "apiserver.key", "-out", "apiserver.crt", "-days", "3650", "-subj", "/", "-addext", "subjectAltName = DNS:calico-api.calico-apiserver.svc") + if out, err := r.RunCmd(cmd); err != nil { + klog.Error("failed to generate certificate for calico api server, output:", out.Output(), "error: ", err) + return nil + } + + cmd = exec.Command("sudo", kubectl, "create", "secret", fmt.Sprintf("--kubeconfig=%s", path.Join(vmpath.GuestPersistentDir, "kubeconfig")), "-n", "calico-apiserver", "generic", "calico-apiserver-certs", "--from-file=apiserver.key", "--from-file=apiserver.crt") + if out, err := r.RunCmd(cmd); err != nil { + klog.Error("failed to generate secret for calico api server, output:", out.Output(), "error: ", err) + return nil + } + cmd = exec.Command("sudo", "/bin/bash", "-c", kubectl, "patch", "apiservice", "v3.projectcalico.org", fmt.Sprintf("--kubeconfig=%s", path.Join(vmpath.GuestPersistentDir, "kubeconfig")), "-p", + `kubectl patch apiservice v3.projectcalico.org -p \ + "{\"spec\": {\"caBundle\": \"$(kubectl get secret -n calico-apiserver calico-apiserver-certs -o go-template='{{ index .data "apiserver.crt" }}')\"}}"`, + ) + if out, err := r.RunCmd(cmd); err != nil { + klog.Error("failed to patch calico api server for calico api server, output:", out.Output(), "error: ", err) + return nil + } + return nil + } // CIDR returns the default CIDR used by this CNI @@ -94,3 +157,6 @@ func (c Calico) CIDR() string { // Calico docs specify 192.168.0.0/16 - but we do this for compatibility with other CNI's. return DefaultPodCIDR } +func apiServerMainifestPath() string { + return path.Join(vmpath.GuestEphemeralDir, "calico-apiserver.yaml") +} From 52b38aebaaa7d0faa37054c2c3415cbc0db7ecd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=8C=A6=E5=8D=97=E8=B7=AF=E4=B9=8B=E8=8A=B1?= <46831212+ComradeProgrammer@users.noreply.github.com> Date: Wed, 2 Oct 2024 15:58:47 +0200 Subject: [PATCH 2/2] Apply suggestions from code review Co-authored-by: Steven Powell <44844360+spowelljr@users.noreply.github.com> --- hack/update/calico_version/update_calico_version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hack/update/calico_version/update_calico_version.go b/hack/update/calico_version/update_calico_version.go index c620c554a277..0780cd577a0b 100644 --- a/hack/update/calico_version/update_calico_version.go +++ b/hack/update/calico_version/update_calico_version.go @@ -58,8 +58,8 @@ func main() { } func updateYAML(version string) { - // for calico we are going to install both calico and calico api server via mainifest - // first we update the calico itself + // for Calico we are going to update both Calico and Calico API server mainifests + // first we update the Calico manifest res, err := http.Get(fmt.Sprintf("https://raw.githubusercontent.com/projectcalico/calico/%s/manifests/calico.yaml", version)) if err != nil { klog.Fatalf("failed to get calico.yaml: %v", err) @@ -81,11 +81,11 @@ func updateYAML(version string) { if err := os.WriteFile("../../../pkg/minikube/cni/calico.yaml", yaml, 0644); err != nil { klog.Fatalf("failed to write to YAML file: %v", err) } - // the second step is the calico api server + // then we update the Calico API server manifest // doc: https://docs.tigera.io/calico/latest/operations/install-apiserver resAPIServer, err := http.Get(fmt.Sprintf("https://raw.githubusercontent.com/projectcalico/calico/%s/manifests/apiserver.yaml", version)) if err != nil { - klog.Fatalf("failed to get calico.yaml: %v", err) + klog.Fatalf("failed to get apiserver.yaml: %v", err) } defer resAPIServer.Body.Close() yamlAPIServer, err := io.ReadAll(resAPIServer.Body)