From ee5fa9f532522060429d3160f8b31688a77eec25 Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Mon, 22 Apr 2019 15:41:46 +0300 Subject: [PATCH 1/4] Deploy Hetzner CCM when cloud-provider=external Signed-off-by: Artiom Diomin --- .golangci.yml | 1 + pkg/config/cluster.go | 49 +++-- pkg/installer/installation/install.go | 6 +- pkg/templates/externalccm/ccm.go | 35 ++++ pkg/templates/externalccm/hetzner.go | 220 ++++++++++++++++++++++ pkg/templates/machinecontroller/helper.go | 4 +- pkg/upgrader/upgrade/upgrade.go | 4 +- 7 files changed, 298 insertions(+), 21 deletions(-) create mode 100644 pkg/templates/externalccm/ccm.go create mode 100644 pkg/templates/externalccm/hetzner.go diff --git a/.golangci.yml b/.golangci.yml index de3bebe9a..c38475a33 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -26,3 +26,4 @@ issues: - "don't use underscores in Go names; func SetDefaults_ClusterNetwork should be SetDefaultsClusterNetwork" - "don't use underscores in Go names; func SetDefaults_Features should be SetDefaultsFeatures" - "don't use underscores in Go names; func SetDefaults_MachineController should be SetDefaultsMachineController" + - "G101: Potential hardcoded credentials" diff --git a/pkg/config/cluster.go b/pkg/config/cluster.go index d8e8bc265..9871b8db8 100644 --- a/pkg/config/cluster.go +++ b/pkg/config/cluster.go @@ -27,6 +27,23 @@ import ( "github.com/pkg/errors" ) +// ENV variable names with credential in them that machine-controller expects to see +const ( + AWSAccessKeyID = "AWS_ACCESS_KEY_ID" + AWSSecretAccessKey = "AWS_SECRET_ACCESS_KEY" + DigitalOceanTokenKey = "DO_TOKEN" + GoogleServiceAccountKey = "GOOGLE_SERVICE_ACCOUNT" + HetznerTokenKey = "HZ_TOKEN" + OpenStackAuthURL = "OS_AUTH_URL" + OpenStackDomainName = "OS_DOMAIN_NAME" + OpenStackPassword = "OS_PASSWORD" + OpenStackTenantName = "OS_TENANT_NAME" + OpenStackUserName = "OS_USER_NAME" + VSphereAddress = "VSPHERE_ADDRESS" + VSpherePasswords = "VSPHERE_PASSWORD" + VSphereUsername = "VSPHERE_USERNAME" +) + // Cluster describes our entire configuration. type Cluster struct { Name string `json:"name"` @@ -462,8 +479,8 @@ func (p ProviderName) ProviderCredentials() (map[string]string, error) { return nil, err } if envCreds.AccessKeyID != "" && envCreds.SecretAccessKey != "" { - creds["AWS_ACCESS_KEY_ID"] = envCreds.AccessKeyID - creds["AWS_SECRET_ACCESS_KEY"] = envCreds.SecretAccessKey + creds[AWSAccessKeyID] = envCreds.AccessKeyID + creds[AWSSecretAccessKey] = envCreds.SecretAccessKey return creds, nil } @@ -474,44 +491,44 @@ func (p ProviderName) ProviderCredentials() (map[string]string, error) { return nil, err } if configCreds.AccessKeyID != "" && configCreds.SecretAccessKey != "" { - creds["AWS_ACCESS_KEY_ID"] = configCreds.AccessKeyID - creds["AWS_SECRET_ACCESS_KEY"] = configCreds.SecretAccessKey + creds[AWSAccessKeyID] = configCreds.AccessKeyID + creds[AWSSecretAccessKey] = configCreds.SecretAccessKey return creds, nil } return nil, errors.New("error parsing aws credentials") case ProviderNameOpenStack: return parseCredentialVariables([]ProviderEnvironmentVariable{ - {Name: "OS_AUTH_URL"}, - {Name: "OS_USERNAME", MachineControllerName: "OS_USER_NAME"}, - {Name: "OS_PASSWORD"}, - {Name: "OS_DOMAIN_NAME"}, - {Name: "OS_TENANT_NAME"}, + {Name: OpenStackAuthURL}, + {Name: "OS_USERNAME", MachineControllerName: OpenStackUserName}, + {Name: OpenStackPassword}, + {Name: OpenStackDomainName}, + {Name: OpenStackTenantName}, }) case ProviderNameHetzner: return parseCredentialVariables([]ProviderEnvironmentVariable{ - {Name: "HCLOUD_TOKEN", MachineControllerName: "HZ_TOKEN"}, + {Name: "HCLOUD_TOKEN", MachineControllerName: HetznerTokenKey}, }) case ProviderNameDigitalOcean: return parseCredentialVariables([]ProviderEnvironmentVariable{ - {Name: "DIGITALOCEAN_TOKEN", MachineControllerName: "DO_TOKEN"}, + {Name: "DIGITALOCEAN_TOKEN", MachineControllerName: DigitalOceanTokenKey}, }) case ProviderNameGCE: gsa, err := parseCredentialVariables([]ProviderEnvironmentVariable{ - {Name: "GOOGLE_CREDENTIALS", MachineControllerName: "GOOGLE_SERVICE_ACCOUNT"}, + {Name: "GOOGLE_CREDENTIALS", MachineControllerName: GoogleServiceAccountKey}, }) if err != nil { return nil, errors.WithStack(err) } // encode it before sending to secret to be consumed by // machine-controller, as machine-controller assumes it will be double encoded - gsa["GOOGLE_SERVICE_ACCOUNT"] = base64.StdEncoding.EncodeToString([]byte(gsa["GOOGLE_SERVICE_ACCOUNT"])) + gsa[GoogleServiceAccountKey] = base64.StdEncoding.EncodeToString([]byte(gsa[GoogleServiceAccountKey])) return gsa, nil case ProviderNameVSphere: return parseCredentialVariables([]ProviderEnvironmentVariable{ - {Name: "VSPHERE_ADDRESS"}, - {Name: "VSPHERE_USERNAME"}, - {Name: "VSPHERE_PASSWORD"}, + {Name: VSphereAddress}, + {Name: VSphereUsername}, + {Name: VSpherePasswords}, }) } diff --git a/pkg/installer/installation/install.go b/pkg/installer/installation/install.go index e33429e48..f59ee9d63 100644 --- a/pkg/installer/installation/install.go +++ b/pkg/installer/installation/install.go @@ -22,6 +22,7 @@ import ( "github.com/kubermatic/kubeone/pkg/certificate" "github.com/kubermatic/kubeone/pkg/features" "github.com/kubermatic/kubeone/pkg/task" + "github.com/kubermatic/kubeone/pkg/templates/externalccm" "github.com/kubermatic/kubeone/pkg/templates/machinecontroller" "github.com/kubermatic/kubeone/pkg/util" ) @@ -42,9 +43,10 @@ func Install(ctx *util.Context) error { {Fn: saveKubeconfig, ErrMsg: "unable to save kubeconfig to the local machine", Retries: 3}, {Fn: util.BuildKubernetesClientset, ErrMsg: "unable to build kubernetes clientset", Retries: 3}, {Fn: features.Activate, ErrMsg: "unable to activate features"}, - {Fn: applyCanalCNI, ErrMsg: "failed to install cni plugin canal", Retries: 3}, + {Fn: externalccm.Ensure, ErrMsg: "failed to install external CCM"}, {Fn: patchCoreDNS, ErrMsg: "failed to patch CoreDNS", Retries: 3}, - {Fn: machinecontroller.EnsureMachineController, ErrMsg: "failed to install machine-controller", Retries: 3}, + {Fn: applyCanalCNI, ErrMsg: "failed to install cni plugin canal", Retries: 3}, + {Fn: machinecontroller.Ensure, ErrMsg: "failed to install machine-controller", Retries: 3}, {Fn: machinecontroller.WaitReady, ErrMsg: "failed to wait for machine-controller", Retries: 3}, {Fn: createWorkerMachines, ErrMsg: "failed to create worker machines", Retries: 3}, } diff --git a/pkg/templates/externalccm/ccm.go b/pkg/templates/externalccm/ccm.go new file mode 100644 index 000000000..ab6393aaa --- /dev/null +++ b/pkg/templates/externalccm/ccm.go @@ -0,0 +1,35 @@ +package externalccm + +import ( + "context" + + "github.com/kubermatic/kubeone/pkg/config" + "github.com/kubermatic/kubeone/pkg/util" + + "k8s.io/apimachinery/pkg/runtime" + dynclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +// Ensure external CCM when +func Ensure(ctx *util.Context) error { + if !ctx.Cluster.Provider.External { + return nil + } + + ctx.Logger.Info("Ensure external CCM is up to date") + + switch ctx.Cluster.Provider.Name { + case config.ProviderNameHetzner: + return ensureHetzner(ctx) + default: + ctx.Logger.Infof("External CCM for %q not yet supported, skipping", ctx.Cluster.Provider.Name) + return nil + } +} + +func simpleCreateOrUpdate(ctx context.Context, client dynclient.Client, obj runtime.Object) error { + okFunc := func(runtime.Object) error { return nil } + _, err := controllerutil.CreateOrUpdate(ctx, client, obj, okFunc) + return err +} diff --git a/pkg/templates/externalccm/hetzner.go b/pkg/templates/externalccm/hetzner.go new file mode 100644 index 000000000..4e3bce188 --- /dev/null +++ b/pkg/templates/externalccm/hetzner.go @@ -0,0 +1,220 @@ +package externalccm + +import ( + "context" + "strings" + + "github.com/Masterminds/semver" + "github.com/pkg/errors" + + "github.com/kubermatic/kubeone/pkg/config" + "github.com/kubermatic/kubeone/pkg/templates/machinecontroller" + "github.com/kubermatic/kubeone/pkg/util" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +const ( + hetznerCCMVersion = "v1.3.0" + hetznerSAName = "cloud-controller-manager" + hetznerDeploymentName = "hcloud-cloud-controller-manager" +) + +func ensureHetzner(ctx *util.Context) error { + if ctx.DynamicClient == nil { + return errors.New("kubernetes client not initialized") + } + + bgctx := context.Background() + + sa := hetznerServiceAccount() + if err := simpleCreateOrUpdate(bgctx, ctx.DynamicClient, sa); err != nil { + return errors.Wrap(err, "failed to ensure hetzner CCM ServiceAccount") + } + + crb := hetznerClusterRoleBinding() + if err := simpleCreateOrUpdate(bgctx, ctx.DynamicClient, crb); err != nil { + return errors.Wrap(err, "failed to ensure hetzner CCM ClusterRoleBinding") + } + + dep := hetznerDeployment() + _, err := controllerutil.CreateOrUpdate(bgctx, ctx.DynamicClient, dep, func(runtime.Object) error { + if dep.ObjectMeta.CreationTimestamp.IsZero() { + // let it create deployment + return nil + } + + if len(dep.Spec.Template.Spec.Containers) != 1 { + return errors.New("unable to choose a CCM container, as number of containers > 1") + } + + want, err := semver.NewConstraint("<= " + hetznerCCMVersion) + if err != nil { + return errors.Wrap(err, "failed to parse hetzner CCM version constraint") + } + + imageSpec := strings.SplitN(dep.Spec.Template.Spec.Containers[0].Image, ":", 2) + if len(imageSpec) != 2 { + return errors.New("unable to greb hetzner CCM image version") + } + + existing, err := semver.NewVersion(imageSpec[1]) + if err != nil { + return errors.Wrap(err, "failed to parse deployed hetzner CCM version") + } + + if !want.Check(existing) { + return errors.New("newer version deployed, skipping") + } + + // let it update deployment + return nil + }) + + if err != nil { + ctx.Logger.Warnf("unable to ensure hetzner CCM Deployment: %v, skipping", err) + } + + return nil +} + +func hetznerServiceAccount() *corev1.ServiceAccount { + return &corev1.ServiceAccount{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ServiceAccount", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: hetznerSAName, + Namespace: metav1.NamespaceSystem, + }, + } +} + +func hetznerClusterRoleBinding() *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "rbac.authorization.k8s.io/v1", + Kind: "ClusterRoleBinding", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "system:cloud-controller-manager", + }, + RoleRef: rbacv1.RoleRef{ + Name: "cluster-admin", + Kind: "ClusterRole", + APIGroup: rbacv1.GroupName, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: hetznerSAName, + Namespace: metav1.NamespaceSystem, + }, + }, + } +} + +func hetznerDeployment() *appsv1.Deployment { + var ( + replicas int32 = 1 + revisions int32 = 2 + ) + + return &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "Deployment", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: hetznerDeploymentName, + Namespace: metav1.NamespaceSystem, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + RevisionHistoryLimit: &revisions, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "hcloud-cloud-controller-manager", + }, + }, + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RecreateDeploymentStrategyType, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "scheduler.alpha.kubernetes.io/critical-pod": "", + }, + Labels: map[string]string{ + "app": "hcloud-cloud-controller-manager", + }, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: hetznerSAName, + Tolerations: []corev1.Toleration{ + { + Key: "node-role.kubernetes.io/master", + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + }, + { + Key: "node.cloudprovider.kubernetes.io/uninitialized", + Value: "true", + Effect: corev1.TaintEffectNoSchedule, + }, + { + Key: "CriticalAddonsOnly", + Operator: corev1.TolerationOpExists, + }, + }, + Containers: []corev1.Container{ + { + Name: "hcloud-cloud-controller-manager", + Image: "hetznercloud/hcloud-cloud-controller-manager:" + hetznerCCMVersion, + Command: []string{ + "/bin/hcloud-cloud-controller-manager", + "--cloud-provider=hcloud", + "--leader-elect=false", + "--allow-untagged-cloud", + }, + Env: []corev1.EnvVar{ + { + Name: "NODE_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "spec.nodeName", + }, + }, + }, + { + Name: "HCLOUD_TOKEN", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: machinecontroller.MachineControllerCredentialsSecretName, + }, + Key: config.HetznerTokenKey, + }, + }, + }, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("50Mi"), + }, + }, + }, + }, + }, + }, + }, + } +} diff --git a/pkg/templates/machinecontroller/helper.go b/pkg/templates/machinecontroller/helper.go index 3728fd885..b2c675fc1 100644 --- a/pkg/templates/machinecontroller/helper.go +++ b/pkg/templates/machinecontroller/helper.go @@ -35,8 +35,8 @@ func simpleCreateOrUpdate(ctx context.Context, client dynclient.Client, obj runt return err } -// EnsureMachineController install/update machine-controller -func EnsureMachineController(ctx *util.Context) error { +// Ensure install/update machine-controller +func Ensure(ctx *util.Context) error { if !*ctx.Cluster.MachineController.Deploy { ctx.Logger.Info("Skipping machine-controller deployment because it was disabled in configuration.") return nil diff --git a/pkg/upgrader/upgrade/upgrade.go b/pkg/upgrader/upgrade/upgrade.go index 6e7af55d6..d22c5aac4 100644 --- a/pkg/upgrader/upgrade/upgrade.go +++ b/pkg/upgrader/upgrade/upgrade.go @@ -24,6 +24,7 @@ import ( "github.com/kubermatic/kubeone/pkg/certificate" "github.com/kubermatic/kubeone/pkg/features" "github.com/kubermatic/kubeone/pkg/task" + "github.com/kubermatic/kubeone/pkg/templates/externalccm" "github.com/kubermatic/kubeone/pkg/templates/machinecontroller" "github.com/kubermatic/kubeone/pkg/util" ) @@ -52,7 +53,8 @@ func Upgrade(ctx *util.Context) error { {Fn: upgradeFollower, ErrMsg: "unable to upgrade follower control plane", Retries: 3}, {Fn: features.Activate, ErrMsg: "unable to activate features"}, {Fn: certificate.DownloadCA, ErrMsg: "unable to download ca from leader", Retries: 3}, - {Fn: machinecontroller.EnsureMachineController, ErrMsg: "failed to update machine-controller", Retries: 3}, + {Fn: externalccm.Ensure, ErrMsg: "failed to install external CCM"}, + {Fn: machinecontroller.Ensure, ErrMsg: "failed to update machine-controller", Retries: 3}, {Fn: machinecontroller.WaitReady, ErrMsg: "failed to wait for machine-controller", Retries: 3}, {Fn: upgradeMachineDeployments, ErrMsg: "unable to upgrade MachineDeployments", Retries: 3}, } From cc27e1b5f7b0086ee726a518034a401fb308c5eb Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Mon, 22 Apr 2019 17:36:15 +0300 Subject: [PATCH 2/4] Deploy DigitalOcean CCM when cloud-provider=external Signed-off-by: Artiom Diomin --- pkg/templates/externalccm/ccm.go | 44 +++- pkg/templates/externalccm/digitalocean.go | 246 ++++++++++++++++++++++ pkg/templates/externalccm/hetzner.go | 42 +--- 3 files changed, 297 insertions(+), 35 deletions(-) create mode 100644 pkg/templates/externalccm/digitalocean.go diff --git a/pkg/templates/externalccm/ccm.go b/pkg/templates/externalccm/ccm.go index ab6393aaa..75b9d2029 100644 --- a/pkg/templates/externalccm/ccm.go +++ b/pkg/templates/externalccm/ccm.go @@ -2,16 +2,21 @@ package externalccm import ( "context" + "strings" + + "github.com/Masterminds/semver" + "github.com/pkg/errors" "github.com/kubermatic/kubeone/pkg/config" "github.com/kubermatic/kubeone/pkg/util" + appsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/runtime" dynclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) -// Ensure external CCM when +// Ensure external CCM deployen if Provider.External func Ensure(ctx *util.Context) error { if !ctx.Cluster.Provider.External { return nil @@ -22,6 +27,8 @@ func Ensure(ctx *util.Context) error { switch ctx.Cluster.Provider.Name { case config.ProviderNameHetzner: return ensureHetzner(ctx) + case config.ProviderNameDigitalOcean: + return ensureDigitalOcean(ctx) default: ctx.Logger.Infof("External CCM for %q not yet supported, skipping", ctx.Cluster.Provider.Name) return nil @@ -33,3 +40,38 @@ func simpleCreateOrUpdate(ctx context.Context, client dynclient.Client, obj runt _, err := controllerutil.CreateOrUpdate(ctx, client, obj, okFunc) return err } + +func mutateDeploymentWithVersionCheck(want *semver.Constraints) func(obj runtime.Object) error { + return func(obj runtime.Object) error { + dep, ok := obj.(*appsv1.Deployment) + if !ok { + return errors.Errorf("unknown object type %T passed", obj) + } + + if dep.ObjectMeta.CreationTimestamp.IsZero() { + // let it create deployment + return nil + } + + if len(dep.Spec.Template.Spec.Containers) != 1 { + return errors.New("unable to choose a CCM container, as number of containers > 1") + } + + imageSpec := strings.SplitN(dep.Spec.Template.Spec.Containers[0].Image, ":", 2) + if len(imageSpec) != 2 { + return errors.New("unable to grab CCM image version") + } + + existing, err := semver.NewVersion(imageSpec[1]) + if err != nil { + return errors.Wrap(err, "failed to parse deployed CCM version") + } + + if !want.Check(existing) { + return errors.New("newer version deployed, skipping") + } + + // OK to update the deployment + return nil + } +} diff --git a/pkg/templates/externalccm/digitalocean.go b/pkg/templates/externalccm/digitalocean.go new file mode 100644 index 000000000..39c614f32 --- /dev/null +++ b/pkg/templates/externalccm/digitalocean.go @@ -0,0 +1,246 @@ +package externalccm + +import ( + "context" + + "github.com/Masterminds/semver" + "github.com/pkg/errors" + + "github.com/kubermatic/kubeone/pkg/config" + "github.com/kubermatic/kubeone/pkg/templates/machinecontroller" + "github.com/kubermatic/kubeone/pkg/util" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +const ( + digitaloceanCCMVersion = "v0.1.9" + digitaloceanSAName = "cloud-controller-manager" + digitaloceanDeploymentName = "digitalocean-cloud-controller-manager" +) + +func ensureDigitalOcean(ctx *util.Context) error { + if ctx.DynamicClient == nil { + return errors.New("kubernetes client not initialized") + } + + bgctx := context.Background() + + sa := doServiceAccount() + if err := simpleCreateOrUpdate(bgctx, ctx.DynamicClient, sa); err != nil { + return errors.Wrap(err, "failed to ensure digitalocean CCM ServiceAccount") + } + + cr := doClusterRole() + if err := simpleCreateOrUpdate(bgctx, ctx.DynamicClient, cr); err != nil { + return errors.Wrap(err, "failed to ensure digitalocean CCM ClusterRole") + } + + crb := doClusterRoleBinding() + if err := simpleCreateOrUpdate(bgctx, ctx.DynamicClient, crb); err != nil { + return errors.Wrap(err, "failed to ensure digitalocean CCM ClusterRoleBinding") + } + + dep := doDeployment() + want, err := semver.NewConstraint("<= " + digitaloceanCCMVersion) + if err != nil { + return errors.Wrap(err, "failed to parse digital CCM version constraint") + } + + _, err = controllerutil.CreateOrUpdate(bgctx, + ctx.DynamicClient, + dep, + mutateDeploymentWithVersionCheck(want)) + if err != nil { + ctx.Logger.Warnf("unable to ensure digitalocean CCM Deployment: %v, skipping", err) + } + return nil +} + +func doServiceAccount() *corev1.ServiceAccount { + return &corev1.ServiceAccount{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ServiceAccount", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: digitaloceanSAName, + Namespace: metav1.NamespaceSystem, + }, + } +} + +func doClusterRole() *rbacv1.ClusterRole { + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "rbac.authorization.k8s.io/v1", + Kind: "ClusterRole", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "system:cloud-controller-manager", + Annotations: map[string]string{ + "rbac.authorization.kubernetes.io/autoupdate": "true", + }, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"events"}, + Verbs: []string{"create", "patch", "update"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"nodes"}, + Verbs: []string{"*"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"nodes/status"}, + Verbs: []string{"patch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"services"}, + Verbs: []string{"list", "patch", "update", "watch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"services/status"}, + Verbs: []string{"list", "patch", "update", "watch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"serviceaccounts"}, + Verbs: []string{"create"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"persistentvolumes"}, + Verbs: []string{"get", "list", "update", "watch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"endpoints"}, + Verbs: []string{"create", "get", "list", "watch", "update"}, + }, + }, + } +} + +func doClusterRoleBinding() *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "rbac.authorization.k8s.io/v1", + Kind: "ClusterRoleBinding", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "system:cloud-controller-manager", + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Name: "system:cloud-controller-manager", + Kind: "ClusterRole", + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: digitaloceanSAName, + Namespace: metav1.NamespaceSystem, + }, + }, + } +} + +func doDeployment() *appsv1.Deployment { + var ( + replicas int32 = 1 + revisions int32 = 2 + ) + + return &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "Deployment", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: digitaloceanDeploymentName, + Namespace: metav1.NamespaceSystem, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + RevisionHistoryLimit: &revisions, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "digitalocean-cloud-controller-manager", + }, + }, + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RecreateDeploymentStrategyType, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "scheduler.alpha.kubernetes.io/critical-pod": "", + }, + Labels: map[string]string{ + "app": "digitalocean-cloud-controller-manager", + }, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: digitaloceanSAName, + Tolerations: []corev1.Toleration{ + { + Key: "node-role.kubernetes.io/master", + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + }, + { + Key: "node.cloudprovider.kubernetes.io/uninitialized", + Value: "true", + Effect: corev1.TaintEffectNoSchedule, + }, + { + Key: "CriticalAddonsOnly", + Operator: corev1.TolerationOpExists, + }, + }, + Containers: []corev1.Container{ + { + Name: "digitalocean-cloud-controller-manager", + Image: "digitalocean/digitalocean-cloud-controller-manager" + digitaloceanCCMVersion, + Command: []string{ + "/bin/digitalocean-cloud-controller-manager", + "--cloud-provider=digitalocean", + "--leader-elect=false", + }, + Env: []corev1.EnvVar{ + { + Name: "DO_ACCESS_TOKEN", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: machinecontroller.MachineControllerCredentialsSecretName, + }, + Key: config.DigitalOceanTokenKey, + }, + }, + }, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("50Mi"), + }, + }, + }, + }, + }, + }, + }, + } +} diff --git a/pkg/templates/externalccm/hetzner.go b/pkg/templates/externalccm/hetzner.go index 4e3bce188..b7a049971 100644 --- a/pkg/templates/externalccm/hetzner.go +++ b/pkg/templates/externalccm/hetzner.go @@ -2,7 +2,6 @@ package externalccm import ( "context" - "strings" "github.com/Masterminds/semver" "github.com/pkg/errors" @@ -16,7 +15,6 @@ import ( rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) @@ -44,39 +42,15 @@ func ensureHetzner(ctx *util.Context) error { } dep := hetznerDeployment() - _, err := controllerutil.CreateOrUpdate(bgctx, ctx.DynamicClient, dep, func(runtime.Object) error { - if dep.ObjectMeta.CreationTimestamp.IsZero() { - // let it create deployment - return nil - } - - if len(dep.Spec.Template.Spec.Containers) != 1 { - return errors.New("unable to choose a CCM container, as number of containers > 1") - } - - want, err := semver.NewConstraint("<= " + hetznerCCMVersion) - if err != nil { - return errors.Wrap(err, "failed to parse hetzner CCM version constraint") - } - - imageSpec := strings.SplitN(dep.Spec.Template.Spec.Containers[0].Image, ":", 2) - if len(imageSpec) != 2 { - return errors.New("unable to greb hetzner CCM image version") - } - - existing, err := semver.NewVersion(imageSpec[1]) - if err != nil { - return errors.Wrap(err, "failed to parse deployed hetzner CCM version") - } - - if !want.Check(existing) { - return errors.New("newer version deployed, skipping") - } - - // let it update deployment - return nil - }) + want, err := semver.NewConstraint("<= " + hetznerCCMVersion) + if err != nil { + return errors.Wrap(err, "failed to parse digital CCM version constraint") + } + _, err = controllerutil.CreateOrUpdate(bgctx, + ctx.DynamicClient, + dep, + mutateDeploymentWithVersionCheck(want)) if err != nil { ctx.Logger.Warnf("unable to ensure hetzner CCM Deployment: %v, skipping", err) } From 38b1d76a0af9d06b3eb33562ee5a0d8ccebd675a Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Mon, 22 Apr 2019 17:44:46 +0300 Subject: [PATCH 3/4] Add forgotten boilerplate headers Signed-off-by: Artiom Diomin --- pkg/templates/externalccm/ccm.go | 16 ++++++++++++++++ pkg/templates/externalccm/digitalocean.go | 16 ++++++++++++++++ pkg/templates/externalccm/hetzner.go | 16 ++++++++++++++++ 3 files changed, 48 insertions(+) diff --git a/pkg/templates/externalccm/ccm.go b/pkg/templates/externalccm/ccm.go index 75b9d2029..bf5b3b9c9 100644 --- a/pkg/templates/externalccm/ccm.go +++ b/pkg/templates/externalccm/ccm.go @@ -1,3 +1,19 @@ +/* +Copyright 2019 The KubeOne Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package externalccm import ( diff --git a/pkg/templates/externalccm/digitalocean.go b/pkg/templates/externalccm/digitalocean.go index 39c614f32..731a6c6ad 100644 --- a/pkg/templates/externalccm/digitalocean.go +++ b/pkg/templates/externalccm/digitalocean.go @@ -1,3 +1,19 @@ +/* +Copyright 2019 The KubeOne Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package externalccm import ( diff --git a/pkg/templates/externalccm/hetzner.go b/pkg/templates/externalccm/hetzner.go index b7a049971..23766eb6d 100644 --- a/pkg/templates/externalccm/hetzner.go +++ b/pkg/templates/externalccm/hetzner.go @@ -1,3 +1,19 @@ +/* +Copyright 2019 The KubeOne Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package externalccm import ( From 5be2895f755c1a7c93f3dfb04bb5184adbd86070 Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Mon, 22 Apr 2019 23:20:11 +0300 Subject: [PATCH 4/4] fix DO image tag Signed-off-by: Artiom Diomin --- pkg/templates/externalccm/digitalocean.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/templates/externalccm/digitalocean.go b/pkg/templates/externalccm/digitalocean.go index 731a6c6ad..3079c0ab9 100644 --- a/pkg/templates/externalccm/digitalocean.go +++ b/pkg/templates/externalccm/digitalocean.go @@ -228,7 +228,7 @@ func doDeployment() *appsv1.Deployment { Containers: []corev1.Container{ { Name: "digitalocean-cloud-controller-manager", - Image: "digitalocean/digitalocean-cloud-controller-manager" + digitaloceanCCMVersion, + Image: "digitalocean/digitalocean-cloud-controller-manager:" + digitaloceanCCMVersion, Command: []string{ "/bin/digitalocean-cloud-controller-manager", "--cloud-provider=digitalocean",