From 1b78c47f0136659b1dc1ba3b9886a81adc65521b Mon Sep 17 00:00:00 2001 From: Michal Fojtik Date: Fri, 5 Oct 2018 16:01:46 +0200 Subject: [PATCH 1/6] ignition: render bootstrap manifests using operator images In order to switch from kube-core-operator renderer we need to make sure that all manifests (bootstrap + phase 2) are provided from the openshift kube/openshift operators. All control plane operators from openshift implement a rendering logic that lay down secrets, configs and YAML definitions for workload resources as well as static pods for initial pivot. --- pkg/asset/ignition/bootstrap/bootstrap.go | 2 - .../ignition/bootstrap/content/bootkube.go | 43 ++++++------------- .../content/bootkube/cvo-overrides.go | 12 ------ 3 files changed, 14 insertions(+), 43 deletions(-) diff --git a/pkg/asset/ignition/bootstrap/bootstrap.go b/pkg/asset/ignition/bootstrap/bootstrap.go index 61cc881be1c..8e6c2227803 100644 --- a/pkg/asset/ignition/bootstrap/bootstrap.go +++ b/pkg/asset/ignition/bootstrap/bootstrap.go @@ -40,7 +40,6 @@ type bootstrapTemplateData struct { EtcdCluster string EtcdctlImage string HyperkubeImage string - KubeCoreRenderImage string ReleaseImage string } @@ -160,7 +159,6 @@ func (a *Bootstrap) getTemplateData(installConfig *types.InstallConfig) (*bootst CloudProvider: getCloudProvider(installConfig), CloudProviderConfig: getCloudProviderConfig(installConfig), DebugConfig: "", - KubeCoreRenderImage: "quay.io/coreos/kube-core-renderer-dev:375423a332f2c12b79438fc6a6da6e448e28ec0f", EtcdCertSignerImage: "quay.io/coreos/kube-etcd-signer-server:678cc8e6841e2121ebfdb6e2db568fce290b67d6", EtcdctlImage: "quay.io/coreos/etcd:v3.2.14", BootkubeImage: "quay.io/coreos/bootkube:v0.10.0", diff --git a/pkg/asset/ignition/bootstrap/content/bootkube.go b/pkg/asset/ignition/bootstrap/content/bootkube.go index 7a906eb4367..1282390f15c 100644 --- a/pkg/asset/ignition/bootstrap/content/bootkube.go +++ b/pkg/asset/ignition/bootstrap/content/bootkube.go @@ -31,7 +31,7 @@ var ( BootkubeShFileTemplate = template.Must(template.New("bootkube.sh").Parse(`#!/usr/bin/env bash set -e -mkdir --parents /etc/kubernetes/manifests/ +mkdir --parents /etc/kubernetes/{manifests,bootstrap-configs,bootstrap-manifests} MACHINE_CONFIG_OPERATOR_IMAGE=$(podman run --rm {{.ReleaseImage}} image machine-config-operator) MACHINE_CONFIG_CONTROLLER_IMAGE=$(podman run --rm {{.ReleaseImage}} image machine-config-controller) @@ -57,22 +57,7 @@ then cp --recursive cvo-bootstrap/manifests . fi -if [ ! -d kco-bootstrap ] -then - echo "Rendering Kubernetes core manifests..." - - # shellcheck disable=SC2154 - podman run \ - --volume "$PWD:/assets:z" \ - --volume /etc/kubernetes:/etc/kubernetes:z \ - "{{.KubeCoreRenderImage}}" \ - --config=/assets/kco-config.yaml \ - --output=/assets/kco-bootstrap - - cp --recursive kco-bootstrap/bootstrap-configs /etc/kubernetes/bootstrap-configs - cp --recursive kco-bootstrap/bootstrap-manifests . - cp --recursive kco-bootstrap/manifests . -fi +mkdir --parents ./{bootstrap-manifests,manifests} if [ ! -d kube-apiserver-bootstrap ] then @@ -84,14 +69,14 @@ then "${KUBE_APISERVER_OPERATOR_IMAGE}" \ /usr/bin/cluster-kube-apiserver-operator render \ --manifest-etcd-serving-ca=etcd-client-ca.crt \ + --manifest-etcd-server-urls={{.EtcdCluster}} \ --asset-input-dir=/assets/tls \ --asset-output-dir=/assets/kube-apiserver-bootstrap \ --config-output-file=/assets/kube-apiserver-bootstrap/config - # TODO: copy the bootstrap manifests to replace kube-core-operator - cp --recursive kube-apiserver-bootstrap/manifests/00_openshift-kube-apiserver-ns.yaml manifests/00_openshift-kube-apiserver-ns.yaml - cp --recursive kube-apiserver-bootstrap/manifests/secret-* manifests/ - cp --recursive kube-apiserver-bootstrap/manifests/configmap-* manifests/ + cp kube-apiserver-bootstrap/config /etc/kubernetes/bootstrap-configs/kube-apiserver-config.yaml + cp kube-apiserver-bootstrap/bootstrap-manifests/* bootstrap-manifests/ + cp kube-apiserver-bootstrap/manifests/* manifests/ fi if [ ! -d kube-controller-manager-bootstrap ] @@ -107,9 +92,9 @@ then --asset-output-dir=/assets/kube-controller-manager-bootstrap \ --config-output-file=/assets/kube-controller-manager-bootstrap/config - # TODO: copy the bootstrap manifests to replace kube-core-operator - cp --recursive kube-controller-manager-bootstrap/manifests/00_openshift-kube-controller-manager-ns.yaml manifests/00_openshift-kube-controller-manager-ns.yaml - cp --recursive kube-controller-manager-bootstrap/manifests/configmap-* manifests/ + cp kube-controller-manager-bootstrap/config /etc/kubernetes/bootstrap-configs/kube-controller-manager-config.yaml + cp kube-controller-manager-bootstrap/bootstrap-manifests/* bootstrap-manifests/ + cp kube-controller-manager-bootstrap/manifests/* manifests/ fi if [ ! -d kube-scheduler-bootstrap ] @@ -125,9 +110,9 @@ then --asset-output-dir=/assets/kube-scheduler-bootstrap \ --config-output-file=/assets/kube-scheduler-bootstrap/config - # TODO: copy the bootstrap manifests to replace kube-core-operator - cp --recursive kube-scheduler-bootstrap/manifests/00_openshift-kube-scheduler-ns.yaml manifests/00_openshift-kube-scheduler-ns.yaml - cp --recursive kube-scheduler-bootstrap/manifests/configmap-* manifests/ + cp kube-scheduler-bootstrap/config /etc/kubernetes/bootstrap-configs/kube-scheduler-config.yaml + cp kube-scheduler-bootstrap/bootstrap-manifests/* bootstrap-manifests/ + cp kube-scheduler-bootstrap/manifests/* manifests/ fi if [ ! -d mco-bootstrap ] @@ -152,8 +137,8 @@ then # 1. read the controller config rendered by MachineConfigOperator # 2. read the default MachineConfigPools rendered by MachineConfigOperator # 3. read any additional MachineConfigs that are needed for the default MachineConfigPools. - mkdir --parents /etc/mcc/bootstrap/ - cp --recursive mco-bootstrap/manifests /etc/mcc/bootstrap/manifests + mkdir --parents /etc/mcc/bootstrap/manifests /etc/kubernetes/manifests/ + cp mco-bootstrap/manifests/* /etc/mcc/bootstrap/manifests/ cp mco-bootstrap/machineconfigoperator-bootstrap-pod.yaml /etc/kubernetes/manifests/ # /etc/ssl/mcs/tls.{crt, key} are locations for MachineConfigServer's tls assets. diff --git a/pkg/asset/manifests/content/bootkube/cvo-overrides.go b/pkg/asset/manifests/content/bootkube/cvo-overrides.go index efcdff7e9b6..ed29b8b9389 100644 --- a/pkg/asset/manifests/content/bootkube/cvo-overrides.go +++ b/pkg/asset/manifests/content/bootkube/cvo-overrides.go @@ -19,18 +19,6 @@ upstream: http://localhost:8080/graph channel: fast clusterID: {{.CVOClusterID}} overrides: -- kind: Deployment # this conflicts with kube-core-operator - namespace: openshift-core-operators - name: openshift-cluster-kube-apiserver-operator - unmanaged: true -- kind: Deployment # this conflicts with kube-core-operator - namespace: openshift-core-operators - name: openshift-cluster-kube-scheduler-operator - unmanaged: true -- kind: Deployment # this conflicts with kube-core-operator - namespace: openshift-core-operators - name: openshift-cluster-kube-controller-manager-operator - unmanaged: true - kind: Deployment # this conflicts with kube-core-operator namespace: openshift-cluster-network-operator name: cluster-network-operator From c9b0e2ff454355700fb84582d33331cb45bf9246 Mon Sep 17 00:00:00 2001 From: Michal Fojtik Date: Mon, 8 Oct 2018 14:16:20 +0200 Subject: [PATCH 2/6] manifests: stop using kube core operator In order to switch to OpenShift operators and bootstrap we need to remove the kube-core-operator rendering and not rely on the assets provided by this operator. The new assets are provided by kube/openshift operator renderers. --- pkg/asset/ignition/bootstrap/bootstrap.go | 8 +- .../tectonic/kube-core-00-appversion.go | 21 --- .../content/tectonic/kube-core-00-operator.go | 65 --------- pkg/asset/manifests/kube-core-operator.go | 133 ------------------ pkg/asset/manifests/operators.go | 7 +- pkg/asset/manifests/tectonic.go | 2 - 6 files changed, 3 insertions(+), 233 deletions(-) delete mode 100644 pkg/asset/manifests/content/tectonic/kube-core-00-appversion.go delete mode 100644 pkg/asset/manifests/content/tectonic/kube-core-00-operator.go delete mode 100644 pkg/asset/manifests/kube-core-operator.go diff --git a/pkg/asset/ignition/bootstrap/bootstrap.go b/pkg/asset/ignition/bootstrap/bootstrap.go index 8e6c2227803..5373d9bc316 100644 --- a/pkg/asset/ignition/bootstrap/bootstrap.go +++ b/pkg/asset/ignition/bootstrap/bootstrap.go @@ -74,7 +74,6 @@ func (a *Bootstrap) Dependencies() []asset.Asset { &kubeconfig.Kubelet{}, &manifests.Manifests{}, &manifests.Tectonic{}, - &manifests.KubeCoreOperator{}, } } @@ -170,18 +169,13 @@ func (a *Bootstrap) getTemplateData(installConfig *types.InstallConfig) (*bootst func (a *Bootstrap) addBootstrapFiles(dependencies asset.Parents) { kubeletKubeconfig := &kubeconfig.Kubelet{} - kubeCoreOperator := &manifests.KubeCoreOperator{} - dependencies.Get(kubeletKubeconfig, kubeCoreOperator) + dependencies.Get(kubeletKubeconfig) a.Config.Storage.Files = append( a.Config.Storage.Files, ignition.FileFromBytes("/etc/kubernetes/kubeconfig", 0600, kubeletKubeconfig.Files()[0].Data), ignition.FileFromBytes("/var/lib/kubelet/kubeconfig", 0600, kubeletKubeconfig.Files()[0].Data), ) - a.Config.Storage.Files = append( - a.Config.Storage.Files, - ignition.FilesFromAsset(rootDir, 0644, kubeCoreOperator)..., - ) a.Config.Storage.Files = append( a.Config.Storage.Files, ignition.FileFromString("/opt/tectonic/report-progress.sh", 0555, content.ReportShFileContents), diff --git a/pkg/asset/manifests/content/tectonic/kube-core-00-appversion.go b/pkg/asset/manifests/content/tectonic/kube-core-00-appversion.go deleted file mode 100644 index a67ba4d09ae..00000000000 --- a/pkg/asset/manifests/content/tectonic/kube-core-00-appversion.go +++ /dev/null @@ -1,21 +0,0 @@ -package tectonic - -const ( - // AppVersionKubeCore is the variable/constant representing the contents of the respective file - AppVersionKubeCore = ` ---- -apiVersion: tco.coreos.com/v1 -kind: AppVersion -metadata: - name: kube-core - namespace: tectonic-system - labels: - managed-by-channel-operator: "true" -spec: - paused: false -status: - paused: false -upgradereq: 0 -upgradecomp: 0 -` -) diff --git a/pkg/asset/manifests/content/tectonic/kube-core-00-operator.go b/pkg/asset/manifests/content/tectonic/kube-core-00-operator.go deleted file mode 100644 index 909177805d6..00000000000 --- a/pkg/asset/manifests/content/tectonic/kube-core-00-operator.go +++ /dev/null @@ -1,65 +0,0 @@ -package tectonic - -import ( - "text/template" -) - -var ( - // KubeCoreOperator is the variable/constant representing the contents of the respective file - KubeCoreOperator = template.Must(template.New("kube-core-00-operator.yaml").Parse(` -apiVersion: apps/v1beta2 -kind: Deployment -metadata: - name: kube-core-operator - namespace: kube-system - labels: - k8s-app: kube-core-operator - managed-by-channel-operator: "true" -spec: - replicas: 1 - selector: - matchLabels: - k8s-app: kube-core-operator - template: - metadata: - labels: - k8s-app: kube-core-operator - tectonic-app-version-name: kube-core - spec: - containers: - - name: kube-core-operator - image: {{.KubeCoreOperatorImage}} - imagePullPolicy: Always - args: - - --config=/etc/cluster-config/kco-config.yaml - resources: - limits: - cpu: 20m - memory: 50Mi - requests: - cpu: 20m - memory: 50Mi - volumeMounts: - - name: cluster-config - mountPath: /etc/cluster-config - imagePullSecrets: - - name: coreos-pull-secret - nodeSelector: - node-role.kubernetes.io/master: "" - restartPolicy: Always - securityContext: - runAsNonRoot: true - runAsUser: 65534 - tolerations: - - key: "node-role.kubernetes.io/master" - operator: "Exists" - effect: "NoSchedule" - volumes: - - name: cluster-config - configMap: - name: cluster-config-v1 - items: - - key: kco-config - path: kco-config.yaml -`)) -) diff --git a/pkg/asset/manifests/kube-core-operator.go b/pkg/asset/manifests/kube-core-operator.go deleted file mode 100644 index 3b384b2c8d4..00000000000 --- a/pkg/asset/manifests/kube-core-operator.go +++ /dev/null @@ -1,133 +0,0 @@ -package manifests - -import ( - "fmt" - "strings" - - "github.com/apparentlymart/go-cidr/cidr" - kubecore "github.com/coreos/tectonic-config/config/kube-core" - "github.com/ghodss/yaml" - "github.com/pkg/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/openshift/installer/pkg/asset" - "github.com/openshift/installer/pkg/asset/installconfig" - "github.com/openshift/installer/pkg/types" -) - -const ( - authConfigOIDCClientID = "tectonic-kubectl" - authConfigOIDCGroupsClaim = "groups" - authConfigOIDCUsernameClaim = "email" - networkConfigAdvertiseAddress = "0.0.0.0" -) - -// KubeCoreOperator generates the kube-core-operator.yaml files -type KubeCoreOperator struct { - Config *kubecore.OperatorConfig - File *asset.File -} - -var _ asset.WritableAsset = (*KubeCoreOperator)(nil) - -// Name returns a human friendly name for the operator -func (kco *KubeCoreOperator) Name() string { - return "Kube Core Operator" -} - -// Dependencies returns all of the dependencies directly needed by an -// KubeCoreOperator asset. -func (kco *KubeCoreOperator) Dependencies() []asset.Asset { - return []asset.Asset{ - &installconfig.InstallConfig{}, - } -} - -// Generate generates the kube-core-operator-config.yml files -func (kco *KubeCoreOperator) Generate(dependencies asset.Parents) error { - installConfig := &installconfig.InstallConfig{} - dependencies.Get(installConfig) - - clusterIP, err := cidr.Host(&installConfig.Config.Networking.ServiceCIDR.IPNet, 10) - if err != nil { - return errors.Wrapf(err, "failed to create %s config from InstallConfig", kco.Name()) - } - - kco.Config = &kubecore.OperatorConfig{ - TypeMeta: metav1.TypeMeta{ - APIVersion: kubecore.APIVersion, - Kind: kubecore.Kind, - }, - ClusterConfig: kubecore.ClusterConfig{ - APIServerURL: getAPIServerURL(installConfig.Config), - }, - AuthConfig: kubecore.AuthConfig{ - OIDCClientID: authConfigOIDCClientID, - OIDCIssuerURL: getOicdIssuerURL(installConfig.Config), - OIDCGroupsClaim: authConfigOIDCGroupsClaim, - OIDCUsernameClaim: authConfigOIDCUsernameClaim, - }, - DNSConfig: kubecore.DNSConfig{ - ClusterIP: clusterIP.String(), - }, - CloudProviderConfig: kubecore.CloudProviderConfig{ - CloudConfigPath: "", - CloudProviderProfile: k8sCloudProvider(installConfig.Config.Platform), - }, - RoutingConfig: kubecore.RoutingConfig{ - Subdomain: getBaseAddress(installConfig.Config), - }, - NetworkConfig: kubecore.NetworkConfig{ - ClusterCIDR: installConfig.Config.Networking.PodCIDR.String(), - ServiceCIDR: installConfig.Config.Networking.ServiceCIDR.String(), - AdvertiseAddress: networkConfigAdvertiseAddress, - EtcdServers: strings.Join(getEtcdServersURLs(installConfig.Config), ","), - }, - } - - data, err := yaml.Marshal(kco.Config) - if err != nil { - return errors.Wrapf(err, "failed to create %s config from InstallConfig", kco.Name()) - } - kco.File = &asset.File{ - Filename: "kco-config.yaml", - Data: data, - } - - return nil -} - -// Files returns the files generated by the asset. -func (kco *KubeCoreOperator) Files() []*asset.File { - if kco.File != nil { - return []*asset.File{kco.File} - } - return []*asset.File{} -} - -func getEtcdServersURLs(ic *types.InstallConfig) []string { - var urls []string - for i := 0; i < ic.MasterCount(); i++ { - urls = append(urls, fmt.Sprintf("https://%s-etcd-%d.%s:2379", ic.ObjectMeta.Name, i, ic.BaseDomain)) - } - return urls -} - -func getOicdIssuerURL(ic *types.InstallConfig) string { - return fmt.Sprintf("https://%s.%s/identity", ic.ObjectMeta.Name, ic.BaseDomain) -} - -func getBaseAddress(ic *types.InstallConfig) string { - return fmt.Sprintf("%s.%s", ic.ObjectMeta.Name, ic.BaseDomain) -} - -// Converts a platform to the cloudProvider that k8s understands -func k8sCloudProvider(platform types.Platform) string { - if platform.AWS != nil { - return "aws" - } - if platform.Libvirt != nil { - //return "libvirt" - } - return "" -} diff --git a/pkg/asset/manifests/operators.go b/pkg/asset/manifests/operators.go index c56a1ac4631..bc58a3fbe03 100644 --- a/pkg/asset/manifests/operators.go +++ b/pkg/asset/manifests/operators.go @@ -44,7 +44,6 @@ func (m *Manifests) Name() string { func (m *Manifests) Dependencies() []asset.Asset { return []asset.Asset{ &installconfig.InstallConfig{}, - &KubeCoreOperator{}, &networkOperator{}, &kubeAddonOperator{}, &machineAPIOperator{}, @@ -70,16 +69,14 @@ func (m *Manifests) Dependencies() []asset.Asset { // Generate generates the respective operator config.yml files func (m *Manifests) Generate(dependencies asset.Parents) error { - kco := &KubeCoreOperator{} no := &networkOperator{} addon := &kubeAddonOperator{} mao := &machineAPIOperator{} installConfig := &installconfig.InstallConfig{} - dependencies.Get(kco, no, addon, mao, installConfig) + dependencies.Get(no, addon, mao, installConfig) - // kco+no+mao go to kube-system config map + // no+mao go to kube-system config map m.KubeSysConfig = configMap("kube-system", "cluster-config-v1", genericData{ - "kco-config": string(kco.Files()[0].Data), "network-config": string(no.Files()[0].Data), "install-config": string(installConfig.Files()[0].Data), "mao-config": string(mao.Files()[0].Data), diff --git a/pkg/asset/manifests/tectonic.go b/pkg/asset/manifests/tectonic.go index 61fd65fba3c..c040993cb1c 100644 --- a/pkg/asset/manifests/tectonic.go +++ b/pkg/asset/manifests/tectonic.go @@ -63,8 +63,6 @@ func (t *Tectonic) Generate(dependencies asset.Parents) error { "99_binding-discovery.yaml": []byte(content.BindingDiscovery), "99_kube-addon-00-appversion.yaml": []byte(content.AppVersionKubeAddon), "99_kube-addon-01-operator.yaml": applyTemplateData(content.KubeAddonOperator, templateData), - "99_kube-core-00-appversion.yaml": []byte(content.AppVersionKubeCore), - "99_kube-core-00-operator.yaml": applyTemplateData(content.KubeCoreOperator, templateData), "99_openshift-cluster-api_cluster.yaml": clusterk8sio.Raw, "99_openshift-cluster-api_worker-machineset.yaml": worker.MachineSetRaw, "99_openshift-cluster-api_worker-user-data-secret.yaml": worker.UserDataSecretRaw, From 5e3a99bb0ae5ad421ba93cbf970516038f0aac4d Mon Sep 17 00:00:00 2001 From: Michal Fojtik Date: Mon, 8 Oct 2018 14:17:03 +0200 Subject: [PATCH 3/6] ignition: use correct images for control plane Instead of hard-coding images in installer, use the origin-release to extract the correct image names for bootstrap manifests. --- pkg/asset/ignition/bootstrap/bootstrap.go | 2 -- pkg/asset/ignition/bootstrap/content/bootkube.go | 6 ++++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/asset/ignition/bootstrap/bootstrap.go b/pkg/asset/ignition/bootstrap/bootstrap.go index 5373d9bc316..115ae9d13d1 100644 --- a/pkg/asset/ignition/bootstrap/bootstrap.go +++ b/pkg/asset/ignition/bootstrap/bootstrap.go @@ -39,7 +39,6 @@ type bootstrapTemplateData struct { EtcdCertSignerImage string EtcdCluster string EtcdctlImage string - HyperkubeImage string ReleaseImage string } @@ -162,7 +161,6 @@ func (a *Bootstrap) getTemplateData(installConfig *types.InstallConfig) (*bootst EtcdctlImage: "quay.io/coreos/etcd:v3.2.14", BootkubeImage: "quay.io/coreos/bootkube:v0.10.0", ReleaseImage: releaseImage, - HyperkubeImage: "openshift/origin-node:latest", EtcdCluster: strings.Join(etcdEndpoints, ","), }, nil } diff --git a/pkg/asset/ignition/bootstrap/content/bootkube.go b/pkg/asset/ignition/bootstrap/content/bootkube.go index 1282390f15c..01b59aca131 100644 --- a/pkg/asset/ignition/bootstrap/content/bootkube.go +++ b/pkg/asset/ignition/bootstrap/content/bootkube.go @@ -42,6 +42,9 @@ KUBE_APISERVER_OPERATOR_IMAGE=$(podman run --rm {{.ReleaseImage}} image cluster- KUBE_CONTROLLER_MANAGER_OPERATOR_IMAGE=$(podman run --rm {{.ReleaseImage}} image cluster-kube-controller-manager-operator) KUBE_SCHEDULER_OPERATOR_IMAGE=$(podman run --rm {{.ReleaseImage}} image cluster-kube-scheduler-operator) +OPENSHIFT_HYPERSHIFT_IMAGE=$(podman run --rm {{.ReleaseImage}} image hypershift) +OPENSHIFT_HYPERKUBE_IMAGE=$(podman run --rm {{.ReleaseImage}} image hyperkube) + if [ ! -d cvo-bootstrap ] then echo "Rendering Cluster Version Operator Manifests..." @@ -70,6 +73,7 @@ then /usr/bin/cluster-kube-apiserver-operator render \ --manifest-etcd-serving-ca=etcd-client-ca.crt \ --manifest-etcd-server-urls={{.EtcdCluster}} \ + --manifest-image=${OPENSHIFT_HYPERSHIFT_IMAGE} \ --asset-input-dir=/assets/tls \ --asset-output-dir=/assets/kube-apiserver-bootstrap \ --config-output-file=/assets/kube-apiserver-bootstrap/config @@ -88,6 +92,7 @@ then --volume "$PWD:/assets:z" \ "${KUBE_CONTROLLER_MANAGER_OPERATOR_IMAGE}" \ /usr/bin/cluster-kube-controller-manager-operator render \ + --manifest-image=${OPENSHIFT_HYPERKUBE_IMAGE} \ --asset-input-dir=/assets/tls \ --asset-output-dir=/assets/kube-controller-manager-bootstrap \ --config-output-file=/assets/kube-controller-manager-bootstrap/config @@ -106,6 +111,7 @@ then --volume "$PWD:/assets:z" \ "${KUBE_SCHEDULER_OPERATOR_IMAGE}" \ /usr/bin/cluster-kube-scheduler-operator render \ + --manifest-image=${OPENSHIFT_HYPERKUBE_IMAGE} \ --asset-input-dir=/assets/tls \ --asset-output-dir=/assets/kube-scheduler-bootstrap \ --config-output-file=/assets/kube-scheduler-bootstrap/config From 4789da76565689d98617fcfc110c41c2daad2f77 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Tue, 9 Oct 2018 16:18:26 +0200 Subject: [PATCH 4/6] tls: add system:masters to kubelet client certs Add the 'system:masters' into kubelet cert key so kubelet is able to reach out to openshift kube apiserver successfully. This is temporary and the "pod" team should figure out how to make kubelet bootstrapping more secure using minimal permissions, preferably using the 'system:node:*' service accounts to align with upstream. --- pkg/asset/tls/kubeletcertkey.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/asset/tls/kubeletcertkey.go b/pkg/asset/tls/kubeletcertkey.go index b6d41889c96..b8632e2025e 100644 --- a/pkg/asset/tls/kubeletcertkey.go +++ b/pkg/asset/tls/kubeletcertkey.go @@ -29,7 +29,9 @@ func (a *KubeletCertKey) Generate(dependencies asset.Parents) error { dependencies.Get(kubeCA) cfg := &CertCfg{ - Subject: pkix.Name{CommonName: "system:serviceaccount:kube-system:default", Organization: []string{"system:serviceaccounts:kube-system"}}, + // system:masters is a hack to get the kubelet up without kube-core + // TODO(node): make kubelet bootstrapping secure with minimal permissions eventually switching to system:node:* CommonName + Subject: pkix.Name{CommonName: "system:serviceaccount:kube-system:default", Organization: []string{"system:serviceaccounts:kube-system", "system:masters"}}, KeyUsages: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, Validity: ValidityThirtyMinutes, From 06eb6f5f90b2ece7196c916ef717d76d585004a1 Mon Sep 17 00:00:00 2001 From: Michal Fojtik Date: Wed, 17 Oct 2018 10:50:58 +0200 Subject: [PATCH 5/6] ignition/bootstrap: add checkpointer and kube-proxy manifests In order to complete the bootkube start sequence we need to provide checkpointer pod that has to be observed as "ready" by the bootkube. Before these manifests were provided by kube-core operator and before we have the checkpointer operator that provide these, we have to install them directly temporarely. The kube-proxy manifests are needed for the tectonic-network operator in order to talk to the bootkube control plane. These should be owned by networking team and moved to their operator. --- pkg/asset/ignition/bootstrap/bootstrap.go | 32 ++ .../ignition/bootstrap/content/bootkube.go | 6 + .../bootstrap/content/bootkube_temporary.go | 431 ++++++++++++++++++ 3 files changed, 469 insertions(+) create mode 100644 pkg/asset/ignition/bootstrap/content/bootkube_temporary.go diff --git a/pkg/asset/ignition/bootstrap/bootstrap.go b/pkg/asset/ignition/bootstrap/bootstrap.go index 115ae9d13d1..f9d6b6975ce 100644 --- a/pkg/asset/ignition/bootstrap/bootstrap.go +++ b/pkg/asset/ignition/bootstrap/bootstrap.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "os" + "path/filepath" "strings" "text/template" @@ -94,6 +95,7 @@ func (a *Bootstrap) Generate(dependencies asset.Parents) error { a.addBootstrapFiles(dependencies) a.addBootkubeFiles(dependencies, templateData) + a.addTemporaryBootkubeFiles(templateData) a.addTectonicFiles(dependencies, templateData) a.addTLSCertFiles(dependencies) @@ -199,6 +201,36 @@ func (a *Bootstrap) addBootkubeFiles(dependencies asset.Parents, templateData *b ) } +func (a *Bootstrap) addTemporaryBootkubeFiles(templateData *bootstrapTemplateData) { + podCheckpointerBootstrapDir := filepath.Join(rootDir, "pod-checkpointer-operator-bootstrap") + for name, data := range content.PodCheckpointerBootkubeManifests { + a.Config.Storage.Files = append( + a.Config.Storage.Files, + ignition.FileFromString(filepath.Join(podCheckpointerBootstrapDir, name), 0644, data), + ) + } + + kubeProxyBootstrapDir := filepath.Join(rootDir, "kube-proxy-operator-bootstrap") + for name, data := range content.KubeProxyBootkubeManifests { + a.Config.Storage.Files = append( + a.Config.Storage.Files, + ignition.FileFromString(filepath.Join(kubeProxyBootstrapDir, name), 0644, data), + ) + } + + kubeDNSBootstrapDir := filepath.Join(rootDir, "kube-dns-operator-bootstrap") + for name, data := range content.KubeDNSBootkubeManifests { + a.Config.Storage.Files = append( + a.Config.Storage.Files, + ignition.FileFromString(filepath.Join(kubeDNSBootstrapDir, name), 0644, data), + ) + } + a.Config.Storage.Files = append( + a.Config.Storage.Files, + ignition.FileFromString(filepath.Join(kubeDNSBootstrapDir, "kube-dns-svc.yaml"), 0644, applyTemplateData(content.BootkubeKubeDNSService, templateData)), + ) +} + func (a *Bootstrap) addTectonicFiles(dependencies asset.Parents, templateData *bootstrapTemplateData) { tectonic := &manifests.Tectonic{} dependencies.Get(tectonic) diff --git a/pkg/asset/ignition/bootstrap/content/bootkube.go b/pkg/asset/ignition/bootstrap/content/bootkube.go index 01b59aca131..0f5251022e6 100644 --- a/pkg/asset/ignition/bootstrap/content/bootkube.go +++ b/pkg/asset/ignition/bootstrap/content/bootkube.go @@ -121,6 +121,12 @@ then cp kube-scheduler-bootstrap/manifests/* manifests/ fi +# TODO: Remove this when checkpointer, kube-proxy and kube-dns are properly rendered by corresponding operators. +echo "Installing temporary bootstrap manifests..." +cp pod-checkpointer-operator-bootstrap/* manifests/ +cp kube-proxy-operator-bootstrap/* manifests/ +cp kube-dns-operator-bootstrap/* manifests/ + if [ ! -d mco-bootstrap ] then echo "Rendering MCO manifests..." diff --git a/pkg/asset/ignition/bootstrap/content/bootkube_temporary.go b/pkg/asset/ignition/bootstrap/content/bootkube_temporary.go new file mode 100644 index 00000000000..7a75ecf8d8b --- /dev/null +++ b/pkg/asset/ignition/bootstrap/content/bootkube_temporary.go @@ -0,0 +1,431 @@ +package content + +import "text/template" + +// PodCheckpointerBootkubeManifests is a map of manifests needed by pod checkpointer to install. +// TODO: This must move to pod-checkpointer-operator renderer. +var PodCheckpointerBootkubeManifests = map[string]string{ + "checkpointer-role.yaml": bootkubeCheckpointerRole, + "checkpointer-role-binding.yaml": bootkubeCheckpointerRoleBinding, + "checkpointer-sa.yaml": bootkubeCheckpointerSA, + "checkpointer-daemonset.yaml": bootkubeCheckpointerDaemonset, +} + +// KubeProxyBootkubeManifests is a map of manifests needed by kube-proxy to install. +// TODO: This must move to networking operator renderer. +var KubeProxyBootkubeManifests = map[string]string{ + "kube-proxy-kube-system-rbac-role-binding.yaml": bootkubeKubeSystemRBACRoleBinding, + "kube-proxy-role-binding.yaml": bootkubeKubeProxyRoleBinding, + "kube-proxy-service-account.yaml": bootkubeKubeProxySA, + "kube-proxy-daemonset.yaml": bootkubeKubeProxyDaemonset, +} + +// KubeDNSBootkubeManifests is a map of manifests needed by kube-dns to install. +// TODO: This must move to networking operator renderer. +var KubeDNSBootkubeManifests = map[string]string{ + "kube-dns-deployment.yaml": bootkubeKubeDNSDeployment, +} + +// BootkubeKubeDNSService is a template for kube-dns service. +var BootkubeKubeDNSService = template.Must(template.New("bootkube.sh").Parse(` +apiVersion: v1 +kind: Service +metadata: + name: kube-dns + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: KubeDNS +spec: + selector: + k8s-app: kube-dns + clusterIP: {{.ClusterDNSIP}} + ports: + - name: dns + port: 53 + protocol: UDP + targetPort: 53 + - name: dns-tcp + port: 53 + protocol: TCP + targetPort: 53 +`)) + +const ( + bootkubeCheckpointerRole = ` +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: pod-checkpointer + namespace: kube-system +rules: +- apiGroups: [""] # "" indicates the core API group + resources: ["pods"] + verbs: ["get", "watch", "list"] +- apiGroups: [""] # "" indicates the core API group + resources: ["secrets", "configmaps"] + verbs: ["get"] +` + + bootkubeCheckpointerRoleBinding = ` +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: pod-checkpointer + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pod-checkpointer +subjects: +- kind: ServiceAccount + name: pod-checkpointer + namespace: kube-system +` + + bootkubeCheckpointerSA = ` +apiVersion: v1 +kind: ServiceAccount +metadata: + namespace: kube-system + name: pod-checkpointer +` + + bootkubeCheckpointerDaemonset = ` +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + k8s-app: pod-checkpointer + tier: control-plane + name: pod-checkpointer + namespace: kube-system +spec: + selector: + matchLabels: + tier: control-plane + k8s-app: pod-checkpointer + template: + metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" + checkpointer.alpha.coreos.com/checkpoint: "true" + labels: + k8s-app: pod-checkpointer + tier: control-plane + spec: + containers: + - command: + - /checkpoint + - --lock-file=/var/run/lock/pod-checkpointer.lock + - --kubeconfig=/etc/checkpointer/kubeconfig + - --checkpoint-grace-period=5m + - --container-runtime-endpoint=unix:///var/run/crio/crio.sock + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: quay.io/coreos/pod-checkpointer:9dc83e1ab3bc36ca25c9f7c18ddef1b91d4a0558 + imagePullPolicy: Always + name: pod-checkpointer + securityContext: + privileged: true + volumeMounts: + - mountPath: /etc/checkpointer + name: kubeconfig + - mountPath: /etc/kubernetes + name: etc-kubernetes + - mountPath: /var/run + name: var-run + serviceAccountName: pod-checkpointer + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + restartPolicy: Always + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + volumes: + - name: kubeconfig + secret: + secretName: controller-manager-kubeconfig + - hostPath: + path: /etc/kubernetes + name: etc-kubernetes + - hostPath: + path: /var/run + name: var-run + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate +` + + bootkubeKubeSystemRBACRoleBinding = ` +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:default-sa +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: default + namespace: kube-system +` + + bootkubeKubeProxySA = ` +apiVersion: v1 +kind: ServiceAccount +metadata: + namespace: kube-system + name: kube-proxy +` + + bootkubeKubeProxyDaemonset = ` +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + k8s-app: kube-proxy + tier: node + name: kube-proxy + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: kube-proxy + tier: node + template: + metadata: + labels: + k8s-app: kube-proxy + tier: node + spec: + containers: + - command: + - ./hyperkube + - proxy + - --cluster-cidr=10.3.0.0/16 + - --hostname-override=$(NODE_NAME) + - --kubeconfig=/etc/kubernetes/kubeconfig + - --proxy-mode=iptables + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: quay.io/coreos/hyperkube:v1.9.3_coreos.0 + name: kube-proxy + securityContext: + privileged: true + volumeMounts: + - mountPath: /etc/ssl/certs + name: ssl-certs-host + readOnly: true + - mountPath: /etc/kubernetes + name: kubeconfig + readOnly: true + hostNetwork: true + serviceAccountName: kube-proxy + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /etc/ssl/certs + name: ssl-certs-host + - name: kubeconfig + secret: + defaultMode: 420 + secretName: controller-manager-kubeconfig + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate +` + + bootkubeKubeProxyRoleBinding = ` +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kube-proxy +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:node-proxier # Automatically created system role. +subjects: +- kind: ServiceAccount + name: kube-proxy + namespace: kube-system +` + bootkubeKubeDNSDeployment = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kube-dns + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" +spec: + # replicas: not specified here: + # 1. In order to make Addon Manager do not reconcile this replicas parameter. + # 2. Default is 1. + # 3. Will be tuned in real time if DNS horizontal auto-scaling is turned on. + strategy: + rollingUpdate: + maxSurge: 10% + maxUnavailable: 0 + selector: + matchLabels: + k8s-app: kube-dns + template: + metadata: + labels: + k8s-app: kube-dns + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + containers: + - name: kubedns + image: "gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.8" + resources: + # TODO: Set memory limits when we've profiled the container for large + # clusters, then set request = limit to keep this container in + # guaranteed class. Currently, this container falls into the + # "burstable" category so the kubelet doesn't backoff from restarting it. + limits: + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + livenessProbe: + httpGet: + path: /healthcheck/kubedns + port: 10054 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /readiness + port: 8081 + scheme: HTTP + # we poll on pod startup for the Kubernetes master service and + # only setup the /readiness HTTP server once that's available. + initialDelaySeconds: 3 + timeoutSeconds: 5 + args: + - --domain=cluster.local. + - --dns-port=10053 + - --config-dir=/kube-dns-config + - --v=2 + env: + - name: PROMETHEUS_PORT + value: "10055" + ports: + - containerPort: 10053 + name: dns-local + protocol: UDP + - containerPort: 10053 + name: dns-tcp-local + protocol: TCP + - containerPort: 10055 + name: metrics + protocol: TCP + volumeMounts: + - name: kube-dns-config + mountPath: /kube-dns-config + - name: dnsmasq + image: "gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.8" + livenessProbe: + httpGet: + path: /healthcheck/dnsmasq + port: 10054 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + args: + - -v=2 + - -logtostderr + - -configDir=/etc/k8s/dns/dnsmasq-nanny + - -restartDnsmasq=true + - -- + - -k + - --cache-size=1000 + - --log-facility=- + - --server=/cluster.local/127.0.0.1#10053 + - --server=/in-addr.arpa/127.0.0.1#10053 + - --server=/ip6.arpa/127.0.0.1#10053 + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + # see: https://github.com/kubernetes/kubernetes/issues/29055 for details + resources: + requests: + cpu: 150m + memory: 20Mi + volumeMounts: + - name: kube-dns-config + mountPath: /etc/k8s/dns/dnsmasq-nanny + - name: sidecar + image: "gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.8" + livenessProbe: + httpGet: + path: /metrics + port: 10054 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + args: + - --v=2 + - --logtostderr + - --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.cluster.local,5,SRV + - --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.cluster.local,5,SRV + ports: + - containerPort: 10054 + name: metrics + protocol: TCP + resources: + requests: + memory: 20Mi + cpu: 10m + dnsPolicy: Default # Don't use cluster DNS. + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" + nodeSelector: + node-role.kubernetes.io/master: "" + volumes: + - name: kube-dns-config + configMap: + name: kube-dns + optional: true +` +) From 6a1b896681f32ceb24431e8dcbd7dc4d17d62dd3 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Thu, 18 Oct 2018 10:38:51 +0200 Subject: [PATCH 6/6] bootkube: add installer config overrides for the control plane --- pkg/asset/ignition/bootstrap/bootstrap.go | 7 ++++ .../ignition/bootstrap/content/bootkube.go | 33 +++++++++++++++++-- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/pkg/asset/ignition/bootstrap/bootstrap.go b/pkg/asset/ignition/bootstrap/bootstrap.go index f9d6b6975ce..618f3253930 100644 --- a/pkg/asset/ignition/bootstrap/bootstrap.go +++ b/pkg/asset/ignition/bootstrap/bootstrap.go @@ -183,6 +183,7 @@ func (a *Bootstrap) addBootstrapFiles(dependencies asset.Parents) { } func (a *Bootstrap) addBootkubeFiles(dependencies asset.Parents, templateData *bootstrapTemplateData) { + bootkubeConfigOverridesDir := filepath.Join(rootDir, "bootkube-config-overrides") adminKubeconfig := &kubeconfig.Admin{} manifests := &manifests.Manifests{} dependencies.Get(adminKubeconfig, manifests) @@ -191,6 +192,12 @@ func (a *Bootstrap) addBootkubeFiles(dependencies asset.Parents, templateData *b a.Config.Storage.Files, ignition.FileFromString("/opt/tectonic/bootkube.sh", 0555, applyTemplateData(content.BootkubeShFileTemplate, templateData)), ) + for _, o := range content.BootkubeConfigOverrides { + a.Config.Storage.Files = append( + a.Config.Storage.Files, + ignition.FileFromString(filepath.Join(bootkubeConfigOverridesDir, o.Name()), 0600, applyTemplateData(o, templateData)), + ) + } a.Config.Storage.Files = append( a.Config.Storage.Files, ignition.FilesFromAsset(rootDir, 0600, adminKubeconfig)..., diff --git a/pkg/asset/ignition/bootstrap/content/bootkube.go b/pkg/asset/ignition/bootstrap/content/bootkube.go index 0f5251022e6..46cd07244f5 100644 --- a/pkg/asset/ignition/bootstrap/content/bootkube.go +++ b/pkg/asset/ignition/bootstrap/content/bootkube.go @@ -76,7 +76,8 @@ then --manifest-image=${OPENSHIFT_HYPERSHIFT_IMAGE} \ --asset-input-dir=/assets/tls \ --asset-output-dir=/assets/kube-apiserver-bootstrap \ - --config-output-file=/assets/kube-apiserver-bootstrap/config + --config-output-file=/assets/kube-apiserver-bootstrap/config \ + --config-override-files=/assets/bootkube-config-overrides/kube-apiserver-config-overrides.yaml cp kube-apiserver-bootstrap/config /etc/kubernetes/bootstrap-configs/kube-apiserver-config.yaml cp kube-apiserver-bootstrap/bootstrap-manifests/* bootstrap-manifests/ @@ -95,7 +96,8 @@ then --manifest-image=${OPENSHIFT_HYPERKUBE_IMAGE} \ --asset-input-dir=/assets/tls \ --asset-output-dir=/assets/kube-controller-manager-bootstrap \ - --config-output-file=/assets/kube-controller-manager-bootstrap/config + --config-output-file=/assets/kube-controller-manager-bootstrap/config \ + --config-override-files=/assets/bootkube-config-overrides/kube-controller-manager-config-overrides.yaml cp kube-controller-manager-bootstrap/config /etc/kubernetes/bootstrap-configs/kube-controller-manager-config.yaml cp kube-controller-manager-bootstrap/bootstrap-manifests/* bootstrap-manifests/ @@ -229,3 +231,30 @@ podman run \ start --asset-dir=/assets `)) ) + +var ( + // BootkubeConfigOverrides contains the configuration override files passed to the render commands of the components. + // These are supposed to be customized by the installer where the config differs from the operator render default. + BootkubeConfigOverrides = []*template.Template{ + KubeApiserverConfigOverridesTemplate, + KubeControllerManagerConfigOverridesTemplate, + } +) + +var ( + // KubeApiserverConfigOverridesTemplate are overrides that the installer passes to the default config of the + // kube-apiserver rendered by the cluster-kube-apiserver-operator. + KubeApiserverConfigOverridesTemplate = template.Must(template.New("kube-apiserver-config-overrides.yaml").Parse(` +apiVersion: kubecontrolplane.config.openshift.io/v1 +kind: KubeAPIServerConfig +kubeletClientInfo: + ca: "" # kubelet uses self-signed serving certs. TODO: fix kubelet pki +`)) + + // KubeControllerManagerConfigOverridesTemplate are overrides that the installer passes to the default config of the + // kube-controller-manager rendered by the cluster-kube-controller-manager-operator. + KubeControllerManagerConfigOverridesTemplate = template.Must(template.New("kube-controller-manager-config-overrides.yaml").Parse(` +apiVersion: kubecontrolplane.config.openshift.io/v1 +kind: KubeControllerManagerConfig +`)) +)