From 90c55fbafaeb8fa1792af7872d740fe137af728a Mon Sep 17 00:00:00 2001 From: Dave McCormick Date: Tue, 10 Sep 2019 13:29:04 +0100 Subject: [PATCH] move Kiam to a plugin (#1726) * Create a new kiam plugin with IAMPolicy Remove Experimental Kiam Support Code Allow plugins to create ca's and use them to sign other certs. Note, the CA must be created first. Improve logging around plugin extensions. * Undo changes to cloud-config-controller * no need for or with one value * remove go modules files --- builtin/files/cluster.yaml.tmpl | 93 +++-- .../kiam/manifests/agent-daemonset.yaml | 103 ++++++ .../kiam/manifests/agent-tls-secret.yaml | 10 + .../server-cluster-role-binding.yaml | 12 + .../kiam/manifests/server-cluster-role.yaml | 21 ++ .../kiam/manifests/server-daemonset.yaml | 106 ++++++ .../kiam/manifests/server-tls-secret.yaml | 10 + .../kiam/manifests/service-account.yaml | 5 + .../files/plugins/kiam/manifests/service.yaml | 15 + builtin/files/plugins/kiam/plugin.yaml | 102 ++++++ .../stack-templates/control-plane.json.tmpl | 4 +- .../files/userdata/cloud-config-controller | 336 +----------------- cmd/render.go | 15 +- credential/encrypted_assets.go | 86 +---- credential/encrypted_assets_test.go | 20 +- credential/generator.go | 46 +-- credential/pki.go | 52 ++- .../step-6-configure-add-ons.md | 4 +- pkg/api/cluster.go | 6 - pkg/api/keypair_spec.go | 8 + pkg/api/types.go | 21 -- pkg/model/cluster_test.go | 14 - pkg/model/credentials.go | 4 +- pkg/model/credentials_test.go | 2 +- pkg/model/stack_test.go | 6 +- pki/keypair.go | 26 ++ pki/pki.go | 33 +- pki/x509.go | 35 -- plugin/clusterextension/extras.go | 93 ++++- test/helper/helper.go | 3 +- test/integration/maincluster_test.go | 79 ---- 31 files changed, 646 insertions(+), 724 deletions(-) create mode 100644 builtin/files/plugins/kiam/manifests/agent-daemonset.yaml create mode 100644 builtin/files/plugins/kiam/manifests/agent-tls-secret.yaml create mode 100644 builtin/files/plugins/kiam/manifests/server-cluster-role-binding.yaml create mode 100644 builtin/files/plugins/kiam/manifests/server-cluster-role.yaml create mode 100644 builtin/files/plugins/kiam/manifests/server-daemonset.yaml create mode 100644 builtin/files/plugins/kiam/manifests/server-tls-secret.yaml create mode 100644 builtin/files/plugins/kiam/manifests/service-account.yaml create mode 100644 builtin/files/plugins/kiam/manifests/service.yaml create mode 100644 builtin/files/plugins/kiam/plugin.yaml diff --git a/builtin/files/cluster.yaml.tmpl b/builtin/files/cluster.yaml.tmpl index 0a5e683df..9528f6e13 100644 --- a/builtin/files/cluster.yaml.tmpl +++ b/builtin/files/cluster.yaml.tmpl @@ -240,7 +240,7 @@ kmsKeyArn: "{{.KMSKeyARN}}" # # If you specify a name for the role, kube-aws will create it without a random id suffix (AWS default). # # Given a role of `yourManagedRole`, this will create an IAM role named `${clusterName}-${region}-yourManagedRole` # # to follow the recommendation in AWS documentation http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-role.html -# # There are several cases you might need a stable name such as with .experimental.kube2IamSupport or .experimental.kiamSupport. See #297 for more information. +# # There are several cases you might need a stable name such as with .experimental.kube2IamSupport or kiam plugin. See #297 for more information. # # ATTENTION: Consider limiting number of characters in clusterName and iam.role.name to avoid the resulting IAM # # role name's length from exceeding the AWS limit: 64. See https://github.com/kubernetes-incubator/kube-aws/issues/347 # # It will have attached a customer Managed Policy that you can modify afterwards if you need more permissions for your cluster. @@ -362,7 +362,7 @@ worker: # # If you specify a name for the role, kube-aws will create it without a random id suffix (AWS default). # # Given a role of `yourManagedRole`, this will create an IAM role named `${clusterName}-${region}-yourManagedRole` # # to follow the recommendation in AWS documentation http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-role.html -# # There are several cases you might need a stable name such as with .experimental.kube2IamSupport or .experimental.kiamSupport. See #297 for more information. +# # There are several cases you might need a stable name such as with .experimental.kube2IamSupport or kiam plugin. See #297 for more information. # # ATTENTION: Consider limiting number of characters in clusterName and iam.role.name to avoid the resulting IAM # # role name's length from exceeding the AWS limit: 64. See https://github.com/kubernetes-incubator/kube-aws/issues/347 # # It will have attached a customer Managed Policy that you can modify afterwards if you need more permissions for your cluster. @@ -1444,38 +1444,6 @@ experimental: ephemeralImageStorage: enabled: false - # When enabled this will install the kiam daemon set using the repo from `kiamImage` - # It will also grant sts:assumeRole permission to the IAM role for controller nodes (only controller nodes where the kiam server is hosted need this, worker nodes do not) - # This will use SSL certificates generated during `kube-aws render credentials`, ensure this has been run with kube-aws v0.9.9+ - # This is intended to be used in combination with .controller.iam.role.name. See #297 for more information. - kiamSupport: - enabled: false - # image: - # repo: quay.io/uswitch/kiam - # tag: v3.2 - # rktPullDocker: false - # sessionDuration: 30m - # serverAddresses: - # serverAddress: localhost:443 - # agentAddress: kiam-server:443 - # Optional resource change for kiam servers/agents can be done via using the resources block below and changing the values. - # Values below are the default if not set. - # serverResources: - # requests: - # cpu: 50m - # memory: 10Mi - # limits: - # cpu: 200m - # memory: 200Mi - # agentResources: - # requests: - # cpu: 10m - # memory: 5Mi - # limits: - # cpu: 200m - # memory: 100Mi - - # When enabled this will install the kube2iam daemon set using the repo from `kube2iamImage` # It will also grant sts:assumeRole permission to the IAM role for controller nodes. # This is intended to be used in combination with .controller.iam.role.name. See #297 for more information. @@ -1607,18 +1575,45 @@ kubeAwsPlugins: selector: prometheus: monitoring - # upgradeHelper - assists when rolling out new versions of kubernetes - # It actively disables old controllers and temporarily removes mutating/validating webhooks whilst - # the upgraded controller is starting up. - # NOTE: You will normally not need this plugin - so ONLY enable if you are experiencing issues testing migrating across versions. - # It will only kill controller that are a different release from currently spinning up version, e.g. :- - # it will kill v1.13.2 controllers when rolling out v1.14.0 - # it will NOT kill v1.14.0 controllers when rolling out v1.14.3 - upgradeHelper: - enabled: false - # disableWebhooks can be used to turn off the webhook feature if required - disableWebhooks: true - # disableWorkerCommunicationChecks can be used to disable the check that all workers checking in with this master - # so that cfn-signal can be generated and upgrade process move forward - # (you don't want a single bad worker to cause an important upgrade to fail) - disableWorkerCommunicationChecks: true + # upgradeHelper - assists when rolling out new versions of kubernetes + # It actively disables old controllers and temporarily removes mutating/validating webhooks whilst + # the upgraded controller is starting up. + # NOTE: You will normally not need this plugin - so ONLY enable if you are experiencing issues testing migrating across versions. + # It will only kill controller that are a different release from currently spinning up version, e.g. :- + # it will kill v1.13.2 controllers when rolling out v1.14.0 + # it will NOT kill v1.14.0 controllers when rolling out v1.14.3 + upgradeHelper: + enabled: false + # disableWebhooks can be used to turn off the webhook feature if required + disableWebhooks: true + # disableWorkerCommunicationChecks can be used to disable the check that all workers checking in with this master + # so that cfn-signal can be generated and upgrade process move forward + # (you don't want a single bad worker to cause an important upgrade to fail) + disableWorkerCommunicationChecks: true + + # kiam - deploys kiam server and agent components + # kiam provides individual pod access policies and tokens for restricting pod access to AWS resources + # It will grant sts:assumeRole permission to the IAM role for controller nodes (only controller nodes where the kiam server is hosted need this, worker nodes do not) + # This is intended to be used in combination with .controller.iam.role.name. See #297 for more information. + kiam: + enabled: false + # image: quay.io/uswitch/kiam + # tag: v3.2 + # sessionDuration: 30m + # server: + # address: localhost:443 + # resources: + # requests: + # cpu: 50m + # limits: + # cpu: 200m + # memory: 200Mi + # agent: + # address: kiam-server:443 + # resources: + # requests: + # cpu: 10m + # memory: 5Mi + # limits: + # cpu: 200m + # memory: 100Mi diff --git a/builtin/files/plugins/kiam/manifests/agent-daemonset.yaml b/builtin/files/plugins/kiam/manifests/agent-daemonset.yaml new file mode 100644 index 000000000..c66656143 --- /dev/null +++ b/builtin/files/plugins/kiam/manifests/agent-daemonset.yaml @@ -0,0 +1,103 @@ +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + namespace: kube-system + name: kiam-agent +spec: + updateStrategy: + rollingUpdate: + maxUnavailable: 100% + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9620" + labels: + app: kiam + role: agent + spec: + priorityClassName: system-node-critical + tolerations: + - operator: Exists + effect: NoSchedule + - operator: Exists + effect: NoExecute + - operator: Exists + key: CriticalAddonsOnly + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node.kubernetes.io/role + operator: NotIn + values: + - master + volumes: + - name: ssl-certs + hostPath: + path: /usr/share/ca-certificates + - name: tls + secret: + secretName: kiam-agent-tls + - name: xtables + hostPath: + path: /run/xtables.lock + containers: + - name: kiam + securityContext: + capabilities: + add: ["NET_ADMIN"] + image: {{ .Values.image }}:{{ .Values.tag }} + command: + - {{ if checkVersion ">= 3.0" .Values.tag }}/kiam{{ else }}/agent{{ end }} + args: + {{if checkVersion ">= 3.0" .Values.tag -}} + - agent + - --whitelist-route-regexp=.* + - --gateway-timeout-creation=1s + {{ end -}} + - --iptables + {{- if .Config.Cluster.Kubernetes.Networking.AmazonVPC.Enabled }} + - --host-interface=!eni0 + {{- else if eq .Config.Cluster.Kubernetes.Networking.SelfHosting.Type "canal" }} + - --host-interface=cali+ + {{- else}} + - --host-interface=cni0 + {{- end }} + - --json-log + - --port=8181 + - --cert=/etc/kiam/tls/tls.crt + - --key=/etc/kiam/tls/tls.key + - --ca=/etc/kiam/tls/ca.crt + - --server-address={{ .Values.agent.address }} + - --prometheus-listen-addr=0.0.0.0:9620 + - --prometheus-sync-interval=5s + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + volumeMounts: + - mountPath: /etc/ssl/certs + name: ssl-certs + - mountPath: /etc/kiam/tls + name: tls + - mountPath: /var/run/xtables.lock + name: xtables + livenessProbe: + httpGet: + path: /ping + port: 8181 + initialDelaySeconds: 3 + periodSeconds: 3 + resources: + requests: + cpu: {{ .Values.agent.resources.requests.cpu }} + memory: {{ .Values.agent.resources.requests.memory }} + limits: + cpu: {{ .Values.agent.resources.limits.cpu }} + memory: {{ .Values.agent.resources.limits.memory }} \ No newline at end of file diff --git a/builtin/files/plugins/kiam/manifests/agent-tls-secret.yaml b/builtin/files/plugins/kiam/manifests/agent-tls-secret.yaml new file mode 100644 index 000000000..7c9b190aa --- /dev/null +++ b/builtin/files/plugins/kiam/manifests/agent-tls-secret.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: kiam-agent-tls + namespace: kube-system +type: kubernetes.io/tls +data: + tls.crt: {{ insertTemplateFile "credentials/kiam-agent.pem" . | b64enc }} + tls.key: {{ insertTemplateFile "credentials/kiam-agent-key.pem" . | b64enc }} + ca.crt: {{ insertTemplateFile "credentials/kiam-ca.pem" . | b64enc }} \ No newline at end of file diff --git a/builtin/files/plugins/kiam/manifests/server-cluster-role-binding.yaml b/builtin/files/plugins/kiam/manifests/server-cluster-role-binding.yaml new file mode 100644 index 000000000..916985861 --- /dev/null +++ b/builtin/files/plugins/kiam/manifests/server-cluster-role-binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: kiam-server +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kiam-server +subjects: +- kind: ServiceAccount + name: kiam-server + namespace: kube-system \ No newline at end of file diff --git a/builtin/files/plugins/kiam/manifests/server-cluster-role.yaml b/builtin/files/plugins/kiam/manifests/server-cluster-role.yaml new file mode 100644 index 000000000..610dff6d6 --- /dev/null +++ b/builtin/files/plugins/kiam/manifests/server-cluster-role.yaml @@ -0,0 +1,21 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: kiam-server +rules: +- apiGroups: + - "" + resources: + - namespaces + - pods + verbs: + - watch + - get + - list +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch \ No newline at end of file diff --git a/builtin/files/plugins/kiam/manifests/server-daemonset.yaml b/builtin/files/plugins/kiam/manifests/server-daemonset.yaml new file mode 100644 index 000000000..0a3bc1869 --- /dev/null +++ b/builtin/files/plugins/kiam/manifests/server-daemonset.yaml @@ -0,0 +1,106 @@ +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + namespace: kube-system + name: kiam-server +spec: + updateStrategy: + rollingUpdate: + maxUnavailable: 100% + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9620" + labels: + app: kiam + role: server + spec: + priorityClassName: system-node-critical + tolerations: + - operator: Exists + effect: NoSchedule + - operator: Exists + effect: NoExecute + - operator: Exists + key: CriticalAddonsOnly + serviceAccountName: kiam-server + nodeSelector: + node.kubernetes.io/role: "master" + volumes: + - name: ssl-certs + hostPath: + path: /usr/share/ca-certificates + - name: tls + secret: + secretName: kiam-server-tls + containers: + - name: kiam + image: {{ .Values.image }}:{{ .Values.tag }} + command: + - {{ if checkVersion ">= 3.0" .Values.tag }}/kiam{{ else }}/server{{ end }} + args: + {{ if checkVersion ">= 3.0" .Values.tag -}} + - server + {{ end -}} + - --json-log + - --bind=0.0.0.0:443 + - --cert=/etc/kiam/tls/tls.crt + - --key=/etc/kiam/tls/tls.key + - --ca=/etc/kiam/tls/ca.crt + - --role-base-arn-autodetect + - --sync=1m + - --prometheus-listen-addr=0.0.0.0:9620 + - --prometheus-sync-interval=5s + - --session-duration={{ .Values.sessionDuration }} + volumeMounts: + - mountPath: /etc/ssl/certs + name: ssl-certs + - mountPath: /etc/kiam/tls + name: tls + livenessProbe: + exec: + command: + {{ if checkVersion ">= 3.0" .Values.tag -}} + - /kiam + - health + - --gateway-timeout-creation=1s + {{ else -}} + - /health + - --server-address-refresh=2s + {{ end -}} + - --cert=/etc/kiam/tls/tls.crt + - --key=/etc/kiam/tls/tls.key + - --ca=/etc/kiam/tls/ca.crt + - --server-address={{ .Values.server.address }} + - --timeout=5s + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 10 + readinessProbe: + exec: + command: + {{ if checkVersion ">= 3.0" .Values.tag -}} + - /kiam + - health + - --gateway-timeout-creation=1s + {{ else -}} + - /health + - --server-address-refresh=2s + {{ end -}} + - --cert=/etc/kiam/tls/tls.crt + - --key=/etc/kiam/tls/tls.key + - --ca=/etc/kiam/tls/ca.crt + - --server-address={{ .Values.server.address }} + - --timeout=5s + initialDelaySeconds: 3 + periodSeconds: 10 + timeoutSeconds: 10 + resources: + requests: + cpu: {{ .Values.server.resources.requests.cpu }} + memory: {{ .Values.server.resources.requests.memory }} + limits: + cpu: {{ .Values.server.resources.limits.cpu }} + memory: {{ .Values.server.resources.limits.memory }} \ No newline at end of file diff --git a/builtin/files/plugins/kiam/manifests/server-tls-secret.yaml b/builtin/files/plugins/kiam/manifests/server-tls-secret.yaml new file mode 100644 index 000000000..ece545933 --- /dev/null +++ b/builtin/files/plugins/kiam/manifests/server-tls-secret.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: kiam-server-tls + namespace: kube-system +type: kubernetes.io/tls +data: + tls.crt: {{ insertTemplateFile "credentials/kiam-server.pem" . | b64enc }} + tls.key: {{ insertTemplateFile "credentials/kiam-server-key.pem" . | b64enc }} + ca.crt: {{ insertTemplateFile "credentials/kiam-ca.pem" . | b64enc }} \ No newline at end of file diff --git a/builtin/files/plugins/kiam/manifests/service-account.yaml b/builtin/files/plugins/kiam/manifests/service-account.yaml new file mode 100644 index 000000000..4c52d4096 --- /dev/null +++ b/builtin/files/plugins/kiam/manifests/service-account.yaml @@ -0,0 +1,5 @@ +kind: ServiceAccount +apiVersion: v1 +metadata: + name: kiam-server + namespace: kube-system \ No newline at end of file diff --git a/builtin/files/plugins/kiam/manifests/service.yaml b/builtin/files/plugins/kiam/manifests/service.yaml new file mode 100644 index 000000000..8e93ec0f8 --- /dev/null +++ b/builtin/files/plugins/kiam/manifests/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: kiam-server + namespace: kube-system +spec: + clusterIP: None + selector: + app: kiam + role: server + ports: + - name: grpclb + port: 443 + targetPort: 443 + protocol: TCP \ No newline at end of file diff --git a/builtin/files/plugins/kiam/plugin.yaml b/builtin/files/plugins/kiam/plugin.yaml new file mode 100644 index 000000000..901379ac2 --- /dev/null +++ b/builtin/files/plugins/kiam/plugin.yaml @@ -0,0 +1,102 @@ +metadata: + name: kiam + version: 0.1.0 +spec: + cluster: + values: + image: quay.io/uswitch/kiam + tag: v3.2 + sessionDuration: 30m + server: + address: localhost:443 + resources: + requests: + cpu: 50m + memory: 10Mi + limits: + cpu: 200m + memory: 200Mi + agent: + address: kiam-server:443 + resources: + requests: + cpu: 10m + memory: 5Mi + limits: + cpu: 200m + memory: 100Mi + + kubernetes: + manifests: + - source: + path: manifests/agent-tls-secret.yaml + - source: + path: manifests/server-tls-secret.yaml + - source: + path: manifests/server-daemonset.yaml + - source: + path: manifests/agent-daemonset.yaml + - source: + path: manifests/service.yaml + - source: + path: manifests/service-account.yaml + - source: + path: manifests/server-cluster-role.yaml + - source: + path: manifests/server-cluster-role-binding.yaml + + pki: + keypairs: + - name: kiam-ca + commonName: kiam-ca + organization: kube-aws-kiam + duration: 43800h + usages: + - ca + - name: kiam-server + commonName: kiam-server + organization: kube-aws-kiam + dnsNames: + - kiam-server + - kiam-server:443 + - localhost + - localhost:443 + - localhost:9610 + duration: 8760h + usages: + - server + - client + signer: kiam-ca + - name: kiam-agent + commonName: kiam-agent + organization: kube-aws-kiam + duration: 8760h + usages: + - client + signer: kiam-ca + + cloudformation: + stacks: + controlPlane: + resources: + content: | + { + "IAMManagedPolicyKiam" : { + "Type" : "AWS::IAM::ManagedPolicy", + "Properties" : { + "Description" : "Policy for managing Kiam on kube-aws controllers", + "Path" : "/", + "Roles":[ { "Ref" : "IAMRoleController" } ], + "PolicyDocument" : { + "Version":"2012-10-17", + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect":"Allow", + "Resource":"*" + } + ] + } + } + } + } \ No newline at end of file diff --git a/builtin/files/stack-templates/control-plane.json.tmpl b/builtin/files/stack-templates/control-plane.json.tmpl index 37c1fa871..075009c9d 100644 --- a/builtin/files/stack-templates/control-plane.json.tmpl +++ b/builtin/files/stack-templates/control-plane.json.tmpl @@ -314,7 +314,7 @@ "Resource": [ "*" ] }, {{end}} - {{if or .Experimental.Kube2IamSupport.Enabled .Experimental.KIAMSupport.Enabled }} + {{if .Experimental.Kube2IamSupport.Enabled }} { "Action": "sts:AssumeRole", "Effect":"Allow", @@ -389,7 +389,7 @@ } } }, - {{if and .Experimental.KIAMSupport.Enabled .KubeResourcesAutosave.Enabled }} + {{if and .KubeResourcesAutosave.Enabled ((index .PluginConfigs "kiam").Enabled) }} "IAMManagedPolicyResourcesAutoSave" : { "Type" : "AWS::IAM::ManagedPolicy", "Properties" : { diff --git a/builtin/files/userdata/cloud-config-controller b/builtin/files/userdata/cloud-config-controller index 067db20b3..05c76eacc 100644 --- a/builtin/files/userdata/cloud-config-controller +++ b/builtin/files/userdata/cloud-config-controller @@ -1106,22 +1106,6 @@ write_files: deploy "${mfdir}/kube-node-drainer-ds.yaml" \ "${mfdir}/kube-node-drainer-asg-status-updater-de.yaml" - {{ end -}} - {{if .Experimental.KIAMSupport.Enabled -}} - # KIAM - kiam_tls_dir=/etc/kubernetes/ssl/kiam - vols="${vols} -v $kiam_tls_dir:$kiam_tls_dir" - - kubectl create secret generic kiam-server-tls -n kube-system \ - --from-file=$kiam_tls_dir/ca.pem \ - --from-file=$kiam_tls_dir/server.pem \ - --from-file=$kiam_tls_dir/server-key.pem --dry-run -o yaml | kubectl apply -n kube-system -f - - kubectl create secret generic kiam-agent-tls -n kube-system \ - --from-file=$kiam_tls_dir/ca.pem \ - --from-file=$kiam_tls_dir/agent.pem \ - --from-file=$kiam_tls_dir/agent-key.pem --dry-run -o yaml | kubectl apply -n kube-system -f - - deploy "${mfdir}/kiam-all.yaml" - {{ end -}} {{if .Experimental.Kube2IamSupport.Enabled -}} # KUBE2IAM @@ -2479,7 +2463,7 @@ write_files: replicas: 1 template: metadata: - {{if .Experimental.KIAMSupport.Enabled -}} + {{if (index .PluginConfigs "kiam").Enabled -}} annotations: iam.amazonaws.com/role: "{{$.ClusterName}}-IAMRoleResourcesAutoSave" {{ end -}} @@ -2602,7 +2586,7 @@ write_files: 'echo decrypting assets shopt -s nullglob set -o pipefail - for encKey in /etc/kubernetes/{ssl,additional-configs,auth}/{,kiam/}*.enc; do + for encKey in /etc/kubernetes/{ssl,additional-configs,auth}/*.enc; do if [ ! -f $encKey ]; then echo skipping non-existent file: $encKey 1>&2 continue @@ -5604,30 +5588,6 @@ write_files: mountPath: /dev {{end}} -{{if .Experimental.KIAMSupport.Enabled }} - - path: /etc/kubernetes/ssl/kiam/ca.pem - encoding: gzip+base64 - content: {{.AssetsConfig.KIAMCACert}} - - - path: /etc/kubernetes/ssl/kiam/server-key.pem{{if .AssetsEncryptionEnabled}}.enc{{end}} - encoding: gzip+base64 - content: {{.AssetsConfig.KIAMServerKey}} - - - path: /etc/kubernetes/ssl/kiam/server.pem - encoding: gzip+base64 - content: {{.AssetsConfig.KIAMServerCert}} - - - path: /etc/kubernetes/ssl/kiam/agent-key.pem{{if .AssetsEncryptionEnabled}}.enc{{end}} - encoding: gzip+base64 - content: {{.AssetsConfig.KIAMAgentKey}} - - - path: /etc/kubernetes/ssl/kiam/agent.pem - encoding: gzip+base64 - content: {{.AssetsConfig.KIAMAgentCert}} - -{{end}} - - # Without the namespace annotation the pod will be unable to assume any roles - path: /srv/kubernetes/manifests/kube-system-ns.yaml content: | @@ -5642,298 +5602,6 @@ write_files: {{ $key }}: {{ $value }} {{- end }} -{{if .Experimental.KIAMSupport.Enabled }} - - path: /srv/kubernetes/manifests/kiam-all.yaml - content: | - apiVersion: extensions/v1beta1 - kind: DaemonSet - metadata: - namespace: kube-system - name: kiam-server - spec: - updateStrategy: - rollingUpdate: - maxUnavailable: 100% - type: RollingUpdate - template: - metadata: - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9620" - labels: - app: kiam - role: server - spec: - priorityClassName: system-node-critical - tolerations: - - operator: Exists - effect: NoSchedule - - operator: Exists - effect: NoExecute - - operator: Exists - key: CriticalAddonsOnly - serviceAccountName: kiam-server - nodeSelector: - node.kubernetes.io/role: "master" - volumes: - - name: ssl-certs - hostPath: - path: /usr/share/ca-certificates - - name: tls - secret: - secretName: kiam-server-tls - containers: - - name: kiam - image: {{.Experimental.KIAMSupport.Image.RepoWithTag}} - command: - - {{if checkVersion ">= 3.0" .Experimental.KIAMSupport.Image.Tag}}/kiam{{else}}/server{{end}} - args: - {{if checkVersion ">= 3.0" .Experimental.KIAMSupport.Image.Tag -}} - - server - {{ end -}} - - --json-log - - --bind=0.0.0.0:443 - - --cert=/etc/kiam/tls/server.pem - - --key=/etc/kiam/tls/server-key.pem - - --ca=/etc/kiam/tls/ca.pem - - --role-base-arn-autodetect - - --sync=1m - - --prometheus-listen-addr=0.0.0.0:9620 - - --prometheus-sync-interval=5s - - --session-duration={{.Experimental.KIAMSupport.SessionDuration}} - volumeMounts: - - mountPath: /etc/ssl/certs - name: ssl-certs - - mountPath: /etc/kiam/tls - name: tls - livenessProbe: - exec: - command: - {{if checkVersion ">= 3.0" .Experimental.KIAMSupport.Image.Tag -}} - - /kiam - - health - - --gateway-timeout-creation=1s - {{ else -}} - - /health - - --server-address-refresh=2s - {{ end -}} - - --cert=/etc/kiam/tls/server.pem - - --key=/etc/kiam/tls/server-key.pem - - --ca=/etc/kiam/tls/ca.pem - - --server-address={{.Experimental.KIAMSupport.ServerAddresses.ServerAddress}} - - --timeout=5s - initialDelaySeconds: 10 - periodSeconds: 10 - timeoutSeconds: 10 - readinessProbe: - exec: - command: - {{if checkVersion ">= 3.0" .Experimental.KIAMSupport.Image.Tag -}} - - /kiam - - health - - --gateway-timeout-creation=1s - {{ else -}} - - /health - - --server-address-refresh=2s - {{ end -}} - - --cert=/etc/kiam/tls/server.pem - - --key=/etc/kiam/tls/server-key.pem - - --ca=/etc/kiam/tls/ca.pem - - --server-address={{.Experimental.KIAMSupport.ServerAddresses.ServerAddress}} - - --timeout=5s - initialDelaySeconds: 3 - periodSeconds: 10 - timeoutSeconds: 10 - resources: - requests: - cpu: {{ if .Experimental.KIAMSupport.ServerResources.Requests.Cpu }}{{ .Experimental.KIAMSupport.ServerResources.Requests.Cpu }}{{ else }}50m{{ end }} - memory: {{ if .Experimental.KIAMSupport.ServerResources.Requests.Memory }}{{ .Experimental.KIAMSupport.ServerResources.Requests.Memory }}{{ else }}10Mi{{ end }} - limits: - cpu: {{ if .Experimental.KIAMSupport.ServerResources.Limits.Cpu }}{{ .Experimental.KIAMSupport.ServerResources.Limits.Cpu }}{{ else }}200m{{ end }} - memory: {{ if .Experimental.KIAMSupport.ServerResources.Limits.Memory }}{{ .Experimental.KIAMSupport.ServerResources.Limits.Memory }}{{ else }}200Mi{{ end }} - --- - apiVersion: v1 - kind: Service - metadata: - name: kiam-server - namespace: kube-system - spec: - clusterIP: None - selector: - app: kiam - role: server - ports: - - name: grpclb - port: 443 - targetPort: 443 - protocol: TCP - --- - --- - kind: ServiceAccount - apiVersion: v1 - metadata: - name: kiam-server - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1beta1 - kind: ClusterRole - metadata: - name: kiam-read - rules: - - apiGroups: - - "" - resources: - - namespaces - - pods - verbs: - - watch - - get - - list - --- - apiVersion: rbac.authorization.k8s.io/v1beta1 - kind: ClusterRole - metadata: - name: kiam-write - rules: - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - --- - apiVersion: rbac.authorization.k8s.io/v1beta1 - kind: ClusterRoleBinding - metadata: - name: kiam-server - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: kiam-read - subjects: - - kind: ServiceAccount - name: kiam-server - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1beta1 - kind: ClusterRoleBinding - metadata: - name: kiam-write - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: kiam-write - subjects: - - kind: ServiceAccount - name: kiam-server - namespace: kube-system - --- - apiVersion: extensions/v1beta1 - kind: DaemonSet - metadata: - namespace: kube-system - name: kiam-agent - spec: - updateStrategy: - rollingUpdate: - maxUnavailable: 100% - type: RollingUpdate - template: - metadata: - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9620" - labels: - app: kiam - role: agent - spec: - priorityClassName: system-node-critical - tolerations: - - operator: Exists - effect: NoSchedule - - operator: Exists - effect: NoExecute - - operator: Exists - key: CriticalAddonsOnly - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node.kubernetes.io/role - operator: NotIn - values: - - master - volumes: - - name: ssl-certs - hostPath: - path: /usr/share/ca-certificates - - name: tls - secret: - secretName: kiam-agent-tls - - name: xtables - hostPath: - path: /run/xtables.lock - containers: - - name: kiam - securityContext: - capabilities: - add: ["NET_ADMIN"] - image: {{.Experimental.KIAMSupport.Image.RepoWithTag}} - command: - - {{if checkVersion ">= 3.0" .Experimental.KIAMSupport.Image.Tag}}/kiam{{else}}/agent{{end}} - args: - {{if checkVersion ">= 3.0" .Experimental.KIAMSupport.Image.Tag -}} - - agent - - --whitelist-route-regexp=.* - - --gateway-timeout-creation=1s - {{ end -}} - - --iptables - {{- if .Kubernetes.Networking.AmazonVPC.Enabled }} - - --host-interface=!eni0 - {{- else if eq .Kubernetes.Networking.SelfHosting.Type "canal" }} - - --host-interface=cali+ - {{- else}} - - --host-interface=cni0 - {{- end }} - - --json-log - - --port=8181 - - --cert=/etc/kiam/tls/agent.pem - - --key=/etc/kiam/tls/agent-key.pem - - --ca=/etc/kiam/tls/ca.pem - - --server-address={{.Experimental.KIAMSupport.ServerAddresses.AgentAddress}} - - --prometheus-listen-addr=0.0.0.0:9620 - - --prometheus-sync-interval=5s - env: - - name: HOST_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - volumeMounts: - - mountPath: /etc/ssl/certs - name: ssl-certs - - mountPath: /etc/kiam/tls - name: tls - - mountPath: /var/run/xtables.lock - name: xtables - livenessProbe: - httpGet: - path: /ping - port: 8181 - initialDelaySeconds: 3 - periodSeconds: 3 - resources: - requests: - cpu: {{ if .Experimental.KIAMSupport.AgentResources.Requests.Cpu }}{{ .Experimental.KIAMSupport.AgentResources.Requests.Cpu }}{{ else }}10m{{ end }} - memory: {{ if .Experimental.KIAMSupport.AgentResources.Requests.Memory }}{{ .Experimental.KIAMSupport.AgentResources.Requests.Memory }}{{ else }}5Mi{{ end }} - limits: - cpu: {{ if .Experimental.KIAMSupport.AgentResources.Limits.Cpu }}{{ .Experimental.KIAMSupport.AgentResources.Limits.Cpu }}{{ else }}200m{{ end }} - memory: {{ if .Experimental.KIAMSupport.AgentResources.Limits.Memory }}{{ .Experimental.KIAMSupport.AgentResources.Limits.Memory }}{{ else }}100Mi{{ end }} -{{end}} - - path: /opt/bin/retry owner: root:root permissions: 0755 diff --git a/cmd/render.go b/cmd/render.go index ebad1e23b..7c317ce8a 100644 --- a/cmd/render.go +++ b/cmd/render.go @@ -53,23 +53,23 @@ func init() { cmdRenderCredentials.Flags().StringVar(&renderCredentialsOpts.ApiServerKeyPath, "apiserver-key-path", "", "path to pem-encoded apiserver RSA key") cmdRenderCredentials.Flags().StringVar(&renderCredentialsOpts.EtcdClientKeyPath, "etcd-client-key-path", "", "path to pem-encoded etcd client RSA key") cmdRenderCredentials.Flags().StringVar(&renderCredentialsOpts.EtcdKeyPath, "etcd-key-path", "", "path to pem-encoded etcd RSA key") - cmdRenderCredentials.Flags().StringVar(&renderCredentialsOpts.KiamAgentKeyPath, "kiam-agent-key-path", "", "path to pem-encoded kiam agent RSA key") - cmdRenderCredentials.Flags().StringVar(&renderCredentialsOpts.KiamServerKeyPath, "kiam-server-key-path", "", "path to pem-encoded kiam server RSA key") cmdRenderCredentials.Flags().StringVar(&renderCredentialsOpts.KubeControllerManagerKeyPath, "kube-controller-manager-key-path", "", "path to pem-encoded kube controller manager RSA key") cmdRenderCredentials.Flags().StringVar(&renderCredentialsOpts.KubeSchedulerKeyPath, "kube-scheduler-key-path", "", "path to pem-encoded kube scheduler RSA key") cmdRenderCredentials.Flags().StringVar(&renderCredentialsOpts.ServiceAccountKeyPath, "service-account-key-path", "", "path to pem-encoded service account RSA key") cmdRenderCredentials.Flags().StringVar(&renderCredentialsOpts.WorkerKeyPath, "worker-key-path", "", "path to pem-encoded worker RSA key") - cmdRenderCredentials.Flags().BoolVar(&renderCredentialsOpts.KIAM, "kiam", true, "generate TLS assets for kiam") cmdRenderCredentials.Flags().BoolVar(&renderCredentialsOpts.AwsDebug, "aws-debug", false, "Log debug information from aws-sdk-go library") } func runCmdRender(_ *cobra.Command, args []string) error { - logger.Warn("'kube-aws render' is deprecated. See 'kube-aws render --help' for usage") if len(args) != 0 { return fmt.Errorf("render takes no arguments\n") } + if err := runCmdRenderStack(cmdRenderCredentials, args); err != nil { + return err + } + if _, err := os.Stat(renderCredentialsOpts.CaKeyPath); os.IsNotExist(err) { renderCredentialsOpts.GenerateCA = true } @@ -77,10 +77,6 @@ func runCmdRender(_ *cobra.Command, args []string) error { return err } - if err := runCmdRenderStack(cmdRenderCredentials, args); err != nil { - return err - } - return nil } @@ -103,5 +99,8 @@ Next steps: } func runCmdRenderCredentials(_ *cobra.Command, _ []string) error { + if _, err := os.Stat(renderCredentialsOpts.CaKeyPath); os.IsNotExist(err) { + renderCredentialsOpts.GenerateCA = true + } return root.RenderCredentials(configPath, renderCredentialsOpts) } diff --git a/credential/encrypted_assets.go b/credential/encrypted_assets.go index 2f7247e46..0daca49db 100644 --- a/credential/encrypted_assets.go +++ b/credential/encrypted_assets.go @@ -38,11 +38,6 @@ type RawAssetsOnMemory struct { EtcdKey []byte EtcdClientKey []byte EtcdTrustedCA []byte - KIAMServerCert []byte - KIAMServerKey []byte - KIAMAgentCert []byte - KIAMAgentKey []byte - KIAMCACert []byte ServiceAccountKey []byte // Other assets. @@ -74,11 +69,6 @@ type RawAssetsOnDisk struct { EtcdKey PlaintextFile EtcdClientKey PlaintextFile EtcdTrustedCA PlaintextFile - KIAMServerCert PlaintextFile - KIAMServerKey PlaintextFile - KIAMAgentCert PlaintextFile - KIAMAgentKey PlaintextFile - KIAMCACert PlaintextFile ServiceAccountKey PlaintextFile // Other assets. @@ -110,11 +100,6 @@ type EncryptedAssetsOnDisk struct { EtcdKey EncryptedFile EtcdClientKey EncryptedFile EtcdTrustedCA EncryptedFile - KIAMServerCert EncryptedFile - KIAMServerKey EncryptedFile - KIAMAgentCert EncryptedFile - KIAMAgentKey EncryptedFile - KIAMCACert EncryptedFile ServiceAccountKey EncryptedFile // Other encrypted assets. @@ -146,11 +131,6 @@ type CompactAssets struct { EtcdClientKey string EtcdKey string EtcdTrustedCA string - KIAMServerCert string - KIAMServerKey string - KIAMAgentCert string - KIAMAgentKey string - KIAMCACert string ServiceAccountKey string // Encrypted -> gzip -> base64 encoded assets. @@ -161,7 +141,7 @@ type CompactAssets struct { EncryptionConfig string } -func ReadRawAssets(dirname string, manageCertificates bool, caKeyRequiredOnController bool, kiamEnabled bool) (*RawAssetsOnDisk, error) { +func ReadRawAssets(dirname string, manageCertificates bool, caKeyRequiredOnController bool) (*RawAssetsOnDisk, error) { defaultTokensFile := "" defaultServiceAccountKey := "<<<" + filepath.Join(dirname, "apiserver-key.pem") defaultTLSBootstrapToken, err := RandomTokenString() @@ -219,14 +199,6 @@ func ReadRawAssets(dirname string, manageCertificates bool, caKeyRequiredOnContr if caKeyRequiredOnController { files = append(files, entry{name: "worker-ca-key.pem", data: &r.WorkerCAKey, defaultValue: nil, expiryCheck: true}) } - - if kiamEnabled { - files = append(files, entry{name: "kiam-server-key.pem", data: &r.KIAMServerKey, defaultValue: nil, expiryCheck: false}) - files = append(files, entry{name: "kiam-server.pem", data: &r.KIAMServerCert, defaultValue: nil, expiryCheck: true}) - files = append(files, entry{name: "kiam-agent-key.pem", data: &r.KIAMAgentKey, defaultValue: nil, expiryCheck: false}) - files = append(files, entry{name: "kiam-agent.pem", data: &r.KIAMAgentCert, defaultValue: nil, expiryCheck: true}) - files = append(files, entry{name: "kiam-ca.pem", data: &r.KIAMCACert, defaultValue: nil, expiryCheck: true}) - } } for _, file := range files { @@ -253,7 +225,7 @@ func ReadRawAssets(dirname string, manageCertificates bool, caKeyRequiredOnContr return r, nil } -func ReadOrEncryptAssets(dirname string, manageCertificates bool, caKeyRequiredOnController bool, kiamEnabled bool, store Store) (*EncryptedAssetsOnDisk, error) { +func ReadOrEncryptAssets(dirname string, manageCertificates bool, caKeyRequiredOnController bool, store Store) (*EncryptedAssetsOnDisk, error) { defaultTokensFile := "" defaultServiceAccountKey := "<<<" + filepath.Join(dirname, "apiserver-key.pem") defaultTLSBootstrapToken, err := RandomTokenString() @@ -308,14 +280,6 @@ func ReadOrEncryptAssets(dirname string, manageCertificates bool, caKeyRequiredO if caKeyRequiredOnController { files = append(files, entry{name: "worker-ca-key.pem", data: &r.WorkerCAKey, defaultValue: nil, readEncrypted: true, expiryCheck: false}) } - - if kiamEnabled { - files = append(files, entry{name: "kiam-server-key.pem", data: &r.KIAMServerKey, defaultValue: nil, readEncrypted: true, expiryCheck: false}) - files = append(files, entry{name: "kiam-server.pem", data: &r.KIAMServerCert, defaultValue: nil, readEncrypted: false, expiryCheck: true}) - files = append(files, entry{name: "kiam-agent-key.pem", data: &r.KIAMAgentKey, defaultValue: nil, readEncrypted: true, expiryCheck: false}) - files = append(files, entry{name: "kiam-agent.pem", data: &r.KIAMAgentCert, defaultValue: nil, readEncrypted: false, expiryCheck: true}) - files = append(files, entry{name: "kiam-ca.pem", data: &r.KIAMCACert, defaultValue: nil, readEncrypted: false, expiryCheck: true}) - } } for _, file := range files { @@ -353,7 +317,7 @@ func ReadOrEncryptAssets(dirname string, manageCertificates bool, caKeyRequiredO return r, nil } -func (r *RawAssetsOnMemory) WriteToDir(dirname string, includeCAKey bool, kiamEnabled bool) error { +func (r *RawAssetsOnMemory) WriteToDir(dirname string, includeCAKey bool) error { type asset struct { name string data []byte @@ -397,16 +361,6 @@ func (r *RawAssetsOnMemory) WriteToDir(dirname string, includeCAKey bool, kiamEn ) } - if kiamEnabled { - assets = append(assets, - asset{"kiam-server-key.pem", r.KIAMServerKey, true, ""}, - asset{"kiam-server.pem", r.KIAMServerCert, true, ""}, - asset{"kiam-agent-key.pem", r.KIAMAgentKey, true, ""}, - asset{"kiam-agent.pem", r.KIAMAgentCert, true, ""}, - asset{"kiam-ca.pem", r.KIAMCACert, true, "ca.pem"}, - ) - } - for _, asset := range assets { path := filepath.Join(dirname, asset.name) @@ -482,7 +436,7 @@ func (r *RawAssetsOnMemory) WriteToDir(dirname string, includeCAKey bool, kiamEn return nil } -func (r *EncryptedAssetsOnDisk) WriteToDir(dirname string, kiamEnabled bool) error { +func (r *EncryptedAssetsOnDisk) WriteToDir(dirname string) error { type asset struct { name string data EncryptedFile @@ -510,20 +464,10 @@ func (r *EncryptedAssetsOnDisk) WriteToDir(dirname string, kiamEnabled bool) err {"apiserver-aggregator-key.pem", r.APIServerAggregatorKey}, {"apiserver-aggregator.pem", r.APIServerAggregatorCert}, {"service-account-key.pem", r.ServiceAccountKey}, - {"tokens.csv", r.AuthTokens}, {"kubelet-tls-bootstrap-token", r.TLSBootstrapToken}, {"encryption-config.yaml", r.EncryptionConfig}, } - if kiamEnabled { - assets = append(assets, - asset{"kiam-server-key.pem", r.KIAMServerKey}, - asset{"kiam-server.pem", r.KIAMServerCert}, - asset{"kiam-agent-key.pem", r.KIAMAgentKey}, - asset{"kiam-agent.pem", r.KIAMAgentCert}, - asset{"kiam-ca.pem", r.KIAMCACert}, - ) - } for _, asset := range assets { if asset.name != "ca-key.pem" { @@ -574,11 +518,6 @@ func (r *RawAssetsOnDisk) Compact() (*CompactAssets, error) { EtcdTrustedCA: compact(r.EtcdTrustedCA), APIServerAggregatorCert: compact(r.APIServerAggregatorCert), APIServerAggregatorKey: compact(r.APIServerAggregatorKey), - KIAMAgentCert: compact(r.KIAMAgentCert), - KIAMAgentKey: compact(r.KIAMAgentKey), - KIAMServerCert: compact(r.KIAMServerCert), - KIAMServerKey: compact(r.KIAMServerKey), - KIAMCACert: compact(r.KIAMCACert), ServiceAccountKey: compact(r.ServiceAccountKey), AuthTokens: compact(r.AuthTokens), @@ -631,11 +570,6 @@ func (r *EncryptedAssetsOnDisk) Compact() (*CompactAssets, error) { EtcdTrustedCA: compact(r.EtcdTrustedCA), APIServerAggregatorCert: compact(r.APIServerAggregatorCert), APIServerAggregatorKey: compact(r.APIServerAggregatorKey), - KIAMAgentKey: compact(r.KIAMAgentKey), - KIAMAgentCert: compact(r.KIAMAgentCert), - KIAMServerKey: compact(r.KIAMServerKey), - KIAMServerCert: compact(r.KIAMServerCert), - KIAMCACert: compact(r.KIAMCACert), ServiceAccountKey: compact(r.ServiceAccountKey), AuthTokens: compact(r.AuthTokens), @@ -679,14 +613,14 @@ func NewKMSConfig(kmsKeyARN string, encSvc KMSEncryptionService, session *sessio } } -func ReadOrCreateEncryptedAssets(tlsAssetsDir string, manageCertificates bool, caKeyRequiredOnController bool, kiamEnabled bool, kmsConfig KMSConfig) (*EncryptedAssetsOnDisk, error) { +func ReadOrCreateEncryptedAssets(tlsAssetsDir string, manageCertificates bool, caKeyRequiredOnController bool, kmsConfig KMSConfig) (*EncryptedAssetsOnDisk, error) { store := kmsConfig.Store() - return ReadOrEncryptAssets(tlsAssetsDir, manageCertificates, caKeyRequiredOnController, kiamEnabled, store) + return ReadOrEncryptAssets(tlsAssetsDir, manageCertificates, caKeyRequiredOnController, store) } -func ReadOrCreateCompactAssets(assetsDir string, manageCertificates bool, caKeyRequiredOnController bool, kiamEnabled bool, kmsConfig KMSConfig) (*CompactAssets, error) { - encryptedAssets, err := ReadOrCreateEncryptedAssets(assetsDir, manageCertificates, caKeyRequiredOnController, kiamEnabled, kmsConfig) +func ReadOrCreateCompactAssets(assetsDir string, manageCertificates bool, caKeyRequiredOnController bool, kmsConfig KMSConfig) (*CompactAssets, error) { + encryptedAssets, err := ReadOrCreateEncryptedAssets(assetsDir, manageCertificates, caKeyRequiredOnController, kmsConfig) if err != nil { return nil, fmt.Errorf("failed to read/create encrypted assets: %v", err) } @@ -699,8 +633,8 @@ func ReadOrCreateCompactAssets(assetsDir string, manageCertificates bool, caKeyR return compactAssets, nil } -func ReadOrCreateUnencryptedCompactAssets(assetsDir string, manageCertificates bool, caKeyRequiredOnController bool, kiamEnabled bool) (*CompactAssets, error) { - unencryptedAssets, err := ReadRawAssets(assetsDir, manageCertificates, caKeyRequiredOnController, kiamEnabled) +func ReadOrCreateUnencryptedCompactAssets(assetsDir string, manageCertificates bool, caKeyRequiredOnController bool) (*CompactAssets, error) { + unencryptedAssets, err := ReadRawAssets(assetsDir, manageCertificates, caKeyRequiredOnController) if err != nil { return nil, fmt.Errorf("failed to read/create encrypted assets: %v", err) } diff --git a/credential/encrypted_assets_test.go b/credential/encrypted_assets_test.go index 2c719c193..e350c611d 100644 --- a/credential/encrypted_assets_test.go +++ b/credential/encrypted_assets_test.go @@ -10,6 +10,7 @@ import ( "strings" "fmt" + "github.com/aws/aws-sdk-go/service/kms" "github.com/kubernetes-incubator/kube-aws/test/helper" ) @@ -64,7 +65,7 @@ func TestReadOrCreateCompactAssets(t *testing.T) { // See https://github.com/kubernetes-incubator/kube-aws/issues/107 t.Run("CachedToPreventUnnecessaryNodeReplacement", func(t *testing.T) { - created, err := ReadOrCreateCompactAssets(dir, true, true, true, kmsConfig) + created, err := ReadOrCreateCompactAssets(dir, true, true, kmsConfig) if err != nil { t.Errorf("failed to read or update compact assets in %s : %v", dir, err) @@ -74,7 +75,7 @@ func TestReadOrCreateCompactAssets(t *testing.T) { // This depends on TestDummyEncryptService which ensures dummy encrypt service to produce different ciphertext for each encryption // created == read means that encrypted assets were loaded from cached files named *.pem.enc, instead of re-encrypting raw assets named *.pem files // TODO Use some kind of mocking framework for tests like this - read, err := ReadOrCreateCompactAssets(dir, true, true, true, kmsConfig) + read, err := ReadOrCreateCompactAssets(dir, true, true, kmsConfig) if err != nil { t.Errorf("failed to read or update compact assets in %s : %v", dir, err) @@ -90,7 +91,7 @@ func TestReadOrCreateCompactAssets(t *testing.T) { }) t.Run("RemoveFilesToRegenerate", func(t *testing.T) { - original, err := ReadOrCreateCompactAssets(dir, true, true, true, kmsConfig) + original, err := ReadOrCreateCompactAssets(dir, true, true, kmsConfig) if err != nil { t.Errorf("failed to read the original encrypted assets : %v", err) @@ -101,7 +102,7 @@ func TestReadOrCreateCompactAssets(t *testing.T) { "admin-key.pem.enc", "worker-key.pem.enc", "apiserver-key.pem.enc", "etcd-key.pem.enc", "etcd-client-key.pem.enc", "worker-ca-key.pem.enc", "kube-controller-manager-key.pem.enc", "kube-scheduler-key.pem.enc", - "kiam-agent-key.pem.enc", "kiam-server-key.pem.enc", "apiserver-aggregator-key.pem.enc", + "apiserver-aggregator-key.pem.enc", } for _, filename := range files { @@ -111,7 +112,7 @@ func TestReadOrCreateCompactAssets(t *testing.T) { } } - regenerated, err := ReadOrCreateCompactAssets(dir, true, true, true, kmsConfig) + regenerated, err := ReadOrCreateCompactAssets(dir, true, true, kmsConfig) if err != nil { t.Errorf("failed to read the regenerated encrypted assets : %v", err) @@ -127,9 +128,6 @@ func TestReadOrCreateCompactAssets(t *testing.T) { {"KubeSchedulerCert", original.KubeSchedulerCert, regenerated.KubeSchedulerCert}, {"EtcdClientCert", original.EtcdClientCert, regenerated.EtcdClientCert}, {"EtcdCert", original.EtcdCert, regenerated.EtcdCert}, - {"KIAMAgentCert", original.KIAMAgentCert, regenerated.KIAMAgentCert}, - {"KIAMServerCert", original.KIAMServerCert, regenerated.KIAMServerCert}, - {"KIAMCACert", original.KIAMCACert, regenerated.KIAMCACert}, {"APIServerAggregatorCert", original.APIServerAggregatorCert, regenerated.APIServerAggregatorCert}, } { if v[1] != v[2] { @@ -146,8 +144,6 @@ func TestReadOrCreateCompactAssets(t *testing.T) { {"KubeSchedulerKey", original.KubeSchedulerKey, regenerated.KubeSchedulerKey}, {"EtcdClientKey", original.EtcdClientKey, regenerated.EtcdClientKey}, {"EtcdKey", original.EtcdKey, regenerated.EtcdKey}, - {"KIAMAgentKey", original.KIAMAgentKey, regenerated.KIAMAgentKey}, - {"KIAMServerKey", original.KIAMServerKey, regenerated.KIAMServerKey}, {"APIServerAggregatorKey", original.APIServerAggregatorKey, regenerated.APIServerAggregatorKey}, } { if v[1] == v[2] { @@ -167,13 +163,13 @@ func TestReadOrCreateCompactAssets(t *testing.T) { func TestReadOrCreateUnEncryptedCompactAssets(t *testing.T) { run := func(dir string, caKeyRequiredOnController bool, t *testing.T) { t.Run("CachedToPreventUnnecessaryNodeReplacementOnUnencrypted", func(t *testing.T) { - created, err := ReadOrCreateUnencryptedCompactAssets(dir, true, caKeyRequiredOnController, true) + created, err := ReadOrCreateUnencryptedCompactAssets(dir, true, caKeyRequiredOnController) if err != nil { t.Errorf("failed to read or update compact assets in %s : %v", dir, err) } - read, err := ReadOrCreateUnencryptedCompactAssets(dir, true, caKeyRequiredOnController, true) + read, err := ReadOrCreateUnencryptedCompactAssets(dir, true, caKeyRequiredOnController) if err != nil { t.Errorf("failed to read or update compact assets in %s : %v", dir, err) diff --git a/credential/generator.go b/credential/generator.go index d063a0c42..36e69fb92 100644 --- a/credential/generator.go +++ b/credential/generator.go @@ -31,8 +31,6 @@ type GeneratorOptions struct { GenerateCA bool CaCertPath string CommonName string - // KIAM is set to true when you want kube-aws to render TLS assets for uswitch/kiam - KIAM bool // Paths for private certificate keys. AdminKeyPath string ApiServerAggregatorKeyPath string @@ -40,8 +38,6 @@ type GeneratorOptions struct { CaKeyPath string EtcdClientKeyPath string EtcdKeyPath string - KiamAgentKeyPath string - KiamServerKeyPath string KubeControllerManagerKeyPath string KubeSchedulerKeyPath string ServiceAccountKeyPath string @@ -87,13 +83,13 @@ func (c Generator) GenerateAssetsOnDisk(dir string, o GeneratorOptions) (*RawAss logger.Info("--> Writing to the storage") alsoWriteCAKey := o.GenerateCA || c.ManageCertificates - if err := assets.WriteToDir(dir, alsoWriteCAKey, o.KIAM); err != nil { + if err := assets.WriteToDir(dir, alsoWriteCAKey); err != nil { return nil, fmt.Errorf("Error creating assets: %v", err) } { logger.Info("--> Verifying the result") - verified, err := ReadRawAssets(dir, c.ManageCertificates, c.ManageCertificates, o.KIAM) + verified, err := ReadRawAssets(dir, c.ManageCertificates, c.ManageCertificates) if err != nil { return nil, fmt.Errorf("failed verifying the result: %v", err) @@ -132,8 +128,6 @@ func (c Generator) GenerateAssetsOnMemory(caKey *rsa.PrivateKey, caCert *x509.Ce generatorOptions.AdminKeyPath: nil, generatorOptions.EtcdKeyPath: nil, generatorOptions.EtcdClientKeyPath: nil, - generatorOptions.KiamAgentKeyPath: nil, - generatorOptions.KiamServerKeyPath: nil, generatorOptions.ServiceAccountKeyPath: nil, generatorOptions.ApiServerAggregatorKeyPath: nil, } @@ -284,41 +278,5 @@ func (c Generator) GenerateAssetsOnMemory(caKey *rsa.PrivateKey, caCert *x509.Ce EncryptionConfig: []byte(encryptionConfig), } - if generatorOptions.KIAM { - // See https://github.com/uswitch/kiam/blob/master/docs/agent.json - agentConfig := pki.ClientCertConfig{ - CommonName: "Kiam Agent", - Duration: certDuration, - } - kiamAgentCert, err := pki.NewSignedClientCertificate(agentConfig, privateKeys[generatorOptions.KiamAgentKeyPath], caCert, caKey) - if err != nil { - return nil, err - } - // See https://github.com/uswitch/kiam/blob/master/docs/server.json - serverConfig := pki.ClientCertConfig{ - CommonName: "Kiam Server", - DNSNames: append( - []string{ - "kiam-server", - "kiam-server:443", - "localhost", - "localhost:443", - "localhost:9610", - }, - ), - Duration: certDuration, - } - kiamServerCert, err := pki.NewSignedKIAMCertificate(serverConfig, privateKeys[generatorOptions.KiamServerKeyPath], caCert, caKey) - if err != nil { - return nil, err - } - - r.KIAMCACert = pki.EncodeCertificatePEM(caCert) - r.KIAMAgentCert = pki.EncodeCertificatePEM(kiamAgentCert) - r.KIAMAgentKey = pki.EncodePrivateKeyPEM(privateKeys[generatorOptions.KiamAgentKeyPath]) - r.KIAMServerCert = pki.EncodeCertificatePEM(kiamServerCert) - r.KIAMServerKey = pki.EncodePrivateKeyPEM(privateKeys[generatorOptions.KiamServerKeyPath]) - } - return r, nil } diff --git a/credential/pki.go b/credential/pki.go index e03e3c4b2..44d5e8e6f 100644 --- a/credential/pki.go +++ b/credential/pki.go @@ -1,10 +1,13 @@ package credential import ( - "github.com/kubernetes-incubator/kube-aws/pkg/api" - "github.com/kubernetes-incubator/kube-aws/pki" + "fmt" "io/ioutil" "os" + + "github.com/kubernetes-incubator/kube-aws/logger" + "github.com/kubernetes-incubator/kube-aws/pkg/api" + "github.com/kubernetes-incubator/kube-aws/pki" ) type ProtectedPKI struct { @@ -19,47 +22,60 @@ func NewProtectedPKI(enc Encryptor) *ProtectedPKI { } } -func (pki *ProtectedPKI) fileExists(path string) bool { - _, err := os.Stat(path) - return err == nil -} - -func (pki *ProtectedPKI) write(path string, data []byte) error { - return ioutil.WriteFile(path, data, 0644) -} - -func (pki *ProtectedPKI) CreateKeyaPair(spec api.KeyPairSpec) error { - keypair, err := pki.GenerateKeyPair(spec) +func (ppki *ProtectedPKI) CreateKeyaPair(spec api.KeyPairSpec) error { + var signer *pki.KeyPair + if spec.Signer != "" { + signerCert, err := ioutil.ReadFile(spec.SignerCertPath()) + if err != nil { + return fmt.Errorf("failed to read signer certificate %s for creating %s: %v", spec.SignerCertPath(), spec.Name, err) + } + signerKey, err := ioutil.ReadFile(spec.SignerKeyPath()) + if err != nil { + return fmt.Errorf("failed to read signer key %s for creating %s: %v", spec.SignerKeyPath(), spec.Name, err) + } + signer, err = pki.KeyPairFromPEMs(spec.Signer, signerCert, signerKey) + } + keypair, err := ppki.GenerateKeyPair(spec, signer) if err != nil { return err } keypath := spec.KeyPath() keypem := keypair.KeyInPEM() - if _, err := CreateEncryptedFile(keypath, keypem, pki); err != nil { + logger.Infof("Writing key pem file %s", keypath) + if err := ioutil.WriteFile(keypath, keypem, 0644); err != nil { return err } crtpath := spec.CertPath() crtpem := keypair.CertInPEM() - if err := pki.write(crtpath, crtpem); err != nil { + logger.Infof("Writing certificate pem file %s", crtpath) + if err := ioutil.WriteFile(crtpath, crtpem, 0644); err != nil { return err } return nil } -func (pki *ProtectedPKI) EnsureKeyPairsCreated(specs []api.KeyPairSpec) error { +func (ppki *ProtectedPKI) EnsureKeyPairsCreated(specs []api.KeyPairSpec) error { for _, spec := range specs { keypath := spec.KeyPath() shapath := spec.KeyPath() + ".fingerprint" encpath := spec.EncryptedKeyPath() crtpath := spec.CertPath() - if !pki.fileExists(keypath) && !pki.fileExists(encpath) && !pki.fileExists(shapath) && !pki.fileExists(crtpath) { - if err := pki.CreateKeyaPair(spec); err != nil { + if !fileExists(keypath) && !fileExists(encpath) && !fileExists(shapath) && !fileExists(crtpath) { + if err := ppki.CreateKeyaPair(spec); err != nil { return err } } } return nil } + +func fileExists(path string) bool { + info, err := os.Stat(path) + if os.IsNotExist(err) { + return false + } + return !info.IsDir() +} diff --git a/docs/getting-started/step-6-configure-add-ons.md b/docs/getting-started/step-6-configure-add-ons.md index d9553751b..d2941a23c 100644 --- a/docs/getting-started/step-6-configure-add-ons.md +++ b/docs/getting-started/step-6-configure-add-ons.md @@ -53,8 +53,8 @@ To allow kube2iam or kiam deployed to worker and controller nodes to assume targ } ``` - To add the policy to controller nodes, set `experimental.kube2IamSupport.enabled` or `experimental.kiamSupport.enabled` to `true` in your `cluster.yaml` (but not both). - For worker nodes, it is `worker.nodePools[].kube2IamSupport.enabled` or `worker.nodePools[].kiamSupport.enabled`. + To add the policy to controller nodes, set `experimental.kube2IamSupport.enabled` or `kubeAwsPlugins.kiam.enabled` to `true` in your `cluster.yaml` (but not both). + For worker nodes, it is `worker.nodePools[].kube2IamSupport.enabled`. 2. Target IAM roles needs to change trust relationships to allow kube-aws worker/controller IAM role to assume the target roles. diff --git a/pkg/api/cluster.go b/pkg/api/cluster.go index c20fb83eb..51f0cc609 100644 --- a/pkg/api/cluster.go +++ b/pkg/api/cluster.go @@ -79,12 +79,6 @@ func NewDefaultCluster() *Cluster { Disk: "xvdb", Filesystem: "xfs", }, - KIAMSupport: KIAMSupport{ - Enabled: false, - Image: Image{Repo: "quay.io/uswitch/kiam", Tag: "v3.2", RktPullDocker: false}, - SessionDuration: "30m", - ServerAddresses: KIAMServerAddresses{ServerAddress: "localhost:443", AgentAddress: "kiam-server:443"}, - }, Kube2IamSupport: Kube2IamSupport{ Enabled: false, }, diff --git a/pkg/api/keypair_spec.go b/pkg/api/keypair_spec.go index a77bdae23..85c9799a4 100644 --- a/pkg/api/keypair_spec.go +++ b/pkg/api/keypair_spec.go @@ -29,3 +29,11 @@ func (spec KeyPairSpec) KeyPath() string { func (spec KeyPairSpec) CertPath() string { return filepath.Join("credentials", fmt.Sprintf("%s.pem", spec.Name)) } + +func (spec KeyPairSpec) SignerCertPath() string { + return filepath.Join("credentials", fmt.Sprintf("%s.pem", spec.Signer)) +} + +func (spec KeyPairSpec) SignerKeyPath() string { + return filepath.Join("credentials", fmt.Sprintf("%s-key.pem", spec.Signer)) +} diff --git a/pkg/api/types.go b/pkg/api/types.go index f8e9107ab..7b166ffa4 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -1,7 +1,5 @@ package api -import "fmt" - type Worker struct { APIEndpointName string `yaml:"apiEndpointName,omitempty"` NodePools []WorkerNodePool `yaml:"nodePools,omitempty"` @@ -25,7 +23,6 @@ type Experimental struct { AwsEnvironment AwsEnvironment `yaml:"awsEnvironment"` AwsNodeLabels AwsNodeLabels `yaml:"awsNodeLabels"` EphemeralImageStorage EphemeralImageStorage `yaml:"ephemeralImageStorage"` - KIAMSupport KIAMSupport `yaml:"kiamSupport,omitempty"` Kube2IamSupport Kube2IamSupport `yaml:"kube2IamSupport,omitempty"` GpuSupport GpuSupport `yaml:"gpuSupport,omitempty"` KubeletOpts string `yaml:"kubeletOpts,omitempty"` @@ -43,10 +40,6 @@ func (c Experimental) Validate(name string) error { return err } - if c.Kube2IamSupport.Enabled && c.KIAMSupport.Enabled { - return fmt.Errorf("at '%s', you can enable kube2IamSupport or kiamSupport, but not both", name) - } - return nil } @@ -115,20 +108,6 @@ type EphemeralImageStorage struct { Filesystem string `yaml:"filesystem"` } -type KIAMSupport struct { - Enabled bool `yaml:"enabled"` - Image Image `yaml:"image,omitempty"` - SessionDuration string `yaml:"sessionDuration,omitempty"` - ServerAddresses KIAMServerAddresses `yaml:"serverAddresses,omitempty"` - ServerResources ComputeResources `yaml:"serverResources,omitempty"` - AgentResources ComputeResources `yaml:"agentResources,omitempty"` -} - -type KIAMServerAddresses struct { - ServerAddress string `yaml:"serverAddress,omitempty"` - AgentAddress string `yaml:"agentAddress,omitempty"` -} - type Kube2IamSupport struct { Enabled bool `yaml:"enabled"` } diff --git a/pkg/model/cluster_test.go b/pkg/model/cluster_test.go index 4b2868abe..991b04116 100644 --- a/pkg/model/cluster_test.go +++ b/pkg/model/cluster_test.go @@ -1415,20 +1415,6 @@ kubernetesVersion: v1.10.2 } } -func TestKube2IamKiamClash(t *testing.T) { - config := ` -experimental: - kube2IamSupport: - enabled: true - kiamSupport: - enabled: true -` - confBody := singleAzConfigYaml + config - _, err := ClusterFromBytes([]byte(confBody)) - if err == nil || !strings.Contains(err.Error(), "not both") { - t.Errorf("expected config to cause error as kube2iam and kiam cannot be enabled together: %s\n%s", err, confBody) - } -} func TestKMSArnValidateRegion(t *testing.T) { config := `keyName: test-key-name s3URI: s3://mybucket/mydir diff --git a/pkg/model/credentials.go b/pkg/model/credentials.go index 4900ac0da..337cebee3 100644 --- a/pkg/model/credentials.go +++ b/pkg/model/credentials.go @@ -14,14 +14,14 @@ func LoadCredentials(sess *session.Session, cfg *Config, opts api.StackTemplateO func (s *Context) LoadCredentials(cfg *Config, opts api.StackTemplateOptions) (*credential.CompactAssets, error) { if cfg.AssetsEncryptionEnabled() { kmsConfig := credential.NewKMSConfig(cfg.KMSKeyARN, s.ProvidedEncryptService, s.Session) - compactAssets, err := credential.ReadOrCreateCompactAssets(opts.AssetsDir, cfg.ManageCertificates, true, cfg.Experimental.KIAMSupport.Enabled, kmsConfig) + compactAssets, err := credential.ReadOrCreateCompactAssets(opts.AssetsDir, cfg.ManageCertificates, true, kmsConfig) if err != nil { return nil, err } return compactAssets, nil } else { - rawAssets, err := credential.ReadOrCreateUnencryptedCompactAssets(opts.AssetsDir, cfg.ManageCertificates, true, cfg.Experimental.KIAMSupport.Enabled) + rawAssets, err := credential.ReadOrCreateUnencryptedCompactAssets(opts.AssetsDir, cfg.ManageCertificates, true) if err != nil { return nil, err } diff --git a/pkg/model/credentials_test.go b/pkg/model/credentials_test.go index c177453f4..6cd7ccfd3 100644 --- a/pkg/model/credentials_test.go +++ b/pkg/model/credentials_test.go @@ -23,7 +23,7 @@ func genAssets(t *testing.T) *credential.RawAssetsOnMemory { } cfg, err := Compile(c, api.ClusterOptions{}) r := NewCredentialGenerator(cfg) - assets, err := r.GenerateAssetsOnMemory(caKey, caCert, credential.GeneratorOptions{KIAM: true}) + assets, err := r.GenerateAssetsOnMemory(caKey, caCert, credential.GeneratorOptions{}) if err != nil { t.Fatalf("failed generating assets: %v", err) } diff --git a/pkg/model/stack_test.go b/pkg/model/stack_test.go index daa54aa6b..92d42fa58 100644 --- a/pkg/model/stack_test.go +++ b/pkg/model/stack_test.go @@ -15,12 +15,13 @@ import ( "strings" "testing" + "os" + "path/filepath" + "github.com/go-yaml/yaml" "github.com/kubernetes-incubator/kube-aws/credential" "github.com/kubernetes-incubator/kube-aws/pkg/api" "github.com/kubernetes-incubator/kube-aws/plugin/clusterextension" - "os" - "path/filepath" ) /* @@ -756,7 +757,6 @@ func clusterToStackForTesting(c *api.Cluster, opts api.StackTemplateOptions) (*S genopts := credential.GeneratorOptions{ GenerateCA: true, - KIAM: true, } if _, err := sess.GenerateAssetsOnDisk(compiled, opts.AssetsDir, genopts); err != nil { diff --git a/pki/keypair.go b/pki/keypair.go index ae23e6e18..385b30223 100644 --- a/pki/keypair.go +++ b/pki/keypair.go @@ -1,5 +1,31 @@ package pki +import ( + "crypto/rsa" + "crypto/x509" + "fmt" +) + +func KeyPairFromPEMs(id string, certpem []byte, keypem []byte) (*KeyPair, error) { + var cert *x509.Certificate + var key *rsa.PrivateKey + var err error + if cert, err = DecodeCertificatePEM(certpem); err != nil { + return nil, fmt.Errorf("failed to decode certificate pem: %v", err) + } + if key, err = DecodePrivateKeyPEM(keypem); err != nil { + return nil, fmt.Errorf("failed to decode private key pem: %v", err) + } + kp := KeyPair{ + Key: key, + Cert: cert, + id: id, + keyPem: keypem, + certPem: certpem, + } + return &kp, nil +} + func (keypair *KeyPair) KeyInPEM() []byte { if keypair.keyPem == nil { keypair.keyPem = EncodePrivateKeyPEM(keypair.Key) diff --git a/pki/pki.go b/pki/pki.go index dbd9ff10a..a8b1e3061 100644 --- a/pki/pki.go +++ b/pki/pki.go @@ -2,15 +2,18 @@ package pki import ( "crypto/rand" + "crypto/rsa" "crypto/x509" "crypto/x509/pkix" "errors" "fmt" - "github.com/kubernetes-incubator/kube-aws/pkg/api" "math" "math/big" "net" "time" + + "github.com/kubernetes-incubator/kube-aws/logger" + "github.com/kubernetes-incubator/kube-aws/pkg/api" ) type PKI struct { @@ -20,7 +23,8 @@ func NewPKI() *PKI { return &PKI{} } -func (pki *PKI) GenerateKeyPair(spec api.KeyPairSpec) (*KeyPair, error) { +func (pki *PKI) GenerateKeyPair(spec api.KeyPairSpec, signer *KeyPair) (*KeyPair, error) { + logger.Debugf("GenerateKeyPair - spec: %+v", spec) key, err := NewPrivateKey() if err != nil { return nil, err @@ -62,6 +66,7 @@ func (pki *PKI) GenerateKeyPair(spec api.KeyPairSpec) (*KeyPair, error) { ips[i] = net.ParseIP(ipStr) } + logger.Debugf("Generating x509 certificate template") tmpl := x509.Certificate{ SerialNumber: serialNumber, Subject: pkix.Name{ @@ -78,16 +83,23 @@ func (pki *PKI) GenerateKeyPair(spec api.KeyPairSpec) (*KeyPair, error) { IsCA: isCA, } - caCert := &tmpl - caKey := key - - if spec.Signer != "" { - // TODO load signer key and cert - caKey = nil - caCert = nil + // handle self-signed/CA certificates or certs signed by a CA + var signerCert *x509.Certificate + var signerKey *rsa.PrivateKey + if signer == nil { + if spec.Signer != "" { + return nil, fmt.Errorf("The certificate spec includes a signer but singer KeyPair is missing") + } + logger.Debugf("This certificate is going to be self-signed!") + signerCert = &tmpl + signerKey = key + } else { + signerCert = signer.Cert + signerKey = signer.Key } - certAsn1DERData, err := x509.CreateCertificate(rand.Reader, &tmpl, caCert, key.Public(), caKey) + logger.Debugf("Creating x509 certificate...") + certAsn1DERData, err := x509.CreateCertificate(rand.Reader, &tmpl, signerCert, key.Public(), signerKey) if err != nil { return nil, err } @@ -96,5 +108,6 @@ func (pki *PKI) GenerateKeyPair(spec api.KeyPairSpec) (*KeyPair, error) { return nil, err } + logger.Debugf("returning keypair..") return &KeyPair{Key: key, Cert: cert}, nil } diff --git a/pki/x509.go b/pki/x509.go index 8388a2424..c2853b0b5 100644 --- a/pki/x509.go +++ b/pki/x509.go @@ -131,38 +131,3 @@ func NewSignedClientCertificate(cfg ClientCertConfig, key *rsa.PrivateKey, caCer } return x509.ParseCertificate(certDERBytes) } - -func NewSignedKIAMCertificate(cfg ClientCertConfig, key *rsa.PrivateKey, caCert *x509.Certificate, caKey *rsa.PrivateKey) (*x509.Certificate, error) { - ips := make([]net.IP, len(cfg.IPAddresses)) - for i, ipStr := range cfg.IPAddresses { - ips[i] = net.ParseIP(ipStr) - } - - serial, err := rand.Int(rand.Reader, new(big.Int).SetInt64(math.MaxInt64)) - if err != nil { - return nil, err - } - - if cfg.Duration <= 0 { - return nil, errors.New("signed client cert duration must not be negative or zero") - } - - certTmpl := x509.Certificate{ - Subject: pkix.Name{ - CommonName: cfg.CommonName, - Organization: append(caCert.Subject.Organization, cfg.Organization...), - }, - DNSNames: cfg.DNSNames, - IPAddresses: ips, - SerialNumber: serial, - NotBefore: caCert.NotBefore, - NotAfter: time.Now().Add(cfg.Duration).UTC(), - KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, - } - certDERBytes, err := x509.CreateCertificate(rand.Reader, &certTmpl, caCert, key.Public(), caKey) - if err != nil { - return nil, err - } - return x509.ParseCertificate(certDERBytes) -} diff --git a/plugin/clusterextension/extras.go b/plugin/clusterextension/extras.go index 2507bfa80..30aee3c55 100644 --- a/plugin/clusterextension/extras.go +++ b/plugin/clusterextension/extras.go @@ -57,14 +57,12 @@ func (e ClusterExtension) KeyPairSpecs() []api.KeyPairSpec { } func (e ClusterExtension) RootStack(config interface{}) (*stack, error) { - logger.Debugf("Generating Plugin extras for root cloudformation stack") return e.stackExt("root", config, func(p *api.Plugin) api.Stack { return p.Spec.Cluster.CloudFormation.Stacks.Root }) } func (e ClusterExtension) NetworkStack(config interface{}) (*stack, error) { - logger.Debugf("Generating Plugin extras for network cloudformation stack") return e.stackExt("network", config, func(p *api.Plugin) api.Stack { return p.Spec.Cluster.CloudFormation.Stacks.Network }) @@ -126,7 +124,6 @@ func (e ClusterExtension) stackExt(name string, config interface{}, src func(p * tags := map[string]interface{}{} err := e.foreachEnabledPlugins(func(p *api.Plugin, pc *api.PluginConfig) error { - logger.Debugf("extras.go stackExt() foreachEnabledPlugins extending stack %s: %+v into %+v", name, pc.Values, p.Spec.Cluster.Values) values, err := pluginutil.MergeValues(p.Spec.Cluster.Values, pc.Values) if err != nil { return err @@ -139,6 +136,9 @@ func (e ClusterExtension) stackExt(name string, config interface{}, src func(p * if err != nil { return fmt.Errorf("failed to load additional resources for %s stack: %v", name, err) } + if l := len(m); l > 0 { + logger.Infof("plugin %s extended stack %s with %d resources", p.Name, name, l) + } for k, v := range m { resources[k] = v } @@ -147,6 +147,9 @@ func (e ClusterExtension) stackExt(name string, config interface{}, src func(p * if err != nil { return fmt.Errorf("failed to load additional outputs for %s stack: %v", name, err) } + if l := len(m); l > 0 { + logger.Infof("plugin %s extended stack %s with %d outputs", p.Name, name, l) + } for k, v := range m { outputs[k] = v } @@ -155,6 +158,9 @@ func (e ClusterExtension) stackExt(name string, config interface{}, src func(p * if err != nil { return fmt.Errorf("failed to load additional tags for %s stack: %v", name, err) } + if l := len(m); l > 0 { + logger.Infof("plugin %s extended stack %s with %d tags", p.Name, name, l) + } for k, v := range m { tags[k] = v } @@ -311,6 +317,7 @@ func (e ClusterExtension) Worker(config interface{}) (*worker, error) { for _, p := range e.plugins { if enabled, pc := p.EnabledIn(e.Configs); enabled { + logger.Debugf("Adding worker extensions from plugin %s", p.Name) values, err := pluginutil.MergeValues(p.Spec.Cluster.Values, pc.Values) if err != nil { return nil, err @@ -321,37 +328,63 @@ func (e ClusterExtension) Worker(config interface{}) (*worker, error) { if err != nil { return nil, fmt.Errorf("failed adding systemd units to worker: %v", err) } + if l := len(extraUnits); l > 0 { + logger.Infof("plugin %s added %d extra worker systemd units", p.Name, l) + } systemdUnits = append(systemdUnits, extraUnits...) extraArchivedFiles, extraFiles, extraConfigSetFiles, err := renderMachineFilesAndConfigSets(render, p.Spec.Cluster.Roles.Worker.Files) if err != nil { return nil, fmt.Errorf("failed adding files to worker: %v", err) } + if l := len(extraArchivedFiles); l > 0 { + logger.Infof("plugin %s added %d extra worker extra archive files", p.Name, l) + } + if l := len(extraFiles); l > 0 { + logger.Infof("plugin %s added %d extra worker extra files", p.Name, l) + } + if l := len(extraConfigSetFiles); l > 0 { + logger.Infof("plugin %s added %d extra worker extra config-set files", p.Name, l) + } archivedFiles = append(archivedFiles, extraArchivedFiles...) files = append(files, extraFiles...) configsets[p.Name] = map[string]map[string]interface{}{ "files": extraConfigSetFiles, } + if l := len(p.Spec.Cluster.Machine.Roles.Worker.IAM.Policy.Statements); l > 0 { + logger.Infof("plugin %s added %d extra worker iam policies", p.Name, l) + } iamStatements = append(iamStatements, p.Spec.Cluster.Machine.Roles.Worker.IAM.Policy.Statements...) + if l := len(p.Spec.Cluster.Machine.Roles.Worker.Kubelet.NodeLabels); l > 0 { + logger.Infof("plugin %s added %d extra worker node labels", p.Name, l) + } for k, v := range p.Spec.Cluster.Machine.Roles.Worker.Kubelet.NodeLabels { nodeLabels[k] = v } + if l := len(p.Spec.Cluster.Machine.Roles.Worker.Kubelet.FeatureGates); l > 0 { + logger.Infof("plugin %s added %d extra worker kubelet feature gates", p.Name, l) + } for k, v := range p.Spec.Cluster.Machine.Roles.Worker.Kubelet.FeatureGates { featureGates[k] = v } if p.Spec.Cluster.Machine.Roles.Worker.Kubelet.Kubeconfig != "" { + logger.Infof("plugin %s changed the worker kubeconfig", p.Name) kubeconfig = p.Spec.Cluster.Machine.Roles.Worker.Kubelet.Kubeconfig } if len(p.Spec.Cluster.Machine.Roles.Controller.Kubelet.Mounts) > 0 { + logger.Infof("plugin %s added %d worker kubelet mounts", p.Name, len(p.Spec.Cluster.Machine.Roles.Controller.Kubelet.Mounts)) kubeletMounts = append(kubeletMounts, p.Spec.Cluster.Machine.Roles.Controller.Kubelet.Mounts...) } extraKubeletFlags, err := getFlags(render, p.Spec.Cluster.Kubernetes.Kubelet.Flags) + if l := len(extraKubeletFlags); l > 0 { + logger.Infof("plugin %s added %d extra worker kubelet command-line flags", p.Name, l) + } if err != nil { return nil, err } @@ -374,14 +407,12 @@ func (e ClusterExtension) Worker(config interface{}) (*worker, error) { } func (e ClusterExtension) ControlPlaneStack(config interface{}) (*stack, error) { - logger.Debugf("Generating Plugin extras for control-plane cloudformation stack") return e.stackExt("control-plane", config, func(p *api.Plugin) api.Stack { return p.Spec.Cluster.CloudFormation.Stacks.ControlPlane }) } func (e ClusterExtension) EtcdStack(config interface{}) (*stack, error) { - logger.Debugf("Generating Plugin extras for etcd cloudformation stack") return e.stackExt("etcd", config, func(p *api.Plugin) api.Stack { return p.Spec.Cluster.CloudFormation.Stacks.Etcd }) @@ -396,7 +427,7 @@ func renderKubernetesManifests(pluginName string, r *plugincontents.TemplateRend for _, m := range mspecs { rendered, ma, err := regularOrConfigSetFile(m.RemoteFileSpec, r) if err != nil { - return files, manifests, configsetFiles, nil + return files, manifests, configsetFiles, fmt.Errorf("Failed to render plugin kubernetes manifest: %v", err) } var name string if m.Name == "" { @@ -508,6 +539,7 @@ func (e ClusterExtension) Controller(clusterConfig interface{}) (*controller, er for _, p := range e.plugins { //fmt.Fprintf(os.Stderr, "plugin=%+v configs=%+v", p, e.configs) if enabled, pc := p.EnabledIn(e.Configs); enabled { + logger.Debugf("Adding controller extensions from plugin %s", p.Name) values, err := pluginutil.MergeValues(p.Spec.Cluster.Values, pc.Values) if err != nil { return nil, err @@ -518,59 +550,98 @@ func (e ClusterExtension) Controller(clusterConfig interface{}) (*controller, er if err != nil { return nil, err } + if l := len(extraApiServerFlags); l > 0 { + logger.Infof("plugin %s added %d extra controller api server command-line flags", p.Name, l) + } apiServerFlags = append(apiServerFlags, extraApiServerFlags...) extraControllerManagerFlags, err := getFlags(render, p.Spec.Cluster.Kubernetes.ControllerManager.Flags) if err != nil { return nil, err } + if l := len(extraControllerManagerFlags); l > 0 { + logger.Infof("plugin %s added %d extra controller controller-manager command-line flags", p.Name, l) + } controllerFlags = append(controllerFlags, extraControllerManagerFlags...) extraKubeSchedulerFlags, err := getFlags(render, p.Spec.Cluster.Kubernetes.KubeScheduler.Flags) if err != nil { return nil, err } + if l := len(extraKubeSchedulerFlags); l > 0 { + logger.Infof("plugin %s added %d extra controller scheduler command-line flags", p.Name, l) + } kubeSchedulerFlags = append(kubeSchedulerFlags, extraKubeSchedulerFlags...) extraKubeletFlags, err := getFlags(render, p.Spec.Cluster.Kubernetes.Kubelet.Flags) if err != nil { return nil, err } + if l := len(extraKubeSchedulerFlags); l > 0 { + logger.Infof("plugin %s added %d extra controller kubelet command-line flags", p.Name, l) + } kubeletFlags = append(kubeletFlags, extraKubeletFlags...) for key, value := range p.Spec.Cluster.Kubernetes.KubeProxy.Config { kubeProxyConfig[key] = value } + if l := len(p.Spec.Cluster.Kubernetes.KubeProxy.Config); l > 0 { + logger.Infof("plugin %s added %d extra controller kube-proxy configuration keys", p.Name, l) + } apiServerVolumes = append(apiServerVolumes, p.Spec.Cluster.Kubernetes.APIServer.Volumes...) + if l := len(p.Spec.Cluster.Kubernetes.APIServer.Volumes); l > 0 { + logger.Infof("plugin %s added %d extra controller volumes", p.Name, l) + } extraUnits, err := renderMachineSystemdUnits(render, p.Spec.Cluster.Machine.Roles.Controller.Systemd.Units) if err != nil { return nil, fmt.Errorf("failed adding systemd units to etcd: %v", err) } + if l := len(extraUnits); l > 0 { + logger.Infof("plugin %s added %d extra controller systemd units", p.Name, l) + } systemdUnits = append(systemdUnits, extraUnits...) extraArchivedFiles, extraFiles, extraConfigSetFiles, err := renderMachineFilesAndConfigSets(render, p.Spec.Cluster.Roles.Controller.Files) if err != nil { return nil, fmt.Errorf("failed adding files to controller: %v", err) } + if l := len(extraArchivedFiles); l > 0 { + logger.Infof("plugin %s added %d extra controller extra archive files", p.Name, l) + } + if l := len(extraFiles); l > 0 { + logger.Infof("plugin %s added %d extra controller extra files", p.Name, l) + } + if l := len(extraConfigSetFiles); l > 0 { + logger.Infof("plugin %s added %d extra controller extra config-set files", p.Name, l) + } archivedFiles = append(archivedFiles, extraArchivedFiles...) files = append(files, extraFiles...) + if l := len(p.Spec.Cluster.Machine.Roles.Controller.IAM.Policy.Statements); l > 0 { + logger.Infof("plugin %s added %d extra controller iam policies", p.Name, l) + } iamStatements = append(iamStatements, p.Spec.Cluster.Machine.Roles.Controller.IAM.Policy.Statements...) + if l := len(p.Spec.Cluster.Machine.Roles.Controller.Kubelet.NodeLabels); l > 0 { + logger.Infof("plugin %s added %d extra controller node labels", p.Name, l) + } for k, v := range p.Spec.Cluster.Machine.Roles.Controller.Kubelet.NodeLabels { nodeLabels[k] = v } if p.Spec.Cluster.Machine.Roles.Controller.Kubelet.Kubeconfig != "" { + logger.Infof("plugin %s changed the controller kubeconfig", p.Name) kubeconfig = p.Spec.Cluster.Machine.Roles.Controller.Kubelet.Kubeconfig } if len(p.Spec.Cluster.Machine.Roles.Controller.Kubelet.Mounts) > 0 { + logger.Infof("plugin %s added %d controller kubelet mounts", p.Name, len(p.Spec.Cluster.Machine.Roles.Controller.Kubelet.Mounts)) kubeletMounts = append(kubeletMounts, p.Spec.Cluster.Machine.Roles.Controller.Kubelet.Mounts...) } + logger.Debugf("Rendering Controller files and manifests...") extraFiles, extraManifests, manifestConfigSetFiles, err := renderKubernetesManifests(p.Name, render, p.Spec.Cluster.Kubernetes.Manifests) if err != nil { return nil, fmt.Errorf("failed adding kubernetes manifests to controller: %v", err) @@ -581,6 +652,9 @@ func (e ClusterExtension) Controller(clusterConfig interface{}) (*controller, er for k, v := range manifestConfigSetFiles { extraConfigSetFiles[k] = v } + if l := len(extraManifests); l > 0 { + logger.Infof("plugin %s added %d extra kubernetes manifests", p.Name, l) + } configsets[p.Name] = map[string]map[string]interface{}{ "files": extraConfigSetFiles, } @@ -618,6 +692,7 @@ func (e ClusterExtension) Etcd(clusterConfig interface{}) (*etcd, error) { for _, p := range e.plugins { if enabled, pc := p.EnabledIn(e.Configs); enabled { + logger.Debugf("Adding etcd extensions from plugin %s", p.Name) values, err := pluginutil.MergeValues(p.Spec.Cluster.Values, pc.Values) if err != nil { return nil, err @@ -628,12 +703,18 @@ func (e ClusterExtension) Etcd(clusterConfig interface{}) (*etcd, error) { if err != nil { return nil, fmt.Errorf("failed adding systemd units to etcd: %v", err) } + if l := len(extraUnits); l > 0 { + logger.Infof("plugin %s added %d extra etcd systemd units", p.Name, l) + } systemdUnits = append(systemdUnits, extraUnits...) extraFiles, err := simpleRenderMachineFiles(render, p.Spec.Cluster.Roles.Etcd.Files) if err != nil { return nil, fmt.Errorf("failed adding files to etcd: %v", err) } + if l := len(extraFiles); l > 0 { + logger.Infof("plugin %s added %d extra etcd files", p.Name, l) + } files = append(files, extraFiles...) iamStatements = append(iamStatements, p.Spec.Cluster.Roles.Etcd.IAM.Policy.Statements...) diff --git a/test/helper/helper.go b/test/helper/helper.go index dd439cbb0..9cc9480ac 100644 --- a/test/helper/helper.go +++ b/test/helper/helper.go @@ -57,7 +57,7 @@ func withDummyCredentials(alsoWriteCAKey bool, fn func(dir string)) { // config/temp, nodepool/config/temp, test/integration/temp defer os.RemoveAll(dir) - for _, pairName := range []string{"ca", "apiserver", "kube-controller-manager", "kube-scheduler", "worker", "admin", "etcd", "etcd-client", "kiam-agent", "kiam-server", "apiserver-aggregator"} { + for _, pairName := range []string{"ca", "apiserver", "kube-controller-manager", "kube-scheduler", "worker", "admin", "etcd", "etcd-client", "apiserver-aggregator"} { certFile := fmt.Sprintf("%s/%s.pem", dir, pairName) if err := ioutil.WriteFile(certFile, []byte(dummyCert), 0644); err != nil { panic(err) @@ -81,7 +81,6 @@ func withDummyCredentials(alsoWriteCAKey bool, fn func(dir string)) { symlinks := []symlink{ {"ca.pem", "worker-ca.pem"}, {"ca.pem", "etcd-trusted-ca.pem"}, - {"ca.pem", "kiam-ca.pem"}, } if alsoWriteCAKey { diff --git a/test/integration/maincluster_test.go b/test/integration/maincluster_test.go index 260b6e0ed..33ea38d8a 100644 --- a/test/integration/maincluster_test.go +++ b/test/integration/maincluster_test.go @@ -121,12 +121,6 @@ func TestMainClusterConfig(t *testing.T) { Disk: "xvdb", Filesystem: "xfs", }, - KIAMSupport: api.KIAMSupport{ - Enabled: false, - Image: api.Image{Repo: "quay.io/uswitch/kiam", Tag: "v3.2", RktPullDocker: false}, - SessionDuration: "30m", - ServerAddresses: api.KIAMServerAddresses{ServerAddress: "localhost:443", AgentAddress: "kiam-server:443"}, - }, Kube2IamSupport: api.Kube2IamSupport{ Enabled: false, }, @@ -1263,8 +1257,6 @@ experimental: enabled: true ephemeralImageStorage: enabled: true - kiamSupport: - enabled: false kube2IamSupport: enabled: true gpuSupport: @@ -1349,12 +1341,6 @@ worker: Disk: "xvdb", Filesystem: "xfs", }, - KIAMSupport: api.KIAMSupport{ - Enabled: false, - Image: api.Image{Repo: "quay.io/uswitch/kiam", Tag: "v3.2", RktPullDocker: false}, - SessionDuration: "30m", - ServerAddresses: api.KIAMServerAddresses{ServerAddress: "localhost:443", AgentAddress: "kiam-server:443"}, - }, Kube2IamSupport: api.Kube2IamSupport{ Enabled: true, }, @@ -1504,71 +1490,6 @@ worker: }, }, }, - { - context: "WithExperimentalFeatureKiam", - configYaml: minimalValidConfigYaml + ` -experimental: - kiamSupport: - enabled: true - image: - repo: quay.io/uswitch/kiam - tag: v2.6 - sessionDuration: 30m - serverAddresses: - serverAddress: localhost:443 - agentAddress: kiam-server:443 -worker: - nodePools: - - name: pool1 -`, - assertConfig: []ConfigTester{ - func(c *config.Config, t *testing.T) { - expected := api.KIAMSupport{ - Enabled: true, - Image: api.Image{Repo: "quay.io/uswitch/kiam", Tag: "v2.6", RktPullDocker: false}, - SessionDuration: "30m", - ServerAddresses: api.KIAMServerAddresses{ServerAddress: "localhost:443", AgentAddress: "kiam-server:443"}, - } - - actual := c.Experimental - - if !reflect.DeepEqual(expected, actual.KIAMSupport) { - t.Errorf("experimental settings didn't match : expected=%+v actual=%+v", expected, actual) - } - - p := c.NodePools[0] - if reflect.DeepEqual(expected, p.Experimental.KIAMSupport) { - t.Errorf("experimental settings shouldn't be inherited to a node pool but it did : toplevel=%v nodepool=%v", expected, p.Experimental) - } - }, - }, - }, - { - context: "WithExperimentalFeatureKiamForWorkerNodePool", - configYaml: minimalValidConfigYaml + ` -worker: - nodePools: - - name: pool1 - kiamSupport: - enabled: true -`, - assertConfig: []ConfigTester{ - func(c *config.Config, t *testing.T) { - expected := api.Experimental{ - KIAMSupport: api.KIAMSupport{ - Enabled: true, - Image: api.Image{Repo: "quay.io/uswitch/kiam", Tag: "v3.2", RktPullDocker: false}, - SessionDuration: "30m", - ServerAddresses: api.KIAMServerAddresses{ServerAddress: "localhost:443", AgentAddress: "kiam-server:443"}, - }, - } - p := c.NodePools[0] - if reflect.DeepEqual(expected, p.Experimental) { - t.Errorf("experimental settings for node pool didn't match : expected=%v actual=%v", expected, p.Experimental) - } - }, - }, - }, { context: "WithKube2IamSupport", configYaml: minimalValidConfigYaml + `