diff --git a/builtin/files/cluster.yaml.tmpl b/builtin/files/cluster.yaml.tmpl index 0a5e683df..9528f6e13 100644 --- a/builtin/files/cluster.yaml.tmpl +++ b/builtin/files/cluster.yaml.tmpl @@ -240,7 +240,7 @@ kmsKeyArn: "{{.KMSKeyARN}}" # # If you specify a name for the role, kube-aws will create it without a random id suffix (AWS default). # # Given a role of `yourManagedRole`, this will create an IAM role named `${clusterName}-${region}-yourManagedRole` # # to follow the recommendation in AWS documentation http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-role.html -# # There are several cases you might need a stable name such as with .experimental.kube2IamSupport or .experimental.kiamSupport. See #297 for more information. +# # There are several cases you might need a stable name such as with .experimental.kube2IamSupport or kiam plugin. See #297 for more information. # # ATTENTION: Consider limiting number of characters in clusterName and iam.role.name to avoid the resulting IAM # # role name's length from exceeding the AWS limit: 64. See https://github.com/kubernetes-incubator/kube-aws/issues/347 # # It will have attached a customer Managed Policy that you can modify afterwards if you need more permissions for your cluster. @@ -362,7 +362,7 @@ worker: # # If you specify a name for the role, kube-aws will create it without a random id suffix (AWS default). # # Given a role of `yourManagedRole`, this will create an IAM role named `${clusterName}-${region}-yourManagedRole` # # to follow the recommendation in AWS documentation http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-role.html -# # There are several cases you might need a stable name such as with .experimental.kube2IamSupport or .experimental.kiamSupport. See #297 for more information. +# # There are several cases you might need a stable name such as with .experimental.kube2IamSupport or kiam plugin. See #297 for more information. # # ATTENTION: Consider limiting number of characters in clusterName and iam.role.name to avoid the resulting IAM # # role name's length from exceeding the AWS limit: 64. See https://github.com/kubernetes-incubator/kube-aws/issues/347 # # It will have attached a customer Managed Policy that you can modify afterwards if you need more permissions for your cluster. @@ -1444,38 +1444,6 @@ experimental: ephemeralImageStorage: enabled: false - # When enabled this will install the kiam daemon set using the repo from `kiamImage` - # It will also grant sts:assumeRole permission to the IAM role for controller nodes (only controller nodes where the kiam server is hosted need this, worker nodes do not) - # This will use SSL certificates generated during `kube-aws render credentials`, ensure this has been run with kube-aws v0.9.9+ - # This is intended to be used in combination with .controller.iam.role.name. See #297 for more information. - kiamSupport: - enabled: false - # image: - # repo: quay.io/uswitch/kiam - # tag: v3.2 - # rktPullDocker: false - # sessionDuration: 30m - # serverAddresses: - # serverAddress: localhost:443 - # agentAddress: kiam-server:443 - # Optional resource change for kiam servers/agents can be done via using the resources block below and changing the values. - # Values below are the default if not set. - # serverResources: - # requests: - # cpu: 50m - # memory: 10Mi - # limits: - # cpu: 200m - # memory: 200Mi - # agentResources: - # requests: - # cpu: 10m - # memory: 5Mi - # limits: - # cpu: 200m - # memory: 100Mi - - # When enabled this will install the kube2iam daemon set using the repo from `kube2iamImage` # It will also grant sts:assumeRole permission to the IAM role for controller nodes. # This is intended to be used in combination with .controller.iam.role.name. See #297 for more information. @@ -1607,18 +1575,45 @@ kubeAwsPlugins: selector: prometheus: monitoring - # upgradeHelper - assists when rolling out new versions of kubernetes - # It actively disables old controllers and temporarily removes mutating/validating webhooks whilst - # the upgraded controller is starting up. - # NOTE: You will normally not need this plugin - so ONLY enable if you are experiencing issues testing migrating across versions. - # It will only kill controller that are a different release from currently spinning up version, e.g. :- - # it will kill v1.13.2 controllers when rolling out v1.14.0 - # it will NOT kill v1.14.0 controllers when rolling out v1.14.3 - upgradeHelper: - enabled: false - # disableWebhooks can be used to turn off the webhook feature if required - disableWebhooks: true - # disableWorkerCommunicationChecks can be used to disable the check that all workers checking in with this master - # so that cfn-signal can be generated and upgrade process move forward - # (you don't want a single bad worker to cause an important upgrade to fail) - disableWorkerCommunicationChecks: true + # upgradeHelper - assists when rolling out new versions of kubernetes + # It actively disables old controllers and temporarily removes mutating/validating webhooks whilst + # the upgraded controller is starting up. + # NOTE: You will normally not need this plugin - so ONLY enable if you are experiencing issues testing migrating across versions. + # It will only kill controller that are a different release from currently spinning up version, e.g. :- + # it will kill v1.13.2 controllers when rolling out v1.14.0 + # it will NOT kill v1.14.0 controllers when rolling out v1.14.3 + upgradeHelper: + enabled: false + # disableWebhooks can be used to turn off the webhook feature if required + disableWebhooks: true + # disableWorkerCommunicationChecks can be used to disable the check that all workers checking in with this master + # so that cfn-signal can be generated and upgrade process move forward + # (you don't want a single bad worker to cause an important upgrade to fail) + disableWorkerCommunicationChecks: true + + # kiam - deploys kiam server and agent components + # kiam provides individual pod access policies and tokens for restricting pod access to AWS resources + # It will grant sts:assumeRole permission to the IAM role for controller nodes (only controller nodes where the kiam server is hosted need this, worker nodes do not) + # This is intended to be used in combination with .controller.iam.role.name. See #297 for more information. + kiam: + enabled: false + # image: quay.io/uswitch/kiam + # tag: v3.2 + # sessionDuration: 30m + # server: + # address: localhost:443 + # resources: + # requests: + # cpu: 50m + # limits: + # cpu: 200m + # memory: 200Mi + # agent: + # address: kiam-server:443 + # resources: + # requests: + # cpu: 10m + # memory: 5Mi + # limits: + # cpu: 200m + # memory: 100Mi diff --git a/builtin/files/plugins/kiam/manifests/agent-daemonset.yaml b/builtin/files/plugins/kiam/manifests/agent-daemonset.yaml new file mode 100644 index 000000000..c66656143 --- /dev/null +++ b/builtin/files/plugins/kiam/manifests/agent-daemonset.yaml @@ -0,0 +1,103 @@ +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + namespace: kube-system + name: kiam-agent +spec: + updateStrategy: + rollingUpdate: + maxUnavailable: 100% + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9620" + labels: + app: kiam + role: agent + spec: + priorityClassName: system-node-critical + tolerations: + - operator: Exists + effect: NoSchedule + - operator: Exists + effect: NoExecute + - operator: Exists + key: CriticalAddonsOnly + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node.kubernetes.io/role + operator: NotIn + values: + - master + volumes: + - name: ssl-certs + hostPath: + path: /usr/share/ca-certificates + - name: tls + secret: + secretName: kiam-agent-tls + - name: xtables + hostPath: + path: /run/xtables.lock + containers: + - name: kiam + securityContext: + capabilities: + add: ["NET_ADMIN"] + image: {{ .Values.image }}:{{ .Values.tag }} + command: + - {{ if checkVersion ">= 3.0" .Values.tag }}/kiam{{ else }}/agent{{ end }} + args: + {{if checkVersion ">= 3.0" .Values.tag -}} + - agent + - --whitelist-route-regexp=.* + - --gateway-timeout-creation=1s + {{ end -}} + - --iptables + {{- if .Config.Cluster.Kubernetes.Networking.AmazonVPC.Enabled }} + - --host-interface=!eni0 + {{- else if eq .Config.Cluster.Kubernetes.Networking.SelfHosting.Type "canal" }} + - --host-interface=cali+ + {{- else}} + - --host-interface=cni0 + {{- end }} + - --json-log + - --port=8181 + - --cert=/etc/kiam/tls/tls.crt + - --key=/etc/kiam/tls/tls.key + - --ca=/etc/kiam/tls/ca.crt + - --server-address={{ .Values.agent.address }} + - --prometheus-listen-addr=0.0.0.0:9620 + - --prometheus-sync-interval=5s + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + volumeMounts: + - mountPath: /etc/ssl/certs + name: ssl-certs + - mountPath: /etc/kiam/tls + name: tls + - mountPath: /var/run/xtables.lock + name: xtables + livenessProbe: + httpGet: + path: /ping + port: 8181 + initialDelaySeconds: 3 + periodSeconds: 3 + resources: + requests: + cpu: {{ .Values.agent.resources.requests.cpu }} + memory: {{ .Values.agent.resources.requests.memory }} + limits: + cpu: {{ .Values.agent.resources.limits.cpu }} + memory: {{ .Values.agent.resources.limits.memory }} \ No newline at end of file diff --git a/builtin/files/plugins/kiam/manifests/agent-tls-secret.yaml b/builtin/files/plugins/kiam/manifests/agent-tls-secret.yaml new file mode 100644 index 000000000..7c9b190aa --- /dev/null +++ b/builtin/files/plugins/kiam/manifests/agent-tls-secret.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: kiam-agent-tls + namespace: kube-system +type: kubernetes.io/tls +data: + tls.crt: {{ insertTemplateFile "credentials/kiam-agent.pem" . | b64enc }} + tls.key: {{ insertTemplateFile "credentials/kiam-agent-key.pem" . | b64enc }} + ca.crt: {{ insertTemplateFile "credentials/kiam-ca.pem" . | b64enc }} \ No newline at end of file diff --git a/builtin/files/plugins/kiam/manifests/server-cluster-role-binding.yaml b/builtin/files/plugins/kiam/manifests/server-cluster-role-binding.yaml new file mode 100644 index 000000000..916985861 --- /dev/null +++ b/builtin/files/plugins/kiam/manifests/server-cluster-role-binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: kiam-server +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kiam-server +subjects: +- kind: ServiceAccount + name: kiam-server + namespace: kube-system \ No newline at end of file diff --git a/builtin/files/plugins/kiam/manifests/server-cluster-role.yaml b/builtin/files/plugins/kiam/manifests/server-cluster-role.yaml new file mode 100644 index 000000000..610dff6d6 --- /dev/null +++ b/builtin/files/plugins/kiam/manifests/server-cluster-role.yaml @@ -0,0 +1,21 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: kiam-server +rules: +- apiGroups: + - "" + resources: + - namespaces + - pods + verbs: + - watch + - get + - list +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch \ No newline at end of file diff --git a/builtin/files/plugins/kiam/manifests/server-daemonset.yaml b/builtin/files/plugins/kiam/manifests/server-daemonset.yaml new file mode 100644 index 000000000..0a3bc1869 --- /dev/null +++ b/builtin/files/plugins/kiam/manifests/server-daemonset.yaml @@ -0,0 +1,106 @@ +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + namespace: kube-system + name: kiam-server +spec: + updateStrategy: + rollingUpdate: + maxUnavailable: 100% + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9620" + labels: + app: kiam + role: server + spec: + priorityClassName: system-node-critical + tolerations: + - operator: Exists + effect: NoSchedule + - operator: Exists + effect: NoExecute + - operator: Exists + key: CriticalAddonsOnly + serviceAccountName: kiam-server + nodeSelector: + node.kubernetes.io/role: "master" + volumes: + - name: ssl-certs + hostPath: + path: /usr/share/ca-certificates + - name: tls + secret: + secretName: kiam-server-tls + containers: + - name: kiam + image: {{ .Values.image }}:{{ .Values.tag }} + command: + - {{ if checkVersion ">= 3.0" .Values.tag }}/kiam{{ else }}/server{{ end }} + args: + {{ if checkVersion ">= 3.0" .Values.tag -}} + - server + {{ end -}} + - --json-log + - --bind=0.0.0.0:443 + - --cert=/etc/kiam/tls/tls.crt + - --key=/etc/kiam/tls/tls.key + - --ca=/etc/kiam/tls/ca.crt + - --role-base-arn-autodetect + - --sync=1m + - --prometheus-listen-addr=0.0.0.0:9620 + - --prometheus-sync-interval=5s + - --session-duration={{ .Values.sessionDuration }} + volumeMounts: + - mountPath: /etc/ssl/certs + name: ssl-certs + - mountPath: /etc/kiam/tls + name: tls + livenessProbe: + exec: + command: + {{ if checkVersion ">= 3.0" .Values.tag -}} + - /kiam + - health + - --gateway-timeout-creation=1s + {{ else -}} + - /health + - --server-address-refresh=2s + {{ end -}} + - --cert=/etc/kiam/tls/tls.crt + - --key=/etc/kiam/tls/tls.key + - --ca=/etc/kiam/tls/ca.crt + - --server-address={{ .Values.server.address }} + - --timeout=5s + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 10 + readinessProbe: + exec: + command: + {{ if checkVersion ">= 3.0" .Values.tag -}} + - /kiam + - health + - --gateway-timeout-creation=1s + {{ else -}} + - /health + - --server-address-refresh=2s + {{ end -}} + - --cert=/etc/kiam/tls/tls.crt + - --key=/etc/kiam/tls/tls.key + - --ca=/etc/kiam/tls/ca.crt + - --server-address={{ .Values.server.address }} + - --timeout=5s + initialDelaySeconds: 3 + periodSeconds: 10 + timeoutSeconds: 10 + resources: + requests: + cpu: {{ .Values.server.resources.requests.cpu }} + memory: {{ .Values.server.resources.requests.memory }} + limits: + cpu: {{ .Values.server.resources.limits.cpu }} + memory: {{ .Values.server.resources.limits.memory }} \ No newline at end of file diff --git a/builtin/files/plugins/kiam/manifests/server-tls-secret.yaml b/builtin/files/plugins/kiam/manifests/server-tls-secret.yaml new file mode 100644 index 000000000..ece545933 --- /dev/null +++ b/builtin/files/plugins/kiam/manifests/server-tls-secret.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: kiam-server-tls + namespace: kube-system +type: kubernetes.io/tls +data: + tls.crt: {{ insertTemplateFile "credentials/kiam-server.pem" . | b64enc }} + tls.key: {{ insertTemplateFile "credentials/kiam-server-key.pem" . | b64enc }} + ca.crt: {{ insertTemplateFile "credentials/kiam-ca.pem" . | b64enc }} \ No newline at end of file diff --git a/builtin/files/plugins/kiam/manifests/service-account.yaml b/builtin/files/plugins/kiam/manifests/service-account.yaml new file mode 100644 index 000000000..4c52d4096 --- /dev/null +++ b/builtin/files/plugins/kiam/manifests/service-account.yaml @@ -0,0 +1,5 @@ +kind: ServiceAccount +apiVersion: v1 +metadata: + name: kiam-server + namespace: kube-system \ No newline at end of file diff --git a/builtin/files/plugins/kiam/manifests/service.yaml b/builtin/files/plugins/kiam/manifests/service.yaml new file mode 100644 index 000000000..8e93ec0f8 --- /dev/null +++ b/builtin/files/plugins/kiam/manifests/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: kiam-server + namespace: kube-system +spec: + clusterIP: None + selector: + app: kiam + role: server + ports: + - name: grpclb + port: 443 + targetPort: 443 + protocol: TCP \ No newline at end of file diff --git a/builtin/files/plugins/kiam/plugin.yaml b/builtin/files/plugins/kiam/plugin.yaml new file mode 100644 index 000000000..901379ac2 --- /dev/null +++ b/builtin/files/plugins/kiam/plugin.yaml @@ -0,0 +1,102 @@ +metadata: + name: kiam + version: 0.1.0 +spec: + cluster: + values: + image: quay.io/uswitch/kiam + tag: v3.2 + sessionDuration: 30m + server: + address: localhost:443 + resources: + requests: + cpu: 50m + memory: 10Mi + limits: + cpu: 200m + memory: 200Mi + agent: + address: kiam-server:443 + resources: + requests: + cpu: 10m + memory: 5Mi + limits: + cpu: 200m + memory: 100Mi + + kubernetes: + manifests: + - source: + path: manifests/agent-tls-secret.yaml + - source: + path: manifests/server-tls-secret.yaml + - source: + path: manifests/server-daemonset.yaml + - source: + path: manifests/agent-daemonset.yaml + - source: + path: manifests/service.yaml + - source: + path: manifests/service-account.yaml + - source: + path: manifests/server-cluster-role.yaml + - source: + path: manifests/server-cluster-role-binding.yaml + + pki: + keypairs: + - name: kiam-ca + commonName: kiam-ca + organization: kube-aws-kiam + duration: 43800h + usages: + - ca + - name: kiam-server + commonName: kiam-server + organization: kube-aws-kiam + dnsNames: + - kiam-server + - kiam-server:443 + - localhost + - localhost:443 + - localhost:9610 + duration: 8760h + usages: + - server + - client + signer: kiam-ca + - name: kiam-agent + commonName: kiam-agent + organization: kube-aws-kiam + duration: 8760h + usages: + - client + signer: kiam-ca + + cloudformation: + stacks: + controlPlane: + resources: + content: | + { + "IAMManagedPolicyKiam" : { + "Type" : "AWS::IAM::ManagedPolicy", + "Properties" : { + "Description" : "Policy for managing Kiam on kube-aws controllers", + "Path" : "/", + "Roles":[ { "Ref" : "IAMRoleController" } ], + "PolicyDocument" : { + "Version":"2012-10-17", + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect":"Allow", + "Resource":"*" + } + ] + } + } + } + } \ No newline at end of file diff --git a/builtin/files/stack-templates/control-plane.json.tmpl b/builtin/files/stack-templates/control-plane.json.tmpl index 37c1fa871..075009c9d 100644 --- a/builtin/files/stack-templates/control-plane.json.tmpl +++ b/builtin/files/stack-templates/control-plane.json.tmpl @@ -314,7 +314,7 @@ "Resource": [ "*" ] }, {{end}} - {{if or .Experimental.Kube2IamSupport.Enabled .Experimental.KIAMSupport.Enabled }} + {{if .Experimental.Kube2IamSupport.Enabled }} { "Action": "sts:AssumeRole", "Effect":"Allow", @@ -389,7 +389,7 @@ } } }, - {{if and .Experimental.KIAMSupport.Enabled .KubeResourcesAutosave.Enabled }} + {{if and .KubeResourcesAutosave.Enabled ((index .PluginConfigs "kiam").Enabled) }} "IAMManagedPolicyResourcesAutoSave" : { "Type" : "AWS::IAM::ManagedPolicy", "Properties" : { diff --git a/builtin/files/userdata/cloud-config-controller b/builtin/files/userdata/cloud-config-controller index 067db20b3..05c76eacc 100644 --- a/builtin/files/userdata/cloud-config-controller +++ b/builtin/files/userdata/cloud-config-controller @@ -1106,22 +1106,6 @@ write_files: deploy "${mfdir}/kube-node-drainer-ds.yaml" \ "${mfdir}/kube-node-drainer-asg-status-updater-de.yaml" - {{ end -}} - {{if .Experimental.KIAMSupport.Enabled -}} - # KIAM - kiam_tls_dir=/etc/kubernetes/ssl/kiam - vols="${vols} -v $kiam_tls_dir:$kiam_tls_dir" - - kubectl create secret generic kiam-server-tls -n kube-system \ - --from-file=$kiam_tls_dir/ca.pem \ - --from-file=$kiam_tls_dir/server.pem \ - --from-file=$kiam_tls_dir/server-key.pem --dry-run -o yaml | kubectl apply -n kube-system -f - - kubectl create secret generic kiam-agent-tls -n kube-system \ - --from-file=$kiam_tls_dir/ca.pem \ - --from-file=$kiam_tls_dir/agent.pem \ - --from-file=$kiam_tls_dir/agent-key.pem --dry-run -o yaml | kubectl apply -n kube-system -f - - deploy "${mfdir}/kiam-all.yaml" - {{ end -}} {{if .Experimental.Kube2IamSupport.Enabled -}} # KUBE2IAM @@ -2479,7 +2463,7 @@ write_files: replicas: 1 template: metadata: - {{if .Experimental.KIAMSupport.Enabled -}} + {{if (index .PluginConfigs "kiam").Enabled -}} annotations: iam.amazonaws.com/role: "{{$.ClusterName}}-IAMRoleResourcesAutoSave" {{ end -}} @@ -2602,7 +2586,7 @@ write_files: 'echo decrypting assets shopt -s nullglob set -o pipefail - for encKey in /etc/kubernetes/{ssl,additional-configs,auth}/{,kiam/}*.enc; do + for encKey in /etc/kubernetes/{ssl,additional-configs,auth}/*.enc; do if [ ! -f $encKey ]; then echo skipping non-existent file: $encKey 1>&2 continue @@ -5604,30 +5588,6 @@ write_files: mountPath: /dev {{end}} -{{if .Experimental.KIAMSupport.Enabled }} - - path: /etc/kubernetes/ssl/kiam/ca.pem - encoding: gzip+base64 - content: {{.AssetsConfig.KIAMCACert}} - - - path: /etc/kubernetes/ssl/kiam/server-key.pem{{if .AssetsEncryptionEnabled}}.enc{{end}} - encoding: gzip+base64 - content: {{.AssetsConfig.KIAMServerKey}} - - - path: /etc/kubernetes/ssl/kiam/server.pem - encoding: gzip+base64 - content: {{.AssetsConfig.KIAMServerCert}} - - - path: /etc/kubernetes/ssl/kiam/agent-key.pem{{if .AssetsEncryptionEnabled}}.enc{{end}} - encoding: gzip+base64 - content: {{.AssetsConfig.KIAMAgentKey}} - - - path: /etc/kubernetes/ssl/kiam/agent.pem - encoding: gzip+base64 - content: {{.AssetsConfig.KIAMAgentCert}} - -{{end}} - - # Without the namespace annotation the pod will be unable to assume any roles - path: /srv/kubernetes/manifests/kube-system-ns.yaml content: | @@ -5642,298 +5602,6 @@ write_files: {{ $key }}: {{ $value }} {{- end }} -{{if .Experimental.KIAMSupport.Enabled }} - - path: /srv/kubernetes/manifests/kiam-all.yaml - content: | - apiVersion: extensions/v1beta1 - kind: DaemonSet - metadata: - namespace: kube-system - name: kiam-server - spec: - updateStrategy: - rollingUpdate: - maxUnavailable: 100% - type: RollingUpdate - template: - metadata: - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9620" - labels: - app: kiam - role: server - spec: - priorityClassName: system-node-critical - tolerations: - - operator: Exists - effect: NoSchedule - - operator: Exists - effect: NoExecute - - operator: Exists - key: CriticalAddonsOnly - serviceAccountName: kiam-server - nodeSelector: - node.kubernetes.io/role: "master" - volumes: - - name: ssl-certs - hostPath: - path: /usr/share/ca-certificates - - name: tls - secret: - secretName: kiam-server-tls - containers: - - name: kiam - image: {{.Experimental.KIAMSupport.Image.RepoWithTag}} - command: - - {{if checkVersion ">= 3.0" .Experimental.KIAMSupport.Image.Tag}}/kiam{{else}}/server{{end}} - args: - {{if checkVersion ">= 3.0" .Experimental.KIAMSupport.Image.Tag -}} - - server - {{ end -}} - - --json-log - - --bind=0.0.0.0:443 - - --cert=/etc/kiam/tls/server.pem - - --key=/etc/kiam/tls/server-key.pem - - --ca=/etc/kiam/tls/ca.pem - - --role-base-arn-autodetect - - --sync=1m - - --prometheus-listen-addr=0.0.0.0:9620 - - --prometheus-sync-interval=5s - - --session-duration={{.Experimental.KIAMSupport.SessionDuration}} - volumeMounts: - - mountPath: /etc/ssl/certs - name: ssl-certs - - mountPath: /etc/kiam/tls - name: tls - livenessProbe: - exec: - command: - {{if checkVersion ">= 3.0" .Experimental.KIAMSupport.Image.Tag -}} - - /kiam - - health - - --gateway-timeout-creation=1s - {{ else -}} - - /health - - --server-address-refresh=2s - {{ end -}} - - --cert=/etc/kiam/tls/server.pem - - --key=/etc/kiam/tls/server-key.pem - - --ca=/etc/kiam/tls/ca.pem - - --server-address={{.Experimental.KIAMSupport.ServerAddresses.ServerAddress}} - - --timeout=5s - initialDelaySeconds: 10 - periodSeconds: 10 - timeoutSeconds: 10 - readinessProbe: - exec: - command: - {{if checkVersion ">= 3.0" .Experimental.KIAMSupport.Image.Tag -}} - - /kiam - - health - - --gateway-timeout-creation=1s - {{ else -}} - - /health - - --server-address-refresh=2s - {{ end -}} - - --cert=/etc/kiam/tls/server.pem - - --key=/etc/kiam/tls/server-key.pem - - --ca=/etc/kiam/tls/ca.pem - - --server-address={{.Experimental.KIAMSupport.ServerAddresses.ServerAddress}} - - --timeout=5s - initialDelaySeconds: 3 - periodSeconds: 10 - timeoutSeconds: 10 - resources: - requests: - cpu: {{ if .Experimental.KIAMSupport.ServerResources.Requests.Cpu }}{{ .Experimental.KIAMSupport.ServerResources.Requests.Cpu }}{{ else }}50m{{ end }} - memory: {{ if .Experimental.KIAMSupport.ServerResources.Requests.Memory }}{{ .Experimental.KIAMSupport.ServerResources.Requests.Memory }}{{ else }}10Mi{{ end }} - limits: - cpu: {{ if .Experimental.KIAMSupport.ServerResources.Limits.Cpu }}{{ .Experimental.KIAMSupport.ServerResources.Limits.Cpu }}{{ else }}200m{{ end }} - memory: {{ if .Experimental.KIAMSupport.ServerResources.Limits.Memory }}{{ .Experimental.KIAMSupport.ServerResources.Limits.Memory }}{{ else }}200Mi{{ end }} - --- - apiVersion: v1 - kind: Service - metadata: - name: kiam-server - namespace: kube-system - spec: - clusterIP: None - selector: - app: kiam - role: server - ports: - - name: grpclb - port: 443 - targetPort: 443 - protocol: TCP - --- - --- - kind: ServiceAccount - apiVersion: v1 - metadata: - name: kiam-server - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1beta1 - kind: ClusterRole - metadata: - name: kiam-read - rules: - - apiGroups: - - "" - resources: - - namespaces - - pods - verbs: - - watch - - get - - list - --- - apiVersion: rbac.authorization.k8s.io/v1beta1 - kind: ClusterRole - metadata: - name: kiam-write - rules: - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - --- - apiVersion: rbac.authorization.k8s.io/v1beta1 - kind: ClusterRoleBinding - metadata: - name: kiam-server - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: kiam-read - subjects: - - kind: ServiceAccount - name: kiam-server - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1beta1 - kind: ClusterRoleBinding - metadata: - name: kiam-write - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: kiam-write - subjects: - - kind: ServiceAccount - name: kiam-server - namespace: kube-system - --- - apiVersion: extensions/v1beta1 - kind: DaemonSet - metadata: - namespace: kube-system - name: kiam-agent - spec: - updateStrategy: - rollingUpdate: - maxUnavailable: 100% - type: RollingUpdate - template: - metadata: - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9620" - labels: - app: kiam - role: agent - spec: - priorityClassName: system-node-critical - tolerations: - - operator: Exists - effect: NoSchedule - - operator: Exists - effect: NoExecute - - operator: Exists - key: CriticalAddonsOnly - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node.kubernetes.io/role - operator: NotIn - values: - - master - volumes: - - name: ssl-certs - hostPath: - path: /usr/share/ca-certificates - - name: tls - secret: - secretName: kiam-agent-tls - - name: xtables - hostPath: - path: /run/xtables.lock - containers: - - name: kiam - securityContext: - capabilities: - add: ["NET_ADMIN"] - image: {{.Experimental.KIAMSupport.Image.RepoWithTag}} - command: - - {{if checkVersion ">= 3.0" .Experimental.KIAMSupport.Image.Tag}}/kiam{{else}}/agent{{end}} - args: - {{if checkVersion ">= 3.0" .Experimental.KIAMSupport.Image.Tag -}} - - agent - - --whitelist-route-regexp=.* - - --gateway-timeout-creation=1s - {{ end -}} - - --iptables - {{- if .Kubernetes.Networking.AmazonVPC.Enabled }} - - --host-interface=!eni0 - {{- else if eq .Kubernetes.Networking.SelfHosting.Type "canal" }} - - --host-interface=cali+ - {{- else}} - - --host-interface=cni0 - {{- end }} - - --json-log - - --port=8181 - - --cert=/etc/kiam/tls/agent.pem - - --key=/etc/kiam/tls/agent-key.pem - - --ca=/etc/kiam/tls/ca.pem - - --server-address={{.Experimental.KIAMSupport.ServerAddresses.AgentAddress}} - - --prometheus-listen-addr=0.0.0.0:9620 - - --prometheus-sync-interval=5s - env: - - name: HOST_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - volumeMounts: - - mountPath: /etc/ssl/certs - name: ssl-certs - - mountPath: /etc/kiam/tls - name: tls - - mountPath: /var/run/xtables.lock - name: xtables - livenessProbe: - httpGet: - path: /ping - port: 8181 - initialDelaySeconds: 3 - periodSeconds: 3 - resources: - requests: - cpu: {{ if .Experimental.KIAMSupport.AgentResources.Requests.Cpu }}{{ .Experimental.KIAMSupport.AgentResources.Requests.Cpu }}{{ else }}10m{{ end }} - memory: {{ if .Experimental.KIAMSupport.AgentResources.Requests.Memory }}{{ .Experimental.KIAMSupport.AgentResources.Requests.Memory }}{{ else }}5Mi{{ end }} - limits: - cpu: {{ if .Experimental.KIAMSupport.AgentResources.Limits.Cpu }}{{ .Experimental.KIAMSupport.AgentResources.Limits.Cpu }}{{ else }}200m{{ end }} - memory: {{ if .Experimental.KIAMSupport.AgentResources.Limits.Memory }}{{ .Experimental.KIAMSupport.AgentResources.Limits.Memory }}{{ else }}100Mi{{ end }} -{{end}} - - path: /opt/bin/retry owner: root:root permissions: 0755 diff --git a/cmd/render.go b/cmd/render.go index ebad1e23b..7c317ce8a 100644 --- a/cmd/render.go +++ b/cmd/render.go @@ -53,23 +53,23 @@ func init() { cmdRenderCredentials.Flags().StringVar(&renderCredentialsOpts.ApiServerKeyPath, "apiserver-key-path", "", "path to pem-encoded apiserver RSA key") cmdRenderCredentials.Flags().StringVar(&renderCredentialsOpts.EtcdClientKeyPath, "etcd-client-key-path", "", "path to pem-encoded etcd client RSA key") cmdRenderCredentials.Flags().StringVar(&renderCredentialsOpts.EtcdKeyPath, "etcd-key-path", "", "path to pem-encoded etcd RSA key") - cmdRenderCredentials.Flags().StringVar(&renderCredentialsOpts.KiamAgentKeyPath, "kiam-agent-key-path", "", "path to pem-encoded kiam agent RSA key") - cmdRenderCredentials.Flags().StringVar(&renderCredentialsOpts.KiamServerKeyPath, "kiam-server-key-path", "", "path to pem-encoded kiam server RSA key") cmdRenderCredentials.Flags().StringVar(&renderCredentialsOpts.KubeControllerManagerKeyPath, "kube-controller-manager-key-path", "", "path to pem-encoded kube controller manager RSA key") cmdRenderCredentials.Flags().StringVar(&renderCredentialsOpts.KubeSchedulerKeyPath, "kube-scheduler-key-path", "", "path to pem-encoded kube scheduler RSA key") cmdRenderCredentials.Flags().StringVar(&renderCredentialsOpts.ServiceAccountKeyPath, "service-account-key-path", "", "path to pem-encoded service account RSA key") cmdRenderCredentials.Flags().StringVar(&renderCredentialsOpts.WorkerKeyPath, "worker-key-path", "", "path to pem-encoded worker RSA key") - cmdRenderCredentials.Flags().BoolVar(&renderCredentialsOpts.KIAM, "kiam", true, "generate TLS assets for kiam") cmdRenderCredentials.Flags().BoolVar(&renderCredentialsOpts.AwsDebug, "aws-debug", false, "Log debug information from aws-sdk-go library") } func runCmdRender(_ *cobra.Command, args []string) error { - logger.Warn("'kube-aws render' is deprecated. See 'kube-aws render --help' for usage") if len(args) != 0 { return fmt.Errorf("render takes no arguments\n") } + if err := runCmdRenderStack(cmdRenderCredentials, args); err != nil { + return err + } + if _, err := os.Stat(renderCredentialsOpts.CaKeyPath); os.IsNotExist(err) { renderCredentialsOpts.GenerateCA = true } @@ -77,10 +77,6 @@ func runCmdRender(_ *cobra.Command, args []string) error { return err } - if err := runCmdRenderStack(cmdRenderCredentials, args); err != nil { - return err - } - return nil } @@ -103,5 +99,8 @@ Next steps: } func runCmdRenderCredentials(_ *cobra.Command, _ []string) error { + if _, err := os.Stat(renderCredentialsOpts.CaKeyPath); os.IsNotExist(err) { + renderCredentialsOpts.GenerateCA = true + } return root.RenderCredentials(configPath, renderCredentialsOpts) } diff --git a/credential/encrypted_assets.go b/credential/encrypted_assets.go index 2f7247e46..0daca49db 100644 --- a/credential/encrypted_assets.go +++ b/credential/encrypted_assets.go @@ -38,11 +38,6 @@ type RawAssetsOnMemory struct { EtcdKey []byte EtcdClientKey []byte EtcdTrustedCA []byte - KIAMServerCert []byte - KIAMServerKey []byte - KIAMAgentCert []byte - KIAMAgentKey []byte - KIAMCACert []byte ServiceAccountKey []byte // Other assets. @@ -74,11 +69,6 @@ type RawAssetsOnDisk struct { EtcdKey PlaintextFile EtcdClientKey PlaintextFile EtcdTrustedCA PlaintextFile - KIAMServerCert PlaintextFile - KIAMServerKey PlaintextFile - KIAMAgentCert PlaintextFile - KIAMAgentKey PlaintextFile - KIAMCACert PlaintextFile ServiceAccountKey PlaintextFile // Other assets. @@ -110,11 +100,6 @@ type EncryptedAssetsOnDisk struct { EtcdKey EncryptedFile EtcdClientKey EncryptedFile EtcdTrustedCA EncryptedFile - KIAMServerCert EncryptedFile - KIAMServerKey EncryptedFile - KIAMAgentCert EncryptedFile - KIAMAgentKey EncryptedFile - KIAMCACert EncryptedFile ServiceAccountKey EncryptedFile // Other encrypted assets. @@ -146,11 +131,6 @@ type CompactAssets struct { EtcdClientKey string EtcdKey string EtcdTrustedCA string - KIAMServerCert string - KIAMServerKey string - KIAMAgentCert string - KIAMAgentKey string - KIAMCACert string ServiceAccountKey string // Encrypted -> gzip -> base64 encoded assets. @@ -161,7 +141,7 @@ type CompactAssets struct { EncryptionConfig string } -func ReadRawAssets(dirname string, manageCertificates bool, caKeyRequiredOnController bool, kiamEnabled bool) (*RawAssetsOnDisk, error) { +func ReadRawAssets(dirname string, manageCertificates bool, caKeyRequiredOnController bool) (*RawAssetsOnDisk, error) { defaultTokensFile := "" defaultServiceAccountKey := "<<<" + filepath.Join(dirname, "apiserver-key.pem") defaultTLSBootstrapToken, err := RandomTokenString() @@ -219,14 +199,6 @@ func ReadRawAssets(dirname string, manageCertificates bool, caKeyRequiredOnContr if caKeyRequiredOnController { files = append(files, entry{name: "worker-ca-key.pem", data: &r.WorkerCAKey, defaultValue: nil, expiryCheck: true}) } - - if kiamEnabled { - files = append(files, entry{name: "kiam-server-key.pem", data: &r.KIAMServerKey, defaultValue: nil, expiryCheck: false}) - files = append(files, entry{name: "kiam-server.pem", data: &r.KIAMServerCert, defaultValue: nil, expiryCheck: true}) - files = append(files, entry{name: "kiam-agent-key.pem", data: &r.KIAMAgentKey, defaultValue: nil, expiryCheck: false}) - files = append(files, entry{name: "kiam-agent.pem", data: &r.KIAMAgentCert, defaultValue: nil, expiryCheck: true}) - files = append(files, entry{name: "kiam-ca.pem", data: &r.KIAMCACert, defaultValue: nil, expiryCheck: true}) - } } for _, file := range files { @@ -253,7 +225,7 @@ func ReadRawAssets(dirname string, manageCertificates bool, caKeyRequiredOnContr return r, nil } -func ReadOrEncryptAssets(dirname string, manageCertificates bool, caKeyRequiredOnController bool, kiamEnabled bool, store Store) (*EncryptedAssetsOnDisk, error) { +func ReadOrEncryptAssets(dirname string, manageCertificates bool, caKeyRequiredOnController bool, store Store) (*EncryptedAssetsOnDisk, error) { defaultTokensFile := "" defaultServiceAccountKey := "<<<" + filepath.Join(dirname, "apiserver-key.pem") defaultTLSBootstrapToken, err := RandomTokenString() @@ -308,14 +280,6 @@ func ReadOrEncryptAssets(dirname string, manageCertificates bool, caKeyRequiredO if caKeyRequiredOnController { files = append(files, entry{name: "worker-ca-key.pem", data: &r.WorkerCAKey, defaultValue: nil, readEncrypted: true, expiryCheck: false}) } - - if kiamEnabled { - files = append(files, entry{name: "kiam-server-key.pem", data: &r.KIAMServerKey, defaultValue: nil, readEncrypted: true, expiryCheck: false}) - files = append(files, entry{name: "kiam-server.pem", data: &r.KIAMServerCert, defaultValue: nil, readEncrypted: false, expiryCheck: true}) - files = append(files, entry{name: "kiam-agent-key.pem", data: &r.KIAMAgentKey, defaultValue: nil, readEncrypted: true, expiryCheck: false}) - files = append(files, entry{name: "kiam-agent.pem", data: &r.KIAMAgentCert, defaultValue: nil, readEncrypted: false, expiryCheck: true}) - files = append(files, entry{name: "kiam-ca.pem", data: &r.KIAMCACert, defaultValue: nil, readEncrypted: false, expiryCheck: true}) - } } for _, file := range files { @@ -353,7 +317,7 @@ func ReadOrEncryptAssets(dirname string, manageCertificates bool, caKeyRequiredO return r, nil } -func (r *RawAssetsOnMemory) WriteToDir(dirname string, includeCAKey bool, kiamEnabled bool) error { +func (r *RawAssetsOnMemory) WriteToDir(dirname string, includeCAKey bool) error { type asset struct { name string data []byte @@ -397,16 +361,6 @@ func (r *RawAssetsOnMemory) WriteToDir(dirname string, includeCAKey bool, kiamEn ) } - if kiamEnabled { - assets = append(assets, - asset{"kiam-server-key.pem", r.KIAMServerKey, true, ""}, - asset{"kiam-server.pem", r.KIAMServerCert, true, ""}, - asset{"kiam-agent-key.pem", r.KIAMAgentKey, true, ""}, - asset{"kiam-agent.pem", r.KIAMAgentCert, true, ""}, - asset{"kiam-ca.pem", r.KIAMCACert, true, "ca.pem"}, - ) - } - for _, asset := range assets { path := filepath.Join(dirname, asset.name) @@ -482,7 +436,7 @@ func (r *RawAssetsOnMemory) WriteToDir(dirname string, includeCAKey bool, kiamEn return nil } -func (r *EncryptedAssetsOnDisk) WriteToDir(dirname string, kiamEnabled bool) error { +func (r *EncryptedAssetsOnDisk) WriteToDir(dirname string) error { type asset struct { name string data EncryptedFile @@ -510,20 +464,10 @@ func (r *EncryptedAssetsOnDisk) WriteToDir(dirname string, kiamEnabled bool) err {"apiserver-aggregator-key.pem", r.APIServerAggregatorKey}, {"apiserver-aggregator.pem", r.APIServerAggregatorCert}, {"service-account-key.pem", r.ServiceAccountKey}, - {"tokens.csv", r.AuthTokens}, {"kubelet-tls-bootstrap-token", r.TLSBootstrapToken}, {"encryption-config.yaml", r.EncryptionConfig}, } - if kiamEnabled { - assets = append(assets, - asset{"kiam-server-key.pem", r.KIAMServerKey}, - asset{"kiam-server.pem", r.KIAMServerCert}, - asset{"kiam-agent-key.pem", r.KIAMAgentKey}, - asset{"kiam-agent.pem", r.KIAMAgentCert}, - asset{"kiam-ca.pem", r.KIAMCACert}, - ) - } for _, asset := range assets { if asset.name != "ca-key.pem" { @@ -574,11 +518,6 @@ func (r *RawAssetsOnDisk) Compact() (*CompactAssets, error) { EtcdTrustedCA: compact(r.EtcdTrustedCA), APIServerAggregatorCert: compact(r.APIServerAggregatorCert), APIServerAggregatorKey: compact(r.APIServerAggregatorKey), - KIAMAgentCert: compact(r.KIAMAgentCert), - KIAMAgentKey: compact(r.KIAMAgentKey), - KIAMServerCert: compact(r.KIAMServerCert), - KIAMServerKey: compact(r.KIAMServerKey), - KIAMCACert: compact(r.KIAMCACert), ServiceAccountKey: compact(r.ServiceAccountKey), AuthTokens: compact(r.AuthTokens), @@ -631,11 +570,6 @@ func (r *EncryptedAssetsOnDisk) Compact() (*CompactAssets, error) { EtcdTrustedCA: compact(r.EtcdTrustedCA), APIServerAggregatorCert: compact(r.APIServerAggregatorCert), APIServerAggregatorKey: compact(r.APIServerAggregatorKey), - KIAMAgentKey: compact(r.KIAMAgentKey), - KIAMAgentCert: compact(r.KIAMAgentCert), - KIAMServerKey: compact(r.KIAMServerKey), - KIAMServerCert: compact(r.KIAMServerCert), - KIAMCACert: compact(r.KIAMCACert), ServiceAccountKey: compact(r.ServiceAccountKey), AuthTokens: compact(r.AuthTokens), @@ -679,14 +613,14 @@ func NewKMSConfig(kmsKeyARN string, encSvc KMSEncryptionService, session *sessio } } -func ReadOrCreateEncryptedAssets(tlsAssetsDir string, manageCertificates bool, caKeyRequiredOnController bool, kiamEnabled bool, kmsConfig KMSConfig) (*EncryptedAssetsOnDisk, error) { +func ReadOrCreateEncryptedAssets(tlsAssetsDir string, manageCertificates bool, caKeyRequiredOnController bool, kmsConfig KMSConfig) (*EncryptedAssetsOnDisk, error) { store := kmsConfig.Store() - return ReadOrEncryptAssets(tlsAssetsDir, manageCertificates, caKeyRequiredOnController, kiamEnabled, store) + return ReadOrEncryptAssets(tlsAssetsDir, manageCertificates, caKeyRequiredOnController, store) } -func ReadOrCreateCompactAssets(assetsDir string, manageCertificates bool, caKeyRequiredOnController bool, kiamEnabled bool, kmsConfig KMSConfig) (*CompactAssets, error) { - encryptedAssets, err := ReadOrCreateEncryptedAssets(assetsDir, manageCertificates, caKeyRequiredOnController, kiamEnabled, kmsConfig) +func ReadOrCreateCompactAssets(assetsDir string, manageCertificates bool, caKeyRequiredOnController bool, kmsConfig KMSConfig) (*CompactAssets, error) { + encryptedAssets, err := ReadOrCreateEncryptedAssets(assetsDir, manageCertificates, caKeyRequiredOnController, kmsConfig) if err != nil { return nil, fmt.Errorf("failed to read/create encrypted assets: %v", err) } @@ -699,8 +633,8 @@ func ReadOrCreateCompactAssets(assetsDir string, manageCertificates bool, caKeyR return compactAssets, nil } -func ReadOrCreateUnencryptedCompactAssets(assetsDir string, manageCertificates bool, caKeyRequiredOnController bool, kiamEnabled bool) (*CompactAssets, error) { - unencryptedAssets, err := ReadRawAssets(assetsDir, manageCertificates, caKeyRequiredOnController, kiamEnabled) +func ReadOrCreateUnencryptedCompactAssets(assetsDir string, manageCertificates bool, caKeyRequiredOnController bool) (*CompactAssets, error) { + unencryptedAssets, err := ReadRawAssets(assetsDir, manageCertificates, caKeyRequiredOnController) if err != nil { return nil, fmt.Errorf("failed to read/create encrypted assets: %v", err) } diff --git a/credential/encrypted_assets_test.go b/credential/encrypted_assets_test.go index 2c719c193..e350c611d 100644 --- a/credential/encrypted_assets_test.go +++ b/credential/encrypted_assets_test.go @@ -10,6 +10,7 @@ import ( "strings" "fmt" + "github.com/aws/aws-sdk-go/service/kms" "github.com/kubernetes-incubator/kube-aws/test/helper" ) @@ -64,7 +65,7 @@ func TestReadOrCreateCompactAssets(t *testing.T) { // See https://github.com/kubernetes-incubator/kube-aws/issues/107 t.Run("CachedToPreventUnnecessaryNodeReplacement", func(t *testing.T) { - created, err := ReadOrCreateCompactAssets(dir, true, true, true, kmsConfig) + created, err := ReadOrCreateCompactAssets(dir, true, true, kmsConfig) if err != nil { t.Errorf("failed to read or update compact assets in %s : %v", dir, err) @@ -74,7 +75,7 @@ func TestReadOrCreateCompactAssets(t *testing.T) { // This depends on TestDummyEncryptService which ensures dummy encrypt service to produce different ciphertext for each encryption // created == read means that encrypted assets were loaded from cached files named *.pem.enc, instead of re-encrypting raw assets named *.pem files // TODO Use some kind of mocking framework for tests like this - read, err := ReadOrCreateCompactAssets(dir, true, true, true, kmsConfig) + read, err := ReadOrCreateCompactAssets(dir, true, true, kmsConfig) if err != nil { t.Errorf("failed to read or update compact assets in %s : %v", dir, err) @@ -90,7 +91,7 @@ func TestReadOrCreateCompactAssets(t *testing.T) { }) t.Run("RemoveFilesToRegenerate", func(t *testing.T) { - original, err := ReadOrCreateCompactAssets(dir, true, true, true, kmsConfig) + original, err := ReadOrCreateCompactAssets(dir, true, true, kmsConfig) if err != nil { t.Errorf("failed to read the original encrypted assets : %v", err) @@ -101,7 +102,7 @@ func TestReadOrCreateCompactAssets(t *testing.T) { "admin-key.pem.enc", "worker-key.pem.enc", "apiserver-key.pem.enc", "etcd-key.pem.enc", "etcd-client-key.pem.enc", "worker-ca-key.pem.enc", "kube-controller-manager-key.pem.enc", "kube-scheduler-key.pem.enc", - "kiam-agent-key.pem.enc", "kiam-server-key.pem.enc", "apiserver-aggregator-key.pem.enc", + "apiserver-aggregator-key.pem.enc", } for _, filename := range files { @@ -111,7 +112,7 @@ func TestReadOrCreateCompactAssets(t *testing.T) { } } - regenerated, err := ReadOrCreateCompactAssets(dir, true, true, true, kmsConfig) + regenerated, err := ReadOrCreateCompactAssets(dir, true, true, kmsConfig) if err != nil { t.Errorf("failed to read the regenerated encrypted assets : %v", err) @@ -127,9 +128,6 @@ func TestReadOrCreateCompactAssets(t *testing.T) { {"KubeSchedulerCert", original.KubeSchedulerCert, regenerated.KubeSchedulerCert}, {"EtcdClientCert", original.EtcdClientCert, regenerated.EtcdClientCert}, {"EtcdCert", original.EtcdCert, regenerated.EtcdCert}, - {"KIAMAgentCert", original.KIAMAgentCert, regenerated.KIAMAgentCert}, - {"KIAMServerCert", original.KIAMServerCert, regenerated.KIAMServerCert}, - {"KIAMCACert", original.KIAMCACert, regenerated.KIAMCACert}, {"APIServerAggregatorCert", original.APIServerAggregatorCert, regenerated.APIServerAggregatorCert}, } { if v[1] != v[2] { @@ -146,8 +144,6 @@ func TestReadOrCreateCompactAssets(t *testing.T) { {"KubeSchedulerKey", original.KubeSchedulerKey, regenerated.KubeSchedulerKey}, {"EtcdClientKey", original.EtcdClientKey, regenerated.EtcdClientKey}, {"EtcdKey", original.EtcdKey, regenerated.EtcdKey}, - {"KIAMAgentKey", original.KIAMAgentKey, regenerated.KIAMAgentKey}, - {"KIAMServerKey", original.KIAMServerKey, regenerated.KIAMServerKey}, {"APIServerAggregatorKey", original.APIServerAggregatorKey, regenerated.APIServerAggregatorKey}, } { if v[1] == v[2] { @@ -167,13 +163,13 @@ func TestReadOrCreateCompactAssets(t *testing.T) { func TestReadOrCreateUnEncryptedCompactAssets(t *testing.T) { run := func(dir string, caKeyRequiredOnController bool, t *testing.T) { t.Run("CachedToPreventUnnecessaryNodeReplacementOnUnencrypted", func(t *testing.T) { - created, err := ReadOrCreateUnencryptedCompactAssets(dir, true, caKeyRequiredOnController, true) + created, err := ReadOrCreateUnencryptedCompactAssets(dir, true, caKeyRequiredOnController) if err != nil { t.Errorf("failed to read or update compact assets in %s : %v", dir, err) } - read, err := ReadOrCreateUnencryptedCompactAssets(dir, true, caKeyRequiredOnController, true) + read, err := ReadOrCreateUnencryptedCompactAssets(dir, true, caKeyRequiredOnController) if err != nil { t.Errorf("failed to read or update compact assets in %s : %v", dir, err) diff --git a/credential/generator.go b/credential/generator.go index d063a0c42..36e69fb92 100644 --- a/credential/generator.go +++ b/credential/generator.go @@ -31,8 +31,6 @@ type GeneratorOptions struct { GenerateCA bool CaCertPath string CommonName string - // KIAM is set to true when you want kube-aws to render TLS assets for uswitch/kiam - KIAM bool // Paths for private certificate keys. AdminKeyPath string ApiServerAggregatorKeyPath string @@ -40,8 +38,6 @@ type GeneratorOptions struct { CaKeyPath string EtcdClientKeyPath string EtcdKeyPath string - KiamAgentKeyPath string - KiamServerKeyPath string KubeControllerManagerKeyPath string KubeSchedulerKeyPath string ServiceAccountKeyPath string @@ -87,13 +83,13 @@ func (c Generator) GenerateAssetsOnDisk(dir string, o GeneratorOptions) (*RawAss logger.Info("--> Writing to the storage") alsoWriteCAKey := o.GenerateCA || c.ManageCertificates - if err := assets.WriteToDir(dir, alsoWriteCAKey, o.KIAM); err != nil { + if err := assets.WriteToDir(dir, alsoWriteCAKey); err != nil { return nil, fmt.Errorf("Error creating assets: %v", err) } { logger.Info("--> Verifying the result") - verified, err := ReadRawAssets(dir, c.ManageCertificates, c.ManageCertificates, o.KIAM) + verified, err := ReadRawAssets(dir, c.ManageCertificates, c.ManageCertificates) if err != nil { return nil, fmt.Errorf("failed verifying the result: %v", err) @@ -132,8 +128,6 @@ func (c Generator) GenerateAssetsOnMemory(caKey *rsa.PrivateKey, caCert *x509.Ce generatorOptions.AdminKeyPath: nil, generatorOptions.EtcdKeyPath: nil, generatorOptions.EtcdClientKeyPath: nil, - generatorOptions.KiamAgentKeyPath: nil, - generatorOptions.KiamServerKeyPath: nil, generatorOptions.ServiceAccountKeyPath: nil, generatorOptions.ApiServerAggregatorKeyPath: nil, } @@ -284,41 +278,5 @@ func (c Generator) GenerateAssetsOnMemory(caKey *rsa.PrivateKey, caCert *x509.Ce EncryptionConfig: []byte(encryptionConfig), } - if generatorOptions.KIAM { - // See https://github.com/uswitch/kiam/blob/master/docs/agent.json - agentConfig := pki.ClientCertConfig{ - CommonName: "Kiam Agent", - Duration: certDuration, - } - kiamAgentCert, err := pki.NewSignedClientCertificate(agentConfig, privateKeys[generatorOptions.KiamAgentKeyPath], caCert, caKey) - if err != nil { - return nil, err - } - // See https://github.com/uswitch/kiam/blob/master/docs/server.json - serverConfig := pki.ClientCertConfig{ - CommonName: "Kiam Server", - DNSNames: append( - []string{ - "kiam-server", - "kiam-server:443", - "localhost", - "localhost:443", - "localhost:9610", - }, - ), - Duration: certDuration, - } - kiamServerCert, err := pki.NewSignedKIAMCertificate(serverConfig, privateKeys[generatorOptions.KiamServerKeyPath], caCert, caKey) - if err != nil { - return nil, err - } - - r.KIAMCACert = pki.EncodeCertificatePEM(caCert) - r.KIAMAgentCert = pki.EncodeCertificatePEM(kiamAgentCert) - r.KIAMAgentKey = pki.EncodePrivateKeyPEM(privateKeys[generatorOptions.KiamAgentKeyPath]) - r.KIAMServerCert = pki.EncodeCertificatePEM(kiamServerCert) - r.KIAMServerKey = pki.EncodePrivateKeyPEM(privateKeys[generatorOptions.KiamServerKeyPath]) - } - return r, nil } diff --git a/credential/pki.go b/credential/pki.go index e03e3c4b2..44d5e8e6f 100644 --- a/credential/pki.go +++ b/credential/pki.go @@ -1,10 +1,13 @@ package credential import ( - "github.com/kubernetes-incubator/kube-aws/pkg/api" - "github.com/kubernetes-incubator/kube-aws/pki" + "fmt" "io/ioutil" "os" + + "github.com/kubernetes-incubator/kube-aws/logger" + "github.com/kubernetes-incubator/kube-aws/pkg/api" + "github.com/kubernetes-incubator/kube-aws/pki" ) type ProtectedPKI struct { @@ -19,47 +22,60 @@ func NewProtectedPKI(enc Encryptor) *ProtectedPKI { } } -func (pki *ProtectedPKI) fileExists(path string) bool { - _, err := os.Stat(path) - return err == nil -} - -func (pki *ProtectedPKI) write(path string, data []byte) error { - return ioutil.WriteFile(path, data, 0644) -} - -func (pki *ProtectedPKI) CreateKeyaPair(spec api.KeyPairSpec) error { - keypair, err := pki.GenerateKeyPair(spec) +func (ppki *ProtectedPKI) CreateKeyaPair(spec api.KeyPairSpec) error { + var signer *pki.KeyPair + if spec.Signer != "" { + signerCert, err := ioutil.ReadFile(spec.SignerCertPath()) + if err != nil { + return fmt.Errorf("failed to read signer certificate %s for creating %s: %v", spec.SignerCertPath(), spec.Name, err) + } + signerKey, err := ioutil.ReadFile(spec.SignerKeyPath()) + if err != nil { + return fmt.Errorf("failed to read signer key %s for creating %s: %v", spec.SignerKeyPath(), spec.Name, err) + } + signer, err = pki.KeyPairFromPEMs(spec.Signer, signerCert, signerKey) + } + keypair, err := ppki.GenerateKeyPair(spec, signer) if err != nil { return err } keypath := spec.KeyPath() keypem := keypair.KeyInPEM() - if _, err := CreateEncryptedFile(keypath, keypem, pki); err != nil { + logger.Infof("Writing key pem file %s", keypath) + if err := ioutil.WriteFile(keypath, keypem, 0644); err != nil { return err } crtpath := spec.CertPath() crtpem := keypair.CertInPEM() - if err := pki.write(crtpath, crtpem); err != nil { + logger.Infof("Writing certificate pem file %s", crtpath) + if err := ioutil.WriteFile(crtpath, crtpem, 0644); err != nil { return err } return nil } -func (pki *ProtectedPKI) EnsureKeyPairsCreated(specs []api.KeyPairSpec) error { +func (ppki *ProtectedPKI) EnsureKeyPairsCreated(specs []api.KeyPairSpec) error { for _, spec := range specs { keypath := spec.KeyPath() shapath := spec.KeyPath() + ".fingerprint" encpath := spec.EncryptedKeyPath() crtpath := spec.CertPath() - if !pki.fileExists(keypath) && !pki.fileExists(encpath) && !pki.fileExists(shapath) && !pki.fileExists(crtpath) { - if err := pki.CreateKeyaPair(spec); err != nil { + if !fileExists(keypath) && !fileExists(encpath) && !fileExists(shapath) && !fileExists(crtpath) { + if err := ppki.CreateKeyaPair(spec); err != nil { return err } } } return nil } + +func fileExists(path string) bool { + info, err := os.Stat(path) + if os.IsNotExist(err) { + return false + } + return !info.IsDir() +} diff --git a/docs/getting-started/step-6-configure-add-ons.md b/docs/getting-started/step-6-configure-add-ons.md index d9553751b..d2941a23c 100644 --- a/docs/getting-started/step-6-configure-add-ons.md +++ b/docs/getting-started/step-6-configure-add-ons.md @@ -53,8 +53,8 @@ To allow kube2iam or kiam deployed to worker and controller nodes to assume targ } ``` - To add the policy to controller nodes, set `experimental.kube2IamSupport.enabled` or `experimental.kiamSupport.enabled` to `true` in your `cluster.yaml` (but not both). - For worker nodes, it is `worker.nodePools[].kube2IamSupport.enabled` or `worker.nodePools[].kiamSupport.enabled`. + To add the policy to controller nodes, set `experimental.kube2IamSupport.enabled` or `kubeAwsPlugins.kiam.enabled` to `true` in your `cluster.yaml` (but not both). + For worker nodes, it is `worker.nodePools[].kube2IamSupport.enabled`. 2. Target IAM roles needs to change trust relationships to allow kube-aws worker/controller IAM role to assume the target roles. diff --git a/pkg/api/cluster.go b/pkg/api/cluster.go index c20fb83eb..51f0cc609 100644 --- a/pkg/api/cluster.go +++ b/pkg/api/cluster.go @@ -79,12 +79,6 @@ func NewDefaultCluster() *Cluster { Disk: "xvdb", Filesystem: "xfs", }, - KIAMSupport: KIAMSupport{ - Enabled: false, - Image: Image{Repo: "quay.io/uswitch/kiam", Tag: "v3.2", RktPullDocker: false}, - SessionDuration: "30m", - ServerAddresses: KIAMServerAddresses{ServerAddress: "localhost:443", AgentAddress: "kiam-server:443"}, - }, Kube2IamSupport: Kube2IamSupport{ Enabled: false, }, diff --git a/pkg/api/keypair_spec.go b/pkg/api/keypair_spec.go index a77bdae23..85c9799a4 100644 --- a/pkg/api/keypair_spec.go +++ b/pkg/api/keypair_spec.go @@ -29,3 +29,11 @@ func (spec KeyPairSpec) KeyPath() string { func (spec KeyPairSpec) CertPath() string { return filepath.Join("credentials", fmt.Sprintf("%s.pem", spec.Name)) } + +func (spec KeyPairSpec) SignerCertPath() string { + return filepath.Join("credentials", fmt.Sprintf("%s.pem", spec.Signer)) +} + +func (spec KeyPairSpec) SignerKeyPath() string { + return filepath.Join("credentials", fmt.Sprintf("%s-key.pem", spec.Signer)) +} diff --git a/pkg/api/types.go b/pkg/api/types.go index f8e9107ab..7b166ffa4 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -1,7 +1,5 @@ package api -import "fmt" - type Worker struct { APIEndpointName string `yaml:"apiEndpointName,omitempty"` NodePools []WorkerNodePool `yaml:"nodePools,omitempty"` @@ -25,7 +23,6 @@ type Experimental struct { AwsEnvironment AwsEnvironment `yaml:"awsEnvironment"` AwsNodeLabels AwsNodeLabels `yaml:"awsNodeLabels"` EphemeralImageStorage EphemeralImageStorage `yaml:"ephemeralImageStorage"` - KIAMSupport KIAMSupport `yaml:"kiamSupport,omitempty"` Kube2IamSupport Kube2IamSupport `yaml:"kube2IamSupport,omitempty"` GpuSupport GpuSupport `yaml:"gpuSupport,omitempty"` KubeletOpts string `yaml:"kubeletOpts,omitempty"` @@ -43,10 +40,6 @@ func (c Experimental) Validate(name string) error { return err } - if c.Kube2IamSupport.Enabled && c.KIAMSupport.Enabled { - return fmt.Errorf("at '%s', you can enable kube2IamSupport or kiamSupport, but not both", name) - } - return nil } @@ -115,20 +108,6 @@ type EphemeralImageStorage struct { Filesystem string `yaml:"filesystem"` } -type KIAMSupport struct { - Enabled bool `yaml:"enabled"` - Image Image `yaml:"image,omitempty"` - SessionDuration string `yaml:"sessionDuration,omitempty"` - ServerAddresses KIAMServerAddresses `yaml:"serverAddresses,omitempty"` - ServerResources ComputeResources `yaml:"serverResources,omitempty"` - AgentResources ComputeResources `yaml:"agentResources,omitempty"` -} - -type KIAMServerAddresses struct { - ServerAddress string `yaml:"serverAddress,omitempty"` - AgentAddress string `yaml:"agentAddress,omitempty"` -} - type Kube2IamSupport struct { Enabled bool `yaml:"enabled"` } diff --git a/pkg/model/cluster_test.go b/pkg/model/cluster_test.go index 4b2868abe..991b04116 100644 --- a/pkg/model/cluster_test.go +++ b/pkg/model/cluster_test.go @@ -1415,20 +1415,6 @@ kubernetesVersion: v1.10.2 } } -func TestKube2IamKiamClash(t *testing.T) { - config := ` -experimental: - kube2IamSupport: - enabled: true - kiamSupport: - enabled: true -` - confBody := singleAzConfigYaml + config - _, err := ClusterFromBytes([]byte(confBody)) - if err == nil || !strings.Contains(err.Error(), "not both") { - t.Errorf("expected config to cause error as kube2iam and kiam cannot be enabled together: %s\n%s", err, confBody) - } -} func TestKMSArnValidateRegion(t *testing.T) { config := `keyName: test-key-name s3URI: s3://mybucket/mydir diff --git a/pkg/model/credentials.go b/pkg/model/credentials.go index 4900ac0da..337cebee3 100644 --- a/pkg/model/credentials.go +++ b/pkg/model/credentials.go @@ -14,14 +14,14 @@ func LoadCredentials(sess *session.Session, cfg *Config, opts api.StackTemplateO func (s *Context) LoadCredentials(cfg *Config, opts api.StackTemplateOptions) (*credential.CompactAssets, error) { if cfg.AssetsEncryptionEnabled() { kmsConfig := credential.NewKMSConfig(cfg.KMSKeyARN, s.ProvidedEncryptService, s.Session) - compactAssets, err := credential.ReadOrCreateCompactAssets(opts.AssetsDir, cfg.ManageCertificates, true, cfg.Experimental.KIAMSupport.Enabled, kmsConfig) + compactAssets, err := credential.ReadOrCreateCompactAssets(opts.AssetsDir, cfg.ManageCertificates, true, kmsConfig) if err != nil { return nil, err } return compactAssets, nil } else { - rawAssets, err := credential.ReadOrCreateUnencryptedCompactAssets(opts.AssetsDir, cfg.ManageCertificates, true, cfg.Experimental.KIAMSupport.Enabled) + rawAssets, err := credential.ReadOrCreateUnencryptedCompactAssets(opts.AssetsDir, cfg.ManageCertificates, true) if err != nil { return nil, err } diff --git a/pkg/model/credentials_test.go b/pkg/model/credentials_test.go index c177453f4..6cd7ccfd3 100644 --- a/pkg/model/credentials_test.go +++ b/pkg/model/credentials_test.go @@ -23,7 +23,7 @@ func genAssets(t *testing.T) *credential.RawAssetsOnMemory { } cfg, err := Compile(c, api.ClusterOptions{}) r := NewCredentialGenerator(cfg) - assets, err := r.GenerateAssetsOnMemory(caKey, caCert, credential.GeneratorOptions{KIAM: true}) + assets, err := r.GenerateAssetsOnMemory(caKey, caCert, credential.GeneratorOptions{}) if err != nil { t.Fatalf("failed generating assets: %v", err) } diff --git a/pkg/model/stack_test.go b/pkg/model/stack_test.go index daa54aa6b..92d42fa58 100644 --- a/pkg/model/stack_test.go +++ b/pkg/model/stack_test.go @@ -15,12 +15,13 @@ import ( "strings" "testing" + "os" + "path/filepath" + "github.com/go-yaml/yaml" "github.com/kubernetes-incubator/kube-aws/credential" "github.com/kubernetes-incubator/kube-aws/pkg/api" "github.com/kubernetes-incubator/kube-aws/plugin/clusterextension" - "os" - "path/filepath" ) /* @@ -756,7 +757,6 @@ func clusterToStackForTesting(c *api.Cluster, opts api.StackTemplateOptions) (*S genopts := credential.GeneratorOptions{ GenerateCA: true, - KIAM: true, } if _, err := sess.GenerateAssetsOnDisk(compiled, opts.AssetsDir, genopts); err != nil { diff --git a/pki/keypair.go b/pki/keypair.go index ae23e6e18..385b30223 100644 --- a/pki/keypair.go +++ b/pki/keypair.go @@ -1,5 +1,31 @@ package pki +import ( + "crypto/rsa" + "crypto/x509" + "fmt" +) + +func KeyPairFromPEMs(id string, certpem []byte, keypem []byte) (*KeyPair, error) { + var cert *x509.Certificate + var key *rsa.PrivateKey + var err error + if cert, err = DecodeCertificatePEM(certpem); err != nil { + return nil, fmt.Errorf("failed to decode certificate pem: %v", err) + } + if key, err = DecodePrivateKeyPEM(keypem); err != nil { + return nil, fmt.Errorf("failed to decode private key pem: %v", err) + } + kp := KeyPair{ + Key: key, + Cert: cert, + id: id, + keyPem: keypem, + certPem: certpem, + } + return &kp, nil +} + func (keypair *KeyPair) KeyInPEM() []byte { if keypair.keyPem == nil { keypair.keyPem = EncodePrivateKeyPEM(keypair.Key) diff --git a/pki/pki.go b/pki/pki.go index dbd9ff10a..a8b1e3061 100644 --- a/pki/pki.go +++ b/pki/pki.go @@ -2,15 +2,18 @@ package pki import ( "crypto/rand" + "crypto/rsa" "crypto/x509" "crypto/x509/pkix" "errors" "fmt" - "github.com/kubernetes-incubator/kube-aws/pkg/api" "math" "math/big" "net" "time" + + "github.com/kubernetes-incubator/kube-aws/logger" + "github.com/kubernetes-incubator/kube-aws/pkg/api" ) type PKI struct { @@ -20,7 +23,8 @@ func NewPKI() *PKI { return &PKI{} } -func (pki *PKI) GenerateKeyPair(spec api.KeyPairSpec) (*KeyPair, error) { +func (pki *PKI) GenerateKeyPair(spec api.KeyPairSpec, signer *KeyPair) (*KeyPair, error) { + logger.Debugf("GenerateKeyPair - spec: %+v", spec) key, err := NewPrivateKey() if err != nil { return nil, err @@ -62,6 +66,7 @@ func (pki *PKI) GenerateKeyPair(spec api.KeyPairSpec) (*KeyPair, error) { ips[i] = net.ParseIP(ipStr) } + logger.Debugf("Generating x509 certificate template") tmpl := x509.Certificate{ SerialNumber: serialNumber, Subject: pkix.Name{ @@ -78,16 +83,23 @@ func (pki *PKI) GenerateKeyPair(spec api.KeyPairSpec) (*KeyPair, error) { IsCA: isCA, } - caCert := &tmpl - caKey := key - - if spec.Signer != "" { - // TODO load signer key and cert - caKey = nil - caCert = nil + // handle self-signed/CA certificates or certs signed by a CA + var signerCert *x509.Certificate + var signerKey *rsa.PrivateKey + if signer == nil { + if spec.Signer != "" { + return nil, fmt.Errorf("The certificate spec includes a signer but singer KeyPair is missing") + } + logger.Debugf("This certificate is going to be self-signed!") + signerCert = &tmpl + signerKey = key + } else { + signerCert = signer.Cert + signerKey = signer.Key } - certAsn1DERData, err := x509.CreateCertificate(rand.Reader, &tmpl, caCert, key.Public(), caKey) + logger.Debugf("Creating x509 certificate...") + certAsn1DERData, err := x509.CreateCertificate(rand.Reader, &tmpl, signerCert, key.Public(), signerKey) if err != nil { return nil, err } @@ -96,5 +108,6 @@ func (pki *PKI) GenerateKeyPair(spec api.KeyPairSpec) (*KeyPair, error) { return nil, err } + logger.Debugf("returning keypair..") return &KeyPair{Key: key, Cert: cert}, nil } diff --git a/pki/x509.go b/pki/x509.go index 8388a2424..c2853b0b5 100644 --- a/pki/x509.go +++ b/pki/x509.go @@ -131,38 +131,3 @@ func NewSignedClientCertificate(cfg ClientCertConfig, key *rsa.PrivateKey, caCer } return x509.ParseCertificate(certDERBytes) } - -func NewSignedKIAMCertificate(cfg ClientCertConfig, key *rsa.PrivateKey, caCert *x509.Certificate, caKey *rsa.PrivateKey) (*x509.Certificate, error) { - ips := make([]net.IP, len(cfg.IPAddresses)) - for i, ipStr := range cfg.IPAddresses { - ips[i] = net.ParseIP(ipStr) - } - - serial, err := rand.Int(rand.Reader, new(big.Int).SetInt64(math.MaxInt64)) - if err != nil { - return nil, err - } - - if cfg.Duration <= 0 { - return nil, errors.New("signed client cert duration must not be negative or zero") - } - - certTmpl := x509.Certificate{ - Subject: pkix.Name{ - CommonName: cfg.CommonName, - Organization: append(caCert.Subject.Organization, cfg.Organization...), - }, - DNSNames: cfg.DNSNames, - IPAddresses: ips, - SerialNumber: serial, - NotBefore: caCert.NotBefore, - NotAfter: time.Now().Add(cfg.Duration).UTC(), - KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, - } - certDERBytes, err := x509.CreateCertificate(rand.Reader, &certTmpl, caCert, key.Public(), caKey) - if err != nil { - return nil, err - } - return x509.ParseCertificate(certDERBytes) -} diff --git a/plugin/clusterextension/extras.go b/plugin/clusterextension/extras.go index 2507bfa80..30aee3c55 100644 --- a/plugin/clusterextension/extras.go +++ b/plugin/clusterextension/extras.go @@ -57,14 +57,12 @@ func (e ClusterExtension) KeyPairSpecs() []api.KeyPairSpec { } func (e ClusterExtension) RootStack(config interface{}) (*stack, error) { - logger.Debugf("Generating Plugin extras for root cloudformation stack") return e.stackExt("root", config, func(p *api.Plugin) api.Stack { return p.Spec.Cluster.CloudFormation.Stacks.Root }) } func (e ClusterExtension) NetworkStack(config interface{}) (*stack, error) { - logger.Debugf("Generating Plugin extras for network cloudformation stack") return e.stackExt("network", config, func(p *api.Plugin) api.Stack { return p.Spec.Cluster.CloudFormation.Stacks.Network }) @@ -126,7 +124,6 @@ func (e ClusterExtension) stackExt(name string, config interface{}, src func(p * tags := map[string]interface{}{} err := e.foreachEnabledPlugins(func(p *api.Plugin, pc *api.PluginConfig) error { - logger.Debugf("extras.go stackExt() foreachEnabledPlugins extending stack %s: %+v into %+v", name, pc.Values, p.Spec.Cluster.Values) values, err := pluginutil.MergeValues(p.Spec.Cluster.Values, pc.Values) if err != nil { return err @@ -139,6 +136,9 @@ func (e ClusterExtension) stackExt(name string, config interface{}, src func(p * if err != nil { return fmt.Errorf("failed to load additional resources for %s stack: %v", name, err) } + if l := len(m); l > 0 { + logger.Infof("plugin %s extended stack %s with %d resources", p.Name, name, l) + } for k, v := range m { resources[k] = v } @@ -147,6 +147,9 @@ func (e ClusterExtension) stackExt(name string, config interface{}, src func(p * if err != nil { return fmt.Errorf("failed to load additional outputs for %s stack: %v", name, err) } + if l := len(m); l > 0 { + logger.Infof("plugin %s extended stack %s with %d outputs", p.Name, name, l) + } for k, v := range m { outputs[k] = v } @@ -155,6 +158,9 @@ func (e ClusterExtension) stackExt(name string, config interface{}, src func(p * if err != nil { return fmt.Errorf("failed to load additional tags for %s stack: %v", name, err) } + if l := len(m); l > 0 { + logger.Infof("plugin %s extended stack %s with %d tags", p.Name, name, l) + } for k, v := range m { tags[k] = v } @@ -311,6 +317,7 @@ func (e ClusterExtension) Worker(config interface{}) (*worker, error) { for _, p := range e.plugins { if enabled, pc := p.EnabledIn(e.Configs); enabled { + logger.Debugf("Adding worker extensions from plugin %s", p.Name) values, err := pluginutil.MergeValues(p.Spec.Cluster.Values, pc.Values) if err != nil { return nil, err @@ -321,37 +328,63 @@ func (e ClusterExtension) Worker(config interface{}) (*worker, error) { if err != nil { return nil, fmt.Errorf("failed adding systemd units to worker: %v", err) } + if l := len(extraUnits); l > 0 { + logger.Infof("plugin %s added %d extra worker systemd units", p.Name, l) + } systemdUnits = append(systemdUnits, extraUnits...) extraArchivedFiles, extraFiles, extraConfigSetFiles, err := renderMachineFilesAndConfigSets(render, p.Spec.Cluster.Roles.Worker.Files) if err != nil { return nil, fmt.Errorf("failed adding files to worker: %v", err) } + if l := len(extraArchivedFiles); l > 0 { + logger.Infof("plugin %s added %d extra worker extra archive files", p.Name, l) + } + if l := len(extraFiles); l > 0 { + logger.Infof("plugin %s added %d extra worker extra files", p.Name, l) + } + if l := len(extraConfigSetFiles); l > 0 { + logger.Infof("plugin %s added %d extra worker extra config-set files", p.Name, l) + } archivedFiles = append(archivedFiles, extraArchivedFiles...) files = append(files, extraFiles...) configsets[p.Name] = map[string]map[string]interface{}{ "files": extraConfigSetFiles, } + if l := len(p.Spec.Cluster.Machine.Roles.Worker.IAM.Policy.Statements); l > 0 { + logger.Infof("plugin %s added %d extra worker iam policies", p.Name, l) + } iamStatements = append(iamStatements, p.Spec.Cluster.Machine.Roles.Worker.IAM.Policy.Statements...) + if l := len(p.Spec.Cluster.Machine.Roles.Worker.Kubelet.NodeLabels); l > 0 { + logger.Infof("plugin %s added %d extra worker node labels", p.Name, l) + } for k, v := range p.Spec.Cluster.Machine.Roles.Worker.Kubelet.NodeLabels { nodeLabels[k] = v } + if l := len(p.Spec.Cluster.Machine.Roles.Worker.Kubelet.FeatureGates); l > 0 { + logger.Infof("plugin %s added %d extra worker kubelet feature gates", p.Name, l) + } for k, v := range p.Spec.Cluster.Machine.Roles.Worker.Kubelet.FeatureGates { featureGates[k] = v } if p.Spec.Cluster.Machine.Roles.Worker.Kubelet.Kubeconfig != "" { + logger.Infof("plugin %s changed the worker kubeconfig", p.Name) kubeconfig = p.Spec.Cluster.Machine.Roles.Worker.Kubelet.Kubeconfig } if len(p.Spec.Cluster.Machine.Roles.Controller.Kubelet.Mounts) > 0 { + logger.Infof("plugin %s added %d worker kubelet mounts", p.Name, len(p.Spec.Cluster.Machine.Roles.Controller.Kubelet.Mounts)) kubeletMounts = append(kubeletMounts, p.Spec.Cluster.Machine.Roles.Controller.Kubelet.Mounts...) } extraKubeletFlags, err := getFlags(render, p.Spec.Cluster.Kubernetes.Kubelet.Flags) + if l := len(extraKubeletFlags); l > 0 { + logger.Infof("plugin %s added %d extra worker kubelet command-line flags", p.Name, l) + } if err != nil { return nil, err } @@ -374,14 +407,12 @@ func (e ClusterExtension) Worker(config interface{}) (*worker, error) { } func (e ClusterExtension) ControlPlaneStack(config interface{}) (*stack, error) { - logger.Debugf("Generating Plugin extras for control-plane cloudformation stack") return e.stackExt("control-plane", config, func(p *api.Plugin) api.Stack { return p.Spec.Cluster.CloudFormation.Stacks.ControlPlane }) } func (e ClusterExtension) EtcdStack(config interface{}) (*stack, error) { - logger.Debugf("Generating Plugin extras for etcd cloudformation stack") return e.stackExt("etcd", config, func(p *api.Plugin) api.Stack { return p.Spec.Cluster.CloudFormation.Stacks.Etcd }) @@ -396,7 +427,7 @@ func renderKubernetesManifests(pluginName string, r *plugincontents.TemplateRend for _, m := range mspecs { rendered, ma, err := regularOrConfigSetFile(m.RemoteFileSpec, r) if err != nil { - return files, manifests, configsetFiles, nil + return files, manifests, configsetFiles, fmt.Errorf("Failed to render plugin kubernetes manifest: %v", err) } var name string if m.Name == "" { @@ -508,6 +539,7 @@ func (e ClusterExtension) Controller(clusterConfig interface{}) (*controller, er for _, p := range e.plugins { //fmt.Fprintf(os.Stderr, "plugin=%+v configs=%+v", p, e.configs) if enabled, pc := p.EnabledIn(e.Configs); enabled { + logger.Debugf("Adding controller extensions from plugin %s", p.Name) values, err := pluginutil.MergeValues(p.Spec.Cluster.Values, pc.Values) if err != nil { return nil, err @@ -518,59 +550,98 @@ func (e ClusterExtension) Controller(clusterConfig interface{}) (*controller, er if err != nil { return nil, err } + if l := len(extraApiServerFlags); l > 0 { + logger.Infof("plugin %s added %d extra controller api server command-line flags", p.Name, l) + } apiServerFlags = append(apiServerFlags, extraApiServerFlags...) extraControllerManagerFlags, err := getFlags(render, p.Spec.Cluster.Kubernetes.ControllerManager.Flags) if err != nil { return nil, err } + if l := len(extraControllerManagerFlags); l > 0 { + logger.Infof("plugin %s added %d extra controller controller-manager command-line flags", p.Name, l) + } controllerFlags = append(controllerFlags, extraControllerManagerFlags...) extraKubeSchedulerFlags, err := getFlags(render, p.Spec.Cluster.Kubernetes.KubeScheduler.Flags) if err != nil { return nil, err } + if l := len(extraKubeSchedulerFlags); l > 0 { + logger.Infof("plugin %s added %d extra controller scheduler command-line flags", p.Name, l) + } kubeSchedulerFlags = append(kubeSchedulerFlags, extraKubeSchedulerFlags...) extraKubeletFlags, err := getFlags(render, p.Spec.Cluster.Kubernetes.Kubelet.Flags) if err != nil { return nil, err } + if l := len(extraKubeSchedulerFlags); l > 0 { + logger.Infof("plugin %s added %d extra controller kubelet command-line flags", p.Name, l) + } kubeletFlags = append(kubeletFlags, extraKubeletFlags...) for key, value := range p.Spec.Cluster.Kubernetes.KubeProxy.Config { kubeProxyConfig[key] = value } + if l := len(p.Spec.Cluster.Kubernetes.KubeProxy.Config); l > 0 { + logger.Infof("plugin %s added %d extra controller kube-proxy configuration keys", p.Name, l) + } apiServerVolumes = append(apiServerVolumes, p.Spec.Cluster.Kubernetes.APIServer.Volumes...) + if l := len(p.Spec.Cluster.Kubernetes.APIServer.Volumes); l > 0 { + logger.Infof("plugin %s added %d extra controller volumes", p.Name, l) + } extraUnits, err := renderMachineSystemdUnits(render, p.Spec.Cluster.Machine.Roles.Controller.Systemd.Units) if err != nil { return nil, fmt.Errorf("failed adding systemd units to etcd: %v", err) } + if l := len(extraUnits); l > 0 { + logger.Infof("plugin %s added %d extra controller systemd units", p.Name, l) + } systemdUnits = append(systemdUnits, extraUnits...) extraArchivedFiles, extraFiles, extraConfigSetFiles, err := renderMachineFilesAndConfigSets(render, p.Spec.Cluster.Roles.Controller.Files) if err != nil { return nil, fmt.Errorf("failed adding files to controller: %v", err) } + if l := len(extraArchivedFiles); l > 0 { + logger.Infof("plugin %s added %d extra controller extra archive files", p.Name, l) + } + if l := len(extraFiles); l > 0 { + logger.Infof("plugin %s added %d extra controller extra files", p.Name, l) + } + if l := len(extraConfigSetFiles); l > 0 { + logger.Infof("plugin %s added %d extra controller extra config-set files", p.Name, l) + } archivedFiles = append(archivedFiles, extraArchivedFiles...) files = append(files, extraFiles...) + if l := len(p.Spec.Cluster.Machine.Roles.Controller.IAM.Policy.Statements); l > 0 { + logger.Infof("plugin %s added %d extra controller iam policies", p.Name, l) + } iamStatements = append(iamStatements, p.Spec.Cluster.Machine.Roles.Controller.IAM.Policy.Statements...) + if l := len(p.Spec.Cluster.Machine.Roles.Controller.Kubelet.NodeLabels); l > 0 { + logger.Infof("plugin %s added %d extra controller node labels", p.Name, l) + } for k, v := range p.Spec.Cluster.Machine.Roles.Controller.Kubelet.NodeLabels { nodeLabels[k] = v } if p.Spec.Cluster.Machine.Roles.Controller.Kubelet.Kubeconfig != "" { + logger.Infof("plugin %s changed the controller kubeconfig", p.Name) kubeconfig = p.Spec.Cluster.Machine.Roles.Controller.Kubelet.Kubeconfig } if len(p.Spec.Cluster.Machine.Roles.Controller.Kubelet.Mounts) > 0 { + logger.Infof("plugin %s added %d controller kubelet mounts", p.Name, len(p.Spec.Cluster.Machine.Roles.Controller.Kubelet.Mounts)) kubeletMounts = append(kubeletMounts, p.Spec.Cluster.Machine.Roles.Controller.Kubelet.Mounts...) } + logger.Debugf("Rendering Controller files and manifests...") extraFiles, extraManifests, manifestConfigSetFiles, err := renderKubernetesManifests(p.Name, render, p.Spec.Cluster.Kubernetes.Manifests) if err != nil { return nil, fmt.Errorf("failed adding kubernetes manifests to controller: %v", err) @@ -581,6 +652,9 @@ func (e ClusterExtension) Controller(clusterConfig interface{}) (*controller, er for k, v := range manifestConfigSetFiles { extraConfigSetFiles[k] = v } + if l := len(extraManifests); l > 0 { + logger.Infof("plugin %s added %d extra kubernetes manifests", p.Name, l) + } configsets[p.Name] = map[string]map[string]interface{}{ "files": extraConfigSetFiles, } @@ -618,6 +692,7 @@ func (e ClusterExtension) Etcd(clusterConfig interface{}) (*etcd, error) { for _, p := range e.plugins { if enabled, pc := p.EnabledIn(e.Configs); enabled { + logger.Debugf("Adding etcd extensions from plugin %s", p.Name) values, err := pluginutil.MergeValues(p.Spec.Cluster.Values, pc.Values) if err != nil { return nil, err @@ -628,12 +703,18 @@ func (e ClusterExtension) Etcd(clusterConfig interface{}) (*etcd, error) { if err != nil { return nil, fmt.Errorf("failed adding systemd units to etcd: %v", err) } + if l := len(extraUnits); l > 0 { + logger.Infof("plugin %s added %d extra etcd systemd units", p.Name, l) + } systemdUnits = append(systemdUnits, extraUnits...) extraFiles, err := simpleRenderMachineFiles(render, p.Spec.Cluster.Roles.Etcd.Files) if err != nil { return nil, fmt.Errorf("failed adding files to etcd: %v", err) } + if l := len(extraFiles); l > 0 { + logger.Infof("plugin %s added %d extra etcd files", p.Name, l) + } files = append(files, extraFiles...) iamStatements = append(iamStatements, p.Spec.Cluster.Roles.Etcd.IAM.Policy.Statements...) diff --git a/test/helper/helper.go b/test/helper/helper.go index dd439cbb0..9cc9480ac 100644 --- a/test/helper/helper.go +++ b/test/helper/helper.go @@ -57,7 +57,7 @@ func withDummyCredentials(alsoWriteCAKey bool, fn func(dir string)) { // config/temp, nodepool/config/temp, test/integration/temp defer os.RemoveAll(dir) - for _, pairName := range []string{"ca", "apiserver", "kube-controller-manager", "kube-scheduler", "worker", "admin", "etcd", "etcd-client", "kiam-agent", "kiam-server", "apiserver-aggregator"} { + for _, pairName := range []string{"ca", "apiserver", "kube-controller-manager", "kube-scheduler", "worker", "admin", "etcd", "etcd-client", "apiserver-aggregator"} { certFile := fmt.Sprintf("%s/%s.pem", dir, pairName) if err := ioutil.WriteFile(certFile, []byte(dummyCert), 0644); err != nil { panic(err) @@ -81,7 +81,6 @@ func withDummyCredentials(alsoWriteCAKey bool, fn func(dir string)) { symlinks := []symlink{ {"ca.pem", "worker-ca.pem"}, {"ca.pem", "etcd-trusted-ca.pem"}, - {"ca.pem", "kiam-ca.pem"}, } if alsoWriteCAKey { diff --git a/test/integration/maincluster_test.go b/test/integration/maincluster_test.go index 260b6e0ed..33ea38d8a 100644 --- a/test/integration/maincluster_test.go +++ b/test/integration/maincluster_test.go @@ -121,12 +121,6 @@ func TestMainClusterConfig(t *testing.T) { Disk: "xvdb", Filesystem: "xfs", }, - KIAMSupport: api.KIAMSupport{ - Enabled: false, - Image: api.Image{Repo: "quay.io/uswitch/kiam", Tag: "v3.2", RktPullDocker: false}, - SessionDuration: "30m", - ServerAddresses: api.KIAMServerAddresses{ServerAddress: "localhost:443", AgentAddress: "kiam-server:443"}, - }, Kube2IamSupport: api.Kube2IamSupport{ Enabled: false, }, @@ -1263,8 +1257,6 @@ experimental: enabled: true ephemeralImageStorage: enabled: true - kiamSupport: - enabled: false kube2IamSupport: enabled: true gpuSupport: @@ -1349,12 +1341,6 @@ worker: Disk: "xvdb", Filesystem: "xfs", }, - KIAMSupport: api.KIAMSupport{ - Enabled: false, - Image: api.Image{Repo: "quay.io/uswitch/kiam", Tag: "v3.2", RktPullDocker: false}, - SessionDuration: "30m", - ServerAddresses: api.KIAMServerAddresses{ServerAddress: "localhost:443", AgentAddress: "kiam-server:443"}, - }, Kube2IamSupport: api.Kube2IamSupport{ Enabled: true, }, @@ -1504,71 +1490,6 @@ worker: }, }, }, - { - context: "WithExperimentalFeatureKiam", - configYaml: minimalValidConfigYaml + ` -experimental: - kiamSupport: - enabled: true - image: - repo: quay.io/uswitch/kiam - tag: v2.6 - sessionDuration: 30m - serverAddresses: - serverAddress: localhost:443 - agentAddress: kiam-server:443 -worker: - nodePools: - - name: pool1 -`, - assertConfig: []ConfigTester{ - func(c *config.Config, t *testing.T) { - expected := api.KIAMSupport{ - Enabled: true, - Image: api.Image{Repo: "quay.io/uswitch/kiam", Tag: "v2.6", RktPullDocker: false}, - SessionDuration: "30m", - ServerAddresses: api.KIAMServerAddresses{ServerAddress: "localhost:443", AgentAddress: "kiam-server:443"}, - } - - actual := c.Experimental - - if !reflect.DeepEqual(expected, actual.KIAMSupport) { - t.Errorf("experimental settings didn't match : expected=%+v actual=%+v", expected, actual) - } - - p := c.NodePools[0] - if reflect.DeepEqual(expected, p.Experimental.KIAMSupport) { - t.Errorf("experimental settings shouldn't be inherited to a node pool but it did : toplevel=%v nodepool=%v", expected, p.Experimental) - } - }, - }, - }, - { - context: "WithExperimentalFeatureKiamForWorkerNodePool", - configYaml: minimalValidConfigYaml + ` -worker: - nodePools: - - name: pool1 - kiamSupport: - enabled: true -`, - assertConfig: []ConfigTester{ - func(c *config.Config, t *testing.T) { - expected := api.Experimental{ - KIAMSupport: api.KIAMSupport{ - Enabled: true, - Image: api.Image{Repo: "quay.io/uswitch/kiam", Tag: "v3.2", RktPullDocker: false}, - SessionDuration: "30m", - ServerAddresses: api.KIAMServerAddresses{ServerAddress: "localhost:443", AgentAddress: "kiam-server:443"}, - }, - } - p := c.NodePools[0] - if reflect.DeepEqual(expected, p.Experimental) { - t.Errorf("experimental settings for node pool didn't match : expected=%v actual=%v", expected, p.Experimental) - } - }, - }, - }, { context: "WithKube2IamSupport", configYaml: minimalValidConfigYaml + `