move Kiam to a plugin

builtin/files/cluster.yaml.tmpl

@@ -240,7 +240,7 @@ kmsKeyArn: "{{.KMSKeyARN}}"
 #      # If you specify a name for the role, kube-aws will create it without a random id suffix (AWS default).
 #      # Given a role of `yourManagedRole`, this will create an IAM role named `${clusterName}-${region}-yourManagedRole`
 #      # to follow the recommendation in AWS documentation  http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-role.html
-#      # There are several cases you might need a stable name such as with .experimental.kube2IamSupport or .experimental.kiamSupport. See #297 for more information.
+#      # There are several cases you might need a stable name such as with .experimental.kube2IamSupport or kiam plugin. See #297 for more information.
 #      # ATTENTION: Consider limiting number of characters in clusterName and iam.role.name to avoid the resulting IAM
 #      # role name's length from exceeding the AWS limit: 64. See https://github.com/kubernetes-incubator/kube-aws/issues/347
 #      # It will have attached a customer Managed Policy that you can modify afterwards if you need more permissions for your cluster.
@@ -362,7 +362,7 @@ worker:
 #          # If you specify a name for the role, kube-aws will create it without a random id suffix (AWS default).
 #          # Given a role of `yourManagedRole`, this will create an IAM role named `${clusterName}-${region}-yourManagedRole`
 #          # to follow the recommendation in AWS documentation  http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-role.html
-#          # There are several cases you might need a stable name such as with .experimental.kube2IamSupport or .experimental.kiamSupport. See #297 for more information.
+#          # There are several cases you might need a stable name such as with .experimental.kube2IamSupport or kiam plugin. See #297 for more information.
 #          # ATTENTION: Consider limiting number of characters in clusterName and iam.role.name to avoid the resulting IAM
 #          # role name's length from exceeding the AWS limit: 64. See https://github.com/kubernetes-incubator/kube-aws/issues/347
 #          # It will have attached a customer Managed Policy that you can modify afterwards if you need more permissions for your cluster.
@@ -1444,38 +1444,6 @@ experimental:
   ephemeralImageStorage:
     enabled: false
 
-  # When enabled this will install the kiam daemon set using the repo from `kiamImage`
-  # It will also grant sts:assumeRole permission to the IAM role for controller nodes (only controller nodes where the kiam server is hosted need this, worker nodes do not)
-  # This will use SSL certificates generated during `kube-aws render credentials`, ensure this has been run with kube-aws v0.9.9+
-  # This is intended to be used in combination with .controller.iam.role.name. See #297 for more information.
-  kiamSupport:
-    enabled: false
-    # image:
-    #  repo: quay.io/uswitch/kiam
-    #  tag: v3.2
-    #  rktPullDocker: false
-    # sessionDuration: 30m
-    # serverAddresses:
-    #  serverAddress: localhost:443
-    #  agentAddress: kiam-server:443
-    # Optional resource change for kiam servers/agents can be done via using the resources block below and changing the values.
-    # Values below are the default if not set.
-    # serverResources:
-    #   requests:
-    #     cpu: 50m
-    #     memory: 10Mi
-    #   limits:
-    #     cpu: 200m
-    #     memory: 200Mi
-    # agentResources:
-    #   requests:
-    #     cpu: 10m
-    #     memory: 5Mi
-    #   limits:
-    #     cpu: 200m
-    #     memory: 100Mi
-
-
   # When enabled this will install the kube2iam daemon set using the repo from `kube2iamImage`
   # It will also grant sts:assumeRole permission to the IAM role for controller nodes.
   # This is intended to be used in combination with .controller.iam.role.name. See #297 for more information.
@@ -1607,18 +1575,45 @@ kubeAwsPlugins:
       selector:
         prometheus: monitoring
 
-    # upgradeHelper - assists when rolling out new versions of kubernetes
-    # It actively disables old controllers and temporarily removes mutating/validating webhooks whilst
-    # the upgraded controller is starting up.
-    # NOTE: You will normally not need this plugin - so ONLY enable if you are experiencing issues testing migrating across versions.
-    # It will only kill controller that are a different release from currently spinning up version, e.g. :-
-    #   it will kill v1.13.2 controllers when rolling out v1.14.0
-    #   it will NOT kill v1.14.0 controllers when rolling out v1.14.3
-    upgradeHelper:
-      enabled: false
-      # disableWebhooks can be used to turn off the webhook feature if required
-      disableWebhooks: true
-      # disableWorkerCommunicationChecks can be used to disable the check that all workers checking in with this master
-      # so that cfn-signal can be generated and upgrade process move forward
-      # (you don't want a single bad worker to cause an important upgrade to fail)
-      disableWorkerCommunicationChecks: true
+  # upgradeHelper - assists when rolling out new versions of kubernetes
+  # It actively disables old controllers and temporarily removes mutating/validating webhooks whilst
+  # the upgraded controller is starting up.
+  # NOTE: You will normally not need this plugin - so ONLY enable if you are experiencing issues testing migrating across versions.
+  # It will only kill controller that are a different release from currently spinning up version, e.g. :-
+  #   it will kill v1.13.2 controllers when rolling out v1.14.0
+  #   it will NOT kill v1.14.0 controllers when rolling out v1.14.3
+  upgradeHelper:
+    enabled: false
+    # disableWebhooks can be used to turn off the webhook feature if required
+    disableWebhooks: true
+    # disableWorkerCommunicationChecks can be used to disable the check that all workers checking in with this master
+    # so that cfn-signal can be generated and upgrade process move forward
+    # (you don't want a single bad worker to cause an important upgrade to fail)
+    disableWorkerCommunicationChecks: true
+
+  # kiam - deploys kiam server and agent components
+  # kiam provides individual pod access policies and tokens for restricting pod access to AWS resources
+  # It will grant sts:assumeRole permission to the IAM role for controller nodes (only controller nodes where the kiam server is hosted need this, worker nodes do not)
+  # This is intended to be used in combination with .controller.iam.role.name. See #297 for more information.
+  kiam:
+    enabled: false
+    # image: quay.io/uswitch/kiam
+    # tag: v3.2
+    # sessionDuration: 30m
+    # server:
+    #  address: localhost:443
+    #  resources:
+    #    requests:
+    #      cpu: 50m
+    #    limits:
+    #      cpu: 200m
+    #      memory: 200Mi
+    # agent:
+    #  address: kiam-server:443
+    #  resources:
+    #    requests:
+    #      cpu: 10m
+    #      memory: 5Mi
+    #    limits:
+    #      cpu: 200m
+    #      memory: 100Mi

builtin/files/plugins/kiam/manifests/agent-daemonset.yaml

@@ -0,0 +1,103 @@
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  namespace: kube-system
+  name: kiam-agent
+spec:
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 100%
+    type: RollingUpdate
+  template:
+    metadata:
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "9620"
+      labels:
+        app: kiam
+        role: agent
+    spec:
+      priorityClassName: system-node-critical
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      - operator: Exists
+        effect: NoExecute
+      - operator: Exists
+        key: CriticalAddonsOnly
+      hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: node.kubernetes.io/role
+                operator: NotIn
+                values:
+                - master
+      volumes:
+        - name: ssl-certs
+          hostPath:
+            path: /usr/share/ca-certificates
+        - name: tls
+          secret:
+            secretName: kiam-agent-tls
+        - name: xtables
+          hostPath:
+            path: /run/xtables.lock
+      containers:
+        - name: kiam
+          securityContext:
+            capabilities:
+              add: ["NET_ADMIN"]
+          image: {{ .Values.image }}:{{ .Values.tag }}
+          command:
+            - {{ if checkVersion ">= 3.0" .Values.tag }}/kiam{{ else }}/agent{{ end }}
+          args:
+            {{if checkVersion ">= 3.0" .Values.tag -}}
+            - agent
+            - --whitelist-route-regexp=.*
+            - --gateway-timeout-creation=1s
+            {{ end -}}
+            - --iptables
+      {{- if .Config.Cluster.Kubernetes.Networking.AmazonVPC.Enabled }}
+            - --host-interface=!eni0
+      {{- else if eq .Config.Cluster.Kubernetes.Networking.SelfHosting.Type "canal" }}
+            - --host-interface=cali+
+      {{- else}}
+            - --host-interface=cni0
+      {{- end }}
+            - --json-log
+            - --port=8181
+            - --cert=/etc/kiam/tls/tls.crt
+            - --key=/etc/kiam/tls/tls.key
+            - --ca=/etc/kiam/tls/ca.crt
+            - --server-address={{ .Values.agent.address }}
+            - --prometheus-listen-addr=0.0.0.0:9620
+            - --prometheus-sync-interval=5s
+          env:
+            - name: HOST_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+          volumeMounts:
+            - mountPath: /etc/ssl/certs
+              name: ssl-certs
+            - mountPath: /etc/kiam/tls
+              name: tls
+            - mountPath: /var/run/xtables.lock
+              name: xtables
+          livenessProbe:
+            httpGet:
+              path: /ping
+              port: 8181
+            initialDelaySeconds: 3
+            periodSeconds: 3
+          resources:
+            requests:
+              cpu: {{ .Values.agent.resources.requests.cpu }}
+              memory: {{ .Values.agent.resources.requests.memory }}
+            limits:
+              cpu: {{ .Values.agent.resources.limits.cpu }}
+              memory: {{ .Values.agent.resources.limits.memory }}

builtin/files/plugins/kiam/manifests/agent-tls-secret.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: kiam-agent-tls
+  namespace: kube-system
+type: kubernetes.io/tls
+data:
+  tls.crt: {{ insertTemplateFile "credentials/kiam-agent.pem" . | b64enc }}
+  tls.key: {{ insertTemplateFile "credentials/kiam-agent-key.pem" . | b64enc }}
+  ca.crt: {{ insertTemplateFile "credentials/kiam-ca.pem" . | b64enc }}

builtin/files/plugins/kiam/manifests/server-cluster-role-binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: kiam-server
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kiam-server
+subjects:
+- kind: ServiceAccount
+  name: kiam-server
+  namespace: kube-system

builtin/files/plugins/kiam/manifests/server-cluster-role.yaml

@@ -0,0 +1,21 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: kiam-server
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  - pods
+  verbs:
+  - watch
+  - get
+  - list
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch

builtin/files/plugins/kiam/manifests/server-daemonset.yaml

@@ -0,0 +1,106 @@
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  namespace: kube-system
+  name: kiam-server
+spec:
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 100%
+    type: RollingUpdate
+  template:
+    metadata:
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "9620"
+      labels:
+        app: kiam
+        role: server
+    spec:
+      priorityClassName: system-node-critical
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      - operator: Exists
+        effect: NoExecute
+      - operator: Exists
+        key: CriticalAddonsOnly
+      serviceAccountName: kiam-server
+      nodeSelector:
+        node.kubernetes.io/role: "master"
+      volumes:
+        - name: ssl-certs
+          hostPath:
+            path: /usr/share/ca-certificates
+        - name: tls
+          secret:
+            secretName: kiam-server-tls
+      containers:
+        - name: kiam
+          image: {{ .Values.image }}:{{ .Values.tag }}
+          command:
+            - {{ if checkVersion ">= 3.0" .Values.tag }}/kiam{{ else }}/server{{ end }}
+          args:
+            {{ if checkVersion ">= 3.0" .Values.tag -}}
+            - server
+            {{ end -}}
+            - --json-log
+            - --bind=0.0.0.0:443
+            - --cert=/etc/kiam/tls/tls.crt
+            - --key=/etc/kiam/tls/tls.key
+            - --ca=/etc/kiam/tls/ca.crt
+            - --role-base-arn-autodetect
+            - --sync=1m
+            - --prometheus-listen-addr=0.0.0.0:9620
+            - --prometheus-sync-interval=5s
+            - --session-duration={{ .Values.sessionDuration }}
+          volumeMounts:
+            - mountPath: /etc/ssl/certs
+              name: ssl-certs
+            - mountPath: /etc/kiam/tls
+              name: tls
+          livenessProbe:
+            exec:
+              command:
+              {{ if checkVersion ">= 3.0" .Values.tag -}}
+              - /kiam
+              - health
+              - --gateway-timeout-creation=1s
+              {{ else -}}
+              - /health
+              - --server-address-refresh=2s
+              {{ end -}}
+              - --cert=/etc/kiam/tls/tls.crt
+              - --key=/etc/kiam/tls/tls.key
+              - --ca=/etc/kiam/tls/ca.crt
+              - --server-address={{ .Values.server.address }}
+              - --timeout=5s
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 10
+          readinessProbe:
+            exec:
+              command:
+              {{ if checkVersion ">= 3.0" .Values.tag -}}
+              - /kiam
+              - health
+              - --gateway-timeout-creation=1s
+              {{ else -}}
+              - /health
+              - --server-address-refresh=2s
+              {{ end -}}
+              - --cert=/etc/kiam/tls/tls.crt
+              - --key=/etc/kiam/tls/tls.key
+              - --ca=/etc/kiam/tls/ca.crt
+              - --server-address={{ .Values.server.address }}
+              - --timeout=5s
+            initialDelaySeconds: 3
+            periodSeconds: 10
+            timeoutSeconds: 10
+          resources:
+            requests:
+              cpu: {{ .Values.server.resources.requests.cpu }}
+              memory: {{ .Values.server.resources.requests.memory }}
+            limits:
+              cpu: {{ .Values.server.resources.limits.cpu }}
+              memory: {{ .Values.server.resources.limits.memory }}

builtin/files/plugins/kiam/manifests/server-tls-secret.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: kiam-server-tls
+  namespace: kube-system
+type: kubernetes.io/tls
+data:
+  tls.crt: {{ insertTemplateFile "credentials/kiam-server.pem" . | b64enc }}
+  tls.key: {{ insertTemplateFile "credentials/kiam-server-key.pem" . | b64enc }}
+  ca.crt: {{ insertTemplateFile "credentials/kiam-ca.pem" . | b64enc }}

builtin/files/plugins/kiam/manifests/service-account.yaml

@@ -0,0 +1,5 @@
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: kiam-server
+  namespace: kube-system