From af5c83a0091f360fab07f498d46a7734e9842764 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Wed, 15 Nov 2023 11:36:15 +0700 Subject: [PATCH] Add confing for autoscaler in EKS --- configs/aws-asg-policy.json | 29 +++ configs/cluster-autoscaler-autodiscover.yaml | 179 +++++++++++++++++++ docs/aws-eks.md | 13 +- 3 files changed, 220 insertions(+), 1 deletion(-) create mode 100644 configs/aws-asg-policy.json create mode 100644 configs/cluster-autoscaler-autodiscover.yaml diff --git a/configs/aws-asg-policy.json b/configs/aws-asg-policy.json new file mode 100644 index 0000000..d29a27d --- /dev/null +++ b/configs/aws-asg-policy.json @@ -0,0 +1,29 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeScalingActivities", + "autoscaling:DescribeTags", + "ec2:DescribeInstanceTypes", + "ec2:DescribeLaunchTemplateVersions" + ], + "Resource": ["*"] + }, + { + "Effect": "Allow", + "Action": [ + "autoscaling:SetDesiredCapacity", + "autoscaling:TerminateInstanceInAutoScalingGroup", + "ec2:DescribeImages", + "ec2:GetInstanceTypesFromInstanceRequirements", + "eks:DescribeNodegroup" + ], + "Resource": ["*"] + } + ] + } \ No newline at end of file diff --git a/configs/cluster-autoscaler-autodiscover.yaml b/configs/cluster-autoscaler-autodiscover.yaml new file mode 100644 index 0000000..1e2fae8 --- /dev/null +++ b/configs/cluster-autoscaler-autodiscover.yaml @@ -0,0 +1,179 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler + name: cluster-autoscaler + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cluster-autoscaler + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +rules: +- apiGroups: [""] + resources: ["events", "endpoints"] + verbs: ["create", "patch"] +- apiGroups: [""] + resources: ["pods/eviction"] + verbs: ["create"] +- apiGroups: [""] + resources: ["pods/status"] + verbs: ["update"] +- apiGroups: [""] + resources: ["endpoints"] + resourceNames: ["cluster-autoscaler"] + verbs: ["get", "update"] +- apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update"] +- apiGroups: [""] + resources: + - "namespaces" + - "pods" + - "services" + - "replicationcontrollers" + - "persistentvolumeclaims" + - "persistentvolumes" + verbs: ["watch", "list", "get"] +- apiGroups: ["extensions"] + resources: ["replicasets", "daemonsets"] + verbs: ["watch", "list", "get"] +- apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["watch", "list"] +- apiGroups: ["apps"] + resources: ["statefulsets", "replicasets", "daemonsets"] + verbs: ["watch", "list", "get"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "csinodes", "csidrivers", "csistoragecapacities"] + verbs: ["watch", "list", "get"] +- apiGroups: ["batch", "extensions"] + resources: ["jobs"] + verbs: ["get", "list", "watch", "patch"] +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create"] +- apiGroups: ["coordination.k8s.io"] + resourceNames: ["cluster-autoscaler"] + resources: ["leases"] + verbs: ["get", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["create", "list", "watch"] +- apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"] + verbs: ["delete", "get", "update", "watch"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cluster-autoscaler + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-autoscaler +subjects: +- kind: ServiceAccount + name: cluster-autoscaler + namespace: kube-system + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cluster-autoscaler +subjects: +- kind: ServiceAccount + name: cluster-autoscaler + namespace: kube-system + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + app: cluster-autoscaler +spec: + replicas: 1 + selector: + matchLabels: + app: cluster-autoscaler + template: + metadata: + labels: + app: cluster-autoscaler + annotations: + prometheus.io/scrape: 'true' + prometheus.io/port: '8085' + spec: + priorityClassName: system-cluster-critical + securityContext: + runAsNonRoot: true + runAsUser: 65534 + fsGroup: 65534 + seccompProfile: + type: RuntimeDefault + serviceAccountName: cluster-autoscaler + containers: + - image: registry.k8s.io/autoscaling/cluster-autoscaler:v1.26.2 + name: cluster-autoscaler + resources: + limits: + cpu: 100m + memory: 600Mi + requests: + cpu: 100m + memory: 600Mi + command: + - ./cluster-autoscaler + - --v=4 + - --stderrthreshold=info + - --cloud-provider=aws + - --skip-nodes-with-local-storage=false + - --expander=least-waste + - --node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/sandbox + volumeMounts: + - name: ssl-certs + mountPath: /etc/ssl/certs/ca-certificates.crt # /etc/ssl/certs/ca-bundle.crt for Amazon Linux Worker Nodes + readOnly: true + imagePullPolicy: "Always" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + volumes: + - name: ssl-certs + hostPath: + path: "/etc/ssl/certs/ca-bundle.crt" diff --git a/docs/aws-eks.md b/docs/aws-eks.md index a67f67b..801191f 100644 --- a/docs/aws-eks.md +++ b/docs/aws-eks.md @@ -30,7 +30,8 @@ An example command below. See the [eksctl docs](https://eksctl.io/usage/creating --nodegroup-name=hub-node \ --node-type=t2.medium \ --nodes=1 --nodes-min=1 --nodes-max=5 \ - --version 1.27 + --version 1.27 \ + --asg-access ``` TODO: Add autoscaling config @@ -72,6 +73,16 @@ but below are the relevant bits. Note that `eksctl` "should" set up an OIDC prov --- +## Install Node Autoscaler + +To run the autoscaler on EKS, we need to create a policy and install Auto-Discovery Setup, which is the preferred method to configure [Cluster Autoscaler in EKS](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#auto-discovery-setup). + + ```sh + aws iam create-policy --policy-name ${CLUSTER_NAME}-asg --policy-document file://configs/aws-asg-policy.json + # Update the cluster name ins case you changed in configs/cluster-autoscaler-autodiscover.yaml file + kubectl apply -f configs/cluster-autoscaler-autodiscover.yaml + ``` + ## Install the EBS CSI Addon for dynamic EBS provisioning Best to walk through the [AWS docs](https://docs.aws.amazon.com/eks/latest/userguide/ebs-csi.html) about this