From 2f3aa84ff39bb9dec6d05a278ca14a75607676e7 Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Tue, 13 Apr 2021 02:26:01 +0300 Subject: [PATCH 1/4] Add prebootstrap command to set containerd as the default --- manager/generate_eks.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/manager/generate_eks.py b/manager/generate_eks.py index d09872f70f..47c720f18b 100644 --- a/manager/generate_eks.py +++ b/manager/generate_eks.py @@ -42,6 +42,13 @@ def default_nodegroup(cluster_config): "evictionHard": {"memory.available": "200Mi", "nodefs.available": "5%"}, "registryPullQPS": 10, }, + "preBootstrapCommands": [ + "export CRICTL_VERSION=v1.21.0", + "wget https://github.com/kubernetes-sigs/cri-tools/releases/download/$CRICTL_VERSION/crictl-$CRICTL_VERSION-linux-amd64.tar.gz", + "sudo tar zxvf crictl-$CRICTL_VERSION-linux-amd64.tar.gz -C /usr/local/bin", + "rm -f crictl-$CRICTL_VERSION-linux-amd64.tar.gz", + "crictl config --set runtime-endpoint=unix:///run/containerd/containerd.sock --set image-endpoint=unix:///run/containerd/containerd.sock", + ], } From e431a6e31bc58ea9b6ac04766d5bb1d177900dad Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Tue, 13 Apr 2021 02:36:13 +0300 Subject: [PATCH 2/4] Also set pull-image-on-create flag --- manager/generate_eks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manager/generate_eks.py b/manager/generate_eks.py index 47c720f18b..b1b90015a1 100644 --- a/manager/generate_eks.py +++ b/manager/generate_eks.py @@ -47,7 +47,7 @@ def default_nodegroup(cluster_config): "wget https://github.com/kubernetes-sigs/cri-tools/releases/download/$CRICTL_VERSION/crictl-$CRICTL_VERSION-linux-amd64.tar.gz", "sudo tar zxvf crictl-$CRICTL_VERSION-linux-amd64.tar.gz -C /usr/local/bin", "rm -f crictl-$CRICTL_VERSION-linux-amd64.tar.gz", - "crictl config --set runtime-endpoint=unix:///run/containerd/containerd.sock --set image-endpoint=unix:///run/containerd/containerd.sock", + "crictl config --set runtime-endpoint=unix:///run/containerd/containerd.sock --set image-endpoint=unix:///run/containerd/containerd.sock --set pull-image-on-create=true", ], } From 61190410dc5f30339c5ce3969e2952dbfc26dcd6 Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Tue, 13 Apr 2021 19:51:16 +0300 Subject: [PATCH 3/4] Changes to EKS for CRI --- manager/generate_eks.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/manager/generate_eks.py b/manager/generate_eks.py index b1b90015a1..bc3f130fa5 100644 --- a/manager/generate_eks.py +++ b/manager/generate_eks.py @@ -22,6 +22,7 @@ # kubelet config schema: # https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/kubelet/config/v1beta1/types.go def default_nodegroup(cluster_config): + crictl_version = "v1.21.0" return { "iam": { "withAddonPolicies": {"autoScaler": True}, @@ -43,10 +44,8 @@ def default_nodegroup(cluster_config): "registryPullQPS": 10, }, "preBootstrapCommands": [ - "export CRICTL_VERSION=v1.21.0", - "wget https://github.com/kubernetes-sigs/cri-tools/releases/download/$CRICTL_VERSION/crictl-$CRICTL_VERSION-linux-amd64.tar.gz", - "sudo tar zxvf crictl-$CRICTL_VERSION-linux-amd64.tar.gz -C /usr/local/bin", - "rm -f crictl-$CRICTL_VERSION-linux-amd64.tar.gz", + "yum install containerd -y", + f"wget https://github.com/kubernetes-sigs/cri-tools/releases/download/{crictl_version}/crictl-{crictl_version}-linux-amd64.tar.gz && sudo tar zxvf crictl-{crictl_version}-linux-amd64.tar.gz -C /usr/local/bin && rm -f crictl-{crictl_version}-linux-amd64.tar.gz", "crictl config --set runtime-endpoint=unix:///run/containerd/containerd.sock --set image-endpoint=unix:///run/containerd/containerd.sock --set pull-image-on-create=true", ], } From b74fd9307e6554265d2fc841d07718bc9f9277e7 Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Wed, 14 Apr 2021 21:18:14 +0300 Subject: [PATCH 4/4] Add research files/code-blocks --- 10-eksclt.al2.conf | 24 +++++++ Makefile | 2 +- containerd.conf | 133 ++++++++++++++++++++++++++++++++++++++ images/manager/Dockerfile | 2 +- manager/generate_eks.py | 6 +- 5 files changed, 161 insertions(+), 6 deletions(-) create mode 100644 10-eksclt.al2.conf create mode 100644 containerd.conf diff --git a/10-eksclt.al2.conf b/10-eksclt.al2.conf new file mode 100644 index 0000000000..6d31998d66 --- /dev/null +++ b/10-eksclt.al2.conf @@ -0,0 +1,24 @@ +# eksctl-specific systemd drop-in unit for kubelet, for Amazon Linux 2 (AL2) + +[Service] +# Local metadata parameters: REGION, AWS_DEFAULT_REGION +EnvironmentFile=/etc/eksctl/metadata.env +# Global and static parameters: CLUSTER_DNS, NODE_LABELS, NODE_TAINTS +EnvironmentFile=/etc/eksctl/kubelet.env +# Local non-static parameters: NODE_IP, INSTANCE_ID +EnvironmentFile=/etc/eksctl/kubelet.local.env + +ExecStart= +ExecStart=/usr/bin/kubelet \ + --node-ip=${NODE_IP} \ + --node-labels=${NODE_LABELS},alpha.eksctl.io/instance-id=${INSTANCE_ID} \ + --max-pods=${MAX_PODS} \ + --register-node=true --register-with-taints=${NODE_TAINTS} \ + --cloud-provider=aws \ + --container-runtime=docker \ + --network-plugin=cni \ + --cni-bin-dir=/opt/cni/bin \ + --cni-conf-dir=/etc/cni/net.d \ + --pod-infra-container-image=${AWS_EKS_ECR_ACCOUNT}.dkr.ecr.${AWS_DEFAULT_REGION}.${AWS_SERVICES_DOMAIN}/eks/pause:3.3-eksbuild.1 \ + --kubeconfig=/etc/eksctl/kubeconfig.yaml \ + --config=/etc/eksctl/kubelet.yaml \ No newline at end of file diff --git a/Makefile b/Makefile index 8eb70fa8f4..35416e3c99 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ kubectl: @eval $$(python3 ./manager/cluster_config_env.py ./dev/config/cluster.yaml) && eksctl utils write-kubeconfig --cluster="$$CORTEX_CLUSTER_NAME" --region="$$CORTEX_REGION" | (grep -v "saved kubeconfig as" | grep -v "using region" | grep -v "eksctl version" || true) cluster-up: - @$(MAKE) images-all + # @$(MAKE) images-all @$(MAKE) cli @kill $(shell pgrep -f rerun) >/dev/null 2>&1 || true @eval $$(python3 ./manager/cluster_config_env.py ./dev/config/cluster.yaml) && ./bin/cortex cluster up ./dev/config/cluster.yaml --configure-env="$$CORTEX_CLUSTER_NAME" diff --git a/containerd.conf b/containerd.conf new file mode 100644 index 0000000000..c16413bf3c --- /dev/null +++ b/containerd.conf @@ -0,0 +1,133 @@ +version = 2 +root = "/var/lib/containerd" +state = "/run/containerd" +plugin_dir = "" +disabled_plugins = [] +required_plugins = [] +oom_score = 0 + +[grpc] + address = "/run/containerd/containerd.sock" + tcp_address = "" + tcp_tls_cert = "" + tcp_tls_key = "" + uid = 0 + gid = 0 + max_recv_message_size = 16777216 + max_send_message_size = 16777216 + +[ttrpc] + address = "" + uid = 0 + gid = 0 + +[debug] + address = "" + uid = 0 + gid = 0 + level = "" + +[metrics] + address = "" + grpc_histogram = false + +[cgroup] + path = "" + +[timeouts] + "io.containerd.timeout.shim.cleanup" = "5s" + "io.containerd.timeout.shim.load" = "5s" + "io.containerd.timeout.shim.shutdown" = "3s" + "io.containerd.timeout.task.state" = "2s" + +[plugins] + [plugins."io.containerd.gc.v1.scheduler"] + pause_threshold = 0.02 + deletion_threshold = 0 + mutation_threshold = 100 + schedule_delay = "0s" + startup_delay = "100ms" + [plugins."io.containerd.grpc.v1.cri"] + disable_tcp_service = true + stream_server_address = "127.0.0.1" + stream_server_port = "0" + stream_idle_timeout = "4h0m0s" + enable_selinux = false + selinux_category_range = 1024 + sandbox_image = "k8s.gcr.io/pause:3.2" + stats_collect_period = 10 + systemd_cgroup = false + enable_tls_streaming = false + max_container_log_line_size = 16384 + disable_cgroup = false + disable_apparmor = false + restrict_oom_score_adj = false + max_concurrent_downloads = 3 + disable_proc_mount = false + unset_seccomp_profile = "" + tolerate_missing_hugetlb_controller = true + disable_hugetlb_controller = true + ignore_image_defined_volumes = false + [plugins."io.containerd.grpc.v1.cri".containerd] + snapshotter = "overlayfs" + default_runtime_name = "runc" + no_pivot = false + disable_snapshot_annotations = true + discard_unpacked_layers = false + [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime] + runtime_type = "" + runtime_engine = "" + runtime_root = "" + privileged_without_host_devices = false + base_runtime_spec = "" + [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime] + runtime_type = "" + runtime_engine = "" + runtime_root = "" + privileged_without_host_devices = false + base_runtime_spec = "" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + runtime_engine = "" + runtime_root = "" + privileged_without_host_devices = false + base_runtime_spec = "" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + [plugins."io.containerd.grpc.v1.cri".cni] + bin_dir = "/opt/cni/bin" + conf_dir = "/etc/cni/net.d" + max_conf_num = 1 + conf_template = "" + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + [plugins."io.containerd.grpc.v1.cri".image_decryption] + key_model = "" + [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming] + tls_cert_file = "" + tls_key_file = "" + [plugins."io.containerd.internal.v1.opt"] + path = "/opt/containerd" + [plugins."io.containerd.internal.v1.restart"] + interval = "10s" + [plugins."io.containerd.metadata.v1.bolt"] + content_sharing_policy = "shared" + [plugins."io.containerd.monitor.v1.cgroups"] + no_prometheus = false + [plugins."io.containerd.runtime.v1.linux"] + shim = "containerd-shim" + runtime = "runc" + runtime_root = "" + no_shim = false + shim_debug = false + [plugins."io.containerd.runtime.v2.task"] + platforms = ["linux/amd64"] + [plugins."io.containerd.service.v1.diff-service"] + default = ["walking"] + [plugins."io.containerd.snapshotter.v1.devmapper"] + root_path = "" + pool_name = "" + base_image_size = "" + async_remove = false \ No newline at end of file diff --git a/images/manager/Dockerfile b/images/manager/Dockerfile index 905cba0731..51fd0becf2 100644 --- a/images/manager/Dockerfile +++ b/images/manager/Dockerfile @@ -15,7 +15,7 @@ RUN pip install --upgrade pip && \ RUN apk add --no-cache bash curl gettext jq openssl -RUN curl --location "https://github.com/weaveworks/eksctl/releases/download/0.40.0/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp && \ +RUN curl --location "https://github.com/weaveworks/eksctl/releases/download/0.44.0/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp && \ mv /tmp/eksctl /usr/local/bin RUN curl -o aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.18.9/2020-11-02/bin/linux/amd64/aws-iam-authenticator && \ diff --git a/manager/generate_eks.py b/manager/generate_eks.py index bc3f130fa5..9330713a8e 100644 --- a/manager/generate_eks.py +++ b/manager/generate_eks.py @@ -22,7 +22,6 @@ # kubelet config schema: # https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/kubelet/config/v1beta1/types.go def default_nodegroup(cluster_config): - crictl_version = "v1.21.0" return { "iam": { "withAddonPolicies": {"autoScaler": True}, @@ -45,12 +44,11 @@ def default_nodegroup(cluster_config): }, "preBootstrapCommands": [ "yum install containerd -y", - f"wget https://github.com/kubernetes-sigs/cri-tools/releases/download/{crictl_version}/crictl-{crictl_version}-linux-amd64.tar.gz && sudo tar zxvf crictl-{crictl_version}-linux-amd64.tar.gz -C /usr/local/bin && rm -f crictl-{crictl_version}-linux-amd64.tar.gz", - "crictl config --set runtime-endpoint=unix:///run/containerd/containerd.sock --set image-endpoint=unix:///run/containerd/containerd.sock --set pull-image-on-create=true", + "truncate -s-1 /etc/systemd/system/kubelet.service.d/10-eksclt.al2.conf", + "echo -n ' --container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock' >> /etc/systemd/system/kubelet.service.d/10-eksclt.al2.conf", ], } - def merge_override(a, b): "merges b into a" for key in b: