Merge branch 'aws:master' into helm_dir_update

README.md

@@ -410,6 +410,17 @@ Specifies the cluster name to tag allocated ENIs with. See the "Cluster Name tag
 
 ---
 
+#### `CLUSTER_ENDPOINT` (v1.12.1+)
+
+Type: String
+
+Default: `""`
+
+Specifies the cluster endpoint to use for connecting to the api-server without relying on kube-proxy. 
+This is an optional configuration parameter that can improve the initialization time of the AWS VPC CNI.
+
+---
+
 #### `ENABLE_POD_ENI` (v1.7.0+)
 
 Type: Boolean as a String

charts/cni-metrics-helper/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.11
+version: 0.1.12
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

charts/cni-metrics-helper/templates/deployment.yaml

@@ -19,6 +19,9 @@ spec:
 {{- range $key, $value := .Values.env }}
         - name: {{ $key }}
           value: {{ $value | quote }}
+{{- end }}
+{{- if .Values.resources }}
+        resources: {{ toYaml .Values.resources | nindent 10 }}
 {{- end }}
         name: cni-metrics-helper
         image: "{{- if .Values.image.override }}{{- .Values.image.override }}{{- else }}{{- .Values.image.account }}.dkr.ecr.{{- .Values.image.region }}.{{- .Values.image.domain }}/cni-metrics-helper:{{- .Values.image.tag }}{{- end}}"

cmd/aws-k8s-agent/main.go

@@ -29,13 +29,13 @@ func main() {
 }
 
 func _main() int {
-	//Do not add anything before initializing logger
+	// Do not add anything before initializing logger
 	log := logger.Get()
 
 	log.Infof("Starting L-IPAMD %s  ...", version.Version)
 	version.RegisterMetric()
 
-	//Check API Server Connectivity
+	// Check API Server Connectivity
 	if err := k8sapi.CheckAPIServerConnectivity(); err != nil {
 		log.Errorf("Failed to check API server connectivity: %s", err)
 		return 1

docs/troubleshooting.md

@@ -220,6 +220,21 @@ cni v1.10.x introduced 2 new env variables - ENABLE_IPv4 and ENABLE_IPv6. The ab
 kubectl apply -f https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/release-1.10/config/master/aws-k8s-cni.yaml
 ```
 
+## CNI Compatibility
+
+The [CNI image](../scripts/dockerfiles/Dockerfile.release) built for the `aws-node` manifest uses Amazon Linux 2 as the base image. Support for other Linux distributions (custom AMIs) is best-effort. Known issues with other Linux distributions are captured here:
+
+- **iptables** - iptables is installed by default in `aws-node` container images. Newer distributions of RHEL (RHEL 8.x+), Ubuntu (Ubuntu 20.x+), etc. have moved to using `nftables`. This leads to issues such as [this](https://github.com/aws/amazon-vpc-cni-k8s/issues/1847) when running IPAMD.
+
+  To resolve this issue on distributions that use `nftables`, there are currently two options:
+  1. Uninstall `nftables` and install `iptables-legacy` in base distribution
+  2. Build a custom CNI image based on `nftables`, such as:
+  ```
+  from $ACCOUNT.dkr.ecr.$REGION.amazonaws.com/amazon-k8s-cni:$IMAGE_TAG
+  run yum install iptables-nft -y
+  run cd /usr/sbin && rm iptables && ln -s xtables-nft-multi iptables
+  ```
+
 ## cni-metrics-helper
 
 See the [cni-metrics-helper README](../cmd/cni-metrics-helper/README.md).

pkg/k8sapi/k8sutils.go

@@ -23,11 +23,11 @@ import (
 var log = logger.Get()
 
 func InitializeRestMapper() (meta.RESTMapper, error) {
-	restCfg, err := ctrl.GetConfig()
-	restCfg.Burst = 200
+	restCfg, err := getRestConfig()
 	if err != nil {
 		return nil, err
 	}
+	restCfg.Burst = 200
 	mapper, err := apiutil.NewDynamicRESTMapper(restCfg)
 	if err != nil {
 		return nil, err
@@ -37,7 +37,7 @@ func InitializeRestMapper() (meta.RESTMapper, error) {
 
 // CreateKubeClient creates a k8s client
 func CreateKubeClient(mapper meta.RESTMapper) (client.Client, error) {
-	restCfg, err := ctrl.GetConfig()
+	restCfg, err := getRestConfig()
 	if err != nil {
 		return nil, err
 	}
@@ -56,12 +56,12 @@ func CreateKubeClient(mapper meta.RESTMapper) (client.Client, error) {
 
 // CreateKubeClient creates a k8s client
 func CreateCachedKubeClient(rawK8SClient client.Client, mapper meta.RESTMapper) (client.Client, error) {
-	restCfg, err := ctrl.GetConfig()
-	restCfg.Burst = 100
-
+	restCfg, err := getRestConfig()
 	if err != nil {
 		return nil, err
 	}
+	restCfg.Burst = 100
+
 	vpcCniScheme := runtime.NewScheme()
 	clientgoscheme.AddToScheme(vpcCniScheme)
 	eniconfigscheme.AddToScheme(vpcCniScheme)
@@ -89,7 +89,7 @@ func CreateCachedKubeClient(rawK8SClient client.Client, mapper meta.RESTMapper)
 }
 func GetKubeClientSet() (kubernetes.Interface, error) {
 	// creates the in-cluster config
-	config, err := rest.InClusterConfig()
+	config, err := getRestConfig()
 	if err != nil {
 		return nil, err
 	}
@@ -103,7 +103,7 @@ func GetKubeClientSet() (kubernetes.Interface, error) {
 }
 
 func CheckAPIServerConnectivity() error {
-	restCfg, err := ctrl.GetConfig()
+	restCfg, err := getRestConfig()
 	if err != nil {
 		return err
 	}
@@ -130,3 +130,14 @@ func CheckAPIServerConnectivity() error {
 		return true, nil
 	})
 }
+
+func getRestConfig() (*rest.Config, error) {
+	restCfg, err := ctrl.GetConfig()
+	if err != nil {
+		return nil, err
+	}
+	if endpoint, ok := os.LookupEnv("CLUSTER_ENDPOINT"); ok {
+		restCfg.Host = endpoint
+	}
+	return restCfg, nil
+}

scripts/lib/canary.sh

@@ -16,7 +16,7 @@ fi
 
 if [[ -z "${SKIP_MAKE_TEST_BINARIES}" ]]; then
   echo "making ginkgo test binaries"
-  (cd $SCRIPT_DIR/../test && make build-test-binaries)
+  (cd $SCRIPT_DIR/../ && make build-test-binaries)
 else
   echo "skipping making ginkgo test binaries"
 fi

scripts/update-cni-images.sh

@@ -14,20 +14,20 @@ AWS_K8S_CNI_MANIFEST="$SCRIPTS_DIR/../config/master/aws-k8s-cni.yaml"
 MANIFEST_IMG_VERSION=`grep "image:" $AWS_K8S_CNI_MANIFEST | cut -d ":" -f3 | cut -d "\"" -f1 | head -1`
 
 # Replace the images in aws-k8s-cni.yaml with the tester images when environment variables are set
-if [[ -z $AWS_K8S_CNI ]]; then
-    echo "Applying latest CNI image from aws-k8s-cni manifest"
+if [[ -z $AMAZON_K8S_CNI ]]; then
+    echo "Using latest CNI image from aws-k8s-cni manifest"
 else
     echo "Replacing CNI image in aws-k8s-cni manifest with $AMAZON_K8S_CNI"
     sed -i'.bak' "s,602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:$MANIFEST_IMG_VERSION,$AMAZON_K8S_CNI," "$AWS_K8S_CNI_MANIFEST"
 fi
-if [[ -z $AWS_K8S_CNI_INIT ]]; then
-    echo "Applying latest CNI init image from aws-k8s-cni manifest"
+if [[ -z $AMAZON_K8S_CNI_INIT ]]; then
+    echo "Using latest CNI init image from aws-k8s-cni manifest"
 else
-    echo "Replacing CNI image in aws-k8s-cni manifest with $AMAZON_K8S_CNI"
+    echo "Replacing CNI init image in aws-k8s-cni manifest with $AMAZON_K8S_CNI_INIT"
     sed -i'.bak' "s,602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni-init:$MANIFEST_IMG_VERSION,$AMAZON_K8S_CNI_INIT," "$AWS_K8S_CNI_MANIFEST"
 fi
 
-echo "Applying aws-k8s-cni.yaml manifest to aws-node daemonset"
+echo "Applying amazon-vpc-cni-k8s/config/master/aws-k8s-cni.yaml manifest"
 kubectl apply -f $AWS_K8S_CNI_MANIFEST
 
 check_ds_rollout "aws-node" "kube-system" "4m"