Update kops cluster config used in weekly tests (#1862)

* Change to kops cluster creation scripts

* Add logging for retry attempt

* Switch kops cluster to use docker container runtime

Co-authored-by: Jayanth Varavani <1111446+jayanthvn@users.noreply.github.com>

.github/workflows/weekly-cron-test.yml

@@ -2,7 +2,7 @@ name: Weekly CNI tests
 
 on:
  schedule:
- - cron: "0 16 * * 1" # every Monday
+ - cron: "0 16 * * 2" # every Tuesday
 
 permissions:
  contents: read

scripts/lib/cluster.sh

@@ -66,14 +66,18 @@ function up-test-cluster() {
 }
 
 function up-kops-cluster {
- aws s3api create-bucket --bucket kops-cni-test-eks --region $AWS_DEFAULT_REGION --create-bucket-configuration LocationConstraint=$AWS_DEFAULT_REGION
- curl -LO https://github.com/kubernetes/kops/releases/download/$(curl -s https://api.github.com/repos/kubernetes/kops/releases/latest | grep tag_name | cut -d '"' -f 4)/kops-linux-amd64
+ KOPS_S3_BUCKET=kops-cni-test-eks-$AWS_ACCOUNT_ID
+ echo "Using $KOPS_S3_BUCKET as kops state store"
+ aws s3api create-bucket --bucket $KOPS_S3_BUCKET --region $AWS_DEFAULT_REGION --create-bucket-configuration LocationConstraint=$AWS_DEFAULT_REGION
+ kops_version=$(curl -s https://api.github.com/repos/kubernetes/kops/releases/latest | grep tag_name | cut -d '"' -f 4)
+ echo "Using kops version $kops_version"
+ curl -LO https://github.com/kubernetes/kops/releases/download/$kops_version/kops-linux-amd64
  chmod +x kops-linux-amd64
  mkdir -p ~/kops_bin
  KOPS_BIN=~/kops_bin/kops
  mv kops-linux-amd64 $KOPS_BIN
  CLUSTER_NAME=kops-cni-test-cluster-${TEST_ID}.k8s.local
- export KOPS_STATE_STORE=s3://kops-cni-test-eks
+ export KOPS_STATE_STORE=s3://${KOPS_S3_BUCKET}
 
  SSH_KEYS=~/.ssh/devopsinuse
  if [ ! -f "$SSH_KEYS" ]
@@ -87,6 +91,7 @@ function up-kops-cluster {
  $KOPS_BIN create cluster \
  --zones ${AWS_DEFAULT_REGION}a,${AWS_DEFAULT_REGION}b \
  --networking amazon-vpc-routed-eni \
+ --container-runtime docker \
  --node-count 2 \
  --ssh-public-key=~/.ssh/devopsinuse.pub \
  --kubernetes-version ${K8S_VERSION} \
@@ -95,10 +100,13 @@ function up-kops-cluster {
  sleep 100
  $KOPS_BIN export kubeconfig --admin
  sleep 10
- while [[ ! $($KOPS_BIN validate cluster | grep "is ready") ]]
+ MAX_RETRIES=15
+ RETRY_ATTEMPT=0
+ while [[ ! $($KOPS_BIN validate cluster | grep "is ready") && $RETRY_ATTEMPT -lt $MAX_RETRIES ]]
  do
- sleep 5
- echo "Waiting for cluster validation"
+ sleep 60
+ let RETRY_ATTEMPT=RETRY_ATTEMPT+1
+ echo "In attempt# $RETRY_ATTEMPT, waiting for cluster validation"
  done
  kubectl apply -f https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/${MANIFEST_CNI_VERSION}/config/master/cni-metrics-helper.yaml
 }

scripts/lib/integration.sh

@@ -11,12 +11,10 @@ function run_kops_conformance() {
  done
  echo "Updated!"
 
- GOPATH=$(go env GOPATH)
- go install github.com/onsi/ginkgo/ginkgo
- wget -qO- https://dl.k8s.io/v$K8S_VERSION/kubernetes-test.tar.gz | tar -zxvf - --strip-components=4 -C /tmp kubernetes/platforms/linux/amd64/e2e.test
+ wget -qO- https://dl.k8s.io/v$K8S_VERSION/kubernetes-test-linux-amd64.tar.gz | tar -zxvf - --strip-components=3 -C /tmp kubernetes/test/bin/e2e.test
 
- $GOPATH/bin/ginkgo -p --focus="Conformance" --failFast --flakeAttempts 2 \
- --skip="(should support remote command execution over websockets)|(should support retrieving logs from the container over websockets)|(Basic StatefulSet functionality [StatefulSetBasic])|\[Slow\]|\[Serial\]" /tmp/e2e.test -- --kubeconfig=$KUBECONFIG
+ /tmp/e2e.test --ginkgo.focus="Conformance" --kubeconfig=$KUBECONFIG --ginkgo.failFast --ginkgo.flakeAttempts 2 \
+ --ginkgo.skip="(should support remote command execution over websockets)|(should support retrieving logs from the container over websockets)|(Basic StatefulSet functionality [StatefulSetBasic])|\[Slow\]|\[Serial\]"
 
  /tmp/e2e.test --ginkgo.focus="\[Serial\].*Conformance" --kubeconfig=$KUBECONFIG --ginkgo.failFast --ginkgo.flakeAttempts 2 \
  --ginkgo.skip="(should support remote command execution over websockets)|(should support retrieving logs from the container over websockets)|\[Slow\]"

scripts/run-integration-tests.sh

@@ -127,10 +127,11 @@ aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --usernam
 ensure_ecr_repo "$AWS_ACCOUNT_ID" "$AWS_ECR_REPO_NAME"
 ensure_ecr_repo "$AWS_ACCOUNT_ID" "$AWS_INIT_ECR_REPO_NAME"
 
-# Check to see if the image already exists in the Docker repository, and if
+# Check to see if the image already exists in the ECR repository, and if
 # not, check out the CNI source code for that image tag, build the CNI
 # image and push it to the Docker repository
-if [[ $(docker images -q "$IMAGE_NAME:$TEST_IMAGE_VERSION" 2> /dev/null) ]]; then
+ecr_image_query_result=$(aws ecr batch-get-image --repository-name=amazon-k8s-cni --image-ids imageTag=$TEST_IMAGE_VERSION --query 'images[].imageId.imageTag' --region us-west-2)
+if [[ $ecr_image_query_result != "[]" ]]; then
  echo "CNI image $IMAGE_NAME:$TEST_IMAGE_VERSION already exists in repository. Skipping image build..."
  DOCKER_BUILD_DURATION=0
 else

scripts/test/config/perf-cluster.yml

@@ -25,7 +25,7 @@ managedNodeGroups:
 
  - name: cni-test-multi-node-mng
  instanceType: m5.xlarge
- desiredCapacity: 1
+ desiredCapacity: 99
  minSize: 1
  maxSize: 100
  iam: