Skip to content

Commit

Permalink
Update Jenkins Scripts for Testbed Migration (#6751)
Browse files Browse the repository at this point in the history 
* Upgraded Windows images to restore CI on the latest kubernetes cluster
* Updated external node e2e test and jenkins script for testbed migration.

Signed-off-by: Shuyang Xin <gavinx@vmware.com>
  • Loading branch information
@XinShuYang
XinShuYang authored Oct 28, 2024
1 parent 71d57f1 commit 47ce51e
Show file tree
Hide file tree
Showing 3 changed files with 19 additions and 26 deletions.
15 changes: 10 additions & 5 deletions ci/jenkins/test-vm.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,10 @@ Run K8s e2e community tests (Conformance & Network Policy) or Antrea e2e tests o
# VM configuration
declare -A LINUX_HOSTS_TO_IP
declare -A WINDOWS_HOSTS_TO_IP
declare -a LIN_HOSTNAMES=("vmbmtest0-1" "vmbmtest0-redhat-0")
declare -a WIN_HOSTNAMES=("vmbmtest0-win-0")
declare -A LINUX_HOSTS_TO_USERNAME=(["vmbmtest0-1"]="ubuntu" ["vmbmtest0-redhat-0"]="root")
declare -A WINDOWS_HOSTS_TO_USERNAME=(["vmbmtest0-win-0"]="Administrator")
declare -a LIN_HOSTNAMES=("vmbm0-1" "vmbm0-redhat-0")
declare -a WIN_HOSTNAMES=("vmbm0-win-0")
declare -A LINUX_HOSTS_TO_USERNAME=(["vmbm0-1"]="ubuntu" ["vmbm0-redhat-0"]="root")
declare -A WINDOWS_HOSTS_TO_USERNAME=(["vmbm0-win-0"]="Administrator")

# To run kubectl cmds
export KUBECONFIG=${KUBECONFIG_PATH}
Expand Down Expand Up @@ -143,10 +143,15 @@ function apply_antrea {
fi
TEMP_ANTREA_TAR="antrea-image.tar"
docker save antrea/antrea-agent-ubuntu:latest antrea/antrea-controller-ubuntu:latest -o $TEMP_ANTREA_TAR
ctr -n k8s.io image import $TEMP_ANTREA_TAR
kubectl get nodes --selector=kubernetes.io/os=linux --no-headers=true -o custom-columns=IP:.status.addresses[0].address | while read -r IP; do
rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" $TEMP_ANTREA_TAR jenkins@${IP}:${WORKDIR}/$TEMP_ANTREA_TAR
ssh -o StrictHostKeyChecking=no -n jenkins@${IP} "crictl rmi --prune; crictl ps --state Exited; ctr -n=k8s.io images import ${WORKDIR}/$TEMP_ANTREA_TAR" || true
done
rm $TEMP_ANTREA_TAR
echo "====== Applying Antrea yaml ======"
./hack/generate-manifest.sh --feature-gates ExternalNode=true,SupportBundleCollection=true --extra-helm-values "controller.apiNodePort=32767" > ${WORKDIR}/antrea.yml
IP=$(kubectl get nodes --selector=node-role.kubernetes.io/control-plane= --no-headers=true -o custom-columns=IP:.status.addresses[0].address)
rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" ${WORKDIR}/*.yml jenkins@${IP}:${WORKDIR}/
kubectl apply -f ${WORKDIR}/antrea.yml
}

Expand Down
28 changes: 8 additions & 20 deletions ci/jenkins/test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -429,12 +429,10 @@ function deliver_antrea_linux {
DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-linux-all.sh --pull
docker save -o antrea-ubuntu.tar antrea/antrea-agent-ubuntu:latest antrea/antrea-controller-ubuntu:latest
echo "===== Pull necessary images on Control-Plane node ====="
harbor_images=("agnhost:2.13" "nginx:1.15-alpine")
antrea_images=("e2eteam/agnhost:2.13" "docker.io/library/nginx:1.15-alpine")
common_images=("registry.k8s.io/e2e-test-images/agnhost:2.29")
k8s_images=("registry.k8s.io/e2e-test-images/agnhost:2.45" "registry.k8s.io/e2e-test-images/jessie-dnsutils:1.5" "registry.k8s.io/e2e-test-images/nginx:1.14-2")
conformance_images=("k8sprow.azurecr.io/kubernetes-e2e-test-images/agnhost:2.45" "k8sprow.azurecr.io/kubernetes-e2e-test-images/jessie-dnsutils:1.5" "k8sprow.azurecr.io/kubernetes-e2e-test-images/nginx:1.14-2")
e2e_images=("toolbox:1.3-0" "nginx:1.21.6-alpine")
common_images=("registry.k8s.io/e2e-test-images/agnhost:2.40")
k8s_images=("registry.k8s.io/e2e-test-images/agnhost:2.52" "registry.k8s.io/e2e-test-images/jessie-dnsutils:1.5" "registry.k8s.io/e2e-test-images/nginx:1.14-2")
conformance_images=("k8sprow.azurecr.io/kubernetes-e2e-test-images/agnhost:2.52" "k8sprow.azurecr.io/kubernetes-e2e-test-images/jessie-dnsutils:1.5" "k8sprow.azurecr.io/kubernetes-e2e-test-images/nginx:1.14-2")
e2e_images=("toolbox:1.4-0" "nginx:1.21.6-alpine")

echo "===== Deliver Antrea YAML to Controller nodes ====="
IP=$(kubectl get nodes -o wide --no-headers=true | awk -v role="$CONTROL_PLANE_NODE_ROLE" '$3 ~ role {print $6}')
Expand All @@ -456,9 +454,6 @@ function deliver_antrea_linux {
rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" antrea-ubuntu.tar jenkins@${IP}:${WORKDIR}/antrea-ubuntu.tar
ssh -o StrictHostKeyChecking=no -n jenkins@${IP} "${CLEAN_STALE_IMAGES_CONTAINERD}; ${PRINT_CONTAINERD_STATUS}; ctr -n=k8s.io images import ${WORKDIR}/antrea-ubuntu.tar" || true

for i in "${!harbor_images[@]}"; do
ssh -o StrictHostKeyChecking=no -n jenkins@${IP} "ctr -n=k8s.io images pull --user ${DOCKER_USERNAME}:${DOCKER_PASSWORD} ${DOCKER_REGISTRY}/antrea/${harbor_images[i]} && ctr -n=k8s.io images tag ${DOCKER_REGISTRY}/antrea/${harbor_images[i]} ${antrea_images[i]}" || true
done
# Pull necessary images in advance to avoid transient error
for image in "${common_images[@]}"; do
ssh -o StrictHostKeyChecking=no -n jenkins@${IP} "ctr -n=k8s.io images pull --user ${DOCKER_USERNAME}:${DOCKER_PASSWORD} ${image}" || true
Expand Down Expand Up @@ -488,22 +483,15 @@ function deliver_antrea_windows {
sed -i 's/if (!(Test-Path $AntreaAgentConfigPath))/if ($true)/' hack/windows/Helper.psm1
kubectl get nodes -o wide --no-headers=true | awk -v role="$CONTROL_PLANE_NODE_ROLE" '$3 !~ role && $1 ~ /win/ {print $1}' | while read WORKER_NAME; do
revert_snapshot_windows ${WORKER_NAME}
# Some tests need us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.13 image but it is not for windows/amd64 10.0.17763
# Use e2eteam/agnhost:2.13 instead
harbor_images=("agnhost:2.13" "agnhost:2.13" "agnhost:2.29" "e2eteam-jessie-dnsutils:1.0" "e2eteam-pause:3.2")
antrea_images=("e2eteam/agnhost:2.13" "us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.13" "registry.k8s.io/e2e-test-images/agnhost:2.29" "e2eteam/jessie-dnsutils:1.0" "e2eteam/pause:3.2")
k8s_images=("registry.k8s.io/e2e-test-images/agnhost:2.45" "registry.k8s.io/e2e-test-images/jessie-dnsutils:1.5" "registry.k8s.io/e2e-test-images/nginx:1.14-2" "registry.k8s.io/pause:3.8")
conformance_images=("k8sprow.azurecr.io/kubernetes-e2e-test-images/agnhost:2.45" "k8sprow.azurecr.io/kubernetes-e2e-test-images/jessie-dnsutils:1.5" "k8sprow.azurecr.io/kubernetes-e2e-test-images/nginx:1.14-2" "k8sprow.azurecr.io/kubernetes-e2e-test-images/pause:3.8")
e2e_images=("toolbox:1.3-0")
k8s_images=("registry.k8s.io/e2e-test-images/agnhost:2.52" "registry.k8s.io/e2e-test-images/jessie-dnsutils:1.5" "registry.k8s.io/e2e-test-images/nginx:1.14-2" "registry.k8s.io/pause:3.10")
conformance_images=("k8sprow.azurecr.io/kubernetes-e2e-test-images/agnhost:2.52" "k8sprow.azurecr.io/kubernetes-e2e-test-images/jessie-dnsutils:1.5" "k8sprow.azurecr.io/kubernetes-e2e-test-images/nginx:1.14-2" "registry.k8s.io/e2e-test-images/pause:3.10")
e2e_images=("${DOCKER_REGISTRY}/antrea/toolbox:1.4-0" "registry.k8s.io/e2e-test-images/agnhost:2.40")
# Pull necessary images in advance to avoid transient error
for i in "${!harbor_images[@]}"; do
ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "ctr -n k8s.io images pull --user ${DOCKER_USERNAME}:${DOCKER_PASSWORD} ${DOCKER_REGISTRY}/antrea/${harbor_images[i]} && ctr -n k8s.io images tag ${DOCKER_REGISTRY}/antrea/${harbor_images[i]} ${antrea_images[i]}" || true
done
for i in "${!k8s_images[@]}"; do
ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "ctr -n k8s.io images pull --user ${DOCKER_USERNAME}:${DOCKER_PASSWORD} ${k8s_images[i]} && ctr -n k8s.io images tag ${k8s_images[i]} ${conformance_images[i]}" || true
done
for image in "${e2e_images[@]}"; do
ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "ctr -n k8s.io images pull --user ${DOCKER_USERNAME}:${DOCKER_PASSWORD} ${DOCKER_REGISTRY}/antrea/${image}" || true
ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "ctr -n k8s.io images pull --user ${DOCKER_USERNAME}:${DOCKER_PASSWORD} ${image}" || true
done

for i in `seq 2`; do
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/vmagent_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -525,7 +525,7 @@ func testANPOnVMs(t *testing.T, data *TestData, vmList []vmInfo, osType string)
})
// Test FQDN rules in ANP
t.Run("testANPOnExternalNodeWithFQDN", func(t *testing.T) {
testANPWithFQDN(t, data, "anp-vmagent-fqdn", namespace, *appliedToVM, []string{"www.facebook.com"}, []string{"docs.google.com"}, []string{"github.com"})
testANPWithFQDN(t, data, "anp-vmagent-fqdn", namespace, *appliedToVM, []string{"docs.amazon.com"}, []string{"docs.google.com"}, []string{"github.com"})
})
}

Expand Down

0 comments on commit 47ce51e

Please sign in to comment.