Merge pull request #224 from networkservicemesh/calico_a

Add Calico
networkservicemesh · Mar 25, 2022 · afae298 · afae298
2 parents 4fb47ca + 38199a4
commit afae298
Show file tree

Hide file tree

Showing 22 changed files with 363 additions and 115 deletions.
diff --git a/.cloudtest_calico.yaml b/.cloudtest_calico.yaml
@@ -0,0 +1,31 @@
+---
+version: 1.0
+root: "./.tests/cloud_test_calico/"
+timeout: 7200  # 2 hour total total timeout
+shuffle-enabled: true
+statistics:
+  enabled: true
+  interval: 60  # 60 seconds for statistics
+import:
+  - cloudtest/packet.yaml
+  - cloudtest/tests.yaml
+
+retest:  # Allow to do test re-run if some kind of failures are detected, line CNI network plugin errors.
+  count: 1  # Allow 5 times to do restart
+  warmup-time: 15  # Put 15 seconds warmup for cluster instance to be used again.
+  allowed-retests: 2  # If cluster instance have few attempts with retest requests one after another, we need to restart cluster.
+  pattern:
+    - "NetworkPlugin cni failed to set up pod"    # Error in AWS dur to leak of IPs or not ability to assign them.
+    - "etcdserver: request timed out"  # Error in any could, reason unknown.
+    - "unable to establish connection to VPP (VPP API socket file /run/vpp/api.sock does not exist)"  # a VPP is not started, it will be re-started in general, but will cause test fail.
+    # Sometimes (rarely) docker registry is unavailable for a moment
+    - "Error response from daemon: Get https://.*docker.io/.*: dial tcp: lookup registry"
+    - "Error response from daemon: Get https://.*docker.io/.*: net/http: request canceled while waiting for connection"
+    - "Failed create pod sandbox"
+reporting:
+  junit-report: "results/junit.xml"
+health-check:
+  - message: "Branch is not up to date"
+    interval: 60  # 1 minute
+    run: |
+      echo "Health check!"
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -66,8 +66,12 @@ jobs:
           git diff --name-only --exit-code go.sum || ( echo "Run go tidy" && false )
 
   packet:
-    name: packet
+    name: packet (CNI ${{ matrix.CNI }})
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        CNI: ["default", "calico"]
     steps:
       - name: Set up /bin permissions   # 1. Set up /bin permissions
         run: |
@@ -87,31 +91,42 @@ jobs:
         with:
           repository: networkservicemesh/deployments-k8s
           path: networkservicemesh/deployments-k8s
-      - name: Checkout files            # 5. Checkout files
+      - name: Compute suffix            # 5. Compute suffix for cloudtest input and output paths
+        id: suffix
+        run: |
+          if [[ "${CNI}" == "calico" ]]; then
+            echo ::set-output name=val::_calico
+          fi
+        env:
+          CNI: ${{ matrix.CNI }}
+      - name: Checkout files            # 6. Checkout files
         uses: actions/checkout@v2
         with:
           path: ${{ github.repository }}
-      - name: Run tests with cloudtest  # 6. Run tests with cloudtest
+      - name: Run tests with cloudtest  # 7. Run tests with cloudtest
         working-directory: ${{ github.repository }}
         run: |
-          cloudtest
+          cloudtest --config=.cloudtest${suffix}.yaml
         env:
           PACKET_AUTH_TOKEN: ${{ secrets.PACKET_AUTH_TOKEN }}
           PACKET_PROJECT_ID: 383890d0-f5d1-4de1-881a-4d1ede549d18
           KUBERNETES_VERSION: ${{ secrets.NSM_KUBERNETES_VERSION }}
-      - name: Publish test report       # 7. Publish test report
+          CNI: ${{ matrix.CNI }}
+          suffix: ${{ steps.suffix.outputs.val }}
+      - name: Publish test report       # 8. Publish test report
         uses: mikepenz/action-junit-report@v2.1.0
         if: ${{ always() }}
         with:
-          report_paths: "**/cloud_test/results/junit.xml"
+          report_paths: "**/cloud_test${{ steps.suffix.outputs.val }}/results/junit.xml"
           suite_regex: "Test*"
           github_token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Upload logs               # 8. Upload logs
+          check_name: "JUnit Test Report (CNI ${{ matrix.CNI }})"
+      - name: Upload logs               # 9. Upload logs
         uses: actions/upload-artifact@v2
         if: ${{ always() }}
         with:
           name: logs-${{ github.run_number }}
-          path: ${{ github.repository }}/.tests/cloud_test/
+          path: ${{ github.repository }}/.tests/
 
   packet-cleanup:
     name: packet cleanup

diff --git a/cloudtest/packet.yaml b/cloudtest/packet.yaml
@@ -25,13 +25,15 @@ providers:
           os: "ubuntu_20_04"
           billing-cycle: "hourly"
           port-vlans:
-            eth3: 1044
+            eth1: 3000  # calico VLAN
+            eth3: 1044  # SR-IOV VLAN
         - name: "Worker"
           host-name: "SR-IOV-Worker-${CLUSTER_NAME}"
           os: "ubuntu_20_04"
           billing-cycle: "hourly"
           port-vlans:
-            eth3: 1044
+            eth1: 3000  # calico VLAN
+            eth3: 1044  # SR-IOV VLAN
       hardware-reservations:
         - 2cf78481-53b0-46c8-a084-6e9815acdb0b
         - 2361d3c2-f694-4fa7-a683-a9f69e2abe7c

diff --git a/scripts/calico/deploy-calico.sh b/scripts/calico/deploy-calico.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+function on_error() {
+  kubectl describe pods --all-namespaces
+  exit 1
+}
+trap 'on_error' ERR
+
+kubectl apply -f https://projectcalico.docs.tigera.io/v3.22/manifests/tigera-operator.yaml
+kubectl apply -f https://raw.githubusercontent.com/projectcalico/vpp-dataplane/master/yaml/calico/installation-default.yaml
+kubectl apply -k scripts/calico
+
+kubectl rollout status -n calico-vpp-dataplane ds/calico-vpp-node --timeout=10m
diff --git a/scripts/calico/kustomization.yaml b/scripts/calico/kustomization.yaml
@@ -0,0 +1,9 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - https://raw.githubusercontent.com/projectcalico/vpp-dataplane/master/yaml/generated/calico-vpp-nohuge.yaml
+
+patchesStrategicMerge:
+  - patch.yaml
diff --git a/scripts/calico/patch.yaml b/scripts/calico/patch.yaml
@@ -0,0 +1,8 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: calico-vpp-config
+  namespace: calico-vpp-dataplane
+data:
+  vpp_dataplane_interface: eno2
diff --git a/scripts/calico/setup-interfaces.sh b/scripts/calico/setup-interfaces.sh
@@ -0,0 +1,44 @@
+#!/bin/bash
+
+set -e
+
+cdr2mask ()
+{
+   # Number of args to shift, 255..255, first non-255 byte, zeroes
+   set -- $(( 5 - ("$1" / 8) )) 255 255 255 255 $(( (255 << (8 - ("$1" % 8))) & 255 )) 0 0 0
+   if [[ "$1" -gt 1 ]]
+   then
+     shift "$1"
+   else
+     shift
+   fi
+   echo "${1-0}"."${2-0}"."${3-0}"."${4-0}"
+}
+
+iface="$1"
+ip="$2"
+cidr="$3"
+mask=$(cdr2mask "$3")
+
+# Unbond interface and set IP address
+cd /etc/network/
+awk -v pattern="iface $1 inet" -v ip="$2" -v mask="$mask" '
+	$0 ~ pattern {
+		printf "%s static\n",pattern;
+		printf "    address %s\n",ip;
+		printf "    netmask %s\n",mask;
+		getline;
+		while ($0 != "") {
+			if ($1=="bond-master") {
+				next;
+				break
+			};
+			print;
+			getline
+		}
+	} 1
+' interfaces > interfaces.tmp && mv interfaces.tmp interfaces
+cd
+ifenslave -d bond0 "${iface}"
+ip addr change "${ip}/${cidr}" dev "${iface}"
+ip link set up dev "${iface}"
diff --git a/scripts/calico/setup-node-ip.sh b/scripts/calico/setup-node-ip.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -e
+
+ip="$1"
+
+sed -Ei "s/(.*)\"/\1 --node-ip=${ip}\"/g" /var/lib/kubelet/kubeadm-flags.env
+systemctl restart kubelet
diff --git a/scripts/create-kubernetes-cluster.sh b/scripts/create-kubernetes-cluster.sh
@@ -1,82 +1,143 @@
 #!/bin/bash -x
-# shellcheck disable=SC2086
+# shellcheck disable=SC2086,SC2029
 
 master_ip=$1
 worker_ip=$2
 sshkey=$3
 
-SSH_OPTS="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o IdentitiesOnly=yes -i ${sshkey}"
-
-function wait_pids() {
-  pids="$1"
-  message="$2"
-  for pid in ${pids}; do
-    echo "waiting for PID ${pid}"
-    wait ${pid}
-    code=$?
-    if test $code -ne 0; then
-      echo "${message}: process exited with code $code, aborting..." && return 1
-    fi
-  done
-  return 0
-}
-
-# Setup SR-IOV
+SSH_CONFIG="ssh_config"
+SSH_OPTS="-F ${SSH_CONFIG} -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o IdentitiesOnly=yes -i ${sshkey}"
+
+if [[ "$CNI" == "calico" ]]; then # calico
+  # Use a new 10.0.0.${base_ip}/30 subnet to prevent IP addresses collisions
+  # ${base_ip} should be <= 248, because 10.0.0.252/30 subnet is reserved for manual testing
+  base_ip=$(( GITHUB_RUN_NUMBER % 63 * 4 ))
+
+  CALICO_MASTER_IP="10.0.0.$(( base_ip + 1 ))"
+  CALICO_WORKER_IP="10.0.0.$(( base_ip + 2 ))"
+  CALICO_CIDR_PREFIX="30"
+  CALICO_INTERFACE="eno2"
+fi
+
+ENVS="KUBERNETES_VERSION CNI"
+
+# wait_pids pid_1 ... pid_n
+source scripts/include/wait-pids.sh
+# wait_start ip_1 ... ip_n
+source scripts/include/wait-start.sh
+
+# 0. Setup SendEnv on the local side.
+cp /etc/ssh/ssh_config ${SSH_CONFIG} || exit 1
+echo "Host *
+	SendEnv ${ENVS}" >> ${SSH_CONFIG} || exit 2
+
+wait_start ${master_ip} ${worker_ip} || exit 3
+
+# 1. Setup AcceptEnv on the servers sides and wait for sshd to restart.
+scp ${SSH_OPTS} scripts/setup-sshd.sh root@${master_ip}:setup-sshd.sh || exit 11
+scp ${SSH_OPTS} scripts/setup-sshd.sh root@${worker_ip}:setup-sshd.sh || exit 12
+
+pids=""
+ssh ${SSH_OPTS} root@${master_ip} ./setup-sshd.sh "${ENVS}" &
+pids+=" $!"
+ssh ${SSH_OPTS} root@${worker_ip} ./setup-sshd.sh "${ENVS}" &
+pids+=" $!"
+wait_pids "${pids}" "sshd config failed" || exit 13
+
+wait_start ${master_ip} ${worker_ip} || exit 14
+
+## 2. Setup SR-IOV.
 pids=""
 /bin/bash scripts/sriov/setup-SRIOV.sh "${master_ip}" "${worker_ip}" "${SSH_OPTS}" &
 pids+=" $!"
-wait_pids "${pids}" "SR-IOV config failed" || exit 1
+wait_pids "${pids}" "SR-IOV config failed" || exit 21
+
+if [[ "$CNI" == "calico" ]]; then # calico
+  # 3. Create Calico scripts directory on nodes.
+  ssh ${SSH_OPTS} root@${master_ip} mkdir -p calico || exit 31
+  ssh ${SSH_OPTS} root@${worker_ip} mkdir -p calico || exit 32
+
+  # 4. Setup Calico interfaces.
+  scp ${SSH_OPTS} scripts/calico/setup-interfaces.sh root@${master_ip}:calico/setup-interfaces.sh || exit 41
+  scp ${SSH_OPTS} scripts/calico/setup-interfaces.sh root@${worker_ip}:calico/setup-interfaces.sh || exit 42
 
-# Create k8s scripts directory on nodes
-ssh ${SSH_OPTS} root@${master_ip} mkdir k8s
-ssh ${SSH_OPTS} root@${worker_ip} mkdir k8s
+  pids=""
+  ssh ${SSH_OPTS} root@${master_ip} ./calico/setup-interfaces.sh "${CALICO_INTERFACE}" "${CALICO_MASTER_IP}" "${CALICO_CIDR_PREFIX}" &
+  pids+=" $!"
+  ssh ${SSH_OPTS} root@${worker_ip} ./calico/setup-interfaces.sh "${CALICO_INTERFACE}" "${CALICO_WORKER_IP}" "${CALICO_CIDR_PREFIX}" &
+  pids+=" $!"
+  wait_pids "${pids}" "setup Calico interfaces failed" || exit 43
+fi
 
-# Setup docker ulimit
-scp ${SSH_OPTS} scripts/k8s/docker-ulimit.sh root@${master_ip}:k8s/docker-ulimit.sh || exit 2
-scp ${SSH_OPTS} scripts/k8s/docker-ulimit.sh root@${worker_ip}:k8s/docker-ulimit.sh || exit 3
+# 5. Create k8s scripts directory on nodes.
+ssh ${SSH_OPTS} root@${master_ip} mkdir -p k8s || exit 51
+ssh ${SSH_OPTS} root@${worker_ip} mkdir -p k8s || exit 52
+
+# 6. Config docker.
+scp ${SSH_OPTS} scripts/k8s/config-docker.sh root@${master_ip}:k8s/config-docker.sh || exit 61
+scp ${SSH_OPTS} scripts/k8s/config-docker.sh root@${worker_ip}:k8s/config-docker.sh || exit 62
 
 pids=""
-ssh ${SSH_OPTS} root@${master_ip} ./k8s/docker-ulimit.sh &
+ssh ${SSH_OPTS} root@${master_ip} ./k8s/config-docker.sh &
 pids+=" $!"
-ssh ${SSH_OPTS} root@${worker_ip} ./k8s/docker-ulimit.sh &
+ssh ${SSH_OPTS} root@${worker_ip} ./k8s/config-docker.sh &
 pids+=" $!"
-wait_pids "${pids}" "kubernetes install failed" || exit 4
+wait_pids "${pids}" "docker config failed" || exit 63
 
-# Install kubeadm, kubelet and kubectl
-scp ${SSH_OPTS} scripts/k8s/install-kubernetes.sh root@${master_ip}:k8s/install-kubernetes.sh || exit 5
-scp ${SSH_OPTS} scripts/k8s/install-kubernetes.sh root@${worker_ip}:k8s/install-kubernetes.sh || exit 6
+# 7. Install kubeadm, kubelet and kubectl.
+scp ${SSH_OPTS} scripts/k8s/install-kubernetes.sh root@${master_ip}:k8s/install-kubernetes.sh || exit 71
+scp ${SSH_OPTS} scripts/k8s/install-kubernetes.sh root@${worker_ip}:k8s/install-kubernetes.sh || exit 72
 
 pids=""
-ssh ${SSH_OPTS} root@${master_ip} ./k8s/install-kubernetes.sh ${KUBERNETES_VERSION} &
+ssh ${SSH_OPTS} root@${master_ip} ./k8s/install-kubernetes.sh &
 pids+=" $!"
-ssh ${SSH_OPTS} root@${worker_ip} ./k8s/install-kubernetes.sh ${KUBERNETES_VERSION} &
+ssh ${SSH_OPTS} root@${worker_ip} ./k8s/install-kubernetes.sh &
 pids+=" $!"
-wait_pids "${pids}" "kubernetes install failed" || exit 7
+wait_pids "${pids}" "kubernetes install failed" || exit 73
 
-# master: start kubernetes and create join script
-# worker: download kubernetes images
-scp ${SSH_OPTS} scripts/k8s/start-master.sh root@${master_ip}:k8s/start-master.sh || exit 8
-scp ${SSH_OPTS} scripts/k8s/download-worker-images.sh root@${worker_ip}:k8s/download-worker-images.sh || exit 9
+# 8.
+#    master: start kubernetes and create join script.
+#    worker: download kubernetes images.
+scp ${SSH_OPTS} scripts/k8s/start-master.sh root@${master_ip}:k8s/start-master.sh || exit 81
+scp ${SSH_OPTS} scripts/k8s/download-worker-images.sh root@${worker_ip}:k8s/download-worker-images.sh || exit 82
 
 pids=""
-ssh ${SSH_OPTS} root@${master_ip} ./k8s/start-master.sh ${KUBERNETES_VERSION} &
+ssh ${SSH_OPTS} root@${master_ip} ./k8s/start-master.sh ${master_ip} ${CALICO_MASTER_IP} &
 pids+=" $!"
 ssh ${SSH_OPTS} root@${worker_ip} ./k8s/download-worker-images.sh &
 pids+=" $!"
-wait_pids "${pids}" "node setup failed" || exit 10
+wait_pids "${pids}" "nodes setup failed" || exit 83
 
-# Download worker join script
+# 9. Download, upload and run worker join script.
 mkdir -p /tmp/${master_ip}
-scp ${SSH_OPTS} root@${master_ip}:k8s/join-cluster.sh /tmp/${master_ip}/join-cluster.sh || exit 11
-chmod +x /tmp/${master_ip}/join-cluster.sh || exit 12
+scp ${SSH_OPTS} root@${master_ip}:k8s/join-cluster.sh /tmp/${master_ip}/join-cluster.sh || exit 91
+chmod +x /tmp/${master_ip}/join-cluster.sh || exit 92
 
-# Upload and run worker join script
-scp ${SSH_OPTS} /tmp/${master_ip}/join-cluster.sh root@${worker_ip}:k8s/join-cluster.sh || exit 13
+scp ${SSH_OPTS} /tmp/${master_ip}/join-cluster.sh root@${worker_ip}:k8s/join-cluster.sh || exit 93
 
 pids=""
 ssh ${SSH_OPTS} root@${worker_ip} ./k8s/join-cluster.sh &
 pids+=" $!"
-wait_pids "${pids}" "worker join failed" || exit 14
+wait_pids "${pids}" "worker join failed" || exit 94
+
+# 10. Save KUBECONFIG to file.
+scp ${SSH_OPTS} root@${master_ip}:.kube/config ${KUBECONFIG} || exit 101
+
+if [[ "$CNI" == "calico" ]]; then # calico
+  # 11. Setup cluster nodes IPs.
+  scp ${SSH_OPTS} scripts/calico/setup-node-ip.sh root@${master_ip}:calico/setup-node-ip.sh || exit 111
+  scp ${SSH_OPTS} scripts/calico/setup-node-ip.sh root@${worker_ip}:calico/setup-node-ip.sh || exit 112
+
+  pids=""
+  ssh ${SSH_OPTS} root@${master_ip} ./calico/setup-node-ip.sh "${CALICO_MASTER_IP}" &
+  pids+=" $!"
+  ssh ${SSH_OPTS} root@${worker_ip} ./calico/setup-node-ip.sh "${CALICO_WORKER_IP}" &
+  pids+=" $!"
+  wait_pids "${pids}" "nodes IPs setup failed" || exit 113
+
+  # 12. Deploy Calico CNI.
+  /bin/bash scripts/calico/deploy-calico.sh || exit 121
+fi
 
-echo "Save KUBECONFIG to file"
-scp ${SSH_OPTS} root@${master_ip}:.kube/config ${KUBECONFIG} || exit 15
+# Get pods
+kubectl get pods --all-namespaces