fix test

Signed-off-by: yiyiyimu <wosoyoung@gmail.com>

.github/workflows/chaos.yml

@@ -1,6 +1,6 @@
 name: Chaos Test
 
-on: [push, pull_request]
+on: [pull_request]
 
 jobs:
   build:
@@ -14,7 +14,7 @@ jobs:
 
       - name: Creating minikube cluster
         run: |
-          bash ./t/chaos/utils.sh startMinikube
+          bash ./t/chaos/utils.sh start_minikube
 
       - name: Print cluster information
         run: |
@@ -28,29 +28,20 @@ jobs:
       - name: Deploy Etcd Operator
         run: |
           git clone https://github.com/yiyiyimu/etcd-operator.git --depth 1
-          cd etcd-operator
-          bash example/rbac/create_role.sh
-          kubectl create -f example/deployment.yaml
-          bash ./t/chaos/utils.sh ensure_pods_ready etcd-operator "True"
-          kubectl create -f example/example-etcd-cluster.yaml
-          bash ./t/chaos/utils.sh ensure_pods_ready etcd "True True True"
+          bash etcd-operator/example/rbac/create_role.sh
+          kubectl create -f etcd-operator/example/deployment.yaml
+          bash ./t/chaos/utils.sh ensure_pods_ready etcd-operator "True" 30
+          kubectl create -f etcd-operator/example/example-etcd-cluster.yaml
+          bash ./t/chaos/utils.sh ensure_pods_ready etcd "True True True" 30
 
-      - uses: actions/checkout@v2
       - name: Deploy APISIX
         run: |
-          DNS_IP=$(kubectl get svc -n kube-system -l k8s-app=kube-dns -o 'jsonpath={..spec.clusterIP}')
-          echo "
-          dns_resolver:
-            - ${DNS_IP}
-          etcd:
-            host:
-              - "http://etcd-cluster-client.default.svc.cluster.local:2379"
-          " > conf/config.yaml
-          example/rbac/create_role.sh
-          kubectl create configmap apisix-gw-config.yaml --from-file=../conf/config.yaml
-          kubectl apply -f deployment.yaml
-          kubectl apply -f service.yaml
-          while [[ $(kubectl get pods -l app=apisix-gw -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != "True True" ]]; do echo "Waiting for pod running" && sleep 10; done
+          bash ./t/chaos/utils.sh modify_config
+          kubectl create configmap apisix-gw-config.yaml --from-file=./conf/config.yaml
+          kubectl apply -f ./kubernetes/deployment.yaml
+          kubectl apply -f ./kubernetes/service.yaml
+          bash ./t/chaos/utils.sh ensure_pods_ready apisix-gw "True" 30
+          nohup kubectl port-forward svc/apisix-gw-lb 9080:9080 >/dev/null 2>&1 &
 
       - name: Deploy Chaos mesh
         run: |

kubernetes/deployment.yaml

@@ -23,7 +23,7 @@ metadata:
   name: apisix-gw-deployment
   # namespace: default
 spec:
-  replicas: 2
+  replicas: 1
   selector:
     matchLabels:
       app: apisix-gw

kubernetes/service.yaml

@@ -41,5 +41,3 @@ spec:
   type: NodePort
   externalTrafficPolicy: Local
   # sessionAffinity: None
-  externalIPs:
-    - 10.253.0.11

t/chaos/go.mod

@@ -1,5 +1,8 @@
 module github.com/apache/apisix/t/chaos
 
-require github.com/gavv/httpexpect/v2 v2.1.0
+require (
+	github.com/gavv/httpexpect/v2 v2.1.0
+	github.com/onsi/gomega v1.9.0
+)
 
 go 1.14

t/chaos/go.sum

@@ -36,6 +36,8 @@ github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo=
 github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.9.0 h1:R1uwffexN6Pr340GtYRIdZmAiN4J+iw6WG4wog1DUXg=
+github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/savsgio/gotils v0.0.0-20200117113501-90175b0fbe3f h1:PgA+Olipyj258EIEYnpFFONrrCcAIWNUNoFhUfMqAGY=
@@ -78,6 +80,8 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpbl
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
@@ -87,5 +91,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 moul.io/http2curl v1.0.1-0.20190925090545-5cd742060b0e h1:C7q+e9M5nggAvWfVg9Nl66kebKeuJlP3FD58V4RR5wo=
 moul.io/http2curl v1.0.1-0.20190925090545-5cd742060b0e/go.mod h1:nejbQVfXh96n9dSF6cH3Jsk/QI1Z2oEL7sSI2ifXFNA=

t/chaos/kill-etcd.yaml

@@ -19,11 +19,11 @@ apiVersion: chaos-mesh.org/v1alpha1
 kind: PodChaos
 metadata:
   name: kill-etcd
-  namespace: chaos-testing
 spec:
   action: pod-kill
   mode: all
-  value: ""
   selector:
     labelSelectors:
       "app": "etcd"
+  scheduler:
+      cron: "@every 10m"

t/chaos/kill-etcd_test.go

@@ -18,16 +18,20 @@
 package chaos
 
 import (
+	"fmt"
 	"net/http"
 	"os/exec"
+	"strings"
 	"testing"
+	"time"
 
 	"github.com/gavv/httpexpect/v2"
+	. "github.com/onsi/gomega"
 )
 
 var (
 	token = "edd1c9f034335f136f87ad84b625c8f1"
-	host  = "http://10.253.0.11:9080"
+	host  = "http://127.0.0.1:9080"
 )
 
 type httpTestCase struct {
@@ -37,6 +41,7 @@ type httpTestCase struct {
 	Body         string
 	Headers      map[string]string
 	ExpectStatus int
+	ExpectBody   string
 }
 
 func caseCheck(tc httpTestCase) {
@@ -51,6 +56,7 @@ func caseCheck(tc httpTestCase) {
 	}
 
 	if req == nil {
+		fmt.Printf("%#v", tc)
 		panic("fail to init request")
 	}
 	for key, val := range tc.Headers {
@@ -64,16 +70,24 @@ func caseCheck(tc httpTestCase) {
 	if tc.ExpectStatus != 0 {
 		resp.Status(tc.ExpectStatus)
 	}
+
+	if tc.ExpectBody != "" {
+		resp.Body().Contains(tc.ExpectBody)
+	}
 }
 
 func setRoute(e *httpexpect.Expect, expectStatus int) {
 	caseCheck(httpTestCase{
 		E:       e,
+		Method:  http.MethodPut,
 		Path:    "/apisix/admin/routes/1",
 		Headers: map[string]string{"X-API-KEY": token},
 		Body: `{
 			"uri": "/hello",
 			"host": "foo.com",
+			"plugins": {
+				"prometheus": {}
+			},
 			"upstream": {
 				"nodes": {
 					"bar.org": 1
@@ -88,28 +102,72 @@ func setRoute(e *httpexpect.Expect, expectStatus int) {
 func getRoute(e *httpexpect.Expect, expectStatus int) {
 	caseCheck(httpTestCase{
 		E:            e,
+		Method:       http.MethodGet,
 		Path:         "/hello",
 		Headers:      map[string]string{"Host": "foo.com"},
 		ExpectStatus: expectStatus,
 	})
 }
 
+func deleteRoute(e *httpexpect.Expect, expectStatus int) {
+	caseCheck(httpTestCase{
+		E:            e,
+		Method:       http.MethodDelete,
+		Path:         "/apisix/admin/routes/1",
+		Headers:      map[string]string{"X-API-KEY": token},
+		ExpectStatus: expectStatus,
+	})
+}
+
+func getPrometheusMetric(e *httpexpect.Expect, expectEtcd int) {
+	caseCheck(httpTestCase{
+		E:          e,
+		Method:     http.MethodGet,
+		Path:       "/apisix/prometheus/metrics",
+		ExpectBody: fmt.Sprintf("apisix_etcd_reachable %d", expectEtcd),
+	})
+}
+
+func runCommand(t *testing.T, cmd string) string {
+	out, err := exec.Command("bash", "-c", cmd).CombinedOutput()
+	if err != nil {
+		t.Fatalf("fail to run command %s: %s, %s", cmd, err.Error(), out)
+	}
+	return string(out)
+}
+
 func TestGetSuccessWhenEtcdKilled(t *testing.T) {
+	g := NewWithT(t)
 	e := httpexpect.New(t, host)
 
 	// check if everything works
-	setRoute(e, http.StatusOK)
+	setRoute(e, http.StatusCreated)
 	getRoute(e, http.StatusOK)
+	getPrometheusMetric(e, 1)
+
+	podName := runCommand(t, "kubectl get pod -l app=apisix-gw -o 'jsonpath={..metadata.name}'")
+	t.Run("error log not contains etcd error", func(t *testing.T) {
+		errorLog := runCommand(t, fmt.Sprintf("kubectl exec -it %s -- cat logs/error.log", podName))
+		g.Expect(strings.Contains(errorLog, "failed to fetch data from etcd")).To(BeFalse())
+	})
 
 	// TODO: use client-go
 	// apply chaos to kill all etcd pods
-	_, err := exec.Command("kubectl apply kill-etcd.yaml").CombinedOutput()
-	if err != nil {
-		panic("fail to apply chaos yaml")
-	}
+	t.Run("kill all etcd pods", func(t *testing.T) {
+		_ = runCommand(t, "kubectl apply -f kill-etcd.yaml")
+		time.Sleep(3 * time.Second)
+	})
 
 	// fail to set route since etcd is all killed
 	// while get route could still succeed
 	setRoute(e, http.StatusInternalServerError)
 	getRoute(e, http.StatusOK)
+	getPrometheusMetric(e, 0)
+
+	t.Run("error log contains etcd error", func(t *testing.T) {
+		errorLog := runCommand(t, fmt.Sprintf("kubectl exec -it %s -- cat logs/error.log", podName))
+		g.Expect(strings.Contains(errorLog, "failed to fetch data from etcd")).To(BeTrue())
+	})
+
+	//deleteRoute(e, http.StatusOK)
 }

t/chaos/utils.sh

@@ -19,30 +19,41 @@
 
 set -ex
 
-startMinikube() {
+start_minikube() {
     curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl"
     chmod +x ./kubectl
     sudo mv ./kubectl /usr/local/bin/kubectl
 
     curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube_latest_amd64.deb
     sudo dpkg -i --force-architecture minikube_latest_amd64.deb
-    sudo mv ./minikube /usr/local/bin/minikube
     minikube start
 }
 
+modify_config() {
+    DNS_IP=$(kubectl get svc -n kube-system -l k8s-app=kube-dns -o 'jsonpath={..spec.clusterIP}')
+    echo "dns_resolver:
+  - ${DNS_IP}
+etcd:
+  host:
+    - \"http://etcd-cluster-client.default.svc.cluster.local:2379\" " > ./conf/config.yaml
+}
+
 ensure_pods_ready() {
     local app=$1
     local status=$2
+    local retries=$3
 
     count=0
-    while [ -n "$(kubectl get pods -l app=${app} -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != ${status})" ];
+    while [[ $(kubectl get pods -l app=${app} -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != ${status} ]];
     do
         echo "Waiting for pod running" && sleep 10;
 
         ((count=count+1))
-        # waiting for 300s, or timeout
-        if [ $count -gt 30 ]; then
+        if [ $count -gt ${retries} ]; then
             printf "Waiting for pod status running timeout\n"
+            kubectl describe pod -l app=${app}
+            printf "\n\n"
+            kubectl logs -l app=${app}
             exit 1
         fi
     done