add stability e2e group and a basic case (#1986)

ci/e2e_kind.groovy

@@ -0,0 +1,220 @@
+//
+// Jenkins pipeline for Kind e2e job.
+//
+// This script is written in declarative syntax. Refer to
+// https://jenkins.io/doc/book/pipeline/syntax/ for more details.
+//
+// Note that parameters of the job is configured in this script.
+//
+
+import groovy.transform.Field
+
+@Field
+def podYAML = '''
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: tidb-operator-e2e
+spec:
+  containers:
+  - name: main
+    image: gcr.io/k8s-testimages/kubekins-e2e:v20200311-1e25827-master
+    command:
+    - runner.sh
+    # Clean containers on TERM signal in root process to avoid cgroup leaking.
+    # https://github.com/pingcap/tidb-operator/issues/1603#issuecomment-582402196
+    - exec
+    - bash
+    - -c
+    - |
+      function clean() {
+        echo "info: clean all containers to avoid cgroup leaking"
+        docker kill $(docker ps -q) || true
+        docker system prune -af || true
+      }
+      trap clean TERM
+      sleep 1d & wait
+    # we need privileged mode in order to do docker in docker
+    securityContext:
+      privileged: true
+    env:
+    - name: DOCKER_IN_DOCKER_ENABLED
+      value: "true"
+    resources:
+      requests:
+        memory: "8000Mi"
+        cpu: 8000m
+        ephemeral-storage: "50Gi"
+      limits:
+        memory: "8000Mi"
+        cpu: 8000m
+        ephemeral-storage: "50Gi"
+    # kind needs /lib/modules and cgroups from the host
+    volumeMounts:
+    - mountPath: /lib/modules
+      name: modules
+      readOnly: true
+    - mountPath: /sys/fs/cgroup
+      name: cgroup
+    # dind expects /var/lib/docker to be volume
+    - name: docker-root
+      mountPath: /var/lib/docker
+    # legacy docker path for cr.io/k8s-testimages/kubekins-e2e
+    - name: docker-graph
+      mountPath: /docker-graph
+  volumes:
+  - name: modules
+    hostPath:
+      path: /lib/modules
+      type: Directory
+  - name: cgroup
+    hostPath:
+      path: /sys/fs/cgroup
+      type: Directory
+  - name: docker-root
+    emptyDir: {}
+  - name: docker-graph
+    emptyDir: {}
+  tolerations:
+  - effect: NoSchedule
+    key: tidb-operator
+    operator: Exists
+  affinity:
+    # running on nodes for tidb-operator only
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: ci.pingcap.com
+            operator: In
+            values:
+            - tidb-operator
+    podAntiAffinity:
+      preferredDuringSchedulingIgnoredDuringExecution:
+      - weight: 100
+        podAffinityTerm:
+          labelSelector:
+            matchExpressions:
+            - key: app
+              operator: In
+              values:
+              - tidb-operator-e2e
+          topologyKey: kubernetes.io/hostname
+'''
+
+// Able to override default values in Jenkins job via environment variables.
+if (!env.DEFAULT_GIT_REF) {
+    env.DEFAULT_GIT_REF = "master"
+}
+
+if (!env.DEFAULT_GINKGO_NODES) {
+    env.DEFAULT_GINKGO_NODES = "8"
+}
+
+if (!env.DEFAULT_E2E_ARGS) {
+    env.DEFAULT_E2E_ARGS = ""
+}
+
+if (!env.DEFAULT_DOCKER_IO_MIRROR) {
+    env.DEFAULT_DOCKER_IO_MIRROR = ""
+}
+
+if (!env.DEFAULT_QUAY_IO_MIRROR) {
+    env.DEFAULT_QUAY_IO_MIRROR = ""
+}
+
+if (!env.DEFAULT_GCR_IO_MIRROR) {
+    env.DEFAULT_GCR_IO_MIRROR = ""
+}
+
+pipeline {
+    agent {
+        kubernetes {
+            yaml podYAML
+            defaultContainer "main"
+            customWorkspace "/home/jenkins/agent/workspace/go/src/github.com/pingcap/tidb-operator"
+        }
+    }
+
+    options {
+        timeout(time: 3, unit: 'HOURS') 
+    }
+
+    parameters {
+        string(name: 'GIT_URL', defaultValue: 'git@github.com:pingcap/tidb-operator.git', description: 'git repo url')
+        string(name: 'GIT_REF', defaultValue: env.DEFAULT_GIT_REF, description: 'git ref spec to checkout, e.g. master, release-1.1')
+        string(name: 'PR_ID', defaultValue: '', description: 'pull request ID, this will override GIT_REF if set, e.g. 1889')
+        string(name: 'GINKGO_NODES', defaultValue: env.DEFAULT_GINKGO_NODES, description: 'the number of ginkgo nodes')
+        string(name: 'E2E_ARGS', defaultValue: env.DEFAULT_E2E_ARGS, description: "e2e args, e.g. --ginkgo.focus='\\[Stability\\]'")
+        string(name: 'DOCKER_IO_MIRROR', defaultValue: env.DEFAULT_DOCKER_IO_MIRROR, description: "docker mirror for docker.io")
+        string(name: 'QUAY_IO_MIRROR', defaultValue: env.DEFAULT_QUAY_IO_MIRROR, description: "mirror for quay.io")
+        string(name: 'GCR_IO_MIRROR', defaultValue: env.DEFAULT_GCR_IO_MIRROR, description: "mirror for gcr.io")
+    }
+
+    environment {
+        GIT_REF = ''
+        ARTIFACTS = "${env.WORKSPACE}/artifacts"
+    }
+
+    stages {
+        stage("Prepare") {
+            steps {
+                // The declarative model for Jenkins Pipelines has a restricted
+                // subset of syntax that it allows in the stage blocks. We use
+                // script step to bypass the restriction.
+                // https://jenkins.io/doc/book/pipeline/syntax/#script
+                script {
+                    GIT_REF = params.GIT_REF
+                    if (params.PR_ID != "") {
+                        GIT_REF = "refs/remotes/origin/pr/${params.PR_ID}/head"
+                    }
+                }
+                echo "env.NODE_NAME: ${env.NODE_NAME}"
+                echo "env.WORKSPACE: ${env.WORKSPACE}"
+                echo "GIT_REF: ${GIT_REF}"
+                echo "ARTIFACTS: ${ARTIFACTS}"
+            }
+        }
+
+        stage("Checkout") {
+            steps {
+                checkout scm: [
+                        $class: 'GitSCM',
+                        branches: [[name: GIT_REF]],
+                        userRemoteConfigs: [[
+                            credentialsId: 'github-sre-bot-ssh',
+                            refspec: '+refs/heads/*:refs/remotes/origin/* +refs/pull/*:refs/remotes/origin/pr/*',
+                            url: "${params.GIT_URL}",
+                        ]]
+                    ]
+            }
+        }
+
+        stage("Run") {
+            steps {
+                sh """
+                #!/bin/bash
+                export GINKGO_NODES=${params.GINKGO_NODES}
+                export REPORT_DIR=${ARTIFACTS}
+                export DOCKER_IO_MIRROR=${params.DOCKER_IO_MIRROR}
+                export QUAY_IO_MIRROR=${params.QUAY_IO_MIRROR}
+                export GCR_IO_MIRROR=${params.GCR_IO_MIRROR}
+                echo "info: begin to run e2e"
+                ./hack/e2e.sh -- ${params.E2E_ARGS}
+                """
+            }
+        }
+    }
+
+    post {
+        always {
+            dir(ARTIFACTS) {
+                archiveArtifacts artifacts: "**", allowEmptyArchive: true
+                junit testResults: "*.xml", allowEmptyResults: true
+            }
+        }
+    }
+}
+
+// vim: et sw=4 ts=4

ci/pingcap_tidb_operator_build_kind.groovy

@@ -238,13 +238,13 @@ def call(BUILD_BRANCH, CREDENTIALS_ID, CODECOV_CREDENTIALS_ID) {
 		def MIRRORS = "DOCKER_IO_MIRROR=http://172.16.4.143:5000 QUAY_IO_MIRROR=http://172.16.4.143:5001"
 		def builds = [:]
 		builds["E2E v1.12.10"] = {
-			build("${MIRRORS} RUNNER_SUITE_NAME=e2e-v1.12 IMAGE_TAG=${GITHASH} SKIP_BUILD=y GINKGO_NODES=6 KUBE_VERSION=v1.12.10 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.12.10_ ./hack/e2e.sh -- --preload-images --ginkgo.skip='\\[Serial\\]'", artifacts)
+			build("${MIRRORS} RUNNER_SUITE_NAME=e2e-v1.12 IMAGE_TAG=${GITHASH} SKIP_BUILD=y GINKGO_NODES=6 KUBE_VERSION=v1.12.10 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.12.10_ ./hack/e2e.sh -- --preload-images", artifacts)
 		}
 		builds["E2E v1.12.10 AdvancedStatefulSet"] = {
-			build("${MIRRORS} RUNNER_SUITE_NAME=e2e-v1.12-advanced-statefulset IMAGE_TAG=${GITHASH} SKIP_BUILD=y GINKGO_NODES=6 KUBE_VERSION=v1.12.10 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.12.10_advanced_statefulset ./hack/e2e.sh -- --preload-images --ginkgo.skip='\\[Serial\\]' --operator-features AdvancedStatefulSet=true", artifacts)
+			build("${MIRRORS} RUNNER_SUITE_NAME=e2e-v1.12-advanced-statefulset IMAGE_TAG=${GITHASH} SKIP_BUILD=y GINKGO_NODES=6 KUBE_VERSION=v1.12.10 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.12.10_advanced_statefulset ./hack/e2e.sh -- --preload-images --operator-features AdvancedStatefulSet=true", artifacts)
 		}
 		builds["E2E v1.17.0"] = {
-			build("${MIRRORS} RUNNER_SUITE_NAME=e2e-v1.17 IMAGE_TAG=${GITHASH} SKIP_BUILD=y GINKGO_NODES=6 KUBE_VERSION=v1.17.0 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.17.0_ ./hack/e2e.sh -- -preload-images --ginkgo.skip='\\[Serial\\]'", artifacts)
+			build("${MIRRORS} RUNNER_SUITE_NAME=e2e-v1.17 IMAGE_TAG=${GITHASH} SKIP_BUILD=y GINKGO_NODES=6 KUBE_VERSION=v1.17.0 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.17.0_ ./hack/e2e.sh -- -preload-images", artifacts)
 		}
 		builds["E2E v1.12.10 Serial"] = {
 			build("${MIRRORS} RUNNER_SUITE_NAME=e2e-v1.12-serial IMAGE_TAG=${GITHASH} SKIP_BUILD=y KUBE_VERSION=v1.12.10 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.12.10_serial_ ./hack/e2e.sh -- --preload-images --ginkgo.focus='\\[Serial\\]' --install-operator=false", artifacts)

hack/lib.sh

@@ -154,7 +154,7 @@ function hack::wait_for_success() {
 }
 
 #
-# Concatenates the elements with an separator between them.
+# Concatenates the elements with a separator between them.
 #
 # Usage: hack::join ',' a b c
 #

tests/e2e/e2e.go

@@ -274,6 +274,11 @@ func RunE2ETests(t *testing.T) {
 
 	gomega.RegisterFailHandler(e2elog.Fail)
 
+	// Disable serial and stability tests by default unless they are explicitly requested.
+	if config.GinkgoConfig.FocusString == "" && config.GinkgoConfig.SkipString == "" {
+		config.GinkgoConfig.SkipString = `\[Stability\]|\[Serial\]`
+	}
+
 	// Run tests through the Ginkgo runner with output to console + JUnit for Jenkins
 	var r []ginkgo.Reporter
 	if framework.TestContext.ReportDir != "" {

tests/e2e/tidbcluster/serial.go

@@ -62,6 +62,7 @@ func mustToString(set sets.Int32) string {
 	return string(b)
 }
 
+// Serial specs describe tests which cannot run in parallel.
 var _ = ginkgo.Describe("[tidb-operator][Serial]", func() {
 	f := framework.NewDefaultFramework("serial")
 
@@ -490,8 +491,8 @@ var _ = ginkgo.Describe("[tidb-operator][Serial]", func() {
 		})
 		framework.ExpectNoError(err)
 
-		ginkgo.By("Make sure pods are not affected")
-		err = utilpod.WaitForPodsAreNotAffected(c, podListBeforeUpgrade.Items, time.Minute*3)
+		ginkgo.By("Make sure pods are not changed")
+		err = utilpod.WaitForPodsAreChanged(c, podListBeforeUpgrade.Items, time.Minute*3)
 		framework.ExpectEqual(err, wait.ErrWaitTimeout, "Pods was not affeteced after the operator is upgraded")
 	})