cherry pick missing PRs into release-1.1 (#1953)

* Support IAM role for backup CRD (#1861) * backup: support kms decryption secret (#1908) * fix API docs html * use /hack/e2e.sh to run a single node kind cluster for develop (#1749) * Backup: make tikv support add serviceaccount and switch rclone env_auth to true (#1930) Co-authored-by: 尹亮 <30903849+shuijing198799@users.noreply.github.com> Co-authored-by: weekface <weekface@gmail.com>
pingcap · Mar 17, 2020 · 403fe05 · 403fe05
1 parent 9ae7ae2
commit 403fe05
Show file tree

Hide file tree

Showing 21 changed files with 508 additions and 133 deletions.
diff --git a/docs/api-references/docs.html b/docs/api-references/docs.html
diff --git a/hack/e2e.sh b/hack/e2e.sh
@@ -53,6 +53,7 @@ Environments:
     KUBECONFIG            path to the kubeconfig file, defaults: ~/.kube/config
     SKIP_BUILD            skip building binaries
     SKIP_IMAGE_BUILD      skip build and push images
+    SKIP_IMAGE_LOAD       skip load images
     SKIP_UP               skip starting the cluster
     SKIP_DOWN             skip shutting down the cluster
     SKIP_TEST             skip running the test
@@ -76,6 +77,7 @@ Environments:
     GINKGO_PARALLEL       if set to `y`, will run specs in parallel, the number of nodes will be the number of cpus
     GINKGO_NO_COLOR       if set to `y`, suppress color output in default reporter
     RUNNER_SUITE_NAME     the suite name of runner
+    SKIP_GINKGO           if set to `y`, skip ginkgo
 
 Examples:
 
@@ -179,6 +181,7 @@ CLUSTER=${CLUSTER:-tidb-operator}
 KUBECONFIG=${KUBECONFIG:-~/.kube/config}
 SKIP_BUILD=${SKIP_BUILD:-}
 SKIP_IMAGE_BUILD=${SKIP_IMAGE_BUILD:-}
+SKIP_IMAGE_LOAD=${SKIP_IMAGE_LOAD:-}
 SKIP_UP=${SKIP_UP:-}
 SKIP_DOWN=${SKIP_DOWN:-}
 SKIP_TEST=${SKIP_TEST:-}
@@ -199,6 +202,7 @@ KUBE_WORKERS=${KUBE_WORKERS:-3}
 DOCKER_IO_MIRROR=${DOCKER_IO_MIRROR:-}
 GCR_IO_MIRROR=${GCR_IO_MIRROR:-}
 QUAY_IO_MIRROR=${QUAY_IO_MIRROR:-}
+SKIP_GINKGO=${SKIP_GINKGO:-}
 RUNNER_SUITE_NAME=${RUNNER_SUITE_NAME:-}
 
 echo "PROVIDER: $PROVIDER"
@@ -485,6 +489,14 @@ else
     exit 1
 fi
 
+if [ "${HOSTNAME:-}" == "tidb-operator-dev" -a ! -f /usr/local/bin/helm ]; then
+    ln -s $OUTPUT_BIN/helm /usr/local/bin/helm
+fi
+
+if [ "${HOSTNAME:-}" == "tidb-operator-dev" -a ! -f /usr/local/bin/kind ]; then
+    ln -s $KIND_BIN /usr/local/bin/kind
+fi
+
 # Environments for hack/run-e2e.sh
 export PROVIDER
 export CLUSTER
@@ -494,6 +506,8 @@ export GCP_REGION
 export GCP_ZONE
 export GCP_CREDENTIALS
 export IMAGE_TAG
+export SKIP_GINKGO
+export SKIP_IMAGE_LOAD
 export TIDB_OPERATOR_IMAGE=$DOCKER_REGISTRY/pingcap/tidb-operator:${IMAGE_TAG}
 export E2E_IMAGE=$DOCKER_REGISTRY/pingcap/tidb-operator-e2e:${IMAGE_TAG}
 export PATH=$OUTPUT_BIN:$PATH

diff --git a/hack/run-e2e.sh b/hack/run-e2e.sh
@@ -29,6 +29,7 @@ GCP_REGION=${GCP_REGION:-}
 GCP_ZONE=${GCP_ZONE:-}
 GCP_CREDENTIALS=${GCP_CREDENTIALS:-}
 IMAGE_TAG=${IMAGE_TAG:-}
+SKIP_IMAGE_LOAD=${SKIP_IMAGE_LOAD:-}
 TIDB_OPERATOR_IMAGE=${TIDB_OPERATOR_IMAGE:-localhost:5000/pingcap/tidb-operator:latest}
 E2E_IMAGE=${E2E_IMAGE:-localhost:5000/pingcap/tidb-operator-e2e:latest}
 KUBECONFIG=${KUBECONFIG:-$HOME/.kube/config}
@@ -41,6 +42,7 @@ GINKGO_PARALLEL=${GINKGO_PARALLEL:-n} # set to 'y' to run tests in parallel
 # in parallel
 GINKGO_NO_COLOR=${GINKGO_NO_COLOR:-n}
 GINKGO_STREAM=${GINKGO_STREAM:-y}
+SKIP_GINKGO=${SKIP_GINKGO:-}
 
 if [ -z "$KUBECONFIG" ]; then
     echo "error: KUBECONFIG is required"
@@ -284,10 +286,18 @@ if [ -z "$KUBECONTEXT" ]; then
     echo "info: current kubeconfig context is '$KUBECONTEXT'"
 fi
 
-e2e::image_load
+if [ -z "$SKIP_IMAGE_LOAD" ]; then
+    e2e::image_load
+fi
+
 e2e::setup_local_pvs
 e2e::setup_helm_server
 
+if [ -n "$SKIP_GINKGO" ]; then
+    echo "info: skipping ginkgo"
+    exit 0
+fi
+
 echo "info: start to run e2e process"
 
 ginkgo_args=()

diff --git a/images/tidb-backup-manager/Dockerfile b/images/tidb-backup-manager/Dockerfile
@@ -1,6 +1,6 @@
 FROM pingcap/tidb-enterprise-tools:latest
-
-ARG VERSION=v1.48.0
+ARG VERSION=v1.51.0
+ARG SHUSH_VERSION=v1.4.0
 RUN apk update && apk add ca-certificates
 
 RUN wget -nv https://github.com/ncw/rclone/releases/download/${VERSION}/rclone-${VERSION}-linux-amd64.zip \
@@ -15,6 +15,10 @@ RUN wget -nv http://download.pingcap.org/br-latest-linux-amd64.tar.gz \
     && chmod 755 /usr/local/bin/br \
     && rm -rf br-latest-linux-amd64.tar.gz
 
+RUN wget -nv https://github.com/realestate-com-au/shush/releases/download/${SHUSH_VERSION}/shush_linux_amd64 \
+  && mv shush_linux_amd64 /usr/local/bin/shush \
+  && chmod 755 /usr/local/bin/shush
+
 COPY bin/tidb-backup-manager /tidb-backup-manager
 COPY entrypoint.sh /entrypoint.sh
 

diff --git a/images/tidb-backup-manager/entrypoint.sh b/images/tidb-backup-manager/entrypoint.sh
@@ -19,7 +19,7 @@ echo "Create rclone.conf file."
 cat <<EOF > /tmp/rclone.conf
 [s3]
 type = s3
-env_auth = false
+env_auth = true
 provider =  ${S3_PROVIDER}
 access_key_id = ${AWS_ACCESS_KEY_ID}
 secret_access_key = ${AWS_SECRET_ACCESS_KEY:-$AWS_SECRET_KEY}
@@ -51,33 +51,40 @@ else
 fi
 
 BACKUP_BIN=/tidb-backup-manager
+if [[ -n "${AWS_DEFAULT_REGION}"]]; then
+	EXEC_COMMAND="exec"
+else
+	EXEC_COMMAND="/usr/local/bin/shush exec --"
+fi
+
+cat /tmp/rclone.conf
 
 # exec command
 case "$1" in
     backup)
         shift 1
         echo "$BACKUP_BIN backup $@"
-        exec $BACKUP_BIN backup "$@"
+        $EXEC_COMMAND $BACKUP_BIN backup "$@"
         ;;
     export)
         shift 1
         echo "$BACKUP_BIN export $@"
-        exec $BACKUP_BIN export "$@"
+        $EXEC_COMMAND $BACKUP_BIN export "$@"
         ;;
     restore)
         shift 1
         echo "$BACKUP_BIN restore $@"
-        exec $BACKUP_BIN restore "$@"
+        $EXEC_COMMAND $BACKUP_BIN restore "$@"
         ;;
     import)
         shift 1
         echo "$BACKUP_BIN import $@"
-        exec $BACKUP_BIN import "$@"
+        $EXEC_COMMAND $BACKUP_BIN import "$@"
         ;;
     clean)
         shift 1
         echo "$BACKUP_BIN clean $@"
-        exec $BACKUP_BIN clean "$@"
+        $EXEC_COMMAND $BACKUP_BIN clean "$@"
         ;;
     *)
         echo "Usage: $0 {backup|restore|clean}"

diff --git a/manifests/backup/backup-aws-s3-br.yaml b/manifests/backup/backup-aws-s3-br.yaml
@@ -0,0 +1,34 @@
+---
+apiVersion: pingcap.com/v1alpha1
+kind: Backup
+metadata:
+  name: demo1-backup-s3
+  namespace: test1
+  # annotations:
+    # iam.amazonaws.com/role: "arn:aws:iam::123456789:role"
+spec:
+  # backupType: full
+  # useKMS: false
+  # serviceAccount: myServiceAccount
+  br:
+    cluster: myCluster
+    # clusterNamespce: <backup-namespace>
+    # enableTLSClient: true
+    # logLevel: info
+    # statusAddr: <status-addr>
+    # concurrency: 4
+    # rateLimit: 0
+    # timeAgo: <time>
+    # checksum: true
+    # sendCredToTikv: true
+  from:
+    host: 172.30.6.56
+    secretName: mySecret
+    # port: 4000
+    # user: root
+  s3:
+    provider: aws
+    region: us-west-2
+    bucket: backup
+    prefix: test1-demo1
+    # secretName: aws-secret
diff --git a/manifests/backup/backup-s3-br.yaml b/manifests/backup/backup-s3-br.yaml
@@ -4,20 +4,28 @@ kind: Backup
 metadata:
   name: demo1-backup-s3
   namespace: test1
+  # annotations:
+    # iam.amazonaws.com/role: "arn:aws:iam::123456789:role"
 spec:
-  #backupType: full
+  # backupType: full
+  # useKMS: false
+  # serviceAccount: myServiceAccount
   br:
-    pd: 10.233.40.168:2379
-    # ca: <ca>
-    # cert: <cert>
-    # key: <key>
+    cluster: myCluster
+    # clusterNamespce: <backup-namespace>
+    # enableTLSClient: true
     # logLevel: info
     # statusAddr: <status-addr>
     # concurrency: 4
     # rateLimit: 0
     # timeAgo: <time>
     # checksum: true
     # sendCredToTikv: true
+  from:
+    host: 172.30.6.56
+    secretName: mySecret
+    # port: 4000
+    # user: root
   s3:
     provider: ceph
     endpoint: http://10.233.57.220

diff --git a/manifests/backup/backup-schedule-aws-s3-br.yaml b/manifests/backup/backup-schedule-aws-s3-br.yaml
@@ -0,0 +1,39 @@
+---
+apiVersion: pingcap.com/v1alpha1
+kind: BackupSchedule
+metadata:
+  name: demo1-backup-schedule-s3
+  namespace: test1
+  # annotations:
+    # iam.amazonaws.com/role: "arn:aws:iam::123456789:role"
+spec:
+  #maxBackups: 5
+  #pause: true
+  maxReservedTime: "3h"
+  schedule: "*/2 * * * *"
+  backupTemplate:
+    #backupType: full
+    # useKMS: false
+    # serviceAccount: myServiceAccount
+    br:
+      cluster: myCluster
+      # clusterNamespce: backupNamespace
+      # enableTLSClient:  true
+      # logLevel: info
+      # statusAddr: <status-addr>
+      # concurrency: 4
+      # rateLimit: 0
+      # timeAgo: <time>
+      # checksum: true
+      # sendCredToTikv: true
+    from:
+      host:         172.30.6.56
+      secretName:   mysecret
+      # port:         4000
+      # user:         root
+    s3:
+      provider: aws
+      region: us-west-2
+      bucket: backup
+      prefix: test1-demo1
+      # secretName: aws-secret
diff --git a/manifests/backup/backup-schedule-s3-br.yaml b/manifests/backup/backup-schedule-s3-br.yaml
@@ -4,25 +4,33 @@ kind: BackupSchedule
 metadata:
   name: demo1-backup-schedule-s3
   namespace: test1
+  # annotations:
+    # iam.amazonaws.com/role: "arn:aws:iam::123456789:role"
 spec:
   #maxBackups: 5
   #pause: true
   maxReservedTime: "3h"
   schedule: "*/2 * * * *"
   backupTemplate:
     #backupType: full
+    # useKMS: false
+    # serviceAccount: myServiceAccount
     br:
-      pd: 10.233.40.168:2379
-      # ca: <ca>
-      # cert: <cert>
-      # key: <key>
+      cluster: myCluster
+      # clusterNamespce: backupNamespace
+      # enableTLSClient:  true
       # logLevel: info
       # statusAddr: <status-addr>
       # concurrency: 4
       # rateLimit: 0
       # timeAgo: <time>
       # checksum: true
       # sendCredToTikv: true
+    from:
+      host:         172.30.6.56
+      secretName:   mysecret
+      # port:         4000
+      # user:         root
     s3:
       provider: ceph
       endpoint: http://10.233.57.220

diff --git a/manifests/backup/restore-aws-s3-br.yaml b/manifests/backup/restore-aws-s3-br.yaml
@@ -0,0 +1,36 @@
+---
+apiVersion: pingcap.com/v1alpha1
+kind: Restore
+metadata:
+  name: demo1-restore-s3-br
+  namespace: test1
+  # annotations:
+    # iam.amazonaws.com/role: "arn:aws:iam::123456789:role"
+spec:
+  # backupType: full
+  # useKMS: false
+  # serviceAccount: myServiceAccount
+  br:
+    cluster: myCluster
+    # clusterNamespce: <restore-namespace>
+    # enableTLSClient: true
+    # db: <db-name>
+    # table: <table-name>
+    # logLevel: info
+    # statusAddr: <status-addr>
+    # concurrency: 4
+    # rateLimit: 0
+    # timeAgo: <time>
+    # checksum: true
+    # sendCredToTikv: true
+  to:
+    host: 172.30.6.56
+    secretName: mySecret
+    # port: 4000
+    # user: root
+  s3:
+    provider: aws
+    region: us-west-2
+    bucket: backup
+    prefix: test1-demo1
+    # secretName: aws-secret
diff --git a/manifests/backup/restore-s3-br.yaml b/manifests/backup/restore-s3-br.yaml
@@ -4,22 +4,30 @@ kind: Restore
 metadata:
   name: demo1-restore-s3-br
   namespace: test1
+  # annotations:
+    # iam.amazonaws.com/role: "arn:aws:iam::123456789:role"
 spec:
   # backupType: full
+  # useKMS: false
+  # serviceAccount: myServiceAccount
   br:
-    pd: 10.233.40.168:2379
+    cluster: myCluster
+    # clusterNamespce: <restore-namespace>
+    # enableTLSClient: true
     # db: <db-name>
     # table: <table-name>
-    # ca: <ca>
-    # cert: <cert>
-    # key: <key>
     # logLevel: info
     # statusAddr: <status-addr>
     # concurrency: 4
     # rateLimit: 0
     # timeAgo: <time>
     # checksum: true
     # sendCredToTikv: true
+  to:
+    host: 172.30.6.56
+    secretName: mySecret
+    # port: 4000
+    # user: root
   s3:
     provider: ceph
     endpoint: http://10.233.57.220