From f4891971d0ec901e0d93db45384204d3f1d59e00 Mon Sep 17 00:00:00 2001 From: "D.K" Date: Tue, 15 Dec 2020 10:49:39 -0800 Subject: [PATCH 01/10] Add Version to Bootstrap Image This change adds a version to the bootstrap images. This will allow for an easier transition between bootstrap images. This field is starting at 0 and will be incremented to 1 for the change to golang 1.15 Signed-off-by: D.K --- Makefile | 127 ++++++-------------------- docker/base/Dockerfile | 6 +- docker/base/Dockerfile.mariadb | 5 +- docker/base/Dockerfile.mariadb103 | 5 +- docker/base/Dockerfile.mysql56 | 5 +- docker/base/Dockerfile.mysql80 | 5 +- docker/base/Dockerfile.percona | 5 +- docker/base/Dockerfile.percona57 | 5 +- docker/base/Dockerfile.percona80 | 5 +- docker/bootstrap/build.sh | 6 +- docker/lite/Dockerfile.alpine | 5 +- docker/lite/Dockerfile.mariadb | 5 +- docker/lite/Dockerfile.mariadb103 | 5 +- docker/lite/Dockerfile.mysql56 | 5 +- docker/lite/Dockerfile.mysql57 | 5 +- docker/lite/Dockerfile.mysql80 | 5 +- docker/lite/Dockerfile.percona | 5 +- docker/lite/Dockerfile.percona57 | 5 +- docker/lite/Dockerfile.percona80 | 5 +- docker/lite/Dockerfile.testing | 5 +- docker/lite/Dockerfile.ubi7.mysql57 | 5 +- docker/lite/Dockerfile.ubi7.mysql80 | 5 +- docker/lite/Dockerfile.ubi7.percona57 | 5 +- docker/lite/Dockerfile.ubi7.percona80 | 5 +- docker/local/Dockerfile | 5 +- docker/test/run.sh | 5 +- test.go | 7 +- 27 files changed, 130 insertions(+), 131 deletions(-) diff --git a/Makefile b/Makefile index e032ff1e8e7..6907f33e16c 100644 --- a/Makefile +++ b/Makefile @@ -190,119 +190,53 @@ $(PROTO_GO_OUTS): install_protoc-gen-go proto/*.proto # This rule builds the bootstrap images for all flavors. DOCKER_IMAGES_FOR_TEST = mariadb mariadb103 mysql56 mysql57 mysql80 percona percona57 percona80 DOCKER_IMAGES = common $(DOCKER_IMAGES_FOR_TEST) +BOOTSTRAP_VERSION=0 docker_bootstrap: - for i in $(DOCKER_IMAGES); do echo "building bootstrap image: $$i"; docker/bootstrap/build.sh $$i || exit 1; done + for i in $(DOCKER_IMAGES); do echo "building bootstrap image: $$i"; docker/bootstrap/build.sh $$i ${BOOTSTRAP_VERSION} || exit 1; done docker_bootstrap_test: - flavors='$(DOCKER_IMAGES_FOR_TEST)' && ./test.go -pull=false -parallel=2 -flavor=$${flavors// /,} + flavors='$(DOCKER_IMAGES_FOR_TEST)' && ./test.go -pull=false -parallel=2 -bootstrap-version=${BOOTSTRAP_VERSION} -flavor=$${flavors// /,} docker_bootstrap_push: - for i in $(DOCKER_IMAGES); do echo "pushing bootstrap image: $$i"; docker push vitess/bootstrap:$$i || exit 1; done + for i in $(DOCKER_IMAGES); do echo "pushing bootstrap image: ${BOOTSTRAP_VERSION}-$$i"; docker push vitess/bootstrap:${BOOTSTRAP_VERSION}-$$i || exit 1; done # Use this target to update the local copy of your images with the one on Dockerhub. docker_bootstrap_pull: - for i in $(DOCKER_IMAGES); do echo "pulling bootstrap image: $$i"; docker pull vitess/bootstrap:$$i || exit 1; done + for i in $(DOCKER_IMAGES); do echo "pulling bootstrap image: $$i"; docker pull vitess/bootstrap:${BOOTSTRAP_VERSION}-$$i || exit 1; done -docker_base: - # Fix permissions before copying files, to avoid AUFS bug. - chmod -R o=g * - docker build -f docker/base/Dockerfile -t vitess/base . - -docker_base_mysql56: - chmod -R o=g * - docker build -f docker/base/Dockerfile.mysql56 -t vitess/base:mysql56 . -docker_base_mysql80: - chmod -R o=g * - docker build -f docker/base/Dockerfile.mysql80 -t vitess/base:mysql80 . - -docker_base_mariadb: - chmod -R o=g * - docker build -f docker/base/Dockerfile.mariadb -t vitess/base:mariadb . - -docker_base_mariadb103: - chmod -R o=g * - docker build -f docker/base/Dockerfile.mariadb -t vitess/base:mariadb103 . +define build_docker_image + # Fix permissions before copying files, to avoid AUFS bug. + ${info Building ${2}} + chmod -R o=g *; + docker build -f ${1} -t ${2} --build-arg bootstrap_version=${BOOTSTRAP_VERSION} .; +endef -docker_base_percona: - chmod -R o=g * - docker build -f docker/base/Dockerfile.percona -t vitess/base:percona . +docker_base: + ${call build_docker_image,docker/base/Dockerfile,vitess/base} -docker_base_percona57: - chmod -R o=g * - docker build -f docker/base/Dockerfile.percona57 -t vitess/base:percona57 . +DOCKER_BASE_SUFFIX = mysql56 mysql80 mariadb mariadb103 percona percona57 percona80 +DOCKER_BASE_TARGETS = $(addprefix docker_base_, $(DOCKER_BASE_SUFFIX)) +$(DOCKER_BASE_TARGETS): docker_base_%: + ${call build_docker_image,docker/base/Dockerfile.$*,vitess/base:$*} -docker_base_percona80: - chmod -R o=g * - docker build -f docker/base/Dockerfile.percona80 -t vitess/base:percona80 . +docker_base_all: docker_base $(DOCKER_BASE_TARGETS) docker_lite: - chmod -R o=g * - docker build -f docker/lite/Dockerfile -t vitess/lite . - -docker_lite_mysql56: - chmod -R o=g * - docker build -f docker/lite/Dockerfile.mysql56 -t vitess/lite:mysql56 . - -docker_lite_mysql57: - chmod -R o=g * - docker build -f docker/lite/Dockerfile.mysql57 -t vitess/lite:mysql57 . - -docker_lite_ubi7.mysql57: - chmod -R o=g * - docker build -f docker/lite/Dockerfile.ubi7.mysql57 -t vitess/lite:ubi7.mysql57 . - -docker_lite_mysql80: - chmod -R o=g * - docker build -f docker/lite/Dockerfile.mysql80 -t vitess/lite:mysql80 . - -docker_lite_ubi7.mysql80: - chmod -R o=g * - docker build -f docker/lite/Dockerfile.ubi7.mysql80 -t vitess/lite:ubi7.mysql80 . + ${call build_docker_image,docker/lite/Dockerfile,vitess/lite} -docker_lite_mariadb: - chmod -R o=g * - docker build -f docker/lite/Dockerfile.mariadb -t vitess/lite:mariadb . +DOCKER_LITE_SUFFIX = mysql56 mysql57 ubi7.mysql57 mysql80 ubi7.mysql80 mariadb mariadb103 percona percona57 ubi7.percona57 percona80 ubi7.percona80 alpine testing +DOCKER_LITE_TARGETS = $(addprefix docker_lite_,$(DOCKER_LITE_SUFFIX)) +$(DOCKER_LITE_TARGETS): docker_lite_%: + ${call build_docker_image,docker/lite/Dockerfile.$*,vitess/lite:$*} -docker_lite_mariadb103: - chmod -R o=g * - docker build -f docker/lite/Dockerfile.mariadb103 -t vitess/lite:mariadb103 . - -docker_lite_percona: - chmod -R o=g * - docker build -f docker/lite/Dockerfile.percona -t vitess/lite:percona . - -docker_lite_percona57: - chmod -R o=g * - docker build -f docker/lite/Dockerfile.percona57 -t vitess/lite:percona57 . - -docker_lite_ubi7.percona57: - chmod -R o=g * - docker build -f docker/lite/Dockerfile.ubi7.percona57 -t vitess/lite:ubi7.percona57 . - -docker_lite_percona80: - chmod -R o=g * - docker build -f docker/lite/Dockerfile.percona80 -t vitess/lite:percona80 . - -docker_lite_ubi7.percona80: - chmod -R o=g * - docker build -f docker/lite/Dockerfile.ubi7.percona80 -t vitess/lite:ubi7.percona80 . - -docker_lite_alpine: - chmod -R o=g * - docker build -f docker/lite/Dockerfile.alpine -t vitess/lite:alpine . - -docker_lite_testing: - chmod -R o=g * - docker build -f docker/lite/Dockerfile.testing -t vitess/lite:testing . +docker_lite_all: docker_lite $(DOCKER_LITE_TARGETS) docker_local: - chmod -R o=g * - docker build -f docker/local/Dockerfile -t vitess/local . + ${call build_docker_image,docker/local/Dockerfile,vitess/local} docker_mini: - chmod -R o=g * - docker build -f docker/mini/Dockerfile -t vitess/mini . + ${call build_docker_image,docker/mini/Dockerfile,vitess/mini} # This rule loads the working copy of the code into a bootstrap image, # and then runs the tests inside Docker. @@ -327,15 +261,6 @@ release: docker_base echo "git push origin v$(VERSION)" echo "Also, don't forget the upload releases/v$(VERSION).tar.gz file to GitHub releases" -packages: docker_base - @if [ -z "$VERSION" ]; then \ - echo "Set the env var VERSION with the release version"; exit 1;\ - fi - mkdir -p releases - docker build -f docker/packaging/Dockerfile -t vitess/packaging . - docker run --rm -v ${PWD}/releases:/vt/releases --env VERSION=$(VERSION) vitess/packaging --package /vt/releases -t deb --deb-no-default-config-files - docker run --rm -v ${PWD}/releases:/vt/releases --env VERSION=$(VERSION) vitess/packaging --package /vt/releases -t rpm - tools: echo $$(date): Installing dependencies ./bootstrap.sh diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile index bc81925d79b..a00b380b946 100644 --- a/docker/base/Dockerfile +++ b/docker/base/Dockerfile @@ -20,7 +20,11 @@ # therefore we need to have the symlinked "Dockerfile" in there as well. # TODO(mberlin): Remove the symlink and this note once # https://github.com/docker/hub-feedback/issues/292 is fixed. -FROM vitess/bootstrap:mysql57 + +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-mysql57" + +FROM "${image}" # Allows some docker builds to disable CGO ARG CGO_ENABLED=0 diff --git a/docker/base/Dockerfile.mariadb b/docker/base/Dockerfile.mariadb index 42e09d1a9d2..9f11742546a 100644 --- a/docker/base/Dockerfile.mariadb +++ b/docker/base/Dockerfile.mariadb @@ -1,4 +1,7 @@ -FROM vitess/bootstrap:mariadb +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-mariadb" + +FROM "${image}" # Allows some docker builds to disable CGO ARG CGO_ENABLED=0 diff --git a/docker/base/Dockerfile.mariadb103 b/docker/base/Dockerfile.mariadb103 index cc86d5aede5..8b15baec736 100644 --- a/docker/base/Dockerfile.mariadb103 +++ b/docker/base/Dockerfile.mariadb103 @@ -1,4 +1,7 @@ -FROM vitess/bootstrap:mariadb103 +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-mariadb103" + +FROM "${image}" # Allows some docker builds to disable CGO ARG CGO_ENABLED=0 diff --git a/docker/base/Dockerfile.mysql56 b/docker/base/Dockerfile.mysql56 index efdbade8ced..2d49c861562 100644 --- a/docker/base/Dockerfile.mysql56 +++ b/docker/base/Dockerfile.mysql56 @@ -1,4 +1,7 @@ -FROM vitess/bootstrap:mysql56 +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-mysql56" + +FROM "${image}" # Allows some docker builds to disable CGO ARG CGO_ENABLED=0 diff --git a/docker/base/Dockerfile.mysql80 b/docker/base/Dockerfile.mysql80 index 4a798700060..64f87ef90ff 100644 --- a/docker/base/Dockerfile.mysql80 +++ b/docker/base/Dockerfile.mysql80 @@ -1,4 +1,7 @@ -FROM vitess/bootstrap:mysql80 +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-mysql80" + +FROM "${image}" # Allows some docker builds to disable CGO ARG CGO_ENABLED=0 diff --git a/docker/base/Dockerfile.percona b/docker/base/Dockerfile.percona index 5b488680126..e1361c8a147 100644 --- a/docker/base/Dockerfile.percona +++ b/docker/base/Dockerfile.percona @@ -1,4 +1,7 @@ -FROM vitess/bootstrap:percona +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-percona" + +FROM "${image}" # Allows some docker builds to disable CGO ARG CGO_ENABLED=0 diff --git a/docker/base/Dockerfile.percona57 b/docker/base/Dockerfile.percona57 index 4a3aa6b0c7c..bc7071de851 100644 --- a/docker/base/Dockerfile.percona57 +++ b/docker/base/Dockerfile.percona57 @@ -1,4 +1,7 @@ -FROM vitess/bootstrap:percona57 +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-percona57" + +FROM "${image}" # Allows some docker builds to disable CGO ARG CGO_ENABLED=0 diff --git a/docker/base/Dockerfile.percona80 b/docker/base/Dockerfile.percona80 index f5997a16e1a..f6bc311ce72 100644 --- a/docker/base/Dockerfile.percona80 +++ b/docker/base/Dockerfile.percona80 @@ -1,4 +1,7 @@ -FROM vitess/bootstrap:percona80 +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-percona80" + +FROM "${image}" # Allows some docker builds to disable CGO ARG CGO_ENABLED=0 diff --git a/docker/bootstrap/build.sh b/docker/bootstrap/build.sh index 699f0627618..286e5035a08 100755 --- a/docker/bootstrap/build.sh +++ b/docker/bootstrap/build.sh @@ -15,12 +15,14 @@ # limitations under the License. flavor=$1 - if [[ -z "$flavor" ]]; then echo "Flavor must be specified as first argument." exit 1 fi +# Set default version of 0 +version="${2:-0}" + if [[ ! -f bootstrap.sh ]]; then echo "This script should be run from the root of the Vitess source tree - e.g. ~/src/vitess.io/vitess" exit 1 @@ -32,5 +34,5 @@ chmod -R o=g * arch=$(uname -m) [ "$arch" == "aarch64" ] && [ $flavor != "common" ] && arch_ext='-arm64v8' if [ -f "docker/bootstrap/Dockerfile.$flavor$arch_ext" ]; then - docker build --no-cache -f docker/bootstrap/Dockerfile.$flavor$arch_ext -t vitess/bootstrap:$flavor$arch_ext . + docker build --no-cache -f docker/bootstrap/Dockerfile.$flavor$arch_ext -t vitess/bootstrap:$version-$flavor$arch_ext . fi diff --git a/docker/lite/Dockerfile.alpine b/docker/lite/Dockerfile.alpine index aac7caba428..1c7912b7029 100644 --- a/docker/lite/Dockerfile.alpine +++ b/docker/lite/Dockerfile.alpine @@ -17,7 +17,10 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -FROM vitess/bootstrap:mariadb103 AS builder +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-mariadb103" + +FROM "${image}" AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER diff --git a/docker/lite/Dockerfile.mariadb b/docker/lite/Dockerfile.mariadb index 1617d899fe2..17f51e79366 100644 --- a/docker/lite/Dockerfile.mariadb +++ b/docker/lite/Dockerfile.mariadb @@ -17,7 +17,10 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -FROM vitess/bootstrap:mariadb AS builder +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-mariadb" + +FROM "${image}" AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER diff --git a/docker/lite/Dockerfile.mariadb103 b/docker/lite/Dockerfile.mariadb103 index 079ab6ff4a7..afa02b1c22c 100644 --- a/docker/lite/Dockerfile.mariadb103 +++ b/docker/lite/Dockerfile.mariadb103 @@ -17,7 +17,10 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -FROM vitess/bootstrap:mariadb103 AS builder +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-mariadb103" + +FROM "${image}" AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER diff --git a/docker/lite/Dockerfile.mysql56 b/docker/lite/Dockerfile.mysql56 index ec361bec520..3ae98ce0c35 100644 --- a/docker/lite/Dockerfile.mysql56 +++ b/docker/lite/Dockerfile.mysql56 @@ -17,7 +17,10 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -FROM vitess/bootstrap:mysql56 AS builder +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-mysql56" + +FROM "${image}" AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER diff --git a/docker/lite/Dockerfile.mysql57 b/docker/lite/Dockerfile.mysql57 index 5ff2d21514d..5f844c2c8fc 100644 --- a/docker/lite/Dockerfile.mysql57 +++ b/docker/lite/Dockerfile.mysql57 @@ -17,7 +17,10 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -FROM vitess/bootstrap:mysql57 AS builder +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-mysql57" + +FROM "${image}" AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER diff --git a/docker/lite/Dockerfile.mysql80 b/docker/lite/Dockerfile.mysql80 index c395d24f37a..ab91b9010a9 100644 --- a/docker/lite/Dockerfile.mysql80 +++ b/docker/lite/Dockerfile.mysql80 @@ -17,7 +17,10 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -FROM vitess/bootstrap:mysql80 AS builder +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-mysql80" + +FROM "${image}" AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER diff --git a/docker/lite/Dockerfile.percona b/docker/lite/Dockerfile.percona index e0c5eac8a78..5e315c791d3 100644 --- a/docker/lite/Dockerfile.percona +++ b/docker/lite/Dockerfile.percona @@ -17,7 +17,10 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -FROM vitess/bootstrap:percona AS builder +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-percona" + +FROM "${image}" AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER diff --git a/docker/lite/Dockerfile.percona57 b/docker/lite/Dockerfile.percona57 index e8cc430475c..be333d9441d 100644 --- a/docker/lite/Dockerfile.percona57 +++ b/docker/lite/Dockerfile.percona57 @@ -17,7 +17,10 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -FROM vitess/bootstrap:percona57 AS builder +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-percona57" + +FROM "${image}" AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER diff --git a/docker/lite/Dockerfile.percona80 b/docker/lite/Dockerfile.percona80 index 0bb057d6ed2..7388db5063b 100644 --- a/docker/lite/Dockerfile.percona80 +++ b/docker/lite/Dockerfile.percona80 @@ -17,7 +17,10 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -FROM vitess/bootstrap:percona80 AS builder +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-percona80" + +FROM "${image}" AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER diff --git a/docker/lite/Dockerfile.testing b/docker/lite/Dockerfile.testing index 523d50e3c77..948c04f3d69 100644 --- a/docker/lite/Dockerfile.testing +++ b/docker/lite/Dockerfile.testing @@ -17,7 +17,10 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -FROM vitess/bootstrap:mysql57 AS builder +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-mysql57" + +FROM "${image}" AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER diff --git a/docker/lite/Dockerfile.ubi7.mysql57 b/docker/lite/Dockerfile.ubi7.mysql57 index c8675f48382..6807a403685 100644 --- a/docker/lite/Dockerfile.ubi7.mysql57 +++ b/docker/lite/Dockerfile.ubi7.mysql57 @@ -17,7 +17,10 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -FROM vitess/bootstrap:mysql57 AS builder +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-mysql57" + +FROM "${image}" AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER diff --git a/docker/lite/Dockerfile.ubi7.mysql80 b/docker/lite/Dockerfile.ubi7.mysql80 index 9c9be8c68f5..54e1d44c33e 100644 --- a/docker/lite/Dockerfile.ubi7.mysql80 +++ b/docker/lite/Dockerfile.ubi7.mysql80 @@ -17,7 +17,10 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -FROM vitess/bootstrap:mysql80 AS builder +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-mysql80" + +FROM "${image}" AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER diff --git a/docker/lite/Dockerfile.ubi7.percona57 b/docker/lite/Dockerfile.ubi7.percona57 index a630262f006..625cc1806c6 100644 --- a/docker/lite/Dockerfile.ubi7.percona57 +++ b/docker/lite/Dockerfile.ubi7.percona57 @@ -17,7 +17,10 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -FROM vitess/bootstrap:percona57 AS builder +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-percona57" + +FROM "${image}" AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER diff --git a/docker/lite/Dockerfile.ubi7.percona80 b/docker/lite/Dockerfile.ubi7.percona80 index 6e993a85ebd..6b8cf4c3b9c 100644 --- a/docker/lite/Dockerfile.ubi7.percona80 +++ b/docker/lite/Dockerfile.ubi7.percona80 @@ -17,7 +17,10 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -FROM vitess/bootstrap:percona80 AS builder +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-percona80" + +FROM "${image}" AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER diff --git a/docker/local/Dockerfile b/docker/local/Dockerfile index a4e76e1a7d1..01b62ec2836 100644 --- a/docker/local/Dockerfile +++ b/docker/local/Dockerfile @@ -1,4 +1,7 @@ -FROM vitess/bootstrap:common +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-common" + +FROM "${image}" RUN apt-get update RUN apt-get install -y sudo curl vim jq mysql-server diff --git a/docker/test/run.sh b/docker/test/run.sh index 842c4512b1b..29d853c06ae 100755 --- a/docker/test/run.sh +++ b/docker/test/run.sh @@ -98,7 +98,8 @@ while true ; do done # Positional flags. flavor=$1 -cmd=$2 +version=${2:-0} +cmd=$3 args= if [[ -z "$flavor" ]]; then @@ -115,7 +116,7 @@ if [[ ! -f bootstrap.sh ]]; then exit 1 fi -image=vitess/bootstrap:$flavor +image=vitess/bootstrap:$version-$flavor if [[ -n "$existing_cache_image" ]]; then image=$existing_cache_image fi diff --git a/test.go b/test.go index 284e747078a..fa414eb9e2a 100755 --- a/test.go +++ b/test.go @@ -74,7 +74,8 @@ For example: // Flags var ( - flavor = flag.String("flavor", "mysql57", "comma-separated bootstrap flavor(s) to run against (when using Docker mode). Available flavors: all,"+flavors) + flavor = flag.String("flavor", "mysql57", "comma-separated bootstrap flavor(s) to run against (when using Docker mode). Available flavors: all,"+flavors) + bootstrapVersion = flag.String("bootstrap-version", "0", "the version identifier to use for the docker images") runCount = flag.Int("runs", 1, "run each test this many times") retryMax = flag.Int("retry", 3, "max number of retries, to detect flaky tests") logPass = flag.Bool("log-pass", false, "log test output even if it passes") @@ -311,7 +312,7 @@ func main() { wg.Add(1) go func(flavor string) { defer wg.Done() - image := "vitess/bootstrap:" + flavor + image := "vitess/bootstrap:" + bootstrapVersion + "-" + flavor pullTime := time.Now() log.Printf("Pulling %v...", image) cmd := exec.Command("docker", "pull", image) @@ -389,7 +390,7 @@ func main() { for _, flavor := range flavors { start := time.Now() log.Printf("Creating Docker cache image for flavor '%s'...", flavor) - if out, err := exec.Command("docker/test/run.sh", "--create_docker_cache", cacheImage(flavor), flavor, "make build").CombinedOutput(); err != nil { + if out, err := exec.Command("docker/test/run.sh", "--create_docker_cache", cacheImage(flavor), flavor, bootstrapVersion, "make build").CombinedOutput(); err != nil { log.Fatalf("Failed to create Docker cache image for flavor '%s': %v\n%s", flavor, err, out) } log.Printf("Creating Docker cache image took %v", round(time.Since(start))) From 09c9d3f4a36da2f33a71ca25cc97d9e14a477f79 Mon Sep 17 00:00:00 2001 From: "D.K" Date: Tue, 15 Dec 2020 10:57:36 -0800 Subject: [PATCH 02/10] Fixing test driver error Must de-reference paramaters Signed-off-by: D.K --- test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test.go b/test.go index fa414eb9e2a..d07bb08fd7f 100755 --- a/test.go +++ b/test.go @@ -312,7 +312,7 @@ func main() { wg.Add(1) go func(flavor string) { defer wg.Done() - image := "vitess/bootstrap:" + bootstrapVersion + "-" + flavor + image := "vitess/bootstrap:" + *bootstrapVersion + "-" + flavor pullTime := time.Now() log.Printf("Pulling %v...", image) cmd := exec.Command("docker", "pull", image) @@ -390,7 +390,7 @@ func main() { for _, flavor := range flavors { start := time.Now() log.Printf("Creating Docker cache image for flavor '%s'...", flavor) - if out, err := exec.Command("docker/test/run.sh", "--create_docker_cache", cacheImage(flavor), flavor, bootstrapVersion, "make build").CombinedOutput(); err != nil { + if out, err := exec.Command("docker/test/run.sh", "--create_docker_cache", cacheImage(flavor), flavor, *bootstrapVersion, "make build").CombinedOutput(); err != nil { log.Fatalf("Failed to create Docker cache image for flavor '%s': %v\n%s", flavor, err, out) } log.Printf("Creating Docker cache image took %v", round(time.Since(start))) From e1eb1ced357e98e4418f8dd7fd010214b277b386 Mon Sep 17 00:00:00 2001 From: "D.K" Date: Wed, 16 Dec 2020 15:13:54 -0800 Subject: [PATCH 03/10] formatting test.go changes Signed-off-by: D.K Signed-off-by: D.K. --- test.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/test.go b/test.go index d07bb08fd7f..9c378b799bc 100755 --- a/test.go +++ b/test.go @@ -76,20 +76,20 @@ For example: var ( flavor = flag.String("flavor", "mysql57", "comma-separated bootstrap flavor(s) to run against (when using Docker mode). Available flavors: all,"+flavors) bootstrapVersion = flag.String("bootstrap-version", "0", "the version identifier to use for the docker images") - runCount = flag.Int("runs", 1, "run each test this many times") - retryMax = flag.Int("retry", 3, "max number of retries, to detect flaky tests") - logPass = flag.Bool("log-pass", false, "log test output even if it passes") - timeout = flag.Duration("timeout", 10*time.Minute, "timeout for each test") - pull = flag.Bool("pull", true, "re-pull the bootstrap image, in case it's been updated") - docker = flag.Bool("docker", true, "run tests with Docker") - useDockerCache = flag.Bool("use_docker_cache", false, "if true, create a temporary Docker image to cache the source code and the binaries generated by 'make build'. Used for execution on Travis CI.") - shard = flag.Int("shard", -1, "if N>=0, run the tests whose Shard field matches N") - tag = flag.String("tag", "", "if provided, only run tests with the given tag. Can't be combined with -shard or explicit test list") - exclude = flag.String("exclude", "", "if provided, exclude tests containing any of the given tags (comma delimited)") - keepData = flag.Bool("keep-data", false, "don't delete the per-test VTDATAROOT subfolders") - printLog = flag.Bool("print-log", false, "print the log of each failed test (or all tests if -log-pass) to the console") - follow = flag.Bool("follow", false, "print test output as it runs, instead of waiting to see if it passes or fails") - parallel = flag.Int("parallel", 1, "number of tests to run in parallel") + runCount = flag.Int("runs", 1, "run each test this many times") + retryMax = flag.Int("retry", 3, "max number of retries, to detect flaky tests") + logPass = flag.Bool("log-pass", false, "log test output even if it passes") + timeout = flag.Duration("timeout", 10*time.Minute, "timeout for each test") + pull = flag.Bool("pull", true, "re-pull the bootstrap image, in case it's been updated") + docker = flag.Bool("docker", true, "run tests with Docker") + useDockerCache = flag.Bool("use_docker_cache", false, "if true, create a temporary Docker image to cache the source code and the binaries generated by 'make build'. Used for execution on Travis CI.") + shard = flag.Int("shard", -1, "if N>=0, run the tests whose Shard field matches N") + tag = flag.String("tag", "", "if provided, only run tests with the given tag. Can't be combined with -shard or explicit test list") + exclude = flag.String("exclude", "", "if provided, exclude tests containing any of the given tags (comma delimited)") + keepData = flag.Bool("keep-data", false, "don't delete the per-test VTDATAROOT subfolders") + printLog = flag.Bool("print-log", false, "print the log of each failed test (or all tests if -log-pass) to the console") + follow = flag.Bool("follow", false, "print test output as it runs, instead of waiting to see if it passes or fails") + parallel = flag.Int("parallel", 1, "number of tests to run in parallel") remoteStats = flag.String("remote-stats", "", "url to send remote stats") ) From bbb41868fc74f407f6fe532b07fd166a3d58fe60 Mon Sep 17 00:00:00 2001 From: "D.K" Date: Wed, 16 Dec 2020 15:29:13 -0800 Subject: [PATCH 04/10] Worksflows run tests directly Signed-off-by: D.K Signed-off-by: D.K. --- test.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test.go b/test.go index 9c378b799bc..b496b7dab88 100755 --- a/test.go +++ b/test.go @@ -131,9 +131,10 @@ type Test struct { // Tags is a list of tags that can be used to filter tests. Tags []string - name string - flavor string - runIndex int + name string + flavor string + bootstrapVersion string + runIndex int pass, fail int } @@ -187,7 +188,7 @@ func (t *Test) run(dir, dataDir string) ([]byte, error) { args = []string{"--use_docker_cache", cacheImage(t.flavor), t.flavor, testArgs} } else { // If there is no cache, we have to call 'make build' before each test. - args = []string{t.flavor, "make build && " + testArgs} + args = []string{t.flavor, t.bootstrapVersion, "make build && " + testArgs} } cmd = exec.Command(path.Join(dir, "docker/test/run.sh"), args...) @@ -355,6 +356,7 @@ func main() { for _, t := range tests { test := *t test.flavor = flavor + test.bootstrapVersion = *bootstrapVersion dup = append(dup, &test) } } From f31ef001128d9c8b9cd0f56052f9df9b472116b1 Mon Sep 17 00:00:00 2001 From: "D.K" Date: Wed, 16 Dec 2020 16:50:32 -0800 Subject: [PATCH 05/10] Adding Xvfb into list of required packages Signed-off-by: D.K --- docker/bootstrap/Dockerfile.common | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/bootstrap/Dockerfile.common b/docker/bootstrap/Dockerfile.common index 54402e358ef..b7ede777bb6 100644 --- a/docker/bootstrap/Dockerfile.common +++ b/docker/bootstrap/Dockerfile.common @@ -15,6 +15,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins software-properties-common \ unzip \ zip \ + xvfb \ && rm -rf /var/lib/apt/lists/* # Set up Vitess environment (equivalent to '. dev.env') From 1e05bd419763bdb249b77b6fc19c7d550391e6a6 Mon Sep 17 00:00:00 2001 From: "D.K" Date: Wed, 16 Dec 2020 18:17:15 -0800 Subject: [PATCH 06/10] Adding versions bootstraps to the bootstrap Need to build the bootstraps off a bootstrap Signed-off-by: D.K --- docker/bootstrap/Dockerfile.mariadb | 5 ++++- docker/bootstrap/Dockerfile.mariadb103 | 5 ++++- docker/bootstrap/Dockerfile.mysql56 | 5 ++++- docker/bootstrap/Dockerfile.mysql57 | 5 ++++- docker/bootstrap/Dockerfile.mysql57-arm64v8 | 5 ++++- docker/bootstrap/Dockerfile.mysql80 | 5 ++++- docker/bootstrap/Dockerfile.percona | 5 ++++- docker/bootstrap/Dockerfile.percona57 | 5 ++++- docker/bootstrap/Dockerfile.percona80 | 5 ++++- docker/bootstrap/build.sh | 2 +- 10 files changed, 37 insertions(+), 10 deletions(-) diff --git a/docker/bootstrap/Dockerfile.mariadb b/docker/bootstrap/Dockerfile.mariadb index 90d9dd1e201..bc125dd82fb 100644 --- a/docker/bootstrap/Dockerfile.mariadb +++ b/docker/bootstrap/Dockerfile.mariadb @@ -1,4 +1,7 @@ -FROM vitess/bootstrap:common +ARG bootstrap_version +ARG image="vitess/bootstrap:${bootstrap_version}-common" + +FROM "${image}" # Install MariaDB 10 RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --recv-keys 9334A25F8507EFA5 && break; done && \ diff --git a/docker/bootstrap/Dockerfile.mariadb103 b/docker/bootstrap/Dockerfile.mariadb103 index 5d15356f419..a7365d3fe98 100644 --- a/docker/bootstrap/Dockerfile.mariadb103 +++ b/docker/bootstrap/Dockerfile.mariadb103 @@ -1,4 +1,7 @@ -FROM vitess/bootstrap:common +ARG bootstrap_version +ARG image="vitess/bootstrap:${bootstrap_version}-common" + +FROM "${image}" # Install MariaDB 10.3 RUN apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com 0xF1656F24C74CD1D8 \ diff --git a/docker/bootstrap/Dockerfile.mysql56 b/docker/bootstrap/Dockerfile.mysql56 index 46ee7b8553a..9905e5b0999 100644 --- a/docker/bootstrap/Dockerfile.mysql56 +++ b/docker/bootstrap/Dockerfile.mysql56 @@ -1,4 +1,7 @@ -FROM vitess/bootstrap:common +ARG bootstrap_version +ARG image="vitess/bootstrap:${bootstrap_version}-common" + +FROM "${image}" # Install MySQL 5.6 # diff --git a/docker/bootstrap/Dockerfile.mysql57 b/docker/bootstrap/Dockerfile.mysql57 index 4889e74f4e1..3ebae48a4d7 100644 --- a/docker/bootstrap/Dockerfile.mysql57 +++ b/docker/bootstrap/Dockerfile.mysql57 @@ -1,4 +1,7 @@ -FROM vitess/bootstrap:common +ARG bootstrap_version +ARG image="vitess/bootstrap:${bootstrap_version}-common" + +FROM "${image}" # Install MySQL 5.7 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gnupg dirmngr ca-certificates && \ diff --git a/docker/bootstrap/Dockerfile.mysql57-arm64v8 b/docker/bootstrap/Dockerfile.mysql57-arm64v8 index 6687307bdd8..69304140dda 100644 --- a/docker/bootstrap/Dockerfile.mysql57-arm64v8 +++ b/docker/bootstrap/Dockerfile.mysql57-arm64v8 @@ -33,7 +33,10 @@ RUN apt-get update && \ make -j4 && \ make install -FROM vitess/bootstrap:common +ARG bootstrap_version +ARG image="vitess/bootstrap:${bootstrap_version}-common" + +FROM "${image}" # Install MySQL 5.7 RUN add-apt-repository 'deb http://ftp.debian.org/debian sid main' && \ diff --git a/docker/bootstrap/Dockerfile.mysql80 b/docker/bootstrap/Dockerfile.mysql80 index 543453e911a..b2a60cdf5c0 100644 --- a/docker/bootstrap/Dockerfile.mysql80 +++ b/docker/bootstrap/Dockerfile.mysql80 @@ -1,4 +1,7 @@ -FROM vitess/bootstrap:common +ARG bootstrap_version +ARG image="vitess/bootstrap:${bootstrap_version}-common" + +FROM "${image}" # Install MySQL 8.0 RUN for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver ha.pool.sks-keyservers.net 8C718D3B5072E1F5 && break; done && \ diff --git a/docker/bootstrap/Dockerfile.percona b/docker/bootstrap/Dockerfile.percona index d2666c6b137..f91f1aaa3cd 100644 --- a/docker/bootstrap/Dockerfile.percona +++ b/docker/bootstrap/Dockerfile.percona @@ -1,4 +1,7 @@ -FROM vitess/bootstrap:common +ARG bootstrap_version +ARG image="vitess/bootstrap:${bootstrap_version}-common" + +FROM "${image}" # Install Percona 5.6 # diff --git a/docker/bootstrap/Dockerfile.percona57 b/docker/bootstrap/Dockerfile.percona57 index 4bac477d764..f5ff6612fda 100644 --- a/docker/bootstrap/Dockerfile.percona57 +++ b/docker/bootstrap/Dockerfile.percona57 @@ -1,4 +1,7 @@ -FROM vitess/bootstrap:common +ARG bootstrap_version +ARG image="vitess/bootstrap:${bootstrap_version}-common" + +FROM "${image}" # Install Percona 5.7 RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --recv-keys 9334A25F8507EFA5 && break; done && \ diff --git a/docker/bootstrap/Dockerfile.percona80 b/docker/bootstrap/Dockerfile.percona80 index d649416a6b6..2442f3649c9 100644 --- a/docker/bootstrap/Dockerfile.percona80 +++ b/docker/bootstrap/Dockerfile.percona80 @@ -1,4 +1,7 @@ -FROM vitess/bootstrap:common +ARG bootstrap_version +ARG image="vitess/bootstrap:${bootstrap_version}-common" + +FROM "${image}" # Install Percona 8.0 RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --recv-keys 9334A25F8507EFA5 && break; done \ diff --git a/docker/bootstrap/build.sh b/docker/bootstrap/build.sh index 286e5035a08..7d0d558c194 100755 --- a/docker/bootstrap/build.sh +++ b/docker/bootstrap/build.sh @@ -34,5 +34,5 @@ chmod -R o=g * arch=$(uname -m) [ "$arch" == "aarch64" ] && [ $flavor != "common" ] && arch_ext='-arm64v8' if [ -f "docker/bootstrap/Dockerfile.$flavor$arch_ext" ]; then - docker build --no-cache -f docker/bootstrap/Dockerfile.$flavor$arch_ext -t vitess/bootstrap:$version-$flavor$arch_ext . + docker build --no-cache -f docker/bootstrap/Dockerfile.$flavor$arch_ext -t vitess/bootstrap:$version-$flavor$arch_ext --build-arg bootstrap_version=$version . fi From 47460c6086190761e5c5160a7c5f3e8a1a9008ea Mon Sep 17 00:00:00 2001 From: "D.K" Date: Thu, 17 Dec 2020 08:05:07 -0800 Subject: [PATCH 07/10] Updating Chrome Dependendecies Signed-off-by: D.K --- bootstrap.sh | 2 +- docker/bootstrap/Dockerfile.common | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/bootstrap.sh b/bootstrap.sh index ff8f702c0a3..a3c78f73a5d 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -276,7 +276,7 @@ install_all() { # chromedriver if [ "$BUILD_CHROME" == 1 ] ; then - install_dep "chromedriver" "73.0.3683.20" "$VTROOT/dist/chromedriver" install_chromedriver + install_dep "chromedriver" "87.0.4280.88" "$VTROOT/dist/chromedriver" install_chromedriver fi echo diff --git a/docker/bootstrap/Dockerfile.common b/docker/bootstrap/Dockerfile.common index b7ede777bb6..807cd4a26bc 100644 --- a/docker/bootstrap/Dockerfile.common +++ b/docker/bootstrap/Dockerfile.common @@ -16,6 +16,9 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins unzip \ zip \ xvfb \ + && curl -sS -o - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ + && echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list \ + && apt-get -y update && apt-get -y install google-chrome-stable \ && rm -rf /var/lib/apt/lists/* # Set up Vitess environment (equivalent to '. dev.env') From 8ea86e3bbf4da60c6ba173efb55cca9644914bf4 Mon Sep 17 00:00:00 2001 From: "D.K" Date: Thu, 17 Dec 2020 08:28:42 -0800 Subject: [PATCH 08/10] Reverting back to chromium Moving back to chromium and setting chromedriver to match the upstream chromium version in Debian Signed-off-by: D.K --- bootstrap.sh | 2 +- docker/bootstrap/Dockerfile.common | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index a3c78f73a5d..0ec75d442cb 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -276,7 +276,7 @@ install_all() { # chromedriver if [ "$BUILD_CHROME" == 1 ] ; then - install_dep "chromedriver" "87.0.4280.88" "$VTROOT/dist/chromedriver" install_chromedriver + install_dep "chromedriver" "83.0.4103.116" "$VTROOT/dist/chromedriver" install_chromedriver fi echo diff --git a/docker/bootstrap/Dockerfile.common b/docker/bootstrap/Dockerfile.common index 807cd4a26bc..b7ede777bb6 100644 --- a/docker/bootstrap/Dockerfile.common +++ b/docker/bootstrap/Dockerfile.common @@ -16,9 +16,6 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins unzip \ zip \ xvfb \ - && curl -sS -o - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ - && echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list \ - && apt-get -y update && apt-get -y install google-chrome-stable \ && rm -rf /var/lib/apt/lists/* # Set up Vitess environment (equivalent to '. dev.env') From 0c8c74670bbb6aaf6364c7d9317275d275f9cd89 Mon Sep 17 00:00:00 2001 From: "D.K" Date: Thu, 17 Dec 2020 09:43:01 -0800 Subject: [PATCH 09/10] Trying Closest version from Chrome for driver Signed-off-by: D.K --- bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap.sh b/bootstrap.sh index 0ec75d442cb..0c459e51e44 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -276,7 +276,7 @@ install_all() { # chromedriver if [ "$BUILD_CHROME" == 1 ] ; then - install_dep "chromedriver" "83.0.4103.116" "$VTROOT/dist/chromedriver" install_chromedriver + install_dep "chromedriver" "83.0.4103.14" "$VTROOT/dist/chromedriver" install_chromedriver fi echo From a1dffac4295280dceea070d3423cf5f176dae7c4 Mon Sep 17 00:00:00 2001 From: "D.K" Date: Thu, 17 Dec 2020 11:02:09 -0800 Subject: [PATCH 10/10] Adding helper target to increment bootstrap version Signed-off-by: D.K --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 6907f33e16c..c1c0313e050 100644 --- a/Makefile +++ b/Makefile @@ -191,6 +191,10 @@ $(PROTO_GO_OUTS): install_protoc-gen-go proto/*.proto DOCKER_IMAGES_FOR_TEST = mariadb mariadb103 mysql56 mysql57 mysql80 percona percona57 percona80 DOCKER_IMAGES = common $(DOCKER_IMAGES_FOR_TEST) BOOTSTRAP_VERSION=0 +ensure_bootstrap_version: + find docker/ -type f -exec sed -i "s/^\(ARG bootstrap_version\)=.*/\1=${BOOTSTRAP_VERSION}/" {} \; + sed -i 's/\(^.*flag.String(\"bootstrap-version\",\) *\"[^\"]\+\"/\1 \"${BOOTSTRAP_VERSION}\"/' test.go + docker_bootstrap: for i in $(DOCKER_IMAGES); do echo "building bootstrap image: $$i"; docker/bootstrap/build.sh $$i ${BOOTSTRAP_VERSION} || exit 1; done