diff --git a/Makefile b/Makefile index e032ff1e8e7..c1c0313e050 100644 --- a/Makefile +++ b/Makefile @@ -190,119 +190,57 @@ $(PROTO_GO_OUTS): install_protoc-gen-go proto/*.proto # This rule builds the bootstrap images for all flavors. DOCKER_IMAGES_FOR_TEST = mariadb mariadb103 mysql56 mysql57 mysql80 percona percona57 percona80 DOCKER_IMAGES = common $(DOCKER_IMAGES_FOR_TEST) +BOOTSTRAP_VERSION=0 +ensure_bootstrap_version: + find docker/ -type f -exec sed -i "s/^\(ARG bootstrap_version\)=.*/\1=${BOOTSTRAP_VERSION}/" {} \; + sed -i 's/\(^.*flag.String(\"bootstrap-version\",\) *\"[^\"]\+\"/\1 \"${BOOTSTRAP_VERSION}\"/' test.go + docker_bootstrap: - for i in $(DOCKER_IMAGES); do echo "building bootstrap image: $$i"; docker/bootstrap/build.sh $$i || exit 1; done + for i in $(DOCKER_IMAGES); do echo "building bootstrap image: $$i"; docker/bootstrap/build.sh $$i ${BOOTSTRAP_VERSION} || exit 1; done docker_bootstrap_test: - flavors='$(DOCKER_IMAGES_FOR_TEST)' && ./test.go -pull=false -parallel=2 -flavor=$${flavors// /,} + flavors='$(DOCKER_IMAGES_FOR_TEST)' && ./test.go -pull=false -parallel=2 -bootstrap-version=${BOOTSTRAP_VERSION} -flavor=$${flavors// /,} docker_bootstrap_push: - for i in $(DOCKER_IMAGES); do echo "pushing bootstrap image: $$i"; docker push vitess/bootstrap:$$i || exit 1; done + for i in $(DOCKER_IMAGES); do echo "pushing bootstrap image: ${BOOTSTRAP_VERSION}-$$i"; docker push vitess/bootstrap:${BOOTSTRAP_VERSION}-$$i || exit 1; done # Use this target to update the local copy of your images with the one on Dockerhub. docker_bootstrap_pull: - for i in $(DOCKER_IMAGES); do echo "pulling bootstrap image: $$i"; docker pull vitess/bootstrap:$$i || exit 1; done - -docker_base: - # Fix permissions before copying files, to avoid AUFS bug. - chmod -R o=g * - docker build -f docker/base/Dockerfile -t vitess/base . + for i in $(DOCKER_IMAGES); do echo "pulling bootstrap image: $$i"; docker pull vitess/bootstrap:${BOOTSTRAP_VERSION}-$$i || exit 1; done -docker_base_mysql56: - chmod -R o=g * - docker build -f docker/base/Dockerfile.mysql56 -t vitess/base:mysql56 . -docker_base_mysql80: - chmod -R o=g * - docker build -f docker/base/Dockerfile.mysql80 -t vitess/base:mysql80 . - -docker_base_mariadb: - chmod -R o=g * - docker build -f docker/base/Dockerfile.mariadb -t vitess/base:mariadb . - -docker_base_mariadb103: - chmod -R o=g * - docker build -f docker/base/Dockerfile.mariadb -t vitess/base:mariadb103 . +define build_docker_image + # Fix permissions before copying files, to avoid AUFS bug. + ${info Building ${2}} + chmod -R o=g *; + docker build -f ${1} -t ${2} --build-arg bootstrap_version=${BOOTSTRAP_VERSION} .; +endef -docker_base_percona: - chmod -R o=g * - docker build -f docker/base/Dockerfile.percona -t vitess/base:percona . +docker_base: + ${call build_docker_image,docker/base/Dockerfile,vitess/base} -docker_base_percona57: - chmod -R o=g * - docker build -f docker/base/Dockerfile.percona57 -t vitess/base:percona57 . +DOCKER_BASE_SUFFIX = mysql56 mysql80 mariadb mariadb103 percona percona57 percona80 +DOCKER_BASE_TARGETS = $(addprefix docker_base_, $(DOCKER_BASE_SUFFIX)) +$(DOCKER_BASE_TARGETS): docker_base_%: + ${call build_docker_image,docker/base/Dockerfile.$*,vitess/base:$*} -docker_base_percona80: - chmod -R o=g * - docker build -f docker/base/Dockerfile.percona80 -t vitess/base:percona80 . +docker_base_all: docker_base $(DOCKER_BASE_TARGETS) docker_lite: - chmod -R o=g * - docker build -f docker/lite/Dockerfile -t vitess/lite . - -docker_lite_mysql56: - chmod -R o=g * - docker build -f docker/lite/Dockerfile.mysql56 -t vitess/lite:mysql56 . - -docker_lite_mysql57: - chmod -R o=g * - docker build -f docker/lite/Dockerfile.mysql57 -t vitess/lite:mysql57 . - -docker_lite_ubi7.mysql57: - chmod -R o=g * - docker build -f docker/lite/Dockerfile.ubi7.mysql57 -t vitess/lite:ubi7.mysql57 . - -docker_lite_mysql80: - chmod -R o=g * - docker build -f docker/lite/Dockerfile.mysql80 -t vitess/lite:mysql80 . - -docker_lite_ubi7.mysql80: - chmod -R o=g * - docker build -f docker/lite/Dockerfile.ubi7.mysql80 -t vitess/lite:ubi7.mysql80 . + ${call build_docker_image,docker/lite/Dockerfile,vitess/lite} -docker_lite_mariadb: - chmod -R o=g * - docker build -f docker/lite/Dockerfile.mariadb -t vitess/lite:mariadb . +DOCKER_LITE_SUFFIX = mysql56 mysql57 ubi7.mysql57 mysql80 ubi7.mysql80 mariadb mariadb103 percona percona57 ubi7.percona57 percona80 ubi7.percona80 alpine testing +DOCKER_LITE_TARGETS = $(addprefix docker_lite_,$(DOCKER_LITE_SUFFIX)) +$(DOCKER_LITE_TARGETS): docker_lite_%: + ${call build_docker_image,docker/lite/Dockerfile.$*,vitess/lite:$*} -docker_lite_mariadb103: - chmod -R o=g * - docker build -f docker/lite/Dockerfile.mariadb103 -t vitess/lite:mariadb103 . - -docker_lite_percona: - chmod -R o=g * - docker build -f docker/lite/Dockerfile.percona -t vitess/lite:percona . - -docker_lite_percona57: - chmod -R o=g * - docker build -f docker/lite/Dockerfile.percona57 -t vitess/lite:percona57 . - -docker_lite_ubi7.percona57: - chmod -R o=g * - docker build -f docker/lite/Dockerfile.ubi7.percona57 -t vitess/lite:ubi7.percona57 . - -docker_lite_percona80: - chmod -R o=g * - docker build -f docker/lite/Dockerfile.percona80 -t vitess/lite:percona80 . - -docker_lite_ubi7.percona80: - chmod -R o=g * - docker build -f docker/lite/Dockerfile.ubi7.percona80 -t vitess/lite:ubi7.percona80 . - -docker_lite_alpine: - chmod -R o=g * - docker build -f docker/lite/Dockerfile.alpine -t vitess/lite:alpine . - -docker_lite_testing: - chmod -R o=g * - docker build -f docker/lite/Dockerfile.testing -t vitess/lite:testing . +docker_lite_all: docker_lite $(DOCKER_LITE_TARGETS) docker_local: - chmod -R o=g * - docker build -f docker/local/Dockerfile -t vitess/local . + ${call build_docker_image,docker/local/Dockerfile,vitess/local} docker_mini: - chmod -R o=g * - docker build -f docker/mini/Dockerfile -t vitess/mini . + ${call build_docker_image,docker/mini/Dockerfile,vitess/mini} # This rule loads the working copy of the code into a bootstrap image, # and then runs the tests inside Docker. @@ -327,15 +265,6 @@ release: docker_base echo "git push origin v$(VERSION)" echo "Also, don't forget the upload releases/v$(VERSION).tar.gz file to GitHub releases" -packages: docker_base - @if [ -z "$VERSION" ]; then \ - echo "Set the env var VERSION with the release version"; exit 1;\ - fi - mkdir -p releases - docker build -f docker/packaging/Dockerfile -t vitess/packaging . - docker run --rm -v ${PWD}/releases:/vt/releases --env VERSION=$(VERSION) vitess/packaging --package /vt/releases -t deb --deb-no-default-config-files - docker run --rm -v ${PWD}/releases:/vt/releases --env VERSION=$(VERSION) vitess/packaging --package /vt/releases -t rpm - tools: echo $$(date): Installing dependencies ./bootstrap.sh diff --git a/bootstrap.sh b/bootstrap.sh index ff8f702c0a3..0c459e51e44 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -276,7 +276,7 @@ install_all() { # chromedriver if [ "$BUILD_CHROME" == 1 ] ; then - install_dep "chromedriver" "73.0.3683.20" "$VTROOT/dist/chromedriver" install_chromedriver + install_dep "chromedriver" "83.0.4103.14" "$VTROOT/dist/chromedriver" install_chromedriver fi echo diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile index bc81925d79b..a00b380b946 100644 --- a/docker/base/Dockerfile +++ b/docker/base/Dockerfile @@ -20,7 +20,11 @@ # therefore we need to have the symlinked "Dockerfile" in there as well. # TODO(mberlin): Remove the symlink and this note once # https://github.com/docker/hub-feedback/issues/292 is fixed. -FROM vitess/bootstrap:mysql57 + +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-mysql57" + +FROM "${image}" # Allows some docker builds to disable CGO ARG CGO_ENABLED=0 diff --git a/docker/base/Dockerfile.mariadb b/docker/base/Dockerfile.mariadb index 42e09d1a9d2..9f11742546a 100644 --- a/docker/base/Dockerfile.mariadb +++ b/docker/base/Dockerfile.mariadb @@ -1,4 +1,7 @@ -FROM vitess/bootstrap:mariadb +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-mariadb" + +FROM "${image}" # Allows some docker builds to disable CGO ARG CGO_ENABLED=0 diff --git a/docker/base/Dockerfile.mariadb103 b/docker/base/Dockerfile.mariadb103 index cc86d5aede5..8b15baec736 100644 --- a/docker/base/Dockerfile.mariadb103 +++ b/docker/base/Dockerfile.mariadb103 @@ -1,4 +1,7 @@ -FROM vitess/bootstrap:mariadb103 +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-mariadb103" + +FROM "${image}" # Allows some docker builds to disable CGO ARG CGO_ENABLED=0 diff --git a/docker/base/Dockerfile.mysql56 b/docker/base/Dockerfile.mysql56 index efdbade8ced..2d49c861562 100644 --- a/docker/base/Dockerfile.mysql56 +++ b/docker/base/Dockerfile.mysql56 @@ -1,4 +1,7 @@ -FROM vitess/bootstrap:mysql56 +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-mysql56" + +FROM "${image}" # Allows some docker builds to disable CGO ARG CGO_ENABLED=0 diff --git a/docker/base/Dockerfile.mysql80 b/docker/base/Dockerfile.mysql80 index 4a798700060..64f87ef90ff 100644 --- a/docker/base/Dockerfile.mysql80 +++ b/docker/base/Dockerfile.mysql80 @@ -1,4 +1,7 @@ -FROM vitess/bootstrap:mysql80 +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-mysql80" + +FROM "${image}" # Allows some docker builds to disable CGO ARG CGO_ENABLED=0 diff --git a/docker/base/Dockerfile.percona b/docker/base/Dockerfile.percona index 5b488680126..e1361c8a147 100644 --- a/docker/base/Dockerfile.percona +++ b/docker/base/Dockerfile.percona @@ -1,4 +1,7 @@ -FROM vitess/bootstrap:percona +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-percona" + +FROM "${image}" # Allows some docker builds to disable CGO ARG CGO_ENABLED=0 diff --git a/docker/base/Dockerfile.percona57 b/docker/base/Dockerfile.percona57 index 4a3aa6b0c7c..bc7071de851 100644 --- a/docker/base/Dockerfile.percona57 +++ b/docker/base/Dockerfile.percona57 @@ -1,4 +1,7 @@ -FROM vitess/bootstrap:percona57 +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-percona57" + +FROM "${image}" # Allows some docker builds to disable CGO ARG CGO_ENABLED=0 diff --git a/docker/base/Dockerfile.percona80 b/docker/base/Dockerfile.percona80 index f5997a16e1a..f6bc311ce72 100644 --- a/docker/base/Dockerfile.percona80 +++ b/docker/base/Dockerfile.percona80 @@ -1,4 +1,7 @@ -FROM vitess/bootstrap:percona80 +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-percona80" + +FROM "${image}" # Allows some docker builds to disable CGO ARG CGO_ENABLED=0 diff --git a/docker/bootstrap/Dockerfile.common b/docker/bootstrap/Dockerfile.common index 54402e358ef..b7ede777bb6 100644 --- a/docker/bootstrap/Dockerfile.common +++ b/docker/bootstrap/Dockerfile.common @@ -15,6 +15,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins software-properties-common \ unzip \ zip \ + xvfb \ && rm -rf /var/lib/apt/lists/* # Set up Vitess environment (equivalent to '. dev.env') diff --git a/docker/bootstrap/Dockerfile.mariadb b/docker/bootstrap/Dockerfile.mariadb index 90d9dd1e201..bc125dd82fb 100644 --- a/docker/bootstrap/Dockerfile.mariadb +++ b/docker/bootstrap/Dockerfile.mariadb @@ -1,4 +1,7 @@ -FROM vitess/bootstrap:common +ARG bootstrap_version +ARG image="vitess/bootstrap:${bootstrap_version}-common" + +FROM "${image}" # Install MariaDB 10 RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --recv-keys 9334A25F8507EFA5 && break; done && \ diff --git a/docker/bootstrap/Dockerfile.mariadb103 b/docker/bootstrap/Dockerfile.mariadb103 index 5d15356f419..a7365d3fe98 100644 --- a/docker/bootstrap/Dockerfile.mariadb103 +++ b/docker/bootstrap/Dockerfile.mariadb103 @@ -1,4 +1,7 @@ -FROM vitess/bootstrap:common +ARG bootstrap_version +ARG image="vitess/bootstrap:${bootstrap_version}-common" + +FROM "${image}" # Install MariaDB 10.3 RUN apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com 0xF1656F24C74CD1D8 \ diff --git a/docker/bootstrap/Dockerfile.mysql56 b/docker/bootstrap/Dockerfile.mysql56 index 46ee7b8553a..9905e5b0999 100644 --- a/docker/bootstrap/Dockerfile.mysql56 +++ b/docker/bootstrap/Dockerfile.mysql56 @@ -1,4 +1,7 @@ -FROM vitess/bootstrap:common +ARG bootstrap_version +ARG image="vitess/bootstrap:${bootstrap_version}-common" + +FROM "${image}" # Install MySQL 5.6 # diff --git a/docker/bootstrap/Dockerfile.mysql57 b/docker/bootstrap/Dockerfile.mysql57 index 4889e74f4e1..3ebae48a4d7 100644 --- a/docker/bootstrap/Dockerfile.mysql57 +++ b/docker/bootstrap/Dockerfile.mysql57 @@ -1,4 +1,7 @@ -FROM vitess/bootstrap:common +ARG bootstrap_version +ARG image="vitess/bootstrap:${bootstrap_version}-common" + +FROM "${image}" # Install MySQL 5.7 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gnupg dirmngr ca-certificates && \ diff --git a/docker/bootstrap/Dockerfile.mysql57-arm64v8 b/docker/bootstrap/Dockerfile.mysql57-arm64v8 index 6687307bdd8..69304140dda 100644 --- a/docker/bootstrap/Dockerfile.mysql57-arm64v8 +++ b/docker/bootstrap/Dockerfile.mysql57-arm64v8 @@ -33,7 +33,10 @@ RUN apt-get update && \ make -j4 && \ make install -FROM vitess/bootstrap:common +ARG bootstrap_version +ARG image="vitess/bootstrap:${bootstrap_version}-common" + +FROM "${image}" # Install MySQL 5.7 RUN add-apt-repository 'deb http://ftp.debian.org/debian sid main' && \ diff --git a/docker/bootstrap/Dockerfile.mysql80 b/docker/bootstrap/Dockerfile.mysql80 index 543453e911a..b2a60cdf5c0 100644 --- a/docker/bootstrap/Dockerfile.mysql80 +++ b/docker/bootstrap/Dockerfile.mysql80 @@ -1,4 +1,7 @@ -FROM vitess/bootstrap:common +ARG bootstrap_version +ARG image="vitess/bootstrap:${bootstrap_version}-common" + +FROM "${image}" # Install MySQL 8.0 RUN for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver ha.pool.sks-keyservers.net 8C718D3B5072E1F5 && break; done && \ diff --git a/docker/bootstrap/Dockerfile.percona b/docker/bootstrap/Dockerfile.percona index d2666c6b137..f91f1aaa3cd 100644 --- a/docker/bootstrap/Dockerfile.percona +++ b/docker/bootstrap/Dockerfile.percona @@ -1,4 +1,7 @@ -FROM vitess/bootstrap:common +ARG bootstrap_version +ARG image="vitess/bootstrap:${bootstrap_version}-common" + +FROM "${image}" # Install Percona 5.6 # diff --git a/docker/bootstrap/Dockerfile.percona57 b/docker/bootstrap/Dockerfile.percona57 index 4bac477d764..f5ff6612fda 100644 --- a/docker/bootstrap/Dockerfile.percona57 +++ b/docker/bootstrap/Dockerfile.percona57 @@ -1,4 +1,7 @@ -FROM vitess/bootstrap:common +ARG bootstrap_version +ARG image="vitess/bootstrap:${bootstrap_version}-common" + +FROM "${image}" # Install Percona 5.7 RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --recv-keys 9334A25F8507EFA5 && break; done && \ diff --git a/docker/bootstrap/Dockerfile.percona80 b/docker/bootstrap/Dockerfile.percona80 index d649416a6b6..2442f3649c9 100644 --- a/docker/bootstrap/Dockerfile.percona80 +++ b/docker/bootstrap/Dockerfile.percona80 @@ -1,4 +1,7 @@ -FROM vitess/bootstrap:common +ARG bootstrap_version +ARG image="vitess/bootstrap:${bootstrap_version}-common" + +FROM "${image}" # Install Percona 8.0 RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --recv-keys 9334A25F8507EFA5 && break; done \ diff --git a/docker/bootstrap/build.sh b/docker/bootstrap/build.sh index 699f0627618..7d0d558c194 100755 --- a/docker/bootstrap/build.sh +++ b/docker/bootstrap/build.sh @@ -15,12 +15,14 @@ # limitations under the License. flavor=$1 - if [[ -z "$flavor" ]]; then echo "Flavor must be specified as first argument." exit 1 fi +# Set default version of 0 +version="${2:-0}" + if [[ ! -f bootstrap.sh ]]; then echo "This script should be run from the root of the Vitess source tree - e.g. ~/src/vitess.io/vitess" exit 1 @@ -32,5 +34,5 @@ chmod -R o=g * arch=$(uname -m) [ "$arch" == "aarch64" ] && [ $flavor != "common" ] && arch_ext='-arm64v8' if [ -f "docker/bootstrap/Dockerfile.$flavor$arch_ext" ]; then - docker build --no-cache -f docker/bootstrap/Dockerfile.$flavor$arch_ext -t vitess/bootstrap:$flavor$arch_ext . + docker build --no-cache -f docker/bootstrap/Dockerfile.$flavor$arch_ext -t vitess/bootstrap:$version-$flavor$arch_ext --build-arg bootstrap_version=$version . fi diff --git a/docker/lite/Dockerfile.alpine b/docker/lite/Dockerfile.alpine index aac7caba428..1c7912b7029 100644 --- a/docker/lite/Dockerfile.alpine +++ b/docker/lite/Dockerfile.alpine @@ -17,7 +17,10 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -FROM vitess/bootstrap:mariadb103 AS builder +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-mariadb103" + +FROM "${image}" AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER diff --git a/docker/lite/Dockerfile.mariadb b/docker/lite/Dockerfile.mariadb index 1617d899fe2..17f51e79366 100644 --- a/docker/lite/Dockerfile.mariadb +++ b/docker/lite/Dockerfile.mariadb @@ -17,7 +17,10 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -FROM vitess/bootstrap:mariadb AS builder +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-mariadb" + +FROM "${image}" AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER diff --git a/docker/lite/Dockerfile.mariadb103 b/docker/lite/Dockerfile.mariadb103 index 079ab6ff4a7..afa02b1c22c 100644 --- a/docker/lite/Dockerfile.mariadb103 +++ b/docker/lite/Dockerfile.mariadb103 @@ -17,7 +17,10 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -FROM vitess/bootstrap:mariadb103 AS builder +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-mariadb103" + +FROM "${image}" AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER diff --git a/docker/lite/Dockerfile.mysql56 b/docker/lite/Dockerfile.mysql56 index ec361bec520..3ae98ce0c35 100644 --- a/docker/lite/Dockerfile.mysql56 +++ b/docker/lite/Dockerfile.mysql56 @@ -17,7 +17,10 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -FROM vitess/bootstrap:mysql56 AS builder +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-mysql56" + +FROM "${image}" AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER diff --git a/docker/lite/Dockerfile.mysql57 b/docker/lite/Dockerfile.mysql57 index 5ff2d21514d..5f844c2c8fc 100644 --- a/docker/lite/Dockerfile.mysql57 +++ b/docker/lite/Dockerfile.mysql57 @@ -17,7 +17,10 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -FROM vitess/bootstrap:mysql57 AS builder +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-mysql57" + +FROM "${image}" AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER diff --git a/docker/lite/Dockerfile.mysql80 b/docker/lite/Dockerfile.mysql80 index c395d24f37a..ab91b9010a9 100644 --- a/docker/lite/Dockerfile.mysql80 +++ b/docker/lite/Dockerfile.mysql80 @@ -17,7 +17,10 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -FROM vitess/bootstrap:mysql80 AS builder +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-mysql80" + +FROM "${image}" AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER diff --git a/docker/lite/Dockerfile.percona b/docker/lite/Dockerfile.percona index e0c5eac8a78..5e315c791d3 100644 --- a/docker/lite/Dockerfile.percona +++ b/docker/lite/Dockerfile.percona @@ -17,7 +17,10 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -FROM vitess/bootstrap:percona AS builder +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-percona" + +FROM "${image}" AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER diff --git a/docker/lite/Dockerfile.percona57 b/docker/lite/Dockerfile.percona57 index e8cc430475c..be333d9441d 100644 --- a/docker/lite/Dockerfile.percona57 +++ b/docker/lite/Dockerfile.percona57 @@ -17,7 +17,10 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -FROM vitess/bootstrap:percona57 AS builder +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-percona57" + +FROM "${image}" AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER diff --git a/docker/lite/Dockerfile.percona80 b/docker/lite/Dockerfile.percona80 index 0bb057d6ed2..7388db5063b 100644 --- a/docker/lite/Dockerfile.percona80 +++ b/docker/lite/Dockerfile.percona80 @@ -17,7 +17,10 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -FROM vitess/bootstrap:percona80 AS builder +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-percona80" + +FROM "${image}" AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER diff --git a/docker/lite/Dockerfile.testing b/docker/lite/Dockerfile.testing index 523d50e3c77..948c04f3d69 100644 --- a/docker/lite/Dockerfile.testing +++ b/docker/lite/Dockerfile.testing @@ -17,7 +17,10 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -FROM vitess/bootstrap:mysql57 AS builder +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-mysql57" + +FROM "${image}" AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER diff --git a/docker/lite/Dockerfile.ubi7.mysql57 b/docker/lite/Dockerfile.ubi7.mysql57 index b96397f91b3..c5f41eb4497 100644 --- a/docker/lite/Dockerfile.ubi7.mysql57 +++ b/docker/lite/Dockerfile.ubi7.mysql57 @@ -17,7 +17,10 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -FROM vitess/bootstrap:mysql57 AS builder +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-mysql57" + +FROM "${image}" AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER diff --git a/docker/lite/Dockerfile.ubi7.mysql80 b/docker/lite/Dockerfile.ubi7.mysql80 index 0eefe70573c..0052a60afc8 100644 --- a/docker/lite/Dockerfile.ubi7.mysql80 +++ b/docker/lite/Dockerfile.ubi7.mysql80 @@ -17,7 +17,10 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -FROM vitess/bootstrap:mysql80 AS builder +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-mysql80" + +FROM "${image}" AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER diff --git a/docker/lite/Dockerfile.ubi7.percona57 b/docker/lite/Dockerfile.ubi7.percona57 index c3e0f2c074e..4feedde76ba 100644 --- a/docker/lite/Dockerfile.ubi7.percona57 +++ b/docker/lite/Dockerfile.ubi7.percona57 @@ -17,7 +17,10 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -FROM vitess/bootstrap:percona57 AS builder +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-percona57" + +FROM "${image}" AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER diff --git a/docker/lite/Dockerfile.ubi7.percona80 b/docker/lite/Dockerfile.ubi7.percona80 index 557e7a34555..e163bc62384 100644 --- a/docker/lite/Dockerfile.ubi7.percona80 +++ b/docker/lite/Dockerfile.ubi7.percona80 @@ -17,7 +17,10 @@ # ensure images contain the right binaries. # Use a temporary layer for the build stage. -FROM vitess/bootstrap:percona80 AS builder +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-percona80" + +FROM "${image}" AS builder # Allows docker builds to set the BUILD_NUMBER ARG BUILD_NUMBER diff --git a/docker/local/Dockerfile b/docker/local/Dockerfile index a4e76e1a7d1..01b62ec2836 100644 --- a/docker/local/Dockerfile +++ b/docker/local/Dockerfile @@ -1,4 +1,7 @@ -FROM vitess/bootstrap:common +ARG bootstrap_version=0 +ARG image="vitess/bootstrap:${bootstrap_version}-common" + +FROM "${image}" RUN apt-get update RUN apt-get install -y sudo curl vim jq mysql-server diff --git a/docker/test/run.sh b/docker/test/run.sh index 842c4512b1b..29d853c06ae 100755 --- a/docker/test/run.sh +++ b/docker/test/run.sh @@ -98,7 +98,8 @@ while true ; do done # Positional flags. flavor=$1 -cmd=$2 +version=${2:-0} +cmd=$3 args= if [[ -z "$flavor" ]]; then @@ -115,7 +116,7 @@ if [[ ! -f bootstrap.sh ]]; then exit 1 fi -image=vitess/bootstrap:$flavor +image=vitess/bootstrap:$version-$flavor if [[ -n "$existing_cache_image" ]]; then image=$existing_cache_image fi diff --git a/test.go b/test.go index 284e747078a..b496b7dab88 100755 --- a/test.go +++ b/test.go @@ -74,21 +74,22 @@ For example: // Flags var ( - flavor = flag.String("flavor", "mysql57", "comma-separated bootstrap flavor(s) to run against (when using Docker mode). Available flavors: all,"+flavors) - runCount = flag.Int("runs", 1, "run each test this many times") - retryMax = flag.Int("retry", 3, "max number of retries, to detect flaky tests") - logPass = flag.Bool("log-pass", false, "log test output even if it passes") - timeout = flag.Duration("timeout", 10*time.Minute, "timeout for each test") - pull = flag.Bool("pull", true, "re-pull the bootstrap image, in case it's been updated") - docker = flag.Bool("docker", true, "run tests with Docker") - useDockerCache = flag.Bool("use_docker_cache", false, "if true, create a temporary Docker image to cache the source code and the binaries generated by 'make build'. Used for execution on Travis CI.") - shard = flag.Int("shard", -1, "if N>=0, run the tests whose Shard field matches N") - tag = flag.String("tag", "", "if provided, only run tests with the given tag. Can't be combined with -shard or explicit test list") - exclude = flag.String("exclude", "", "if provided, exclude tests containing any of the given tags (comma delimited)") - keepData = flag.Bool("keep-data", false, "don't delete the per-test VTDATAROOT subfolders") - printLog = flag.Bool("print-log", false, "print the log of each failed test (or all tests if -log-pass) to the console") - follow = flag.Bool("follow", false, "print test output as it runs, instead of waiting to see if it passes or fails") - parallel = flag.Int("parallel", 1, "number of tests to run in parallel") + flavor = flag.String("flavor", "mysql57", "comma-separated bootstrap flavor(s) to run against (when using Docker mode). Available flavors: all,"+flavors) + bootstrapVersion = flag.String("bootstrap-version", "0", "the version identifier to use for the docker images") + runCount = flag.Int("runs", 1, "run each test this many times") + retryMax = flag.Int("retry", 3, "max number of retries, to detect flaky tests") + logPass = flag.Bool("log-pass", false, "log test output even if it passes") + timeout = flag.Duration("timeout", 10*time.Minute, "timeout for each test") + pull = flag.Bool("pull", true, "re-pull the bootstrap image, in case it's been updated") + docker = flag.Bool("docker", true, "run tests with Docker") + useDockerCache = flag.Bool("use_docker_cache", false, "if true, create a temporary Docker image to cache the source code and the binaries generated by 'make build'. Used for execution on Travis CI.") + shard = flag.Int("shard", -1, "if N>=0, run the tests whose Shard field matches N") + tag = flag.String("tag", "", "if provided, only run tests with the given tag. Can't be combined with -shard or explicit test list") + exclude = flag.String("exclude", "", "if provided, exclude tests containing any of the given tags (comma delimited)") + keepData = flag.Bool("keep-data", false, "don't delete the per-test VTDATAROOT subfolders") + printLog = flag.Bool("print-log", false, "print the log of each failed test (or all tests if -log-pass) to the console") + follow = flag.Bool("follow", false, "print test output as it runs, instead of waiting to see if it passes or fails") + parallel = flag.Int("parallel", 1, "number of tests to run in parallel") remoteStats = flag.String("remote-stats", "", "url to send remote stats") ) @@ -130,9 +131,10 @@ type Test struct { // Tags is a list of tags that can be used to filter tests. Tags []string - name string - flavor string - runIndex int + name string + flavor string + bootstrapVersion string + runIndex int pass, fail int } @@ -186,7 +188,7 @@ func (t *Test) run(dir, dataDir string) ([]byte, error) { args = []string{"--use_docker_cache", cacheImage(t.flavor), t.flavor, testArgs} } else { // If there is no cache, we have to call 'make build' before each test. - args = []string{t.flavor, "make build && " + testArgs} + args = []string{t.flavor, t.bootstrapVersion, "make build && " + testArgs} } cmd = exec.Command(path.Join(dir, "docker/test/run.sh"), args...) @@ -311,7 +313,7 @@ func main() { wg.Add(1) go func(flavor string) { defer wg.Done() - image := "vitess/bootstrap:" + flavor + image := "vitess/bootstrap:" + *bootstrapVersion + "-" + flavor pullTime := time.Now() log.Printf("Pulling %v...", image) cmd := exec.Command("docker", "pull", image) @@ -354,6 +356,7 @@ func main() { for _, t := range tests { test := *t test.flavor = flavor + test.bootstrapVersion = *bootstrapVersion dup = append(dup, &test) } } @@ -389,7 +392,7 @@ func main() { for _, flavor := range flavors { start := time.Now() log.Printf("Creating Docker cache image for flavor '%s'...", flavor) - if out, err := exec.Command("docker/test/run.sh", "--create_docker_cache", cacheImage(flavor), flavor, "make build").CombinedOutput(); err != nil { + if out, err := exec.Command("docker/test/run.sh", "--create_docker_cache", cacheImage(flavor), flavor, *bootstrapVersion, "make build").CombinedOutput(); err != nil { log.Fatalf("Failed to create Docker cache image for flavor '%s': %v\n%s", flavor, err, out) } log.Printf("Creating Docker cache image took %v", round(time.Since(start)))