Merge pull request #137 from tinyspeck/slack-vitess-2019.08.26.r0

Slack vitess 2019.08.26.r0

.travis.yml

@@ -28,7 +28,7 @@ services:
 
 language: go
 go:
-  - 1.11.x
+  - 1.12.x
 go_import_path: vitess.io/vitess
 env:
   global:

bootstrap.sh

@@ -20,24 +20,15 @@
 # 0. Initialization and helper methods.
 # 1. Installation of dependencies.
 # 2. Installation of Go tools and vendored Go dependencies.
-# 3. Detection of installed MySQL and setting MYSQL_FLAVOR.
-# 4. Installation of development related steps e.g. creating Git hooks.
 
 BUILD_TESTS=${BUILD_TESTS:-1}
+BUILD_PYTHON=${BUILD_PYTHON:-1}
+BUILD_JAVA=${BUILD_JAVA:-1}
 
 #
 # 0. Initialization and helper methods.
 #
 
-# Run parallel make, based on number of cores available.
-case $(uname) in
-  Linux)  NB_CORES=$(grep -c '^processor' /proc/cpuinfo);;
-  Darwin) NB_CORES=$(sysctl hw.ncpu | awk '{ print $2 }');;
-esac
-if [ -n "$NB_CORES" ]; then
-  export MAKEFLAGS="-j$((NB_CORES+1)) -l${NB_CORES}"
-fi
-
 function fail() {
   echo "ERROR: $1"
   exit 1
@@ -48,19 +39,27 @@ function fail() {
 go version &>/dev/null  || fail "Go is not installed or is not on \$PATH"
 [[ "$(go version 2>&1)" =~ go1\.[1-9][1-9] ]] || fail "Go is not version 1.11+"
 
+# Create main directories.
+mkdir -p "$VTROOT/dist"
+mkdir -p "$VTROOT/bin"
+mkdir -p "$VTROOT/lib"
+mkdir -p "$VTROOT/vthook"
+
+# Install git hooks.
+echo "creating git hooks"
+mkdir -p "$VTTOP/.git/hooks"
+ln -sf "$VTTOP/misc/git/pre-commit" "$VTTOP/.git/hooks/pre-commit"
+ln -sf "$VTTOP/misc/git/commit-msg" "$VTTOP/.git/hooks/commit-msg"
+(cd "$VTTOP" && git config core.hooksPath "$VTTOP/.git/hooks")
+
+
 # Set up the proper GOPATH for go get below.
 if [ "$BUILD_TESTS" == 1 ] ; then
     source ./dev.env
 else
     source ./build.env
 fi
 
-# Create main directories.
-mkdir -p "$VTROOT/dist"
-mkdir -p "$VTROOT/bin"
-mkdir -p "$VTROOT/lib"
-mkdir -p "$VTROOT/vthook"
-
 if [ "$BUILD_TESTS" == 1 ] ; then
     # Set up required soft links.
     # TODO(mberlin): Which of these can be deleted?
@@ -144,12 +143,13 @@ function install_grpc() {
   PIP=$grpc_virtualenv/bin/pip
   $PIP install --upgrade pip
   $PIP install --upgrade --ignore-installed virtualenv
+  $PIP install mysql-connector-python
 
   grpcio_ver=$version
   $PIP install --upgrade grpcio=="$grpcio_ver" grpcio-tools=="$grpcio_ver"
 }
 
-if [ "$BUILD_TESTS" == 1 ] ; then
+if [ "$BUILD_PYTHON" == 1 ] ; then
     install_dep "gRPC" "1.16.0" "$VTROOT/dist/grpc" install_grpc
 fi
 
@@ -186,9 +186,11 @@ function install_zookeeper() {
   zip -d "lib/$zk-fatjar.jar" 'META-INF/*.SF' 'META-INF/*.RSA' 'META-INF/*SF' || true # needed for >=3.4.10 <3.5
   rm -rf "$zk" "$zk.tar.gz"
 }
-zk_ver=${ZK_VERSION:-3.4.14}
-install_dep "Zookeeper" "$zk_ver" "$VTROOT/dist/vt-zookeeper-$zk_ver" install_zookeeper
 
+zk_ver=${ZK_VERSION:-3.4.14}
+if [ "$BUILD_JAVA" == 1 ] ; then
+  install_dep "Zookeeper" "$zk_ver" "$VTROOT/dist/vt-zookeeper-$zk_ver" install_zookeeper
+fi
 
 # Download and install etcd, link etcd binary into our root.
 function install_etcd() {
@@ -252,7 +254,7 @@ function install_pymock() {
   popd >/dev/null
 }
 pymock_version=1.0.1
-if [ "$BUILD_TESTS" == 1 ] ; then
+if [ "$BUILD_PYTHON" == 1 ] ; then
     install_dep "py-mock" "$pymock_version" "$VTROOT/dist/py-mock-$pymock_version" install_pymock
 fi
 
@@ -267,7 +269,7 @@ function install_selenium() {
   # instead of go/dist/selenium/lib/python3.5/site-packages and then can't find module 'pip._vendor.requests'
   PYTHONPATH='' $PIP install selenium
 }
-if [ "$BUILD_TESTS" == 1 ] ; then
+if [ "$BUILD_PYTHON" == 1 ] ; then
     install_dep "Selenium" "latest" "$VTROOT/dist/selenium" install_selenium
 fi
 
@@ -281,10 +283,13 @@ function install_chromedriver() {
   unzip -o -q chromedriver_linux64.zip -d "$dist"
   rm chromedriver_linux64.zip
 }
-if [ "$BUILD_TESTS" == 1 ] ; then
+if [ "$BUILD_PYTHON" == 1 ] ; then
     install_dep "chromedriver" "73.0.3683.20" "$VTROOT/dist/chromedriver" install_chromedriver
 fi
 
+if [ "$BUILD_PYTHON" == 1 ] ; then
+  PYTHONPATH='' $PIP install mysql-connector-python
+fi
 
 #
 # 2. Installation of Go tools and vendored Go dependencies.
@@ -324,59 +329,5 @@ go get -u $gotools || fail "Failed to download some Go tools with 'go get'. Plea
 echo "Updating govendor dependencies..."
 govendor sync || fail "Failed to download/update dependencies with govendor. Please re-run bootstrap.sh in case of transient errors."
 
-
-#
-# 3. Detection of installed MySQL and setting MYSQL_FLAVOR.
-#
-
-
-# find mysql and prepare to use libmysqlclient
-
-if [ "$BUILD_TESTS" == 1 ] ; then
-  if [ -z "$MYSQL_FLAVOR" ]; then
-    export MYSQL_FLAVOR=MySQL56
-    echo "MYSQL_FLAVOR environment variable not set. Using default: $MYSQL_FLAVOR"
-  fi
-  case "$MYSQL_FLAVOR" in
-    "MySQL56" | "MySQL80")
-      myversion="$("$VT_MYSQL_ROOT/bin/mysql" --version)"
-      [[ "$myversion" =~ Distrib\ 5\.[67] || "$myversion" =~ Ver\ 8\. ]] || fail "Couldn't find MySQL 5.6+ in $VT_MYSQL_ROOT. Set VT_MYSQL_ROOT to override search location."
-      echo "Found MySQL 5.6+ installation in $VT_MYSQL_ROOT."
-      ;;
-
-    "MariaDB" | "MariaDB103")
-      myversion="$("$VT_MYSQL_ROOT/bin/mysql" --version)"
-      [[ "$myversion" =~ MariaDB ]] || fail "Couldn't find MariaDB in $VT_MYSQL_ROOT. Set VT_MYSQL_ROOT to override search location."
-      echo "Found MariaDB installation in $VT_MYSQL_ROOT."
-      ;;
-
-    *)
-      fail "Unsupported MYSQL_FLAVOR $MYSQL_FLAVOR"
-      ;;
-
-  esac
-  # save the flavor that was used in bootstrap, so it can be restored
-  # every time dev.env is sourced.
-  echo "$MYSQL_FLAVOR" > "$VTROOT/dist/MYSQL_FLAVOR"
-fi
-
-#
-# 4. Installation of development related steps e.g. creating Git hooks.
-#
-
-if [ "$BUILD_TESTS" == 1 ] ; then
- # Create the Git hooks.
- echo "creating git hooks"
- mkdir -p "$VTTOP/.git/hooks"
- ln -sf "$VTTOP/misc/git/pre-commit" "$VTTOP/.git/hooks/pre-commit"
- ln -sf "$VTTOP/misc/git/prepare-commit-msg.bugnumber" "$VTTOP/.git/hooks/prepare-commit-msg"
- ln -sf "$VTTOP/misc/git/commit-msg" "$VTTOP/.git/hooks/commit-msg"
- (cd "$VTTOP" && git config core.hooksPath "$VTTOP/.git/hooks")
- echo
- echo "bootstrap finished - run 'source dev.env' in your shell before building."
-else
- echo
- echo "bootstrap finished - run 'source build.env' in your shell before building."
-fi
-
-
+echo
+echo "bootstrap finished - run 'source dev.env' or 'source build.env' in your shell before building."

build.env

@@ -36,4 +36,3 @@ export VTROOT
 if [[ "$VTTOP" == "${VTTOP/\/src\/vitess.io\/vitess/}" ]]; then
   echo "WARNING: VTTOP($VTTOP) does not contain src/vitess.io/vitess"
 fi
-

config/mycnf/master_mariadb.cnf

@@ -1,16 +1,8 @@
+# This file is auto-included when MariaDB (any version) is detected.
+
 # enable strict mode so it's safe to compare sequence numbers across different server IDs.
 gtid_strict_mode = 1
 innodb_stats_persistent = 0
-innodb_support_xa = 0
-
-# Semi-sync replication is required for automated unplanned failover
-# (when the master goes away). Here we just load the plugin so it's
-# available if desired, but it's disabled at startup.
-#
-# If the -enable_semi_sync flag is used, VTTablet will enable semi-sync
-# at the proper time when replication is set up, or when masters are
-# promoted or demoted.
-plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so
 
 # When semi-sync is enabled, don't allow fallback to async
 # if you get no ack, or have no slaves. This is necessary to

config/mycnf/master_mariadb100.cnf

@@ -0,0 +1,12 @@
+# This file is auto-included when MariaDB 10.0 is detected.
+
+innodb_support_xa = 0
+
+# Semi-sync replication is required for automated unplanned failover
+# (when the master goes away). Here we just load the plugin so it's
+# available if desired, but it's disabled at startup.
+#
+# If the -enable_semi_sync flag is used, VTTablet will enable semi-sync
+# at the proper time when replication is set up, or when masters are
+# promoted or demoted.
+plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so

config/mycnf/master_mariadb101.cnf

@@ -0,0 +1,12 @@
+# This file is auto-included when MariaDB 10.1 is detected.
+
+innodb_support_xa = 0
+
+# Semi-sync replication is required for automated unplanned failover
+# (when the master goes away). Here we just load the plugin so it's
+# available if desired, but it's disabled at startup.
+#
+# If the -enable_semi_sync flag is used, VTTablet will enable semi-sync
+# at the proper time when replication is set up, or when masters are
+# promoted or demoted.
+plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so

config/mycnf/master_mariadb102.cnf

@@ -0,0 +1,12 @@
+# This file is auto-included when MariaDB 10.2 is detected.
+
+innodb_support_xa = 0
+
+# Semi-sync replication is required for automated unplanned failover
+# (when the master goes away). Here we just load the plugin so it's
+# available if desired, but it's disabled at startup.
+#
+# If the -enable_semi_sync flag is used, VTTablet will enable semi-sync
+# at the proper time when replication is set up, or when masters are
+# promoted or demoted.
+plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so

config/mycnf/master_mariadb103.cnf

@@ -1,3 +1,5 @@
+# This file is auto-included when MariaDB 10.3 is detected.
+
 # enable strict mode so it's safe to compare sequence numbers across different server IDs.
 gtid_strict_mode = 1
 innodb_stats_persistent = 0

config/mycnf/master_mysql56.cnf

@@ -1,3 +1,5 @@
+# This file is auto-included when MySQL 5.6 is detected.
+
 # Options for enabling GTID
 # https://dev.mysql.com/doc/refman/5.6/en/replication-gtids-howto.html
 gtid_mode = ON

config/mycnf/master_mysql57.cnf

@@ -0,0 +1,32 @@
+# This file is auto-included when MySQL 5.7 is detected.
+
+# Options for enabling GTID
+# https://dev.mysql.com/doc/refman/5.6/en/replication-gtids-howto.html
+gtid_mode = ON
+log_bin
+log_slave_updates
+enforce_gtid_consistency
+innodb_use_native_aio = 0
+
+# Crash-safe replication settings.
+master_info_repository = TABLE
+relay_log_info_repository = TABLE
+relay_log_purge = 1
+relay_log_recovery = 1
+
+# Semi-sync replication is required for automated unplanned failover
+# (when the master goes away). Here we just load the plugin so it's
+# available if desired, but it's disabled at startup.
+#
+# If the -enable_semi_sync flag is used, VTTablet will enable semi-sync
+# at the proper time when replication is set up, or when masters are
+# promoted or demoted.
+plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisync_slave.so
+
+# When semi-sync is enabled, don't allow fallback to async
+# if you get no ack, or have no slaves. This is necessary to
+# prevent alternate futures when doing a failover in response to
+# a master that becomes unresponsive.
+rpl_semi_sync_master_timeout = 1000000000000000000
+rpl_semi_sync_master_wait_no_slave = 1
+

config/mycnf/master_mysql80.cnf

@@ -1,3 +1,5 @@
+# This file is auto-included when MySQL 8.0 is detected.
+
 # Options for enabling GTID
 # https://dev.mysql.com/doc/refman/5.6/en/replication-gtids-howto.html
 gtid_mode = ON

dev.env

@@ -95,16 +95,6 @@ if [[ "$VT_MYSQL_ROOT" == "" ]]; then
   fi
 fi
 
-# restore MYSQL_FLAVOR, saved by bootstrap.sh
-if [ -r "$VTROOT/dist/MYSQL_FLAVOR" ]; then
-  MYSQL_FLAVOR=$(cat "$VTROOT/dist/MYSQL_FLAVOR")
-  export MYSQL_FLAVOR
-fi
-
-# mysql cgo library config
-if [ -z "$MYSQL_FLAVOR" ]; then
-  export MYSQL_FLAVOR=MariaDB
-fi
 PKG_CONFIG_PATH=$(prepend_path "$PKG_CONFIG_PATH" "$VTROOT/lib")
 export PKG_CONFIG_PATH