Local example 101: idempotent on existing clusters (#13373)

* local example 101: idempotent Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * grep --quiet Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * failing if waiting for super_read_only=false times out Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --------- Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>
vitessio · Jun 27, 2023 · aa0fdff · aa0fdff
1 parent d4c5096
commit aa0fdff
Show file tree

Hide file tree

Showing 3 changed files with 50 additions and 9 deletions.
diff --git a/examples/common/lib/utils.sh b/examples/common/lib/utils.sh
@@ -38,9 +38,9 @@ function wait_for_shard_tablets() {
  done;
 
  cur_tablets=$(vtctldclient GetTablets --keyspace "${keyspace}" --shard "${shard}" | wc -l)
- if [[ ${cur_tablets} -lt ${num_tablets} ]]; then
+ if [[ ${cur_tablets} -lt ${num_tablets} ]]; then
  fail "Timed out after ${wait_secs} seconds waiting for tablets to come up in ${keyspace}/${shard}"
- fi
+ fi
 }
 
 # Wait for a primary tablet to be elected and become healthy and serving
@@ -56,17 +56,43 @@ function wait_for_healthy_shard_primary() {
  local wait_secs=180
 
  for _ in $(seq 1 ${wait_secs}); do
- if ! vtctldclient --server=localhost:15999 GetShard "${keyspace}/${shard}" | grep -qi "${unhealthy_indicator}"; then
+ if ! vtctldclient --server=localhost:15999 GetShard "${keyspace}/${shard}" | grep -qi "${unhealthy_indicator}"; then
  break
  fi
  sleep 1
  done;
 
- if vtctldclient --server=localhost:15999 GetShard "${keyspace}/${shard}" | grep -qi "${unhealthy_indicator}"; then
+ if vtctldclient --server=localhost:15999 GetShard "${keyspace}/${shard}" | grep -qi "${unhealthy_indicator}"; then
  fail "Timed out after ${wait_secs} seconds waiting for a primary tablet to be elected and become healthy in ${keyspace}/${shard}"
  fi
 }
 
+
+# Wait for a primary tablet to be writeable, ie read_only=0 and super_read_only=0
+function wait_for_writeable_shard_primary() {
+ if [[ -z ${1} || -z ${2} ]]; then
+ fail "A keyspace and shard must be specified when waiting for the shard's primary to be healthy"
+ fi
+ local keyspace=${1}
+ local shard=${2}
+ local wait_secs=30
+
+ PRIMARY_TABLET="$(vtctldclient --server=localhost:15999 GetTablets --keyspace "$keyspace" --shard "$shard" | grep -w "primary" | awk '{print $1}')"
+ if [ -z "$PRIMARY_TABLET" ] ; then
+ fail "Cannot determine primary tablet for keyspace/shard $keyspace/$shard"
+ fi
+
+ for _ in $(seq 1 ${wait_secs}); do
+ if vtctldclient --server=localhost:15999 GetFullStatus "$PRIMARY_TABLET" | grep "super_read_only" | grep --quiet "false" ; then
+ break
+ fi
+ sleep 1
+ done
+ if vtctldclient --server=localhost:15999 GetFullStatus "$PRIMARY_TABLET" | grep "super_read_only" | grep --quiet "true" ; then
+ fail "Timed out after ${wait_secs} seconds waiting for a primary tablet $PRIMARY_TABLET to be writeable in ${keyspace}/${shard}"
+ fi
+}
+
 # Wait for the shard primary tablet's VReplication engine to open.
 # There is currently no API call or client command that can be specifically used
 # to check the VReplication engine's status (no vars in /debug/vars etc. either).
@@ -109,6 +135,7 @@ function wait_for_healthy_shard() {
 
  wait_for_shard_tablets "${keyspace}" "${shard}" "${num_tablets}"
  wait_for_healthy_shard_primary "${keyspace}" "${shard}"
+ wait_for_writeable_shard_primary "${keyspace}" "${shard}"
  wait_for_shard_vreplication_engine "${keyspace}" "${shard}"
 }
 

diff --git a/examples/local/101_initial_cluster.sh b/examples/local/101_initial_cluster.sh
@@ -40,11 +40,16 @@ fi
 # start vtctld
 CELL=zone1 ../common/scripts/vtctld-up.sh
 
-# Create the keyspace with the sidecar database name and set the
-# correct durability policy. Please see the comment above for
-# more context on using a custom sidecar database name in your
-# Vitess clusters.
-vtctldclient --server localhost:15999 CreateKeyspace --sidecar-db-name="${SIDECAR_DB_NAME}" --durability-policy=semi_sync commerce || fail "Failed to create and configure the commerce keyspace"
+if vtctldclient GetKeyspace commerce > /dev/null 2>&1 ; then
+ # Keyspace already exists: we could be running this 101 example on an non-empty VTDATAROOT
+ vtctldclient --server localhost:15999 SetKeyspaceDurabilityPolicy --durability-policy=semi_sync commerce || fail "Failed to set keyspace durability policy on the commerce keyspace"
+else
+ # Create the keyspace with the sidecar database name and set the
+ # correct durability policy. Please see the comment above for
+ # more context on using a custom sidecar database name in your
+ # Vitess clusters.
+ vtctldclient --server localhost:15999 CreateKeyspace --sidecar-db-name="${SIDECAR_DB_NAME}" --durability-policy=semi_sync commerce || fail "Failed to create and configure the commerce keyspace"
+fi
 
 # start vttablets for keyspace commerce
 for i in 100 101 102; do

diff --git a/test/local_example.sh b/test/local_example.sh
@@ -18,6 +18,10 @@
 # It should be kept in sync with the steps in https://vitess.io/docs/get-started/local/
 # So we can detect if a regression affecting a tutorial is introduced.
 
+killall_vtdataroot() {
+ pkill -9 -e -f '(vtdataroot|VTDATAROOT)' # kill Vitess processes
+}
+
 source build.env
 
 set -xeo pipefail
@@ -28,7 +32,12 @@ unset VTROOT # ensure that the examples can run without VTROOT now.
 source ../common/env.sh # Required so that "mysql" works from alias
 
 ./101_initial_cluster.sh
+sleep 5 # Give vtgate time to really start.
 
+killall_vtdataroot
+# verify local example is able to start on an existing setup
+
+./101_initial_cluster.sh
 sleep 5 # Give vtgate time to really start.
 
 mysql < ../common/insert_commerce_data.sql