Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Move all examples to vtctldclient #14226

Merged
merged 2 commits into from
Oct 11, 2023
Merged

Move all examples to vtctldclient #14226

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3f0e36b
Move all examples to vtctldclient
mattlord Oct 10, 2023
3fa576f
Address review comments
mattlord Oct 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions examples/backups/start_cluster.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,8 @@ done
vtctldclient PlannedReparentShard commerce/0 --new-primary "zone1-100"

# create the schema for commerce
vtctlclient ApplySchema -- --sql-file ./create_commerce_schema.sql commerce || fail "Could not apply schema for the commerce keyspace"
vtctlclient ApplyVSchema -- --vschema_file ../local/vschema_commerce_seq.json commerce || fail "Could not apply vschema for the commerce keyspace"
vtctldclient ApplySchema --sql-file ./create_commerce_schema.sql commerce || fail "Could not apply schema for the commerce keyspace"
vtctldclient ApplyVSchema --vschema-file ../local/vschema_commerce_seq.json commerce || fail "Could not apply vschema for the commerce keyspace"

# Create keyspace and set the semi_sync durability policy.
vtctldclient CreateKeyspace --durability-policy=semi_sync customer || fail "Failed to create and configure the customer keyspace"
Expand All @@ -67,8 +67,8 @@ for shard in "-80" "80-"; do
done

# create the schema for customer
vtctlclient ApplySchema -- --sql-file ./create_customer_schema.sql customer || fail "Could not apply schema for the customer keyspace"
vtctlclient ApplyVSchema -- --vschema_file ../local/vschema_customer_sharded.json customer || fail "Could not apply vschema for the customer keyspace"
vtctldclient ApplySchema --sql-file ./create_customer_schema.sql customer || fail "Could not apply schema for the customer keyspace"
vtctldclient ApplyVSchema --vschema-file ../local/vschema_customer_sharded.json customer || fail "Could not apply vschema for the customer keyspace"


# start vtgate
Expand Down
4 changes: 2 additions & 2 deletions examples/backups/stop_tablets.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
source ../common/env.sh

for tablet in 100 200 300; do
if vtctlclient --action_timeout 1s --server localhost:15999 GetTablet zone1-$tablet >/dev/null 2>&1; then
if vtctldclient --action_timeout 1s --server localhost:15999 GetTablet zone1-$tablet >/dev/null 2>&1; then
# The zero tablet is up. Try to shutdown 0-2 tablet + mysqlctl
for i in 0 1 2; do
uid=$(($tablet + $i))
Expand All @@ -29,7 +29,7 @@ for tablet in 100 200 300; do
echo "Shutting down mysql zone1-$uid"
CELL=zone1 TABLET_UID=$uid ../common/scripts/mysqlctl-down.sh
echo "Removing tablet directory zone1-$uid"
vtctlclient DeleteTablet -- --allow_primary=true zone1-$uid
vtctldclient DeleteTablets --allow-primary=true zone1-$uid
mattlord marked this conversation as resolved.
Show resolved Hide resolved
rm -Rf $VTDATAROOT/vt_0000000$uid
done
fi
Expand Down
2 changes: 1 addition & 1 deletion examples/backups/take_backups.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,5 +20,5 @@
source ../common/env.sh

for shard in "customer/-80" "customer/80-" "commerce/0"; do
vtctlclient BackupShard "${shard}" || fail "Failed to backup shard: ${shard}"
vtctldclient BackupShard "${shard}" || fail "Failed to backup shard: ${shard}"
done
8 changes: 4 additions & 4 deletions examples/backups/upgrade_cluster.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ for i in 101 102; do
echo "Shutting down mysql zone1-$i"
CELL=zone1 TABLET_UID=$i ../common/scripts/mysqlctl-down.sh
echo "Removing tablet directory zone1-$i"
vtctlclient DeleteTablet -- --allow_primary=true zone1-$i
vtctldclient DeleteTablets --allow-primary=true zone1-$i
rm -Rf $VTDATAROOT/vt_0000000$i
echo "Starting tablet zone1-$i again"
CELL=zone1 TABLET_UID=$i ../common/scripts/mysqlctl-up.sh
Expand All @@ -39,7 +39,7 @@ for i in 201 202; do
echo "Shutting down mysql zone1-$i"
CELL=zone1 TABLET_UID=$i ../common/scripts/mysqlctl-down.sh
echo "Removing tablet directory zone1-$i"
vtctlclient DeleteTablet -- --allow_primary=true zone1-$i
vtctldclient DeleteTablets --allow-primary=true zone1-$i
rm -Rf $VTDATAROOT/vt_0000000$i
echo "Starting tablet zone1-$i again"
CELL=zone1 TABLET_UID=$i ../common/scripts/mysqlctl-up.sh
Expand All @@ -52,7 +52,7 @@ for i in 301 302; do
echo "Shutting down mysql zone1-$i"
CELL=zone1 TABLET_UID=$i ../common/scripts/mysqlctl-down.sh
echo "Removing tablet directory zone1-$i"
vtctlclient DeleteTablet -- --allow_primary=true zone1-$i
vtctldclient DeleteTablets --allow-primary=true zone1-$i
rm -Rf $VTDATAROOT/vt_0000000$i
echo "Starting tablet zone1-$i again"
CELL=zone1 TABLET_UID=$i ../common/scripts/mysqlctl-up.sh
Expand Down Expand Up @@ -94,4 +94,4 @@ SHARD=-80 CELL=zone1 KEYSPACE=customer TABLET_UID=200 ../common/scripts/vttablet

echo "Restarting tablet zone1-300"
CELL=zone1 TABLET_UID=300 ../common/scripts/vttablet-down.sh
SHARD=80- CELL=zone1 KEYSPACE=customer TABLET_UID=300 ../common/scripts/vttablet-up.sh
SHARD=80- CELL=zone1 KEYSPACE=customer TABLET_UID=300 ../common/scripts/vttablet-up.sh
5 changes: 2 additions & 3 deletions examples/common/env.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,11 @@ fi

# mysqld might be in /usr/sbin which will not be in the default PATH
PATH="/usr/sbin:$PATH"
for binary in mysqld etcd etcdctl curl vtctlclient vtctldclient vttablet vtgate vtctld mysqlctl; do
for binary in mysqld etcd etcdctl curl vtctldclient vttablet vtgate vtctld mysqlctl; do
command -v "$binary" > /dev/null || fail "${binary} is not installed in PATH. See https://vitess.io/docs/get-started/local/ for install instructions."
done;

# vtctlclient has a separate alias setup below
# vtctldclient has a separate alias setup below
for binary in vttablet vtgate vtctld mysqlctl vtorc vtctl; do
alias $binary="$binary --config-file-not-found-handling=ignore"
done;
Expand Down Expand Up @@ -77,7 +77,6 @@ mkdir -p "${VTDATAROOT}/tmp"
# such as ~/.my.cnf

alias mysql="command mysql --no-defaults -h 127.0.0.1 -P 15306"
alias vtctlclient="command vtctlclient --server localhost:15999 --log_dir ${VTDATAROOT}/tmp --alsologtostderr --config-file-not-found-handling=ignore"
alias vtctldclient="command vtctldclient --server localhost:15999"

# Make sure aliases are expanded in non-interactive shell
Expand Down
4 changes: 2 additions & 2 deletions examples/common/lib/utils.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,13 +108,13 @@ function wait_for_shard_vreplication_engine() {
local wait_secs=90

for _ in $(seq 1 ${wait_secs}); do
if vtctlclient --server=localhost:15999 Workflow -- "${keyspace}" listall &>/dev/null; then
if vtctldclient --server=localhost:15999 workflow --keyspace "${keyspace}" list &>/dev/null; then
break
fi
sleep 1
done;

if ! vtctlclient --server=localhost:15999 Workflow -- "${keyspace}" listall &>/dev/null; then
if ! vtctldclient --server=localhost:15999 workflow --keyspace "${keyspace}" list &>/dev/null; then
fail "Timed out after ${wait_secs} seconds waiting for the primary tablet's VReplication engine to open in ${keyspace}/${shard}"
fi
}
Expand Down
12 changes: 1 addition & 11 deletions examples/compose/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,17 +222,7 @@ The vreplication container included performs the following actions;
4. Prints out helpful debug information for you.
```
vitess/examples/compose$ docker-compose logs -f vreplication
vreplication_1 | + /vt/bin/vtctlclient --server vtctld:15999 VReplicationExec local-0000000101 'insert into _vt.vreplication (db_name, source, pos, max_tps, max_replication_lag, tablet_types, time_updated, transaction_timestamp, state) values('\''commerce'\'', '\''keyspace:\"ext_commerce\" shard:\"0\" filter:<rules:<match:\"/.*\" > > on_ddl:EXEC_IGNORE '\'', '\'''\'', 9999, 9999, '\''primary'\'', 0, 0, '\''Running'\'')'
vreplication_1 | + /vt/bin/vtctlclient --server vtctld:15999 VReplicationExec local-0000000101 'select * from _vt.vreplication'
vreplication_1 | +----+----------+--------------------------------+-----+----------+---------+---------------------+------+--------------+--------------+-----------------------+---------+---------+----------+
vreplication_1 | | id | workflow | source | pos | stop_pos | max_tps | max_replication_lag | cell | tablet_types | time_updated | transaction_timestamp | state | message | db_name |
vreplication_1 | +----+----------+--------------------------------+-----+----------+---------+---------------------+------+--------------+--------------+-----------------------+---------+---------+----------+
vreplication_1 | | 1 | | keyspace:"ext_commerce" | | | 9999 | 9999 | | primary | 0 | 0 | Running | | commerce |
vreplication_1 | | | | shard:"0" | | | | | | | | | | | |
vreplication_1 | | | | filter:<rules:<match:"/.*" > > | | | | | | | | | | | |
vreplication_1 | | | | on_ddl:EXEC_IGNORE | | | | | | | | | | | |
vreplication_1 | +----+----------+--------------------------------+-----+----------+---------+---------------------+------+--------------+--------------+-----------------------+---------+---------+----------+
compose_vreplication_1 exited with code 0
...
```

### Connect to vgate and run queries
Expand Down
28 changes: 11 additions & 17 deletions examples/compose/externaldb_vreplication.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,19 +19,19 @@ set -ex
VTCTLD_SERVER=${VTCTLD_SERVER:-'vtctld:15999'}

# Wait until source and destination primaries are available
until (/vt/bin/vtctlclient --server $VTCTLD_SERVER ListAllTablets | grep "ext_" | grep "primary" ); do
until (/vt/bin/vtctldclient --server $VTCTLD_SERVER GetTablets | grep "ext_" | grep "primary" ); do
echo 'waiting for external primary..';
sleep 1;
done

until (/vt/bin/vtctlclient --server $VTCTLD_SERVER ListAllTablets | grep -v "ext_" | grep "primary" ); do
until (/vt/bin/vtctldclient --server $VTCTLD_SERVER GetTablets | grep -v "ext_" | grep "primary" ); do
echo 'waiting for managed primary..';
sleep 1;
done


# Get source and destination tablet and shard information
TABLET_INFO=$(/vt/bin/vtctlclient --server $VTCTLD_SERVER ListAllTablets)
TABLET_INFO=$(/vt/bin/vtctldclient --server $VTCTLD_SERVER GetTablets)
source_alias=$(echo "$TABLET_INFO "| grep "ext_" | grep "primary" | awk '{ print $1 }')
dest_alias=$(echo "$TABLET_INFO "| grep -v "ext_" | grep "primary" | awk '{ print $1 }')
source_keyspace=$(echo "$TABLET_INFO "| grep "ext_" | grep "primary" | awk '{ print $2 }')
Expand All @@ -43,33 +43,27 @@ dest_tablet=$(echo "$TABLET_INFO "| grep -v "ext_" | grep "primary" | awk '{ pri


# Disable foreign_key checks on destination
/vt/bin/vtctlclient --server $VTCTLD_SERVER ExecuteFetchAsDba $dest_alias 'SET GLOBAL FOREIGN_KEY_CHECKS=0;'
/vt/bin/vtctldclient --server $VTCTLD_SERVER ExecuteFetchAsDba $dest_alias 'SET GLOBAL FOREIGN_KEY_CHECKS=0;'

# Get source_sql mode
source_sql_mode=$(/vt/bin/vtctlclient --server $VTCTLD_SERVER ExecuteFetchAsDba $source_alias 'SELECT @@GLOBAL.sql_mode' | awk 'NR==4 {print $2}')
source_sql_mode=$(/vt/bin/vtctldclient --server $VTCTLD_SERVER ExecuteFetchAsDba $source_alias 'SELECT @@GLOBAL.sql_mode' | awk 'NR==4 {print $2}')

# Apply source sql_mode to destination
# The intention is to avoid replication errors
/vt/bin/vtctlclient --server $VTCTLD_SERVER ExecuteFetchAsDba $dest_alias "SET GLOBAL sql_mode='$source_sql_mode';"
/vt/bin/vtctldclient --server $VTCTLD_SERVER ExecuteFetchAsDba $dest_alias "SET GLOBAL sql_mode='$source_sql_mode';"
mattlord marked this conversation as resolved.
Show resolved Hide resolved

# Verify sql_mode matches
[ $source_sql_mode == $(/vt/bin/vtctlclient --server $VTCTLD_SERVER ExecuteFetchAsDba $dest_alias 'SELECT @@GLOBAL.sql_mode' | awk 'NR==4 {print $2}') ] && \
[ $source_sql_mode == $(/vt/bin/vtctldclient --server $VTCTLD_SERVER ExecuteFetchAsDba $dest_alias 'SELECT @@GLOBAL.sql_mode' | awk 'NR==4 {print $2}') ] && \
echo "Source and Destination sql_mode Match." || echo "sql_mode MisMatch"

until /vt/bin/vtctlclient --server $VTCTLD_SERVER GetSchema $dest_alias; do
until /vt/bin/vtctldclient --server $VTCTLD_SERVER GetSchema $dest_alias; do
echo "Waiting for destination schema to be ready..";
sleep 3;
done

# Copy schema from source to destination shard
/vt/bin/vtctlclient --server $VTCTLD_SERVER CopySchemaShard $source_tablet $dest_tablet || true

# Verify schema
/vt/bin/vtctlclient --server $VTCTLD_SERVER GetSchema $dest_alias

# Start vreplication
/vt/bin/vtctlclient --server $VTCTLD_SERVER VReplicationExec $dest_alias 'insert into _vt.vreplication (db_name, source, pos, max_tps, max_replication_lag, tablet_types, time_updated, transaction_timestamp, state) values('"'"''"$dest_keyspace"''"'"', '"'"'keyspace:\"'"$source_keyspace"'\" shard:\"'"$source_shard"'\" filter:<rules:<match:\"/.*\" > > on_ddl:EXEC_IGNORE '"'"', '"'"''"'"', 9999, 9999, '"'"'primary'"'"', 0, 0, '"'"'Running'"'"')'
# Start vreplication workflow
/vt/bin/vtctldclient --server $VTCTLD_SERVER MoveTables --workflow ext_commerce2commerce --target-keyspace $dest_keyspace create --source-keyspace $source_keyspace --all-tables

# Check vreplication status
/vt/bin/vtctlclient --server $VTCTLD_SERVER VReplicationExec $dest_alias 'select * from _vt.vreplication'
/vt/bin/vtctldclient --server $VTCTLD_SERVER MoveTables --workflow ext_commerce2commerce --target-keyspace $dest_keyspace show

4 changes: 2 additions & 2 deletions examples/compose/lvtctl.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,5 +20,5 @@ if [[ "$OSTYPE" == "msys" ]]; then
tty=winpty
fi

# This is a convenience script to run vtctlclient against the local example.
exec $tty docker-compose exec ${CS:-vtctld} /vt/bin/vtctlclient --server vtctld:15999 "$@"
# This is a convenience script to run vtctldclient against the local example.
exec $tty docker-compose exec ${CS:-vtctld} /vt/bin/vtctldclient --server vtctld:15999 "$@"
12 changes: 6 additions & 6 deletions examples/compose/schemaload.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,23 +26,23 @@ sleep $sleeptime

if [ ! -f schema_run ]; then
while true; do
vtctlclient --server vtctld:$GRPC_PORT GetTablet $targettab && break
vtctldclient --server vtctld:$GRPC_PORT GetTablet $targettab && break
sleep 1
done
if [ "$external_db" = "0" ]; then
for schema_file in $schema_files; do
echo "Applying Schema ${schema_file} to ${KEYSPACE}"
vtctlclient --server vtctld:$GRPC_PORT -- ApplySchema --sql-file /script/tables/${schema_file} $KEYSPACE || \
vtctlclient --server vtctld:$GRPC_PORT -- ApplySchema --sql "$(cat /script/tables/${schema_file})" $KEYSPACE || true
vtctldclient --server vtctld:$GRPC_PORT ApplySchema --sql-file /script/tables/${schema_file} $KEYSPACE || \
vtctldclient --server vtctld:$GRPC_PORT ApplySchema --sql "$(cat /script/tables/${schema_file})" $KEYSPACE || true
done
fi
echo "Applying VSchema ${vschema_file} to ${KEYSPACE}"

vtctlclient --server vtctld:$GRPC_PORT -- ApplyVSchema --vschema_file /script/${vschema_file} $KEYSPACE || \
vtctlclient --server vtctld:$GRPC_PORT -- ApplyVSchema --vschema "$(cat /script/${vschema_file})" $KEYSPACE
vtctldclient --server vtctld:$GRPC_PORT ApplyVSchema --vschema-file /script/${vschema_file} $KEYSPACE || \
vtctldclient --server vtctld:$GRPC_PORT ApplyVSchema --vschema "$(cat /script/${vschema_file})" $KEYSPACE

echo "List All Tablets"
vtctlclient --server vtctld:$GRPC_PORT ListAllTablets
vtctldclient --server vtctld:$GRPC_PORT GetTablets

if [ -n "$load_file" ]; then
# vtgate can take a REALLY long time to come up fully
Expand Down
2 changes: 1 addition & 1 deletion examples/compose/vtcompose/vtcompose.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -534,7 +534,7 @@ func generateDefaultShard(tabAlias int, shard string, keyspaceData keyspaceInfo,
path: /services/init_shard_primary%[2]d
value:
image: vitess/lite:${VITESS_TAG:-latest}
command: ["sh", "-c", "/vt/bin/vtctlclient %[5]s InitShardPrimary -force %[4]s/%[3]s %[6]s-%[2]d "]
command: ["sh", "-c", "/vt/bin/vtctldclient %[5]s InitShardPrimary --force %[4]s/%[3]s %[6]s-%[2]d "]
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

simply -f also works if we want to demo that in the examples

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more. 

vtctldclient InitShardPrimary -f
E1010 20:01:59.194594   92668 main.go:56] unknown shorthand flag: 'f' in -f

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ah shoot, sorry. we should make that work (later)

%[1]s
`, dependsOn, aliases[0], shard, keyspaceData.keyspace, opts.topologyFlags, opts.cell)
}
Expand Down
2 changes: 1 addition & 1 deletion examples/compose/vttablet-up.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ sleep $sleeptime

# Create the cell
# https://vitess.io/blog/2020-04-27-life-of-a-cluster/
$VTROOT/bin/vtctlclient --server vtctld:$GRPC_PORT -- AddCellInfo --root vitess/$CELL --server_address consul1:8500 $CELL || true
$VTROOT/bin/vtctldclient --server vtctld:$GRPC_PORT AddCellInfo --root vitess/$CELL --server-address consul1:8500 $CELL || true

#Populate external db conditional args
if [ $tablet_role = "externalprimary" ]; then
Expand Down
23 changes: 13 additions & 10 deletions examples/local/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,35 +19,38 @@ mysql --table < ../common/select_commerce_data.sql
./201_customer_tablets.sh

# Initiate move tables
vtctlclient MoveTables -- --source commerce --tables 'customer,corder' Create customer.commerce2customer
vtctldclient MoveTables --workflow commerce2customer --target-keyspace customer create --source-keyspace commerce --tables "customer,corder"

# Validate
vtctlclient VDiff customer.commerce2customer
vtctldclient vdiff --workflow commerce2customer --target-keyspace customer create
vtctldclient vdiff --workflow commerce2customer --target-keyspace customer show last

# Cut-over
vtctlclient MoveTables -- --tablet_types=rdonly,replica SwitchTraffic customer.commerce2customer
vtctlclient MoveTables -- --tablet_types=primary SwitchTraffic customer.commerce2customer
vtctldclient MoveTables --workflow commerce2customer --target-keyspace customer switchtraffic --tablet-types "rdonly,replica"
vtctldclient MoveTables --workflow commerce2customer --target-keyspace customer switchtraffic --tablet-types primary

# Clean-up
vtctlclient MoveTables Complete customer.commerce2customer
vtctldclient MoveTables --workflow commerce2customer --target-keyspace customer complete

# Prepare for resharding
./301_customer_sharded.sh
./302_new_shards.sh

# Reshard
vtctlclient Reshard -- --source_shards '0' --target_shards '-80,80-' Create customer.cust2cust
vtctldclient Reshard --workflow cust2cust --target-keyspace customer create --source-shards '0' --target-shards '-80,80-'

# Validate
vtctlclient VDiff customer.cust2cust
vtctldclient vdiff --workflow cust2cust --target-keyspace customer create
vtctldclient vdiff --workflow cust2cust --target-keyspace customer show last

# Cut-over
vtctlclient Reshard -- --tablet_types=rdonly,replica SwitchTraffic customer.cust2cust
vtctlclient Reshard -- --tablet_types=primary SwitchTraffic customer.cust2cust
vtctldclient Reshard --workflow cust2cust --target-keyspace customer switchtraffic --tablet-types "rdonly,replica"
vtctldclient Reshard --workflow cust2cust --target-keyspace customer switchtraffic --tablet-types primary

# Down shard 0
vtctldclient Reshard --workflow cust2cust --target-keyspace customer complete
./306_down_shard_0.sh
vtctlclient DeleteShard -- --force --recursive customer/0
vtctldclient DeleteShards --force --recursive customer/0

# Down cluster
./401_teardown.sh
Expand Down
Loading
Loading