Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Test implementation of Kubernetes with a Vitess cluster #1

Merged
merged 15 commits into from
Nov 1, 2020
Merged

Test implementation of Kubernetes with a Vitess cluster #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
375c853
Addition of Vitess sharding init
frouioui Oct 16, 2020
eaaafc0
Addition of SQL and YAML files for basic Kubernetes Vitess Sharded cl…
frouioui Oct 17, 2020
1e3ebed
Improvement of the Makefile with working commands to locally run shar…
frouioui Oct 17, 2020
2a7714d
Modification of the Makefile for Vitess Cluster
frouioui Oct 24, 2020
741b3d2
Addition of the basic vschema file for the user table region sharding
frouioui Oct 27, 2020
f627af4
Addition of the region sharding upgraded files
frouioui Oct 27, 2020
db1868b
Addition of better makefile commands for the region sharding of the u…
frouioui Oct 27, 2020
f964a8f
Addition of a Kubernetes ConfigMap for country based sharding
frouioui Oct 31, 2020
69f3f4b
Copy of the vitess cluster secrets into a Kubernetes Secret config file
frouioui Oct 31, 2020
f0e8b64
Addition of a mount volume into VTCTLD pods for the country configura…
frouioui Oct 31, 2020
26a6eb9
Addition of the extraVolume for the users keyspace shards to mount co…
frouioui Oct 31, 2020
854b696
Improvement of the Makefile,Readme and sharded cluster k8s config file
frouioui Oct 31, 2020
53c6d51
Improvement of the README's command list and fix of a small issue in …
frouioui Nov 1, 2020
a251d98
Fix typo in readme
frouioui Nov 1, 2020
6856b55
Modified mysql cluster folder name
frouioui Nov 1, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,4 @@
# Test folder
# /test
/test/gridfs_main.go
/test_mysql/vitess
/mysql/vitess
114 changes: 114 additions & 0 deletions mysql/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
##
## Tagenal
## Tsinghua Univeristy
##

V_KEYSPACE_USERS = users
V_KEYSPACE_CONFIG = config

DATABASE_FOLDER_PATH = ./database

VITESS_OPERATOR_PATH = ./vitess/examples/operator
VITESS_COMMON_PATH = ./vitess/examples/common

# Aliases
MYSQL_CLIENT = mysql -h 127.0.0.1 -P 15306 -u user
VTCTL_CLIENT = vtctlclient -server=localhost:15999

# Unsharded commands
INIT_USERS_TABLES = $(VTCTL_CLIENT) ApplySchema -sql="$(shell cat $(DATABASE_FOLDER_PATH)/init/init_users.sql)" $(V_KEYSPACE_USERS)
INIT_USERS_VSCHEMA = $(VTCTL_CLIENT) ApplyVSchema -vschema='$(shell cat $(DATABASE_FOLDER_PATH)/vschema/vschema_users_initial.json)' $(V_KEYSPACE_USERS)

# Sharded commands
SHARD_INIT_CONFIG_USERS_SEQUENCES_SQL = $(VTCTL_CLIENT) ApplySchema -sql="$(shell cat $(DATABASE_FOLDER_PATH)/init/init_users_seq.sql)" $(V_KEYSPACE_CONFIG)
SHARD_INIT_CONFIG_USERS_SEQUENCES_VSCHEMA = $(VTCTL_CLIENT) ApplyVSchema -vschema='$(shell cat $(DATABASE_FOLDER_PATH)/vschema/vschema_config_seq.json)' $(V_KEYSPACE_CONFIG)
SHARD_ALTER_USERS_TABLES_SQL = $(VTCTL_CLIENT) ApplySchema -sql="$(shell cat $(DATABASE_FOLDER_PATH)/init/alter_users_tables_sharding.sql)" $(V_KEYSPACE_USERS)
SHARD_INIT_USERS_VSCHEMA = $(VTCTL_CLIENT) ApplyVSchema -vschema='$(shell cat $(DATABASE_FOLDER_PATH)/vschema/vschema_users_shard.json)' $(V_KEYSPACE_USERS)
SHARD_INIT_RESHARD_USERS = $(VTCTL_CLIENT) Reshard $(V_KEYSPACE_USERS).user2user '-' '-80,80-'
SHARD_VERIFY_USERS_SHARDING_PROCESS = $(VTCTL_CLIENT) VDiff $(V_KEYSPACE_USERS).user2user
SHARD_SWITCH_READ_REPLICA_USERS = $(VTCTL_CLIENT) SwitchReads -tablet_type=replica $(V_KEYSPACE_USERS).user2user
SHARD_SWITCH_READ_RDONLY_USERS = $(VTCTL_CLIENT) SwitchReads -tablet_type=rdonly $(V_KEYSPACE_USERS).user2user
SHARD_SWITCH_WRITE_USERS = $(VTCTL_CLIENT) SwitchWrites $(V_KEYSPACE_USERS).user2user

# Region sharding commands
REGION_SHARD_INIT_CONFIG_USERS_VSCHEMA = $(VTCTL_CLIENT) ApplyVSchema -vschema='$(shell cat $(DATABASE_FOLDER_PATH)/vschema/region/vschema_users_shard_region.json)' $(V_KEYSPACE_USERS)
REGION_SHARD_INIT_CONFIG_USER_LOOKUP_VINDEX = $(VTCTL_CLIENT) CreateLookupVindex -tablet_types=REPLICA $(V_KEYSPACE_USERS) '$(shell cat $(DATABASE_FOLDER_PATH)/vschema/region/vschema_users_shard_lookup_vindex.json)'
REGION_SHARD_EXTERNALIZE_USER_LOOKUP_VINDEX = $(VTCTL_CLIENT) ExternalizeVindex $(V_KEYSPACE_USERS).user_region_lookup

GET_USERS_VSCHEMA = $(VTCTL_CLIENT) GetVSchema $(V_KEYSPACE_USERS)

list_vtctld:
kubectl get pods --selector="planetscale.com/component=vtctld" -o custom-columns=":metadata.name"

start_minikube:
minikube start --cpus=12 --memory=14000 --disk-size=80g

start_minikube_dashboard:
minikube dashboard

clone_vitess_github:
git clone git@github.com:vitessio/vitess.git

install_vitess_operator:
kubectl apply -f $(VITESS_OPERATOR_PATH)/operator.yaml

init_kubernetes_unsharded_database:
kubectl apply -f kubernetes/vitess_cluster_secret.yaml
kubectl apply -f kubernetes/vitess_cluster_config.yaml
kubectl apply -f kubernetes/init_cluster_vitess.yaml

port_forwarding_vitess:
./script/port_forwarding.sh

init_unsharded_database:
$(INIT_USERS_TABLES)
$(INIT_USERS_VSCHEMA)

init_users_increment_sequence:
$(SHARD_INIT_CONFIG_USERS_SEQUENCES_SQL)
$(SHARD_INIT_CONFIG_USERS_SEQUENCES_VSCHEMA)
$(INIT_USERS_VSCHEMA)
$(SHARD_ALTER_USERS_TABLES_SQL)
$(SHARD_INIT_USERS_VSCHEMA)

init_sharded_database:
@echo Wait ...
@sleep 5
kubectl apply -f kubernetes/init_cluster_vitess_sharded.yaml

init_region_sharding_users:
$(REGION_SHARD_INIT_CONFIG_USERS_VSCHEMA)
$(REGION_SHARD_INIT_CONFIG_USER_LOOKUP_VINDEX)
@echo Wait ...
@sleep 5
$(REGION_SHARD_EXTERNALIZE_USER_LOOKUP_VINDEX)

resharding_process_users:
$(SHARD_INIT_RESHARD_USERS)
$(SHARD_VERIFY_USERS_SHARDING_PROCESS)
$(SHARD_SWITCH_READ_REPLICA_USERS)
$(SHARD_SWITCH_READ_RDONLY_USERS)
$(SHARD_SWITCH_WRITE_USERS)

reshard_users_db:
$(SHARD_INIT_RESHARD_USERS)

copy_locations_json_to_k8s:
kubectl cp ./database/locations/locations.json $(shell kubectl get pods --selector="planetscale.com/component=vtctld" -o custom-columns=":metadata.name"):/tmp/countries.json
kubectl cp ./database/locations/locations.json $(shell kubectl get pods --selector="planetscale.com/component=vtgate" -o custom-columns=":metadata.name"):/tmp/countries.json

show_vttablets:
kubectl get pods --selector="planetscale.com/component=vttablet" -o custom-columns=":metadata.name"

show_vitess_tablets:
echo "show vitess_tablets;" | $(MYSQL_CLIENT) --table

insert_few_user_row:
$(MYSQL_CLIENT) < ./database/insert/insert_data_users.sql

show_user_table:
@$(MYSQL_CLIENT) --table < ./database/select/select_user.sql
@$(MYSQL_CLIENT) --table < ./database/select/select_user_shard_1.sql
@$(MYSQL_CLIENT) --table < ./database/select/select_user_shard_2.sql

.PHONY: set_aliases
Loading