FB8-87, FB8-119: Supporting dynamic scheduling in MTS (facebook#999) (f…

…acebook#999)

Summary:
Jira ticket: https://jira.percona.com/browse/FB8-87
Jira ticket: https://jira.percona.com/browse/FB8-119

Reference commit: facebook@5846968
Reference commit: facebook@d6217c9

Statically assigning shards to slave workers can cause imbalance if a few shards are hotter than others. We should be able to check the imbalance among slave workers and dynamically reassign shards.
Pull Request resolved: facebook#999

Differential Revision: D14883857

mysql-test/r/dd_is_compatibility_ci.result

@@ -201,6 +201,7 @@ ROUTINES
 SCHEMATA
 SCHEMATA_EXTENSIONS
 SCHEMA_PRIVILEGES
+SLAVE_DB_LOAD
 STATISTICS
 ST_GEOMETRY_COLUMNS
 ST_SPATIAL_REFERENCE_SYSTEMS

mysql-test/r/dd_is_compatibility_cs.result

@@ -202,6 +202,7 @@ ROUTINES
 SCHEMATA
 SCHEMATA_EXTENSIONS
 SCHEMA_PRIVILEGES
+SLAVE_DB_LOAD
 STATISTICS
 ST_GEOMETRY_COLUMNS
 ST_SPATIAL_REFERENCE_SYSTEMS

mysql-test/r/information_schema_ci.result

@@ -94,6 +94,7 @@ ROUTINES
 SCHEMATA
 SCHEMATA_EXTENSIONS
 SCHEMA_PRIVILEGES
+SLAVE_DB_LOAD
 STATISTICS
 ST_GEOMETRY_COLUMNS
 ST_SPATIAL_REFERENCE_SYSTEMS
@@ -900,7 +901,7 @@ AND table_name COLLATE utf8_general_ci not like 'innodb_%'
 AND table_name COLLATE utf8_general_ci not like 'rocksdb_%'
 GROUP BY TABLE_SCHEMA;
 TABLE_SCHEMA	count(*)
-information_schema	50
+information_schema	51
 mysql	35
 create table t1 (i int, j int);
 create trigger trg1 before insert on t1 for each row
@@ -1383,6 +1384,7 @@ ROUTINES	information_schema.ROUTINES	1
 SCHEMATA	information_schema.SCHEMATA	1
 SCHEMATA_EXTENSIONS	information_schema.SCHEMATA_EXTENSIONS	1
 SCHEMA_PRIVILEGES	information_schema.SCHEMA_PRIVILEGES	1
+SLAVE_DB_LOAD	information_schema.SLAVE_DB_LOAD	1
 STATISTICS	information_schema.STATISTICS	1
 ST_GEOMETRY_COLUMNS	information_schema.ST_GEOMETRY_COLUMNS	1
 ST_SPATIAL_REFERENCE_SYSTEMS	information_schema.ST_SPATIAL_REFERENCE_SYSTEMS	1
@@ -2527,6 +2529,7 @@ ROUTINES	ROUTINE_SCHEMA
 SCHEMATA	SCHEMA_NAME
 SCHEMATA_EXTENSIONS	SCHEMA_NAME
 SCHEMA_PRIVILEGES	TABLE_SCHEMA
+SLAVE_DB_LOAD	DB
 STATISTICS	TABLE_SCHEMA
 ST_GEOMETRY_COLUMNS	TABLE_SCHEMA
 ST_SPATIAL_REFERENCE_SYSTEMS	SRS_NAME
@@ -2597,6 +2600,7 @@ ROUTINES	ROUTINE_SCHEMA
 SCHEMATA	SCHEMA_NAME
 SCHEMATA_EXTENSIONS	SCHEMA_NAME
 SCHEMA_PRIVILEGES	TABLE_SCHEMA
+SLAVE_DB_LOAD	DB
 STATISTICS	TABLE_SCHEMA
 ST_GEOMETRY_COLUMNS	TABLE_SCHEMA
 ST_SPATIAL_REFERENCE_SYSTEMS	SRS_NAME

mysql-test/r/information_schema_cs.result

@@ -94,6 +94,7 @@ ROUTINES
 SCHEMATA
 SCHEMATA_EXTENSIONS
 SCHEMA_PRIVILEGES
+SLAVE_DB_LOAD
 STATISTICS
 ST_GEOMETRY_COLUMNS
 ST_SPATIAL_REFERENCE_SYSTEMS
@@ -900,7 +901,7 @@ AND table_name COLLATE utf8_general_ci not like 'innodb_%'
 AND table_name COLLATE utf8_general_ci not like 'rocksdb_%'
 GROUP BY TABLE_SCHEMA;
 TABLE_SCHEMA	count(*)
-information_schema	50
+information_schema	51
 mysql	35
 create table t1 (i int, j int);
 create trigger trg1 before insert on t1 for each row
@@ -1383,6 +1384,7 @@ ROUTINES	information_schema.ROUTINES	1
 SCHEMATA	information_schema.SCHEMATA	1
 SCHEMATA_EXTENSIONS	information_schema.SCHEMATA_EXTENSIONS	1
 SCHEMA_PRIVILEGES	information_schema.SCHEMA_PRIVILEGES	1
+SLAVE_DB_LOAD	information_schema.SLAVE_DB_LOAD	1
 STATISTICS	information_schema.STATISTICS	1
 ST_GEOMETRY_COLUMNS	information_schema.ST_GEOMETRY_COLUMNS	1
 ST_SPATIAL_REFERENCE_SYSTEMS	information_schema.ST_SPATIAL_REFERENCE_SYSTEMS	1
@@ -2527,6 +2529,7 @@ ROUTINES	ROUTINE_SCHEMA
 SCHEMATA	SCHEMA_NAME
 SCHEMATA_EXTENSIONS	SCHEMA_NAME
 SCHEMA_PRIVILEGES	TABLE_SCHEMA
+SLAVE_DB_LOAD	DB
 STATISTICS	TABLE_SCHEMA
 ST_GEOMETRY_COLUMNS	TABLE_SCHEMA
 ST_SPATIAL_REFERENCE_SYSTEMS	SRS_NAME
@@ -2597,6 +2600,7 @@ ROUTINES	ROUTINE_SCHEMA
 SCHEMATA	SCHEMA_NAME
 SCHEMATA_EXTENSIONS	SCHEMA_NAME
 SCHEMA_PRIVILEGES	TABLE_SCHEMA
+SLAVE_DB_LOAD	DB
 STATISTICS	TABLE_SCHEMA
 ST_GEOMETRY_COLUMNS	TABLE_SCHEMA
 ST_SPATIAL_REFERENCE_SYSTEMS	SRS_NAME

mysql-test/r/mysqld--help-notwin.result

@@ -769,6 +769,13 @@ The following options may be given as the first argument:
  --min-examined-row-limit=# 
  Don't write queries to slow log that examine fewer rows
  than that
+ --mts-dynamic-rebalance 
+ Shuffle DB's within workers periodically for load
+ balancing
+ --mts-imbalance-threshold[=#] 
+ Threshold to trigger worker thread rebalancing. This
+ parameter denotes the percent load on the most loaded
+ worker.
  --myisam-block-size=# 
  Block size to be used for MyISAM index pages
  --myisam-data-pointer-size=# 
@@ -2511,6 +2518,8 @@ max-user-connections 0
 max-write-lock-count 18446744073709551615
 memlock FALSE
 min-examined-row-limit 0
+mts-dynamic-rebalance FALSE
+mts-imbalance-threshold 90
 myisam-block-size 1024
 myisam-data-pointer-size 6
 myisam-max-sort-file-size 9223372036853727232

mysql-test/r/mysqlshow_ci.result

@@ -158,6 +158,7 @@ Database: information_schema
 | SCHEMA_PRIVILEGES                     |
 | SCHEMATA                              |
 | SCHEMATA_EXTENSIONS                   |
+| SLAVE_DB_LOAD                         |
 | ST_GEOMETRY_COLUMNS                   |
 | ST_SPATIAL_REFERENCE_SYSTEMS          |
 | ST_UNITS_OF_MEASURE                   |
@@ -259,6 +260,7 @@ Database: INFORMATION_SCHEMA
 | SCHEMA_PRIVILEGES                     |
 | SCHEMATA                              |
 | SCHEMATA_EXTENSIONS                   |
+| SLAVE_DB_LOAD                         |
 | ST_GEOMETRY_COLUMNS                   |
 | ST_SPATIAL_REFERENCE_SYSTEMS          |
 | ST_UNITS_OF_MEASURE                   |

mysql-test/r/mysqlshow_cs.result

@@ -159,6 +159,7 @@ Database: information_schema
 | SCHEMATA                              |
 | SCHEMATA_EXTENSIONS                   |
 | SCHEMA_PRIVILEGES                     |
+| SLAVE_DB_LOAD                         |
 | STATISTICS                            |
 | ST_GEOMETRY_COLUMNS                   |
 | ST_SPATIAL_REFERENCE_SYSTEMS          |
@@ -261,6 +262,7 @@ Database: INFORMATION_SCHEMA
 | SCHEMATA                              |
 | SCHEMATA_EXTENSIONS                   |
 | SCHEMA_PRIVILEGES                     |
+| SLAVE_DB_LOAD                         |
 | STATISTICS                            |
 | ST_GEOMETRY_COLUMNS                   |
 | ST_SPATIAL_REFERENCE_SYSTEMS          |

mysql-test/suite/funcs_1/r/is_columns_is_cs.result

@@ -442,6 +442,9 @@ def	information_schema	SCHEMA_PRIVILEGES	TABLE_CATALOG	2		NO	varchar	170	512	NUL
 def	information_schema	SCHEMA_PRIVILEGES	TABLE_SCHEMA	3		NO	varchar	21	64	NULL	NULL	NULL	utf8mb3	utf8mb3_general_ci	varchar(64)			select			NULL
 def	information_schema	SCHEMA_PRIVILEGES	PRIVILEGE_TYPE	4		NO	varchar	21	64	NULL	NULL	NULL	utf8mb3	utf8mb3_general_ci	varchar(64)			select			NULL
 def	information_schema	SCHEMA_PRIVILEGES	IS_GRANTABLE	5		NO	varchar	1	3	NULL	NULL	NULL	utf8mb3	utf8mb3_general_ci	varchar(3)			select			NULL
+def	information_schema	SLAVE_DB_LOAD	DB	1		NO	varchar	64	192	NULL	NULL	NULL	utf8mb3	utf8mb3_general_ci	varchar(192)			select			NULL
+def	information_schema	SLAVE_DB_LOAD	WORKER	2		NO	bigint	NULL	NULL	NULL	NULL	NULL	NULL	NULL	bigint unsigned			select			NULL
+def	information_schema	SLAVE_DB_LOAD	DB_LOAD	3		NO	bigint	NULL	NULL	NULL	NULL	NULL	NULL	NULL	bigint unsigned			select			NULL
 def	information_schema	STATISTICS	TABLE_CATALOG	1	NULL	NO	varchar	64	192	NULL	NULL	NULL	utf8mb3	utf8mb3_bin	varchar(64)			select			NULL
 def	information_schema	STATISTICS	TABLE_SCHEMA	2	NULL	NO	varchar	64	192	NULL	NULL	NULL	utf8mb3	utf8mb3_bin	varchar(64)			select			NULL
 def	information_schema	STATISTICS	TABLE_NAME	3	NULL	NO	varchar	64	192	NULL	NULL	NULL	utf8mb3	utf8mb3_bin	varchar(64)			select			NULL
@@ -1113,6 +1116,9 @@ NULL	information_schema	SCHEMATA	SQL_PATH	binary	0	0	NULL	NULL	binary(0)
 3.0476	information_schema	SCHEMA_PRIVILEGES	TABLE_SCHEMA	varchar	21	64	utf8mb3	utf8mb3_general_ci	varchar(64)
 3.0476	information_schema	SCHEMA_PRIVILEGES	PRIVILEGE_TYPE	varchar	21	64	utf8mb3	utf8mb3_general_ci	varchar(64)
 3.0000	information_schema	SCHEMA_PRIVILEGES	IS_GRANTABLE	varchar	1	3	utf8mb3	utf8mb3_general_ci	varchar(3)
+3.0000	information_schema	SLAVE_DB_LOAD	DB	varchar	64	192	utf8mb3	utf8mb3_general_ci	varchar(192)
+NULL	information_schema	SLAVE_DB_LOAD	WORKER	bigint	NULL	NULL	NULL	NULL	bigint unsigned
+NULL	information_schema	SLAVE_DB_LOAD	DB_LOAD	bigint	NULL	NULL	NULL	NULL	bigint unsigned
 3.0000	information_schema	STATISTICS	TABLE_CATALOG	varchar	64	192	utf8mb3	utf8mb3_bin	varchar(64)
 3.0000	information_schema	STATISTICS	TABLE_SCHEMA	varchar	64	192	utf8mb3	utf8mb3_bin	varchar(64)
 3.0000	information_schema	STATISTICS	TABLE_NAME	varchar	64	192	utf8mb3	utf8mb3_bin	varchar(64)

mysql-test/suite/funcs_1/r/is_tables_is.result

@@ -754,6 +754,29 @@ user_comment
 Separator	-----------------------------------------------------
 TABLE_CATALOG	def
 TABLE_SCHEMA	information_schema
+TABLE_NAME	SLAVE_DB_LOAD
+TABLE_TYPE	SYSTEM VIEW
+ENGINE	NULL
+VERSION	10
+ROW_FORMAT	NULL
+TABLE_ROWS	#TBLR#
+AVG_ROW_LENGTH	#ARL#
+DATA_LENGTH	#DL#
+MAX_DATA_LENGTH	#MDL#
+INDEX_LENGTH	#IL#
+DATA_FREE	#DF#
+AUTO_INCREMENT	#AI#
+CREATE_TIME	#CRT#
+UPDATE_TIME	#UT#
+CHECK_TIME	#CT#
+TABLE_COLLATION	NULL
+CHECKSUM	NULL
+CREATE_OPTIONS	#CO#
+TABLE_COMMENT	#TC#
+user_comment	
+Separator	-----------------------------------------------------
+TABLE_CATALOG	def
+TABLE_SCHEMA	information_schema
 TABLE_NAME	STATISTICS
 TABLE_TYPE	SYSTEM VIEW
 ENGINE	NULL
@@ -1901,6 +1924,29 @@ user_comment
 Separator	-----------------------------------------------------
 TABLE_CATALOG	def
 TABLE_SCHEMA	information_schema
+TABLE_NAME	SLAVE_DB_LOAD
+TABLE_TYPE	SYSTEM VIEW
+ENGINE	NULL
+VERSION	10
+ROW_FORMAT	NULL
+TABLE_ROWS	#TBLR#
+AVG_ROW_LENGTH	#ARL#
+DATA_LENGTH	#DL#
+MAX_DATA_LENGTH	#MDL#
+INDEX_LENGTH	#IL#
+DATA_FREE	#DF#
+AUTO_INCREMENT	#AI#
+CREATE_TIME	#CRT#
+UPDATE_TIME	#UT#
+CHECK_TIME	#CT#
+TABLE_COLLATION	NULL
+CHECKSUM	NULL
+CREATE_OPTIONS	#CO#
+TABLE_COMMENT	#TC#
+user_comment	
+Separator	-----------------------------------------------------
+TABLE_CATALOG	def
+TABLE_SCHEMA	information_schema
 TABLE_NAME	STATISTICS
 TABLE_TYPE	SYSTEM VIEW
 ENGINE	NULL

mysql-test/suite/information_schema/r/information_schema_db.result

@@ -55,6 +55,7 @@ ROUTINES
 SCHEMATA
 SCHEMATA_EXTENSIONS
 SCHEMA_PRIVILEGES
+SLAVE_DB_LOAD
 STATISTICS
 ST_GEOMETRY_COLUMNS
 ST_SPATIAL_REFERENCE_SYSTEMS

mysql-test/suite/rpl/r/rpl_slave_db_load_table.result

@@ -0,0 +1,32 @@
+include/master-slave.inc
+Warnings:
+Note	####	Sending passwords in plain text without SSL/TLS is extremely insecure.
+Note	####	Storing MySQL user name or password information in the connection metadata repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START REPLICA; see the 'START REPLICA Syntax' in the MySQL Manual for more information.
+[connection master]
+include/rpl_restart_server.inc [server_number=2]
+[connection slave]
+set global mts_dynamic_rebalance=TRUE;
+set global replica_parallel_workers=2;
+set global debug="+d,skip_checkpoint_load_reset";
+include/start_slave.inc
+select * from information_schema.slave_db_load;
+DB	WORKER	DB_LOAD
+[connection master]
+create table t1 (a int);
+insert into t1 values(1);
+insert into t1 values(1);
+insert into t1 values(1);
+insert into t1 values(1);
+insert into t1 values(1);
+include/sync_slave_sql_with_master.inc
+[connection slave]
+select * from information_schema.slave_db_load;
+DB	WORKER	DB_LOAD
+test	1	6
+include/stop_slave.inc
+set global mts_dynamic_rebalance=0;
+set global debug="-d,skip_checkpoint_load_reset";
+include/start_slave.inc
+[connection master]
+drop table t1;
+include/rpl_end.inc

mysql-test/suite/rpl/t/rpl_slave_db_load_table-slave.opt

@@ -0,0 +1 @@
+--replica_parallel_type='DATABASE' --replica_preserve_commit_order=OFF

mysql-test/suite/rpl/t/rpl_slave_db_load_table.test

@@ -0,0 +1,47 @@
+# This test verifies the functionality of slave_db_load table.
+
+source include/have_debug.inc;
+source include/master-slave.inc;
+
+let rpl_server_number=2;
+source include/rpl_restart_server.inc;
+
+source include/rpl_connection_slave.inc;
+let $old_mts_dynamic_rebalance= `select @@global.mts_dynamic_rebalance`;
+let $old_replica_parallel_workers= `select @@global.replica_parallel_workers`;
+set global mts_dynamic_rebalance=TRUE;
+set global replica_parallel_workers=2;
+set global debug="+d,skip_checkpoint_load_reset";
+source include/start_slave.inc;
+
+# Load should be empty
+select * from information_schema.slave_db_load;
+
+source include/rpl_connection_master.inc;
+create table t1 (a int);
+let $num_inserts=5;
+while ($num_inserts)
+{
+  insert into t1 values(1);
+  dec $num_inserts;
+}
+
+source include/sync_slave_sql_with_master.inc;
+
+source include/rpl_connection_slave.inc;
+# Load of test should be 6 (1 create and 5 inserts)
+select * from information_schema.slave_db_load;
+
+# restore varaibles
+source include/stop_slave.inc;
+eval set global mts_dynamic_rebalance=$old_mts_dynamic_rebalance;
+--disable_query_log
+eval set global replica_parallel_workers=$old_replica_parallel_workers;
+--enable_query_log
+set global debug="-d,skip_checkpoint_load_reset";
+source include/start_slave.inc;
+
+source include/rpl_connection_master.inc;
+drop table t1;
+
+source include/rpl_end.inc;

mysql-test/suite/sys_vars/r/mts_dynamic_rebalance_basic.result

@@ -0,0 +1,21 @@
+set @save.mts_dynamic_rebalance= @@global.mts_dynamic_rebalance;
+select @@session.mts_dynamic_rebalance;
+ERROR HY000: Variable 'mts_dynamic_rebalance' is a GLOBAL variable
+select variable_name from performance_schema.global_variables where variable_name='$var';
+variable_name
+select variable_name from performance_schema.session_variables where variable_name='$var';
+variable_name
+set @@global.mts_dynamic_rebalance= false;
+select @@global.mts_dynamic_rebalance;
+@@global.mts_dynamic_rebalance
+0
+set @@global.mts_dynamic_rebalance= 1.1;
+ERROR 42000: Incorrect argument type to variable 'mts_dynamic_rebalance'
+set @@global.mts_dynamic_rebalance= "foo";
+ERROR 42000: Variable 'mts_dynamic_rebalance' can't be set to the value of 'foo'
+set @@global.mts_dynamic_rebalance= false;
+set @@global.mts_dynamic_rebalance= true;
+select @@global.mts_dynamic_rebalance as "truncated to the maximum";
+truncated to the maximum
+1
+set @@global.mts_dynamic_rebalance= @save.mts_dynamic_rebalance;

mysql-test/suite/sys_vars/r/mts_imbalance_threshold_basic.result

@@ -0,0 +1,23 @@
+set @save.mts_imbalance_threshold= @@global.mts_imbalance_threshold;
+select @@session.mts_imbalance_threshold;
+ERROR HY000: Variable 'mts_imbalance_threshold' is a GLOBAL variable
+select variable_name from performance_schema.global_variables where variable_name='$var';
+variable_name
+select variable_name from performance_schema.session_variables where variable_name='$var';
+variable_name
+set @@global.mts_imbalance_threshold= 0.0;
+select @@global.mts_imbalance_threshold;
+@@global.mts_imbalance_threshold
+0.000000
+set @@global.mts_imbalance_threshold= test;
+ERROR 42000: Incorrect argument type to variable 'mts_imbalance_threshold'
+set @@global.mts_imbalance_threshold= "foo";
+ERROR 42000: Incorrect argument type to variable 'mts_imbalance_threshold'
+set @@global.mts_imbalance_threshold= 0;
+set @@global.mts_imbalance_threshold= cast(-1 as unsigned int);
+Warnings:
+Warning	1292	Truncated incorrect mts_imbalance_threshold value: '1.8446744073709552e19'
+select @@global.mts_imbalance_threshold as "truncated to the maximum";
+truncated to the maximum
+100.000000
+set @@global.mts_imbalance_threshold= @save.mts_imbalance_threshold;