From 552e8f1ee2aa99b322073e8e39b5f177a95884a9 Mon Sep 17 00:00:00 2001 From: Abhinav Sharma Date: Wed, 26 Feb 2020 15:32:27 -0800 Subject: [PATCH] New status variable gtid_committed to denote trxs committed to storage engine (#1110) (#1110) Summary: Gtid_executed can give a false sense of how many transactions are committed because it's updated after they are written to the binlog. There are scenarios where there might be a significant delay between write to binlog and commit in the storage engine, like while using semi-sync protocol. This change introduces another status variable gtid_committed that is updated after storage engine commit. 8.0 porting notes: gtid_executed is now updated at the end of group commit. That makes 'gtid_executed' what 'gtid_committed' was designed to be. That's why this patch adds 'gtid_committed' as an alias for the existing 'gtid_executed' variable. Reference Patch: https://github.com/facebook/mysql-5.6/commit/14fdbb5004b Pull Request resolved: https://github.com/facebook/mysql-5.6/pull/1110 Reviewed By: lloyd Differential Revision: D20121951 Pulled By: abhinav04sharma --- mysql-test/include/mtr_check.sql | 1 + .../r/binlog_persist_only_variables.result | 4 ++ .../r/binlog_persist_variables.result | 5 ++ .../t/binlog_persist_only_variables.test | 9 +++- .../t/binlog_persist_variables.test | 7 ++- .../rpl_gtid/r/rpl_gtid_committed.result | 24 ++++++++++ .../rpl_gtid/t/rpl_gtid_committed-master.opt | 1 + .../rpl_gtid/t/rpl_gtid_committed-slave.opt | 1 + .../suite/rpl_gtid/t/rpl_gtid_committed.cnf | 8 ++++ .../suite/rpl_gtid/t/rpl_gtid_committed.test | 46 +++++++++++++++++++ .../sys_vars/r/gtid_committed_basic.result | 11 +++++ .../sys_vars/t/gtid_committed_basic.test | 31 +++++++++++++ mysql-test/t/all_persisted_variables.test | 1 + sql/sys_vars.cc | 5 ++ 14 files changed, 151 insertions(+), 3 deletions(-) create mode 100644 mysql-test/suite/rpl_gtid/r/rpl_gtid_committed.result create mode 100644 mysql-test/suite/rpl_gtid/t/rpl_gtid_committed-master.opt create mode 100644 mysql-test/suite/rpl_gtid/t/rpl_gtid_committed-slave.opt create mode 100644 mysql-test/suite/rpl_gtid/t/rpl_gtid_committed.cnf create mode 100644 mysql-test/suite/rpl_gtid/t/rpl_gtid_committed.test create mode 100644 mysql-test/suite/sys_vars/r/gtid_committed_basic.result create mode 100644 mysql-test/suite/sys_vars/t/gtid_committed_basic.test diff --git a/mysql-test/include/mtr_check.sql b/mysql-test/include/mtr_check.sql index 80cd538bee3e..1c5323f3bd24 100644 --- a/mysql-test/include/mtr_check.sql +++ b/mysql-test/include/mtr_check.sql @@ -78,6 +78,7 @@ BEGIN SELECT * FROM performance_schema.global_variables WHERE variable_name NOT IN ('timestamp', 'server_uuid', 'gtid_executed', 'gtid_purged', + 'gtid_committed', 'group_replication_group_name', 'keyring_file_data', 'innodb_thread_sleep_delay') diff --git a/mysql-test/suite/binlog_nogtid/r/binlog_persist_only_variables.result b/mysql-test/suite/binlog_nogtid/r/binlog_persist_only_variables.result index 2c7ce27df54f..d230e014fe54 100644 --- a/mysql-test/suite/binlog_nogtid/r/binlog_persist_only_variables.result +++ b/mysql-test/suite/binlog_nogtid/r/binlog_persist_only_variables.result @@ -59,6 +59,8 @@ SET PERSIST_ONLY binlog_transaction_dependency_tracking = @@GLOBAL.binlog_transa SET PERSIST_ONLY binlog_trx_meta_data = @@GLOBAL.binlog_trx_meta_data; SET PERSIST_ONLY enable_binlog_hlc = @@GLOBAL.enable_binlog_hlc; SET PERSIST_ONLY enforce_gtid_consistency = @@GLOBAL.enforce_gtid_consistency; +SET PERSIST_ONLY gtid_committed = @@GLOBAL.gtid_committed; +ERROR HY000: Variable 'gtid_committed' is a non persistent read only variable SET PERSIST_ONLY gtid_executed = @@GLOBAL.gtid_executed; ERROR HY000: Variable 'gtid_executed' is a non persistent read only variable SET PERSIST_ONLY gtid_executed_compression_period = @@GLOBAL.gtid_executed_compression_period; @@ -196,6 +198,8 @@ RESET PERSIST binlog_transaction_dependency_tracking; RESET PERSIST binlog_trx_meta_data; RESET PERSIST enable_binlog_hlc; RESET PERSIST enforce_gtid_consistency; +RESET PERSIST gtid_committed; +ERROR HY000: Variable gtid_committed does not exist in persisted config file RESET PERSIST gtid_executed; ERROR HY000: Variable gtid_executed does not exist in persisted config file RESET PERSIST gtid_executed_compression_period; diff --git a/mysql-test/suite/binlog_nogtid/r/binlog_persist_variables.result b/mysql-test/suite/binlog_nogtid/r/binlog_persist_variables.result index 842440f5b07a..74fbe01623e6 100644 --- a/mysql-test/suite/binlog_nogtid/r/binlog_persist_variables.result +++ b/mysql-test/suite/binlog_nogtid/r/binlog_persist_variables.result @@ -62,6 +62,8 @@ SET PERSIST binlog_transaction_dependency_tracking = @@GLOBAL.binlog_transaction SET PERSIST binlog_trx_meta_data = @@GLOBAL.binlog_trx_meta_data; SET PERSIST enable_binlog_hlc = @@GLOBAL.enable_binlog_hlc; SET PERSIST enforce_gtid_consistency = @@GLOBAL.enforce_gtid_consistency; +SET PERSIST gtid_committed = @@GLOBAL.gtid_committed; +ERROR HY000: Variable 'gtid_committed' is a read only variable SET PERSIST gtid_executed = @@GLOBAL.gtid_executed; ERROR HY000: Variable 'gtid_executed' is a read only variable SET PERSIST gtid_executed_compression_period = @@GLOBAL.gtid_executed_compression_period; @@ -209,6 +211,9 @@ RESET PERSIST IF EXISTS binlog_transaction_dependency_tracking; RESET PERSIST IF EXISTS binlog_trx_meta_data; RESET PERSIST IF EXISTS enable_binlog_hlc; RESET PERSIST IF EXISTS enforce_gtid_consistency; +RESET PERSIST IF EXISTS gtid_committed; +Warnings: +Warning 3615 Variable gtid_committed does not exist in persisted config file RESET PERSIST IF EXISTS gtid_executed; Warnings: Warning 3615 Variable gtid_executed does not exist in persisted config file diff --git a/mysql-test/suite/binlog_nogtid/t/binlog_persist_only_variables.test b/mysql-test/suite/binlog_nogtid/t/binlog_persist_only_variables.test index f8dfc85eddbe..af5f28e59fb2 100644 --- a/mysql-test/suite/binlog_nogtid/t/binlog_persist_only_variables.test +++ b/mysql-test/suite/binlog_nogtid/t/binlog_persist_only_variables.test @@ -74,7 +74,7 @@ while ( $varid <= $countvars ) --let $varnames= `SELECT varname FROM rplvars WHERE id=$varid` # Following variables are non persistent read only variables. - if (`SELECT '$varnames' IN ('gtid_executed', 'gtid_owned', 'log_bin', 'log_bin_basename', 'log_bin_index', 'relay_log', 'relay_log_basename', 'relay_log_index', 'relay_log_info_file', 'slave_load_tmpdir', 'log_bin_use_v1_row_events')`) + if (`SELECT '$varnames' IN ('gtid_committed', 'gtid_executed', 'gtid_owned', 'log_bin', 'log_bin_basename', 'log_bin_index', 'relay_log', 'relay_log_basename', 'relay_log_index', 'relay_log_info_file', 'slave_load_tmpdir', 'log_bin_use_v1_row_events')`) { --error ER_INCORRECT_GLOBAL_LOCAL_VAR } @@ -94,6 +94,11 @@ while ( $varid <= $countvars ) --echo --let $expected= `SELECT $countvars - 11` +# gtid_committed needs to be excluded +if (`SELECT COUNT(*) FROM performance_schema.global_variables WHERE variable_name = 'gtid_committed'`) +{ + --dec $expected +} --let $assert_text= 'Expect \$expected persisted variables in persisted_variables table.' --let $assert_cond= [SELECT COUNT(*) as count FROM performance_schema.persisted_variables, count, 1] = $expected --source include/assert.inc @@ -129,7 +134,7 @@ while ( $varid <= $countvars ) { --let $varnames= `SELECT varname FROM rplvars WHERE id=$varid` - if (`SELECT '$varnames' IN ('gtid_executed', 'gtid_owned', 'log_bin', 'log_bin_basename', 'log_bin_index', 'relay_log', 'relay_log_basename', 'relay_log_index', 'relay_log_info_file', 'slave_load_tmpdir', 'log_bin_use_v1_row_events')`) + if (`SELECT '$varnames' IN ('gtid_committed', 'gtid_executed', 'gtid_owned', 'log_bin', 'log_bin_basename', 'log_bin_index', 'relay_log', 'relay_log_basename', 'relay_log_index', 'relay_log_info_file', 'slave_load_tmpdir', 'log_bin_use_v1_row_events')`) { --error ER_VAR_DOES_NOT_EXIST } diff --git a/mysql-test/suite/binlog_nogtid/t/binlog_persist_variables.test b/mysql-test/suite/binlog_nogtid/t/binlog_persist_variables.test index ead6914e10da..7d0a4cd736d7 100644 --- a/mysql-test/suite/binlog_nogtid/t/binlog_persist_variables.test +++ b/mysql-test/suite/binlog_nogtid/t/binlog_persist_variables.test @@ -75,7 +75,7 @@ while ( $varid <= $countvars ) --let $varnames= `SELECT varname FROM rplvars WHERE id=$varid;` # Following variables are either non persistent or read only variables. - if (`SELECT '$varnames' IN ('binlog_row_event_max_size', 'binlog_gtid_simple_recovery', 'gtid_executed', 'gtid_next', 'gtid_owned', 'log_bin', 'log_bin_basename', 'log_bin_index', 'log_slave_updates', 'relay_log', 'relay_log_basename', 'relay_log_index', 'relay_log_index', 'relay_log_info_file', 'relay_log_recovery', 'relay_log_space_limit', 'slave_load_tmpdir', 'slave_load_tmpdir', 'slave_skip_errors', 'log_bin_use_v1_row_events', 'binlog_rotate_encryption_master_key_at_startup')`) + if (`SELECT '$varnames' IN ('binlog_row_event_max_size', 'binlog_gtid_simple_recovery', 'gtid_committed', 'gtid_executed', 'gtid_next', 'gtid_owned', 'log_bin', 'log_bin_basename', 'log_bin_index', 'log_slave_updates', 'relay_log', 'relay_log_basename', 'relay_log_index', 'relay_log_index', 'relay_log_info_file', 'relay_log_recovery', 'relay_log_space_limit', 'slave_load_tmpdir', 'slave_load_tmpdir', 'slave_skip_errors', 'log_bin_use_v1_row_events', 'binlog_rotate_encryption_master_key_at_startup')`) { --error ER_INCORRECT_GLOBAL_LOCAL_VAR } @@ -86,6 +86,11 @@ while ( $varid <= $countvars ) --echo --let $expected= `SELECT $countvars - 18` +# gtid_committed needs to be excluded +if (`SELECT COUNT(*) FROM performance_schema.global_variables WHERE variable_name = 'gtid_committed'`) +{ + --dec $expected +} --let $assert_text= 'Expect \$expected persisted variables in persisted_variables table.' --let $assert_cond= [SELECT COUNT(*) as count FROM performance_schema.persisted_variables, count, 1] = $expected --source include/assert.inc diff --git a/mysql-test/suite/rpl_gtid/r/rpl_gtid_committed.result b/mysql-test/suite/rpl_gtid/r/rpl_gtid_committed.result new file mode 100644 index 000000000000..024774f00db3 --- /dev/null +++ b/mysql-test/suite/rpl_gtid/r/rpl_gtid_committed.result @@ -0,0 +1,24 @@ +include/master-slave.inc +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +[connection master] +[connection master] +[connection slave] +SET @@global.debug = '+d,before_semi_sync_reply'; +[connection master] +CREATE TABLE t1 (a INT); +[connection slave] +SET DEBUG_SYNC = 'now WAIT_FOR semi_sync_reply_reached'; +SET @@global.debug = '-d,before_semi_sync_reply'; +[connection master1] +include/assert.inc ["gtid_executed should be empty"] +include/assert.inc ["gtid_committed should be empty"] +[connection slave] +SET DEBUG_SYNC = 'now SIGNAL semi_sync_reply_continue'; +[connection master] +include/assert.inc ["gtid_executed should be $master_uuid:1"] +include/assert.inc ["gtid_committed should be $master_uuid:1"] +DROP TABLE t1; +call mtr.add_suppression("A message intended for a client cannot be sent there as no client-session is attached"); +include/rpl_end.inc diff --git a/mysql-test/suite/rpl_gtid/t/rpl_gtid_committed-master.opt b/mysql-test/suite/rpl_gtid/t/rpl_gtid_committed-master.opt new file mode 100644 index 000000000000..8c9e13fe2c4a --- /dev/null +++ b/mysql-test/suite/rpl_gtid/t/rpl_gtid_committed-master.opt @@ -0,0 +1 @@ +$SEMISYNC_MASTER_PLUGIN_OPT $SEMISYNC_MASTER_PLUGIN_LOAD diff --git a/mysql-test/suite/rpl_gtid/t/rpl_gtid_committed-slave.opt b/mysql-test/suite/rpl_gtid/t/rpl_gtid_committed-slave.opt new file mode 100644 index 000000000000..7585c48909a2 --- /dev/null +++ b/mysql-test/suite/rpl_gtid/t/rpl_gtid_committed-slave.opt @@ -0,0 +1 @@ +$SEMISYNC_SLAVE_PLUGIN_OPT $SEMISYNC_SLAVE_PLUGIN_LOAD diff --git a/mysql-test/suite/rpl_gtid/t/rpl_gtid_committed.cnf b/mysql-test/suite/rpl_gtid/t/rpl_gtid_committed.cnf new file mode 100644 index 000000000000..10b884192228 --- /dev/null +++ b/mysql-test/suite/rpl_gtid/t/rpl_gtid_committed.cnf @@ -0,0 +1,8 @@ +!include ../my.cnf + +[mysqld.1] +rpl_semi_sync_master_enabled=1 +rpl_semi_sync_master_timeout=100000000 + +[mysqld.2] +rpl_semi_sync_slave_enabled=1 diff --git a/mysql-test/suite/rpl_gtid/t/rpl_gtid_committed.test b/mysql-test/suite/rpl_gtid/t/rpl_gtid_committed.test new file mode 100644 index 000000000000..30704e50b43c --- /dev/null +++ b/mysql-test/suite/rpl_gtid/t/rpl_gtid_committed.test @@ -0,0 +1,46 @@ +--source include/have_debug_sync.inc +--source include/have_semisync_plugin.inc +--source include/master-slave.inc + +--source include/rpl_connection_master.inc +--let $master_uuid = `SELECT @@server_uuid;` + +--source include/rpl_connection_slave.inc +SET @@global.debug = '+d,before_semi_sync_reply'; + +--source include/rpl_connection_master.inc +--send CREATE TABLE t1 (a INT) + +--source include/rpl_connection_slave.inc +SET DEBUG_SYNC = 'now WAIT_FOR semi_sync_reply_reached'; +SET @@global.debug = '-d,before_semi_sync_reply'; + +--source include/rpl_connection_master1.inc +--let $assert_text = "gtid_executed should be empty" +--let $assert_cond = "[SELECT @@global.gtid_executed]" = "" +--source include/assert.inc + +# gtid_committed is an alias of gtid_executed. +--let $assert_text = "gtid_committed should be empty" +--let $assert_cond = "[SELECT @@global.gtid_committed]" = "" +--source include/assert.inc + +--source include/rpl_connection_slave.inc +SET DEBUG_SYNC = 'now SIGNAL semi_sync_reply_continue'; + +--source include/rpl_connection_master.inc +--reap +--let $assert_text = "gtid_executed should be \$master_uuid:1" +--let $assert_cond = "[SELECT @@global.gtid_executed]" = "$master_uuid:1" +--source include/assert.inc + +# gtid_committed is an alias of gtid_executed. +--let $assert_text = "gtid_committed should be \$master_uuid:1" +--let $assert_cond = "[SELECT @@global.gtid_committed]" = "$master_uuid:1" +--source include/assert.inc + +DROP TABLE t1; + +call mtr.add_suppression("A message intended for a client cannot be sent there as no client-session is attached"); + +--source include/rpl_end.inc diff --git a/mysql-test/suite/sys_vars/r/gtid_committed_basic.result b/mysql-test/suite/sys_vars/r/gtid_committed_basic.result new file mode 100644 index 000000000000..c844b96c9f3f --- /dev/null +++ b/mysql-test/suite/sys_vars/r/gtid_committed_basic.result @@ -0,0 +1,11 @@ +RESET MASTER; +SET @start_global_value = @@global.gtid_committed; +include/assert.inc ["gtid_committed is empty initially"] +include/assert.inc ["@@global.gtid_committed is readable"] +SELECT @@session.gtid_committed; +ERROR HY000: Variable 'gtid_committed' is a GLOBAL variable +Expected error 'Variable is a GLOBAL variable' +include/assert.inc ["gtid_committed in global variables"] +include/assert.inc ["gtid_committed in session variables"] +include/assert.inc ["gtid_committed in performance_schema.global_variables"] +include/assert.inc ["gtid_committed in performance_schema.session_variables"] diff --git a/mysql-test/suite/sys_vars/t/gtid_committed_basic.test b/mysql-test/suite/sys_vars/t/gtid_committed_basic.test new file mode 100644 index 000000000000..42bda33e6f20 --- /dev/null +++ b/mysql-test/suite/sys_vars/t/gtid_committed_basic.test @@ -0,0 +1,31 @@ +# Clear gtid_committed. +RESET MASTER; + +SET @start_global_value = @@global.gtid_committed; +--let $assert_text = "gtid_committed is empty initially" +--let $assert_cond = "[SELECT @start_global_value]" = "" +--source include/assert.inc + +--let $assert_text = "@@global.gtid_committed is readable" +--let $assert_cond = "[SELECT @@global.gtid_committed]" = "" +--source include/assert.inc + +--error ER_INCORRECT_GLOBAL_LOCAL_VAR +SELECT @@session.gtid_committed; +--echo Expected error 'Variable is a GLOBAL variable' + +--let $assert_text = "gtid_committed in global variables" +--let $assert_cond = "[SHOW GLOBAL VARIABLES LIKE \'gtid_committed\', Variable_name, 1]" = "gtid_committed" +--source include/assert.inc + +--let $assert_text = "gtid_committed in session variables" +--let $assert_cond = "[SHOW SESSION VARIABLES LIKE \'gtid_committed\', Variable_name, 1]" = "gtid_committed" +--source include/assert.inc + +--let $assert_text = "gtid_committed in performance_schema.global_variables" +--let $assert_cond = "[SELECT Variable_Name FROM performance_schema.global_variables WHERE Variable_Name = \'gtid_committed\', Variable_Name, 1]" = "gtid_committed"; +--source include/assert.inc + +--let $assert_text = "gtid_committed in performance_schema.session_variables" +--let $assert_cond = "[SELECT Variable_Name FROM performance_schema.session_variables WHERE Variable_Name = \'gtid_committed\', Variable_Name, 1]" = "gtid_committed"; +--source include/assert.inc diff --git a/mysql-test/t/all_persisted_variables.test b/mysql-test/t/all_persisted_variables.test index e0f63e4adc2b..28899db111ce 100644 --- a/mysql-test/t/all_persisted_variables.test +++ b/mysql-test/t/all_persisted_variables.test @@ -48,6 +48,7 @@ let $total_persistent_vars= `SELECT $total_global_vars - 207`; let $total_excluded_vars=`SELECT COUNT(*) FROM performance_schema.global_variables WHERE variable_name in ( 'binlog_file_basedir', 'binlog_index_basedir', +'gtid_committed', 'innodb_buffer_pool_populate', 'per_user_session_var_user_name_delimiter', '_list_end_' diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 3a90f6fa7cb1..7d9303efdca6 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -6661,6 +6661,11 @@ static Sys_var_gtid_executed Sys_gtid_executed( "binary log. The session variable contains the set of GTIDs " "in the current, ongoing transaction."); +static Sys_var_gtid_executed Sys_gtid_committed( + "gtid_committed", + "The global variable contains the set of GTIDs committed in the storage " + "engine"); + static bool check_gtid_purged(sys_var *self, THD *thd, set_var *var) { DBUG_TRACE;