New status variable gtid_committed to denote trxs committed to storag…

…e engine (facebook#1110) (facebook#1110)

Summary:
Gtid_executed can give a false sense of how many transactions are
committed because it's updated after they are written to the binlog. There are
scenarios where there might be a significant delay between write to binlog and
commit in the storage engine, like while using semi-sync protocol. This change
introduces another status variable gtid_committed that is updated after storage
engine commit.

8.0 porting notes:
gtid_executed is now updated at the end of group commit. That makes
'gtid_executed' what 'gtid_committed' was designed to be. That's why
this patch adds 'gtid_committed' as an alias for the existing
'gtid_executed' variable.

Reference Patch: facebook@14fdbb5004b

Pull Request resolved: facebook#1110

Reviewed By: lloyd

Differential Revision: D20121951

Pulled By: abhinav04sharma

mysql-test/include/mtr_check.sql

@@ -113,6 +113,7 @@ BEGIN
   SELECT /*+SET_VAR(use_secondary_engine=OFF)*/ * FROM performance_schema.global_variables
     WHERE variable_name NOT IN ('timestamp', 'server_uuid',
                                 'gtid_executed', 'gtid_purged',
+                                'gtid_committed',
                                 'group_replication_group_name',
                                 'keyring_file_data',
                                 'innodb_thread_sleep_delay')

mysql-test/suite/binlog_nogtid/r/binlog_persist_only_variables.result

@@ -32,6 +32,7 @@ AND (VARIABLE_NAME NOT LIKE 'rocksdb%')
 ORDER BY VARIABLE_NAME;
 CREATE TABLE non_persisted (name VARCHAR(100) PRIMARY KEY);
 INSERT INTO non_persisted VALUES
+('gtid_committed'),
 ('gtid_executed'), ('gtid_owned'), ('gtid_purged'), ('log_bin'),
 ('log_bin_basename'), ('log_bin_index'), ('relay_log'), ('relay_log_basename'),
 ('relay_log_index'), ('relay_log_info_file'), ('replica_load_tmpdir'),
@@ -93,6 +94,8 @@ SET PERSIST_ONLY binlog_trx_meta_data = @@GLOBAL.binlog_trx_meta_data;
 SET PERSIST_ONLY enable_binlog_hlc = @@GLOBAL.enable_binlog_hlc;
 SET PERSIST_ONLY enforce_gtid_consistency = @@GLOBAL.enforce_gtid_consistency;
 SET PERSIST_ONLY group_replication_consistency = @@GLOBAL.group_replication_consistency;
+SET PERSIST_ONLY gtid_committed = @@GLOBAL.gtid_committed;
+ERROR HY000: Variable 'gtid_committed' is a non persistent read only variable
 SET PERSIST_ONLY gtid_executed = @@GLOBAL.gtid_executed;
 ERROR HY000: Variable 'gtid_executed' is a non persistent read only variable
 SET PERSIST_ONLY gtid_executed_compression_period = @@GLOBAL.gtid_executed_compression_period;
@@ -346,6 +349,8 @@ RESET PERSIST binlog_trx_meta_data;
 RESET PERSIST enable_binlog_hlc;
 RESET PERSIST enforce_gtid_consistency;
 RESET PERSIST group_replication_consistency;
+RESET PERSIST gtid_committed;
+ERROR HY000: Variable gtid_committed does not exist in persisted config file
 RESET PERSIST gtid_executed;
 ERROR HY000: Variable gtid_executed does not exist in persisted config file
 RESET PERSIST gtid_executed_compression_period;

mysql-test/suite/binlog_nogtid/r/binlog_persist_variables.result

@@ -75,6 +75,8 @@ SET PERSIST binlog_trx_meta_data = @@GLOBAL.binlog_trx_meta_data;
 SET PERSIST enable_binlog_hlc = @@GLOBAL.enable_binlog_hlc;
 SET PERSIST enforce_gtid_consistency = @@GLOBAL.enforce_gtid_consistency;
 SET PERSIST group_replication_consistency = @@GLOBAL.group_replication_consistency;
+SET PERSIST gtid_committed = @@GLOBAL.gtid_committed;
+ERROR HY000: Variable 'gtid_committed' is a read only variable
 SET PERSIST gtid_executed = @@GLOBAL.gtid_executed;
 ERROR HY000: Variable 'gtid_executed' is a read only variable
 SET PERSIST gtid_executed_compression_period = @@GLOBAL.gtid_executed_compression_period;
@@ -329,6 +331,9 @@ RESET PERSIST IF EXISTS binlog_trx_meta_data;
 RESET PERSIST IF EXISTS enable_binlog_hlc;
 RESET PERSIST IF EXISTS enforce_gtid_consistency;
 RESET PERSIST IF EXISTS group_replication_consistency;
+RESET PERSIST IF EXISTS gtid_committed;
+Warnings:
+Warning	3615	Variable gtid_committed does not exist in persisted config file
 RESET PERSIST IF EXISTS gtid_executed;
 Warnings:
 Warning	3615	Variable gtid_executed does not exist in persisted config file

mysql-test/suite/binlog_nogtid/t/binlog_persist_only_variables.test

@@ -65,6 +65,7 @@ INSERT INTO rplvars (varname, varvalue)
 
 CREATE TABLE non_persisted (name VARCHAR(100) PRIMARY KEY);
 INSERT INTO non_persisted VALUES
+  ('gtid_committed'),
   ('gtid_executed'), ('gtid_owned'), ('gtid_purged'), ('log_bin'),
   ('log_bin_basename'), ('log_bin_index'), ('relay_log'), ('relay_log_basename'),
   ('relay_log_index'), ('relay_log_info_file'), ('replica_load_tmpdir'),
@@ -123,6 +124,7 @@ while ( $varid <= $countvars )
 --echo
 # Add variables that need to be excluded
 let $expected=`SELECT $countvars - 13 - (SELECT COUNT(*) FROM performance_schema.global_variables WHERE variable_name in (
+'gtid_committed',
 '_list_end_'
 ))`;
 --let $assert_text= 'Expect \$expected persisted variables in persisted_variables table.'

mysql-test/suite/binlog_nogtid/t/binlog_persist_variables.test

@@ -80,7 +80,7 @@ while ( $varid <= $countvars )
   --let $varnames = `SELECT varname FROM rplvars WHERE id=$varid;`
 
   # The following variables are either non persistent or read only variables.
-  if (`SELECT '$varnames' IN ('binlog_row_event_max_size', 'binlog_gtid_simple_recovery', 'gtid_executed', 'gtid_next', 'gtid_owned', 'gtid_purged', 'log_bin', 'log_bin_basename', 'log_bin_index', 'log_replica_updates', 'log_slave_updates', 'relay_log', 'relay_log_basename', 'relay_log_index', 'relay_log_index', 'relay_log_info_file', 'relay_log_recovery', 'relay_log_space_limit', 'replica_load_tmpdir', 'slave_load_tmpdir', 'replica_skip_errors', 'slave_skip_errors', 'log_bin_use_v1_row_events', 'binlog_rotate_encryption_master_key_at_startup', 'skip_replica_start', 'skip_slave_start')`)
+  if (`SELECT '$varnames' IN ('gtid_committed', 'binlog_row_event_max_size', 'binlog_gtid_simple_recovery', 'gtid_executed', 'gtid_next', 'gtid_owned', 'gtid_purged', 'log_bin', 'log_bin_basename', 'log_bin_index', 'log_replica_updates', 'log_slave_updates', 'relay_log', 'relay_log_basename', 'relay_log_index', 'relay_log_index', 'relay_log_info_file', 'relay_log_recovery', 'relay_log_space_limit', 'replica_load_tmpdir', 'slave_load_tmpdir', 'replica_skip_errors', 'slave_skip_errors', 'log_bin_use_v1_row_events', 'binlog_rotate_encryption_master_key_at_startup', 'skip_replica_start', 'skip_slave_start')`)
   {
     --error ER_INCORRECT_GLOBAL_LOCAL_VAR
   }
@@ -92,6 +92,7 @@ while ( $varid <= $countvars )
 --echo
 # Add variables that need to be excluded
 let $expected=`SELECT $countvars - 24 - (SELECT COUNT(*) FROM performance_schema.global_variables WHERE variable_name in (
+'gtid_committed',
 '_list_end_'
 ))`;
 --let $assert_text= 'Expect \$expected persisted variables in persisted_variables table.'

mysql-test/suite/rpl_gtid/r/rpl_gtid_committed.result

@@ -0,0 +1,24 @@
+include/master-slave.inc
+Warnings:
+Note	####	Sending passwords in plain text without SSL/TLS is extremely insecure.
+Note	####	Storing MySQL user name or password information in the connection metadata repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START REPLICA; see the 'START REPLICA Syntax' in the MySQL Manual for more information.
+[connection master]
+[connection master]
+[connection slave]
+SET @@global.debug = '+d,before_semi_sync_reply';
+[connection master]
+CREATE TABLE t1 (a INT);
+[connection slave]
+SET DEBUG_SYNC = 'now WAIT_FOR semi_sync_reply_reached';
+SET @@global.debug = '-d,before_semi_sync_reply';
+[connection master1]
+include/assert.inc ["gtid_executed should be empty"]
+include/assert.inc ["gtid_committed should be empty"]
+[connection slave]
+SET DEBUG_SYNC = 'now SIGNAL semi_sync_reply_continue';
+[connection master]
+include/assert.inc ["gtid_executed should be $master_uuid:1"]
+include/assert.inc ["gtid_committed should be $master_uuid:1"]
+DROP TABLE t1;
+call mtr.add_suppression("A message intended for a client cannot be sent there as no client-session is attached");
+include/rpl_end.inc

mysql-test/suite/rpl_gtid/t/rpl_gtid_committed-master.opt

@@ -0,0 +1 @@
+$SEMISYNC_MASTER_PLUGIN_OPT $SEMISYNC_MASTER_PLUGIN_LOAD

mysql-test/suite/rpl_gtid/t/rpl_gtid_committed-slave.opt

@@ -0,0 +1 @@
+$SEMISYNC_SLAVE_PLUGIN_OPT $SEMISYNC_SLAVE_PLUGIN_LOAD

mysql-test/suite/rpl_gtid/t/rpl_gtid_committed.cnf

@@ -0,0 +1,8 @@
+!include ../my.cnf
+
+[mysqld.1]
+rpl_semi_sync_master_enabled=1
+rpl_semi_sync_master_timeout=100000000
+
+[mysqld.2]
+rpl_semi_sync_slave_enabled=1

mysql-test/suite/rpl_gtid/t/rpl_gtid_committed.test

@@ -0,0 +1,46 @@
+--source include/have_debug_sync.inc
+--source include/have_semisync_plugin.inc
+--source include/master-slave.inc
+
+--source include/rpl_connection_master.inc
+--let $master_uuid = `SELECT @@server_uuid;`
+
+--source include/rpl_connection_slave.inc
+SET @@global.debug = '+d,before_semi_sync_reply';
+
+--source include/rpl_connection_master.inc
+--send CREATE TABLE t1 (a INT)
+
+--source include/rpl_connection_slave.inc
+SET DEBUG_SYNC = 'now WAIT_FOR semi_sync_reply_reached';
+SET @@global.debug = '-d,before_semi_sync_reply';
+
+--source include/rpl_connection_master1.inc
+--let $assert_text = "gtid_executed should be empty"
+--let $assert_cond = "[SELECT @@global.gtid_executed]" = ""
+--source include/assert.inc
+
+# gtid_committed is an alias of gtid_executed.
+--let $assert_text = "gtid_committed should be empty"
+--let $assert_cond = "[SELECT @@global.gtid_committed]" = ""
+--source include/assert.inc
+
+--source include/rpl_connection_slave.inc
+SET DEBUG_SYNC = 'now SIGNAL semi_sync_reply_continue';
+
+--source include/rpl_connection_master.inc
+--reap
+--let $assert_text = "gtid_executed should be \$master_uuid:1"
+--let $assert_cond = "[SELECT @@global.gtid_executed]" = "$master_uuid:1"
+--source include/assert.inc
+
+# gtid_committed is an alias of gtid_executed.
+--let $assert_text = "gtid_committed should be \$master_uuid:1"
+--let $assert_cond = "[SELECT @@global.gtid_committed]" = "$master_uuid:1"
+--source include/assert.inc
+
+DROP TABLE t1;
+
+call mtr.add_suppression("A message intended for a client cannot be sent there as no client-session is attached");
+
+--source include/rpl_end.inc

mysql-test/suite/sys_vars/r/gtid_committed_basic.result

@@ -0,0 +1,11 @@
+RESET MASTER;
+SET @start_global_value = @@global.gtid_committed;
+include/assert.inc ["gtid_committed is empty initially"]
+include/assert.inc ["@@global.gtid_committed is readable"]
+SELECT @@session.gtid_committed;
+ERROR HY000: Variable 'gtid_committed' is a GLOBAL variable
+Expected error 'Variable is a GLOBAL variable'
+include/assert.inc ["gtid_committed in global variables"]
+include/assert.inc ["gtid_committed in session variables"]
+include/assert.inc ["gtid_committed in performance_schema.global_variables"]
+include/assert.inc ["gtid_committed in performance_schema.session_variables"]

mysql-test/suite/sys_vars/t/gtid_committed_basic.test

@@ -0,0 +1,31 @@
+# Clear gtid_committed.
+RESET MASTER;
+
+SET @start_global_value = @@global.gtid_committed;
+--let $assert_text = "gtid_committed is empty initially"
+--let $assert_cond = "[SELECT @start_global_value]" = ""
+--source include/assert.inc
+
+--let $assert_text = "@@global.gtid_committed is readable"
+--let $assert_cond = "[SELECT @@global.gtid_committed]" = ""
+--source include/assert.inc
+
+--error ER_INCORRECT_GLOBAL_LOCAL_VAR
+SELECT @@session.gtid_committed;
+--echo Expected error 'Variable is a GLOBAL variable'
+
+--let $assert_text = "gtid_committed in global variables"
+--let $assert_cond = "[SHOW GLOBAL VARIABLES LIKE \'gtid_committed\', Variable_name, 1]" = "gtid_committed"
+--source include/assert.inc
+
+--let $assert_text = "gtid_committed in session variables"
+--let $assert_cond = "[SHOW SESSION VARIABLES LIKE \'gtid_committed\', Variable_name, 1]" = "gtid_committed"
+--source include/assert.inc
+
+--let $assert_text = "gtid_committed in performance_schema.global_variables"
+--let $assert_cond = "[SELECT Variable_Name FROM performance_schema.global_variables WHERE Variable_Name = \'gtid_committed\', Variable_Name, 1]" = "gtid_committed";
+--source include/assert.inc
+
+--let $assert_text = "gtid_committed in performance_schema.session_variables"
+--let $assert_cond = "[SELECT Variable_Name FROM performance_schema.session_variables WHERE Variable_Name = \'gtid_committed\', Variable_Name, 1]" = "gtid_committed";
+--source include/assert.inc

mysql-test/t/all_persisted_variables.test

@@ -48,6 +48,7 @@ let $total_persistent_vars= `SELECT $total_global_vars - 214`;
 let $total_excluded_vars=`SELECT COUNT(*) FROM performance_schema.global_variables WHERE variable_name in (
 'binlog_file_basedir',
 'binlog_index_basedir',
+'gtid_committed',
 'innodb_buffer_pool_populate',
 'innodb_log_flush_events',
 'innodb_log_recent_closed_size',

sql/sys_vars.cc

@@ -7229,6 +7229,11 @@ static Sys_var_gtid_executed Sys_gtid_executed(
     "binary log. The session variable contains the set of GTIDs "
     "in the current, ongoing transaction.");
 
+static Sys_var_gtid_executed Sys_gtid_committed(
+    "gtid_committed",
+    "The global variable contains the set of GTIDs committed in the storage "
+    "engine");
+
 static bool check_gtid_purged(sys_var *self, THD *thd, set_var *var) {
   DBUG_TRACE;