Option to shutdown master if semi-sync is being switched off while th…

…ere are active un-acked trxns

Summary:
Semi-sync master can be switched off manually or because of a timeout while
waiting for an ack.

This change introduces a new variable rpl_semi_sync_master_crash_if_active_trxs
which when set to true will cause the master to shutdown (after printing an
error msg) if semi-sync is being switched off while there are active un-acked
transactions. This prevents any un-acked trx from committing on the
master.

Reviewed By: hermanlee

Differential Revision: D5841339

fbshipit-source-id: c378441

mysql-test/suite/rpl/r/rpl_semi_sync_master_crash_if_active_trxs.result

@@ -0,0 +1,41 @@
+include/master-slave.inc
+Warnings:
+Note	####	Sending passwords in plain text without SSL/TLS is extremely insecure.
+Note	####	Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
+[connection master]
+call mtr.add_suppression("Timeout waiting for reply of binlog*");
+call mtr.add_suppression("Force shutdown: Semi-sync master is being switched off while there are active un-acked transactions");
+create table t1(a int) engine=innodb;
+include/stop_slave.inc
+SET GLOBAL DEBUG='+d,dont_send_semi_sync_reply';
+include/start_slave.inc
+insert into t1 values(1);
+ERROR HY000: Lost connection to MySQL server during query
+SET GLOBAL DEBUG='-d,dont_send_semi_sync_reply';
+# Restart the master server
+include/rpl_start_server.inc [server_number=1]
+include/stop_slave.inc
+SET GLOBAL DEBUG='+d,dont_send_semi_sync_reply';
+include/start_slave.inc
+set @@global.rpl_semi_sync_master_timeout= 1000000;
+insert into t1 values(2);
+set @@global.rpl_semi_sync_master_enabled= OFF;
+ERROR HY000: Lost connection to MySQL server during query
+ERROR HY000: Lost connection to MySQL server during query
+SET GLOBAL DEBUG='-d,dont_send_semi_sync_reply';
+# Restart the master server
+include/rpl_start_server.inc [server_number=1]
+"Table at the master"
+select * from t1;
+a
+1
+2
+"Table at the slave"
+select * from t1;
+a
+1
+2
+set @@global.rpl_semi_sync_master_enabled= OFF;
+set @@global.rpl_semi_sync_master_enabled= ON;
+drop table t1;
+include/rpl_end.inc

mysql-test/suite/rpl/t/rpl_semi_sync_master_crash_if_active_trxs.cnf

@@ -0,0 +1,7 @@
+!include ../my.cnf
+[mysqld.1]
+rpl_semi_sync_master_enabled=1
+rpl_semi_sync_master_timeout=1000 # 1 second
+rpl_semi_sync_master_crash_if_active_trxs=1
+[mysqld.2]
+rpl_semi_sync_slave_enabled=1

mysql-test/suite/rpl/t/rpl_semi_sync_master_crash_if_active_trxs.test

@@ -0,0 +1,115 @@
+source include/master-slave.inc;
+source include/have_debug.inc;
+
+
+call mtr.add_suppression("Timeout waiting for reply of binlog*");
+call mtr.add_suppression("Force shutdown: Semi-sync master is being switched off while there are active un-acked transactions");
+
+connection master;
+create table t1(a int) engine=innodb;
+sync_slave_with_master;
+
+# Disable ack on slave
+connection slave;
+source include/stop_slave.inc;
+SET GLOBAL DEBUG='+d,dont_send_semi_sync_reply';
+source include/start_slave.inc;
+
+##
+## Ack timeout when there are active un-acked trxs should shutdown the master
+##
+connection master;
+enable_reconnect;
+exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect;
+# The master will die while executing this stmt because slave will not ack it
+error 2013;
+insert into t1 values(1);
+
+# Wait for master to die
+source include/wait_until_disconnected.inc;
+
+# Resume normal slave operations i.e start acking
+connection slave;
+SET GLOBAL DEBUG='-d,dont_send_semi_sync_reply';
+
+# Restart master
+echo # Restart the master server;
+let $rpl_server_number= 1;
+source include/rpl_start_server.inc;
+disable_reconnect;
+
+connection master;
+sync_slave_with_master;
+
+##
+## Turning off semi-sync when there are active un-acked trxs should shutdown the
+## master
+##
+
+# Disable ack on the slave
+connection slave;
+source include/stop_slave.inc;
+SET GLOBAL DEBUG='+d,dont_send_semi_sync_reply';
+source include/start_slave.inc;
+
+connection master;
+# Set timeout to a large value so that we have a chance to switch semi-sync off
+# manually
+set @@global.rpl_semi_sync_master_timeout= 1000000;
+send insert into t1 values(2); # this stmt will hang
+
+# Wait till the insert is processed on the slave
+connection slave;
+let $wait_condition= SELECT count(*) = 1 from t1 where a = 2;
+source include/wait_condition.inc;
+
+connection master1;
+enable_reconnect;
+exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect;
+error 2013;
+set @@global.rpl_semi_sync_master_enabled= OFF; # master will shutdown
+
+# Wait for master to die
+source include/wait_until_disconnected.inc;
+
+connection master;
+error 2013;
+reap;
+
+# Resume normal slave operations i.e start acking
+connection slave;
+SET GLOBAL DEBUG='-d,dont_send_semi_sync_reply';
+
+# Restart master
+echo # Restart the master server;
+let $rpl_server_number= 1;
+source include/rpl_start_server.inc;
+disable_reconnect;
+
+connection master;
+sync_slave_with_master;
+
+# Will contain 2 rows because master recovery rolls trxs forward
+echo "Table at the master";
+connection master;
+select * from t1;
+
+# Will contain 2 rows because slaves received all trxs
+echo "Table at the slave";
+connection slave;
+select * from t1;
+
+##
+## Turning off semi-sync when there are no active un-acked trxs should succeed
+##
+connection master;
+set @@global.rpl_semi_sync_master_enabled= OFF; # should not fail
+set @@global.rpl_semi_sync_master_enabled= ON;
+
+# Cleanup
+connection master;
+drop table t1;
+sync_slave_with_master;
+
+
+source include/rpl_end.inc;

mysql-test/suite/sys_vars/r/rpl_semi_sync_master_crash_if_active_trxs_basic.result

@@ -0,0 +1,21 @@
+set @save.rpl_semi_sync_master_crash_if_active_trxs= @@global.rpl_semi_sync_master_crash_if_active_trxs;
+select @@session.rpl_semi_sync_master_crash_if_active_trxs;
+ERROR HY000: Variable 'rpl_semi_sync_master_crash_if_active_trxs' is a GLOBAL variable
+select variable_name from information_schema.global_variables where variable_name='$var';
+variable_name
+select variable_name from information_schema.session_variables where variable_name='$var';
+variable_name
+set @@global.rpl_semi_sync_master_crash_if_active_trxs= false;
+select @@global.rpl_semi_sync_master_crash_if_active_trxs;
+@@global.rpl_semi_sync_master_crash_if_active_trxs
+0
+set @@global.rpl_semi_sync_master_crash_if_active_trxs= 1.1;
+ERROR 42000: Incorrect argument type to variable 'rpl_semi_sync_master_crash_if_active_trxs'
+set @@global.rpl_semi_sync_master_crash_if_active_trxs= "foo";
+ERROR 42000: Variable 'rpl_semi_sync_master_crash_if_active_trxs' can't be set to the value of 'foo'
+set @@global.rpl_semi_sync_master_crash_if_active_trxs= false;
+set @@global.rpl_semi_sync_master_crash_if_active_trxs= true;
+select @@global.rpl_semi_sync_master_crash_if_active_trxs as "truncated to the maximum";
+truncated to the maximum
+1
+set @@global.rpl_semi_sync_master_crash_if_active_trxs= @save.rpl_semi_sync_master_crash_if_active_trxs;

mysql-test/suite/sys_vars/t/rpl_semi_sync_master_crash_if_active_trxs_basic.test

@@ -0,0 +1,38 @@
+--source include/not_embedded.inc
+
+let $var= rpl_semi_sync_master_crash_if_active_trxs;
+eval set @save.$var= @@global.$var;
+
+#
+# exists as global only
+#
+--error ER_INCORRECT_GLOBAL_LOCAL_VAR
+eval select @@session.$var;
+
+select variable_name from information_schema.global_variables where variable_name='$var';
+select variable_name from information_schema.session_variables where variable_name='$var';
+
+#
+# show that it's writable
+#
+let $value= false;
+eval set @@global.$var= $value;
+eval select @@global.$var;
+
+#
+# incorrect value
+#
+--error ER_WRONG_TYPE_FOR_VAR
+eval set @@global.$var= 1.1;
+--error ER_WRONG_VALUE_FOR_VAR
+eval set @@global.$var= "foo";
+
+#
+# min/max values
+#
+eval set @@global.$var= false;
+eval set @@global.$var= true;
+eval select @@global.$var as "truncated to the maximum";
+
+# cleanup
+eval set @@global.$var= @save.$var;

plugin/semisync/semisync_master.cc

@@ -28,6 +28,7 @@
 /* This indicates whether semi-synchronous replication is enabled. */
 char rpl_semi_sync_master_enabled;
 unsigned long rpl_semi_sync_master_timeout;
+char rpl_semi_sync_master_crash_if_active_trxs;
 unsigned long rpl_semi_sync_master_trace_level;
 char rpl_semi_sync_master_status                    = 0;
 unsigned long rpl_semi_sync_master_yes_transactions = 0;
@@ -908,6 +909,13 @@ int ReplSemiSyncMaster::switch_off()
 {
   const char *kWho = "ReplSemiSyncMaster::switch_off";
 
+  if (rpl_semi_sync_master_crash_if_active_trxs && !active_tranxs_->is_empty())
+  {
+    sql_print_error("Force shutdown: Semi-sync master is being switched off "
+                    "while there are active un-acked transactions");
+    exit(0);
+  }
+
   function_enter(kWho);
   state_ = false;
 

plugin/semisync/semisync_master.h

@@ -645,6 +645,7 @@ extern char rpl_semi_sync_master_enabled;
 extern char rpl_semi_sync_master_status;
 extern unsigned long rpl_semi_sync_master_clients;
 extern unsigned long rpl_semi_sync_master_timeout;
+extern char rpl_semi_sync_master_crash_if_active_trxs;
 extern unsigned long rpl_semi_sync_master_trace_level;
 extern unsigned long rpl_semi_sync_master_yes_transactions;
 extern unsigned long rpl_semi_sync_master_no_transactions;

plugin/semisync/semisync_master_plugin.cc

@@ -254,6 +254,13 @@ static MYSQL_SYSVAR_ULONG(timeout, rpl_semi_sync_master_timeout,
   fix_rpl_semi_sync_master_timeout,	// update
   10000, 0, ~0UL, 1);
 
+static MYSQL_SYSVAR_BOOL(crash_if_active_trxs,
+  rpl_semi_sync_master_crash_if_active_trxs,
+  PLUGIN_VAR_OPCMDARG,
+ "Crash if there is an attempt to switch off semi-sync master while there are "
+ "active un-acked transactions",
+  NULL, NULL, 0);
+
 static MYSQL_SYSVAR_BOOL(wait_no_slave, rpl_semi_sync_master_wait_no_slave,
   PLUGIN_VAR_OPCMDARG,
  "Wait until timeout when no semi-synchronous replication slave available (enabled by default). ",
@@ -277,6 +284,7 @@ static MYSQL_SYSVAR_STR(histogram_trx_wait_step_size,
 static SYS_VAR* semi_sync_master_system_vars[]= {
   MYSQL_SYSVAR(enabled),
   MYSQL_SYSVAR(timeout),
+  MYSQL_SYSVAR(crash_if_active_trxs),
   MYSQL_SYSVAR(wait_no_slave),
   MYSQL_SYSVAR(trace_level),
   MYSQL_SYSVAR(histogram_trx_wait_step_size),

plugin/semisync/semisync_slave_plugin.cc

@@ -18,6 +18,7 @@
 
 #include "semisync_slave.h"
 #include <mysql.h>
+#include <debug_sync.h>
 
 static ReplSemiSyncSlave repl_semisync;
 
@@ -64,6 +65,7 @@ int repl_semi_slave_queue_event(Binlog_relay_IO_param *param,
 {
   if (rpl_semi_sync_slave_status && semi_sync_need_reply)
   {
+      DBUG_EXECUTE_IF("dont_send_semi_sync_reply", { return 0; });
     /*
       We deliberately ignore the error in slaveReply, such error
       should not cause the slave IO thread to stop, and the error