BUG#13874553: rpl.rpl_stop_slave fails sporadically on pb2

The test case makes use of the fine DEBUG_SYNC facility. Furthermore,
since it needs synchronization on internal threads (dump and SQL
threads) the server code has DEBUG_SYNC commands internally deployed
and activated through the DBUG_EXECUTE_IF macro. The internal
DBUG_SYNC commands are then controlled from the test case through the
DEBUG variable.

There were three problems around the DEBUG + DEBUG_SYNC facility
usage:

1. When signaling the SQL thread to continue, the test would reset
   immediately the DEBUG_SYNC variable. This could mean that the SQL
   thread might loose the signal and continue to wait forever;

2. A similar scenario was happening with the dump thread on the
   master. This thread was instructed to wait, and later it would be
   signaled to continue, but immediately after the DEBUG_SYNC would be
   reset. This could lead to the dump thread missing the signal and
   wait forever;

3. The test was not cleaning itself up with respect to the
   instrumentation of the dump thread. This would leave the
   conditional execution of an internal DEBUG_SYNC command active
   (through the usage of DBUG_EXECUTE_IF). 

We fix #1 and #2 by waiting for the threads to receive the signal and
only then issue the reset. We fix #3 by reseting the DEBUG variable,
thus deactivating the dump thread internal DEBUG_SYNC command.

mysql-test/extra/rpl_tests/rpl_stop_slave.test

@@ -42,6 +42,10 @@ send STOP SLAVE SQL_THREAD;
 connection slave1;
 --echo # To resume slave SQL thread
 SET DEBUG_SYNC= 'now SIGNAL signal.continue';
+# wait for the sql thread to come out of the
+# waiting phase before resetting the signals
+--let $wait_condition= select count(*)=0 from information_schema.processlist where state LIKE '%debug sync point%' and info LIKE '%INSERT INTO t1 VALUES (1)%'
+--source include/wait_condition.inc
 SET DEBUG_SYNC= 'RESET';
 
 --echo

mysql-test/suite/rpl/r/rpl_stop_slave.result

@@ -93,10 +93,12 @@ DROP TABLE t1, t2;
 CREATE TABLE t1 (c1 INT KEY, c2 INT) ENGINE=InnoDB;
 CREATE TABLE t2 (c1 INT) ENGINE=MyISAM;
 INSERT INTO t1 VALUES(1, 1);
+include/stop_slave.inc
 [connection master]
+include/stop_dump_threads.inc
 SET GLOBAL debug= 'd,dump_thread_wait_before_send_xid';
 [connection slave]
-include/restart_slave.inc
+include/start_slave.inc
 BEGIN;
 UPDATE t1 SET c2 = 2 WHERE c1 = 1;
 [connection master]
@@ -115,8 +117,11 @@ SET DEBUG_SYNC= 'RESET';
 [connection slave]
 include/wait_for_slave_to_stop.inc
 [connection slave1]
+[connection master]
+SET GLOBAL debug= '$debug_save';
+include/stop_dump_threads.inc
+[connection slave1]
 include/start_slave.inc
 [connection master]
 DROP TABLE t1, t2;
-SET GLOBAL debug= $debug_save;
 include/rpl_end.inc

mysql-test/suite/rpl/t/rpl_stop_slave.test

@@ -72,14 +72,17 @@ CREATE TABLE t2 (c1 INT) ENGINE=MyISAM;
 INSERT INTO t1 VALUES(1, 1);
 
 sync_slave_with_master;
+--source include/stop_slave.inc
 
 --source include/rpl_connection_master.inc
+# make sure that there are no zombie threads
+--source include/stop_dump_threads.inc
 
 let $debug_save= `SELECT @@GLOBAL.debug`;
 SET GLOBAL debug= 'd,dump_thread_wait_before_send_xid';
 
 --source include/rpl_connection_slave.inc
-source include/restart_slave_sql.inc;
+--source include/start_slave.inc
 
 BEGIN;
 UPDATE t1 SET c2 = 2 WHERE c1 = 1;
@@ -91,6 +94,10 @@ INSERT INTO t2 VALUES(1);
 UPDATE t1 SET c2 = 3 WHERE c1 = 1;
 COMMIT;
 
+# wait for the dump thread reach the sync point
+--let $wait_condition= select count(*)=1 from information_schema.processlist where state LIKE '%debug sync point%' and command='Binlog Dump'
+--source include/wait_condition.inc
+
 --source include/rpl_connection_slave1.inc
 let $show_statement= SHOW PROCESSLIST;
 let $field= Info;
@@ -103,17 +110,35 @@ send STOP SLAVE;
 ROLLBACK;
 
 --source include/rpl_connection_master.inc
+
 SET DEBUG_SYNC= 'now SIGNAL signal.continue';
+# wait for the dump thread to come out of the
+# waiting phase before resetting the signals
+--let $wait_condition= select count(*)=0 from information_schema.processlist where state LIKE '%debug sync point%' and command='Binlog Dump'
+--source include/wait_condition.inc
 SET DEBUG_SYNC= 'RESET';
 
 --source include/rpl_connection_slave.inc
 source include/wait_for_slave_to_stop.inc;
 
 --source include/rpl_connection_slave1.inc
 reap;
+
+# Slave has stopped, thence lets make sure that
+# we kill the zombie dump threads. Also, make
+# sure that we disable the DBUG_EXECUTE_IF
+# that would set the dump thread to wait
+--source include/rpl_connection_master.inc
+SET GLOBAL debug= '$debug_save';
+# make sure that there are no zombie threads
+--source include/stop_dump_threads.inc
+
+--source include/rpl_connection_slave1.inc
+# now the dump thread on the master will start
+# from a clean slate, i.e. without the
+# DBUG_EXECUTE_IF set
 source include/start_slave.inc;
 
 --source include/rpl_connection_master.inc
 DROP TABLE t1, t2;
-SET GLOBAL debug= $debug_save;
 --source include/rpl_end.inc