BUG#20136704 --SLAVE-PRESERVE-COMMIT-ORDER CAUSES SLAVE TO DEADLOCK AND

             BREAK FOR SOME QUERIE

A corner case caused slave hang when slave_preserve_commit_order is ON.

ANALYSIS
========
CREATE TABLE t1(c1 INT PRIMARY KEY, c2 INT, INDEX(c2)) ENGINE = InnoDB;
INSERT INTO t1 VALUES(1, NULL),(2, 2), (3, NULL), (4, 4), (5, NULL), (6, 6);

INSERT INTO t1 VALUES(7, NULL);
DELETE FROM t1 WHERE c2 <= 3;

On master, the INSERT statement acquires its row lock before DELETE STATEMENT.
Since the INSERT doesn't block the DELETE, Both statements may have same
commit parent which mean they can be applied parallel on slave.

On slave, they will be applied parallel if MTS is ON. There is a chance that
the DELETE acquires its lock before the INSERT. The DELETE holds a gap lock
on INDEX(c2) which blocks the INSERT statement. The INSERT cannot be applied
unless DELETE is committed or rolled back. And meanwhile the DELETE is waiting
for the INSERT to commit since slave_preserve_commit_order is ON. That is
a deadlock and hangs the slave. Here we call the deadlock as a order commit
deadlock.

FIX
===
A deadlock checking mechanism is introduced. Every time when a transaction
needs to wait for another transaction to release a row lock, innodb will
call a slave function to check if there is an order commit deadlock. If
it founds an order commit deadlock, It will set a deadlock flag to the
slave worker which is holding the row lock. Thereafter, the worker will
roll its transaction back and retry it again.

include/mysql/service_thd_engine_lock.h

@@ -0,0 +1,47 @@
+/* Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA */
+
+#ifndef MYSQL_SERVICE_THD_EGINE_LOCK_INCLUDED
+#define MYSQL_SERVICE_THD_EGINE_LOCK_INCLUDED
+
+/**
+  @file include/mysql/service_thd_engine_lock.h
+  This service provides functions for storage engines to report
+  lock related activities.
+
+  SYNOPSIS
+  thd_row_lock_wait() - call it just when the engine find a transaction should wait
+                        another transaction to realease a row lock
+  thd                   The session which is waiting  for the row lock to release.
+  thd_wait_for          The session which is holding the row lock.
+*/
+
+#ifdef __cplusplus
+class THD;
+#else
+#define THD void
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+  void thd_report_row_lock_wait(THD* self, THD *wait_for);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

mysql-test/include/rpl_diff.inc

@@ -113,7 +113,7 @@ while ($_rpl_diff_servers)
 
   # Execute statement
   --let $_rpl_diff_file= $MYSQLTEST_VARDIR/tmp/_rpl_diff_server-$_rpl_diff_server_i.tmp
-  --exec $MYSQL --defaults-group-suffix=.$_rpl_diff_server_i $_rpl_diff_database < $_rpl_diff_statement_file > $_rpl_diff_file
+  --exec $MYSQL $_rpl_diff_database < $_rpl_diff_statement_file > $_rpl_diff_file
 
   # Compare
   if (!$_rpl_diff_first)

mysql-test/suite/rpl/r/rpl_mts_slave_preserve_commit_order_deadlock.result

@@ -0,0 +1,169 @@
+include/master-slave.inc
+Warnings:
+Note	####	Sending passwords in plain text without SSL/TLS is extremely insecure.
+Note	####	Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
+[connection master]
+CREATE TABLE t1(c1 INT PRIMARY KEY, c2 INT, INDEX(c2)) ENGINE = InnoDB;
+INSERT INTO t1 VALUES(1, NULL),(2, 2), (3, NULL), (4, 4), (5, NULL), (6, 6);
+include/sync_slave_sql_with_master.inc
+include/stop_slave_sql.inc
+SET @saved_slave_parallel_type = @@GLOBAL.slave_parallel_type;
+SET @saved_slave_parallel_workers = @@GLOBAL.slave_parallel_workers;
+SET @saved_slave_preserve_commit_order = @@GLOBAL.slave_preserve_commit_order;
+SET @saved_innodb_lock_wait_timeout = @@GLOBAL.innodb_lock_wait_timeout;
+SET @saved_slave_transaction_retries = @@GLOBAL.slave_transaction_retries;
+SET GLOBAL slave_transaction_retries = 2;
+SET GLOBAL slave_parallel_type = "LOGICAL_CLOCK";
+SET GLOBAL slave_parallel_workers = 3;
+SET GLOBAL slave_preserve_commit_order = ON;
+SET GLOBAL innodb_lock_wait_timeout = 1000;
+#
+# Case 1: Verify slave can find the deadlock when DELETE is waiting
+# for its turn to commit
+#
+[connection master]
+INSERT INTO t1 VALUES(10, 10);
+SET debug = "d,set_commit_parent_100";
+INSERT INTO t1 VALUES(11, NULL);
+DELETE FROM t1 WHERE c2 <= 3;
+[connection slave]
+BEGIN;
+INSERT INTO t1 VALUES(11, 11);
+[connection slave1]
+include/start_slave_sql.inc
+[connection slave]
+ROLLBACK;
+include/rpl_diff.inc
+[connection master]
+SET debug = "";
+TRUNCATE t1;
+INSERT INTO t1 VALUES(1, NULL),(2, 2), (3, NULL), (4, 4), (5, NULL), (6, 6);
+include/sync_slave_sql_with_master.inc
+include/stop_slave_sql.inc
+#
+# Case 2: Verify slave can find the deadlock when it begins to applying
+# second DELETE statement.
+#
+[connection master]
+INSERT INTO t1 VALUES(20, NULL);
+SET debug = "d,set_commit_parent_100";
+INSERT INTO t1 VALUES(21, NULL);
+BEGIN;
+INSERT INTO t1 VALUES(22, 22);
+DELETE FROM t1 WHERE c2 <= 3;
+INSERT INTO t1 VALUES(23, 23);
+INSERT INTO t1 VALUES(24, 24);
+INSERT INTO t1 VALUES(25, 25);
+COMMIT;
+[connection slave]
+BEGIN;
+INSERT INTO t1 VALUES(21, 21);
+[connection server_2_1]
+BEGIN;
+INSERT INTO t1 VALUES(23, 23);
+[connection slave1]
+include/start_slave_sql.inc
+[connection slave]
+ROLLBACK;
+[connection server_2_1]
+ROLLBACK;
+include/rpl_diff.inc
+[connection master]
+SET debug = "";
+TRUNCATE t1;
+INSERT INTO t1 VALUES(1, NULL),(2, 2), (3, NULL), (4, 4), (5, NULL), (32, 32);
+include/sync_slave_sql_with_master.inc
+include/stop_slave_sql.inc
+#
+# Test case 3: Verify the worker can handle it correctly when it is
+# retrying a transaction.
+#
+[connection master]
+INSERT INTO t1 VALUES(30, NULL);
+SET debug = "d,set_commit_parent_100";
+INSERT INTO t1 VALUES(31, NULL);
+INSERT INTO t1 VALUES(33, NULL);
+DELETE FROM t1 WHERE c2 <= 3;
+[connection slave]
+BEGIN;
+INSERT INTO t1 VALUES(31, 31);
+[connection server_2_1]
+BEGIN;
+INSERT INTO t1 VALUES(33, 33);
+[connection slave1]
+include/start_slave_sql.inc
+[connection slave]
+ROLLBACK;
+[connection server_2_1]
+ROLLBACK;
+include/rpl_diff.inc
+SET debug = "";
+#
+# Test Case 4: Innodb internal transaction deadlock
+#
+[connection master]
+CREATE TABLE t2 LIKE mysql.innodb_table_stats;
+include/sync_slave_sql_with_master.inc
+include/stop_slave_sql.inc
+CALL mtr.add_suppression(".*InnoDB: Cannot save table statistics for table.*");
+LOCK TABLE t1 WRITE;
+[connection master]
+TRUNCATE t2;
+SET debug = "d,set_commit_parent_100";
+ANALYZE TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	OK
+INSERT INTO t2 SELECT * FROM mysql.innodb_table_stats;
+[connection slave]
+include/start_slave_sql.inc
+UNLOCK TABLES;
+#
+# Test Case 5: It won't cause transaction rollback if
+#              slave_preserve_commit_order is OFF
+#
+[connection master]
+SET debug = "";
+TRUNCATE t1;
+INSERT INTO t1 VALUES(1, NULL),(2, 2), (3, NULL), (4, 4), (5, NULL), (6, 6);
+include/sync_slave_sql_with_master.inc
+include/stop_slave_sql.inc
+[connection master]
+INSERT INTO t1 VALUES(50, 50);
+SET debug = "d,set_commit_parent_100";
+INSERT INTO t1 VALUES(51, NULL);
+BEGIN;
+INSERT INTO t1 VALUES(52, 52);
+DELETE FROM t1 WHERE c2 <= 3;
+INSERT INTO t1 VALUES(53, 53);
+INSERT INTO t1 VALUES(54, 54);
+INSERT INTO t1 VALUES(55, 55);
+COMMIT;
+[connection slave]
+BEGIN;
+INSERT INTO t1 VALUES(51, 51);
+[connection server_2_1]
+BEGIN;
+INSERT INTO t1 VALUES(53, 53);
+[connection slave1]
+SET GLOBAL slave_preserve_commit_order = OFF;
+SET GLOBAL slave_transaction_retries = 0;
+include/start_slave_sql.inc
+[connection slave]
+ROLLBACK;
+[connection server_2_1]
+ROLLBACK;
+include/rpl_diff.inc
+#
+# Deinitialize
+#
+[connection master]
+SET debug = "";
+DROP TABLE t1, t2;
+include/sync_slave_sql_with_master.inc
+include/stop_slave.inc
+SET GLOBAL slave_transaction_retries = @saved_slave_transaction_retries;
+SET GLOBAL slave_parallel_type = @saved_slave_parallel_type;
+SET GLOBAL slave_parallel_workers = @saved_slave_parallel_workers;
+SET GLOBAL slave_preserve_commit_order = @saved_slave_preserve_commit_order;
+SET GLOBAL innodb_lock_wait_timeout = @saved_innodb_lock_wait_timeout;
+include/rpl_end.inc