Skip to content

Commit 61e2172

Browse files
author
Libing Song
committed
BUG#20136704 --SLAVE-PRESERVE-COMMIT-ORDER CAUSES SLAVE TO DEADLOCK AND
 BREAK FOR SOME QUERIE A corner case caused slave hang when slave_preserve_commit_order is ON. ANALYSIS ======== CREATE TABLE t1(c1 INT PRIMARY KEY, c2 INT, INDEX(c2)) ENGINE = InnoDB; INSERT INTO t1 VALUES(1, NULL),(2, 2), (3, NULL), (4, 4), (5, NULL), (6, 6); INSERT INTO t1 VALUES(7, NULL); DELETE FROM t1 WHERE c2 <= 3; On master, the INSERT statement acquires its row lock before DELETE STATEMENT. Since the INSERT doesn't block the DELETE, Both statements may have same commit parent which mean they can be applied parallel on slave. On slave, they will be applied parallel if MTS is ON. There is a chance that the DELETE acquires its lock before the INSERT. The DELETE holds a gap lock on INDEX(c2) which blocks the INSERT statement. The INSERT cannot be applied unless DELETE is committed or rolled back. And meanwhile the DELETE is waiting for the INSERT to commit since slave_preserve_commit_order is ON. That is a deadlock and hangs the slave. Here we call the deadlock as a order commit deadlock. FIX === A deadlock checking mechanism is introduced. Every time when a transaction needs to wait for another transaction to release a row lock, innodb will call a slave function to check if there is an order commit deadlock. If it founds an order commit deadlock, It will set a deadlock flag to the slave worker which is holding the row lock. Thereafter, the worker will roll its transaction back and retry it again.
1 parent 72177d5 commit 61e2172

12 files changed

+714
-24
lines changed

‎include/mysql/service_thd_engine_lock.h

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/* Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
2+
3+
This program is free software; you can redistribute it and/or modify
4+
it under the terms of the GNU General Public License as published by
5+
the Free Software Foundation; version 2 of the License.
6+
7+
This program is distributed in the hope that it will be useful,
8+
but WITHOUT ANY WARRANTY; without even the implied warranty of
9+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10+
GNU General Public License for more details.
11+
12+
You should have received a copy of the GNU General Public License
13+
along with this program; if not, write to the Free Software
14+
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */
15+
16+
#ifndef MYSQL_SERVICE_THD_EGINE_LOCK_INCLUDED
17+
#define MYSQL_SERVICE_THD_EGINE_LOCK_INCLUDED
18+
19+
/**
20+
@file include/mysql/service_thd_engine_lock.h
21+
This service provides functions for storage engines to report
22+
lock related activities.
23+
24+
SYNOPSIS
25+
thd_row_lock_wait() - call it just when the engine find a transaction should wait
26+
another transaction to realease a row lock
27+
thd The session which is waiting for the row lock to release.
28+
thd_wait_for The session which is holding the row lock.
29+
*/
30+
31+
#ifdef __cplusplus
32+
class THD;
33+
#else
34+
#define THD void
35+
#endif
36+
37+
#ifdef __cplusplus
38+
extern "C" {
39+
#endif
40+
41+
void thd_report_row_lock_wait(THD* self, THD *wait_for);
42+
43+
#ifdef __cplusplus
44+
}
45+
#endif
46+
47+
#endif

‎mysql-test/include/rpl_diff.inc

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ while ($_rpl_diff_servers)
113113

114114
# Execute statement
115115
--let $_rpl_diff_file= $MYSQLTEST_VARDIR/tmp/_rpl_diff_server-$_rpl_diff_server_i.tmp
116-
--exec $MYSQL --defaults-group-suffix=.$_rpl_diff_server_i $_rpl_diff_database < $_rpl_diff_statement_file > $_rpl_diff_file
116+
--exec $MYSQL $_rpl_diff_database < $_rpl_diff_statement_file > $_rpl_diff_file
117117

118118
# Compare
119119
if (!$_rpl_diff_first)

‎mysql-test/suite/rpl/r/rpl_mts_slave_preserve_commit_order_deadlock.result

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
include/master-slave.inc
2+
Warnings:
3+
Note #### Sending passwords in plain text without SSL/TLS is extremely insecure.
4+
Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
5+
[connection master]
6+
CREATE TABLE t1(c1 INT PRIMARY KEY, c2 INT, INDEX(c2)) ENGINE = InnoDB;
7+
INSERT INTO t1 VALUES(1, NULL),(2, 2), (3, NULL), (4, 4), (5, NULL), (6, 6);
8+
include/sync_slave_sql_with_master.inc
9+
include/stop_slave_sql.inc
10+
SET @saved_slave_parallel_type = @@GLOBAL.slave_parallel_type;
11+
SET @saved_slave_parallel_workers = @@GLOBAL.slave_parallel_workers;
12+
SET @saved_slave_preserve_commit_order = @@GLOBAL.slave_preserve_commit_order;
13+
SET @saved_innodb_lock_wait_timeout = @@GLOBAL.innodb_lock_wait_timeout;
14+
SET @saved_slave_transaction_retries = @@GLOBAL.slave_transaction_retries;
15+
SET GLOBAL slave_transaction_retries = 2;
16+
SET GLOBAL slave_parallel_type = "LOGICAL_CLOCK";
17+
SET GLOBAL slave_parallel_workers = 3;
18+
SET GLOBAL slave_preserve_commit_order = ON;
19+
SET GLOBAL innodb_lock_wait_timeout = 1000;
20+
#
21+
# Case 1: Verify slave can find the deadlock when DELETE is waiting
22+
# for its turn to commit
23+
#
24+
[connection master]
25+
INSERT INTO t1 VALUES(10, 10);
26+
SET debug = "d,set_commit_parent_100";
27+
INSERT INTO t1 VALUES(11, NULL);
28+
DELETE FROM t1 WHERE c2 <= 3;
29+
[connection slave]
30+
BEGIN;
31+
INSERT INTO t1 VALUES(11, 11);
32+
[connection slave1]
33+
include/start_slave_sql.inc
34+
[connection slave]
35+
ROLLBACK;
36+
include/rpl_diff.inc
37+
[connection master]
38+
SET debug = "";
39+
TRUNCATE t1;
40+
INSERT INTO t1 VALUES(1, NULL),(2, 2), (3, NULL), (4, 4), (5, NULL), (6, 6);
41+
include/sync_slave_sql_with_master.inc
42+
include/stop_slave_sql.inc
43+
#
44+
# Case 2: Verify slave can find the deadlock when it begins to applying
45+
# second DELETE statement.
46+
#
47+
[connection master]
48+
INSERT INTO t1 VALUES(20, NULL);
49+
SET debug = "d,set_commit_parent_100";
50+
INSERT INTO t1 VALUES(21, NULL);
51+
BEGIN;
52+
INSERT INTO t1 VALUES(22, 22);
53+
DELETE FROM t1 WHERE c2 <= 3;
54+
INSERT INTO t1 VALUES(23, 23);
55+
INSERT INTO t1 VALUES(24, 24);
56+
INSERT INTO t1 VALUES(25, 25);
57+
COMMIT;
58+
[connection slave]
59+
BEGIN;
60+
INSERT INTO t1 VALUES(21, 21);
61+
[connection server_2_1]
62+
BEGIN;
63+
INSERT INTO t1 VALUES(23, 23);
64+
[connection slave1]
65+
include/start_slave_sql.inc
66+
[connection slave]
67+
ROLLBACK;
68+
[connection server_2_1]
69+
ROLLBACK;
70+
include/rpl_diff.inc
71+
[connection master]
72+
SET debug = "";
73+
TRUNCATE t1;
74+
INSERT INTO t1 VALUES(1, NULL),(2, 2), (3, NULL), (4, 4), (5, NULL), (32, 32);
75+
include/sync_slave_sql_with_master.inc
76+
include/stop_slave_sql.inc
77+
#
78+
# Test case 3: Verify the worker can handle it correctly when it is
79+
# retrying a transaction.
80+
#
81+
[connection master]
82+
INSERT INTO t1 VALUES(30, NULL);
83+
SET debug = "d,set_commit_parent_100";
84+
INSERT INTO t1 VALUES(31, NULL);
85+
INSERT INTO t1 VALUES(33, NULL);
86+
DELETE FROM t1 WHERE c2 <= 3;
87+
[connection slave]
88+
BEGIN;
89+
INSERT INTO t1 VALUES(31, 31);
90+
[connection server_2_1]
91+
BEGIN;
92+
INSERT INTO t1 VALUES(33, 33);
93+
[connection slave1]
94+
include/start_slave_sql.inc
95+
[connection slave]
96+
ROLLBACK;
97+
[connection server_2_1]
98+
ROLLBACK;
99+
include/rpl_diff.inc
100+
SET debug = "";
101+
#
102+
# Test Case 4: Innodb internal transaction deadlock
103+
#
104+
[connection master]
105+
CREATE TABLE t2 LIKE mysql.innodb_table_stats;
106+
include/sync_slave_sql_with_master.inc
107+
include/stop_slave_sql.inc
108+
CALL mtr.add_suppression(".*InnoDB: Cannot save table statistics for table.*");
109+
LOCK TABLE t1 WRITE;
110+
[connection master]
111+
TRUNCATE t2;
112+
SET debug = "d,set_commit_parent_100";
113+
ANALYZE TABLE t1;
114+
Table Op Msg_type Msg_text
115+
test.t1 analyze status OK
116+
INSERT INTO t2 SELECT * FROM mysql.innodb_table_stats;
117+
[connection slave]
118+
include/start_slave_sql.inc
119+
UNLOCK TABLES;
120+
#
121+
# Test Case 5: It won't cause transaction rollback if
122+
# slave_preserve_commit_order is OFF
123+
#
124+
[connection master]
125+
SET debug = "";
126+
TRUNCATE t1;
127+
INSERT INTO t1 VALUES(1, NULL),(2, 2), (3, NULL), (4, 4), (5, NULL), (6, 6);
128+
include/sync_slave_sql_with_master.inc
129+
include/stop_slave_sql.inc
130+
[connection master]
131+
INSERT INTO t1 VALUES(50, 50);
132+
SET debug = "d,set_commit_parent_100";
133+
INSERT INTO t1 VALUES(51, NULL);
134+
BEGIN;
135+
INSERT INTO t1 VALUES(52, 52);
136+
DELETE FROM t1 WHERE c2 <= 3;
137+
INSERT INTO t1 VALUES(53, 53);
138+
INSERT INTO t1 VALUES(54, 54);
139+
INSERT INTO t1 VALUES(55, 55);
140+
COMMIT;
141+
[connection slave]
142+
BEGIN;
143+
INSERT INTO t1 VALUES(51, 51);
144+
[connection server_2_1]
145+
BEGIN;
146+
INSERT INTO t1 VALUES(53, 53);
147+
[connection slave1]
148+
SET GLOBAL slave_preserve_commit_order = OFF;
149+
SET GLOBAL slave_transaction_retries = 0;
150+
include/start_slave_sql.inc
151+
[connection slave]
152+
ROLLBACK;
153+
[connection server_2_1]
154+
ROLLBACK;
155+
include/rpl_diff.inc
156+
#
157+
# Deinitialize
158+
#
159+
[connection master]
160+
SET debug = "";
161+
DROP TABLE t1, t2;
162+
include/sync_slave_sql_with_master.inc
163+
include/stop_slave.inc
164+
SET GLOBAL slave_transaction_retries = @saved_slave_transaction_retries;
165+
SET GLOBAL slave_parallel_type = @saved_slave_parallel_type;
166+
SET GLOBAL slave_parallel_workers = @saved_slave_parallel_workers;
167+
SET GLOBAL slave_preserve_commit_order = @saved_slave_preserve_commit_order;
168+
SET GLOBAL innodb_lock_wait_timeout = @saved_innodb_lock_wait_timeout;
169+
include/rpl_end.inc

0 commit comments

Comments
 (0)