Skip to content

Commit 39c70ca

Browse files
bhatvinayfacebook-github-bot
bhatvinay
authored and
facebook-github-bot
committed
ensure that promotion is a sync point on new leader
Summary: Newly elected raft leader makes sure that all trxs from the previous leader is committed by sql appliers. It then switches the server's trx logs from apply-log-* to binary-log-*. To other part of the system this looks like a rotation, but the necessary sync calls are not made here. So, if the server (or os) restarts, then the storage engine could lose the commit markers of the last batch of trxs. This will result in silent data drift. This diff fixes the problem by making an explicit call to ha_flush_logs() before switching the server's trx logs Reviewed By: anirbanr-fb Differential Revision: D27582002 fbshipit-source-id: dc932ed247c
1 parent f12eea3 commit 39c70ca

File tree

3 files changed

+160
-0
lines changed

3 files changed

+160
-0
lines changed

‎mysql-test/suite/rpl_raft/r/rpl_raft_leader_election_crash.result

+72
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
include/raft_3_node.inc
2+
Warnings:
3+
Note #### Sending passwords in plain text without SSL/TLS is extremely insecure.
4+
Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
5+
Warnings:
6+
Note #### Sending passwords in plain text without SSL/TLS is extremely insecure.
7+
Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
8+
[connection master]
9+
"server_1 is the initial leader"
10+
select variable_value from information_schema.global_status where variable_name = 'Rpl_raft_role';
11+
variable_value
12+
LEADER
13+
"Stopping slave applier on all peers"
14+
stop slave sql_thread;
15+
Warnings:
16+
Note 1255 Slave already has been stopped
17+
stop slave sql_thread;
18+
stop slave sql_thread;
19+
"Writing data on leader"
20+
create table t1 (a int primary key auto_increment) engine = innodb;
21+
insert into t1 values();
22+
insert into t1 values();
23+
insert into t1 values();
24+
select * from t1;
25+
a
26+
1
27+
2
28+
3
29+
"Setting debug symbol so that server_2 crashed on becoming a leader after switching logs"
30+
set global debug="+d,crash_after_point_binlog_to_binlog";
31+
"Stopping sql appliers on server_2"
32+
stop slave sql_thread
33+
insert into t1 values();
34+
insert into t1 values();
35+
insert into t1 values();
36+
select * from t1;
37+
a
38+
1
39+
2
40+
3
41+
4
42+
5
43+
6
44+
"Transfering leadership: server_1 -> server_2"
45+
set @@global.rpl_raft_new_leader_uuid = 'uuid2';
46+
select sleep(1);
47+
sleep(1)
48+
0
49+
"Restarting server_2"
50+
include/rpl_start_server.inc [server_number=2]
51+
"Checking table values in server_2"
52+
connection server_2
53+
select * from t1;
54+
a
55+
1
56+
2
57+
3
58+
4
59+
5
60+
6
61+
select sleep(20);
62+
sleep(20)
63+
0
64+
"Make server_1 the leader"
65+
connection server_1
66+
sleep(10)
67+
0
68+
Warnings:
69+
Note 1254 Slave is already running
70+
include/sync_slave_sql_with_master.inc
71+
include/sync_slave_sql_with_master.inc
72+
include/rpl_end.inc

‎mysql-test/suite/rpl_raft/t/rpl_raft_leader_election_crash.test

+80
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
2+
source ../include/raft_3_node.inc;
3+
4+
connection server_1;
5+
let $uuid1= `select variable_value from information_schema.global_status where variable_name = 'Rpl_raft_peer_uuid'`;
6+
7+
connection server_2;
8+
let $uuid2= `select variable_value from information_schema.global_status where variable_name = 'Rpl_raft_peer_uuid'`;
9+
10+
connection server_3;
11+
let $uuid3= `select variable_value from information_schema.global_status where variable_name = 'Rpl_raft_peer_uuid'`;
12+
13+
echo "server_1 is the initial leader";
14+
connection server_1;
15+
select variable_value from information_schema.global_status where variable_name = 'Rpl_raft_role';
16+
17+
echo "Stopping slave applier on all peers";
18+
connection server_1;
19+
stop slave sql_thread;
20+
connection server_2;
21+
stop slave sql_thread;
22+
connection server_3;
23+
stop slave sql_thread;
24+
25+
echo "Writing data on leader";
26+
connection server_1;
27+
create table t1 (a int primary key auto_increment) engine = innodb;
28+
insert into t1 values();
29+
insert into t1 values();
30+
insert into t1 values();
31+
select * from t1;
32+
33+
echo "Setting debug symbol so that server_2 crashed on becoming a leader after switching logs";
34+
connection server_2;
35+
set global debug="+d,crash_after_point_binlog_to_binlog";
36+
37+
echo "Stopping sql appliers on server_2"
38+
stop slave sql_thread;
39+
40+
connection server_1;
41+
insert into t1 values();
42+
insert into t1 values();
43+
insert into t1 values();
44+
select * from t1;
45+
echo "Transfering leadership: server_1 -> server_2";
46+
--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
47+
replace_result $uuid2 uuid2;
48+
eval set @@global.rpl_raft_new_leader_uuid = '$uuid2';
49+
50+
select sleep(1);
51+
echo "Restarting server_2";
52+
let $rpl_server_number = 2;
53+
--source include/rpl_start_server.inc
54+
echo "Checking table values in server_2"
55+
connection server_2;
56+
select * from t1;
57+
58+
select sleep(20);
59+
60+
# Cleanup
61+
--disable_query_log
62+
63+
echo "Make server_1 the leader"
64+
connection server_1;
65+
set global rpl_raft_start_election = 1;
66+
select sleep(10);
67+
drop table t1;
68+
69+
connection server_2;
70+
start slave sql_thread;
71+
72+
connection server_3;
73+
start slave sql_thread;
74+
75+
let $sync_slave_connection= server_2;
76+
source include/sync_slave_sql_with_master.inc;
77+
let $sync_slave_connection= server_3;
78+
source include/sync_slave_sql_with_master.inc;
79+
80+
source include/rpl_end.inc;

‎sql/binlog.cc

+8
Original file line numberDiff line numberDiff line change
@@ -8066,6 +8066,12 @@ int binlog_change_to_binlog()
80668066
int error= 0;
80678067
uint64_t prev_hlc= 0;
80688068

8069+
// Flush logs to ensure that storage engine has flushed and fsynced the last
8070+
// batch of transactions. This is important because the act of switching trx
8071+
// logs from "apply-logs-*" to "binary-logs-*" looks like a rotation to other
8072+
// parts of the system and rotation is always a 'sync' point
8073+
ha_flush_logs(NULL);
8074+
80698075
mysql_mutex_lock(mysql_bin_log.get_log_lock());
80708076
dump_log.lock();
80718077
mysql_bin_log.lock_index();
@@ -8185,6 +8191,8 @@ int binlog_change_to_binlog()
81858191
dump_log.unlock();
81868192
mysql_mutex_unlock(mysql_bin_log.get_log_lock());
81878193

8194+
DBUG_EXECUTE_IF("crash_after_point_binlog_to_binlog", DBUG_SUICIDE(););
8195+
81888196
DBUG_RETURN(error);
81898197
}
81908198

0 commit comments

Comments
 (0)