Skip to content

Commit 316e0e5

Browse files
abhinav04sharmafacebook-github-bot
abhinav04sharma
authored and
facebook-github-bot
committed
Add trx meta data in the binlog using comments in rows query event
Summary: If enabled, For every trx before the commit event is written we write a rows query event which contains meta data in JSON format wrapped inside a comment. Storing timestamps in the meta data to calculate slave lag in millisecond granularity. Reviewed By: tianx Differential Revision: D5220355 fbshipit-source-id: a336f92
1 parent d26efe0 commit 316e0e5

17 files changed

+463
-4
lines changed

‎client/mysqlbinlog.cc‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1510,6 +1510,13 @@ Exit_status process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev,
15101510
}
15111511
}
15121512
case ROWS_QUERY_LOG_EVENT:
1513+
// case: if events contains trx meta data print it else fall through
1514+
// like a normal rows query event
1515+
if (((Rows_query_log_event*) ev)->has_trx_meta_data())
1516+
{
1517+
ev->print(result_file, print_event_info);
1518+
break;
1519+
}
15131520
case WRITE_ROWS_EVENT:
15141521
case DELETE_ROWS_EVENT:
15151522
case UPDATE_ROWS_EVENT:

‎mysql-test/r/mysqld--help-notwin-profiling.result‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,10 @@ The following options may be given as the first argument:
147147
non-transactional engines for the binary log. If you
148148
often use statements updating a great number of rows, you
149149
can increase this to get more performance
150+
--binlog-trx-meta-data
151+
Log meta data about every trx in the binary log. This
152+
information is logged as a comment in a Rows_query_log
153+
event in JSON format.
150154
--binlogging-impossible-mode=name
151155
On a fatal error when statements cannot be binlogged the
152156
behaviour can be ignore the error and let the master
@@ -1728,6 +1732,7 @@ binlog-row-event-max-size 8192
17281732
binlog-row-image FULL
17291733
binlog-rows-query-log-events FALSE
17301734
binlog-stmt-cache-size 32768
1735+
binlog-trx-meta-data FALSE
17311736
binlogging-impossible-mode IGNORE_ERROR
17321737
block-create-memory FALSE
17331738
block-create-myisam FALSE

‎mysql-test/r/mysqld--help-notwin.result‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,10 @@ The following options may be given as the first argument:
147147
non-transactional engines for the binary log. If you
148148
often use statements updating a great number of rows, you
149149
can increase this to get more performance
150+
--binlog-trx-meta-data
151+
Log meta data about every trx in the binary log. This
152+
information is logged as a comment in a Rows_query_log
153+
event in JSON format.
150154
--binlogging-impossible-mode=name
151155
On a fatal error when statements cannot be binlogged the
152156
behaviour can be ignore the error and let the master
@@ -1726,6 +1730,7 @@ binlog-row-event-max-size 8192
17261730
binlog-row-image FULL
17271731
binlog-rows-query-log-events FALSE
17281732
binlog-stmt-cache-size 32768
1733+
binlog-trx-meta-data FALSE
17291734
binlogging-impossible-mode IGNORE_ERROR
17301735
block-create-memory FALSE
17311736
block-create-myisam FALSE

‎mysql-test/suite/rpl/r/binlog_trx_meta_data.result‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
include/master-slave.inc
2+
Warnings:
3+
Note #### Sending passwords in plain text without SSL/TLS is extremely insecure.
4+
Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
5+
[connection master]
6+
set @save.binlog_trx_meta_data= @@global.binlog_trx_meta_data;
7+
set @@global.binlog_trx_meta_data= true;
8+
set @save.binlog_trx_meta_data= @@global.binlog_trx_meta_data;
9+
set @@global.binlog_trx_meta_data= true;
10+
include/sync_slave_sql_with_master.inc
11+
include/sync_slave_sql_with_master.inc
12+
meta count(*)
13+
# /*::TRX_META_DATA::{"timestamps":[ts]}*/ 134
14+
meta count(*)
15+
# /*::TRX_META_DATA::{"timestamps":[ts,ts]}*/ 134
16+
include/rpl_end.inc

‎mysql-test/suite/rpl/t/binlog_trx_meta_data.test‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
source include/master-slave.inc;
2+
source include/have_binlog_format_row.inc;
3+
4+
# Enable trx meta data
5+
connection master;
6+
set @save.binlog_trx_meta_data= @@global.binlog_trx_meta_data;
7+
set @@global.binlog_trx_meta_data= true;
8+
9+
connection slave;
10+
set @save.binlog_trx_meta_data= @@global.binlog_trx_meta_data;
11+
set @@global.binlog_trx_meta_data= true;
12+
13+
# Generate load
14+
connection master;
15+
let $skip_cleanup= 1;
16+
let $sync_with_master= 1;
17+
let $databases=2;
18+
let $iter=5;
19+
source extra/rpl_tests/rpl_parallel_simple_load.test;
20+
enable_result_log;
21+
22+
# Check binlogs for meta data
23+
connection master;
24+
let $MYSQLD_DATADIR = `select @@datadir`;
25+
let $MYSQLD_SECURE_FILE_DIR = `select @@secure_file_priv`;
26+
exec $MYSQL_BINLOG -v -v $MYSQLD_DATADIR/master-bin.0* | grep "::TRX_META_DATA::" | sed 's/\"[0-9]*\"/ts/g' > $MYSQLD_SECURE_FILE_DIR/meta_data.dat;
27+
set sql_log_bin=0;
28+
create table test.meta_data(meta text);
29+
eval load data infile '$MYSQLD_SECURE_FILE_DIR/meta_data.dat' into table test.meta_data;
30+
select meta, count(*) from test.meta_data;
31+
drop table test.meta_data;
32+
set sql_log_bin=1;
33+
remove_file $MYSQLD_SECURE_FILE_DIR/meta_data.dat;
34+
35+
connection slave;
36+
let $MYSQLD_DATADIR = `select @@datadir`;
37+
let $MYSQLD_SECURE_FILE_DIR = `select @@secure_file_priv`;
38+
exec $MYSQL_BINLOG -v -v $MYSQLD_DATADIR/slave-bin.0* | grep "::TRX_META_DATA::" | sed 's/\"[0-9]*\"/ts/g' > $MYSQLD_SECURE_FILE_DIR/meta_data.dat;
39+
set sql_log_bin=0;
40+
create table test.meta_data(meta text);
41+
eval load data infile '$MYSQLD_SECURE_FILE_DIR/meta_data.dat' into table test.meta_data;
42+
select meta, count(*) from test.meta_data;
43+
drop table test.meta_data;
44+
set sql_log_bin=1;
45+
remove_file $MYSQLD_SECURE_FILE_DIR/meta_data.dat;
46+
47+
# Consistency check
48+
connection master;
49+
let $include_silent=1;
50+
source extra/rpl_tests/rpl_parallel_simple_load_consistency.test;
51+
let $include_silent=;
52+
53+
# Cleanup
54+
connection master;
55+
source extra/rpl_tests/rpl_parallel_simple_load_cleanup.test;
56+
connection slave;
57+
set @@global.binlog_trx_meta_data= @save.binlog_trx_meta_data;
58+
connection master;
59+
set @@global.binlog_trx_meta_data= @save.binlog_trx_meta_data;
60+
61+
source include/rpl_end.inc;

‎mysql-test/suite/sys_vars/r/binlog_trx_meta_data_basic.result‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
SET @start_value = @@global.binlog_trx_meta_data;
2+
SELECT @start_value;
3+
@start_value
4+
0
5+
SET @@global.binlog_trx_meta_data = DEFAULT;
6+
SELECT @@global.binlog_trx_meta_data = TRUE;
7+
@@global.binlog_trx_meta_data = TRUE
8+
0
9+
SET @@global.binlog_trx_meta_data = ON;
10+
SELECT @@global.binlog_trx_meta_data;
11+
@@global.binlog_trx_meta_data
12+
1
13+
SET @@global.binlog_trx_meta_data = OFF;
14+
SELECT @@global.binlog_trx_meta_data;
15+
@@global.binlog_trx_meta_data
16+
0
17+
SET @@global.binlog_trx_meta_data = 2;
18+
ERROR 42000: Variable 'binlog_trx_meta_data' can't be set to the value of '2'
19+
SET @@global.binlog_trx_meta_data = -1;
20+
ERROR 42000: Variable 'binlog_trx_meta_data' can't be set to the value of '-1'
21+
SET @@global.binlog_trx_meta_data = TRUEF;
22+
ERROR 42000: Variable 'binlog_trx_meta_data' can't be set to the value of 'TRUEF'
23+
SET @@global.binlog_trx_meta_data = TRUE_F;
24+
ERROR 42000: Variable 'binlog_trx_meta_data' can't be set to the value of 'TRUE_F'
25+
SET @@global.binlog_trx_meta_data = FALSE0;
26+
ERROR 42000: Variable 'binlog_trx_meta_data' can't be set to the value of 'FALSE0'
27+
SET @@global.binlog_trx_meta_data = OON;
28+
ERROR 42000: Variable 'binlog_trx_meta_data' can't be set to the value of 'OON'
29+
SET @@global.binlog_trx_meta_data = ONN;
30+
ERROR 42000: Variable 'binlog_trx_meta_data' can't be set to the value of 'ONN'
31+
SET @@global.binlog_trx_meta_data = OOFF;
32+
ERROR 42000: Variable 'binlog_trx_meta_data' can't be set to the value of 'OOFF'
33+
SET @@global.binlog_trx_meta_data = 0FF;
34+
ERROR 42000: Variable 'binlog_trx_meta_data' can't be set to the value of '0FF'
35+
SET @@global.binlog_trx_meta_data = ' ';
36+
ERROR 42000: Variable 'binlog_trx_meta_data' can't be set to the value of ' '
37+
SET @@global.binlog_trx_meta_data = " ";
38+
ERROR 42000: Variable 'binlog_trx_meta_data' can't be set to the value of ' '
39+
SET @@global.binlog_trx_meta_data = '';
40+
ERROR 42000: Variable 'binlog_trx_meta_data' can't be set to the value of ''
41+
SET @@session.binlog_trx_meta_data = OFF;
42+
ERROR HY000: Variable 'binlog_trx_meta_data' is a GLOBAL variable and should be set with SET GLOBAL
43+
SELECT @@session.binlog_trx_meta_data;
44+
ERROR HY000: Variable 'binlog_trx_meta_data' is a GLOBAL variable
45+
SELECT IF(@@global.binlog_trx_meta_data, "ON", "OFF") = VARIABLE_VALUE
46+
FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES
47+
WHERE VARIABLE_NAME='binlog_trx_meta_data';
48+
IF(@@global.binlog_trx_meta_data, "ON", "OFF") = VARIABLE_VALUE
49+
1
50+
SET @@global.binlog_trx_meta_data = 0;
51+
SELECT @@global.binlog_trx_meta_data;
52+
@@global.binlog_trx_meta_data
53+
0
54+
SET @@global.binlog_trx_meta_data = 1;
55+
SELECT @@global.binlog_trx_meta_data;
56+
@@global.binlog_trx_meta_data
57+
1
58+
SET @@global.binlog_trx_meta_data = TRUE;
59+
SELECT @@global.binlog_trx_meta_data;
60+
@@global.binlog_trx_meta_data
61+
1
62+
SET @@global.binlog_trx_meta_data = FALSE;
63+
SELECT @@global.binlog_trx_meta_data;
64+
@@global.binlog_trx_meta_data
65+
0
66+
SET @@global.binlog_trx_meta_data = ON;
67+
SELECT @@binlog_trx_meta_data = @@global.binlog_trx_meta_data;
68+
@@binlog_trx_meta_data = @@global.binlog_trx_meta_data
69+
1
70+
SET binlog_trx_meta_data = ON;
71+
ERROR HY000: Variable 'binlog_trx_meta_data' is a GLOBAL variable and should be set with SET GLOBAL
72+
SET local.binlog_trx_meta_data = OFF;
73+
ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'binlog_trx_meta_data = OFF' at line 1
74+
SELECT local.binlog_trx_meta_data;
75+
ERROR 42S02: Unknown table 'local' in field list
76+
SET global.binlog_trx_meta_data = ON;
77+
ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'binlog_trx_meta_data = ON' at line 1
78+
SELECT global.binlog_trx_meta_data;
79+
ERROR 42S02: Unknown table 'global' in field list
80+
SELECT binlog_trx_meta_data = @@session.binlog_trx_meta_data;
81+
ERROR 42S22: Unknown column 'binlog_trx_meta_data' in 'field list'
82+
SET @@global.binlog_trx_meta_data = @start_value;
83+
SELECT @@global.binlog_trx_meta_data;
84+
@@global.binlog_trx_meta_data
85+
0

‎mysql-test/suite/sys_vars/t/binlog_trx_meta_data_basic.test‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
--source include/have_innodb.inc
2+
--source include/load_sysvars.inc
3+
4+
SET @start_value = @@global.binlog_trx_meta_data;
5+
SELECT @start_value;
6+
7+
8+
SET @@global.binlog_trx_meta_data = DEFAULT;
9+
SELECT @@global.binlog_trx_meta_data = TRUE;
10+
11+
12+
SET @@global.binlog_trx_meta_data = ON;
13+
SELECT @@global.binlog_trx_meta_data;
14+
SET @@global.binlog_trx_meta_data = OFF;
15+
SELECT @@global.binlog_trx_meta_data;
16+
17+
--Error ER_WRONG_VALUE_FOR_VAR
18+
SET @@global.binlog_trx_meta_data = 2;
19+
--Error ER_WRONG_VALUE_FOR_VAR
20+
SET @@global.binlog_trx_meta_data = -1;
21+
--Error ER_WRONG_VALUE_FOR_VAR
22+
SET @@global.binlog_trx_meta_data = TRUEF;
23+
--Error ER_WRONG_VALUE_FOR_VAR
24+
SET @@global.binlog_trx_meta_data = TRUE_F;
25+
--Error ER_WRONG_VALUE_FOR_VAR
26+
SET @@global.binlog_trx_meta_data = FALSE0;
27+
--Error ER_WRONG_VALUE_FOR_VAR
28+
SET @@global.binlog_trx_meta_data = OON;
29+
--Error ER_WRONG_VALUE_FOR_VAR
30+
SET @@global.binlog_trx_meta_data = ONN;
31+
--Error ER_WRONG_VALUE_FOR_VAR
32+
SET @@global.binlog_trx_meta_data = OOFF;
33+
--Error ER_WRONG_VALUE_FOR_VAR
34+
SET @@global.binlog_trx_meta_data = 0FF;
35+
--Error ER_WRONG_VALUE_FOR_VAR
36+
SET @@global.binlog_trx_meta_data = ' ';
37+
--Error ER_WRONG_VALUE_FOR_VAR
38+
SET @@global.binlog_trx_meta_data = " ";
39+
--Error ER_WRONG_VALUE_FOR_VAR
40+
SET @@global.binlog_trx_meta_data = '';
41+
42+
43+
--Error ER_GLOBAL_VARIABLE
44+
SET @@session.binlog_trx_meta_data = OFF;
45+
--Error ER_INCORRECT_GLOBAL_LOCAL_VAR
46+
SELECT @@session.binlog_trx_meta_data;
47+
48+
49+
SELECT IF(@@global.binlog_trx_meta_data, "ON", "OFF") = VARIABLE_VALUE
50+
FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES
51+
WHERE VARIABLE_NAME='binlog_trx_meta_data';
52+
53+
54+
SET @@global.binlog_trx_meta_data = 0;
55+
SELECT @@global.binlog_trx_meta_data;
56+
SET @@global.binlog_trx_meta_data = 1;
57+
SELECT @@global.binlog_trx_meta_data;
58+
59+
SET @@global.binlog_trx_meta_data = TRUE;
60+
SELECT @@global.binlog_trx_meta_data;
61+
SET @@global.binlog_trx_meta_data = FALSE;
62+
SELECT @@global.binlog_trx_meta_data;
63+
64+
SET @@global.binlog_trx_meta_data = ON;
65+
SELECT @@binlog_trx_meta_data = @@global.binlog_trx_meta_data;
66+
67+
--Error ER_GLOBAL_VARIABLE
68+
SET binlog_trx_meta_data = ON;
69+
--Error ER_PARSE_ERROR
70+
SET local.binlog_trx_meta_data = OFF;
71+
--Error ER_UNKNOWN_TABLE
72+
SELECT local.binlog_trx_meta_data;
73+
--Error ER_PARSE_ERROR
74+
SET global.binlog_trx_meta_data = ON;
75+
--Error ER_UNKNOWN_TABLE
76+
SELECT global.binlog_trx_meta_data;
77+
--Error ER_BAD_FIELD_ERROR
78+
SELECT binlog_trx_meta_data = @@session.binlog_trx_meta_data;
79+
80+
SET @@global.binlog_trx_meta_data = @start_value;
81+
SELECT @@global.binlog_trx_meta_data;

0 commit comments

Comments
 (0)