-
Notifications
You must be signed in to change notification settings - Fork 713
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
FB8-116, FB8-152, FB8-83, FB8-125: Implement trx metadata in binlog (#…
…965) Summary: Jira ticket: https://jira.percona.com/browse/FB8-116 Jira ticket: https://jira.percona.com/browse/FB8-152 Jira ticket: https://jira.percona.com/browse/FB8-83 Jira ticket: https://jira.percona.com/browse/FB8-125 Reference commit: 316e0e5 Reference commit: a85b8d3 Reference commit: 21edb26 Reference commit: 96de6c4 Reference commit: e1cb53ba0f9 Reference commit: bd25e29 Reference commit: e392a18d40e Reference commit: fa4f588a248 Reference commit: 68f8c9a58fa Reference commit: 4519ab421f8 Reference commit: 42bdfc8 Reference commit: 3950491 Reference commit: bc17d30 Reference commit: eb7b1df Reference commit: 6a27bd42dfe Reference commit: 9503743bf59 Porting notes: The deadlock/ASAN bugfixes were skipped - I wasn't able to reproduce the issue mentioned in the commit messages even without the patches, and the pathes themselves caused early guaranteed crashes in the server. -------- 316e0e5 -------- If enabled, For every trx before the commit event is written we write a rows query event which contains meta data in JSON format wrapped inside a comment. Storing timestamps in the meta data to calculate slave lag in millisecond granularity. -------- a85b8d3 -------- Supporting binlog trx meta data for SBR. The meta data event is written right before every query event in a trx representing a statement. -------- 21edb26 -------- Milli-seconds behind master was not calculated correctly for MTS. With this change we calculate it just like seconds behind master. One difference between these is that sec timestamp is available for every event but msec timestamp is only available once per query/rows event in form of trx metadata inside a Rows_query_event. -------- bd25e29 -------- A separate event for trx metadata was causing an unreasonable increase in binlog size. Also, now we're not writing rows query event if there are no comments in the query and `log_only_query_comments` is enabled. -------- eb7b1df -------- Log meta data event before logging the rows query event for a statement. Pull Request resolved: #965 Reviewed By: yizhang82 Differential Revision: D14652380 Pulled By: yizhang82 fbshipit-source-id: ef13b78
- Loading branch information
1 parent
3fb11bd
commit 3847e40
Showing
40 changed files
with
1,333 additions
and
88 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,65 @@ | ||
# Init | ||
--source include/rpl_connection_master.inc | ||
--source include/sync_slave_sql_with_master.inc | ||
--source include/rpl_connection_master.inc | ||
flush logs; | ||
--source include/sync_slave_sql_with_master.inc | ||
--source include/rpl_connection_master.inc | ||
let $master_last_log = query_get_value("SHOW MASTER STATUS", "File", 1); | ||
eval purge binary logs to '$master_last_log'; | ||
--source include/rpl_connection_slave.inc | ||
flush logs; | ||
let $slave_last_log = query_get_value("SHOW MASTER STATUS", "File", 1); | ||
eval purge binary logs to '$slave_last_log'; | ||
|
||
# Generate load | ||
--source include/rpl_connection_master.inc | ||
disable_result_log; | ||
disable_query_log; | ||
create table t1(a int auto_increment primary key) engine=innodb; | ||
# the following loop will generate 300 meta data events, 1 for each statement | ||
# the total number of meta data events will be 301, +1 for create stmt above | ||
let $iter= 10; | ||
while ($iter > 0) | ||
{ | ||
begin; | ||
eval /*insert { hello: { world } }*/insert into t1 values(); | ||
eval /*update { { hello : world } }*/update t1 set a= a + 1; | ||
/*delete { }*/delete from t1; | ||
commit; | ||
dec $iter; | ||
} | ||
enable_result_log; | ||
enable_query_log; | ||
--source include/sync_slave_sql_with_master.inc | ||
|
||
# Check binlogs for meta data | ||
--source include/rpl_connection_master.inc | ||
disable_query_log; | ||
let $MYSQLD_DATADIR = `select @@datadir`; | ||
let $MYSQLD_SECURE_FILE_DIR = `select @@secure_file_priv`; | ||
exec $MYSQL_BINLOG -v -v $MYSQLD_DATADIR/master-bin.0* | grep "::TRX_META_DATA::" | sed 's/\"[0-9]*\"/ts/g' > $MYSQLD_SECURE_FILE_DIR/meta_data.dat; | ||
set sql_log_bin=0; | ||
create table test.meta_data(meta text); | ||
eval load data infile '$MYSQLD_SECURE_FILE_DIR/meta_data.dat' into table test.meta_data; | ||
select meta, count(*) from test.meta_data group by meta; | ||
drop table test.meta_data; | ||
set sql_log_bin=1; | ||
remove_file $MYSQLD_SECURE_FILE_DIR/meta_data.dat; | ||
|
||
--source include/rpl_connection_slave.inc | ||
let $MYSQLD_DATADIR = `select @@datadir`; | ||
let $MYSQLD_SECURE_FILE_DIR = `select @@secure_file_priv`; | ||
exec $MYSQL_BINLOG -v -v $MYSQLD_DATADIR/slave-bin.0* | grep "::TRX_META_DATA::" | sed 's/\"[0-9]*\"/ts/g' > $MYSQLD_SECURE_FILE_DIR/meta_data.dat; | ||
set sql_log_bin=0; | ||
create table test.meta_data(meta text); | ||
eval load data infile '$MYSQLD_SECURE_FILE_DIR/meta_data.dat' into table test.meta_data; | ||
select meta, count(*) from test.meta_data group by meta; | ||
drop table test.meta_data; | ||
set sql_log_bin=1; | ||
remove_file $MYSQLD_SECURE_FILE_DIR/meta_data.dat; | ||
enable_query_log; | ||
|
||
# Cleanup | ||
--source include/rpl_connection_master.inc | ||
drop table t1; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,78 @@ | ||
include/master-slave.inc | ||
Warnings: | ||
Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. | ||
Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. | ||
[connection master] | ||
set @save.binlog_trx_meta_data= @@global.binlog_trx_meta_data; | ||
set @@global.binlog_trx_meta_data= true; | ||
set @save.binlog_trx_meta_data= @@global.binlog_trx_meta_data; | ||
set @@global.binlog_trx_meta_data= true; | ||
drop database if exists test2; | ||
drop database if exists test3; | ||
drop database if exists test4; | ||
create database test2 db_metadata '{"shard":"test2_shard", "replicaset":"test2_replicaset"}'; | ||
create database test3 db_metadata '{"shard":"test3_shard", "replicaset":"test3_replicaset"}'; | ||
use test; | ||
create table t1 (a int primary key, b char(8)); | ||
insert into t1 values(1, 'a'); | ||
insert into t1 values(2, 'b'); | ||
use test2; | ||
create table t1 (a int primary key, b char(8)); | ||
insert into t1 values(1, 'a'); | ||
insert into t1 values(2, 'b'); | ||
use test3; | ||
create table t1 (a int primary key, b char(8)); | ||
insert into t1 values(1, 'a'); | ||
insert into t1 values(2, 'b'); | ||
alter database test3 db_metadata '{"shard":"test3_shard_updated", "replicaset":"test3_replicaset_updated"}'; | ||
insert into t1 values(3, 'c'); | ||
insert into t1 values(4, 'd'); | ||
create database test4 db_metadata '{"shard":"newly_added_shard_for_test4", "replicaset":"new_replicaset_for_test4"}'; | ||
insert into t1 values(5, 'e'); | ||
insert into t1 values(6, 'f'); | ||
use test4; | ||
create table t1 (a int primary key, b char(8)); | ||
insert into t1 values(1, 'a'); | ||
insert into t1 values(2, 'b'); | ||
# /*::TRX_META_DATA::{"ts":[ts]}*/ | ||
# /*::TRX_META_DATA::{"ts":[ts]}*/ | ||
# /*::TRX_META_DATA::{"ts":[ts]}*/ | ||
# /*::TRX_META_DATA::{"shard":"test2_shard","replicaset":"test2_replicaset","ts":[ts]}*/ | ||
# /*::TRX_META_DATA::{"shard":"test2_shard","replicaset":"test2_replicaset","ts":[ts]}*/ | ||
# /*::TRX_META_DATA::{"shard":"test2_shard","replicaset":"test2_replicaset","ts":[ts]}*/ | ||
# /*::TRX_META_DATA::{"shard":"test3_shard","replicaset":"test3_replicaset","ts":[ts]}*/ | ||
# /*::TRX_META_DATA::{"shard":"test3_shard","replicaset":"test3_replicaset","ts":[ts]}*/ | ||
# /*::TRX_META_DATA::{"shard":"test3_shard","replicaset":"test3_replicaset","ts":[ts]}*/ | ||
# /*::TRX_META_DATA::{"shard":"test3_shard_updated","replicaset":"test3_replicaset_updated","ts":[ts]}*/ | ||
# /*::TRX_META_DATA::{"shard":"test3_shard_updated","replicaset":"test3_replicaset_updated","ts":[ts]}*/ | ||
# /*::TRX_META_DATA::{"shard":"test3_shard_updated","replicaset":"test3_replicaset_updated","ts":[ts]}*/ | ||
# /*::TRX_META_DATA::{"shard":"test3_shard_updated","replicaset":"test3_replicaset_updated","ts":[ts]}*/ | ||
# /*::TRX_META_DATA::{"shard":"newly_added_shard_for_test4","replicaset":"new_replicaset_for_test4","ts":[ts]}*/ | ||
# /*::TRX_META_DATA::{"shard":"newly_added_shard_for_test4","replicaset":"new_replicaset_for_test4","ts":[ts]}*/ | ||
# /*::TRX_META_DATA::{"shard":"newly_added_shard_for_test4","replicaset":"new_replicaset_for_test4","ts":[ts]}*/ | ||
include/sync_slave_sql_with_master.inc | ||
# /*::TRX_META_DATA::{"ts":[ts,ts]}*/ | ||
# /*::TRX_META_DATA::{"ts":[ts,ts]}*/ | ||
# /*::TRX_META_DATA::{"ts":[ts,ts]}*/ | ||
# /*::TRX_META_DATA::{"shard":"test2_shard","replicaset":"test2_replicaset","ts":[ts,ts]}*/ | ||
# /*::TRX_META_DATA::{"shard":"test2_shard","replicaset":"test2_replicaset","ts":[ts,ts]}*/ | ||
# /*::TRX_META_DATA::{"shard":"test2_shard","replicaset":"test2_replicaset","ts":[ts,ts]}*/ | ||
# /*::TRX_META_DATA::{"shard":"test3_shard","replicaset":"test3_replicaset","ts":[ts,ts]}*/ | ||
# /*::TRX_META_DATA::{"shard":"test3_shard","replicaset":"test3_replicaset","ts":[ts,ts]}*/ | ||
# /*::TRX_META_DATA::{"shard":"test3_shard","replicaset":"test3_replicaset","ts":[ts,ts]}*/ | ||
# /*::TRX_META_DATA::{"shard":"test3_shard_updated","replicaset":"test3_replicaset_updated","ts":[ts,ts]}*/ | ||
# /*::TRX_META_DATA::{"shard":"test3_shard_updated","replicaset":"test3_replicaset_updated","ts":[ts,ts]}*/ | ||
# /*::TRX_META_DATA::{"shard":"test3_shard_updated","replicaset":"test3_replicaset_updated","ts":[ts,ts]}*/ | ||
# /*::TRX_META_DATA::{"shard":"test3_shard_updated","replicaset":"test3_replicaset_updated","ts":[ts,ts]}*/ | ||
# /*::TRX_META_DATA::{"shard":"newly_added_shard_for_test4","replicaset":"new_replicaset_for_test4","ts":[ts,ts]}*/ | ||
# /*::TRX_META_DATA::{"shard":"newly_added_shard_for_test4","replicaset":"new_replicaset_for_test4","ts":[ts,ts]}*/ | ||
# /*::TRX_META_DATA::{"shard":"newly_added_shard_for_test4","replicaset":"new_replicaset_for_test4","ts":[ts,ts]}*/ | ||
use test; | ||
drop table if exists t1; | ||
drop database if exists test2; | ||
drop database if exists test3; | ||
drop database if exists test4; | ||
include/sync_slave_sql_with_master.inc | ||
set @@global.binlog_trx_meta_data= @save.binlog_trx_meta_data; | ||
set @@global.binlog_trx_meta_data= @save.binlog_trx_meta_data; | ||
include/rpl_end.inc |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
include/master-slave.inc | ||
Warnings: | ||
Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. | ||
Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. | ||
[connection master] | ||
[connection master] | ||
set @@global.binlog_trx_meta_data= true; | ||
[connection slave] | ||
set @@global.binlog_trx_meta_data= true; | ||
[connection master] | ||
include/sync_slave_sql_with_master.inc | ||
[connection master] | ||
flush logs; | ||
include/sync_slave_sql_with_master.inc | ||
[connection master] | ||
purge binary logs to 'master-bin.000002'; | ||
[connection slave] | ||
flush logs; | ||
purge binary logs to 'slave-bin.000002'; | ||
[connection master] | ||
include/sync_slave_sql_with_master.inc | ||
[connection master] | ||
meta count(*) | ||
# /*::TRX_META_DATA::{"ts":[ts]}*/ 31 | ||
[connection slave] | ||
meta count(*) | ||
# /*::TRX_META_DATA::{"ts":[ts,ts]}*/ 31 | ||
[connection master] | ||
drop table t1; | ||
[connection master] | ||
set @@global.binlog_trx_meta_data= 0; | ||
[connection slave] | ||
set @@global.binlog_trx_meta_data= 0; | ||
include/rpl_end.inc |
Oops, something went wrong.