-
Notifications
You must be signed in to change notification settings - Fork 713
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Optimize Rows_query_log_event logging to save disk space
Summary: A separate event for trx metadata was causing an unreasonable increase in binlog size. Also, now we're not writing rows query event if there are no comments in the query and `log_only_query_comments` is enabled Reviewed By: anirbanr-fb Differential Revision: D7002217 fbshipit-source-id: 1c3ab92
- Loading branch information
1 parent
9df52ea
commit bd25e29
Showing
19 changed files
with
430 additions
and
253 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,7 @@ | ||
| SET @@SESSION.BINLOG_ROWS_QUERY_LOG_EVENTS = 1; | ||
| SET @@GLOBAL.LOG_ONLY_QUERY_COMMENTS = 0; | ||
| CREATE TABLE t1 (a VARCHAR(256)); | ||
| INSERT INTO t1 VALUES ('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'); | ||
| include/assert.inc [Logged query must match the one issued.] | ||
| DROP TABLE t1; | ||
| SET @@GLOBAL.LOG_ONLY_QUERY_COMMENTS = DEFAULT; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,62 @@ | ||
| # Init | ||
| connection master; | ||
| flush logs; | ||
| let $master_last_log = query_get_value("SHOW MASTER STATUS", "File", 1); | ||
| eval purge binary logs to '$master_last_log'; | ||
| connection slave; | ||
| flush logs; | ||
| let $slave_last_log = query_get_value("SHOW MASTER STATUS", "File", 1); | ||
| eval purge binary logs to '$slave_last_log'; | ||
|
|
||
| # Generate load | ||
| connection master; | ||
| disable_result_log; | ||
| disable_query_log; | ||
| create table t1(a int auto_increment primary key) engine=innodb; | ||
| # the following loop will generate 300 meta data events, 1 for each statement | ||
| # the total number of meta data events will be 301, +1 for create stmt above | ||
| let $iter= 10; | ||
| while ($iter > 0) | ||
| { | ||
| begin; | ||
| eval /*insert*/insert into t1 values(); | ||
| eval /*update*/update t1 set a= a + 1; | ||
| /*delete*/delete from t1; | ||
| commit; | ||
| dec $iter; | ||
| } | ||
| enable_result_log; | ||
| enable_query_log; | ||
| sync_slave_with_master; | ||
|
|
||
| # Check binlogs for meta data | ||
| connection master; | ||
| disable_query_log; | ||
| let $MYSQLD_DATADIR = `select @@datadir`; | ||
| let $MYSQLD_SECURE_FILE_DIR = `select @@secure_file_priv`; | ||
| exec $MYSQL_BINLOG -v -v $MYSQLD_DATADIR/master-bin.0* | grep "::TRX_META_DATA::" | sed 's/\"[0-9]*\"/ts/g' > $MYSQLD_SECURE_FILE_DIR/meta_data.dat; | ||
| set sql_log_bin=0; | ||
| create table test.meta_data(meta text); | ||
| eval load data infile '$MYSQLD_SECURE_FILE_DIR/meta_data.dat' into table test.meta_data; | ||
| select meta, count(*) from test.meta_data group by meta; | ||
| drop table test.meta_data; | ||
| set sql_log_bin=1; | ||
| remove_file $MYSQLD_SECURE_FILE_DIR/meta_data.dat; | ||
|
|
||
| connection slave; | ||
| let $MYSQLD_DATADIR = `select @@datadir`; | ||
| let $MYSQLD_SECURE_FILE_DIR = `select @@secure_file_priv`; | ||
| exec $MYSQL_BINLOG -v -v $MYSQLD_DATADIR/slave-bin.0* | grep "::TRX_META_DATA::" | sed 's/\"[0-9]*\"/ts/g' > $MYSQLD_SECURE_FILE_DIR/meta_data.dat; | ||
| set sql_log_bin=0; | ||
| create table test.meta_data(meta text); | ||
| eval load data infile '$MYSQLD_SECURE_FILE_DIR/meta_data.dat' into table test.meta_data; | ||
| select meta, count(*) from test.meta_data group by meta; | ||
| drop table test.meta_data; | ||
| set sql_log_bin=1; | ||
| remove_file $MYSQLD_SECURE_FILE_DIR/meta_data.dat; | ||
| enable_query_log; | ||
|
|
||
| # Cleanup | ||
| connection master; | ||
| drop table t1; | ||
| sync_slave_with_master; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,69 @@ | ||
| include/master-slave.inc | ||
| Warnings: | ||
| Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. | ||
| Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. | ||
| [connection master] | ||
| set @@global.binlog_trx_meta_data= true; | ||
| set @@global.binlog_trx_meta_data= true; | ||
| set @@session.binlog_rows_query_log_events= OFF; | ||
| set @@global.binlog_rows_query_log_events= OFF; | ||
| include/stop_slave.inc | ||
| include/start_slave.inc | ||
| flush logs; | ||
| purge binary logs to 'master-bin.000002'; | ||
| flush logs; | ||
| purge binary logs to 'slave-bin.000002'; | ||
| meta count(*) | ||
| # /*::TRX_META_DATA::{"ts":[ts]}*/ 31 | ||
| meta count(*) | ||
| # /*::TRX_META_DATA::{"ts":[ts,ts]}*/ 31 | ||
| drop table t1; | ||
| set @@session.binlog_rows_query_log_events= ON; | ||
| set @@global.log_only_query_comments= OFF; | ||
| set @@global.binlog_rows_query_log_events= ON; | ||
| set @@global.log_only_query_comments= OFF; | ||
| include/stop_slave.inc | ||
| include/start_slave.inc | ||
| flush logs; | ||
| purge binary logs to 'master-bin.000003'; | ||
| flush logs; | ||
| purge binary logs to 'slave-bin.000003'; | ||
| meta count(*) | ||
| # /*::TRX_META_DATA::{"ts":[ts]}*/ 1 | ||
| # /*::TRX_META_DATA::{"ts":[ts]}*//*delete*/delete from t1 10 | ||
| # /*::TRX_META_DATA::{"ts":[ts]}*//*insert*/insert into t1 values() 10 | ||
| # /*::TRX_META_DATA::{"ts":[ts]}*//*update*/update t1 set a= a + 1 10 | ||
| meta count(*) | ||
| # /*::TRX_META_DATA::{"ts":[ts,ts]}*/ 1 | ||
| # /*::TRX_META_DATA::{"ts":[ts,ts]}*//*delete*/delete from t1 10 | ||
| # /*::TRX_META_DATA::{"ts":[ts,ts]}*//*insert*/insert into t1 values() 10 | ||
| # /*::TRX_META_DATA::{"ts":[ts,ts]}*//*update*/update t1 set a= a + 1 10 | ||
| drop table t1; | ||
| set @@session.binlog_rows_query_log_events= ON; | ||
| set @@global.log_only_query_comments= ON; | ||
| set @@global.binlog_rows_query_log_events= ON; | ||
| set @@global.log_only_query_comments= ON; | ||
| include/stop_slave.inc | ||
| include/start_slave.inc | ||
| flush logs; | ||
| purge binary logs to 'master-bin.000004'; | ||
| flush logs; | ||
| purge binary logs to 'slave-bin.000004'; | ||
| meta count(*) | ||
| # /*::TRX_META_DATA::{"ts":[ts]}*/ 1 | ||
| # /*::TRX_META_DATA::{"ts":[ts]}*//*delete*/ 10 | ||
| # /*::TRX_META_DATA::{"ts":[ts]}*//*insert*/ 10 | ||
| # /*::TRX_META_DATA::{"ts":[ts]}*//*update*/ 10 | ||
| meta count(*) | ||
| # /*::TRX_META_DATA::{"ts":[ts,ts]}*/ 1 | ||
| # /*::TRX_META_DATA::{"ts":[ts,ts]}*//*delete*/ 10 | ||
| # /*::TRX_META_DATA::{"ts":[ts,ts]}*//*insert*/ 10 | ||
| # /*::TRX_META_DATA::{"ts":[ts,ts]}*//*update*/ 10 | ||
| drop table t1; | ||
| set @@global.binlog_trx_meta_data= default; | ||
| set @@session.binlog_rows_query_log_events= default; | ||
| set @@global.log_only_query_comments= default; | ||
| set @@global.binlog_trx_meta_data= default; | ||
| set @@global.binlog_rows_query_log_events= default; | ||
| set @@global.log_only_query_comments= default; | ||
| include/rpl_end.inc |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,69 +1,18 @@ | ||
| source include/master-slave.inc; | ||
| source include/not_binlog_format_row.inc; # we have binlog_trx_meta_data_rbr.test for this | ||
|
|
||
| # Enable trx meta data | ||
| connection master; | ||
| set @save.binlog_trx_meta_data= @@global.binlog_trx_meta_data; | ||
| set @@global.binlog_trx_meta_data= true; | ||
|
|
||
| connection slave; | ||
| set @save.binlog_trx_meta_data= @@global.binlog_trx_meta_data; | ||
| set @@global.binlog_trx_meta_data= true; | ||
|
|
||
| # Generate load | ||
| connection master; | ||
| disable_result_log; | ||
| disable_query_log; | ||
| create table t1(a int) engine=innodb; | ||
| # the following loop will generate 300 meta data events, 1 for each statement | ||
| # the total number of meta data events will be 301, +1 for create stmt above | ||
| let $iter= 100; | ||
| while ($iter > 0) | ||
| { | ||
| begin; | ||
| eval insert into t1 values($iter); | ||
| eval update t1 set a= $iter + 1; | ||
| delete from t1; | ||
| commit; | ||
| dec $iter; | ||
| } | ||
| enable_result_log; | ||
| enable_query_log; | ||
| sync_slave_with_master; | ||
|
|
||
| # Check binlogs for meta data | ||
| connection master; | ||
| disable_query_log; | ||
| let $MYSQLD_DATADIR = `select @@datadir`; | ||
| let $MYSQLD_SECURE_FILE_DIR = `select @@secure_file_priv`; | ||
| exec $MYSQL_BINLOG -v -v $MYSQLD_DATADIR/master-bin.0* | grep "::TRX_META_DATA::" | sed 's/\"[0-9]*\"/ts/g' > $MYSQLD_SECURE_FILE_DIR/meta_data.dat; | ||
| set sql_log_bin=0; | ||
| create table test.meta_data(meta text); | ||
| eval load data infile '$MYSQLD_SECURE_FILE_DIR/meta_data.dat' into table test.meta_data; | ||
| select meta, count(*) from test.meta_data; | ||
| drop table test.meta_data; | ||
| set sql_log_bin=1; | ||
| remove_file $MYSQLD_SECURE_FILE_DIR/meta_data.dat; | ||
|
|
||
| connection slave; | ||
| let $MYSQLD_DATADIR = `select @@datadir`; | ||
| let $MYSQLD_SECURE_FILE_DIR = `select @@secure_file_priv`; | ||
| exec $MYSQL_BINLOG -v -v $MYSQLD_DATADIR/slave-bin.0* | grep "::TRX_META_DATA::" | sed 's/\"[0-9]*\"/ts/g' > $MYSQLD_SECURE_FILE_DIR/meta_data.dat; | ||
| set sql_log_bin=0; | ||
| create table test.meta_data(meta text); | ||
| eval load data infile '$MYSQLD_SECURE_FILE_DIR/meta_data.dat' into table test.meta_data; | ||
| select meta, count(*) from test.meta_data; | ||
| drop table test.meta_data; | ||
| set sql_log_bin=1; | ||
| remove_file $MYSQLD_SECURE_FILE_DIR/meta_data.dat; | ||
| enable_query_log; | ||
| source suite/rpl/include/binlog_trx_meta_data.inc; | ||
|
|
||
| # Cleanup | ||
| connection master; | ||
| drop table t1; | ||
| sync_slave_with_master; | ||
| set @@global.binlog_trx_meta_data= default; | ||
| connection slave; | ||
| set @@global.binlog_trx_meta_data= @save.binlog_trx_meta_data; | ||
| connection master; | ||
| set @@global.binlog_trx_meta_data= @save.binlog_trx_meta_data; | ||
| set @@global.binlog_trx_meta_data= default; | ||
|
|
||
| source include/rpl_end.inc; |
Oops, something went wrong.