use new FlushWAL API in MyRocks

Summary:
RocksDB has recently added a FlushWAL API which will improve upon the
performance of MySQL 2PC (more details here facebook/rocksdb#2345).
This patch adds support for using the FlushWAL API in MyRocks and also matches flush_log_at_trx_commit with innodb_flush_log_at_trx_commit behaviour.

Finally, it updates the submodule to include the removal of an unneeded
assertion in the write path, which was tripped by this change.

Reviewed By: yoshinorim

Differential Revision: D5503719

fbshipit-source-id: c29f0a2

mysql-test/r/mysqld--help-notwin-profiling.result

@@ -1147,6 +1147,9 @@ The following options may be given as the first argument:
  Enable or disable ROCKSDB_COMPACTION_STATS plugin.
  Possible values are ON, OFF, FORCE (don't start if the
  plugin fails to load).
+ --rocksdb-concurrent-prepare 
+ DBOptions::concurrent_prepare for RocksDB
+ (Defaults to on; use --skip-rocksdb-concurrent-prepare to disable.)
  --rocksdb-create-checkpoint=name 
  Checkpoint directory
  --rocksdb-create-if-missing 
@@ -1285,6 +1288,9 @@ The following options may be given as the first argument:
  DBOptions::log_file_time_to_roll for RocksDB
  --rocksdb-manifest-preallocation-size=# 
  DBOptions::manifest_preallocation_size for RocksDB
+ --rocksdb-manual-wal-flush 
+ DBOptions::manual_wal_flush for RocksDB
+ (Defaults to on; use --skip-rocksdb-manual-wal-flush to disable.)
  --rocksdb-master-skip-tx-api 
  Skipping holding any lock on row access. Not effective on
  slave.
@@ -2077,6 +2083,7 @@ rocksdb-compaction-sequential-deletes-count-sd FALSE
 rocksdb-compaction-sequential-deletes-file-size 0
 rocksdb-compaction-sequential-deletes-window 0
 rocksdb-compaction-stats ON
+rocksdb-concurrent-prepare TRUE
 rocksdb-create-checkpoint 
 rocksdb-create-if-missing TRUE
 rocksdb-create-missing-column-families FALSE
@@ -2120,6 +2127,7 @@ rocksdb-lock-wait-timeout 1
 rocksdb-locks ON
 rocksdb-log-file-time-to-roll 0
 rocksdb-manifest-preallocation-size 4194304
+rocksdb-manual-wal-flush TRUE
 rocksdb-master-skip-tx-api FALSE
 rocksdb-max-background-jobs 2
 rocksdb-max-log-file-size 0

mysql-test/r/mysqld--help-notwin.result

@@ -1145,6 +1145,9 @@ The following options may be given as the first argument:
  Enable or disable ROCKSDB_COMPACTION_STATS plugin.
  Possible values are ON, OFF, FORCE (don't start if the
  plugin fails to load).
+ --rocksdb-concurrent-prepare 
+ DBOptions::concurrent_prepare for RocksDB
+ (Defaults to on; use --skip-rocksdb-concurrent-prepare to disable.)
  --rocksdb-create-checkpoint=name 
  Checkpoint directory
  --rocksdb-create-if-missing 
@@ -1283,6 +1286,9 @@ The following options may be given as the first argument:
  DBOptions::log_file_time_to_roll for RocksDB
  --rocksdb-manifest-preallocation-size=# 
  DBOptions::manifest_preallocation_size for RocksDB
+ --rocksdb-manual-wal-flush 
+ DBOptions::manual_wal_flush for RocksDB
+ (Defaults to on; use --skip-rocksdb-manual-wal-flush to disable.)
  --rocksdb-master-skip-tx-api 
  Skipping holding any lock on row access. Not effective on
  slave.
@@ -2074,6 +2080,7 @@ rocksdb-compaction-sequential-deletes-count-sd FALSE
 rocksdb-compaction-sequential-deletes-file-size 0
 rocksdb-compaction-sequential-deletes-window 0
 rocksdb-compaction-stats ON
+rocksdb-concurrent-prepare TRUE
 rocksdb-create-checkpoint 
 rocksdb-create-if-missing TRUE
 rocksdb-create-missing-column-families FALSE
@@ -2117,6 +2124,7 @@ rocksdb-lock-wait-timeout 1
 rocksdb-locks ON
 rocksdb-log-file-time-to-roll 0
 rocksdb-manifest-preallocation-size 4194304
+rocksdb-manual-wal-flush TRUE
 rocksdb-master-skip-tx-api FALSE
 rocksdb-max-background-jobs 2
 rocksdb-max-log-file-size 0

mysql-test/suite/rocksdb/r/2pc_group_commit.result

@@ -4,20 +4,20 @@ CREATE DATABASE mysqlslap;
 USE mysqlslap;
 CREATE TABLE t1(id BIGINT AUTO_INCREMENT, value BIGINT, PRIMARY KEY(id)) ENGINE=rocksdb;
 # 2PC enabled, MyRocks durability enabled
-SET GLOBAL rocksdb_enable_2pc=0;
+SET GLOBAL rocksdb_enable_2pc=1;
 SET GLOBAL rocksdb_flush_log_at_trx_commit=1;
 ## 2PC + durability + single thread
 select variable_value into @c from information_schema.global_status where variable_name='rocksdb_wal_group_syncs';
 select case when variable_value-@c = 1000 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_wal_group_syncs';
 case when variable_value-@c = 1000 then 'true' else 'false' end
-false
+true
 ## 2PC + durability + group commit
 select variable_value into @c from information_schema.global_status where variable_name='rocksdb_wal_group_syncs';
 select case when variable_value-@c > 0 and variable_value-@c < 10000 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_wal_group_syncs';
 case when variable_value-@c > 0 and variable_value-@c < 10000 then 'true' else 'false' end
-false
+true
 # 2PC enabled, MyRocks durability disabled
-SET GLOBAL rocksdb_enable_2pc=0;
+SET GLOBAL rocksdb_enable_2pc=1;
 SET GLOBAL rocksdb_flush_log_at_trx_commit=0;
 select variable_value into @c from information_schema.global_status where variable_name='rocksdb_wal_group_syncs';
 select case when variable_value-@c = 0 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_wal_group_syncs';
@@ -28,16 +28,16 @@ select case when variable_value-@c = 0 then 'true' else 'false' end from informa
 case when variable_value-@c = 0 then 'true' else 'false' end
 true
 # 2PC disabled, MyRocks durability enabled
-SET GLOBAL rocksdb_enable_2pc=1;
+SET GLOBAL rocksdb_enable_2pc=0;
 SET GLOBAL rocksdb_flush_log_at_trx_commit=1;
 select variable_value into @c from information_schema.global_status where variable_name='rocksdb_wal_group_syncs';
 select case when variable_value-@c = 0 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_wal_group_syncs';
 case when variable_value-@c = 0 then 'true' else 'false' end
-false
+true
 select variable_value into @c from information_schema.global_status where variable_name='rocksdb_wal_group_syncs';
 select case when variable_value-@c = 0 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_wal_group_syncs';
 case when variable_value-@c = 0 then 'true' else 'false' end
-false
+true
 SET GLOBAL rocksdb_enable_2pc=1;
 SET GLOBAL rocksdb_flush_log_at_trx_commit=1;
 DROP TABLE t1;

mysql-test/suite/rocksdb/r/rocksdb.result

@@ -880,6 +880,7 @@ rocksdb_compaction_sequential_deletes	0
 rocksdb_compaction_sequential_deletes_count_sd	OFF
 rocksdb_compaction_sequential_deletes_file_size	0
 rocksdb_compaction_sequential_deletes_window	0
+rocksdb_concurrent_prepare	ON
 rocksdb_create_checkpoint	
 rocksdb_create_if_missing	ON
 rocksdb_create_missing_column_families	OFF
@@ -917,6 +918,7 @@ rocksdb_lock_scanned_rows	OFF
 rocksdb_lock_wait_timeout	1
 rocksdb_log_file_time_to_roll	0
 rocksdb_manifest_preallocation_size	4194304
+rocksdb_manual_wal_flush	ON
 rocksdb_master_skip_tx_api	OFF
 rocksdb_max_background_jobs	2
 rocksdb_max_log_file_size	0

mysql-test/suite/rocksdb/r/write_sync.result

@@ -2,35 +2,26 @@ SET GLOBAL rocksdb_write_disable_wal=false;
 SET GLOBAL rocksdb_write_ignore_missing_column_families=true;
 create table aaa (id int primary key, i int) engine rocksdb;
 set @save_rocksdb_flush_log_at_trx_commit=@@global.rocksdb_flush_log_at_trx_commit;
-SET GLOBAL rocksdb_flush_log_at_trx_commit=0;
+SET GLOBAL rocksdb_flush_log_at_trx_commit=1;
 select variable_value into @a from information_schema.global_status where variable_name='rocksdb_wal_synced';
 insert aaa(id, i) values(1,1);
 select variable_value-@a from information_schema.global_status where variable_name='rocksdb_wal_synced';
 variable_value-@a
-0
-insert aaa(id, i) values(2,1);
-select variable_value-@a from information_schema.global_status where variable_name='rocksdb_wal_synced';
-variable_value-@a
-0
-insert aaa(id, i) values(3,1);
-select variable_value-@a from information_schema.global_status where variable_name='rocksdb_wal_synced';
-variable_value-@a
-0
-SET GLOBAL rocksdb_flush_log_at_trx_commit=1;
-insert aaa(id, i) values(4,1);
-select variable_value-@a from information_schema.global_status where variable_name='rocksdb_wal_synced';
-variable_value-@a
 1
-insert aaa(id, i) values(5,1);
+insert aaa(id, i) values(2,1);
 select variable_value-@a from information_schema.global_status where variable_name='rocksdb_wal_synced';
 variable_value-@a
 2
-insert aaa(id, i) values(6,1);
+insert aaa(id, i) values(3,1);
 select variable_value-@a from information_schema.global_status where variable_name='rocksdb_wal_synced';
 variable_value-@a
 3
+SET GLOBAL rocksdb_flush_log_at_trx_commit=0;
+select variable_value into @a from information_schema.global_status where variable_name='rocksdb_wal_synced';
+insert aaa(id, i) values(4,1);
 SET GLOBAL rocksdb_flush_log_at_trx_commit=2;
-insert aaa(id, i) values(7,1);
+select variable_value into @a from information_schema.global_status where variable_name='rocksdb_wal_synced';
+insert aaa(id, i) values(5,1);
 truncate table aaa;
 drop table aaa;
 set @@global.rocksdb_flush_log_at_trx_commit=@save_rocksdb_flush_log_at_trx_commit;

mysql-test/suite/rocksdb/t/2pc_group_commit.test

@@ -13,7 +13,7 @@ USE mysqlslap;
 CREATE TABLE t1(id BIGINT AUTO_INCREMENT, value BIGINT, PRIMARY KEY(id)) ENGINE=rocksdb;
 
 --echo # 2PC enabled, MyRocks durability enabled
-SET GLOBAL rocksdb_enable_2pc=0;
+SET GLOBAL rocksdb_enable_2pc=1;
 SET GLOBAL rocksdb_flush_log_at_trx_commit=1;
 
 --echo ## 2PC + durability + single thread
@@ -28,7 +28,7 @@ select case when variable_value-@c > 0 and variable_value-@c < 10000 then 'true'
 
 
 --echo # 2PC enabled, MyRocks durability disabled
-SET GLOBAL rocksdb_enable_2pc=0;
+SET GLOBAL rocksdb_enable_2pc=1;
 SET GLOBAL rocksdb_flush_log_at_trx_commit=0;
 
 select variable_value into @c from information_schema.global_status where variable_name='rocksdb_wal_group_syncs';
@@ -41,7 +41,7 @@ select case when variable_value-@c = 0 then 'true' else 'false' end from informa
 
 
 --echo # 2PC disabled, MyRocks durability enabled
-SET GLOBAL rocksdb_enable_2pc=1;
+SET GLOBAL rocksdb_enable_2pc=0;
 SET GLOBAL rocksdb_flush_log_at_trx_commit=1;
 
 select variable_value into @c from information_schema.global_status where variable_name='rocksdb_wal_group_syncs';

mysql-test/suite/rocksdb/t/write_sync.test

@@ -6,8 +6,8 @@ SET GLOBAL rocksdb_write_ignore_missing_column_families=true;
 create table aaa (id int primary key, i int) engine rocksdb;
 
 set @save_rocksdb_flush_log_at_trx_commit=@@global.rocksdb_flush_log_at_trx_commit;
-SET GLOBAL rocksdb_flush_log_at_trx_commit=0;
---exec sleep 30
+SET GLOBAL rocksdb_flush_log_at_trx_commit=1;
+--exec sleep 5
 select variable_value into @a from information_schema.global_status where variable_name='rocksdb_wal_synced';
 insert aaa(id, i) values(1,1);
 select variable_value-@a from information_schema.global_status where variable_name='rocksdb_wal_synced';
@@ -16,19 +16,22 @@ select variable_value-@a from information_schema.global_status where variable_na
 insert aaa(id, i) values(3,1);
 select variable_value-@a from information_schema.global_status where variable_name='rocksdb_wal_synced';
 
-SET GLOBAL rocksdb_flush_log_at_trx_commit=1;
+SET GLOBAL rocksdb_flush_log_at_trx_commit=0;
+--exec sleep 5
+select variable_value into @a from information_schema.global_status where variable_name='rocksdb_wal_synced';
 insert aaa(id, i) values(4,1);
-select variable_value-@a from information_schema.global_status where variable_name='rocksdb_wal_synced';
-insert aaa(id, i) values(5,1);
-select variable_value-@a from information_schema.global_status where variable_name='rocksdb_wal_synced';
-insert aaa(id, i) values(6,1);
-select variable_value-@a from information_schema.global_status where variable_name='rocksdb_wal_synced';
+
+let $status_var=rocksdb_wal_synced;
+let $status_var_value=`select @a+1`;
+source include/wait_for_status_var.inc;
 
 SET GLOBAL rocksdb_flush_log_at_trx_commit=2;
-insert aaa(id, i) values(7,1);
+--exec sleep 5
+select variable_value into @a from information_schema.global_status where variable_name='rocksdb_wal_synced';
+insert aaa(id, i) values(5,1);
 
 let $status_var=rocksdb_wal_synced;
-let $status_var_value=`select @a+4`;
+let $status_var_value=`select @a+1`;
 source include/wait_for_status_var.inc;
 
 truncate table aaa;

mysql-test/suite/rocksdb_sys_vars/r/rocksdb_concurrent_prepare_basic.result

@@ -0,0 +1,14 @@
+CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO valid_values VALUES(1);
+INSERT INTO valid_values VALUES(1024);
+CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO invalid_values VALUES('\'aaa\'');
+SET @start_global_value = @@global.ROCKSDB_CONCURRENT_PREPARE;
+SELECT @start_global_value;
+@start_global_value
+1
+"Trying to set variable @@global.ROCKSDB_CONCURRENT_PREPARE to 444. It should fail because it is readonly."
+SET @@global.ROCKSDB_CONCURRENT_PREPARE   = 444;
+ERROR HY000: Variable 'rocksdb_concurrent_prepare' is a read only variable
+DROP TABLE valid_values;
+DROP TABLE invalid_values;

mysql-test/suite/rocksdb_sys_vars/r/rocksdb_manual_wal_flush_basic.result

@@ -0,0 +1,14 @@
+CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO valid_values VALUES(1);
+INSERT INTO valid_values VALUES(1024);
+CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO invalid_values VALUES('\'aaa\'');
+SET @start_global_value = @@global.ROCKSDB_MANUAL_WAL_FLUSH;
+SELECT @start_global_value;
+@start_global_value
+1
+"Trying to set variable @@global.ROCKSDB_MANUAL_WAL_FLUSH to 444. It should fail because it is readonly."
+SET @@global.ROCKSDB_MANUAL_WAL_FLUSH   = 444;
+ERROR HY000: Variable 'rocksdb_manual_wal_flush' is a read only variable
+DROP TABLE valid_values;
+DROP TABLE invalid_values;

mysql-test/suite/rocksdb_sys_vars/t/rocksdb_concurrent_prepare_basic.test

@@ -0,0 +1,16 @@
+--source include/have_rocksdb.inc
+
+CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO valid_values VALUES(1);
+INSERT INTO valid_values VALUES(1024);
+
+CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO invalid_values VALUES('\'aaa\'');
+
+--let $sys_var=ROCKSDB_CONCURRENT_PREPARE
+--let $read_only=1
+--let $session=0
+--source ../include/rocksdb_sys_var.inc
+
+DROP TABLE valid_values;
+DROP TABLE invalid_values;

mysql-test/suite/rocksdb_sys_vars/t/rocksdb_manual_wal_flush_basic.test

@@ -0,0 +1,16 @@
+--source include/have_rocksdb.inc
+
+CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO valid_values VALUES(1);
+INSERT INTO valid_values VALUES(1024);
+
+CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO invalid_values VALUES('\'aaa\'');
+
+--let $sys_var=ROCKSDB_MANUAL_WAL_FLUSH
+--let $read_only=1
+--let $session=0
+--source ../include/rocksdb_sys_var.inc
+
+DROP TABLE valid_values;
+DROP TABLE invalid_values;