Skip to content

Commit

Permalink
Merge pull request #4401 from ldonoso/PS-7798-5.7
Browse files Browse the repository at this point in the history
PS 7798 [5.7] - Merge fb-prod202102, fb-prod202103, fb-prod202104
  • Loading branch information
ldonoso authored Jul 14, 2021
2 parents cd09cec + daf394a commit 3983aa6
Show file tree
Hide file tree
Showing 21 changed files with 775 additions and 98 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -964,6 +964,7 @@ rocksdb_lock_scanned_rows OFF
rocksdb_lock_wait_timeout 1
rocksdb_log_file_time_to_roll 0
rocksdb_manifest_preallocation_size 4194304
rocksdb_manual_compaction_bottommost_level kForceOptimized
rocksdb_manual_compaction_threads 0
rocksdb_manual_wal_flush ON
rocksdb_master_skip_tx_api OFF
Expand Down
12 changes: 6 additions & 6 deletions mysql-test/suite/rocksdb/r/drop_cf_during_create_table.result
Original file line number Diff line number Diff line change
Expand Up @@ -18,16 +18,16 @@ CF_NAME STAT_TYPE VALUE
cf_primary_key NUM_IMMUTABLE_MEM_TABLE 0
cf_primary_key MEM_TABLE_FLUSH_PENDING 0
cf_primary_key COMPACTION_PENDING 0
cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 728
cf_primary_key CUR_SIZE_ALL_MEM_TABLES 728
cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 2048
cf_primary_key CUR_SIZE_ALL_MEM_TABLES 2048
cf_primary_key NUM_ENTRIES_ACTIVE_MEM_TABLE 0
cf_primary_key NUM_ENTRIES_IMM_MEM_TABLES 0
cf_primary_key NON_BLOCK_CACHE_SST_MEM_USAGE 0
cf_primary_key NUM_LIVE_VERSIONS 1
cf_primary_key NUM_IMMUTABLE_MEM_TABLE_FLUSHED 0
cf_primary_key NUM_RUNNING_FLUSHES 0
cf_primary_key NUM_RUNNING_COMPACTIONS 0
cf_primary_key SIZE_ALL_MEM_TABLES 728
cf_primary_key SIZE_ALL_MEM_TABLES 2048
cf_primary_key NUM_DELETES_ACTIVE_MEM_TABLE 0
cf_primary_key NUM_DELETES_IMM_MEM_TABLES 0
cf_primary_key ESTIMATE_NUM_KEYS 0
Expand Down Expand Up @@ -55,16 +55,16 @@ CF_NAME STAT_TYPE VALUE
cf_primary_key NUM_IMMUTABLE_MEM_TABLE 0
cf_primary_key MEM_TABLE_FLUSH_PENDING 0
cf_primary_key COMPACTION_PENDING 0
cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 728
cf_primary_key CUR_SIZE_ALL_MEM_TABLES 728
cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 2048
cf_primary_key CUR_SIZE_ALL_MEM_TABLES 2048
cf_primary_key NUM_ENTRIES_ACTIVE_MEM_TABLE 0
cf_primary_key NUM_ENTRIES_IMM_MEM_TABLES 0
cf_primary_key NON_BLOCK_CACHE_SST_MEM_USAGE 0
cf_primary_key NUM_LIVE_VERSIONS 1
cf_primary_key NUM_IMMUTABLE_MEM_TABLE_FLUSHED 0
cf_primary_key NUM_RUNNING_FLUSHES 0
cf_primary_key NUM_RUNNING_COMPACTIONS 0
cf_primary_key SIZE_ALL_MEM_TABLES 728
cf_primary_key SIZE_ALL_MEM_TABLES 2048
cf_primary_key NUM_DELETES_ACTIVE_MEM_TABLE 0
cf_primary_key NUM_DELETES_IMM_MEM_TABLES 0
cf_primary_key ESTIMATE_NUM_KEYS 0
Expand Down
6 changes: 3 additions & 3 deletions mysql-test/suite/rocksdb/r/drop_cf_during_drop_table.result
Original file line number Diff line number Diff line change
Expand Up @@ -29,16 +29,16 @@ CF_NAME STAT_TYPE VALUE
cf_primary_key NUM_IMMUTABLE_MEM_TABLE 0
cf_primary_key MEM_TABLE_FLUSH_PENDING 0
cf_primary_key COMPACTION_PENDING 0
cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 728
cf_primary_key CUR_SIZE_ALL_MEM_TABLES 728
cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 2048
cf_primary_key CUR_SIZE_ALL_MEM_TABLES 2048
cf_primary_key NUM_ENTRIES_ACTIVE_MEM_TABLE 0
cf_primary_key NUM_ENTRIES_IMM_MEM_TABLES 0
cf_primary_key NON_BLOCK_CACHE_SST_MEM_USAGE 0
cf_primary_key NUM_LIVE_VERSIONS 1
cf_primary_key NUM_IMMUTABLE_MEM_TABLE_FLUSHED 0
cf_primary_key NUM_RUNNING_FLUSHES 0
cf_primary_key NUM_RUNNING_COMPACTIONS 0
cf_primary_key SIZE_ALL_MEM_TABLES 728
cf_primary_key SIZE_ALL_MEM_TABLES 2048
cf_primary_key NUM_DELETES_ACTIVE_MEM_TABLE 0
cf_primary_key NUM_DELETES_IMM_MEM_TABLES 0
cf_primary_key ESTIMATE_NUM_KEYS 0
Expand Down
42 changes: 21 additions & 21 deletions mysql-test/suite/rocksdb/r/drop_cf_during_manual_compaction.result
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,16 @@ CF_NAME STAT_TYPE VALUE
cf_primary_key NUM_IMMUTABLE_MEM_TABLE 0
cf_primary_key MEM_TABLE_FLUSH_PENDING 0
cf_primary_key COMPACTION_PENDING 0
cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 728
cf_primary_key CUR_SIZE_ALL_MEM_TABLES 728
cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 2048
cf_primary_key CUR_SIZE_ALL_MEM_TABLES 2048
cf_primary_key NUM_ENTRIES_ACTIVE_MEM_TABLE 0
cf_primary_key NUM_ENTRIES_IMM_MEM_TABLES 0
cf_primary_key NON_BLOCK_CACHE_SST_MEM_USAGE 0
cf_primary_key NUM_LIVE_VERSIONS 1
cf_primary_key NUM_IMMUTABLE_MEM_TABLE_FLUSHED 0
cf_primary_key NUM_RUNNING_FLUSHES 0
cf_primary_key NUM_RUNNING_COMPACTIONS 0
cf_primary_key SIZE_ALL_MEM_TABLES 728
cf_primary_key SIZE_ALL_MEM_TABLES 2048
cf_primary_key NUM_DELETES_ACTIVE_MEM_TABLE 0
cf_primary_key NUM_DELETES_IMM_MEM_TABLES 0
cf_primary_key ESTIMATE_NUM_KEYS 0
Expand All @@ -30,16 +30,16 @@ CF_NAME STAT_TYPE VALUE
cf_primary_key NUM_IMMUTABLE_MEM_TABLE 0
cf_primary_key MEM_TABLE_FLUSH_PENDING 0
cf_primary_key COMPACTION_PENDING 0
cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 728
cf_primary_key CUR_SIZE_ALL_MEM_TABLES 728
cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 2048
cf_primary_key CUR_SIZE_ALL_MEM_TABLES 2048
cf_primary_key NUM_ENTRIES_ACTIVE_MEM_TABLE 0
cf_primary_key NUM_ENTRIES_IMM_MEM_TABLES 0
cf_primary_key NON_BLOCK_CACHE_SST_MEM_USAGE 0
cf_primary_key NUM_LIVE_VERSIONS 1
cf_primary_key NUM_IMMUTABLE_MEM_TABLE_FLUSHED 0
cf_primary_key NUM_RUNNING_FLUSHES 0
cf_primary_key NUM_RUNNING_COMPACTIONS 0
cf_primary_key SIZE_ALL_MEM_TABLES 728
cf_primary_key SIZE_ALL_MEM_TABLES 2048
cf_primary_key NUM_DELETES_ACTIVE_MEM_TABLE 0
cf_primary_key NUM_DELETES_IMM_MEM_TABLES 0
cf_primary_key ESTIMATE_NUM_KEYS 0
Expand All @@ -51,16 +51,16 @@ CF_NAME STAT_TYPE VALUE
cf_primary_key NUM_IMMUTABLE_MEM_TABLE 0
cf_primary_key MEM_TABLE_FLUSH_PENDING 0
cf_primary_key COMPACTION_PENDING 0
cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 728
cf_primary_key CUR_SIZE_ALL_MEM_TABLES 728
cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 2048
cf_primary_key CUR_SIZE_ALL_MEM_TABLES 2048
cf_primary_key NUM_ENTRIES_ACTIVE_MEM_TABLE 0
cf_primary_key NUM_ENTRIES_IMM_MEM_TABLES 0
cf_primary_key NON_BLOCK_CACHE_SST_MEM_USAGE 0
cf_primary_key NUM_LIVE_VERSIONS 1
cf_primary_key NUM_IMMUTABLE_MEM_TABLE_FLUSHED 0
cf_primary_key NUM_RUNNING_FLUSHES 0
cf_primary_key NUM_RUNNING_COMPACTIONS 0
cf_primary_key SIZE_ALL_MEM_TABLES 728
cf_primary_key SIZE_ALL_MEM_TABLES 2048
cf_primary_key NUM_DELETES_ACTIVE_MEM_TABLE 0
cf_primary_key NUM_DELETES_IMM_MEM_TABLES 0
cf_primary_key ESTIMATE_NUM_KEYS 0
Expand All @@ -73,16 +73,16 @@ CF_NAME STAT_TYPE VALUE
cf_primary_key NUM_IMMUTABLE_MEM_TABLE 0
cf_primary_key MEM_TABLE_FLUSH_PENDING 0
cf_primary_key COMPACTION_PENDING 0
cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 728
cf_primary_key CUR_SIZE_ALL_MEM_TABLES 728
cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 2048
cf_primary_key CUR_SIZE_ALL_MEM_TABLES 2048
cf_primary_key NUM_ENTRIES_ACTIVE_MEM_TABLE 0
cf_primary_key NUM_ENTRIES_IMM_MEM_TABLES 0
cf_primary_key NON_BLOCK_CACHE_SST_MEM_USAGE 0
cf_primary_key NUM_LIVE_VERSIONS 1
cf_primary_key NUM_IMMUTABLE_MEM_TABLE_FLUSHED 0
cf_primary_key NUM_RUNNING_FLUSHES 0
cf_primary_key NUM_RUNNING_COMPACTIONS 0
cf_primary_key SIZE_ALL_MEM_TABLES 728
cf_primary_key SIZE_ALL_MEM_TABLES 2048
cf_primary_key NUM_DELETES_ACTIVE_MEM_TABLE 0
cf_primary_key NUM_DELETES_IMM_MEM_TABLES 0
cf_primary_key ESTIMATE_NUM_KEYS 0
Expand All @@ -100,16 +100,16 @@ CF_NAME STAT_TYPE VALUE
cf_primary_key NUM_IMMUTABLE_MEM_TABLE 0
cf_primary_key MEM_TABLE_FLUSH_PENDING 0
cf_primary_key COMPACTION_PENDING 0
cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 728
cf_primary_key CUR_SIZE_ALL_MEM_TABLES 728
cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 2048
cf_primary_key CUR_SIZE_ALL_MEM_TABLES 2048
cf_primary_key NUM_ENTRIES_ACTIVE_MEM_TABLE 0
cf_primary_key NUM_ENTRIES_IMM_MEM_TABLES 0
cf_primary_key NON_BLOCK_CACHE_SST_MEM_USAGE 0
cf_primary_key NUM_LIVE_VERSIONS 1
cf_primary_key NUM_IMMUTABLE_MEM_TABLE_FLUSHED 0
cf_primary_key NUM_RUNNING_FLUSHES 0
cf_primary_key NUM_RUNNING_COMPACTIONS 0
cf_primary_key SIZE_ALL_MEM_TABLES 728
cf_primary_key SIZE_ALL_MEM_TABLES 2048
cf_primary_key NUM_DELETES_ACTIVE_MEM_TABLE 0
cf_primary_key NUM_DELETES_IMM_MEM_TABLES 0
cf_primary_key ESTIMATE_NUM_KEYS 0
Expand All @@ -123,16 +123,16 @@ CF_NAME STAT_TYPE VALUE
cf_primary_key NUM_IMMUTABLE_MEM_TABLE 0
cf_primary_key MEM_TABLE_FLUSH_PENDING 0
cf_primary_key COMPACTION_PENDING 0
cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 728
cf_primary_key CUR_SIZE_ALL_MEM_TABLES 728
cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 2048
cf_primary_key CUR_SIZE_ALL_MEM_TABLES 2048
cf_primary_key NUM_ENTRIES_ACTIVE_MEM_TABLE 0
cf_primary_key NUM_ENTRIES_IMM_MEM_TABLES 0
cf_primary_key NON_BLOCK_CACHE_SST_MEM_USAGE 0
cf_primary_key NUM_LIVE_VERSIONS 1
cf_primary_key NUM_IMMUTABLE_MEM_TABLE_FLUSHED 0
cf_primary_key NUM_RUNNING_FLUSHES 0
cf_primary_key NUM_RUNNING_COMPACTIONS 0
cf_primary_key SIZE_ALL_MEM_TABLES 728
cf_primary_key SIZE_ALL_MEM_TABLES 2048
cf_primary_key NUM_DELETES_ACTIVE_MEM_TABLE 0
cf_primary_key NUM_DELETES_IMM_MEM_TABLES 0
cf_primary_key ESTIMATE_NUM_KEYS 0
Expand All @@ -145,16 +145,16 @@ CF_NAME STAT_TYPE VALUE
cf_primary_key NUM_IMMUTABLE_MEM_TABLE 0
cf_primary_key MEM_TABLE_FLUSH_PENDING 0
cf_primary_key COMPACTION_PENDING 0
cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 728
cf_primary_key CUR_SIZE_ALL_MEM_TABLES 728
cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 2048
cf_primary_key CUR_SIZE_ALL_MEM_TABLES 2048
cf_primary_key NUM_ENTRIES_ACTIVE_MEM_TABLE 0
cf_primary_key NUM_ENTRIES_IMM_MEM_TABLES 0
cf_primary_key NON_BLOCK_CACHE_SST_MEM_USAGE 0
cf_primary_key NUM_LIVE_VERSIONS 1
cf_primary_key NUM_IMMUTABLE_MEM_TABLE_FLUSHED 0
cf_primary_key NUM_RUNNING_FLUSHES 0
cf_primary_key NUM_RUNNING_COMPACTIONS 0
cf_primary_key SIZE_ALL_MEM_TABLES 728
cf_primary_key SIZE_ALL_MEM_TABLES 2048
cf_primary_key NUM_DELETES_ACTIVE_MEM_TABLE 0
cf_primary_key NUM_DELETES_IMM_MEM_TABLES 0
cf_primary_key ESTIMATE_NUM_KEYS 0
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
CREATE PROCEDURE compact_start()
BEGIN
select variable_value into @c from performance_schema.global_status where variable_name='rocksdb_compact_write_bytes';
END//
CREATE PROCEDURE compact_end()
BEGIN
select case when variable_value-@c > 0 then 'true' else 'false' end as checked from performance_schema.global_status where variable_name='rocksdb_compact_write_bytes';
END//
CREATE TABLE t1 (
a int not null,
b int not null,
primary key (a,b),
key (b)
) ENGINE=RocksDB;
DELETE FROM t1;
set @@session.rocksdb_manual_compaction_bottommost_level='kSkip';
call compact_start();
set @@global.rocksdb_compact_cf = 'default';
call compact_end();
checked
true
set @@session.rocksdb_manual_compaction_bottommost_level='kSkip';
call compact_start();
set @@global.rocksdb_compact_cf = 'default';
call compact_end();
checked
false
set @@session.rocksdb_manual_compaction_bottommost_level='kForceOptimized';
call compact_start();
set @@global.rocksdb_compact_cf = 'default';
call compact_end();
checked
true
DROP PROCEDURE compact_start;
DROP PROCEDURE compact_end;
drop table t1;
6 changes: 3 additions & 3 deletions mysql-test/suite/rocksdb/r/restart_during_drop_cf.result
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,16 @@ CF_NAME STAT_TYPE VALUE
cf_primary_key NUM_IMMUTABLE_MEM_TABLE 0
cf_primary_key MEM_TABLE_FLUSH_PENDING 0
cf_primary_key COMPACTION_PENDING 0
cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 728
cf_primary_key CUR_SIZE_ALL_MEM_TABLES 728
cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 2048
cf_primary_key CUR_SIZE_ALL_MEM_TABLES 2048
cf_primary_key NUM_ENTRIES_ACTIVE_MEM_TABLE 0
cf_primary_key NUM_ENTRIES_IMM_MEM_TABLES 0
cf_primary_key NON_BLOCK_CACHE_SST_MEM_USAGE 0
cf_primary_key NUM_LIVE_VERSIONS 1
cf_primary_key NUM_IMMUTABLE_MEM_TABLE_FLUSHED 0
cf_primary_key NUM_RUNNING_FLUSHES 0
cf_primary_key NUM_RUNNING_COMPACTIONS 0
cf_primary_key SIZE_ALL_MEM_TABLES 728
cf_primary_key SIZE_ALL_MEM_TABLES 2048
cf_primary_key NUM_DELETES_ACTIVE_MEM_TABLE 0
cf_primary_key NUM_DELETES_IMM_MEM_TABLES 0
cf_primary_key ESTIMATE_NUM_KEYS 0
Expand Down
1 change: 1 addition & 0 deletions mysql-test/suite/rocksdb/r/rocksdb.result
Original file line number Diff line number Diff line change
Expand Up @@ -966,6 +966,7 @@ rocksdb_lock_scanned_rows OFF
rocksdb_lock_wait_timeout 1
rocksdb_log_file_time_to_roll 0
rocksdb_manifest_preallocation_size 4194304
rocksdb_manual_compaction_bottommost_level kForceOptimized
rocksdb_manual_compaction_threads 0
rocksdb_manual_wal_flush ON
rocksdb_master_skip_tx_api OFF
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
--loose_rocksdb_max_subcompactions=1
--loose_rocksdb_default_cf_options=target_file_size_base=100k;max_bytes_for_level_multiplier=1;max_bytes_for_level_base=1m;target_file_size_multiplier=1

48 changes: 48 additions & 0 deletions mysql-test/suite/rocksdb/t/manual_compaction_bottommost_level.test
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
--source include/have_rocksdb.inc

DELIMITER //;
CREATE PROCEDURE compact_start()
BEGIN
select variable_value into @c from performance_schema.global_status where variable_name='rocksdb_compact_write_bytes';
END//
CREATE PROCEDURE compact_end()
BEGIN
select case when variable_value-@c > 0 then 'true' else 'false' end as checked from performance_schema.global_status where variable_name='rocksdb_compact_write_bytes';
END//
DELIMITER ;//

CREATE TABLE t1 (
a int not null,
b int not null,
primary key (a,b),
key (b)
) ENGINE=RocksDB;

# Populate tables
let $max = 1000;
let $table = t1;
--source suite/rocksdb/include/drop_table_repopulate_table.inc

# compact to move all data in files in Lmax level
set @@session.rocksdb_manual_compaction_bottommost_level='kSkip';
call compact_start();
set @@global.rocksdb_compact_cf = 'default';
call compact_end(); # should return true as compaction of lower levels is performed

# skip compaction of files in bottommost level i.e. Lmax->Lmax compaction and ensure rocksdb_compact_write_byte
# does not increase after compaction
set @@session.rocksdb_manual_compaction_bottommost_level='kSkip';
call compact_start();
set @@global.rocksdb_compact_cf = 'default';
call compact_end(); # should return false as files only in bottommost layer

# restore 'kForceOptimized' default setting and check that value of rocksdb_compact_write_byte increses after compaction
set @@session.rocksdb_manual_compaction_bottommost_level='kForceOptimized';
call compact_start();
set @@global.rocksdb_compact_cf = 'default';
call compact_end(); # should return true

# cleanup
DROP PROCEDURE compact_start;
DROP PROCEDURE compact_end;
drop table t1;
Loading

0 comments on commit 3983aa6

Please sign in to comment.