From 16b3858b9dbc84c075368b467aefcba4b6b23229 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Fri, 9 Jul 2021 12:19:47 +0200 Subject: [PATCH 01/72] Fix incorrect bloom filter full key check Upstream commit ID : fb-mysql-5.6.35/64ba7e29cdc2d444e21e100acdbe2351e1c88384 PS-7798 : Merge fb-prod202102 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 065b73b46dabb7c5d41fb4622622e47928432a8d Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Fri, 9 Jul 2021 12:31:16 +0200 Subject: [PATCH 02/72] Support show_query_digest with SHOW ENGINE STATUS Upstream commit ID : fb-mysql-5.6.35/0fb83bb7569838026b912af23de52077244693b3 PS-7798 : Merge fb-prod202102 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 84c7c6f7520efa35647c43dd8326daedfe685a3a Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Fri, 26 Mar 2021 17:10:59 -0700 Subject: [PATCH 03/72] Bump rocksdb to 6.19 Upstream commit ID : fb-mysql-5.6.35/f896415fa0d31ca956fbe2e63ee0eed544879f96 PS-7798 : Merge fb-prod202102 Summary: This re-introduces Luqun's get_rocksdb_files.sh (which was reverted due to RocksDB perf regression in 6.17) to support building RocksDB 6.17+ and updates RocksDB to 6.19. The regression in 6.17 was reverted in RocksDB 6.18 so we should be able to proceed with new version. update-submodule: rocksdb Reviewed By: Pushapgl Differential Revision: D27378086 fbshipit-source-id: dd0e0eb7ba3 --- storage/rocksdb/get_rocksdb_files.sh | 21 ++++++++++++--------- storage/rocksdb/ha_rocksdb.cc | 6 ++++-- storage/rocksdb/rocksdb | 2 +- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/storage/rocksdb/get_rocksdb_files.sh b/storage/rocksdb/get_rocksdb_files.sh index dbaab0f7c462..a71279b960a6 100755 --- a/storage/rocksdb/get_rocksdb_files.sh +++ b/storage/rocksdb/get_rocksdb_files.sh @@ -14,14 +14,17 @@ rm $MKFILE # create build_version.cc file. Only create one if it doesn't exists or if it is different # this is so that we don't rebuild mysqld every time bv=rocksdb/util/build_version.cc -date=$(date +%F) -git_sha=$(pushd rocksdb >/dev/null && git rev-parse HEAD 2>/dev/null && popd >/dev/null) -if [ ! -f $bv ] || [ -z $git_sha ] || [ ! `grep $git_sha $bv` ] +build_date=$(date +%F) +pushd rocksdb>/dev/null +git_sha=$(git rev-parse HEAD 2>/dev/null) +git_tag=$(git symbolic-ref -q --short HEAD || \ + git describe --tags --exact-match 2>/dev/null) +git_mod=$(git diff-index HEAD --quiet 2>/dev/null; echo $?) +git_date=$(git log -1 --date=format:"%Y-%m-%d %T" --format="%ad" 2>/dev/null) +popd>/dev/null +if [ ! -f $bv ] || [ -z $git_sha ] || [ ! `grep -q $git_sha $bv` ] then -echo "#include \"build_version.h\" -const char* rocksdb_build_git_sha = -\"rocksdb_build_git_sha:$git_sha\"; -const char* rocksdb_build_git_date = -\"rocksdb_build_git_date:$date\"; -const char* rocksdb_build_compile_date = __DATE__;" > $bv +sed -e s/@GIT_SHA@/$git_sha/ -e s:@GIT_TAG@:"$git_tag": \ + -e s/@GIT_MOD@/"$git_mod"/ -e s/@BUILD_DATE@/"$build_date"/ \ + -e s/@GIT_DATE@/"$git_date"/ rocksdb/util/build_version.cc.in > $bv fi diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index 8546d645e5e0..dbb6f58ebe61 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -4039,7 +4039,8 @@ static int rocksdb_commit_by_xid(handlerton *const hton, XID *const xid) { DBUG_ASSERT(xid != nullptr); DBUG_ASSERT(commit_latency_stats != nullptr); - rocksdb::StopWatchNano timer(rocksdb::Env::Default(), true); + auto clock = rocksdb::Env::Default()->GetSystemClock().get(); + rocksdb::StopWatchNano timer(clock, true); const auto name = rdb_xid_to_string(*xid); DBUG_ASSERT(!name.empty()); @@ -4152,7 +4153,8 @@ static int rocksdb_commit(handlerton *const hton, THD *const thd, DBUG_ASSERT(thd != nullptr); DBUG_ASSERT(commit_latency_stats != nullptr); - rocksdb::StopWatchNano timer(rocksdb::Env::Default(), true); + auto clock = rocksdb::Env::Default()->GetSystemClock().get(); + rocksdb::StopWatchNano timer(clock, true); /* note: h->external_lock(F_UNLCK) is called after this function is called) */ Rdb_transaction *&tx = get_tx_from_thd(thd); diff --git a/storage/rocksdb/rocksdb b/storage/rocksdb/rocksdb index 09c7e96eac1a..0f8c041ea7bb 160000 --- a/storage/rocksdb/rocksdb +++ b/storage/rocksdb/rocksdb @@ -1 +1 @@ -Subproject commit 09c7e96eac1ab983f97ce9e0406730b8014b3398 +Subproject commit 0f8c041ea7bb458caa5ec0dbeef9fa42d0b97482 From 465d2e592ef53646b078e270cd3e3e887969f267 Mon Sep 17 00:00:00 2001 From: Manuel Ung Date: Wed, 10 Mar 2021 14:04:42 -0800 Subject: [PATCH 04/72] Check for errors during inplace_populate_sk Upstream commit ID : fb-mysql-5.6.35/f12eea3370f5c147305afdf0727b355bcd2cf4ec PS-7798 : Merge fb-prod202103 Summary: In `inplace_populate_sk`, `ha_index_init` can return an error if the query has been killed, and if that happens, it's not safe to continue with calling `index_first`/`index_next`. The fix is to check for errors and return early if needed. Reviewed By: luqun Differential Revision: D26960027 fbshipit-source-id: aaf7bd24f9d --- storage/rocksdb/ha_rocksdb.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index dbb6f58ebe61..77008526a875 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -13043,7 +13043,8 @@ int ha_rocksdb::inplace_populate_sk( keys. */ const uint pk = pk_index(table, m_tbl_def); - ha_index_init(pk, true); + res = ha_index_init(pk, true); + if (res) DBUG_RETURN(res); /* Scan each record in the primary key in order */ for (res = index_first(table->record[0]); res == 0; From 7405f20dc06238fd80eea6e108e7d4705b85f807 Mon Sep 17 00:00:00 2001 From: Rahul Kumar Date: Thu, 8 Apr 2021 16:12:10 -0700 Subject: [PATCH 05/72] Adds capability to set different BottommostLevelCompaction option for MyRocks Manual Compaction Upstream commit ID : fb-mysql-5.6.35/e00196f49460de6eeb91ee6f1e761da576b93ceb PS-7798 : Merge fb-prod202103 Summary: Currently BottommostLevelCompaction is set to default kForceOptimized for manual compaction. This is fine usually. However, for privacy triggered compactions, it does not make much sense, because it compacts Lmax SST files that do not have any overlapping key updates at all. This adds a session variable to specify the option for bottommost level compaction before triggering manual compaction. Porting from 8.0.20 branch Reviewed By: yoshinorim Differential Revision: D27780545 fbshipit-source-id: e5b9e7c6012 --- .../r-native-partitioning/rocksdb.result | 1 + .../manual_compaction_bottommost_level.result | 36 ++++++ mysql-test/suite/rocksdb/r/rocksdb.result | 1 + ...ual_compaction_bottommost_level-master.opt | 3 + .../t/manual_compaction_bottommost_level.test | 48 ++++++++ ...l_compaction_bottommost_level_basic.result | 114 ++++++++++++++++++ ...ual_compaction_bottommost_level_basic.test | 18 +++ storage/rocksdb/ha_rocksdb.cc | 42 ++++++- storage/rocksdb/rdb_threads.h | 10 +- 9 files changed, 264 insertions(+), 9 deletions(-) create mode 100644 mysql-test/suite/rocksdb/r/manual_compaction_bottommost_level.result create mode 100644 mysql-test/suite/rocksdb/t/manual_compaction_bottommost_level-master.opt create mode 100644 mysql-test/suite/rocksdb/t/manual_compaction_bottommost_level.test create mode 100644 mysql-test/suite/rocksdb_sys_vars/r/rocksdb_manual_compaction_bottommost_level_basic.result create mode 100644 mysql-test/suite/rocksdb_sys_vars/t/rocksdb_manual_compaction_bottommost_level_basic.test diff --git a/mysql-test/suite/rocksdb/r-native-partitioning/rocksdb.result b/mysql-test/suite/rocksdb/r-native-partitioning/rocksdb.result index c2f2827f75c8..d3b2f9c45551 100644 --- a/mysql-test/suite/rocksdb/r-native-partitioning/rocksdb.result +++ b/mysql-test/suite/rocksdb/r-native-partitioning/rocksdb.result @@ -964,6 +964,7 @@ rocksdb_lock_scanned_rows OFF rocksdb_lock_wait_timeout 1 rocksdb_log_file_time_to_roll 0 rocksdb_manifest_preallocation_size 4194304 +rocksdb_manual_compaction_bottommost_level kForceOptimized rocksdb_manual_compaction_threads 0 rocksdb_manual_wal_flush ON rocksdb_master_skip_tx_api OFF diff --git a/mysql-test/suite/rocksdb/r/manual_compaction_bottommost_level.result b/mysql-test/suite/rocksdb/r/manual_compaction_bottommost_level.result new file mode 100644 index 000000000000..bd8c031ef4ab --- /dev/null +++ b/mysql-test/suite/rocksdb/r/manual_compaction_bottommost_level.result @@ -0,0 +1,36 @@ +CREATE PROCEDURE compact_start() +BEGIN +select variable_value into @c from performance_schema.global_status where variable_name='rocksdb_compact_write_bytes'; +END// +CREATE PROCEDURE compact_end() +BEGIN +select case when variable_value-@c > 0 then 'true' else 'false' end as checked from performance_schema.global_status where variable_name='rocksdb_compact_write_bytes'; +END// +CREATE TABLE t1 ( +a int not null, +b int not null, +primary key (a,b), +key (b) +) ENGINE=RocksDB; +DELETE FROM t1; +set @@session.rocksdb_manual_compaction_bottommost_level='kSkip'; +call compact_start(); +set @@global.rocksdb_compact_cf = 'default'; +call compact_end(); +checked +true +set @@session.rocksdb_manual_compaction_bottommost_level='kSkip'; +call compact_start(); +set @@global.rocksdb_compact_cf = 'default'; +call compact_end(); +checked +false +set @@session.rocksdb_manual_compaction_bottommost_level='kForceOptimized'; +call compact_start(); +set @@global.rocksdb_compact_cf = 'default'; +call compact_end(); +checked +true +DROP PROCEDURE compact_start; +DROP PROCEDURE compact_end; +drop table t1; diff --git a/mysql-test/suite/rocksdb/r/rocksdb.result b/mysql-test/suite/rocksdb/r/rocksdb.result index 3c3274c67bc8..50680ea4959e 100644 --- a/mysql-test/suite/rocksdb/r/rocksdb.result +++ b/mysql-test/suite/rocksdb/r/rocksdb.result @@ -966,6 +966,7 @@ rocksdb_lock_scanned_rows OFF rocksdb_lock_wait_timeout 1 rocksdb_log_file_time_to_roll 0 rocksdb_manifest_preallocation_size 4194304 +rocksdb_manual_compaction_bottommost_level kForceOptimized rocksdb_manual_compaction_threads 0 rocksdb_manual_wal_flush ON rocksdb_master_skip_tx_api OFF diff --git a/mysql-test/suite/rocksdb/t/manual_compaction_bottommost_level-master.opt b/mysql-test/suite/rocksdb/t/manual_compaction_bottommost_level-master.opt new file mode 100644 index 000000000000..14406151c688 --- /dev/null +++ b/mysql-test/suite/rocksdb/t/manual_compaction_bottommost_level-master.opt @@ -0,0 +1,3 @@ +--loose_rocksdb_max_subcompactions=1 +--loose_rocksdb_default_cf_options=target_file_size_base=100k;max_bytes_for_level_multiplier=1;max_bytes_for_level_base=1m;target_file_size_multiplier=1 + diff --git a/mysql-test/suite/rocksdb/t/manual_compaction_bottommost_level.test b/mysql-test/suite/rocksdb/t/manual_compaction_bottommost_level.test new file mode 100644 index 000000000000..7a0df1f939c7 --- /dev/null +++ b/mysql-test/suite/rocksdb/t/manual_compaction_bottommost_level.test @@ -0,0 +1,48 @@ +--source include/have_rocksdb.inc + +DELIMITER //; +CREATE PROCEDURE compact_start() +BEGIN + select variable_value into @c from performance_schema.global_status where variable_name='rocksdb_compact_write_bytes'; +END// +CREATE PROCEDURE compact_end() +BEGIN + select case when variable_value-@c > 0 then 'true' else 'false' end as checked from performance_schema.global_status where variable_name='rocksdb_compact_write_bytes'; +END// +DELIMITER ;// + +CREATE TABLE t1 ( + a int not null, + b int not null, + primary key (a,b), + key (b) +) ENGINE=RocksDB; + +# Populate tables +let $max = 1000; +let $table = t1; +--source suite/rocksdb/include/drop_table_repopulate_table.inc + +# compact to move all data in files in Lmax level +set @@session.rocksdb_manual_compaction_bottommost_level='kSkip'; +call compact_start(); +set @@global.rocksdb_compact_cf = 'default'; +call compact_end(); # should return true as compaction of lower levels is performed + +# skip compaction of files in bottommost level i.e. Lmax->Lmax compaction and ensure rocksdb_compact_write_byte +# does not increase after compaction +set @@session.rocksdb_manual_compaction_bottommost_level='kSkip'; +call compact_start(); +set @@global.rocksdb_compact_cf = 'default'; +call compact_end(); # should return false as files only in bottommost layer + +# restore 'kForceOptimized' default setting and check that value of rocksdb_compact_write_byte increses after compaction +set @@session.rocksdb_manual_compaction_bottommost_level='kForceOptimized'; +call compact_start(); +set @@global.rocksdb_compact_cf = 'default'; +call compact_end(); # should return true + +# cleanup +DROP PROCEDURE compact_start; +DROP PROCEDURE compact_end; +drop table t1; diff --git a/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_manual_compaction_bottommost_level_basic.result b/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_manual_compaction_bottommost_level_basic.result new file mode 100644 index 000000000000..dd666bd1cbe3 --- /dev/null +++ b/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_manual_compaction_bottommost_level_basic.result @@ -0,0 +1,114 @@ +CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO valid_values VALUES('kSkip'); +INSERT INTO valid_values VALUES('kIfHaveCompactionFilter'); +INSERT INTO valid_values VALUES('kForce'); +INSERT INTO valid_values VALUES('kForceOptimized'); +CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO invalid_values VALUES('\'aaa\''); +SET @start_global_value = @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +SELECT @start_global_value; +@start_global_value +kForceOptimized +SET @start_session_value = @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +SELECT @start_session_value; +@start_session_value +kForceOptimized +'# Setting to valid values in global scope#' +"Trying to set variable @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL to kSkip" +SET @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = kSkip; +SELECT @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kSkip +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = DEFAULT; +SELECT @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +"Trying to set variable @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL to kIfHaveCompactionFilter" +SET @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = kIfHaveCompactionFilter; +SELECT @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kIfHaveCompactionFilter +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = DEFAULT; +SELECT @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +"Trying to set variable @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL to kForce" +SET @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = kForce; +SELECT @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForce +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = DEFAULT; +SELECT @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +"Trying to set variable @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL to kForceOptimized" +SET @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = kForceOptimized; +SELECT @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = DEFAULT; +SELECT @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +'# Setting to valid values in session scope#' +"Trying to set variable @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL to kSkip" +SET @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = kSkip; +SELECT @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kSkip +"Setting the session scope variable back to default" +SET @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = DEFAULT; +SELECT @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +"Trying to set variable @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL to kIfHaveCompactionFilter" +SET @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = kIfHaveCompactionFilter; +SELECT @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kIfHaveCompactionFilter +"Setting the session scope variable back to default" +SET @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = DEFAULT; +SELECT @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +"Trying to set variable @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL to kForce" +SET @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = kForce; +SELECT @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForce +"Setting the session scope variable back to default" +SET @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = DEFAULT; +SELECT @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +"Trying to set variable @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL to kForceOptimized" +SET @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = kForceOptimized; +SELECT @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +"Setting the session scope variable back to default" +SET @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = DEFAULT; +SELECT @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +'# Testing with invalid values in global scope #' +"Trying to set variable @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL to 'aaa'" +SET @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = 'aaa'; +Got one of the listed errors +SELECT @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +SET @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = @start_global_value; +SELECT @@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@global.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +SET @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL = @start_session_value; +SELECT @@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL; +@@session.ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +kForceOptimized +DROP TABLE valid_values; +DROP TABLE invalid_values; diff --git a/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_manual_compaction_bottommost_level_basic.test b/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_manual_compaction_bottommost_level_basic.test new file mode 100644 index 000000000000..bd716ec0f328 --- /dev/null +++ b/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_manual_compaction_bottommost_level_basic.test @@ -0,0 +1,18 @@ +--source include/have_rocksdb.inc + +CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO valid_values VALUES('kSkip'); +INSERT INTO valid_values VALUES('kIfHaveCompactionFilter'); +INSERT INTO valid_values VALUES('kForce'); +INSERT INTO valid_values VALUES('kForceOptimized'); + +CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO invalid_values VALUES('\'aaa\''); + +--let $sys_var=ROCKSDB_MANUAL_COMPACTION_BOTTOMMOST_LEVEL +--let $read_only=0 +--let $session=1 +--source ../include/rocksdb_sys_var.inc + +DROP TABLE valid_values; +DROP TABLE invalid_values; diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index 77008526a875..76ed862636ef 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -141,10 +141,12 @@ static handler *rocksdb_create_handler(my_core::handlerton *hton, my_core::MEM_ROOT *mem_root); static rocksdb::CompactRangeOptions getCompactRangeOptions( - int concurrency = 0) { + int concurrency = 0, + rocksdb::BottommostLevelCompaction bottommost_level_compaction = + rocksdb::BottommostLevelCompaction::kForceOptimized) { rocksdb::CompactRangeOptions compact_range_options; compact_range_options.bottommost_level_compaction = - rocksdb::BottommostLevelCompaction::kForceOptimized; + bottommost_level_compaction; compact_range_options.exclusive_manual_compaction = false; if (concurrency > 0) { compact_range_options.max_subcompactions = concurrency; @@ -847,6 +849,16 @@ static TYPELIB info_log_level_typelib = { array_elements(info_log_level_names) - 1, "info_log_level_typelib", info_log_level_names, nullptr}; +/* This enum needs to be kept up to date with rocksdb::BottommostLevelCompaction + */ +static const char *bottommost_level_compaction_names[] = { + "kSkip", "kIfHaveCompactionFilter", "kForce", "kForceOptimized", NullS}; + +static TYPELIB bottommost_level_compaction_typelib = { + array_elements(bottommost_level_compaction_names) - 1, + "bottommost_level_compaction_typelib", bottommost_level_compaction_names, + nullptr}; + static void rocksdb_set_rocksdb_info_log_level( THD *const thd, struct st_mysql_sys_var *const var, void *const var_ptr, const void *const save) { @@ -1200,6 +1212,15 @@ static MYSQL_THDVAR_INT( /* default rocksdb.dboption max_subcompactions */ 0, /* min */ 0, /* max */ 128, 0); +static MYSQL_THDVAR_ENUM( + manual_compaction_bottommost_level, PLUGIN_VAR_RQCMDARG, + "Option for bottommost level compaction during manual " + "compaction", + nullptr, nullptr, + /* default */ + (ulong)rocksdb::BottommostLevelCompaction::kForceOptimized, + &bottommost_level_compaction_typelib); + static MYSQL_SYSVAR_BOOL( create_if_missing, *reinterpret_cast(&rocksdb_db_options->create_if_missing), @@ -2339,6 +2360,7 @@ static struct st_mysql_sys_var *rocksdb_system_variables[] = { MYSQL_SYSVAR(debug_manual_compaction_delay), MYSQL_SYSVAR(max_manual_compactions), MYSQL_SYSVAR(manual_compaction_threads), + MYSQL_SYSVAR(manual_compaction_bottommost_level), MYSQL_SYSVAR(rollback_on_timeout), MYSQL_SYSVAR(enable_insert_with_update_caching), @@ -2384,8 +2406,13 @@ static int rocksdb_compact_column_family(THD *const thd, auto cfh = cf_manager.get_cf(cf_name); if (cfh != nullptr && rdb != nullptr) { + rocksdb::BottommostLevelCompaction bottommost_level_compaction = + (rocksdb::BottommostLevelCompaction)THDVAR( + thd, manual_compaction_bottommost_level); + int mc_id = rdb_mc_thread.request_manual_compaction( - cfh, nullptr, nullptr, THDVAR(thd, manual_compaction_threads)); + cfh, nullptr, nullptr, THDVAR(thd, manual_compaction_threads), + bottommost_level_compaction); if (mc_id == -1) { my_error(ER_INTERNAL_ERROR, MYF(0), "Can't schedule more manual compactions. " @@ -14203,8 +14230,9 @@ void Rdb_manual_compaction_thread::run() { // it is cancelled by CancelAllBackgroundWork, then status is // set to shutdownInProgress. const rocksdb::Status s = - rdb->CompactRange(getCompactRangeOptions(mcr.concurrency), mcr.cf.get(), - mcr.start, mcr.limit); + rdb->CompactRange(getCompactRangeOptions( + mcr.concurrency, mcr.bottommost_level_compaction), + mcr.cf.get(), mcr.start, mcr.limit); rocksdb_manual_compactions_running--; if (s.ok()) { @@ -14266,7 +14294,8 @@ void Rdb_manual_compaction_thread::clear_manual_compaction_request( int Rdb_manual_compaction_thread::request_manual_compaction( std::shared_ptr cf, rocksdb::Slice *start, - rocksdb::Slice *limit, int concurrency) { + rocksdb::Slice *limit, int concurrency, + rocksdb::BottommostLevelCompaction bottommost_level_compaction) { int mc_id = -1; RDB_MUTEX_LOCK_CHECK(m_mc_mutex); if (m_requests.size() >= rocksdb_max_manual_compactions) { @@ -14280,6 +14309,7 @@ int Rdb_manual_compaction_thread::request_manual_compaction( mcr.start = start; mcr.limit = limit; mcr.concurrency = concurrency; + mcr.bottommost_level_compaction = bottommost_level_compaction; m_requests.insert(std::make_pair(mcr.mc_id, mcr)); RDB_MUTEX_UNLOCK_CHECK(m_mc_mutex); return mc_id; diff --git a/storage/rocksdb/rdb_threads.h b/storage/rocksdb/rdb_threads.h index d5fc4e89b1d8..019e13777cb7 100644 --- a/storage/rocksdb/rdb_threads.h +++ b/storage/rocksdb/rdb_threads.h @@ -147,6 +147,8 @@ class Rdb_manual_compaction_thread : public Rdb_thread { rocksdb::Slice *start; rocksdb::Slice *limit; int concurrency = 0; + rocksdb::BottommostLevelCompaction bottommost_level_compaction = + rocksdb::BottommostLevelCompaction::kForceOptimized; }; int m_latest_mc_id; @@ -163,9 +165,11 @@ class Rdb_manual_compaction_thread : public Rdb_thread { } virtual void run() override; - int request_manual_compaction(std::shared_ptr cf, - rocksdb::Slice *start, rocksdb::Slice *limit, - int concurrency = 0); + int request_manual_compaction( + std::shared_ptr cf, rocksdb::Slice *start, + rocksdb::Slice *limit, int concurrency = 0, + rocksdb::BottommostLevelCompaction bottommost_level_compaction = + rocksdb::BottommostLevelCompaction::kForceOptimized); bool is_manual_compaction_finished(int mc_id); void clear_manual_compaction_request(int mc_id, bool init_only = false); void clear_all_manual_compaction_requests(); From dbd503972b8a88bd4c62ea6a5056d6298a9560ff Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Fri, 9 Jul 2021 14:05:31 +0200 Subject: [PATCH 06/72] Use float string comparison when comparing entire record Upstream commit ID : fb-mysql-5.6.35/997d25216c3f968fa0439bcba90d25ffd2bbb695 PS-7798 : Merge fb-prod202102 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 69c4e6ed535012628bfcbe129867ad931887cb0d Mon Sep 17 00:00:00 2001 From: Pushap Goyal Date: Wed, 21 Apr 2021 13:22:47 -0700 Subject: [PATCH 07/72] Port: Added new SQL function GET_INDEX_SIZE_BY_PREFIX Upstream commit ID : fb-mysql-5.6.35/e4f83712bced1e13a4a25da545719677d20890ab PS-7798 : Merge fb-prod202104 Summary: Port of D27419569. **Porting Notes** Core Data Demand Efficiency team wants to track physical size of the assoc types and fbtypes, as well as existing logical size. Tracking physical size of the specific assoc/fb types is not available as of now. ***Change*** Added the new function GET_INDEX_SIZE_BY_PREFIX which returns the index size by prefix. Currently we support only numeric keys. Reviewed By: yizhang82 Differential Revision: D27919708 fbshipit-source-id: c1ba5976cbe --- storage/rocksdb/ha_rocksdb.cc | 78 +++++++++++++++++++++-------------- storage/rocksdb/ha_rocksdb.h | 5 +++ 2 files changed, 53 insertions(+), 30 deletions(-) diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index 76ed862636ef..dfcf73768538 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -11889,6 +11889,47 @@ ha_rows ha_rocksdb::records_in_range(uint inx, key_range *const min_key, const Rdb_key_def &kd = *m_key_descr_arr[inx]; + auto disk_size = kd.m_stats.m_actual_disk_size; + if (disk_size == 0) disk_size = kd.m_stats.m_data_size; + auto rows = kd.m_stats.m_rows; + if (rows == 0 || disk_size == 0) { + rows = 1; + disk_size = ROCKSDB_ASSUMED_KEY_VALUE_DISK_SIZE; + } + ulonglong total_size = 0; + ulonglong total_row = 0; + records_in_range_internal(inx, min_key, max_key, disk_size, rows, &total_size, + &total_row); + ret = total_row; + /* + GetApproximateSizes() gives estimates so ret might exceed stats.records. + MySQL then decides to use full index scan rather than range scan, which + is not efficient for most cases. + To prevent this, changing estimated records slightly smaller than + stats.records. + */ + if (ret >= stats.records) { + ret = stats.records * 0.99; + } + + if (rocksdb_debug_optimizer_n_rows > 0) { + ret = rocksdb_debug_optimizer_n_rows; + } else if (ret == 0) { + ret = 1; + } + + DBUG_RETURN(ret); +} + +void ha_rocksdb::records_in_range_internal(uint inx, key_range *const min_key, + key_range *const max_key, + int64 disk_size, int64 rows, + ulonglong *total_size, + ulonglong *row_count) { + DBUG_ENTER_FUNC(); + + const Rdb_key_def &kd = *m_key_descr_arr[inx]; + uint size1 = 0; if (min_key) { size1 = kd.pack_index_tuple(table, m_pack_buffer, m_sk_packed_tuple, @@ -11922,50 +11963,27 @@ ha_rows ha_rocksdb::records_in_range(uint inx, key_range *const min_key, // right bound being successor() of the left one, e.g. "t.key>10 AND t.key<11" if (slice1.compare(slice2) >= 0) { // It's not possible to get slice2 > slice1 - DBUG_ASSERT(slice1.compare(slice2) == 0); - DBUG_RETURN(HA_EXIT_SUCCESS); + assert(slice1.compare(slice2) == 0); + DBUG_VOID_RETURN; } rocksdb::Range r(kd.m_is_reverse_cf ? slice2 : slice1, kd.m_is_reverse_cf ? slice1 : slice2); uint64_t sz = 0; - auto disk_size = kd.m_stats.m_actual_disk_size; - if (disk_size == 0) disk_size = kd.m_stats.m_data_size; - auto rows = kd.m_stats.m_rows; - if (rows == 0 || disk_size == 0) { - rows = 1; - disk_size = ROCKSDB_ASSUMED_KEY_VALUE_DISK_SIZE; - } #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" // Getting statistics, including from Memtables uint8_t include_flags = rocksdb::DB::INCLUDE_FILES; rdb->GetApproximateSizes(kd.get_cf(), &r, 1, &sz, include_flags); - ret = rows * ((double)sz / (double)disk_size); + *row_count = rows * ((double)sz / (double)disk_size); + *total_size = sz; uint64_t memTableCount; rdb->GetApproximateMemTableStats(kd.get_cf(), r, &memTableCount, &sz); - ret += memTableCount; - - /* - GetApproximateSizes() gives estimates so ret might exceed stats.records. - MySQL then decides to use full index scan rather than range scan, which - is not efficient for most cases. - To prevent this, changing estimated records slightly smaller than - stats.records. - */ - if (ret >= stats.records) { - ret = stats.records * 0.99; - } - - if (rocksdb_debug_optimizer_n_rows > 0) { - ret = rocksdb_debug_optimizer_n_rows; - } else if (ret == 0) { - ret = 1; - } - - DBUG_RETURN(ret); + *row_count += memTableCount; + *total_size += sz; + DBUG_VOID_RETURN; } void ha_rocksdb::update_create_info(HA_CREATE_INFO *const create_info) { diff --git a/storage/rocksdb/ha_rocksdb.h b/storage/rocksdb/ha_rocksdb.h index 0a0b78ce3e71..e916bc84ddb7 100644 --- a/storage/rocksdb/ha_rocksdb.h +++ b/storage/rocksdb/ha_rocksdb.h @@ -376,6 +376,11 @@ class ha_rocksdb : public my_core::handler { // the buffer size should be at least 2*Rdb_key_def::INDEX_NUMBER_SIZE rocksdb::Range get_range(const int i, uchar buf[]) const; + void records_in_range_internal(uint inx, key_range *const min_key, + key_range *const max_key, int64 disk_size, + int64 rows, ulonglong *total_size, + ulonglong *row_count); + /* Perf timers for data reads */ From 7a231e1c5b417ca4271d80b7d8ed967409577e5a Mon Sep 17 00:00:00 2001 From: Luke Lu Date: Tue, 27 Apr 2021 18:34:02 -0700 Subject: [PATCH 08/72] Updating RocksDB submodule to 6.20.2 for prod202103 Upstream commit ID : fb-mysql-5.6.35/ccfbfe798da49c4ea49483048647b9128850ef5d PS-7798 : Merge fb-prod202104 Summary: As title, stay on the latest rocksdb branch for each full feature as per Yoshi's recommendation. update-submodule: rocksdb Reviewed By: yoshinorim Differential Revision: D28049296 fbshipit-source-id: b74b75b101f --- storage/rocksdb/rocksdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/rocksdb/rocksdb b/storage/rocksdb/rocksdb index 0f8c041ea7bb..c56ad3c60a61 160000 --- a/storage/rocksdb/rocksdb +++ b/storage/rocksdb/rocksdb @@ -1 +1 @@ -Subproject commit 0f8c041ea7bb458caa5ec0dbeef9fa42d0b97482 +Subproject commit c56ad3c60a61bc39159c6fa1a112f6301cd86c89 From 8d61849a3b549b1a0b5be8a53d3b4c216ea7e9bc Mon Sep 17 00:00:00 2001 From: Luqun Lou Date: Mon, 3 May 2021 10:59:19 -0700 Subject: [PATCH 09/72] rebase multiple cf MTR due to memtable size difference Upstream commit ID : fb-mysql-5.6.35/dcf392a57223ff46cefe6bf07130a3bfd8c40cc9 PS-7798 : Merge fb-prod202104 Summary: see D28049296 (https://github.com/facebook/mysql-5.6/commit/ccfbfe798da49c4ea49483048647b9128850ef5d) discussion, there is a rocksdb change which cause this memtable size difference Reviewed By: yizhang82 Differential Revision: D28155396 fbshipit-source-id: 3267cd53d68 --- .../r/drop_cf_during_create_table.result | 12 +++--- .../r/drop_cf_during_drop_table.result | 6 +-- .../r/drop_cf_during_manual_compaction.result | 42 +++++++++---------- .../rocksdb/r/restart_during_drop_cf.result | 6 +-- 4 files changed, 33 insertions(+), 33 deletions(-) diff --git a/mysql-test/suite/rocksdb/r/drop_cf_during_create_table.result b/mysql-test/suite/rocksdb/r/drop_cf_during_create_table.result index 1c1a2d42c231..c3a78da4ebee 100644 --- a/mysql-test/suite/rocksdb/r/drop_cf_during_create_table.result +++ b/mysql-test/suite/rocksdb/r/drop_cf_during_create_table.result @@ -18,8 +18,8 @@ CF_NAME STAT_TYPE VALUE cf_primary_key NUM_IMMUTABLE_MEM_TABLE 0 cf_primary_key MEM_TABLE_FLUSH_PENDING 0 cf_primary_key COMPACTION_PENDING 0 -cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 728 -cf_primary_key CUR_SIZE_ALL_MEM_TABLES 728 +cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 2048 +cf_primary_key CUR_SIZE_ALL_MEM_TABLES 2048 cf_primary_key NUM_ENTRIES_ACTIVE_MEM_TABLE 0 cf_primary_key NUM_ENTRIES_IMM_MEM_TABLES 0 cf_primary_key NON_BLOCK_CACHE_SST_MEM_USAGE 0 @@ -27,7 +27,7 @@ cf_primary_key NUM_LIVE_VERSIONS 1 cf_primary_key NUM_IMMUTABLE_MEM_TABLE_FLUSHED 0 cf_primary_key NUM_RUNNING_FLUSHES 0 cf_primary_key NUM_RUNNING_COMPACTIONS 0 -cf_primary_key SIZE_ALL_MEM_TABLES 728 +cf_primary_key SIZE_ALL_MEM_TABLES 2048 cf_primary_key NUM_DELETES_ACTIVE_MEM_TABLE 0 cf_primary_key NUM_DELETES_IMM_MEM_TABLES 0 cf_primary_key ESTIMATE_NUM_KEYS 0 @@ -55,8 +55,8 @@ CF_NAME STAT_TYPE VALUE cf_primary_key NUM_IMMUTABLE_MEM_TABLE 0 cf_primary_key MEM_TABLE_FLUSH_PENDING 0 cf_primary_key COMPACTION_PENDING 0 -cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 728 -cf_primary_key CUR_SIZE_ALL_MEM_TABLES 728 +cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 2048 +cf_primary_key CUR_SIZE_ALL_MEM_TABLES 2048 cf_primary_key NUM_ENTRIES_ACTIVE_MEM_TABLE 0 cf_primary_key NUM_ENTRIES_IMM_MEM_TABLES 0 cf_primary_key NON_BLOCK_CACHE_SST_MEM_USAGE 0 @@ -64,7 +64,7 @@ cf_primary_key NUM_LIVE_VERSIONS 1 cf_primary_key NUM_IMMUTABLE_MEM_TABLE_FLUSHED 0 cf_primary_key NUM_RUNNING_FLUSHES 0 cf_primary_key NUM_RUNNING_COMPACTIONS 0 -cf_primary_key SIZE_ALL_MEM_TABLES 728 +cf_primary_key SIZE_ALL_MEM_TABLES 2048 cf_primary_key NUM_DELETES_ACTIVE_MEM_TABLE 0 cf_primary_key NUM_DELETES_IMM_MEM_TABLES 0 cf_primary_key ESTIMATE_NUM_KEYS 0 diff --git a/mysql-test/suite/rocksdb/r/drop_cf_during_drop_table.result b/mysql-test/suite/rocksdb/r/drop_cf_during_drop_table.result index 1ccb2bf5b2af..4abf1fce84b6 100644 --- a/mysql-test/suite/rocksdb/r/drop_cf_during_drop_table.result +++ b/mysql-test/suite/rocksdb/r/drop_cf_during_drop_table.result @@ -29,8 +29,8 @@ CF_NAME STAT_TYPE VALUE cf_primary_key NUM_IMMUTABLE_MEM_TABLE 0 cf_primary_key MEM_TABLE_FLUSH_PENDING 0 cf_primary_key COMPACTION_PENDING 0 -cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 728 -cf_primary_key CUR_SIZE_ALL_MEM_TABLES 728 +cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 2048 +cf_primary_key CUR_SIZE_ALL_MEM_TABLES 2048 cf_primary_key NUM_ENTRIES_ACTIVE_MEM_TABLE 0 cf_primary_key NUM_ENTRIES_IMM_MEM_TABLES 0 cf_primary_key NON_BLOCK_CACHE_SST_MEM_USAGE 0 @@ -38,7 +38,7 @@ cf_primary_key NUM_LIVE_VERSIONS 1 cf_primary_key NUM_IMMUTABLE_MEM_TABLE_FLUSHED 0 cf_primary_key NUM_RUNNING_FLUSHES 0 cf_primary_key NUM_RUNNING_COMPACTIONS 0 -cf_primary_key SIZE_ALL_MEM_TABLES 728 +cf_primary_key SIZE_ALL_MEM_TABLES 2048 cf_primary_key NUM_DELETES_ACTIVE_MEM_TABLE 0 cf_primary_key NUM_DELETES_IMM_MEM_TABLES 0 cf_primary_key ESTIMATE_NUM_KEYS 0 diff --git a/mysql-test/suite/rocksdb/r/drop_cf_during_manual_compaction.result b/mysql-test/suite/rocksdb/r/drop_cf_during_manual_compaction.result index 1a67b10dd307..cf317cba582b 100644 --- a/mysql-test/suite/rocksdb/r/drop_cf_during_manual_compaction.result +++ b/mysql-test/suite/rocksdb/r/drop_cf_during_manual_compaction.result @@ -6,8 +6,8 @@ CF_NAME STAT_TYPE VALUE cf_primary_key NUM_IMMUTABLE_MEM_TABLE 0 cf_primary_key MEM_TABLE_FLUSH_PENDING 0 cf_primary_key COMPACTION_PENDING 0 -cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 728 -cf_primary_key CUR_SIZE_ALL_MEM_TABLES 728 +cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 2048 +cf_primary_key CUR_SIZE_ALL_MEM_TABLES 2048 cf_primary_key NUM_ENTRIES_ACTIVE_MEM_TABLE 0 cf_primary_key NUM_ENTRIES_IMM_MEM_TABLES 0 cf_primary_key NON_BLOCK_CACHE_SST_MEM_USAGE 0 @@ -15,7 +15,7 @@ cf_primary_key NUM_LIVE_VERSIONS 1 cf_primary_key NUM_IMMUTABLE_MEM_TABLE_FLUSHED 0 cf_primary_key NUM_RUNNING_FLUSHES 0 cf_primary_key NUM_RUNNING_COMPACTIONS 0 -cf_primary_key SIZE_ALL_MEM_TABLES 728 +cf_primary_key SIZE_ALL_MEM_TABLES 2048 cf_primary_key NUM_DELETES_ACTIVE_MEM_TABLE 0 cf_primary_key NUM_DELETES_IMM_MEM_TABLES 0 cf_primary_key ESTIMATE_NUM_KEYS 0 @@ -30,8 +30,8 @@ CF_NAME STAT_TYPE VALUE cf_primary_key NUM_IMMUTABLE_MEM_TABLE 0 cf_primary_key MEM_TABLE_FLUSH_PENDING 0 cf_primary_key COMPACTION_PENDING 0 -cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 728 -cf_primary_key CUR_SIZE_ALL_MEM_TABLES 728 +cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 2048 +cf_primary_key CUR_SIZE_ALL_MEM_TABLES 2048 cf_primary_key NUM_ENTRIES_ACTIVE_MEM_TABLE 0 cf_primary_key NUM_ENTRIES_IMM_MEM_TABLES 0 cf_primary_key NON_BLOCK_CACHE_SST_MEM_USAGE 0 @@ -39,7 +39,7 @@ cf_primary_key NUM_LIVE_VERSIONS 1 cf_primary_key NUM_IMMUTABLE_MEM_TABLE_FLUSHED 0 cf_primary_key NUM_RUNNING_FLUSHES 0 cf_primary_key NUM_RUNNING_COMPACTIONS 0 -cf_primary_key SIZE_ALL_MEM_TABLES 728 +cf_primary_key SIZE_ALL_MEM_TABLES 2048 cf_primary_key NUM_DELETES_ACTIVE_MEM_TABLE 0 cf_primary_key NUM_DELETES_IMM_MEM_TABLES 0 cf_primary_key ESTIMATE_NUM_KEYS 0 @@ -51,8 +51,8 @@ CF_NAME STAT_TYPE VALUE cf_primary_key NUM_IMMUTABLE_MEM_TABLE 0 cf_primary_key MEM_TABLE_FLUSH_PENDING 0 cf_primary_key COMPACTION_PENDING 0 -cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 728 -cf_primary_key CUR_SIZE_ALL_MEM_TABLES 728 +cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 2048 +cf_primary_key CUR_SIZE_ALL_MEM_TABLES 2048 cf_primary_key NUM_ENTRIES_ACTIVE_MEM_TABLE 0 cf_primary_key NUM_ENTRIES_IMM_MEM_TABLES 0 cf_primary_key NON_BLOCK_CACHE_SST_MEM_USAGE 0 @@ -60,7 +60,7 @@ cf_primary_key NUM_LIVE_VERSIONS 1 cf_primary_key NUM_IMMUTABLE_MEM_TABLE_FLUSHED 0 cf_primary_key NUM_RUNNING_FLUSHES 0 cf_primary_key NUM_RUNNING_COMPACTIONS 0 -cf_primary_key SIZE_ALL_MEM_TABLES 728 +cf_primary_key SIZE_ALL_MEM_TABLES 2048 cf_primary_key NUM_DELETES_ACTIVE_MEM_TABLE 0 cf_primary_key NUM_DELETES_IMM_MEM_TABLES 0 cf_primary_key ESTIMATE_NUM_KEYS 0 @@ -73,8 +73,8 @@ CF_NAME STAT_TYPE VALUE cf_primary_key NUM_IMMUTABLE_MEM_TABLE 0 cf_primary_key MEM_TABLE_FLUSH_PENDING 0 cf_primary_key COMPACTION_PENDING 0 -cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 728 -cf_primary_key CUR_SIZE_ALL_MEM_TABLES 728 +cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 2048 +cf_primary_key CUR_SIZE_ALL_MEM_TABLES 2048 cf_primary_key NUM_ENTRIES_ACTIVE_MEM_TABLE 0 cf_primary_key NUM_ENTRIES_IMM_MEM_TABLES 0 cf_primary_key NON_BLOCK_CACHE_SST_MEM_USAGE 0 @@ -82,7 +82,7 @@ cf_primary_key NUM_LIVE_VERSIONS 1 cf_primary_key NUM_IMMUTABLE_MEM_TABLE_FLUSHED 0 cf_primary_key NUM_RUNNING_FLUSHES 0 cf_primary_key NUM_RUNNING_COMPACTIONS 0 -cf_primary_key SIZE_ALL_MEM_TABLES 728 +cf_primary_key SIZE_ALL_MEM_TABLES 2048 cf_primary_key NUM_DELETES_ACTIVE_MEM_TABLE 0 cf_primary_key NUM_DELETES_IMM_MEM_TABLES 0 cf_primary_key ESTIMATE_NUM_KEYS 0 @@ -100,8 +100,8 @@ CF_NAME STAT_TYPE VALUE cf_primary_key NUM_IMMUTABLE_MEM_TABLE 0 cf_primary_key MEM_TABLE_FLUSH_PENDING 0 cf_primary_key COMPACTION_PENDING 0 -cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 728 -cf_primary_key CUR_SIZE_ALL_MEM_TABLES 728 +cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 2048 +cf_primary_key CUR_SIZE_ALL_MEM_TABLES 2048 cf_primary_key NUM_ENTRIES_ACTIVE_MEM_TABLE 0 cf_primary_key NUM_ENTRIES_IMM_MEM_TABLES 0 cf_primary_key NON_BLOCK_CACHE_SST_MEM_USAGE 0 @@ -109,7 +109,7 @@ cf_primary_key NUM_LIVE_VERSIONS 1 cf_primary_key NUM_IMMUTABLE_MEM_TABLE_FLUSHED 0 cf_primary_key NUM_RUNNING_FLUSHES 0 cf_primary_key NUM_RUNNING_COMPACTIONS 0 -cf_primary_key SIZE_ALL_MEM_TABLES 728 +cf_primary_key SIZE_ALL_MEM_TABLES 2048 cf_primary_key NUM_DELETES_ACTIVE_MEM_TABLE 0 cf_primary_key NUM_DELETES_IMM_MEM_TABLES 0 cf_primary_key ESTIMATE_NUM_KEYS 0 @@ -123,8 +123,8 @@ CF_NAME STAT_TYPE VALUE cf_primary_key NUM_IMMUTABLE_MEM_TABLE 0 cf_primary_key MEM_TABLE_FLUSH_PENDING 0 cf_primary_key COMPACTION_PENDING 0 -cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 728 -cf_primary_key CUR_SIZE_ALL_MEM_TABLES 728 +cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 2048 +cf_primary_key CUR_SIZE_ALL_MEM_TABLES 2048 cf_primary_key NUM_ENTRIES_ACTIVE_MEM_TABLE 0 cf_primary_key NUM_ENTRIES_IMM_MEM_TABLES 0 cf_primary_key NON_BLOCK_CACHE_SST_MEM_USAGE 0 @@ -132,7 +132,7 @@ cf_primary_key NUM_LIVE_VERSIONS 1 cf_primary_key NUM_IMMUTABLE_MEM_TABLE_FLUSHED 0 cf_primary_key NUM_RUNNING_FLUSHES 0 cf_primary_key NUM_RUNNING_COMPACTIONS 0 -cf_primary_key SIZE_ALL_MEM_TABLES 728 +cf_primary_key SIZE_ALL_MEM_TABLES 2048 cf_primary_key NUM_DELETES_ACTIVE_MEM_TABLE 0 cf_primary_key NUM_DELETES_IMM_MEM_TABLES 0 cf_primary_key ESTIMATE_NUM_KEYS 0 @@ -145,8 +145,8 @@ CF_NAME STAT_TYPE VALUE cf_primary_key NUM_IMMUTABLE_MEM_TABLE 0 cf_primary_key MEM_TABLE_FLUSH_PENDING 0 cf_primary_key COMPACTION_PENDING 0 -cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 728 -cf_primary_key CUR_SIZE_ALL_MEM_TABLES 728 +cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 2048 +cf_primary_key CUR_SIZE_ALL_MEM_TABLES 2048 cf_primary_key NUM_ENTRIES_ACTIVE_MEM_TABLE 0 cf_primary_key NUM_ENTRIES_IMM_MEM_TABLES 0 cf_primary_key NON_BLOCK_CACHE_SST_MEM_USAGE 0 @@ -154,7 +154,7 @@ cf_primary_key NUM_LIVE_VERSIONS 1 cf_primary_key NUM_IMMUTABLE_MEM_TABLE_FLUSHED 0 cf_primary_key NUM_RUNNING_FLUSHES 0 cf_primary_key NUM_RUNNING_COMPACTIONS 0 -cf_primary_key SIZE_ALL_MEM_TABLES 728 +cf_primary_key SIZE_ALL_MEM_TABLES 2048 cf_primary_key NUM_DELETES_ACTIVE_MEM_TABLE 0 cf_primary_key NUM_DELETES_IMM_MEM_TABLES 0 cf_primary_key ESTIMATE_NUM_KEYS 0 diff --git a/mysql-test/suite/rocksdb/r/restart_during_drop_cf.result b/mysql-test/suite/rocksdb/r/restart_during_drop_cf.result index 263457e34876..3654306963af 100644 --- a/mysql-test/suite/rocksdb/r/restart_during_drop_cf.result +++ b/mysql-test/suite/rocksdb/r/restart_during_drop_cf.result @@ -7,8 +7,8 @@ CF_NAME STAT_TYPE VALUE cf_primary_key NUM_IMMUTABLE_MEM_TABLE 0 cf_primary_key MEM_TABLE_FLUSH_PENDING 0 cf_primary_key COMPACTION_PENDING 0 -cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 728 -cf_primary_key CUR_SIZE_ALL_MEM_TABLES 728 +cf_primary_key CUR_SIZE_ACTIVE_MEM_TABLE 2048 +cf_primary_key CUR_SIZE_ALL_MEM_TABLES 2048 cf_primary_key NUM_ENTRIES_ACTIVE_MEM_TABLE 0 cf_primary_key NUM_ENTRIES_IMM_MEM_TABLES 0 cf_primary_key NON_BLOCK_CACHE_SST_MEM_USAGE 0 @@ -16,7 +16,7 @@ cf_primary_key NUM_LIVE_VERSIONS 1 cf_primary_key NUM_IMMUTABLE_MEM_TABLE_FLUSHED 0 cf_primary_key NUM_RUNNING_FLUSHES 0 cf_primary_key NUM_RUNNING_COMPACTIONS 0 -cf_primary_key SIZE_ALL_MEM_TABLES 728 +cf_primary_key SIZE_ALL_MEM_TABLES 2048 cf_primary_key NUM_DELETES_ACTIVE_MEM_TABLE 0 cf_primary_key NUM_DELETES_IMM_MEM_TABLES 0 cf_primary_key ESTIMATE_NUM_KEYS 0 From c91cd8b44bebfa845c51abd456930d578817a980 Mon Sep 17 00:00:00 2001 From: Andrew Kryczka Date: Sat, 8 May 2021 23:04:33 -0700 Subject: [PATCH 10/72] Register comparators in mysql_ldb Upstream commit ID : fb-mysql-5.6.35/d45c30966752a04ca931c86552e00df3ef933bd7 PS-7798 : Merge fb-prod202104 Summary: Previously the `mysql_ldb` commands could not operate on a MyRocks instance due to wrong comparator provided. In particular, MyRocks column families can use a mix of "RocksDB_SE_v3.10" and "rev:RocksDB_SE_v3.10"; however, the provided comparator was always "RocksDB_SE_v3.10". This diff registers both possible comparators, which allows each column family's proper comparator to be deserialized from the OPTIONS file. The user must pass `--try_load_options` to their `ldb` command. Reviewed By: yizhang82 Differential Revision: D28313731 fbshipit-source-id: e9219a1199d --- storage/rocksdb/tools/mysql_ldb.cc | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/storage/rocksdb/tools/mysql_ldb.cc b/storage/rocksdb/tools/mysql_ldb.cc index 08efcf5ddf52..2dd38575915b 100644 --- a/storage/rocksdb/tools/mysql_ldb.cc +++ b/storage/rocksdb/tools/mysql_ldb.cc @@ -5,13 +5,31 @@ // #include "../rdb_comparator.h" #include "rocksdb/ldb_tool.h" +#include "rocksdb/utilities/object_registry.h" int main(int argc, char **argv) { - rocksdb::Options db_options; - const myrocks::Rdb_pk_comparator pk_comparator; - db_options.comparator = &pk_comparator; + // Register the comparators so they can be loaded from OPTIONS file when + // `--try_load_options` is provided. + rocksdb::ObjectLibrary::Default()->Register( + myrocks::Rdb_pk_comparator().Name(), + rocksdb::FactoryFunc( + [](const std::string & /* uri */, + std::unique_ptr * /* res_guard */, + std::string * /* err_msg */) { + static myrocks::Rdb_pk_comparator cmp; + return &cmp; + })); + rocksdb::ObjectLibrary::Default()->Register( + myrocks::Rdb_rev_comparator().Name(), + rocksdb::FactoryFunc( + [](const std::string & /* uri */, + std::unique_ptr * /* res_guard */, + std::string * /* err_msg */) { + static myrocks::Rdb_rev_comparator cmp; + return &cmp; + })); rocksdb::LDBTool tool; - tool.Run(argc, argv, db_options); + tool.Run(argc, argv); return 0; } From 220ab8bdbffdd582b8a8f351c6c6fe53cf1a3bd4 Mon Sep 17 00:00:00 2001 From: Yoshinori Matsunobu Date: Thu, 20 May 2021 10:54:58 -0700 Subject: [PATCH 11/72] Aborting process on all RDB_IO_ERROR_TX_COMMIT errors on commit() Upstream commit ID : fb-mysql-5.6.35/ba9d330448aa82009df5303bef21f952e49d2547 PS-7798 : Merge fb-prod202104 Summary: When hitting some I/O errors in RocksDB (e.g. out of space), current RocksDB may return error codes other than kIOError at engine commit phase, while not returning any error at prepare phase, depending on how it hits errors. Our expectation is any engine commit failure of the two phase commit protocol should abort the process, since binlog commit is already done. This diff makes sure to call abort() to terminate the process on RDB_IO_ERROR_TX_COMMIT at commit phase, not just RDB_IO_ERROR_DICT_COMMIT. This prevents from a case that a transaction gets written to binlog (and replicated) but it is rolled back in MyRocks engine due to commit failure, which is an inconsistent state. With this diff, binlog and engine will be consistent based on binlog state (either rolled forward or rolled back, aligned to binlog state). This diff makes another change that it no longer aborts on systems errors on MyRocks Prepare. Prior to this diff, kIOError at Prepare() terminated the process, but it was not necessary since transactions could be safely rolled back on Prepare() error. Clients hitting errors at Prepare() will get ER_INTERNAL_ERROR with RocksDB error status. Default ER_ERROR_DURING_COMMIT is not returned. This was because ER_ERROR_DURING_COMMIT does not take string arguments so it is difficult for users to understand why it failed. Reviewed By: hermanlee Differential Revision: D28576723 fbshipit-source-id: 1b0b6893f30 --- .../rocksdb_rpl/r/tx_system_failure.result | 240 ++++++++++++++++++ .../rocksdb_rpl/t/tx_system_failure.test | 37 +++ .../t/tx_system_failure_commit.inc | 53 ++++ .../t/tx_system_failure_prepare.inc | 17 ++ storage/rocksdb/ha_rocksdb.cc | 48 +++- 5 files changed, 385 insertions(+), 10 deletions(-) create mode 100644 mysql-test/suite/rocksdb_rpl/r/tx_system_failure.result create mode 100644 mysql-test/suite/rocksdb_rpl/t/tx_system_failure.test create mode 100644 mysql-test/suite/rocksdb_rpl/t/tx_system_failure_commit.inc create mode 100644 mysql-test/suite/rocksdb_rpl/t/tx_system_failure_prepare.inc diff --git a/mysql-test/suite/rocksdb_rpl/r/tx_system_failure.result b/mysql-test/suite/rocksdb_rpl/r/tx_system_failure.result new file mode 100644 index 000000000000..884c922a4f74 --- /dev/null +++ b/mysql-test/suite/rocksdb_rpl/r/tx_system_failure.result @@ -0,0 +1,240 @@ +include/master-slave.inc +[connection master] +call mtr.add_suppression("Attempting backtrace"); +call mtr.add_suppression("failed to write to WAL"); +call mtr.add_suppression("aborting on WAL write error"); +create table t1 (id int primary key, value int) engine = rocksdb; +insert into t1 values (1,1),(2,2),(3,3); +set session debug= "+d,myrocks_prepare_io_error"; +insert into t1 values (4,4); +ERROR HY000: Internal error: RocksDB error on COMMIT (Prepare/merge): IO error: +begin; +insert into t1 values (5,5); +insert into t1 values (6,6); +commit; +ERROR HY000: Internal error: RocksDB error on COMMIT (Prepare/merge): IO error: +set session debug= "-d,myrocks_prepare_io_error"; +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +drop table t1; +create table t1 (id int primary key, value int) engine = rocksdb; +insert into t1 values (1,1),(2,2),(3,3); +set session debug= "+d,myrocks_prepare_incomplete"; +insert into t1 values (4,4); +ERROR HY000: Internal error: RocksDB error on COMMIT (Prepare/merge): Result incomplete: +begin; +insert into t1 values (5,5); +insert into t1 values (6,6); +commit; +ERROR HY000: Internal error: RocksDB error on COMMIT (Prepare/merge): Result incomplete: +set session debug= "-d,myrocks_prepare_incomplete"; +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +drop table t1; +[connection master] +create table t1 (id int primary key, value int) engine = rocksdb; +insert into t1 values (1,1),(2,2),(3,3); +set session debug= "+d,myrocks_commit_io_error"; +insert into t1 values (4,4); +ERROR HY000: Lost connection to MySQL server during query +include/rpl_reconnect.inc +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +4 4 +include/sync_slave_sql_with_master.inc +[connection slave] +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +4 4 +[connection master] +set session debug= "+d,myrocks_commit_io_error"; +begin; +insert into t1 values (5,5); +insert into t1 values (6,6); +commit; +ERROR HY000: Lost connection to MySQL server during query +include/rpl_reconnect.inc +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +include/sync_slave_sql_with_master.inc +[connection slave] +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +[connection master] +drop table t1; +include/sync_slave_sql_with_master.inc +[connection master] +create table t1 (id int primary key, value int) engine = rocksdb; +insert into t1 values (1,1),(2,2),(3,3); +set session debug= "+d,myrocks_commit_incomplete"; +insert into t1 values (4,4); +ERROR HY000: Lost connection to MySQL server during query +include/rpl_reconnect.inc +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +4 4 +include/sync_slave_sql_with_master.inc +[connection slave] +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +4 4 +[connection master] +set session debug= "+d,myrocks_commit_incomplete"; +begin; +insert into t1 values (5,5); +insert into t1 values (6,6); +commit; +ERROR HY000: Lost connection to MySQL server during query +include/rpl_reconnect.inc +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +include/sync_slave_sql_with_master.inc +[connection slave] +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +[connection master] +drop table t1; +include/sync_slave_sql_with_master.inc +[connection master] +create table t1 (id int primary key, value int) engine = rocksdb; +insert into t1 values (1,1),(2,2),(3,3); +set session debug= "+d,myrocks_commit_merge_io_error"; +insert into t1 values (4,4); +ERROR HY000: Lost connection to MySQL server during query +include/rpl_reconnect.inc +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +4 4 +include/sync_slave_sql_with_master.inc +[connection slave] +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +4 4 +[connection master] +set session debug= "+d,myrocks_commit_merge_io_error"; +begin; +insert into t1 values (5,5); +insert into t1 values (6,6); +commit; +ERROR HY000: Lost connection to MySQL server during query +include/rpl_reconnect.inc +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +include/sync_slave_sql_with_master.inc +[connection slave] +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +[connection master] +drop table t1; +include/sync_slave_sql_with_master.inc +[connection master] +create table t1 (id int primary key, value int) engine = rocksdb; +insert into t1 values (1,1),(2,2),(3,3); +set session debug= "+d,myrocks_commit_merge_incomplete"; +insert into t1 values (4,4); +ERROR HY000: Lost connection to MySQL server during query +include/rpl_reconnect.inc +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +4 4 +include/sync_slave_sql_with_master.inc +[connection slave] +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +4 4 +[connection master] +set session debug= "+d,myrocks_commit_merge_incomplete"; +begin; +insert into t1 values (5,5); +insert into t1 values (6,6); +commit; +ERROR HY000: Lost connection to MySQL server during query +include/rpl_reconnect.inc +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +include/sync_slave_sql_with_master.inc +[connection slave] +select * from t1 order by id; +id value +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +[connection master] +drop table t1; +include/sync_slave_sql_with_master.inc +include/rpl_end.inc diff --git a/mysql-test/suite/rocksdb_rpl/t/tx_system_failure.test b/mysql-test/suite/rocksdb_rpl/t/tx_system_failure.test new file mode 100644 index 000000000000..bcadf521acc2 --- /dev/null +++ b/mysql-test/suite/rocksdb_rpl/t/tx_system_failure.test @@ -0,0 +1,37 @@ +--disable_warnings +--source include/have_rocksdb.inc +--source include/have_binlog_format_row.inc +--source include/not_asan.inc +--source include/have_debug.inc +--source include/not_valgrind.inc +--source include/master-slave.inc + + +call mtr.add_suppression("Attempting backtrace"); +call mtr.add_suppression("failed to write to WAL"); +call mtr.add_suppression("aborting on WAL write error"); + +# systems errors on prepare should just roll back +let $myrocks_debug_set_code = "+d,myrocks_prepare_io_error"; +let $myrocks_debug_unset_code = "-d,myrocks_prepare_io_error"; +--source tx_system_failure_prepare.inc + +let $myrocks_debug_set_code = "+d,myrocks_prepare_incomplete"; +let $myrocks_debug_unset_code = "-d,myrocks_prepare_incomplete"; +--source tx_system_failure_prepare.inc + +# systems errors on commit should abort, then roll forward, +# replication should consistent +let $myrocks_debug_set_code = "+d,myrocks_commit_io_error"; +--source tx_system_failure_commit.inc + +let $myrocks_debug_set_code = "+d,myrocks_commit_incomplete"; +--source tx_system_failure_commit.inc + +let $myrocks_debug_set_code = "+d,myrocks_commit_merge_io_error"; +--source tx_system_failure_commit.inc + +let $myrocks_debug_set_code = "+d,myrocks_commit_merge_incomplete"; +--source tx_system_failure_commit.inc + +--source include/rpl_end.inc diff --git a/mysql-test/suite/rocksdb_rpl/t/tx_system_failure_commit.inc b/mysql-test/suite/rocksdb_rpl/t/tx_system_failure_commit.inc new file mode 100644 index 000000000000..2ffc0879ae27 --- /dev/null +++ b/mysql-test/suite/rocksdb_rpl/t/tx_system_failure_commit.inc @@ -0,0 +1,53 @@ +--source include/rpl_connection_master.inc +create table t1 (id int primary key, value int) engine = rocksdb; + +insert into t1 values (1,1),(2,2),(3,3); + +# auto commit failure at engine commit, crash and restart (roll forward with binlog) +--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +eval set session debug= $myrocks_debug_set_code; +--error 2013 +insert into t1 values (4,4); + +--enable_reconnect +--source include/wait_until_connected_again.inc + +# Primary instance restart needs this, otherwise rpl_end fails. +--let $rpl_server_number = 1 +--source include/rpl_reconnect.inc + +select * from t1 order by id; +source include/sync_slave_sql_with_master.inc; + +--source include/rpl_connection_slave.inc +select * from t1 order by id; + + +--source include/rpl_connection_master.inc +# tx failure at engine commit, crash and restart (roll forward with binlog) +--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +eval set session debug= $myrocks_debug_set_code; +begin; +insert into t1 values (5,5); +insert into t1 values (6,6); +--error 2013 +commit; + +--enable_reconnect +--source include/wait_until_connected_again.inc + +# Primary instance restart needs this, otherwise rpl_end fails. +--let $rpl_server_number = 1 +--source include/rpl_reconnect.inc + +select * from t1 order by id; +source include/sync_slave_sql_with_master.inc; + +--source include/rpl_connection_slave.inc +select * from t1 order by id; + + +--source include/rpl_connection_master.inc +drop table t1; +source include/sync_slave_sql_with_master.inc; + diff --git a/mysql-test/suite/rocksdb_rpl/t/tx_system_failure_prepare.inc b/mysql-test/suite/rocksdb_rpl/t/tx_system_failure_prepare.inc new file mode 100644 index 000000000000..bb72dcb5f309 --- /dev/null +++ b/mysql-test/suite/rocksdb_rpl/t/tx_system_failure_prepare.inc @@ -0,0 +1,17 @@ +create table t1 (id int primary key, value int) engine = rocksdb; + +insert into t1 values (1,1),(2,2),(3,3); + +eval set session debug= $myrocks_debug_set_code; +--error ER_INTERNAL_ERROR +insert into t1 values (4,4); + +begin; +insert into t1 values (5,5); +insert into t1 values (6,6); +--error ER_INTERNAL_ERROR +commit; +eval set session debug= $myrocks_debug_unset_code; +select * from t1 order by id; + +drop table t1; diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index dfcf73768538..dcc775b8d0fc 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -3318,6 +3318,19 @@ class Rdb_transaction { } }; +#ifndef NDEBUG +// simulate that RocksDB has reported corrupted data +static void dbug_change_status_to_corrupted(rocksdb::Status *status) { + *status = rocksdb::Status::Corruption(); +} +static void dbug_change_status_to_io_error(rocksdb::Status *status) { + *status = rocksdb::Status::IOError(); +} +static void dbug_change_status_to_incomplete(rocksdb::Status *status) { + *status = rocksdb::Status::Incomplete(); +} +#endif + /* This is a rocksdb transaction. Its members represent the current transaction, which consists of: @@ -3372,14 +3385,23 @@ class Rdb_transaction_impl : public Rdb_transaction { rocksdb::Status s; s = merge_auto_incr_map(m_rocksdb_tx->GetWriteBatch()->GetWriteBatch()); +#ifndef NDEBUG + DBUG_EXECUTE_IF("myrocks_prepare_io_error", + dbug_change_status_to_io_error(&s);); + DBUG_EXECUTE_IF("myrocks_prepare_incomplete", + dbug_change_status_to_incomplete(&s);); +#endif if (!s.ok()) { - rdb_handle_io_error(s, RDB_IO_ERROR_TX_COMMIT); + std::string msg = + "RocksDB error on COMMIT (Prepare/merge): " + s.ToString(); + my_error(ER_INTERNAL_ERROR, MYF(0), msg.c_str()); return false; } s = m_rocksdb_tx->Prepare(); if (!s.ok()) { - rdb_handle_io_error(s, RDB_IO_ERROR_TX_COMMIT); + std::string msg = "RocksDB error on COMMIT (Prepare): " + s.ToString(); + my_error(ER_INTERNAL_ERROR, MYF(0), msg.c_str()); return false; } return true; @@ -3390,6 +3412,12 @@ class Rdb_transaction_impl : public Rdb_transaction { rocksdb::Status s; s = merge_auto_incr_map(m_rocksdb_tx->GetWriteBatch()->GetWriteBatch()); +#ifndef NDEBUG + DBUG_EXECUTE_IF("myrocks_commit_merge_io_error", + dbug_change_status_to_io_error(&s);); + DBUG_EXECUTE_IF("myrocks_commit_merge_incomplete", + dbug_change_status_to_incomplete(&s);); +#endif if (!s.ok()) { rdb_handle_io_error(s, RDB_IO_ERROR_TX_COMMIT); res = true; @@ -3398,6 +3426,12 @@ class Rdb_transaction_impl : public Rdb_transaction { release_snapshot(); s = m_rocksdb_tx->Commit(); +#ifndef NDEBUG + DBUG_EXECUTE_IF("myrocks_commit_io_error", + dbug_change_status_to_io_error(&s);); + DBUG_EXECUTE_IF("myrocks_commit_incomplete", + dbug_change_status_to_incomplete(&s);); +#endif if (!s.ok()) { rdb_handle_io_error(s, RDB_IO_ERROR_TX_COMMIT); res = true; @@ -5588,13 +5622,6 @@ static inline void rocksdb_smart_next(bool seek_backward, } } -#if !defined(DBUG_OFF) -// simulate that RocksDB has reported corrupted data -static void dbug_change_status_to_corrupted(rocksdb::Status *status) { - *status = rocksdb::Status::Corruption(); -} -#endif - // If the iterator is not valid it might be because of EOF but might be due // to IOError or corruption. The good practice is always check it. // https://github.com/facebook/rocksdb/wiki/Iterator#error-handling @@ -14592,8 +14619,9 @@ void rdb_handle_io_error(const rocksdb::Status status, abort(); } else if (!status.ok()) { switch (err_type) { + case RDB_IO_ERROR_TX_COMMIT: case RDB_IO_ERROR_DICT_COMMIT: { - rdb_log_status_error(status, "Failed to write to WAL (dictionary)"); + rdb_log_status_error(status, "Failed to write to WAL (non kIOError)"); /* NO_LINT_DEBUG */ sql_print_error("MyRocks: aborting on WAL write error."); abort(); From 2c30efad91abf6a58821b398d471f12bc354157d Mon Sep 17 00:00:00 2001 From: Luke Lu Date: Wed, 26 May 2021 12:18:01 -0700 Subject: [PATCH 12/72] Updating RocksDB submodule to 6.21.fb Upstream commit ID : fb-mysql-5.6.35/6e53ec0ef23f6a16f966669bd58b72a7a9ced698 PS-7798 : Merge fb-prod202104 Summary: pull in the latest rocksdb version for next full feature (prod202104) update-submodule: rocksdb Reviewed By: yoshinorim Differential Revision: D28689774 fbshipit-source-id: 455d6d93e54 --- storage/rocksdb/rocksdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/rocksdb/rocksdb b/storage/rocksdb/rocksdb index c56ad3c60a61..c7f8ae9f1774 160000 --- a/storage/rocksdb/rocksdb +++ b/storage/rocksdb/rocksdb @@ -1 +1 @@ -Subproject commit c56ad3c60a61bc39159c6fa1a112f6301cd86c89 +Subproject commit c7f8ae9f17743774ac48e2ffc6e150eafcfae416 From 5cc99aac8768d768dd830de78fb342151b26be2c Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:29 +0200 Subject: [PATCH 13/72] Support for dumping raft logs to vanilla async replicas Upstream commit ID : fb-mysql-5.6.35/2451c64901e59e10be92843da298733710cb2bd7 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 48f1492cb8874026d37cb40adafb2ca36d74c9e7 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:29 +0200 Subject: [PATCH 14/72] reinitializing previous_gtid_set_map in binlog_change_to_apply Upstream commit ID : fb-mysql-5.6.35/623780af3ee8295a98ce407ab46a48e865b9e95e PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 1206f7c723ff6e15668df0a52aa06de4d56b243d Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:29 +0200 Subject: [PATCH 15/72] Adding previous_raft_opid in previous HLC metadata event Upstream commit ID : fb-mysql-5.6.35/c607a5a9104ee50b87efd2326f741bded737f70e PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 1916f38116c4b01a28decfe7f0817a95173eb165 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:29 +0200 Subject: [PATCH 16/72] Ensuring proper column names to match expectations on Automation Upstream commit ID : fb-mysql-5.6.35/e40167380581697596b398e832d2d35d132f5bc6 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From fde8495df92e69e6d3b8b28b55e008410454d71c Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:29 +0200 Subject: [PATCH 17/72] Enable val_opid setting for prev_opid so that followers don't use 0,0 Upstream commit ID : fb-mysql-5.6.35/0b215f8c7eb777e6e4d2202b621c40519c03b0c3 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 9be3bd16e1de91326d3e986797363f39d0d01cfb Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:29 +0200 Subject: [PATCH 18/72] cur_log_ext should be incremented only on reaching consensus Upstream commit ID : fb-mysql-5.6.35/e4a49506500c0b8c46b2340b8c15afe3e64e8ed5 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 9ef5a929aede199921958126341990344acd4cf1 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:29 +0200 Subject: [PATCH 19/72] Properly adding raft term and index to apply side binlog metadata event Upstream commit ID : fb-mysql-5.6.35/6b1db2cfa9c1c8fb60e01d09c6b7c2d8cdc2dcae PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 3c71cd8fb6916fb90ffa4d610de027061ef64740 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:29 +0200 Subject: [PATCH 20/72] recover raft logs by removing partial trxs Upstream commit ID : fb-mysql-5.6.35/6ab1c7c1ac92693062ccfbcb1de1552cbf0cb162 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 643b2c0903825711a3751f05e74cf472408d9131 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:29 +0200 Subject: [PATCH 21/72] fix adjust_linfo_offsets() to take care of relay_log/mysql_bin_log Upstream commit ID : fb-mysql-5.6.35/d8d5fed47c5509ee5fab4eb596f3378e877f46e3 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From f33608b6c18e5e08ef0822d02ec057b3f9f876c9 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:29 +0200 Subject: [PATCH 22/72] ABS enhancements and fixes Upstream commit ID : fb-mysql-5.6.35/4cd2cff73fa4d821c1f7bd2ecfcd9f17f98e9fe1 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 3e025ce4eecc33db397f53be3568db7d01942735 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:29 +0200 Subject: [PATCH 23/72] Small fix to setting trans markers in new_file_impl() Upstream commit ID : fb-mysql-5.6.35/9486d32143a49416e399cfa1526d7f9a7c738e31 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 262e9bb6fbace2a5e0b28f557b2348d0e76b139f Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:29 +0200 Subject: [PATCH 24/72] Execute free floating metadata event in sync mode Upstream commit ID : fb-mysql-5.6.35/50a8e298885ba5ddf0415d11eee528cd31ccf90b PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From fad24148925b979aaa1c42358f8ffe423f62b8c1 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:29 +0200 Subject: [PATCH 25/72] error line logging only for server startup Upstream commit ID : fb-mysql-5.6.35/c018587221e551162ac587eab2fcbbd0960fe372 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 1f3566f1fb54276a6403c85a962a16f2485f8e57 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:29 +0200 Subject: [PATCH 26/72] Fix purged gtid calculation in dump threads Upstream commit ID : fb-mysql-5.6.35/77b9160bcaa58d5e2cb9ce262efc6064ca7642b3 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 195305e7b7902cb0e1e40315707a32cbe641a2b9 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:29 +0200 Subject: [PATCH 27/72] Increasing config change IO_CACHE size to follow actual config change Upstream commit ID : fb-mysql-5.6.35/9d01043b1df6de6537cf223a7b62987d8649de5f PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From e26f5ce81980c13d9f6c27083a80decbbb009350 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:29 +0200 Subject: [PATCH 28/72] Adding raft binlog event type and another Metadata event sub-type Upstream commit ID : fb-mysql-5.6.35/9fc96c4116396d458d2a818fe024c8edc2f2b5bb PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 2e115cf84a73c47e101bdaf2a87a6085c1a4fa6e Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:29 +0200 Subject: [PATCH 29/72] Preventing deadlock in mysql_binlog_send due to wrong lock order Upstream commit ID : fb-mysql-5.6.35/5c3580412734164d26340e6825aa2c018c27f23d PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 50529a4788d46251f04ea4c5cd0264003be00f76 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:29 +0200 Subject: [PATCH 30/72] clear raft tests Upstream commit ID : fb-mysql-5.6.35/896b8facb96ebbff7ca5b645c556d5d34453f397 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 1c808c0abb6119f0e454145221d82a2069e42644 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:29 +0200 Subject: [PATCH 31/72] fix memory corruptions reorted by valgrind Upstream commit ID : fb-mysql-5.6.35/54db17753f7dff688da017b015d962a678226231 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 6e47d4c396a726f352700af752332de2345f0df5 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:29 +0200 Subject: [PATCH 32/72] Fixing rpl_raft_dump_raft_logs and rpl_raft_leader_election_simple Upstream commit ID : fb-mysql-5.6.35/262b9bbb600e6711033b7f9fb6901f10ff906cb2 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 73e88a6c2ddac5630d684e67019b5be8cd04c6e3 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:29 +0200 Subject: [PATCH 33/72] Adding timestamps for raft rotates which happen in the context of listener thread. Upstream commit ID : fb-mysql-5.6.35/832493787ef0bc1d89df5ce0676de31cb48d87b6 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From ad811ec8e1226d0d6b12b5d2008df419ad88de7f Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:29 +0200 Subject: [PATCH 34/72] Checking for inited bool to make sure global_init_info was successful Upstream commit ID : fb-mysql-5.6.35/8585ff9b21d4f610dd8e9875b9610c57a2d83018 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 0623a3c6748c2a6b45e5c8ea2c3574ba8dda4906 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:29 +0200 Subject: [PATCH 35/72] Util method to get string representation of listener queue callbacks Upstream commit ID : fb-mysql-5.6.35/6e357dbd6912acfbd7d03db4a0bcfff4e85e1a9c PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 6e33dabec65b83ac43a25ce73e7f414b60122d70 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:30 +0200 Subject: [PATCH 36/72] Do not allow change master in raft mode Upstream commit ID : fb-mysql-5.6.35/ca88863463f9545116063177619bae3cb642939a PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From fa605138a2f7ff91a4f902c57195c2253c212e42 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:30 +0200 Subject: [PATCH 37/72] Changing the lock order of binlog change to apply and binlog change to binlog for dump log Upstream commit ID : fb-mysql-5.6.35/9eca3910e72d09022d4839a611d7a92c16a03c30 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 2bf0ecfe3f36057cbd46570b1253521dcc139fc9 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:30 +0200 Subject: [PATCH 38/72] Fixes around raft log truncation Upstream commit ID : fb-mysql-5.6.35/ca1ba80c94739645793e2a9b0d821f9819268ccc PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From d3626fb99c9dc97f8d611725a1a6c9a12602d905 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:30 +0200 Subject: [PATCH 39/72] handle partial rotate events correctly on rstart Upstream commit ID : fb-mysql-5.6.35/fe787140db7168223d3f283368a868eed7319db3 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 71d80135333a2433a5cc348bc236517dc725955b Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:30 +0200 Subject: [PATCH 40/72] Fixed truncation in rbr_column_type_mismatch_whitelist when log_column_names is enabled Upstream commit ID : fb-mysql-5.6.35/104e734cd0cb4fd38cb453f45e5371caf4017c26 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 8b8264e6f85ca73ccef5b44e661c674801872301 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:30 +0200 Subject: [PATCH 41/72] Support for code coverage Upstream commit ID : fb-mysql-5.6.35/9dbfb25924d11bff21eebf52bdb7d4da08ee4b44 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 8b771e23c7a8536c55caa50bf2b8c500dcddaf98 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:30 +0200 Subject: [PATCH 42/72] Making index statistics dynamically allocated Upstream commit ID : fb-mysql-5.6.35/ba59b57b7ee443d7c9c1b61e23234f89d5d1d884 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From b82fcf2c99bbab3e31bfacb37ae8f3263a7e36b4 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:30 +0200 Subject: [PATCH 43/72] Rebaseline mysqld--help-not-win-profiling Upstream commit ID : fb-mysql-5.6.35/9787032bb91e49666d8193dd1b26528000649c77 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 33616b9a401e5606304aae95fe88c96837aa9c55 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:30 +0200 Subject: [PATCH 44/72] Add coverage-src-filter option to code coverage Upstream commit ID : fb-mysql-5.6.35/a48bec5d59b380eb9243c251cf8f068a5466f1a6 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From d045af2940b0dd7732ceac7359ca5455e81ee2a2 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:30 +0200 Subject: [PATCH 45/72] Add exponential backoff for smart restart Upstream commit ID : fb-mysql-5.6.35/0b3d1c39dda5af059e512d04beb0cdd4add246a6 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 89410555270d55b974e2198b9d432bcd695734f7 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:30 +0200 Subject: [PATCH 46/72] ensure that promotion is a sync point on new leader Upstream commit ID : fb-mysql-5.6.35/39c70cafbf23feb3330cfa7c4210b5fd8f186ae5 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 867017711bf710f1244edcc0704641d98a3df6c0 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:30 +0200 Subject: [PATCH 47/72] Fixed flaky admission_control_yield test Upstream commit ID : fb-mysql-5.6.35/81ae98e6a8c934823c16936bcd5681db6b46cb17 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 1bcace4d4bfba138e9f2d29b966c1f33f3107eac Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:30 +0200 Subject: [PATCH 48/72] Add locking when reading index cardinality in InnoDB Upstream commit ID : fb-mysql-5.6.35/acd99073327aaa7c4121696d1996a135caf75555 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 81641a11669bdf9f89712a52f05889d1d8fe9b47 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:30 +0200 Subject: [PATCH 49/72] Support for "with gtid" clause in show raft logs command Upstream commit ID : fb-mysql-5.6.35/f9fd0ce3069b0a95bcdc3f6efc823b8fb56845a8 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 3a8763e62168c9564f481c9272aa76d4ecbc0578 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:30 +0200 Subject: [PATCH 50/72] Fix before image when binlog entries are written during idempotent recovery Upstream commit ID : fb-mysql-5.6.35/d4626033dd8a8e46bee8be0ec0c338994d1cea48 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 67e02c5ddc4f8e0ea8a4e9fa5cb9dba9244384a7 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:30 +0200 Subject: [PATCH 51/72] Changes required for MySQL privacy plugin Upstream commit ID : fb-mysql-5.6.35/2901d9a9463c89e6bd24e918fe02844850f8d981 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 6cd9aa4352f007d0b2a4bc6ae8afa14dd4ffb5a0 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:30 +0200 Subject: [PATCH 52/72] Fix oss build errors Upstream commit ID : fb-mysql-5.6.35/05007850d65b3a69cbaa3e9442aea33abe9250a8 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From e1505e9cb9b24267e97d4ee96ad3ae6cbd85b873 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:30 +0200 Subject: [PATCH 53/72] Add argument to KILL QUERY statement to specify the reason why kill happened Upstream commit ID : fb-mysql-5.6.35/354ea91e0d9ccf471ea08d8724eef7d44291db04 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 9b8a7ae7f14db7400e5dfd0d0b688aa00eb3f63d Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:30 +0200 Subject: [PATCH 54/72] Make MySQL error code consistent between 5.6.35 and 8.0.20 Upstream commit ID : fb-mysql-5.6.35/e19d09073012954be372d32ba93b38bd497f0685 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From ab9a874a07d20d3cd94ca7105da0fccaa163da0f Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:30 +0200 Subject: [PATCH 55/72] Removing the mutex from the signal side of ABS Upstream commit ID : fb-mysql-5.6.35/7cdcea348f9ff0c0d3c3b0e8729476229f090819 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From eca5a46fbd04fc9e6089eff0feae4b49afbb48b4 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:30 +0200 Subject: [PATCH 56/72] show_raft_status should take LOCK_status Upstream commit ID : fb-mysql-5.6.35/077b8d254efd92cc7c055c8b915ba6262e18cb6e PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 8b6e26b1babd404cbebfc015b1216c8df48dde3a Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:30 +0200 Subject: [PATCH 57/72] Adding a DISABLE SAFE MUTEX option to CMAKE like 8.0 Upstream commit ID : fb-mysql-5.6.35/2378c1e8c1e59d55c0659e97316cae5ac787e81b PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From b92de222e548416eb6aefc571888ee7b396c32a0 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:30 +0200 Subject: [PATCH 58/72] Fix a valgrind error with get_index_size_by_prefix(non-integer-field) Upstream commit ID : fb-mysql-5.6.35/ddda374d0a518c9e13e9fa4563a55ba67e321e24 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From a7cfa0a353005cedcd01c6c547aea5d6c1e92947 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:30 +0200 Subject: [PATCH 59/72] amend MySQL to support the PPF SQL cache Upstream commit ID : fb-mysql-5.6.35/01e535506ca5c4565a9f51cadec9b84d2a748e29 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From f7509d6572400bced4601d92e0fbd7685123bce8 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:30 +0200 Subject: [PATCH 60/72] Add error message for purpose policy check fail Upstream commit ID : fb-mysql-5.6.35/73b7ec9184c70acf7199b217778fffb23563e7b2 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From cfa5df0ab6de8a2776b0577d7cce573cfc2330ab Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:30 +0200 Subject: [PATCH 61/72] Add new THD function to populate query response attribute Upstream commit ID : fb-mysql-5.6.35/1b80de18f6b009aa75efcb128f26d3f89912f3dd PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From b6e93cafb37f980649d051c6c67ae7d26615b0bb Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:30 +0200 Subject: [PATCH 62/72] Adding --malloc-conf option in mysqld_safe Upstream commit ID : fb-mysql-5.6.35/05a8c91dcb787bc3a423765ba05c1b2b858d33de PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From ae786670ae1e908a774f448747a4025cafb82956 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:30 +0200 Subject: [PATCH 63/72] Adding jemalloc_tcache_bytes status variable Upstream commit ID : fb-mysql-5.6.35/bae825b5b8f332b16ba9e8065d43e1fa7a364867 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From dee06f13a6ee77d9233e8224fe9785f816219d1a Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:30 +0200 Subject: [PATCH 64/72] Fixed flaky adaptive deadlock test Upstream commit ID : fb-mysql-5.6.35/47fc3c7282e0726761c50431f9e046daa533e2aa PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 2889b484d04cf0c4a909b142da1990d9e918269a Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:30 +0200 Subject: [PATCH 65/72] Gracefully exit mysqld_safe loop during backoff Upstream commit ID : fb-mysql-5.6.35/39ce50d9bffe13c8e89c279ef0810e9ecb16f2dc PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 9aae327cd057c603dd79e79f5d2774334fc53d96 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:31 +0200 Subject: [PATCH 66/72] Safety checks in cannot_safely_rollback to prevent heap-use-after-free Upstream commit ID : fb-mysql-5.6.35/baaf5dbe0ab754d87a9721e3cb9ecbc876efe464 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 2189c0a46b0cb95b58c8d965690c0511fe7eded8 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:31 +0200 Subject: [PATCH 67/72] Install extra headers needed for plugins Upstream commit ID : fb-mysql-5.6.35/c807ef482fe4e998a804622165264eedf18d62d6 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 9321ca5b28a3b5992675b905e62117d94729f4ff Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:31 +0200 Subject: [PATCH 68/72] Propagate initial max_db_connections value Upstream commit ID : fb-mysql-5.6.35/0f1f275a0255b88b43b4e20b23b9498558fd9b60 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 274118a79153af6a99635ad104ad95106bb7ff22 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:31 +0200 Subject: [PATCH 69/72] properly set write throttling params when set via command line Upstream commit ID : fb-mysql-5.6.35/124deb25ff19049b287d824273ed1a072ca3fccc PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 54f39058dac34dd2b4c4d93a00600e5a7509cfc3 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:31 +0200 Subject: [PATCH 70/72] Fix deadlocks in STMT dep repl caused due to innodb gap locks and commit ordering Upstream commit ID : fb-mysql-5.6.35/ce8c4740a0c0dfed0d219b7f1d30d05da8ffb40c PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From 8d37b7359fd39bbc758374b59c0eb713fbd15460 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:31 +0200 Subject: [PATCH 71/72] Support original_caller in query/connection attributes Upstream commit ID : fb-mysql-5.6.35/f43c69d0e5b4bb9c655698933f14fc017306de53 PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server From b31ec68f49b89e3752c950230486bbd4fd360a80 Mon Sep 17 00:00:00 2001 From: Luis Donoso Date: Mon, 12 Jul 2021 15:48:31 +0200 Subject: [PATCH 72/72] Fix heap overflow in group_relay_log_name handling Upstream commit ID : fb-mysql-5.6.35/4edc56e8b7f473ea35bc6629335221c51a5aac2d PS-7798 : Merge fb-prod202104 This is a NULL cherry-pick to Percona Server to create the commit placeholder for the corresponding upstream commit. Reason: Patch not taken into Percona Server