From ba5897ada899a3d72887ce764bee108076f694ff Mon Sep 17 00:00:00 2001 From: Changyu Bi Date: Tue, 19 Sep 2023 08:33:05 -0700 Subject: [PATCH] Fix stress test failure due to write fault injections and disable write fault injection (#11859) Summary: This PR contains two fixes: 1. disable write fault injection since it caused several other kinds of internal stress test failures. I'll try to fix those separately before enabling it again. 2. Fix segfault like ``` https://github.com/facebook/rocksdb/issues/5 0x000000000083dc43 in rocksdb::port::Mutex::Lock (this=0x30) at internal_repo_rocksdb/repo/port/port_posix.cc:80 80 internal_repo_rocksdb/repo/port/port_posix.cc: No such file or directory. https://github.com/facebook/rocksdb/issues/6 0x0000000000465142 in rocksdb::MutexLock::MutexLock (mu=0x30, this=) at internal_repo_rocksdb/repo/util/mutexlock.h:37 37 internal_repo_rocksdb/repo/util/mutexlock.h: No such file or directory. https://github.com/facebook/rocksdb/issues/7 rocksdb::FaultInjectionTestFS::DisableWriteErrorInjection (this=0x0) at internal_repo_rocksdb/repo/utilities/fault_injection_fs.h:505 505 internal_repo_rocksdb/repo/utilities/fault_injection_fs.h: No such file or directory. ``` Pull Request resolved: https://github.com/facebook/rocksdb/pull/11859 Test Plan: db_stress with no fault injection: `./db_stress --write_fault_one_in=0 --read_fault_one_in=0 --open_metadata_write_fault_one_in=0 --open_read_fault_one_in=0 --open_write_fault_one_in=0 --sync_fault_injection=0` Reviewed By: jaykorean Differential Revision: D49408247 Pulled By: cbi42 fbshipit-source-id: 0ca01f20e6e81bf52af77818b50d562ef7462165 --- db_stress_tool/db_stress_test_base.cc | 4 +++- tools/db_crashtest.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/db_stress_tool/db_stress_test_base.cc b/db_stress_tool/db_stress_test_base.cc index daaa66f92cb..c79c0e237a1 100644 --- a/db_stress_tool/db_stress_test_base.cc +++ b/db_stress_tool/db_stress_test_base.cc @@ -2693,7 +2693,9 @@ void StressTest::Open(SharedState* shared, bool reopen) { // If this is for DB reopen, write error injection may have been enabled. // Disable it here in case there is no open fault injection. - fault_fs_guard->DisableWriteErrorInjection(); + if (fault_fs_guard) { + fault_fs_guard->DisableWriteErrorInjection(); + } if (!FLAGS_use_txn) { // Determine whether we need to inject file metadata write failures // during DB reopen. If it does, enable it. diff --git a/tools/db_crashtest.py b/tools/db_crashtest.py index 09bb11d1a00..49e9094f116 100644 --- a/tools/db_crashtest.py +++ b/tools/db_crashtest.py @@ -179,7 +179,7 @@ "max_key_len": 3, "key_len_percent_dist": "1,30,69", "read_fault_one_in": lambda: random.choice([0, 32, 1000]), - "write_fault_one_in": lambda: random.choice([0, 500]), + "write_fault_one_in": 0, "open_metadata_write_fault_one_in": lambda: random.choice([0, 0, 8]), "open_write_fault_one_in": lambda: random.choice([0, 0, 16]), "open_read_fault_one_in": lambda: random.choice([0, 0, 32]), @@ -681,7 +681,7 @@ def finalize_and_sanitize(src_params): dest_params["verify_file_checksums_one_in"] = 0 if dest_params["write_fault_one_in"] > 0: # background work may be disabled while DB is resuming after some error - dest_params["max_write_buffer_number"] = max(dest_params["max_write_buffer_number"], 6) + dest_params["max_write_buffer_number"] = max(dest_params["max_write_buffer_number"], 10) return dest_params