Fix stress test failure due to write fault injections and disable wri…

…te fault injection (#11859)

Summary:
This PR contains two fixes:

1. disable write fault injection since it caused several other kinds of internal stress test failures. I'll try to fix those separately before enabling it again.
2. Fix segfault like
```
#5  0x000000000083dc43 in rocksdb::port::Mutex::Lock (this=0x30) at internal_repo_rocksdb/repo/port/port_posix.cc:80
80	internal_repo_rocksdb/repo/port/port_posix.cc: No such file or directory.
#6  0x0000000000465142 in rocksdb::MutexLock::MutexLock (mu=0x30, this=<optimized out>) at internal_repo_rocksdb/repo/util/mutexlock.h:37
37	internal_repo_rocksdb/repo/util/mutexlock.h: No such file or directory.
#7  rocksdb::FaultInjectionTestFS::DisableWriteErrorInjection (this=0x0) at internal_repo_rocksdb/repo/utilities/fault_injection_fs.h:505
505	internal_repo_rocksdb/repo/utilities/fault_injection_fs.h: No such file or directory.
```

Pull Request resolved: #11859

Test Plan: db_stress with no fault injection: `./db_stress --write_fault_one_in=0 --read_fault_one_in=0 --open_metadata_write_fault_one_in=0 --open_read_fault_one_in=0 --open_write_fault_one_in=0 --sync_fault_injection=0`

Reviewed By: jaykorean

Differential Revision: D49408247

Pulled By: cbi42

fbshipit-source-id: 0ca01f20e6e81bf52af77818b50d562ef7462165

db_stress_tool/db_stress_test_base.cc

@@ -2693,7 +2693,9 @@ void StressTest::Open(SharedState* shared, bool reopen) {
 
     // If this is for DB reopen, write error injection may have been enabled.
     // Disable it here in case there is no open fault injection.
-    fault_fs_guard->DisableWriteErrorInjection();
+    if (fault_fs_guard) {
+      fault_fs_guard->DisableWriteErrorInjection();
+    }
     if (!FLAGS_use_txn) {
       // Determine whether we need to inject file metadata write failures
       // during DB reopen. If it does, enable it.

tools/db_crashtest.py

@@ -179,7 +179,7 @@
     "max_key_len": 3,
     "key_len_percent_dist": "1,30,69",
     "read_fault_one_in": lambda: random.choice([0, 32, 1000]),
-    "write_fault_one_in": lambda: random.choice([0, 500]),
+    "write_fault_one_in": 0,
     "open_metadata_write_fault_one_in": lambda: random.choice([0, 0, 8]),
     "open_write_fault_one_in": lambda: random.choice([0, 0, 16]),
     "open_read_fault_one_in": lambda: random.choice([0, 0, 32]),
@@ -681,7 +681,7 @@ def finalize_and_sanitize(src_params):
         dest_params["verify_file_checksums_one_in"] = 0
     if dest_params["write_fault_one_in"] > 0:
         # background work may be disabled while DB is resuming after some error
-        dest_params["max_write_buffer_number"] = max(dest_params["max_write_buffer_number"], 6)
+        dest_params["max_write_buffer_number"] = max(dest_params["max_write_buffer_number"], 10)
     return dest_params