diff --git a/db/error_handler.cc b/db/error_handler.cc index 21c3a686f57..6797371b940 100644 --- a/db/error_handler.cc +++ b/db/error_handler.cc @@ -556,6 +556,7 @@ Status ErrorHandler::ClearBGError() { // old_bg_error is only for notifying listeners, so may not be checked old_bg_error.PermitUncheckedError(); // Clear and check the recovery IO and BG error + is_db_stopped_.store(false, std::memory_order_release); bg_error_ = Status::OK(); recovery_error_ = IOStatus::OK(); bg_error_.PermitUncheckedError(); @@ -671,11 +672,14 @@ const Status& ErrorHandler::StartRecoverFromRetryableBGIOError( // wait the previous recover thread to finish and create a new thread // to recover from the bg error. db_mutex_->Unlock(); + TEST_SYNC_POINT( + "StartRecoverFromRetryableBGIOError:BeforeWaitingForOtherThread"); old_recovery_thread->join(); + TEST_SYNC_POINT( + "StartRecoverFromRetryableBGIOError:AfterWaitingForOtherThread"); db_mutex_->Lock(); } - TEST_SYNC_POINT("StartRecoverFromRetryableBGIOError::in_progress"); recovery_thread_.reset( new port::Thread(&ErrorHandler::RecoverFromRetryableBGIOError, this)); @@ -689,6 +693,7 @@ const Status& ErrorHandler::StartRecoverFromRetryableBGIOError( // Automatic recover from Retryable BG IO error. Must be called after db // mutex is released. void ErrorHandler::RecoverFromRetryableBGIOError() { + assert(recovery_in_prog_); TEST_SYNC_POINT("RecoverFromRetryableBGIOError:BeforeStart"); TEST_SYNC_POINT("RecoverFromRetryableBGIOError:BeforeStart2"); InstrumentedMutexLock l(db_mutex_); @@ -754,22 +759,12 @@ void ErrorHandler::RecoverFromRetryableBGIOError() { // recover from the retryable IO error and no other BG errors. Clean // the bg_error and notify user. TEST_SYNC_POINT("RecoverFromRetryableBGIOError:RecoverSuccess"); - Status old_bg_error = bg_error_; - is_db_stopped_.store(false, std::memory_order_release); - bg_error_ = Status::OK(); - bg_error_.PermitUncheckedError(); - EventHelpers::NotifyOnErrorRecoveryEnd( - db_options_.listeners, old_bg_error, bg_error_, db_mutex_); if (bg_error_stats_ != nullptr) { RecordTick(bg_error_stats_.get(), ERROR_HANDLER_AUTORESUME_SUCCESS_COUNT); RecordInHistogram(bg_error_stats_.get(), ERROR_HANDLER_AUTORESUME_RETRY_COUNT, retry_count); } - recovery_in_prog_ = false; - if (soft_error_no_bg_work_) { - soft_error_no_bg_work_ = false; - } return; } else { // In this case: 1) recovery_error_ is more serious or not retryable diff --git a/db/error_handler_fs_test.cc b/db/error_handler_fs_test.cc index 56a0f57aa0e..56d2fcfc098 100644 --- a/db/error_handler_fs_test.cc +++ b/db/error_handler_fs_test.cc @@ -155,9 +155,9 @@ class ErrorHandlerFSListener : public EventListener { FaultInjectionTestFS* fault_fs_; }; -TEST_F(DBErrorHandlingFSTest, FLushWriteError) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); +TEST_F(DBErrorHandlingFSTest, FlushWriteError) { + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -200,9 +200,9 @@ TEST_F(DBErrorHandlingFSTest, FLushWriteError) { // All the NoSpace IOError will be handled as the regular BG Error no matter the // retryable flag is set of not. So the auto resume for retryable IO Error will // not be triggered. Also, it is mapped as hard error. -TEST_F(DBErrorHandlingFSTest, FLushWriteNoSpaceError) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); +TEST_F(DBErrorHandlingFSTest, FlushWriteNoSpaceError) { + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -244,9 +244,9 @@ TEST_F(DBErrorHandlingFSTest, FLushWriteNoSpaceError) { Destroy(options); } -TEST_F(DBErrorHandlingFSTest, FLushWriteRetryableError) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); +TEST_F(DBErrorHandlingFSTest, FlushWriteRetryableError) { + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -318,9 +318,9 @@ TEST_F(DBErrorHandlingFSTest, FLushWriteRetryableError) { Destroy(options); } -TEST_F(DBErrorHandlingFSTest, FLushWriteFileScopeError) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); +TEST_F(DBErrorHandlingFSTest, FlushWriteFileScopeError) { + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -402,9 +402,9 @@ TEST_F(DBErrorHandlingFSTest, FLushWriteFileScopeError) { Destroy(options); } -TEST_F(DBErrorHandlingFSTest, FLushWALWriteRetryableError) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); +TEST_F(DBErrorHandlingFSTest, FlushWALWriteRetryableError) { + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -446,9 +446,9 @@ TEST_F(DBErrorHandlingFSTest, FLushWALWriteRetryableError) { Destroy(options); } -TEST_F(DBErrorHandlingFSTest, FLushWALAtomicWriteRetryableError) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); +TEST_F(DBErrorHandlingFSTest, FlushWALAtomicWriteRetryableError) { + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -492,9 +492,9 @@ TEST_F(DBErrorHandlingFSTest, FLushWALAtomicWriteRetryableError) { } // The flush error is injected before we finish the table build -TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableError1) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); +TEST_F(DBErrorHandlingFSTest, FlushWritNoWALRetryableError1) { + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -548,9 +548,9 @@ TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableError1) { } // The retryable IO error is injected before we sync table -TEST_F(DBErrorHandlingFSTest, FLushWriteNoWALRetryableError2) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); +TEST_F(DBErrorHandlingFSTest, FlushWriteNoWALRetryableError2) { + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -592,9 +592,9 @@ TEST_F(DBErrorHandlingFSTest, FLushWriteNoWALRetryableError2) { } // The retryable IO error is injected before we close the table file -TEST_F(DBErrorHandlingFSTest, FLushWriteNoWALRetryableError3) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); +TEST_F(DBErrorHandlingFSTest, FlushWriteNoWALRetryableError3) { + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -636,8 +636,8 @@ TEST_F(DBErrorHandlingFSTest, FLushWriteNoWALRetryableError3) { } TEST_F(DBErrorHandlingFSTest, ManifestWriteError) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -677,8 +677,8 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteError) { } TEST_F(DBErrorHandlingFSTest, ManifestWriteRetryableError) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -720,8 +720,8 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteRetryableError) { } TEST_F(DBErrorHandlingFSTest, ManifestWriteFileScopeError) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -766,8 +766,8 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteFileScopeError) { } TEST_F(DBErrorHandlingFSTest, ManifestWriteNoWALRetryableError) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -811,8 +811,8 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteNoWALRetryableError) { } TEST_F(DBErrorHandlingFSTest, DoubleManifestWriteError) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -863,8 +863,8 @@ TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteError) { ROCKSDB_GTEST_SKIP("Test requires non-mock environment"); return; } - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -933,8 +933,8 @@ TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteError) { } TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteRetryableError) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -1005,8 +1005,8 @@ TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteRetryableError) { } TEST_F(DBErrorHandlingFSTest, CompactionWriteError) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -1047,8 +1047,8 @@ TEST_F(DBErrorHandlingFSTest, CompactionWriteError) { } TEST_F(DBErrorHandlingFSTest, DISABLED_CompactionWriteRetryableError) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -1094,8 +1094,8 @@ TEST_F(DBErrorHandlingFSTest, DISABLED_CompactionWriteRetryableError) { } TEST_F(DBErrorHandlingFSTest, DISABLED_CompactionWriteFileScopeError) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -1186,8 +1186,8 @@ TEST_F(DBErrorHandlingFSTest, AutoRecoverFlushError) { ROCKSDB_GTEST_SKIP("Test requires non-mock environment"); return; } - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -1231,8 +1231,8 @@ TEST_F(DBErrorHandlingFSTest, AutoRecoverFlushError) { } TEST_F(DBErrorHandlingFSTest, FailRecoverFlushError) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -1260,8 +1260,8 @@ TEST_F(DBErrorHandlingFSTest, WALWriteError) { ROCKSDB_GTEST_SKIP("Test requires non-mock environment"); return; } - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -1333,8 +1333,8 @@ TEST_F(DBErrorHandlingFSTest, WALWriteError) { } TEST_F(DBErrorHandlingFSTest, WALWriteRetryableError) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -1428,8 +1428,8 @@ TEST_F(DBErrorHandlingFSTest, MultiCFWALWriteError) { ROCKSDB_GTEST_SKIP("Test requires non-mock environment"); return; } - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -1760,10 +1760,10 @@ TEST_F(DBErrorHandlingFSTest, MultiDBVariousErrors) { // to soft error and trigger auto resume. During auto resume, SwitchMemtable // is disabled to avoid small SST tables. Write can still be applied before // the bg error is cleaned unless the memtable is full. -TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableErrorAutoRecover1) { +TEST_F(DBErrorHandlingFSTest, FlushWritNoWALRetryableErrorAutoRecover1) { // Activate the FS before the first resume - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -1784,7 +1784,7 @@ TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableErrorAutoRecover1) { ASSERT_OK(Put(Key(1), "val1", wo)); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( {{"RecoverFromRetryableBGIOError:LoopOut", - "FLushWritNoWALRetryableeErrorAutoRecover1:1"}}); + "FlushWritNoWALRetryableeErrorAutoRecover1:1"}}); SyncPoint::GetInstance()->SetCallBack( "BuildTable:BeforeFinishBuildTable", [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); @@ -1793,7 +1793,7 @@ TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableErrorAutoRecover1) { s = Flush(); ASSERT_EQ("val1", Get(Key(1))); ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); - TEST_SYNC_POINT("FLushWritNoWALRetryableeErrorAutoRecover1:1"); + TEST_SYNC_POINT("FlushWritNoWALRetryableeErrorAutoRecover1:1"); ASSERT_EQ("val1", Get(Key(1))); ASSERT_EQ("val1", Get(Key(1))); SyncPoint::GetInstance()->DisableProcessing(); @@ -1830,10 +1830,94 @@ TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableErrorAutoRecover1) { Destroy(options); } -TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableErrorAutoRecover2) { +TEST_F(DBErrorHandlingFSTest, MultipleRecoveryThreads) { + // This test creates a scenario where second write's recovery can get started + // while mutex is released for a short period during + // NotifyOnErrorRecoveryEnd() from the first write's recovery. This is to make + // sure RecoverFromRetryableBGIOError() from the second write's recovery + // thread does not start with recovery_in_prog_ = false; + + std::shared_ptr listener = + std::make_shared(); + Options options = GetDefaultOptions(); + options.env = fault_env_.get(); + options.create_if_missing = true; + options.listeners.emplace_back(listener); + options.max_bgerror_resume_count = 100; + options.bgerror_resume_retry_interval = 1000000; // 1 second + options.statistics = CreateDBStatistics(); + + listener->EnableAutoRecovery(false); + DestroyAndReopen(options); + + IOStatus error_msg = IOStatus::IOError("Retryable IO Error"); + error_msg.SetRetryable(true); + + WriteOptions wo = WriteOptions(); + wo.disableWAL = true; + fault_fs_->SetFilesystemActive(false, error_msg); + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"NotifyOnErrorRecoveryEnd:MutexUnlocked:1", + "MultipleRecoveryThreads:1"}, + {"MultipleRecoveryThreads:2", + "NotifyOnErrorRecoveryEnd:MutexUnlocked:2"}, + {"StartRecoverFromRetryableBGIOError:BeforeWaitingForOtherThread", + "MultipleRecoveryThreads:3"}, + {"RecoverFromRetryableBGIOError:RecoverSuccess", + "MultipleRecoveryThreads:4"}, + {"MultipleRecoveryThreads:4", + "StartRecoverFromRetryableBGIOError:AfterWaitingForOtherThread"}}); + SyncPoint::GetInstance()->EnableProcessing(); + + // First write with read fault injected and recovery will start + { + ASSERT_OK(Put(Key(1), "val1", wo)); + Status s = Flush(); + ASSERT_NOK(s); + } + // Remove read fault injection so that first recovery can go through + fault_fs_->SetFilesystemActive(true); + + // At this point, first recovery is now at NotifyOnErrorRecoveryEnd. Mutex is + // released. + TEST_SYNC_POINT("MultipleRecoveryThreads:1"); + + ROCKSDB_NAMESPACE::port::Thread second_write([&] { + // Second write with read fault injected + fault_fs_->SetFilesystemActive(false, error_msg); + ASSERT_OK(Put(Key(2), "val2", wo)); + Status s = Flush(); + ASSERT_NOK(s); + }); + // Second bg thread before waiting for the first thread's recovery thread + TEST_SYNC_POINT("MultipleRecoveryThreads:3"); + // First thread's recovery thread continues + TEST_SYNC_POINT("MultipleRecoveryThreads:2"); + // Wait for the first thread's recovery to finish + // (this sets recovery_in_prog_ = false) + // And second thread continues and starts recovery thread + TEST_SYNC_POINT("MultipleRecoveryThreads:4"); + second_write.join(); + // Remove error injection so that second thread recovery can go through + fault_fs_->SetFilesystemActive(true); + + // Set up sync point so that we can wait for the recovery thread to finish + ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( + {{"RecoverFromRetryableBGIOError:RecoverSuccess", + "MultipleRecoveryThreads:6"}}); + + // Wait for the second thread's recovery to be done + TEST_SYNC_POINT("MultipleRecoveryThreads:6"); + + SyncPoint::GetInstance()->DisableProcessing(); + SyncPoint::GetInstance()->ClearAllCallBacks(); + Destroy(options); +} + +TEST_F(DBErrorHandlingFSTest, FlushWritNoWALRetryableErrorAutoRecover2) { // Activate the FS before the first resume - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -1891,10 +1975,10 @@ TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableErrorAutoRecover2) { // Auto resume fromt the flush retryable IO error. Activate the FS before the // first resume. Resume is successful -TEST_F(DBErrorHandlingFSTest, FLushWritRetryableErrorAutoRecover1) { +TEST_F(DBErrorHandlingFSTest, FlushWritRetryableErrorAutoRecover1) { // Activate the FS before the first resume - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -1933,10 +2017,10 @@ TEST_F(DBErrorHandlingFSTest, FLushWritRetryableErrorAutoRecover1) { // Auto resume fromt the flush retryable IO error and set the retry limit count. // Never activate the FS and auto resume should fail at the end -TEST_F(DBErrorHandlingFSTest, FLushWritRetryableErrorAutoRecover2) { +TEST_F(DBErrorHandlingFSTest, FlushWritRetryableErrorAutoRecover2) { // Fail all the resume and let user to resume - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -1953,18 +2037,18 @@ TEST_F(DBErrorHandlingFSTest, FLushWritRetryableErrorAutoRecover2) { ASSERT_OK(Put(Key(1), "val1")); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency( - {{"FLushWritRetryableeErrorAutoRecover2:0", + {{"FlushWritRetryableeErrorAutoRecover2:0", "RecoverFromRetryableBGIOError:BeforeStart"}, {"RecoverFromRetryableBGIOError:LoopOut", - "FLushWritRetryableeErrorAutoRecover2:1"}}); + "FlushWritRetryableeErrorAutoRecover2:1"}}); SyncPoint::GetInstance()->SetCallBack( "BuildTable:BeforeFinishBuildTable", [&](void*) { fault_fs_->SetFilesystemActive(false, error_msg); }); SyncPoint::GetInstance()->EnableProcessing(); s = Flush(); ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError); - TEST_SYNC_POINT("FLushWritRetryableeErrorAutoRecover2:0"); - TEST_SYNC_POINT("FLushWritRetryableeErrorAutoRecover2:1"); + TEST_SYNC_POINT("FlushWritRetryableeErrorAutoRecover2:0"); + TEST_SYNC_POINT("FlushWritRetryableeErrorAutoRecover2:1"); fault_fs_->SetFilesystemActive(true); ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks(); SyncPoint::GetInstance()->DisableProcessing(); @@ -1986,8 +2070,8 @@ TEST_F(DBErrorHandlingFSTest, FLushWritRetryableErrorAutoRecover2) { // Fail the first resume and let the second resume be successful. TEST_F(DBErrorHandlingFSTest, ManifestWriteRetryableErrorAutoRecover) { // Fail the first resume and let the second resume be successful - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -2039,8 +2123,8 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteRetryableErrorAutoRecover) { TEST_F(DBErrorHandlingFSTest, ManifestWriteNoWALRetryableErrorAutoRecover) { // Fail the first resume and let the second resume be successful - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -2094,8 +2178,8 @@ TEST_F(DBErrorHandlingFSTest, ManifestWriteNoWALRetryableErrorAutoRecover) { TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteRetryableErrorAutoRecover) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -2184,8 +2268,8 @@ TEST_F(DBErrorHandlingFSTest, CompactionWriteRetryableErrorAutoRecover) { // compaction, the FS is set to active and compaction is successful, so // the test will hit the CompactionJob::FinishCompactionOutputFile1 sync // point. - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -2238,8 +2322,8 @@ TEST_F(DBErrorHandlingFSTest, CompactionWriteRetryableErrorAutoRecover) { } TEST_F(DBErrorHandlingFSTest, WALWriteRetryableErrorAutoRecover1) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -2341,8 +2425,8 @@ TEST_F(DBErrorHandlingFSTest, WALWriteRetryableErrorAutoRecover1) { TEST_F(DBErrorHandlingFSTest, WALWriteRetryableErrorAutoRecover2) { // Fail the first recover and try second time. - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -2442,10 +2526,10 @@ TEST_F(DBErrorHandlingFSTest, WALWriteRetryableErrorAutoRecover2) { // Fail auto resume from a flush retryable error and verify that // OnErrorRecoveryEnd listener callback is called -TEST_F(DBErrorHandlingFSTest, FLushWritRetryableErrorAbortRecovery) { +TEST_F(DBErrorHandlingFSTest, FlushWritRetryableErrorAbortRecovery) { // Activate the FS before the first resume - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -2736,9 +2820,9 @@ TEST_F(DBErrorHandlingFSTest, CompactionReadRetryableErrorAutoRecover) { class DBErrorHandlingFencingTest : public DBErrorHandlingFSTest, public testing::WithParamInterface {}; -TEST_P(DBErrorHandlingFencingTest, FLushWriteFenced) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); +TEST_P(DBErrorHandlingFencingTest, FlushWriteFenced) { + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -2765,8 +2849,8 @@ TEST_P(DBErrorHandlingFencingTest, FLushWriteFenced) { } TEST_P(DBErrorHandlingFencingTest, ManifestWriteFenced) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -2800,8 +2884,8 @@ TEST_P(DBErrorHandlingFencingTest, ManifestWriteFenced) { } TEST_P(DBErrorHandlingFencingTest, CompactionWriteFenced) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; @@ -2841,8 +2925,8 @@ TEST_P(DBErrorHandlingFencingTest, CompactionWriteFenced) { } TEST_P(DBErrorHandlingFencingTest, WALWriteFenced) { - std::shared_ptr listener( - new ErrorHandlerFSListener()); + std::shared_ptr listener = + std::make_shared(); Options options = GetDefaultOptions(); options.env = fault_env_.get(); options.create_if_missing = true; diff --git a/db/event_helpers.cc b/db/event_helpers.cc index 700c5f22c79..65f6a5a4861 100644 --- a/db/event_helpers.cc +++ b/db/event_helpers.cc @@ -230,6 +230,8 @@ void EventHelpers::NotifyOnErrorRecoveryEnd( db_mutex->AssertHeld(); // release lock while notifying events db_mutex->Unlock(); + TEST_SYNC_POINT("NotifyOnErrorRecoveryEnd:MutexUnlocked:1"); + TEST_SYNC_POINT("NotifyOnErrorRecoveryEnd:MutexUnlocked:2"); for (auto& listener : listeners) { BackgroundErrorRecoveryInfo info; info.old_bg_error = old_bg_error;