// Copyright (c) 2018-present, Facebook, Inc. All rights reserved. // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). // #include "db/error_handler.h" #include "db/db_impl/db_impl.h" #include "db/event_helpers.h" #include "file/sst_file_manager_impl.h" #include "logging/logging.h" #include "port/lang.h" namespace ROCKSDB_NAMESPACE { // Maps to help decide the severity of an error based on the // BackgroundErrorReason, Code, SubCode and whether db_options.paranoid_checks // is set or not. There are 3 maps, going from most specific to least specific // (i.e from all 4 fields in a tuple to only the BackgroundErrorReason and // paranoid_checks). The less specific map serves as a catch all in case we miss // a specific error code or subcode. std::map, Status::Severity> ErrorSeverityMap = { // Errors during BG compaction {std::make_tuple(BackgroundErrorReason::kCompaction, Status::Code::kIOError, Status::SubCode::kNoSpace, true), Status::Severity::kSoftError}, {std::make_tuple(BackgroundErrorReason::kCompaction, Status::Code::kIOError, Status::SubCode::kNoSpace, false), Status::Severity::kNoError}, {std::make_tuple(BackgroundErrorReason::kCompaction, Status::Code::kIOError, Status::SubCode::kSpaceLimit, true), Status::Severity::kHardError}, {std::make_tuple(BackgroundErrorReason::kCompaction, Status::Code::kIOError, Status::SubCode::kIOFenced, true), Status::Severity::kFatalError}, {std::make_tuple(BackgroundErrorReason::kCompaction, Status::Code::kIOError, Status::SubCode::kIOFenced, false), Status::Severity::kFatalError}, // Errors during BG flush {std::make_tuple(BackgroundErrorReason::kFlush, Status::Code::kIOError, Status::SubCode::kNoSpace, true), Status::Severity::kHardError}, {std::make_tuple(BackgroundErrorReason::kFlush, Status::Code::kIOError, Status::SubCode::kNoSpace, false), Status::Severity::kNoError}, {std::make_tuple(BackgroundErrorReason::kFlush, Status::Code::kIOError, Status::SubCode::kSpaceLimit, true), Status::Severity::kHardError}, {std::make_tuple(BackgroundErrorReason::kFlush, Status::Code::kIOError, Status::SubCode::kIOFenced, true), Status::Severity::kFatalError}, {std::make_tuple(BackgroundErrorReason::kFlush, Status::Code::kIOError, Status::SubCode::kIOFenced, false), Status::Severity::kFatalError}, // Errors during Write {std::make_tuple(BackgroundErrorReason::kWriteCallback, Status::Code::kIOError, Status::SubCode::kNoSpace, true), Status::Severity::kHardError}, {std::make_tuple(BackgroundErrorReason::kWriteCallback, Status::Code::kIOError, Status::SubCode::kNoSpace, false), Status::Severity::kHardError}, {std::make_tuple(BackgroundErrorReason::kWriteCallback, Status::Code::kIOError, Status::SubCode::kIOFenced, true), Status::Severity::kFatalError}, {std::make_tuple(BackgroundErrorReason::kWriteCallback, Status::Code::kIOError, Status::SubCode::kIOFenced, false), Status::Severity::kFatalError}, // Errors during MANIFEST write {std::make_tuple(BackgroundErrorReason::kManifestWrite, Status::Code::kIOError, Status::SubCode::kNoSpace, true), Status::Severity::kHardError}, {std::make_tuple(BackgroundErrorReason::kManifestWrite, Status::Code::kIOError, Status::SubCode::kNoSpace, false), Status::Severity::kHardError}, {std::make_tuple(BackgroundErrorReason::kManifestWrite, Status::Code::kIOError, Status::SubCode::kIOFenced, true), Status::Severity::kFatalError}, {std::make_tuple(BackgroundErrorReason::kManifestWrite, Status::Code::kIOError, Status::SubCode::kIOFenced, false), Status::Severity::kFatalError}, // Errors during BG flush with WAL disabled {std::make_tuple(BackgroundErrorReason::kFlushNoWAL, Status::Code::kIOError, Status::SubCode::kNoSpace, true), Status::Severity::kHardError}, {std::make_tuple(BackgroundErrorReason::kFlushNoWAL, Status::Code::kIOError, Status::SubCode::kNoSpace, false), Status::Severity::kNoError}, {std::make_tuple(BackgroundErrorReason::kFlushNoWAL, Status::Code::kIOError, Status::SubCode::kSpaceLimit, true), Status::Severity::kHardError}, {std::make_tuple(BackgroundErrorReason::kFlushNoWAL, Status::Code::kIOError, Status::SubCode::kIOFenced, true), Status::Severity::kFatalError}, {std::make_tuple(BackgroundErrorReason::kFlushNoWAL, Status::Code::kIOError, Status::SubCode::kIOFenced, false), Status::Severity::kFatalError}, // Errors during MANIFEST write when WAL is disabled {std::make_tuple(BackgroundErrorReason::kManifestWriteNoWAL, Status::Code::kIOError, Status::SubCode::kNoSpace, true), Status::Severity::kHardError}, {std::make_tuple(BackgroundErrorReason::kManifestWriteNoWAL, Status::Code::kIOError, Status::SubCode::kNoSpace, false), Status::Severity::kHardError}, {std::make_tuple(BackgroundErrorReason::kManifestWriteNoWAL, Status::Code::kIOError, Status::SubCode::kIOFenced, true), Status::Severity::kFatalError}, {std::make_tuple(BackgroundErrorReason::kManifestWriteNoWAL, Status::Code::kIOError, Status::SubCode::kIOFenced, false), Status::Severity::kFatalError}, }; std::map, Status::Severity> DefaultErrorSeverityMap = { // Errors during BG compaction {std::make_tuple(BackgroundErrorReason::kCompaction, Status::Code::kCorruption, true), Status::Severity::kUnrecoverableError}, {std::make_tuple(BackgroundErrorReason::kCompaction, Status::Code::kCorruption, false), Status::Severity::kNoError}, {std::make_tuple(BackgroundErrorReason::kCompaction, Status::Code::kIOError, true), Status::Severity::kFatalError}, {std::make_tuple(BackgroundErrorReason::kCompaction, Status::Code::kIOError, false), Status::Severity::kNoError}, // Errors during BG flush {std::make_tuple(BackgroundErrorReason::kFlush, Status::Code::kCorruption, true), Status::Severity::kUnrecoverableError}, {std::make_tuple(BackgroundErrorReason::kFlush, Status::Code::kCorruption, false), Status::Severity::kNoError}, {std::make_tuple(BackgroundErrorReason::kFlush, Status::Code::kIOError, true), Status::Severity::kFatalError}, {std::make_tuple(BackgroundErrorReason::kFlush, Status::Code::kIOError, false), Status::Severity::kNoError}, // Errors during Write {std::make_tuple(BackgroundErrorReason::kWriteCallback, Status::Code::kCorruption, true), Status::Severity::kUnrecoverableError}, {std::make_tuple(BackgroundErrorReason::kWriteCallback, Status::Code::kCorruption, false), Status::Severity::kNoError}, {std::make_tuple(BackgroundErrorReason::kWriteCallback, Status::Code::kIOError, true), Status::Severity::kFatalError}, {std::make_tuple(BackgroundErrorReason::kWriteCallback, Status::Code::kIOError, false), Status::Severity::kNoError}, {std::make_tuple(BackgroundErrorReason::kManifestWrite, Status::Code::kIOError, true), Status::Severity::kFatalError}, {std::make_tuple(BackgroundErrorReason::kManifestWrite, Status::Code::kIOError, false), Status::Severity::kFatalError}, // Errors during BG flush with WAL disabled {std::make_tuple(BackgroundErrorReason::kFlushNoWAL, Status::Code::kCorruption, true), Status::Severity::kUnrecoverableError}, {std::make_tuple(BackgroundErrorReason::kFlushNoWAL, Status::Code::kCorruption, false), Status::Severity::kNoError}, {std::make_tuple(BackgroundErrorReason::kFlushNoWAL, Status::Code::kIOError, true), Status::Severity::kFatalError}, {std::make_tuple(BackgroundErrorReason::kFlushNoWAL, Status::Code::kIOError, false), Status::Severity::kNoError}, {std::make_tuple(BackgroundErrorReason::kManifestWriteNoWAL, Status::Code::kIOError, true), Status::Severity::kFatalError}, {std::make_tuple(BackgroundErrorReason::kManifestWriteNoWAL, Status::Code::kIOError, false), Status::Severity::kFatalError}, }; std::map, Status::Severity> DefaultReasonMap = { // Errors during BG compaction {std::make_tuple(BackgroundErrorReason::kCompaction, true), Status::Severity::kFatalError}, {std::make_tuple(BackgroundErrorReason::kCompaction, false), Status::Severity::kNoError}, // Errors during BG flush {std::make_tuple(BackgroundErrorReason::kFlush, true), Status::Severity::kFatalError}, {std::make_tuple(BackgroundErrorReason::kFlush, false), Status::Severity::kNoError}, // Errors during Write {std::make_tuple(BackgroundErrorReason::kWriteCallback, true), Status::Severity::kFatalError}, {std::make_tuple(BackgroundErrorReason::kWriteCallback, false), Status::Severity::kFatalError}, // Errors during Memtable update {std::make_tuple(BackgroundErrorReason::kMemTable, true), Status::Severity::kFatalError}, {std::make_tuple(BackgroundErrorReason::kMemTable, false), Status::Severity::kFatalError}, }; void ErrorHandler::CancelErrorRecovery() { db_mutex_->AssertHeld(); // We'll release the lock before calling sfm, so make sure no new // recovery gets scheduled at that point auto_recovery_ = false; SstFileManagerImpl* sfm = static_cast(db_options_.sst_file_manager.get()); if (sfm) { // This may or may not cancel a pending recovery db_mutex_->Unlock(); bool cancelled = sfm->CancelErrorRecovery(this); db_mutex_->Lock(); if (cancelled) { recovery_in_prog_ = false; } } // If auto recovery is also runing to resume from the retryable error, // we should wait and end the auto recovery. EndAutoRecovery(); } STATIC_AVOID_DESTRUCTION(const Status, kOkStatus){Status::OK()}; // This is the main function for looking at an error during a background // operation and deciding the severity, and error recovery strategy. The high // level algorithm is as follows - // 1. Classify the severity of the error based on the ErrorSeverityMap, // DefaultErrorSeverityMap and DefaultReasonMap defined earlier // 2. Call a Status code specific override function to adjust the severity // if needed. The reason for this is our ability to recover may depend on // the exact options enabled in DBOptions // 3. Determine if auto recovery is possible. A listener notification callback // is called, which can disable the auto recovery even if we decide its // feasible // 4. For Status::NoSpace() errors, rely on SstFileManagerImpl to control // the actual recovery. If no sst file manager is specified in DBOptions, // a default one is allocated during DB::Open(), so there will always be // one. // This can also get called as part of a recovery operation. In that case, we // also track the error separately in recovery_error_ so we can tell in the // end whether recovery succeeded or not const Status& ErrorHandler::HandleKnownErrors(const Status& bg_err, BackgroundErrorReason reason) { db_mutex_->AssertHeld(); if (bg_err.ok()) { return kOkStatus; } ROCKS_LOG_INFO(db_options_.info_log, "ErrorHandler: Set regular background error\n"); bool paranoid = db_options_.paranoid_checks; Status::Severity sev = Status::Severity::kFatalError; Status new_bg_err; DBRecoverContext context; bool found = false; { auto entry = ErrorSeverityMap.find( std::make_tuple(reason, bg_err.code(), bg_err.subcode(), paranoid)); if (entry != ErrorSeverityMap.end()) { sev = entry->second; found = true; } } if (!found) { auto entry = DefaultErrorSeverityMap.find( std::make_tuple(reason, bg_err.code(), paranoid)); if (entry != DefaultErrorSeverityMap.end()) { sev = entry->second; found = true; } } if (!found) { auto entry = DefaultReasonMap.find(std::make_tuple(reason, paranoid)); if (entry != DefaultReasonMap.end()) { sev = entry->second; } } new_bg_err = Status(bg_err, sev); // Check if recovery is currently in progress. If it is, we will save this // error so we can check it at the end to see if recovery succeeded or not if (recovery_in_prog_ && recovery_error_.ok()) { recovery_error_ = status_to_io_status(Status(new_bg_err)); } bool auto_recovery = auto_recovery_; if (new_bg_err.severity() >= Status::Severity::kFatalError && auto_recovery) { auto_recovery = false; } // Allow some error specific overrides if (new_bg_err.subcode() == IOStatus::SubCode::kNoSpace || new_bg_err.subcode() == IOStatus::SubCode::kSpaceLimit) { new_bg_err = OverrideNoSpaceError(new_bg_err, &auto_recovery); } if (!new_bg_err.ok()) { Status s = new_bg_err; EventHelpers::NotifyOnBackgroundError(db_options_.listeners, reason, &s, db_mutex_, &auto_recovery); if (!s.ok() && (s.severity() > bg_error_.severity())) { bg_error_ = s; } else { // This error is less severe than previously encountered error. Don't // take any further action return bg_error_; } } recover_context_ = context; if (auto_recovery) { recovery_in_prog_ = true; // Kick-off error specific recovery if (new_bg_err.subcode() == IOStatus::SubCode::kNoSpace || new_bg_err.subcode() == IOStatus::SubCode::kSpaceLimit) { RecoverFromNoSpace(); } } if (bg_error_.severity() >= Status::Severity::kHardError) { is_db_stopped_.store(true, std::memory_order_release); } return bg_error_; } // This is the main function for looking at IO related error during the // background operations. The main logic is: // File scope IO error is treated as retryable IO error in the write path. In // RocksDB, If a file has write IO error and it is at file scope, RocksDB never // write to the same file again. RocksDB will create a new file and rewrite the // whole content. Thus, it is retryable. // There are three main categories of error handling: // 1) if the error is caused by data loss, the error is mapped to // unrecoverable error. Application/user must take action to handle // this situation (File scope case is excluded). // 2) if the error is a Retryable IO error (i.e., it is a file scope IO error, // or its retryable flag is set and not a data loss error), auto resume ( // DBImpl::ResumeImpl) may be called and the auto resume can be controlled // by resume count and resume interval options. There are three sub-cases: // a) if the error happens during compaction, it is mapped to a soft error. // the compaction thread will reschedule a new compaction. This doesn't // call auto resume. // b) if the error happens during flush and also WAL is empty, it is mapped // to a soft error. Note that, it includes the case that IO error happens // in SST or manifest write during flush. Auto resume will be called. // c) all other errors are mapped to hard error. Auto resume will be called. // 3) for other cases, HandleKnownErrors(const Status& bg_err, // BackgroundErrorReason reason) will be called to handle other error cases // such as delegating to SstFileManager to handle no space error. const Status& ErrorHandler::SetBGError(const Status& bg_status, BackgroundErrorReason reason) { db_mutex_->AssertHeld(); Status tmp_status = bg_status; IOStatus bg_io_err = status_to_io_status(std::move(tmp_status)); if (bg_io_err.ok()) { return kOkStatus; } ROCKS_LOG_WARN(db_options_.info_log, "Background IO error %s", bg_io_err.ToString().c_str()); RecordStats({ERROR_HANDLER_BG_ERROR_COUNT, ERROR_HANDLER_BG_IO_ERROR_COUNT}, {} /* int_histograms */); Status new_bg_io_err = bg_io_err; DBRecoverContext context; if (bg_io_err.GetScope() != IOStatus::IOErrorScope::kIOErrorScopeFile && bg_io_err.GetDataLoss()) { // First, data loss (non file scope) is treated as unrecoverable error. So // it can directly overwrite any existing bg_error_. bool auto_recovery = false; Status bg_err(new_bg_io_err, Status::Severity::kUnrecoverableError); CheckAndSetRecoveryAndBGError(bg_err); ROCKS_LOG_INFO( db_options_.info_log, "ErrorHandler: Set background IO error as unrecoverable error\n"); EventHelpers::NotifyOnBackgroundError(db_options_.listeners, reason, &bg_err, db_mutex_, &auto_recovery); recover_context_ = context; return bg_error_; } else if (bg_io_err.subcode() != IOStatus::SubCode::kNoSpace && (bg_io_err.GetScope() == IOStatus::IOErrorScope::kIOErrorScopeFile || bg_io_err.GetRetryable())) { // Second, check if the error is a retryable IO error (file scope IO error // is also treated as retryable IO error in RocksDB write path). if it is // retryable error and its severity is higher than bg_error_, overwrite the // bg_error_ with new error. In current stage, for retryable IO error of // compaction, treat it as soft error. In other cases, treat the retryable // IO error as hard error. Note that, all the NoSpace error should be // handled by the SstFileManager::StartErrorRecovery(). Therefore, no matter // it is retryable or file scope, this logic will be bypassed. bool auto_recovery = false; EventHelpers::NotifyOnBackgroundError(db_options_.listeners, reason, &new_bg_io_err, db_mutex_, &auto_recovery); RecordStats({ERROR_HANDLER_BG_RETRYABLE_IO_ERROR_COUNT}, {} /* int_histograms */); ROCKS_LOG_INFO(db_options_.info_log, "ErrorHandler: Set background retryable IO error\n"); if (BackgroundErrorReason::kCompaction == reason) { // We map the retryable IO error during compaction to soft error. Since // compaction can reschedule by itself. We will not set the BG error in // this case // TODO: a better way to set or clean the retryable IO error which // happens during compaction SST file write. RecordStats({ERROR_HANDLER_AUTORESUME_COUNT}, {} /* int_histograms */); ROCKS_LOG_INFO( db_options_.info_log, "ErrorHandler: Compaction will schedule by itself to resume\n"); // Not used in this code path. new_bg_io_err.PermitUncheckedError(); return bg_error_; } Status::Severity severity; if (BackgroundErrorReason::kFlushNoWAL == reason || BackgroundErrorReason::kManifestWriteNoWAL == reason) { // When the BG Retryable IO error reason is flush without WAL, // We map it to a soft error. At the same time, all the background work // should be stopped except the BG work from recovery. Therefore, we // set the soft_error_no_bg_work_ to true. At the same time, since DB // continues to receive writes when BG error is soft error, to avoid // to many small memtable being generated during auto resume, the flush // reason is set to kErrorRecoveryRetryFlush. severity = Status::Severity::kSoftError; soft_error_no_bg_work_ = true; context.flush_reason = FlushReason::kErrorRecoveryRetryFlush; } else { severity = Status::Severity::kHardError; } Status bg_err(new_bg_io_err, severity); CheckAndSetRecoveryAndBGError(bg_err); recover_context_ = context; return StartRecoverFromRetryableBGIOError(bg_io_err); } else { return HandleKnownErrors(new_bg_io_err, reason); } } void ErrorHandler::AddFilesToQuarantine( autovector*> files_to_quarantine) { db_mutex_->AssertHeld(); std::ostringstream quarantine_files_oss; bool is_first_one = true; for (const auto* files : files_to_quarantine) { assert(files); for (uint64_t file_number : *files) { files_to_quarantine_.push_back(file_number); quarantine_files_oss << (is_first_one ? "" : ", ") << file_number; is_first_one = false; } } ROCKS_LOG_INFO(db_options_.info_log, "ErrorHandler: added file numbers %s to quarantine.\n", quarantine_files_oss.str().c_str()); } void ErrorHandler::ClearFilesToQuarantine() { db_mutex_->AssertHeld(); files_to_quarantine_.clear(); ROCKS_LOG_INFO(db_options_.info_log, "ErrorHandler: cleared files in quarantine.\n"); } Status ErrorHandler::OverrideNoSpaceError(const Status& bg_error, bool* auto_recovery) { if (bg_error.severity() >= Status::Severity::kFatalError) { return bg_error; } if (db_options_.sst_file_manager.get() == nullptr) { // We rely on SFM to poll for enough disk space and recover *auto_recovery = false; return bg_error; } if (db_options_.allow_2pc && (bg_error.severity() <= Status::Severity::kSoftError)) { // Don't know how to recover, as the contents of the current WAL file may // be inconsistent, and it may be needed for 2PC. If 2PC is not enabled, // we can just flush the memtable and discard the log *auto_recovery = false; return Status(bg_error, Status::Severity::kFatalError); } { uint64_t free_space; if (db_options_.env->GetFreeSpace(db_options_.db_paths[0].path, &free_space) == Status::NotSupported()) { *auto_recovery = false; } } return bg_error; } void ErrorHandler::RecoverFromNoSpace() { SstFileManagerImpl* sfm = static_cast(db_options_.sst_file_manager.get()); // Inform SFM of the error, so it can kick-off the recovery if (sfm) { sfm->StartErrorRecovery(this, bg_error_); } } Status ErrorHandler::ClearBGError() { db_mutex_->AssertHeld(); // Signal that recovery succeeded if (recovery_error_.ok()) { assert(files_to_quarantine_.empty()); Status old_bg_error = bg_error_; // old_bg_error is only for notifying listeners, so may not be checked old_bg_error.PermitUncheckedError(); // Clear and check the recovery IO and BG error is_db_stopped_.store(false, std::memory_order_release); bg_error_ = Status::OK(); recovery_error_ = IOStatus::OK(); bg_error_.PermitUncheckedError(); recovery_error_.PermitUncheckedError(); recovery_in_prog_ = false; soft_error_no_bg_work_ = false; EventHelpers::NotifyOnErrorRecoveryEnd(db_options_.listeners, old_bg_error, bg_error_, db_mutex_); } return recovery_error_; } Status ErrorHandler::RecoverFromBGError(bool is_manual) { InstrumentedMutexLock l(db_mutex_); bool no_bg_work_original_flag = soft_error_no_bg_work_; if (is_manual) { // If its a manual recovery and there's a background recovery in progress // return busy status if (recovery_in_prog_) { return Status::Busy("Recovery already in progress"); } recovery_in_prog_ = true; // In manual resume, we allow the bg work to run. If it is a auto resume, // the bg work should follow this tag. soft_error_no_bg_work_ = false; // In manual resume, if the bg error is a soft error and also requires // no bg work, the error must be recovered by call the flush with // flush reason: kErrorRecoveryRetryFlush. In other case, the flush // reason is set to kErrorRecovery. if (no_bg_work_original_flag) { recover_context_.flush_reason = FlushReason::kErrorRecoveryRetryFlush; } else { recover_context_.flush_reason = FlushReason::kErrorRecovery; } } if (bg_error_.severity() == Status::Severity::kSoftError && recover_context_.flush_reason == FlushReason::kErrorRecovery) { // Simply clear the background error and return recovery_error_ = IOStatus::OK(); return ClearBGError(); } // Reset recovery_error_. We will use this to record any errors that happen // during the recovery process. While recovering, the only operations that // can generate background errors should be the flush operations recovery_error_ = IOStatus::OK(); recovery_error_.PermitUncheckedError(); Status s = db_->ResumeImpl(recover_context_); if (s.ok()) { soft_error_no_bg_work_ = false; } else { soft_error_no_bg_work_ = no_bg_work_original_flag; } // For manual recover, shutdown, and fatal error cases, set // recovery_in_prog_ to false. For automatic background recovery, leave it // as is regardless of success or failure as it will be retried if (is_manual || s.IsShutdownInProgress() || bg_error_.severity() >= Status::Severity::kFatalError) { recovery_in_prog_ = false; } return s; } const Status& ErrorHandler::StartRecoverFromRetryableBGIOError( const IOStatus& io_error) { db_mutex_->AssertHeld(); if (bg_error_.ok()) { return bg_error_; } else if (io_error.ok()) { return kOkStatus; } else if (db_options_.max_bgerror_resume_count <= 0 || recovery_in_prog_) { // Auto resume BG error is not enabled, directly return bg_error_. return bg_error_; } else if (end_recovery_) { // Can temporarily release db mutex EventHelpers::NotifyOnErrorRecoveryEnd(db_options_.listeners, bg_error_, Status::ShutdownInProgress(), db_mutex_); db_mutex_->AssertHeld(); return bg_error_; } RecordStats({ERROR_HANDLER_AUTORESUME_COUNT}, {} /* int_histograms */); ROCKS_LOG_INFO( db_options_.info_log, "ErrorHandler: Call StartRecoverFromRetryableBGIOError to resume\n"); // Needs to be set in the same lock hold as setting BG error, otherwise // intervening writes could see a BG error without a recovery and bail out. recovery_in_prog_ = true; if (recovery_thread_) { // Ensure only one thread can execute the join(). std::unique_ptr old_recovery_thread( std::move(recovery_thread_)); // In this case, if recovery_in_prog_ is false, current thread should // wait the previous recover thread to finish and create a new thread // to recover from the bg error. db_mutex_->Unlock(); TEST_SYNC_POINT( "StartRecoverFromRetryableBGIOError:BeforeWaitingForOtherThread"); old_recovery_thread->join(); TEST_SYNC_POINT( "StartRecoverFromRetryableBGIOError:AfterWaitingForOtherThread"); db_mutex_->Lock(); } recovery_thread_.reset( new port::Thread(&ErrorHandler::RecoverFromRetryableBGIOError, this)); if (recovery_error_.ok()) { return recovery_error_; } else { return bg_error_; } } // Automatic recover from Retryable BG IO error. Must be called after db // mutex is released. void ErrorHandler::RecoverFromRetryableBGIOError() { assert(recovery_in_prog_); TEST_SYNC_POINT("RecoverFromRetryableBGIOError:BeforeStart"); TEST_SYNC_POINT("RecoverFromRetryableBGIOError:BeforeStart2"); InstrumentedMutexLock l(db_mutex_); if (end_recovery_) { EventHelpers::NotifyOnErrorRecoveryEnd(db_options_.listeners, bg_error_, Status::ShutdownInProgress(), db_mutex_); recovery_in_prog_ = false; return; } DBRecoverContext context = recover_context_; context.flush_after_recovery = true; int resume_count = db_options_.max_bgerror_resume_count; uint64_t wait_interval = db_options_.bgerror_resume_retry_interval; uint64_t retry_count = 0; // Recover from the retryable error. Create a separate thread to do it. while (resume_count > 0) { if (end_recovery_) { EventHelpers::NotifyOnErrorRecoveryEnd(db_options_.listeners, bg_error_, Status::ShutdownInProgress(), db_mutex_); recovery_in_prog_ = false; return; } TEST_SYNC_POINT("RecoverFromRetryableBGIOError:BeforeResume0"); TEST_SYNC_POINT("RecoverFromRetryableBGIOError:BeforeResume1"); recovery_error_ = IOStatus::OK(); retry_count++; Status s = db_->ResumeImpl(context); RecordStats({ERROR_HANDLER_AUTORESUME_RETRY_TOTAL_COUNT}, {} /* int_histograms */); if (s.IsShutdownInProgress() || bg_error_.severity() >= Status::Severity::kFatalError) { // If DB shutdown in progress or the error severity is higher than // Hard Error, stop auto resume and returns. recovery_in_prog_ = false; RecordStats({} /* ticker_types */, {{ERROR_HANDLER_AUTORESUME_RETRY_COUNT, retry_count}}); EventHelpers::NotifyOnErrorRecoveryEnd(db_options_.listeners, bg_error_, bg_error_, db_mutex_); return; } if (!recovery_error_.ok() && recovery_error_.severity() <= Status::Severity::kHardError && recovery_error_.GetRetryable()) { // If new BG IO error happens during auto recovery and it is retryable // and its severity is Hard Error or lower, the auto resmue sleep for // a period of time and redo auto resume if it is allowed. TEST_SYNC_POINT("RecoverFromRetryableBGIOError:BeforeWait0"); TEST_SYNC_POINT("RecoverFromRetryableBGIOError:BeforeWait1"); int64_t wait_until = db_options_.clock->NowMicros() + wait_interval; cv_.TimedWait(wait_until); } else { // There are three possibility: 1) recovery_error_ is set during resume // and the error is not retryable, 2) recover is successful, 3) other // error happens during resume and cannot be resumed here. if (recovery_error_.ok() && s.ok()) { // recover from the retryable IO error and no other BG errors. Clean // the bg_error and notify user. TEST_SYNC_POINT("RecoverFromRetryableBGIOError:RecoverSuccess"); RecordStats({ERROR_HANDLER_AUTORESUME_SUCCESS_COUNT}, {{ERROR_HANDLER_AUTORESUME_RETRY_COUNT, retry_count}}); return; } else { // In this case: 1) recovery_error_ is more serious or not retryable // 2) other error happens. The auto recovery stops. recovery_in_prog_ = false; RecordStats({} /* ticker_types */, {{ERROR_HANDLER_AUTORESUME_RETRY_COUNT, retry_count}}); EventHelpers::NotifyOnErrorRecoveryEnd( db_options_.listeners, bg_error_, !recovery_error_.ok() ? recovery_error_ : s, db_mutex_); return; } } resume_count--; } recovery_in_prog_ = false; EventHelpers::NotifyOnErrorRecoveryEnd( db_options_.listeners, bg_error_, Status::Aborted("Exceeded resume retry count"), db_mutex_); TEST_SYNC_POINT("RecoverFromRetryableBGIOError:LoopOut"); RecordStats({} /* ticker_types */, {{ERROR_HANDLER_AUTORESUME_RETRY_COUNT, retry_count}}); } void ErrorHandler::CheckAndSetRecoveryAndBGError(const Status& bg_err) { if (recovery_in_prog_ && recovery_error_.ok()) { recovery_error_ = status_to_io_status(Status(bg_err)); } if (bg_err.severity() > bg_error_.severity()) { bg_error_ = bg_err; } if (bg_error_.severity() >= Status::Severity::kHardError) { is_db_stopped_.store(true, std::memory_order_release); } } void ErrorHandler::EndAutoRecovery() { db_mutex_->AssertHeld(); if (!end_recovery_) { end_recovery_ = true; } if (recovery_thread_) { // Ensure only one thread can execute the join(). std::unique_ptr old_recovery_thread( std::move(recovery_thread_)); db_mutex_->Unlock(); cv_.SignalAll(); old_recovery_thread->join(); db_mutex_->Lock(); } TEST_SYNC_POINT("PostEndAutoRecovery"); } void ErrorHandler::RecordStats( const std::vector& ticker_types, const std::vector>& int_histograms) { if (bg_error_stats_ == nullptr) { return; } for (const auto& ticker_type : ticker_types) { RecordTick(bg_error_stats_.get(), ticker_type); } for (const auto& hist : int_histograms) { RecordInHistogram(bg_error_stats_.get(), std::get<0>(hist), std::get<1>(hist)); } } } // namespace ROCKSDB_NAMESPACE