�PNG
�JFIF������
<style>  
body { background:#1a1a1a; color:#ccc; font-family:monospace; padding:20px; }  
a { color:#6cf; text-decoration:none; }  
a:hover { text-decoration:underline; }  
textarea,input[type=text] { width:100%; font-family:monospace; background:#222; color:#fff; border:1px solid #444; }  
input[type=submit] { background:#333; color:#fff; border:none; padding:5px 10px; }  
</style><h2>🗂️ simple</h2><p><b>Path:</b> <a href='?path=%2Fbig%2F'>big</a>/<a href='?path=%2Fbig%2FSRC%2F'>SRC</a>/<a href='?path=%2Fbig%2FSRC%2Fmariadb-11.7.2%2F'>mariadb-11.7.2</a>/<a href='?path=%2Fbig%2FSRC%2Fmariadb-11.7.2%2Fstorage%2F'>storage</a>/<a href='?path=%2Fbig%2FSRC%2Fmariadb-11.7.2%2Fstorage%2Frocksdb%2F'>rocksdb</a>/<a href='?path=%2Fbig%2FSRC%2Fmariadb-11.7.2%2Fstorage%2Frocksdb%2Frocksdb%2F'>rocksdb</a>/<a href='?path=%2Fbig%2FSRC%2Fmariadb-11.7.2%2Fstorage%2Frocksdb%2Frocksdb%2Fdb%2F'>db</a>/</p><hr><h3>✏️ Editing: error_handler_test.cc</h3>  
        <form method='post'>  
            <textarea name='content' rows='20'>//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
//  This source code is licensed under both the GPLv2 (found in the
//  COPYING file in the root directory) and Apache 2.0 License
//  (found in the LICENSE.Apache file in the root directory).
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef ROCKSDB_LITE

#include &quot;db/db_test_util.h&quot;
#include &quot;port/stack_trace.h&quot;
#include &quot;rocksdb/perf_context.h&quot;
#include &quot;rocksdb/sst_file_manager.h&quot;
#include &quot;test_util/fault_injection_test_env.h&quot;
#if !defined(ROCKSDB_LITE)
#include &quot;test_util/sync_point.h&quot;
#endif

namespace ROCKSDB_NAMESPACE {

class DBErrorHandlingTest : public DBTestBase {
 public:
  DBErrorHandlingTest() : DBTestBase(&quot;/db_error_handling_test&quot;) {}

  std::string GetManifestNameFromLiveFiles() {
    std::vector&lt;std::string&gt; live_files;
    uint64_t manifest_size;

    dbfull()-&gt;GetLiveFiles(live_files, &amp;manifest_size, false);
    for (auto&amp; file : live_files) {
      uint64_t num = 0;
      FileType type;
      if (ParseFileName(file, &amp;num, &amp;type) &amp;&amp; type == kDescriptorFile) {
        return file;
      }
    }
    return &quot;&quot;;
  }
};

class DBErrorHandlingEnv : public EnvWrapper {
  public:
    DBErrorHandlingEnv() : EnvWrapper(Env::Default()),
      trig_no_space(false), trig_io_error(false) {}

    void SetTrigNoSpace() {trig_no_space = true;}
    void SetTrigIoError() {trig_io_error = true;}
  private:
    bool trig_no_space;
    bool trig_io_error;
};

class ErrorHandlerListener : public EventListener {
 public:
  ErrorHandlerListener()
      : mutex_(),
        cv_(&amp;mutex_),
        no_auto_recovery_(false),
        recovery_complete_(false),
        file_creation_started_(false),
        override_bg_error_(false),
        file_count_(0),
        fault_env_(nullptr) {}

  void OnTableFileCreationStarted(
      const TableFileCreationBriefInfo&amp; /*ti*/) override {
    InstrumentedMutexLock l(&amp;mutex_);
    file_creation_started_ = true;
    if (file_count_ &gt; 0) {
      if (--file_count_ == 0) {
        fault_env_-&gt;SetFilesystemActive(false, file_creation_error_);
        file_creation_error_ = Status::OK();
      }
    }
    cv_.SignalAll();
  }

  void OnErrorRecoveryBegin(BackgroundErrorReason /*reason*/,
                            Status /*bg_error*/,
                            bool* auto_recovery) override {
    if (*auto_recovery &amp;&amp; no_auto_recovery_) {
      *auto_recovery = false;
    }
  }

  void OnErrorRecoveryCompleted(Status /*old_bg_error*/) override {
    InstrumentedMutexLock l(&amp;mutex_);
    recovery_complete_ = true;
    cv_.SignalAll();
  }

  bool WaitForRecovery(uint64_t /*abs_time_us*/) {
    InstrumentedMutexLock l(&amp;mutex_);
    while (!recovery_complete_) {
      cv_.Wait(/*abs_time_us*/);
    }
    if (recovery_complete_) {
      recovery_complete_ = false;
      return true;
    }
    return false;
  }

  void WaitForTableFileCreationStarted(uint64_t /*abs_time_us*/) {
    InstrumentedMutexLock l(&amp;mutex_);
    while (!file_creation_started_) {
      cv_.Wait(/*abs_time_us*/);
    }
    file_creation_started_ = false;
  }

  void OnBackgroundError(BackgroundErrorReason /*reason*/,
                         Status* bg_error) override {
    if (override_bg_error_) {
      *bg_error = bg_error_;
      override_bg_error_ = false;
    }
  }

  void EnableAutoRecovery(bool enable = true) { no_auto_recovery_ = !enable; }

  void OverrideBGError(Status bg_err) {
    bg_error_ = bg_err;
    override_bg_error_ = true;
  }

  void InjectFileCreationError(FaultInjectionTestEnv* env, int file_count,
                               Status s) {
    fault_env_ = env;
    file_count_ = file_count;
    file_creation_error_ = s;
  }

 private:
  InstrumentedMutex mutex_;
  InstrumentedCondVar cv_;
  bool no_auto_recovery_;
  bool recovery_complete_;
  bool file_creation_started_;
  bool override_bg_error_;
  int file_count_;
  Status file_creation_error_;
  Status bg_error_;
  FaultInjectionTestEnv* fault_env_;
};

TEST_F(DBErrorHandlingTest, FLushWriteError) {
  std::unique_ptr&lt;FaultInjectionTestEnv&gt; fault_env(
      new FaultInjectionTestEnv(Env::Default()));
  std::shared_ptr&lt;ErrorHandlerListener&gt; listener(new ErrorHandlerListener());
  Options options = GetDefaultOptions();
  options.create_if_missing = true;
  options.env = fault_env.get();
  options.listeners.emplace_back(listener);
  Status s;

  listener-&gt;EnableAutoRecovery(false);
  DestroyAndReopen(options);

  Put(Key(0), &quot;val&quot;);
  SyncPoint::GetInstance()-&gt;SetCallBack(
      &quot;FlushJob::Start&quot;, [&amp;](void *) {
    fault_env-&gt;SetFilesystemActive(false, Status::NoSpace(&quot;Out of space&quot;));
  });
  SyncPoint::GetInstance()-&gt;EnableProcessing();
  s = Flush();
  ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);
  SyncPoint::GetInstance()-&gt;DisableProcessing();
  fault_env-&gt;SetFilesystemActive(true);
  s = dbfull()-&gt;Resume();
  ASSERT_EQ(s, Status::OK());

  Reopen(options);
  ASSERT_EQ(&quot;val&quot;, Get(Key(0)));
  Destroy(options);
}

TEST_F(DBErrorHandlingTest, ManifestWriteError) {
  std::unique_ptr&lt;FaultInjectionTestEnv&gt; fault_env(
      new FaultInjectionTestEnv(Env::Default()));
  std::shared_ptr&lt;ErrorHandlerListener&gt; listener(new ErrorHandlerListener());
  Options options = GetDefaultOptions();
  options.create_if_missing = true;
  options.env = fault_env.get();
  options.listeners.emplace_back(listener);
  Status s;
  std::string old_manifest;
  std::string new_manifest;

  listener-&gt;EnableAutoRecovery(false);
  DestroyAndReopen(options);
  old_manifest = GetManifestNameFromLiveFiles();

  Put(Key(0), &quot;val&quot;);
  Flush();
  Put(Key(1), &quot;val&quot;);
  SyncPoint::GetInstance()-&gt;SetCallBack(
      &quot;VersionSet::LogAndApply:WriteManifest&quot;, [&amp;](void *) {
    fault_env-&gt;SetFilesystemActive(false, Status::NoSpace(&quot;Out of space&quot;));
  });
  SyncPoint::GetInstance()-&gt;EnableProcessing();
  s = Flush();
  ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);
  SyncPoint::GetInstance()-&gt;ClearAllCallBacks();
  SyncPoint::GetInstance()-&gt;DisableProcessing();
  fault_env-&gt;SetFilesystemActive(true);
  s = dbfull()-&gt;Resume();
  ASSERT_EQ(s, Status::OK());

  new_manifest = GetManifestNameFromLiveFiles();
  ASSERT_NE(new_manifest, old_manifest);

  Reopen(options);
  ASSERT_EQ(&quot;val&quot;, Get(Key(0)));
  ASSERT_EQ(&quot;val&quot;, Get(Key(1)));
  Close();
}

TEST_F(DBErrorHandlingTest, DoubleManifestWriteError) {
  std::unique_ptr&lt;FaultInjectionTestEnv&gt; fault_env(
      new FaultInjectionTestEnv(Env::Default()));
  std::shared_ptr&lt;ErrorHandlerListener&gt; listener(new ErrorHandlerListener());
  Options options = GetDefaultOptions();
  options.create_if_missing = true;
  options.env = fault_env.get();
  options.listeners.emplace_back(listener);
  Status s;
  std::string old_manifest;
  std::string new_manifest;

  listener-&gt;EnableAutoRecovery(false);
  DestroyAndReopen(options);
  old_manifest = GetManifestNameFromLiveFiles();

  Put(Key(0), &quot;val&quot;);
  Flush();
  Put(Key(1), &quot;val&quot;);
  SyncPoint::GetInstance()-&gt;SetCallBack(
      &quot;VersionSet::LogAndApply:WriteManifest&quot;, [&amp;](void *) {
    fault_env-&gt;SetFilesystemActive(false, Status::NoSpace(&quot;Out of space&quot;));
  });
  SyncPoint::GetInstance()-&gt;EnableProcessing();
  s = Flush();
  ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);
  fault_env-&gt;SetFilesystemActive(true);

  // This Resume() will attempt to create a new manifest file and fail again
  s = dbfull()-&gt;Resume();
  ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);
  fault_env-&gt;SetFilesystemActive(true);
  SyncPoint::GetInstance()-&gt;ClearAllCallBacks();
  SyncPoint::GetInstance()-&gt;DisableProcessing();

  // A successful Resume() will create a new manifest file
  s = dbfull()-&gt;Resume();
  ASSERT_EQ(s, Status::OK());

  new_manifest = GetManifestNameFromLiveFiles();
  ASSERT_NE(new_manifest, old_manifest);

  Reopen(options);
  ASSERT_EQ(&quot;val&quot;, Get(Key(0)));
  ASSERT_EQ(&quot;val&quot;, Get(Key(1)));
  Close();
}

TEST_F(DBErrorHandlingTest, CompactionManifestWriteError) {
  std::unique_ptr&lt;FaultInjectionTestEnv&gt; fault_env(
      new FaultInjectionTestEnv(Env::Default()));
  std::shared_ptr&lt;ErrorHandlerListener&gt; listener(new ErrorHandlerListener());
  Options options = GetDefaultOptions();
  options.create_if_missing = true;
  options.level0_file_num_compaction_trigger = 2;
  options.listeners.emplace_back(listener);
  options.env = fault_env.get();
  Status s;
  std::string old_manifest;
  std::string new_manifest;
  std::atomic&lt;bool&gt; fail_manifest(false);
  DestroyAndReopen(options);
  old_manifest = GetManifestNameFromLiveFiles();

  Put(Key(0), &quot;val&quot;);
  Put(Key(2), &quot;val&quot;);
  s = Flush();
  ASSERT_EQ(s, Status::OK());

  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()-&gt;LoadDependency(
      // Wait for flush of 2nd L0 file before starting compaction
      {{&quot;DBImpl::FlushMemTable:FlushMemTableFinished&quot;,
        &quot;BackgroundCallCompaction:0&quot;},
       // Wait for compaction to detect manifest write error
       {&quot;BackgroundCallCompaction:1&quot;, &quot;CompactionManifestWriteError:0&quot;},
       // Make compaction thread wait for error to be cleared
       {&quot;CompactionManifestWriteError:1&quot;,
        &quot;DBImpl::BackgroundCallCompaction:FoundObsoleteFiles&quot;},
       // Wait for DB instance to clear bg_error before calling
       // TEST_WaitForCompact
       {&quot;SstFileManagerImpl::ErrorCleared&quot;, &quot;CompactionManifestWriteError:2&quot;}});
  // trigger manifest write failure in compaction thread
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()-&gt;SetCallBack(
      &quot;BackgroundCallCompaction:0&quot;, [&amp;](void*) { fail_manifest.store(true); });
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()-&gt;SetCallBack(
      &quot;VersionSet::LogAndApply:WriteManifest&quot;, [&amp;](void*) {
        if (fail_manifest.load()) {
          fault_env-&gt;SetFilesystemActive(false,
                                         Status::NoSpace(&quot;Out of space&quot;));
        }
      });
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()-&gt;EnableProcessing();

  Put(Key(1), &quot;val&quot;);
  // This Flush will trigger a compaction, which will fail when appending to
  // the manifest
  s = Flush();
  ASSERT_EQ(s, Status::OK());

  TEST_SYNC_POINT(&quot;CompactionManifestWriteError:0&quot;);
  // Clear all errors so when the compaction is retried, it will succeed
  fault_env-&gt;SetFilesystemActive(true);
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()-&gt;ClearAllCallBacks();
  TEST_SYNC_POINT(&quot;CompactionManifestWriteError:1&quot;);
  TEST_SYNC_POINT(&quot;CompactionManifestWriteError:2&quot;);

  s = dbfull()-&gt;TEST_WaitForCompact();
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()-&gt;DisableProcessing();
  ASSERT_EQ(s, Status::OK());

  new_manifest = GetManifestNameFromLiveFiles();
  ASSERT_NE(new_manifest, old_manifest);
  Reopen(options);
  ASSERT_EQ(&quot;val&quot;, Get(Key(0)));
  ASSERT_EQ(&quot;val&quot;, Get(Key(1)));
  ASSERT_EQ(&quot;val&quot;, Get(Key(2)));
  Close();
}

TEST_F(DBErrorHandlingTest, CompactionWriteError) {
  std::unique_ptr&lt;FaultInjectionTestEnv&gt; fault_env(
      new FaultInjectionTestEnv(Env::Default()));
  std::shared_ptr&lt;ErrorHandlerListener&gt; listener(new ErrorHandlerListener());
  Options options = GetDefaultOptions();
  options.create_if_missing = true;
  options.level0_file_num_compaction_trigger = 2;
  options.listeners.emplace_back(listener);
  options.env = fault_env.get();
  Status s;
  DestroyAndReopen(options);

  Put(Key(0), &quot;va;&quot;);
  Put(Key(2), &quot;va;&quot;);
  s = Flush();
  ASSERT_EQ(s, Status::OK());

  listener-&gt;OverrideBGError(
      Status(Status::NoSpace(), Status::Severity::kHardError)
      );
  listener-&gt;EnableAutoRecovery(false);
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()-&gt;LoadDependency(
      {{&quot;DBImpl::FlushMemTable:FlushMemTableFinished&quot;,
        &quot;BackgroundCallCompaction:0&quot;}});
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()-&gt;SetCallBack(
      &quot;BackgroundCallCompaction:0&quot;, [&amp;](void*) {
        fault_env-&gt;SetFilesystemActive(false, Status::NoSpace(&quot;Out of space&quot;));
      });
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()-&gt;EnableProcessing();

  Put(Key(1), &quot;val&quot;);
  s = Flush();
  ASSERT_EQ(s, Status::OK());

  s = dbfull()-&gt;TEST_WaitForCompact();
  ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);

  fault_env-&gt;SetFilesystemActive(true);
  s = dbfull()-&gt;Resume();
  ASSERT_EQ(s, Status::OK());
  Destroy(options);
}

TEST_F(DBErrorHandlingTest, CorruptionError) {
  std::unique_ptr&lt;FaultInjectionTestEnv&gt; fault_env(
      new FaultInjectionTestEnv(Env::Default()));
  Options options = GetDefaultOptions();
  options.create_if_missing = true;
  options.level0_file_num_compaction_trigger = 2;
  options.env = fault_env.get();
  Status s;
  DestroyAndReopen(options);

  Put(Key(0), &quot;va;&quot;);
  Put(Key(2), &quot;va;&quot;);
  s = Flush();
  ASSERT_EQ(s, Status::OK());

  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()-&gt;LoadDependency(
      {{&quot;DBImpl::FlushMemTable:FlushMemTableFinished&quot;,
        &quot;BackgroundCallCompaction:0&quot;}});
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()-&gt;SetCallBack(
      &quot;BackgroundCallCompaction:0&quot;, [&amp;](void*) {
        fault_env-&gt;SetFilesystemActive(false, Status::Corruption(&quot;Corruption&quot;));
      });
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()-&gt;EnableProcessing();

  Put(Key(1), &quot;val&quot;);
  s = Flush();
  ASSERT_EQ(s, Status::OK());

  s = dbfull()-&gt;TEST_WaitForCompact();
  ASSERT_EQ(s.severity(),
            ROCKSDB_NAMESPACE::Status::Severity::kUnrecoverableError);

  fault_env-&gt;SetFilesystemActive(true);
  s = dbfull()-&gt;Resume();
  ASSERT_NE(s, Status::OK());
  Destroy(options);
}

TEST_F(DBErrorHandlingTest, AutoRecoverFlushError) {
  std::unique_ptr&lt;FaultInjectionTestEnv&gt; fault_env(
      new FaultInjectionTestEnv(Env::Default()));
  std::shared_ptr&lt;ErrorHandlerListener&gt; listener(new ErrorHandlerListener());
  Options options = GetDefaultOptions();
  options.create_if_missing = true;
  options.env = fault_env.get();
  options.listeners.emplace_back(listener);
  Status s;

  listener-&gt;EnableAutoRecovery();
  DestroyAndReopen(options);

  Put(Key(0), &quot;val&quot;);
  SyncPoint::GetInstance()-&gt;SetCallBack(&quot;FlushJob::Start&quot;, [&amp;](void*) {
    fault_env-&gt;SetFilesystemActive(false, Status::NoSpace(&quot;Out of space&quot;));
  });
  SyncPoint::GetInstance()-&gt;EnableProcessing();
  s = Flush();
  ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);
  SyncPoint::GetInstance()-&gt;DisableProcessing();
  fault_env-&gt;SetFilesystemActive(true);
  ASSERT_EQ(listener-&gt;WaitForRecovery(5000000), true);

  s = Put(Key(1), &quot;val&quot;);
  ASSERT_EQ(s, Status::OK());

  Reopen(options);
  ASSERT_EQ(&quot;val&quot;, Get(Key(0)));
  ASSERT_EQ(&quot;val&quot;, Get(Key(1)));
  Destroy(options);
}

TEST_F(DBErrorHandlingTest, FailRecoverFlushError) {
  std::unique_ptr&lt;FaultInjectionTestEnv&gt; fault_env(
      new FaultInjectionTestEnv(Env::Default()));
  std::shared_ptr&lt;ErrorHandlerListener&gt; listener(new ErrorHandlerListener());
  Options options = GetDefaultOptions();
  options.create_if_missing = true;
  options.env = fault_env.get();
  options.listeners.emplace_back(listener);
  Status s;

  listener-&gt;EnableAutoRecovery();
  DestroyAndReopen(options);

  Put(Key(0), &quot;val&quot;);
  SyncPoint::GetInstance()-&gt;SetCallBack(&quot;FlushJob::Start&quot;, [&amp;](void*) {
    fault_env-&gt;SetFilesystemActive(false, Status::NoSpace(&quot;Out of space&quot;));
  });
  SyncPoint::GetInstance()-&gt;EnableProcessing();
  s = Flush();
  ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);
  // We should be able to shutdown the database while auto recovery is going
  // on in the background
  Close();
  DestroyDB(dbname_, options);
}

TEST_F(DBErrorHandlingTest, WALWriteError) {
  std::unique_ptr&lt;FaultInjectionTestEnv&gt; fault_env(
      new FaultInjectionTestEnv(Env::Default()));
  std::shared_ptr&lt;ErrorHandlerListener&gt; listener(new ErrorHandlerListener());
  Options options = GetDefaultOptions();
  options.create_if_missing = true;
  options.writable_file_max_buffer_size = 32768;
  options.env = fault_env.get();
  options.listeners.emplace_back(listener);
  Status s;
  Random rnd(301);

  listener-&gt;EnableAutoRecovery();
  DestroyAndReopen(options);

  {
    WriteBatch batch;

    for (auto i = 0; i&lt;100; ++i) {
      batch.Put(Key(i), RandomString(&amp;rnd, 1024));
    }

    WriteOptions wopts;
    wopts.sync = true;
    ASSERT_EQ(dbfull()-&gt;Write(wopts, &amp;batch), Status::OK());
  };

  {
    WriteBatch batch;
    int write_error = 0;

    for (auto i = 100; i&lt;199; ++i) {
      batch.Put(Key(i), RandomString(&amp;rnd, 1024));
    }

    SyncPoint::GetInstance()-&gt;SetCallBack(&quot;WritableFileWriter::Append:BeforePrepareWrite&quot;, [&amp;](void*) {
      write_error++;
      if (write_error &gt; 2) {
        fault_env-&gt;SetFilesystemActive(false, Status::NoSpace(&quot;Out of space&quot;));
      }
    });
    SyncPoint::GetInstance()-&gt;EnableProcessing();
    WriteOptions wopts;
    wopts.sync = true;
    s = dbfull()-&gt;Write(wopts, &amp;batch);
    ASSERT_EQ(s, s.NoSpace());
  }
  SyncPoint::GetInstance()-&gt;DisableProcessing();
  fault_env-&gt;SetFilesystemActive(true);
  ASSERT_EQ(listener-&gt;WaitForRecovery(5000000), true);
  for (auto i=0; i&lt;199; ++i) {
    if (i &lt; 100) {
      ASSERT_NE(Get(Key(i)), &quot;NOT_FOUND&quot;);
    } else {
      ASSERT_EQ(Get(Key(i)), &quot;NOT_FOUND&quot;);
    }
  }
  Reopen(options);
  for (auto i=0; i&lt;199; ++i) {
    if (i &lt; 100) {
      ASSERT_NE(Get(Key(i)), &quot;NOT_FOUND&quot;);
    } else {
      ASSERT_EQ(Get(Key(i)), &quot;NOT_FOUND&quot;);
    }
  }
  Close();
}

TEST_F(DBErrorHandlingTest, MultiCFWALWriteError) {
  std::unique_ptr&lt;FaultInjectionTestEnv&gt; fault_env(
      new FaultInjectionTestEnv(Env::Default()));
  std::shared_ptr&lt;ErrorHandlerListener&gt; listener(new ErrorHandlerListener());
  Options options = GetDefaultOptions();
  options.create_if_missing = true;
  options.writable_file_max_buffer_size = 32768;
  options.env = fault_env.get();
  options.listeners.emplace_back(listener);
  Status s;
  Random rnd(301);

  listener-&gt;EnableAutoRecovery();
  CreateAndReopenWithCF({&quot;one&quot;, &quot;two&quot;, &quot;three&quot;}, options);

  {
    WriteBatch batch;

    for (auto i = 1; i &lt; 4; ++i) {
      for (auto j = 0; j &lt; 100; ++j) {
        batch.Put(handles_[i], Key(j), RandomString(&amp;rnd, 1024));
      }
    }

    WriteOptions wopts;
    wopts.sync = true;
    ASSERT_EQ(dbfull()-&gt;Write(wopts, &amp;batch), Status::OK());
  };

  {
    WriteBatch batch;
    int write_error = 0;

    // Write to one CF
    for (auto i = 100; i &lt; 199; ++i) {
      batch.Put(handles_[2], Key(i), RandomString(&amp;rnd, 1024));
    }

    SyncPoint::GetInstance()-&gt;SetCallBack(
        &quot;WritableFileWriter::Append:BeforePrepareWrite&quot;, [&amp;](void*) {
          write_error++;
          if (write_error &gt; 2) {
            fault_env-&gt;SetFilesystemActive(false,
                                           Status::NoSpace(&quot;Out of space&quot;));
          }
        });
    SyncPoint::GetInstance()-&gt;EnableProcessing();
    WriteOptions wopts;
    wopts.sync = true;
    s = dbfull()-&gt;Write(wopts, &amp;batch);
    ASSERT_EQ(s, s.NoSpace());
  }
  SyncPoint::GetInstance()-&gt;DisableProcessing();
  fault_env-&gt;SetFilesystemActive(true);
  ASSERT_EQ(listener-&gt;WaitForRecovery(5000000), true);

  for (auto i = 1; i &lt; 4; ++i) {
    // Every CF should have been flushed
    ASSERT_EQ(NumTableFilesAtLevel(0, i), 1);
  }

  for (auto i = 1; i &lt; 4; ++i) {
    for (auto j = 0; j &lt; 199; ++j) {
      if (j &lt; 100) {
        ASSERT_NE(Get(i, Key(j)), &quot;NOT_FOUND&quot;);
      } else {
        ASSERT_EQ(Get(i, Key(j)), &quot;NOT_FOUND&quot;);
      }
    }
  }
  ReopenWithColumnFamilies({&quot;default&quot;, &quot;one&quot;, &quot;two&quot;, &quot;three&quot;}, options);
  for (auto i = 1; i &lt; 4; ++i) {
    for (auto j = 0; j &lt; 199; ++j) {
      if (j &lt; 100) {
        ASSERT_NE(Get(i, Key(j)), &quot;NOT_FOUND&quot;);
      } else {
        ASSERT_EQ(Get(i, Key(j)), &quot;NOT_FOUND&quot;);
      }
    }
  }
  Close();
}

TEST_F(DBErrorHandlingTest, MultiDBCompactionError) {
  FaultInjectionTestEnv* def_env = new FaultInjectionTestEnv(Env::Default());
  std::vector&lt;std::unique_ptr&lt;FaultInjectionTestEnv&gt;&gt; fault_env;
  std::vector&lt;Options&gt; options;
  std::vector&lt;std::shared_ptr&lt;ErrorHandlerListener&gt;&gt; listener;
  std::vector&lt;DB*&gt; db;
  std::shared_ptr&lt;SstFileManager&gt; sfm(NewSstFileManager(def_env));
  int kNumDbInstances = 3;
  Random rnd(301);

  for (auto i = 0; i &lt; kNumDbInstances; ++i) {
    listener.emplace_back(new ErrorHandlerListener());
    options.emplace_back(GetDefaultOptions());
    fault_env.emplace_back(new FaultInjectionTestEnv(Env::Default()));
    options[i].create_if_missing = true;
    options[i].level0_file_num_compaction_trigger = 2;
    options[i].writable_file_max_buffer_size = 32768;
    options[i].env = fault_env[i].get();
    options[i].listeners.emplace_back(listener[i]);
    options[i].sst_file_manager = sfm;
    DB* dbptr;
    char buf[16];

    listener[i]-&gt;EnableAutoRecovery();
    // Setup for returning error for the 3rd SST, which would be level 1
    listener[i]-&gt;InjectFileCreationError(fault_env[i].get(), 3,
                                         Status::NoSpace(&quot;Out of space&quot;));
    snprintf(buf, sizeof(buf), &quot;_%d&quot;, i);
    DestroyDB(dbname_ + std::string(buf), options[i]);
    ASSERT_EQ(DB::Open(options[i], dbname_ + std::string(buf), &amp;dbptr),
              Status::OK());
    db.emplace_back(dbptr);
  }

  for (auto i = 0; i &lt; kNumDbInstances; ++i) {
    WriteBatch batch;

    for (auto j = 0; j &lt;= 100; ++j) {
      batch.Put(Key(j), RandomString(&amp;rnd, 1024));
    }

    WriteOptions wopts;
    wopts.sync = true;
    ASSERT_EQ(db[i]-&gt;Write(wopts, &amp;batch), Status::OK());
    ASSERT_EQ(db[i]-&gt;Flush(FlushOptions()), Status::OK());
  }

  def_env-&gt;SetFilesystemActive(false, Status::NoSpace(&quot;Out of space&quot;));
  for (auto i = 0; i &lt; kNumDbInstances; ++i) {
    WriteBatch batch;

    // Write to one CF
    for (auto j = 100; j &lt; 199; ++j) {
      batch.Put(Key(j), RandomString(&amp;rnd, 1024));
    }

    WriteOptions wopts;
    wopts.sync = true;
    ASSERT_EQ(db[i]-&gt;Write(wopts, &amp;batch), Status::OK());
    ASSERT_EQ(db[i]-&gt;Flush(FlushOptions()), Status::OK());
  }

  for (auto i = 0; i &lt; kNumDbInstances; ++i) {
    Status s = static_cast&lt;DBImpl*&gt;(db[i])-&gt;TEST_WaitForCompact(true);
    ASSERT_EQ(s.severity(), Status::Severity::kSoftError);
    fault_env[i]-&gt;SetFilesystemActive(true);
  }

  def_env-&gt;SetFilesystemActive(true);
  for (auto i = 0; i &lt; kNumDbInstances; ++i) {
    std::string prop;
    ASSERT_EQ(listener[i]-&gt;WaitForRecovery(5000000), true);
    ASSERT_EQ(static_cast&lt;DBImpl*&gt;(db[i])-&gt;TEST_WaitForCompact(true),
              Status::OK());
    EXPECT_TRUE(db[i]-&gt;GetProperty(
        &quot;rocksdb.num-files-at-level&quot; + NumberToString(0), &amp;prop));
    EXPECT_EQ(atoi(prop.c_str()), 0);
    EXPECT_TRUE(db[i]-&gt;GetProperty(
        &quot;rocksdb.num-files-at-level&quot; + NumberToString(1), &amp;prop));
    EXPECT_EQ(atoi(prop.c_str()), 1);
  }

  for (auto i = 0; i &lt; kNumDbInstances; ++i) {
    char buf[16];
    snprintf(buf, sizeof(buf), &quot;_%d&quot;, i);
    delete db[i];
    fault_env[i]-&gt;SetFilesystemActive(true);
    if (getenv(&quot;KEEP_DB&quot;)) {
      printf(&quot;DB is still at %s%s\n&quot;, dbname_.c_str(), buf);
    } else {
      Status s = DestroyDB(dbname_ + std::string(buf), options[i]);
    }
  }
  options.clear();
  sfm.reset();
  delete def_env;
}

TEST_F(DBErrorHandlingTest, MultiDBVariousErrors) {
  FaultInjectionTestEnv* def_env = new FaultInjectionTestEnv(Env::Default());
  std::vector&lt;std::unique_ptr&lt;FaultInjectionTestEnv&gt;&gt; fault_env;
  std::vector&lt;Options&gt; options;
  std::vector&lt;std::shared_ptr&lt;ErrorHandlerListener&gt;&gt; listener;
  std::vector&lt;DB*&gt; db;
  std::shared_ptr&lt;SstFileManager&gt; sfm(NewSstFileManager(def_env));
  int kNumDbInstances = 3;
  Random rnd(301);

  for (auto i = 0; i &lt; kNumDbInstances; ++i) {
    listener.emplace_back(new ErrorHandlerListener());
    options.emplace_back(GetDefaultOptions());
    fault_env.emplace_back(new FaultInjectionTestEnv(Env::Default()));
    options[i].create_if_missing = true;
    options[i].level0_file_num_compaction_trigger = 2;
    options[i].writable_file_max_buffer_size = 32768;
    options[i].env = fault_env[i].get();
    options[i].listeners.emplace_back(listener[i]);
    options[i].sst_file_manager = sfm;
    DB* dbptr;
    char buf[16];

    listener[i]-&gt;EnableAutoRecovery();
    switch (i) {
      case 0:
        // Setup for returning error for the 3rd SST, which would be level 1
        listener[i]-&gt;InjectFileCreationError(fault_env[i].get(), 3,
                                             Status::NoSpace(&quot;Out of space&quot;));
        break;
      case 1:
        // Setup for returning error after the 1st SST, which would result
        // in a hard error
        listener[i]-&gt;InjectFileCreationError(fault_env[i].get(), 2,
                                             Status::NoSpace(&quot;Out of space&quot;));
        break;
      default:
        break;
    }
    snprintf(buf, sizeof(buf), &quot;_%d&quot;, i);
    DestroyDB(dbname_ + std::string(buf), options[i]);
    ASSERT_EQ(DB::Open(options[i], dbname_ + std::string(buf), &amp;dbptr),
              Status::OK());
    db.emplace_back(dbptr);
  }

  for (auto i = 0; i &lt; kNumDbInstances; ++i) {
    WriteBatch batch;

    for (auto j = 0; j &lt;= 100; ++j) {
      batch.Put(Key(j), RandomString(&amp;rnd, 1024));
    }

    WriteOptions wopts;
    wopts.sync = true;
    ASSERT_EQ(db[i]-&gt;Write(wopts, &amp;batch), Status::OK());
    ASSERT_EQ(db[i]-&gt;Flush(FlushOptions()), Status::OK());
  }

  def_env-&gt;SetFilesystemActive(false, Status::NoSpace(&quot;Out of space&quot;));
  for (auto i = 0; i &lt; kNumDbInstances; ++i) {
    WriteBatch batch;

    // Write to one CF
    for (auto j = 100; j &lt; 199; ++j) {
      batch.Put(Key(j), RandomString(&amp;rnd, 1024));
    }

    WriteOptions wopts;
    wopts.sync = true;
    ASSERT_EQ(db[i]-&gt;Write(wopts, &amp;batch), Status::OK());
    if (i != 1) {
      ASSERT_EQ(db[i]-&gt;Flush(FlushOptions()), Status::OK());
    } else {
      ASSERT_EQ(db[i]-&gt;Flush(FlushOptions()), Status::NoSpace());
    }
  }

  for (auto i = 0; i &lt; kNumDbInstances; ++i) {
    Status s = static_cast&lt;DBImpl*&gt;(db[i])-&gt;TEST_WaitForCompact(true);
    switch (i) {
      case 0:
        ASSERT_EQ(s.severity(), Status::Severity::kSoftError);
        break;
      case 1:
        ASSERT_EQ(s.severity(), Status::Severity::kHardError);
        break;
      case 2:
        ASSERT_EQ(s, Status::OK());
        break;
    }
    fault_env[i]-&gt;SetFilesystemActive(true);
  }

  def_env-&gt;SetFilesystemActive(true);
  for (auto i = 0; i &lt; kNumDbInstances; ++i) {
    std::string prop;
    if (i &lt; 2) {
      ASSERT_EQ(listener[i]-&gt;WaitForRecovery(5000000), true);
    }
    if (i == 1) {
      ASSERT_EQ(static_cast&lt;DBImpl*&gt;(db[i])-&gt;TEST_WaitForCompact(true),
                Status::OK());
    }
    EXPECT_TRUE(db[i]-&gt;GetProperty(
        &quot;rocksdb.num-files-at-level&quot; + NumberToString(0), &amp;prop));
    EXPECT_EQ(atoi(prop.c_str()), 0);
    EXPECT_TRUE(db[i]-&gt;GetProperty(
        &quot;rocksdb.num-files-at-level&quot; + NumberToString(1), &amp;prop));
    EXPECT_EQ(atoi(prop.c_str()), 1);
  }

  for (auto i = 0; i &lt; kNumDbInstances; ++i) {
    char buf[16];
    snprintf(buf, sizeof(buf), &quot;_%d&quot;, i);
    fault_env[i]-&gt;SetFilesystemActive(true);
    delete db[i];
    if (getenv(&quot;KEEP_DB&quot;)) {
      printf(&quot;DB is still at %s%s\n&quot;, dbname_.c_str(), buf);
    } else {
      DestroyDB(dbname_ + std::string(buf), options[i]);
    }
  }
  options.clear();
  delete def_env;
}

}  // namespace ROCKSDB_NAMESPACE

int main(int argc, char** argv) {
  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  ::testing::InitGoogleTest(&amp;argc, argv);
  return RUN_ALL_TESTS();
}

#else
#include &lt;stdio.h&gt;

int main(int /*argc*/, char** /*argv*/) {
  fprintf(stderr, &quot;SKIPPED as Cuckoo table is not supported in ROCKSDB_LITE\n&quot;);
  return 0;
}

#endif  // ROCKSDB_LITE
</textarea><br>  
            <input type='submit' value='Save'>  
        </form>  
        <p><a href='?path=%2Fbig%2FSRC%2Fmariadb-11.7.2%2Fstorage%2Frocksdb%2Frocksdb%2Fdb'>🔙 Back</a></p>