Added (gauge) metric "rocksdb_read_only" (#14469)

jsteemann · KVS85 · web-flow · commit e88ab99617b8 · 2021-07-08T16:37:45.000+03:00
* APM-107: Added metric "rocksdb_read_only" to determine whether RocksDB is
  currently in read-only mode due to a background error. The metric will have
  a value of "1" if RocksDB is in read-only mode and "0" if RocksDB is in
  normal operations mode. If the metric value is "1" it means all writes into
  RocksDB will fail, so inspecting the logfiles and acting on the actual error
  situation is required.

Co-authored-by: Vadim &lt;vadim@arangodb.com&gt;
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,6 +1,13 @@
 v3.7.13 (XXXX-XX-XX)
 --------------------
 
+* APM-107: Added metric "rocksdb_read_only" to determine whether RocksDB is
+  currently in read-only mode due to a background error. The metric will have a
+  value of "1" if RocksDB is in read-only mode and "0" if RocksDB is in normal
+  operations mode. If the metric value is "1" it means all writes into RocksDB
+  will fail, so inspecting the logfiles and acting on the actual error situation
+  is required.
+
 * Add following term ids, which prevents old synchronous replication requests to
   be accepted after a follower was dropped and has gotten in sync again.
   This makes the chaos tests which delay synchronous replication requests more
diff --git a/arangod/RocksDBEngine/RocksDBBackgroundErrorListener.cpp b/arangod/RocksDBEngine/RocksDBBackgroundErrorListener.cpp
@@ -28,6 +28,9 @@
 
 namespace arangodb {
 
+RocksDBBackgroundErrorListener::RocksDBBackgroundErrorListener()
+    : _called(false) {}
+
 RocksDBBackgroundErrorListener::~RocksDBBackgroundErrorListener() = default;
 
 void RocksDBBackgroundErrorListener::OnBackgroundError(rocksdb::BackgroundErrorReason reason,
@@ -37,10 +40,8 @@ void RocksDBBackgroundErrorListener::OnBackgroundError(rocksdb::BackgroundErrorR
     return;
   }
 
-  if (!_called) {
-    _called = true;
-
-    std::string operation = "unknown";
+  if (!_called.exchange(true)) {
+    char const* operation = "unknown";
     switch (reason) {
       case rocksdb::BackgroundErrorReason::kFlush: {
         operation = "flush";
@@ -67,4 +68,11 @@ void RocksDBBackgroundErrorListener::OnBackgroundError(rocksdb::BackgroundErrorR
   }
 }
 
+void RocksDBBackgroundErrorListener::OnErrorRecoveryCompleted(rocksdb::Status /* old_bg_error */) {
+  _called.store(false, std::memory_order_relaxed);
+    
+  LOG_TOPIC("8ff56", WARN, Logger::ROCKSDB)
+      << "RocksDB resuming operations after background error";
+}
+
 }  // namespace arangodb
diff --git a/arangod/RocksDBEngine/RocksDBBackgroundErrorListener.h b/arangod/RocksDBEngine/RocksDBBackgroundErrorListener.h
@@ -27,16 +27,24 @@
 #include <rocksdb/db.h>
 #include <rocksdb/listener.h>
 
+#include <atomic>
+
 namespace arangodb {
 
 class RocksDBBackgroundErrorListener : public rocksdb::EventListener {
  public:
+  RocksDBBackgroundErrorListener();
   virtual ~RocksDBBackgroundErrorListener();
 
   void OnBackgroundError(rocksdb::BackgroundErrorReason reason, rocksdb::Status* error) override;
+  void OnErrorRecoveryCompleted(rocksdb::Status /* old_bg_error */) override;
+
+  bool called() const noexcept {
+    return _called.load(std::memory_order_relaxed);
+  }
 
  private:
-  bool _called = false;
+  std::atomic<bool> _called;
 };  // class RocksDBThrottle
 
 }  // namespace arangodb
diff --git a/arangod/RocksDBEngine/RocksDBEngine.cpp b/arangod/RocksDBEngine/RocksDBEngine.cpp
@@ -642,7 +642,8 @@ void RocksDBEngine::start() {
     _options.listeners.push_back(_shaListener);
   }  // if
 
-  _options.listeners.push_back(std::make_shared<RocksDBBackgroundErrorListener>());
+  _errorListener = std::make_shared<RocksDBBackgroundErrorListener>();
+  _options.listeners.push_back(_errorListener);
 
   if (opts._totalWriteBufferSize > 0) {
     _options.db_write_buffer_size = opts._totalWriteBufferSize;
@@ -2468,6 +2469,10 @@ void RocksDBEngine::getStatistics(VPackBuilder& builder) const {
     builder.add("rocksdbengine.throttle.bps", VPackValue(_listener->GetThrottle()));
   }  // if
 
+  if (_errorListener) {
+    builder.add("rocksdb.read-only", VPackValue(_errorListener->called() ? 1 : 0));
+  }
+
   builder.close();
 }
 
diff --git a/arangod/RocksDBEngine/RocksDBEngine.h b/arangod/RocksDBEngine/RocksDBEngine.h
@@ -52,6 +52,7 @@ namespace arangodb {
 
 class PhysicalCollection;
 class PhysicalView;
+class RocksDBBackgroundErrorListener;
 class RocksDBBackgroundThread;
 class RocksDBEventListener;
 class RocksDBKey;
@@ -497,6 +498,8 @@ class RocksDBEngine final : public StorageEngine {
   // optional code to notice when rocksdb creates or deletes .ssh files.  Currently
   //  uses that input to create or delete parallel sha256 files
   std::shared_ptr<RocksDBEventListener> _shaListener;
+  
+  std::shared_ptr<RocksDBBackgroundErrorListener> _errorListener;
 
   arangodb::basics::ReadWriteLock _purgeLock;
 
