Fix a rare shutdown race in RocksDBShaCalculatorThread

arangodb · KVS85 · Sep 3, 2021 · Aug 26, 2021 · Aug 26, 2021 · Aug 26, 2021
commit bad6d6936335ad48f8f78c40d8bb3147dfd84ea1
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,6 +1,8 @@
 v3.8.1 (XXXX-XX-XX)
 -------------------
 
+* Fix a rare shutdown race in RocksDBShaCalculatorThread. 
+
 * Reduce internal priority of AQL execution. This prevents possible deadlocks
   with modification operations in a cluster and replicationFactor >= 2, and can
   also improve responsiveness under high load of AQL queries.

diff --git a/arangod/RocksDBEngine/Listeners/RocksDBShaCalculator.cpp b/arangod/RocksDBEngine/Listeners/RocksDBShaCalculator.cpp
@@ -269,11 +269,19 @@ RocksDBShaCalculator::RocksDBShaCalculator(application_features::ApplicationServ
 
 /// @brief Shutdown the background thread only if it was ever started
 RocksDBShaCalculator::~RocksDBShaCalculator() {
+  waitForShutdown();
+}
+
+void RocksDBShaCalculator::waitForShutdown() {
+  // send shutdown signal to SHA thread
+  _shaThread.beginShutdown();
+
   _shaThread.signalLoop();
   CONDITION_LOCKER(locker, _threadDone);
   if (_shaThread.isRunning()) {
     _threadDone.wait();
   }
+  // now we are sure the SHA thread is not running anymore
 }
 
 void RocksDBShaCalculator::OnFlushCompleted(rocksdb::DB* db,

@@ -89,7 +89,9 @@ class RocksDBShaCalculator : public rocksdb::EventListener {
 
   void OnCompactionCompleted(rocksdb::DB* db, const rocksdb::CompactionJobInfo& ci) override;
 
-  void beginShutdown() { _shaThread.beginShutdown(); };
+  void beginShutdown() { _shaThread.beginShutdown(); }
+
+  void waitForShutdown();
 
  protected:
   /// thread to perform sha256 and file deletes in background

@@ -915,7 +915,7 @@ void RocksDBEngine::beginShutdown() {
   // signal the event listener that we are going to shut down soon
   if (_shaListener != nullptr) {
     _shaListener->beginShutdown();
-  }  // if
+  } 
 }
 
 void RocksDBEngine::stop() {
@@ -924,6 +924,10 @@ void RocksDBEngine::stop() {
   // in case we missed the beginShutdown somehow, call it again
   replicationManager()->beginShutdown();
   replicationManager()->dropAll();
+
+  if (_shaListener != nullptr) {
+    _shaListener->waitForShutdown();
+  }
 
   if (_backgroundThread) {
     // stop the press