diff --git a/CHANGELOG b/CHANGELOG index 49e4c410590b..8ec67a583843 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,8 @@ v3.8.2 (XXXX-XX-XX) ------------------- +* Fix a rare shutdown race in RocksDBShaCalculatorThread. + * Reduce internal priority of AQL execution. This prevents possible deadlocks with modification operations in a cluster and replicationFactor >= 2, and can also improve responsiveness under high load of AQL queries. diff --git a/arangod/RocksDBEngine/Listeners/RocksDBShaCalculator.cpp b/arangod/RocksDBEngine/Listeners/RocksDBShaCalculator.cpp index 2c107c18f314..ee22adc09f19 100644 --- a/arangod/RocksDBEngine/Listeners/RocksDBShaCalculator.cpp +++ b/arangod/RocksDBEngine/Listeners/RocksDBShaCalculator.cpp @@ -269,11 +269,22 @@ RocksDBShaCalculator::RocksDBShaCalculator(application_features::ApplicationServ /// @brief Shutdown the background thread only if it was ever started RocksDBShaCalculator::~RocksDBShaCalculator() { + // when we get here the background thread should have been stopped + // already. + TRI_ASSERT(!_shaThread.isRunning()); + waitForShutdown(); +} + +void RocksDBShaCalculator::waitForShutdown() { + // send shutdown signal to SHA thread + _shaThread.beginShutdown(); + _shaThread.signalLoop(); CONDITION_LOCKER(locker, _threadDone); if (_shaThread.isRunning()) { _threadDone.wait(); } + // now we are sure the SHA thread is not running anymore } void RocksDBShaCalculator::OnFlushCompleted(rocksdb::DB* db, diff --git a/arangod/RocksDBEngine/Listeners/RocksDBShaCalculator.h b/arangod/RocksDBEngine/Listeners/RocksDBShaCalculator.h index a14974782491..1562063b70b0 100644 --- a/arangod/RocksDBEngine/Listeners/RocksDBShaCalculator.h +++ b/arangod/RocksDBEngine/Listeners/RocksDBShaCalculator.h @@ -89,7 +89,9 @@ class RocksDBShaCalculator : public rocksdb::EventListener { void OnCompactionCompleted(rocksdb::DB* db, const rocksdb::CompactionJobInfo& ci) override; - void beginShutdown() { _shaThread.beginShutdown(); }; + void beginShutdown() { _shaThread.beginShutdown(); } + + void waitForShutdown(); protected: /// thread to perform sha256 and file deletes in background diff --git a/arangod/RocksDBEngine/RocksDBEngine.cpp b/arangod/RocksDBEngine/RocksDBEngine.cpp index cdb9784459d7..a78e5abb79f0 100644 --- a/arangod/RocksDBEngine/RocksDBEngine.cpp +++ b/arangod/RocksDBEngine/RocksDBEngine.cpp @@ -915,7 +915,7 @@ void RocksDBEngine::beginShutdown() { // signal the event listener that we are going to shut down soon if (_shaListener != nullptr) { _shaListener->beginShutdown(); - } // if + } } void RocksDBEngine::stop() { @@ -924,6 +924,10 @@ void RocksDBEngine::stop() { // in case we missed the beginShutdown somehow, call it again replicationManager()->beginShutdown(); replicationManager()->dropAll(); + + if (_shaListener != nullptr) { + _shaListener->waitForShutdown(); + } if (_backgroundThread) { // stop the press