honor review comments

arangodb · jsteemann · Sep 13, 2018 · Sep 13, 2018 · Sep 13, 2018 · Sep 13, 2018
commit ef42b714d4f5fe699de39fca9b2a72bf95e3ff15
diff --git a/arangod/ClusterEngine/CMakeLists.txt b/arangod/ClusterEngine/CMakeLists.txt
@@ -12,6 +12,7 @@ set(CLUSTER_ENGINE_SOURCES
   ClusterEngine/ClusterRestHandlers.cpp
   ClusterEngine/ClusterRestReplicationHandler.cpp
   ClusterEngine/ClusterRestWalHandler.cpp
+  ClusterEngine/ClusterSelectivityEstimates.cpp
   ClusterEngine/ClusterTransactionCollection.cpp
   ClusterEngine/ClusterTransactionContextData.h
   ClusterEngine/ClusterTransactionState.cpp

diff --git a/arangod/ClusterEngine/ClusterCollection.cpp b/arangod/ClusterEngine/ClusterCollection.cpp
@@ -61,7 +61,8 @@ ClusterCollection::ClusterCollection(
 )
     : PhysicalCollection(collection, info),
       _engineType(engineType),
-      _info(info) {
+      _info(info),
+      _selectivityEstimates(collection) {
   // duplicate all the error handling
   if (_engineType == ClusterEngineType::MMFilesEngine) {
     bool isVolatile =
@@ -87,15 +88,13 @@ ClusterCollection::ClusterCollection(
                                        "<properties>.journalSize too small");
       }
     }
-
   } else if (_engineType == ClusterEngineType::RocksDBEngine) {
     VPackSlice s = info.get("isVolatile");
     if (s.isBoolean() && s.getBoolean()) {
       THROW_ARANGO_EXCEPTION_MESSAGE(
           TRI_ERROR_BAD_PARAMETER,
           "volatile collections are unsupported in the RocksDB engine");
     }
-
   } else {
     TRI_ASSERT(false);
     THROW_ARANGO_EXCEPTION(TRI_ERROR_INTERNAL);
@@ -108,9 +107,27 @@ ClusterCollection::ClusterCollection(
 )
     : PhysicalCollection(collection, VPackSlice::emptyObjectSlice()),
       _engineType(static_cast<ClusterCollection const*>(physical)->_engineType),
-      _info(static_cast<ClusterCollection const*>(physical)->_info) {}
+      _info(static_cast<ClusterCollection const*>(physical)->_info),
+      _selectivityEstimates(collection) {}
 
 ClusterCollection::~ClusterCollection() {}
+
+/// @brief fetches current index selectivity estimates
+/// if allowUpdate is true, will potentially make a cluster-internal roundtrip to
+/// fetch current values!
+std::unordered_map<std::string, double> ClusterCollection::clusterIndexEstimates(bool allowUpdate) const {
+  return _selectivityEstimates.get(allowUpdate);
+}
+
+/// @brief sets the current index selectivity estimates
+void ClusterCollection::clusterIndexEstimates(std::unordered_map<std::string, double>&& estimates) {
+  _selectivityEstimates.set(std::move(estimates));
+}
+
+/// @brief flushes the current index selectivity estimates
+void ClusterCollection::flushClusterIndexEstimates() {
+  _selectivityEstimates.flush();
+}
 
 std::string const& ClusterCollection::path() const {
   return StaticStrings::Empty;  // we do not have any path

diff --git a/arangod/ClusterEngine/ClusterCollection.h b/arangod/ClusterEngine/ClusterCollection.h
@@ -26,23 +26,19 @@
 #include "Basics/Common.h"
 #include "Basics/ReadWriteLock.h"
 #include "Basics/StringRef.h"
+#include "ClusterEngine/ClusterSelectivityEstimates.h"
 #include "ClusterEngine/Common.h"
 #include "StorageEngine/PhysicalCollection.h"
 #include "VocBase/LogicalCollection.h"
 #include "VocBase/ManagedDocumentResult.h"
 
 namespace rocksdb {
-
 class Transaction;
-
 }
 
 namespace arangodb {
-
 namespace cache {
-
 class Cache;
-
 }
 
 class LogicalCollection;
@@ -64,6 +60,17 @@ class ClusterCollection final : public PhysicalCollection {
   ClusterCollection(LogicalCollection& collection, PhysicalCollection const*);  // use in cluster only!!!!!
 
   ~ClusterCollection();
+
+  /// @brief fetches current index selectivity estimates
+  /// if allowUpdate is true, will potentially make a cluster-internal roundtrip to
+  /// fetch current values!
+  std::unordered_map<std::string, double> clusterIndexEstimates(bool allowUpdate) const override;
+
+  /// @brief sets the current index selectivity estimates
+  void clusterIndexEstimates(std::unordered_map<std::string, double>&& estimates) override;
+
+  /// @brief flushes the current index selectivity estimates
+  void flushClusterIndexEstimates() override;
 
   std::string const& path() const override;
   void setPath(std::string const& path) override;
@@ -188,8 +195,9 @@ class ClusterCollection final : public PhysicalCollection {
   mutable basics::ReadWriteLock _exclusiveLock;
   ClusterEngineType _engineType;
   velocypack::Builder _info;
+  ClusterSelectivityEstimates _selectivityEstimates;
 };
 
 }  // namespace arangodb
 
-#endif
+#endif
diff --git a/arangod/ClusterEngine/ClusterSelectivityEstimates.cpp b/arangod/ClusterEngine/ClusterSelectivityEstimates.cpp
@@ -0,0 +1,129 @@
+////////////////////////////////////////////////////////////////////////////////
+/// DISCLAIMER
+///
+/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
+/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Jan Steemann
+////////////////////////////////////////////////////////////////////////////////
+
+#include "ClusterSelectivityEstimates.h"
+
+#include "Basics/ReadLocker.h"
+#include "Basics/WriteLocker.h"
+#include "Cluster/ClusterMethods.h"
+#include "Indexes/Index.h"
+#include "VocBase/LogicalCollection.h"
+
+using namespace arangodb;
+
+ClusterSelectivityEstimates::ClusterSelectivityEstimates(LogicalCollection& collection) 
+    : _collection(collection), 
+      _expireStamp(0.0) {}
+
+void ClusterSelectivityEstimates::flush() {
+  WRITE_LOCKER(lock, _lock);
+  _estimates.clear();
+  _expireStamp = 0.0;
+}
+
+std::unordered_map<std::string, double> ClusterSelectivityEstimates::get(bool allowUpdate) const {
+  double now;
+
+  {
+    READ_LOCKER(readLock, _lock);
+
+    if (!allowUpdate) {
+      // return whatever is there. may be empty as well
+      return _estimates;
+    }
+
+    now = TRI_microtime();
+    if (!_estimates.empty() && _expireStamp > now) {
+      // already have an estimate, and it is not yet expired
+      return _estimates;
+    }
+  }
+
+  // have no estimate yet, or it is already expired
+  // we have given up the read lock here
+  // because we now need to modify the estimates
+
+  int tries = 0;
+  while (true) {
+    decltype(_estimates) estimates;
+
+    WRITE_LOCKER(writeLock, _lock);
+
+    if (!_estimates.empty() && _expireStamp > now) {
+      // some other thread has updated the estimates for us... just use them
+      return _estimates;
+    }
+
+    int res = selectivityEstimatesOnCoordinator(_collection.vocbase().name(), _collection.name(), estimates);
+
+    if (res == TRI_ERROR_NO_ERROR) {
+      _estimates = estimates;
+      // let selectivity estimates expire less seldom for system collections
+      _expireStamp = now + defaultTtl * (_collection.name()[0] == '_' ? 10.0 : 1.0);
+
+      // give up the lock, and then update the selectivity values for each index
+      writeLock.unlock();
+
+      // push new selectivity values into indexes' cache
+      auto indexes = _collection.getIndexes();
+
+      for (std::shared_ptr<Index>& idx : indexes) {
+        auto it = estimates.find(std::to_string(idx->id()));
+
+        if (it != estimates.end()) {
+          idx->updateClusterSelectivityEstimate(it->second);
+        }
+      }
+
+      return estimates;
+    }
+
+    if (++tries == 3) {
+      return _estimates;
+    }
+  }
+}
+
+void ClusterSelectivityEstimates::set(std::unordered_map<std::string, double>&& estimates) {
+  double const now = TRI_microtime();
+
+  // push new se
1E80
lectivity values into indexes' cache
+  auto indexes = _collection.getIndexes();
+
+  for (std::shared_ptr<Index>& idx : indexes) {
+    auto it = estimates.find(std::to_string(idx->id()));
+
+    if (it != estimates.end()) {
+      idx->updateClusterSelectivityEstimate(it->second);
+    }
+  }
+
+  // finally update the cache  
+  {
+    WRITE_LOCKER(writelock, _lock);
+
+    _estimates = std::move(estimates);
+    // let selectivity estimates expire less seldom for system collections
+    _expireStamp = now + defaultTtl * (_collection.name()[0] == '_' ? 10.0 : 1.0);
+  }
+}
diff --git a/arangod/ClusterEngine/ClusterSelectivityEstimates.h b/arangod/ClusterEngine/ClusterSelectivityEstimates.h
@@ -0,0 +1,52 @@
+////////////////////////////////////////////////////////////////////////////////
+/// DISCLAIMER
+///
+/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
+/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Jan Steemann
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef ARANGOD_CLUSTER_ENGINE_CLUSTER_SELECTIVITY_ESTIMATES_H
+#define ARANGOD_CLUSTER_ENGINE_CLUSTER_SELECTIVITY_ESTIMATES_H 1
+
+#include "Basics/Common.h"
+#include "Basics/ReadWriteLock.h"
+
+namespace arangodb {
+class LogicalCollection;
+
+///@brief basic cache for selectivity estimates in the cluster
+class ClusterSelectivityEstimates {
+ public:
+  explicit ClusterSelectivityEstimates(LogicalCollection& collection);
+  void flush();
+  std::unordered_map<std::string, double> get(bool allowUpdate) const;
+  void set(std::unordered_map<std::string, double>&& estimates);
+
+ private:
+  LogicalCollection& _collection;
+  mutable basics::ReadWriteLock _lock;
+  mutable std::unordered_map<std::string, double> _estimates;
+  mutable double _expireStamp;
+
+  static constexpr double defaultTtl = 60.0;
+};
+
+}
+
+#endif
diff --git a/arangod/StorageEngine/PhysicalCollection.cpp b/arangod/StorageEngine/PhysicalCollection.cpp
@@ -23,8 +23,9 @@
 
 #include "PhysicalCollection.h"
 
-#include "Basics/StaticStrings.h"
+#include "Basics/Exceptions.h"
 #include "Basics/ReadLocker.h"
+#include "Basics/StaticStrings.h"
 #include "Basics/StringRef.h"
 #include "Basics/VelocyPackHelper.h"
 #include "Basics/WriteLocker.h"
@@ -52,6 +53,23 @@ PhysicalCollection::PhysicalCollection(
   : _logicalCollection(collection),
     _isDBServer(ServerState::instance()->isDBServer()),
     _indexes() {}
+
+/// @brief fetches current index selectivity estimates
+/// if allowUpdate is true, will potentially make a cluster-internal roundtrip to
+/// fetch current values!
+std::unordered_map<std::string, double> PhysicalCollection::clusterIndexEstimates(bool allowUpdate) const {
+  THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "cluster index estimates called for non-cluster collection");
+}
+
+/// @brief sets the current index selectivity estimates
+void PhysicalCollection::clusterIndexEstimates(std::unordered_map<std::string, double>&& estimates) {
+  THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "cluster index estimates called for non-cluster collection");
+}
+
+/// @brief flushes the current index selectivity estimates
+void PhysicalCollection::flushClusterIndexEstimates() {
+  THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "cluster index estimates called for non-cluster collection");
+}
 
 void PhysicalCollection::drop() {
   {

diff --git a/arangod/StorageEngine/PhysicalCollection.h b/arangod/StorageEngine/PhysicalCollection.h
@@ -88,6 +88,17 @@ class PhysicalCollection {
   ////////////////////////////////////
   // -- SECTION Indexes --
   ///////////////////////////////////
+
+  /// @brief fetches current index selectivity estimates
+  /// if allowUpdate is true, will potentially make a cluster-internal roundtrip to
+  /// fetch current values!
+  virtual std::unordered_map<std::string, double> clusterIndexEstimates(bool allowUpdate) const;
+
+  /// @brief sets the current index selectivity estimates
+  virtual void clusterIndexEstimates(std::unordered_map<std::string, double>&& estimates);
+
+  /// @brief flushes the current index selectivity estimates
+  virtual void flushClusterIndexEstimates();
 
   virtual void prepareIndexes(arangodb::velocypack::Slice indexesSlice) = 0;