Merge remote-tracking branch 'origin/devel' into feature/zkd-index

arangodb · maierlars · Jan 5, 2022 · Mar 3, 2021 · Mar 4, 2021 · Mar 4, 2021
commit 5f83c9f9c7e8033675ded81c80ee6b5ac92d1b37
diff --git a/CHANGELOG b/CHANGELOG
@@ -18,6 +18,38 @@ devel
   The index implements the relation <=, == and >= natively. Strict relations are
   emulated using post filtering. Ranges can be unbounded on one or both sides.
 
+* Add ReplicatedLogs column family.
+
+* Add optimization rule for AqlAnalyzer.
+
+* Fixed issue #14592: IS_NULL(@x) isn't recognized as a constant expression.
+
+* Fixed issue BTS-539 "Unsynchronized query kill while it's being finalized in
+  another thread was uncovered through `test-kill.js` of `communication_ssl`
+  suite". Fixed possible (but unlikely) crash when killing an AQL query.
+
+* Append physical compaction of log collection to every Raft log
+  compaction (BTS-542).
+
+* Change optimization level for debug builds back to `-O0` (from `-Og`)
+  because `-Og` seems to cause debuggability issues in some environments.
+
+* Fixed issue BTS-536 "Upgrading without rest-server is aborted by error".
+  Now stating `--server.rest-server false` does not require the additional
+  `--console` argument for upgrading a server.
+
+* Fixed various problems in GEO_INTERSECTS: wrong results, not implemented
+  cases and numerically unstable behaviour. In particular, the case of
+  the intersection of two polygons in which one is an S2LngLatRect
+  is fixed (BTS-475).
+
+* Fixed ES-867 and ES-922: removed eslint from NPM packages descriptions and
+  updated netmask package to non-vulnerable version.
+
+* Web UI: Fixes a logical error which occured after re-visiting the logs view
+  which lead to not displaying the logs view and its entries correctly
+  (BTS-507).
+
 * Fixed a bug, where Coordinators handled plan changes for databases
   in heartbeat thread in wrong order. Databases could be listed, but
   not used.

diff --git a/arangod/RocksDBEngine/CMakeLists.txt b/arangod/RocksDBEngine/CMakeLists.txt
@@ -72,9 +72,9 @@ set(ROCKSDB_SOURCES
   RocksDBEngine/RocksDBLogValue.cpp
   RocksDBEngine/RocksDBMetaCollection.cpp
   RocksDBEngine/RocksDBMetadata.cpp
-  RocksDBEngine/RocksDBTransactionMethods.cpp
   RocksDBEngine/RocksDBOptimizerRules.cpp
   RocksDBEngine/RocksDBOptionFeature.cpp
+  RocksDBEngine/RocksDBPersistedLog.cpp
   RocksDBEngine/RocksDBPrimaryIndex.cpp
   RocksDBEngine/RocksDBRecoveryManager.cpp
   RocksDBEngine/RocksDBReplicationCommon.cpp
@@ -90,6 +90,7 @@ set(ROCKSDB_SOURCES
   RocksDBEngine/RocksDBSettingsManager.cpp
   RocksDBEngine/RocksDBSyncThread.cpp
   RocksDBEngine/RocksDBTransactionCollection.cpp
+  RocksDBEngine/RocksDBTransactionMethods.cpp
   RocksDBEngine/RocksDBTransactionState.cpp
   RocksDBEngine/RocksDBTtlIndex.cpp
   RocksDBEngine/RocksDBTypes.cpp

diff --git a/arangod/RocksDBEngine/RocksDBColumnFamilyManager.cpp b/arangod/RocksDBEngine/RocksDBColumnFamilyManager.cpp
@@ -30,17 +30,19 @@
 
 namespace arangodb {
 
-std::array<char const*, arangodb::RocksDBColumnFamilyManager::numberOfColumnFamilies>
-    RocksDBColumnFamilyManager::_internalNames = {"default",      "Documents",
-                                                  "PrimaryIndex", "EdgeIndex",
-                                                  "VPackIndex",   "GeoIndex",
-                                                  "FulltextIndex", "ZkdIndex"};
+std::array<char const*, arangodb::RocksDBColumnFamilyManager::numberOfColumnFamilies> RocksDBColumnFamilyManager::_internalNames =
+    {"default",       "Documents",      "PrimaryIndex",
+     "EdgeIndex",     "VPackIndex",     "GeoIndex",
+     "FulltextIndex", "ReplicatedLogs", "ZkdIndex"};
+
 std::array<char const*, arangodb::RocksDBColumnFamilyManager::numberOfColumnFamilies> RocksDBColumnFamilyManager::_externalNames =
-    {"definitions", "documents", "primary", "edge", "vpack", "geo", "fulltext", "zkd"};
+    {"definitions", "documents", "primary",         "edge", "vpack",
+     "geo",         "fulltext",  "replicated-logs", "zkd"};
 
 std::array<rocksdb::ColumnFamilyHandle*, RocksDBColumnFamilyManager::numberOfColumnFamilies>
-    RocksDBColumnFamilyManager::_handles = {nullptr, nullptr, nullptr, nullptr,
-                                            nullptr, nullptr, nullptr, nullptr};
+    RocksDBColumnFamilyManager::_handles = {nullptr, nullptr, nullptr,
+                                            nullptr, nullptr, nullptr,
+                                            nullptr, nullptr, nullptr};
 
 rocksdb::ColumnFamilyHandle* RocksDBColumnFamilyManager::_defaultHandle = nullptr;
 

diff --git a/arangod/RocksDBEngine/RocksDBColumnFamilyManager.h b/arangod/RocksDBEngine/RocksDBColumnFamilyManager.h
@@ -45,7 +45,8 @@ struct RocksDBColumnFamilyManager {
     VPackIndex = 4,  // persistent, "skiplist", "hash"
     GeoIndex = 5,
     FulltextIndex = 6,
-    ZkdIndex = 7,
+    ReplicatedLogs = 7,
+    ZkdIndex = 8,
 
     Invalid = 1024  // special placeholder
   };
@@ -55,8 +56,8 @@ struct RocksDBColumnFamilyManager {
     External   // for display to users
   };
 
-  static constexpr size_t minNumberOfColumnFamilies = 8;
-  static constexpr size_t numberOfColumnFamilies = 8;
+  static constexpr size_t minNumberOfColumnFamilies = 7;
+  static constexpr size_t numberOfColumnFamilies = 9;
 
   static void initialize();
 

diff --git a/arangod/RocksDBEngine/RocksDBEngine.cpp b/arangod/RocksDBEngine/RocksDBEngine.cpp
@@ -706,9 +706,9 @@ void RocksDBEngine::start() {
   addFamily(RocksDBColumnFamilyManager::Family::VPackIndex);
   addFamily(RocksDBColumnFamilyManager::Family::GeoIndex);
   addFamily(RocksDBColumnFamilyManager::Family::FulltextIndex);
+  addFamily(RocksDBColumnFamilyManager::Family::ReplicatedLogs);
   addFamily(RocksDBColumnFamilyManager::Family::ZkdIndex);
 
-
   size_t const minNumberOfColumnFamilies = RocksDBColumnFamilyManager::minNumberOfColumnFamilies;
   bool dbExisted = false;
   {
@@ -746,15 +746,17 @@ void RocksDBEngine::start() {
           << "found existing column families: " << names;
       auto const replicatedLogsName = RocksDBColumnFamilyManager::name(
           RocksDBColumnFamilyManager::Family::ReplicatedLogs);
+      auto const zkdIndexName = RocksDBColumnFamilyManager::name(
+          RocksDBColumnFamilyManager::Family::ReplicatedLogs);
 
       for (auto const& it : cfFamilies) {
         auto it2 = std::find(existingColumnFamilies.begin(),
                              existingColumnFamilies.end(), it.name);
         if (it2 == existingColumnFamilies.end()) {
 
-          if (it.name == replicatedLogsName) {
+          if (it.name == replicatedLogsName || it.name == zkdIndexName) {
             LOG_TOPIC("293c3", INFO, Logger::STARTUP)
-                << "column family " << replicatedLogsName
+                << "column family " << it.name
                 << " is missing and will be created.";
             continue;
           }
@@ -834,8 +836,10 @@ void RocksDBEngine::start() {
                                   cfHandles[5]);
   RocksDBColumnFamilyManager::set(RocksDBColumnFamilyManager::Family::FulltextIndex,
                                   cfHandles[6]);
-  RocksDBColumnFamilyManager::set(RocksDBColumnFamilyManager::Family::ZkdIndex,
+  RocksDBColumnFamilyManager::set(RocksDBColumnFamilyManager::Family::ReplicatedLogs,
                                   cfHandles[7]);
+  RocksDBColumnFamilyManager::set(RocksDBColumnFamilyManager::Family::ZkdIndex,
+                                  cfHandles[8]);
   TRI_ASSERT(RocksDBColumnFamilyManager::get(RocksDBColumnFamilyManager::Family::Definitions)
                  ->GetID() == 0);
 

diff --git a/arangod/RocksDBEngine/RocksDBKey.h b/arangod/RocksDBEngine/RocksDBKey.h
@@ -178,6 +178,12 @@ class RocksDBKey {
   void constructZkdIndexValue(uint64_t objectId, const zkd::byte_string& value);
   void constructZkdIndexValue(uint64_t objectId, const zkd::byte_string& value, LocalDocumentId documentId);
 
+  //////////////////////////////////////////////////////////////////////////////
+  /// @brief Create a fully-specified key for revision tree for a collection
+  //////////////////////////////////////////////////////////////////////////////
+  void constructLogEntry(uint64_t objectId, replication2::LogIndex idx);
+
+
  public:
   //////////////////////////////////////////////////////////////////////////////
   /// @brief Extracts the type from a key
@@ -288,6 +294,12 @@ class RocksDBKey {
   //////////////////////////////////////////////////////////////////////////////
   static zkd::byte_string_view zkdIndexValue(rocksdb::Slice const& slice);
 
+  //////////////////////////////////////////////////////////////////////////////
+  /// @brief Extracts log index from key
+  //////////////////////////////////////////////////////////////////////////////
+  static replication2::LogIndex logIndex(RocksDBKey const&);
+  static replication2::LogIndex logIndex(rocksdb::Slice const&);
+
   /// size of internal objectID
   static constexpr size_t objectIdSize() { return sizeof(uint64_t); }
 

diff --git a/arangod/RocksDBEngine/RocksDBKeyBounds.cpp b/arangod/RocksDBEngine/RocksDBKeyBounds.cpp
@@ -236,6 +236,8 @@ rocksdb::ColumnFamilyHandle* RocksDBKeyBounds::columnFamily() const {
       return RocksDBColumnFamilyManager::get(RocksDBColumnFamilyManager::Family::GeoIndex);
     case RocksDBEntryType::ZkdIndexValue:
       return RocksDBColumnFamilyManager::get(RocksDBColumnFamilyManager::Family::ZkdIndex);
+    case RocksDBEntryType::LogEntry:
+      return RocksDBColumnFamilyManager::get(RocksDBColumnFamilyManager::Family::ReplicatedLogs);
     case RocksDBEntryType::Database:
     case RocksDBEntryType::Collection:
     case RocksDBEntryType::CounterValue:

diff --git a/arangod/RocksDBEngine/RocksDBOptionFeature.cpp b/arangod/RocksDBEngine/RocksDBOptionFeature.cpp
@@ -636,7 +636,8 @@ rocksdb::ColumnFamilyOptions RocksDBOptionFeature::columnFamilyOptions(
     case RocksDBColumnFamilyManager::Family::PrimaryIndex:
     case RocksDBColumnFamilyManager::Family::GeoIndex:
     case RocksDBColumnFamilyManager::Family::FulltextIndex:
-    case RocksDBColumnFamilyManager::Family::ZkdIndex: {
+    case RocksDBColumnFamilyManager::Family::ZkdIndex:
+    case RocksDBColumnFamilyManager::Family::ReplicatedLogs: {
       // fixed 8 byte object id prefix
       options.prefix_extractor = std::shared_ptr<rocksdb::SliceTransform const>(
           rocksdb::NewFixedPrefixTransform(RocksDBKey::objectIdSize()));

diff --git a/arangod/RocksDBEngine/RocksDBTypes.cpp b/arangod/RocksDBEngine/RocksDBTypes.cpp
@@ -161,6 +161,10 @@ char const* arangodb::rocksDBEntryTypeName(arangodb::RocksDBEntryType type) {
       return "ZkdIndexValue";
     case arangodb::RocksDBEntryType::UniqueZkdIndexValue:
       return "UniqueZkdIndexValue";
+    case RocksDBEntryType::LogEntry:
+      return "ReplicatedLogEntry";
+    case RocksDBEntryType::ReplicatedLog:
+      return "ReplicatedLog";
   }
   return "Invalid";
 }
@@ -263,6 +267,10 @@ rocksdb::Slice const& arangodb::rocksDBSlice(RocksDBEntryType const& type) {
       return ZdkIndexValue;
     case RocksDBEntryType::UniqueZkdIndexValue:
       return UniqueZdkIndexValue;
+    case RocksDBEntryType::LogEntry:
+      return LogEntry;
+    case RocksDBEntryType::ReplicatedLog:
+      return ReplicatedLog;
   }
 
   return Placeholder;  // avoids warning - errorslice instead ?!

diff --git a/arangod/RocksDBEngine/RocksDBTypes.h b/arangod/RocksDBEngine/RocksDBTypes.h
@@ -52,12 +52,13 @@ enum class RocksDBEntryType : char {
   KeyGeneratorValue = '=',
   View = '>',
   GeoIndexValue = '?',
-  ZkdIndexValue = 'z',
-  UniqueZkdIndexValue = 'Z',
+  LogEntry = 'L',
   // RevisionTreeValue = '@', // pre-3.8 GA revision trees. do not use or reuse!
   // RevisionTreeValue = '/', // pre-3.8 GA revision trees. do not use or reuse!
   RevisionTreeValue = '*',
-  ReplicatedLog = 'l'
+  ReplicatedLog = 'l',
+  ZkdIndexValue = 'z',
+  UniqueZkdIndexValue = 'Z',
 };
 
 char const* rocksDBEntryTypeName(RocksDBEntryType);