arangodb · maierlars · Feb 20, 2023 · Feb 17, 2023 · Feb 19, 2023 · Feb 19, 2023
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,6 +1,10 @@
 devel
 -----
 
+* Updated ArangoDB Starter to 0.15.7.
+
+* Updated OpenSSL to 1.1.1t and OpenLDAP to 2.6.4.
+
 * Made all transactions used by the gharial API on coordinators and a few
   others marked "globally managed". This fixes an issue where
   transaction conflicts could lead to a silent out of sync situation

diff --git a/VERSIONS b/VERSIONS
@@ -1,13 +1,13 @@
 CXX_STANDARD "20"
-STARTER_REV "0.15.7-preview-1"
+STARTER_REV "0.15.7"
 SYNCER_REV "v2.14.0"
 GCC_LINUX "11.2.1_git20220219-r2"
 MSVC_WINDOWS "2022"
 LLVM_CLANG_MACOS "14"
 MACOS_MIN "10.15"
-OPENSSL_LINUX   "1.1.1s"
-OPENSSL_MACOS   "1.1.1s"
-OPENSSL_WINDOWS "1.1.1s"
+OPENSSL_LINUX   "1.1.1t"
+OPENSSL_MACOS   "1.1.1t"
+OPENSSL_WINDOWS "1.1.1t"
 USE_RCLONE "true"
 RCLONE_VERSION "1.59.0"
 MINIMAL_DEBUG_INFO "On"

diff --git a/arangod/Aql/DocumentProducingHelper.cpp b/arangod/Aql/DocumentProducingHelper.cpp
@@ -153,7 +153,7 @@ IndexIterator::DocumentCallback aql::buildDocumentCallback(
 }
 
 template<bool checkUniqueness>
-std::function<bool(LocalDocumentId const& token)> aql::getNullCallback(
+IndexIterator::LocalDocumentIdCallback aql::getNullCallback(
     DocumentProducingFunctionContext& context) {
   TRI_ASSERT(!context.hasFilter());
 
@@ -435,6 +435,7 @@ IndexIterator::CoveringCallback aql::getCallback(
                     IndexIteratorCoveringData& covering) {
     TRI_ASSERT(context.getAllowCoveringIndexOptimization());
     if constexpr (checkUniqueness) {
+      TRI_ASSERT(token.isSet());
       if (!context.checkUniqueness(token)) {
         // Document already found, skip it
         return false;

diff --git a/arangod/Aql/ExecutionBlockImpl.tpp b/arangod/Aql/ExecutionBlockImpl.tpp
@@ -690,8 +690,9 @@ static SkipRowsRangeVariant constexpr skipRowsType() {
 #ifdef USE_ENTERPRISE
                   arangodb::iresearch::OffsetMaterializeExecutor,
 #endif
-                  MaterializeExecutor<void>,
-                  MaterializeExecutor<std::string const&>>) ||
+                  MaterializeExecutor<void, false>,
+                  MaterializeExecutor<std::string const&, true>,
+                  MaterializeExecutor<std::string const&, false>>) ||
           IsSearchExecutor<Executor>::value,
       "Unexpected executor for SkipVariants::EXECUTOR");
 

diff --git a/arangod/Aql/ExecutionNode.cpp b/arangod/Aql/ExecutionNode.cpp
@@ -2827,18 +2827,23 @@ ExecutionNode* AsyncNode::clone(ExecutionPlan* plan, bool withDependencies,
 }
 
 namespace {
-constexpr std::string_view MATERIALIZE_NODE_IN_NM_DOC_PARAM = "inNmDocId";
-constexpr std::string_view MATERIALIZE_NODE_OUT_VARIABLE_PARAM = "outVariable";
-constexpr std::string_view MATERIALIZE_NODE_MULTI_NODE_PARAM = "multiNode";
+constexpr std::string_view kMaterializeNodeInNmDocParam = "inNmDocId";
+constexpr std::string_view kMaterializeNodeOutVariableParam = "outVariable";
+constexpr std::string_view kMaterializeNodeMultiNodeParam = "multiNode";
+constexpr std::string_view kMaterializeNodeInLocalDocIdParam = "inLocalDocId";
 }  // namespace
 
 MaterializeNode* materialize::createMaterializeNode(
     ExecutionPlan* plan, arangodb::velocypack::Slice const base) {
-  auto isMulti = base.get(MATERIALIZE_NODE_MULTI_NODE_PARAM);
+  auto isMulti = base.get(kMaterializeNodeMultiNodeParam);
   if (isMulti.isBoolean() && isMulti.getBoolean()) {
     return new MaterializeMultiNode(plan, base);
   }
-  return new MaterializeSingleNode(plan, base);
+  auto inLocalDocId = base.get(kMaterializeNodeInLocalDocIdParam);
+  if (inLocalDocId.isBoolean() && inLocalDocId.getBoolean()) {
+    return new MaterializeSingleNode<true>(plan, base);
+  }
+  return new MaterializeSingleNode<false>(plan, base);
 }
 
 MaterializeNode::MaterializeNode(ExecutionPlan* plan, ExecutionNodeId id,
@@ -2852,27 +2857,19 @@ MaterializeNode::MaterializeNode(ExecutionPlan* plan,
                                  arangodb::velocypack::Slice const& base)
     : ExecutionNode(plan, base),
       _inNonMaterializedDocId(aql::Variable::varFromVPack(
-          plan->getAst(), base, MATERIALIZE_NODE_IN_NM_DOC_PARAM, true)),
+          plan->getAst(), base, kMaterializeNodeInNmDocParam, true)),
       _outVariable(aql::Variable::varFromVPack(
-          plan->getAst(), base, MATERIALIZE_NODE_OUT_VARIABLE_PARAM)) {}
+          plan->getAst(), base, kMaterializeNodeOutVariableParam)) {}
 
 void MaterializeNode::doToVelocyPack(velocypack::Builder& nodes,
                                      unsigned /*flags*/) const {
-  nodes.add(VPackValue(MATERIALIZE_NODE_IN_NM_DOC_PARAM));
+  nodes.add(VPackValue(kMaterializeNodeInNmDocParam));
   _inNonMaterializedDocId->toVelocyPack(nodes);
 
-  nodes.add(VPackValue(MATERIALIZE_NODE_OUT_VARIABLE_PARAM));
+  nodes.add(VPackValue(kMaterializeNodeOutVariableParam));
   _outVariable->toVelocyPack(nodes);
 }
 
-auto MaterializeNode::getReadableInputRegisters(
-    RegisterId const inNmDocId) const -> RegIdSet {
-  // TODO (Dronplane) for index exectutor here we possibly will
-  // return additional SearchDoc register. So will keep this function for time
-  // being.
-  return RegIdSet{inNmDocId};
-}
-
 CostEstimate MaterializeNode::estimateCost() const {
   if (_dependencies.empty()) {
     // we should always have dependency as we need input for materializing
@@ -2907,7 +2904,7 @@ void MaterializeMultiNode::doToVelocyPack(velocypack::Builder& nodes,
                                           unsigned flags) const {
   // call base class method
   MaterializeNode::doToVelocyPack(nodes, flags);
-  nodes.add(MATERIALIZE_NODE_MULTI_NODE_PARAM, velocypack::Value(true));
+  nodes.add(kMaterializeNodeMultiNodeParam, velocypack::Value(true));
 }
 
 std::unique_ptr<ExecutionBlock> MaterializeMultiNode::createBlock(
@@ -2929,7 +2926,7 @@ std::unique_ptr<ExecutionBlock> MaterializeMultiNode::createBlock(
     outDocumentRegId = it->second.registerId;
   }
 
-  auto readableInputRegisters = getReadableInputRegisters(inNmDocIdRegId);
+  RegIdSet readableInputRegisters{inNmDocIdRegId};
   auto writableOutputRegisters = RegIdSet{outDocumentRegId};
 
   auto registerInfos = createRegisterInfos(std::move(readableInputRegisters),
@@ -2938,7 +2935,7 @@ std::unique_ptr<ExecutionBlock> MaterializeMultiNode::createBlock(
   auto executorInfos = MaterializerExecutorInfos<void>(
       inNmDocIdRegId, outDocumentRegId, engine.getQuery());
 
-  return std::make_unique<ExecutionBlockImpl<MaterializeExecutor<void>>>(
+  return std::make_unique<ExecutionBlockImpl<MaterializeExecutor<void, false>>>(
       &engine, this, std::move(registerInfos), std::move(executorInfos));
 }
 
@@ -2961,47 +2958,53 @@ ExecutionNode* MaterializeMultiNode::clone(ExecutionPlan* plan,
   return cloneHelper(std::move(c), withDependencies, withProperties);
 }
 
-MaterializeSingleNode::MaterializeSingleNode(ExecutionPlan* plan,
-                                             ExecutionNodeId id,
-                                             aql::Collection const* collection,
-                                             aql::Variable const& inDocId,
-                                             aql::Variable const& outVariable)
+template<bool localDocumentId>
+MaterializeSingleNode<localDocumentId>::MaterializeSingleNode(
+    ExecutionPlan* plan, ExecutionNodeId id, aql::Collection const* collection,
+    aql::Variable const& inDocId, aql::Variable const& outVariable)
     : MaterializeNode(plan, id, inDocId, outVariable),
       CollectionAccessingNode(collection) {}
 
-MaterializeSingleNode::MaterializeSingleNode(
+template<bool localDocumentId>
+MaterializeSingleNode<localDocumentId>::MaterializeSingleNode(
     ExecutionPlan* plan, arangodb::velocypack::Slice const& base)
     : MaterializeNode(plan, base), CollectionAccessingNode(plan, base) {}
 
-void MaterializeSingleNode::doToVelocyPack(velocypack::Builder& nodes,
-                                           unsigned flags) const {
+template<bool localDocumentId>
+void MaterializeSingleNode<localDocumentId>::doToVelocyPack(
+    velocypack::Builder& nodes, unsigned flags) const {
   // call base class method
   MaterializeNode::doToVelocyPack(nodes, flags);
 
   // add collection information
   CollectionAccessingNode::toVelocyPack(nodes, flags);
+  nodes.add(kMaterializeNodeInLocalDocIdParam, localDocumentId);
 }
 
-std::unique_ptr<ExecutionBlock> MaterializeSingleNode::createBlock(
+template<bool localDocumentId>
+std::unique_ptr<ExecutionBlock>
+MaterializeSingleNode<localDocumentId>::createBlock(
     ExecutionEngine& engine,
     std::unordered_map<ExecutionNode*, ExecutionBlock*> const&) const {
   ExecutionNode const* previousNode = getFirstDependency();
   TRI_ASSERT(previousNode != nullptr);
-  RegisterId inNmDocIdRegId;
-  {
-    auto it = getRegisterPlan()->varInfo.find(_inNonMaterializedDocId->id);
-    TRI_ASSERT(it != getRegisterPlan()->varInfo.end());
-    inNmDocIdRegId = it->second.registerId;
-  }
   RegisterId outDocumentRegId;
   {
     auto it = getRegisterPlan()->varInfo.find(_outVariable->id);
     TRI_ASSERT(it != getRegisterPlan()->varInfo.end());
     outDocumentRegId = it->second.registerId;
   }
   auto const& name = collection()->name();
-
-  auto readableInputRegisters = getReadableInputRegisters(inNmDocIdRegId);
+  RegisterId inNmDocIdRegId;
+  {
+    auto it = getRegisterPlan()->varInfo.find(_inNonMaterializedDocId->id);
+    TRI_ASSERT(it != getRegisterPlan()->varInfo.end());
+    inNmDocIdRegId = it->second.registerId;
+  }
+  RegIdSet readableInputRegisters;
+  if (inNmDocIdRegId.isValid()) {
+    readableInputRegisters.emplace(inNmDocIdRegId);
+  }
   auto writableOutputRegisters = RegIdSet{outDocumentRegId};
 
   auto registerInfos = createRegisterInfos(std::move(readableInputRegisters),
@@ -3010,13 +3013,13 @@ std::unique_ptr<ExecutionBlock> MaterializeSingleNode::createBlock(
   auto executorInfos = MaterializerExecutorInfos<decltype(name)>(
       inNmDocIdRegId, outDocumentRegId, engine.getQuery(), name);
   return std::make_unique<
-      ExecutionBlockImpl<MaterializeExecutor<decltype(name)>>>(
+      ExecutionBlockImpl<MaterializeExecutor<decltype(name), localDocumentId>>>(
       &engine, this, std::move(registerInfos), std::move(executorInfos));
 }
 
-ExecutionNode* MaterializeSingleNode::clone(ExecutionPlan* plan,
-                                            bool withDependencies,
-                                            bool withProperties) const {
+template<bool localDocumentId>
+ExecutionNode* MaterializeSingleNode<localDocumentId>::clone(
+    ExecutionPlan* plan, bool withDependencies, bool withProperties) const {
   TRI_ASSERT(plan);
 
   auto* outVariable = _outVariable;
@@ -3028,8 +3031,11 @@ ExecutionNode* MaterializeSingleNode::clone(ExecutionPlan* plan,
         plan->getAst()->variables()->createVariable(inNonMaterializedDocId);
   }
 
-  auto c = std::make_unique<MaterializeSingleNode>(
+  auto c = std::make_unique<MaterializeSingleNode<localDocumentId>>(
       plan, _id, collection(), *inNonMaterializedDocId, *outVariable);
   CollectionAccessingNode::cloneInto(*c);
   return cloneHelper(std::move(c), withDependencies, withProperties);
 }
+
+template class arangodb::aql::materialize::MaterializeSingleNode<false>;
+template class arangodb::aql::materialize::MaterializeSingleNode<true>;
diff --git a/arangod/Aql/ExecutionNode.h b/arangod/Aql/ExecutionNode.h
@@ -1170,8 +1170,6 @@ class MaterializeNode : public ExecutionNode {
  void doToVelocyPack(arangodb::velocypack::Builder& nodes,
                       unsigned flags) const override;
 
-  auto getReadableInputRegisters(RegisterId inNmDocId) const -> RegIdSet;
-
  protected:
   /// @brief input variable non-materialized document ids
   aql::Variable const* _inNonMaterializedDocId;
@@ -1205,6 +1203,7 @@ class MaterializeMultiNode : public MaterializeNode {
                       unsigned flags) const override final;
 };
 
+template<bool localDocumentId>
 class MaterializeSingleNode : public MaterializeNode,
                               public CollectionAccessingNode {
  public:

diff --git a/arangod/Aql/IndexExecutor.cpp b/arangod/Aql/IndexExecutor.cpp
@@ -106,10 +106,21 @@ IndexIterator::CoveringCallback getCallback(
     transaction::Methods::IndexHandle const& index,
     IndexNode::IndexValuesVars const& outNonMaterializedIndVars,
     IndexNode::IndexValuesRegisters const& outNonMaterializedIndRegs) {
-  return [&context, &index, &outNonMaterializedIndVars,
-          &outNonMaterializedIndRegs](LocalDocumentId const& token,
-                                      IndexIteratorCoveringData& covering) {
-    if constexpr (checkUniqueness) {
+  auto impl = [&context, &index, &outNonMaterializedIndVars,
+               &outNonMaterializedIndRegs]<typename TokenType>(
+                  TokenType&& token, IndexIteratorCoveringData& covering) {
+    constexpr bool isLocalDocumentId =
+        std::is_same_v<LocalDocumentId, std::decay_t<TokenType>>;
+    // can't be a static_assert as this implementation is still possible
+    // just can't be used right now as for restriction in late materialization
+    // rule
+#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
+    if constexpr (checkUniqueness && !isLocalDocumentId) {
+      TRI_ASSERT(false);
+    }
+#endif
+    if constexpr (checkUniqueness && isLocalDocumentId) {
+      TRI_ASSERT(token.isSet());
       if (!context.checkUniqueness(token)) {
         // Document already found, skip it
         return false;
@@ -135,11 +146,17 @@ IndexIterator::CoveringCallback getCallback(
     OutputAqlItemRow& output = context.getOutputRow();
     RegisterId registerId = context.getOutputRegister();
 
-    // move a document id
-    AqlValue v(AqlValueHintUInt(token.id()));
-    AqlValueGuard guard{v, true};
     TRI_ASSERT(!output.isFull());
-    output.moveValueInto(registerId, input, guard);
+    // move a document id
+    if constexpr (isLocalDocumentId) {
+      TRI_ASSERT(token.isSet());
+      AqlValue v(AqlValueHintUInt(token.id()));
+      AqlValueGuard guard{v, false};
+      output.moveValueInto(registerId, input, guard);
+    } else {
+      AqlValueGuard guard{token, true};
+      output.moveValueInto(registerId, input, guard);
+    }
 
     // hash/skiplist/persistent
     if (covering.isArray()) {
@@ -170,6 +187,7 @@ IndexIterator::CoveringCallback getCallback(
     output.advanceRow();
     return true;
   };
+  return impl;
 }
 
 }  // namespace

diff --git a/arangod/Aql/IndexNode.cpp b/arangod/Aql/IndexNode.cpp
@@ -192,7 +192,7 @@ IndexNode::IndexNode(ExecutionPlan* plan,
       _outNonMaterializedIndVars.second.try_emplace(var, fieldNumber);
     }
   }
-
+  _options.forLateMaterialization = isLateMaterialized();
   prepareProjections();
 }
 
@@ -555,6 +555,7 @@ void IndexNode::setLateMaterialized(aql::Variable const* docIdVariable,
     _outNonMaterializedIndVars.second.try_emplace(indVars.second.var,
                                                   indVars.second.indexFieldNum);
   }
+  _options.forLateMaterialization = true;
 }
 
 transaction::Methods::IndexHandle IndexNode::getSingleIndex() const {

diff --git a/arangod/Aql/IndexNodeOptimizerRules.cpp b/arangod/Aql/IndexNodeOptimizerRules.cpp
@@ -122,6 +122,10 @@ void arangodb::aql::lateDocumentMaterializationRule(
       auto& indexes = indexNode->getIndexes();
       TRI_ASSERT(!indexes.empty());
       if (indexes.size() != 1) {
+        // When enabling this please consider how inverted index would
+        // operate together with persistent as first produces
+        // SearchDocs but latter LocalDocumentIds. Usage of
+        // two separate variables might be the simplest solution
         continue;  // several indexes are not supported
       }
       auto& index = indexes.front();
@@ -326,15 +330,23 @@ void arangodb::aql::lateDocumentMaterializationRule(
 
         // we could apply late materialization
         // 1. We need to notify index node - it should not materialize
-        // documents, but produce only localDocIds
+        // documents, but produce only localDocIds or SearchDocs
+        // in case of inverted index
         indexNode->setLateMaterialized(localDocIdTmp, index->id(),
                                        uniqueVariables);
         // 2. We need to add materializer after limit node to do materialization
         // insert a materialize node
-        auto* materializeNode = plan->registerNode(
-            std::make_unique<materialize::MaterializeSingleNode>(
+        auto makeMaterializer = [&]() -> std::unique_ptr<ExecutionNode> {
+          if (index->type() == Index::TRI_IDX_TYPE_INVERTED_INDEX) {
+            return std::make_unique<materialize::MaterializeSingleNode<false>>(
                 plan.get(), plan->nextId(), indexNode->collection(),
-                *localDocIdTmp, *var));
+                *localDocIdTmp, *var);
+          }
+          return std::make_unique<materialize::MaterializeSingleNode<true>>(
+              plan.get(), plan->nextId(), indexNode->collection(),
+              *localDocIdTmp, *var);
+        };
+        auto* materializeNode = plan->registerNode(makeMaterializer());
         TRI_ASSERT(materializeNode);
 
         // on cluster we need to materialize node stay close to sort node on db