diff --git a/arangod/Aql/AqlFunctionsInternalCache.h b/arangod/Aql/AqlFunctionsInternalCache.h index 6262d436fdd4..f89b7e7c51e0 100644 --- a/arangod/Aql/AqlFunctionsInternalCache.h +++ b/arangod/Aql/AqlFunctionsInternalCache.h @@ -25,13 +25,11 @@ #include "Aql/AqlValue.h" #include "Basics/Common.h" -#include +#include "VocBase/Validators.h" #include #include - - namespace arangodb { namespace transaction { @@ -49,6 +47,8 @@ class AqlFunctionsInternalCache final { AqlFunctionsInternalCache() = default; ~AqlFunctionsInternalCache(); + AqlFunctionsInternalCache(AqlFunctionsInternalCache&&) = default; + void clear() noexcept; icu::RegexMatcher* buildRegexMatcher(char const* ptr, size_t length, bool caseInsensitive); @@ -97,4 +97,3 @@ class AqlFunctionsInternalCache final { } // namespace aql } // namespace arangodb - diff --git a/arangod/Aql/EngineInfoContainerDBServerServerBased.cpp b/arangod/Aql/EngineInfoContainerDBServerServerBased.cpp index e23ee4bc3613..efe9cae7ee18 100644 --- a/arangod/Aql/EngineInfoContainerDBServerServerBased.cpp +++ b/arangod/Aql/EngineInfoContainerDBServerServerBased.cpp @@ -87,7 +87,8 @@ EngineInfoContainerDBServerServerBased::TraverserEngineShardLists::TraverserEngi } #endif _edgeCollections.emplace_back( - getAllLocalShards(shardMapping, server, col->shardIds(restrictToShards))); + getAllLocalShards(shardMapping, server, col->shardIds(restrictToShards), + col->isSatellite() && node->isSmart())); } // Extract vertices auto const& vertices = _node->vertexColls(); @@ -102,21 +103,26 @@ EngineInfoContainerDBServerServerBased::TraverserEngineShardLists::TraverserEngi _inaccessible.insert(std::to_string(col->id().id())); } #endif - auto shards = getAllLocalShards(shardMapping, server, col->shardIds(restrictToShards)); + auto shards = getAllLocalShards(shardMapping, server, col->shardIds(restrictToShards), + col->isSatellite() && node->isSmart()); _vertexCollections.try_emplace(col->name(), std::move(shards)); } } std::vector EngineInfoContainerDBServerServerBased::TraverserEngineShardLists::getAllLocalShards( std::unordered_map const& shardMapping, - ServerID const& server, std::shared_ptr> shardIds) { + ServerID const& server, std::shared_ptr> shardIds, bool colIsSatellite) { std::vector localShards; for (auto const& shard : *shardIds) { auto const& it = shardMapping.find(shard); TRI_ASSERT(it != shardMapping.end()); if (it->second == server) { localShards.emplace_back(shard); + // Guaranteed that the traversal will be executed on this server. _hasShard = true; + } else if (colIsSatellite) { + // The satellite does not force run of a traversal here. + localShards.emplace_back(shard); } } return localShards; diff --git a/arangod/Aql/EngineInfoContainerDBServerServerBased.h b/arangod/Aql/EngineInfoContainerDBServerServerBased.h index aced12378b49..f51384287c29 100644 --- a/arangod/Aql/EngineInfoContainerDBServerServerBased.h +++ b/arangod/Aql/EngineInfoContainerDBServerServerBased.h @@ -81,7 +81,8 @@ class EngineInfoContainerDBServerServerBased { private: std::vector getAllLocalShards(std::unordered_map const& shardMapping, ServerID const& server, - std::shared_ptr> shardIds); + std::shared_ptr> shardIds, + bool colIsSatellite); private: // The graph node we need to serialize diff --git a/arangod/Aql/FixedVarExpressionContext.cpp b/arangod/Aql/FixedVarExpressionContext.cpp index 5fb6eee13cdf..a7ec095da965 100644 --- a/arangod/Aql/FixedVarExpressionContext.cpp +++ b/arangod/Aql/FixedVarExpressionContext.cpp @@ -56,14 +56,18 @@ void FixedVarExpressionContext::setVariableValue(Variable const* var, AqlValue c _vars.try_emplace(var, value); } +void FixedVarExpressionContext::clearVariableValue(Variable const* var) { + _vars.erase(var); +} + void FixedVarExpressionContext::serializeAllVariables(velocypack::Options const& opts, velocypack::Builder& builder) const { TRI_ASSERT(builder.isOpenArray()); for (auto const& it : _vars) { builder.openArray(); it.first->toVelocyPack(builder); - it.second.toVelocyPack(&opts, builder, /*resolveExternals*/true, - /*allowUnindexed*/false); + it.second.toVelocyPack(&opts, builder, /*resolveExternals*/ true, + /*allowUnindexed*/ false); builder.close(); } } diff --git a/arangod/Aql/FixedVarExpressionContext.h b/arangod/Aql/FixedVarExpressionContext.h index f77fc2c89333..9960be5fc4a2 100644 --- a/arangod/Aql/FixedVarExpressionContext.h +++ b/arangod/Aql/FixedVarExpressionContext.h @@ -38,8 +38,7 @@ class AqlItemBlock; class FixedVarExpressionContext final : public QueryExpressionContext { public: - explicit FixedVarExpressionContext(transaction::Methods& trx, - QueryContext& query, + explicit FixedVarExpressionContext(transaction::Methods& trx, QueryContext& query, AqlFunctionsInternalCache& cache); ~FixedVarExpressionContext() override = default; @@ -51,8 +50,14 @@ class FixedVarExpressionContext final : public QueryExpressionContext { void clearVariableValues(); + // @brief This method will set the given variable to the given AQL value + // if the variable already holds a value, this method will keep the old value. void setVariableValue(Variable const*, AqlValue const&); + // @brief This method will only clear the given variable, and keep + // all others intact. If the variable does not exist, this is a noop. + void clearVariableValue(Variable const*); + void serializeAllVariables(velocypack::Options const& opts, arangodb::velocypack::Builder&) const; diff --git a/arangod/Aql/GraphNode.cpp b/arangod/Aql/GraphNode.cpp index be76af1636b8..57daaf107a1a 100644 --- a/arangod/Aql/GraphNode.cpp +++ b/arangod/Aql/GraphNode.cpp @@ -90,7 +90,7 @@ TRI_edge_direction_e parseDirection(AstNode const* node) { return uint64ToDirection(dirNode->getIntValue()); } -} +} // namespace GraphNode::GraphNode(ExecutionPlan* plan, ExecutionNodeId id, TRI_vocbase_t* vocbase, AstNode const* direction, @@ -107,7 +107,6 @@ GraphNode::GraphNode(ExecutionPlan* plan, ExecutionNodeId id, _optionsBuilt(false), _isSmart(false), _isDisjoint(false), - _isHybrid(false), _options(std::move(options)) { // Direction is already the correct Integer. // Is not inserted by user but by enum. @@ -301,9 +300,10 @@ GraphNode::GraphNode(ExecutionPlan* plan, arangodb::velocypack::Slice const& bas _defaultDirection(uint64ToDirection(arangodb::basics::VelocyPackHelper::stringUInt64( base.get("defaultDirection")))), _optionsBuilt(false), - _isSmart(arangodb::basics::VelocyPackHelper::getBooleanValue(base, "isSmart", false)), - _isDisjoint(arangodb::basics::VelocyPackHelper::getBooleanValue(base, "isDisjoint", false)) { - + _isSmart(arangodb::basics::VelocyPackHelper::getBooleanValue(base, StaticStrings::IsSmart, + false)), + _isDisjoint(arangodb::basics::VelocyPackHelper::getBooleanValue(base, StaticStrings::IsDisjoint, + false)) { if (!ServerState::instance()->isDBServer()) { // Graph Information. Do we need to reload the graph here? std::string graphName; @@ -357,7 +357,8 @@ GraphNode::GraphNode(ExecutionPlan* plan, arangodb::velocypack::Slice const& bas auto getAqlCollectionFromName = [&](std::string const& name) -> aql::Collection& { // if the collection was already existent in the query, addCollection will // just return it. - return *query.collections().add(name, AccessMode::Type::READ, aql::Collection::Hint::Collection); + return *query.collections().add(name, AccessMode::Type::READ, + aql::Collection::Hint::Collection); }; auto vPackDirListIter = VPackArrayIterator(dirList); @@ -397,7 +398,8 @@ GraphNode::GraphNode(ExecutionPlan* plan, arangodb::velocypack::Slice const& bas "graph needs a translation from collection to shard names"); } for (auto const& item : VPackObjectIterator(collectionToShard)) { - _collectionToShard.insert({item.key.copyString(), item.value.copyString()}); + _collectionToShard.insert({item.key.copyString(), + std::vector{item.value.copyString()}}); } // Out variables @@ -510,9 +512,11 @@ std::string const& GraphNode::collectionToShardName(std::string const& collName) auto found = _collectionToShard.find(collName); if (found == _collectionToShard.end()) { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL_AQL, "unable to find shard '" + collName + "' in query shard map"); + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL_AQL, + "unable to find shard '" + collName + + "' in query shard map"); } - return found->second; + return found->second.front(); } void GraphNode::toVelocyPackHelper(VPackBuilder& nodes, unsigned flags, @@ -581,7 +585,8 @@ void GraphNode::toVelocyPackHelper(VPackBuilder& nodes, unsigned flags, { VPackObjectBuilder guard(&nodes); for (auto const& item : _collectionToShard) { - nodes.add(item.first, VPackValue(item.second)); + TRI_ASSERT(item.second.size() == 1); + nodes.add(item.first, VPackValue(item.second.front())); } } @@ -714,8 +719,8 @@ void GraphNode::enhanceEngineInfo(VPackBuilder& builder) const { } #endif -void GraphNode::addEdgeCollection(Collections const& collections, std::string const& name, - TRI_edge_direction_e dir) { +void GraphNode::addEdgeCollection(Collections const& collections, + std::string const& name, TRI_edge_direction_e dir) { auto aqlCollection = collections.get(name); if (aqlCollection != nullptr) { addEdgeCollection(*aqlCollection, dir); @@ -811,8 +816,6 @@ bool GraphNode::isSmart() const { return _isSmart; } bool GraphNode::isDisjoint() const { return _isDisjoint; } -bool GraphNode::isHybrid() const { return _isHybrid; } - TRI_vocbase_t* GraphNode::vocbase() const { return _vocbase; } Variable const* GraphNode::vertexOutVariable() const { diff --git a/arangod/Aql/GraphNode.h b/arangod/Aql/GraphNode.h index 73934c70eb96..e1abbdce0334 100644 --- a/arangod/Aql/GraphNode.h +++ b/arangod/Aql/GraphNode.h @@ -23,12 +23,12 @@ #pragma once -#include "Aql/types.h" #include "Aql/Condition.h" #include "Aql/ExecutionNode.h" #include "Aql/ExecutionNodeId.h" #include "Aql/GraphNode.h" #include "Aql/Graphs.h" +#include "Aql/types.h" #include "Cluster/ClusterTypes.h" #include "VocBase/LogicalCollection.h" #include "VocBase/voc-types.h" @@ -81,10 +81,13 @@ class GraphNode : public ExecutionNode { GraphNode(ExecutionPlan* plan, arangodb::velocypack::Slice const& base); public: + // QueryPlan decided that we use this graph as a satellite bool isUsedAsSatellite() const; + // Defines whether a GraphNode can fully be pushed down to a DBServer bool isLocalGraphNode() const; + // Will wait as soon as any of our collections is a satellite (in sync) void waitForSatelliteIfRequired(ExecutionEngine const* engine) const; - + // Can be fully pushed down to a DBServer and is available on all DBServers bool isEligibleAsSatelliteTraversal() const; protected: @@ -122,9 +125,6 @@ class GraphNode : public ExecutionNode { /// @brief flag, if the graph is a Disjoint SmartGraph (Enterprise Edition only!) bool isDisjoint() const; - /// @brief flag, if the graph is a Hybrid SmartGraph (Enterprise Edition only!) - bool isHybrid() const; - /// @brief return the database TRI_vocbase_t* vocbase() const; @@ -192,20 +192,21 @@ class GraphNode : public ExecutionNode { void injectVertexCollection(aql::Collection& other); std::vector collections() const; - void setCollectionToShard(std::map const& map) { + void setCollectionToShard(std::unordered_map> const& map) { _collectionToShard = map; } void addCollectionToShard(std::string const& coll, std::string const& shard) { // NOTE: Do not replace this by emplace or insert. // This is also used to overwrite the existing entry. - _collectionToShard[coll] = shard; + _collectionToShard[coll] = std::vector{shard}; } public: graph::Graph const* graph() const noexcept; private: - void addEdgeCollection(aql::Collections const& collections, std::string const& name, TRI_edge_direction_e dir); + void addEdgeCollection(aql::Collections const& collections, + std::string const& name, TRI_edge_direction_e dir); void addEdgeCollection(aql::Collection& collection, TRI_edge_direction_e dir); void addVertexCollection(aql::Collections const& collections, std::string const& name); void addVertexCollection(aql::Collection& collection); @@ -258,9 +259,6 @@ class GraphNode : public ExecutionNode { /// @brief flag, if graph is smart *and* disjoint (Enterprise Edition only!) bool _isDisjoint; - /// @brief flag, if graph is smart *and* hybrid (Enterprise Edition only!) - bool _isHybrid; - /// @brief The directions edges are followed std::vector _directions; @@ -271,7 +269,7 @@ class GraphNode : public ExecutionNode { std::unordered_map _engines; /// @brief list of shards involved, required for one-shard-databases - std::map _collectionToShard; + std::unordered_map> _collectionToShard; }; } // namespace aql diff --git a/arangod/Aql/KShortestPathsNode.cpp b/arangod/Aql/KShortestPathsNode.cpp index f03cd7dd96e1..478d0d40daf3 100644 --- a/arangod/Aql/KShortestPathsNode.cpp +++ b/arangod/Aql/KShortestPathsNode.cpp @@ -342,14 +342,25 @@ std::unique_ptr KShortestPathsNode::createBlock( if (shortestPathType() == arangodb::graph::ShortestPathType::Type::KPaths) { arangodb::graph::TwoSidedEnumeratorOptions enumeratorOptions{opts->minDepth, opts->maxDepth}; + PathValidatorOptions validatorOptions(opts->tmpVar(), opts->getExpressionCtx()); if (!ServerState::instance()->isCoordinator()) { // Create IndexAccessor for BaseProviderOptions (TODO: Location need to // be changed in the future) create BaseProviderOptions - BaseProviderOptions forwardProviderOptions(opts->tmpVar(), buildUsedIndexes(), + + // TODO FIXME START - only tmp workaround - we'll provide an empty depth based index info (also add an alias for large type) + std::pair, std::unordered_map>> usedIndexes{}; + usedIndexes.first = buildUsedIndexes(); + + std::pair, std::unordered_map>> reversedUsedIndexes{}; + reversedUsedIndexes.first = buildReverseUsedIndexes(); + // TODO FIXME END + + BaseProviderOptions forwardProviderOptions(opts->tmpVar(), usedIndexes, + opts->getExpressionCtx(), opts->collectionToShard()); - BaseProviderOptions backwardProviderOptions(opts->tmpVar(), - buildReverseUsedIndexes(), + BaseProviderOptions backwardProviderOptions(opts->tmpVar(), reversedUsedIndexes, + opts->getExpressionCtx(), opts->collectionToShard()); if (opts->query().queryOptions().getTraversalProfileLevel() == @@ -361,7 +372,8 @@ std::unique_ptr KShortestPathsNode::createBlock( opts->query().resourceMonitor()}, SingleServerProvider{opts->query(), backwardProviderOptions, opts->query().resourceMonitor()}, - std::move(enumeratorOptions), opts->query().resourceMonitor()); + std::move(enumeratorOptions), std::move(validatorOptions), + opts->query().resourceMonitor()); auto executorInfos = KShortestPathsExecutorInfos(outputRegister, engine.getQuery(), @@ -377,7 +389,8 @@ std::unique_ptr KShortestPathsNode::createBlock( opts->query().resourceMonitor()}, ProviderTracer{opts->query(), backwardProviderOptions, opts->query().resourceMonitor()}, - std::move(enumeratorOptions), opts->query().resourceMonitor()); + std::move(enumeratorOptions), std::move(validatorOptions), + opts->query().resourceMonitor()); auto executorInfos = KShortestPathsExecutorInfos(outputRegister, engine.getQuery(), @@ -401,7 +414,8 @@ std::unique_ptr KShortestPathsNode::createBlock( opts->query().resourceMonitor()}, ClusterProvider{opts->query(), backwardProviderOptions, opts->query().resourceMonitor()}, - std::move(enumeratorOptions), opts->query().resourceMonitor()); + std::move(enumeratorOptions), std::move(validatorOptions), + opts->query().resourceMonitor()); auto executorInfos = KShortestPathsExecutorInfos(outputRegister, engine.getQuery(), @@ -417,7 +431,8 @@ std::unique_ptr KShortestPathsNode::createBlock( opts->query().resourceMonitor()}, ProviderTracer{opts->query(), backwardProviderOptions, opts->query().resourceMonitor()}, - std::move(enumeratorOptions), opts->query().resourceMonitor()); + std::move(enumeratorOptions), std::move(validatorOptions), + opts->query().resourceMonitor()); auto executorInfos = KShortestPathsExecutorInfos(outputRegister, engine.getQuery(), @@ -475,7 +490,7 @@ void KShortestPathsNode::kShortestPathsCloneHelper(ExecutionPlan& plan, c._fromCondition = _fromCondition->clone(_plan->getAst()); c._toCondition = _toCondition->clone(_plan->getAst()); } - + void KShortestPathsNode::replaceVariables(std::unordered_map const& replacements) { if (_inStartVariable != nullptr) { _inStartVariable = Variable::replace(_inStartVariable, replacements); @@ -524,7 +539,8 @@ std::vector KShortestPathsNode::buildUsedIndexes } indexAccessors.emplace_back(indexToUse, - _toCondition->clone(options()->query().ast()), 0); + _toCondition->clone(options()->query().ast()), + 0, nullptr); break; } case TRI_EDGE_OUT: { @@ -539,7 +555,8 @@ std::vector KShortestPathsNode::buildUsedIndexes } indexAccessors.emplace_back(indexToUse, - _fromCondition->clone(options()->query().ast()), 0); + _fromCondition->clone(options()->query().ast()), + 0, nullptr); break; } case TRI_EDGE_ANY: @@ -571,7 +588,8 @@ std::vector KShortestPathsNode::buildReverseUsed } indexAccessors.emplace_back(indexToUse, - _fromCondition->clone(options()->query().ast()), 0); + _fromCondition->clone(options()->query().ast()), + 0, nullptr); break; } case TRI_EDGE_OUT: { @@ -586,7 +604,8 @@ std::vector KShortestPathsNode::buildReverseUsed } indexAccessors.emplace_back(indexToUse, - _toCondition->clone(options()->query().ast()), 0); + _toCondition->clone(options()->query().ast()), + 0, nullptr); break; } case TRI_EDGE_ANY: diff --git a/arangod/Aql/QueryExpressionContext.cpp b/arangod/Aql/QueryExpressionContext.cpp index 7f03b0ee4e7a..29b362eb6331 100644 --- a/arangod/Aql/QueryExpressionContext.cpp +++ b/arangod/Aql/QueryExpressionContext.cpp @@ -23,9 +23,9 @@ #include "QueryExpressionContext.h" +#include "Aql/AqlFunctionsInternalCache.h" #include "Aql/AqlValue.h" #include "Aql/QueryContext.h" -#include "Aql/AqlFunctionsInternalCache.h" #include "Transaction/Methods.h" using namespace arangodb; @@ -63,10 +63,8 @@ TRI_vocbase_t& QueryExpressionContext::vocbase() const { return _trx.vocbase(); } -transaction::Methods& QueryExpressionContext::trx() const { - return _trx; -} +transaction::Methods& QueryExpressionContext::trx() const { return _trx; } -bool QueryExpressionContext::killed() const { - return _query.killed(); -} +bool QueryExpressionContext::killed() const { return _query.killed(); } + +QueryContext& QueryExpressionContext::query() { return _query; } diff --git a/arangod/Aql/QueryExpressionContext.h b/arangod/Aql/QueryExpressionContext.h index a9b214c549bb..d4b06e7d8f14 100644 --- a/arangod/Aql/QueryExpressionContext.h +++ b/arangod/Aql/QueryExpressionContext.h @@ -34,11 +34,9 @@ class AqlFunctionsInternalCache; class QueryExpressionContext : public ExpressionContext { public: - explicit QueryExpressionContext(transaction::Methods& trx, - QueryContext& query, + explicit QueryExpressionContext(transaction::Methods& trx, QueryContext& query, AqlFunctionsInternalCache& cache) noexcept - : ExpressionContext(), - _trx(trx), _query(query), _aqlFunctionsInternalCache(cache) {} + : ExpressionContext(), _trx(trx), _query(query), _aqlFunctionsInternalCache(cache) {} void registerWarning(ErrorCode errorCode, char const* msg) override final; void registerError(ErrorCode errorCode, char const* msg) override final; @@ -47,7 +45,8 @@ class QueryExpressionContext : public ExpressionContext { bool caseInsensitive) override final; icu::RegexMatcher* buildLikeMatcher(char const* ptr, size_t length, bool caseInsensitive) override final; - icu::RegexMatcher* buildSplitMatcher(AqlValue splitExpression, velocypack::Options const* opts, + icu::RegexMatcher* buildSplitMatcher(AqlValue splitExpression, + velocypack::Options const* opts, bool& isEmptyExpression) override final; arangodb::ValidatorBase* buildValidator(arangodb::velocypack::Slice const&) override final; @@ -56,6 +55,7 @@ class QueryExpressionContext : public ExpressionContext { /// may be inaccessible on some platforms transaction::Methods& trx() const override final; bool killed() const override final; + QueryContext& query(); private: transaction::Methods& _trx; diff --git a/arangod/Aql/ShardLocking.cpp b/arangod/Aql/ShardLocking.cpp index 81b83c98d147..db13a958695f 100644 --- a/arangod/Aql/ShardLocking.cpp +++ b/arangod/Aql/ShardLocking.cpp @@ -63,7 +63,7 @@ void ShardLocking::addNode(ExecutionNode const* baseNode, size_t snippetId, } auto const graphIsUsedAsSatellite = graphNode->isUsedAsSatellite(); auto const isUsedAsSatellite = [&](auto const& col) { - return graphIsUsedAsSatellite || (pushToSingleServer && col->isSatellite()); + return graphIsUsedAsSatellite || (pushToSingleServer && col->isSatellite()) || (col->isSatellite() && graphNode->isSmart()); }; // Add all Edge Collections to the Transactions, Traversals do never write for (auto const& col : graphNode->edgeColls()) { @@ -75,6 +75,7 @@ void ShardLocking::addNode(ExecutionNode const* baseNode, size_t snippetId, for (auto const& col : graphNode->vertexColls()) { updateLocking(col, AccessMode::Type::READ, snippetId, {}, isUsedAsSatellite(col)); } + break; } case ExecutionNode::ENUMERATE_COLLECTION: diff --git a/arangod/CMakeLists.txt b/arangod/CMakeLists.txt index 914cdb1f0713..7e2ff53586cd 100644 --- a/arangod/CMakeLists.txt +++ b/arangod/CMakeLists.txt @@ -198,6 +198,7 @@ set(LIB_ARANGO_GRAPH_SOURCES Graph/PathManagement/PathStore.cpp Graph/PathManagement/PathStoreTracer.cpp Graph/PathManagement/PathValidator.cpp + Graph/PathManagement/PathValidatorOptions.cpp Graph/Options/OneSidedEnumeratorOptions.cpp Graph/Options/TwoSidedEnumeratorOptions.cpp Graph/Providers/ClusterProvider.cpp diff --git a/arangod/Cluster/ClusterEdgeCursor.cpp b/arangod/Cluster/ClusterEdgeCursor.cpp index 4a34d7beaab7..a255cd1dfda6 100644 --- a/arangod/Cluster/ClusterEdgeCursor.cpp +++ b/arangod/Cluster/ClusterEdgeCursor.cpp @@ -38,7 +38,7 @@ using namespace arangodb; using namespace arangodb::graph; using namespace arangodb::traverser; -ClusterEdgeCursor::ClusterEdgeCursor(graph::BaseOptions const* opts) +ClusterEdgeCursor::ClusterEdgeCursor(graph::BaseOptions const* opts) : _position(0), _opts(opts), _cache(static_cast(opts->cache())), @@ -46,8 +46,8 @@ ClusterEdgeCursor::ClusterEdgeCursor(graph::BaseOptions const* opts) TRI_ASSERT(_cache != nullptr); if (_cache == nullptr) { - THROW_ARANGO_EXCEPTION_MESSAGE( - TRI_ERROR_INTERNAL, "no cache present for cluster edge cursor"); + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, + "no cache present for cluster edge cursor"); } } @@ -66,7 +66,7 @@ void ClusterEdgeCursor::readAll(EdgeCursor::Callback const& callback) { callback(EdgeDocumentToken(edge), edge, _position); } } - + ClusterTraverserEdgeCursor::ClusterTraverserEdgeCursor(traverser::TraverserOptions const* opts) : ClusterEdgeCursor(opts) {} @@ -75,7 +75,8 @@ traverser::TraverserOptions const* ClusterTraverserEdgeCursor::traverserOptions( return dynamic_cast(_opts); } -void ClusterTraverserEdgeCursor::rearm(arangodb::velocypack::StringRef vertexId, uint64_t depth) { +void ClusterTraverserEdgeCursor::rearm(arangodb::velocypack::StringRef vertexId, + uint64_t depth) { _edgeList.clear(); _position = 0; @@ -83,18 +84,20 @@ void ClusterTraverserEdgeCursor::rearm(arangodb::velocypack::StringRef vertexId, TRI_ASSERT(trx != nullptr); TRI_ASSERT(_cache != nullptr); - Result res = fetchEdgesFromEngines(*trx, *_cache, traverserOptions()->getExpressionCtx(), vertexId, depth, _edgeList); + Result res = fetchEdgesFromEngines(*trx, *_cache, traverserOptions()->getExpressionCtx(), + vertexId, depth, _edgeList); if (res.fail()) { THROW_ARANGO_EXCEPTION(res); } _httpRequests += _cache->engines()->size(); } -ClusterShortestPathEdgeCursor::ClusterShortestPathEdgeCursor(graph::BaseOptions const* opts, bool backward) - : ClusterEdgeCursor(opts), - _backward(backward) {} +ClusterShortestPathEdgeCursor::ClusterShortestPathEdgeCursor(graph::BaseOptions const* opts, + bool backward) + : ClusterEdgeCursor(opts), _backward(backward) {} -void ClusterShortestPathEdgeCursor::rearm(arangodb::velocypack::StringRef vertexId, uint64_t /*depth*/) { +void ClusterShortestPathEdgeCursor::rearm(arangodb::velocypack::StringRef vertexId, + uint64_t /*depth*/) { _edgeList.clear(); _position = 0; @@ -102,7 +105,8 @@ void ClusterShortestPathEdgeCursor::rearm(arangodb::velocypack::StringRef vertex transaction::BuilderLeaser b(trx); b->add(VPackValuePair(vertexId.data(), vertexId.length(), VPackValueType::String)); - Result res = fetchEdgesFromEngines(*trx, *_cache, b->slice(), _backward, _edgeList, _cache->insertedDocuments()); + Result res = fetchEdgesFromEngines(*trx, *_cache, b->slice(), _backward, + _edgeList, _cache->insertedDocuments()); if (res.fail()) { THROW_ARANGO_EXCEPTION(res); } diff --git a/arangod/Cluster/TraverserEngine.cpp b/arangod/Cluster/TraverserEngine.cpp index 30fcaebd70a8..71f370a4bd33 100644 --- a/arangod/Cluster/TraverserEngine.cpp +++ b/arangod/Cluster/TraverserEngine.cpp @@ -117,7 +117,7 @@ BaseEngine::BaseEngine(TRI_vocbase_t& vocbase, TRI_ASSERT(shardList.isArray()); for (VPackSlice const shard : VPackArrayIterator(shardList)) { TRI_ASSERT(shard.isString()); - _query.collections().add(shard.copyString(), AccessMode::Type::READ, aql::Collection::Hint::Collection); + _query.collections().add(shard.copyString(), AccessMode::Type::READ, aql::Collection::Hint::Shard); } } @@ -415,6 +415,10 @@ void TraverserEngine::smartSearch(VPackSlice, VPackBuilder&) { THROW_ARANGO_EXCEPTION(TRI_ERROR_ONLY_ENTERPRISE); } +void TraverserEngine::smartSearchNew(VPackSlice, VPackBuilder&) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_ONLY_ENTERPRISE); +} + void TraverserEngine::smartSearchBFS(VPackSlice, VPackBuilder&) { THROW_ARANGO_EXCEPTION(TRI_ERROR_ONLY_ENTERPRISE); } diff --git a/arangod/Cluster/TraverserEngine.h b/arangod/Cluster/TraverserEngine.h index f5b04b9fe752..2c084ceba92a 100644 --- a/arangod/Cluster/TraverserEngine.h +++ b/arangod/Cluster/TraverserEngine.h @@ -117,6 +117,8 @@ class BaseTraverserEngine : public BaseEngine { virtual void smartSearch(arangodb::velocypack::Slice, arangodb::velocypack::Builder&) = 0; + virtual void smartSearchNew(arangodb::velocypack::Slice, arangodb::velocypack::Builder&) = 0; + virtual void smartSearchBFS(arangodb::velocypack::Slice, arangodb::velocypack::Builder&) = 0; @@ -180,6 +182,8 @@ class TraverserEngine : public BaseTraverserEngine { void smartSearch(arangodb::velocypack::Slice, arangodb::velocypack::Builder&) override; + void smartSearchNew(arangodb::velocypack::Slice, arangodb::velocypack::Builder&) override; + void smartSearchBFS(arangodb::velocypack::Slice, arangodb::velocypack::Builder&) override; void smartSearchWeighted(arangodb::velocypack::Slice, arangodb::velocypack::Builder&) override; diff --git a/arangod/Graph/BaseOptions.cpp b/arangod/Graph/BaseOptions.cpp index e8d0a9330e99..d72cec2f3155 100644 --- a/arangod/Graph/BaseOptions.cpp +++ b/arangod/Graph/BaseOptions.cpp @@ -365,7 +365,8 @@ void BaseOptions::serializeVariables(VPackBuilder& builder) const { _expressionCtx.serializeAllVariables(_query.vpackOptions(), builder); } -void BaseOptions::setCollectionToShard(std::map const& in) { +void BaseOptions::setCollectionToShard( + std::unordered_map> const& in) { _collectionToShard = std::move(in); } @@ -462,6 +463,10 @@ void BaseOptions::injectTestCache(std::unique_ptr&& testCache) { _cache = std::move(testCache); } +arangodb::aql::FixedVarExpressionContext& BaseOptions::getExpressionCtx() { + return _expressionCtx; +} + arangodb::aql::FixedVarExpressionContext const& BaseOptions::getExpressionCtx() const { return _expressionCtx; } diff --git a/arangod/Graph/BaseOptions.h b/arangod/Graph/BaseOptions.h index 1ec81522948a..1150dbb40f2e 100644 --- a/arangod/Graph/BaseOptions.h +++ b/arangod/Graph/BaseOptions.h @@ -117,7 +117,7 @@ struct BaseOptions { void serializeVariables(arangodb::velocypack::Builder&) const; - void setCollectionToShard(std::map const&); + void setCollectionToShard(std::unordered_map> const&); bool produceVertices() const { return _produceVertices; } @@ -157,7 +157,7 @@ struct BaseOptions { void activateCache(bool enableDocumentCache, std::unordered_map const* engines); - std::map const& collectionToShard() const { + std::unordered_map> const& collectionToShard() const { return _collectionToShard; } @@ -178,6 +178,8 @@ struct BaseOptions { bool refactor() const { return _refactor; } aql::Variable const* tmpVar(); // TODO check public + arangodb::aql::FixedVarExpressionContext& getExpressionCtx(); + arangodb::aql::FixedVarExpressionContext const& getExpressionCtx() const; protected: @@ -215,7 +217,7 @@ struct BaseOptions { std::unique_ptr _cache; // @brief - translations for one-shard-databases - std::map _collectionToShard; + std::unordered_map> _collectionToShard; /// @brief a value of 1 (which is the default) means "no parallelism" size_t _parallelism; diff --git a/arangod/Graph/Cache/RefactoredTraverserCache.cpp b/arangod/Graph/Cache/RefactoredTraverserCache.cpp index d8eb1f3ee436..0a47ad38d89a 100644 --- a/arangod/Graph/Cache/RefactoredTraverserCache.cpp +++ b/arangod/Graph/Cache/RefactoredTraverserCache.cpp @@ -53,8 +53,7 @@ using namespace arangodb::graph; namespace { bool isWithClauseMissing(arangodb::basics::Exception const& ex) { - if (ServerState::instance()->isDBServer() && - ex.code() == TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND) { + if (ServerState::instance()->isDBServer() && ex.code() == TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND) { // on a DB server, we could have got here only in the OneShard case. // in this case turn the rather misleading "collection or view not found" // error into a nicer "collection not known to traversal, please add WITH" @@ -68,19 +67,20 @@ bool isWithClauseMissing(arangodb::basics::Exception const& ex) { return false; } -} // namespace +} // namespace RefactoredTraverserCache::RefactoredTraverserCache( arangodb::transaction::Methods* trx, aql::QueryContext* query, arangodb::ResourceMonitor& resourceMonitor, arangodb::aql::TraversalStats& stats, - std::map const& collectionToShardMap) + std::unordered_map> const& collectionToShardMap) : _query(query), _trx(trx), _stringHeap(resourceMonitor, 4096), /* arbitrary block-size may be adjusted for performance */ _collectionToShardMap(collectionToShardMap), _resourceMonitor(resourceMonitor), - _allowImplicitCollections(ServerState::instance()->isSingleServer() && - !_query->vocbase().server().getFeature().requireWith()) { + _allowImplicitCollections( + ServerState::instance()->isSingleServer() && + !_query->vocbase().server().getFeature().requireWith()) { TRI_ASSERT(!ServerState::instance()->isCoordinator()); } @@ -106,17 +106,20 @@ bool RefactoredTraverserCache::appendEdge(EdgeDocumentToken const& idToken, return false; } - auto res = col->getPhysical()->read( - _trx, idToken.localDocumentId(), [&](LocalDocumentId const&, VPackSlice edge) -> bool { - // NOTE: Do not count this as Primary Index Scan, we counted it in the - // edge Index before copying... - if constexpr (std::is_same_v) { - result = aql::AqlValue(edge); - } else if constexpr (std::is_same_v) { - result.add(edge); - } - return true; - }).ok(); + auto res = + col->getPhysical() + ->read(_trx, idToken.localDocumentId(), + [&](LocalDocumentId const&, VPackSlice edge) -> bool { + // NOTE: Do not count this as Primary Index Scan, we counted + // it in the edge Index before copying... + if constexpr (std::is_same_v) { + result = aql::AqlValue(edge); + } else if constexpr (std::is_same_v) { + result.add(edge); + } + return true; + }) + .ok(); if (ADB_UNLIKELY(!res)) { // We already had this token, inconsistent state. Return NULL in Production LOG_TOPIC("daac5", ERR, arangodb::Logger::GRAPHS) @@ -139,20 +142,7 @@ ResultT> RefactoredTraverserCache::extractCollect } std::string colName = idHashed.substr(0, pos).toString(); - if (_collectionToShardMap.empty()) { - TRI_ASSERT(!ServerState::instance()->isDBServer()); - return std::make_pair(colName, pos); - } - auto it = _collectionToShardMap.find(colName); - if (it == _collectionToShardMap.end()) { - // Connected to a vertex where we do not know the Shard to. - return Result{TRI_ERROR_QUERY_COLLECTION_LOCK_FAILED, - "collection not known to traversal: '" + colName + - "'. please add 'WITH " + colName + - "' as the first line in your AQL"}; - } - // We have translated to a Shard - return std::make_pair(it->second, pos); + return std::make_pair(colName, pos); } template @@ -163,43 +153,68 @@ bool RefactoredTraverserCache::appendVertex(aql::TraversalStats& stats, if (collectionNameResult.fail()) { THROW_ARANGO_EXCEPTION(collectionNameResult.result()); } - - std::string const& collectionName = collectionNameResult.get().first; - try { - transaction::AllowImplicitCollectionsSwitcher disallower(_trx->state()->options(), _allowImplicitCollections); - - Result res = _trx->documentFastPathLocal( - collectionName, - id.substr(collectionNameResult.get().second + 1).stringRef(), - [&](LocalDocumentId const&, VPackSlice doc) -> bool { - stats.addScannedIndex(1); - // copying... - if constexpr (std::is_same_v) { - result = aql::AqlValue(doc); - } else if constexpr (std::is_same_v) { - result.add(doc); - } - return true; - }); - if (res.ok()) { - return true; + auto findDocumentInShard = [&](std::string const& collectionName) -> bool { + try { + transaction::AllowImplicitCollectionsSwitcher disallower(_trx->state()->options(), + _allowImplicitCollections); + + Result res = _trx->documentFastPathLocal( + collectionName, id.substr(collectionNameResult.get().second + 1).stringRef(), + [&](LocalDocumentId const&, VPackSlice doc) -> bool { + stats.addScannedIndex(1); + // copying... + if constexpr (std::is_same_v) { + result = aql::AqlValue(doc); + } else if constexpr (std::is_same_v) { + result.add(doc); + } + return true; + }); + if (res.ok()) { + return true; + } + + if (!res.is(TRI_ERROR_ARANGO_DOCUMENT_NOT_FOUND)) { + // ok we are in a rather bad state. Better throw and abort. + THROW_ARANGO_EXCEPTION(res); + } + } catch (basics::Exception const& ex) { + if (isWithClauseMissing(ex)) { + // turn the error into a different error + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_COLLECTION_LOCK_FAILED, + "collection not known to traversal: '" + collectionName + + "'. please add 'WITH " + collectionName + + "' as the first line in your AQL"); + } + // rethrow original error + throw; } + return false; + }; - if (!res.is(TRI_ERROR_ARANGO_DOCUMENT_NOT_FOUND)) { - // ok we are in a rather bad state. Better throw and abort. - THROW_ARANGO_EXCEPTION(res); + std::string const& collectionName = collectionNameResult.get().first; + if (_collectionToShardMap.empty()) { + TRI_ASSERT(!ServerState::instance()->isDBServer()); + if (findDocumentInShard(collectionName)) { + return true; } - } catch (basics::Exception const& ex) { - if (isWithClauseMissing(ex)) { - // turn the error into a different error + } else { + auto it = _collectionToShardMap.find(collectionName); + if (it == _collectionToShardMap.end()) { + // Connected to a vertex where we do not know the Shard to. THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_COLLECTION_LOCK_FAILED, - "collection not known to traversal: '" + - collectionName + "'. please add 'WITH " + collectionName + - "' as the first line in your AQL"); + "collection not known to traversal: '" + collectionName + + "'. please add 'WITH " + collectionName + + "' as the first line in your AQL"); + } + for (auto const& shard : it->second) { + if (findDocumentInShard(shard)) { + // Short circuit, as soon as one shard contains this document + // we can return it. + return true; + } } - // rethrow original error - throw; } // Register a warning. It is okay though but helps the user diff --git a/arangod/Graph/Cache/RefactoredTraverserCache.h b/arangod/Graph/Cache/RefactoredTraverserCache.h index 495fad9509da..ca4a9f688a2d 100644 --- a/arangod/Graph/Cache/RefactoredTraverserCache.h +++ b/arangod/Graph/Cache/RefactoredTraverserCache.h @@ -62,11 +62,10 @@ struct EdgeDocumentToken; class RefactoredTraverserCache { public: - explicit RefactoredTraverserCache(arangodb::transaction::Methods* trx, - aql::QueryContext* query, - arangodb::ResourceMonitor& resourceMonitor, - arangodb::aql::TraversalStats& stats, - std::map const& collectionToShardMap); + explicit RefactoredTraverserCache( + arangodb::transaction::Methods* trx, aql::QueryContext* query, + arangodb::ResourceMonitor& resourceMonitor, arangodb::aql::TraversalStats& stats, + std::unordered_map> const& collectionToShardMap); ~RefactoredTraverserCache(); RefactoredTraverserCache(RefactoredTraverserCache const&) = delete; @@ -81,8 +80,7 @@ class RefactoredTraverserCache { /// @brief Inserts the real document stored within the token /// into the given builder. ////////////////////////////////////////////////////////////////////////////// - void insertEdgeIntoResult(graph::EdgeDocumentToken const& etkn, - velocypack::Builder& builder); + void insertEdgeIntoResult(graph::EdgeDocumentToken const& etkn, velocypack::Builder& builder); ////////////////////////////////////////////////////////////////////////////// /// @brief Inserts the real document identified by the _id string @@ -148,9 +146,9 @@ class RefactoredTraverserCache { std::unordered_set _persistedStrings; private: - std::map const& _collectionToShardMap; + std::unordered_map> const& _collectionToShardMap; arangodb::ResourceMonitor& _resourceMonitor; - + /// @brief whether or not to allow adding of previously unknown collections /// during the traversal bool const _allowImplicitCollections; diff --git a/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.cpp b/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.cpp index 31015a005ef9..930cd63ee9a5 100644 --- a/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.cpp +++ b/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.cpp @@ -42,10 +42,11 @@ namespace { IndexIteratorOptions defaultIndexIteratorOptions; } -RefactoredSingleServerEdgeCursor::LookupInfo::LookupInfo(transaction::Methods::IndexHandle idx, - aql::AstNode* condition, - std::optional memberToUpdate) +RefactoredSingleServerEdgeCursor::LookupInfo::LookupInfo( + transaction::Methods::IndexHandle idx, aql::AstNode* condition, + std::optional memberToUpdate, aql::Expression* expression) : _idxHandle(std::move(idx)), + _expression(expression), _indexCondition(condition), _cursor(nullptr), _conditionMemberToUpdate(memberToUpdate) {} @@ -58,6 +59,10 @@ RefactoredSingleServerEdgeCursor::LookupInfo::LookupInfo(LookupInfo&& other) noe RefactoredSingleServerEdgeCursor::LookupInfo::~LookupInfo() = default; +aql::Expression* RefactoredSingleServerEdgeCursor::LookupInfo::getExpression() { + return _expression; +} + void RefactoredSingleServerEdgeCursor::LookupInfo::rearmVertex( VertexType vertex, transaction::Methods* trx, arangodb::aql::Variable const* tmpVar) { auto& node = _indexCondition; @@ -74,6 +79,25 @@ void RefactoredSingleServerEdgeCursor::LookupInfo::rearmVertex( TRI_ASSERT(idNode->isValueType(aql::VALUE_TYPE_STRING)); // must edit node in place; TODO replace node? // TODO i think there is now a mutable String node available + TEMPORARILY_UNLOCK_NODE(idNode); + idNode->setStringValue(vertex.data(), vertex.length()); + } else { + // If we have to inject the vertex value it has to be within + // the last member of the condition. + // We only get into this case iff the index used does + // not cover _from resp. _to. + // inject _from/_to value + auto expressionNode = _expression->nodeForModification(); + + TRI_ASSERT(expressionNode->numMembers() > 0); + auto dirCmp = expressionNode->getMemberUnchecked(expressionNode->numMembers() - 1); + TRI_ASSERT(dirCmp->type == aql::NODE_TYPE_OPERATOR_BINARY_EQ); + TRI_ASSERT(dirCmp->numMembers() == 2); + + auto idNode = dirCmp->getMemberUnchecked(1); + TRI_ASSERT(idNode->type == aql::NODE_TYPE_VALUE); + TRI_ASSERT(idNode->isValueType(aql::VALUE_TYPE_STRING)); + TEMPORARILY_UNLOCK_NODE(idNode); idNode->setStringValue(vertex.data(), vertex.length()); } @@ -100,15 +124,28 @@ IndexIterator& RefactoredSingleServerEdgeCursor::LookupInfo::cursor() { } RefactoredSingleServerEdgeCursor::RefactoredSingleServerEdgeCursor( - arangodb::transaction::Methods* trx, - arangodb::aql::Variable const* tmpVar, std::vector const& indexConditions) - : _tmpVar(tmpVar), _currentCursor(0), _trx(trx) { + transaction::Methods* trx, arangodb::aql::Variable const* tmpVar, + std::vector const& globalIndexConditions, + std::unordered_map> const& depthBasedIndexConditions, + arangodb::aql::FixedVarExpressionContext& expressionContext) + : _tmpVar(tmpVar), _currentCursor(0), _trx(trx), _expressionCtx(expressionContext) { // We need at least one indexCondition, otherwise nothing to serve - TRI_ASSERT(!indexConditions.empty()); - _lookupInfo.reserve(indexConditions.size()); - for (auto const& idxCond : indexConditions) { + TRI_ASSERT(!globalIndexConditions.empty()); + _lookupInfo.reserve(globalIndexConditions.size()); + _depthLookupInfo.reserve(depthBasedIndexConditions.size()); + for (auto const& idxCond : globalIndexConditions) { _lookupInfo.emplace_back(idxCond.indexHandle(), idxCond.getCondition(), - idxCond.getMemberToUpdate()); + idxCond.getMemberToUpdate(), idxCond.getExpression()); + } + for (auto const& obj : depthBasedIndexConditions) { + auto& [depth, idxCondArray] = obj; + + std::vector tmpLookupVec; + for (auto const& idxCond : idxCondArray) { + tmpLookupVec.emplace_back(idxCond.indexHandle(), idxCond.getCondition(), + idxCond.getMemberToUpdate(), idxCond.getExpression()); + } + _depthLookupInfo.try_emplace(depth, std::move(tmpLookupVec)); } } @@ -133,9 +170,22 @@ void RefactoredSingleServerEdgeCursor::rearm(VertexType vertex, uint64_t /*depth } } -void RefactoredSingleServerEdgeCursor::readAll(aql::TraversalStats& stats, +void RefactoredSingleServerEdgeCursor::readAll(SingleServerProvider& provider, + aql::TraversalStats& stats, size_t depth, Callback const& callback) { TRI_ASSERT(!_lookupInfo.empty()); + VPackBuilder tmpBuilder; + + auto handleExpression = [&](aql::Expression* expression, + EdgeDocumentToken edgeToken, VPackSlice edge) { + if (edge.isString()) { + tmpBuilder.clear(); + provider.insertEdgeIntoResult(edgeToken, tmpBuilder); + edge = tmpBuilder.slice(); + } + return evaluateExpression(expression, edge); + }; + for (_currentCursor = 0; _currentCursor < _lookupInfo.size(); ++_currentCursor) { auto& cursor = _lookupInfo[_currentCursor].cursor(); LogicalCollection* collection = cursor.collection(); @@ -148,31 +198,85 @@ void RefactoredSingleServerEdgeCursor::readAll(aql::TraversalStats& stats, return false; } #endif - callback(EdgeDocumentToken(cid, token), edge, _currentCursor); + + EdgeDocumentToken edgeToken(cid, token); + // eval depth-based expression first if available + bool foundDepthInfo = + (_depthLookupInfo.find(depth) == _depthLookupInfo.end()) ? false : true; + if (foundDepthInfo && _depthLookupInfo.at(depth).size() > 0 && + _depthLookupInfo.at(depth)[_currentCursor].getExpression() != nullptr) { + if (!handleExpression(_depthLookupInfo.at(depth)[_currentCursor].getExpression(), + edgeToken, edge)) { + stats.incrFiltered(); + return false; + } + } else { + // eval global expression if available AND only if no depth specific expression needs to be handled before + if (_lookupInfo[_currentCursor].getExpression() != nullptr) { + if (!handleExpression(_lookupInfo[_currentCursor].getExpression(), + edgeToken, edge)) { + stats.incrFiltered(); + return false; + } + } + } + + callback(std::move(edgeToken), edge, _currentCursor); return true; }); } else { cursor.all([&](LocalDocumentId const& token) { - return collection->getPhysical()->read(_trx, token, [&](LocalDocumentId const&, VPackSlice edgeDoc) { - stats.addScannedIndex(1); + return collection->getPhysical() + ->read(_trx, token, + [&](LocalDocumentId const&, VPackSlice edgeDoc) { + stats.addScannedIndex(1); #ifdef USE_ENTERPRISE - if (_trx->skipInaccessible()) { - // TODO: we only need to check one of these - VPackSlice from = transaction::helpers::extractFromFromDocument(edgeDoc); - VPackSlice to = transaction::helpers::extractToFromDocument(edgeDoc); - if (CheckInaccessible(_trx, from) || CheckInaccessible(_trx, to)) { - return false; - } - } + if (_trx->skipInaccessible()) { + // TODO: we only need to check one of these + VPackSlice from = + transaction::helpers::extractFromFromDocument(edgeDoc); + VPackSlice to = + transaction::helpers::extractToFromDocument(edgeDoc); + if (CheckInaccessible(_trx, from) || CheckInaccessible(_trx, to)) { + return false; + } + } #endif - callback(EdgeDocumentToken(cid, token), edgeDoc, _currentCursor); - return true; - }).ok(); + // eval expression if available + if (_lookupInfo[_currentCursor].getExpression() != nullptr) { + bool result = + evaluateExpression(_lookupInfo[_currentCursor].getExpression(), + edgeDoc); + if (!result) { + stats.incrFiltered(); + return false; + } + } + callback(EdgeDocumentToken(cid, token), edgeDoc, _currentCursor); + return true; + }) + .ok(); }); } } } -arangodb::transaction::Methods* RefactoredSingleServerEdgeCursor::trx() const { - return _trx; +bool RefactoredSingleServerEdgeCursor::evaluateExpression(arangodb::aql::Expression* expression, + VPackSlice value) { + if (expression == nullptr) { + return true; + } + + TRI_ASSERT(value.isObject() || value.isNull()); + + aql::AqlValue edgeVal(aql::AqlValueHintDocumentNoCopy(value.begin())); + _expressionCtx.setVariableValue(_tmpVar, edgeVal); + TRI_DEFER(_expressionCtx.clearVariableValue(_tmpVar)); + + bool mustDestroy = false; + aql::AqlValue res = expression->execute(&_expressionCtx, mustDestroy); + aql::AqlValueGuard guard(res, mustDestroy); + TRI_ASSERT(res.isBoolean()); + + return res.toBoolean(); } diff --git a/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.h b/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.h index 6335bee3a6c6..249d6c0d317e 100644 --- a/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.h +++ b/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.h @@ -24,7 +24,9 @@ #pragma once +#include "Aql/AqlFunctionsInternalCache.h" #include "Aql/Expression.h" +#include "Aql/FixedVarExpressionContext.h" #include "Aql/QueryContext.h" #include "Indexes/IndexIterator.h" #include "Transaction/Methods.h" @@ -51,11 +53,13 @@ struct IndexAccessor; struct EdgeDocumentToken; +struct SingleServerProvider; + class RefactoredSingleServerEdgeCursor { public: struct LookupInfo { LookupInfo(transaction::Methods::IndexHandle idx, aql::AstNode* condition, - std::optional memberToUpdate); + std::optional memberToUpdate, aql::Expression* expression); ~LookupInfo(); LookupInfo(LookupInfo const&) = delete; @@ -66,12 +70,12 @@ class RefactoredSingleServerEdgeCursor { arangodb::aql::Variable const* tmpVar); IndexIterator& cursor(); + aql::Expression* getExpression(); private: - // This struct does only take responsibility for the expression // NOTE: The expression can be nullptr! transaction::Methods::IndexHandle _idxHandle; - std::unique_ptr _expression; + aql::Expression* _expression; aql::AstNode* _indexCondition; std::unique_ptr _cursor; @@ -83,9 +87,11 @@ class RefactoredSingleServerEdgeCursor { enum Direction { FORWARD, BACKWARD }; public: - RefactoredSingleServerEdgeCursor(arangodb::transaction::Methods* trx, - arangodb::aql::Variable const* tmpVar, - std::vector const& indexConditions); + RefactoredSingleServerEdgeCursor( + transaction::Methods* trx, arangodb::aql::Variable const* tmpVar, + std::vector const& globalIndexConditions, + std::unordered_map> const& depthBasedIndexConditions, + arangodb::aql::FixedVarExpressionContext& expressionContext); ~RefactoredSingleServerEdgeCursor(); using Callback = @@ -95,17 +101,18 @@ class RefactoredSingleServerEdgeCursor { aql::Variable const* _tmpVar; size_t _currentCursor; std::vector _lookupInfo; + std::unordered_map> _depthLookupInfo; - arangodb::transaction::Methods* _trx; + transaction::Methods* _trx; + arangodb::aql::FixedVarExpressionContext& _expressionCtx; public: - void readAll(aql::TraversalStats& stats, Callback const& callback); + void readAll(SingleServerProvider& provider, aql::TraversalStats& stats, + size_t depth, Callback const& callback); void rearm(VertexType vertex, uint64_t depth); - private: - [[nodiscard]] transaction::Methods* trx() const; + bool evaluateExpression(arangodb::aql::Expression* expression, VPackSlice value); }; } // namespace graph } // namespace arangodb - diff --git a/arangod/Graph/EdgeDocumentToken.h b/arangod/Graph/EdgeDocumentToken.h index abeff524e397..8935cfdffc4c 100644 --- a/arangod/Graph/EdgeDocumentToken.h +++ b/arangod/Graph/EdgeDocumentToken.h @@ -118,12 +118,15 @@ struct EdgeDocumentToken { #ifdef ARANGODB_ENABLE_MAINTAINER_MODE TRI_ASSERT(_type == TokenType::COORDINATOR); #endif - return velocypack::Slice(_data.vpack).binaryEquals(velocypack::Slice(other._data.vpack)); + return velocypack::Slice(_data.vpack) + .binaryEquals(velocypack::Slice(other._data.vpack)); } bool equalsLocal(EdgeDocumentToken const& other) const { #ifdef ARANGODB_ENABLE_MAINTAINER_MODE - TRI_ASSERT(_type == TokenType::LOCAL); + // For local the cid and localDocumentId have illegal values on NONE + // and can be compared with real values + TRI_ASSERT(_type == TokenType::LOCAL || _type == TokenType::NONE); #endif return _data.document.cid == other.cid() && _data.document.localDocumentId == other.localDocumentId(); @@ -136,6 +139,14 @@ struct EdgeDocumentToken { return equalsLocal(other); } + bool isValid() const { + if (ServerState::instance()->isCoordinator()) { + return _data.vpack != nullptr; + } + return _data.document.cid != DataSourceId::none() && + _data.document.localDocumentId != LocalDocumentId::none(); + } + size_t hash() const { if (ServerState::instance()->isCoordinator()) { auto vslice = arangodb::velocypack::Slice(vpack()); diff --git a/arangod/Graph/Enumerators/OneSidedEnumerator.cpp b/arangod/Graph/Enumerators/OneSidedEnumerator.cpp index 7febe88af686..ff6e67361f86 100644 --- a/arangod/Graph/Enumerators/OneSidedEnumerator.cpp +++ b/arangod/Graph/Enumerators/OneSidedEnumerator.cpp @@ -36,7 +36,9 @@ #include "Graph/Providers/ProviderTracer.h" #include "Graph/Providers/SingleServerProvider.h" #include "Graph/Queues/FifoQueue.h" +#include "Graph/Queues/LifoQueue.h" #include "Graph/Queues/QueueTracer.h" +#include "Graph/algorithm-aliases.h" #include #include @@ -46,37 +48,34 @@ using namespace arangodb; using namespace arangodb::graph; -template -OneSidedEnumerator::OneSidedEnumerator( - ProviderType&& forwardProvider, OneSidedEnumeratorOptions&& options, - arangodb::ResourceMonitor& resourceMonitor) +template +OneSidedEnumerator::OneSidedEnumerator(Provider&& forwardProvider, + OneSidedEnumeratorOptions&& options, + PathValidatorOptions validatorOptions, + arangodb::ResourceMonitor& resourceMonitor) : _options(std::move(options)), _queue(resourceMonitor), _provider(std::move(forwardProvider)), - _validator(_interior), - _interior(resourceMonitor), - _resultPath{_provider} {} + _validator(_provider, _interior, std::move(validatorOptions)), + _interior(resourceMonitor) {} -template -OneSidedEnumerator::~OneSidedEnumerator() { -} +template +OneSidedEnumerator::~OneSidedEnumerator() = default; -template -auto OneSidedEnumerator::destroyEngines() - -> void { +template +auto OneSidedEnumerator::destroyEngines() -> void { _provider.destroyEngines(); } -template -void OneSidedEnumerator::clear() { +template +void OneSidedEnumerator::clear() { _interior.reset(); _queue.clear(); _results.clear(); } -template -auto OneSidedEnumerator::computeNeighbourhoodOfNextVertex() - -> void { +template +auto OneSidedEnumerator::computeNeighbourhoodOfNextVertex() -> void { // Pull next element from Queue // Do 1 step search TRI_ASSERT(!_queue.isEmpty()); @@ -91,13 +90,31 @@ auto OneSidedEnumerator:: TRI_ASSERT(_queue.hasProcessableElement()); } - auto step = _queue.pop(); - auto posPrevious = _interior.append(step); - + auto tmp = _queue.pop(); + auto posPrevious = _interior.append(std::move(tmp)); + auto& step = _interior.getStepReference(posPrevious); + + // only explore here if we're responsible + if (!step.isResponsible(_provider.trx())) { + // This server cannot decide on this specific vertex. + // Include it in results, to report back that we + // found this undecided path + _results.push_back(&step); + return; + } ValidationResult res = _validator.validatePath(step); - if ((step.getDepth() >= _options.getMinDepth()) && !res.isFiltered()) { + + // TODO: Adjust log output + LOG_TOPIC("78155", TRACE, Logger::GRAPHS) + << std::boolalpha << " Validated Vertex: " << step.getVertex().getID() + << " filtered " << res.isFiltered() << " pruned " << res.isPruned() + << " depth " << _options.getMinDepth() << " <= " << step.getDepth() + << "<= " << _options.getMaxDepth(); + if (step.getDepth() >= _options.getMinDepth() && !res.isFiltered()) { // Include it in results. - _results.push_back(step); + _results.push_back(&step); + } else { + _stats.incrFiltered(); } if (step.getDepth() < _options.getMaxDepth() && !res.isPruned()) { @@ -111,8 +128,8 @@ auto OneSidedEnumerator:: * @return true There will be no further path. * @return false There is a chance that there is more data available. */ -template -bool OneSidedEnumerator::isDone() const { +template +bool OneSidedEnumerator::isDone() const { return _results.empty() && searchDone(); } @@ -125,10 +142,10 @@ bool OneSidedEnumerator:: * * @param source The source vertex to start the paths */ -template -void OneSidedEnumerator::reset(VertexRef source) { +template +void OneSidedEnumerator::reset(VertexRef source, size_t depth) { clear(); - auto firstStep = _provider.startVertex(source); + auto firstStep = _provider.startVertex(source, depth); _queue.append(std::move(firstStep)); } @@ -145,32 +162,23 @@ void OneSidedEnumerator:: * @return true Found and written a path, result is modified. * @return false No path found, result has not been changed. */ -template -bool OneSidedEnumerator::getNextPath(VPackBuilder& result) { +template +auto OneSidedEnumerator::getNextPath() + -> std::unique_ptr { while (!isDone()) { searchMoreResults(); while (!_results.empty()) { - auto const& vertex = _results.back(); - - // Performance Optimization: - // It seems to be pointless to first push - // everything in to the _resultPath object - // and then iterate again to return the path - // we should be able to return the path in the first go. - _resultPath.clear(); - _interior.buildPath(vertex, _resultPath); - TRI_ASSERT(!_resultPath.isEmpty()); + auto* step = _results.back(); _results.pop_back(); - _resultPath.toVelocyPack(result); - return true; + return std::make_unique(step, _provider, _interior); } } - return false; + return nullptr; } -template -void OneSidedEnumerator::searchMoreResults() { +template +void OneSidedEnumerator::searchMoreResults() { while (_results.empty() && !searchDone()) { // TODO: check && !_queue.isEmpty() _resultsFetched = false; computeNeighbourhoodOfNextVertex(); @@ -186,8 +194,8 @@ void OneSidedEnumerator:: * @return false No path found. */ -template -bool OneSidedEnumerator::skipPath() { +template +bool OneSidedEnumerator::skipPath() { while (!isDone()) { searchMoreResults(); @@ -200,21 +208,19 @@ bool OneSidedEnumerator:: return false; } -template -auto OneSidedEnumerator::searchDone() const - -> bool { +template +auto OneSidedEnumerator::searchDone() const -> bool { return _queue.isEmpty(); } -template -auto OneSidedEnumerator::fetchResults() - -> void { +template +auto OneSidedEnumerator::fetchResults() -> void { if (!_resultsFetched && !_results.empty()) { std::vector looseEnds{}; - for (auto& vertex : _results) { - if (!vertex.isProcessable()) { - looseEnds.emplace_back(&vertex); + for (auto* vertex : _results) { + if (!vertex->isProcessable()) { + looseEnds.emplace_back(vertex); } } @@ -235,22 +241,31 @@ auto OneSidedEnumerator:: _resultsFetched = true; } -template -auto OneSidedEnumerator::stealStats() - -> aql::TraversalStats { - aql::TraversalStats stats = _provider.stealStats(); - return stats; +template +auto OneSidedEnumerator::stealStats() -> aql::TraversalStats { + _stats += _provider.stealStats(); + + auto t = _stats; + // Placement new of stats, do not reallocate space. + _stats.~TraversalStats(); + new (&_stats) aql::TraversalStats{}; + return t; } /* SingleServerProvider Section */ -template class ::arangodb::graph::OneSidedEnumerator< - ::arangodb::graph::FifoQueue<::arangodb::graph::SingleServerProvider::Step>, - ::arangodb::graph::PathStore, SingleServerProvider, - ::arangodb::graph::PathValidator, VertexUniquenessLevel::PATH>>; - -template class ::arangodb::graph::OneSidedEnumerator< - ::arangodb::graph::QueueTracer<::arangodb::graph::FifoQueue<::arangodb::graph::SingleServerProvider::Step>>, - ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore>, - ::arangodb::graph::ProviderTracer, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore>, VertexUniquenessLevel::PATH>>; +// Breadth First Search +template class ::arangodb::graph::OneSidedEnumerator>; +template class ::arangodb::graph::OneSidedEnumerator>; +template class ::arangodb::graph::OneSidedEnumerator>; +template class ::arangodb::graph::OneSidedEnumerator>; +template class ::arangodb::graph::OneSidedEnumerator>; +template class ::arangodb::graph::OneSidedEnumerator>; + +// Depth First Search +template class ::arangodb::graph::OneSidedEnumerator>; +template class ::arangodb::graph::OneSidedEnumerator>; +template class ::arangodb::graph::OneSidedEnumerator>; +template class ::arangodb::graph::OneSidedEnumerator>; +template class ::arangodb::graph::OneSidedEnumerator>; +template class ::arangodb::graph::OneSidedEnumerator>; diff --git a/arangod/Graph/Enumerators/OneSidedEnumerator.h b/arangod/Graph/Enumerators/OneSidedEnumerator.h index 465dea72c907..8e821fca2a1a 100644 --- a/arangod/Graph/Enumerators/OneSidedEnumerator.h +++ b/arangod/Graph/Enumerators/OneSidedEnumerator.h @@ -28,6 +28,8 @@ #include "Basics/ResourceUsage.h" +#include "Aql/TraversalStats.h" +#include "Graph/Enumerators/OneSidedEnumeratorInterface.h" #include "Graph/Options/OneSidedEnumeratorOptions.h" #include "Graph/PathManagement/SingleProviderPathResult.h" #include "Transaction/Methods.h" @@ -36,10 +38,6 @@ namespace arangodb { -namespace aql { -class TraversalStats; -} - namespace velocypack { class Builder; class HashedStringRef; @@ -48,31 +46,33 @@ class HashedStringRef; namespace graph { struct OneSidedEnumeratorOptions; +class PathValidatorOptions; -template -class PathResult; - -template -class OneSidedEnumerator { +template +class OneSidedEnumerator : public TraversalEnumerator { public: - using Step = typename ProviderType::Step; // public due to tracer access + using Step = typename Configuration::Step; // public due to tracer access + using Provider = typename Configuration::Provider; + using Store = typename Configuration::Store; + + using ResultPathType = SingleProviderPathResult; private: using VertexRef = arangodb::velocypack::HashedStringRef; - using Shell = std::multiset; - using ResultList = std::vector; + using ResultList = std::vector; using GraphOptions = arangodb::graph::OneSidedEnumeratorOptions; public: - OneSidedEnumerator(ProviderType&& forwardProvider, OneSidedEnumeratorOptions&& options, + OneSidedEnumerator(Provider&& provider, OneSidedEnumeratorOptions&& options, + PathValidatorOptions pathValidatorOptions, arangodb::ResourceMonitor& resourceMonitor); OneSidedEnumerator(OneSidedEnumerator const& other) = delete; OneSidedEnumerator(OneSidedEnumerator&& other) noexcept = default; ~OneSidedEnumerator(); - void clear(); + void clear() override; /** * @brief Quick test if the finder can prove there is no more data available. @@ -80,7 +80,7 @@ class OneSidedEnumerator { * @return true There will be no further path. * @return false There is a chance that there is more data available. */ - [[nodiscard]] bool isDone() const; + [[nodiscard]] bool isDone() const override; /** * @brief Reset to new source and target vertices. @@ -90,8 +90,9 @@ class OneSidedEnumerator { * call of reset. * * @param source The source vertex to start the paths + * @param depth The depth we're starting the search at */ - void reset(VertexRef source); + void reset(VertexRef source, size_t depth = 0) override; /** * @brief Get the next path, if available written into the result build. @@ -106,7 +107,7 @@ class OneSidedEnumerator { * @return true Found and written a path, result is modified. * @return false No path found, result has not been changed. */ - bool getNextPath(arangodb::velocypack::Builder& result); + auto getNextPath() -> std::unique_ptr override; /** * @brief Skip the next Path, like getNextPath, but does not return the path. @@ -115,14 +116,14 @@ class OneSidedEnumerator { * @return false No path found. */ - bool skipPath(); - auto destroyEngines() -> void; + bool skipPath() override; + auto destroyEngines() -> void override; /** * @brief Return statistics generated since * the last time this method was called. */ - auto stealStats() -> aql::TraversalStats; + auto stealStats() -> aql::TraversalStats override; private: [[nodiscard]] auto searchDone() const -> bool; @@ -143,16 +144,15 @@ class OneSidedEnumerator { GraphOptions _options; ResultList _results{}; bool _resultsFetched{false}; + aql::TraversalStats _stats{}; // The next elements to process - QueueType _queue; - ProviderType _provider; - PathValidatorType _validator; + typename Configuration::Queue _queue; + typename Configuration::Provider _provider; + typename Configuration::Validator _validator; // This stores all paths processed - PathStoreType _interior; - - SingleProviderPathResult _resultPath; + typename Configuration::Store _interior; }; } // namespace graph } // namespace arangodb diff --git a/arangod/Graph/Enumerators/OneSidedEnumeratorInterface.h b/arangod/Graph/Enumerators/OneSidedEnumeratorInterface.h new file mode 100644 index 000000000000..7cd217066d62 --- /dev/null +++ b/arangod/Graph/Enumerators/OneSidedEnumeratorInterface.h @@ -0,0 +1,74 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2021 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +/// @author Heiko Kernbach +//////////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include +#include +#include + +namespace arangodb { + +namespace velocypack { +class HashedStringRef; +class Builder; +} // namespace velocypack + +namespace aql { +class TraversalStats; +} + +namespace graph { + +// TODO Temporary, just Make everything compile +class PathResultInterface { + public: + PathResultInterface() {} + virtual ~PathResultInterface() {} + + virtual auto toVelocyPack(arangodb::velocypack::Builder& builder) -> void = 0; + virtual auto toSchreierEntry(arangodb::velocypack::Builder& builder, + size_t& currentLength) + -> void = 0; +}; + +class TraversalEnumerator { + public: + using VertexRef = arangodb::velocypack::HashedStringRef; + TraversalEnumerator(){}; + virtual ~TraversalEnumerator() {} + + virtual void clear() = 0; + [[nodiscard]] virtual bool isDone() const = 0; + + virtual void reset(VertexRef source, size_t depth = 0) = 0; + virtual auto getNextPath() -> std::unique_ptr = 0; + virtual bool skipPath() = 0; + virtual auto destroyEngines() -> void = 0; + + virtual auto stealStats() -> aql::TraversalStats = 0; +}; + +} // namespace graph +} // namespace arangodb diff --git a/arangod/Graph/Enumerators/TwoSidedEnumerator.cpp b/arangod/Graph/Enumerators/TwoSidedEnumerator.cpp index 51ea0ed1a842..10df092a905d 100644 --- a/arangod/Graph/Enumerators/TwoSidedEnumerator.cpp +++ b/arangod/Graph/Enumerators/TwoSidedEnumerator.cpp @@ -54,12 +54,12 @@ using namespace arangodb::graph; template TwoSidedEnumerator::Ball::Ball( Direction dir, ProviderType&& provider, GraphOptions const& options, - arangodb::ResourceMonitor& resourceMonitor) + PathValidatorOptions validatorOptions, arangodb::ResourceMonitor& resourceMonitor) : _resourceMonitor(resourceMonitor), _interior(resourceMonitor), _queue(resourceMonitor), _provider(std::move(provider)), - _validator(_interior), + _validator(_provider, _interior, std::move(validatorOptions)), _direction(dir), _minDepth(options.getMinDepth()) {} @@ -67,9 +67,10 @@ template ::Ball::~Ball() = default; template -void TwoSidedEnumerator::Ball::reset(VertexRef center) { +void TwoSidedEnumerator::Ball::reset( + VertexRef center, size_t depth) { clear(); - auto firstStep = _provider.startVertex(center); + auto firstStep = _provider.startVertex(center, depth); _shell.emplace(std::move(firstStep)); } @@ -144,9 +145,9 @@ auto TwoSidedEnumerator:: // Notes for the future: // Vertices are now fetched. Thnink about other less-blocking and batch-wise // fetching (e.g. re-fetch at some later point). - // TODO: Discuss how to optimize here. Currently we'll mark looseEnds in fetch as fetched. - // This works, but we might create a batch limit here in the future. - // Also discuss: Do we want (re-)fetch logic here? + // TODO: Discuss how to optimize here. Currently we'll mark looseEnds in + // fetch as fetched. This works, but we might create a batch limit here in + // the future. Also discuss: Do we want (re-)fetch logic here? // TODO: maybe we can combine this with prefetching of paths } } @@ -245,10 +246,13 @@ auto TwoSidedEnumerator:: template TwoSidedEnumerator::TwoSidedEnumerator( ProviderType&& forwardProvider, ProviderType&& backwardProvider, - TwoSidedEnumeratorOptions&& options, arangodb::ResourceMonitor& resourceMonitor) + TwoSidedEnumeratorOptions&& options, PathValidatorOptions validatorOptions, + arangodb::ResourceMonitor& resourceMonitor) : _options(std::move(options)), - _left{Direction::FORWARD, std::move(forwardProvider), _options, resourceMonitor}, - _right{Direction::BACKWARD, std::move(backwardProvider), _options, resourceMonitor}, + _left{Direction::FORWARD, std::move(forwardProvider), _options, + validatorOptions, resourceMonitor}, + _right{Direction::BACKWARD, std::move(backwardProvider), _options, + std::move(validatorOptions), resourceMonitor}, _resultPath{_left.provider(), _right.provider()} {} template @@ -293,7 +297,7 @@ bool TwoSidedEnumerator:: */ template void TwoSidedEnumerator::reset( - VertexRef source, VertexRef target) { + VertexRef source, VertexRef target, size_t depth) { _results.clear(); _left.reset(source); _right.reset(target); @@ -428,23 +432,25 @@ auto TwoSidedEnumerator:: template class ::arangodb::graph::TwoSidedEnumerator< ::arangodb::graph::FifoQueue<::arangodb::graph::SingleServerProvider::Step>, ::arangodb::graph::PathStore, SingleServerProvider, - ::arangodb::graph::PathValidator, VertexUniquenessLevel::PATH>>; + ::arangodb::graph::PathValidator, VertexUniquenessLevel::PATH>>; template class ::arangodb::graph::TwoSidedEnumerator< ::arangodb::graph::QueueTracer<::arangodb::graph::FifoQueue<::arangodb::graph::SingleServerProvider::Step>>, ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore>, ::arangodb::graph::ProviderTracer, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore>, VertexUniquenessLevel::PATH>>; + ::arangodb::graph::PathValidator<::arangodb::graph::ProviderTracer, + ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore>, VertexUniquenessLevel::PATH>>; /* ClusterProvider Section */ template class ::arangodb::graph::TwoSidedEnumerator< ::arangodb::graph::FifoQueue<::arangodb::graph::ClusterProvider::Step>, ::arangodb::graph::PathStore, ClusterProvider, - ::arangodb::graph::PathValidator, VertexUniquenessLevel::PATH>>; + ::arangodb::graph::PathValidator, VertexUniquenessLevel::PATH>>; template class ::arangodb::graph::TwoSidedEnumerator< ::arangodb::graph::QueueTracer<::arangodb::graph::FifoQueue<::arangodb::graph::ClusterProvider::Step>>, ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore>, ::arangodb::graph::ProviderTracer, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore>, VertexUniquenessLevel::PATH>>; + ::arangodb::graph::PathValidator<::arangodb::graph::ProviderTracer, + ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore>, VertexUniquenessLevel::PATH>>; diff --git a/arangod/Graph/Enumerators/TwoSidedEnumerator.h b/arangod/Graph/Enumerators/TwoSidedEnumerator.h index 2dddc3d83792..141afd35e95d 100644 --- a/arangod/Graph/Enumerators/TwoSidedEnumerator.h +++ b/arangod/Graph/Enumerators/TwoSidedEnumerator.h @@ -47,6 +47,7 @@ class HashedStringRef; namespace graph { +class PathValidatorOptions; struct TwoSidedEnumeratorOptions; template @@ -69,10 +70,10 @@ class TwoSidedEnumerator { class Ball { public: Ball(Direction dir, ProviderType&& provider, GraphOptions const& options, - arangodb::ResourceMonitor& resourceMonitor); + PathValidatorOptions validatorOptions, arangodb::ResourceMonitor& resourceMonitor); ~Ball(); auto clear() -> void; - auto reset(VertexRef center) -> void; + auto reset(VertexRef center, size_t depth = 0) -> void; auto startNextDepth() -> void; [[nodiscard]] auto noPathLeft() const -> bool; [[nodiscard]] auto getDepth() const -> size_t; @@ -82,7 +83,8 @@ class TwoSidedEnumerator { auto buildPath(Step const& vertexInShell, PathResult& path) -> void; - auto matchResultsInShell(Step const& match, ResultList& results, PathValidatorType const& otherSideValidator) -> void; + auto matchResultsInShell(Step const& match, ResultList& results, + PathValidatorType const& otherSideValidator) -> void; auto computeNeighbourhoodOfNextVertex(Ball& other, ResultList& results) -> void; // Ensure that we have fetched all vertices @@ -118,7 +120,7 @@ class TwoSidedEnumerator { public: TwoSidedEnumerator(ProviderType&& forwardProvider, ProviderType&& backwardProvider, - TwoSidedEnumeratorOptions&& options, + TwoSidedEnumeratorOptions&& options, PathValidatorOptions validatorOptions, arangodb::ResourceMonitor& resourceMonitor); TwoSidedEnumerator(TwoSidedEnumerator const& other) = delete; TwoSidedEnumerator(TwoSidedEnumerator&& other) noexcept = default; @@ -145,7 +147,7 @@ class TwoSidedEnumerator { * @param source The source vertex to start the paths * @param target The target vertex to end the paths */ - void reset(VertexRef source, VertexRef target); + void reset(VertexRef source, VertexRef target, size_t depth = 0); /** * @brief Get the next path, if available written into the result build. diff --git a/arangod/Graph/Graph.cpp b/arangod/Graph/Graph.cpp index 0ab3270d36c9..60aab621acd1 100644 --- a/arangod/Graph/Graph.cpp +++ b/arangod/Graph/Graph.cpp @@ -451,7 +451,6 @@ void EdgeDefinition::toVelocyPack(VPackBuilder& builder) const { builder.add(VPackValue(to)); } builder.close(); // array - builder.add("type", VPackValue(getType())); } ResultT EdgeDefinition::createFromVelocypack(VPackSlice edgeDefinition, std::set const& satCollections) { @@ -572,7 +571,6 @@ void EdgeDefinition::addToBuilder(VPackBuilder& builder) const { builder.add(VPackValue(to)); } builder.close(); // to - builder.add(StaticStrings::GraphEdgeDefinitionType, VPackValue(getType())); builder.close(); // obj } @@ -775,11 +773,6 @@ bool Graph::isDisjoint() const { return false; } -bool Graph::isHybrid() const { - TRI_ASSERT(_satelliteColls.empty()); - return false; -} - bool Graph::isSatellite() const { return _isSatellite; } void Graph::createCollectionOptions(VPackBuilder& builder, bool waitForSync) const { diff --git a/arangod/Graph/Graph.h b/arangod/Graph/Graph.h index 7991934cb612..9d424ead1082 100644 --- a/arangod/Graph/Graph.h +++ b/arangod/Graph/Graph.h @@ -207,7 +207,6 @@ class Graph { virtual bool isSmart() const; virtual bool isDisjoint() const; virtual bool isSatellite() const; - virtual bool isHybrid() const; virtual bool needsToBeSatellite(std::string const& edge) const; uint64_t numberOfShards() const; diff --git a/arangod/Graph/GraphManager.cpp b/arangod/Graph/GraphManager.cpp index ea8e8260f3ae..80a49c90cdba 100644 --- a/arangod/Graph/GraphManager.cpp +++ b/arangod/Graph/GraphManager.cpp @@ -534,7 +534,7 @@ Result GraphManager::ensureCollections(Graph* graph, bool waitForSync) const { return res; } else { // not found the collection, need to create it later - if (graph->isHybrid() && graph->needsToBeSatellite(edgeColl)) { // check for satellites + if (graph->needsToBeSatellite(edgeColl)) { // check for satellites satelliteEdgeCollectionsToCreate.emplace(edgeColl); } else { edgeCollectionsToCreate.emplace(edgeColl); @@ -555,7 +555,9 @@ Result GraphManager::ensureCollections(Graph* graph, bool waitForSync) const { return res; } else { if (edgeCollectionsToCreate.find(vertexColl) == edgeCollectionsToCreate.end()) { - if (graph->isHybrid() && graph->satelliteCollections().find(vertexColl) != graph->satelliteCollections().end()) { + if (!graph->satelliteCollections().empty() && + graph->satelliteCollections().find(vertexColl) != + graph->satelliteCollections().end()) { satelliteDocumentCollectionsToCreate.emplace(vertexColl); } else { documentCollectionsToCreate.emplace(vertexColl); @@ -622,7 +624,7 @@ Result GraphManager::ensureCollections(Graph* graph, bool waitForSync) const { // new builder VPackBuilder satOptionsBuilder; - if (graph->isHybrid()) { + if (!satelliteDocumentCollectionsToCreate.empty() || !satelliteEdgeCollectionsToCreate.empty()) { satOptionsBuilder.openObject(); graph->createSatelliteCollectionOptions(satOptionsBuilder, waitForSync); satOptionsBuilder.close(); diff --git a/arangod/Graph/PathManagement/PathStore.cpp b/arangod/Graph/PathManagement/PathStore.cpp index b49c8d69b13d..e6402d410764 100644 --- a/arangod/Graph/PathManagement/PathStore.cpp +++ b/arangod/Graph/PathManagement/PathStore.cpp @@ -23,7 +23,6 @@ #include "PathStore.h" #include "Graph/PathManagement/PathResult.h" -#include "Graph/PathManagement/SingleProviderPathResult.h" #include "Graph/Providers/ClusterProvider.h" #include "Graph/Providers/ProviderTracer.h" @@ -78,7 +77,7 @@ size_t PathStore::append(Step step) { } template -Step PathStore::get(size_t position) const { +Step PathStore::getStep(size_t position) const { TRI_ASSERT(position <= size()); Step step = _schreier.at(position); LOG_TOPIC("45bf5", TRACE, Logger::GRAPHS) @@ -87,6 +86,16 @@ Step PathStore::get(size_t position) const { return step; } +template +Step& PathStore::getStepReference(size_t position) { + TRI_ASSERT(position <= size()); + auto& step = _schreier.at(position); + LOG_TOPIC("45bf6", TRACE, Logger::GRAPHS) + << " Get step: " << step.toString(); + + return step; +} + template template auto PathStore::buildPath(Step const& vertex, PathResultType& path) const -> void { @@ -159,6 +168,26 @@ auto PathStore::visitReversePath(Step const& step, } } +template +auto PathStore::modifyReversePath(Step& step, + std::function const& visitor) +-> bool { + Step * walker = &step; + // Guaranteed to make progress, as the schreier vector contains a loop-free tree. + while (true) { + bool cont = visitor(*walker); + if (!cont) { + // Aborted + return false; + } + if (walker->isFirst()) { + // Visited the full path + return true; + } + walker = &_schreier.at(walker->getPrevious()); + } +} + /* SingleServerProvider Section */ template class PathStore; @@ -167,11 +196,6 @@ template void PathStore::buildPath& path) const; -template void PathStore::buildPath< - SingleProviderPathResult>( - SingleServerProvider::Step const& vertex, - SingleProviderPathResult& path) const; - template void PathStore::reverseBuildPath( SingleServerProvider::Step const& vertex, PathResult& path) const; @@ -183,12 +207,6 @@ template void PathStore::buildPath< ProviderTracer::Step const& vertex, PathResult, ProviderTracer::Step>& path) const; -template void PathStore::buildPath< - SingleProviderPathResult, ProviderTracer::Step>>( - ProviderTracer::Step const& vertex, - SingleProviderPathResult, - ProviderTracer::Step>& path) const; - template void PathStore::Step>::reverseBuildPath>( ProviderTracer::Step const& vertex, PathResult, ProviderTracer::Step>& path) const; diff --git a/arangod/Graph/PathManagement/PathStore.h b/arangod/Graph/PathManagement/PathStore.h index 8ec934790c33..a24eb1195f15 100644 --- a/arangod/Graph/PathManagement/PathStore.h +++ b/arangod/Graph/PathManagement/PathStore.h @@ -71,7 +71,10 @@ class PathStore { size_t append(Step step); // @briefs Method returns a step at given position - Step get(size_t position) const; + Step getStep(size_t position) const; + + // @briefs Method returns a step reference at given position + Step& getStepReference(size_t position); // @brief returns the current vector size size_t size() const { return _schreier.size(); } @@ -79,11 +82,14 @@ class PathStore { auto visitReversePath(Step const& step, std::function const& visitor) const -> bool; + auto modifyReversePath(Step& step, std::function const& visitor) -> bool; + template auto buildPath(Step const& vertex, PathResultType& path) const -> void; template - auto reverseBuildPath(Step const& vertex, PathResult& path) const -> void; + auto reverseBuildPath(Step const& vertex, PathResult& path) const + -> void; private: /// @brief schreier vector to store the visited vertices @@ -94,4 +100,3 @@ class PathStore { } // namespace graph } // namespace arangodb - diff --git a/arangod/Graph/PathManagement/PathStoreTracer.cpp b/arangod/Graph/PathManagement/PathStoreTracer.cpp index f32478e01554..280bc7331e68 100644 --- a/arangod/Graph/PathManagement/PathStoreTracer.cpp +++ b/arangod/Graph/PathManagement/PathStoreTracer.cpp @@ -64,10 +64,17 @@ size_t PathStoreTracer::append(Step step) { } template -typename PathStoreImpl::Step PathStoreTracer::get(size_t position) const { +typename PathStoreImpl::Step PathStoreTracer::getStep(size_t position) const { double start = TRI_microtime(); - TRI_DEFER(_stats["get"].addTiming(TRI_microtime() - start)); - return _impl.get(position); + TRI_DEFER(_stats["getStep"].addTiming(TRI_microtime() - start)); + return _impl.getStep(position); +} + +template +typename PathStoreImpl::Step& PathStoreTracer::getStepReference(size_t position) { + double start = TRI_microtime(); + TRI_DEFER(_stats["getStepReference"].addTiming(TRI_microtime() - start)); + return _impl.getStepReference(position); } template @@ -104,6 +111,13 @@ auto PathStoreTracer::visitReversePath( return _impl.visitReversePath(step, visitor); } +template +auto PathStoreTracer::modifyReversePath(Step& step, const std::function& visitor) -> bool { + double start = TRI_microtime(); + TRI_DEFER(_stats["modifyReversePath"].addTiming(TRI_microtime() - start)); + return _impl.modifyReversePath(step, visitor); +} + /* SingleServerProvider Section */ template class ::arangodb::graph::PathStoreTracer>; @@ -114,11 +128,6 @@ template void ::arangodb::graph::PathStoreTracer::Step const& vertex, PathResult, ProviderTracer::Step>& path) const; -template void ::arangodb::graph::PathStoreTracer>::buildPath< - SingleProviderPathResult, ProviderTracer::Step>>( - ProviderTracer::Step const& vertex, - SingleProviderPathResult, ProviderTracer::Step>& path) const; - template void arangodb::graph::PathStoreTracer>::reverseBuildPath>( ProviderTracer::Step const& vertex, PathResult, ProviderTracer::Step>& path) const; diff --git a/arangod/Graph/PathManagement/PathStoreTracer.h b/arangod/Graph/PathManagement/PathStoreTracer.h index 288bd019f92a..a1895924b64a 100644 --- a/arangod/Graph/PathManagement/PathStoreTracer.h +++ b/arangod/Graph/PathManagement/PathStoreTracer.h @@ -60,7 +60,8 @@ class PathStoreTracer { // returns the index of inserted element size_t append(Step step); - auto get(size_t position) const -> Step; + auto getStep(size_t position) const -> Step; + auto getStepReference(size_t position) -> Step&; // @brief returns the current vector size size_t size() const; @@ -75,6 +76,8 @@ class PathStoreTracer { auto visitReversePath(Step const& step, std::function const& visitor) const -> bool; + auto modifyReversePath(Step& step, std::function const& visitor) -> bool; + private: PathStoreImpl _impl; diff --git a/arangod/Graph/PathManagement/PathValidator.cpp b/arangod/Graph/PathManagement/PathValidator.cpp index 28f0a8d4d3a2..9c9d7f7a90be 100644 --- a/arangod/Graph/PathManagement/PathValidator.cpp +++ b/arangod/Graph/PathManagement/PathValidator.cpp @@ -22,9 +22,12 @@ //////////////////////////////////////////////////////////////////////////////// #include "PathValidator.h" +#include "Aql/AstNode.h" +#include "Aql/PruneExpressionEvaluator.h" #include "Graph/PathManagement/PathStore.h" #include "Graph/PathManagement/PathStoreTracer.h" #include "Graph/Providers/ClusterProvider.h" +#include "Graph/Providers/ProviderTracer.h" #include "Graph/Providers/SingleServerProvider.h" #include "Graph/Types/ValidationResult.h" @@ -33,13 +36,26 @@ using namespace arangodb; using namespace arangodb::graph; -template -PathValidator::PathValidator(PathStore const& store) - : _store(store) {} +template +PathValidator::PathValidator( + ProviderType& provider, PathStore const& store, PathValidatorOptions opts) + : _store(store), _provider(provider), _options(std::move(opts)) {} + +template +PathValidator::~PathValidator() = default; + +template +auto PathValidator::validatePath( + typename PathStore::Step const& step) -> ValidationResult { + auto ctx = _options.getExpressionContext(); + // Reset variables + ctx.clearVariableValues(); + auto res = evaluateVertexCondition(step); + if (res.isFiltered() && res.isPruned()) { + // Can give up here. This Value is not used + return res; + } -template -auto PathValidator::validatePath(typename PathStore::Step const& step) - -> ValidationResult { if constexpr (vertexUniqueness == VertexUniquenessLevel::PATH) { _uniqueVertices.clear(); // Reserving here is pointless, we will test paths that increase by at most 1 entry. @@ -52,25 +68,24 @@ auto PathValidator::validatePath(typename PathStore return added; }); if (!success) { - return ValidationResult{ValidationResult::Type::FILTER}; + res.combine(ValidationResult::Type::FILTER); } - return ValidationResult{ValidationResult::Type::TAKE}; } if constexpr (vertexUniqueness == VertexUniquenessLevel::GLOBAL) { auto const& [unused, added] = _uniqueVertices.emplace(step.getVertexIdentifier()); // If this add fails, we need to exclude this path if (!added) { - return ValidationResult{ValidationResult::Type::FILTER}; + res.combine(ValidationResult::Type::FILTER); } - return ValidationResult{ValidationResult::Type::TAKE}; } - return ValidationResult{ValidationResult::Type::TAKE}; + return res; } -template -auto PathValidator::validatePath( +template +auto PathValidator::validatePath( typename PathStore::Step const& step, - PathValidator const& otherValidator) -> ValidationResult { + PathValidator const& otherValidator) + -> ValidationResult { if constexpr (vertexUniqueness == VertexUniquenessLevel::PATH) { // For PATH: take _uniqueVertices of otherValidator, and run Visitor of other side, check if one vertex is duplicate. auto const& otherUniqueVertices = otherValidator.exposeUniqueVertices(); @@ -106,22 +121,126 @@ auto PathValidator::validatePath( return ValidationResult{ValidationResult::Type::TAKE}; } -template -auto PathValidator::exposeUniqueVertices() const +template +auto PathValidator::exposeUniqueVertices() const -> ::arangodb::containers::HashSet, std::equal_to> const& { return _uniqueVertices; } +template +auto PathValidator::evaluateVertexRestriction( + typename PathStore::Step const& step) -> bool { + if (step.isFirst()) { + // first step => always allowed + return true; + } + + auto allowedCollections = _options.getAllowedVertexCollections(); + if (allowedCollections.empty()) { + // all allowed + return true; + } + + auto collectionName = step.getCollectionName(); + if (std::find(allowedCollections.begin(), allowedCollections.end(), + collectionName) != allowedCollections.end()) { + // found in allowed collections => allowed + return true; + } + + // not allowed + return false; +} + +template +auto PathValidator::evaluateVertexCondition( + typename PathStore::Step const& step) -> ValidationResult { + // evaluate if vertex collection is allowed + bool isAllowed = evaluateVertexRestriction(step); + if (!isAllowed) { + return ValidationResult{ValidationResult::Type::FILTER}; + } + + auto expr = _options.getVertexExpression(step.getDepth()); + if (expr != nullptr) { + _tmpObjectBuilder.clear(); + _provider.addVertexToBuilder(step.getVertex(), _tmpObjectBuilder); + + // evaluate expression + bool satifiesCondition = evaluateExpression(expr, _tmpObjectBuilder.slice()); + if (!satifiesCondition) { + return ValidationResult{ValidationResult::Type::FILTER}; + } + } + return ValidationResult{ValidationResult::Type::TAKE}; +} + +template +auto PathValidator::evaluateExpression( + arangodb::aql::Expression* expression, VPackSlice value) -> bool { + if (expression == nullptr) { + return true; + } + + TRI_ASSERT(value.isObject() || value.isNull()); + auto tmpVar = _options.getTempVar(); + bool mustDestroy = false; + auto ctx = _options.getExpressionContext(); + aql::AqlValue tmpVal{value}; + ctx.setVariableValue(tmpVar, tmpVal); + aql::AqlValue res = expression->execute(&ctx, mustDestroy); + aql::AqlValueGuard guard{res, mustDestroy}; + TRI_ASSERT(res.isBoolean()); + return res.toBoolean(); +} + +template +void PathValidator::setPruneEvaluator( + std::unique_ptr eval) { + _pruneEvaluator = std::move(eval); +} + +template +void PathValidator::setPostFilterEvaluator( + std::unique_ptr eval) { + _postFilterEvaluator = std::move(eval); +} + +template +void PathValidator::reset() { + if constexpr (vertexUniqueness != VertexUniquenessLevel::NONE) { + _uniqueVertices.clear(); + } +} + namespace arangodb { namespace graph { /* SingleServerProvider Section */ -template class PathValidator, VertexUniquenessLevel::PATH>; -template class PathValidator>, VertexUniquenessLevel::PATH>; +template class PathValidator, VertexUniquenessLevel::NONE>; +template class PathValidator, + PathStoreTracer::Step>>, VertexUniquenessLevel::NONE>; + +template class PathValidator, VertexUniquenessLevel::PATH>; +template class PathValidator, + PathStoreTracer::Step>>, VertexUniquenessLevel::PATH>; + +template class PathValidator, VertexUniquenessLevel::GLOBAL>; +template class PathValidator, + PathStoreTracer::Step>>, VertexUniquenessLevel::GLOBAL>; /* ClusterProvider Section */ -template class PathValidator, VertexUniquenessLevel::PATH>; -template class PathValidator>, VertexUniquenessLevel::PATH>; +template class PathValidator, VertexUniquenessLevel::NONE>; +template class PathValidator, + PathStoreTracer::Step>>, VertexUniquenessLevel::NONE>; + +template class PathValidator, VertexUniquenessLevel::PATH>; +template class PathValidator, + PathStoreTracer::Step>>, VertexUniquenessLevel::PATH>; + +template class PathValidator, VertexUniquenessLevel::GLOBAL>; +template class PathValidator, + PathStoreTracer::Step>>, VertexUniquenessLevel::GLOBAL>; } // namespace graph } // namespace arangodb diff --git a/arangod/Graph/PathManagement/PathValidator.h b/arangod/Graph/PathManagement/PathValidator.h index 0ca712af659e..a3971bf50d0e 100644 --- a/arangod/Graph/PathManagement/PathValidator.h +++ b/arangod/Graph/PathManagement/PathValidator.h @@ -25,36 +25,75 @@ #include #include "Containers/HashSet.h" +#include "Graph/PathManagement/PathValidatorOptions.h" #include "Graph/Types/UniquenessLevel.h" +#include + namespace arangodb { +namespace aql { +class PruneExpressionEvaluator; +} namespace graph { class ValidationResult; -template +/* + * TODO: + * + * - EdgeUniqueness, like vertex Uniqueness, and sometimes Edge and Vertex Uniqueness enforce each other. + * (e.g. => VertexUniqueness == PATH => EdgeUniquess PATH || NONE. + * + * - Prune Condition. + * - PostFilter Condition. + * - Path Conditions. Vertex, Edge maybe in LookupInfo + * (e.g. p.vertices[*].name ALL == "HANS") + */ +template class PathValidator { using VertexRef = arangodb::velocypack::HashedStringRef; public: - PathValidator(PathStore const& store); + PathValidator(Provider& provider, PathStore const& store, PathValidatorOptions opts); + ~PathValidator(); auto validatePath(typename PathStore::Step const& step) -> ValidationResult; auto validatePath(typename PathStore::Step const& step, - PathValidator const& otherValidator) + PathValidator const& otherValidator) -> ValidationResult; + void setPruneEvaluator(std::unique_ptr eval); + + void setPostFilterEvaluator(std::unique_ptr eval); + + void reset(); + private: PathStore const& _store; + Provider& _provider; // Only for applied vertex uniqueness // TODO: Figure out if we can make this Member template dependend // e.g. std::enable_if ::arangodb::containers::HashSet, std::equal_to> _uniqueVertices; + PathValidatorOptions _options; + + std::unique_ptr _pruneEvaluator; + + std::unique_ptr _postFilterEvaluator; + + arangodb::velocypack::Builder _tmpObjectBuilder; + private: + auto evaluateVertexCondition(typename PathStore::Step const&) -> ValidationResult; + auto evaluateVertexRestriction(typename PathStore::Step const& step) -> bool; + auto exposeUniqueVertices() const -> ::arangodb::containers::HashSet, std::equal_to> const&; + + auto evaluateExpression(arangodb::aql::Expression* expression, + arangodb::velocypack::Slice value) -> bool; }; } // namespace graph } // namespace arangodb diff --git a/arangod/Graph/PathManagement/PathValidatorOptions.cpp b/arangod/Graph/PathManagement/PathValidatorOptions.cpp new file mode 100644 index 000000000000..81935f1931a1 --- /dev/null +++ b/arangod/Graph/PathManagement/PathValidatorOptions.cpp @@ -0,0 +1,79 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2021-2021 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +//////////////////////////////////////////////////////////////////////////////// + +#include "PathValidatorOptions.h" + +#include "Aql/Expression.h" +#include "Aql/QueryContext.h" + +using namespace arangodb; +using namespace arangodb::graph; + +PathValidatorOptions::PathValidatorOptions(aql::Variable const* tmpVar, + arangodb::aql::FixedVarExpressionContext& expressionContext) + : _tmpVar(tmpVar), _expressionCtx(expressionContext) {} + +void PathValidatorOptions::setAllVerticesExpression(std::unique_ptr expression) { + // All edge expression should not be set before + TRI_ASSERT(_allVerticesExpression == nullptr); + _allVerticesExpression = std::move(expression); +} + +void PathValidatorOptions::setVertexExpression(uint64_t depth, + std::unique_ptr expression) { + // Should not respecifiy the condition on a certain depth + TRI_ASSERT(_vertexExpressionOnDepth.find(depth) == _vertexExpressionOnDepth.end()); + _vertexExpressionOnDepth.emplace(depth, std::move(expression)); +} + +aql::Expression* PathValidatorOptions::getVertexExpression(uint64_t depth) const { + auto const& it = _vertexExpressionOnDepth.find(depth); + if (it != _vertexExpressionOnDepth.end()) { + return it->second.get(); + } + return _allVerticesExpression.get(); +} + +void PathValidatorOptions::addAllowedVertexCollection(std::string const& collectionName) { + TRI_ASSERT(std::find(_allowedVertexCollections.begin(), + _allowedVertexCollections.end(), + collectionName) == _allowedVertexCollections.end()); + _allowedVertexCollections.emplace_back(collectionName); +} + +void PathValidatorOptions::addAllowedVertexCollections(std::vector const& collectionNames) { + for (auto& name : collectionNames) { + addAllowedVertexCollection(name); + } +} + +std::vector const& PathValidatorOptions::getAllowedVertexCollections() const { + return _allowedVertexCollections; +} + +aql::Variable const* PathValidatorOptions::getTempVar() const { + return _tmpVar; +} + +aql::FixedVarExpressionContext& PathValidatorOptions::getExpressionContext() { + return _expressionCtx; +} diff --git a/arangod/Graph/PathManagement/PathValidatorOptions.h b/arangod/Graph/PathManagement/PathValidatorOptions.h new file mode 100644 index 000000000000..14c5096146f4 --- /dev/null +++ b/arangod/Graph/PathManagement/PathValidatorOptions.h @@ -0,0 +1,88 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2021-2021 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +//////////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include +#include +#include + +#include "Aql/AqlFunctionsInternalCache.h" +#include "Aql/FixedVarExpressionContext.h" +#include "Transaction/Methods.h" + +namespace arangodb { +namespace aql { +class Expression; +class ExpressionContext; +class QueryContext; +struct Variable; +} // namespace aql + +namespace graph { +class PathValidatorOptions { + public: + PathValidatorOptions(aql::Variable const* tmpVar, + arangodb::aql::FixedVarExpressionContext& expressionContext); + ~PathValidatorOptions() = default; + PathValidatorOptions(PathValidatorOptions&&) = default; + PathValidatorOptions(PathValidatorOptions const&) = default; + + /** + * @brief Set the expression that needs to hold true for ALL vertices on the path. + */ + void setAllVerticesExpression(std::unique_ptr expression); + + /** + * @brief Set the expression that needs to hold true for the vertex on the + * given depth. NOTE: This will overrule the ALL vertex expression, so make + * sure this expression contains everything the ALL expression covers. + */ + void setVertexExpression(uint64_t depth, std::unique_ptr expression); + + /** + * @brief Get the Expression a vertex needs to hold if defined on the given + * depth. It may return a nullptr if all vertices are valid. + * Caller does NOT take responsibilty. Do not delete this pointer. + */ + aql::Expression* getVertexExpression(uint64_t depth) const; + + void addAllowedVertexCollection(std::string const& collectionName); + + void addAllowedVertexCollections(std::vector const& collectionNames); + + std::vector const& getAllowedVertexCollections() const; + + aql::Variable const* getTempVar() const; + + aql::FixedVarExpressionContext& getExpressionContext(); + + private: + std::shared_ptr _allVerticesExpression; + std::unordered_map> _vertexExpressionOnDepth; + aql::Variable const* _tmpVar; + arangodb::aql::FixedVarExpressionContext& _expressionCtx; + + std::vector _allowedVertexCollections; +}; +} // namespace graph +} // namespace arangodb diff --git a/arangod/Graph/PathManagement/SingleProviderPathResult.cpp b/arangod/Graph/PathManagement/SingleProviderPathResult.cpp index 86732a8d9a8c..c1597ed4d74d 100644 --- a/arangod/Graph/PathManagement/SingleProviderPathResult.cpp +++ b/arangod/Graph/PathManagement/SingleProviderPathResult.cpp @@ -25,6 +25,8 @@ #include "SingleProviderPathResult.h" #include "Basics/StaticStrings.h" +#include "Graph/PathManagement/PathStore.h" +#include "Graph/PathManagement/PathStoreTracer.h" #include "Graph/Providers/ClusterProvider.h" #include "Graph/Providers/ProviderTracer.h" #include "Graph/Providers/SingleServerProvider.h" @@ -35,43 +37,55 @@ using namespace arangodb; using namespace arangodb::graph; -template -SingleProviderPathResult::SingleProviderPathResult(ProviderType& provider) - : _provider(provider) {} +template +SingleProviderPathResult::SingleProviderPathResult( + Step* step, ProviderType& provider, PathStoreType& store) + : _step(step), _provider(provider), _store(store) { + TRI_ASSERT(_step != nullptr); +} -template -auto SingleProviderPathResult::clear() -> void { +template +auto SingleProviderPathResult::clear() -> void { _vertices.clear(); _edges.clear(); } -template -auto SingleProviderPathResult::appendVertex(typename Step::Vertex v) +template +auto SingleProviderPathResult::appendVertex(typename Step::Vertex v) -> void { _vertices.push_back(std::move(v)); } -template -auto SingleProviderPathResult::prependVertex(typename Step::Vertex v) +template +auto SingleProviderPathResult::prependVertex(typename Step::Vertex v) -> void { _vertices.insert(_vertices.begin(), std::move(v)); } -template -auto SingleProviderPathResult::appendEdge(typename Step::Edge e) +template +auto SingleProviderPathResult::appendEdge(typename Step::Edge e) -> void { _edges.push_back(std::move(e)); } -template -auto SingleProviderPathResult::prependEdge(typename Step::Edge e) +template +auto SingleProviderPathResult::prependEdge(typename Step::Edge e) -> void { _edges.insert(_edges.begin(), std::move(e)); } -template -auto SingleProviderPathResult::toVelocyPack(arangodb::velocypack::Builder& builder) - -> void { +template +auto SingleProviderPathResult::toVelocyPack( + arangodb::velocypack::Builder& builder) -> void { + if (_vertices.empty()) { + _store.visitReversePath(*_step, [&](Step const& s) -> bool { + prependVertex(s.getVertex()); + if (s.getEdge().isValid()) { + prependEdge(s.getEdge()); + } + return true; + }); + } VPackObjectBuilder path{&builder}; { builder.add(VPackValue(StaticStrings::GraphQueryVertices)); @@ -92,24 +106,78 @@ auto SingleProviderPathResult::toVelocyPack(arangodb::velocy } } -template -auto SingleProviderPathResult::isEmpty() const -> bool { - return _vertices.empty(); +template +auto SingleProviderPathResult::toSchreierEntry( + arangodb::velocypack::Builder& result, size_t& currentLength) -> void { + size_t prevIndex = 0; + + auto writeStepToBuilder = [&](Step& step) { + VPackArrayBuilder arrayGuard(&result); + // Create a VPackValue based on the char* inside the HashedStringRef, will save us a String copy each time. + result.add(VPackValue(step.getVertex().getID().begin())); + + // Index position of previous step + result.add(VPackValue(prevIndex)); + // The depth of the current step + result.add(VPackValue(step.getDepth())); + + bool isResponsible = step.isResponsible(_provider.trx()); + // Print if we have a loose end here (looseEnd := this server is NOT responsible) + result.add(VPackValue(!isResponsible)); + if (isResponsible) { + // This server needs to provide the data for the vertex + _provider.addVertexToBuilder(step.getVertex(), result); + } else { + result.add(VPackSlice::nullSlice()); + } + + _provider.addEdgeToBuilder(step.getEdge(), result); + }; // TODO: Create method instead of lambda + + std::vector toWrite{}; + + _store.modifyReversePath(*_step, [&](Step& step) -> bool { + if (!step.hasLocalSchreierIndex()) { + toWrite.emplace_back(&step); + return true; + } else { + prevIndex = step.getLocalSchreierIndex(); + return false; + } + }); + + for (auto it = toWrite.rbegin(); it != toWrite.rend(); it++) { + TRI_ASSERT(!(*it)->hasLocalSchreierIndex()); + writeStepToBuilder(**it); + + prevIndex = currentLength; + (*it)->setLocalSchreierIndex(currentLength++); + } +} + +template +auto SingleProviderPathResult::isEmpty() const + -> bool { + return false; } /* SingleServerProvider Section */ template class ::arangodb::graph::SingleProviderPathResult< - ::arangodb::graph::SingleServerProvider, ::arangodb::graph::SingleServerProvider::Step>; + ::arangodb::graph::SingleServerProvider, ::arangodb::graph::PathStore<::arangodb::graph::SingleServerProvider::Step>, + ::arangodb::graph::SingleServerProvider::Step>; template class ::arangodb::graph::SingleProviderPathResult< ::arangodb::graph::ProviderTracer<::arangodb::graph::SingleServerProvider>, + ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::graph::SingleServerProvider::Step>>, ::arangodb::graph::SingleServerProvider::Step>; /* ClusterProvider Section */ - template class ::arangodb::graph::SingleProviderPathResult< - ::arangodb::graph::ClusterProvider, ::arangodb::graph::ClusterProvider::Step>; + ::arangodb::graph::ClusterProvider, ::arangodb::graph::PathStore<::arangodb::graph::ClusterProvider::Step>, + ::arangodb::graph::ClusterProvider::Step>; template class ::arangodb::graph::SingleProviderPathResult< - ::arangodb::graph::ProviderTracer<::arangodb::graph::ClusterProvider>, ::arangodb::graph::ClusterProvider::Step>; \ No newline at end of file + ::arangodb::graph::ProviderTracer<::arangodb::graph::ClusterProvider>, + ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::graph::ClusterProvider::Step>>, + ::arangodb::graph::ClusterProvider::Step>; diff --git a/arangod/Graph/PathManagement/SingleProviderPathResult.h b/arangod/Graph/PathManagement/SingleProviderPathResult.h index 9d0fd94b359f..13b7b3021b69 100644 --- a/arangod/Graph/PathManagement/SingleProviderPathResult.h +++ b/arangod/Graph/PathManagement/SingleProviderPathResult.h @@ -27,7 +27,11 @@ #include #include "Containers/HashSet.h" +// TODO Temporary include +#include "Graph/Enumerators/OneSidedEnumeratorInterface.h" + #include +#include namespace arangodb { @@ -37,27 +41,37 @@ class Builder; namespace graph { -template -class SingleProviderPathResult { +template +class SingleProviderPathResult : public PathResultInterface { using VertexRef = arangodb::velocypack::HashedStringRef; public: - SingleProviderPathResult(ProviderType& provider); + SingleProviderPathResult(Step* step, ProviderType& provider, PathStoreType& store); auto clear() -> void; auto appendVertex(typename Step::Vertex v) -> void; auto prependVertex(typename Step::Vertex v) -> void; auto appendEdge(typename Step::Edge e) -> void; auto prependEdge(typename Step::Edge e) -> void; - auto toVelocyPack(arangodb::velocypack::Builder& builder) -> void; - + + auto toVelocyPack(arangodb::velocypack::Builder& builder) -> void override; + + /** + * @brief Appends this path as a SchreierVector entry into the given builder + */ + auto toSchreierEntry(arangodb::velocypack::Builder& builder, size_t& currentLength) + -> void override; + auto isEmpty() const -> bool; private: + Step* _step; + std::vector _vertices; std::vector _edges; - + // Provider for the path ProviderType& _provider; + PathStoreType& _store; }; } // namespace graph } // namespace arangodb diff --git a/arangod/Graph/Providers/BaseProviderOptions.cpp b/arangod/Graph/Providers/BaseProviderOptions.cpp index 12e42e959f4a..b23bef9404c1 100644 --- a/arangod/Graph/Providers/BaseProviderOptions.cpp +++ b/arangod/Graph/Providers/BaseProviderOptions.cpp @@ -27,11 +27,20 @@ using namespace arangodb; using namespace arangodb::graph; IndexAccessor::IndexAccessor(transaction::Methods::IndexHandle idx, - aql::AstNode* condition, std::optional memberToUpdate) - : _idx(idx), _indexCondition(condition), _memberToUpdate(memberToUpdate) {} + aql::AstNode* condition, std::optional memberToUpdate, + std::shared_ptr expression) + : _idx(idx), _indexCondition(condition), _memberToUpdate(memberToUpdate) { + if (expression != nullptr) { + _expression = std::move(expression); + } +} aql::AstNode* IndexAccessor::getCondition() const { return _indexCondition; } +aql::Expression* IndexAccessor::getExpression() const { + return _expression.get(); +} + transaction::Methods::IndexHandle IndexAccessor::indexHandle() const { return _idx; } @@ -40,25 +49,34 @@ std::optional IndexAccessor::getMemberToUpdate() const { return _memberToUpdate; } -BaseProviderOptions::BaseProviderOptions(aql::Variable const* tmpVar, - std::vector indexInfo, - std::map const& collectionToShardMap) +BaseProviderOptions::BaseProviderOptions( + aql::Variable const* tmpVar, + std::pair, std::unordered_map>> indexInfo, + aql::FixedVarExpressionContext& expressionContext, + std::unordered_map> const& collectionToShardMap) : _temporaryVariable(tmpVar), _indexInformation(std::move(indexInfo)), + _expressionContext(expressionContext), _collectionToShardMap(collectionToShardMap) {} aql::Variable const* BaseProviderOptions::tmpVar() const { return _temporaryVariable; } -std::vector const& BaseProviderOptions::indexInformations() const { +// first is global index information, second is depth-based index information. +std::pair, std::unordered_map>> const& +BaseProviderOptions::indexInformations() const { return _indexInformation; } -std::map const& BaseProviderOptions::collectionToShardMap() const { +std::unordered_map> const& BaseProviderOptions::collectionToShardMap() const { return _collectionToShardMap; } +aql::FixedVarExpressionContext& BaseProviderOptions::expressionContext() const { + return _expressionContext; +} + ClusterBaseProviderOptions::ClusterBaseProviderOptions( std::shared_ptr cache, std::unordered_map const* engines, bool backward) @@ -77,4 +95,4 @@ bool ClusterBaseProviderOptions::isBackward() const { return _backward; } std::unordered_map const* ClusterBaseProviderOptions::engines() const { TRI_ASSERT(_engines != nullptr); return _engines; -} \ No newline at end of file +} diff --git a/arangod/Graph/Providers/BaseProviderOptions.h b/arangod/Graph/Providers/BaseProviderOptions.h index 6f2ce9bed657..ae84dc818a5b 100644 --- a/arangod/Graph/Providers/BaseProviderOptions.h +++ b/arangod/Graph/Providers/BaseProviderOptions.h @@ -23,6 +23,7 @@ #pragma once +#include "Aql/Expression.h" #include "Aql/FixedVarExpressionContext.h" #include "Cluster/ClusterInfo.h" #include "Graph/Cache/RefactoredClusterTraverserCache.h" @@ -41,9 +42,11 @@ namespace graph { struct IndexAccessor { IndexAccessor(transaction::Methods::IndexHandle idx, aql::AstNode* condition, - std::optional memberToUpdate); + std::optional memberToUpdate, + std::shared_ptr expression); aql::AstNode* getCondition() const; + aql::Expression* getExpression() const; transaction::Methods::IndexHandle indexHandle() const; std::optional getMemberToUpdate() const; @@ -51,27 +54,39 @@ struct IndexAccessor { transaction::Methods::IndexHandle _idx; aql::AstNode* _indexCondition; std::optional _memberToUpdate; + + // TODO: This needs to be changed BEFORE merge + std::shared_ptr _expression; }; struct BaseProviderOptions { public: - BaseProviderOptions(aql::Variable const* tmpVar, std::vector indexInfo, - std::map const& collectionToShardMap); + BaseProviderOptions( + aql::Variable const* tmpVar, + std::pair, std::unordered_map>> indexInfo, + aql::FixedVarExpressionContext& expressionContext, + std::unordered_map> const& collectionToShardMap); aql::Variable const* tmpVar() const; - std::vector const& indexInformations() const; + std::pair, std::unordered_map>> const& indexInformations() const; + + std::unordered_map> const& collectionToShardMap() const; - std::map const& collectionToShardMap() const; + aql::FixedVarExpressionContext& expressionContext() const; private: // The temporary Variable used in the Indexes aql::Variable const* _temporaryVariable; // One entry per collection, ShardTranslation needs // to be done by Provider - std::vector _indexInformation; + std::pair, std::unordered_map>> _indexInformation; + + // The context of AQL variables. These variables are set from the outside. + // and the caller needs to make sure the reference stays valid + aql::FixedVarExpressionContext& _expressionContext; // CollectionName to ShardMap, used if the Traversal is pushed down to DBServer - std::map const& _collectionToShardMap; + std::unordered_map> const& _collectionToShardMap; }; struct ClusterBaseProviderOptions { @@ -96,4 +111,3 @@ struct ClusterBaseProviderOptions { } // namespace graph } // namespace arangodb - diff --git a/arangod/Graph/Providers/BaseStep.h b/arangod/Graph/Providers/BaseStep.h index 81bc49531500..c9a49139d98c 100644 --- a/arangod/Graph/Providers/BaseStep.h +++ b/arangod/Graph/Providers/BaseStep.h @@ -23,9 +23,12 @@ #pragma once +#include + #include namespace arangodb { + namespace graph { template @@ -34,6 +37,7 @@ class BaseStep { BaseStep() : _previous{std::numeric_limits::max()}, _depth{0} {} BaseStep(size_t prev) : _previous{prev}, _depth{0} {} + BaseStep(size_t prev, size_t depth) : _previous{prev}, _depth{depth} {} size_t getPrevious() const { return _previous; } @@ -46,13 +50,26 @@ class BaseStep { return static_cast(this)->isLooseEnd(); } - size_t getDepth() const { - return _depth; + size_t getDepth() const { return _depth; } + + ResultT> extractCollectionName( + arangodb::velocypack::HashedStringRef const& idHashed) const { + size_t pos = idHashed.find('/'); + if (pos == std::string::npos) { + // Invalid input. If we get here somehow we managed to store invalid + // _from/_to values or the traverser did a let an illegal start through + TRI_ASSERT(false); + return Result{TRI_ERROR_GRAPH_INVALID_EDGE, + "edge contains invalid value " + idHashed.toString()}; + } + + std::string colName = idHashed.substr(0, pos).toString(); + return std::make_pair(colName, pos); } private: - size_t const _previous; - size_t const _depth; + size_t _previous; + size_t _depth; }; } // namespace graph } // namespace arangodb diff --git a/arangod/Graph/Providers/ClusterProvider.cpp b/arangod/Graph/Providers/ClusterProvider.cpp index 6a643cf17bf6..8e4425c08742 100644 --- a/arangod/Graph/Providers/ClusterProvider.cpp +++ b/arangod/Graph/Providers/ClusterProvider.cpp @@ -68,7 +68,6 @@ VertexType getEdgeDestination(arangodb::velocypack::Slice edge, VertexType const } } // namespace - namespace arangodb { namespace graph { auto operator<<(std::ostream& out, ClusterProvider::Step const& step) -> std::ostream& { @@ -94,9 +93,11 @@ VertexType const& ClusterProvider::Step::Vertex::getID() const { } EdgeType const& ClusterProvider::Step::Edge::getID() const { return _edge; } -bool ClusterProvider::Step::Edge::isValid() const { - return !_edge.empty(); -}; +bool ClusterProvider::Step::Edge::isValid() const { return !_edge.empty(); }; + +bool ClusterProvider::Step::isResponsible(transaction::Methods* trx) const { + return true; +} ClusterProvider::ClusterProvider(arangodb::aql::QueryContext& queryContext, ClusterBaseProviderOptions opts, @@ -107,17 +108,16 @@ ClusterProvider::ClusterProvider(arangodb::aql::QueryContext& queryContext, _opts(std::move(opts)), _stats{} {} -ClusterProvider::~ClusterProvider() { - clear(); -} +ClusterProvider::~ClusterProvider() { clear(); } void ClusterProvider::clear() { for (auto const& entry : _vertexConnectedEdges) { - _resourceMonitor->decreaseMemoryUsage(costPerVertexOrEdgeType + (entry.second.size() * (costPerVertexOrEdgeType * 2))); + _resourceMonitor->decreaseMemoryUsage( + costPerVertexOrEdgeType + (entry.second.size() * (costPerVertexOrEdgeType * 2))); } } -auto ClusterProvider::startVertex(VertexType vertex) -> Step { +auto ClusterProvider::startVertex(VertexType vertex, size_t depth) -> Step { LOG_TOPIC("da308", TRACE, Logger::GRAPHS) << " Start Vertex:" << vertex; // Create the default initial step. return Step(_opts.getCache()->persistString(vertex)); @@ -203,9 +203,8 @@ void ClusterProvider::fetchVerticesFromEngines(std::vector const& looseEn _opts.getCache()->datalake().add(std::move(payload)); } - /* TODO: Needs to be taken care of as soon as we enable shortest paths for ClusterProvider - bool forShortestPath = true; - if (!forShortestPath) { + /* TODO: Needs to be taken care of as soon as we enable shortest paths for + ClusterProvider bool forShortestPath = true; if (!forShortestPath) { // Fill everything we did not find with NULL for (auto const& v : vertexIds) { result.try_emplace(v, VPackSlice::nullSlice()); @@ -222,8 +221,10 @@ void ClusterProvider::fetchVerticesFromEngines(std::vector const& looseEn for (auto& lE : looseEnds) { if (!_opts.getCache()->isVertexCached(lE->getVertexIdentifier())) { // if we end up here, we we're not able to cache the requested vertex (e.g. it does not exist) - _query->warnings().registerWarning(TRI_ERROR_ARANGO_DOCUMENT_NOT_FOUND, lE->getVertexIdentifier().toString()); - _opts.getCache()->cacheVertex(std::move(lE->getVertexIdentifier()), VPackSlice::nullSlice()); + _query->warnings().registerWarning(TRI_ERROR_ARANGO_DOCUMENT_NOT_FOUND, + lE->getVertexIdentifier().toString()); + _opts.getCache()->cacheVertex(std::move(lE->getVertexIdentifier()), + VPackSlice::nullSlice()); } result.emplace_back(std::move(lE)); } @@ -249,18 +250,19 @@ void ClusterProvider::destroyEngines() { auto const* engines = _opts.engines(); for (auto const& engine : *engines) { _stats.addHttpRequests(1); - auto res = network::sendRequestRetry(pool, "server:" + engine.first, fuerte::RestVerb::Delete, - "/_internal/traverser/" + - arangodb::basics::StringUtils::itoa(engine.second), - VPackBuffer(), options) - .get(); + auto res = + network::sendRequestRetry(pool, "server:" + engine.first, fuerte::RestVerb::Delete, + "/_internal/traverser/" + + arangodb::basics::StringUtils::itoa(engine.second), + VPackBuffer(), options) + .get(); if (res.error != fuerte::Error::NoError) { // Note If there was an error on server side we do not have // CL_COMM_SENT LOG_TOPIC("d31a5", ERR, arangodb::Logger::GRAPHS) - << "Could not destroy all traversal engines: " - << TRI_errno_string(network::fuerteToArangoErrorCode(res)); + << "Could not destroy all traversal engines: " + << TRI_errno_string(network::fuerteToArangoErrorCode(res)); } } } @@ -339,7 +341,8 @@ Result ClusterProvider::fetchEdgesFromEngines(VertexType const& vertex) { arangodb::velocypack::HashedStringRef edgeIdRef(edge.get(StaticStrings::IdString)); - auto edgeToEmplace = std::make_pair(edgeIdRef, VertexType{getEdgeDestination(edge, vertex)}); + auto edgeToEmplace = + std::make_pair(edgeIdRef, VertexType{getEdgeDestination(edge, vertex)}); connectedEdges.emplace_back(edgeToEmplace); } @@ -351,7 +354,8 @@ Result ClusterProvider::fetchEdgesFromEngines(VertexType const& vertex) { // Note: This disables the TRI_DEFER futures.clear(); - _resourceMonitor->increaseMemoryUsage(costPerVertexOrEdgeType + (connectedEdges.size() * (costPerVertexOrEdgeType * 2))); + _resourceMonitor->increaseMemoryUsage( + costPerVertexOrEdgeType + (connectedEdges.size() * (costPerVertexOrEdgeType * 2))); _vertexConnectedEdges.emplace(vertex, std::move(connectedEdges)); return TRI_ERROR_NO_ERROR; @@ -398,12 +402,12 @@ auto ClusterProvider::expand(Step const& step, size_t previous, TRI_ASSERT(_opts.getCache()->isVertexCached(vertex.getID())); TRI_ASSERT(_vertexConnectedEdges.find(vertex.getID()) != _vertexConnectedEdges.end()); for (auto const& relation : _vertexConnectedEdges.at(vertex.getID())) { - bool fetched = _vertexConnectedEdges.find(relation.second) != _vertexConnectedEdges.end(); + bool fetched = + _vertexConnectedEdges.find(relation.second) != _vertexConnectedEdges.end(); callback(Step{relation.second, relation.first, previous, fetched}); } } - void ClusterProvider::addVertexToBuilder(Step::Vertex const& vertex, arangodb::velocypack::Builder& builder) { TRI_ASSERT(_opts.getCache()->isVertexCached(vertex.getID())); diff --git a/arangod/Graph/Providers/ClusterProvider.h b/arangod/Graph/Providers/ClusterProvider.h index 76ebb47fafdc..e10dd2f1a9c5 100644 --- a/arangod/Graph/Providers/ClusterProvider.h +++ b/arangod/Graph/Providers/ClusterProvider.h @@ -121,6 +121,29 @@ class ClusterProvider { VertexType getVertexIdentifier() const { return _vertex.getID(); } + std::string getCollectionName() const { + auto collectionNameResult = extractCollectionName(_vertex.getID()); + if (collectionNameResult.fail()) { + THROW_ARANGO_EXCEPTION(collectionNameResult.result()); + } + return collectionNameResult.get().first; + }; + + /* to be moved-out later into seperate step */ + void setLocalSchreierIndex(size_t index) { + TRI_ASSERT(index != std::numeric_limits::max()); + TRI_ASSERT(!hasLocalSchreierIndex()); + _localSchreierIndex = index; + } + + bool hasLocalSchreierIndex() const { + return _localSchreierIndex != std::numeric_limits::max(); + } + + std::size_t getLocalSchreierIndex() const { return _localSchreierIndex; } + + bool isResponsible(transaction::Methods* trx) const; + friend auto operator<<(std::ostream& out, Step const& step) -> std::ostream&; private: @@ -130,6 +153,7 @@ class ClusterProvider { Vertex _vertex; Edge _edge; bool _fetched; + size_t _localSchreierIndex = std::numeric_limits::max(); // to be removed later }; public: @@ -143,7 +167,7 @@ class ClusterProvider { void clear(); - auto startVertex(VertexType vertex) -> Step; + auto startVertex(VertexType vertex, size_t depth = 0) -> Step; auto fetch(std::vector const& looseEnds) -> futures::Future>; auto expand(Step const& from, size_t previous, std::function const& callback) -> void; diff --git a/arangod/Graph/Providers/ProviderTracer.cpp b/arangod/Graph/Providers/ProviderTracer.cpp index 2efcce0bd0c6..e78e41dfae7f 100644 --- a/arangod/Graph/Providers/ProviderTracer.cpp +++ b/arangod/Graph/Providers/ProviderTracer.cpp @@ -37,7 +37,7 @@ template ProviderTracer::ProviderTracer(arangodb::aql::QueryContext& queryContext, Options opts, arangodb::ResourceMonitor& resourceMonitor) - : _impl{queryContext, opts, resourceMonitor} {} + : _impl{queryContext, std::move(opts), resourceMonitor} {} template ProviderTracer::~ProviderTracer() { @@ -48,10 +48,10 @@ ProviderTracer::~ProviderTracer() { } template -typename ProviderImpl::Step ProviderTracer::startVertex(VertexType vertex) { +typename ProviderImpl::Step ProviderTracer::startVertex(VertexType vertex, size_t depth) { double start = TRI_microtime(); TRI_DEFER(_stats["startVertex"].addTiming(TRI_microtime() - start)); - return _impl.startVertex(vertex); + return _impl.startVertex(vertex, depth); } template diff --git a/arangod/Graph/Providers/ProviderTracer.h b/arangod/Graph/Providers/ProviderTracer.h index 13ed9cf33aaf..3c6f852ec37c 100644 --- a/arangod/Graph/Providers/ProviderTracer.h +++ b/arangod/Graph/Providers/ProviderTracer.h @@ -59,7 +59,7 @@ class ProviderTracer { ProviderTracer& operator=(ProviderTracer const&) = delete; ProviderTracer& operator=(ProviderTracer&&) = default; - auto startVertex(VertexType vertex) -> Step; + auto startVertex(VertexType vertex, size_t depth = 0) -> Step; auto fetch(std::vector const& looseEnds) -> futures::Future>; // rocks auto expand(Step const& from, size_t previous, std::function callback) -> void; // index diff --git a/arangod/Graph/Providers/SingleServerProvider.cpp b/arangod/Graph/Providers/SingleServerProvider.cpp index 245f6abf1d8c..46ac1b1bb44f 100644 --- a/arangod/Graph/Providers/SingleServerProvider.cpp +++ b/arangod/Graph/Providers/SingleServerProvider.cpp @@ -47,6 +47,9 @@ auto operator<<(std::ostream& out, SingleServerProvider::Step const& step) -> st SingleServerProvider::Step::Step(VertexType v) : _vertex(v), _edge() {} +SingleServerProvider::Step::Step(VertexType v, size_t depth) + : BaseStep(std::numeric_limits::max(), depth), _vertex(v), _edge() {} + SingleServerProvider::Step::Step(VertexType v, EdgeDocumentToken edge, size_t prev) : BaseStep(prev), _vertex(v), _edge(std::move(edge)) {} @@ -64,12 +67,24 @@ EdgeDocumentToken const& SingleServerProvider::Step::Edge::getID() const { } bool SingleServerProvider::Step::Edge::isValid() const { - return getID().localDocumentId() != DataSourceId::none(); + return getID().isValid(); +}; + +#ifndef USE_ENTERPRISE +bool SingleServerProvider::Step::isResponsible(transaction::Methods* trx) const { + return true; }; +#endif + + void SingleServerProvider::addEdgeToBuilder(Step::Edge const& edge, arangodb::velocypack::Builder& builder) { - insertEdgeIntoResult(edge.getID(), builder); + if (edge.isValid()) { + insertEdgeIntoResult(edge.getID(), builder); + } else { + builder.add(VPackSlice::nullSlice()); + } }; void SingleServerProvider::Step::Edge::addToBuilder(SingleServerProvider& provider, @@ -86,7 +101,7 @@ SingleServerProvider::SingleServerProvider(arangodb::aql::QueryContext& queryCon _opts.collectionToShardMap()), _stats{} { // activateCache(false); // TODO CHECK RefactoredTraverserCache (will be discussed in the future, need to do benchmarks if affordable) - _cursor = buildCursor(); + _cursor = buildCursor(opts.expressionContext()); } void SingleServerProvider::activateCache(bool enableDocumentCache) { @@ -108,13 +123,13 @@ void SingleServerProvider::activateCache(bool enableDocumentCache) { // _cache = new RefactoredTraverserCache(query()); } -auto SingleServerProvider::startVertex(VertexType vertex) -> Step { +auto SingleServerProvider::startVertex(VertexType vertex, size_t depth) -> Step { LOG_TOPIC("78156", TRACE, Logger::GRAPHS) << " Start Vertex:" << vertex; // Create default initial step // Note: Refactor naming, Strings in our cache here are not allowed to be removed. - return Step(_cache.persistString(vertex)); + return Step(_cache.persistString(vertex), depth); } auto SingleServerProvider::fetch(std::vector const& looseEnds) @@ -134,21 +149,28 @@ auto SingleServerProvider::expand(Step const& step, size_t previous, TRI_ASSERT(!step.isLooseEnd()); auto const& vertex = step.getVertex(); TRI_ASSERT(_cursor != nullptr); + LOG_TOPIC("c9169", TRACE, Logger::GRAPHS) + << " Expanding " << vertex.getID(); _cursor->rearm(vertex.getID(), 0); - _cursor->readAll(_stats, [&](EdgeDocumentToken&& eid, VPackSlice edge, size_t /*cursorIdx*/) -> void { - VertexType id = _cache.persistString(([&]() -> auto { - if (edge.isString()) { - return VertexType(edge); - } else { - VertexType other(transaction::helpers::extractFromFromDocument(edge)); - if (other == vertex.getID()) { // TODO: Check getId - discuss - other = VertexType(transaction::helpers::extractToFromDocument(edge)); - } - return other; - } - })()); - callback(Step{id, std::move(eid), previous}); - }); + _cursor->readAll( + *this, _stats, step.getDepth(), + [&](EdgeDocumentToken&& eid, VPackSlice edge, size_t /*cursorIdx*/) -> void { + VertexType id = _cache.persistString(([&]() -> auto { + if (edge.isString()) { + return VertexType(edge); + } else { + VertexType other(transaction::helpers::extractFromFromDocument(edge)); + if (other == vertex.getID()) { // TODO: Check getId - discuss + other = VertexType(transaction::helpers::extractToFromDocument(edge)); + } + return other; + } + })()); + // TODO: Adjust log output + LOG_TOPIC("c9168", TRACE, Logger::GRAPHS) + << " Neighbor of " << vertex.getID() << " -> " << id; + callback(Step{id, std::move(eid), previous, step.getDepth() + 1}); + }); } void SingleServerProvider::addVertexToBuilder(Step::Vertex const& vertex, @@ -161,12 +183,17 @@ void SingleServerProvider::insertEdgeIntoResult(EdgeDocumentToken edge, _cache.insertEdgeIntoResult(edge, builder); } -std::unique_ptr SingleServerProvider::buildCursor() { - return std::make_unique(trx(), _opts.tmpVar(), - _opts.indexInformations()); +std::unique_ptr SingleServerProvider::buildCursor( + arangodb::aql::FixedVarExpressionContext& expressionContext) { + return std::make_unique( + trx(), _opts.tmpVar(), _opts.indexInformations().first, + _opts.indexInformations().second, expressionContext); } arangodb::transaction::Methods* SingleServerProvider::trx() { + TRI_ASSERT(_trx != nullptr); + TRI_ASSERT(_trx->state() != nullptr); + TRI_ASSERT(_trx->transactionContextPtr() != nullptr); return _trx.get(); } diff --git a/arangod/Graph/Providers/SingleServerProvider.h b/arangod/Graph/Providers/SingleServerProvider.h index 3f2ee786362f..d35e28b8f7ab 100644 --- a/arangod/Graph/Providers/SingleServerProvider.h +++ b/arangod/Graph/Providers/SingleServerProvider.h @@ -68,7 +68,7 @@ struct SingleServerProvider { explicit Vertex(VertexType v) : _vertex(v) {} VertexType const& getID() const; - + bool operator<(Vertex const& other) const noexcept { return _vertex < other._vertex; } @@ -98,6 +98,7 @@ struct SingleServerProvider { Step(VertexType v); Step(VertexType v, EdgeDocumentToken edge, size_t prev); Step(VertexType v, EdgeDocumentToken edge, size_t prev, size_t depth); + Step(VertexType v, size_t depth); ~Step(); bool operator<(Step const& other) const noexcept { @@ -115,15 +116,40 @@ struct SingleServerProvider { VertexType getVertexIdentifier() const { return _vertex.getID(); } + std::string getCollectionName() const { + auto collectionNameResult = extractCollectionName(_vertex.getID()); + if (collectionNameResult.fail()) { + THROW_ARANGO_EXCEPTION(collectionNameResult.result()); + } + return collectionNameResult.get().first; + }; + + void setLocalSchreierIndex(size_t index) { + TRI_ASSERT(index != std::numeric_limits::max()); + TRI_ASSERT(!hasLocalSchreierIndex()); + _localSchreierIndex = index; + } + + bool hasLocalSchreierIndex() const { + return _localSchreierIndex != std::numeric_limits::max(); + } + + std::size_t getLocalSchreierIndex() const { + return _localSchreierIndex; + } + + bool isResponsible(transaction::Methods* trx) const; + friend auto operator<<(std::ostream& out, Step const& step) -> std::ostream&; private: Vertex _vertex; Edge _edge; + size_t _localSchreierIndex = std::numeric_limits::max(); }; public: - SingleServerProvider(arangodb::aql::QueryContext& queryContext, BaseProviderOptions opts, + SingleServerProvider(arangodb::aql::QueryContext& queryContext, Options opts, arangodb::ResourceMonitor& resourceMonitor); SingleServerProvider(SingleServerProvider const&) = delete; SingleServerProvider(SingleServerProvider&&) = default; @@ -131,7 +157,7 @@ struct SingleServerProvider { SingleServerProvider& operator=(SingleServerProvider const&) = delete; - auto startVertex(VertexType vertex) -> Step; + auto startVertex(VertexType vertex, size_t depth = 0) -> Step; auto fetch(std::vector const& looseEnds) -> futures::Future>; // rocks auto expand(Step const& from, size_t previous, @@ -151,7 +177,8 @@ struct SingleServerProvider { private: void activateCache(bool enableDocumentCache); - std::unique_ptr buildCursor(); + std::unique_ptr buildCursor( + arangodb::aql::FixedVarExpressionContext& expressionContext); private: // Unique_ptr to have this class movable, and to keep reference of trx() @@ -163,9 +190,8 @@ struct SingleServerProvider { BaseProviderOptions _opts; RefactoredTraverserCache _cache; - + arangodb::aql::TraversalStats _stats; }; } // namespace graph } // namespace arangodb - diff --git a/arangod/Graph/SingleServerEdgeCursor.cpp b/arangod/Graph/SingleServerEdgeCursor.cpp index 6e6d94c082a2..912711975846 100644 --- a/arangod/Graph/SingleServerEdgeCursor.cpp +++ b/arangod/Graph/SingleServerEdgeCursor.cpp @@ -127,7 +127,7 @@ bool SingleServerEdgeCursor::advanceCursor(IndexIterator*& cursor, bool SingleServerEdgeCursor::next(EdgeCursor::Callback const& callback) { // fills callback with next EdgeDocumentToken and Slice that contains the - // ohter side of the edge (we are standing on a node and want to iterate all + // other side of the edge (we are standing on a node and want to iterate all // connected edges TRI_ASSERT(!_cursors.empty()); @@ -225,6 +225,7 @@ void SingleServerEdgeCursor::readAll(EdgeCursor::Callback const& callback) { return false; } #endif + _opts->cache()->increaseCounter(); callback(EdgeDocumentToken(cid, token), edge, cursorId); return true; }); diff --git a/arangod/Graph/TraverserCache.cpp b/arangod/Graph/TraverserCache.cpp index e11bfeb41fdb..5ab0431a2b9c 100644 --- a/arangod/Graph/TraverserCache.cpp +++ b/arangod/Graph/TraverserCache.cpp @@ -28,8 +28,8 @@ #include "Basics/StringHeap.h" #include "Basics/VelocyPackHelper.h" #include "Cluster/ServerState.h" -#include "Graph/EdgeDocumentToken.h" #include "Graph/BaseOptions.h" +#include "Graph/EdgeDocumentToken.h" #include "Logger/LogMacros.h" #include "Logger/Logger.h" #include "Logger/LoggerStream.h" @@ -42,8 +42,8 @@ #include "VocBase/ManagedDocumentResult.h" #include -#include #include +#include #include #include @@ -51,11 +51,11 @@ using namespace arangodb; using namespace arangodb::graph; namespace { -constexpr size_t costPerPersistedString = sizeof(void*) + sizeof(arangodb::velocypack::HashedStringRef); +constexpr size_t costPerPersistedString = + sizeof(void*) + sizeof(arangodb::velocypack::HashedStringRef); bool isWithClauseMissing(arangodb::basics::Exception const& ex) { - if (ServerState::instance()->isDBServer() && - ex.code() == TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND) { + if (ServerState::instance()->isDBServer() && ex.code() == TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND) { // on a DB server, we could have got here only in the OneShard case. // in this case turn the rather misleading "collection or view not found" // error into a nicer "collection not known to traversal, please add WITH" @@ -70,7 +70,7 @@ bool isWithClauseMissing(arangodb::basics::Exception const& ex) { return false; } -} // namespace +} // namespace TraverserCache::TraverserCache(aql::QueryContext& query, BaseOptions* opts) : _query(query), @@ -79,12 +79,11 @@ TraverserCache::TraverserCache(aql::QueryContext& query, BaseOptions* opts) _filteredDocuments(0), _stringHeap(query.resourceMonitor(), 4096), /* arbitrary block-size, may be adjusted for performance */ _baseOptions(opts), - _allowImplicitCollections(ServerState::instance()->isSingleServer() && - !_query.vocbase().server().getFeature().requireWith()) {} + _allowImplicitCollections( + ServerState::instance()->isSingleServer() && + !_query.vocbase().server().getFeature().requireWith()) {} -TraverserCache::~TraverserCache() { - clear(); -} +TraverserCache::~TraverserCache() { clear(); } void TraverserCache::clear() { _query.resourceMonitor().decreaseMemoryUsage(_persistedStrings.size() * ::costPerPersistedString); @@ -119,7 +118,8 @@ VPackSlice TraverserCache::lookupToken(EdgeDocumentToken const& idToken) { return VPackSlice(_mmdr.vpack()); } -bool TraverserCache::appendVertex(arangodb::velocypack::StringRef id, arangodb::velocypack::Builder& result) { +bool TraverserCache::appendVertex(arangodb::velocypack::StringRef id, + arangodb::velocypack::Builder& result) { if (!_baseOptions->produceVertices()) { // this traversal does not produce any vertices result.add(arangodb::velocypack::Slice::nullSlice()); @@ -141,20 +141,24 @@ bool TraverserCache::appendVertex(arangodb::velocypack::StringRef id, arangodb:: if (!map.empty()) { auto found = map.find(collectionName); if (found != map.end()) { - collectionName = found->second; + // Old API, could only convey exactly one Shard. + TRI_ASSERT(found->second.size() == 1); + collectionName = found->second.front(); } } try { - transaction::AllowImplicitCollectionsSwitcher disallower(_trx->state()->options(), _allowImplicitCollections); + transaction::AllowImplicitCollectionsSwitcher disallower(_trx->state()->options(), + _allowImplicitCollections); + + Result res = _trx->documentFastPathLocal(collectionName, id.substr(pos + 1), + [&](LocalDocumentId const&, VPackSlice doc) { + ++_insertedDocuments; + // copying... + result.add(doc); + return true; + }); - Result res = _trx->documentFastPathLocal(collectionName, id.substr(pos + 1), [&](LocalDocumentId const&, VPackSlice doc) { - ++_insertedDocuments; - // copying... - result.add(doc); - return true; - }); - if (res.ok()) { return true; } @@ -165,13 +169,13 @@ bool TraverserCache::appendVertex(arangodb::velocypack::StringRef id, arangodb:: } } catch (basics::Exception const& ex) { if (isWithClauseMissing(ex)) { - // turn the error into a different error + // turn the error into a different error THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_COLLECTION_LOCK_FAILED, - "collection not known to traversal: '" + - collectionName + "'. please add 'WITH " + collectionName + - "' as the first line in your AQL"); + "collection not known to traversal: '" + collectionName + + "'. please add 'WITH " + collectionName + + "' as the first line in your AQL"); } - // rethrow original error + // rethrow original error throw; } @@ -185,7 +189,8 @@ bool TraverserCache::appendVertex(arangodb::velocypack::StringRef id, arangodb:: return false; } -bool TraverserCache::appendVertex(arangodb::velocypack::StringRef id, arangodb::aql::AqlValue& result) { +bool TraverserCache::appendVertex(arangodb::velocypack::StringRef id, + arangodb::aql::AqlValue& result) { result = arangodb::aql::AqlValue(arangodb::aql::AqlValueHintNull()); if (!_baseOptions->produceVertices()) { @@ -207,20 +212,25 @@ bool TraverserCache::appendVertex(arangodb::velocypack::StringRef id, arangodb:: if (!map.empty()) { auto found = map.find(collectionName); if (found != map.end()) { - collectionName = found->second; + // Old API, could only convey exactly one Shard. + TRI_ASSERT(found->second.size() == 1); + collectionName = found->second.front(); } } try { - transaction::AllowImplicitCollectionsSwitcher disallower(_trx->state()->options(), _allowImplicitCollections); + transaction::AllowImplicitCollectionsSwitcher disallower(_trx->state()->options(), + _allowImplicitCollections); + + Result res = + _trx->documentFastPathLocal(collectionName, id.substr(pos + 1), + [&](LocalDocumentId const&, VPackSlice doc) { + ++_insertedDocuments; + // copying... + result = arangodb::aql::AqlValue(doc); + return true; + }); - Result res = _trx->documentFastPathLocal(collectionName, id.substr(pos + 1), [&](LocalDocumentId const&, VPackSlice doc) { - ++_insertedDocuments; - // copying... - result = arangodb::aql::AqlValue(doc); - return true; - }); - if (res.ok()) { return true; } @@ -231,11 +241,11 @@ bool TraverserCache::appendVertex(arangodb::velocypack::StringRef id, arangodb:: } } catch (basics::Exception const& ex) { if (isWithClauseMissing(ex)) { - // turn the error into a different error + // turn the error into a different error THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_COLLECTION_LOCK_FAILED, - "collection not known to traversal: '" + - collectionName + "'. please add 'WITH " + collectionName + - "' as the first line in your AQL"); + "collection not known to traversal: '" + collectionName + + "'. please add 'WITH " + collectionName + + "' as the first line in your AQL"); } // rethrow original error throw; @@ -262,7 +272,9 @@ aql::AqlValue TraverserCache::fetchEdgeAqlResult(EdgeDocumentToken const& idToke } arangodb::velocypack::StringRef TraverserCache::persistString(arangodb::velocypack::StringRef idString) { - return persistString(arangodb::velocypack::HashedStringRef(idString.data(), static_cast(idString.size()))).stringRef(); + return persistString(arangodb::velocypack::HashedStringRef( + idString.data(), static_cast(idString.size()))) + .stringRef(); } arangodb::velocypack::HashedStringRef TraverserCache::persistString(arangodb::velocypack::HashedStringRef idString) { @@ -273,9 +285,9 @@ arangodb::velocypack::HashedStringRef TraverserCache::persistString(arangodb::ve auto res = _stringHeap.registerString(idString); { ResourceUsageScope guard(_query.resourceMonitor(), ::costPerPersistedString); - + _persistedStrings.emplace(res); - + // now make the TraverserCache responsible for memory tracking guard.steal(); } diff --git a/arangod/Graph/Types/ValidationResult.cpp b/arangod/Graph/Types/ValidationResult.cpp index ca269aaf610c..a23fe21fb18c 100644 --- a/arangod/Graph/Types/ValidationResult.cpp +++ b/arangod/Graph/Types/ValidationResult.cpp @@ -35,6 +35,8 @@ bool ValidationResult::isFiltered() const noexcept { return _type == Type::FILTER; } +void ValidationResult::combine(Type t) noexcept { _type = std::max(_type, t); } + std::ostream& arangodb::graph::operator<<(std::ostream& stream, ValidationResult const& res) { switch (res._type) { case ValidationResult::Type::TAKE: diff --git a/arangod/Graph/Types/ValidationResult.h b/arangod/Graph/Types/ValidationResult.h index 09ebef670492..41695b1ddc9c 100644 --- a/arangod/Graph/Types/ValidationResult.h +++ b/arangod/Graph/Types/ValidationResult.h @@ -39,6 +39,8 @@ class ValidationResult { bool isPruned() const noexcept; bool isFiltered() const noexcept; + void combine(Type t) noexcept; + private: Type _type; }; diff --git a/arangod/Graph/algorithm-aliases.h b/arangod/Graph/algorithm-aliases.h index 2faa74fcdf19..b76872a64475 100644 --- a/arangod/Graph/algorithm-aliases.h +++ b/arangod/Graph/algorithm-aliases.h @@ -28,7 +28,10 @@ #include "Graph/Queues/FifoQueue.h" #include "Graph/Queues/LifoQueue.h" +#include "Graph/Queues/QueueTracer.h" +#include "Graph/PathManagement/PathStore.h" +#include "Graph/PathManagement/PathStoreTracer.h" #include "Graph/PathManagement/PathValidator.h" #include "Graph/Providers/ProviderTracer.h" #include "Graph/Types/UniquenessLevel.h" @@ -40,54 +43,60 @@ namespace graph { template using KPathEnumerator = TwoSidedEnumerator, PathStore, Provider, - PathValidator, VertexUniquenessLevel::PATH>>; + PathValidator, VertexUniquenessLevel::PATH>>; // K_PATH implementation using Tracing template using TracedKPathEnumerator = TwoSidedEnumerator>, PathStoreTracer>, ProviderTracer, - PathValidator>, VertexUniquenessLevel::PATH>>; + PathValidator, PathStoreTracer>, VertexUniquenessLevel::PATH>>; + +template +struct BFSConfiguration { + using Provider = + typename std::conditional, ProviderType>::type; + using Step = typename Provider::Step; + using Queue = + typename std::conditional>, FifoQueue>::type; + using Store = + typename std::conditional>, PathStore>::type; + using Validator = PathValidator; +}; + +template +struct DFSConfiguration { + using Provider = + typename std::conditional, ProviderType>::type; + using Step = typename Provider::Step; + using Queue = + typename std::conditional>, LifoQueue>::type; + using Store = + typename std::conditional>, PathStore>::type; + using Validator = PathValidator; +}; // BFS Traversal Enumerator implementation -template +template using BFSEnumerator = - OneSidedEnumerator, PathStore, Provider, - PathValidator, VertexUniquenessLevel::PATH>>; + OneSidedEnumerator>; // BFS Traversal Enumerator implementation using Tracing -template +template using TracedBFSEnumerator = - OneSidedEnumerator>, - PathStoreTracer>, ProviderTracer, - PathValidator>, VertexUniquenessLevel::PATH>>; + OneSidedEnumerator>; // BFS Traversal Enumerator implementation using Tracing (without provider tracing) -template -using TracedBFSEnumeratorWOPT = - OneSidedEnumerator>, - PathStoreTracer>, Provider, - PathValidator>, VertexUniquenessLevel::PATH>>; // DFS Traversal Enumerator implementation -template +template using DFSEnumerator = - OneSidedEnumerator, PathStore, Provider, - PathValidator, VertexUniquenessLevel::PATH>>; + OneSidedEnumerator>; // DFS Traversal Enumerator implementation using Tracing -template +template using TracedDFSEnumerator = - OneSidedEnumerator>, - PathStoreTracer>, ProviderTracer, - PathValidator>, VertexUniquenessLevel::PATH>>; - -// DFS Traversal Enumerator implementation using Tracing -template -using TracedDFSEnumeratorWOPT = - OneSidedEnumerator>, - PathStoreTracer>, Provider, - PathValidator>, VertexUniquenessLevel::PATH>>; + OneSidedEnumerator>; } // namespace graph } // namespace arangodb diff --git a/arangod/InternalRestHandler/InternalRestTraverserHandler.cpp b/arangod/InternalRestHandler/InternalRestTraverserHandler.cpp index 15268b542be9..753cc29d9ea2 100644 --- a/arangod/InternalRestHandler/InternalRestTraverserHandler.cpp +++ b/arangod/InternalRestHandler/InternalRestTraverserHandler.cpp @@ -232,7 +232,7 @@ void InternalRestTraverserHandler::queryEngine() { // Safe cast BaseTraverserEngines are all of type TRAVERSER auto eng = static_cast(engine); TRI_ASSERT(eng != nullptr); - eng->smartSearch(body, result); + eng->smartSearchNew(body, result); } else if (option == "smartSearchBFS") { if (engine->getType() != BaseEngine::EngineType::TRAVERSER) { generateError(ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER, diff --git a/arangod/Transaction/Methods.h b/arangod/Transaction/Methods.h index 69cc9c45c546..7fc1a6c001e3 100644 --- a/arangod/Transaction/Methods.h +++ b/arangod/Transaction/Methods.h @@ -483,7 +483,7 @@ class Methods { /// @brief the transaction context std::shared_ptr _transactionContext; - + bool _mainTransaction; private: diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 0ec13848ceda..01b39e9a861d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -62,6 +62,8 @@ set(ARANGODB_TESTS_COMMON_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Mocks/PreparedResponseConnectionPool.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Mocks/StorageEngineMock.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Mocks/Servers.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Mocks/MockGraph.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Mocks/MockGraphProvider.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Basics/icu-helper.cpp ${CMAKE_CURRENT_SOURCE_DIR}/IResearch/AgencyMock.cpp ${CMAKE_CURRENT_SOURCE_DIR}/IResearch/common.cpp @@ -229,8 +231,6 @@ set(ARANGODB_TESTS_SOURCES Graph/KPathFinderTest.cpp Graph/DFSFinderTest.cpp Graph/KShortestPathsFinder.cpp - Graph/MockGraph.cpp - Graph/MockGraphProvider.cpp Graph/PathStoreTest.cpp Graph/PathValidatorTest.cpp Graph/FifoQueueTest.cpp @@ -325,6 +325,7 @@ endif() target_include_directories(arangodbtests PRIVATE ${INCLUDE_DIRECTORIES} + ${CMAKE_CURRENT_SOURCE_DIR}/Mocks/ ) if(MSVC AND USE_FAIL_ON_WARNINGS) diff --git a/tests/Graph/DFSFinderTest.cpp b/tests/Graph/DFSFinderTest.cpp index fd1148652c4d..c3ad3e87a23c 100644 --- a/tests/Graph/DFSFinderTest.cpp +++ b/tests/Graph/DFSFinderTest.cpp @@ -57,8 +57,8 @@ namespace tests { namespace graph { class DFSFinderTest : public ::testing::TestWithParam { - // using DFSFinder = DFSEnumerator; - using DFSFinder = TracedDFSEnumeratorWOPT; + // using DFSFinder = DFSEnumerator; + using DFSFinder = TracedDFSEnumerator; protected: bool activateLogging{false}; @@ -68,11 +68,23 @@ class DFSFinderTest : public ::testing::TestWithParamnewTrxContext()}; + aql::Variable _tmpVar{"tmp", 0, false}; + arangodb::aql::AqlFunctionsInternalCache _functionsCache{}; + arangodb::aql::FixedVarExpressionContext _expressionContext{_trx, *_query.get(), + _functionsCache}; + DFSFinderTest() { if (activateLogging) { Logger::GRAPHS.setLogLevel(LogLevel::TRACE); } + // Important Note: + // Tests are using a LifoQueue. In those tests we do guarantee fetching in order + // e.g. (1) expands to (2), (3), (4) + // we will first traverse (4), then (3), then (2) + /* a chain 1->2->3->4 */ mockGraph.addEdge(1, 2); mockGraph.addEdge(2, 3); @@ -158,8 +170,11 @@ class DFSFinderTest : public ::testing::TestWithParam DFSFinder { arangodb::graph::OneSidedEnumeratorOptions options{minDepth, maxDepth}; - return DFSFinder{MockGraphProvider(mockGraph, *_query.get(), looseEndBehaviour(), false), - std::move(options), resourceMonitor}; + PathValidatorOptions validatorOpts{&_tmpVar, _expressionContext}; + return DFSFinder({*_query.get(), + MockGraphProviderOptions{mockGraph, looseEndBehaviour(), false}, + resourceMonitor}, + std::move(options), std::move(validatorOpts), resourceMonitor); } auto vId(size_t nr) -> std::string { @@ -248,8 +263,9 @@ TEST_P(DFSFinderTest, no_path_exists) { EXPECT_FALSE(finder.isDone()); { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); pathEquals(result.slice(), {91}); pathStructureValid(result.slice(), 0); EXPECT_TRUE(finder.isDone()); @@ -258,7 +274,7 @@ TEST_P(DFSFinderTest, no_path_exists) { { result.clear(); // Try again to make sure we stay at non-existing - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); EXPECT_TRUE(result.isEmpty()); EXPECT_TRUE(finder.isDone()); @@ -282,8 +298,10 @@ TEST_P(DFSFinderTest, path_depth_0) { EXPECT_FALSE(finder.isDone()); { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathEquals(result.slice(), {1}); pathStructureValid(result.slice(), 0); EXPECT_TRUE(finder.isDone()); @@ -292,7 +310,7 @@ TEST_P(DFSFinderTest, path_depth_0) { { result.clear(); // Try again to make sure we stay at non-existing - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); EXPECT_TRUE(result.isEmpty()); EXPECT_TRUE(finder.isDone()); @@ -315,8 +333,9 @@ TEST_P(DFSFinderTest, path_depth_1) { EXPECT_FALSE(finder.isDone()); { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); pathStructureValid(result.slice(), 1); pathEquals(result.slice(), {1, 2}); @@ -327,7 +346,7 @@ TEST_P(DFSFinderTest, path_depth_1) { { result.clear(); // Try again to make sure we stay at non-existing - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); EXPECT_TRUE(result.isEmpty()); EXPECT_TRUE(finder.isDone()); @@ -351,8 +370,10 @@ TEST_P(DFSFinderTest, path_depth_2) { EXPECT_FALSE(finder.isDone()); { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 2); pathEquals(result.slice(), {1, 2, 3}); @@ -362,7 +383,7 @@ TEST_P(DFSFinderTest, path_depth_2) { { result.clear(); // Try again to make sure we stay at non-existing - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); EXPECT_TRUE(result.isEmpty()); EXPECT_TRUE(finder.isDone()); @@ -385,8 +406,10 @@ TEST_P(DFSFinderTest, path_depth_3) { EXPECT_FALSE(finder.isDone()); { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 3); pathEquals(result.slice(), {1, 2, 3, 4}); @@ -396,7 +419,7 @@ TEST_P(DFSFinderTest, path_depth_3) { { result.clear(); // Try again to make sure we stay at non-existing - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); EXPECT_TRUE(result.isEmpty()); EXPECT_TRUE(finder.isDone()); @@ -420,24 +443,30 @@ TEST_P(DFSFinderTest, path_diamond) { EXPECT_FALSE(finder.isDone()); { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 2); EXPECT_FALSE(finder.isDone()); } { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 2); EXPECT_FALSE(finder.isDone()); } { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 2); EXPECT_TRUE(finder.isDone()); @@ -446,7 +475,7 @@ TEST_P(DFSFinderTest, path_diamond) { { result.clear(); // Try again to make sure we stay at non-existing - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); EXPECT_TRUE(result.isEmpty()); EXPECT_TRUE(finder.isDone()); @@ -469,8 +498,10 @@ TEST_P(DFSFinderTest, path_depth_1_to_2) { EXPECT_FALSE(finder.isDone()); { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 1); pathEquals(result.slice(), {10, 12}); @@ -479,8 +510,10 @@ TEST_P(DFSFinderTest, path_depth_1_to_2) { { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 2); pathEquals(result.slice(), {10, 12, 13}); EXPECT_FALSE(finder.isDone()); @@ -488,8 +521,10 @@ TEST_P(DFSFinderTest, path_depth_1_to_2) { { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 2); pathEquals(result.slice(), {10, 12, 11}); EXPECT_FALSE(finder.isDone()); @@ -497,8 +532,10 @@ TEST_P(DFSFinderTest, path_depth_1_to_2) { { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 1); pathEquals(result.slice(), {10, 11}); EXPECT_TRUE(finder.isDone()); @@ -507,7 +544,7 @@ TEST_P(DFSFinderTest, path_depth_1_to_2) { { result.clear(); // Try again to make sure we stay at non-existing - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); EXPECT_TRUE(result.isEmpty()); EXPECT_TRUE(finder.isDone()); @@ -524,8 +561,10 @@ TEST_P(DFSFinderTest, path_depth_1_to_2_skip) { EXPECT_FALSE(finder.isDone()); { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 1); pathEquals(result.slice(), {10, 12}); @@ -541,8 +580,10 @@ TEST_P(DFSFinderTest, path_depth_1_to_2_skip) { { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 2); pathEquals(result.slice(), {10, 12, 11}); EXPECT_FALSE(finder.isDone()); @@ -550,8 +591,10 @@ TEST_P(DFSFinderTest, path_depth_1_to_2_skip) { { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 1); pathEquals(result.slice(), {10, 11}); EXPECT_TRUE(finder.isDone()); @@ -560,7 +603,177 @@ TEST_P(DFSFinderTest, path_depth_1_to_2_skip) { { result.clear(); // Try again to make sure we stay at non-existing - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); + EXPECT_FALSE(hasPath); + EXPECT_TRUE(result.isEmpty()); + EXPECT_TRUE(finder.isDone()); + } +} + +TEST_P(DFSFinderTest, path_loop) { + VPackBuilder result; + auto finder = pathFinder(1, 10); + + // Source and target are direct neighbors, there is only one path between them + auto source = vId(20); + + finder.reset(toHashedStringRef(source)); + + EXPECT_FALSE(finder.isDone()); + { + result.clear(); + auto hasPath = finder.getNextPath(); + EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + + pathStructureValid(result.slice(), 1); + pathEquals(result.slice(), {20, 21}); + + EXPECT_FALSE(finder.isDone()); + } + + { + result.clear(); + auto hasPath = finder.getNextPath(); + EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + + pathStructureValid(result.slice(), 2); + pathEquals(result.slice(), {20, 21, 22}); + + EXPECT_FALSE(finder.isDone()); + } + + { + result.clear(); + auto hasPath = finder.getNextPath(); + EXPECT_FALSE(hasPath); + EXPECT_TRUE(result.isEmpty()); + EXPECT_TRUE(finder.isDone()); + } +} + +TEST_P(DFSFinderTest, triangle_loop) { + VPackBuilder result; + auto finder = pathFinder(1, 10); + auto source = vId(30); + + finder.reset(toHashedStringRef(source)); + + EXPECT_FALSE(finder.isDone()); + { + result.clear(); + auto hasPath = finder.getNextPath(); + EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + + pathStructureValid(result.slice(), 1); + pathEquals(result.slice(), {30, 31}); + + EXPECT_FALSE(finder.isDone()); + } + + { + result.clear(); + auto hasPath = finder.getNextPath(); + EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + + pathStructureValid(result.slice(), 2); + pathEquals(result.slice(), {30, 31, 32}); + + EXPECT_FALSE(finder.isDone()); + } + + { + result.clear(); + auto hasPath = finder.getNextPath(); + EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + + pathStructureValid(result.slice(), 3); + pathEquals(result.slice(), {30, 31, 32, 34}); + + EXPECT_FALSE(finder.isDone()); + } + + { + result.clear(); + auto hasPath = finder.getNextPath(); + EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + + pathStructureValid(result.slice(), 3); + pathEquals(result.slice(), {30, 31, 32, 33}); + + EXPECT_FALSE(finder.isDone()); + } + + { + result.clear(); + // Try again to make sure we stay at non-existing + auto hasPath = finder.getNextPath(); + EXPECT_FALSE(hasPath); + EXPECT_TRUE(result.isEmpty()); + EXPECT_TRUE(finder.isDone()); + } +} + +TEST_P(DFSFinderTest, triangle_loop_skip) { + VPackBuilder result; + auto finder = pathFinder(1, 10); + auto source = vId(30); + + finder.reset(toHashedStringRef(source)); + + EXPECT_FALSE(finder.isDone()); + { + result.clear(); + auto hasPath = finder.getNextPath(); + EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + + pathStructureValid(result.slice(), 1); + pathEquals(result.slice(), {30, 31}); + + EXPECT_FALSE(finder.isDone()); + } + + { + result.clear(); + auto hasPath = finder.getNextPath(); + EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + + pathStructureValid(result.slice(), 2); + pathEquals(result.slice(), {30, 31, 32}); + + EXPECT_FALSE(finder.isDone()); + } + + { + result.clear(); + bool skipped = finder.skipPath(); + EXPECT_TRUE(skipped); + EXPECT_FALSE(finder.isDone()); + } + + { + result.clear(); + auto hasPath = finder.getNextPath(); + EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + + pathStructureValid(result.slice(), 3); + pathEquals(result.slice(), {30, 31, 32, 33}); + + EXPECT_FALSE(finder.isDone()); + } + + { + result.clear(); + // Try again to make sure we stay at non-existing + auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); EXPECT_TRUE(result.isEmpty()); EXPECT_TRUE(finder.isDone()); @@ -569,9 +782,6 @@ TEST_P(DFSFinderTest, path_depth_1_to_2_skip) { /* TODO: Add more tests * - path_depth_2_to_3 - * - path_loop - * - triangle_loop - * - triangle_loop_skip * - many_neighbours_source * - many_neighbours_target */ diff --git a/tests/Graph/FifoQueueTest.cpp b/tests/Graph/FifoQueueTest.cpp index 0eab67b09c03..5379d2c1d4e5 100644 --- a/tests/Graph/FifoQueueTest.cpp +++ b/tests/Graph/FifoQueueTest.cpp @@ -24,6 +24,7 @@ #include "Basics/GlobalResourceMonitor.h" #include "Basics/ResourceUsage.h" #include "Basics/StringUtils.h" +#include "Basics/ResultT.h" #include "Graph/Providers/BaseStep.h" #include "Graph/Queues/FifoQueue.h" diff --git a/tests/Graph/GenericGraphProviderTest.cpp b/tests/Graph/GenericGraphProviderTest.cpp index bb6e939ce3b9..8a00d3e64e43 100644 --- a/tests/Graph/GenericGraphProviderTest.cpp +++ b/tests/Graph/GenericGraphProviderTest.cpp @@ -64,11 +64,14 @@ class GraphProviderTest : public ::testing::Test { std::unique_ptr server{nullptr}; std::unique_ptr query{nullptr}; std::unique_ptr> clusterEngines{nullptr}; + std::unique_ptr _trx{}; arangodb::GlobalResourceMonitor global{}; arangodb::ResourceMonitor resourceMonitor{global}; + arangodb::aql::AqlFunctionsInternalCache _functionsCache{}; + std::unique_ptr _expressionContext; - std::map _emptyShardMap{}; + std::unordered_map> _emptyShardMap{}; GraphProviderTest() {} ~GraphProviderTest() {} @@ -86,8 +89,9 @@ class GraphProviderTest : public ::testing::Test { // We now have collections "v" and "e" query = singleServer->getQuery("RETURN 1", {"v", "e"}); - return MockGraphProvider(graph, *query.get(), - MockGraphProvider::LooseEndBehaviour::NEVER); + return MockGraphProvider(*query.get(), + MockGraphProviderOptions{graph, MockGraphProvider::LooseEndBehaviour::NEVER}, + resourceMonitor); } if constexpr (std::is_same_v) { s = std::make_unique(); @@ -97,15 +101,24 @@ class GraphProviderTest : public ::testing::Test { // We now have collections "v" and "e" query = singleServer->getQuery("RETURN 1", {"v", "e"}); + _trx = std::make_unique(query->newTrxContext()); auto edgeIndexHandle = singleServer->getEdgeIndexHandle("e"); auto tmpVar = singleServer->generateTempVar(query.get()); auto indexCondition = singleServer->buildOutboundCondition(query.get(), tmpVar); - std::vector usedIndexes{ - IndexAccessor{edgeIndexHandle, indexCondition, 0}}; + std::vector usedIndexes{}; + usedIndexes.emplace_back(IndexAccessor{edgeIndexHandle, indexCondition, 0, nullptr}); - BaseProviderOptions opts(tmpVar, std::move(usedIndexes), _emptyShardMap); + _expressionContext = + std::make_unique(*_trx.get(), *query, + _functionsCache); + + BaseProviderOptions opts( + tmpVar, + std::make_pair(std::move(usedIndexes), + std::unordered_map>{}), + *_expressionContext.get(), _emptyShardMap); return SingleServerProvider(*query.get(), std::move(opts), resourceMonitor); } if constexpr (std::is_same_v) { @@ -122,10 +135,9 @@ class GraphProviderTest : public ::testing::Test { server.getSystemDatabase()); arangodb::aql::Query fakeQuery(ctx, queryString, nullptr); try { - fakeQuery.collections().add("s9880", AccessMode::Type::READ, - arangodb::aql::Collection::Hint::Shard); - } catch(...) { - + fakeQuery.collections().add("s9880", AccessMode::Type::READ, + arangodb::aql::Collection::Hint::Shard); + } catch (...) { } fakeQuery.prepareQuery(SerializationFormat::SHADOWROWS); auto ast = fakeQuery.ast(); @@ -137,10 +149,10 @@ class GraphProviderTest : public ::testing::Test { opts.setVariable(tmpVar); auto const* access = - ast->createNodeAttributeAccess(tmpVarRef, - StaticStrings::FromString.c_str(), - StaticStrings::FromString.length()); - auto const* cond = ast->createNodeBinaryOperator(NODE_TYPE_OPERATOR_BINARY_EQ, access, tmpIdNode); + ast->createNodeAttributeAccess(tmpVarRef, StaticStrings::FromString.c_str(), + StaticStrings::FromString.length()); + auto const* cond = + ast->createNodeBinaryOperator(NODE_TYPE_OPERATOR_BINARY_EQ, access, tmpIdNode); auto fromCondition = ast->createNodeNaryOperator(NODE_TYPE_OPERATOR_NARY_AND); fromCondition->addMember(cond); opts.addLookupInfo(fakeQuery.plan(), "s9880", StaticStrings::FromString, fromCondition); @@ -158,9 +170,13 @@ class GraphProviderTest : public ::testing::Test { std::tie(preparedResponses, engineId) = graph.simulateApi(server, expectedVerticesEdgesBundleToFetch, opts); // Note: Please don't remove for debugging purpose. - /*for (auto const& resp : preparedResponses) { - LOG_DEVEL << resp.generateResponse()->copyPayload().get()->toString(); - }*/ +#if 0 + for (auto const& resp : preparedResponses) { + auto payload = resp.generateResponse()->copyPayload(); + VPackSlice husti(payload->data()); + LOG_DEVEL << husti.toJson(); + } +#endif } server = std::make_unique(true, false); @@ -189,8 +205,7 @@ class GraphProviderTest : public ::testing::Test { clusterEngines = std::make_unique>(); clusterEngines->emplace("PRMR_0001", engineId); - auto clusterCache = - std::make_shared(resourceMonitor); + auto clusterCache = std::make_shared(resourceMonitor); ClusterBaseProviderOptions opts(clusterCache, clusterEngines.get(), false); return ClusterProvider(*query.get(), std::move(opts), resourceMonitor); @@ -299,8 +314,7 @@ TYPED_TEST(GraphProviderTest, should_enumerate_all_edges) { std::unordered_set found{}; std::unordered_map>> const& expectedVerticesEdgesBundleToFetch = { - {0, {}} - }; + {0, {}}}; auto testee = this->makeProvider(g, expectedVerticesEdgesBundleToFetch); std::string startString = g.vertexToId(0); VPackHashedStringRef startH{startString.c_str(), diff --git a/tests/Graph/GraphMockProviderInstances.cpp b/tests/Graph/GraphMockProviderInstances.cpp index 617b3cd7163b..fbb6e3a8a9a4 100644 --- a/tests/Graph/GraphMockProviderInstances.cpp +++ b/tests/Graph/GraphMockProviderInstances.cpp @@ -25,47 +25,62 @@ #include "Graph/Enumerators/OneSidedEnumerator.cpp" #include "Graph/Enumerators/TwoSidedEnumerator.cpp" #include "Graph/PathManagement/PathResult.cpp" -#include "Graph/PathManagement/SingleProviderPathResult.cpp" #include "Graph/PathManagement/PathStore.cpp" +#include "Graph/PathManagement/SingleProviderPathResult.cpp" #include "Graph/Queues/FifoQueue.h" #include "Graph/Queues/LifoQueue.h" #include "Graph/PathManagement/PathStoreTracer.cpp" #include "Graph/PathManagement/PathStoreTracer.h" #include "Graph/PathManagement/PathValidator.cpp" +#include "Graph/Providers/ProviderTracer.cpp" #include "Graph/Queues/QueueTracer.cpp" template class ::arangodb::graph::PathResult<::arangodb::tests::graph::MockGraphProvider, ::arangodb::tests::graph::MockGraphProvider::Step>; template class ::arangodb::graph::SingleProviderPathResult< - ::arangodb::tests::graph::MockGraphProvider, ::arangodb::tests::graph::MockGraphProvider::Step>; + ::arangodb::tests::graph::MockGraphProvider, + ::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, + ::arangodb::tests::graph::MockGraphProvider::Step>; + +template class ::arangodb::graph::SingleProviderPathResult< + ::arangodb::tests::graph::MockGraphProvider, + ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>>, + ::arangodb::tests::graph::MockGraphProvider::Step>; + +template class ::arangodb::graph::ProviderTracer<::arangodb::tests::graph::MockGraphProvider>; template class ::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>; template class ::arangodb::graph::PathValidator< + ::arangodb::tests::graph::MockGraphProvider, ::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, VertexUniquenessLevel::PATH>; template class ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>>; template class ::arangodb::graph::PathValidator< + ::arangodb::tests::graph::MockGraphProvider, ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>>, VertexUniquenessLevel::PATH>; template class ::arangodb::graph::TwoSidedEnumerator< ::arangodb::graph::FifoQueue<::arangodb::tests::graph::MockGraphProvider::Step>, ::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, ::arangodb::tests::graph::MockGraphProvider, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, VertexUniquenessLevel::PATH>>; + ::arangodb::graph::PathValidator<::arangodb::tests::graph::MockGraphProvider, + ::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, VertexUniquenessLevel::PATH>>; -template class ::arangodb::graph::OneSidedEnumerator< - ::arangodb::graph::LifoQueue<::arangodb::tests::graph::MockGraphProvider::Step>, - ::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, ::arangodb::tests::graph::MockGraphProvider, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, VertexUniquenessLevel::PATH>>; +// BFS with PATH uniquness +template class ::arangodb::graph::OneSidedEnumerator<::arangodb::graph::BFSConfiguration< + ::arangodb::tests::graph::MockGraphProvider, ::arangodb::graph::VertexUniquenessLevel::PATH, false>>; +template class ::arangodb::graph::OneSidedEnumerator<::arangodb::graph::BFSConfiguration< + ::arangodb::tests::graph::MockGraphProvider, ::arangodb::graph::VertexUniquenessLevel::PATH, true>>; -template class ::arangodb::graph::OneSidedEnumerator< - ::arangodb::graph::FifoQueue<::arangodb::tests::graph::MockGraphProvider::Step>, - ::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, ::arangodb::tests::graph::MockGraphProvider, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, VertexUniquenessLevel::PATH>>; +// DFS with PATH uniquness +template class ::arangodb::graph::OneSidedEnumerator<::arangodb::graph::DFSConfiguration< + ::arangodb::tests::graph::MockGraphProvider, ::arangodb::graph::VertexUniquenessLevel::PATH, false>>; +template class ::arangodb::graph::OneSidedEnumerator<::arangodb::graph::DFSConfiguration< + ::arangodb::tests::graph::MockGraphProvider, ::arangodb::graph::VertexUniquenessLevel::PATH, true>>; template class ::arangodb::graph::QueueTracer<::arangodb::graph::FifoQueue<::arangodb::tests::graph::MockGraphProvider::Step>>; @@ -74,35 +89,13 @@ template class ::arangodb::graph::QueueTracer<::arangodb::graph::LifoQueue<::ara template class ::arangodb::graph::TwoSidedEnumerator< ::arangodb::graph::QueueTracer<::arangodb::graph::FifoQueue<::arangodb::tests::graph::MockGraphProvider::Step>>, ::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, ::arangodb::tests::graph::MockGraphProvider, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, VertexUniquenessLevel::PATH>>; - -template class ::arangodb::graph::OneSidedEnumerator< - ::arangodb::graph::QueueTracer<::arangodb::graph::LifoQueue<::arangodb::tests::graph::MockGraphProvider::Step>>, - ::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, ::arangodb::tests::graph::MockGraphProvider, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, VertexUniquenessLevel::PATH>>; - -template class ::arangodb::graph::OneSidedEnumerator< - ::arangodb::graph::QueueTracer<::arangodb::graph::FifoQueue<::arangodb::tests::graph::MockGraphProvider::Step>>, - ::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, ::arangodb::tests::graph::MockGraphProvider, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, VertexUniquenessLevel::PATH>>; + ::arangodb::graph::PathValidator<::arangodb::tests::graph::MockGraphProvider, + ::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, VertexUniquenessLevel::PATH>>; template class ::arangodb::graph::TwoSidedEnumerator< ::arangodb::graph::QueueTracer<::arangodb::graph::FifoQueue<::arangodb::tests::graph::MockGraphProvider::Step>>, ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>>, ::arangodb::tests::graph::MockGraphProvider, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>>, + ::arangodb::graph::PathValidator<::arangodb::tests::graph::MockGraphProvider, + ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>>, VertexUniquenessLevel::PATH>>; - -template class ::arangodb::graph::OneSidedEnumerator< - ::arangodb::graph::QueueTracer<::arangodb::graph::LifoQueue<::arangodb::tests::graph::MockGraphProvider::Step>>, - ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>>, - ::arangodb::tests::graph::MockGraphProvider, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>>, - VertexUniquenessLevel::PATH>>; - -template class ::arangodb::graph::OneSidedEnumerator< - ::arangodb::graph::QueueTracer<::arangodb::graph::FifoQueue<::arangodb::tests::graph::MockGraphProvider::Step>>, - ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>>, - ::arangodb::tests::graph::MockGraphProvider, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>>, - VertexUniquenessLevel::PATH>>; \ No newline at end of file diff --git a/tests/Graph/GraphTestTools.h b/tests/Graph/GraphTestTools.h index fe1d517cc588..9a6ea406e2c9 100644 --- a/tests/Graph/GraphTestTools.h +++ b/tests/Graph/GraphTestTools.h @@ -83,6 +83,7 @@ struct GraphTestSetup // setup required application features features.emplace_back(server.addFeature(), false); features.emplace_back(server.addFeature(), false); + features.emplace_back(server.addFeature(), false); features.emplace_back(server.addFeature(), false); features.emplace_back(server.addFeature(), false); server.getFeature().setEngineTesting(&engine); diff --git a/tests/Graph/KPathFinderTest.cpp b/tests/Graph/KPathFinderTest.cpp index 68c92ddd7936..9340096c6704 100644 --- a/tests/Graph/KPathFinderTest.cpp +++ b/tests/Graph/KPathFinderTest.cpp @@ -69,6 +69,13 @@ class KPathFinderTest arangodb::GlobalResourceMonitor global{}; arangodb::ResourceMonitor resourceMonitor{global}; + // PathValidatorOptions parts (used for API not under test here) + aql::Variable _tmpVar{"tmp", 0, false}; + arangodb::aql::AqlFunctionsInternalCache _functionsCache{}; + + arangodb::transaction::Methods _trx{_query->newTrxContext()}; + arangodb::aql::FixedVarExpressionContext _expressionContext{_trx, *_query.get(), + _functionsCache}; KPathFinderTest() { if (activateLogging) { Logger::GRAPHS.setLogLevel(LogLevel::TRACE); @@ -159,9 +166,15 @@ class KPathFinderTest auto pathFinder(size_t minDepth, size_t maxDepth) -> KPathFinder { arangodb::graph::TwoSidedEnumeratorOptions options{minDepth, maxDepth}; - return KPathFinder{MockGraphProvider(mockGraph, *_query.get(), looseEndBehaviour(), false), - MockGraphProvider(mockGraph, *_query.get(), looseEndBehaviour(), true), - std::move(options), resourceMonitor}; + PathValidatorOptions validatorOpts{&_tmpVar, _expressionContext}; + return KPathFinder{ + MockGraphProvider(*_query.get(), + MockGraphProviderOptions{mockGraph, looseEndBehaviour(), false}, + resourceMonitor), + MockGraphProvider(*_query.get(), + MockGraphProviderOptions{mockGraph, looseEndBehaviour(), true}, + resourceMonitor), + std::move(options), std::move(validatorOpts), resourceMonitor}; } auto vId(size_t nr) -> std::string { diff --git a/tests/Graph/LifoQueueTest.cpp b/tests/Graph/LifoQueueTest.cpp index 33b116e952ab..7eb89ec19b31 100644 --- a/tests/Graph/LifoQueueTest.cpp +++ b/tests/Graph/LifoQueueTest.cpp @@ -24,6 +24,7 @@ #include "Basics/GlobalResourceMonitor.h" #include "Basics/ResourceUsage.h" #include "Basics/StringUtils.h" +#include "Basics/ResultT.h" #include "Graph/Providers/BaseStep.h" #include "Graph/Queues/LifoQueue.h" diff --git a/tests/Graph/PathValidatorTest.cpp b/tests/Graph/PathValidatorTest.cpp index fd4c37fe411c..12eb80bbd837 100644 --- a/tests/Graph/PathValidatorTest.cpp +++ b/tests/Graph/PathValidatorTest.cpp @@ -22,6 +22,10 @@ #include "gtest/gtest.h" +#include "../Mocks/Servers.h" + +#include "Aql/Ast.h" +#include "Aql/Query.h" #include "Basics/GlobalResourceMonitor.h" #include "Basics/ResourceUsage.h" #include "Basics/StringHeap.h" @@ -31,62 +35,68 @@ #include "Graph/Providers/BaseStep.h" #include "Graph/Types/UniquenessLevel.h" +#include "./MockGraph.h" +#include "./MockGraphProvider.h" + using namespace arangodb; using namespace arangodb::graph; using namespace arangodb::velocypack; +namespace { +aql::AstNode* InitializeReference(aql::Ast& ast, aql::Variable& var) { + ast.scopes()->start(aql::ScopeType::AQL_SCOPE_MAIN); + ast.scopes()->addVariable(&var); + aql::AstNode* a = ast.createNodeReference("tmp"); + ast.scopes()->endCurrent(); + return a; +} +} // namespace + namespace arangodb { namespace tests { namespace graph_path_validator_test { static_assert(GTEST_HAS_TYPED_TEST, "We need typed tests for the following:"); -class Step : public arangodb::graph::BaseStep { - public: - using Vertex = HashedStringRef; - using Edge = HashedStringRef; - - HashedStringRef _id; - - Step(HashedStringRef id, size_t previous) - : arangodb::graph::BaseStep{previous}, _id{id} {}; - - ~Step() = default; - - bool operator==(Step const& other) { return _id == other._id; } - - std::string toString() const { - return " _id: " + _id.toString() + - ", _previous: " + basics::StringUtils::itoa(getPrevious()); - } - - bool isProcessable() const { return true; } - HashedStringRef getVertex() const { return _id; } - HashedStringRef getEdge() const { return _id; } +using Step = typename graph::MockGraphProvider::Step; - arangodb::velocypack::HashedStringRef getVertexIdentifier() const { - return getVertex(); - } -}; - -using TypesToTest = - ::testing::Types, VertexUniquenessLevel::NONE>, - PathValidator, VertexUniquenessLevel::PATH>, - PathValidator, VertexUniquenessLevel::GLOBAL>>; +using TypesToTest = ::testing::Types< + PathValidator, VertexUniquenessLevel::NONE>, + PathValidator, VertexUniquenessLevel::PATH>, + PathValidator, VertexUniquenessLevel::GLOBAL>>; template class PathValidatorTest : public ::testing::Test { + protected: + graph::MockGraph mockGraph; + mocks::MockAqlServer _server{true}; + + std::unique_ptr _query{_server.createFakeQuery()}; + + private: arangodb::GlobalResourceMonitor _global{}; arangodb::ResourceMonitor _resourceMonitor{_global}; + std::unique_ptr _provider; + PathStore _pathStore{_resourceMonitor}; StringHeap _heap{_resourceMonitor, 4096}; + // Expression Parts + arangodb::transaction::Methods _trx{_query->newTrxContext()}; + aql::Ast* _ast{_query->ast()}; + aql::Variable _tmpVar{"tmp", 0, false}; + aql::AstNode* _varNode{::InitializeReference(*_ast, _tmpVar)}; + + arangodb::aql::AqlFunctionsInternalCache _functionsCache{}; + arangodb::aql::FixedVarExpressionContext _expressionContext{_trx, *_query, _functionsCache}; + PathValidatorOptions _opts{&_tmpVar, _expressionContext}; + protected: VertexUniquenessLevel getVertexUniquness() { - if constexpr (std::is_same_v, VertexUniquenessLevel::NONE>>) { + if constexpr (std::is_same_v, VertexUniquenessLevel::NONE>>) { return VertexUniquenessLevel::NONE; - } else if constexpr (std::is_same_v, VertexUniquenessLevel::PATH>>) { + } else if constexpr (std::is_same_v, VertexUniquenessLevel::PATH>>) { return VertexUniquenessLevel::PATH; } else { return VertexUniquenessLevel::GLOBAL; @@ -95,61 +105,104 @@ class PathValidatorTest : public ::testing::Test { PathStore& store() { return _pathStore; } - ValidatorType testee() { return ValidatorType{this->store()}; } - Step makeStep(size_t id, size_t previous) { - std::string idStr = basics::StringUtils::itoa(id); - HashedStringRef hStr(idStr.data(), static_cast(idStr.length())); - return Step(_heap.registerString(hStr), previous); + aql::Variable const* tmpVar() { return &_tmpVar; } + + aql::Query* query() { return _query.get(); } + + ValidatorType testee() { + ensureProvider(); + return ValidatorType{*_provider.get(), this->store(), _opts}; + } + + Step startPath(size_t id) { + ensureProvider(); + auto base = mockGraph.vertexToId(id); + HashedStringRef ref{base.c_str(), static_cast(base.length())}; + auto hStr = _heap.registerString(ref); + return _provider->startVertex(hStr); + } + + // Get and modify the options used in the Validator testee(). + // Make sure to Modify them before calling testee() method. + PathValidatorOptions& options() { return _opts; } + + std::vector expandPath(Step previous) { + // We at least have called startPath before, this ensured the provider + TRI_ASSERT(_provider != nullptr); + size_t prev = _pathStore.append(previous); + std::vector result; + _provider->expand(previous, prev, + [&result](Step s) { result.emplace_back(s); }); + return result; + } + + // Add a path defined by the given vector. We start a first() and end in last() + void addPath(std::vector path) { + // This function can only add paths of length 1 or more + TRI_ASSERT(path.size() >= 2); + for (size_t i = 0; i < path.size() - 1; ++i) { + mockGraph.addEdge(path.at(i), path.at(i + 1)); + } + } + + /* + * generates a condition #TMP._key == '' + */ + std::unique_ptr conditionKeyMatches(std::string const& toMatch) { + auto expectedKey = _ast->createNodeValueString(toMatch.c_str(), toMatch.length()); + auto keyAccess = + _ast->createNodeAttributeAccess(_varNode, StaticStrings::KeyString.c_str(), + StaticStrings::KeyString.length()); + // This condition cannot be fulfilled + auto condition = _ast->createNodeBinaryOperator(aql::AstNodeType::NODE_TYPE_OPERATOR_BINARY_EQ, + keyAccess, expectedKey); + return std::make_unique(_ast, condition); + } + + private: + void ensureProvider() { + if (_provider == nullptr) { + _provider = std::make_unique( + *_query.get(), + graph::MockGraphProviderOptions{mockGraph, graph::MockGraphProviderOptions::LooseEndBehaviour::NEVER, + false}, + _resourceMonitor); + } } }; TYPED_TEST_CASE(PathValidatorTest, TypesToTest); TYPED_TEST(PathValidatorTest, it_should_honor_uniqueness_on_single_path_first_duplicate) { - auto&& ps = this->store(); + // We add a loop that ends in the start vertex (0) again. + this->addPath({0, 1, 2, 3, 0}); auto validator = this->testee(); - size_t lastIndex = std::numeric_limits::max(); + Step s = this->startPath(0); { - Step s = this->makeStep(0, lastIndex); auto res = validator.validatePath(s); // The start vertex is always valid EXPECT_FALSE(res.isFiltered()); EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 0); - } - - // We add a loop that ends in the start vertex (0) again. - { - Step s = this->makeStep(1, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 1); - } - { - Step s = this->makeStep(2, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 2); } - { - Step s = this->makeStep(3, lastIndex); + // The next 3 steps are good to take. + for (size_t i = 0; i < 3; ++i) { + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1) + << "Not enough connections after step " << s.getVertexIdentifier(); + s = neighbors.at(0); auto res = validator.validatePath(s); EXPECT_FALSE(res.isFiltered()); EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 3); } - // Add duplicate vertex on Path + // Now we move to the duplicate vertex { - Step s = this->makeStep(0, lastIndex); + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1); + s = neighbors.at(0); auto res = validator.validatePath(s); + if (this->getVertexUniquness() == VertexUniquenessLevel::NONE) { // No uniqueness check, take the vertex EXPECT_FALSE(res.isFiltered()); @@ -163,49 +216,35 @@ TYPED_TEST(PathValidatorTest, it_should_honor_uniqueness_on_single_path_first_du } TYPED_TEST(PathValidatorTest, it_should_honor_uniqueness_on_single_path_last_duplicate) { - auto&& ps = this->store(); + // We add a loop that loops on the last vertex(3). + this->addPath({0, 1, 2, 3, 3}); auto validator = this->testee(); - size_t lastIndex = std::numeric_limits::max(); + Step s = this->startPath(0); { - Step s = this->makeStep(0, lastIndex); auto res = validator.validatePath(s); // The start vertex is always valid EXPECT_FALSE(res.isFiltered()); EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 0); - } - // We add a loop on the last vertex of the path (3) - { - Step s = this->makeStep(1, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 1); - } - { - Step s = this->makeStep(2, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 2); } - { - Step s = this->makeStep(3, lastIndex); + // The next 3 steps are good to take. + for (size_t i = 0; i < 3; ++i) { + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1) + << "Not enough connections after step " << s.getVertexIdentifier(); + s = neighbors.at(0); auto res = validator.validatePath(s); EXPECT_FALSE(res.isFiltered()); EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 3); } - // Add duplicate vertex on Path + // Now we move to the duplicate vertex { - Step s = this->makeStep(3, lastIndex); + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1); + s = neighbors.at(0); auto res = validator.validatePath(s); + if (this->getVertexUniquness() == VertexUniquenessLevel::NONE) { // No uniqueness check, take the vertex EXPECT_FALSE(res.isFiltered()); @@ -219,49 +258,35 @@ TYPED_TEST(PathValidatorTest, it_should_honor_uniqueness_on_single_path_last_dup } TYPED_TEST(PathValidatorTest, it_should_honor_uniqueness_on_single_path_interior_duplicate) { - auto&& ps = this->store(); + // We add a loop that loops on the last vertex(2). + this->addPath({0, 1, 2, 3, 2}); auto validator = this->testee(); - size_t lastIndex = std::numeric_limits::max(); + Step s = this->startPath(0); { - Step s = this->makeStep(0, lastIndex); auto res = validator.validatePath(s); // The start vertex is always valid EXPECT_FALSE(res.isFiltered()); EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 0); - } - // We add a loop that ends in one interior vertex (2) again. - { - Step s = this->makeStep(1, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 1); } - { - Step s = this->makeStep(2, lastIndex); + // The next 3 steps are good to take. + for (size_t i = 0; i < 3; ++i) { + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1) + << "Not enough connections after step " << s.getVertexIdentifier(); + s = neighbors.at(0); auto res = validator.validatePath(s); EXPECT_FALSE(res.isFiltered()); EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 2); - } - { - Step s = this->makeStep(3, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 3); } - // Add duplicate vertex on Path + // Now we move to the duplicate vertex { - Step s = this->makeStep(2, lastIndex); + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1); + s = neighbors.at(0); auto res = validator.validatePath(s); + if (this->getVertexUniquness() == VertexUniquenessLevel::NONE) { // No uniqueness check, take the vertex EXPECT_FALSE(res.isFiltered()); @@ -275,160 +300,201 @@ TYPED_TEST(PathValidatorTest, it_should_honor_uniqueness_on_single_path_interior } TYPED_TEST(PathValidatorTest, it_should_honor_uniqueness_on_global_paths_last_duplicate) { - auto&& ps = this->store(); + // We add a two paths, that share the same start and end vertex (3) + this->addPath({0, 1, 2, 3}); + this->addPath({0, 4, 5, 3}); + auto validator = this->testee(); - size_t lastIndex = std::numeric_limits::max(); + Step s = this->startPath(0); { - Step s = this->makeStep(0, lastIndex); auto res = validator.validatePath(s); // The start vertex is always valid EXPECT_FALSE(res.isFiltered()); EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 0); - } - // We add two paths, each without a loop. - // Both paths share a common vertex besides the start. - - // First path 0 -> 1 -> 2 -> 3 - { - Step s = this->makeStep(1, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 1); - } - { - Step s = this->makeStep(2, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 2); - } - { - Step s = this->makeStep(3, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 3); } - // First path 0 -> 4 -> 5 - lastIndex = 0; - { - Step s = this->makeStep(4, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 4); - } + auto branch = this->expandPath(s); + // 1 and 4, we do not care on the ordering. + ASSERT_EQ(branch.size(), 2); { - Step s = this->makeStep(5, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 5); + { + // Test the branch vertex itself + s = branch.at(0); + auto res = validator.validatePath(s); + EXPECT_FALSE(res.isFiltered()); + EXPECT_FALSE(res.isPruned()); + } + // The first branch is good until the end + for (size_t i = 0; i < 2; ++i) { + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1) + << "Not enough connections after step " << s.getVertexIdentifier(); + s = neighbors.at(0); + auto res = validator.validatePath(s); + EXPECT_FALSE(res.isFiltered()); + EXPECT_FALSE(res.isPruned()); + } } - - // Add duplicate vertex (3) which is the last on the first path { - Step s = this->makeStep(3, lastIndex); - auto res = validator.validatePath(s); - if (this->getVertexUniquness() != VertexUniquenessLevel::GLOBAL) { - // The vertex is visited twice, but not on same path. - // As long as we are not GLOBAL this is okay. + // The second branch is good but for the last vertex + { + // Test the branch vertex itself + s = branch.at(1); + auto res = validator.validatePath(s); EXPECT_FALSE(res.isFiltered()); EXPECT_FALSE(res.isPruned()); - } else { - // With GLOBAL uniqueness this vertex is illegal - EXPECT_TRUE(res.isFiltered()); - EXPECT_TRUE(res.isPruned()); + } + for (size_t i = 0; i < 1; ++i) { + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1) + << "Not enough connections after step " << s.getVertexIdentifier(); + s = neighbors.at(0); + auto res = validator.validatePath(s); + EXPECT_FALSE(res.isFiltered()); + EXPECT_FALSE(res.isPruned()); + } + + // Now we move to the duplicate vertex + { + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1); + s = neighbors.at(0); + auto res = validator.validatePath(s); + + if (this->getVertexUniquness() != VertexUniquenessLevel::GLOBAL) { + // The vertex is visited twice, but not on same path. + // As long as we are not GLOBAL this is okay. + // No uniqueness check, take the vertex + EXPECT_FALSE(res.isFiltered()); + EXPECT_FALSE(res.isPruned()); + } else { + // With GLOBAL uniqueness this vertex is illegal + EXPECT_TRUE(res.isFiltered()); + EXPECT_TRUE(res.isPruned()); + } } } } TYPED_TEST(PathValidatorTest, it_should_honor_uniqueness_on_global_paths_interior_duplicate) { - auto&& ps = this->store(); + // We add a two paths, that share the same start and end vertex (3) + this->addPath({0, 1, 2, 3}); + this->addPath({0, 4, 5, 1}); + auto validator = this->testee(); - size_t lastIndex = std::numeric_limits::max(); + Step s = this->startPath(0); { - Step s = this->makeStep(0, lastIndex); auto res = validator.validatePath(s); // The start vertex is always valid EXPECT_FALSE(res.isFiltered()); EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 0); } - // We add two paths, each without a loop. - // Both paths share a common vertex besides the start. - // First path 0 -> 1 -> 2 -> 3 - { - Step s = this->makeStep(1, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 1); - } + auto branch = this->expandPath(s); + // 1 and 4, we do need to care on the ordering, this is right now guaranteed. + // If this test fails at any point in time, we can add some code here that + // ensures that we first visit Vertex 1, then Vertex 4 + ASSERT_EQ(branch.size(), 2); { - Step s = this->makeStep(2, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 2); + // The first branch is good until the end + { + // Test the branch vertex itself + s = branch.at(0); + auto res = validator.validatePath(s); + EXPECT_FALSE(res.isFiltered()); + EXPECT_FALSE(res.isPruned()); + } + for (size_t i = 0; i < 2; ++i) { + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1) + << "Not enough connections after step " << s.getVertexIdentifier(); + s = neighbors.at(0); + auto res = validator.validatePath(s); + EXPECT_FALSE(res.isFiltered()); + EXPECT_FALSE(res.isPruned()); + } } { - Step s = this->makeStep(3, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 3); - } + // The second branch is good but for the last vertex + { + // Test the branch vertex itself + s = branch.at(1); + auto res = validator.validatePath(s); + EXPECT_FALSE(res.isFiltered()); + EXPECT_FALSE(res.isPruned()); + } + for (size_t i = 0; i < 1; ++i) { + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1) + << "Not enough connections after step " << s.getVertexIdentifier(); + s = neighbors.at(0); + auto res = validator.validatePath(s); + EXPECT_FALSE(res.isFiltered()); + EXPECT_FALSE(res.isPruned()); + } - // First path 0 -> 4 -> 5 - lastIndex = 0; - { - Step s = this->makeStep(4, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 4); + // Now we move to the duplicate vertex + { + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1); + s = neighbors.at(0); + auto res = validator.validatePath(s); + + if (this->getVertexUniquness() != VertexUniquenessLevel::GLOBAL) { + // The vertex is visited twice, but not on same path. + // As long as we are not GLOBAL this is okay. + // No uniqueness check, take the vertex + EXPECT_FALSE(res.isFiltered()); + EXPECT_FALSE(res.isPruned()); + } else { + // With GLOBAL uniqueness this vertex is illegal + EXPECT_TRUE(res.isFiltered()); + EXPECT_TRUE(res.isPruned()); + } + } } +} + +TYPED_TEST(PathValidatorTest, it_should_test_an_all_vertices_condition) { + this->addPath({0, 1}); + std::string keyToMatch = "1"; + + auto expression = this->conditionKeyMatches(keyToMatch); + auto& opts = this->options(); + opts.setAllVerticesExpression(std::move(expression)); + auto validator = this->testee(); { - Step s = this->makeStep(5, lastIndex); + // Testing x._key == "1" with `{_key: "1"} => Should succeed + Step s = this->startPath(1); auto res = validator.validatePath(s); EXPECT_FALSE(res.isFiltered()); EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 5); } - // Add duplicate vertex (1) which is interior to the first path + // we start a new path, so reset the uniqueness checks + validator.reset(); + { - Step s = this->makeStep(1, lastIndex); - auto res = validator.validatePath(s); - if (this->getVertexUniquness() != VertexUniquenessLevel::GLOBAL) { - // The vertex is visited twice, but not on same path. - // As long as we are not GLOBAL this is okay. - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - } else { - // With GLOBAL uniqueness this vertex is illegal + // Testing x._key == "1" with `{_key: "0"} => Should fail + Step s = this->startPath(0); + { + auto res = validator.validatePath(s); EXPECT_TRUE(res.isFiltered()); EXPECT_TRUE(res.isPruned()); } + + // Testing condition on level 1 (not start) + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1); + s = neighbors.at(0); + { + // Testing x._key == "1" with `{_key: "1"} => Should succeed + auto res = validator.validatePath(s); + EXPECT_FALSE(res.isFiltered()); + EXPECT_FALSE(res.isPruned()); + } } } diff --git a/tests/Graph/SingleServerProviderTest.cpp b/tests/Graph/SingleServerProviderTest.cpp index ebe2c6811719..95298433e0f2 100644 --- a/tests/Graph/SingleServerProviderTest.cpp +++ b/tests/Graph/SingleServerProviderTest.cpp @@ -38,10 +38,22 @@ using namespace arangodb::tests; using namespace arangodb::tests::graph; using namespace arangodb::graph; +namespace { +aql::AstNode* InitializeReference(aql::Ast& ast, aql::Variable& var) { + ast.scopes()->start(aql::ScopeType::AQL_SCOPE_MAIN); + ast.scopes()->addVariable(&var); + aql::AstNode* a = ast.createNodeReference(var.name); + ast.scopes()->endCurrent(); + return a; +} +} // namespace + namespace arangodb { namespace tests { namespace single_server_provider_test { +using Step = SingleServerProvider::Step; + class SingleServerProviderTest : public ::testing::Test { protected: // Only used to mock a singleServer @@ -50,8 +62,17 @@ class SingleServerProviderTest : public ::testing::Test { std::unique_ptr query{nullptr}; arangodb::GlobalResourceMonitor _global{}; arangodb::ResourceMonitor _resourceMonitor{_global}; + arangodb::aql::AqlFunctionsInternalCache _functionsCache{}; + std::unique_ptr _expressionContext; + std::unique_ptr _trx{}; + + // Expression Parts + aql::Variable* _tmpVar{nullptr}; + aql::AstNode* _varNode{nullptr}; - std::map _emptyShardMap{}; + std::unordered_map> _emptyShardMap{}; + + std::string stringToMatch = "0-1"; SingleServerProviderTest() {} ~SingleServerProviderTest() {} @@ -65,20 +86,60 @@ class SingleServerProviderTest : public ::testing::Test { // We now have collections "v" and "e" query = singleServer->getQuery("RETURN 1", {"v", "e"}); + _trx = std::make_unique(query->newTrxContext()); auto edgeIndexHandle = singleServer->getEdgeIndexHandle("e"); - auto tmpVar = singleServer->generateTempVar(query.get()); - auto indexCondition = singleServer->buildOutboundCondition(query.get(), tmpVar); + _tmpVar = singleServer->generateTempVar(query.get()); + + auto indexCondition = singleServer->buildOutboundCondition(query.get(), _tmpVar); + _varNode = ::InitializeReference(*query->ast(), *_tmpVar); - std::vector usedIndexes{IndexAccessor{edgeIndexHandle, indexCondition, 0}}; + std::vector usedIndexes{}; + auto expr = conditionKeyMatches(stringToMatch); + usedIndexes.emplace_back(IndexAccessor{edgeIndexHandle, indexCondition, 0, expr}); - BaseProviderOptions opts(tmpVar, std::move(usedIndexes), _emptyShardMap); + _expressionContext = + std::make_unique(*_trx, *query, _functionsCache); + BaseProviderOptions opts(_tmpVar, + std::make_pair(std::move(usedIndexes), + std::unordered_map>{}), + *_expressionContext.get(), _emptyShardMap); return {*query.get(), std::move(opts), _resourceMonitor}; } + + /* + * generates a condition #TMP._key == '' + */ + std::shared_ptr conditionKeyMatches(std::string const& toMatch) { + auto expectedKey = + query->ast()->createNodeValueString(toMatch.c_str(), toMatch.length()); + auto keyAccess = + query->ast()->createNodeAttributeAccess(_varNode, + StaticStrings::KeyString.c_str(), + StaticStrings::KeyString.length()); + // This condition cannot be fulfilled + auto condition = query->ast()->createNodeBinaryOperator(aql::AstNodeType::NODE_TYPE_OPERATOR_BINARY_EQ, + keyAccess, expectedKey); + return std::make_shared(query->ast(), condition); + } }; -TEST_F(SingleServerProviderTest, it_must_be_described) { - ASSERT_TRUE(true); +TEST_F(SingleServerProviderTest, it_can_provide_edges) { + MockGraph g; + g.addEdge(0, 1, 2); + g.addEdge(0, 2, 3); + g.addEdge(1, 2, 1); + auto testee = makeProvider(g); + auto startVertex = g.vertexToId(0); + HashedStringRef hashedStart{startVertex.c_str(), + static_cast(startVertex.length())}; + Step s = testee.startVertex(hashedStart); + + testee.expand(s, 0, [&](Step next) { + VPackBuilder hund; + testee.addEdgeToBuilder(next.getEdge(), hund); + LOG_DEVEL << next.getVertexIdentifier() << " e: " << hund.toJson(); + }); } } // namespace single_server_provider_test diff --git a/tests/Graph/TraverserCacheTest.cpp b/tests/Graph/TraverserCacheTest.cpp index aaff2441f8ba..de9c24981706 100644 --- a/tests/Graph/TraverserCacheTest.cpp +++ b/tests/Graph/TraverserCacheTest.cpp @@ -55,7 +55,7 @@ class TraverserCacheTest : public ::testing::Test { std::unique_ptr traverserCache{nullptr}; std::shared_ptr queryContext{nullptr}; std::unique_ptr trx{nullptr}; - std::map collectionToShardMap{}; // can be empty, only used in standalone mode + std::unordered_map> collectionToShardMap{}; // can be empty, only used in standalone mode arangodb::ResourceMonitor* _monitor; TraverserCacheTest() : gdb(s.server, "testVocbase") { @@ -65,7 +65,8 @@ class TraverserCacheTest : public ::testing::Test { _monitor = &query->resourceMonitor(); traverserCache = std::make_unique(trx.get(), query.get(), - query->resourceMonitor(), stats, collectionToShardMap); + query->resourceMonitor(), + stats, collectionToShardMap); } ~TraverserCacheTest() = default; @@ -85,7 +86,8 @@ TEST_F(TraverserCacheTest, it_should_return_a_null_aqlvalue_if_vertex_is_not_ava VPackBuilder builder; // NOTE: we do not have the data, so we get null for any vertex - traverserCache->insertVertexIntoResult(stats, arangodb::velocypack::HashedStringRef(id), builder); + traverserCache->insertVertexIntoResult(stats, arangodb::velocypack::HashedStringRef(id), + builder); ASSERT_TRUE(builder.slice().isNull()); auto all = query->warnings().all(); ASSERT_EQ(all.size(), 1); @@ -129,7 +131,8 @@ TEST_F(TraverserCacheTest, it_should_increase_memory_usage_when_persisting_a_str EXPECT_EQ(memoryUsageBefore, _monitor->current()); } -TEST_F(TraverserCacheTest, it_should_not_increase_memory_usage_twice_when_persisting_two_equal_strings) { +TEST_F(TraverserCacheTest, + it_should_not_increase_memory_usage_twice_when_persisting_two_equal_strings) { auto memoryUsageStart = _monitor->current(); auto data = VPackParser::fromJson(R"({"_key":"123", "value":123})"); @@ -168,7 +171,8 @@ TEST_F(TraverserCacheTest, it_should_increase_memory_usage_twice_when_persisting EXPECT_EQ(memoryUsageStart, _monitor->current()); } -TEST_F(TraverserCacheTest, it_should_increase_memory_usage_twice_when_persisting_a_string_clear_persist_again) { +TEST_F(TraverserCacheTest, + it_should_increase_memory_usage_twice_when_persisting_a_string_clear_persist_again) { auto memoryUsageBefore = _monitor->current(); auto data = VPackParser::fromJson(R"({"_key":"123", "value":123})"); @@ -220,7 +224,8 @@ TEST_F(TraverserCacheTest, it_should_insert_a_vertex_into_a_result_builder) { HashedStringRef id{doc.get("_id")}; VPackBuilder builder; - traverserCache->insertVertexIntoResult(stats, arangodb::velocypack::HashedStringRef(id), builder); + traverserCache->insertVertexIntoResult(stats, arangodb::velocypack::HashedStringRef(id), + builder); EXPECT_TRUE(builder.slice().get("_key").isString()); EXPECT_EQ(builder.slice().get("_key").toString(), "0"); @@ -235,7 +240,7 @@ TEST_F(TraverserCacheTest, it_should_insert_an_edge_into_a_result_builder) { graph.addEdge(0, 1); gdb.addGraph(graph); - std::string edgeKey = "0-1"; // (EdgeKey format: -) + std::string edgeKey = "0-1"; // (EdgeKey format: -) std::shared_ptr col = gdb.vocbase.lookupCollection("e"); @@ -244,7 +249,8 @@ TEST_F(TraverserCacheTest, it_should_insert_an_edge_into_a_result_builder) { bool called = false; auto result = col->getPhysical()->read(trx.get(), arangodb::velocypack::StringRef{edgeKey}, - [&fetchedDocumentId, &called, &edgeKey](LocalDocumentId const& ldid, VPackSlice edgeDocument) { + [&fetchedDocumentId, &called, + &edgeKey](LocalDocumentId const& ldid, VPackSlice edgeDocument) { fetchedDocumentId = ldid.id(); called = true; EXPECT_TRUE(edgeDocument.isObject()); @@ -256,8 +262,8 @@ TEST_F(TraverserCacheTest, it_should_insert_an_edge_into_a_result_builder) { ASSERT_TRUE(result.ok()); ASSERT_NE(fetchedDocumentId, 0); - DataSourceId dataSourceId{col->id()}; // valid - LocalDocumentId localDocumentId{fetchedDocumentId}; // valid + DataSourceId dataSourceId{col->id()}; // valid + LocalDocumentId localDocumentId{fetchedDocumentId}; // valid EdgeDocumentToken edt{dataSourceId, localDocumentId}; VPackBuilder builder; @@ -270,7 +276,7 @@ TEST_F(TraverserCacheTest, it_should_insert_an_edge_into_a_result_builder) { // check stats EXPECT_EQ(stats.getHttpRequests(), 0); EXPECT_EQ(stats.getFiltered(), 0); - EXPECT_EQ(stats.getScannedIndex(), 0); // <- see note in method: appendEdge + EXPECT_EQ(stats.getScannedIndex(), 0); // <- see note in method: appendEdge } } // namespace traverser_cache_test diff --git a/tests/Graph/MockGraph.cpp b/tests/Mocks/MockGraph.cpp similarity index 100% rename from tests/Graph/MockGraph.cpp rename to tests/Mocks/MockGraph.cpp diff --git a/tests/Graph/MockGraph.h b/tests/Mocks/MockGraph.h similarity index 100% rename from tests/Graph/MockGraph.h rename to tests/Mocks/MockGraph.h diff --git a/tests/Graph/MockGraphProvider.cpp b/tests/Mocks/MockGraphProvider.cpp similarity index 91% rename from tests/Graph/MockGraphProvider.cpp rename to tests/Mocks/MockGraphProvider.cpp index 60d6bc3f3539..fc08796fcb5f 100644 --- a/tests/Graph/MockGraphProvider.cpp +++ b/tests/Mocks/MockGraphProvider.cpp @@ -59,6 +59,12 @@ MockGraphProvider::Step::Step(size_t prev, VertexType v, EdgeType e, bool isProc _edge(e), _isProcessable(isProcessable) {} +MockGraphProvider::Step::Step(size_t prev, VertexType v, bool isProcessable, size_t depth) + : arangodb::graph::BaseStep{prev, depth}, + _vertex(v), + _edge({}), + _isProcessable(isProcessable) {} + MockGraphProvider::Step::Step(size_t prev, VertexType v, EdgeType e, bool isProcessable, size_t depth) : arangodb::graph::BaseStep{prev, depth}, @@ -66,13 +72,11 @@ MockGraphProvider::Step::Step(size_t prev, VertexType v, EdgeType e, _edge(e), _isProcessable(isProcessable) {} -MockGraphProvider::Step::~Step() {} - -MockGraphProvider::MockGraphProvider(MockGraph const& data, - arangodb::aql::QueryContext& queryContext, - LooseEndBehaviour looseEnds, bool reverse) - : _trx(queryContext.newTrxContext()), _reverse(reverse), _looseEnds(looseEnds), _stats{} { - for (auto const& it : data.edges()) { +MockGraphProvider::MockGraphProvider(arangodb::aql::QueryContext& queryContext, + MockGraphProviderOptions opts, + arangodb::ResourceMonitor&) + : _trx(queryContext.newTrxContext()), _reverse(opts.reverse()), _looseEnds(opts.looseEnds()), _stats{} { + for (auto const& it : opts.data().edges()) { _fromIndex[it._from].push_back(it); _toIndex[it._to].push_back(it); } @@ -91,7 +95,7 @@ auto MockGraphProvider::decideProcessable() const -> bool { } } -auto MockGraphProvider::startVertex(VertexType v) -> Step { +auto MockGraphProvider::startVertex(VertexType v, size_t depth) -> Step { LOG_TOPIC("78156", TRACE, Logger::GRAPHS) << " Start Vertex:" << v; return Step(v, decideProcessable()); diff --git a/tests/Graph/MockGraphProvider.h b/tests/Mocks/MockGraphProvider.h similarity index 75% rename from tests/Graph/MockGraphProvider.h rename to tests/Mocks/MockGraphProvider.h index b6296cd12649..1fac5ecf3b31 100644 --- a/tests/Graph/MockGraphProvider.h +++ b/tests/Mocks/MockGraphProvider.h @@ -56,18 +56,39 @@ class HashedStringRef; namespace tests { namespace graph { + +class MockGraphProviderOptions { + public: + enum class LooseEndBehaviour { NEVER, ALWAYS }; + MockGraphProviderOptions(MockGraph const& data, LooseEndBehaviour looseEnds, + bool reverse = false) + : _data(data), _looseEnds(looseEnds), _reverse(reverse) {} + ~MockGraphProviderOptions() = default; + + LooseEndBehaviour looseEnds() const { return _looseEnds; } + MockGraph const& data() const { return _data; } + bool reverse() const { return _reverse; } + + private: + MockGraph const& _data; + LooseEndBehaviour _looseEnds; + bool _reverse; + ; +}; + class MockGraphProvider { using VertexType = arangodb::velocypack::HashedStringRef; using EdgeType = MockGraph::EdgeDef; public: - enum class LooseEndBehaviour { NEVER, ALWAYS }; + using Options = MockGraphProviderOptions; + using LooseEndBehaviour = typename MockGraphProviderOptions::LooseEndBehaviour; class Step : public arangodb::graph::BaseStep { public: class Vertex { public: - explicit Vertex(VertexType v) : _vertex(v) {}; + explicit Vertex(VertexType v) : _vertex(v){}; VertexType getID() const { return _vertex; } @@ -106,8 +127,9 @@ class MockGraphProvider { Step(VertexType v, bool isProcessable); Step(size_t prev, VertexType v, EdgeType e, bool isProcessable); + Step(size_t prev, VertexType v, bool isProcessable, size_t depth); Step(size_t prev, VertexType v, EdgeType e, bool isProcessable, size_t depth); - ~Step(); + ~Step() = default; bool operator<(Step const& other) const noexcept { return _vertex < other._vertex; @@ -124,6 +146,10 @@ class MockGraphProvider { } } + bool isResponsible(transaction::Methods* trx) const { + return true; + } + Vertex getVertex() const { /*if (!isProcessable()) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, @@ -144,6 +170,28 @@ class MockGraphProvider { VertexType getVertexIdentifier() const { return getVertex().getID(); } + std::string getCollectionName() const { + auto collectionNameResult = extractCollectionName(_vertex.getID()); + if (collectionNameResult.fail()) { + THROW_ARANGO_EXCEPTION(collectionNameResult.result()); + } + return collectionNameResult.get().first; + }; + + void setLocalSchreierIndex(size_t index) { + TRI_ASSERT(index != std::numeric_limits::max()); + TRI_ASSERT(!hasLocalSchreierIndex()); + _localSchreierIndex = index; + } + + bool hasLocalSchreierIndex() const { + return _localSchreierIndex != std::numeric_limits::max(); + } + + std::size_t getLocalSchreierIndex() const { + return _localSchreierIndex; + } + bool isProcessable() const { return _isProcessable; } bool isLooseEnd() const { return !isProcessable(); } @@ -159,11 +207,13 @@ class MockGraphProvider { Vertex _vertex; Edge _edge; bool _isProcessable; + size_t _localSchreierIndex; }; MockGraphProvider() = delete; - MockGraphProvider(MockGraph const& data, arangodb::aql::QueryContext& queryContext, - LooseEndBehaviour looseEnds, bool reverse = false); + MockGraphProvider(arangodb::aql::QueryContext& queryContext, Options opts, + arangodb::ResourceMonitor& resourceMonitor); + MockGraphProvider(MockGraphProvider const&) = delete; // TODO: check "Rule of 5" MockGraphProvider(MockGraphProvider&&) = default; ~MockGraphProvider(); @@ -172,7 +222,7 @@ class MockGraphProvider { MockGraphProvider& operator=(MockGraphProvider&&) = default; void destroyEngines(){}; - auto startVertex(VertexType vertex) -> Step; + auto startVertex(VertexType vertex, size_t depth = 0) -> Step; auto fetch(std::vector const& looseEnds) -> futures::Future>; auto expand(Step const& from, size_t previous) -> std::vector; auto expand(Step const& from, size_t previous, std::function callback) -> void; diff --git a/tests/Mocks/Servers.cpp b/tests/Mocks/Servers.cpp index 3fd63f4904c5..a471ddf1d71c 100644 --- a/tests/Mocks/Servers.cpp +++ b/tests/Mocks/Servers.cpp @@ -35,8 +35,8 @@ #include "Aql/ExecutionEngine.h" #include "Aql/OptimizerRulesFeature.h" #include "Aql/Query.h" -#include "Basics/files.h" #include "Basics/StringUtils.h" +#include "Basics/files.h" #include "Cluster/ActionDescription.h" #include "Cluster/AgencyCache.h" #include "Cluster/ClusterFeature.h" @@ -81,7 +81,6 @@ #include "VocBase/vocbase.h" #include "utils/log.hpp" - #include "Servers.h" #include "TemplateSpecializer.h" @@ -124,6 +123,7 @@ static void SetupDatabaseFeaturePhase(MockServer& server) { SetupBasicFeaturePhase(server); server.addFeature(false); // true ?? server.addFeature(true); + server.addFeature(false); server.addFeature(false); server.addFeature(false); server.addFeature(false); @@ -141,7 +141,7 @@ static void SetupClusterFeaturePhase(MockServer& server) { SetupDatabaseFeaturePhase(server); server.addFeature(false); server.addFeature(false); - + // fake the exit code with which unresolved futures are returned on // shutdown. if we don't do this lots of places in ClusterInfo will // report failures during testing @@ -192,7 +192,7 @@ MockServer::MockServer() MockServer::~MockServer() { stopFeatures(); _server.setStateUnsafe(_oldApplicationServerState); - + arangodb::ServerState::instance()->setRebootId(_oldRebootId); } @@ -207,9 +207,9 @@ void MockServer::init() { _server.setStateUnsafe(arangodb::application_features::ApplicationServer::State::IN_WAIT); arangodb::transaction::Methods::clearDataSourceRegistrationCallbacks(); - // many other places rely on the reboot id being initialized, + // many other places rely on the reboot id being initialized, // so we do it here in a central place - arangodb::ServerState::instance()->setRebootId(arangodb::RebootId{1}); + arangodb::ServerState::instance()->setRebootId(arangodb::RebootId{1}); } void MockServer::startFeatures() { @@ -258,7 +258,7 @@ void MockServer::startFeatures() { for (ApplicationFeature& f : orderedFeatures) { auto info = _features.find(&f); // We only start those features... - if(info != _features.end()) { + if (info != _features.end()) { if (info->second) { try { f.start(); @@ -297,7 +297,7 @@ void MockServer::stopFeatures() { for (ApplicationFeature& f : orderedFeatures) { auto info = _features.find(&f); // We only start those features... - if(info != _features.end()) { + if (info != _features.end()) { if (info->second) { try { f.stop(); @@ -312,7 +312,7 @@ void MockServer::stopFeatures() { for (ApplicationFeature& f : orderedFeatures) { auto info = _features.find(&f); // We only start those features... - if(info != _features.end()) { + if (info != _features.end()) { try { f.unprepare(); } catch (...) { @@ -549,10 +549,14 @@ void MockClusterServer::agencyCreateDatabase(std::string const& name) { st = ts.specialize(current_dbs_string); agencyTrx("/arango/Current/Databases/" + name, st); - _server.getFeature().clusterInfo().waitForPlan( - agencyTrx("/arango/Plan/Version", R"=({"op":"increment"})=")).wait(); - _server.getFeature().clusterInfo().waitForCurrent( - agencyTrx("/arango/Current/Version", R"=({"op":"increment"})=")).wait(); + _server.getFeature() + .clusterInfo() + .waitForPlan(agencyTrx("/arango/Plan/Version", R"=({"op":"increment"})=")) + .wait(); + _server.getFeature() + .clusterInfo() + .waitForCurrent(agencyTrx("/arango/Current/Version", R"=({"op":"increment"})=")) + .wait(); } void MockClusterServer::agencyCreateCollections(std::string const& name) { @@ -605,7 +609,7 @@ std::shared_ptr MockClusterServer::createCollection( std::string cid = "98765" + basics::StringUtils::itoa(type); auto& databaseFeature = _server.getFeature(); auto vocbase = databaseFeature.lookupDatabase(dbName); - + VPackBuilder props; { // This is hand-crafted unfortunately the code does not exist... @@ -618,25 +622,25 @@ std::shared_ptr MockClusterServer::createCollection( { VPackArrayBuilder guard2(&props); auto const primIndex = arangodb::velocypack::Parser::fromJson( - R"({"id":"0","type":"primary","name": + R"({"id":"0","type":"primary","name": "primary","fields":["_key"],"unique":true,"sparse":false })"); -props.add(primIndex->slice()); -if (type == TRI_COL_TYPE_EDGE) { - auto const fromIndex = arangodb::velocypack::Parser::fromJson( - R"({"id":"1","type":"edge","name": + props.add(primIndex->slice()); + if (type == TRI_COL_TYPE_EDGE) { + auto const fromIndex = arangodb::velocypack::Parser::fromJson( + R"({"id":"1","type":"edge","name": "edge_from","fields":["_from"],"unique":false,"sparse":false })"); -props.add(fromIndex->slice()); -auto const toIndex = arangodb::velocypack::Parser::fromJson( -R"({"id":"2","type":"edge","name": + props.add(fromIndex->slice()); + auto const toIndex = arangodb::velocypack::Parser::fromJson( + R"({"id":"2","type":"edge","name": "edge_to","fields":["_to"],"unique":false,"sparse":false})"); -props.add(toIndex->slice()); -} + props.add(toIndex->slice()); + } } } LogicalCollection dummy(*vocbase, props.slice(), true); - + auto shards = std::make_shared(); for (auto const& [shard, server] : shardNameToServerNamePairs) { shards->emplace(shard, std::vector{server}); @@ -650,11 +654,13 @@ props.add(toIndex->slice()); VPackBuilder velocy = dummy.toVelocyPackIgnore(ignoreKeys, LogicalDataSource::Serialization::List); - agencyTrx("/arango/Plan/Collections/" + dbName + "/" + basics::StringUtils::itoa(dummy.planId().id()), velocy.toJson()); + agencyTrx("/arango/Plan/Collections/" + dbName + "/" + + basics::StringUtils::itoa(dummy.planId().id()), + velocy.toJson()); { - /* Hard-Coded section to inject the CURRENT counter part. - * We do not have a shard available here that could generate the values accordingly. - */ + /* Hard-Coded section to inject the CURRENT counter part. + * We do not have a shard available here that could generate the values accordingly. + */ VPackBuilder current; { VPackObjectBuilder report(¤t); @@ -678,7 +684,9 @@ props.add(toIndex->slice()); // NOTE: we omited Indexes } } - agencyTrx("/arango/Current/Collections/" + dbName + "/" + basics::StringUtils::itoa(dummy.planId().id()), current.toJson()); + agencyTrx("/arango/Current/Collections/" + dbName + "/" + + basics::StringUtils::itoa(dummy.planId().id()), + current.toJson()); } _server.getFeature() @@ -762,7 +770,8 @@ void MockDBServer::createShard(std::string const& dbName, std::string shardName, maintenance::ActionDescription ad( std::map{{maintenance::NAME, maintenance::CREATE_COLLECTION}, {maintenance::COLLECTION, - basics::StringUtils::itoa(clusterCollection.planId().id())}, + basics::StringUtils::itoa( + clusterCollection.planId().id())}, {maintenance::SHARD, shardName}, {maintenance::DATABASE, dbName}, {maintenance::SERVER_ID, "PRMR_0001"}, @@ -828,7 +837,7 @@ std::pair MockCoordinator::registerFakedDBServer(std:: { VPackObjectBuilder b(&builder); builder.add("endpoint", VPackValue(fakedEndpoint)); - builder.add("advertisedEndpoint",VPackValue(fakedEndpoint)); + builder.add("advertisedEndpoint", VPackValue(fakedEndpoint)); builder.add("host", VPackValue(fakedHost)); builder.add("version", VPackValue(rest::Version::getNumericServerVersion())); builder.add("versionString", VPackValue(rest::Version::getServerVersion())); diff --git a/tests/Mocks/StorageEngineMock.cpp b/tests/Mocks/StorageEngineMock.cpp index 3d6629ad79dc..e9f303be60b1 100644 --- a/tests/Mocks/StorageEngineMock.cpp +++ b/tests/Mocks/StorageEngineMock.cpp @@ -100,19 +100,19 @@ class EdgeIndexIteratorMock final : public arangodb::IndexIterator { typedef std::unordered_multimap Map; EdgeIndexIteratorMock(arangodb::LogicalCollection* collection, - arangodb::transaction::Methods* trx, arangodb::Index const* index, - Map const& map, std::unique_ptr&& keys) + arangodb::transaction::Methods* trx, + arangodb::Index const* index, Map const& map, + std::unique_ptr&& keys, bool isFrom) : IndexIterator(collection, trx), _map(map), - _begin(_map.begin()), + _begin(_map.end()), _end(_map.end()), _keys(std::move(keys)), - _keysIt(_keys->slice()) {} - - char const* typeName() const override { return "edge-index-iterator-mock"; } + _keysIt(_keys->slice()), + _isFrom(isFrom) {} - bool nextImpl(LocalDocumentIdCallback const& cb, size_t limit) override { - while (limit && _begin != _end && _keysIt.valid()) { + bool prepareNextRange() { + if (_keysIt.valid()) { auto key = _keysIt.value(); if (key.isObject()) { @@ -120,16 +120,63 @@ class EdgeIndexIteratorMock final : public arangodb::IndexIterator { } std::tie(_begin, _end) = _map.equal_range(key.toString()); + _keysIt++; + return true; + } else { + // Just make sure begin and end are equal + _begin = _map.end(); + _end = _map.end(); + return false; + } + } - while (limit && _begin != _end) { - cb(_begin->second); - ++_begin; - --limit; - } + char const* typeName() const override { return "edge-index-iterator-mock"; } + + bool hasExtra() const override { return true; } - ++_keysIt; + bool nextImpl(LocalDocumentIdCallback const& cb, size_t limit) override { + // We can at most return limit + for (size_t l = 0; l < limit; ++l) { + while (_begin == _end) { + if (!prepareNextRange()) { + return false; + } + } + TRI_ASSERT(_begin != _end); + cb(_begin->second); + ++_begin; + } + // Returned due to limit. + if (_begin == _end) { + // Out limit hit the last index entry + // Return false if we do not have another range + return prepareNextRange(); } - return _begin != _end && _keysIt.valid(); + return true; + } + + bool nextExtraImpl(ExtraCallback const& cb, size_t limit) override { + auto res = nextImpl( + [&](arangodb::LocalDocumentId docId) -> bool { + auto res = _collection->getPhysical()->read( + _trx, docId, + [&](arangodb::LocalDocumentId const& token, arangodb::velocypack::Slice doc) -> bool { + // The nextExtra API in EdgeIndex will deliver the _id value of + // the oposite vertex. We simulate that here by reading the real + // document one more time and just extracting this value. + if (_isFrom) { + return cb(token, doc.get(arangodb::StaticStrings::ToString)); + } else { + return cb(token, doc.get(arangodb::StaticStrings::FromString)); + } + }); + // We can only have good responses here. + // Otherwise storage and Index do differ + TRI_ASSERT(res.ok()); + return res.ok(); + }, + limit); + return res; } void resetImpl() override { @@ -144,6 +191,7 @@ class EdgeIndexIteratorMock final : public arangodb::IndexIterator { Map::const_iterator _end; std::unique_ptr _keys; arangodb::velocypack::ArrayIterator _keysIt; + bool _isFrom; }; // EdgeIndexIteratorMock class EdgeIndexMock final : public arangodb::Index { @@ -341,7 +389,7 @@ class EdgeIndexMock final : public arangodb::Index { return std::make_unique(&_collection, trx, this, isFrom ? _edgesFrom : _edgesTo, - std::move(keys)); + std::move(keys), isFrom); } /// @brief create the iterator @@ -371,7 +419,7 @@ class EdgeIndexMock final : public arangodb::Index { return std::make_unique(&_collection, trx, this, isFrom ? _edgesFrom : _edgesTo, - std::move(keys)); + std::move(keys), isFrom); } /// @brief the hash table for _from @@ -455,14 +503,17 @@ class HashIndexMap { struct VPackBuilderComparator { bool operator()(VPackBuilder const& builder1, VPackBuilder const& builder2) const { - return ::arangodb::basics::VelocyPackHelper::compare(builder1.slice(), builder2.slice(), true) == 0; + return ::arangodb::basics::VelocyPackHelper::compare(builder1.slice(), + builder2.slice(), true) == 0; } }; - using ValueMap = std::unordered_multimap; + using ValueMap = + std::unordered_multimap; using DocumentsIndexMap = std::unordered_map; - arangodb::velocypack::Slice getSliceByField(arangodb::velocypack::Slice const& doc, size_t i) { + arangodb::velocypack::Slice getSliceByField(arangodb::velocypack::Slice const& doc, + size_t i) { TRI_ASSERT(i < _fields.size()); TRI_ASSERT(!doc.isNone()); auto slice = doc; @@ -475,7 +526,8 @@ class HashIndexMap { return slice; } - void insertSlice(arangodb::LocalDocumentId const& documentId, arangodb::velocypack::Slice const& slice, size_t i) { + void insertSlice(arangodb::LocalDocumentId const& documentId, + arangodb::velocypack::Slice const& slice, size_t i) { VPackBuilder builder; if (slice.isNone() || slice.isNull()) { builder.add(VPackSlice::nullSlice()); @@ -486,11 +538,13 @@ class HashIndexMap { } public: - HashIndexMap(std::vector> const& fields) : _fields(fields), _valueMaps(fields.size()) { + HashIndexMap(std::vector> const& fields) + : _fields(fields), _valueMaps(fields.size()) { TRI_ASSERT(!_fields.empty()); } - void insert(arangodb::LocalDocumentId const& documentId, arangodb::velocypack::Slice const& doc) { + void insert(arangodb::LocalDocumentId const& documentId, + arangodb::velocypack::Slice const& doc) { VPackBuilder builder; builder.openArray(); auto toClose = true; @@ -510,14 +564,16 @@ class HashIndexMap { if (slice.isNone() || slice.isNull()) { break; } - } else { // expansion + } else { // expansion isExpansion = slice.isArray(); TRI_ASSERT(isExpansion); auto found = false; - for (auto sliceIt = arangodb::velocypack::ArrayIterator(slice); sliceIt != sliceIt.end(); ++sliceIt) { + for (auto sliceIt = arangodb::velocypack::ArrayIterator(slice); + sliceIt != sliceIt.end(); ++sliceIt) { auto subSlice = sliceIt.value(); if (!(subSlice.isNone() || subSlice.isNull())) { - for (auto fieldItForArray = fieldIt; fieldItForArray != _fields[i].end(); ++fieldItForArray) { + for (auto fieldItForArray = fieldIt; + fieldItForArray != _fields[i].end(); ++fieldItForArray) { TRI_ASSERT(subSlice.isObject()); subSlice = subSlice.get(fieldItForArray->name); if (subSlice.isNone() || subSlice.isNull()) { @@ -544,7 +600,8 @@ class HashIndexMap { if (slice.isArray() && i == _fields.size() - 1) { auto found = false; auto wasNull = false; - for (auto sliceIt = arangodb::velocypack::ArrayIterator(slice); sliceIt != sliceIt.end(); ++sliceIt) { + for (auto sliceIt = arangodb::velocypack::ArrayIterator(slice); + sliceIt != sliceIt.end(); ++sliceIt) { auto subSlice = sliceIt.value(); if (!(subSlice.isNone() || subSlice.isNull())) { insertSlice(documentId, subSlice, i); @@ -557,7 +614,7 @@ class HashIndexMap { insertSlice(documentId, VPackSlice::nullSlice(), i); } toClose = false; - } else { // object + } else { // object insertSlice(documentId, slice, i); builder.add(slice); } @@ -569,7 +626,8 @@ class HashIndexMap { _docIndexMap.try_emplace(documentId, std::move(builder)); } - bool remove(arangodb::LocalDocumentId const& documentId, arangodb::velocypack::Slice const& doc) { + bool remove(arangodb::LocalDocumentId const& documentId, + arangodb::velocypack::Slice const& doc) { size_t i = 0; auto documentRemoved = false; for (auto& map : _valueMaps) { @@ -592,7 +650,8 @@ class HashIndexMap { _docIndexMap.clear(); } - std::unordered_map find(std::unique_ptr&& keys) const { + std::unordered_map find( + std::unique_ptr&& keys) const { std::unordered_map found; TRI_ASSERT(keys->slice().isArray()); auto sliceIt = arangodb::velocypack::ArrayIterator(keys->slice()); @@ -605,7 +664,7 @@ class HashIndexMap { return std::unordered_map(); } if (found.empty()) { - std::transform(begin, end, std::inserter(found, found.end()), [] (auto const& item) { + std::transform(begin, end, std::inserter(found, found.end()), [](auto const& item) { return std::make_pair(item.second, &item.first); }); } else { @@ -650,7 +709,7 @@ class HashIndexIteratorMock final : public arangodb::IndexIterator { HashIndexIteratorMock(arangodb::LogicalCollection* collection, arangodb::transaction::Methods* trx, arangodb::Index const* index, HashIndexMap const& map, std::unique_ptr&& keys) - : IndexIterator(collection, trx), _map(map) { + : IndexIterator(collection, trx), _map(map) { _documents = _map.find(std::move(keys)); _begin = _documents.begin(); _end = _documents.end(); @@ -684,9 +743,7 @@ class HashIndexIteratorMock final : public arangodb::IndexIterator { _end = _documents.end(); } - bool hasCovering() const override { - return true; - } + bool hasCovering() const override { return true; } private: HashIndexMap const& _map; @@ -706,8 +763,9 @@ class HashIndexMock final : public arangodb::Index { return nullptr; } - auto const type = arangodb::basics::VelocyPackHelper::getStringRef(typeSlice, - arangodb::velocypack::StringRef()); + auto const type = + arangodb::basics::VelocyPackHelper::getStringRef(typeSlice, + arangodb::velocypack::StringRef()); if (type.compare("hash") != 0) { return nullptr; @@ -781,17 +839,19 @@ class HashIndexMock final : public arangodb::Index { std::vector> const& allIndexes, arangodb::aql::AstNode const* node, arangodb::aql::Variable const* reference, size_t itemsInIndex) const override { - return arangodb::SortedIndexAttributeMatcher::supportsFilterCondition(allIndexes, this, node, reference, itemsInIndex); + return arangodb::SortedIndexAttributeMatcher::supportsFilterCondition( + allIndexes, this, node, reference, itemsInIndex); } Index::SortCosts supportsSortCondition(arangodb::aql::SortCondition const* sortCondition, - arangodb::aql::Variable const* reference, - size_t itemsInIndex) const override { - return arangodb::SortedIndexAttributeMatcher::supportsSortCondition(this, sortCondition, reference, itemsInIndex); + arangodb::aql::Variable const* reference, + size_t itemsInIndex) const override { + return arangodb::SortedIndexAttributeMatcher::supportsSortCondition(this, sortCondition, + reference, itemsInIndex); } - arangodb::aql::AstNode* specializeCondition( - arangodb::aql::AstNode* node, arangodb::aql::Variable const* reference) const override { + arangodb::aql::AstNode* specializeCondition(arangodb::aql::AstNode* node, + arangodb::aql::Variable const* reference) const override { return arangodb::SortedIndexAttributeMatcher::specializeCondition(this, node, reference); } @@ -804,8 +864,7 @@ class HashIndexMock final : public arangodb::Index { if (nullptr == node) { keys->close(); return std::make_unique(&_collection, trx, this, - _hashData, - std::move(keys)); + _hashData, std::move(keys)); } TRI_ASSERT(node->type == arangodb::aql::NODE_TYPE_OPERATOR_NARY_AND); @@ -824,7 +883,7 @@ class HashIndexMock final : public arangodb::Index { auto valNode = comp->getMember(1); if (!(attrNode->type == arangodb::aql::NODE_TYPE_ATTRIBUTE_ACCESS || - attrNode->type == arangodb::aql::NODE_TYPE_EXPANSION)) { + attrNode->type == arangodb::aql::NODE_TYPE_EXPANSION)) { // got value == a.b -> flip sides std::swap(attrNode, valNode); } @@ -834,11 +893,13 @@ class HashIndexMock final : public arangodb::Index { std::vector attributes; if (attrNode->type == arangodb::aql::NODE_TYPE_ATTRIBUTE_ACCESS) { do { - attributes.emplace_back(std::string(attrNode->getStringValue(), attrNode->getStringLength()), false); + attributes.emplace_back(std::string(attrNode->getStringValue(), + attrNode->getStringLength()), + false); attrNode = attrNode->getMember(0); } while (attrNode->type == arangodb::aql::NODE_TYPE_ATTRIBUTE_ACCESS); std::reverse(attributes.begin(), attributes.end()); - } else { // expansion + } else { // expansion TRI_ASSERT(attrNode->type == arangodb::aql::NODE_TYPE_EXPANSION); auto expNode = attrNode; TRI_ASSERT(expNode->numMembers() >= 2); @@ -847,7 +908,9 @@ class HashIndexMock final : public arangodb::Index { attrNode = left->getMember(1); TRI_ASSERT(attrNode->type == arangodb::aql::NODE_TYPE_ATTRIBUTE_ACCESS); do { - attributes.emplace_back(std::string(attrNode->getStringValue(), attrNode->getStringLength()), false); + attributes.emplace_back(std::string(attrNode->getStringValue(), + attrNode->getStringLength()), + false); attrNode = attrNode->getMember(0); } while (attrNode->type == arangodb::aql::NODE_TYPE_ATTRIBUTE_ACCESS); attributes.front().shouldExpand = true; @@ -858,16 +921,19 @@ class HashIndexMock final : public arangodb::Index { TRI_ASSERT(attrNode->type == arangodb::aql::NODE_TYPE_ATTRIBUTE_ACCESS || attrNode->type == arangodb::aql::NODE_TYPE_REFERENCE); while (attrNode->type == arangodb::aql::NODE_TYPE_ATTRIBUTE_ACCESS) { - attributesRight.emplace_back(std::string(attrNode->getStringValue(), attrNode->getStringLength()), false); + attributesRight.emplace_back(std::string(attrNode->getStringValue(), + attrNode->getStringLength()), + false); attrNode = attrNode->getMember(0); } - attributes.insert(attributes.end(), attributesRight.crbegin(), attributesRight.crend()); + attributes.insert(attributes.end(), attributesRight.crbegin(), + attributesRight.crend()); } allAttributes.emplace_back(std::move(attributes), valNode); } size_t nullsCount = 0; for (auto const& f : _fields) { - auto it = std::find_if(allAttributes.cbegin(), allAttributes.cend(), [&f] (auto const& attrs) { + auto it = std::find_if(allAttributes.cbegin(), allAttributes.cend(), [&f](auto const& attrs) { return arangodb::basics::AttributeName::isIdentical(attrs.first, f, true); }); if (it != allAttributes.cend()) { @@ -883,8 +949,7 @@ class HashIndexMock final : public arangodb::Index { keys->close(); return std::make_unique(&_collection, trx, this, - _hashData, - std::move(keys)); + _hashData, std::move(keys)); } HashIndexMock(arangodb::IndexId iid, arangodb::LogicalCollection& collection, @@ -1047,7 +1112,8 @@ bool PhysicalCollectionMock::dropIndex(arangodb::IndexId iid) { return false; } -void PhysicalCollectionMock::figuresSpecific(bool /*details*/, arangodb::velocypack::Builder&) { +void PhysicalCollectionMock::figuresSpecific(bool /*details*/, + arangodb::velocypack::Builder&) { before(); TRI_ASSERT(false); } @@ -1074,9 +1140,10 @@ void PhysicalCollectionMock::getPropertiesVPack(arangodb::velocypack::Builder&) before(); } -arangodb::Result PhysicalCollectionMock::insert( - arangodb::transaction::Methods* trx, arangodb::velocypack::Slice newSlice, - arangodb::ManagedDocumentResult& result, arangodb::OperationOptions& options) { +arangodb::Result PhysicalCollectionMock::insert(arangodb::transaction::Methods* trx, + arangodb::velocypack::Slice newSlice, + arangodb::ManagedDocumentResult& result, + arangodb::OperationOptions& options) { before(); TRI_ASSERT(newSlice.isObject()); @@ -1100,7 +1167,8 @@ arangodb::Result PhysicalCollectionMock::insert( TRI_ASSERT(builder.slice().get(arangodb::StaticStrings::KeyString).isString()); arangodb::velocypack::StringRef key{builder.slice().get(arangodb::StaticStrings::KeyString)}; - arangodb::LocalDocumentId id = ::generateDocumentId(_logicalCollection, revisionId, _lastDocumentId); + arangodb::LocalDocumentId id = + ::generateDocumentId(_logicalCollection, revisionId, _lastDocumentId); auto const& [ref, didInsert] = _documents.emplace(key, DocElement{builder.steal(), id.id()}); TRI_ASSERT(didInsert); @@ -1154,7 +1222,7 @@ arangodb::Result PhysicalCollectionMock::lookupKey( arangodb::transaction::Methods*, arangodb::velocypack::StringRef key, std::pair& result) const { before(); - + auto it = _documents.find(arangodb::velocypack::StringRef{key}); if (it != _documents.end()) { if (_documents.find(arangodb::velocypack::StringRef{key}) != _documents.end()) { @@ -1233,9 +1301,9 @@ void PhysicalCollectionMock::prepareIndexes(arangodb::velocypack::Slice indexesS } } -arangodb::Result PhysicalCollectionMock::read(arangodb::transaction::Methods*, - arangodb::velocypack::StringRef const& key, - arangodb::IndexIterator::DocumentCallback const& cb) const { +arangodb::Result PhysicalCollectionMock::read( + arangodb::transaction::Methods*, arangodb::velocypack::StringRef const& key, + arangodb::IndexIterator::DocumentCallback const& cb) const { before(); auto it = _documents.find(key); if (it != _documents.end()) { @@ -1273,9 +1341,10 @@ bool PhysicalCollectionMock::readDocument(arangodb::transaction::Methods* trx, return false; } -arangodb::Result PhysicalCollectionMock::remove( - arangodb::transaction::Methods& trx, arangodb::velocypack::Slice slice, - arangodb::ManagedDocumentResult& previous, arangodb::OperationOptions& options) { +arangodb::Result PhysicalCollectionMock::remove(arangodb::transaction::Methods& trx, + arangodb::velocypack::Slice slice, + arangodb::ManagedDocumentResult& previous, + arangodb::OperationOptions& options) { before(); auto key = slice.get(arangodb::StaticStrings::KeyString); @@ -1293,10 +1362,11 @@ arangodb::Result PhysicalCollectionMock::remove( return arangodb::Result(TRI_ERROR_ARANGO_DOCUMENT_NOT_FOUND); } -arangodb::Result PhysicalCollectionMock::update( - arangodb::transaction::Methods* trx, arangodb::velocypack::Slice newSlice, - arangodb::ManagedDocumentResult& result, arangodb::OperationOptions& options, - arangodb::ManagedDocumentResult& previous) { +arangodb::Result PhysicalCollectionMock::update(arangodb::transaction::Methods* trx, + arangodb::velocypack::Slice newSlice, + arangodb::ManagedDocumentResult& result, + arangodb::OperationOptions& options, + arangodb::ManagedDocumentResult& previous) { return updateInternal(trx, newSlice, result, options, previous, true); } @@ -1342,7 +1412,8 @@ arangodb::Result PhysicalCollectionMock::updateInternal( TRI_ASSERT(doc.isObject()); arangodb::RevisionId oldRev = arangodb::RevisionId::fromSlice(doc); if (!checkRevision(trx, expectedRev, oldRev)) { - return arangodb::Result(TRI_ERROR_ARANGO_CONFLICT, "_rev values mismatch"); + return arangodb::Result(TRI_ERROR_ARANGO_CONFLICT, + "_rev values mismatch"); } } arangodb::velocypack::Builder builder; @@ -1389,14 +1460,15 @@ arangodb::Result PhysicalCollectionMock::updateProperties(arangodb::velocypack:: } std::shared_ptr StorageEngineMock::buildLinkMock( - arangodb::IndexId id, arangodb::LogicalCollection& collection, VPackSlice const& info) { + arangodb::IndexId id, arangodb::LogicalCollection& collection, VPackSlice const& info) { auto index = std::shared_ptr( - new arangodb::iresearch::IResearchLinkMock(id, collection)); - auto res = static_cast(index.get())->init(info, [](irs::directory& dir) { - if (arangodb::iresearch::IResearchLinkMock::InitCallback != nullptr) { - arangodb::iresearch::IResearchLinkMock::InitCallback(dir); - } - }); + new arangodb::iresearch::IResearchLinkMock(id, collection)); + auto res = static_cast(index.get()) + ->init(info, [](irs::directory& dir) { + if (arangodb::iresearch::IResearchLinkMock::InitCallback != nullptr) { + arangodb::iresearch::IResearchLinkMock::InitCallback(dir); + } + }); if (!res.ok()) { THROW_ARANGO_EXCEPTION(res); @@ -1418,15 +1490,13 @@ StorageEngineMock::StorageEngineMock(arangodb::application_features::Application vocbaseCount(1), _releasedTick(0) {} -arangodb::HealthData StorageEngineMock::healthCheck() { - return {}; -} +arangodb::HealthData StorageEngineMock::healthCheck() { return {}; } arangodb::WalAccess const* StorageEngineMock::walAccess() const { TRI_ASSERT(false); return nullptr; } - + void StorageEngineMock::addOptimizerRules(arangodb::aql::OptimizerRulesFeature& /*feature*/) { before(); // NOOP @@ -1436,9 +1506,7 @@ void StorageEngineMock::addRestHandlers(arangodb::rest::RestHandlerFactory& hand TRI_ASSERT(false); } -void StorageEngineMock::addV8Functions() { - TRI_ASSERT(false); -} +void StorageEngineMock::addV8Functions() { TRI_ASSERT(false); } void StorageEngineMock::changeCollection(TRI_vocbase_t& vocbase, arangodb::LogicalCollection const& collection, @@ -1528,7 +1596,7 @@ arangodb::Result StorageEngineMock::createView(TRI_vocbase_t& vocbase, return arangodb::Result(TRI_ERROR_NO_ERROR); // assume mock view persisted OK } - + arangodb::Result StorageEngineMock::compactAll(bool changeLevels, bool compactBottomMostLevel) { TRI_ASSERT(false); return arangodb::Result(); @@ -1588,8 +1656,8 @@ void StorageEngineMock::getCollectionInfo(TRI_vocbase_t& vocbase, arangodb::Data } ErrorCode StorageEngineMock::getCollectionsAndIndexes(TRI_vocbase_t& vocbase, - arangodb::velocypack::Builder& result, - bool wasCleanShutdown, bool isUpgrade) { + arangodb::velocypack::Builder& result, + bool wasCleanShutdown, bool isUpgrade) { TRI_ASSERT(false); return TRI_ERROR_INTERNAL; } @@ -1627,7 +1695,8 @@ arangodb::velocypack::Builder StorageEngineMock::getReplicationApplierConfigurat return arangodb::velocypack::Builder(); } -ErrorCode StorageEngineMock::getViews(TRI_vocbase_t& vocbase, arangodb::velocypack::Builder& result) { +ErrorCode StorageEngineMock::getViews(TRI_vocbase_t& vocbase, + arangodb::velocypack::Builder& result) { result.openArray(); for (auto& entry : views) { @@ -1656,9 +1725,9 @@ TRI_voc_tick_t StorageEngineMock::recoveryTick() { return recoveryTickResult; } -arangodb::Result StorageEngineMock::lastLogger( - TRI_vocbase_t& vocbase, - uint64_t tickStart, uint64_t tickEnd, arangodb::velocypack::Builder& builderSPtr) { +arangodb::Result StorageEngineMock::lastLogger(TRI_vocbase_t& vocbase, + uint64_t tickStart, uint64_t tickEnd, + arangodb::velocypack::Builder& builderSPtr) { TRI_ASSERT(false); return arangodb::Result(TRI_ERROR_NOT_IMPLEMENTED); } @@ -1701,14 +1770,14 @@ arangodb::Result StorageEngineMock::renameCollection(TRI_vocbase_t& vocbase, return arangodb::Result(TRI_ERROR_INTERNAL); } -ErrorCode StorageEngineMock::saveReplicationApplierConfiguration(TRI_vocbase_t& vocbase, - arangodb::velocypack::Slice slice, - bool doSync) { +ErrorCode StorageEngineMock::saveReplicationApplierConfiguration( + TRI_vocbase_t& vocbase, arangodb::velocypack::Slice slice, bool doSync) { TRI_ASSERT(false); return TRI_ERROR_NO_ERROR; } -ErrorCode StorageEngineMock::saveReplicationApplierConfiguration(arangodb::velocypack::Slice, bool) { +ErrorCode StorageEngineMock::saveReplicationApplierConfiguration(arangodb::velocypack::Slice, + bool) { TRI_ASSERT(false); return TRI_ERROR_NO_ERROR; } @@ -1783,7 +1852,8 @@ arangodb::Result TransactionCollectionMock::lockUsage() { } } - return arangodb::Result(_collection ? TRI_ERROR_NO_ERROR : TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND); + return arangodb::Result(_collection ? TRI_ERROR_NO_ERROR + : TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND); } arangodb::Result TransactionCollectionMock::doLock(arangodb::AccessMode::Type type) { @@ -1818,7 +1888,7 @@ TransactionStateMock::TransactionStateMock(TRI_vocbase_t& vocbase, arangodb::Tra arangodb::Result TransactionStateMock::abortTransaction(arangodb::transaction::Methods* trx) { ++abortTransactionCount; updateStatus(arangodb::transaction::Status::ABORTED); -// releaseUsage(); + // releaseUsage(); resetTransactionId(); return arangodb::Result(); @@ -1827,13 +1897,12 @@ arangodb::Result TransactionStateMock::abortTransaction(arangodb::transaction::M arangodb::Result TransactionStateMock::beginTransaction(arangodb::transaction::Hints hints) { ++beginTransactionCount; _hints = hints; - + arangodb::Result res = useCollections(); - if (res.fail()) { // something is wrong + if (res.fail()) { // something is wrong return res; } - if (!res.ok()) { updateStatus(arangodb::transaction::Status::ABORTED); resetTransactionId(); @@ -1851,7 +1920,7 @@ arangodb::Result TransactionStateMock::commitTransaction(arangodb::transaction:: return arangodb::Result(); } - + uint64_t TransactionStateMock::numCommits() const { return commitTransactionCount; } diff --git a/tests/Transaction/ManagerSetup.h b/tests/Transaction/ManagerSetup.h index 924a3947ed76..6b4b13a4397a 100644 --- a/tests/Transaction/ManagerSetup.h +++ b/tests/Transaction/ManagerSetup.h @@ -44,23 +44,23 @@ namespace arangodb { namespace tests { namespace mocks { - - // ----------------------------------------------------------------------------- - // --SECTION-- setup / tear-down - // ----------------------------------------------------------------------------- - - struct TransactionManagerSetup { - arangodb::tests::mocks::MockAqlServer server; - TransactionManagerSetup() : server(false) { - server.addFeature(true); - server.startFeatures(); - } +// ----------------------------------------------------------------------------- +// --SECTION-- setup / tear-down +// ----------------------------------------------------------------------------- - ~TransactionManagerSetup() = default; - }; +struct TransactionManagerSetup { + arangodb::tests::mocks::MockAqlServer server; -} // namespace tests -} // namespace mocks -} // namespace arangodb + TransactionManagerSetup() : server(false) { + TRI_ASSERT(server.server().hasFeature()); + server.startFeatures(); + } + + ~TransactionManagerSetup() = default; +}; + +} // namespace mocks +} // namespace tests +} // namespace arangodb #endif diff --git a/tests/js/server/aql/aql-graph-traverser.js b/tests/js/server/aql/aql-graph-traverser.js index 0569968691a4..60a502cba83b 100644 --- a/tests/js/server/aql/aql-graph-traverser.js +++ b/tests/js/server/aql/aql-graph-traverser.js @@ -2429,7 +2429,12 @@ function complexFilteringSuite() { // 1 Edge (Tri1->Tri2) // 1 Primary (Tri2) - assertEqual(stats.scannedIndex, 1); + if (isCluster) { + // one edge and one vertex lookup + assertEqual(stats.scannedIndex, 2); + } else { + assertEqual(stats.scannedIndex, 1); + } assertEqual(stats.filtered, 1); }, @@ -2513,7 +2518,7 @@ function complexFilteringSuite() { // 2 Primary lookup B,D // 2 Edge Lookups (2 B) (0 D) // 2 Primary Lookups (C, F) - assertTrue(stats.scannedIndex <= 5); + assertTrue(stats.scannedIndex <= 9); } else { // 2 Edge Lookups (A) // 2 Primary (B, D) for Filtering @@ -2556,7 +2561,7 @@ function complexFilteringSuite() { // 1 Primary lookup A // 2 Primary lookup B,D // 4 Primary Lookups (C, F, E, G) - assertTrue(stats.scannedIndex <= 7); + assertTrue(stats.scannedIndex <= 13); } else { // 2 Edge Lookups (A) // 4 Edge Lookups (2 B) (2 D) @@ -2600,7 +2605,7 @@ function complexFilteringSuite() { // 2 Primary lookup B,D // 2 Edge Lookups (0 B) (2 D) // 2 Primary Lookups (E, G) - assertTrue(stats.scannedIndex <= 5); + assertTrue(stats.scannedIndex <= 8); } else { // 2 Edge Lookups (A) // 2 Primary Lookups for Eval (B, D) @@ -2641,7 +2646,7 @@ function complexFilteringSuite() { // 1 Primary (B) // 2 Edge // 2 Primary (C,F) - assertTrue(stats.scannedIndex <= 4); + assertTrue(stats.scannedIndex <= 7); } else { // 2 Edge Lookups (A) // 2 Edge Lookups (B) @@ -2684,7 +2689,7 @@ function complexFilteringSuite() { // they may be inserted in the vertexToFetch list, which // lazy loads all vertices in it. if (stats.scannedIndex !== 8) { - assertTrue(stats.scannedIndex <= 5); + assertTrue(stats.scannedIndex <= 10); } } else { // 2 Edge Lookups (A) @@ -2739,7 +2744,7 @@ function complexFilteringSuite() { // 2 Primary lookup B,D // 2 Edge Lookups (2 B) (0 D) // 2 Primary Lookups (C, F) - assertTrue(stats.scannedIndex <= 5, stats.scannedIndex); + assertTrue(stats.scannedIndex <= 8, stats.scannedIndex); } else { // Cluster uses a lookup cache. // Pointless in single-server mode @@ -2797,7 +2802,7 @@ function complexFilteringSuite() { // 2 Primary lookup B,D // 2 Edge Lookups (2 B) (0 D) // 2 Primary Lookups (C, F) - assertTrue(stats.scannedIndex <= 5); + assertTrue(stats.scannedIndex <= 8); } else { // Cluster uses a lookup cache. // Pointless in single-server mode @@ -2847,7 +2852,7 @@ function complexFilteringSuite() { // 2 Primary lookup B,D // 2 Edge Lookups (2 B) (0 D) // 2 Primary Lookups (C, F) - assertTrue(stats.scannedIndex <= 7); + assertTrue(stats.scannedIndex <= 13, stats.scannedIndex); } else { // 2 Edge Lookups (A) // 2 Primary (B, D) for Filtering @@ -3619,7 +3624,7 @@ function optimizeQuantifierSuite() { let stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 5); + assertTrue(stats.scannedIndex <= 9, stats.scannedIndex); } else { // With traverser-read-cache // assertEqual(stats.scannedIndex, 9); @@ -3664,7 +3669,7 @@ function optimizeQuantifierSuite() { let stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 4); + assertTrue(stats.scannedIndex <= 8, stats.scannedIndex); } else { // With traverser-read-cache // assertEqual(stats.scannedIndex, 8); @@ -3693,7 +3698,7 @@ function optimizeQuantifierSuite() { stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 4); + assertTrue(stats.scannedIndex <= 8, stats.scannedIndex); } else { // With traverser-read-cache // assertEqual(stats.scannedIndex, 8); @@ -3723,7 +3728,7 @@ function optimizeQuantifierSuite() { let stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 5); + assertTrue(stats.scannedIndex <= 9, stats.scannedIndex); } else { // With traverser-read-cache // assertEqual(stats.scannedIndex, 9); @@ -3768,7 +3773,7 @@ function optimizeQuantifierSuite() { let stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 4); + assertTrue(stats.scannedIndex <= 8, stats.scannedIndex); } else { // With traverser-read-cache // assertEqual(stats.scannedIndex, 8); @@ -3796,7 +3801,7 @@ function optimizeQuantifierSuite() { stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 4); + assertTrue(stats.scannedIndex <= 8); } else { // With traverser-read-cache // assertEqual(stats.scannedIndex, 8); @@ -3827,7 +3832,7 @@ function optimizeQuantifierSuite() { let stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 5); + assertTrue(stats.scannedIndex <= 9, stats.scannedIndex); } else { // With traverser-read-cache // assertEqual(stats.scannedIndex, 9); @@ -3858,7 +3863,7 @@ function optimizeQuantifierSuite() { let stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 3); + assertTrue(stats.scannedIndex <= 7, stats.scannedIndex); } else { // With activated traverser-read-cache: // assertEqual(stats.scannedIndex, 7); @@ -3889,7 +3894,7 @@ function optimizeQuantifierSuite() { let stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 5); + assertTrue(stats.scannedIndex <= 9, stats.scannedIndex); } else { // With traverser-read-cache // assertEqual(stats.scannedIndex, 9); @@ -3921,7 +3926,7 @@ function optimizeQuantifierSuite() { let stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 3); + assertTrue(stats.scannedIndex <= 7, stats.scannedIndex); } else { // With activated traverser-read-cache: // assertEqual(stats.scannedIndex, 7); @@ -3953,7 +3958,7 @@ function optimizeQuantifierSuite() { let stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 5); + assertTrue(stats.scannedIndex <= 9, stats.scannedIndex); } else { // With activated traverser-read-cache: // assertEqual(stats.scannedIndex, 9); @@ -3985,7 +3990,7 @@ function optimizeQuantifierSuite() { let stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 3); + assertTrue(stats.scannedIndex <= 6, stats.scannedIndex); } else { // With activated traverser-read-cache: // assertEqual(stats.scannedIndex, 7);