From e951cba9505dfccae9d7ddc8c7ad611dd51d2f1f Mon Sep 17 00:00:00 2001 From: Heiko Date: Thu, 22 Apr 2021 13:22:00 +0200 Subject: [PATCH 01/72] Feature/hybrid smart graph api (#14037) Added First Draft of Hybrid Smart Graph API --- arangod/Aql/GraphNode.cpp | 3 ++ arangod/Aql/GraphNode.h | 6 +++ arangod/Graph/Graph.cpp | 64 ++++++++++++++++++++++++++++++- arangod/Graph/Graph.h | 41 ++++++++++++++++++-- arangod/Graph/GraphManager.cpp | 42 ++++++++++++++++---- arangod/Graph/GraphOperations.cpp | 2 +- lib/Basics/StaticStrings.cpp | 4 +- lib/Basics/StaticStrings.h | 4 +- 8 files changed, 150 insertions(+), 16 deletions(-) diff --git a/arangod/Aql/GraphNode.cpp b/arangod/Aql/GraphNode.cpp index cc91156163a6..be76af1636b8 100644 --- a/arangod/Aql/GraphNode.cpp +++ b/arangod/Aql/GraphNode.cpp @@ -107,6 +107,7 @@ GraphNode::GraphNode(ExecutionPlan* plan, ExecutionNodeId id, _optionsBuilt(false), _isSmart(false), _isDisjoint(false), + _isHybrid(false), _options(std::move(options)) { // Direction is already the correct Integer. // Is not inserted by user but by enum. @@ -810,6 +811,8 @@ bool GraphNode::isSmart() const { return _isSmart; } bool GraphNode::isDisjoint() const { return _isDisjoint; } +bool GraphNode::isHybrid() const { return _isHybrid; } + TRI_vocbase_t* GraphNode::vocbase() const { return _vocbase; } Variable const* GraphNode::vertexOutVariable() const { diff --git a/arangod/Aql/GraphNode.h b/arangod/Aql/GraphNode.h index 6a7e0518d42e..0531fe39f028 100644 --- a/arangod/Aql/GraphNode.h +++ b/arangod/Aql/GraphNode.h @@ -123,6 +123,9 @@ class GraphNode : public ExecutionNode { /// @brief flag, if the graph is a Disjoint SmartGraph (Enterprise Edition only!) bool isDisjoint() const; + /// @brief flag, if the graph is a Hybrid SmartGraph (Enterprise Edition only!) + bool isHybrid() const; + /// @brief return the database TRI_vocbase_t* vocbase() const; @@ -256,6 +259,9 @@ class GraphNode : public ExecutionNode { /// @brief flag, if graph is smart *and* disjoint (Enterprise Edition only!) bool _isDisjoint; + /// @brief flag, if graph is smart *and* hybrid (Enterprise Edition only!) + bool _isHybrid; + /// @brief The directions edges are followed std::vector _directions; diff --git a/arangod/Graph/Graph.cpp b/arangod/Graph/Graph.cpp index d48523282617..0ab3270d36c9 100644 --- a/arangod/Graph/Graph.cpp +++ b/arangod/Graph/Graph.cpp @@ -116,6 +116,19 @@ Graph::Graph(velocypack::Slice const& slice, ServerDefaults const& serverDefault TRI_ASSERT(!_rev.empty()); if (slice.hasKey(StaticStrings::GraphEdgeDefinitions)) { + // TODO Feature HybridSmartGraphs: Check why we're landing here and not in SmartGraphEE - Cleanup! + if (slice.isObject()) { + if (slice.hasKey(StaticStrings::GraphSatellites) && + slice.get(StaticStrings::GraphSatellites).isArray()) { + + for (VPackSlice it : VPackArrayIterator(slice.get(StaticStrings::GraphSatellites))) { + if (!it.isString()) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_GRAPH_INVALID_PARAMETER); + } + _satelliteColls.emplace(it.copyString()); + } + } + } parseEdgeDefinitions(slice.get(StaticStrings::GraphEdgeDefinitions)); } if (slice.hasKey(StaticStrings::GraphOrphans)) { @@ -146,6 +159,19 @@ Graph::Graph(TRI_vocbase_t& vocbase, std::string&& graphName, TRI_ASSERT(_rev.empty()); if (info.hasKey(StaticStrings::GraphEdgeDefinitions)) { + // TODO Feature HybridSmartGraphs: Check why we're landing here and not in SmartGraphEE - Cleanup! + if (options.isObject()) { + if (options.hasKey(StaticStrings::GraphSatellites) && + options.get(StaticStrings::GraphSatellites).isArray()) { + + for (VPackSlice it : VPackArrayIterator(options.get(StaticStrings::GraphSatellites))) { + if (!it.isString()) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_GRAPH_INVALID_PARAMETER); + } + _satelliteColls.emplace(it.copyString()); + } + } + } parseEdgeDefinitions(info.get(StaticStrings::GraphEdgeDefinitions)); } if (info.hasKey(StaticStrings::GraphOrphans)) { @@ -212,10 +238,18 @@ std::set const& Graph::orphanCollections() const { return _orphanColls; } +std::set const& Graph::satelliteCollections() const { + return _satelliteColls; +} + std::set const& Graph::edgeCollections() const { return _edgeColls; } +bool Graph::needsToBeSatellite(std::string const& edge) const { + return false; +} + std::map const& Graph::edgeDefinitions() const { return _edgeDefs; } @@ -417,9 +451,10 @@ void EdgeDefinition::toVelocyPack(VPackBuilder& builder) const { builder.add(VPackValue(to)); } builder.close(); // array + builder.add("type", VPackValue(getType())); } -ResultT EdgeDefinition::createFromVelocypack(VPackSlice edgeDefinition) { +ResultT EdgeDefinition::createFromVelocypack(VPackSlice edgeDefinition, std::set const& satCollections) { Result res = EdgeDefinition::validateEdgeDefinition(edgeDefinition); if (res.fail()) { return res; @@ -433,9 +468,11 @@ ResultT EdgeDefinition::createFromVelocypack(VPackSlice edgeDefi // duplicates in from and to shouldn't occur, but are safely ignored here for (VPackSlice it : VPackArrayIterator(from)) { + TRI_ASSERT(it.isString()); fromSet.emplace(it.copyString()); } for (VPackSlice it : VPackArrayIterator(to)) { + TRI_ASSERT(it.isString()); toSet.emplace(it.copyString()); } @@ -484,6 +521,19 @@ bool EdgeDefinition::renameCollection(std::string const& oldName, std::string co return renamed; } +auto EdgeDefinition::getType() const -> EdgeDefinitionType { + return _type; +} + +auto EdgeDefinition::setType(EdgeDefinitionType type) -> bool { + TRI_ASSERT(type != EdgeDefinitionType::DEFAULT); + if (_type == EdgeDefinitionType::DEFAULT) { + _type = type; + return true; + } + return false; +} + bool EdgeDefinition::isFromVertexCollectionUsed(std::string const& collectionName) const { if (getFrom().find(collectionName) != getFrom().end()) { return true; @@ -522,6 +572,7 @@ void EdgeDefinition::addToBuilder(VPackBuilder& builder) const { builder.add(VPackValue(to)); } builder.close(); // to + builder.add(StaticStrings::GraphEdgeDefinitionType, VPackValue(getType())); builder.close(); // obj } @@ -590,7 +641,7 @@ ResultT Graph::addEdgeDefinition(EdgeDefinition const& ed } ResultT Graph::addEdgeDefinition(VPackSlice const& edgeDefinitionSlice) { - auto res = EdgeDefinition::createFromVelocypack(edgeDefinitionSlice); + auto res = EdgeDefinition::createFromVelocypack(edgeDefinitionSlice, satelliteCollections()); if (res.fail()) { return std::move(res).result(); @@ -724,6 +775,11 @@ bool Graph::isDisjoint() const { return false; } +bool Graph::isHybrid() const { + TRI_ASSERT(_satelliteColls.empty()); + return false; +} + bool Graph::isSatellite() const { return _isSatellite; } void Graph::createCollectionOptions(VPackBuilder& builder, bool waitForSync) const { @@ -743,6 +799,10 @@ void Graph::createCollectionOptions(VPackBuilder& builder, bool waitForSync) con builder.add(StaticStrings::ReplicationFactor, VPackValue(replicationFactor())); } +void Graph::createSatelliteCollectionOptions(VPackBuilder& builder, bool waitForSync) const { + TRI_ASSERT(false); +} + std::optional> Graph::getEdgeDefinition( std::string const& collectionName) const { auto it = edgeDefinitions().find(collectionName); diff --git a/arangod/Graph/Graph.h b/arangod/Graph/Graph.h index 22cab54f8af1..16034af130e8 100644 --- a/arangod/Graph/Graph.h +++ b/arangod/Graph/Graph.h @@ -46,12 +46,23 @@ struct ServerDefaults; namespace graph { class EdgeDefinition { + public: + enum EdgeDefinitionType { + DEFAULT, + SMART_TO_SMART, + SAT_TO_SAT, + SMART_TO_SAT, + SAT_TO_SMART + }; + public: EdgeDefinition(std::string edgeCollection_, std::set&& from_, - std::set&& to_) + std::set&& to_, + EdgeDefinitionType type = EdgeDefinitionType::DEFAULT) : _edgeCollection(std::move(edgeCollection_)), _from(std::move(from_)), - _to(std::move(to_)) {} + _to(std::move(to_)), + _type(type) {} std::string const& getName() const { return _edgeCollection; } void setName(std::string const& newName) { _edgeCollection = newName; } @@ -70,7 +81,8 @@ class EdgeDefinition { /// types of values. static Result validateEdgeDefinition(const velocypack::Slice& edgeDefinition); - static ResultT createFromVelocypack(velocypack::Slice edgeDefinition); + static ResultT createFromVelocypack(velocypack::Slice edgeDefinition, + std::set const& satCollections); void toVelocyPack(velocypack::Builder&) const; @@ -82,10 +94,23 @@ class EdgeDefinition { bool renameCollection(std::string const& oldName, std::string const& newName); + auto getType() const -> EdgeDefinitionType; + + /* @brief + * Set type of the EdgeDefinition. Only allowed to be called once and only if + * type is DEFAULT. If type has been set, it is not changeable anymore. + * + * @param type Type to be set + * + * @return True if type has been set, returns false in case type has not been set. + */ + auto setType(EdgeDefinitionType type) -> bool; + private: std::string _edgeCollection; std::set _from; std::set _to; + EdgeDefinitionType _type; }; class Graph { @@ -151,6 +176,8 @@ class Graph { virtual void createCollectionOptions(VPackBuilder& builder, bool waitForSync) const; + virtual void createSatelliteCollectionOptions(VPackBuilder& builder, bool waitForSync) const; + public: /// @brief get the cids of all vertexCollections std::set const& vertexCollections() const; @@ -158,6 +185,9 @@ class Graph { /// @brief get the cids of all orphanCollections std::set const& orphanCollections() const; + /// @brief get the cids of all satelliteCollections + std::set const& satelliteCollections() const; + /// @brief get the cids of all edgeCollections std::set const& edgeCollections() const; @@ -178,6 +208,8 @@ class Graph { virtual bool isSmart() const; virtual bool isDisjoint() const; virtual bool isSatellite() const; + virtual bool isHybrid() const; + virtual bool needsToBeSatellite(std::string const& edge) const; uint64_t numberOfShards() const; uint64_t replicationFactor() const; @@ -291,6 +323,9 @@ class Graph { /// @brief the names of all orphanCollections std::set _orphanColls; + /// @brief the names of all orphanCollections + std::set _satelliteColls; + /// @brief the names of all edgeCollections std::set _edgeColls; diff --git a/arangod/Graph/GraphManager.cpp b/arangod/Graph/GraphManager.cpp index e1cdeeefc80b..ea8e8260f3ae 100644 --- a/arangod/Graph/GraphManager.cpp +++ b/arangod/Graph/GraphManager.cpp @@ -459,8 +459,7 @@ OperationResult GraphManager::storeGraph(Graph const& graph, bool waitForSync, Result GraphManager::applyOnAllGraphs(std::function)> const& callback) const { std::string const queryStr{"FOR g IN _graphs RETURN g"}; arangodb::aql::Query query(transaction::StandaloneContext::Create(_vocbase), - arangodb::aql::QueryString{queryStr}, - nullptr); + arangodb::aql::QueryString{queryStr}, nullptr); query.queryOptions().skipAudit = true; aql::QueryResult queryResult = query.executeSync(); @@ -505,7 +504,9 @@ Result GraphManager::applyOnAllGraphs(std::function documentCollectionsToCreate{}; + std::unordered_set satelliteDocumentCollectionsToCreate{}; std::unordered_set edgeCollectionsToCreate{}; + std::unordered_set satelliteEdgeCollectionsToCreate{}; std::unordered_set> existentDocumentCollections{}; std::unordered_set> existentEdgeCollections{}; @@ -533,7 +534,11 @@ Result GraphManager::ensureCollections(Graph* graph, bool waitForSync) const { return res; } else { // not found the collection, need to create it later - edgeCollectionsToCreate.emplace(edgeColl); + if (graph->isHybrid() && graph->needsToBeSatellite(edgeColl)) { // check for satellites + satelliteEdgeCollectionsToCreate.emplace(edgeColl); + } else { + edgeCollectionsToCreate.emplace(edgeColl); + } } } @@ -550,7 +555,11 @@ Result GraphManager::ensureCollections(Graph* graph, bool waitForSync) const { return res; } else { if (edgeCollectionsToCreate.find(vertexColl) == edgeCollectionsToCreate.end()) { - documentCollectionsToCreate.emplace(vertexColl); + if (graph->isHybrid() && graph->satelliteCollections().find(vertexColl) != graph->satelliteCollections().end()) { + satelliteDocumentCollectionsToCreate.emplace(vertexColl); + } else { + documentCollectionsToCreate.emplace(vertexColl); + } } } } @@ -610,6 +619,22 @@ Result GraphManager::ensureCollections(Graph* graph, bool waitForSync) const { collectionsToCreate.emplace_back( CollectionCreationInfo{edgeColl, TRI_COL_TYPE_EDGE, options}); } + + // new builder + VPackBuilder satOptionsBuilder; + if (graph->isHybrid()) { + satOptionsBuilder.openObject(); + graph->createSatelliteCollectionOptions(satOptionsBuilder, waitForSync); + satOptionsBuilder.close(); + for (auto const& satColl : satelliteDocumentCollectionsToCreate) { + collectionsToCreate.emplace_back(CollectionCreationInfo{satColl, TRI_COL_TYPE_DOCUMENT, + satOptionsBuilder.slice()}); + } + for (auto const& satEdgeColl : satelliteEdgeCollectionsToCreate) { + collectionsToCreate.emplace_back(CollectionCreationInfo{satEdgeColl, TRI_COL_TYPE_EDGE, + satOptionsBuilder.slice()}); + } + } if (collectionsToCreate.empty()) { // NOTE: Empty graph is allowed. return TRI_ERROR_NO_ERROR; @@ -650,8 +675,7 @@ Result GraphManager::readGraphKeys(velocypack::Builder& builder) const { Result GraphManager::readGraphByQuery(velocypack::Builder& builder, std::string const& queryStr) const { - arangodb::aql::Query query(ctx(), arangodb::aql::QueryString(queryStr), - nullptr); + arangodb::aql::Query query(ctx(), arangodb::aql::QueryString(queryStr), nullptr); query.queryOptions().skipAudit = true; LOG_TOPIC("f6782", DEBUG, arangodb::Logger::GRAPHS) @@ -670,7 +694,7 @@ Result GraphManager::readGraphByQuery(velocypack::Builder& builder, if (graphsSlice.isNone()) { return Result(TRI_ERROR_OUT_OF_MEMORY); - } + } if (!graphsSlice.isArray()) { LOG_TOPIC("338b7", ERR, arangodb::Logger::GRAPHS) << "cannot read graphs from _graphs collection"; @@ -1037,7 +1061,9 @@ ResultT> GraphManager::buildGraphFromInput(std::string co s = s.get(StaticStrings::ReplicationFactor); if ((s.isNumber() && s.getNumber() == 0) || (s.isString() && s.stringRef() == "satellite")) { - return Result{TRI_ERROR_BAD_PARAMETER, "invalid combination of 'isSmart' and 'satellite' replicationFactor"}; + return Result{TRI_ERROR_BAD_PARAMETER, + "invalid combination of 'isSmart' and 'satellite' " + "replicationFactor"}; } } } diff --git a/arangod/Graph/GraphOperations.cpp b/arangod/Graph/GraphOperations.cpp index fa696e284186..ab632adb8472 100644 --- a/arangod/Graph/GraphOperations.cpp +++ b/arangod/Graph/GraphOperations.cpp @@ -226,7 +226,7 @@ OperationResult GraphOperations::editEdgeDefinition(VPackSlice edgeDefinitionSli bool waitForSync, std::string const& edgeDefinitionName) { OperationOptions options(ExecContext::current()); - auto maybeEdgeDef = EdgeDefinition::createFromVelocypack(edgeDefinitionSlice); + auto maybeEdgeDef = EdgeDefinition::createFromVelocypack(edgeDefinitionSlice, graph().satelliteCollections()); if (!maybeEdgeDef) { return OperationResult{std::move(maybeEdgeDef).result(), options}; } diff --git a/lib/Basics/StaticStrings.cpp b/lib/Basics/StaticStrings.cpp index bf22cc751b4c..b67b58f924ed 100644 --- a/lib/Basics/StaticStrings.cpp +++ b/lib/Basics/StaticStrings.cpp @@ -264,14 +264,17 @@ std::string const StaticStrings::ShardingSingle("single"); // graph attribute names std::string const StaticStrings::GraphCollection("_graphs"); std::string const StaticStrings::IsDisjoint("isDisjoint"); +std::string const StaticStrings::IsHybrid("isHybrid"); std::string const StaticStrings::GraphIsSmart("isSmart"); std::string const StaticStrings::GraphIsSatellite("isSatellite"); +std::string const StaticStrings::GraphSatellites("satellites"); std::string const StaticStrings::GraphFrom("from"); std::string const StaticStrings::GraphTo("to"); std::string const StaticStrings::GraphOptions("options"); std::string const StaticStrings::GraphSmartGraphAttribute( "smartGraphAttribute"); std::string const StaticStrings::GraphEdgeDefinitions("edgeDefinitions"); +std::string const StaticStrings::GraphEdgeDefinitionType("type"); std::string const StaticStrings::GraphOrphans("orphanCollections"); std::string const StaticStrings::GraphInitial("initial"); std::string const StaticStrings::GraphInitialCid("initialCid"); @@ -319,7 +322,6 @@ std::string const StaticStrings::GraphQueryShortestPathType("shortestPathType"); // rest query parameter std::string const StaticStrings::GraphDropCollections("dropCollections"); std::string const StaticStrings::GraphDropCollection("dropCollection"); -std::string const StaticStrings::GraphCreateCollections("createCollections"); std::string const StaticStrings::GraphCreateCollection("createCollection"); // Replication diff --git a/lib/Basics/StaticStrings.h b/lib/Basics/StaticStrings.h index e0ea921aa957..7274a2067e19 100644 --- a/lib/Basics/StaticStrings.h +++ b/lib/Basics/StaticStrings.h @@ -246,7 +246,9 @@ class StaticStrings { // graph attribute names static std::string const GraphCollection; static std::string const IsDisjoint; + static std::string const IsHybrid; static std::string const GraphIsSatellite; + static std::string const GraphSatellites; static std::string const GraphIsSmart; static std::string const GraphFrom; static std::string const GraphTo; @@ -254,9 +256,9 @@ class StaticStrings { static std::string const GraphSmartGraphAttribute; static std::string const GraphDropCollections; static std::string const GraphDropCollection; - static std::string const GraphCreateCollections; static std::string const GraphCreateCollection; static std::string const GraphEdgeDefinitions; + static std::string const GraphEdgeDefinitionType; static std::string const GraphOrphans; static std::string const GraphInitial; static std::string const GraphInitialCid; From 1b4657c86c5469b362c21068c2085f0910b44760 Mon Sep 17 00:00:00 2001 From: Heiko Kernbach Date: Thu, 22 Apr 2021 18:13:19 +0200 Subject: [PATCH 02/72] add satellites to GraphNode --- arangod/Aql/GraphNode.cpp | 32 +++++++++++++++++++++++++++----- arangod/Aql/GraphNode.h | 5 +++++ 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/arangod/Aql/GraphNode.cpp b/arangod/Aql/GraphNode.cpp index be76af1636b8..378fd61ad651 100644 --- a/arangod/Aql/GraphNode.cpp +++ b/arangod/Aql/GraphNode.cpp @@ -140,7 +140,7 @@ GraphNode::GraphNode(ExecutionPlan* plan, ExecutionNodeId id, } std::string n = col->getString(); auto c = ci.getCollection(_vocbase->name(), n); - if (c->isSmart() && !c->isDisjoint()) { + if (c->isSmart() && !c->isDisjoint()) { // HEIKO CHECK _isDisjoint = false; } if (!c->isSmart() || c->distributeShardsLike().empty()) { @@ -247,6 +247,7 @@ GraphNode::GraphNode(ExecutionPlan* plan, ExecutionNodeId id, if (ServerState::instance()->isRunningInCluster()) { _isSmart = _graphObj->isSmart(); _isDisjoint = _graphObj->isDisjoint(); + _isHybrid = _graphObj->isHybrid(); } for (const auto& n : eColls) { @@ -286,6 +287,15 @@ GraphNode::GraphNode(ExecutionPlan* plan, ExecutionNodeId id, for (auto const& v : vColls) { addVertexCollection(collections, v); } + + auto sColls = _graphObj->satelliteCollections(); + length = sColls.size(); + if (length > 0) { + _satelliteColls.reserve(length); + for (auto const& s : sColls) { + addVertexCollection(collections, s); + } + } } } @@ -301,9 +311,12 @@ GraphNode::GraphNode(ExecutionPlan* plan, arangodb::velocypack::Slice const& bas _defaultDirection(uint64ToDirection(arangodb::basics::VelocyPackHelper::stringUInt64( base.get("defaultDirection")))), _optionsBuilt(false), - _isSmart(arangodb::basics::VelocyPackHelper::getBooleanValue(base, "isSmart", false)), - _isDisjoint(arangodb::basics::VelocyPackHelper::getBooleanValue(base, "isDisjoint", false)) { - + _isSmart(arangodb::basics::VelocyPackHelper::getBooleanValue(base, + "isSmart", false)), + _isDisjoint( + arangodb::basics::VelocyPackHelper::getBooleanValue(base, "isDisjoint", false)), + _isHybrid( + arangodb::basics::VelocyPackHelper::getBooleanValue(base, "isHybrid", false)) { if (!ServerState::instance()->isDBServer()) { // Graph Information. Do we need to reload the graph here? std::string graphName; @@ -458,6 +471,7 @@ GraphNode::GraphNode(ExecutionPlan* plan, ExecutionNodeId id, TRI_vocbase_t* voc _optionsBuilt(false), _isSmart(false), _isDisjoint(false), + _isHybrid(false), _directions(std::move(directions)), _options(std::move(options)) { setGraphInfoAndCopyColls(edgeColls, vertexColls); @@ -491,6 +505,7 @@ GraphNode::GraphNode(ExecutionPlan& plan, GraphNode const& other, _optionsBuilt(false), _isSmart(other.isSmart()), _isDisjoint(other.isDisjoint()), + _isHybrid(other.isHybrid()), _directions(other._directions), _options(std::move(options)), _collectionToShard(other._collectionToShard) { @@ -791,7 +806,7 @@ void GraphNode::addVertexCollection(aql::Collection& collection) { std::vector GraphNode::collections() const { std::unordered_set set; - set.reserve(_edgeColls.size() + _vertexColls.size()); + set.reserve(_edgeColls.size() + _vertexColls.size() + _satelliteColls.size()); for (auto const& collPointer : _edgeColls) { set.emplace(collPointer); @@ -799,6 +814,9 @@ std::vector GraphNode::collections() const { for (auto const& collPointer : _vertexColls) { set.emplace(collPointer); } + for (auto const& collPointer : _satelliteColls) { + set.emplace(collPointer); + } std::vector vector; vector.reserve(set.size()); @@ -845,6 +863,10 @@ std::vector const& GraphNode::vertexColls() const { return _vertexColls; } +std::vector const& GraphNode::satelliteColls() const { + return _satelliteColls; +} + graph::Graph const* GraphNode::graph() const noexcept { return _graphObj; } bool GraphNode::isEligibleAsSatelliteTraversal() const { diff --git a/arangod/Aql/GraphNode.h b/arangod/Aql/GraphNode.h index 0531fe39f028..d64cdf609cca 100644 --- a/arangod/Aql/GraphNode.h +++ b/arangod/Aql/GraphNode.h @@ -181,6 +181,8 @@ class GraphNode : public ExecutionNode { std::vector const& vertexColls() const; + std::vector const& satelliteColls() const; + virtual void getConditionVariables(std::vector&) const; /// @brief return any of the collections. @@ -246,6 +248,9 @@ class GraphNode : public ExecutionNode { /// as an edge can also point to another edge, instead of a vertex). std::vector _vertexColls; + /// @brief the satellite collection names + std::vector _satelliteColls; + /// @brief The default direction given in the query TRI_edge_direction_e const _defaultDirection; From a85416cb49a14aaba4f6c475796885a3a4ec0d91 Mon Sep 17 00:00:00 2001 From: Heiko Kernbach Date: Thu, 22 Apr 2021 18:15:36 +0200 Subject: [PATCH 03/72] add satellites to ShardLocking --- arangod/Aql/ShardLocking.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/arangod/Aql/ShardLocking.cpp b/arangod/Aql/ShardLocking.cpp index 81b83c98d147..c752aeff953c 100644 --- a/arangod/Aql/ShardLocking.cpp +++ b/arangod/Aql/ShardLocking.cpp @@ -75,6 +75,16 @@ void ShardLocking::addNode(ExecutionNode const* baseNode, size_t snippetId, for (auto const& col : graphNode->vertexColls()) { updateLocking(col, AccessMode::Type::READ, snippetId, {}, isUsedAsSatellite(col)); } + if (graphNode->isHybrid()) { + // Add all Satellite Collections to the Transactions, Traversals do never + // write, the collections have been adjusted already. Additional Satellite + // Collection can only be used in a hybrid environment + for (auto const& col : graphNode->satelliteColls()) { + // TODO Feature HybridSmartGraphs: think we can just set last param here to true, but double check + updateLocking(col, AccessMode::Type::READ, snippetId, {}, true); + } + } + break; } case ExecutionNode::ENUMERATE_COLLECTION: From fec767cf747c87aa3619d9297ea7fb48d4dcf8f5 Mon Sep 17 00:00:00 2001 From: Heiko Kernbach Date: Thu, 22 Apr 2021 18:34:19 +0200 Subject: [PATCH 04/72] Revert "add satellites to ShardLocking" This reverts commit a85416cb49a14aaba4f6c475796885a3a4ec0d91. --- arangod/Aql/ShardLocking.cpp | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/arangod/Aql/ShardLocking.cpp b/arangod/Aql/ShardLocking.cpp index c752aeff953c..81b83c98d147 100644 --- a/arangod/Aql/ShardLocking.cpp +++ b/arangod/Aql/ShardLocking.cpp @@ -75,16 +75,6 @@ void ShardLocking::addNode(ExecutionNode const* baseNode, size_t snippetId, for (auto const& col : graphNode->vertexColls()) { updateLocking(col, AccessMode::Type::READ, snippetId, {}, isUsedAsSatellite(col)); } - if (graphNode->isHybrid()) { - // Add all Satellite Collections to the Transactions, Traversals do never - // write, the collections have been adjusted already. Additional Satellite - // Collection can only be used in a hybrid environment - for (auto const& col : graphNode->satelliteColls()) { - // TODO Feature HybridSmartGraphs: think we can just set last param here to true, but double check - updateLocking(col, AccessMode::Type::READ, snippetId, {}, true); - } - } - break; } case ExecutionNode::ENUMERATE_COLLECTION: From 5bee2ce856768469c017db8bb5c33628713b4f1c Mon Sep 17 00:00:00 2001 From: Heiko Kernbach Date: Thu, 22 Apr 2021 18:34:27 +0200 Subject: [PATCH 05/72] Revert "add satellites to GraphNode" This reverts commit 1b4657c86c5469b362c21068c2085f0910b44760. --- arangod/Aql/GraphNode.cpp | 32 +++++--------------------------- arangod/Aql/GraphNode.h | 5 ----- 2 files changed, 5 insertions(+), 32 deletions(-) diff --git a/arangod/Aql/GraphNode.cpp b/arangod/Aql/GraphNode.cpp index 378fd61ad651..be76af1636b8 100644 --- a/arangod/Aql/GraphNode.cpp +++ b/arangod/Aql/GraphNode.cpp @@ -140,7 +140,7 @@ GraphNode::GraphNode(ExecutionPlan* plan, ExecutionNodeId id, } std::string n = col->getString(); auto c = ci.getCollection(_vocbase->name(), n); - if (c->isSmart() && !c->isDisjoint()) { // HEIKO CHECK + if (c->isSmart() && !c->isDisjoint()) { _isDisjoint = false; } if (!c->isSmart() || c->distributeShardsLike().empty()) { @@ -247,7 +247,6 @@ GraphNode::GraphNode(ExecutionPlan* plan, ExecutionNodeId id, if (ServerState::instance()->isRunningInCluster()) { _isSmart = _graphObj->isSmart(); _isDisjoint = _graphObj->isDisjoint(); - _isHybrid = _graphObj->isHybrid(); } for (const auto& n : eColls) { @@ -287,15 +286,6 @@ GraphNode::GraphNode(ExecutionPlan* plan, ExecutionNodeId id, for (auto const& v : vColls) { addVertexCollection(collections, v); } - - auto sColls = _graphObj->satelliteCollections(); - length = sColls.size(); - if (length > 0) { - _satelliteColls.reserve(length); - for (auto const& s : sColls) { - addVertexCollection(collections, s); - } - } } } @@ -311,12 +301,9 @@ GraphNode::GraphNode(ExecutionPlan* plan, arangodb::velocypack::Slice const& bas _defaultDirection(uint64ToDirection(arangodb::basics::VelocyPackHelper::stringUInt64( base.get("defaultDirection")))), _optionsBuilt(false), - _isSmart(arangodb::basics::VelocyPackHelper::getBooleanValue(base, - "isSmart", false)), - _isDisjoint( - arangodb::basics::VelocyPackHelper::getBooleanValue(base, "isDisjoint", false)), - _isHybrid( - arangodb::basics::VelocyPackHelper::getBooleanValue(base, "isHybrid", false)) { + _isSmart(arangodb::basics::VelocyPackHelper::getBooleanValue(base, "isSmart", false)), + _isDisjoint(arangodb::basics::VelocyPackHelper::getBooleanValue(base, "isDisjoint", false)) { + if (!ServerState::instance()->isDBServer()) { // Graph Information. Do we need to reload the graph here? std::string graphName; @@ -471,7 +458,6 @@ GraphNode::GraphNode(ExecutionPlan* plan, ExecutionNodeId id, TRI_vocbase_t* voc _optionsBuilt(false), _isSmart(false), _isDisjoint(false), - _isHybrid(false), _directions(std::move(directions)), _options(std::move(options)) { setGraphInfoAndCopyColls(edgeColls, vertexColls); @@ -505,7 +491,6 @@ GraphNode::GraphNode(ExecutionPlan& plan, GraphNode const& other, _optionsBuilt(false), _isSmart(other.isSmart()), _isDisjoint(other.isDisjoint()), - _isHybrid(other.isHybrid()), _directions(other._directions), _options(std::move(options)), _collectionToShard(other._collectionToShard) { @@ -806,7 +791,7 @@ void GraphNode::addVertexCollection(aql::Collection& collection) { std::vector GraphNode::collections() const { std::unordered_set set; - set.reserve(_edgeColls.size() + _vertexColls.size() + _satelliteColls.size()); + set.reserve(_edgeColls.size() + _vertexColls.size()); for (auto const& collPointer : _edgeColls) { set.emplace(collPointer); @@ -814,9 +799,6 @@ std::vector GraphNode::collections() const { for (auto const& collPointer : _vertexColls) { set.emplace(collPointer); } - for (auto const& collPointer : _satelliteColls) { - set.emplace(collPointer); - } std::vector vector; vector.reserve(set.size()); @@ -863,10 +845,6 @@ std::vector const& GraphNode::vertexColls() const { return _vertexColls; } -std::vector const& GraphNode::satelliteColls() const { - return _satelliteColls; -} - graph::Graph const* GraphNode::graph() const noexcept { return _graphObj; } bool GraphNode::isEligibleAsSatelliteTraversal() const { diff --git a/arangod/Aql/GraphNode.h b/arangod/Aql/GraphNode.h index d64cdf609cca..0531fe39f028 100644 --- a/arangod/Aql/GraphNode.h +++ b/arangod/Aql/GraphNode.h @@ -181,8 +181,6 @@ class GraphNode : public ExecutionNode { std::vector const& vertexColls() const; - std::vector const& satelliteColls() const; - virtual void getConditionVariables(std::vector&) const; /// @brief return any of the collections. @@ -248,9 +246,6 @@ class GraphNode : public ExecutionNode { /// as an edge can also point to another edge, instead of a vertex). std::vector _vertexColls; - /// @brief the satellite collection names - std::vector _satelliteColls; - /// @brief The default direction given in the query TRI_edge_direction_e const _defaultDirection; From 0392473f3328a1e562cde00e2e5002d2511b2b5d Mon Sep 17 00:00:00 2001 From: Heiko Date: Wed, 30 Jun 2021 15:21:12 +0200 Subject: [PATCH 06/72] Feature/hybrid smart graph testsuite (#14056) --- arangod/Aql/AqlFunctionsInternalCache.h | 7 +- ...EngineInfoContainerDBServerServerBased.cpp | 12 +- .../EngineInfoContainerDBServerServerBased.h | 3 +- arangod/Aql/FixedVarExpressionContext.cpp | 8 +- arangod/Aql/FixedVarExpressionContext.h | 9 +- arangod/Aql/GraphNode.cpp | 31 +- arangod/Aql/GraphNode.h | 22 +- arangod/Aql/KShortestPathsNode.cpp | 43 +- arangod/Aql/QueryExpressionContext.cpp | 12 +- arangod/Aql/QueryExpressionContext.h | 10 +- arangod/Aql/ShardLocking.cpp | 3 +- arangod/CMakeLists.txt | 1 + arangod/Cluster/ClusterEdgeCursor.cpp | 26 +- arangod/Cluster/TraverserEngine.cpp | 6 +- arangod/Cluster/TraverserEngine.h | 4 + arangod/Graph/BaseOptions.cpp | 7 +- arangod/Graph/BaseOptions.h | 8 +- .../Graph/Cache/RefactoredTraverserCache.cpp | 139 +++-- .../Graph/Cache/RefactoredTraverserCache.h | 16 +- .../RefactoredSingleServerEdgeCursor.cpp | 158 +++++- .../RefactoredSingleServerEdgeCursor.h | 29 +- arangod/Graph/EdgeDocumentToken.h | 15 +- .../Graph/Enumerators/OneSidedEnumerator.cpp | 157 ++--- .../Graph/Enumerators/OneSidedEnumerator.h | 52 +- .../Enumerators/OneSidedEnumeratorInterface.h | 74 +++ .../Graph/Enumerators/TwoSidedEnumerator.cpp | 36 +- .../Graph/Enumerators/TwoSidedEnumerator.h | 12 +- arangod/Graph/Graph.cpp | 7 - arangod/Graph/Graph.h | 1 - arangod/Graph/GraphManager.cpp | 8 +- arangod/Graph/PathManagement/PathStore.cpp | 44 +- arangod/Graph/PathManagement/PathStore.h | 11 +- .../Graph/PathManagement/PathStoreTracer.cpp | 25 +- .../Graph/PathManagement/PathStoreTracer.h | 5 +- .../Graph/PathManagement/PathValidator.cpp | 159 +++++- arangod/Graph/PathManagement/PathValidator.h | 45 +- .../PathManagement/PathValidatorOptions.cpp | 79 +++ .../PathManagement/PathValidatorOptions.h | 88 +++ .../SingleProviderPathResult.cpp | 114 +++- .../PathManagement/SingleProviderPathResult.h | 26 +- .../Graph/Providers/BaseProviderOptions.cpp | 34 +- arangod/Graph/Providers/BaseProviderOptions.h | 30 +- arangod/Graph/Providers/BaseStep.h | 25 +- arangod/Graph/Providers/ClusterProvider.cpp | 54 +- arangod/Graph/Providers/ClusterProvider.h | 26 +- arangod/Graph/Providers/ProviderTracer.cpp | 6 +- arangod/Graph/Providers/ProviderTracer.h | 2 +- .../Graph/Providers/SingleServerProvider.cpp | 71 ++- .../Graph/Providers/SingleServerProvider.h | 38 +- arangod/Graph/SingleServerEdgeCursor.cpp | 3 +- arangod/Graph/TraverserCache.cpp | 98 ++-- arangod/Graph/Types/ValidationResult.cpp | 2 + arangod/Graph/Types/ValidationResult.h | 2 + arangod/Graph/algorithm-aliases.h | 65 ++- .../InternalRestTraverserHandler.cpp | 2 +- arangod/Transaction/Methods.h | 2 +- tests/CMakeLists.txt | 5 +- tests/Graph/DFSFinderTest.cpp | 270 ++++++++- tests/Graph/FifoQueueTest.cpp | 1 + tests/Graph/GenericGraphProviderTest.cpp | 56 +- tests/Graph/GraphMockProviderInstances.cpp | 67 +-- tests/Graph/GraphTestTools.h | 1 + tests/Graph/KPathFinderTest.cpp | 19 +- tests/Graph/LifoQueueTest.cpp | 1 + tests/Graph/PathValidatorTest.cpp | 534 ++++++++++-------- tests/Graph/SingleServerProviderTest.cpp | 75 ++- tests/Graph/TraverserCacheTest.cpp | 28 +- tests/{Graph => Mocks}/MockGraph.cpp | 0 tests/{Graph => Mocks}/MockGraph.h | 0 tests/{Graph => Mocks}/MockGraphProvider.cpp | 20 +- tests/{Graph => Mocks}/MockGraphProvider.h | 62 +- tests/Mocks/Servers.cpp | 73 +-- tests/Mocks/StorageEngineMock.cpp | 273 +++++---- tests/Transaction/ManagerSetup.h | 32 +- tests/js/server/aql/aql-graph-traverser.js | 47 +- 75 files changed, 2459 insertions(+), 1077 deletions(-) create mode 100644 arangod/Graph/Enumerators/OneSidedEnumeratorInterface.h create mode 100644 arangod/Graph/PathManagement/PathValidatorOptions.cpp create mode 100644 arangod/Graph/PathManagement/PathValidatorOptions.h rename tests/{Graph => Mocks}/MockGraph.cpp (100%) rename tests/{Graph => Mocks}/MockGraph.h (100%) rename tests/{Graph => Mocks}/MockGraphProvider.cpp (91%) rename tests/{Graph => Mocks}/MockGraphProvider.h (75%) diff --git a/arangod/Aql/AqlFunctionsInternalCache.h b/arangod/Aql/AqlFunctionsInternalCache.h index 6262d436fdd4..f89b7e7c51e0 100644 --- a/arangod/Aql/AqlFunctionsInternalCache.h +++ b/arangod/Aql/AqlFunctionsInternalCache.h @@ -25,13 +25,11 @@ #include "Aql/AqlValue.h" #include "Basics/Common.h" -#include +#include "VocBase/Validators.h" #include #include - - namespace arangodb { namespace transaction { @@ -49,6 +47,8 @@ class AqlFunctionsInternalCache final { AqlFunctionsInternalCache() = default; ~AqlFunctionsInternalCache(); + AqlFunctionsInternalCache(AqlFunctionsInternalCache&&) = default; + void clear() noexcept; icu::RegexMatcher* buildRegexMatcher(char const* ptr, size_t length, bool caseInsensitive); @@ -97,4 +97,3 @@ class AqlFunctionsInternalCache final { } // namespace aql } // namespace arangodb - diff --git a/arangod/Aql/EngineInfoContainerDBServerServerBased.cpp b/arangod/Aql/EngineInfoContainerDBServerServerBased.cpp index e23ee4bc3613..efe9cae7ee18 100644 --- a/arangod/Aql/EngineInfoContainerDBServerServerBased.cpp +++ b/arangod/Aql/EngineInfoContainerDBServerServerBased.cpp @@ -87,7 +87,8 @@ EngineInfoContainerDBServerServerBased::TraverserEngineShardLists::TraverserEngi } #endif _edgeCollections.emplace_back( - getAllLocalShards(shardMapping, server, col->shardIds(restrictToShards))); + getAllLocalShards(shardMapping, server, col->shardIds(restrictToShards), + col->isSatellite() && node->isSmart())); } // Extract vertices auto const& vertices = _node->vertexColls(); @@ -102,21 +103,26 @@ EngineInfoContainerDBServerServerBased::TraverserEngineShardLists::TraverserEngi _inaccessible.insert(std::to_string(col->id().id())); } #endif - auto shards = getAllLocalShards(shardMapping, server, col->shardIds(restrictToShards)); + auto shards = getAllLocalShards(shardMapping, server, col->shardIds(restrictToShards), + col->isSatellite() && node->isSmart()); _vertexCollections.try_emplace(col->name(), std::move(shards)); } } std::vector EngineInfoContainerDBServerServerBased::TraverserEngineShardLists::getAllLocalShards( std::unordered_map const& shardMapping, - ServerID const& server, std::shared_ptr> shardIds) { + ServerID const& server, std::shared_ptr> shardIds, bool colIsSatellite) { std::vector localShards; for (auto const& shard : *shardIds) { auto const& it = shardMapping.find(shard); TRI_ASSERT(it != shardMapping.end()); if (it->second == server) { localShards.emplace_back(shard); + // Guaranteed that the traversal will be executed on this server. _hasShard = true; + } else if (colIsSatellite) { + // The satellite does not force run of a traversal here. + localShards.emplace_back(shard); } } return localShards; diff --git a/arangod/Aql/EngineInfoContainerDBServerServerBased.h b/arangod/Aql/EngineInfoContainerDBServerServerBased.h index aced12378b49..f51384287c29 100644 --- a/arangod/Aql/EngineInfoContainerDBServerServerBased.h +++ b/arangod/Aql/EngineInfoContainerDBServerServerBased.h @@ -81,7 +81,8 @@ class EngineInfoContainerDBServerServerBased { private: std::vector getAllLocalShards(std::unordered_map const& shardMapping, ServerID const& server, - std::shared_ptr> shardIds); + std::shared_ptr> shardIds, + bool colIsSatellite); private: // The graph node we need to serialize diff --git a/arangod/Aql/FixedVarExpressionContext.cpp b/arangod/Aql/FixedVarExpressionContext.cpp index 5fb6eee13cdf..a7ec095da965 100644 --- a/arangod/Aql/FixedVarExpressionContext.cpp +++ b/arangod/Aql/FixedVarExpressionContext.cpp @@ -56,14 +56,18 @@ void FixedVarExpressionContext::setVariableValue(Variable const* var, AqlValue c _vars.try_emplace(var, value); } +void FixedVarExpressionContext::clearVariableValue(Variable const* var) { + _vars.erase(var); +} + void FixedVarExpressionContext::serializeAllVariables(velocypack::Options const& opts, velocypack::Builder& builder) const { TRI_ASSERT(builder.isOpenArray()); for (auto const& it : _vars) { builder.openArray(); it.first->toVelocyPack(builder); - it.second.toVelocyPack(&opts, builder, /*resolveExternals*/true, - /*allowUnindexed*/false); + it.second.toVelocyPack(&opts, builder, /*resolveExternals*/ true, + /*allowUnindexed*/ false); builder.close(); } } diff --git a/arangod/Aql/FixedVarExpressionContext.h b/arangod/Aql/FixedVarExpressionContext.h index f77fc2c89333..9960be5fc4a2 100644 --- a/arangod/Aql/FixedVarExpressionContext.h +++ b/arangod/Aql/FixedVarExpressionContext.h @@ -38,8 +38,7 @@ class AqlItemBlock; class FixedVarExpressionContext final : public QueryExpressionContext { public: - explicit FixedVarExpressionContext(transaction::Methods& trx, - QueryContext& query, + explicit FixedVarExpressionContext(transaction::Methods& trx, QueryContext& query, AqlFunctionsInternalCache& cache); ~FixedVarExpressionContext() override = default; @@ -51,8 +50,14 @@ class FixedVarExpressionContext final : public QueryExpressionContext { void clearVariableValues(); + // @brief This method will set the given variable to the given AQL value + // if the variable already holds a value, this method will keep the old value. void setVariableValue(Variable const*, AqlValue const&); + // @brief This method will only clear the given variable, and keep + // all others intact. If the variable does not exist, this is a noop. + void clearVariableValue(Variable const*); + void serializeAllVariables(velocypack::Options const& opts, arangodb::velocypack::Builder&) const; diff --git a/arangod/Aql/GraphNode.cpp b/arangod/Aql/GraphNode.cpp index be76af1636b8..57daaf107a1a 100644 --- a/arangod/Aql/GraphNode.cpp +++ b/arangod/Aql/GraphNode.cpp @@ -90,7 +90,7 @@ TRI_edge_direction_e parseDirection(AstNode const* node) { return uint64ToDirection(dirNode->getIntValue()); } -} +} // namespace GraphNode::GraphNode(ExecutionPlan* plan, ExecutionNodeId id, TRI_vocbase_t* vocbase, AstNode const* direction, @@ -107,7 +107,6 @@ GraphNode::GraphNode(ExecutionPlan* plan, ExecutionNodeId id, _optionsBuilt(false), _isSmart(false), _isDisjoint(false), - _isHybrid(false), _options(std::move(options)) { // Direction is already the correct Integer. // Is not inserted by user but by enum. @@ -301,9 +300,10 @@ GraphNode::GraphNode(ExecutionPlan* plan, arangodb::velocypack::Slice const& bas _defaultDirection(uint64ToDirection(arangodb::basics::VelocyPackHelper::stringUInt64( base.get("defaultDirection")))), _optionsBuilt(false), - _isSmart(arangodb::basics::VelocyPackHelper::getBooleanValue(base, "isSmart", false)), - _isDisjoint(arangodb::basics::VelocyPackHelper::getBooleanValue(base, "isDisjoint", false)) { - + _isSmart(arangodb::basics::VelocyPackHelper::getBooleanValue(base, StaticStrings::IsSmart, + false)), + _isDisjoint(arangodb::basics::VelocyPackHelper::getBooleanValue(base, StaticStrings::IsDisjoint, + false)) { if (!ServerState::instance()->isDBServer()) { // Graph Information. Do we need to reload the graph here? std::string graphName; @@ -357,7 +357,8 @@ GraphNode::GraphNode(ExecutionPlan* plan, arangodb::velocypack::Slice const& bas auto getAqlCollectionFromName = [&](std::string const& name) -> aql::Collection& { // if the collection was already existent in the query, addCollection will // just return it. - return *query.collections().add(name, AccessMode::Type::READ, aql::Collection::Hint::Collection); + return *query.collections().add(name, AccessMode::Type::READ, + aql::Collection::Hint::Collection); }; auto vPackDirListIter = VPackArrayIterator(dirList); @@ -397,7 +398,8 @@ GraphNode::GraphNode(ExecutionPlan* plan, arangodb::velocypack::Slice const& bas "graph needs a translation from collection to shard names"); } for (auto const& item : VPackObjectIterator(collectionToShard)) { - _collectionToShard.insert({item.key.copyString(), item.value.copyString()}); + _collectionToShard.insert({item.key.copyString(), + std::vector{item.value.copyString()}}); } // Out variables @@ -510,9 +512,11 @@ std::string const& GraphNode::collectionToShardName(std::string const& collName) auto found = _collectionToShard.find(collName); if (found == _collectionToShard.end()) { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL_AQL, "unable to find shard '" + collName + "' in query shard map"); + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL_AQL, + "unable to find shard '" + collName + + "' in query shard map"); } - return found->second; + return found->second.front(); } void GraphNode::toVelocyPackHelper(VPackBuilder& nodes, unsigned flags, @@ -581,7 +585,8 @@ void GraphNode::toVelocyPackHelper(VPackBuilder& nodes, unsigned flags, { VPackObjectBuilder guard(&nodes); for (auto const& item : _collectionToShard) { - nodes.add(item.first, VPackValue(item.second)); + TRI_ASSERT(item.second.size() == 1); + nodes.add(item.first, VPackValue(item.second.front())); } } @@ -714,8 +719,8 @@ void GraphNode::enhanceEngineInfo(VPackBuilder& builder) const { } #endif -void GraphNode::addEdgeCollection(Collections const& collections, std::string const& name, - TRI_edge_direction_e dir) { +void GraphNode::addEdgeCollection(Collections const& collections, + std::string const& name, TRI_edge_direction_e dir) { auto aqlCollection = collections.get(name); if (aqlCollection != nullptr) { addEdgeCollection(*aqlCollection, dir); @@ -811,8 +816,6 @@ bool GraphNode::isSmart() const { return _isSmart; } bool GraphNode::isDisjoint() const { return _isDisjoint; } -bool GraphNode::isHybrid() const { return _isHybrid; } - TRI_vocbase_t* GraphNode::vocbase() const { return _vocbase; } Variable const* GraphNode::vertexOutVariable() const { diff --git a/arangod/Aql/GraphNode.h b/arangod/Aql/GraphNode.h index 73934c70eb96..e1abbdce0334 100644 --- a/arangod/Aql/GraphNode.h +++ b/arangod/Aql/GraphNode.h @@ -23,12 +23,12 @@ #pragma once -#include "Aql/types.h" #include "Aql/Condition.h" #include "Aql/ExecutionNode.h" #include "Aql/ExecutionNodeId.h" #include "Aql/GraphNode.h" #include "Aql/Graphs.h" +#include "Aql/types.h" #include "Cluster/ClusterTypes.h" #include "VocBase/LogicalCollection.h" #include "VocBase/voc-types.h" @@ -81,10 +81,13 @@ class GraphNode : public ExecutionNode { GraphNode(ExecutionPlan* plan, arangodb::velocypack::Slice const& base); public: + // QueryPlan decided that we use this graph as a satellite bool isUsedAsSatellite() const; + // Defines whether a GraphNode can fully be pushed down to a DBServer bool isLocalGraphNode() const; + // Will wait as soon as any of our collections is a satellite (in sync) void waitForSatelliteIfRequired(ExecutionEngine const* engine) const; - + // Can be fully pushed down to a DBServer and is available on all DBServers bool isEligibleAsSatelliteTraversal() const; protected: @@ -122,9 +125,6 @@ class GraphNode : public ExecutionNode { /// @brief flag, if the graph is a Disjoint SmartGraph (Enterprise Edition only!) bool isDisjoint() const; - /// @brief flag, if the graph is a Hybrid SmartGraph (Enterprise Edition only!) - bool isHybrid() const; - /// @brief return the database TRI_vocbase_t* vocbase() const; @@ -192,20 +192,21 @@ class GraphNode : public ExecutionNode { void injectVertexCollection(aql::Collection& other); std::vector collections() const; - void setCollectionToShard(std::map const& map) { + void setCollectionToShard(std::unordered_map> const& map) { _collectionToShard = map; } void addCollectionToShard(std::string const& coll, std::string const& shard) { // NOTE: Do not replace this by emplace or insert. // This is also used to overwrite the existing entry. - _collectionToShard[coll] = shard; + _collectionToShard[coll] = std::vector{shard}; } public: graph::Graph const* graph() const noexcept; private: - void addEdgeCollection(aql::Collections const& collections, std::string const& name, TRI_edge_direction_e dir); + void addEdgeCollection(aql::Collections const& collections, + std::string const& name, TRI_edge_direction_e dir); void addEdgeCollection(aql::Collection& collection, TRI_edge_direction_e dir); void addVertexCollection(aql::Collections const& collections, std::string const& name); void addVertexCollection(aql::Collection& collection); @@ -258,9 +259,6 @@ class GraphNode : public ExecutionNode { /// @brief flag, if graph is smart *and* disjoint (Enterprise Edition only!) bool _isDisjoint; - /// @brief flag, if graph is smart *and* hybrid (Enterprise Edition only!) - bool _isHybrid; - /// @brief The directions edges are followed std::vector _directions; @@ -271,7 +269,7 @@ class GraphNode : public ExecutionNode { std::unordered_map _engines; /// @brief list of shards involved, required for one-shard-databases - std::map _collectionToShard; + std::unordered_map> _collectionToShard; }; } // namespace aql diff --git a/arangod/Aql/KShortestPathsNode.cpp b/arangod/Aql/KShortestPathsNode.cpp index f03cd7dd96e1..478d0d40daf3 100644 --- a/arangod/Aql/KShortestPathsNode.cpp +++ b/arangod/Aql/KShortestPathsNode.cpp @@ -342,14 +342,25 @@ std::unique_ptr KShortestPathsNode::createBlock( if (shortestPathType() == arangodb::graph::ShortestPathType::Type::KPaths) { arangodb::graph::TwoSidedEnumeratorOptions enumeratorOptions{opts->minDepth, opts->maxDepth}; + PathValidatorOptions validatorOptions(opts->tmpVar(), opts->getExpressionCtx()); if (!ServerState::instance()->isCoordinator()) { // Create IndexAccessor for BaseProviderOptions (TODO: Location need to // be changed in the future) create BaseProviderOptions - BaseProviderOptions forwardProviderOptions(opts->tmpVar(), buildUsedIndexes(), + + // TODO FIXME START - only tmp workaround - we'll provide an empty depth based index info (also add an alias for large type) + std::pair, std::unordered_map>> usedIndexes{}; + usedIndexes.first = buildUsedIndexes(); + + std::pair, std::unordered_map>> reversedUsedIndexes{}; + reversedUsedIndexes.first = buildReverseUsedIndexes(); + // TODO FIXME END + + BaseProviderOptions forwardProviderOptions(opts->tmpVar(), usedIndexes, + opts->getExpressionCtx(), opts->collectionToShard()); - BaseProviderOptions backwardProviderOptions(opts->tmpVar(), - buildReverseUsedIndexes(), + BaseProviderOptions backwardProviderOptions(opts->tmpVar(), reversedUsedIndexes, + opts->getExpressionCtx(), opts->collectionToShard()); if (opts->query().queryOptions().getTraversalProfileLevel() == @@ -361,7 +372,8 @@ std::unique_ptr KShortestPathsNode::createBlock( opts->query().resourceMonitor()}, SingleServerProvider{opts->query(), backwardProviderOptions, opts->query().resourceMonitor()}, - std::move(enumeratorOptions), opts->query().resourceMonitor()); + std::move(enumeratorOptions), std::move(validatorOptions), + opts->query().resourceMonitor()); auto executorInfos = KShortestPathsExecutorInfos(outputRegister, engine.getQuery(), @@ -377,7 +389,8 @@ std::unique_ptr KShortestPathsNode::createBlock( opts->query().resourceMonitor()}, ProviderTracer{opts->query(), backwardProviderOptions, opts->query().resourceMonitor()}, - std::move(enumeratorOptions), opts->query().resourceMonitor()); + std::move(enumeratorOptions), std::move(validatorOptions), + opts->query().resourceMonitor()); auto executorInfos = KShortestPathsExecutorInfos(outputRegister, engine.getQuery(), @@ -401,7 +414,8 @@ std::unique_ptr KShortestPathsNode::createBlock( opts->query().resourceMonitor()}, ClusterProvider{opts->query(), backwardProviderOptions, opts->query().resourceMonitor()}, - std::move(enumeratorOptions), opts->query().resourceMonitor()); + std::move(enumeratorOptions), std::move(validatorOptions), + opts->query().resourceMonitor()); auto executorInfos = KShortestPathsExecutorInfos(outputRegister, engine.getQuery(), @@ -417,7 +431,8 @@ std::unique_ptr KShortestPathsNode::createBlock( opts->query().resourceMonitor()}, ProviderTracer{opts->query(), backwardProviderOptions, opts->query().resourceMonitor()}, - std::move(enumeratorOptions), opts->query().resourceMonitor()); + std::move(enumeratorOptions), std::move(validatorOptions), + opts->query().resourceMonitor()); auto executorInfos = KShortestPathsExecutorInfos(outputRegister, engine.getQuery(), @@ -475,7 +490,7 @@ void KShortestPathsNode::kShortestPathsCloneHelper(ExecutionPlan& plan, c._fromCondition = _fromCondition->clone(_plan->getAst()); c._toCondition = _toCondition->clone(_plan->getAst()); } - + void KShortestPathsNode::replaceVariables(std::unordered_map const& replacements) { if (_inStartVariable != nullptr) { _inStartVariable = Variable::replace(_inStartVariable, replacements); @@ -524,7 +539,8 @@ std::vector KShortestPathsNode::buildUsedIndexes } indexAccessors.emplace_back(indexToUse, - _toCondition->clone(options()->query().ast()), 0); + _toCondition->clone(options()->query().ast()), + 0, nullptr); break; } case TRI_EDGE_OUT: { @@ -539,7 +555,8 @@ std::vector KShortestPathsNode::buildUsedIndexes } indexAccessors.emplace_back(indexToUse, - _fromCondition->clone(options()->query().ast()), 0); + _fromCondition->clone(options()->query().ast()), + 0, nullptr); break; } case TRI_EDGE_ANY: @@ -571,7 +588,8 @@ std::vector KShortestPathsNode::buildReverseUsed } indexAccessors.emplace_back(indexToUse, - _fromCondition->clone(options()->query().ast()), 0); + _fromCondition->clone(options()->query().ast()), + 0, nullptr); break; } case TRI_EDGE_OUT: { @@ -586,7 +604,8 @@ std::vector KShortestPathsNode::buildReverseUsed } indexAccessors.emplace_back(indexToUse, - _toCondition->clone(options()->query().ast()), 0); + _toCondition->clone(options()->query().ast()), + 0, nullptr); break; } case TRI_EDGE_ANY: diff --git a/arangod/Aql/QueryExpressionContext.cpp b/arangod/Aql/QueryExpressionContext.cpp index 7f03b0ee4e7a..29b362eb6331 100644 --- a/arangod/Aql/QueryExpressionContext.cpp +++ b/arangod/Aql/QueryExpressionContext.cpp @@ -23,9 +23,9 @@ #include "QueryExpressionContext.h" +#include "Aql/AqlFunctionsInternalCache.h" #include "Aql/AqlValue.h" #include "Aql/QueryContext.h" -#include "Aql/AqlFunctionsInternalCache.h" #include "Transaction/Methods.h" using namespace arangodb; @@ -63,10 +63,8 @@ TRI_vocbase_t& QueryExpressionContext::vocbase() const { return _trx.vocbase(); } -transaction::Methods& QueryExpressionContext::trx() const { - return _trx; -} +transaction::Methods& QueryExpressionContext::trx() const { return _trx; } -bool QueryExpressionContext::killed() const { - return _query.killed(); -} +bool QueryExpressionContext::killed() const { return _query.killed(); } + +QueryContext& QueryExpressionContext::query() { return _query; } diff --git a/arangod/Aql/QueryExpressionContext.h b/arangod/Aql/QueryExpressionContext.h index a9b214c549bb..d4b06e7d8f14 100644 --- a/arangod/Aql/QueryExpressionContext.h +++ b/arangod/Aql/QueryExpressionContext.h @@ -34,11 +34,9 @@ class AqlFunctionsInternalCache; class QueryExpressionContext : public ExpressionContext { public: - explicit QueryExpressionContext(transaction::Methods& trx, - QueryContext& query, + explicit QueryExpressionContext(transaction::Methods& trx, QueryContext& query, AqlFunctionsInternalCache& cache) noexcept - : ExpressionContext(), - _trx(trx), _query(query), _aqlFunctionsInternalCache(cache) {} + : ExpressionContext(), _trx(trx), _query(query), _aqlFunctionsInternalCache(cache) {} void registerWarning(ErrorCode errorCode, char const* msg) override final; void registerError(ErrorCode errorCode, char const* msg) override final; @@ -47,7 +45,8 @@ class QueryExpressionContext : public ExpressionContext { bool caseInsensitive) override final; icu::RegexMatcher* buildLikeMatcher(char const* ptr, size_t length, bool caseInsensitive) override final; - icu::RegexMatcher* buildSplitMatcher(AqlValue splitExpression, velocypack::Options const* opts, + icu::RegexMatcher* buildSplitMatcher(AqlValue splitExpression, + velocypack::Options const* opts, bool& isEmptyExpression) override final; arangodb::ValidatorBase* buildValidator(arangodb::velocypack::Slice const&) override final; @@ -56,6 +55,7 @@ class QueryExpressionContext : public ExpressionContext { /// may be inaccessible on some platforms transaction::Methods& trx() const override final; bool killed() const override final; + QueryContext& query(); private: transaction::Methods& _trx; diff --git a/arangod/Aql/ShardLocking.cpp b/arangod/Aql/ShardLocking.cpp index 81b83c98d147..db13a958695f 100644 --- a/arangod/Aql/ShardLocking.cpp +++ b/arangod/Aql/ShardLocking.cpp @@ -63,7 +63,7 @@ void ShardLocking::addNode(ExecutionNode const* baseNode, size_t snippetId, } auto const graphIsUsedAsSatellite = graphNode->isUsedAsSatellite(); auto const isUsedAsSatellite = [&](auto const& col) { - return graphIsUsedAsSatellite || (pushToSingleServer && col->isSatellite()); + return graphIsUsedAsSatellite || (pushToSingleServer && col->isSatellite()) || (col->isSatellite() && graphNode->isSmart()); }; // Add all Edge Collections to the Transactions, Traversals do never write for (auto const& col : graphNode->edgeColls()) { @@ -75,6 +75,7 @@ void ShardLocking::addNode(ExecutionNode const* baseNode, size_t snippetId, for (auto const& col : graphNode->vertexColls()) { updateLocking(col, AccessMode::Type::READ, snippetId, {}, isUsedAsSatellite(col)); } + break; } case ExecutionNode::ENUMERATE_COLLECTION: diff --git a/arangod/CMakeLists.txt b/arangod/CMakeLists.txt index 914cdb1f0713..7e2ff53586cd 100644 --- a/arangod/CMakeLists.txt +++ b/arangod/CMakeLists.txt @@ -198,6 +198,7 @@ set(LIB_ARANGO_GRAPH_SOURCES Graph/PathManagement/PathStore.cpp Graph/PathManagement/PathStoreTracer.cpp Graph/PathManagement/PathValidator.cpp + Graph/PathManagement/PathValidatorOptions.cpp Graph/Options/OneSidedEnumeratorOptions.cpp Graph/Options/TwoSidedEnumeratorOptions.cpp Graph/Providers/ClusterProvider.cpp diff --git a/arangod/Cluster/ClusterEdgeCursor.cpp b/arangod/Cluster/ClusterEdgeCursor.cpp index 4a34d7beaab7..a255cd1dfda6 100644 --- a/arangod/Cluster/ClusterEdgeCursor.cpp +++ b/arangod/Cluster/ClusterEdgeCursor.cpp @@ -38,7 +38,7 @@ using namespace arangodb; using namespace arangodb::graph; using namespace arangodb::traverser; -ClusterEdgeCursor::ClusterEdgeCursor(graph::BaseOptions const* opts) +ClusterEdgeCursor::ClusterEdgeCursor(graph::BaseOptions const* opts) : _position(0), _opts(opts), _cache(static_cast(opts->cache())), @@ -46,8 +46,8 @@ ClusterEdgeCursor::ClusterEdgeCursor(graph::BaseOptions const* opts) TRI_ASSERT(_cache != nullptr); if (_cache == nullptr) { - THROW_ARANGO_EXCEPTION_MESSAGE( - TRI_ERROR_INTERNAL, "no cache present for cluster edge cursor"); + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, + "no cache present for cluster edge cursor"); } } @@ -66,7 +66,7 @@ void ClusterEdgeCursor::readAll(EdgeCursor::Callback const& callback) { callback(EdgeDocumentToken(edge), edge, _position); } } - + ClusterTraverserEdgeCursor::ClusterTraverserEdgeCursor(traverser::TraverserOptions const* opts) : ClusterEdgeCursor(opts) {} @@ -75,7 +75,8 @@ traverser::TraverserOptions const* ClusterTraverserEdgeCursor::traverserOptions( return dynamic_cast(_opts); } -void ClusterTraverserEdgeCursor::rearm(arangodb::velocypack::StringRef vertexId, uint64_t depth) { +void ClusterTraverserEdgeCursor::rearm(arangodb::velocypack::StringRef vertexId, + uint64_t depth) { _edgeList.clear(); _position = 0; @@ -83,18 +84,20 @@ void ClusterTraverserEdgeCursor::rearm(arangodb::velocypack::StringRef vertexId, TRI_ASSERT(trx != nullptr); TRI_ASSERT(_cache != nullptr); - Result res = fetchEdgesFromEngines(*trx, *_cache, traverserOptions()->getExpressionCtx(), vertexId, depth, _edgeList); + Result res = fetchEdgesFromEngines(*trx, *_cache, traverserOptions()->getExpressionCtx(), + vertexId, depth, _edgeList); if (res.fail()) { THROW_ARANGO_EXCEPTION(res); } _httpRequests += _cache->engines()->size(); } -ClusterShortestPathEdgeCursor::ClusterShortestPathEdgeCursor(graph::BaseOptions const* opts, bool backward) - : ClusterEdgeCursor(opts), - _backward(backward) {} +ClusterShortestPathEdgeCursor::ClusterShortestPathEdgeCursor(graph::BaseOptions const* opts, + bool backward) + : ClusterEdgeCursor(opts), _backward(backward) {} -void ClusterShortestPathEdgeCursor::rearm(arangodb::velocypack::StringRef vertexId, uint64_t /*depth*/) { +void ClusterShortestPathEdgeCursor::rearm(arangodb::velocypack::StringRef vertexId, + uint64_t /*depth*/) { _edgeList.clear(); _position = 0; @@ -102,7 +105,8 @@ void ClusterShortestPathEdgeCursor::rearm(arangodb::velocypack::StringRef vertex transaction::BuilderLeaser b(trx); b->add(VPackValuePair(vertexId.data(), vertexId.length(), VPackValueType::String)); - Result res = fetchEdgesFromEngines(*trx, *_cache, b->slice(), _backward, _edgeList, _cache->insertedDocuments()); + Result res = fetchEdgesFromEngines(*trx, *_cache, b->slice(), _backward, + _edgeList, _cache->insertedDocuments()); if (res.fail()) { THROW_ARANGO_EXCEPTION(res); } diff --git a/arangod/Cluster/TraverserEngine.cpp b/arangod/Cluster/TraverserEngine.cpp index 30fcaebd70a8..71f370a4bd33 100644 --- a/arangod/Cluster/TraverserEngine.cpp +++ b/arangod/Cluster/TraverserEngine.cpp @@ -117,7 +117,7 @@ BaseEngine::BaseEngine(TRI_vocbase_t& vocbase, TRI_ASSERT(shardList.isArray()); for (VPackSlice const shard : VPackArrayIterator(shardList)) { TRI_ASSERT(shard.isString()); - _query.collections().add(shard.copyString(), AccessMode::Type::READ, aql::Collection::Hint::Collection); + _query.collections().add(shard.copyString(), AccessMode::Type::READ, aql::Collection::Hint::Shard); } } @@ -415,6 +415,10 @@ void TraverserEngine::smartSearch(VPackSlice, VPackBuilder&) { THROW_ARANGO_EXCEPTION(TRI_ERROR_ONLY_ENTERPRISE); } +void TraverserEngine::smartSearchNew(VPackSlice, VPackBuilder&) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_ONLY_ENTERPRISE); +} + void TraverserEngine::smartSearchBFS(VPackSlice, VPackBuilder&) { THROW_ARANGO_EXCEPTION(TRI_ERROR_ONLY_ENTERPRISE); } diff --git a/arangod/Cluster/TraverserEngine.h b/arangod/Cluster/TraverserEngine.h index f5b04b9fe752..2c084ceba92a 100644 --- a/arangod/Cluster/TraverserEngine.h +++ b/arangod/Cluster/TraverserEngine.h @@ -117,6 +117,8 @@ class BaseTraverserEngine : public BaseEngine { virtual void smartSearch(arangodb::velocypack::Slice, arangodb::velocypack::Builder&) = 0; + virtual void smartSearchNew(arangodb::velocypack::Slice, arangodb::velocypack::Builder&) = 0; + virtual void smartSearchBFS(arangodb::velocypack::Slice, arangodb::velocypack::Builder&) = 0; @@ -180,6 +182,8 @@ class TraverserEngine : public BaseTraverserEngine { void smartSearch(arangodb::velocypack::Slice, arangodb::velocypack::Builder&) override; + void smartSearchNew(arangodb::velocypack::Slice, arangodb::velocypack::Builder&) override; + void smartSearchBFS(arangodb::velocypack::Slice, arangodb::velocypack::Builder&) override; void smartSearchWeighted(arangodb::velocypack::Slice, arangodb::velocypack::Builder&) override; diff --git a/arangod/Graph/BaseOptions.cpp b/arangod/Graph/BaseOptions.cpp index e8d0a9330e99..d72cec2f3155 100644 --- a/arangod/Graph/BaseOptions.cpp +++ b/arangod/Graph/BaseOptions.cpp @@ -365,7 +365,8 @@ void BaseOptions::serializeVariables(VPackBuilder& builder) const { _expressionCtx.serializeAllVariables(_query.vpackOptions(), builder); } -void BaseOptions::setCollectionToShard(std::map const& in) { +void BaseOptions::setCollectionToShard( + std::unordered_map> const& in) { _collectionToShard = std::move(in); } @@ -462,6 +463,10 @@ void BaseOptions::injectTestCache(std::unique_ptr&& testCache) { _cache = std::move(testCache); } +arangodb::aql::FixedVarExpressionContext& BaseOptions::getExpressionCtx() { + return _expressionCtx; +} + arangodb::aql::FixedVarExpressionContext const& BaseOptions::getExpressionCtx() const { return _expressionCtx; } diff --git a/arangod/Graph/BaseOptions.h b/arangod/Graph/BaseOptions.h index 1ec81522948a..1150dbb40f2e 100644 --- a/arangod/Graph/BaseOptions.h +++ b/arangod/Graph/BaseOptions.h @@ -117,7 +117,7 @@ struct BaseOptions { void serializeVariables(arangodb::velocypack::Builder&) const; - void setCollectionToShard(std::map const&); + void setCollectionToShard(std::unordered_map> const&); bool produceVertices() const { return _produceVertices; } @@ -157,7 +157,7 @@ struct BaseOptions { void activateCache(bool enableDocumentCache, std::unordered_map const* engines); - std::map const& collectionToShard() const { + std::unordered_map> const& collectionToShard() const { return _collectionToShard; } @@ -178,6 +178,8 @@ struct BaseOptions { bool refactor() const { return _refactor; } aql::Variable const* tmpVar(); // TODO check public + arangodb::aql::FixedVarExpressionContext& getExpressionCtx(); + arangodb::aql::FixedVarExpressionContext const& getExpressionCtx() const; protected: @@ -215,7 +217,7 @@ struct BaseOptions { std::unique_ptr _cache; // @brief - translations for one-shard-databases - std::map _collectionToShard; + std::unordered_map> _collectionToShard; /// @brief a value of 1 (which is the default) means "no parallelism" size_t _parallelism; diff --git a/arangod/Graph/Cache/RefactoredTraverserCache.cpp b/arangod/Graph/Cache/RefactoredTraverserCache.cpp index d8eb1f3ee436..0a47ad38d89a 100644 --- a/arangod/Graph/Cache/RefactoredTraverserCache.cpp +++ b/arangod/Graph/Cache/RefactoredTraverserCache.cpp @@ -53,8 +53,7 @@ using namespace arangodb::graph; namespace { bool isWithClauseMissing(arangodb::basics::Exception const& ex) { - if (ServerState::instance()->isDBServer() && - ex.code() == TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND) { + if (ServerState::instance()->isDBServer() && ex.code() == TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND) { // on a DB server, we could have got here only in the OneShard case. // in this case turn the rather misleading "collection or view not found" // error into a nicer "collection not known to traversal, please add WITH" @@ -68,19 +67,20 @@ bool isWithClauseMissing(arangodb::basics::Exception const& ex) { return false; } -} // namespace +} // namespace RefactoredTraverserCache::RefactoredTraverserCache( arangodb::transaction::Methods* trx, aql::QueryContext* query, arangodb::ResourceMonitor& resourceMonitor, arangodb::aql::TraversalStats& stats, - std::map const& collectionToShardMap) + std::unordered_map> const& collectionToShardMap) : _query(query), _trx(trx), _stringHeap(resourceMonitor, 4096), /* arbitrary block-size may be adjusted for performance */ _collectionToShardMap(collectionToShardMap), _resourceMonitor(resourceMonitor), - _allowImplicitCollections(ServerState::instance()->isSingleServer() && - !_query->vocbase().server().getFeature().requireWith()) { + _allowImplicitCollections( + ServerState::instance()->isSingleServer() && + !_query->vocbase().server().getFeature().requireWith()) { TRI_ASSERT(!ServerState::instance()->isCoordinator()); } @@ -106,17 +106,20 @@ bool RefactoredTraverserCache::appendEdge(EdgeDocumentToken const& idToken, return false; } - auto res = col->getPhysical()->read( - _trx, idToken.localDocumentId(), [&](LocalDocumentId const&, VPackSlice edge) -> bool { - // NOTE: Do not count this as Primary Index Scan, we counted it in the - // edge Index before copying... - if constexpr (std::is_same_v) { - result = aql::AqlValue(edge); - } else if constexpr (std::is_same_v) { - result.add(edge); - } - return true; - }).ok(); + auto res = + col->getPhysical() + ->read(_trx, idToken.localDocumentId(), + [&](LocalDocumentId const&, VPackSlice edge) -> bool { + // NOTE: Do not count this as Primary Index Scan, we counted + // it in the edge Index before copying... + if constexpr (std::is_same_v) { + result = aql::AqlValue(edge); + } else if constexpr (std::is_same_v) { + result.add(edge); + } + return true; + }) + .ok(); if (ADB_UNLIKELY(!res)) { // We already had this token, inconsistent state. Return NULL in Production LOG_TOPIC("daac5", ERR, arangodb::Logger::GRAPHS) @@ -139,20 +142,7 @@ ResultT> RefactoredTraverserCache::extractCollect } std::string colName = idHashed.substr(0, pos).toString(); - if (_collectionToShardMap.empty()) { - TRI_ASSERT(!ServerState::instance()->isDBServer()); - return std::make_pair(colName, pos); - } - auto it = _collectionToShardMap.find(colName); - if (it == _collectionToShardMap.end()) { - // Connected to a vertex where we do not know the Shard to. - return Result{TRI_ERROR_QUERY_COLLECTION_LOCK_FAILED, - "collection not known to traversal: '" + colName + - "'. please add 'WITH " + colName + - "' as the first line in your AQL"}; - } - // We have translated to a Shard - return std::make_pair(it->second, pos); + return std::make_pair(colName, pos); } template @@ -163,43 +153,68 @@ bool RefactoredTraverserCache::appendVertex(aql::TraversalStats& stats, if (collectionNameResult.fail()) { THROW_ARANGO_EXCEPTION(collectionNameResult.result()); } - - std::string const& collectionName = collectionNameResult.get().first; - try { - transaction::AllowImplicitCollectionsSwitcher disallower(_trx->state()->options(), _allowImplicitCollections); - - Result res = _trx->documentFastPathLocal( - collectionName, - id.substr(collectionNameResult.get().second + 1).stringRef(), - [&](LocalDocumentId const&, VPackSlice doc) -> bool { - stats.addScannedIndex(1); - // copying... - if constexpr (std::is_same_v) { - result = aql::AqlValue(doc); - } else if constexpr (std::is_same_v) { - result.add(doc); - } - return true; - }); - if (res.ok()) { - return true; + auto findDocumentInShard = [&](std::string const& collectionName) -> bool { + try { + transaction::AllowImplicitCollectionsSwitcher disallower(_trx->state()->options(), + _allowImplicitCollections); + + Result res = _trx->documentFastPathLocal( + collectionName, id.substr(collectionNameResult.get().second + 1).stringRef(), + [&](LocalDocumentId const&, VPackSlice doc) -> bool { + stats.addScannedIndex(1); + // copying... + if constexpr (std::is_same_v) { + result = aql::AqlValue(doc); + } else if constexpr (std::is_same_v) { + result.add(doc); + } + return true; + }); + if (res.ok()) { + return true; + } + + if (!res.is(TRI_ERROR_ARANGO_DOCUMENT_NOT_FOUND)) { + // ok we are in a rather bad state. Better throw and abort. + THROW_ARANGO_EXCEPTION(res); + } + } catch (basics::Exception const& ex) { + if (isWithClauseMissing(ex)) { + // turn the error into a different error + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_COLLECTION_LOCK_FAILED, + "collection not known to traversal: '" + collectionName + + "'. please add 'WITH " + collectionName + + "' as the first line in your AQL"); + } + // rethrow original error + throw; } + return false; + }; - if (!res.is(TRI_ERROR_ARANGO_DOCUMENT_NOT_FOUND)) { - // ok we are in a rather bad state. Better throw and abort. - THROW_ARANGO_EXCEPTION(res); + std::string const& collectionName = collectionNameResult.get().first; + if (_collectionToShardMap.empty()) { + TRI_ASSERT(!ServerState::instance()->isDBServer()); + if (findDocumentInShard(collectionName)) { + return true; } - } catch (basics::Exception const& ex) { - if (isWithClauseMissing(ex)) { - // turn the error into a different error + } else { + auto it = _collectionToShardMap.find(collectionName); + if (it == _collectionToShardMap.end()) { + // Connected to a vertex where we do not know the Shard to. THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_COLLECTION_LOCK_FAILED, - "collection not known to traversal: '" + - collectionName + "'. please add 'WITH " + collectionName + - "' as the first line in your AQL"); + "collection not known to traversal: '" + collectionName + + "'. please add 'WITH " + collectionName + + "' as the first line in your AQL"); + } + for (auto const& shard : it->second) { + if (findDocumentInShard(shard)) { + // Short circuit, as soon as one shard contains this document + // we can return it. + return true; + } } - // rethrow original error - throw; } // Register a warning. It is okay though but helps the user diff --git a/arangod/Graph/Cache/RefactoredTraverserCache.h b/arangod/Graph/Cache/RefactoredTraverserCache.h index 495fad9509da..ca4a9f688a2d 100644 --- a/arangod/Graph/Cache/RefactoredTraverserCache.h +++ b/arangod/Graph/Cache/RefactoredTraverserCache.h @@ -62,11 +62,10 @@ struct EdgeDocumentToken; class RefactoredTraverserCache { public: - explicit RefactoredTraverserCache(arangodb::transaction::Methods* trx, - aql::QueryContext* query, - arangodb::ResourceMonitor& resourceMonitor, - arangodb::aql::TraversalStats& stats, - std::map const& collectionToShardMap); + explicit RefactoredTraverserCache( + arangodb::transaction::Methods* trx, aql::QueryContext* query, + arangodb::ResourceMonitor& resourceMonitor, arangodb::aql::TraversalStats& stats, + std::unordered_map> const& collectionToShardMap); ~RefactoredTraverserCache(); RefactoredTraverserCache(RefactoredTraverserCache const&) = delete; @@ -81,8 +80,7 @@ class RefactoredTraverserCache { /// @brief Inserts the real document stored within the token /// into the given builder. ////////////////////////////////////////////////////////////////////////////// - void insertEdgeIntoResult(graph::EdgeDocumentToken const& etkn, - velocypack::Builder& builder); + void insertEdgeIntoResult(graph::EdgeDocumentToken const& etkn, velocypack::Builder& builder); ////////////////////////////////////////////////////////////////////////////// /// @brief Inserts the real document identified by the _id string @@ -148,9 +146,9 @@ class RefactoredTraverserCache { std::unordered_set _persistedStrings; private: - std::map const& _collectionToShardMap; + std::unordered_map> const& _collectionToShardMap; arangodb::ResourceMonitor& _resourceMonitor; - + /// @brief whether or not to allow adding of previously unknown collections /// during the traversal bool const _allowImplicitCollections; diff --git a/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.cpp b/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.cpp index 31015a005ef9..930cd63ee9a5 100644 --- a/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.cpp +++ b/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.cpp @@ -42,10 +42,11 @@ namespace { IndexIteratorOptions defaultIndexIteratorOptions; } -RefactoredSingleServerEdgeCursor::LookupInfo::LookupInfo(transaction::Methods::IndexHandle idx, - aql::AstNode* condition, - std::optional memberToUpdate) +RefactoredSingleServerEdgeCursor::LookupInfo::LookupInfo( + transaction::Methods::IndexHandle idx, aql::AstNode* condition, + std::optional memberToUpdate, aql::Expression* expression) : _idxHandle(std::move(idx)), + _expression(expression), _indexCondition(condition), _cursor(nullptr), _conditionMemberToUpdate(memberToUpdate) {} @@ -58,6 +59,10 @@ RefactoredSingleServerEdgeCursor::LookupInfo::LookupInfo(LookupInfo&& other) noe RefactoredSingleServerEdgeCursor::LookupInfo::~LookupInfo() = default; +aql::Expression* RefactoredSingleServerEdgeCursor::LookupInfo::getExpression() { + return _expression; +} + void RefactoredSingleServerEdgeCursor::LookupInfo::rearmVertex( VertexType vertex, transaction::Methods* trx, arangodb::aql::Variable const* tmpVar) { auto& node = _indexCondition; @@ -74,6 +79,25 @@ void RefactoredSingleServerEdgeCursor::LookupInfo::rearmVertex( TRI_ASSERT(idNode->isValueType(aql::VALUE_TYPE_STRING)); // must edit node in place; TODO replace node? // TODO i think there is now a mutable String node available + TEMPORARILY_UNLOCK_NODE(idNode); + idNode->setStringValue(vertex.data(), vertex.length()); + } else { + // If we have to inject the vertex value it has to be within + // the last member of the condition. + // We only get into this case iff the index used does + // not cover _from resp. _to. + // inject _from/_to value + auto expressionNode = _expression->nodeForModification(); + + TRI_ASSERT(expressionNode->numMembers() > 0); + auto dirCmp = expressionNode->getMemberUnchecked(expressionNode->numMembers() - 1); + TRI_ASSERT(dirCmp->type == aql::NODE_TYPE_OPERATOR_BINARY_EQ); + TRI_ASSERT(dirCmp->numMembers() == 2); + + auto idNode = dirCmp->getMemberUnchecked(1); + TRI_ASSERT(idNode->type == aql::NODE_TYPE_VALUE); + TRI_ASSERT(idNode->isValueType(aql::VALUE_TYPE_STRING)); + TEMPORARILY_UNLOCK_NODE(idNode); idNode->setStringValue(vertex.data(), vertex.length()); } @@ -100,15 +124,28 @@ IndexIterator& RefactoredSingleServerEdgeCursor::LookupInfo::cursor() { } RefactoredSingleServerEdgeCursor::RefactoredSingleServerEdgeCursor( - arangodb::transaction::Methods* trx, - arangodb::aql::Variable const* tmpVar, std::vector const& indexConditions) - : _tmpVar(tmpVar), _currentCursor(0), _trx(trx) { + transaction::Methods* trx, arangodb::aql::Variable const* tmpVar, + std::vector const& globalIndexConditions, + std::unordered_map> const& depthBasedIndexConditions, + arangodb::aql::FixedVarExpressionContext& expressionContext) + : _tmpVar(tmpVar), _currentCursor(0), _trx(trx), _expressionCtx(expressionContext) { // We need at least one indexCondition, otherwise nothing to serve - TRI_ASSERT(!indexConditions.empty()); - _lookupInfo.reserve(indexConditions.size()); - for (auto const& idxCond : indexConditions) { + TRI_ASSERT(!globalIndexConditions.empty()); + _lookupInfo.reserve(globalIndexConditions.size()); + _depthLookupInfo.reserve(depthBasedIndexConditions.size()); + for (auto const& idxCond : globalIndexConditions) { _lookupInfo.emplace_back(idxCond.indexHandle(), idxCond.getCondition(), - idxCond.getMemberToUpdate()); + idxCond.getMemberToUpdate(), idxCond.getExpression()); + } + for (auto const& obj : depthBasedIndexConditions) { + auto& [depth, idxCondArray] = obj; + + std::vector tmpLookupVec; + for (auto const& idxCond : idxCondArray) { + tmpLookupVec.emplace_back(idxCond.indexHandle(), idxCond.getCondition(), + idxCond.getMemberToUpdate(), idxCond.getExpression()); + } + _depthLookupInfo.try_emplace(depth, std::move(tmpLookupVec)); } } @@ -133,9 +170,22 @@ void RefactoredSingleServerEdgeCursor::rearm(VertexType vertex, uint64_t /*depth } } -void RefactoredSingleServerEdgeCursor::readAll(aql::TraversalStats& stats, +void RefactoredSingleServerEdgeCursor::readAll(SingleServerProvider& provider, + aql::TraversalStats& stats, size_t depth, Callback const& callback) { TRI_ASSERT(!_lookupInfo.empty()); + VPackBuilder tmpBuilder; + + auto handleExpression = [&](aql::Expression* expression, + EdgeDocumentToken edgeToken, VPackSlice edge) { + if (edge.isString()) { + tmpBuilder.clear(); + provider.insertEdgeIntoResult(edgeToken, tmpBuilder); + edge = tmpBuilder.slice(); + } + return evaluateExpression(expression, edge); + }; + for (_currentCursor = 0; _currentCursor < _lookupInfo.size(); ++_currentCursor) { auto& cursor = _lookupInfo[_currentCursor].cursor(); LogicalCollection* collection = cursor.collection(); @@ -148,31 +198,85 @@ void RefactoredSingleServerEdgeCursor::readAll(aql::TraversalStats& stats, return false; } #endif - callback(EdgeDocumentToken(cid, token), edge, _currentCursor); + + EdgeDocumentToken edgeToken(cid, token); + // eval depth-based expression first if available + bool foundDepthInfo = + (_depthLookupInfo.find(depth) == _depthLookupInfo.end()) ? false : true; + if (foundDepthInfo && _depthLookupInfo.at(depth).size() > 0 && + _depthLookupInfo.at(depth)[_currentCursor].getExpression() != nullptr) { + if (!handleExpression(_depthLookupInfo.at(depth)[_currentCursor].getExpression(), + edgeToken, edge)) { + stats.incrFiltered(); + return false; + } + } else { + // eval global expression if available AND only if no depth specific expression needs to be handled before + if (_lookupInfo[_currentCursor].getExpression() != nullptr) { + if (!handleExpression(_lookupInfo[_currentCursor].getExpression(), + edgeToken, edge)) { + stats.incrFiltered(); + return false; + } + } + } + + callback(std::move(edgeToken), edge, _currentCursor); return true; }); } else { cursor.all([&](LocalDocumentId const& token) { - return collection->getPhysical()->read(_trx, token, [&](LocalDocumentId const&, VPackSlice edgeDoc) { - stats.addScannedIndex(1); + return collection->getPhysical() + ->read(_trx, token, + [&](LocalDocumentId const&, VPackSlice edgeDoc) { + stats.addScannedIndex(1); #ifdef USE_ENTERPRISE - if (_trx->skipInaccessible()) { - // TODO: we only need to check one of these - VPackSlice from = transaction::helpers::extractFromFromDocument(edgeDoc); - VPackSlice to = transaction::helpers::extractToFromDocument(edgeDoc); - if (CheckInaccessible(_trx, from) || CheckInaccessible(_trx, to)) { - return false; - } - } + if (_trx->skipInaccessible()) { + // TODO: we only need to check one of these + VPackSlice from = + transaction::helpers::extractFromFromDocument(edgeDoc); + VPackSlice to = + transaction::helpers::extractToFromDocument(edgeDoc); + if (CheckInaccessible(_trx, from) || CheckInaccessible(_trx, to)) { + return false; + } + } #endif - callback(EdgeDocumentToken(cid, token), edgeDoc, _currentCursor); - return true; - }).ok(); + // eval expression if available + if (_lookupInfo[_currentCursor].getExpression() != nullptr) { + bool result = + evaluateExpression(_lookupInfo[_currentCursor].getExpression(), + edgeDoc); + if (!result) { + stats.incrFiltered(); + return false; + } + } + callback(EdgeDocumentToken(cid, token), edgeDoc, _currentCursor); + return true; + }) + .ok(); }); } } } -arangodb::transaction::Methods* RefactoredSingleServerEdgeCursor::trx() const { - return _trx; +bool RefactoredSingleServerEdgeCursor::evaluateExpression(arangodb::aql::Expression* expression, + VPackSlice value) { + if (expression == nullptr) { + return true; + } + + TRI_ASSERT(value.isObject() || value.isNull()); + + aql::AqlValue edgeVal(aql::AqlValueHintDocumentNoCopy(value.begin())); + _expressionCtx.setVariableValue(_tmpVar, edgeVal); + TRI_DEFER(_expressionCtx.clearVariableValue(_tmpVar)); + + bool mustDestroy = false; + aql::AqlValue res = expression->execute(&_expressionCtx, mustDestroy); + aql::AqlValueGuard guard(res, mustDestroy); + TRI_ASSERT(res.isBoolean()); + + return res.toBoolean(); } diff --git a/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.h b/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.h index 6335bee3a6c6..249d6c0d317e 100644 --- a/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.h +++ b/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.h @@ -24,7 +24,9 @@ #pragma once +#include "Aql/AqlFunctionsInternalCache.h" #include "Aql/Expression.h" +#include "Aql/FixedVarExpressionContext.h" #include "Aql/QueryContext.h" #include "Indexes/IndexIterator.h" #include "Transaction/Methods.h" @@ -51,11 +53,13 @@ struct IndexAccessor; struct EdgeDocumentToken; +struct SingleServerProvider; + class RefactoredSingleServerEdgeCursor { public: struct LookupInfo { LookupInfo(transaction::Methods::IndexHandle idx, aql::AstNode* condition, - std::optional memberToUpdate); + std::optional memberToUpdate, aql::Expression* expression); ~LookupInfo(); LookupInfo(LookupInfo const&) = delete; @@ -66,12 +70,12 @@ class RefactoredSingleServerEdgeCursor { arangodb::aql::Variable const* tmpVar); IndexIterator& cursor(); + aql::Expression* getExpression(); private: - // This struct does only take responsibility for the expression // NOTE: The expression can be nullptr! transaction::Methods::IndexHandle _idxHandle; - std::unique_ptr _expression; + aql::Expression* _expression; aql::AstNode* _indexCondition; std::unique_ptr _cursor; @@ -83,9 +87,11 @@ class RefactoredSingleServerEdgeCursor { enum Direction { FORWARD, BACKWARD }; public: - RefactoredSingleServerEdgeCursor(arangodb::transaction::Methods* trx, - arangodb::aql::Variable const* tmpVar, - std::vector const& indexConditions); + RefactoredSingleServerEdgeCursor( + transaction::Methods* trx, arangodb::aql::Variable const* tmpVar, + std::vector const& globalIndexConditions, + std::unordered_map> const& depthBasedIndexConditions, + arangodb::aql::FixedVarExpressionContext& expressionContext); ~RefactoredSingleServerEdgeCursor(); using Callback = @@ -95,17 +101,18 @@ class RefactoredSingleServerEdgeCursor { aql::Variable const* _tmpVar; size_t _currentCursor; std::vector _lookupInfo; + std::unordered_map> _depthLookupInfo; - arangodb::transaction::Methods* _trx; + transaction::Methods* _trx; + arangodb::aql::FixedVarExpressionContext& _expressionCtx; public: - void readAll(aql::TraversalStats& stats, Callback const& callback); + void readAll(SingleServerProvider& provider, aql::TraversalStats& stats, + size_t depth, Callback const& callback); void rearm(VertexType vertex, uint64_t depth); - private: - [[nodiscard]] transaction::Methods* trx() const; + bool evaluateExpression(arangodb::aql::Expression* expression, VPackSlice value); }; } // namespace graph } // namespace arangodb - diff --git a/arangod/Graph/EdgeDocumentToken.h b/arangod/Graph/EdgeDocumentToken.h index abeff524e397..8935cfdffc4c 100644 --- a/arangod/Graph/EdgeDocumentToken.h +++ b/arangod/Graph/EdgeDocumentToken.h @@ -118,12 +118,15 @@ struct EdgeDocumentToken { #ifdef ARANGODB_ENABLE_MAINTAINER_MODE TRI_ASSERT(_type == TokenType::COORDINATOR); #endif - return velocypack::Slice(_data.vpack).binaryEquals(velocypack::Slice(other._data.vpack)); + return velocypack::Slice(_data.vpack) + .binaryEquals(velocypack::Slice(other._data.vpack)); } bool equalsLocal(EdgeDocumentToken const& other) const { #ifdef ARANGODB_ENABLE_MAINTAINER_MODE - TRI_ASSERT(_type == TokenType::LOCAL); + // For local the cid and localDocumentId have illegal values on NONE + // and can be compared with real values + TRI_ASSERT(_type == TokenType::LOCAL || _type == TokenType::NONE); #endif return _data.document.cid == other.cid() && _data.document.localDocumentId == other.localDocumentId(); @@ -136,6 +139,14 @@ struct EdgeDocumentToken { return equalsLocal(other); } + bool isValid() const { + if (ServerState::instance()->isCoordinator()) { + return _data.vpack != nullptr; + } + return _data.document.cid != DataSourceId::none() && + _data.document.localDocumentId != LocalDocumentId::none(); + } + size_t hash() const { if (ServerState::instance()->isCoordinator()) { auto vslice = arangodb::velocypack::Slice(vpack()); diff --git a/arangod/Graph/Enumerators/OneSidedEnumerator.cpp b/arangod/Graph/Enumerators/OneSidedEnumerator.cpp index 7febe88af686..ff6e67361f86 100644 --- a/arangod/Graph/Enumerators/OneSidedEnumerator.cpp +++ b/arangod/Graph/Enumerators/OneSidedEnumerator.cpp @@ -36,7 +36,9 @@ #include "Graph/Providers/ProviderTracer.h" #include "Graph/Providers/SingleServerProvider.h" #include "Graph/Queues/FifoQueue.h" +#include "Graph/Queues/LifoQueue.h" #include "Graph/Queues/QueueTracer.h" +#include "Graph/algorithm-aliases.h" #include #include @@ -46,37 +48,34 @@ using namespace arangodb; using namespace arangodb::graph; -template -OneSidedEnumerator::OneSidedEnumerator( - ProviderType&& forwardProvider, OneSidedEnumeratorOptions&& options, - arangodb::ResourceMonitor& resourceMonitor) +template +OneSidedEnumerator::OneSidedEnumerator(Provider&& forwardProvider, + OneSidedEnumeratorOptions&& options, + PathValidatorOptions validatorOptions, + arangodb::ResourceMonitor& resourceMonitor) : _options(std::move(options)), _queue(resourceMonitor), _provider(std::move(forwardProvider)), - _validator(_interior), - _interior(resourceMonitor), - _resultPath{_provider} {} + _validator(_provider, _interior, std::move(validatorOptions)), + _interior(resourceMonitor) {} -template -OneSidedEnumerator::~OneSidedEnumerator() { -} +template +OneSidedEnumerator::~OneSidedEnumerator() = default; -template -auto OneSidedEnumerator::destroyEngines() - -> void { +template +auto OneSidedEnumerator::destroyEngines() -> void { _provider.destroyEngines(); } -template -void OneSidedEnumerator::clear() { +template +void OneSidedEnumerator::clear() { _interior.reset(); _queue.clear(); _results.clear(); } -template -auto OneSidedEnumerator::computeNeighbourhoodOfNextVertex() - -> void { +template +auto OneSidedEnumerator::computeNeighbourhoodOfNextVertex() -> void { // Pull next element from Queue // Do 1 step search TRI_ASSERT(!_queue.isEmpty()); @@ -91,13 +90,31 @@ auto OneSidedEnumerator:: TRI_ASSERT(_queue.hasProcessableElement()); } - auto step = _queue.pop(); - auto posPrevious = _interior.append(step); - + auto tmp = _queue.pop(); + auto posPrevious = _interior.append(std::move(tmp)); + auto& step = _interior.getStepReference(posPrevious); + + // only explore here if we're responsible + if (!step.isResponsible(_provider.trx())) { + // This server cannot decide on this specific vertex. + // Include it in results, to report back that we + // found this undecided path + _results.push_back(&step); + return; + } ValidationResult res = _validator.validatePath(step); - if ((step.getDepth() >= _options.getMinDepth()) && !res.isFiltered()) { + + // TODO: Adjust log output + LOG_TOPIC("78155", TRACE, Logger::GRAPHS) + << std::boolalpha << " Validated Vertex: " << step.getVertex().getID() + << " filtered " << res.isFiltered() << " pruned " << res.isPruned() + << " depth " << _options.getMinDepth() << " <= " << step.getDepth() + << "<= " << _options.getMaxDepth(); + if (step.getDepth() >= _options.getMinDepth() && !res.isFiltered()) { // Include it in results. - _results.push_back(step); + _results.push_back(&step); + } else { + _stats.incrFiltered(); } if (step.getDepth() < _options.getMaxDepth() && !res.isPruned()) { @@ -111,8 +128,8 @@ auto OneSidedEnumerator:: * @return true There will be no further path. * @return false There is a chance that there is more data available. */ -template -bool OneSidedEnumerator::isDone() const { +template +bool OneSidedEnumerator::isDone() const { return _results.empty() && searchDone(); } @@ -125,10 +142,10 @@ bool OneSidedEnumerator:: * * @param source The source vertex to start the paths */ -template -void OneSidedEnumerator::reset(VertexRef source) { +template +void OneSidedEnumerator::reset(VertexRef source, size_t depth) { clear(); - auto firstStep = _provider.startVertex(source); + auto firstStep = _provider.startVertex(source, depth); _queue.append(std::move(firstStep)); } @@ -145,32 +162,23 @@ void OneSidedEnumerator:: * @return true Found and written a path, result is modified. * @return false No path found, result has not been changed. */ -template -bool OneSidedEnumerator::getNextPath(VPackBuilder& result) { +template +auto OneSidedEnumerator::getNextPath() + -> std::unique_ptr { while (!isDone()) { searchMoreResults(); while (!_results.empty()) { - auto const& vertex = _results.back(); - - // Performance Optimization: - // It seems to be pointless to first push - // everything in to the _resultPath object - // and then iterate again to return the path - // we should be able to return the path in the first go. - _resultPath.clear(); - _interior.buildPath(vertex, _resultPath); - TRI_ASSERT(!_resultPath.isEmpty()); + auto* step = _results.back(); _results.pop_back(); - _resultPath.toVelocyPack(result); - return true; + return std::make_unique(step, _provider, _interior); } } - return false; + return nullptr; } -template -void OneSidedEnumerator::searchMoreResults() { +template +void OneSidedEnumerator::searchMoreResults() { while (_results.empty() && !searchDone()) { // TODO: check && !_queue.isEmpty() _resultsFetched = false; computeNeighbourhoodOfNextVertex(); @@ -186,8 +194,8 @@ void OneSidedEnumerator:: * @return false No path found. */ -template -bool OneSidedEnumerator::skipPath() { +template +bool OneSidedEnumerator::skipPath() { while (!isDone()) { searchMoreResults(); @@ -200,21 +208,19 @@ bool OneSidedEnumerator:: return false; } -template -auto OneSidedEnumerator::searchDone() const - -> bool { +template +auto OneSidedEnumerator::searchDone() const -> bool { return _queue.isEmpty(); } -template -auto OneSidedEnumerator::fetchResults() - -> void { +template +auto OneSidedEnumerator::fetchResults() -> void { if (!_resultsFetched && !_results.empty()) { std::vector looseEnds{}; - for (auto& vertex : _results) { - if (!vertex.isProcessable()) { - looseEnds.emplace_back(&vertex); + for (auto* vertex : _results) { + if (!vertex->isProcessable()) { + looseEnds.emplace_back(vertex); } } @@ -235,22 +241,31 @@ auto OneSidedEnumerator:: _resultsFetched = true; } -template -auto OneSidedEnumerator::stealStats() - -> aql::TraversalStats { - aql::TraversalStats stats = _provider.stealStats(); - return stats; +template +auto OneSidedEnumerator::stealStats() -> aql::TraversalStats { + _stats += _provider.stealStats(); + + auto t = _stats; + // Placement new of stats, do not reallocate space. + _stats.~TraversalStats(); + new (&_stats) aql::TraversalStats{}; + return t; } /* SingleServerProvider Section */ -template class ::arangodb::graph::OneSidedEnumerator< - ::arangodb::graph::FifoQueue<::arangodb::graph::SingleServerProvider::Step>, - ::arangodb::graph::PathStore, SingleServerProvider, - ::arangodb::graph::PathValidator, VertexUniquenessLevel::PATH>>; - -template class ::arangodb::graph::OneSidedEnumerator< - ::arangodb::graph::QueueTracer<::arangodb::graph::FifoQueue<::arangodb::graph::SingleServerProvider::Step>>, - ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore>, - ::arangodb::graph::ProviderTracer, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore>, VertexUniquenessLevel::PATH>>; +// Breadth First Search +template class ::arangodb::graph::OneSidedEnumerator>; +template class ::arangodb::graph::OneSidedEnumerator>; +template class ::arangodb::graph::OneSidedEnumerator>; +template class ::arangodb::graph::OneSidedEnumerator>; +template class ::arangodb::graph::OneSidedEnumerator>; +template class ::arangodb::graph::OneSidedEnumerator>; + +// Depth First Search +template class ::arangodb::graph::OneSidedEnumerator>; +template class ::arangodb::graph::OneSidedEnumerator>; +template class ::arangodb::graph::OneSidedEnumerator>; +template class ::arangodb::graph::OneSidedEnumerator>; +template class ::arangodb::graph::OneSidedEnumerator>; +template class ::arangodb::graph::OneSidedEnumerator>; diff --git a/arangod/Graph/Enumerators/OneSidedEnumerator.h b/arangod/Graph/Enumerators/OneSidedEnumerator.h index 465dea72c907..8e821fca2a1a 100644 --- a/arangod/Graph/Enumerators/OneSidedEnumerator.h +++ b/arangod/Graph/Enumerators/OneSidedEnumerator.h @@ -28,6 +28,8 @@ #include "Basics/ResourceUsage.h" +#include "Aql/TraversalStats.h" +#include "Graph/Enumerators/OneSidedEnumeratorInterface.h" #include "Graph/Options/OneSidedEnumeratorOptions.h" #include "Graph/PathManagement/SingleProviderPathResult.h" #include "Transaction/Methods.h" @@ -36,10 +38,6 @@ namespace arangodb { -namespace aql { -class TraversalStats; -} - namespace velocypack { class Builder; class HashedStringRef; @@ -48,31 +46,33 @@ class HashedStringRef; namespace graph { struct OneSidedEnumeratorOptions; +class PathValidatorOptions; -template -class PathResult; - -template -class OneSidedEnumerator { +template +class OneSidedEnumerator : public TraversalEnumerator { public: - using Step = typename ProviderType::Step; // public due to tracer access + using Step = typename Configuration::Step; // public due to tracer access + using Provider = typename Configuration::Provider; + using Store = typename Configuration::Store; + + using ResultPathType = SingleProviderPathResult; private: using VertexRef = arangodb::velocypack::HashedStringRef; - using Shell = std::multiset; - using ResultList = std::vector; + using ResultList = std::vector; using GraphOptions = arangodb::graph::OneSidedEnumeratorOptions; public: - OneSidedEnumerator(ProviderType&& forwardProvider, OneSidedEnumeratorOptions&& options, + OneSidedEnumerator(Provider&& provider, OneSidedEnumeratorOptions&& options, + PathValidatorOptions pathValidatorOptions, arangodb::ResourceMonitor& resourceMonitor); OneSidedEnumerator(OneSidedEnumerator const& other) = delete; OneSidedEnumerator(OneSidedEnumerator&& other) noexcept = default; ~OneSidedEnumerator(); - void clear(); + void clear() override; /** * @brief Quick test if the finder can prove there is no more data available. @@ -80,7 +80,7 @@ class OneSidedEnumerator { * @return true There will be no further path. * @return false There is a chance that there is more data available. */ - [[nodiscard]] bool isDone() const; + [[nodiscard]] bool isDone() const override; /** * @brief Reset to new source and target vertices. @@ -90,8 +90,9 @@ class OneSidedEnumerator { * call of reset. * * @param source The source vertex to start the paths + * @param depth The depth we're starting the search at */ - void reset(VertexRef source); + void reset(VertexRef source, size_t depth = 0) override; /** * @brief Get the next path, if available written into the result build. @@ -106,7 +107,7 @@ class OneSidedEnumerator { * @return true Found and written a path, result is modified. * @return false No path found, result has not been changed. */ - bool getNextPath(arangodb::velocypack::Builder& result); + auto getNextPath() -> std::unique_ptr override; /** * @brief Skip the next Path, like getNextPath, but does not return the path. @@ -115,14 +116,14 @@ class OneSidedEnumerator { * @return false No path found. */ - bool skipPath(); - auto destroyEngines() -> void; + bool skipPath() override; + auto destroyEngines() -> void override; /** * @brief Return statistics generated since * the last time this method was called. */ - auto stealStats() -> aql::TraversalStats; + auto stealStats() -> aql::TraversalStats override; private: [[nodiscard]] auto searchDone() const -> bool; @@ -143,16 +144,15 @@ class OneSidedEnumerator { GraphOptions _options; ResultList _results{}; bool _resultsFetched{false}; + aql::TraversalStats _stats{}; // The next elements to process - QueueType _queue; - ProviderType _provider; - PathValidatorType _validator; + typename Configuration::Queue _queue; + typename Configuration::Provider _provider; + typename Configuration::Validator _validator; // This stores all paths processed - PathStoreType _interior; - - SingleProviderPathResult _resultPath; + typename Configuration::Store _interior; }; } // namespace graph } // namespace arangodb diff --git a/arangod/Graph/Enumerators/OneSidedEnumeratorInterface.h b/arangod/Graph/Enumerators/OneSidedEnumeratorInterface.h new file mode 100644 index 000000000000..7cd217066d62 --- /dev/null +++ b/arangod/Graph/Enumerators/OneSidedEnumeratorInterface.h @@ -0,0 +1,74 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2021 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +/// @author Heiko Kernbach +//////////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include +#include +#include + +namespace arangodb { + +namespace velocypack { +class HashedStringRef; +class Builder; +} // namespace velocypack + +namespace aql { +class TraversalStats; +} + +namespace graph { + +// TODO Temporary, just Make everything compile +class PathResultInterface { + public: + PathResultInterface() {} + virtual ~PathResultInterface() {} + + virtual auto toVelocyPack(arangodb::velocypack::Builder& builder) -> void = 0; + virtual auto toSchreierEntry(arangodb::velocypack::Builder& builder, + size_t& currentLength) + -> void = 0; +}; + +class TraversalEnumerator { + public: + using VertexRef = arangodb::velocypack::HashedStringRef; + TraversalEnumerator(){}; + virtual ~TraversalEnumerator() {} + + virtual void clear() = 0; + [[nodiscard]] virtual bool isDone() const = 0; + + virtual void reset(VertexRef source, size_t depth = 0) = 0; + virtual auto getNextPath() -> std::unique_ptr = 0; + virtual bool skipPath() = 0; + virtual auto destroyEngines() -> void = 0; + + virtual auto stealStats() -> aql::TraversalStats = 0; +}; + +} // namespace graph +} // namespace arangodb diff --git a/arangod/Graph/Enumerators/TwoSidedEnumerator.cpp b/arangod/Graph/Enumerators/TwoSidedEnumerator.cpp index 51ea0ed1a842..10df092a905d 100644 --- a/arangod/Graph/Enumerators/TwoSidedEnumerator.cpp +++ b/arangod/Graph/Enumerators/TwoSidedEnumerator.cpp @@ -54,12 +54,12 @@ using namespace arangodb::graph; template TwoSidedEnumerator::Ball::Ball( Direction dir, ProviderType&& provider, GraphOptions const& options, - arangodb::ResourceMonitor& resourceMonitor) + PathValidatorOptions validatorOptions, arangodb::ResourceMonitor& resourceMonitor) : _resourceMonitor(resourceMonitor), _interior(resourceMonitor), _queue(resourceMonitor), _provider(std::move(provider)), - _validator(_interior), + _validator(_provider, _interior, std::move(validatorOptions)), _direction(dir), _minDepth(options.getMinDepth()) {} @@ -67,9 +67,10 @@ template ::Ball::~Ball() = default; template -void TwoSidedEnumerator::Ball::reset(VertexRef center) { +void TwoSidedEnumerator::Ball::reset( + VertexRef center, size_t depth) { clear(); - auto firstStep = _provider.startVertex(center); + auto firstStep = _provider.startVertex(center, depth); _shell.emplace(std::move(firstStep)); } @@ -144,9 +145,9 @@ auto TwoSidedEnumerator:: // Notes for the future: // Vertices are now fetched. Thnink about other less-blocking and batch-wise // fetching (e.g. re-fetch at some later point). - // TODO: Discuss how to optimize here. Currently we'll mark looseEnds in fetch as fetched. - // This works, but we might create a batch limit here in the future. - // Also discuss: Do we want (re-)fetch logic here? + // TODO: Discuss how to optimize here. Currently we'll mark looseEnds in + // fetch as fetched. This works, but we might create a batch limit here in + // the future. Also discuss: Do we want (re-)fetch logic here? // TODO: maybe we can combine this with prefetching of paths } } @@ -245,10 +246,13 @@ auto TwoSidedEnumerator:: template TwoSidedEnumerator::TwoSidedEnumerator( ProviderType&& forwardProvider, ProviderType&& backwardProvider, - TwoSidedEnumeratorOptions&& options, arangodb::ResourceMonitor& resourceMonitor) + TwoSidedEnumeratorOptions&& options, PathValidatorOptions validatorOptions, + arangodb::ResourceMonitor& resourceMonitor) : _options(std::move(options)), - _left{Direction::FORWARD, std::move(forwardProvider), _options, resourceMonitor}, - _right{Direction::BACKWARD, std::move(backwardProvider), _options, resourceMonitor}, + _left{Direction::FORWARD, std::move(forwardProvider), _options, + validatorOptions, resourceMonitor}, + _right{Direction::BACKWARD, std::move(backwardProvider), _options, + std::move(validatorOptions), resourceMonitor}, _resultPath{_left.provider(), _right.provider()} {} template @@ -293,7 +297,7 @@ bool TwoSidedEnumerator:: */ template void TwoSidedEnumerator::reset( - VertexRef source, VertexRef target) { + VertexRef source, VertexRef target, size_t depth) { _results.clear(); _left.reset(source); _right.reset(target); @@ -428,23 +432,25 @@ auto TwoSidedEnumerator:: template class ::arangodb::graph::TwoSidedEnumerator< ::arangodb::graph::FifoQueue<::arangodb::graph::SingleServerProvider::Step>, ::arangodb::graph::PathStore, SingleServerProvider, - ::arangodb::graph::PathValidator, VertexUniquenessLevel::PATH>>; + ::arangodb::graph::PathValidator, VertexUniquenessLevel::PATH>>; template class ::arangodb::graph::TwoSidedEnumerator< ::arangodb::graph::QueueTracer<::arangodb::graph::FifoQueue<::arangodb::graph::SingleServerProvider::Step>>, ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore>, ::arangodb::graph::ProviderTracer, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore>, VertexUniquenessLevel::PATH>>; + ::arangodb::graph::PathValidator<::arangodb::graph::ProviderTracer, + ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore>, VertexUniquenessLevel::PATH>>; /* ClusterProvider Section */ template class ::arangodb::graph::TwoSidedEnumerator< ::arangodb::graph::FifoQueue<::arangodb::graph::ClusterProvider::Step>, ::arangodb::graph::PathStore, ClusterProvider, - ::arangodb::graph::PathValidator, VertexUniquenessLevel::PATH>>; + ::arangodb::graph::PathValidator, VertexUniquenessLevel::PATH>>; template class ::arangodb::graph::TwoSidedEnumerator< ::arangodb::graph::QueueTracer<::arangodb::graph::FifoQueue<::arangodb::graph::ClusterProvider::Step>>, ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore>, ::arangodb::graph::ProviderTracer, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore>, VertexUniquenessLevel::PATH>>; + ::arangodb::graph::PathValidator<::arangodb::graph::ProviderTracer, + ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore>, VertexUniquenessLevel::PATH>>; diff --git a/arangod/Graph/Enumerators/TwoSidedEnumerator.h b/arangod/Graph/Enumerators/TwoSidedEnumerator.h index 2dddc3d83792..141afd35e95d 100644 --- a/arangod/Graph/Enumerators/TwoSidedEnumerator.h +++ b/arangod/Graph/Enumerators/TwoSidedEnumerator.h @@ -47,6 +47,7 @@ class HashedStringRef; namespace graph { +class PathValidatorOptions; struct TwoSidedEnumeratorOptions; template @@ -69,10 +70,10 @@ class TwoSidedEnumerator { class Ball { public: Ball(Direction dir, ProviderType&& provider, GraphOptions const& options, - arangodb::ResourceMonitor& resourceMonitor); + PathValidatorOptions validatorOptions, arangodb::ResourceMonitor& resourceMonitor); ~Ball(); auto clear() -> void; - auto reset(VertexRef center) -> void; + auto reset(VertexRef center, size_t depth = 0) -> void; auto startNextDepth() -> void; [[nodiscard]] auto noPathLeft() const -> bool; [[nodiscard]] auto getDepth() const -> size_t; @@ -82,7 +83,8 @@ class TwoSidedEnumerator { auto buildPath(Step const& vertexInShell, PathResult& path) -> void; - auto matchResultsInShell(Step const& match, ResultList& results, PathValidatorType const& otherSideValidator) -> void; + auto matchResultsInShell(Step const& match, ResultList& results, + PathValidatorType const& otherSideValidator) -> void; auto computeNeighbourhoodOfNextVertex(Ball& other, ResultList& results) -> void; // Ensure that we have fetched all vertices @@ -118,7 +120,7 @@ class TwoSidedEnumerator { public: TwoSidedEnumerator(ProviderType&& forwardProvider, ProviderType&& backwardProvider, - TwoSidedEnumeratorOptions&& options, + TwoSidedEnumeratorOptions&& options, PathValidatorOptions validatorOptions, arangodb::ResourceMonitor& resourceMonitor); TwoSidedEnumerator(TwoSidedEnumerator const& other) = delete; TwoSidedEnumerator(TwoSidedEnumerator&& other) noexcept = default; @@ -145,7 +147,7 @@ class TwoSidedEnumerator { * @param source The source vertex to start the paths * @param target The target vertex to end the paths */ - void reset(VertexRef source, VertexRef target); + void reset(VertexRef source, VertexRef target, size_t depth = 0); /** * @brief Get the next path, if available written into the result build. diff --git a/arangod/Graph/Graph.cpp b/arangod/Graph/Graph.cpp index 0ab3270d36c9..60aab621acd1 100644 --- a/arangod/Graph/Graph.cpp +++ b/arangod/Graph/Graph.cpp @@ -451,7 +451,6 @@ void EdgeDefinition::toVelocyPack(VPackBuilder& builder) const { builder.add(VPackValue(to)); } builder.close(); // array - builder.add("type", VPackValue(getType())); } ResultT EdgeDefinition::createFromVelocypack(VPackSlice edgeDefinition, std::set const& satCollections) { @@ -572,7 +571,6 @@ void EdgeDefinition::addToBuilder(VPackBuilder& builder) const { builder.add(VPackValue(to)); } builder.close(); // to - builder.add(StaticStrings::GraphEdgeDefinitionType, VPackValue(getType())); builder.close(); // obj } @@ -775,11 +773,6 @@ bool Graph::isDisjoint() const { return false; } -bool Graph::isHybrid() const { - TRI_ASSERT(_satelliteColls.empty()); - return false; -} - bool Graph::isSatellite() const { return _isSatellite; } void Graph::createCollectionOptions(VPackBuilder& builder, bool waitForSync) const { diff --git a/arangod/Graph/Graph.h b/arangod/Graph/Graph.h index 7991934cb612..9d424ead1082 100644 --- a/arangod/Graph/Graph.h +++ b/arangod/Graph/Graph.h @@ -207,7 +207,6 @@ class Graph { virtual bool isSmart() const; virtual bool isDisjoint() const; virtual bool isSatellite() const; - virtual bool isHybrid() const; virtual bool needsToBeSatellite(std::string const& edge) const; uint64_t numberOfShards() const; diff --git a/arangod/Graph/GraphManager.cpp b/arangod/Graph/GraphManager.cpp index ea8e8260f3ae..80a49c90cdba 100644 --- a/arangod/Graph/GraphManager.cpp +++ b/arangod/Graph/GraphManager.cpp @@ -534,7 +534,7 @@ Result GraphManager::ensureCollections(Graph* graph, bool waitForSync) const { return res; } else { // not found the collection, need to create it later - if (graph->isHybrid() && graph->needsToBeSatellite(edgeColl)) { // check for satellites + if (graph->needsToBeSatellite(edgeColl)) { // check for satellites satelliteEdgeCollectionsToCreate.emplace(edgeColl); } else { edgeCollectionsToCreate.emplace(edgeColl); @@ -555,7 +555,9 @@ Result GraphManager::ensureCollections(Graph* graph, bool waitForSync) const { return res; } else { if (edgeCollectionsToCreate.find(vertexColl) == edgeCollectionsToCreate.end()) { - if (graph->isHybrid() && graph->satelliteCollections().find(vertexColl) != graph->satelliteCollections().end()) { + if (!graph->satelliteCollections().empty() && + graph->satelliteCollections().find(vertexColl) != + graph->satelliteCollections().end()) { satelliteDocumentCollectionsToCreate.emplace(vertexColl); } else { documentCollectionsToCreate.emplace(vertexColl); @@ -622,7 +624,7 @@ Result GraphManager::ensureCollections(Graph* graph, bool waitForSync) const { // new builder VPackBuilder satOptionsBuilder; - if (graph->isHybrid()) { + if (!satelliteDocumentCollectionsToCreate.empty() || !satelliteEdgeCollectionsToCreate.empty()) { satOptionsBuilder.openObject(); graph->createSatelliteCollectionOptions(satOptionsBuilder, waitForSync); satOptionsBuilder.close(); diff --git a/arangod/Graph/PathManagement/PathStore.cpp b/arangod/Graph/PathManagement/PathStore.cpp index b49c8d69b13d..e6402d410764 100644 --- a/arangod/Graph/PathManagement/PathStore.cpp +++ b/arangod/Graph/PathManagement/PathStore.cpp @@ -23,7 +23,6 @@ #include "PathStore.h" #include "Graph/PathManagement/PathResult.h" -#include "Graph/PathManagement/SingleProviderPathResult.h" #include "Graph/Providers/ClusterProvider.h" #include "Graph/Providers/ProviderTracer.h" @@ -78,7 +77,7 @@ size_t PathStore::append(Step step) { } template -Step PathStore::get(size_t position) const { +Step PathStore::getStep(size_t position) const { TRI_ASSERT(position <= size()); Step step = _schreier.at(position); LOG_TOPIC("45bf5", TRACE, Logger::GRAPHS) @@ -87,6 +86,16 @@ Step PathStore::get(size_t position) const { return step; } +template +Step& PathStore::getStepReference(size_t position) { + TRI_ASSERT(position <= size()); + auto& step = _schreier.at(position); + LOG_TOPIC("45bf6", TRACE, Logger::GRAPHS) + << " Get step: " << step.toString(); + + return step; +} + template template auto PathStore::buildPath(Step const& vertex, PathResultType& path) const -> void { @@ -159,6 +168,26 @@ auto PathStore::visitReversePath(Step const& step, } } +template +auto PathStore::modifyReversePath(Step& step, + std::function const& visitor) +-> bool { + Step * walker = &step; + // Guaranteed to make progress, as the schreier vector contains a loop-free tree. + while (true) { + bool cont = visitor(*walker); + if (!cont) { + // Aborted + return false; + } + if (walker->isFirst()) { + // Visited the full path + return true; + } + walker = &_schreier.at(walker->getPrevious()); + } +} + /* SingleServerProvider Section */ template class PathStore; @@ -167,11 +196,6 @@ template void PathStore::buildPath& path) const; -template void PathStore::buildPath< - SingleProviderPathResult>( - SingleServerProvider::Step const& vertex, - SingleProviderPathResult& path) const; - template void PathStore::reverseBuildPath( SingleServerProvider::Step const& vertex, PathResult& path) const; @@ -183,12 +207,6 @@ template void PathStore::buildPath< ProviderTracer::Step const& vertex, PathResult, ProviderTracer::Step>& path) const; -template void PathStore::buildPath< - SingleProviderPathResult, ProviderTracer::Step>>( - ProviderTracer::Step const& vertex, - SingleProviderPathResult, - ProviderTracer::Step>& path) const; - template void PathStore::Step>::reverseBuildPath>( ProviderTracer::Step const& vertex, PathResult, ProviderTracer::Step>& path) const; diff --git a/arangod/Graph/PathManagement/PathStore.h b/arangod/Graph/PathManagement/PathStore.h index 8ec934790c33..a24eb1195f15 100644 --- a/arangod/Graph/PathManagement/PathStore.h +++ b/arangod/Graph/PathManagement/PathStore.h @@ -71,7 +71,10 @@ class PathStore { size_t append(Step step); // @briefs Method returns a step at given position - Step get(size_t position) const; + Step getStep(size_t position) const; + + // @briefs Method returns a step reference at given position + Step& getStepReference(size_t position); // @brief returns the current vector size size_t size() const { return _schreier.size(); } @@ -79,11 +82,14 @@ class PathStore { auto visitReversePath(Step const& step, std::function const& visitor) const -> bool; + auto modifyReversePath(Step& step, std::function const& visitor) -> bool; + template auto buildPath(Step const& vertex, PathResultType& path) const -> void; template - auto reverseBuildPath(Step const& vertex, PathResult& path) const -> void; + auto reverseBuildPath(Step const& vertex, PathResult& path) const + -> void; private: /// @brief schreier vector to store the visited vertices @@ -94,4 +100,3 @@ class PathStore { } // namespace graph } // namespace arangodb - diff --git a/arangod/Graph/PathManagement/PathStoreTracer.cpp b/arangod/Graph/PathManagement/PathStoreTracer.cpp index f32478e01554..280bc7331e68 100644 --- a/arangod/Graph/PathManagement/PathStoreTracer.cpp +++ b/arangod/Graph/PathManagement/PathStoreTracer.cpp @@ -64,10 +64,17 @@ size_t PathStoreTracer::append(Step step) { } template -typename PathStoreImpl::Step PathStoreTracer::get(size_t position) const { +typename PathStoreImpl::Step PathStoreTracer::getStep(size_t position) const { double start = TRI_microtime(); - TRI_DEFER(_stats["get"].addTiming(TRI_microtime() - start)); - return _impl.get(position); + TRI_DEFER(_stats["getStep"].addTiming(TRI_microtime() - start)); + return _impl.getStep(position); +} + +template +typename PathStoreImpl::Step& PathStoreTracer::getStepReference(size_t position) { + double start = TRI_microtime(); + TRI_DEFER(_stats["getStepReference"].addTiming(TRI_microtime() - start)); + return _impl.getStepReference(position); } template @@ -104,6 +111,13 @@ auto PathStoreTracer::visitReversePath( return _impl.visitReversePath(step, visitor); } +template +auto PathStoreTracer::modifyReversePath(Step& step, const std::function& visitor) -> bool { + double start = TRI_microtime(); + TRI_DEFER(_stats["modifyReversePath"].addTiming(TRI_microtime() - start)); + return _impl.modifyReversePath(step, visitor); +} + /* SingleServerProvider Section */ template class ::arangodb::graph::PathStoreTracer>; @@ -114,11 +128,6 @@ template void ::arangodb::graph::PathStoreTracer::Step const& vertex, PathResult, ProviderTracer::Step>& path) const; -template void ::arangodb::graph::PathStoreTracer>::buildPath< - SingleProviderPathResult, ProviderTracer::Step>>( - ProviderTracer::Step const& vertex, - SingleProviderPathResult, ProviderTracer::Step>& path) const; - template void arangodb::graph::PathStoreTracer>::reverseBuildPath>( ProviderTracer::Step const& vertex, PathResult, ProviderTracer::Step>& path) const; diff --git a/arangod/Graph/PathManagement/PathStoreTracer.h b/arangod/Graph/PathManagement/PathStoreTracer.h index 288bd019f92a..a1895924b64a 100644 --- a/arangod/Graph/PathManagement/PathStoreTracer.h +++ b/arangod/Graph/PathManagement/PathStoreTracer.h @@ -60,7 +60,8 @@ class PathStoreTracer { // returns the index of inserted element size_t append(Step step); - auto get(size_t position) const -> Step; + auto getStep(size_t position) const -> Step; + auto getStepReference(size_t position) -> Step&; // @brief returns the current vector size size_t size() const; @@ -75,6 +76,8 @@ class PathStoreTracer { auto visitReversePath(Step const& step, std::function const& visitor) const -> bool; + auto modifyReversePath(Step& step, std::function const& visitor) -> bool; + private: PathStoreImpl _impl; diff --git a/arangod/Graph/PathManagement/PathValidator.cpp b/arangod/Graph/PathManagement/PathValidator.cpp index 28f0a8d4d3a2..9c9d7f7a90be 100644 --- a/arangod/Graph/PathManagement/PathValidator.cpp +++ b/arangod/Graph/PathManagement/PathValidator.cpp @@ -22,9 +22,12 @@ //////////////////////////////////////////////////////////////////////////////// #include "PathValidator.h" +#include "Aql/AstNode.h" +#include "Aql/PruneExpressionEvaluator.h" #include "Graph/PathManagement/PathStore.h" #include "Graph/PathManagement/PathStoreTracer.h" #include "Graph/Providers/ClusterProvider.h" +#include "Graph/Providers/ProviderTracer.h" #include "Graph/Providers/SingleServerProvider.h" #include "Graph/Types/ValidationResult.h" @@ -33,13 +36,26 @@ using namespace arangodb; using namespace arangodb::graph; -template -PathValidator::PathValidator(PathStore const& store) - : _store(store) {} +template +PathValidator::PathValidator( + ProviderType& provider, PathStore const& store, PathValidatorOptions opts) + : _store(store), _provider(provider), _options(std::move(opts)) {} + +template +PathValidator::~PathValidator() = default; + +template +auto PathValidator::validatePath( + typename PathStore::Step const& step) -> ValidationResult { + auto ctx = _options.getExpressionContext(); + // Reset variables + ctx.clearVariableValues(); + auto res = evaluateVertexCondition(step); + if (res.isFiltered() && res.isPruned()) { + // Can give up here. This Value is not used + return res; + } -template -auto PathValidator::validatePath(typename PathStore::Step const& step) - -> ValidationResult { if constexpr (vertexUniqueness == VertexUniquenessLevel::PATH) { _uniqueVertices.clear(); // Reserving here is pointless, we will test paths that increase by at most 1 entry. @@ -52,25 +68,24 @@ auto PathValidator::validatePath(typename PathStore return added; }); if (!success) { - return ValidationResult{ValidationResult::Type::FILTER}; + res.combine(ValidationResult::Type::FILTER); } - return ValidationResult{ValidationResult::Type::TAKE}; } if constexpr (vertexUniqueness == VertexUniquenessLevel::GLOBAL) { auto const& [unused, added] = _uniqueVertices.emplace(step.getVertexIdentifier()); // If this add fails, we need to exclude this path if (!added) { - return ValidationResult{ValidationResult::Type::FILTER}; + res.combine(ValidationResult::Type::FILTER); } - return ValidationResult{ValidationResult::Type::TAKE}; } - return ValidationResult{ValidationResult::Type::TAKE}; + return res; } -template -auto PathValidator::validatePath( +template +auto PathValidator::validatePath( typename PathStore::Step const& step, - PathValidator const& otherValidator) -> ValidationResult { + PathValidator const& otherValidator) + -> ValidationResult { if constexpr (vertexUniqueness == VertexUniquenessLevel::PATH) { // For PATH: take _uniqueVertices of otherValidator, and run Visitor of other side, check if one vertex is duplicate. auto const& otherUniqueVertices = otherValidator.exposeUniqueVertices(); @@ -106,22 +121,126 @@ auto PathValidator::validatePath( return ValidationResult{ValidationResult::Type::TAKE}; } -template -auto PathValidator::exposeUniqueVertices() const +template +auto PathValidator::exposeUniqueVertices() const -> ::arangodb::containers::HashSet, std::equal_to> const& { return _uniqueVertices; } +template +auto PathValidator::evaluateVertexRestriction( + typename PathStore::Step const& step) -> bool { + if (step.isFirst()) { + // first step => always allowed + return true; + } + + auto allowedCollections = _options.getAllowedVertexCollections(); + if (allowedCollections.empty()) { + // all allowed + return true; + } + + auto collectionName = step.getCollectionName(); + if (std::find(allowedCollections.begin(), allowedCollections.end(), + collectionName) != allowedCollections.end()) { + // found in allowed collections => allowed + return true; + } + + // not allowed + return false; +} + +template +auto PathValidator::evaluateVertexCondition( + typename PathStore::Step const& step) -> ValidationResult { + // evaluate if vertex collection is allowed + bool isAllowed = evaluateVertexRestriction(step); + if (!isAllowed) { + return ValidationResult{ValidationResult::Type::FILTER}; + } + + auto expr = _options.getVertexExpression(step.getDepth()); + if (expr != nullptr) { + _tmpObjectBuilder.clear(); + _provider.addVertexToBuilder(step.getVertex(), _tmpObjectBuilder); + + // evaluate expression + bool satifiesCondition = evaluateExpression(expr, _tmpObjectBuilder.slice()); + if (!satifiesCondition) { + return ValidationResult{ValidationResult::Type::FILTER}; + } + } + return ValidationResult{ValidationResult::Type::TAKE}; +} + +template +auto PathValidator::evaluateExpression( + arangodb::aql::Expression* expression, VPackSlice value) -> bool { + if (expression == nullptr) { + return true; + } + + TRI_ASSERT(value.isObject() || value.isNull()); + auto tmpVar = _options.getTempVar(); + bool mustDestroy = false; + auto ctx = _options.getExpressionContext(); + aql::AqlValue tmpVal{value}; + ctx.setVariableValue(tmpVar, tmpVal); + aql::AqlValue res = expression->execute(&ctx, mustDestroy); + aql::AqlValueGuard guard{res, mustDestroy}; + TRI_ASSERT(res.isBoolean()); + return res.toBoolean(); +} + +template +void PathValidator::setPruneEvaluator( + std::unique_ptr eval) { + _pruneEvaluator = std::move(eval); +} + +template +void PathValidator::setPostFilterEvaluator( + std::unique_ptr eval) { + _postFilterEvaluator = std::move(eval); +} + +template +void PathValidator::reset() { + if constexpr (vertexUniqueness != VertexUniquenessLevel::NONE) { + _uniqueVertices.clear(); + } +} + namespace arangodb { namespace graph { /* SingleServerProvider Section */ -template class PathValidator, VertexUniquenessLevel::PATH>; -template class PathValidator>, VertexUniquenessLevel::PATH>; +template class PathValidator, VertexUniquenessLevel::NONE>; +template class PathValidator, + PathStoreTracer::Step>>, VertexUniquenessLevel::NONE>; + +template class PathValidator, VertexUniquenessLevel::PATH>; +template class PathValidator, + PathStoreTracer::Step>>, VertexUniquenessLevel::PATH>; + +template class PathValidator, VertexUniquenessLevel::GLOBAL>; +template class PathValidator, + PathStoreTracer::Step>>, VertexUniquenessLevel::GLOBAL>; /* ClusterProvider Section */ -template class PathValidator, VertexUniquenessLevel::PATH>; -template class PathValidator>, VertexUniquenessLevel::PATH>; +template class PathValidator, VertexUniquenessLevel::NONE>; +template class PathValidator, + PathStoreTracer::Step>>, VertexUniquenessLevel::NONE>; + +template class PathValidator, VertexUniquenessLevel::PATH>; +template class PathValidator, + PathStoreTracer::Step>>, VertexUniquenessLevel::PATH>; + +template class PathValidator, VertexUniquenessLevel::GLOBAL>; +template class PathValidator, + PathStoreTracer::Step>>, VertexUniquenessLevel::GLOBAL>; } // namespace graph } // namespace arangodb diff --git a/arangod/Graph/PathManagement/PathValidator.h b/arangod/Graph/PathManagement/PathValidator.h index 0ca712af659e..a3971bf50d0e 100644 --- a/arangod/Graph/PathManagement/PathValidator.h +++ b/arangod/Graph/PathManagement/PathValidator.h @@ -25,36 +25,75 @@ #include #include "Containers/HashSet.h" +#include "Graph/PathManagement/PathValidatorOptions.h" #include "Graph/Types/UniquenessLevel.h" +#include + namespace arangodb { +namespace aql { +class PruneExpressionEvaluator; +} namespace graph { class ValidationResult; -template +/* + * TODO: + * + * - EdgeUniqueness, like vertex Uniqueness, and sometimes Edge and Vertex Uniqueness enforce each other. + * (e.g. => VertexUniqueness == PATH => EdgeUniquess PATH || NONE. + * + * - Prune Condition. + * - PostFilter Condition. + * - Path Conditions. Vertex, Edge maybe in LookupInfo + * (e.g. p.vertices[*].name ALL == "HANS") + */ +template class PathValidator { using VertexRef = arangodb::velocypack::HashedStringRef; public: - PathValidator(PathStore const& store); + PathValidator(Provider& provider, PathStore const& store, PathValidatorOptions opts); + ~PathValidator(); auto validatePath(typename PathStore::Step const& step) -> ValidationResult; auto validatePath(typename PathStore::Step const& step, - PathValidator const& otherValidator) + PathValidator const& otherValidator) -> ValidationResult; + void setPruneEvaluator(std::unique_ptr eval); + + void setPostFilterEvaluator(std::unique_ptr eval); + + void reset(); + private: PathStore const& _store; + Provider& _provider; // Only for applied vertex uniqueness // TODO: Figure out if we can make this Member template dependend // e.g. std::enable_if ::arangodb::containers::HashSet, std::equal_to> _uniqueVertices; + PathValidatorOptions _options; + + std::unique_ptr _pruneEvaluator; + + std::unique_ptr _postFilterEvaluator; + + arangodb::velocypack::Builder _tmpObjectBuilder; + private: + auto evaluateVertexCondition(typename PathStore::Step const&) -> ValidationResult; + auto evaluateVertexRestriction(typename PathStore::Step const& step) -> bool; + auto exposeUniqueVertices() const -> ::arangodb::containers::HashSet, std::equal_to> const&; + + auto evaluateExpression(arangodb::aql::Expression* expression, + arangodb::velocypack::Slice value) -> bool; }; } // namespace graph } // namespace arangodb diff --git a/arangod/Graph/PathManagement/PathValidatorOptions.cpp b/arangod/Graph/PathManagement/PathValidatorOptions.cpp new file mode 100644 index 000000000000..81935f1931a1 --- /dev/null +++ b/arangod/Graph/PathManagement/PathValidatorOptions.cpp @@ -0,0 +1,79 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2021-2021 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +//////////////////////////////////////////////////////////////////////////////// + +#include "PathValidatorOptions.h" + +#include "Aql/Expression.h" +#include "Aql/QueryContext.h" + +using namespace arangodb; +using namespace arangodb::graph; + +PathValidatorOptions::PathValidatorOptions(aql::Variable const* tmpVar, + arangodb::aql::FixedVarExpressionContext& expressionContext) + : _tmpVar(tmpVar), _expressionCtx(expressionContext) {} + +void PathValidatorOptions::setAllVerticesExpression(std::unique_ptr expression) { + // All edge expression should not be set before + TRI_ASSERT(_allVerticesExpression == nullptr); + _allVerticesExpression = std::move(expression); +} + +void PathValidatorOptions::setVertexExpression(uint64_t depth, + std::unique_ptr expression) { + // Should not respecifiy the condition on a certain depth + TRI_ASSERT(_vertexExpressionOnDepth.find(depth) == _vertexExpressionOnDepth.end()); + _vertexExpressionOnDepth.emplace(depth, std::move(expression)); +} + +aql::Expression* PathValidatorOptions::getVertexExpression(uint64_t depth) const { + auto const& it = _vertexExpressionOnDepth.find(depth); + if (it != _vertexExpressionOnDepth.end()) { + return it->second.get(); + } + return _allVerticesExpression.get(); +} + +void PathValidatorOptions::addAllowedVertexCollection(std::string const& collectionName) { + TRI_ASSERT(std::find(_allowedVertexCollections.begin(), + _allowedVertexCollections.end(), + collectionName) == _allowedVertexCollections.end()); + _allowedVertexCollections.emplace_back(collectionName); +} + +void PathValidatorOptions::addAllowedVertexCollections(std::vector const& collectionNames) { + for (auto& name : collectionNames) { + addAllowedVertexCollection(name); + } +} + +std::vector const& PathValidatorOptions::getAllowedVertexCollections() const { + return _allowedVertexCollections; +} + +aql::Variable const* PathValidatorOptions::getTempVar() const { + return _tmpVar; +} + +aql::FixedVarExpressionContext& PathValidatorOptions::getExpressionContext() { + return _expressionCtx; +} diff --git a/arangod/Graph/PathManagement/PathValidatorOptions.h b/arangod/Graph/PathManagement/PathValidatorOptions.h new file mode 100644 index 000000000000..14c5096146f4 --- /dev/null +++ b/arangod/Graph/PathManagement/PathValidatorOptions.h @@ -0,0 +1,88 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2021-2021 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +//////////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include +#include +#include + +#include "Aql/AqlFunctionsInternalCache.h" +#include "Aql/FixedVarExpressionContext.h" +#include "Transaction/Methods.h" + +namespace arangodb { +namespace aql { +class Expression; +class ExpressionContext; +class QueryContext; +struct Variable; +} // namespace aql + +namespace graph { +class PathValidatorOptions { + public: + PathValidatorOptions(aql::Variable const* tmpVar, + arangodb::aql::FixedVarExpressionContext& expressionContext); + ~PathValidatorOptions() = default; + PathValidatorOptions(PathValidatorOptions&&) = default; + PathValidatorOptions(PathValidatorOptions const&) = default; + + /** + * @brief Set the expression that needs to hold true for ALL vertices on the path. + */ + void setAllVerticesExpression(std::unique_ptr expression); + + /** + * @brief Set the expression that needs to hold true for the vertex on the + * given depth. NOTE: This will overrule the ALL vertex expression, so make + * sure this expression contains everything the ALL expression covers. + */ + void setVertexExpression(uint64_t depth, std::unique_ptr expression); + + /** + * @brief Get the Expression a vertex needs to hold if defined on the given + * depth. It may return a nullptr if all vertices are valid. + * Caller does NOT take responsibilty. Do not delete this pointer. + */ + aql::Expression* getVertexExpression(uint64_t depth) const; + + void addAllowedVertexCollection(std::string const& collectionName); + + void addAllowedVertexCollections(std::vector const& collectionNames); + + std::vector const& getAllowedVertexCollections() const; + + aql::Variable const* getTempVar() const; + + aql::FixedVarExpressionContext& getExpressionContext(); + + private: + std::shared_ptr _allVerticesExpression; + std::unordered_map> _vertexExpressionOnDepth; + aql::Variable const* _tmpVar; + arangodb::aql::FixedVarExpressionContext& _expressionCtx; + + std::vector _allowedVertexCollections; +}; +} // namespace graph +} // namespace arangodb diff --git a/arangod/Graph/PathManagement/SingleProviderPathResult.cpp b/arangod/Graph/PathManagement/SingleProviderPathResult.cpp index 86732a8d9a8c..c1597ed4d74d 100644 --- a/arangod/Graph/PathManagement/SingleProviderPathResult.cpp +++ b/arangod/Graph/PathManagement/SingleProviderPathResult.cpp @@ -25,6 +25,8 @@ #include "SingleProviderPathResult.h" #include "Basics/StaticStrings.h" +#include "Graph/PathManagement/PathStore.h" +#include "Graph/PathManagement/PathStoreTracer.h" #include "Graph/Providers/ClusterProvider.h" #include "Graph/Providers/ProviderTracer.h" #include "Graph/Providers/SingleServerProvider.h" @@ -35,43 +37,55 @@ using namespace arangodb; using namespace arangodb::graph; -template -SingleProviderPathResult::SingleProviderPathResult(ProviderType& provider) - : _provider(provider) {} +template +SingleProviderPathResult::SingleProviderPathResult( + Step* step, ProviderType& provider, PathStoreType& store) + : _step(step), _provider(provider), _store(store) { + TRI_ASSERT(_step != nullptr); +} -template -auto SingleProviderPathResult::clear() -> void { +template +auto SingleProviderPathResult::clear() -> void { _vertices.clear(); _edges.clear(); } -template -auto SingleProviderPathResult::appendVertex(typename Step::Vertex v) +template +auto SingleProviderPathResult::appendVertex(typename Step::Vertex v) -> void { _vertices.push_back(std::move(v)); } -template -auto SingleProviderPathResult::prependVertex(typename Step::Vertex v) +template +auto SingleProviderPathResult::prependVertex(typename Step::Vertex v) -> void { _vertices.insert(_vertices.begin(), std::move(v)); } -template -auto SingleProviderPathResult::appendEdge(typename Step::Edge e) +template +auto SingleProviderPathResult::appendEdge(typename Step::Edge e) -> void { _edges.push_back(std::move(e)); } -template -auto SingleProviderPathResult::prependEdge(typename Step::Edge e) +template +auto SingleProviderPathResult::prependEdge(typename Step::Edge e) -> void { _edges.insert(_edges.begin(), std::move(e)); } -template -auto SingleProviderPathResult::toVelocyPack(arangodb::velocypack::Builder& builder) - -> void { +template +auto SingleProviderPathResult::toVelocyPack( + arangodb::velocypack::Builder& builder) -> void { + if (_vertices.empty()) { + _store.visitReversePath(*_step, [&](Step const& s) -> bool { + prependVertex(s.getVertex()); + if (s.getEdge().isValid()) { + prependEdge(s.getEdge()); + } + return true; + }); + } VPackObjectBuilder path{&builder}; { builder.add(VPackValue(StaticStrings::GraphQueryVertices)); @@ -92,24 +106,78 @@ auto SingleProviderPathResult::toVelocyPack(arangodb::velocy } } -template -auto SingleProviderPathResult::isEmpty() const -> bool { - return _vertices.empty(); +template +auto SingleProviderPathResult::toSchreierEntry( + arangodb::velocypack::Builder& result, size_t& currentLength) -> void { + size_t prevIndex = 0; + + auto writeStepToBuilder = [&](Step& step) { + VPackArrayBuilder arrayGuard(&result); + // Create a VPackValue based on the char* inside the HashedStringRef, will save us a String copy each time. + result.add(VPackValue(step.getVertex().getID().begin())); + + // Index position of previous step + result.add(VPackValue(prevIndex)); + // The depth of the current step + result.add(VPackValue(step.getDepth())); + + bool isResponsible = step.isResponsible(_provider.trx()); + // Print if we have a loose end here (looseEnd := this server is NOT responsible) + result.add(VPackValue(!isResponsible)); + if (isResponsible) { + // This server needs to provide the data for the vertex + _provider.addVertexToBuilder(step.getVertex(), result); + } else { + result.add(VPackSlice::nullSlice()); + } + + _provider.addEdgeToBuilder(step.getEdge(), result); + }; // TODO: Create method instead of lambda + + std::vector toWrite{}; + + _store.modifyReversePath(*_step, [&](Step& step) -> bool { + if (!step.hasLocalSchreierIndex()) { + toWrite.emplace_back(&step); + return true; + } else { + prevIndex = step.getLocalSchreierIndex(); + return false; + } + }); + + for (auto it = toWrite.rbegin(); it != toWrite.rend(); it++) { + TRI_ASSERT(!(*it)->hasLocalSchreierIndex()); + writeStepToBuilder(**it); + + prevIndex = currentLength; + (*it)->setLocalSchreierIndex(currentLength++); + } +} + +template +auto SingleProviderPathResult::isEmpty() const + -> bool { + return false; } /* SingleServerProvider Section */ template class ::arangodb::graph::SingleProviderPathResult< - ::arangodb::graph::SingleServerProvider, ::arangodb::graph::SingleServerProvider::Step>; + ::arangodb::graph::SingleServerProvider, ::arangodb::graph::PathStore<::arangodb::graph::SingleServerProvider::Step>, + ::arangodb::graph::SingleServerProvider::Step>; template class ::arangodb::graph::SingleProviderPathResult< ::arangodb::graph::ProviderTracer<::arangodb::graph::SingleServerProvider>, + ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::graph::SingleServerProvider::Step>>, ::arangodb::graph::SingleServerProvider::Step>; /* ClusterProvider Section */ - template class ::arangodb::graph::SingleProviderPathResult< - ::arangodb::graph::ClusterProvider, ::arangodb::graph::ClusterProvider::Step>; + ::arangodb::graph::ClusterProvider, ::arangodb::graph::PathStore<::arangodb::graph::ClusterProvider::Step>, + ::arangodb::graph::ClusterProvider::Step>; template class ::arangodb::graph::SingleProviderPathResult< - ::arangodb::graph::ProviderTracer<::arangodb::graph::ClusterProvider>, ::arangodb::graph::ClusterProvider::Step>; \ No newline at end of file + ::arangodb::graph::ProviderTracer<::arangodb::graph::ClusterProvider>, + ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::graph::ClusterProvider::Step>>, + ::arangodb::graph::ClusterProvider::Step>; diff --git a/arangod/Graph/PathManagement/SingleProviderPathResult.h b/arangod/Graph/PathManagement/SingleProviderPathResult.h index 9d0fd94b359f..13b7b3021b69 100644 --- a/arangod/Graph/PathManagement/SingleProviderPathResult.h +++ b/arangod/Graph/PathManagement/SingleProviderPathResult.h @@ -27,7 +27,11 @@ #include #include "Containers/HashSet.h" +// TODO Temporary include +#include "Graph/Enumerators/OneSidedEnumeratorInterface.h" + #include +#include namespace arangodb { @@ -37,27 +41,37 @@ class Builder; namespace graph { -template -class SingleProviderPathResult { +template +class SingleProviderPathResult : public PathResultInterface { using VertexRef = arangodb::velocypack::HashedStringRef; public: - SingleProviderPathResult(ProviderType& provider); + SingleProviderPathResult(Step* step, ProviderType& provider, PathStoreType& store); auto clear() -> void; auto appendVertex(typename Step::Vertex v) -> void; auto prependVertex(typename Step::Vertex v) -> void; auto appendEdge(typename Step::Edge e) -> void; auto prependEdge(typename Step::Edge e) -> void; - auto toVelocyPack(arangodb::velocypack::Builder& builder) -> void; - + + auto toVelocyPack(arangodb::velocypack::Builder& builder) -> void override; + + /** + * @brief Appends this path as a SchreierVector entry into the given builder + */ + auto toSchreierEntry(arangodb::velocypack::Builder& builder, size_t& currentLength) + -> void override; + auto isEmpty() const -> bool; private: + Step* _step; + std::vector _vertices; std::vector _edges; - + // Provider for the path ProviderType& _provider; + PathStoreType& _store; }; } // namespace graph } // namespace arangodb diff --git a/arangod/Graph/Providers/BaseProviderOptions.cpp b/arangod/Graph/Providers/BaseProviderOptions.cpp index 12e42e959f4a..b23bef9404c1 100644 --- a/arangod/Graph/Providers/BaseProviderOptions.cpp +++ b/arangod/Graph/Providers/BaseProviderOptions.cpp @@ -27,11 +27,20 @@ using namespace arangodb; using namespace arangodb::graph; IndexAccessor::IndexAccessor(transaction::Methods::IndexHandle idx, - aql::AstNode* condition, std::optional memberToUpdate) - : _idx(idx), _indexCondition(condition), _memberToUpdate(memberToUpdate) {} + aql::AstNode* condition, std::optional memberToUpdate, + std::shared_ptr expression) + : _idx(idx), _indexCondition(condition), _memberToUpdate(memberToUpdate) { + if (expression != nullptr) { + _expression = std::move(expression); + } +} aql::AstNode* IndexAccessor::getCondition() const { return _indexCondition; } +aql::Expression* IndexAccessor::getExpression() const { + return _expression.get(); +} + transaction::Methods::IndexHandle IndexAccessor::indexHandle() const { return _idx; } @@ -40,25 +49,34 @@ std::optional IndexAccessor::getMemberToUpdate() const { return _memberToUpdate; } -BaseProviderOptions::BaseProviderOptions(aql::Variable const* tmpVar, - std::vector indexInfo, - std::map const& collectionToShardMap) +BaseProviderOptions::BaseProviderOptions( + aql::Variable const* tmpVar, + std::pair, std::unordered_map>> indexInfo, + aql::FixedVarExpressionContext& expressionContext, + std::unordered_map> const& collectionToShardMap) : _temporaryVariable(tmpVar), _indexInformation(std::move(indexInfo)), + _expressionContext(expressionContext), _collectionToShardMap(collectionToShardMap) {} aql::Variable const* BaseProviderOptions::tmpVar() const { return _temporaryVariable; } -std::vector const& BaseProviderOptions::indexInformations() const { +// first is global index information, second is depth-based index information. +std::pair, std::unordered_map>> const& +BaseProviderOptions::indexInformations() const { return _indexInformation; } -std::map const& BaseProviderOptions::collectionToShardMap() const { +std::unordered_map> const& BaseProviderOptions::collectionToShardMap() const { return _collectionToShardMap; } +aql::FixedVarExpressionContext& BaseProviderOptions::expressionContext() const { + return _expressionContext; +} + ClusterBaseProviderOptions::ClusterBaseProviderOptions( std::shared_ptr cache, std::unordered_map const* engines, bool backward) @@ -77,4 +95,4 @@ bool ClusterBaseProviderOptions::isBackward() const { return _backward; } std::unordered_map const* ClusterBaseProviderOptions::engines() const { TRI_ASSERT(_engines != nullptr); return _engines; -} \ No newline at end of file +} diff --git a/arangod/Graph/Providers/BaseProviderOptions.h b/arangod/Graph/Providers/BaseProviderOptions.h index 6f2ce9bed657..ae84dc818a5b 100644 --- a/arangod/Graph/Providers/BaseProviderOptions.h +++ b/arangod/Graph/Providers/BaseProviderOptions.h @@ -23,6 +23,7 @@ #pragma once +#include "Aql/Expression.h" #include "Aql/FixedVarExpressionContext.h" #include "Cluster/ClusterInfo.h" #include "Graph/Cache/RefactoredClusterTraverserCache.h" @@ -41,9 +42,11 @@ namespace graph { struct IndexAccessor { IndexAccessor(transaction::Methods::IndexHandle idx, aql::AstNode* condition, - std::optional memberToUpdate); + std::optional memberToUpdate, + std::shared_ptr expression); aql::AstNode* getCondition() const; + aql::Expression* getExpression() const; transaction::Methods::IndexHandle indexHandle() const; std::optional getMemberToUpdate() const; @@ -51,27 +54,39 @@ struct IndexAccessor { transaction::Methods::IndexHandle _idx; aql::AstNode* _indexCondition; std::optional _memberToUpdate; + + // TODO: This needs to be changed BEFORE merge + std::shared_ptr _expression; }; struct BaseProviderOptions { public: - BaseProviderOptions(aql::Variable const* tmpVar, std::vector indexInfo, - std::map const& collectionToShardMap); + BaseProviderOptions( + aql::Variable const* tmpVar, + std::pair, std::unordered_map>> indexInfo, + aql::FixedVarExpressionContext& expressionContext, + std::unordered_map> const& collectionToShardMap); aql::Variable const* tmpVar() const; - std::vector const& indexInformations() const; + std::pair, std::unordered_map>> const& indexInformations() const; + + std::unordered_map> const& collectionToShardMap() const; - std::map const& collectionToShardMap() const; + aql::FixedVarExpressionContext& expressionContext() const; private: // The temporary Variable used in the Indexes aql::Variable const* _temporaryVariable; // One entry per collection, ShardTranslation needs // to be done by Provider - std::vector _indexInformation; + std::pair, std::unordered_map>> _indexInformation; + + // The context of AQL variables. These variables are set from the outside. + // and the caller needs to make sure the reference stays valid + aql::FixedVarExpressionContext& _expressionContext; // CollectionName to ShardMap, used if the Traversal is pushed down to DBServer - std::map const& _collectionToShardMap; + std::unordered_map> const& _collectionToShardMap; }; struct ClusterBaseProviderOptions { @@ -96,4 +111,3 @@ struct ClusterBaseProviderOptions { } // namespace graph } // namespace arangodb - diff --git a/arangod/Graph/Providers/BaseStep.h b/arangod/Graph/Providers/BaseStep.h index 81bc49531500..c9a49139d98c 100644 --- a/arangod/Graph/Providers/BaseStep.h +++ b/arangod/Graph/Providers/BaseStep.h @@ -23,9 +23,12 @@ #pragma once +#include + #include namespace arangodb { + namespace graph { template @@ -34,6 +37,7 @@ class BaseStep { BaseStep() : _previous{std::numeric_limits::max()}, _depth{0} {} BaseStep(size_t prev) : _previous{prev}, _depth{0} {} + BaseStep(size_t prev, size_t depth) : _previous{prev}, _depth{depth} {} size_t getPrevious() const { return _previous; } @@ -46,13 +50,26 @@ class BaseStep { return static_cast(this)->isLooseEnd(); } - size_t getDepth() const { - return _depth; + size_t getDepth() const { return _depth; } + + ResultT> extractCollectionName( + arangodb::velocypack::HashedStringRef const& idHashed) const { + size_t pos = idHashed.find('/'); + if (pos == std::string::npos) { + // Invalid input. If we get here somehow we managed to store invalid + // _from/_to values or the traverser did a let an illegal start through + TRI_ASSERT(false); + return Result{TRI_ERROR_GRAPH_INVALID_EDGE, + "edge contains invalid value " + idHashed.toString()}; + } + + std::string colName = idHashed.substr(0, pos).toString(); + return std::make_pair(colName, pos); } private: - size_t const _previous; - size_t const _depth; + size_t _previous; + size_t _depth; }; } // namespace graph } // namespace arangodb diff --git a/arangod/Graph/Providers/ClusterProvider.cpp b/arangod/Graph/Providers/ClusterProvider.cpp index 6a643cf17bf6..8e4425c08742 100644 --- a/arangod/Graph/Providers/ClusterProvider.cpp +++ b/arangod/Graph/Providers/ClusterProvider.cpp @@ -68,7 +68,6 @@ VertexType getEdgeDestination(arangodb::velocypack::Slice edge, VertexType const } } // namespace - namespace arangodb { namespace graph { auto operator<<(std::ostream& out, ClusterProvider::Step const& step) -> std::ostream& { @@ -94,9 +93,11 @@ VertexType const& ClusterProvider::Step::Vertex::getID() const { } EdgeType const& ClusterProvider::Step::Edge::getID() const { return _edge; } -bool ClusterProvider::Step::Edge::isValid() const { - return !_edge.empty(); -}; +bool ClusterProvider::Step::Edge::isValid() const { return !_edge.empty(); }; + +bool ClusterProvider::Step::isResponsible(transaction::Methods* trx) const { + return true; +} ClusterProvider::ClusterProvider(arangodb::aql::QueryContext& queryContext, ClusterBaseProviderOptions opts, @@ -107,17 +108,16 @@ ClusterProvider::ClusterProvider(arangodb::aql::QueryContext& queryContext, _opts(std::move(opts)), _stats{} {} -ClusterProvider::~ClusterProvider() { - clear(); -} +ClusterProvider::~ClusterProvider() { clear(); } void ClusterProvider::clear() { for (auto const& entry : _vertexConnectedEdges) { - _resourceMonitor->decreaseMemoryUsage(costPerVertexOrEdgeType + (entry.second.size() * (costPerVertexOrEdgeType * 2))); + _resourceMonitor->decreaseMemoryUsage( + costPerVertexOrEdgeType + (entry.second.size() * (costPerVertexOrEdgeType * 2))); } } -auto ClusterProvider::startVertex(VertexType vertex) -> Step { +auto ClusterProvider::startVertex(VertexType vertex, size_t depth) -> Step { LOG_TOPIC("da308", TRACE, Logger::GRAPHS) << " Start Vertex:" << vertex; // Create the default initial step. return Step(_opts.getCache()->persistString(vertex)); @@ -203,9 +203,8 @@ void ClusterProvider::fetchVerticesFromEngines(std::vector const& looseEn _opts.getCache()->datalake().add(std::move(payload)); } - /* TODO: Needs to be taken care of as soon as we enable shortest paths for ClusterProvider - bool forShortestPath = true; - if (!forShortestPath) { + /* TODO: Needs to be taken care of as soon as we enable shortest paths for + ClusterProvider bool forShortestPath = true; if (!forShortestPath) { // Fill everything we did not find with NULL for (auto const& v : vertexIds) { result.try_emplace(v, VPackSlice::nullSlice()); @@ -222,8 +221,10 @@ void ClusterProvider::fetchVerticesFromEngines(std::vector const& looseEn for (auto& lE : looseEnds) { if (!_opts.getCache()->isVertexCached(lE->getVertexIdentifier())) { // if we end up here, we we're not able to cache the requested vertex (e.g. it does not exist) - _query->warnings().registerWarning(TRI_ERROR_ARANGO_DOCUMENT_NOT_FOUND, lE->getVertexIdentifier().toString()); - _opts.getCache()->cacheVertex(std::move(lE->getVertexIdentifier()), VPackSlice::nullSlice()); + _query->warnings().registerWarning(TRI_ERROR_ARANGO_DOCUMENT_NOT_FOUND, + lE->getVertexIdentifier().toString()); + _opts.getCache()->cacheVertex(std::move(lE->getVertexIdentifier()), + VPackSlice::nullSlice()); } result.emplace_back(std::move(lE)); } @@ -249,18 +250,19 @@ void ClusterProvider::destroyEngines() { auto const* engines = _opts.engines(); for (auto const& engine : *engines) { _stats.addHttpRequests(1); - auto res = network::sendRequestRetry(pool, "server:" + engine.first, fuerte::RestVerb::Delete, - "/_internal/traverser/" + - arangodb::basics::StringUtils::itoa(engine.second), - VPackBuffer(), options) - .get(); + auto res = + network::sendRequestRetry(pool, "server:" + engine.first, fuerte::RestVerb::Delete, + "/_internal/traverser/" + + arangodb::basics::StringUtils::itoa(engine.second), + VPackBuffer(), options) + .get(); if (res.error != fuerte::Error::NoError) { // Note If there was an error on server side we do not have // CL_COMM_SENT LOG_TOPIC("d31a5", ERR, arangodb::Logger::GRAPHS) - << "Could not destroy all traversal engines: " - << TRI_errno_string(network::fuerteToArangoErrorCode(res)); + << "Could not destroy all traversal engines: " + << TRI_errno_string(network::fuerteToArangoErrorCode(res)); } } } @@ -339,7 +341,8 @@ Result ClusterProvider::fetchEdgesFromEngines(VertexType const& vertex) { arangodb::velocypack::HashedStringRef edgeIdRef(edge.get(StaticStrings::IdString)); - auto edgeToEmplace = std::make_pair(edgeIdRef, VertexType{getEdgeDestination(edge, vertex)}); + auto edgeToEmplace = + std::make_pair(edgeIdRef, VertexType{getEdgeDestination(edge, vertex)}); connectedEdges.emplace_back(edgeToEmplace); } @@ -351,7 +354,8 @@ Result ClusterProvider::fetchEdgesFromEngines(VertexType const& vertex) { // Note: This disables the TRI_DEFER futures.clear(); - _resourceMonitor->increaseMemoryUsage(costPerVertexOrEdgeType + (connectedEdges.size() * (costPerVertexOrEdgeType * 2))); + _resourceMonitor->increaseMemoryUsage( + costPerVertexOrEdgeType + (connectedEdges.size() * (costPerVertexOrEdgeType * 2))); _vertexConnectedEdges.emplace(vertex, std::move(connectedEdges)); return TRI_ERROR_NO_ERROR; @@ -398,12 +402,12 @@ auto ClusterProvider::expand(Step const& step, size_t previous, TRI_ASSERT(_opts.getCache()->isVertexCached(vertex.getID())); TRI_ASSERT(_vertexConnectedEdges.find(vertex.getID()) != _vertexConnectedEdges.end()); for (auto const& relation : _vertexConnectedEdges.at(vertex.getID())) { - bool fetched = _vertexConnectedEdges.find(relation.second) != _vertexConnectedEdges.end(); + bool fetched = + _vertexConnectedEdges.find(relation.second) != _vertexConnectedEdges.end(); callback(Step{relation.second, relation.first, previous, fetched}); } } - void ClusterProvider::addVertexToBuilder(Step::Vertex const& vertex, arangodb::velocypack::Builder& builder) { TRI_ASSERT(_opts.getCache()->isVertexCached(vertex.getID())); diff --git a/arangod/Graph/Providers/ClusterProvider.h b/arangod/Graph/Providers/ClusterProvider.h index 76ebb47fafdc..e10dd2f1a9c5 100644 --- a/arangod/Graph/Providers/ClusterProvider.h +++ b/arangod/Graph/Providers/ClusterProvider.h @@ -121,6 +121,29 @@ class ClusterProvider { VertexType getVertexIdentifier() const { return _vertex.getID(); } + std::string getCollectionName() const { + auto collectionNameResult = extractCollectionName(_vertex.getID()); + if (collectionNameResult.fail()) { + THROW_ARANGO_EXCEPTION(collectionNameResult.result()); + } + return collectionNameResult.get().first; + }; + + /* to be moved-out later into seperate step */ + void setLocalSchreierIndex(size_t index) { + TRI_ASSERT(index != std::numeric_limits::max()); + TRI_ASSERT(!hasLocalSchreierIndex()); + _localSchreierIndex = index; + } + + bool hasLocalSchreierIndex() const { + return _localSchreierIndex != std::numeric_limits::max(); + } + + std::size_t getLocalSchreierIndex() const { return _localSchreierIndex; } + + bool isResponsible(transaction::Methods* trx) const; + friend auto operator<<(std::ostream& out, Step const& step) -> std::ostream&; private: @@ -130,6 +153,7 @@ class ClusterProvider { Vertex _vertex; Edge _edge; bool _fetched; + size_t _localSchreierIndex = std::numeric_limits::max(); // to be removed later }; public: @@ -143,7 +167,7 @@ class ClusterProvider { void clear(); - auto startVertex(VertexType vertex) -> Step; + auto startVertex(VertexType vertex, size_t depth = 0) -> Step; auto fetch(std::vector const& looseEnds) -> futures::Future>; auto expand(Step const& from, size_t previous, std::function const& callback) -> void; diff --git a/arangod/Graph/Providers/ProviderTracer.cpp b/arangod/Graph/Providers/ProviderTracer.cpp index 2efcce0bd0c6..e78e41dfae7f 100644 --- a/arangod/Graph/Providers/ProviderTracer.cpp +++ b/arangod/Graph/Providers/ProviderTracer.cpp @@ -37,7 +37,7 @@ template ProviderTracer::ProviderTracer(arangodb::aql::QueryContext& queryContext, Options opts, arangodb::ResourceMonitor& resourceMonitor) - : _impl{queryContext, opts, resourceMonitor} {} + : _impl{queryContext, std::move(opts), resourceMonitor} {} template ProviderTracer::~ProviderTracer() { @@ -48,10 +48,10 @@ ProviderTracer::~ProviderTracer() { } template -typename ProviderImpl::Step ProviderTracer::startVertex(VertexType vertex) { +typename ProviderImpl::Step ProviderTracer::startVertex(VertexType vertex, size_t depth) { double start = TRI_microtime(); TRI_DEFER(_stats["startVertex"].addTiming(TRI_microtime() - start)); - return _impl.startVertex(vertex); + return _impl.startVertex(vertex, depth); } template diff --git a/arangod/Graph/Providers/ProviderTracer.h b/arangod/Graph/Providers/ProviderTracer.h index 13ed9cf33aaf..3c6f852ec37c 100644 --- a/arangod/Graph/Providers/ProviderTracer.h +++ b/arangod/Graph/Providers/ProviderTracer.h @@ -59,7 +59,7 @@ class ProviderTracer { ProviderTracer& operator=(ProviderTracer const&) = delete; ProviderTracer& operator=(ProviderTracer&&) = default; - auto startVertex(VertexType vertex) -> Step; + auto startVertex(VertexType vertex, size_t depth = 0) -> Step; auto fetch(std::vector const& looseEnds) -> futures::Future>; // rocks auto expand(Step const& from, size_t previous, std::function callback) -> void; // index diff --git a/arangod/Graph/Providers/SingleServerProvider.cpp b/arangod/Graph/Providers/SingleServerProvider.cpp index 245f6abf1d8c..46ac1b1bb44f 100644 --- a/arangod/Graph/Providers/SingleServerProvider.cpp +++ b/arangod/Graph/Providers/SingleServerProvider.cpp @@ -47,6 +47,9 @@ auto operator<<(std::ostream& out, SingleServerProvider::Step const& step) -> st SingleServerProvider::Step::Step(VertexType v) : _vertex(v), _edge() {} +SingleServerProvider::Step::Step(VertexType v, size_t depth) + : BaseStep(std::numeric_limits::max(), depth), _vertex(v), _edge() {} + SingleServerProvider::Step::Step(VertexType v, EdgeDocumentToken edge, size_t prev) : BaseStep(prev), _vertex(v), _edge(std::move(edge)) {} @@ -64,12 +67,24 @@ EdgeDocumentToken const& SingleServerProvider::Step::Edge::getID() const { } bool SingleServerProvider::Step::Edge::isValid() const { - return getID().localDocumentId() != DataSourceId::none(); + return getID().isValid(); +}; + +#ifndef USE_ENTERPRISE +bool SingleServerProvider::Step::isResponsible(transaction::Methods* trx) const { + return true; }; +#endif + + void SingleServerProvider::addEdgeToBuilder(Step::Edge const& edge, arangodb::velocypack::Builder& builder) { - insertEdgeIntoResult(edge.getID(), builder); + if (edge.isValid()) { + insertEdgeIntoResult(edge.getID(), builder); + } else { + builder.add(VPackSlice::nullSlice()); + } }; void SingleServerProvider::Step::Edge::addToBuilder(SingleServerProvider& provider, @@ -86,7 +101,7 @@ SingleServerProvider::SingleServerProvider(arangodb::aql::QueryContext& queryCon _opts.collectionToShardMap()), _stats{} { // activateCache(false); // TODO CHECK RefactoredTraverserCache (will be discussed in the future, need to do benchmarks if affordable) - _cursor = buildCursor(); + _cursor = buildCursor(opts.expressionContext()); } void SingleServerProvider::activateCache(bool enableDocumentCache) { @@ -108,13 +123,13 @@ void SingleServerProvider::activateCache(bool enableDocumentCache) { // _cache = new RefactoredTraverserCache(query()); } -auto SingleServerProvider::startVertex(VertexType vertex) -> Step { +auto SingleServerProvider::startVertex(VertexType vertex, size_t depth) -> Step { LOG_TOPIC("78156", TRACE, Logger::GRAPHS) << " Start Vertex:" << vertex; // Create default initial step // Note: Refactor naming, Strings in our cache here are not allowed to be removed. - return Step(_cache.persistString(vertex)); + return Step(_cache.persistString(vertex), depth); } auto SingleServerProvider::fetch(std::vector const& looseEnds) @@ -134,21 +149,28 @@ auto SingleServerProvider::expand(Step const& step, size_t previous, TRI_ASSERT(!step.isLooseEnd()); auto const& vertex = step.getVertex(); TRI_ASSERT(_cursor != nullptr); + LOG_TOPIC("c9169", TRACE, Logger::GRAPHS) + << " Expanding " << vertex.getID(); _cursor->rearm(vertex.getID(), 0); - _cursor->readAll(_stats, [&](EdgeDocumentToken&& eid, VPackSlice edge, size_t /*cursorIdx*/) -> void { - VertexType id = _cache.persistString(([&]() -> auto { - if (edge.isString()) { - return VertexType(edge); - } else { - VertexType other(transaction::helpers::extractFromFromDocument(edge)); - if (other == vertex.getID()) { // TODO: Check getId - discuss - other = VertexType(transaction::helpers::extractToFromDocument(edge)); - } - return other; - } - })()); - callback(Step{id, std::move(eid), previous}); - }); + _cursor->readAll( + *this, _stats, step.getDepth(), + [&](EdgeDocumentToken&& eid, VPackSlice edge, size_t /*cursorIdx*/) -> void { + VertexType id = _cache.persistString(([&]() -> auto { + if (edge.isString()) { + return VertexType(edge); + } else { + VertexType other(transaction::helpers::extractFromFromDocument(edge)); + if (other == vertex.getID()) { // TODO: Check getId - discuss + other = VertexType(transaction::helpers::extractToFromDocument(edge)); + } + return other; + } + })()); + // TODO: Adjust log output + LOG_TOPIC("c9168", TRACE, Logger::GRAPHS) + << " Neighbor of " << vertex.getID() << " -> " << id; + callback(Step{id, std::move(eid), previous, step.getDepth() + 1}); + }); } void SingleServerProvider::addVertexToBuilder(Step::Vertex const& vertex, @@ -161,12 +183,17 @@ void SingleServerProvider::insertEdgeIntoResult(EdgeDocumentToken edge, _cache.insertEdgeIntoResult(edge, builder); } -std::unique_ptr SingleServerProvider::buildCursor() { - return std::make_unique(trx(), _opts.tmpVar(), - _opts.indexInformations()); +std::unique_ptr SingleServerProvider::buildCursor( + arangodb::aql::FixedVarExpressionContext& expressionContext) { + return std::make_unique( + trx(), _opts.tmpVar(), _opts.indexInformations().first, + _opts.indexInformations().second, expressionContext); } arangodb::transaction::Methods* SingleServerProvider::trx() { + TRI_ASSERT(_trx != nullptr); + TRI_ASSERT(_trx->state() != nullptr); + TRI_ASSERT(_trx->transactionContextPtr() != nullptr); return _trx.get(); } diff --git a/arangod/Graph/Providers/SingleServerProvider.h b/arangod/Graph/Providers/SingleServerProvider.h index 3f2ee786362f..d35e28b8f7ab 100644 --- a/arangod/Graph/Providers/SingleServerProvider.h +++ b/arangod/Graph/Providers/SingleServerProvider.h @@ -68,7 +68,7 @@ struct SingleServerProvider { explicit Vertex(VertexType v) : _vertex(v) {} VertexType const& getID() const; - + bool operator<(Vertex const& other) const noexcept { return _vertex < other._vertex; } @@ -98,6 +98,7 @@ struct SingleServerProvider { Step(VertexType v); Step(VertexType v, EdgeDocumentToken edge, size_t prev); Step(VertexType v, EdgeDocumentToken edge, size_t prev, size_t depth); + Step(VertexType v, size_t depth); ~Step(); bool operator<(Step const& other) const noexcept { @@ -115,15 +116,40 @@ struct SingleServerProvider { VertexType getVertexIdentifier() const { return _vertex.getID(); } + std::string getCollectionName() const { + auto collectionNameResult = extractCollectionName(_vertex.getID()); + if (collectionNameResult.fail()) { + THROW_ARANGO_EXCEPTION(collectionNameResult.result()); + } + return collectionNameResult.get().first; + }; + + void setLocalSchreierIndex(size_t index) { + TRI_ASSERT(index != std::numeric_limits::max()); + TRI_ASSERT(!hasLocalSchreierIndex()); + _localSchreierIndex = index; + } + + bool hasLocalSchreierIndex() const { + return _localSchreierIndex != std::numeric_limits::max(); + } + + std::size_t getLocalSchreierIndex() const { + return _localSchreierIndex; + } + + bool isResponsible(transaction::Methods* trx) const; + friend auto operator<<(std::ostream& out, Step const& step) -> std::ostream&; private: Vertex _vertex; Edge _edge; + size_t _localSchreierIndex = std::numeric_limits::max(); }; public: - SingleServerProvider(arangodb::aql::QueryContext& queryContext, BaseProviderOptions opts, + SingleServerProvider(arangodb::aql::QueryContext& queryContext, Options opts, arangodb::ResourceMonitor& resourceMonitor); SingleServerProvider(SingleServerProvider const&) = delete; SingleServerProvider(SingleServerProvider&&) = default; @@ -131,7 +157,7 @@ struct SingleServerProvider { SingleServerProvider& operator=(SingleServerProvider const&) = delete; - auto startVertex(VertexType vertex) -> Step; + auto startVertex(VertexType vertex, size_t depth = 0) -> Step; auto fetch(std::vector const& looseEnds) -> futures::Future>; // rocks auto expand(Step const& from, size_t previous, @@ -151,7 +177,8 @@ struct SingleServerProvider { private: void activateCache(bool enableDocumentCache); - std::unique_ptr buildCursor(); + std::unique_ptr buildCursor( + arangodb::aql::FixedVarExpressionContext& expressionContext); private: // Unique_ptr to have this class movable, and to keep reference of trx() @@ -163,9 +190,8 @@ struct SingleServerProvider { BaseProviderOptions _opts; RefactoredTraverserCache _cache; - + arangodb::aql::TraversalStats _stats; }; } // namespace graph } // namespace arangodb - diff --git a/arangod/Graph/SingleServerEdgeCursor.cpp b/arangod/Graph/SingleServerEdgeCursor.cpp index 6e6d94c082a2..912711975846 100644 --- a/arangod/Graph/SingleServerEdgeCursor.cpp +++ b/arangod/Graph/SingleServerEdgeCursor.cpp @@ -127,7 +127,7 @@ bool SingleServerEdgeCursor::advanceCursor(IndexIterator*& cursor, bool SingleServerEdgeCursor::next(EdgeCursor::Callback const& callback) { // fills callback with next EdgeDocumentToken and Slice that contains the - // ohter side of the edge (we are standing on a node and want to iterate all + // other side of the edge (we are standing on a node and want to iterate all // connected edges TRI_ASSERT(!_cursors.empty()); @@ -225,6 +225,7 @@ void SingleServerEdgeCursor::readAll(EdgeCursor::Callback const& callback) { return false; } #endif + _opts->cache()->increaseCounter(); callback(EdgeDocumentToken(cid, token), edge, cursorId); return true; }); diff --git a/arangod/Graph/TraverserCache.cpp b/arangod/Graph/TraverserCache.cpp index e11bfeb41fdb..5ab0431a2b9c 100644 --- a/arangod/Graph/TraverserCache.cpp +++ b/arangod/Graph/TraverserCache.cpp @@ -28,8 +28,8 @@ #include "Basics/StringHeap.h" #include "Basics/VelocyPackHelper.h" #include "Cluster/ServerState.h" -#include "Graph/EdgeDocumentToken.h" #include "Graph/BaseOptions.h" +#include "Graph/EdgeDocumentToken.h" #include "Logger/LogMacros.h" #include "Logger/Logger.h" #include "Logger/LoggerStream.h" @@ -42,8 +42,8 @@ #include "VocBase/ManagedDocumentResult.h" #include -#include #include +#include #include #include @@ -51,11 +51,11 @@ using namespace arangodb; using namespace arangodb::graph; namespace { -constexpr size_t costPerPersistedString = sizeof(void*) + sizeof(arangodb::velocypack::HashedStringRef); +constexpr size_t costPerPersistedString = + sizeof(void*) + sizeof(arangodb::velocypack::HashedStringRef); bool isWithClauseMissing(arangodb::basics::Exception const& ex) { - if (ServerState::instance()->isDBServer() && - ex.code() == TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND) { + if (ServerState::instance()->isDBServer() && ex.code() == TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND) { // on a DB server, we could have got here only in the OneShard case. // in this case turn the rather misleading "collection or view not found" // error into a nicer "collection not known to traversal, please add WITH" @@ -70,7 +70,7 @@ bool isWithClauseMissing(arangodb::basics::Exception const& ex) { return false; } -} // namespace +} // namespace TraverserCache::TraverserCache(aql::QueryContext& query, BaseOptions* opts) : _query(query), @@ -79,12 +79,11 @@ TraverserCache::TraverserCache(aql::QueryContext& query, BaseOptions* opts) _filteredDocuments(0), _stringHeap(query.resourceMonitor(), 4096), /* arbitrary block-size, may be adjusted for performance */ _baseOptions(opts), - _allowImplicitCollections(ServerState::instance()->isSingleServer() && - !_query.vocbase().server().getFeature().requireWith()) {} + _allowImplicitCollections( + ServerState::instance()->isSingleServer() && + !_query.vocbase().server().getFeature().requireWith()) {} -TraverserCache::~TraverserCache() { - clear(); -} +TraverserCache::~TraverserCache() { clear(); } void TraverserCache::clear() { _query.resourceMonitor().decreaseMemoryUsage(_persistedStrings.size() * ::costPerPersistedString); @@ -119,7 +118,8 @@ VPackSlice TraverserCache::lookupToken(EdgeDocumentToken const& idToken) { return VPackSlice(_mmdr.vpack()); } -bool TraverserCache::appendVertex(arangodb::velocypack::StringRef id, arangodb::velocypack::Builder& result) { +bool TraverserCache::appendVertex(arangodb::velocypack::StringRef id, + arangodb::velocypack::Builder& result) { if (!_baseOptions->produceVertices()) { // this traversal does not produce any vertices result.add(arangodb::velocypack::Slice::nullSlice()); @@ -141,20 +141,24 @@ bool TraverserCache::appendVertex(arangodb::velocypack::StringRef id, arangodb:: if (!map.empty()) { auto found = map.find(collectionName); if (found != map.end()) { - collectionName = found->second; + // Old API, could only convey exactly one Shard. + TRI_ASSERT(found->second.size() == 1); + collectionName = found->second.front(); } } try { - transaction::AllowImplicitCollectionsSwitcher disallower(_trx->state()->options(), _allowImplicitCollections); + transaction::AllowImplicitCollectionsSwitcher disallower(_trx->state()->options(), + _allowImplicitCollections); + + Result res = _trx->documentFastPathLocal(collectionName, id.substr(pos + 1), + [&](LocalDocumentId const&, VPackSlice doc) { + ++_insertedDocuments; + // copying... + result.add(doc); + return true; + }); - Result res = _trx->documentFastPathLocal(collectionName, id.substr(pos + 1), [&](LocalDocumentId const&, VPackSlice doc) { - ++_insertedDocuments; - // copying... - result.add(doc); - return true; - }); - if (res.ok()) { return true; } @@ -165,13 +169,13 @@ bool TraverserCache::appendVertex(arangodb::velocypack::StringRef id, arangodb:: } } catch (basics::Exception const& ex) { if (isWithClauseMissing(ex)) { - // turn the error into a different error + // turn the error into a different error THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_COLLECTION_LOCK_FAILED, - "collection not known to traversal: '" + - collectionName + "'. please add 'WITH " + collectionName + - "' as the first line in your AQL"); + "collection not known to traversal: '" + collectionName + + "'. please add 'WITH " + collectionName + + "' as the first line in your AQL"); } - // rethrow original error + // rethrow original error throw; } @@ -185,7 +189,8 @@ bool TraverserCache::appendVertex(arangodb::velocypack::StringRef id, arangodb:: return false; } -bool TraverserCache::appendVertex(arangodb::velocypack::StringRef id, arangodb::aql::AqlValue& result) { +bool TraverserCache::appendVertex(arangodb::velocypack::StringRef id, + arangodb::aql::AqlValue& result) { result = arangodb::aql::AqlValue(arangodb::aql::AqlValueHintNull()); if (!_baseOptions->produceVertices()) { @@ -207,20 +212,25 @@ bool TraverserCache::appendVertex(arangodb::velocypack::StringRef id, arangodb:: if (!map.empty()) { auto found = map.find(collectionName); if (found != map.end()) { - collectionName = found->second; + // Old API, could only convey exactly one Shard. + TRI_ASSERT(found->second.size() == 1); + collectionName = found->second.front(); } } try { - transaction::AllowImplicitCollectionsSwitcher disallower(_trx->state()->options(), _allowImplicitCollections); + transaction::AllowImplicitCollectionsSwitcher disallower(_trx->state()->options(), + _allowImplicitCollections); + + Result res = + _trx->documentFastPathLocal(collectionName, id.substr(pos + 1), + [&](LocalDocumentId const&, VPackSlice doc) { + ++_insertedDocuments; + // copying... + result = arangodb::aql::AqlValue(doc); + return true; + }); - Result res = _trx->documentFastPathLocal(collectionName, id.substr(pos + 1), [&](LocalDocumentId const&, VPackSlice doc) { - ++_insertedDocuments; - // copying... - result = arangodb::aql::AqlValue(doc); - return true; - }); - if (res.ok()) { return true; } @@ -231,11 +241,11 @@ bool TraverserCache::appendVertex(arangodb::velocypack::StringRef id, arangodb:: } } catch (basics::Exception const& ex) { if (isWithClauseMissing(ex)) { - // turn the error into a different error + // turn the error into a different error THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_COLLECTION_LOCK_FAILED, - "collection not known to traversal: '" + - collectionName + "'. please add 'WITH " + collectionName + - "' as the first line in your AQL"); + "collection not known to traversal: '" + collectionName + + "'. please add 'WITH " + collectionName + + "' as the first line in your AQL"); } // rethrow original error throw; @@ -262,7 +272,9 @@ aql::AqlValue TraverserCache::fetchEdgeAqlResult(EdgeDocumentToken const& idToke } arangodb::velocypack::StringRef TraverserCache::persistString(arangodb::velocypack::StringRef idString) { - return persistString(arangodb::velocypack::HashedStringRef(idString.data(), static_cast(idString.size()))).stringRef(); + return persistString(arangodb::velocypack::HashedStringRef( + idString.data(), static_cast(idString.size()))) + .stringRef(); } arangodb::velocypack::HashedStringRef TraverserCache::persistString(arangodb::velocypack::HashedStringRef idString) { @@ -273,9 +285,9 @@ arangodb::velocypack::HashedStringRef TraverserCache::persistString(arangodb::ve auto res = _stringHeap.registerString(idString); { ResourceUsageScope guard(_query.resourceMonitor(), ::costPerPersistedString); - + _persistedStrings.emplace(res); - + // now make the TraverserCache responsible for memory tracking guard.steal(); } diff --git a/arangod/Graph/Types/ValidationResult.cpp b/arangod/Graph/Types/ValidationResult.cpp index ca269aaf610c..a23fe21fb18c 100644 --- a/arangod/Graph/Types/ValidationResult.cpp +++ b/arangod/Graph/Types/ValidationResult.cpp @@ -35,6 +35,8 @@ bool ValidationResult::isFiltered() const noexcept { return _type == Type::FILTER; } +void ValidationResult::combine(Type t) noexcept { _type = std::max(_type, t); } + std::ostream& arangodb::graph::operator<<(std::ostream& stream, ValidationResult const& res) { switch (res._type) { case ValidationResult::Type::TAKE: diff --git a/arangod/Graph/Types/ValidationResult.h b/arangod/Graph/Types/ValidationResult.h index 09ebef670492..41695b1ddc9c 100644 --- a/arangod/Graph/Types/ValidationResult.h +++ b/arangod/Graph/Types/ValidationResult.h @@ -39,6 +39,8 @@ class ValidationResult { bool isPruned() const noexcept; bool isFiltered() const noexcept; + void combine(Type t) noexcept; + private: Type _type; }; diff --git a/arangod/Graph/algorithm-aliases.h b/arangod/Graph/algorithm-aliases.h index 2faa74fcdf19..b76872a64475 100644 --- a/arangod/Graph/algorithm-aliases.h +++ b/arangod/Graph/algorithm-aliases.h @@ -28,7 +28,10 @@ #include "Graph/Queues/FifoQueue.h" #include "Graph/Queues/LifoQueue.h" +#include "Graph/Queues/QueueTracer.h" +#include "Graph/PathManagement/PathStore.h" +#include "Graph/PathManagement/PathStoreTracer.h" #include "Graph/PathManagement/PathValidator.h" #include "Graph/Providers/ProviderTracer.h" #include "Graph/Types/UniquenessLevel.h" @@ -40,54 +43,60 @@ namespace graph { template using KPathEnumerator = TwoSidedEnumerator, PathStore, Provider, - PathValidator, VertexUniquenessLevel::PATH>>; + PathValidator, VertexUniquenessLevel::PATH>>; // K_PATH implementation using Tracing template using TracedKPathEnumerator = TwoSidedEnumerator>, PathStoreTracer>, ProviderTracer, - PathValidator>, VertexUniquenessLevel::PATH>>; + PathValidator, PathStoreTracer>, VertexUniquenessLevel::PATH>>; + +template +struct BFSConfiguration { + using Provider = + typename std::conditional, ProviderType>::type; + using Step = typename Provider::Step; + using Queue = + typename std::conditional>, FifoQueue>::type; + using Store = + typename std::conditional>, PathStore>::type; + using Validator = PathValidator; +}; + +template +struct DFSConfiguration { + using Provider = + typename std::conditional, ProviderType>::type; + using Step = typename Provider::Step; + using Queue = + typename std::conditional>, LifoQueue>::type; + using Store = + typename std::conditional>, PathStore>::type; + using Validator = PathValidator; +}; // BFS Traversal Enumerator implementation -template +template using BFSEnumerator = - OneSidedEnumerator, PathStore, Provider, - PathValidator, VertexUniquenessLevel::PATH>>; + OneSidedEnumerator>; // BFS Traversal Enumerator implementation using Tracing -template +template using TracedBFSEnumerator = - OneSidedEnumerator>, - PathStoreTracer>, ProviderTracer, - PathValidator>, VertexUniquenessLevel::PATH>>; + OneSidedEnumerator>; // BFS Traversal Enumerator implementation using Tracing (without provider tracing) -template -using TracedBFSEnumeratorWOPT = - OneSidedEnumerator>, - PathStoreTracer>, Provider, - PathValidator>, VertexUniquenessLevel::PATH>>; // DFS Traversal Enumerator implementation -template +template using DFSEnumerator = - OneSidedEnumerator, PathStore, Provider, - PathValidator, VertexUniquenessLevel::PATH>>; + OneSidedEnumerator>; // DFS Traversal Enumerator implementation using Tracing -template +template using TracedDFSEnumerator = - OneSidedEnumerator>, - PathStoreTracer>, ProviderTracer, - PathValidator>, VertexUniquenessLevel::PATH>>; - -// DFS Traversal Enumerator implementation using Tracing -template -using TracedDFSEnumeratorWOPT = - OneSidedEnumerator>, - PathStoreTracer>, Provider, - PathValidator>, VertexUniquenessLevel::PATH>>; + OneSidedEnumerator>; } // namespace graph } // namespace arangodb diff --git a/arangod/InternalRestHandler/InternalRestTraverserHandler.cpp b/arangod/InternalRestHandler/InternalRestTraverserHandler.cpp index 15268b542be9..753cc29d9ea2 100644 --- a/arangod/InternalRestHandler/InternalRestTraverserHandler.cpp +++ b/arangod/InternalRestHandler/InternalRestTraverserHandler.cpp @@ -232,7 +232,7 @@ void InternalRestTraverserHandler::queryEngine() { // Safe cast BaseTraverserEngines are all of type TRAVERSER auto eng = static_cast(engine); TRI_ASSERT(eng != nullptr); - eng->smartSearch(body, result); + eng->smartSearchNew(body, result); } else if (option == "smartSearchBFS") { if (engine->getType() != BaseEngine::EngineType::TRAVERSER) { generateError(ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER, diff --git a/arangod/Transaction/Methods.h b/arangod/Transaction/Methods.h index 69cc9c45c546..7fc1a6c001e3 100644 --- a/arangod/Transaction/Methods.h +++ b/arangod/Transaction/Methods.h @@ -483,7 +483,7 @@ class Methods { /// @brief the transaction context std::shared_ptr _transactionContext; - + bool _mainTransaction; private: diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 0ec13848ceda..01b39e9a861d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -62,6 +62,8 @@ set(ARANGODB_TESTS_COMMON_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Mocks/PreparedResponseConnectionPool.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Mocks/StorageEngineMock.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Mocks/Servers.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Mocks/MockGraph.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Mocks/MockGraphProvider.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Basics/icu-helper.cpp ${CMAKE_CURRENT_SOURCE_DIR}/IResearch/AgencyMock.cpp ${CMAKE_CURRENT_SOURCE_DIR}/IResearch/common.cpp @@ -229,8 +231,6 @@ set(ARANGODB_TESTS_SOURCES Graph/KPathFinderTest.cpp Graph/DFSFinderTest.cpp Graph/KShortestPathsFinder.cpp - Graph/MockGraph.cpp - Graph/MockGraphProvider.cpp Graph/PathStoreTest.cpp Graph/PathValidatorTest.cpp Graph/FifoQueueTest.cpp @@ -325,6 +325,7 @@ endif() target_include_directories(arangodbtests PRIVATE ${INCLUDE_DIRECTORIES} + ${CMAKE_CURRENT_SOURCE_DIR}/Mocks/ ) if(MSVC AND USE_FAIL_ON_WARNINGS) diff --git a/tests/Graph/DFSFinderTest.cpp b/tests/Graph/DFSFinderTest.cpp index fd1148652c4d..c3ad3e87a23c 100644 --- a/tests/Graph/DFSFinderTest.cpp +++ b/tests/Graph/DFSFinderTest.cpp @@ -57,8 +57,8 @@ namespace tests { namespace graph { class DFSFinderTest : public ::testing::TestWithParam { - // using DFSFinder = DFSEnumerator; - using DFSFinder = TracedDFSEnumeratorWOPT; + // using DFSFinder = DFSEnumerator; + using DFSFinder = TracedDFSEnumerator; protected: bool activateLogging{false}; @@ -68,11 +68,23 @@ class DFSFinderTest : public ::testing::TestWithParamnewTrxContext()}; + aql::Variable _tmpVar{"tmp", 0, false}; + arangodb::aql::AqlFunctionsInternalCache _functionsCache{}; + arangodb::aql::FixedVarExpressionContext _expressionContext{_trx, *_query.get(), + _functionsCache}; + DFSFinderTest() { if (activateLogging) { Logger::GRAPHS.setLogLevel(LogLevel::TRACE); } + // Important Note: + // Tests are using a LifoQueue. In those tests we do guarantee fetching in order + // e.g. (1) expands to (2), (3), (4) + // we will first traverse (4), then (3), then (2) + /* a chain 1->2->3->4 */ mockGraph.addEdge(1, 2); mockGraph.addEdge(2, 3); @@ -158,8 +170,11 @@ class DFSFinderTest : public ::testing::TestWithParam DFSFinder { arangodb::graph::OneSidedEnumeratorOptions options{minDepth, maxDepth}; - return DFSFinder{MockGraphProvider(mockGraph, *_query.get(), looseEndBehaviour(), false), - std::move(options), resourceMonitor}; + PathValidatorOptions validatorOpts{&_tmpVar, _expressionContext}; + return DFSFinder({*_query.get(), + MockGraphProviderOptions{mockGraph, looseEndBehaviour(), false}, + resourceMonitor}, + std::move(options), std::move(validatorOpts), resourceMonitor); } auto vId(size_t nr) -> std::string { @@ -248,8 +263,9 @@ TEST_P(DFSFinderTest, no_path_exists) { EXPECT_FALSE(finder.isDone()); { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); pathEquals(result.slice(), {91}); pathStructureValid(result.slice(), 0); EXPECT_TRUE(finder.isDone()); @@ -258,7 +274,7 @@ TEST_P(DFSFinderTest, no_path_exists) { { result.clear(); // Try again to make sure we stay at non-existing - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); EXPECT_TRUE(result.isEmpty()); EXPECT_TRUE(finder.isDone()); @@ -282,8 +298,10 @@ TEST_P(DFSFinderTest, path_depth_0) { EXPECT_FALSE(finder.isDone()); { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathEquals(result.slice(), {1}); pathStructureValid(result.slice(), 0); EXPECT_TRUE(finder.isDone()); @@ -292,7 +310,7 @@ TEST_P(DFSFinderTest, path_depth_0) { { result.clear(); // Try again to make sure we stay at non-existing - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); EXPECT_TRUE(result.isEmpty()); EXPECT_TRUE(finder.isDone()); @@ -315,8 +333,9 @@ TEST_P(DFSFinderTest, path_depth_1) { EXPECT_FALSE(finder.isDone()); { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); pathStructureValid(result.slice(), 1); pathEquals(result.slice(), {1, 2}); @@ -327,7 +346,7 @@ TEST_P(DFSFinderTest, path_depth_1) { { result.clear(); // Try again to make sure we stay at non-existing - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); EXPECT_TRUE(result.isEmpty()); EXPECT_TRUE(finder.isDone()); @@ -351,8 +370,10 @@ TEST_P(DFSFinderTest, path_depth_2) { EXPECT_FALSE(finder.isDone()); { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 2); pathEquals(result.slice(), {1, 2, 3}); @@ -362,7 +383,7 @@ TEST_P(DFSFinderTest, path_depth_2) { { result.clear(); // Try again to make sure we stay at non-existing - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); EXPECT_TRUE(result.isEmpty()); EXPECT_TRUE(finder.isDone()); @@ -385,8 +406,10 @@ TEST_P(DFSFinderTest, path_depth_3) { EXPECT_FALSE(finder.isDone()); { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 3); pathEquals(result.slice(), {1, 2, 3, 4}); @@ -396,7 +419,7 @@ TEST_P(DFSFinderTest, path_depth_3) { { result.clear(); // Try again to make sure we stay at non-existing - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); EXPECT_TRUE(result.isEmpty()); EXPECT_TRUE(finder.isDone()); @@ -420,24 +443,30 @@ TEST_P(DFSFinderTest, path_diamond) { EXPECT_FALSE(finder.isDone()); { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 2); EXPECT_FALSE(finder.isDone()); } { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 2); EXPECT_FALSE(finder.isDone()); } { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 2); EXPECT_TRUE(finder.isDone()); @@ -446,7 +475,7 @@ TEST_P(DFSFinderTest, path_diamond) { { result.clear(); // Try again to make sure we stay at non-existing - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); EXPECT_TRUE(result.isEmpty()); EXPECT_TRUE(finder.isDone()); @@ -469,8 +498,10 @@ TEST_P(DFSFinderTest, path_depth_1_to_2) { EXPECT_FALSE(finder.isDone()); { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 1); pathEquals(result.slice(), {10, 12}); @@ -479,8 +510,10 @@ TEST_P(DFSFinderTest, path_depth_1_to_2) { { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 2); pathEquals(result.slice(), {10, 12, 13}); EXPECT_FALSE(finder.isDone()); @@ -488,8 +521,10 @@ TEST_P(DFSFinderTest, path_depth_1_to_2) { { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 2); pathEquals(result.slice(), {10, 12, 11}); EXPECT_FALSE(finder.isDone()); @@ -497,8 +532,10 @@ TEST_P(DFSFinderTest, path_depth_1_to_2) { { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 1); pathEquals(result.slice(), {10, 11}); EXPECT_TRUE(finder.isDone()); @@ -507,7 +544,7 @@ TEST_P(DFSFinderTest, path_depth_1_to_2) { { result.clear(); // Try again to make sure we stay at non-existing - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); EXPECT_TRUE(result.isEmpty()); EXPECT_TRUE(finder.isDone()); @@ -524,8 +561,10 @@ TEST_P(DFSFinderTest, path_depth_1_to_2_skip) { EXPECT_FALSE(finder.isDone()); { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 1); pathEquals(result.slice(), {10, 12}); @@ -541,8 +580,10 @@ TEST_P(DFSFinderTest, path_depth_1_to_2_skip) { { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 2); pathEquals(result.slice(), {10, 12, 11}); EXPECT_FALSE(finder.isDone()); @@ -550,8 +591,10 @@ TEST_P(DFSFinderTest, path_depth_1_to_2_skip) { { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 1); pathEquals(result.slice(), {10, 11}); EXPECT_TRUE(finder.isDone()); @@ -560,7 +603,177 @@ TEST_P(DFSFinderTest, path_depth_1_to_2_skip) { { result.clear(); // Try again to make sure we stay at non-existing - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); + EXPECT_FALSE(hasPath); + EXPECT_TRUE(result.isEmpty()); + EXPECT_TRUE(finder.isDone()); + } +} + +TEST_P(DFSFinderTest, path_loop) { + VPackBuilder result; + auto finder = pathFinder(1, 10); + + // Source and target are direct neighbors, there is only one path between them + auto source = vId(20); + + finder.reset(toHashedStringRef(source)); + + EXPECT_FALSE(finder.isDone()); + { + result.clear(); + auto hasPath = finder.getNextPath(); + EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + + pathStructureValid(result.slice(), 1); + pathEquals(result.slice(), {20, 21}); + + EXPECT_FALSE(finder.isDone()); + } + + { + result.clear(); + auto hasPath = finder.getNextPath(); + EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + + pathStructureValid(result.slice(), 2); + pathEquals(result.slice(), {20, 21, 22}); + + EXPECT_FALSE(finder.isDone()); + } + + { + result.clear(); + auto hasPath = finder.getNextPath(); + EXPECT_FALSE(hasPath); + EXPECT_TRUE(result.isEmpty()); + EXPECT_TRUE(finder.isDone()); + } +} + +TEST_P(DFSFinderTest, triangle_loop) { + VPackBuilder result; + auto finder = pathFinder(1, 10); + auto source = vId(30); + + finder.reset(toHashedStringRef(source)); + + EXPECT_FALSE(finder.isDone()); + { + result.clear(); + auto hasPath = finder.getNextPath(); + EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + + pathStructureValid(result.slice(), 1); + pathEquals(result.slice(), {30, 31}); + + EXPECT_FALSE(finder.isDone()); + } + + { + result.clear(); + auto hasPath = finder.getNextPath(); + EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + + pathStructureValid(result.slice(), 2); + pathEquals(result.slice(), {30, 31, 32}); + + EXPECT_FALSE(finder.isDone()); + } + + { + result.clear(); + auto hasPath = finder.getNextPath(); + EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + + pathStructureValid(result.slice(), 3); + pathEquals(result.slice(), {30, 31, 32, 34}); + + EXPECT_FALSE(finder.isDone()); + } + + { + result.clear(); + auto hasPath = finder.getNextPath(); + EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + + pathStructureValid(result.slice(), 3); + pathEquals(result.slice(), {30, 31, 32, 33}); + + EXPECT_FALSE(finder.isDone()); + } + + { + result.clear(); + // Try again to make sure we stay at non-existing + auto hasPath = finder.getNextPath(); + EXPECT_FALSE(hasPath); + EXPECT_TRUE(result.isEmpty()); + EXPECT_TRUE(finder.isDone()); + } +} + +TEST_P(DFSFinderTest, triangle_loop_skip) { + VPackBuilder result; + auto finder = pathFinder(1, 10); + auto source = vId(30); + + finder.reset(toHashedStringRef(source)); + + EXPECT_FALSE(finder.isDone()); + { + result.clear(); + auto hasPath = finder.getNextPath(); + EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + + pathStructureValid(result.slice(), 1); + pathEquals(result.slice(), {30, 31}); + + EXPECT_FALSE(finder.isDone()); + } + + { + result.clear(); + auto hasPath = finder.getNextPath(); + EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + + pathStructureValid(result.slice(), 2); + pathEquals(result.slice(), {30, 31, 32}); + + EXPECT_FALSE(finder.isDone()); + } + + { + result.clear(); + bool skipped = finder.skipPath(); + EXPECT_TRUE(skipped); + EXPECT_FALSE(finder.isDone()); + } + + { + result.clear(); + auto hasPath = finder.getNextPath(); + EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + + pathStructureValid(result.slice(), 3); + pathEquals(result.slice(), {30, 31, 32, 33}); + + EXPECT_FALSE(finder.isDone()); + } + + { + result.clear(); + // Try again to make sure we stay at non-existing + auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); EXPECT_TRUE(result.isEmpty()); EXPECT_TRUE(finder.isDone()); @@ -569,9 +782,6 @@ TEST_P(DFSFinderTest, path_depth_1_to_2_skip) { /* TODO: Add more tests * - path_depth_2_to_3 - * - path_loop - * - triangle_loop - * - triangle_loop_skip * - many_neighbours_source * - many_neighbours_target */ diff --git a/tests/Graph/FifoQueueTest.cpp b/tests/Graph/FifoQueueTest.cpp index 0eab67b09c03..5379d2c1d4e5 100644 --- a/tests/Graph/FifoQueueTest.cpp +++ b/tests/Graph/FifoQueueTest.cpp @@ -24,6 +24,7 @@ #include "Basics/GlobalResourceMonitor.h" #include "Basics/ResourceUsage.h" #include "Basics/StringUtils.h" +#include "Basics/ResultT.h" #include "Graph/Providers/BaseStep.h" #include "Graph/Queues/FifoQueue.h" diff --git a/tests/Graph/GenericGraphProviderTest.cpp b/tests/Graph/GenericGraphProviderTest.cpp index bb6e939ce3b9..8a00d3e64e43 100644 --- a/tests/Graph/GenericGraphProviderTest.cpp +++ b/tests/Graph/GenericGraphProviderTest.cpp @@ -64,11 +64,14 @@ class GraphProviderTest : public ::testing::Test { std::unique_ptr server{nullptr}; std::unique_ptr query{nullptr}; std::unique_ptr> clusterEngines{nullptr}; + std::unique_ptr _trx{}; arangodb::GlobalResourceMonitor global{}; arangodb::ResourceMonitor resourceMonitor{global}; + arangodb::aql::AqlFunctionsInternalCache _functionsCache{}; + std::unique_ptr _expressionContext; - std::map _emptyShardMap{}; + std::unordered_map> _emptyShardMap{}; GraphProviderTest() {} ~GraphProviderTest() {} @@ -86,8 +89,9 @@ class GraphProviderTest : public ::testing::Test { // We now have collections "v" and "e" query = singleServer->getQuery("RETURN 1", {"v", "e"}); - return MockGraphProvider(graph, *query.get(), - MockGraphProvider::LooseEndBehaviour::NEVER); + return MockGraphProvider(*query.get(), + MockGraphProviderOptions{graph, MockGraphProvider::LooseEndBehaviour::NEVER}, + resourceMonitor); } if constexpr (std::is_same_v) { s = std::make_unique(); @@ -97,15 +101,24 @@ class GraphProviderTest : public ::testing::Test { // We now have collections "v" and "e" query = singleServer->getQuery("RETURN 1", {"v", "e"}); + _trx = std::make_unique(query->newTrxContext()); auto edgeIndexHandle = singleServer->getEdgeIndexHandle("e"); auto tmpVar = singleServer->generateTempVar(query.get()); auto indexCondition = singleServer->buildOutboundCondition(query.get(), tmpVar); - std::vector usedIndexes{ - IndexAccessor{edgeIndexHandle, indexCondition, 0}}; + std::vector usedIndexes{}; + usedIndexes.emplace_back(IndexAccessor{edgeIndexHandle, indexCondition, 0, nullptr}); - BaseProviderOptions opts(tmpVar, std::move(usedIndexes), _emptyShardMap); + _expressionContext = + std::make_unique(*_trx.get(), *query, + _functionsCache); + + BaseProviderOptions opts( + tmpVar, + std::make_pair(std::move(usedIndexes), + std::unordered_map>{}), + *_expressionContext.get(), _emptyShardMap); return SingleServerProvider(*query.get(), std::move(opts), resourceMonitor); } if constexpr (std::is_same_v) { @@ -122,10 +135,9 @@ class GraphProviderTest : public ::testing::Test { server.getSystemDatabase()); arangodb::aql::Query fakeQuery(ctx, queryString, nullptr); try { - fakeQuery.collections().add("s9880", AccessMode::Type::READ, - arangodb::aql::Collection::Hint::Shard); - } catch(...) { - + fakeQuery.collections().add("s9880", AccessMode::Type::READ, + arangodb::aql::Collection::Hint::Shard); + } catch (...) { } fakeQuery.prepareQuery(SerializationFormat::SHADOWROWS); auto ast = fakeQuery.ast(); @@ -137,10 +149,10 @@ class GraphProviderTest : public ::testing::Test { opts.setVariable(tmpVar); auto const* access = - ast->createNodeAttributeAccess(tmpVarRef, - StaticStrings::FromString.c_str(), - StaticStrings::FromString.length()); - auto const* cond = ast->createNodeBinaryOperator(NODE_TYPE_OPERATOR_BINARY_EQ, access, tmpIdNode); + ast->createNodeAttributeAccess(tmpVarRef, StaticStrings::FromString.c_str(), + StaticStrings::FromString.length()); + auto const* cond = + ast->createNodeBinaryOperator(NODE_TYPE_OPERATOR_BINARY_EQ, access, tmpIdNode); auto fromCondition = ast->createNodeNaryOperator(NODE_TYPE_OPERATOR_NARY_AND); fromCondition->addMember(cond); opts.addLookupInfo(fakeQuery.plan(), "s9880", StaticStrings::FromString, fromCondition); @@ -158,9 +170,13 @@ class GraphProviderTest : public ::testing::Test { std::tie(preparedResponses, engineId) = graph.simulateApi(server, expectedVerticesEdgesBundleToFetch, opts); // Note: Please don't remove for debugging purpose. - /*for (auto const& resp : preparedResponses) { - LOG_DEVEL << resp.generateResponse()->copyPayload().get()->toString(); - }*/ +#if 0 + for (auto const& resp : preparedResponses) { + auto payload = resp.generateResponse()->copyPayload(); + VPackSlice husti(payload->data()); + LOG_DEVEL << husti.toJson(); + } +#endif } server = std::make_unique(true, false); @@ -189,8 +205,7 @@ class GraphProviderTest : public ::testing::Test { clusterEngines = std::make_unique>(); clusterEngines->emplace("PRMR_0001", engineId); - auto clusterCache = - std::make_shared(resourceMonitor); + auto clusterCache = std::make_shared(resourceMonitor); ClusterBaseProviderOptions opts(clusterCache, clusterEngines.get(), false); return ClusterProvider(*query.get(), std::move(opts), resourceMonitor); @@ -299,8 +314,7 @@ TYPED_TEST(GraphProviderTest, should_enumerate_all_edges) { std::unordered_set found{}; std::unordered_map>> const& expectedVerticesEdgesBundleToFetch = { - {0, {}} - }; + {0, {}}}; auto testee = this->makeProvider(g, expectedVerticesEdgesBundleToFetch); std::string startString = g.vertexToId(0); VPackHashedStringRef startH{startString.c_str(), diff --git a/tests/Graph/GraphMockProviderInstances.cpp b/tests/Graph/GraphMockProviderInstances.cpp index 617b3cd7163b..fbb6e3a8a9a4 100644 --- a/tests/Graph/GraphMockProviderInstances.cpp +++ b/tests/Graph/GraphMockProviderInstances.cpp @@ -25,47 +25,62 @@ #include "Graph/Enumerators/OneSidedEnumerator.cpp" #include "Graph/Enumerators/TwoSidedEnumerator.cpp" #include "Graph/PathManagement/PathResult.cpp" -#include "Graph/PathManagement/SingleProviderPathResult.cpp" #include "Graph/PathManagement/PathStore.cpp" +#include "Graph/PathManagement/SingleProviderPathResult.cpp" #include "Graph/Queues/FifoQueue.h" #include "Graph/Queues/LifoQueue.h" #include "Graph/PathManagement/PathStoreTracer.cpp" #include "Graph/PathManagement/PathStoreTracer.h" #include "Graph/PathManagement/PathValidator.cpp" +#include "Graph/Providers/ProviderTracer.cpp" #include "Graph/Queues/QueueTracer.cpp" template class ::arangodb::graph::PathResult<::arangodb::tests::graph::MockGraphProvider, ::arangodb::tests::graph::MockGraphProvider::Step>; template class ::arangodb::graph::SingleProviderPathResult< - ::arangodb::tests::graph::MockGraphProvider, ::arangodb::tests::graph::MockGraphProvider::Step>; + ::arangodb::tests::graph::MockGraphProvider, + ::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, + ::arangodb::tests::graph::MockGraphProvider::Step>; + +template class ::arangodb::graph::SingleProviderPathResult< + ::arangodb::tests::graph::MockGraphProvider, + ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>>, + ::arangodb::tests::graph::MockGraphProvider::Step>; + +template class ::arangodb::graph::ProviderTracer<::arangodb::tests::graph::MockGraphProvider>; template class ::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>; template class ::arangodb::graph::PathValidator< + ::arangodb::tests::graph::MockGraphProvider, ::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, VertexUniquenessLevel::PATH>; template class ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>>; template class ::arangodb::graph::PathValidator< + ::arangodb::tests::graph::MockGraphProvider, ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>>, VertexUniquenessLevel::PATH>; template class ::arangodb::graph::TwoSidedEnumerator< ::arangodb::graph::FifoQueue<::arangodb::tests::graph::MockGraphProvider::Step>, ::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, ::arangodb::tests::graph::MockGraphProvider, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, VertexUniquenessLevel::PATH>>; + ::arangodb::graph::PathValidator<::arangodb::tests::graph::MockGraphProvider, + ::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, VertexUniquenessLevel::PATH>>; -template class ::arangodb::graph::OneSidedEnumerator< - ::arangodb::graph::LifoQueue<::arangodb::tests::graph::MockGraphProvider::Step>, - ::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, ::arangodb::tests::graph::MockGraphProvider, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, VertexUniquenessLevel::PATH>>; +// BFS with PATH uniquness +template class ::arangodb::graph::OneSidedEnumerator<::arangodb::graph::BFSConfiguration< + ::arangodb::tests::graph::MockGraphProvider, ::arangodb::graph::VertexUniquenessLevel::PATH, false>>; +template class ::arangodb::graph::OneSidedEnumerator<::arangodb::graph::BFSConfiguration< + ::arangodb::tests::graph::MockGraphProvider, ::arangodb::graph::VertexUniquenessLevel::PATH, true>>; -template class ::arangodb::graph::OneSidedEnumerator< - ::arangodb::graph::FifoQueue<::arangodb::tests::graph::MockGraphProvider::Step>, - ::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, ::arangodb::tests::graph::MockGraphProvider, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, VertexUniquenessLevel::PATH>>; +// DFS with PATH uniquness +template class ::arangodb::graph::OneSidedEnumerator<::arangodb::graph::DFSConfiguration< + ::arangodb::tests::graph::MockGraphProvider, ::arangodb::graph::VertexUniquenessLevel::PATH, false>>; +template class ::arangodb::graph::OneSidedEnumerator<::arangodb::graph::DFSConfiguration< + ::arangodb::tests::graph::MockGraphProvider, ::arangodb::graph::VertexUniquenessLevel::PATH, true>>; template class ::arangodb::graph::QueueTracer<::arangodb::graph::FifoQueue<::arangodb::tests::graph::MockGraphProvider::Step>>; @@ -74,35 +89,13 @@ template class ::arangodb::graph::QueueTracer<::arangodb::graph::LifoQueue<::ara template class ::arangodb::graph::TwoSidedEnumerator< ::arangodb::graph::QueueTracer<::arangodb::graph::FifoQueue<::arangodb::tests::graph::MockGraphProvider::Step>>, ::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, ::arangodb::tests::graph::MockGraphProvider, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, VertexUniquenessLevel::PATH>>; - -template class ::arangodb::graph::OneSidedEnumerator< - ::arangodb::graph::QueueTracer<::arangodb::graph::LifoQueue<::arangodb::tests::graph::MockGraphProvider::Step>>, - ::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, ::arangodb::tests::graph::MockGraphProvider, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, VertexUniquenessLevel::PATH>>; - -template class ::arangodb::graph::OneSidedEnumerator< - ::arangodb::graph::QueueTracer<::arangodb::graph::FifoQueue<::arangodb::tests::graph::MockGraphProvider::Step>>, - ::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, ::arangodb::tests::graph::MockGraphProvider, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, VertexUniquenessLevel::PATH>>; + ::arangodb::graph::PathValidator<::arangodb::tests::graph::MockGraphProvider, + ::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, VertexUniquenessLevel::PATH>>; template class ::arangodb::graph::TwoSidedEnumerator< ::arangodb::graph::QueueTracer<::arangodb::graph::FifoQueue<::arangodb::tests::graph::MockGraphProvider::Step>>, ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>>, ::arangodb::tests::graph::MockGraphProvider, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>>, + ::arangodb::graph::PathValidator<::arangodb::tests::graph::MockGraphProvider, + ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>>, VertexUniquenessLevel::PATH>>; - -template class ::arangodb::graph::OneSidedEnumerator< - ::arangodb::graph::QueueTracer<::arangodb::graph::LifoQueue<::arangodb::tests::graph::MockGraphProvider::Step>>, - ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>>, - ::arangodb::tests::graph::MockGraphProvider, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>>, - VertexUniquenessLevel::PATH>>; - -template class ::arangodb::graph::OneSidedEnumerator< - ::arangodb::graph::QueueTracer<::arangodb::graph::FifoQueue<::arangodb::tests::graph::MockGraphProvider::Step>>, - ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>>, - ::arangodb::tests::graph::MockGraphProvider, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>>, - VertexUniquenessLevel::PATH>>; \ No newline at end of file diff --git a/tests/Graph/GraphTestTools.h b/tests/Graph/GraphTestTools.h index fe1d517cc588..9a6ea406e2c9 100644 --- a/tests/Graph/GraphTestTools.h +++ b/tests/Graph/GraphTestTools.h @@ -83,6 +83,7 @@ struct GraphTestSetup // setup required application features features.emplace_back(server.addFeature(), false); features.emplace_back(server.addFeature(), false); + features.emplace_back(server.addFeature(), false); features.emplace_back(server.addFeature(), false); features.emplace_back(server.addFeature(), false); server.getFeature().setEngineTesting(&engine); diff --git a/tests/Graph/KPathFinderTest.cpp b/tests/Graph/KPathFinderTest.cpp index 68c92ddd7936..9340096c6704 100644 --- a/tests/Graph/KPathFinderTest.cpp +++ b/tests/Graph/KPathFinderTest.cpp @@ -69,6 +69,13 @@ class KPathFinderTest arangodb::GlobalResourceMonitor global{}; arangodb::ResourceMonitor resourceMonitor{global}; + // PathValidatorOptions parts (used for API not under test here) + aql::Variable _tmpVar{"tmp", 0, false}; + arangodb::aql::AqlFunctionsInternalCache _functionsCache{}; + + arangodb::transaction::Methods _trx{_query->newTrxContext()}; + arangodb::aql::FixedVarExpressionContext _expressionContext{_trx, *_query.get(), + _functionsCache}; KPathFinderTest() { if (activateLogging) { Logger::GRAPHS.setLogLevel(LogLevel::TRACE); @@ -159,9 +166,15 @@ class KPathFinderTest auto pathFinder(size_t minDepth, size_t maxDepth) -> KPathFinder { arangodb::graph::TwoSidedEnumeratorOptions options{minDepth, maxDepth}; - return KPathFinder{MockGraphProvider(mockGraph, *_query.get(), looseEndBehaviour(), false), - MockGraphProvider(mockGraph, *_query.get(), looseEndBehaviour(), true), - std::move(options), resourceMonitor}; + PathValidatorOptions validatorOpts{&_tmpVar, _expressionContext}; + return KPathFinder{ + MockGraphProvider(*_query.get(), + MockGraphProviderOptions{mockGraph, looseEndBehaviour(), false}, + resourceMonitor), + MockGraphProvider(*_query.get(), + MockGraphProviderOptions{mockGraph, looseEndBehaviour(), true}, + resourceMonitor), + std::move(options), std::move(validatorOpts), resourceMonitor}; } auto vId(size_t nr) -> std::string { diff --git a/tests/Graph/LifoQueueTest.cpp b/tests/Graph/LifoQueueTest.cpp index 33b116e952ab..7eb89ec19b31 100644 --- a/tests/Graph/LifoQueueTest.cpp +++ b/tests/Graph/LifoQueueTest.cpp @@ -24,6 +24,7 @@ #include "Basics/GlobalResourceMonitor.h" #include "Basics/ResourceUsage.h" #include "Basics/StringUtils.h" +#include "Basics/ResultT.h" #include "Graph/Providers/BaseStep.h" #include "Graph/Queues/LifoQueue.h" diff --git a/tests/Graph/PathValidatorTest.cpp b/tests/Graph/PathValidatorTest.cpp index fd4c37fe411c..12eb80bbd837 100644 --- a/tests/Graph/PathValidatorTest.cpp +++ b/tests/Graph/PathValidatorTest.cpp @@ -22,6 +22,10 @@ #include "gtest/gtest.h" +#include "../Mocks/Servers.h" + +#include "Aql/Ast.h" +#include "Aql/Query.h" #include "Basics/GlobalResourceMonitor.h" #include "Basics/ResourceUsage.h" #include "Basics/StringHeap.h" @@ -31,62 +35,68 @@ #include "Graph/Providers/BaseStep.h" #include "Graph/Types/UniquenessLevel.h" +#include "./MockGraph.h" +#include "./MockGraphProvider.h" + using namespace arangodb; using namespace arangodb::graph; using namespace arangodb::velocypack; +namespace { +aql::AstNode* InitializeReference(aql::Ast& ast, aql::Variable& var) { + ast.scopes()->start(aql::ScopeType::AQL_SCOPE_MAIN); + ast.scopes()->addVariable(&var); + aql::AstNode* a = ast.createNodeReference("tmp"); + ast.scopes()->endCurrent(); + return a; +} +} // namespace + namespace arangodb { namespace tests { namespace graph_path_validator_test { static_assert(GTEST_HAS_TYPED_TEST, "We need typed tests for the following:"); -class Step : public arangodb::graph::BaseStep { - public: - using Vertex = HashedStringRef; - using Edge = HashedStringRef; - - HashedStringRef _id; - - Step(HashedStringRef id, size_t previous) - : arangodb::graph::BaseStep{previous}, _id{id} {}; - - ~Step() = default; - - bool operator==(Step const& other) { return _id == other._id; } - - std::string toString() const { - return " _id: " + _id.toString() + - ", _previous: " + basics::StringUtils::itoa(getPrevious()); - } - - bool isProcessable() const { return true; } - HashedStringRef getVertex() const { return _id; } - HashedStringRef getEdge() const { return _id; } +using Step = typename graph::MockGraphProvider::Step; - arangodb::velocypack::HashedStringRef getVertexIdentifier() const { - return getVertex(); - } -}; - -using TypesToTest = - ::testing::Types, VertexUniquenessLevel::NONE>, - PathValidator, VertexUniquenessLevel::PATH>, - PathValidator, VertexUniquenessLevel::GLOBAL>>; +using TypesToTest = ::testing::Types< + PathValidator, VertexUniquenessLevel::NONE>, + PathValidator, VertexUniquenessLevel::PATH>, + PathValidator, VertexUniquenessLevel::GLOBAL>>; template class PathValidatorTest : public ::testing::Test { + protected: + graph::MockGraph mockGraph; + mocks::MockAqlServer _server{true}; + + std::unique_ptr _query{_server.createFakeQuery()}; + + private: arangodb::GlobalResourceMonitor _global{}; arangodb::ResourceMonitor _resourceMonitor{_global}; + std::unique_ptr _provider; + PathStore _pathStore{_resourceMonitor}; StringHeap _heap{_resourceMonitor, 4096}; + // Expression Parts + arangodb::transaction::Methods _trx{_query->newTrxContext()}; + aql::Ast* _ast{_query->ast()}; + aql::Variable _tmpVar{"tmp", 0, false}; + aql::AstNode* _varNode{::InitializeReference(*_ast, _tmpVar)}; + + arangodb::aql::AqlFunctionsInternalCache _functionsCache{}; + arangodb::aql::FixedVarExpressionContext _expressionContext{_trx, *_query, _functionsCache}; + PathValidatorOptions _opts{&_tmpVar, _expressionContext}; + protected: VertexUniquenessLevel getVertexUniquness() { - if constexpr (std::is_same_v, VertexUniquenessLevel::NONE>>) { + if constexpr (std::is_same_v, VertexUniquenessLevel::NONE>>) { return VertexUniquenessLevel::NONE; - } else if constexpr (std::is_same_v, VertexUniquenessLevel::PATH>>) { + } else if constexpr (std::is_same_v, VertexUniquenessLevel::PATH>>) { return VertexUniquenessLevel::PATH; } else { return VertexUniquenessLevel::GLOBAL; @@ -95,61 +105,104 @@ class PathValidatorTest : public ::testing::Test { PathStore& store() { return _pathStore; } - ValidatorType testee() { return ValidatorType{this->store()}; } - Step makeStep(size_t id, size_t previous) { - std::string idStr = basics::StringUtils::itoa(id); - HashedStringRef hStr(idStr.data(), static_cast(idStr.length())); - return Step(_heap.registerString(hStr), previous); + aql::Variable const* tmpVar() { return &_tmpVar; } + + aql::Query* query() { return _query.get(); } + + ValidatorType testee() { + ensureProvider(); + return ValidatorType{*_provider.get(), this->store(), _opts}; + } + + Step startPath(size_t id) { + ensureProvider(); + auto base = mockGraph.vertexToId(id); + HashedStringRef ref{base.c_str(), static_cast(base.length())}; + auto hStr = _heap.registerString(ref); + return _provider->startVertex(hStr); + } + + // Get and modify the options used in the Validator testee(). + // Make sure to Modify them before calling testee() method. + PathValidatorOptions& options() { return _opts; } + + std::vector expandPath(Step previous) { + // We at least have called startPath before, this ensured the provider + TRI_ASSERT(_provider != nullptr); + size_t prev = _pathStore.append(previous); + std::vector result; + _provider->expand(previous, prev, + [&result](Step s) { result.emplace_back(s); }); + return result; + } + + // Add a path defined by the given vector. We start a first() and end in last() + void addPath(std::vector path) { + // This function can only add paths of length 1 or more + TRI_ASSERT(path.size() >= 2); + for (size_t i = 0; i < path.size() - 1; ++i) { + mockGraph.addEdge(path.at(i), path.at(i + 1)); + } + } + + /* + * generates a condition #TMP._key == '' + */ + std::unique_ptr conditionKeyMatches(std::string const& toMatch) { + auto expectedKey = _ast->createNodeValueString(toMatch.c_str(), toMatch.length()); + auto keyAccess = + _ast->createNodeAttributeAccess(_varNode, StaticStrings::KeyString.c_str(), + StaticStrings::KeyString.length()); + // This condition cannot be fulfilled + auto condition = _ast->createNodeBinaryOperator(aql::AstNodeType::NODE_TYPE_OPERATOR_BINARY_EQ, + keyAccess, expectedKey); + return std::make_unique(_ast, condition); + } + + private: + void ensureProvider() { + if (_provider == nullptr) { + _provider = std::make_unique( + *_query.get(), + graph::MockGraphProviderOptions{mockGraph, graph::MockGraphProviderOptions::LooseEndBehaviour::NEVER, + false}, + _resourceMonitor); + } } }; TYPED_TEST_CASE(PathValidatorTest, TypesToTest); TYPED_TEST(PathValidatorTest, it_should_honor_uniqueness_on_single_path_first_duplicate) { - auto&& ps = this->store(); + // We add a loop that ends in the start vertex (0) again. + this->addPath({0, 1, 2, 3, 0}); auto validator = this->testee(); - size_t lastIndex = std::numeric_limits::max(); + Step s = this->startPath(0); { - Step s = this->makeStep(0, lastIndex); auto res = validator.validatePath(s); // The start vertex is always valid EXPECT_FALSE(res.isFiltered()); EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 0); - } - - // We add a loop that ends in the start vertex (0) again. - { - Step s = this->makeStep(1, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 1); - } - { - Step s = this->makeStep(2, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 2); } - { - Step s = this->makeStep(3, lastIndex); + // The next 3 steps are good to take. + for (size_t i = 0; i < 3; ++i) { + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1) + << "Not enough connections after step " << s.getVertexIdentifier(); + s = neighbors.at(0); auto res = validator.validatePath(s); EXPECT_FALSE(res.isFiltered()); EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 3); } - // Add duplicate vertex on Path + // Now we move to the duplicate vertex { - Step s = this->makeStep(0, lastIndex); + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1); + s = neighbors.at(0); auto res = validator.validatePath(s); + if (this->getVertexUniquness() == VertexUniquenessLevel::NONE) { // No uniqueness check, take the vertex EXPECT_FALSE(res.isFiltered()); @@ -163,49 +216,35 @@ TYPED_TEST(PathValidatorTest, it_should_honor_uniqueness_on_single_path_first_du } TYPED_TEST(PathValidatorTest, it_should_honor_uniqueness_on_single_path_last_duplicate) { - auto&& ps = this->store(); + // We add a loop that loops on the last vertex(3). + this->addPath({0, 1, 2, 3, 3}); auto validator = this->testee(); - size_t lastIndex = std::numeric_limits::max(); + Step s = this->startPath(0); { - Step s = this->makeStep(0, lastIndex); auto res = validator.validatePath(s); // The start vertex is always valid EXPECT_FALSE(res.isFiltered()); EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 0); - } - // We add a loop on the last vertex of the path (3) - { - Step s = this->makeStep(1, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 1); - } - { - Step s = this->makeStep(2, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 2); } - { - Step s = this->makeStep(3, lastIndex); + // The next 3 steps are good to take. + for (size_t i = 0; i < 3; ++i) { + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1) + << "Not enough connections after step " << s.getVertexIdentifier(); + s = neighbors.at(0); auto res = validator.validatePath(s); EXPECT_FALSE(res.isFiltered()); EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 3); } - // Add duplicate vertex on Path + // Now we move to the duplicate vertex { - Step s = this->makeStep(3, lastIndex); + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1); + s = neighbors.at(0); auto res = validator.validatePath(s); + if (this->getVertexUniquness() == VertexUniquenessLevel::NONE) { // No uniqueness check, take the vertex EXPECT_FALSE(res.isFiltered()); @@ -219,49 +258,35 @@ TYPED_TEST(PathValidatorTest, it_should_honor_uniqueness_on_single_path_last_dup } TYPED_TEST(PathValidatorTest, it_should_honor_uniqueness_on_single_path_interior_duplicate) { - auto&& ps = this->store(); + // We add a loop that loops on the last vertex(2). + this->addPath({0, 1, 2, 3, 2}); auto validator = this->testee(); - size_t lastIndex = std::numeric_limits::max(); + Step s = this->startPath(0); { - Step s = this->makeStep(0, lastIndex); auto res = validator.validatePath(s); // The start vertex is always valid EXPECT_FALSE(res.isFiltered()); EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 0); - } - // We add a loop that ends in one interior vertex (2) again. - { - Step s = this->makeStep(1, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 1); } - { - Step s = this->makeStep(2, lastIndex); + // The next 3 steps are good to take. + for (size_t i = 0; i < 3; ++i) { + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1) + << "Not enough connections after step " << s.getVertexIdentifier(); + s = neighbors.at(0); auto res = validator.validatePath(s); EXPECT_FALSE(res.isFiltered()); EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 2); - } - { - Step s = this->makeStep(3, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 3); } - // Add duplicate vertex on Path + // Now we move to the duplicate vertex { - Step s = this->makeStep(2, lastIndex); + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1); + s = neighbors.at(0); auto res = validator.validatePath(s); + if (this->getVertexUniquness() == VertexUniquenessLevel::NONE) { // No uniqueness check, take the vertex EXPECT_FALSE(res.isFiltered()); @@ -275,160 +300,201 @@ TYPED_TEST(PathValidatorTest, it_should_honor_uniqueness_on_single_path_interior } TYPED_TEST(PathValidatorTest, it_should_honor_uniqueness_on_global_paths_last_duplicate) { - auto&& ps = this->store(); + // We add a two paths, that share the same start and end vertex (3) + this->addPath({0, 1, 2, 3}); + this->addPath({0, 4, 5, 3}); + auto validator = this->testee(); - size_t lastIndex = std::numeric_limits::max(); + Step s = this->startPath(0); { - Step s = this->makeStep(0, lastIndex); auto res = validator.validatePath(s); // The start vertex is always valid EXPECT_FALSE(res.isFiltered()); EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 0); - } - // We add two paths, each without a loop. - // Both paths share a common vertex besides the start. - - // First path 0 -> 1 -> 2 -> 3 - { - Step s = this->makeStep(1, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 1); - } - { - Step s = this->makeStep(2, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 2); - } - { - Step s = this->makeStep(3, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 3); } - // First path 0 -> 4 -> 5 - lastIndex = 0; - { - Step s = this->makeStep(4, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 4); - } + auto branch = this->expandPath(s); + // 1 and 4, we do not care on the ordering. + ASSERT_EQ(branch.size(), 2); { - Step s = this->makeStep(5, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 5); + { + // Test the branch vertex itself + s = branch.at(0); + auto res = validator.validatePath(s); + EXPECT_FALSE(res.isFiltered()); + EXPECT_FALSE(res.isPruned()); + } + // The first branch is good until the end + for (size_t i = 0; i < 2; ++i) { + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1) + << "Not enough connections after step " << s.getVertexIdentifier(); + s = neighbors.at(0); + auto res = validator.validatePath(s); + EXPECT_FALSE(res.isFiltered()); + EXPECT_FALSE(res.isPruned()); + } } - - // Add duplicate vertex (3) which is the last on the first path { - Step s = this->makeStep(3, lastIndex); - auto res = validator.validatePath(s); - if (this->getVertexUniquness() != VertexUniquenessLevel::GLOBAL) { - // The vertex is visited twice, but not on same path. - // As long as we are not GLOBAL this is okay. + // The second branch is good but for the last vertex + { + // Test the branch vertex itself + s = branch.at(1); + auto res = validator.validatePath(s); EXPECT_FALSE(res.isFiltered()); EXPECT_FALSE(res.isPruned()); - } else { - // With GLOBAL uniqueness this vertex is illegal - EXPECT_TRUE(res.isFiltered()); - EXPECT_TRUE(res.isPruned()); + } + for (size_t i = 0; i < 1; ++i) { + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1) + << "Not enough connections after step " << s.getVertexIdentifier(); + s = neighbors.at(0); + auto res = validator.validatePath(s); + EXPECT_FALSE(res.isFiltered()); + EXPECT_FALSE(res.isPruned()); + } + + // Now we move to the duplicate vertex + { + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1); + s = neighbors.at(0); + auto res = validator.validatePath(s); + + if (this->getVertexUniquness() != VertexUniquenessLevel::GLOBAL) { + // The vertex is visited twice, but not on same path. + // As long as we are not GLOBAL this is okay. + // No uniqueness check, take the vertex + EXPECT_FALSE(res.isFiltered()); + EXPECT_FALSE(res.isPruned()); + } else { + // With GLOBAL uniqueness this vertex is illegal + EXPECT_TRUE(res.isFiltered()); + EXPECT_TRUE(res.isPruned()); + } } } } TYPED_TEST(PathValidatorTest, it_should_honor_uniqueness_on_global_paths_interior_duplicate) { - auto&& ps = this->store(); + // We add a two paths, that share the same start and end vertex (3) + this->addPath({0, 1, 2, 3}); + this->addPath({0, 4, 5, 1}); + auto validator = this->testee(); - size_t lastIndex = std::numeric_limits::max(); + Step s = this->startPath(0); { - Step s = this->makeStep(0, lastIndex); auto res = validator.validatePath(s); // The start vertex is always valid EXPECT_FALSE(res.isFiltered()); EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 0); } - // We add two paths, each without a loop. - // Both paths share a common vertex besides the start. - // First path 0 -> 1 -> 2 -> 3 - { - Step s = this->makeStep(1, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 1); - } + auto branch = this->expandPath(s); + // 1 and 4, we do need to care on the ordering, this is right now guaranteed. + // If this test fails at any point in time, we can add some code here that + // ensures that we first visit Vertex 1, then Vertex 4 + ASSERT_EQ(branch.size(), 2); { - Step s = this->makeStep(2, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 2); + // The first branch is good until the end + { + // Test the branch vertex itself + s = branch.at(0); + auto res = validator.validatePath(s); + EXPECT_FALSE(res.isFiltered()); + EXPECT_FALSE(res.isPruned()); + } + for (size_t i = 0; i < 2; ++i) { + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1) + << "Not enough connections after step " << s.getVertexIdentifier(); + s = neighbors.at(0); + auto res = validator.validatePath(s); + EXPECT_FALSE(res.isFiltered()); + EXPECT_FALSE(res.isPruned()); + } } { - Step s = this->makeStep(3, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 3); - } + // The second branch is good but for the last vertex + { + // Test the branch vertex itself + s = branch.at(1); + auto res = validator.validatePath(s); + EXPECT_FALSE(res.isFiltered()); + EXPECT_FALSE(res.isPruned()); + } + for (size_t i = 0; i < 1; ++i) { + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1) + << "Not enough connections after step " << s.getVertexIdentifier(); + s = neighbors.at(0); + auto res = validator.validatePath(s); + EXPECT_FALSE(res.isFiltered()); + EXPECT_FALSE(res.isPruned()); + } - // First path 0 -> 4 -> 5 - lastIndex = 0; - { - Step s = this->makeStep(4, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 4); + // Now we move to the duplicate vertex + { + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1); + s = neighbors.at(0); + auto res = validator.validatePath(s); + + if (this->getVertexUniquness() != VertexUniquenessLevel::GLOBAL) { + // The vertex is visited twice, but not on same path. + // As long as we are not GLOBAL this is okay. + // No uniqueness check, take the vertex + EXPECT_FALSE(res.isFiltered()); + EXPECT_FALSE(res.isPruned()); + } else { + // With GLOBAL uniqueness this vertex is illegal + EXPECT_TRUE(res.isFiltered()); + EXPECT_TRUE(res.isPruned()); + } + } } +} + +TYPED_TEST(PathValidatorTest, it_should_test_an_all_vertices_condition) { + this->addPath({0, 1}); + std::string keyToMatch = "1"; + + auto expression = this->conditionKeyMatches(keyToMatch); + auto& opts = this->options(); + opts.setAllVerticesExpression(std::move(expression)); + auto validator = this->testee(); { - Step s = this->makeStep(5, lastIndex); + // Testing x._key == "1" with `{_key: "1"} => Should succeed + Step s = this->startPath(1); auto res = validator.validatePath(s); EXPECT_FALSE(res.isFiltered()); EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 5); } - // Add duplicate vertex (1) which is interior to the first path + // we start a new path, so reset the uniqueness checks + validator.reset(); + { - Step s = this->makeStep(1, lastIndex); - auto res = validator.validatePath(s); - if (this->getVertexUniquness() != VertexUniquenessLevel::GLOBAL) { - // The vertex is visited twice, but not on same path. - // As long as we are not GLOBAL this is okay. - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - } else { - // With GLOBAL uniqueness this vertex is illegal + // Testing x._key == "1" with `{_key: "0"} => Should fail + Step s = this->startPath(0); + { + auto res = validator.validatePath(s); EXPECT_TRUE(res.isFiltered()); EXPECT_TRUE(res.isPruned()); } + + // Testing condition on level 1 (not start) + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1); + s = neighbors.at(0); + { + // Testing x._key == "1" with `{_key: "1"} => Should succeed + auto res = validator.validatePath(s); + EXPECT_FALSE(res.isFiltered()); + EXPECT_FALSE(res.isPruned()); + } } } diff --git a/tests/Graph/SingleServerProviderTest.cpp b/tests/Graph/SingleServerProviderTest.cpp index ebe2c6811719..95298433e0f2 100644 --- a/tests/Graph/SingleServerProviderTest.cpp +++ b/tests/Graph/SingleServerProviderTest.cpp @@ -38,10 +38,22 @@ using namespace arangodb::tests; using namespace arangodb::tests::graph; using namespace arangodb::graph; +namespace { +aql::AstNode* InitializeReference(aql::Ast& ast, aql::Variable& var) { + ast.scopes()->start(aql::ScopeType::AQL_SCOPE_MAIN); + ast.scopes()->addVariable(&var); + aql::AstNode* a = ast.createNodeReference(var.name); + ast.scopes()->endCurrent(); + return a; +} +} // namespace + namespace arangodb { namespace tests { namespace single_server_provider_test { +using Step = SingleServerProvider::Step; + class SingleServerProviderTest : public ::testing::Test { protected: // Only used to mock a singleServer @@ -50,8 +62,17 @@ class SingleServerProviderTest : public ::testing::Test { std::unique_ptr query{nullptr}; arangodb::GlobalResourceMonitor _global{}; arangodb::ResourceMonitor _resourceMonitor{_global}; + arangodb::aql::AqlFunctionsInternalCache _functionsCache{}; + std::unique_ptr _expressionContext; + std::unique_ptr _trx{}; + + // Expression Parts + aql::Variable* _tmpVar{nullptr}; + aql::AstNode* _varNode{nullptr}; - std::map _emptyShardMap{}; + std::unordered_map> _emptyShardMap{}; + + std::string stringToMatch = "0-1"; SingleServerProviderTest() {} ~SingleServerProviderTest() {} @@ -65,20 +86,60 @@ class SingleServerProviderTest : public ::testing::Test { // We now have collections "v" and "e" query = singleServer->getQuery("RETURN 1", {"v", "e"}); + _trx = std::make_unique(query->newTrxContext()); auto edgeIndexHandle = singleServer->getEdgeIndexHandle("e"); - auto tmpVar = singleServer->generateTempVar(query.get()); - auto indexCondition = singleServer->buildOutboundCondition(query.get(), tmpVar); + _tmpVar = singleServer->generateTempVar(query.get()); + + auto indexCondition = singleServer->buildOutboundCondition(query.get(), _tmpVar); + _varNode = ::InitializeReference(*query->ast(), *_tmpVar); - std::vector usedIndexes{IndexAccessor{edgeIndexHandle, indexCondition, 0}}; + std::vector usedIndexes{}; + auto expr = conditionKeyMatches(stringToMatch); + usedIndexes.emplace_back(IndexAccessor{edgeIndexHandle, indexCondition, 0, expr}); - BaseProviderOptions opts(tmpVar, std::move(usedIndexes), _emptyShardMap); + _expressionContext = + std::make_unique(*_trx, *query, _functionsCache); + BaseProviderOptions opts(_tmpVar, + std::make_pair(std::move(usedIndexes), + std::unordered_map>{}), + *_expressionContext.get(), _emptyShardMap); return {*query.get(), std::move(opts), _resourceMonitor}; } + + /* + * generates a condition #TMP._key == '' + */ + std::shared_ptr conditionKeyMatches(std::string const& toMatch) { + auto expectedKey = + query->ast()->createNodeValueString(toMatch.c_str(), toMatch.length()); + auto keyAccess = + query->ast()->createNodeAttributeAccess(_varNode, + StaticStrings::KeyString.c_str(), + StaticStrings::KeyString.length()); + // This condition cannot be fulfilled + auto condition = query->ast()->createNodeBinaryOperator(aql::AstNodeType::NODE_TYPE_OPERATOR_BINARY_EQ, + keyAccess, expectedKey); + return std::make_shared(query->ast(), condition); + } }; -TEST_F(SingleServerProviderTest, it_must_be_described) { - ASSERT_TRUE(true); +TEST_F(SingleServerProviderTest, it_can_provide_edges) { + MockGraph g; + g.addEdge(0, 1, 2); + g.addEdge(0, 2, 3); + g.addEdge(1, 2, 1); + auto testee = makeProvider(g); + auto startVertex = g.vertexToId(0); + HashedStringRef hashedStart{startVertex.c_str(), + static_cast(startVertex.length())}; + Step s = testee.startVertex(hashedStart); + + testee.expand(s, 0, [&](Step next) { + VPackBuilder hund; + testee.addEdgeToBuilder(next.getEdge(), hund); + LOG_DEVEL << next.getVertexIdentifier() << " e: " << hund.toJson(); + }); } } // namespace single_server_provider_test diff --git a/tests/Graph/TraverserCacheTest.cpp b/tests/Graph/TraverserCacheTest.cpp index aaff2441f8ba..de9c24981706 100644 --- a/tests/Graph/TraverserCacheTest.cpp +++ b/tests/Graph/TraverserCacheTest.cpp @@ -55,7 +55,7 @@ class TraverserCacheTest : public ::testing::Test { std::unique_ptr traverserCache{nullptr}; std::shared_ptr queryContext{nullptr}; std::unique_ptr trx{nullptr}; - std::map collectionToShardMap{}; // can be empty, only used in standalone mode + std::unordered_map> collectionToShardMap{}; // can be empty, only used in standalone mode arangodb::ResourceMonitor* _monitor; TraverserCacheTest() : gdb(s.server, "testVocbase") { @@ -65,7 +65,8 @@ class TraverserCacheTest : public ::testing::Test { _monitor = &query->resourceMonitor(); traverserCache = std::make_unique(trx.get(), query.get(), - query->resourceMonitor(), stats, collectionToShardMap); + query->resourceMonitor(), + stats, collectionToShardMap); } ~TraverserCacheTest() = default; @@ -85,7 +86,8 @@ TEST_F(TraverserCacheTest, it_should_return_a_null_aqlvalue_if_vertex_is_not_ava VPackBuilder builder; // NOTE: we do not have the data, so we get null for any vertex - traverserCache->insertVertexIntoResult(stats, arangodb::velocypack::HashedStringRef(id), builder); + traverserCache->insertVertexIntoResult(stats, arangodb::velocypack::HashedStringRef(id), + builder); ASSERT_TRUE(builder.slice().isNull()); auto all = query->warnings().all(); ASSERT_EQ(all.size(), 1); @@ -129,7 +131,8 @@ TEST_F(TraverserCacheTest, it_should_increase_memory_usage_when_persisting_a_str EXPECT_EQ(memoryUsageBefore, _monitor->current()); } -TEST_F(TraverserCacheTest, it_should_not_increase_memory_usage_twice_when_persisting_two_equal_strings) { +TEST_F(TraverserCacheTest, + it_should_not_increase_memory_usage_twice_when_persisting_two_equal_strings) { auto memoryUsageStart = _monitor->current(); auto data = VPackParser::fromJson(R"({"_key":"123", "value":123})"); @@ -168,7 +171,8 @@ TEST_F(TraverserCacheTest, it_should_increase_memory_usage_twice_when_persisting EXPECT_EQ(memoryUsageStart, _monitor->current()); } -TEST_F(TraverserCacheTest, it_should_increase_memory_usage_twice_when_persisting_a_string_clear_persist_again) { +TEST_F(TraverserCacheTest, + it_should_increase_memory_usage_twice_when_persisting_a_string_clear_persist_again) { auto memoryUsageBefore = _monitor->current(); auto data = VPackParser::fromJson(R"({"_key":"123", "value":123})"); @@ -220,7 +224,8 @@ TEST_F(TraverserCacheTest, it_should_insert_a_vertex_into_a_result_builder) { HashedStringRef id{doc.get("_id")}; VPackBuilder builder; - traverserCache->insertVertexIntoResult(stats, arangodb::velocypack::HashedStringRef(id), builder); + traverserCache->insertVertexIntoResult(stats, arangodb::velocypack::HashedStringRef(id), + builder); EXPECT_TRUE(builder.slice().get("_key").isString()); EXPECT_EQ(builder.slice().get("_key").toString(), "0"); @@ -235,7 +240,7 @@ TEST_F(TraverserCacheTest, it_should_insert_an_edge_into_a_result_builder) { graph.addEdge(0, 1); gdb.addGraph(graph); - std::string edgeKey = "0-1"; // (EdgeKey format: -) + std::string edgeKey = "0-1"; // (EdgeKey format: -) std::shared_ptr col = gdb.vocbase.lookupCollection("e"); @@ -244,7 +249,8 @@ TEST_F(TraverserCacheTest, it_should_insert_an_edge_into_a_result_builder) { bool called = false; auto result = col->getPhysical()->read(trx.get(), arangodb::velocypack::StringRef{edgeKey}, - [&fetchedDocumentId, &called, &edgeKey](LocalDocumentId const& ldid, VPackSlice edgeDocument) { + [&fetchedDocumentId, &called, + &edgeKey](LocalDocumentId const& ldid, VPackSlice edgeDocument) { fetchedDocumentId = ldid.id(); called = true; EXPECT_TRUE(edgeDocument.isObject()); @@ -256,8 +262,8 @@ TEST_F(TraverserCacheTest, it_should_insert_an_edge_into_a_result_builder) { ASSERT_TRUE(result.ok()); ASSERT_NE(fetchedDocumentId, 0); - DataSourceId dataSourceId{col->id()}; // valid - LocalDocumentId localDocumentId{fetchedDocumentId}; // valid + DataSourceId dataSourceId{col->id()}; // valid + LocalDocumentId localDocumentId{fetchedDocumentId}; // valid EdgeDocumentToken edt{dataSourceId, localDocumentId}; VPackBuilder builder; @@ -270,7 +276,7 @@ TEST_F(TraverserCacheTest, it_should_insert_an_edge_into_a_result_builder) { // check stats EXPECT_EQ(stats.getHttpRequests(), 0); EXPECT_EQ(stats.getFiltered(), 0); - EXPECT_EQ(stats.getScannedIndex(), 0); // <- see note in method: appendEdge + EXPECT_EQ(stats.getScannedIndex(), 0); // <- see note in method: appendEdge } } // namespace traverser_cache_test diff --git a/tests/Graph/MockGraph.cpp b/tests/Mocks/MockGraph.cpp similarity index 100% rename from tests/Graph/MockGraph.cpp rename to tests/Mocks/MockGraph.cpp diff --git a/tests/Graph/MockGraph.h b/tests/Mocks/MockGraph.h similarity index 100% rename from tests/Graph/MockGraph.h rename to tests/Mocks/MockGraph.h diff --git a/tests/Graph/MockGraphProvider.cpp b/tests/Mocks/MockGraphProvider.cpp similarity index 91% rename from tests/Graph/MockGraphProvider.cpp rename to tests/Mocks/MockGraphProvider.cpp index 60d6bc3f3539..fc08796fcb5f 100644 --- a/tests/Graph/MockGraphProvider.cpp +++ b/tests/Mocks/MockGraphProvider.cpp @@ -59,6 +59,12 @@ MockGraphProvider::Step::Step(size_t prev, VertexType v, EdgeType e, bool isProc _edge(e), _isProcessable(isProcessable) {} +MockGraphProvider::Step::Step(size_t prev, VertexType v, bool isProcessable, size_t depth) + : arangodb::graph::BaseStep{prev, depth}, + _vertex(v), + _edge({}), + _isProcessable(isProcessable) {} + MockGraphProvider::Step::Step(size_t prev, VertexType v, EdgeType e, bool isProcessable, size_t depth) : arangodb::graph::BaseStep{prev, depth}, @@ -66,13 +72,11 @@ MockGraphProvider::Step::Step(size_t prev, VertexType v, EdgeType e, _edge(e), _isProcessable(isProcessable) {} -MockGraphProvider::Step::~Step() {} - -MockGraphProvider::MockGraphProvider(MockGraph const& data, - arangodb::aql::QueryContext& queryContext, - LooseEndBehaviour looseEnds, bool reverse) - : _trx(queryContext.newTrxContext()), _reverse(reverse), _looseEnds(looseEnds), _stats{} { - for (auto const& it : data.edges()) { +MockGraphProvider::MockGraphProvider(arangodb::aql::QueryContext& queryContext, + MockGraphProviderOptions opts, + arangodb::ResourceMonitor&) + : _trx(queryContext.newTrxContext()), _reverse(opts.reverse()), _looseEnds(opts.looseEnds()), _stats{} { + for (auto const& it : opts.data().edges()) { _fromIndex[it._from].push_back(it); _toIndex[it._to].push_back(it); } @@ -91,7 +95,7 @@ auto MockGraphProvider::decideProcessable() const -> bool { } } -auto MockGraphProvider::startVertex(VertexType v) -> Step { +auto MockGraphProvider::startVertex(VertexType v, size_t depth) -> Step { LOG_TOPIC("78156", TRACE, Logger::GRAPHS) << " Start Vertex:" << v; return Step(v, decideProcessable()); diff --git a/tests/Graph/MockGraphProvider.h b/tests/Mocks/MockGraphProvider.h similarity index 75% rename from tests/Graph/MockGraphProvider.h rename to tests/Mocks/MockGraphProvider.h index b6296cd12649..1fac5ecf3b31 100644 --- a/tests/Graph/MockGraphProvider.h +++ b/tests/Mocks/MockGraphProvider.h @@ -56,18 +56,39 @@ class HashedStringRef; namespace tests { namespace graph { + +class MockGraphProviderOptions { + public: + enum class LooseEndBehaviour { NEVER, ALWAYS }; + MockGraphProviderOptions(MockGraph const& data, LooseEndBehaviour looseEnds, + bool reverse = false) + : _data(data), _looseEnds(looseEnds), _reverse(reverse) {} + ~MockGraphProviderOptions() = default; + + LooseEndBehaviour looseEnds() const { return _looseEnds; } + MockGraph const& data() const { return _data; } + bool reverse() const { return _reverse; } + + private: + MockGraph const& _data; + LooseEndBehaviour _looseEnds; + bool _reverse; + ; +}; + class MockGraphProvider { using VertexType = arangodb::velocypack::HashedStringRef; using EdgeType = MockGraph::EdgeDef; public: - enum class LooseEndBehaviour { NEVER, ALWAYS }; + using Options = MockGraphProviderOptions; + using LooseEndBehaviour = typename MockGraphProviderOptions::LooseEndBehaviour; class Step : public arangodb::graph::BaseStep { public: class Vertex { public: - explicit Vertex(VertexType v) : _vertex(v) {}; + explicit Vertex(VertexType v) : _vertex(v){}; VertexType getID() const { return _vertex; } @@ -106,8 +127,9 @@ class MockGraphProvider { Step(VertexType v, bool isProcessable); Step(size_t prev, VertexType v, EdgeType e, bool isProcessable); + Step(size_t prev, VertexType v, bool isProcessable, size_t depth); Step(size_t prev, VertexType v, EdgeType e, bool isProcessable, size_t depth); - ~Step(); + ~Step() = default; bool operator<(Step const& other) const noexcept { return _vertex < other._vertex; @@ -124,6 +146,10 @@ class MockGraphProvider { } } + bool isResponsible(transaction::Methods* trx) const { + return true; + } + Vertex getVertex() const { /*if (!isProcessable()) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, @@ -144,6 +170,28 @@ class MockGraphProvider { VertexType getVertexIdentifier() const { return getVertex().getID(); } + std::string getCollectionName() const { + auto collectionNameResult = extractCollectionName(_vertex.getID()); + if (collectionNameResult.fail()) { + THROW_ARANGO_EXCEPTION(collectionNameResult.result()); + } + return collectionNameResult.get().first; + }; + + void setLocalSchreierIndex(size_t index) { + TRI_ASSERT(index != std::numeric_limits::max()); + TRI_ASSERT(!hasLocalSchreierIndex()); + _localSchreierIndex = index; + } + + bool hasLocalSchreierIndex() const { + return _localSchreierIndex != std::numeric_limits::max(); + } + + std::size_t getLocalSchreierIndex() const { + return _localSchreierIndex; + } + bool isProcessable() const { return _isProcessable; } bool isLooseEnd() const { return !isProcessable(); } @@ -159,11 +207,13 @@ class MockGraphProvider { Vertex _vertex; Edge _edge; bool _isProcessable; + size_t _localSchreierIndex; }; MockGraphProvider() = delete; - MockGraphProvider(MockGraph const& data, arangodb::aql::QueryContext& queryContext, - LooseEndBehaviour looseEnds, bool reverse = false); + MockGraphProvider(arangodb::aql::QueryContext& queryContext, Options opts, + arangodb::ResourceMonitor& resourceMonitor); + MockGraphProvider(MockGraphProvider const&) = delete; // TODO: check "Rule of 5" MockGraphProvider(MockGraphProvider&&) = default; ~MockGraphProvider(); @@ -172,7 +222,7 @@ class MockGraphProvider { MockGraphProvider& operator=(MockGraphProvider&&) = default; void destroyEngines(){}; - auto startVertex(VertexType vertex) -> Step; + auto startVertex(VertexType vertex, size_t depth = 0) -> Step; auto fetch(std::vector const& looseEnds) -> futures::Future>; auto expand(Step const& from, size_t previous) -> std::vector; auto expand(Step const& from, size_t previous, std::function callback) -> void; diff --git a/tests/Mocks/Servers.cpp b/tests/Mocks/Servers.cpp index 3fd63f4904c5..a471ddf1d71c 100644 --- a/tests/Mocks/Servers.cpp +++ b/tests/Mocks/Servers.cpp @@ -35,8 +35,8 @@ #include "Aql/ExecutionEngine.h" #include "Aql/OptimizerRulesFeature.h" #include "Aql/Query.h" -#include "Basics/files.h" #include "Basics/StringUtils.h" +#include "Basics/files.h" #include "Cluster/ActionDescription.h" #include "Cluster/AgencyCache.h" #include "Cluster/ClusterFeature.h" @@ -81,7 +81,6 @@ #include "VocBase/vocbase.h" #include "utils/log.hpp" - #include "Servers.h" #include "TemplateSpecializer.h" @@ -124,6 +123,7 @@ static void SetupDatabaseFeaturePhase(MockServer& server) { SetupBasicFeaturePhase(server); server.addFeature(false); // true ?? server.addFeature(true); + server.addFeature(false); server.addFeature(false); server.addFeature(false); server.addFeature(false); @@ -141,7 +141,7 @@ static void SetupClusterFeaturePhase(MockServer& server) { SetupDatabaseFeaturePhase(server); server.addFeature(false); server.addFeature(false); - + // fake the exit code with which unresolved futures are returned on // shutdown. if we don't do this lots of places in ClusterInfo will // report failures during testing @@ -192,7 +192,7 @@ MockServer::MockServer() MockServer::~MockServer() { stopFeatures(); _server.setStateUnsafe(_oldApplicationServerState); - + arangodb::ServerState::instance()->setRebootId(_oldRebootId); } @@ -207,9 +207,9 @@ void MockServer::init() { _server.setStateUnsafe(arangodb::application_features::ApplicationServer::State::IN_WAIT); arangodb::transaction::Methods::clearDataSourceRegistrationCallbacks(); - // many other places rely on the reboot id being initialized, + // many other places rely on the reboot id being initialized, // so we do it here in a central place - arangodb::ServerState::instance()->setRebootId(arangodb::RebootId{1}); + arangodb::ServerState::instance()->setRebootId(arangodb::RebootId{1}); } void MockServer::startFeatures() { @@ -258,7 +258,7 @@ void MockServer::startFeatures() { for (ApplicationFeature& f : orderedFeatures) { auto info = _features.find(&f); // We only start those features... - if(info != _features.end()) { + if (info != _features.end()) { if (info->second) { try { f.start(); @@ -297,7 +297,7 @@ void MockServer::stopFeatures() { for (ApplicationFeature& f : orderedFeatures) { auto info = _features.find(&f); // We only start those features... - if(info != _features.end()) { + if (info != _features.end()) { if (info->second) { try { f.stop(); @@ -312,7 +312,7 @@ void MockServer::stopFeatures() { for (ApplicationFeature& f : orderedFeatures) { auto info = _features.find(&f); // We only start those features... - if(info != _features.end()) { + if (info != _features.end()) { try { f.unprepare(); } catch (...) { @@ -549,10 +549,14 @@ void MockClusterServer::agencyCreateDatabase(std::string const& name) { st = ts.specialize(current_dbs_string); agencyTrx("/arango/Current/Databases/" + name, st); - _server.getFeature().clusterInfo().waitForPlan( - agencyTrx("/arango/Plan/Version", R"=({"op":"increment"})=")).wait(); - _server.getFeature().clusterInfo().waitForCurrent( - agencyTrx("/arango/Current/Version", R"=({"op":"increment"})=")).wait(); + _server.getFeature() + .clusterInfo() + .waitForPlan(agencyTrx("/arango/Plan/Version", R"=({"op":"increment"})=")) + .wait(); + _server.getFeature() + .clusterInfo() + .waitForCurrent(agencyTrx("/arango/Current/Version", R"=({"op":"increment"})=")) + .wait(); } void MockClusterServer::agencyCreateCollections(std::string const& name) { @@ -605,7 +609,7 @@ std::shared_ptr MockClusterServer::createCollection( std::string cid = "98765" + basics::StringUtils::itoa(type); auto& databaseFeature = _server.getFeature(); auto vocbase = databaseFeature.lookupDatabase(dbName); - + VPackBuilder props; { // This is hand-crafted unfortunately the code does not exist... @@ -618,25 +622,25 @@ std::shared_ptr MockClusterServer::createCollection( { VPackArrayBuilder guard2(&props); auto const primIndex = arangodb::velocypack::Parser::fromJson( - R"({"id":"0","type":"primary","name": + R"({"id":"0","type":"primary","name": "primary","fields":["_key"],"unique":true,"sparse":false })"); -props.add(primIndex->slice()); -if (type == TRI_COL_TYPE_EDGE) { - auto const fromIndex = arangodb::velocypack::Parser::fromJson( - R"({"id":"1","type":"edge","name": + props.add(primIndex->slice()); + if (type == TRI_COL_TYPE_EDGE) { + auto const fromIndex = arangodb::velocypack::Parser::fromJson( + R"({"id":"1","type":"edge","name": "edge_from","fields":["_from"],"unique":false,"sparse":false })"); -props.add(fromIndex->slice()); -auto const toIndex = arangodb::velocypack::Parser::fromJson( -R"({"id":"2","type":"edge","name": + props.add(fromIndex->slice()); + auto const toIndex = arangodb::velocypack::Parser::fromJson( + R"({"id":"2","type":"edge","name": "edge_to","fields":["_to"],"unique":false,"sparse":false})"); -props.add(toIndex->slice()); -} + props.add(toIndex->slice()); + } } } LogicalCollection dummy(*vocbase, props.slice(), true); - + auto shards = std::make_shared(); for (auto const& [shard, server] : shardNameToServerNamePairs) { shards->emplace(shard, std::vector{server}); @@ -650,11 +654,13 @@ props.add(toIndex->slice()); VPackBuilder velocy = dummy.toVelocyPackIgnore(ignoreKeys, LogicalDataSource::Serialization::List); - agencyTrx("/arango/Plan/Collections/" + dbName + "/" + basics::StringUtils::itoa(dummy.planId().id()), velocy.toJson()); + agencyTrx("/arango/Plan/Collections/" + dbName + "/" + + basics::StringUtils::itoa(dummy.planId().id()), + velocy.toJson()); { - /* Hard-Coded section to inject the CURRENT counter part. - * We do not have a shard available here that could generate the values accordingly. - */ + /* Hard-Coded section to inject the CURRENT counter part. + * We do not have a shard available here that could generate the values accordingly. + */ VPackBuilder current; { VPackObjectBuilder report(¤t); @@ -678,7 +684,9 @@ props.add(toIndex->slice()); // NOTE: we omited Indexes } } - agencyTrx("/arango/Current/Collections/" + dbName + "/" + basics::StringUtils::itoa(dummy.planId().id()), current.toJson()); + agencyTrx("/arango/Current/Collections/" + dbName + "/" + + basics::StringUtils::itoa(dummy.planId().id()), + current.toJson()); } _server.getFeature() @@ -762,7 +770,8 @@ void MockDBServer::createShard(std::string const& dbName, std::string shardName, maintenance::ActionDescription ad( std::map{{maintenance::NAME, maintenance::CREATE_COLLECTION}, {maintenance::COLLECTION, - basics::StringUtils::itoa(clusterCollection.planId().id())}, + basics::StringUtils::itoa( + clusterCollection.planId().id())}, {maintenance::SHARD, shardName}, {maintenance::DATABASE, dbName}, {maintenance::SERVER_ID, "PRMR_0001"}, @@ -828,7 +837,7 @@ std::pair MockCoordinator::registerFakedDBServer(std:: { VPackObjectBuilder b(&builder); builder.add("endpoint", VPackValue(fakedEndpoint)); - builder.add("advertisedEndpoint",VPackValue(fakedEndpoint)); + builder.add("advertisedEndpoint", VPackValue(fakedEndpoint)); builder.add("host", VPackValue(fakedHost)); builder.add("version", VPackValue(rest::Version::getNumericServerVersion())); builder.add("versionString", VPackValue(rest::Version::getServerVersion())); diff --git a/tests/Mocks/StorageEngineMock.cpp b/tests/Mocks/StorageEngineMock.cpp index 3d6629ad79dc..e9f303be60b1 100644 --- a/tests/Mocks/StorageEngineMock.cpp +++ b/tests/Mocks/StorageEngineMock.cpp @@ -100,19 +100,19 @@ class EdgeIndexIteratorMock final : public arangodb::IndexIterator { typedef std::unordered_multimap Map; EdgeIndexIteratorMock(arangodb::LogicalCollection* collection, - arangodb::transaction::Methods* trx, arangodb::Index const* index, - Map const& map, std::unique_ptr&& keys) + arangodb::transaction::Methods* trx, + arangodb::Index const* index, Map const& map, + std::unique_ptr&& keys, bool isFrom) : IndexIterator(collection, trx), _map(map), - _begin(_map.begin()), + _begin(_map.end()), _end(_map.end()), _keys(std::move(keys)), - _keysIt(_keys->slice()) {} - - char const* typeName() const override { return "edge-index-iterator-mock"; } + _keysIt(_keys->slice()), + _isFrom(isFrom) {} - bool nextImpl(LocalDocumentIdCallback const& cb, size_t limit) override { - while (limit && _begin != _end && _keysIt.valid()) { + bool prepareNextRange() { + if (_keysIt.valid()) { auto key = _keysIt.value(); if (key.isObject()) { @@ -120,16 +120,63 @@ class EdgeIndexIteratorMock final : public arangodb::IndexIterator { } std::tie(_begin, _end) = _map.equal_range(key.toString()); + _keysIt++; + return true; + } else { + // Just make sure begin and end are equal + _begin = _map.end(); + _end = _map.end(); + return false; + } + } - while (limit && _begin != _end) { - cb(_begin->second); - ++_begin; - --limit; - } + char const* typeName() const override { return "edge-index-iterator-mock"; } + + bool hasExtra() const override { return true; } - ++_keysIt; + bool nextImpl(LocalDocumentIdCallback const& cb, size_t limit) override { + // We can at most return limit + for (size_t l = 0; l < limit; ++l) { + while (_begin == _end) { + if (!prepareNextRange()) { + return false; + } + } + TRI_ASSERT(_begin != _end); + cb(_begin->second); + ++_begin; + } + // Returned due to limit. + if (_begin == _end) { + // Out limit hit the last index entry + // Return false if we do not have another range + return prepareNextRange(); } - return _begin != _end && _keysIt.valid(); + return true; + } + + bool nextExtraImpl(ExtraCallback const& cb, size_t limit) override { + auto res = nextImpl( + [&](arangodb::LocalDocumentId docId) -> bool { + auto res = _collection->getPhysical()->read( + _trx, docId, + [&](arangodb::LocalDocumentId const& token, arangodb::velocypack::Slice doc) -> bool { + // The nextExtra API in EdgeIndex will deliver the _id value of + // the oposite vertex. We simulate that here by reading the real + // document one more time and just extracting this value. + if (_isFrom) { + return cb(token, doc.get(arangodb::StaticStrings::ToString)); + } else { + return cb(token, doc.get(arangodb::StaticStrings::FromString)); + } + }); + // We can only have good responses here. + // Otherwise storage and Index do differ + TRI_ASSERT(res.ok()); + return res.ok(); + }, + limit); + return res; } void resetImpl() override { @@ -144,6 +191,7 @@ class EdgeIndexIteratorMock final : public arangodb::IndexIterator { Map::const_iterator _end; std::unique_ptr _keys; arangodb::velocypack::ArrayIterator _keysIt; + bool _isFrom; }; // EdgeIndexIteratorMock class EdgeIndexMock final : public arangodb::Index { @@ -341,7 +389,7 @@ class EdgeIndexMock final : public arangodb::Index { return std::make_unique(&_collection, trx, this, isFrom ? _edgesFrom : _edgesTo, - std::move(keys)); + std::move(keys), isFrom); } /// @brief create the iterator @@ -371,7 +419,7 @@ class EdgeIndexMock final : public arangodb::Index { return std::make_unique(&_collection, trx, this, isFrom ? _edgesFrom : _edgesTo, - std::move(keys)); + std::move(keys), isFrom); } /// @brief the hash table for _from @@ -455,14 +503,17 @@ class HashIndexMap { struct VPackBuilderComparator { bool operator()(VPackBuilder const& builder1, VPackBuilder const& builder2) const { - return ::arangodb::basics::VelocyPackHelper::compare(builder1.slice(), builder2.slice(), true) == 0; + return ::arangodb::basics::VelocyPackHelper::compare(builder1.slice(), + builder2.slice(), true) == 0; } }; - using ValueMap = std::unordered_multimap; + using ValueMap = + std::unordered_multimap; using DocumentsIndexMap = std::unordered_map; - arangodb::velocypack::Slice getSliceByField(arangodb::velocypack::Slice const& doc, size_t i) { + arangodb::velocypack::Slice getSliceByField(arangodb::velocypack::Slice const& doc, + size_t i) { TRI_ASSERT(i < _fields.size()); TRI_ASSERT(!doc.isNone()); auto slice = doc; @@ -475,7 +526,8 @@ class HashIndexMap { return slice; } - void insertSlice(arangodb::LocalDocumentId const& documentId, arangodb::velocypack::Slice const& slice, size_t i) { + void insertSlice(arangodb::LocalDocumentId const& documentId, + arangodb::velocypack::Slice const& slice, size_t i) { VPackBuilder builder; if (slice.isNone() || slice.isNull()) { builder.add(VPackSlice::nullSlice()); @@ -486,11 +538,13 @@ class HashIndexMap { } public: - HashIndexMap(std::vector> const& fields) : _fields(fields), _valueMaps(fields.size()) { + HashIndexMap(std::vector> const& fields) + : _fields(fields), _valueMaps(fields.size()) { TRI_ASSERT(!_fields.empty()); } - void insert(arangodb::LocalDocumentId const& documentId, arangodb::velocypack::Slice const& doc) { + void insert(arangodb::LocalDocumentId const& documentId, + arangodb::velocypack::Slice const& doc) { VPackBuilder builder; builder.openArray(); auto toClose = true; @@ -510,14 +564,16 @@ class HashIndexMap { if (slice.isNone() || slice.isNull()) { break; } - } else { // expansion + } else { // expansion isExpansion = slice.isArray(); TRI_ASSERT(isExpansion); auto found = false; - for (auto sliceIt = arangodb::velocypack::ArrayIterator(slice); sliceIt != sliceIt.end(); ++sliceIt) { + for (auto sliceIt = arangodb::velocypack::ArrayIterator(slice); + sliceIt != sliceIt.end(); ++sliceIt) { auto subSlice = sliceIt.value(); if (!(subSlice.isNone() || subSlice.isNull())) { - for (auto fieldItForArray = fieldIt; fieldItForArray != _fields[i].end(); ++fieldItForArray) { + for (auto fieldItForArray = fieldIt; + fieldItForArray != _fields[i].end(); ++fieldItForArray) { TRI_ASSERT(subSlice.isObject()); subSlice = subSlice.get(fieldItForArray->name); if (subSlice.isNone() || subSlice.isNull()) { @@ -544,7 +600,8 @@ class HashIndexMap { if (slice.isArray() && i == _fields.size() - 1) { auto found = false; auto wasNull = false; - for (auto sliceIt = arangodb::velocypack::ArrayIterator(slice); sliceIt != sliceIt.end(); ++sliceIt) { + for (auto sliceIt = arangodb::velocypack::ArrayIterator(slice); + sliceIt != sliceIt.end(); ++sliceIt) { auto subSlice = sliceIt.value(); if (!(subSlice.isNone() || subSlice.isNull())) { insertSlice(documentId, subSlice, i); @@ -557,7 +614,7 @@ class HashIndexMap { insertSlice(documentId, VPackSlice::nullSlice(), i); } toClose = false; - } else { // object + } else { // object insertSlice(documentId, slice, i); builder.add(slice); } @@ -569,7 +626,8 @@ class HashIndexMap { _docIndexMap.try_emplace(documentId, std::move(builder)); } - bool remove(arangodb::LocalDocumentId const& documentId, arangodb::velocypack::Slice const& doc) { + bool remove(arangodb::LocalDocumentId const& documentId, + arangodb::velocypack::Slice const& doc) { size_t i = 0; auto documentRemoved = false; for (auto& map : _valueMaps) { @@ -592,7 +650,8 @@ class HashIndexMap { _docIndexMap.clear(); } - std::unordered_map find(std::unique_ptr&& keys) const { + std::unordered_map find( + std::unique_ptr&& keys) const { std::unordered_map found; TRI_ASSERT(keys->slice().isArray()); auto sliceIt = arangodb::velocypack::ArrayIterator(keys->slice()); @@ -605,7 +664,7 @@ class HashIndexMap { return std::unordered_map(); } if (found.empty()) { - std::transform(begin, end, std::inserter(found, found.end()), [] (auto const& item) { + std::transform(begin, end, std::inserter(found, found.end()), [](auto const& item) { return std::make_pair(item.second, &item.first); }); } else { @@ -650,7 +709,7 @@ class HashIndexIteratorMock final : public arangodb::IndexIterator { HashIndexIteratorMock(arangodb::LogicalCollection* collection, arangodb::transaction::Methods* trx, arangodb::Index const* index, HashIndexMap const& map, std::unique_ptr&& keys) - : IndexIterator(collection, trx), _map(map) { + : IndexIterator(collection, trx), _map(map) { _documents = _map.find(std::move(keys)); _begin = _documents.begin(); _end = _documents.end(); @@ -684,9 +743,7 @@ class HashIndexIteratorMock final : public arangodb::IndexIterator { _end = _documents.end(); } - bool hasCovering() const override { - return true; - } + bool hasCovering() const override { return true; } private: HashIndexMap const& _map; @@ -706,8 +763,9 @@ class HashIndexMock final : public arangodb::Index { return nullptr; } - auto const type = arangodb::basics::VelocyPackHelper::getStringRef(typeSlice, - arangodb::velocypack::StringRef()); + auto const type = + arangodb::basics::VelocyPackHelper::getStringRef(typeSlice, + arangodb::velocypack::StringRef()); if (type.compare("hash") != 0) { return nullptr; @@ -781,17 +839,19 @@ class HashIndexMock final : public arangodb::Index { std::vector> const& allIndexes, arangodb::aql::AstNode const* node, arangodb::aql::Variable const* reference, size_t itemsInIndex) const override { - return arangodb::SortedIndexAttributeMatcher::supportsFilterCondition(allIndexes, this, node, reference, itemsInIndex); + return arangodb::SortedIndexAttributeMatcher::supportsFilterCondition( + allIndexes, this, node, reference, itemsInIndex); } Index::SortCosts supportsSortCondition(arangodb::aql::SortCondition const* sortCondition, - arangodb::aql::Variable const* reference, - size_t itemsInIndex) const override { - return arangodb::SortedIndexAttributeMatcher::supportsSortCondition(this, sortCondition, reference, itemsInIndex); + arangodb::aql::Variable const* reference, + size_t itemsInIndex) const override { + return arangodb::SortedIndexAttributeMatcher::supportsSortCondition(this, sortCondition, + reference, itemsInIndex); } - arangodb::aql::AstNode* specializeCondition( - arangodb::aql::AstNode* node, arangodb::aql::Variable const* reference) const override { + arangodb::aql::AstNode* specializeCondition(arangodb::aql::AstNode* node, + arangodb::aql::Variable const* reference) const override { return arangodb::SortedIndexAttributeMatcher::specializeCondition(this, node, reference); } @@ -804,8 +864,7 @@ class HashIndexMock final : public arangodb::Index { if (nullptr == node) { keys->close(); return std::make_unique(&_collection, trx, this, - _hashData, - std::move(keys)); + _hashData, std::move(keys)); } TRI_ASSERT(node->type == arangodb::aql::NODE_TYPE_OPERATOR_NARY_AND); @@ -824,7 +883,7 @@ class HashIndexMock final : public arangodb::Index { auto valNode = comp->getMember(1); if (!(attrNode->type == arangodb::aql::NODE_TYPE_ATTRIBUTE_ACCESS || - attrNode->type == arangodb::aql::NODE_TYPE_EXPANSION)) { + attrNode->type == arangodb::aql::NODE_TYPE_EXPANSION)) { // got value == a.b -> flip sides std::swap(attrNode, valNode); } @@ -834,11 +893,13 @@ class HashIndexMock final : public arangodb::Index { std::vector attributes; if (attrNode->type == arangodb::aql::NODE_TYPE_ATTRIBUTE_ACCESS) { do { - attributes.emplace_back(std::string(attrNode->getStringValue(), attrNode->getStringLength()), false); + attributes.emplace_back(std::string(attrNode->getStringValue(), + attrNode->getStringLength()), + false); attrNode = attrNode->getMember(0); } while (attrNode->type == arangodb::aql::NODE_TYPE_ATTRIBUTE_ACCESS); std::reverse(attributes.begin(), attributes.end()); - } else { // expansion + } else { // expansion TRI_ASSERT(attrNode->type == arangodb::aql::NODE_TYPE_EXPANSION); auto expNode = attrNode; TRI_ASSERT(expNode->numMembers() >= 2); @@ -847,7 +908,9 @@ class HashIndexMock final : public arangodb::Index { attrNode = left->getMember(1); TRI_ASSERT(attrNode->type == arangodb::aql::NODE_TYPE_ATTRIBUTE_ACCESS); do { - attributes.emplace_back(std::string(attrNode->getStringValue(), attrNode->getStringLength()), false); + attributes.emplace_back(std::string(attrNode->getStringValue(), + attrNode->getStringLength()), + false); attrNode = attrNode->getMember(0); } while (attrNode->type == arangodb::aql::NODE_TYPE_ATTRIBUTE_ACCESS); attributes.front().shouldExpand = true; @@ -858,16 +921,19 @@ class HashIndexMock final : public arangodb::Index { TRI_ASSERT(attrNode->type == arangodb::aql::NODE_TYPE_ATTRIBUTE_ACCESS || attrNode->type == arangodb::aql::NODE_TYPE_REFERENCE); while (attrNode->type == arangodb::aql::NODE_TYPE_ATTRIBUTE_ACCESS) { - attributesRight.emplace_back(std::string(attrNode->getStringValue(), attrNode->getStringLength()), false); + attributesRight.emplace_back(std::string(attrNode->getStringValue(), + attrNode->getStringLength()), + false); attrNode = attrNode->getMember(0); } - attributes.insert(attributes.end(), attributesRight.crbegin(), attributesRight.crend()); + attributes.insert(attributes.end(), attributesRight.crbegin(), + attributesRight.crend()); } allAttributes.emplace_back(std::move(attributes), valNode); } size_t nullsCount = 0; for (auto const& f : _fields) { - auto it = std::find_if(allAttributes.cbegin(), allAttributes.cend(), [&f] (auto const& attrs) { + auto it = std::find_if(allAttributes.cbegin(), allAttributes.cend(), [&f](auto const& attrs) { return arangodb::basics::AttributeName::isIdentical(attrs.first, f, true); }); if (it != allAttributes.cend()) { @@ -883,8 +949,7 @@ class HashIndexMock final : public arangodb::Index { keys->close(); return std::make_unique(&_collection, trx, this, - _hashData, - std::move(keys)); + _hashData, std::move(keys)); } HashIndexMock(arangodb::IndexId iid, arangodb::LogicalCollection& collection, @@ -1047,7 +1112,8 @@ bool PhysicalCollectionMock::dropIndex(arangodb::IndexId iid) { return false; } -void PhysicalCollectionMock::figuresSpecific(bool /*details*/, arangodb::velocypack::Builder&) { +void PhysicalCollectionMock::figuresSpecific(bool /*details*/, + arangodb::velocypack::Builder&) { before(); TRI_ASSERT(false); } @@ -1074,9 +1140,10 @@ void PhysicalCollectionMock::getPropertiesVPack(arangodb::velocypack::Builder&) before(); } -arangodb::Result PhysicalCollectionMock::insert( - arangodb::transaction::Methods* trx, arangodb::velocypack::Slice newSlice, - arangodb::ManagedDocumentResult& result, arangodb::OperationOptions& options) { +arangodb::Result PhysicalCollectionMock::insert(arangodb::transaction::Methods* trx, + arangodb::velocypack::Slice newSlice, + arangodb::ManagedDocumentResult& result, + arangodb::OperationOptions& options) { before(); TRI_ASSERT(newSlice.isObject()); @@ -1100,7 +1167,8 @@ arangodb::Result PhysicalCollectionMock::insert( TRI_ASSERT(builder.slice().get(arangodb::StaticStrings::KeyString).isString()); arangodb::velocypack::StringRef key{builder.slice().get(arangodb::StaticStrings::KeyString)}; - arangodb::LocalDocumentId id = ::generateDocumentId(_logicalCollection, revisionId, _lastDocumentId); + arangodb::LocalDocumentId id = + ::generateDocumentId(_logicalCollection, revisionId, _lastDocumentId); auto const& [ref, didInsert] = _documents.emplace(key, DocElement{builder.steal(), id.id()}); TRI_ASSERT(didInsert); @@ -1154,7 +1222,7 @@ arangodb::Result PhysicalCollectionMock::lookupKey( arangodb::transaction::Methods*, arangodb::velocypack::StringRef key, std::pair& result) const { before(); - + auto it = _documents.find(arangodb::velocypack::StringRef{key}); if (it != _documents.end()) { if (_documents.find(arangodb::velocypack::StringRef{key}) != _documents.end()) { @@ -1233,9 +1301,9 @@ void PhysicalCollectionMock::prepareIndexes(arangodb::velocypack::Slice indexesS } } -arangodb::Result PhysicalCollectionMock::read(arangodb::transaction::Methods*, - arangodb::velocypack::StringRef const& key, - arangodb::IndexIterator::DocumentCallback const& cb) const { +arangodb::Result PhysicalCollectionMock::read( + arangodb::transaction::Methods*, arangodb::velocypack::StringRef const& key, + arangodb::IndexIterator::DocumentCallback const& cb) const { before(); auto it = _documents.find(key); if (it != _documents.end()) { @@ -1273,9 +1341,10 @@ bool PhysicalCollectionMock::readDocument(arangodb::transaction::Methods* trx, return false; } -arangodb::Result PhysicalCollectionMock::remove( - arangodb::transaction::Methods& trx, arangodb::velocypack::Slice slice, - arangodb::ManagedDocumentResult& previous, arangodb::OperationOptions& options) { +arangodb::Result PhysicalCollectionMock::remove(arangodb::transaction::Methods& trx, + arangodb::velocypack::Slice slice, + arangodb::ManagedDocumentResult& previous, + arangodb::OperationOptions& options) { before(); auto key = slice.get(arangodb::StaticStrings::KeyString); @@ -1293,10 +1362,11 @@ arangodb::Result PhysicalCollectionMock::remove( return arangodb::Result(TRI_ERROR_ARANGO_DOCUMENT_NOT_FOUND); } -arangodb::Result PhysicalCollectionMock::update( - arangodb::transaction::Methods* trx, arangodb::velocypack::Slice newSlice, - arangodb::ManagedDocumentResult& result, arangodb::OperationOptions& options, - arangodb::ManagedDocumentResult& previous) { +arangodb::Result PhysicalCollectionMock::update(arangodb::transaction::Methods* trx, + arangodb::velocypack::Slice newSlice, + arangodb::ManagedDocumentResult& result, + arangodb::OperationOptions& options, + arangodb::ManagedDocumentResult& previous) { return updateInternal(trx, newSlice, result, options, previous, true); } @@ -1342,7 +1412,8 @@ arangodb::Result PhysicalCollectionMock::updateInternal( TRI_ASSERT(doc.isObject()); arangodb::RevisionId oldRev = arangodb::RevisionId::fromSlice(doc); if (!checkRevision(trx, expectedRev, oldRev)) { - return arangodb::Result(TRI_ERROR_ARANGO_CONFLICT, "_rev values mismatch"); + return arangodb::Result(TRI_ERROR_ARANGO_CONFLICT, + "_rev values mismatch"); } } arangodb::velocypack::Builder builder; @@ -1389,14 +1460,15 @@ arangodb::Result PhysicalCollectionMock::updateProperties(arangodb::velocypack:: } std::shared_ptr StorageEngineMock::buildLinkMock( - arangodb::IndexId id, arangodb::LogicalCollection& collection, VPackSlice const& info) { + arangodb::IndexId id, arangodb::LogicalCollection& collection, VPackSlice const& info) { auto index = std::shared_ptr( - new arangodb::iresearch::IResearchLinkMock(id, collection)); - auto res = static_cast(index.get())->init(info, [](irs::directory& dir) { - if (arangodb::iresearch::IResearchLinkMock::InitCallback != nullptr) { - arangodb::iresearch::IResearchLinkMock::InitCallback(dir); - } - }); + new arangodb::iresearch::IResearchLinkMock(id, collection)); + auto res = static_cast(index.get()) + ->init(info, [](irs::directory& dir) { + if (arangodb::iresearch::IResearchLinkMock::InitCallback != nullptr) { + arangodb::iresearch::IResearchLinkMock::InitCallback(dir); + } + }); if (!res.ok()) { THROW_ARANGO_EXCEPTION(res); @@ -1418,15 +1490,13 @@ StorageEngineMock::StorageEngineMock(arangodb::application_features::Application vocbaseCount(1), _releasedTick(0) {} -arangodb::HealthData StorageEngineMock::healthCheck() { - return {}; -} +arangodb::HealthData StorageEngineMock::healthCheck() { return {}; } arangodb::WalAccess const* StorageEngineMock::walAccess() const { TRI_ASSERT(false); return nullptr; } - + void StorageEngineMock::addOptimizerRules(arangodb::aql::OptimizerRulesFeature& /*feature*/) { before(); // NOOP @@ -1436,9 +1506,7 @@ void StorageEngineMock::addRestHandlers(arangodb::rest::RestHandlerFactory& hand TRI_ASSERT(false); } -void StorageEngineMock::addV8Functions() { - TRI_ASSERT(false); -} +void StorageEngineMock::addV8Functions() { TRI_ASSERT(false); } void StorageEngineMock::changeCollection(TRI_vocbase_t& vocbase, arangodb::LogicalCollection const& collection, @@ -1528,7 +1596,7 @@ arangodb::Result StorageEngineMock::createView(TRI_vocbase_t& vocbase, return arangodb::Result(TRI_ERROR_NO_ERROR); // assume mock view persisted OK } - + arangodb::Result StorageEngineMock::compactAll(bool changeLevels, bool compactBottomMostLevel) { TRI_ASSERT(false); return arangodb::Result(); @@ -1588,8 +1656,8 @@ void StorageEngineMock::getCollectionInfo(TRI_vocbase_t& vocbase, arangodb::Data } ErrorCode StorageEngineMock::getCollectionsAndIndexes(TRI_vocbase_t& vocbase, - arangodb::velocypack::Builder& result, - bool wasCleanShutdown, bool isUpgrade) { + arangodb::velocypack::Builder& result, + bool wasCleanShutdown, bool isUpgrade) { TRI_ASSERT(false); return TRI_ERROR_INTERNAL; } @@ -1627,7 +1695,8 @@ arangodb::velocypack::Builder StorageEngineMock::getReplicationApplierConfigurat return arangodb::velocypack::Builder(); } -ErrorCode StorageEngineMock::getViews(TRI_vocbase_t& vocbase, arangodb::velocypack::Builder& result) { +ErrorCode StorageEngineMock::getViews(TRI_vocbase_t& vocbase, + arangodb::velocypack::Builder& result) { result.openArray(); for (auto& entry : views) { @@ -1656,9 +1725,9 @@ TRI_voc_tick_t StorageEngineMock::recoveryTick() { return recoveryTickResult; } -arangodb::Result StorageEngineMock::lastLogger( - TRI_vocbase_t& vocbase, - uint64_t tickStart, uint64_t tickEnd, arangodb::velocypack::Builder& builderSPtr) { +arangodb::Result StorageEngineMock::lastLogger(TRI_vocbase_t& vocbase, + uint64_t tickStart, uint64_t tickEnd, + arangodb::velocypack::Builder& builderSPtr) { TRI_ASSERT(false); return arangodb::Result(TRI_ERROR_NOT_IMPLEMENTED); } @@ -1701,14 +1770,14 @@ arangodb::Result StorageEngineMock::renameCollection(TRI_vocbase_t& vocbase, return arangodb::Result(TRI_ERROR_INTERNAL); } -ErrorCode StorageEngineMock::saveReplicationApplierConfiguration(TRI_vocbase_t& vocbase, - arangodb::velocypack::Slice slice, - bool doSync) { +ErrorCode StorageEngineMock::saveReplicationApplierConfiguration( + TRI_vocbase_t& vocbase, arangodb::velocypack::Slice slice, bool doSync) { TRI_ASSERT(false); return TRI_ERROR_NO_ERROR; } -ErrorCode StorageEngineMock::saveReplicationApplierConfiguration(arangodb::velocypack::Slice, bool) { +ErrorCode StorageEngineMock::saveReplicationApplierConfiguration(arangodb::velocypack::Slice, + bool) { TRI_ASSERT(false); return TRI_ERROR_NO_ERROR; } @@ -1783,7 +1852,8 @@ arangodb::Result TransactionCollectionMock::lockUsage() { } } - return arangodb::Result(_collection ? TRI_ERROR_NO_ERROR : TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND); + return arangodb::Result(_collection ? TRI_ERROR_NO_ERROR + : TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND); } arangodb::Result TransactionCollectionMock::doLock(arangodb::AccessMode::Type type) { @@ -1818,7 +1888,7 @@ TransactionStateMock::TransactionStateMock(TRI_vocbase_t& vocbase, arangodb::Tra arangodb::Result TransactionStateMock::abortTransaction(arangodb::transaction::Methods* trx) { ++abortTransactionCount; updateStatus(arangodb::transaction::Status::ABORTED); -// releaseUsage(); + // releaseUsage(); resetTransactionId(); return arangodb::Result(); @@ -1827,13 +1897,12 @@ arangodb::Result TransactionStateMock::abortTransaction(arangodb::transaction::M arangodb::Result TransactionStateMock::beginTransaction(arangodb::transaction::Hints hints) { ++beginTransactionCount; _hints = hints; - + arangodb::Result res = useCollections(); - if (res.fail()) { // something is wrong + if (res.fail()) { // something is wrong return res; } - if (!res.ok()) { updateStatus(arangodb::transaction::Status::ABORTED); resetTransactionId(); @@ -1851,7 +1920,7 @@ arangodb::Result TransactionStateMock::commitTransaction(arangodb::transaction:: return arangodb::Result(); } - + uint64_t TransactionStateMock::numCommits() const { return commitTransactionCount; } diff --git a/tests/Transaction/ManagerSetup.h b/tests/Transaction/ManagerSetup.h index 924a3947ed76..6b4b13a4397a 100644 --- a/tests/Transaction/ManagerSetup.h +++ b/tests/Transaction/ManagerSetup.h @@ -44,23 +44,23 @@ namespace arangodb { namespace tests { namespace mocks { - - // ----------------------------------------------------------------------------- - // --SECTION-- setup / tear-down - // ----------------------------------------------------------------------------- - - struct TransactionManagerSetup { - arangodb::tests::mocks::MockAqlServer server; - TransactionManagerSetup() : server(false) { - server.addFeature(true); - server.startFeatures(); - } +// ----------------------------------------------------------------------------- +// --SECTION-- setup / tear-down +// ----------------------------------------------------------------------------- - ~TransactionManagerSetup() = default; - }; +struct TransactionManagerSetup { + arangodb::tests::mocks::MockAqlServer server; -} // namespace tests -} // namespace mocks -} // namespace arangodb + TransactionManagerSetup() : server(false) { + TRI_ASSERT(server.server().hasFeature()); + server.startFeatures(); + } + + ~TransactionManagerSetup() = default; +}; + +} // namespace mocks +} // namespace tests +} // namespace arangodb #endif diff --git a/tests/js/server/aql/aql-graph-traverser.js b/tests/js/server/aql/aql-graph-traverser.js index 0569968691a4..60a502cba83b 100644 --- a/tests/js/server/aql/aql-graph-traverser.js +++ b/tests/js/server/aql/aql-graph-traverser.js @@ -2429,7 +2429,12 @@ function complexFilteringSuite() { // 1 Edge (Tri1->Tri2) // 1 Primary (Tri2) - assertEqual(stats.scannedIndex, 1); + if (isCluster) { + // one edge and one vertex lookup + assertEqual(stats.scannedIndex, 2); + } else { + assertEqual(stats.scannedIndex, 1); + } assertEqual(stats.filtered, 1); }, @@ -2513,7 +2518,7 @@ function complexFilteringSuite() { // 2 Primary lookup B,D // 2 Edge Lookups (2 B) (0 D) // 2 Primary Lookups (C, F) - assertTrue(stats.scannedIndex <= 5); + assertTrue(stats.scannedIndex <= 9); } else { // 2 Edge Lookups (A) // 2 Primary (B, D) for Filtering @@ -2556,7 +2561,7 @@ function complexFilteringSuite() { // 1 Primary lookup A // 2 Primary lookup B,D // 4 Primary Lookups (C, F, E, G) - assertTrue(stats.scannedIndex <= 7); + assertTrue(stats.scannedIndex <= 13); } else { // 2 Edge Lookups (A) // 4 Edge Lookups (2 B) (2 D) @@ -2600,7 +2605,7 @@ function complexFilteringSuite() { // 2 Primary lookup B,D // 2 Edge Lookups (0 B) (2 D) // 2 Primary Lookups (E, G) - assertTrue(stats.scannedIndex <= 5); + assertTrue(stats.scannedIndex <= 8); } else { // 2 Edge Lookups (A) // 2 Primary Lookups for Eval (B, D) @@ -2641,7 +2646,7 @@ function complexFilteringSuite() { // 1 Primary (B) // 2 Edge // 2 Primary (C,F) - assertTrue(stats.scannedIndex <= 4); + assertTrue(stats.scannedIndex <= 7); } else { // 2 Edge Lookups (A) // 2 Edge Lookups (B) @@ -2684,7 +2689,7 @@ function complexFilteringSuite() { // they may be inserted in the vertexToFetch list, which // lazy loads all vertices in it. if (stats.scannedIndex !== 8) { - assertTrue(stats.scannedIndex <= 5); + assertTrue(stats.scannedIndex <= 10); } } else { // 2 Edge Lookups (A) @@ -2739,7 +2744,7 @@ function complexFilteringSuite() { // 2 Primary lookup B,D // 2 Edge Lookups (2 B) (0 D) // 2 Primary Lookups (C, F) - assertTrue(stats.scannedIndex <= 5, stats.scannedIndex); + assertTrue(stats.scannedIndex <= 8, stats.scannedIndex); } else { // Cluster uses a lookup cache. // Pointless in single-server mode @@ -2797,7 +2802,7 @@ function complexFilteringSuite() { // 2 Primary lookup B,D // 2 Edge Lookups (2 B) (0 D) // 2 Primary Lookups (C, F) - assertTrue(stats.scannedIndex <= 5); + assertTrue(stats.scannedIndex <= 8); } else { // Cluster uses a lookup cache. // Pointless in single-server mode @@ -2847,7 +2852,7 @@ function complexFilteringSuite() { // 2 Primary lookup B,D // 2 Edge Lookups (2 B) (0 D) // 2 Primary Lookups (C, F) - assertTrue(stats.scannedIndex <= 7); + assertTrue(stats.scannedIndex <= 13, stats.scannedIndex); } else { // 2 Edge Lookups (A) // 2 Primary (B, D) for Filtering @@ -3619,7 +3624,7 @@ function optimizeQuantifierSuite() { let stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 5); + assertTrue(stats.scannedIndex <= 9, stats.scannedIndex); } else { // With traverser-read-cache // assertEqual(stats.scannedIndex, 9); @@ -3664,7 +3669,7 @@ function optimizeQuantifierSuite() { let stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 4); + assertTrue(stats.scannedIndex <= 8, stats.scannedIndex); } else { // With traverser-read-cache // assertEqual(stats.scannedIndex, 8); @@ -3693,7 +3698,7 @@ function optimizeQuantifierSuite() { stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 4); + assertTrue(stats.scannedIndex <= 8, stats.scannedIndex); } else { // With traverser-read-cache // assertEqual(stats.scannedIndex, 8); @@ -3723,7 +3728,7 @@ function optimizeQuantifierSuite() { let stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 5); + assertTrue(stats.scannedIndex <= 9, stats.scannedIndex); } else { // With traverser-read-cache // assertEqual(stats.scannedIndex, 9); @@ -3768,7 +3773,7 @@ function optimizeQuantifierSuite() { let stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 4); + assertTrue(stats.scannedIndex <= 8, stats.scannedIndex); } else { // With traverser-read-cache // assertEqual(stats.scannedIndex, 8); @@ -3796,7 +3801,7 @@ function optimizeQuantifierSuite() { stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 4); + assertTrue(stats.scannedIndex <= 8); } else { // With traverser-read-cache // assertEqual(stats.scannedIndex, 8); @@ -3827,7 +3832,7 @@ function optimizeQuantifierSuite() { let stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 5); + assertTrue(stats.scannedIndex <= 9, stats.scannedIndex); } else { // With traverser-read-cache // assertEqual(stats.scannedIndex, 9); @@ -3858,7 +3863,7 @@ function optimizeQuantifierSuite() { let stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 3); + assertTrue(stats.scannedIndex <= 7, stats.scannedIndex); } else { // With activated traverser-read-cache: // assertEqual(stats.scannedIndex, 7); @@ -3889,7 +3894,7 @@ function optimizeQuantifierSuite() { let stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 5); + assertTrue(stats.scannedIndex <= 9, stats.scannedIndex); } else { // With traverser-read-cache // assertEqual(stats.scannedIndex, 9); @@ -3921,7 +3926,7 @@ function optimizeQuantifierSuite() { let stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 3); + assertTrue(stats.scannedIndex <= 7, stats.scannedIndex); } else { // With activated traverser-read-cache: // assertEqual(stats.scannedIndex, 7); @@ -3953,7 +3958,7 @@ function optimizeQuantifierSuite() { let stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 5); + assertTrue(stats.scannedIndex <= 9, stats.scannedIndex); } else { // With activated traverser-read-cache: // assertEqual(stats.scannedIndex, 9); @@ -3985,7 +3990,7 @@ function optimizeQuantifierSuite() { let stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 3); + assertTrue(stats.scannedIndex <= 6, stats.scannedIndex); } else { // With activated traverser-read-cache: // assertEqual(stats.scannedIndex, 7); From 94484e39185ca3a1497ac176dc943d65e1dc8c22 Mon Sep 17 00:00:00 2001 From: Heiko Date: Mon, 5 Jul 2021 10:39:08 +0200 Subject: [PATCH 07/72] renamed evalExpression -> evaluateVertexExpression (#14449) --- arangod/Graph/PathManagement/PathValidator.cpp | 4 ++-- arangod/Graph/PathManagement/PathValidator.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arangod/Graph/PathManagement/PathValidator.cpp b/arangod/Graph/PathManagement/PathValidator.cpp index 9c9d7f7a90be..8bd7e15ce00b 100644 --- a/arangod/Graph/PathManagement/PathValidator.cpp +++ b/arangod/Graph/PathManagement/PathValidator.cpp @@ -167,7 +167,7 @@ auto PathValidator::evaluateVertexCon _provider.addVertexToBuilder(step.getVertex(), _tmpObjectBuilder); // evaluate expression - bool satifiesCondition = evaluateExpression(expr, _tmpObjectBuilder.slice()); + bool satifiesCondition = evaluateVertexExpression(expr, _tmpObjectBuilder.slice()); if (!satifiesCondition) { return ValidationResult{ValidationResult::Type::FILTER}; } @@ -176,7 +176,7 @@ auto PathValidator::evaluateVertexCon } template -auto PathValidator::evaluateExpression( +auto PathValidator::evaluateVertexExpression( arangodb::aql::Expression* expression, VPackSlice value) -> bool { if (expression == nullptr) { return true; diff --git a/arangod/Graph/PathManagement/PathValidator.h b/arangod/Graph/PathManagement/PathValidator.h index a3971bf50d0e..297994eb0c1c 100644 --- a/arangod/Graph/PathManagement/PathValidator.h +++ b/arangod/Graph/PathManagement/PathValidator.h @@ -92,7 +92,7 @@ class PathValidator { auto exposeUniqueVertices() const -> ::arangodb::containers::HashSet, std::equal_to> const&; - auto evaluateExpression(arangodb::aql::Expression* expression, + auto evaluateVertexExpression(arangodb::aql::Expression* expression, arangodb::velocypack::Slice value) -> bool; }; } // namespace graph From 47bf4bb2fbf7c8c8a65a8cf2289354d0d943c39c Mon Sep 17 00:00:00 2001 From: Heiko Date: Mon, 5 Jul 2021 10:40:26 +0200 Subject: [PATCH 08/72] fix init of variables (#14442) --- arangod/Graph/Enumerators/OneSidedEnumerator.cpp | 4 ++-- arangod/Graph/Enumerators/OneSidedEnumerator.h | 7 ++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/arangod/Graph/Enumerators/OneSidedEnumerator.cpp b/arangod/Graph/Enumerators/OneSidedEnumerator.cpp index ff6e67361f86..1cbc21ce5b5d 100644 --- a/arangod/Graph/Enumerators/OneSidedEnumerator.cpp +++ b/arangod/Graph/Enumerators/OneSidedEnumerator.cpp @@ -56,8 +56,8 @@ OneSidedEnumerator::OneSidedEnumerator(Provider&& forwardProvider : _options(std::move(options)), _queue(resourceMonitor), _provider(std::move(forwardProvider)), - _validator(_provider, _interior, std::move(validatorOptions)), - _interior(resourceMonitor) {} + _interior(resourceMonitor), + _validator(_provider, _interior, std::move(validatorOptions)) {} template OneSidedEnumerator::~OneSidedEnumerator() = default; diff --git a/arangod/Graph/Enumerators/OneSidedEnumerator.h b/arangod/Graph/Enumerators/OneSidedEnumerator.h index 8e821fca2a1a..80e2ee7a898b 100644 --- a/arangod/Graph/Enumerators/OneSidedEnumerator.h +++ b/arangod/Graph/Enumerators/OneSidedEnumerator.h @@ -146,13 +146,10 @@ class OneSidedEnumerator : public TraversalEnumerator { bool _resultsFetched{false}; aql::TraversalStats _stats{}; - // The next elements to process - typename Configuration::Queue _queue; + typename Configuration::Queue _queue; // The next elements to process typename Configuration::Provider _provider; + typename Configuration::Store _interior; // This stores all paths processed typename Configuration::Validator _validator; - - // This stores all paths processed - typename Configuration::Store _interior; }; } // namespace graph } // namespace arangodb From 0cc1c0750cb413190a2886bd7a5c562c5cf891c4 Mon Sep 17 00:00:00 2001 From: Heiko Date: Mon, 5 Jul 2021 10:46:19 +0200 Subject: [PATCH 09/72] unify method used in cursor readAll - GORDO-1210 (#14443) --- .../RefactoredSingleServerEdgeCursor.cpp | 77 +++++++++++-------- .../RefactoredSingleServerEdgeCursor.h | 2 +- arangod/RocksDBEngine/RocksDBEdgeIndex.cpp | 7 +- .../aql-graph-traversal-generic-tests.js | 16 +++- 4 files changed, 65 insertions(+), 37 deletions(-) diff --git a/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.cpp b/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.cpp index 930cd63ee9a5..1c8659abc16a 100644 --- a/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.cpp +++ b/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.cpp @@ -176,21 +176,48 @@ void RefactoredSingleServerEdgeCursor::readAll(SingleServerProvider& provider, TRI_ASSERT(!_lookupInfo.empty()); VPackBuilder tmpBuilder; - auto handleExpression = [&](aql::Expression* expression, - EdgeDocumentToken edgeToken, VPackSlice edge) { + auto evaluateEdgeExpressionHelper = [&](aql::Expression* expression, + EdgeDocumentToken edgeToken, VPackSlice edge) { if (edge.isString()) { tmpBuilder.clear(); provider.insertEdgeIntoResult(edgeToken, tmpBuilder); edge = tmpBuilder.slice(); } - return evaluateExpression(expression, edge); + return evaluateEdgeExpression(expression, edge); + }; + + auto evaluateLookupInfos = [&](EdgeDocumentToken const& edgeToken, + VPackSlice const& edge) -> bool { + bool foundDepthInfo = + (_depthLookupInfo.find(depth) == _depthLookupInfo.end()) ? false : true; + if (foundDepthInfo && _depthLookupInfo.at(depth).size() > 0 && + _depthLookupInfo.at(depth)[_currentCursor].getExpression() != nullptr) { + if (!evaluateEdgeExpressionHelper( + _depthLookupInfo.at(depth)[_currentCursor].getExpression(), edgeToken, edge)) { + stats.incrFiltered(); + return false; + } + } else { + // eval global expression if available AND only if no depth specific expression needs to be handled before + if (_lookupInfo[_currentCursor].getExpression() != nullptr) { + if (!evaluateEdgeExpressionHelper(_lookupInfo[_currentCursor].getExpression(), + edgeToken, edge)) { + stats.incrFiltered(); + return false; + } + } + } + + return true; }; for (_currentCursor = 0; _currentCursor < _lookupInfo.size(); ++_currentCursor) { auto& cursor = _lookupInfo[_currentCursor].cursor(); LogicalCollection* collection = cursor.collection(); auto cid = collection->id(); - if (cursor.hasExtra()) { + bool hasExtra = cursor.hasExtra(); + + if (hasExtra) { cursor.allExtra([&](LocalDocumentId const& token, VPackSlice edge) { stats.addScannedIndex(1); #ifdef USE_ENTERPRISE @@ -201,24 +228,9 @@ void RefactoredSingleServerEdgeCursor::readAll(SingleServerProvider& provider, EdgeDocumentToken edgeToken(cid, token); // eval depth-based expression first if available - bool foundDepthInfo = - (_depthLookupInfo.find(depth) == _depthLookupInfo.end()) ? false : true; - if (foundDepthInfo && _depthLookupInfo.at(depth).size() > 0 && - _depthLookupInfo.at(depth)[_currentCursor].getExpression() != nullptr) { - if (!handleExpression(_depthLookupInfo.at(depth)[_currentCursor].getExpression(), - edgeToken, edge)) { - stats.incrFiltered(); - return false; - } - } else { - // eval global expression if available AND only if no depth specific expression needs to be handled before - if (_lookupInfo[_currentCursor].getExpression() != nullptr) { - if (!handleExpression(_lookupInfo[_currentCursor].getExpression(), - edgeToken, edge)) { - stats.incrFiltered(); - return false; - } - } + bool needToRead = evaluateLookupInfos(edgeToken, edge); + if (!needToRead) { + return false; } callback(std::move(edgeToken), edge, _currentCursor); @@ -242,17 +254,14 @@ void RefactoredSingleServerEdgeCursor::readAll(SingleServerProvider& provider, } } #endif - // eval expression if available - if (_lookupInfo[_currentCursor].getExpression() != nullptr) { - bool result = - evaluateExpression(_lookupInfo[_currentCursor].getExpression(), - edgeDoc); - if (!result) { - stats.incrFiltered(); - return false; - } + // eval depth-based expression first if available + EdgeDocumentToken edgeToken(cid, token); + bool needToRead = evaluateLookupInfos(edgeToken, edgeDoc); + if (!needToRead) { + return false; } - callback(EdgeDocumentToken(cid, token), edgeDoc, _currentCursor); + + callback(std::move(edgeToken), edgeDoc, _currentCursor); return true; }) .ok(); @@ -261,8 +270,8 @@ void RefactoredSingleServerEdgeCursor::readAll(SingleServerProvider& provider, } } -bool RefactoredSingleServerEdgeCursor::evaluateExpression(arangodb::aql::Expression* expression, - VPackSlice value) { +bool RefactoredSingleServerEdgeCursor::evaluateEdgeExpression(arangodb::aql::Expression* expression, + VPackSlice value) { if (expression == nullptr) { return true; } diff --git a/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.h b/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.h index 249d6c0d317e..646bd9f79541 100644 --- a/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.h +++ b/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.h @@ -112,7 +112,7 @@ class RefactoredSingleServerEdgeCursor { void rearm(VertexType vertex, uint64_t depth); - bool evaluateExpression(arangodb::aql::Expression* expression, VPackSlice value); + bool evaluateEdgeExpression(arangodb::aql::Expression* expression, VPackSlice value); }; } // namespace graph } // namespace arangodb diff --git a/arangod/RocksDBEngine/RocksDBEdgeIndex.cpp b/arangod/RocksDBEngine/RocksDBEdgeIndex.cpp index 08eca1ddabd7..d9ca02ef5f71 100644 --- a/arangod/RocksDBEngine/RocksDBEdgeIndex.cpp +++ b/arangod/RocksDBEngine/RocksDBEdgeIndex.cpp @@ -111,7 +111,12 @@ class RocksDBEdgeIndexLookupIterator final : public IndexIterator { char const* typeName() const override { return "edge-index-iterator"; } - bool hasExtra() const override { return true; } + bool hasExtra() const override { + TRI_IF_FAILURE("RocksDBEdgeIndex::disableHasExtra") { + return false; + } + return true; + } /// @brief we provide a method to provide the index attribute values /// while scanning the index diff --git a/js/server/modules/@arangodb/testutils/aql-graph-traversal-generic-tests.js b/js/server/modules/@arangodb/testutils/aql-graph-traversal-generic-tests.js index 568871bd479c..3d3a966f0de0 100644 --- a/js/server/modules/@arangodb/testutils/aql-graph-traversal-generic-tests.js +++ b/js/server/modules/@arangodb/testutils/aql-graph-traversal-generic-tests.js @@ -2567,7 +2567,7 @@ function testCompleteGraphDfsUniqueEdgesPathD2(testGraph) { checkResIsValidDfsOf(expectedPathsAsTree, actualPaths); } -function testCompleteGraphDfsUniqueVerticesPathD3(testGraph) { +function completeGraphDfsUniqueVerticesPathD3Helper(testGraph) { assertTrue(testGraph.name().startsWith(protoGraphs.completeGraph.name())); const query = aql` FOR v, e, p IN 0..3 OUTBOUND ${testGraph.vertex('A')} GRAPH ${testGraph.name()} OPTIONS {uniqueVertices: "path"} @@ -2641,6 +2641,19 @@ function testCompleteGraphDfsUniqueVerticesPathD3(testGraph) { checkResIsValidDfsOf(expectedPathsAsTree, actualPaths); } +function testCompleteGraphDfsUniqueVerticesPathD3(testGraph) { + completeGraphDfsUniqueVerticesPathD3Helper(testGraph); +} + +function testCompleteGraphDfsUniqueVerticesPathD3NotHasExtra(testGraph) { + internal.debugSetFailAt("RocksDBEdgeIndex::disableHasExtra"); + try { + completeGraphDfsUniqueVerticesPathD3Helper(testGraph); + } finally { + internal.debugRemoveFailAt("RocksDBEdgeIndex::disableHasExtra"); + } +} + function testCompleteGraphDfsUniqueVerticesUniqueEdgesPathD2(testGraph) { assertTrue(testGraph.name().startsWith(protoGraphs.completeGraph.name())); const query = aql` @@ -5871,6 +5884,7 @@ const testsByGraph = { testCompleteGraphDfsUniqueVerticesPathD1, testCompleteGraphDfsUniqueVerticesPathD2, testCompleteGraphDfsUniqueVerticesPathD3, + testCompleteGraphDfsUniqueVerticesPathD3NotHasExtra, testCompleteGraphDfsUniqueEdgesPathD1, testCompleteGraphDfsUniqueEdgesPathD2, testCompleteGraphDfsUniqueVerticesUniqueEdgesPathD2, From 71d34958d7b5082968a01e6d50757c347f2afd55 Mon Sep 17 00:00:00 2001 From: Heiko Date: Tue, 6 Jul 2021 17:44:30 +0200 Subject: [PATCH 10/72] Feature/hybrid smart graph testsuite extended (#14432) --- arangod/Aql/ExecutionEngine.cpp | 1 + tests/Mocks/MockGraph.cpp | 86 ++++++++++++++++++--------------- tests/Mocks/MockGraph.h | 10 ++-- tests/Mocks/Servers.cpp | 74 ++++++++++++++++------------ tests/Mocks/Servers.h | 18 +++++-- 5 files changed, 109 insertions(+), 80 deletions(-) diff --git a/arangod/Aql/ExecutionEngine.cpp b/arangod/Aql/ExecutionEngine.cpp index ea30d8680bfd..c21a22bd3741 100644 --- a/arangod/Aql/ExecutionEngine.cpp +++ b/arangod/Aql/ExecutionEngine.cpp @@ -632,6 +632,7 @@ void ExecutionEngine::instantiateFromPlan(Query& query, ExecutionEngine* engine = nullptr; #ifdef USE_ENTERPRISE bool const pushToSingleServer = plan.hasAppliedRule( + // Feature HybridSmartGraphs: Check here. static_cast(OptimizerRule::RuleLevel::clusterOneShardRule)); #else bool const pushToSingleServer = false; diff --git a/tests/Mocks/MockGraph.cpp b/tests/Mocks/MockGraph.cpp index 7dba1cc3273e..221fba80f046 100644 --- a/tests/Mocks/MockGraph.cpp +++ b/tests/Mocks/MockGraph.cpp @@ -88,7 +88,9 @@ void MockGraph::addEdge(size_t from, size_t to, double weight) { } void MockGraph::storeData(TRI_vocbase_t& vocbase, std::string const& vertexCollectionName, - std::string const& edgeCollectionName) const { + std::string const& edgeCollectionName, + std::string const& edgeCollectionSecondName, + std::vector const& secondEdges) const { { // Insert vertices arangodb::OperationOptions options; @@ -111,28 +113,36 @@ void MockGraph::storeData(TRI_vocbase_t& vocbase, std::string const& vertexColle EXPECT_TRUE(added == vertices().size()); } - { - // Insert edges - arangodb::OperationOptions options; - arangodb::SingleCollectionTransaction trx(arangodb::transaction::StandaloneContext::Create(vocbase), - edgeCollectionName, - arangodb::AccessMode::Type::WRITE); - EXPECT_TRUE((trx.begin().ok())); - size_t added = 0; - velocypack::Builder b; - for (auto& edge : edges()) { - b.clear(); - edge.addToBuilder(b); - auto res = trx.insert(edgeCollectionName, b.slice(), options); - if (res.fail()) { - LOG_DEVEL << res.errorMessage() << " " << b.toJson(); + auto insertEdges = [&](std::string const& edgeCollectionToInsert, + std::vector const& edges) -> void { + { + // Insert edges + arangodb::OperationOptions options; + arangodb::SingleCollectionTransaction trx( + arangodb::transaction::StandaloneContext::Create(vocbase), + edgeCollectionToInsert, arangodb::AccessMode::Type::WRITE); + EXPECT_TRUE((trx.begin().ok())); + size_t added = 0; + velocypack::Builder b; + for (auto& edge : edges) { + b.clear(); + edge.addToBuilder(b); + auto res = trx.insert(edgeCollectionToInsert, b.slice(), options); + if (res.fail()) { + LOG_DEVEL << res.errorMessage() << " " << b.toJson(); + } + EXPECT_TRUE((res.ok())); + added++; } - EXPECT_TRUE((res.ok())); - added++; + + EXPECT_TRUE((trx.commit().ok())); + EXPECT_TRUE(added == edges.size()); } + }; - EXPECT_TRUE((trx.commit().ok())); - EXPECT_TRUE(added == edges().size()); + insertEdges(edgeCollectionName, edges()); + if (!edgeCollectionSecondName.empty()) { + insertEdges(edgeCollectionSecondName, secondEdges); } } @@ -166,8 +176,6 @@ void MockGraph::prepareServer(MockCoordinator& server) const { getEdgeShardNameServerPairs(), TRI_COL_TYPE_EDGE); } - - template <> // Future: Also engineID's need to be returned here. std::pair, uint64_t> MockGraph::simulateApi( @@ -184,15 +192,15 @@ std::pair, uint64_t> MockG { // init restaqlhandler arangodb::tests::PreparedRequestResponse prep{server.getSystemDatabase()}; - + // generate and add body here VPackBuilder builder; builder.openObject(); builder.add("lockInfo", VPackValue(VPackValueType::Object)); builder.add("read", VPackValue(VPackValueType::Array)); - // append here the collection names (?) <-- TODO: Check RestAqlHandler.cpp:230 - // builder.add(VPackValue(_vertexCollectionName)); + // append here the collection names (?) <-- TODO: Check + // RestAqlHandler.cpp:230 builder.add(VPackValue(_vertexCollectionName)); // builder.add(VPackValue(_edgeCollectionName)); // appending collection shard ids for (auto const& vShard : _vertexShards) { @@ -201,8 +209,8 @@ std::pair, uint64_t> MockG for (auto const& eShard : _edgeShards) { builder.add(VPackValue(eShard.first)); } - builder.close(); // array READ - builder.close(); // object lockInfo + builder.close(); // array READ + builder.close(); // object lockInfo builder.add("options", VPackValue(VPackValueType::Object)); builder.add("ttl", VPackValue(120)); @@ -216,7 +224,7 @@ std::pair, uint64_t> MockG builder.add("traverserEngines", VPackValue(VPackValueType::Array)); - builder.openObject(); // main container + builder.openObject(); // main container builder.add(VPackValue("options")); @@ -228,25 +236,25 @@ std::pair, uint64_t> MockG builder.add(VPackValue("vertices")); builder.openObject(); - for (auto const& vertexTuple: getVertexShardNameServerPairs()) { + for (auto const& vertexTuple : getVertexShardNameServerPairs()) { builder.add(_vertexCollectionName, VPackValue(VPackValueType::Array)); - builder.add(VPackValue(vertexTuple.first)); // shardID - builder.close(); // inner array + builder.add(VPackValue(vertexTuple.first)); // shardID + builder.close(); // inner array } - builder.close(); // vertices + builder.close(); // vertices builder.add(VPackValue("edges")); builder.openArray(); - for (auto const& edgeTuple: getEdgeShardNameServerPairs()) { + for (auto const& edgeTuple : getEdgeShardNameServerPairs()) { builder.openArray(); - builder.add(VPackValue(edgeTuple.first)); // shardID - builder.close(); // inner array + builder.add(VPackValue(edgeTuple.first)); // shardID + builder.close(); // inner array } - builder.close(); // edges - builder.close(); // shards - builder.close(); // main container - builder.close(); // array traverserEngines + builder.close(); // edges + builder.close(); // shards + builder.close(); // main container + builder.close(); // array traverserEngines builder.close(); // object (outer) prep.addBody(builder.slice()); diff --git a/tests/Mocks/MockGraph.h b/tests/Mocks/MockGraph.h index 3ea12b4cdc9f..5305e9ff3519 100644 --- a/tests/Mocks/MockGraph.h +++ b/tests/Mocks/MockGraph.h @@ -121,9 +121,11 @@ class MockGraph { arangodb::graph::BaseOptions& opts) const; void storeData(TRI_vocbase_t& vocbase, std::string const& vertexCollectionName, - std::string const& edgeCollectionName) const; + std::string const& edgeCollectionName, + std::string const& edgeCollectionSecondName = "", + std::vector const& secondEdges = {}) const; - private: + protected: std::vector> const& getVertexShardNameServerPairs() const { return _vertexShards; } @@ -131,13 +133,13 @@ class MockGraph { return _edgeShards; } - private: + protected: std::vector _edges; std::unordered_set _vertices; - std::string _vertexCollectionName{"v"}; std::string _edgeCollectionName{"e"}; + private: std::vector> _vertexShards{ {"s9870", "PRMR_0001"}}; std::vector> _edgeShards{ diff --git a/tests/Mocks/Servers.cpp b/tests/Mocks/Servers.cpp index a471ddf1d71c..1186a089f54d 100644 --- a/tests/Mocks/Servers.cpp +++ b/tests/Mocks/Servers.cpp @@ -594,23 +594,10 @@ void MockClusterServer::agencyDropDatabase(std::string const& name) { .wait(); } -// Create a clusterWide Collection. -// This does NOT create Shards. -std::shared_ptr MockClusterServer::createCollection( - std::string const& dbName, std::string collectionName, - std::vector> shardNameToServerNamePairs, - TRI_col_type_e type) { - /* - std::string cID, uint64_t shards, - uint64_t replicationFactor, uint64_t writeConcern, - bool waitForRep, velocypack::Slice const& slice, - std::string coordinatorId, RebootId rebootId */ - // This is unsafe - std::string cid = "98765" + basics::StringUtils::itoa(type); - auto& databaseFeature = _server.getFeature(); - auto vocbase = databaseFeature.lookupDatabase(dbName); - - VPackBuilder props; +void MockClusterServer::buildCollectionProperties(VPackBuilder& props, + std::string const& collectionName, + std::string const& cid, TRI_col_type_e type, + VPackSlice additionalProperties) { { // This is hand-crafted unfortunately the code does not exist... VPackObjectBuilder guard(&props); @@ -638,24 +625,18 @@ std::shared_ptr MockClusterServer::createCollection( props.add(toIndex->slice()); } } - } - LogicalCollection dummy(*vocbase, props.slice(), true); - auto shards = std::make_shared(); - for (auto const& [shard, server] : shardNameToServerNamePairs) { - shards->emplace(shard, std::vector{server}); + if (additionalProperties.isObject()) { + props.add(VPackObjectIterator(additionalProperties)); + } } - dummy.setShardMap(shards); - - std::unordered_set const ignoreKeys{ - "allowUserKeys", "cid", "globallyUniqueId", "count", - "planId", "version", "objectId"}; - dummy.setStatus(TRI_VOC_COL_STATUS_LOADED); - VPackBuilder velocy = - dummy.toVelocyPackIgnore(ignoreKeys, LogicalDataSource::Serialization::List); +} +void MockClusterServer::injectCollectionToAgency( + std::string const& dbName, VPackBuilder& velocy, DataSourceId const& planId, + std::vector> shardNameToServerNamePairs) { agencyTrx("/arango/Plan/Collections/" + dbName + "/" + - basics::StringUtils::itoa(dummy.planId().id()), + basics::StringUtils::itoa(planId.id()), velocy.toJson()); { /* Hard-Coded section to inject the CURRENT counter part. @@ -685,7 +666,7 @@ std::shared_ptr MockClusterServer::createCollection( } } agencyTrx("/arango/Current/Collections/" + dbName + "/" + - basics::StringUtils::itoa(dummy.planId().id()), + basics::StringUtils::itoa(planId.id()), current.toJson()); } @@ -698,6 +679,35 @@ std::shared_ptr MockClusterServer::createCollection( .clusterInfo() .waitForCurrent(agencyTrx("/arango/Current/Version", R"=({"op":"increment"})=")) .wait(); +} + +// Create a clusterWide Collection. +// This does NOT create Shards. +std::shared_ptr MockClusterServer::createCollection( + std::string const& dbName, std::string collectionName, + std::vector> shardNameToServerNamePairs, + TRI_col_type_e type, VPackSlice additionalProperties) { + std::string cid = std::to_string(_server.getFeature().clusterInfo().uniqid()); + auto& databaseFeature = _server.getFeature(); + auto vocbase = databaseFeature.lookupDatabase(dbName); + + VPackBuilder props; + buildCollectionProperties(props, collectionName, cid, type, additionalProperties); + LogicalCollection dummy(*vocbase, props.slice(), true); + + auto shards = std::make_shared(); + for (auto const& [shard, server] : shardNameToServerNamePairs) { + shards->emplace(shard, std::vector{server}); + } + dummy.setShardMap(shards); + + std::unordered_set const ignoreKeys{ + "allowUserKeys", "cid", "globallyUniqueId", "count", + "planId", "version", "objectId"}; + dummy.setStatus(TRI_VOC_COL_STATUS_LOADED); + VPackBuilder velocy = + dummy.toVelocyPackIgnore(ignoreKeys, LogicalDataSource::Serialization::List); + injectCollectionToAgency(dbName, velocy, dummy.planId(), shardNameToServerNamePairs); ClusterInfo& clusterInfo = server().getFeature().clusterInfo(); return clusterInfo.getCollection(dbName, collectionName); diff --git a/tests/Mocks/Servers.h b/tests/Mocks/Servers.h index a5133c2a4ac6..2a8c2c94465b 100644 --- a/tests/Mocks/Servers.h +++ b/tests/Mocks/Servers.h @@ -202,7 +202,16 @@ class MockClusterServer : public MockServer, std::shared_ptr createCollection( std::string const& dbName, std::string collectionName, std::vector> shardNameToServerNamePairs, - TRI_col_type_e type); + TRI_col_type_e type, + VPackSlice additionalProperties = VPackSlice{VPackSlice::nullSlice()}); + + void buildCollectionProperties(VPackBuilder& props, std::string const& collectionName, + std::string const& cid, TRI_col_type_e type, + VPackSlice additionalProperties); + + void injectCollectionToAgency(std::string const& dbName, VPackBuilder& velocy, + DataSourceId const& planId, + std::vector> shardNameToServerNamePairs); // You can only create specialized types protected: @@ -213,17 +222,16 @@ class MockClusterServer : public MockServer, // Implementation knows the place when all features are included consensus::index_t agencyTrx(std::string const& key, std::string const& value); void agencyCreateDatabase(std::string const& name); - + // creation of collection is separated // as for DBerver at first maintenance should // create database and only after collections // will be populated in plan. void agencyCreateCollections(std::string const& name); - void agencyDropDatabase(std::string const& name); -protected: + + protected: std::unique_ptr _pool; - private: bool _useAgencyMockPool; arangodb::ServerState::RoleEnum _oldRole; int _dummy; From fa37f394eca575444308c230da5936c281fbfa9f Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Wed, 7 Jul 2021 10:15:55 +0200 Subject: [PATCH 11/72] Feature/hybrid smart graph test setup helper (#14455) --- ...EngineInfoContainerDBServerServerBased.cpp | 78 +++++++++++-------- .../EngineInfoContainerDBServerServerBased.h | 43 +++++----- arangod/ClusterEngine/ClusterIndexFactory.cpp | 73 +++++++++-------- arangod/ClusterEngine/ClusterIndexFactory.h | 16 ++-- .../Graph/Enumerators/OneSidedEnumerator.cpp | 1 + .../InternalRestTraverserHandler.cpp | 26 ++++--- .../StorageEngine/EngineSelectorFeature.cpp | 26 ++++--- tests/Mocks/Servers.cpp | 57 +++++++++++--- tests/Mocks/Servers.h | 22 +++++- tests/Mocks/StorageEngineMock.cpp | 34 +++++++- tests/Mocks/StorageEngineMock.h | 35 +++++---- 11 files changed, 263 insertions(+), 148 deletions(-) diff --git a/arangod/Aql/EngineInfoContainerDBServerServerBased.cpp b/arangod/Aql/EngineInfoContainerDBServerServerBased.cpp index a64039a74f41..33158ceec7c9 100644 --- a/arangod/Aql/EngineInfoContainerDBServerServerBased.cpp +++ b/arangod/Aql/EngineInfoContainerDBServerServerBased.cpp @@ -36,8 +36,8 @@ #include "StorageEngine/TransactionState.h" #include "Utils/CollectionNameResolver.h" -#include #include +#include using namespace arangodb; using namespace arangodb::aql; @@ -80,10 +80,13 @@ EngineInfoContainerDBServerServerBased::TraverserEngineShardLists::TraverserEngi auto const& edges = _node->edgeColls(); TRI_ASSERT(!edges.empty()); auto const& restrictToShards = query.queryOptions().restrictToShards; +#ifdef USE_ENTERPRISE + transaction::Methods trx{query.newTrxContext()}; +#endif // Extract the local shards for edge collections. for (auto const& col : edges) { #ifdef USE_ENTERPRISE - if (query.trxForOptimization().isInaccessibleCollection(col->id())) { + if (trx.isInaccessibleCollection(col->id())) { _inaccessible.insert(col->name()); _inaccessible.insert(std::to_string(col->id().id())); } @@ -100,7 +103,7 @@ EngineInfoContainerDBServerServerBased::TraverserEngineShardLists::TraverserEngi // Or if we guarantee to never read vertex data. for (auto const& col : vertices) { #ifdef USE_ENTERPRISE - if (query.trxForOptimization().isInaccessibleCollection(col->id())) { + if (trx.isInaccessibleCollection(col->id())) { _inaccessible.insert(col->name()); _inaccessible.insert(std::to_string(col->id().id())); } @@ -112,8 +115,8 @@ EngineInfoContainerDBServerServerBased::TraverserEngineShardLists::TraverserEngi } std::vector EngineInfoContainerDBServerServerBased::TraverserEngineShardLists::getAllLocalShards( - std::unordered_map const& shardMapping, - ServerID const& server, std::shared_ptr> shardIds, bool colIsSatellite) { + std::unordered_map const& shardMapping, ServerID const& server, + std::shared_ptr> shardIds, bool colIsSatellite) { std::vector localShards; for (auto const& shard : *shardIds) { auto const& it = shardMapping.find(shard); @@ -255,8 +258,7 @@ void EngineInfoContainerDBServerServerBased::closeSnippet(QueryId inputSnippet) } std::vector EngineInfoContainerDBServerServerBased::buildEngineInfo( - QueryId clusterQueryId, - VPackBuilder& infoBuilder, ServerID const& server, + QueryId clusterQueryId, VPackBuilder& infoBuilder, ServerID const& server, std::unordered_map const& nodesById, std::map& nodeAliases) { LOG_TOPIC("4bbe6", DEBUG, arangodb::Logger::AQL) @@ -322,11 +324,11 @@ arangodb::futures::Future EngineInfoContainerDBServerServerBased::buildS auto buildCallback = [this, server, serverDest, didCreateEngine = std::move(didCreateEngine), - &serverToQueryId, &serverToQueryIdLock, &snippetIds, globalId]( - arangodb::futures::Try const& response) -> Result { + &serverToQueryId, &serverToQueryIdLock, &snippetIds, + globalId](arangodb::futures::Try const& response) -> Result { auto const& resolvedResponse = response.get(); auto queryId = globalId; - + std::unique_lock guard{serverToQueryIdLock}; if (resolvedResponse.fail()) { @@ -334,7 +336,7 @@ arangodb::futures::Future EngineInfoContainerDBServerServerBased::buildS LOG_TOPIC("f9a77", DEBUG, Logger::AQL) << server << " responded with " << res.errorNumber() << ": " << res.errorMessage(); - + serverToQueryId.emplace_back(serverDest, globalId); return res; } @@ -351,8 +353,8 @@ arangodb::futures::Future EngineInfoContainerDBServerServerBased::buildS }; return network::sendRequestRetry(pool, serverDest, fuerte::RestVerb::Post, - "/_api/aql/setup", std::move(buffer), options, - std::move(headers)) + "/_api/aql/setup", std::move(buffer), + options, std::move(headers)) .then([buildCallback = std::move(buildCallback)](futures::Try&& resp) mutable { return buildCallback(resp); }); @@ -373,7 +375,8 @@ bool EngineInfoContainerDBServerServerBased::isNotSatelliteLeader(VPackSlice inf return true; } - TRI_ASSERT((infoSlice.get("snippets").isObject() && !infoSlice.get("snippets").isEmptyObject()) || + TRI_ASSERT((infoSlice.get("snippets").isObject() && + !infoSlice.get("snippets").isEmptyObject()) || infoSlice.hasKey("traverserEngines")); return false; @@ -406,7 +409,7 @@ Result EngineInfoContainerDBServerServerBased::buildEngines( TRI_ASSERT(!_closedSnippets.empty() || !_graphNodes.empty()); ErrorCode cleanupReason = TRI_ERROR_CLUSTER_TIMEOUT; - + auto cleanupGuard = scopeGuard([this, &serverToQueryId, &cleanupReason]() { // Fire and forget std::ignore = cleanupEngines(cleanupReason, _query.vocbase().name(), serverToQueryId); @@ -427,19 +430,21 @@ Result EngineInfoContainerDBServerServerBased::buildEngines( options.timeout = network::Timeout(SETUP_TIMEOUT); options.skipScheduler = true; // hack to speed up future.get() options.param("ttl", std::to_string(_query.queryOptions().ttl)); - + TRI_IF_FAILURE("Query::setupTimeout") { - options.timeout = network::Timeout(0.01 + (double) RandomGenerator::interval(uint32_t(10))); + options.timeout = + network::Timeout(0.01 + (double)RandomGenerator::interval(uint32_t(10))); } - + TRI_IF_FAILURE("Query::setupTimeoutFailSequence") { options.timeout = network::Timeout(0.5); } - + /// cluster global query id, under which the query will be registered /// on DB servers from 3.8 onwards. - QueryId clusterQueryId = _query.vocbase().server().getFeature().clusterInfo().uniqid(); - + QueryId clusterQueryId = + _query.vocbase().server().getFeature().clusterInfo().uniqid(); + // decreases lock timeout manually for fast path auto oldLockTimeout = _query.getLockTimeout(); _query.setLockTimeout(FAST_PATH_LOCK_TIMEOUT); @@ -449,7 +454,8 @@ Result EngineInfoContainerDBServerServerBased::buildEngines( for (ServerID const& server : dbServers) { // Build Lookup Infos VPackBuilder infoBuilder; - auto didCreateEngine = buildEngineInfo(clusterQueryId, infoBuilder, server, nodesById, nodeAliases); + auto didCreateEngine = + buildEngineInfo(clusterQueryId, infoBuilder, server, nodesById, nodeAliases); VPackSlice infoSlice = infoBuilder.slice(); if (isNotSatelliteLeader(infoSlice)) { @@ -468,7 +474,7 @@ Result EngineInfoContainerDBServerServerBased::buildEngines( .thenValue([](std::vector>&& responses) -> Result { // We can directly report a non TRI_ERROR_LOCK_TIMEOUT // error as we need to abort after. - // Otherwise we need to report + // Otherwise we need to report Result res{TRI_ERROR_NO_ERROR}; for (auto const& tryRes : responses) { auto response = tryRes.get(); @@ -498,16 +504,18 @@ Result EngineInfoContainerDBServerServerBased::buildEngines( { // in case of fast path failure, we need to cleanup engines - auto requests = cleanupEngines(fastPathResult.get().errorNumber(), _query.vocbase().name(), serverToQueryId); + auto requests = cleanupEngines(fastPathResult.get().errorNumber(), + _query.vocbase().name(), serverToQueryId); // Wait for all requests to complete. // So we know that all Transactions are aborted. // We do NOT care for the actual result. futures::collectAll(requests).wait(); snippetIds.clear(); } - + // we must generate a new query id, because the fast path setup has failed - clusterQueryId = _query.vocbase().server().getFeature().clusterInfo().uniqid(); + clusterQueryId = + _query.vocbase().server().getFeature().clusterInfo().uniqid(); // set back to default lock timeout for slow path fallback _query.setLockTimeout(oldLockTimeout); @@ -521,7 +529,8 @@ Result EngineInfoContainerDBServerServerBased::buildEngines( std::sort(engineInformation.begin(), engineInformation.end(), [](auto const& lhs, auto const& rhs) { // Entry <0> is the Server - return TransactionState::ServerIdLessThan(std::get<0>(lhs), std::get<0>(rhs)); + return TransactionState::ServerIdLessThan(std::get<0>(lhs), + std::get<0>(rhs)); }); #ifdef ARANGODB_ENABLE_MAINTAINER_MODE // Make sure we always maintain the correct ordering of servers @@ -547,7 +556,9 @@ Result EngineInfoContainerDBServerServerBased::buildEngines( overwrittenOptions.add("clusterQueryId", VPackValue(clusterQueryId)); addOptionsPart(overwrittenOptions, server); overwrittenOptions.close(); - auto newRequest = arangodb::velocypack::Collection::merge(infoSlice, overwrittenOptions.slice(), false); + auto newRequest = + arangodb::velocypack::Collection::merge(infoSlice, + overwrittenOptions.slice(), false); auto request = buildSetupRequest(trx, std::move(server), newRequest.slice(), std::move(didCreateEngine), snippetIds, serverToQueryId, @@ -680,7 +691,7 @@ std::vector EngineInfoContainerDBServerServerBased options.database = dbname; options.timeout = network::Timeout(10.0); // Picked arbitrarily options.skipScheduler = true; // hack to speed up future.get() - + // Shutdown query snippets std::string url("/_api/aql/finish/"); VPackBuffer body; @@ -691,8 +702,8 @@ std::vector EngineInfoContainerDBServerServerBased requests.reserve(serverQueryIds.size()); for (auto const& [server, queryId] : serverQueryIds) { requests.emplace_back(network::sendRequestRetry(pool, server, fuerte::RestVerb::Delete, - url + std::to_string(queryId), - /*copy*/ body, options)); + url + std::to_string(queryId), + /*copy*/ body, options)); } _query.incHttpRequests(static_cast(serverQueryIds.size())); @@ -703,8 +714,9 @@ std::vector EngineInfoContainerDBServerServerBased for (auto& gn : _graphNodes) { auto allEngines = gn->engines(); for (auto const& engine : *allEngines) { - requests.emplace_back(network::sendRequestRetry(pool, "server:" + engine.first, fuerte::RestVerb::Delete, - url + basics::StringUtils::itoa(engine.second), noBody, options)); + requests.emplace_back(network::sendRequestRetry( + pool, "server:" + engine.first, fuerte::RestVerb::Delete, + url + basics::StringUtils::itoa(engine.second), noBody, options)); } _query.incHttpRequests(static_cast(allEngines->size())); gn->clearEngines(); diff --git a/arangod/Aql/EngineInfoContainerDBServerServerBased.h b/arangod/Aql/EngineInfoContainerDBServerServerBased.h index 93cc23a4ca2f..83739592330f 100644 --- a/arangod/Aql/EngineInfoContainerDBServerServerBased.h +++ b/arangod/Aql/EngineInfoContainerDBServerServerBased.h @@ -40,7 +40,7 @@ namespace network { class ConnectionPool; struct RequestOptions; struct Response; -} +} // namespace network namespace velocypack { class Builder; @@ -56,6 +56,9 @@ class QuerySnippet; class EngineInfoContainerDBServerServerBased { private: + // TODO Temporary. Do not check in!! + // We need to access the TraverserEngineShardList nothing else. + public: // @brief Local struct to create the // information required to build traverser engines // on DB servers. @@ -123,7 +126,6 @@ class EngineInfoContainerDBServerServerBased { // the given queryid of the coordinator as data provider. void closeSnippet(QueryId inputSnippet); - // Build the Engines for the DBServer // * Creates one Query-Entry for each Snippet per Shard (multiple on the // same DB) @@ -140,28 +142,28 @@ class EngineInfoContainerDBServerServerBased { MapRemoteToSnippet& snippetIds, aql::ServerQueryIdList& serverQueryIds, std::map& nodeAliases); - // Insert a GraphNode that needs to generate TraverserEngines on // the DBServers. The GraphNode itself will retain on the coordinator. void addGraphNode(GraphNode* node, bool pushToSingleServer); private: /** - * @brief Helper method to generate the Request to be send to a specific database server. - * this request contains all the necessary information to create a transaction with correct shard - * locking, as well as QuerySnippets and GraphEngines on the receiver side. - * - * @param clusterQueryId cluster-wide query id (used from 3.8 onwards) - * @param infoBuilder (mutable) the request body will be written into this builder. - * @param server The DatabaseServer we suppose to send the request to, used to identify the shards on this server - * @param nodesById A vector to get Nodes by their id. - * @param nodeAliases (mutable) A map of node-aliases, if a server is responsible for more then one shard we need to duplicate some nodes in the query (e.g. an IndexNode can only access one shard at a time) this list can map cloned node -> original node ids. - * - * @return A vector with one entry per GraphNode in the query (in order) it indicates if this Server has created a GraphEngine for this Node and needs to participate in the GraphOperation or not. - */ - std::vector buildEngineInfo(QueryId clusterQueryId, VPackBuilder& infoBuilder, ServerID const& server, - std::unordered_map const& nodesById, - std::map& nodeAliases); + * @brief Helper method to generate the Request to be send to a specific database server. + * this request contains all the necessary information to create a transaction with correct shard + * locking, as well as QuerySnippets and GraphEngines on the receiver side. + * + * @param clusterQueryId cluster-wide query id (used from 3.8 onwards) + * @param infoBuilder (mutable) the request body will be written into this builder. + * @param server The DatabaseServer we suppose to send the request to, used to identify the shards on this server + * @param nodesById A vector to get Nodes by their id. + * @param nodeAliases (mutable) A map of node-aliases, if a server is responsible for more then one shard we need to duplicate some nodes in the query (e.g. an IndexNode can only access one shard at a time) this list can map cloned node -> original node ids. + * + * @return A vector with one entry per GraphNode in the query (in order) it indicates if this Server has created a GraphEngine for this Node and needs to participate in the GraphOperation or not. + */ + std::vector buildEngineInfo( + QueryId clusterQueryId, VPackBuilder& infoBuilder, ServerID const& server, + std::unordered_map const& nodesById, + std::map& nodeAliases); arangodb::futures::Future buildSetupRequest( transaction::Methods& trx, ServerID const& server, VPackSlice infoSlice, @@ -171,7 +173,6 @@ class EngineInfoContainerDBServerServerBased { [[nodiscard]] bool isNotSatelliteLeader(VPackSlice infoSlice) const; - /** * @brief Will send a shutdown to all engines registered in the list of * queryIds. @@ -186,8 +187,8 @@ class EngineInfoContainerDBServerServerBased { * -> queryid. */ std::vector> cleanupEngines( - ErrorCode errorCode, std::string const& dbname, aql::ServerQueryIdList& serverQueryIds) const; - + ErrorCode errorCode, std::string const& dbname, + aql::ServerQueryIdList& serverQueryIds) const; // Insert the Locking information into the message to be send to DBServers void addLockingPart(arangodb::velocypack::Builder& builder, ServerID const& server) const; diff --git a/arangod/ClusterEngine/ClusterIndexFactory.cpp b/arangod/ClusterEngine/ClusterIndexFactory.cpp index cdf75062ccc6..bab36516d24c 100644 --- a/arangod/ClusterEngine/ClusterIndexFactory.cpp +++ b/arangod/ClusterEngine/ClusterIndexFactory.cpp @@ -52,8 +52,7 @@ struct DefaultIndexFactory : public arangodb::IndexTypeFactory { : IndexTypeFactory(server), _type(type) {} bool equal(arangodb::velocypack::Slice const& lhs, - arangodb::velocypack::Slice const& rhs, - std::string const& dbname) const override { + arangodb::velocypack::Slice const& rhs, std::string const& dbname) const override { auto& clusterEngine = _server.getFeature().engine(); auto* engine = clusterEngine.actualEngine(); @@ -76,15 +75,14 @@ struct DefaultIndexFactory : public arangodb::IndexTypeFactory { auto ct = clusterEngine.engineType(); return std::make_shared(id, collection, ct, - arangodb::Index::type(_type), - definition); + arangodb::Index::type(_type), definition); } - virtual arangodb::Result normalize( // normalize definition - arangodb::velocypack::Builder& normalized, // normalized definition (out-param) - arangodb::velocypack::Slice definition, // source definition - bool isCreation, // definition for index creation - TRI_vocbase_t const& vocbase // index vocbase + virtual arangodb::Result normalize( // normalize definition + arangodb::velocypack::Builder& normalized, // normalized definition (out-param) + arangodb::velocypack::Slice definition, // source definition + bool isCreation, // definition for index creation + TRI_vocbase_t const& vocbase // index vocbase ) const override { auto& clusterEngine = _server.getFeature().engine(); @@ -96,8 +94,8 @@ struct DefaultIndexFactory : public arangodb::IndexTypeFactory { "cannot find storage engine while normalizing index"); } - return engine->indexFactory().factory(_type).normalize( // normalize definition - normalized, definition, isCreation, vocbase // args + return engine->indexFactory().factory(_type).normalize( // normalize definition + normalized, definition, isCreation, vocbase // args ); } }; @@ -113,7 +111,8 @@ struct EdgeIndexFactory : public DefaultIndexFactory { bool isClusterConstructor) const override { if (!isClusterConstructor) { // this index type cannot be created directly - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "cannot create edge index"); + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, + "cannot create edge index"); } auto& clusterEngine = @@ -137,7 +136,8 @@ struct PrimaryIndexFactory : public DefaultIndexFactory { bool isClusterConstructor) const override { if (!isClusterConstructor) { // this index type cannot be created directly - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "cannot create primary index"); + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, + "cannot create primary index"); } auto& clusterEngine = @@ -154,8 +154,8 @@ struct PrimaryIndexFactory : public DefaultIndexFactory { namespace arangodb { -ClusterIndexFactory::ClusterIndexFactory(application_features::ApplicationServer& server) - : IndexFactory(server) { +void ClusterIndexFactory::LinkIndexFactories(application_features::ApplicationServer& server, + IndexFactory& factory) { static const EdgeIndexFactory edgeIndexFactory(server, "edge"); static const DefaultIndexFactory fulltextIndexFactory(server, "fulltext"); static const DefaultIndexFactory geoIndexFactory(server, "geo"); @@ -167,16 +167,21 @@ ClusterIndexFactory::ClusterIndexFactory(application_features::ApplicationServer static const DefaultIndexFactory skiplistIndexFactory(server, "skiplist"); static const DefaultIndexFactory ttlIndexFactory(server, "ttl"); - emplace(edgeIndexFactory._type, edgeIndexFactory); - emplace(fulltextIndexFactory._type, fulltextIndexFactory); - emplace(geoIndexFactory._type, geoIndexFactory); - emplace(geo1IndexFactory._type, geo1IndexFactory); - emplace(geo2IndexFactory._type, geo2IndexFactory); - emplace(hashIndexFactory._type, hashIndexFactory); - emplace(persistentIndexFactory._type, persistentIndexFactory); - emplace(primaryIndexFactory._type, primaryIndexFactory); - emplace(skiplistIndexFactory._type, skiplistIndexFactory); - emplace(ttlIndexFactory._type, ttlIndexFactory); + factory.emplace(edgeIndexFactory._type, edgeIndexFactory); + factory.emplace(fulltextIndexFactory._type, fulltextIndexFactory); + factory.emplace(geoIndexFactory._type, geoIndexFactory); + factory.emplace(geo1IndexFactory._type, geo1IndexFactory); + factory.emplace(geo2IndexFactory._type, geo2IndexFactory); + factory.emplace(hashIndexFactory._type, hashIndexFactory); + factory.emplace(persistentIndexFactory._type, persistentIndexFactory); + factory.emplace(primaryIndexFactory._type, primaryIndexFactory); + factory.emplace(skiplistIndexFactory._type, skiplistIndexFactory); + factory.emplace(ttlIndexFactory._type, ttlIndexFactory); +} + +ClusterIndexFactory::ClusterIndexFactory(application_features::ApplicationServer& server) + : IndexFactory(server) { + LinkIndexFactories(server, *this); } /// @brief index name aliases (e.g. "persistent" => "hash", "skiplist" => @@ -191,11 +196,11 @@ std::unordered_map ClusterIndexFactory::indexAliases() return ae->indexFactory().indexAliases(); } -Result ClusterIndexFactory::enhanceIndexDefinition( // normalize definition - velocypack::Slice const definition, // source definition - velocypack::Builder& normalized, // normalized definition (out-param) - bool isCreation, // definition for index creation - TRI_vocbase_t const& vocbase // index vocbase +Result ClusterIndexFactory::enhanceIndexDefinition( // normalize definition + velocypack::Slice const definition, // source definition + velocypack::Builder& normalized, // normalized definition (out-param) + bool isCreation, // definition for index creation + TRI_vocbase_t const& vocbase // index vocbase ) const { auto& ce = _server.getFeature().engine(); @@ -205,8 +210,8 @@ Result ClusterIndexFactory::enhanceIndexDefinition( // normalize definition return TRI_ERROR_INTERNAL; } - return ae->indexFactory().enhanceIndexDefinition( // normalize definition - definition, normalized, isCreation, vocbase // args + return ae->indexFactory().enhanceIndexDefinition( // normalize definition + definition, normalized, isCreation, vocbase // args ); } @@ -299,8 +304,8 @@ void ClusterIndexFactory::prepareIndexes( indexes.emplace_back(std::move(idx)); } catch (std::exception const& ex) { LOG_TOPIC("7ed52", ERR, arangodb::Logger::ENGINES) - << "error creating index from definition '" << v.toString() << "': " << ex.what(); - + << "error creating index from definition '" << v.toString() + << "': " << ex.what(); } } } diff --git a/arangod/ClusterEngine/ClusterIndexFactory.h b/arangod/ClusterEngine/ClusterIndexFactory.h index 780cf1d7ee94..ee4f683c762f 100644 --- a/arangod/ClusterEngine/ClusterIndexFactory.h +++ b/arangod/ClusterEngine/ClusterIndexFactory.h @@ -29,18 +29,20 @@ namespace arangodb { class ClusterIndexFactory final : public IndexFactory { public: + static void LinkIndexFactories(application_features::ApplicationServer& server, + IndexFactory& factory); explicit ClusterIndexFactory(application_features::ApplicationServer&); ~ClusterIndexFactory() = default; - Result enhanceIndexDefinition( // normalize definition - velocypack::Slice const definition, // source definition - velocypack::Builder& normalized, // normalized definition (out-param) - bool isCreation, // definition for index creation - TRI_vocbase_t const& vocbase // index vocbase + Result enhanceIndexDefinition( // normalize definition + velocypack::Slice const definition, // source definition + velocypack::Builder& normalized, // normalized definition (out-param) + bool isCreation, // definition for index creation + TRI_vocbase_t const& vocbase // index vocbase ) const override; - /// @brief index name aliases (e.g. "persistent" => "hash", "skiplist" => "hash") - /// used to display storage engine capabilities + /// @brief index name aliases (e.g. "persistent" => "hash", "skiplist" => + /// "hash") used to display storage engine capabilities std::unordered_map indexAliases() const override; void fillSystemIndexes(arangodb::LogicalCollection& col, diff --git a/arangod/Graph/Enumerators/OneSidedEnumerator.cpp b/arangod/Graph/Enumerators/OneSidedEnumerator.cpp index 1cbc21ce5b5d..2b573453746f 100644 --- a/arangod/Graph/Enumerators/OneSidedEnumerator.cpp +++ b/arangod/Graph/Enumerators/OneSidedEnumerator.cpp @@ -269,3 +269,4 @@ template class ::arangodb::graph::OneSidedEnumerator>; template class ::arangodb::graph::OneSidedEnumerator>; template class ::arangodb::graph::OneSidedEnumerator>; + diff --git a/arangod/InternalRestHandler/InternalRestTraverserHandler.cpp b/arangod/InternalRestHandler/InternalRestTraverserHandler.cpp index 753cc29d9ea2..981f7b80fae3 100644 --- a/arangod/InternalRestHandler/InternalRestTraverserHandler.cpp +++ b/arangod/InternalRestHandler/InternalRestTraverserHandler.cpp @@ -32,6 +32,7 @@ #include "Rest/GeneralResponse.h" #include "Transaction/StandaloneContext.h" +#include #include #include @@ -85,8 +86,9 @@ RestStatus InternalRestTraverserHandler::execute() { } void InternalRestTraverserHandler::createEngine() { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_NOT_IMPLEMENTED, - "API traversal engine creation no longer supported"); + THROW_ARANGO_EXCEPTION_MESSAGE( + TRI_ERROR_NOT_IMPLEMENTED, + "API traversal engine creation no longer supported"); } void InternalRestTraverserHandler::queryEngine() { @@ -117,7 +119,8 @@ void InternalRestTraverserHandler::queryEngine() { return; } - std::chrono::time_point start = std::chrono::steady_clock::now(); + std::chrono::time_point start = + std::chrono::steady_clock::now(); traverser::BaseEngine* engine = nullptr; while (true) { @@ -127,7 +130,8 @@ void InternalRestTraverserHandler::queryEngine() { break; } generateError(ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER, - "invalid TraverserEngine id - potentially the AQL query was already aborted or timed out"); + "invalid TraverserEngine id - potentially the AQL query " + "was already aborted or timed out"); return; } catch (basics::Exception const& ex) { // it is possible that the engine is already in use @@ -147,18 +151,18 @@ void InternalRestTraverserHandler::queryEngine() { generateError(ResponseCode::SERVER_ERROR, TRI_ERROR_LOCK_TIMEOUT); return; } - } + } TRI_ASSERT(engine != nullptr); auto& registry = _registry; // For the guard - auto cleanup = scopeGuard([registry, &engineId]() { - registry->closeEngine(engineId); - }); + auto cleanup = + scopeGuard([registry, &engineId]() { registry->closeEngine(engineId); }); if (option == "lock") { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_NOT_IMPLEMENTED, - "API for traversal engine locking no longer supported"); + THROW_ARANGO_EXCEPTION_MESSAGE( + TRI_ERROR_NOT_IMPLEMENTED, + "API for traversal engine locking no longer supported"); } VPackBuilder result; @@ -182,7 +186,7 @@ void InternalRestTraverserHandler::queryEngine() { // Safe cast BaseTraverserEngines are all of type TRAVERSER auto eng = static_cast(engine); TRI_ASSERT(eng != nullptr); - + VPackSlice variables = body.get("variables"); eng->injectVariables(variables); diff --git a/arangod/StorageEngine/EngineSelectorFeature.cpp b/arangod/StorageEngine/EngineSelectorFeature.cpp index ea0f4df8b923..77825a441b7a 100644 --- a/arangod/StorageEngine/EngineSelectorFeature.cpp +++ b/arangod/StorageEngine/EngineSelectorFeature.cpp @@ -55,10 +55,10 @@ std::unordered_map createEngineMap() { std::unordered_map map; // rocksdb is not deprecated and the engine of choice map.try_emplace(arangodb::RocksDBEngine::EngineName, - EngineInfo{ std::type_index(typeid(arangodb::RocksDBEngine)), false, true }); + EngineInfo{std::type_index(typeid(arangodb::RocksDBEngine)), false, true}); return map; } -} +} // namespace namespace arangodb { @@ -134,7 +134,8 @@ void EngineSelectorFeature::prepare() { if (selected == engines.end()) { if (_engineName == "mmfiles") { LOG_TOPIC("10eb6", FATAL, Logger::STARTUP) - << "the mmfiles storage engine is unavailable from version v3.7.0 onwards"; + << "the mmfiles storage engine is unavailable from version v3.7.0 " + "onwards"; } else { // should not happen LOG_TOPIC("3e975", FATAL, Logger::STARTUP) @@ -151,8 +152,7 @@ void EngineSelectorFeature::prepare() { "engine."; if (!ServerState::instance()->isCoordinator() && - !basics::FileUtils::isRegularFile(_engineFilePath) && - !_allowDeprecatedDeployments) { + !basics::FileUtils::isRegularFile(_engineFilePath) && !_allowDeprecatedDeployments) { LOG_TOPIC("ca0a7", FATAL, Logger::STARTUP) << "The " << _engineName << " storage engine cannot be used for new deployments."; @@ -173,10 +173,12 @@ void EngineSelectorFeature::prepare() { for (auto& engine : engines) { StorageEngine& e = server().getFeature(engine.second.type); // turn off all other storage engines - LOG_TOPIC("001b6", TRACE, Logger::STARTUP) << "disabling storage engine " << engine.first; + LOG_TOPIC("001b6", TRACE, Logger::STARTUP) + << "disabling storage engine " << engine.first; e.disable(); if (engine.first == _engineName) { - LOG_TOPIC("4a3fc", INFO, Logger::FIXME) << "using storage engine " << engine.first; + LOG_TOPIC("4a3fc", INFO, Logger::FIXME) + << "using storage engine " << engine.first; ce.setActualEngine(&e); } } @@ -239,8 +241,14 @@ void EngineSelectorFeature::unprepare() { _engine = nullptr; if (ServerState::instance()->isCoordinator()) { - ClusterEngine& ce = server().getFeature(); - ce.setActualEngine(nullptr); +#ifdef ARANGODB_USE_GOOGLE_TESTS + if (!arangodb::ClusterEngine::Mocking) { +#endif + ClusterEngine& ce = server().getFeature(); + ce.setActualEngine(nullptr); +#ifdef ARANGODB_USE_GOOGLE_TESTS + } +#endif } } diff --git a/tests/Mocks/Servers.cpp b/tests/Mocks/Servers.cpp index 4fc552aeb292..49e8165c3c06 100644 --- a/tests/Mocks/Servers.cpp +++ b/tests/Mocks/Servers.cpp @@ -181,11 +181,16 @@ static void SetupAqlPhase(MockServer& server) { #endif } -MockServer::MockServer() +MockServer::MockServer(arangodb::ServerState::RoleEnum myRole, bool injectClusterIndexes) : _server(std::make_shared("", "", "", nullptr), nullptr), - _engine(_server), + _engine(_server, injectClusterIndexes), _oldRebootId(0), _started(false) { + _oldRole = arangodb::ServerState::instance()->getRole(); + arangodb::ServerState::instance()->setRole(myRole); + if (arangodb::ServerState::instance()->isCoordinator()) { + arangodb::ClusterEngine::Mocking = true; + } init(); } @@ -193,6 +198,10 @@ MockServer::~MockServer() { stopFeatures(); _server.setStateUnsafe(_oldApplicationServerState); + if (arangodb::ServerState::instance()->isCoordinator()) { + arangodb::ClusterEngine::Mocking = false; + } + arangodb::ServerState::instance()->setRole(_oldRole); arangodb::ServerState::instance()->setRebootId(_oldRebootId); } @@ -454,10 +463,10 @@ std::pair, consensus::index_t> AgencyCache:: consensus::Store& AgencyCache::store() { return _readDB; } -MockClusterServer::MockClusterServer(bool useAgencyMockPool) - : MockServer(), _useAgencyMockPool(useAgencyMockPool) { - _oldRole = arangodb::ServerState::instance()->getRole(); - +MockClusterServer::MockClusterServer(bool useAgencyMockPool, + arangodb::ServerState::RoleEnum newRole, + bool injectClusterIndexes) + : MockServer(newRole, injectClusterIndexes), _useAgencyMockPool(useAgencyMockPool) { // Add features SetupAqlPhase(*this); @@ -478,7 +487,6 @@ MockClusterServer::~MockClusterServer() { ci.shutdownSyncers(); ci.waitForSyncersToStop(); _server.getFeature().shutdownAgencyCache(); - arangodb::ServerState::instance()->setRole(_oldRole); } void MockClusterServer::startFeatures() { @@ -518,6 +526,32 @@ void MockClusterServer::startFeatures() { _server.getFeature().clusterInfo().startSyncers(); } +std::unique_ptr MockClusterServer::createFakeQuery( + bool activateTracing, std::string queryString, + std::function callback) const { + auto bindParams = std::make_shared(); + bindParams->openObject(); + bindParams->close(); + VPackBuilder queryOptions; + queryOptions.openObject(); + if (activateTracing) { + queryOptions.add("profile", VPackValue(int(aql::ProfileLevel::TraceTwo))); + } + queryOptions.close(); + if (queryString.empty()) { + queryString = "RETURN 1"; + } + + aql::QueryString fakeQueryString(queryString); + auto query = std::make_unique( + arangodb::transaction::StandaloneContext::Create(getSystemDatabase()), + fakeQueryString, bindParams, queryOptions.slice()); + callback(*query); + query->prepareQuery(aql::SerializationFormat::SHADOWROWS); + + return query; +} + consensus::index_t MockClusterServer::agencyTrx(std::string const& key, std::string const& value) { // Build an agency transaction: @@ -710,8 +744,7 @@ std::shared_ptr MockClusterServer::createCollection( } MockDBServer::MockDBServer(bool start, bool useAgencyMock) - : MockClusterServer(useAgencyMock) { - arangodb::ServerState::instance()->setRole(arangodb::ServerState::RoleEnum::ROLE_DBSERVER); + : MockClusterServer(useAgencyMock, arangodb::ServerState::RoleEnum::ROLE_DBSERVER) { addFeature(false); // do not start the thread addFeature(false); // do not start the thread if (start) { @@ -824,9 +857,9 @@ void MockDBServer::createShard(std::string const& dbName, std::string shardName, } } -MockCoordinator::MockCoordinator(bool start, bool useAgencyMock) - : MockClusterServer(useAgencyMock) { - arangodb::ServerState::instance()->setRole(arangodb::ServerState::RoleEnum::ROLE_COORDINATOR); +MockCoordinator::MockCoordinator(bool start, bool useAgencyMock, bool injectClusterIndexes) + : MockClusterServer(useAgencyMock, arangodb::ServerState::RoleEnum::ROLE_COORDINATOR, + injectClusterIndexes) { if (start) { MockCoordinator::startFeatures(); MockCoordinator::createDatabase("_system"); diff --git a/tests/Mocks/Servers.h b/tests/Mocks/Servers.h index 2a8c2c94465b..97471b89627b 100644 --- a/tests/Mocks/Servers.h +++ b/tests/Mocks/Servers.h @@ -58,7 +58,13 @@ namespace mocks { /// @brief mock application server with no features added class MockServer { public: - MockServer(); + // Note, setting injectClusterIndexes causes the "create" methods to fail. + // this is all hardly worked around the Cluster engine and needs a proper + // clean up. It is highly recommended to not set injectClusterIndexes unless + // you want to specificly test something that selects an index, but cannot use + // it. Use with care for now. + MockServer(arangodb::ServerState::RoleEnum = arangodb::ServerState::RoleEnum::ROLE_SINGLE, + bool injectClusterIndexes = false); virtual ~MockServer(); application_features::ApplicationServer& server(); @@ -121,6 +127,7 @@ class MockServer { private: bool _started; + arangodb::ServerState::RoleEnum _oldRole; }; /// @brief a server with almost no features added (Metrics are available @@ -213,9 +220,14 @@ class MockClusterServer : public MockServer, DataSourceId const& planId, std::vector> shardNameToServerNamePairs); + std::unique_ptr createFakeQuery( + bool activateTracing = false, std::string queryString = "", + std::function runBeforePrepare = + [](arangodb::aql::Query&) {}) const; // You can only create specialized types protected: - MockClusterServer(bool useAgencyMockConnection); + MockClusterServer(bool useAgencyMockConnection, arangodb::ServerState::RoleEnum role, + bool injectClusterIndexes = false); ~MockClusterServer(); protected: @@ -232,8 +244,9 @@ class MockClusterServer : public MockServer, protected: std::unique_ptr _pool; + + private: bool _useAgencyMockPool; - arangodb::ServerState::RoleEnum _oldRole; int _dummy; }; @@ -251,7 +264,8 @@ class MockDBServer : public MockClusterServer { class MockCoordinator : public MockClusterServer { public: - MockCoordinator(bool startFeatures = true, bool useAgencyMockConnection = true); + MockCoordinator(bool startFeatures = true, bool useAgencyMockConnection = true, + bool injectClusterIndexes = false); ~MockCoordinator(); TRI_vocbase_t* createDatabase(std::string const& name) override; diff --git a/tests/Mocks/StorageEngineMock.cpp b/tests/Mocks/StorageEngineMock.cpp index e9f303be60b1..92b96fbcabd7 100644 --- a/tests/Mocks/StorageEngineMock.cpp +++ b/tests/Mocks/StorageEngineMock.cpp @@ -33,6 +33,7 @@ #include "Cluster/ClusterFeature.h" #include "Cluster/ClusterInfo.h" #include "ClusterEngine/ClusterEngine.h" +#include "ClusterEngine/ClusterIndexFactory.h" #include "IResearch/IResearchCommon.h" #include "IResearch/IResearchFeature.h" #include "IResearch/IResearchLinkCoordinator.h" @@ -479,8 +480,13 @@ class AllIteratorMock final : public arangodb::IndexIterator { }; // AllIteratorMock struct IndexFactoryMock : arangodb::IndexFactory { - IndexFactoryMock(arangodb::application_features::ApplicationServer& server) - : IndexFactory(server) {} + IndexFactoryMock(arangodb::application_features::ApplicationServer& server, bool injectClusterIndexes) + : IndexFactory(server) { + if (injectClusterIndexes) { + arangodb::ClusterIndexFactory::LinkIndexFactories(server, *this); + } + } + virtual void fillSystemIndexes(arangodb::LogicalCollection& col, std::vector>& systemIndexes) const override { // NOOP @@ -1301,6 +1307,24 @@ void PhysicalCollectionMock::prepareIndexes(arangodb::velocypack::Slice indexesS } } +arangodb::IndexEstMap PhysicalCollectionMock::clusterIndexEstimates(bool allowUpdating, + arangodb::TransactionId tid) { + TRI_ASSERT(arangodb::ServerState::instance()->isCoordinator()); + arangodb::IndexEstMap estimates; + for (auto const& it : _indexes) { + std::string id = std::to_string(it->id().id()); + if (it->hasSelectivityEstimate()) { + // Note: This may actually be bad, as this instance cannot + // have documents => The estimate is off. + estimates.emplace(std::move(id), it->selectivityEstimate()); + } else { + // Random hardcoded estimate. We do not actually know anything + estimates.emplace(std::move(id), 0.25); + } + } + return estimates; +} + arangodb::Result PhysicalCollectionMock::read( arangodb::transaction::Methods*, arangodb::velocypack::StringRef const& key, arangodb::IndexIterator::DocumentCallback const& cb) const { @@ -1484,9 +1508,11 @@ std::function StorageEngineMock::recoveryTickCallback = []() -> void {}; /*static*/ std::string StorageEngineMock::versionFilenameResult; -StorageEngineMock::StorageEngineMock(arangodb::application_features::ApplicationServer& server) +StorageEngineMock::StorageEngineMock(arangodb::application_features::ApplicationServer& server, + bool injectClusterIndexes) : StorageEngine(server, "Mock", "", - std::unique_ptr(new IndexFactoryMock(server))), + std::unique_ptr( + new IndexFactoryMock(server, injectClusterIndexes))), vocbaseCount(1), _releasedTick(0) {} diff --git a/tests/Mocks/StorageEngineMock.h b/tests/Mocks/StorageEngineMock.h index eb8f8ed5608e..52f2971e2155 100644 --- a/tests/Mocks/StorageEngineMock.h +++ b/tests/Mocks/StorageEngineMock.h @@ -28,13 +28,13 @@ #include #include "Basics/Result.h" +#include "IResearchLinkMock.h" #include "Indexes/IndexIterator.h" #include "StorageEngine/HealthData.h" #include "StorageEngine/PhysicalCollection.h" #include "StorageEngine/StorageEngine.h" #include "StorageEngine/TransactionCollection.h" #include "StorageEngine/TransactionState.h" -#include "IResearchLinkMock.h" #include "VocBase/Identifiers/IndexId.h" #include "VocBase/Identifiers/LocalDocumentId.h" @@ -96,12 +96,16 @@ class PhysicalCollectionMock : public arangodb::PhysicalCollection { virtual uint64_t numberDocuments(arangodb::transaction::Methods* trx) const override; virtual std::string const& path() const override; virtual void prepareIndexes(arangodb::velocypack::Slice indexesSlice) override; + + arangodb::IndexEstMap clusterIndexEstimates(bool allowUpdating, + arangodb::TransactionId tid) override; + virtual arangodb::Result read(arangodb::transaction::Methods*, arangodb::velocypack::StringRef const& key, arangodb::IndexIterator::DocumentCallback const& cb) const override; virtual arangodb::Result read(arangodb::transaction::Methods* trx, - arangodb::LocalDocumentId const& token, - arangodb::IndexIterator::DocumentCallback const& cb) const override; + arangodb::LocalDocumentId const& token, + arangodb::IndexIterator::DocumentCallback const& cb) const override; virtual bool readDocument(arangodb::transaction::Methods* trx, arangodb::LocalDocumentId const& token, arangodb::ManagedDocumentResult& result) const override; @@ -184,7 +188,8 @@ class StorageEngineMock : public arangodb::StorageEngine { std::map, arangodb::velocypack::Builder> views; std::atomic vocbaseCount; - explicit StorageEngineMock(arangodb::application_features::ApplicationServer& server); + explicit StorageEngineMock(arangodb::application_features::ApplicationServer& server, + bool injectClusterIndexes = false); arangodb::HealthData healthCheck() override; virtual void addOptimizerRules(arangodb::aql::OptimizerRulesFeature& feature) override; virtual void addRestHandlers(arangodb::rest::RestHandlerFactory& handlerFactory) override; @@ -194,7 +199,8 @@ class StorageEngineMock : public arangodb::StorageEngine { bool doSync) override; virtual arangodb::Result changeView(TRI_vocbase_t& vocbase, arangodb::LogicalView const& view, bool doSync) override; - virtual void createCollection(TRI_vocbase_t& vocbase, arangodb::LogicalCollection const& collection) override; + virtual void createCollection(TRI_vocbase_t& vocbase, + arangodb::LogicalCollection const& collection) override; virtual std::unique_ptr createDatabase(arangodb::CreateDatabaseInfo&&, ErrorCode& status) override; virtual arangodb::Result createLoggerState(TRI_vocbase_t*, VPackBuilder&) override; @@ -227,13 +233,15 @@ class StorageEngineMock : public arangodb::StorageEngine { arangodb::velocypack::Builder& result, bool includeIndexes, TRI_voc_tick_t maxTick) override; virtual ErrorCode getCollectionsAndIndexes(TRI_vocbase_t& vocbase, - arangodb::velocypack::Builder& result, - bool wasCleanShutdown, bool isUpgrade) override; + arangodb::velocypack::Builder& result, + bool wasCleanShutdown, bool isUpgrade) override; virtual void getDatabases(arangodb::velocypack::Builder& result) override; virtual void cleanupReplicationContexts() override; - virtual arangodb::velocypack::Builder getReplicationApplierConfiguration(TRI_vocbase_t& vocbase, ErrorCode& result) override; + virtual arangodb::velocypack::Builder getReplicationApplierConfiguration( + TRI_vocbase_t& vocbase, ErrorCode& result) override; virtual arangodb::velocypack::Builder getReplicationApplierConfiguration(ErrorCode& result) override; - virtual ErrorCode getViews(TRI_vocbase_t& vocbase, arangodb::velocypack::Builder& result) override; + virtual ErrorCode getViews(TRI_vocbase_t& vocbase, + arangodb::velocypack::Builder& result) override; virtual arangodb::Result handleSyncKeys(arangodb::DatabaseInitialSyncer& syncer, arangodb::LogicalCollection& col, std::string const& keysId) override; @@ -256,15 +264,16 @@ class StorageEngineMock : public arangodb::StorageEngine { arangodb::LogicalCollection const& collection, std::string const& oldName) override; virtual ErrorCode saveReplicationApplierConfiguration(TRI_vocbase_t& vocbase, - arangodb::velocypack::Slice slice, - bool doSync) override; - virtual ErrorCode saveReplicationApplierConfiguration(arangodb::velocypack::Slice, bool) override; + arangodb::velocypack::Slice slice, + bool doSync) override; + virtual ErrorCode saveReplicationApplierConfiguration(arangodb::velocypack::Slice, + bool) override; virtual std::string versionFilename(TRI_voc_tick_t) const override; virtual void waitForEstimatorSync(std::chrono::milliseconds maxWaitTime) override; virtual arangodb::WalAccess const* walAccess() const override; static std::shared_ptr buildLinkMock( - arangodb::IndexId id, arangodb::LogicalCollection& collection, VPackSlice const& info); + arangodb::IndexId id, arangodb::LogicalCollection& collection, VPackSlice const& info); private: TRI_voc_tick_t _releasedTick; From 087a29c0077d9899eafb98f6833852e596c5befb Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Thu, 8 Jul 2021 15:02:31 +0200 Subject: [PATCH 12/72] Feature/hybrid smart graph glue cpp tests (#14485) --- arangod/VocBase/LogicalCollection.cpp | 4 ++++ arangod/VocBase/LogicalCollection.h | 2 ++ tests/Mocks/MockGraph.cpp | 11 +++++++---- tests/Mocks/MockGraph.h | 4 ++-- tests/Mocks/Servers.cpp | 5 ++++- tests/Mocks/Servers.h | 8 ++++++++ 6 files changed, 27 insertions(+), 7 deletions(-) diff --git a/arangod/VocBase/LogicalCollection.cpp b/arangod/VocBase/LogicalCollection.cpp index 02a8ad487111..9d192ca3aefe 100644 --- a/arangod/VocBase/LogicalCollection.cpp +++ b/arangod/VocBase/LogicalCollection.cpp @@ -1242,6 +1242,10 @@ void LogicalCollection::setInternalValidatorTypes(uint64_t type) { _internalValidatorTypes = type; } +uint64_t LogicalCollection::getInternalValidatorTypes() const { + return _internalValidatorTypes; +} + void LogicalCollection::addInternalValidator(std::unique_ptr validator) { // For the time beeing we only allow ONE internal validator. // This however is a non-necessary restriction and can be leveraged at any diff --git a/arangod/VocBase/LogicalCollection.h b/arangod/VocBase/LogicalCollection.h index b5f9b21a99da..1f6ad5e475a8 100644 --- a/arangod/VocBase/LogicalCollection.h +++ b/arangod/VocBase/LogicalCollection.h @@ -348,6 +348,8 @@ class LogicalCollection : public LogicalDataSource { // (Technically no issue but will have side-effects on shards) void setInternalValidatorTypes(uint64_t type); + uint64_t getInternalValidatorTypes() const; + bool isShard() const noexcept; bool isLocalSmartEdgeCollection() const noexcept; diff --git a/tests/Mocks/MockGraph.cpp b/tests/Mocks/MockGraph.cpp index 221fba80f046..b239a54f782a 100644 --- a/tests/Mocks/MockGraph.cpp +++ b/tests/Mocks/MockGraph.cpp @@ -76,14 +76,17 @@ void MockGraph::VertexDef::addToBuilder(arangodb::velocypack::Builder& builder) builder.close(); } -void MockGraph::addEdge(std::string from, std::string to, double weight) { - _edges.emplace_back(EdgeDef{from, to, weight, _edgeCollectionName}); +MockGraph::EdgeDef MockGraph::addEdge(std::string from, std::string to, double weight) { + EdgeDef newEdge {from, to, weight, _edgeCollectionName}; + _edges.emplace_back(newEdge); _vertices.emplace(std::move(from)); _vertices.emplace(std::move(to)); + + return newEdge; } -void MockGraph::addEdge(size_t from, size_t to, double weight) { - addEdge(getVertexCollectionName() + "/" + basics::StringUtils::itoa(from), +MockGraph::EdgeDef MockGraph::addEdge(size_t from, size_t to, double weight) { + return addEdge(getVertexCollectionName() + "/" + basics::StringUtils::itoa(from), getVertexCollectionName() + "/" + basics::StringUtils::itoa(to), weight); } diff --git a/tests/Mocks/MockGraph.h b/tests/Mocks/MockGraph.h index 5305e9ff3519..bc994d076dd1 100644 --- a/tests/Mocks/MockGraph.h +++ b/tests/Mocks/MockGraph.h @@ -87,8 +87,8 @@ class MockGraph { MockGraph() {} ~MockGraph() {} - void addEdge(std::string from, std::string to, double weight = 1.0); - void addEdge(size_t from, size_t to, double weight = 1.0); + EdgeDef addEdge(std::string from, std::string to, double weight = 1.0); + EdgeDef addEdge(size_t from, size_t to, double weight = 1.0); auto edges() const -> std::vector const& { return _edges; } auto vertices() const -> std::unordered_set const& { diff --git a/tests/Mocks/Servers.cpp b/tests/Mocks/Servers.cpp index 49e8165c3c06..1754183ff374 100644 --- a/tests/Mocks/Servers.cpp +++ b/tests/Mocks/Servers.cpp @@ -717,7 +717,8 @@ std::shared_ptr MockClusterServer::createCollection( std::string const& dbName, std::string collectionName, std::vector> shardNameToServerNamePairs, TRI_col_type_e type, VPackSlice additionalProperties) { - std::string cid = std::to_string(_server.getFeature().clusterInfo().uniqid()); + std::string cid = + std::to_string(_server.getFeature().clusterInfo().uniqid()); auto& databaseFeature = _server.getFeature(); auto vocbase = databaseFeature.lookupDatabase(dbName); @@ -805,6 +806,8 @@ void MockDBServer::createShard(std::string const& dbName, std::string shardName, #ifndef USE_ENTERPRISE props->add(StaticStrings::ReplicationFactor, VPackValue(1)); #endif + props->add(StaticStrings::InternalValidatorTypes, + VPackValue(clusterCollection.getInternalValidatorTypes())); } maintenance::ActionDescription ad( std::map{{maintenance::NAME, maintenance::CREATE_COLLECTION}, diff --git a/tests/Mocks/Servers.h b/tests/Mocks/Servers.h index 97471b89627b..483014ef754d 100644 --- a/tests/Mocks/Servers.h +++ b/tests/Mocks/Servers.h @@ -212,6 +212,14 @@ class MockClusterServer : public MockServer, TRI_col_type_e type, VPackSlice additionalProperties = VPackSlice{VPackSlice::nullSlice()}); +#ifdef USE_ENTERPRISE + std::shared_ptr createSmartCollection( + std::string const& dbName, std::string collectionName, + std::vector> shardNameToServerNamePairs, + TRI_col_type_e type, + VPackSlice additionalProperties = VPackSlice{VPackSlice::nullSlice()}); +#endif + void buildCollectionProperties(VPackBuilder& props, std::string const& collectionName, std::string const& cid, TRI_col_type_e type, VPackSlice additionalProperties); From 7751eec5aa1b3d209fe76da78a86d6d1f7d9c4fa Mon Sep 17 00:00:00 2001 From: Heiko Date: Mon, 12 Jul 2021 12:17:04 +0200 Subject: [PATCH 13/72] Feature/hybrid smart graph additional steptype (#14474) --- arangod/Aql/ExecutionBlockImpl.cpp | 120 ++++++++++-------- arangod/Aql/KShortestPathsExecutor.cpp | 21 +-- arangod/Aql/KShortestPathsNode.cpp | 23 ++-- arangod/CMakeLists.txt | 1 + .../RefactoredSingleServerEdgeCursor.cpp | 78 ++++++++---- .../RefactoredSingleServerEdgeCursor.h | 6 +- .../Graph/Enumerators/OneSidedEnumerator.cpp | 69 +++++++--- .../Graph/Enumerators/TwoSidedEnumerator.cpp | 20 +-- arangod/Graph/PathManagement/PathResult.cpp | 19 ++- arangod/Graph/PathManagement/PathStore.cpp | 76 ++++++++--- .../Graph/PathManagement/PathStoreTracer.cpp | 51 ++++++-- .../Graph/PathManagement/PathValidator.cpp | 55 ++++++-- .../SingleProviderPathResult.cpp | 110 +++++++++------- arangod/Graph/Providers/ClusterProvider.h | 13 -- arangod/Graph/Providers/ProviderTracer.cpp | 22 +++- .../Graph/Providers/SingleServerProvider.cpp | 107 +++++++--------- .../Graph/Providers/SingleServerProvider.h | 104 ++------------- arangod/Graph/Queues/QueueTracer.cpp | 21 ++- .../Graph/Steps/SingleServerProviderStep.cpp | 76 +++++++++++ .../Graph/Steps/SingleServerProviderStep.h | 110 ++++++++++++++++ tests/Graph/GenericGraphProviderTest.cpp | 15 ++- tests/Graph/SingleServerProviderTest.cpp | 5 +- 22 files changed, 714 insertions(+), 408 deletions(-) create mode 100644 arangod/Graph/Steps/SingleServerProviderStep.cpp create mode 100644 arangod/Graph/Steps/SingleServerProviderStep.h diff --git a/arangod/Aql/ExecutionBlockImpl.cpp b/arangod/Aql/ExecutionBlockImpl.cpp index 0f721c5d1ded..286171292068 100644 --- a/arangod/Aql/ExecutionBlockImpl.cpp +++ b/arangod/Aql/ExecutionBlockImpl.cpp @@ -82,6 +82,7 @@ #include "Graph/Providers/SingleServerProvider.h" #include "Graph/Queues/FifoQueue.h" #include "Graph/Queues/QueueTracer.h" +#include "Graph/Steps/SingleServerProviderStep.h" #include "Graph/algorithm-aliases.h" #include @@ -92,11 +93,13 @@ #include /* SingleServerProvider Section */ +using SingleServerProviderStep = ::arangodb::graph::SingleServerProviderStep; + using KPathRefactored = - arangodb::graph::KPathEnumerator; + arangodb::graph::KPathEnumerator>; using KPathRefactoredTracer = - arangodb::graph::TracedKPathEnumerator; + arangodb::graph::TracedKPathEnumerator>; /* ClusterProvider Section */ using KPathRefactoredCluster = @@ -188,7 +191,7 @@ ExecutionBlockImpl::ExecutionBlockImpl(ExecutionEngine* engine, // Break the stack before waiting. // We should not use this here. _stackBeforeWaiting.popCall(); - + if (_exeNode->isCallstackSplitEnabled()) { _callstackSplit = std::make_unique(*this); } @@ -342,7 +345,7 @@ ExecutionBlockImpl::execute(AqlCallStack const& stack) { TRI_IF_FAILURE("ExecutionBlock::getOrSkipSome3") { THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); } - + // check if this block failed already. if (_firstFailure.fail()) { // if so, just return the stored error. @@ -380,13 +383,17 @@ ExecutionBlockImpl::execute(AqlCallStack const& stack) { TRI_ASSERT(_firstFailure.ok()); // store only the first failure we got _firstFailure = {ex.code(), ex.what()}; - LOG_QUERY("7289a", DEBUG) << printBlockInfo() << " local statemachine failed with exception: " << ex.what(); + LOG_QUERY("7289a", DEBUG) + << printBlockInfo() + << " local statemachine failed with exception: " << ex.what(); throw; } catch (std::exception const& ex) { TRI_ASSERT(_firstFailure.ok()); // store only the first failure we got _firstFailure = {TRI_ERROR_INTERNAL, ex.what()}; - LOG_QUERY("2bbd5", DEBUG) << printBlockInfo() << " local statemachine failed with exception: " << ex.what(); + LOG_QUERY("2bbd5", DEBUG) + << printBlockInfo() + << " local statemachine failed with exception: " << ex.what(); // Rewire the error, to be consistent with potentially next caller. THROW_ARANGO_EXCEPTION(_firstFailure); } @@ -527,7 +534,7 @@ auto ExecutionBlockImpl::allocateOutputBlock(AqlCall&& call) } template -void ExecutionBlockImpl::ensureOutputBlock(AqlCall&& call) { +void ExecutionBlockImpl::ensureOutputBlock(AqlCall&& call) { if (_outputItemRow == nullptr || !_outputItemRow->isInitialized()) { _outputItemRow = allocateOutputBlock(std::move(call)); } else { @@ -602,12 +609,12 @@ static SkipRowsRangeVariant constexpr skipRowsType() { useExecutor == (is_one_of_v< Executor, FilterExecutor, ShortestPathExecutor, ReturnExecutor, - KShortestPathsExecutor, - KShortestPathsExecutor, KShortestPathsExecutor, - KShortestPathsExecutor, KShortestPathsExecutor, ParallelUnsortedGatherExecutor, - IdExecutor>, IdExecutor, HashedCollectExecutor, - AccuWindowExecutor, WindowExecutor, IndexExecutor, EnumerateCollectionExecutor, DistinctCollectExecutor, - ConstrainedSortExecutor, CountCollectExecutor, + KShortestPathsExecutor, KShortestPathsExecutor, + KShortestPathsExecutor, KShortestPathsExecutor, + KShortestPathsExecutor, ParallelUnsortedGatherExecutor, + IdExecutor>, IdExecutor, + HashedCollectExecutor, AccuWindowExecutor, WindowExecutor, IndexExecutor, EnumerateCollectionExecutor, + DistinctCollectExecutor, ConstrainedSortExecutor, CountCollectExecutor, #ifdef ARANGODB_USE_GOOGLE_TESTS TestLambdaSkipExecutor, #endif @@ -734,7 +741,8 @@ auto ExecutionBlockImpl::executeFetcher(ExecutionContext& ctx, // SubqueryStart and the partnered SubqueryEnd by *not* // pushing the upstream request. if constexpr (!std::is_same_v) { - ctx.stack.pushCall(createUpstreamCall(std::move(aqlCall), ctx.clientCallList.hasMoreCalls())); + ctx.stack.pushCall(createUpstreamCall(std::move(aqlCall), + ctx.clientCallList.hasMoreCalls())); } auto const result = std::invoke([&]() { @@ -759,23 +767,24 @@ auto ExecutionBlockImpl::executeFetcher(ExecutionContext& ctx, // TODO - we should avoid flooding the queue with too many tasks as that // can significantly delay processing of user REST requests. - + // we can safely ignore the result here, because we will try to // claim the task ourselves anyway. - std::ignore = SchedulerFeature::SCHEDULER->queue( - RequestLane::INTERNAL_LOW, - [block = this, task = _prefetchTask, stack = ctx.stack]() mutable { - if (!task->tryClaim()) { - return; - } - // task is a copy of the PrefetchTask shared_ptr, and we will only - // attempt to execute the task if we successfully claimed the task. - // i.e., it does not matter if this task lingers around in the - // scheduler queue even after the execution block has been destroyed, - // because in this case we will not be able to claim the task and - // simply return early without accessing the block. - task->execute(*block, stack); - }); + std::ignore = + SchedulerFeature::SCHEDULER->queue(RequestLane::INTERNAL_LOW, + [block = this, task = _prefetchTask, + stack = ctx.stack]() mutable { + if (!task->tryClaim()) { + return; + } + // task is a copy of the PrefetchTask shared_ptr, and we will only + // attempt to execute the task if we successfully claimed the task. + // i.e., it does not matter if this task lingers around in the + // scheduler queue even after the execution block has been destroyed, + // because in this case we will not be able to claim the task and + // simply return early without accessing the block. + task->execute(*block, stack); + }); } if constexpr (!std::is_same_v) { @@ -1231,10 +1240,10 @@ auto ExecutionBlockImpl::executeFastForward(typename Fetcher::DataRang template std::tuple ExecutionBlockImpl::executeWithoutTrace(AqlCallStack const& callStack) { - // We can only work on a Stack that has valid calls for all levels. + // We can only work on a Stack that has valid calls for all levels. TRI_ASSERT(callStack.hasAllValidCalls()); ExecutionContext ctx(*this, callStack); - + ExecutorState localExecutorState = ExecutorState::DONE; // We can only have returned the following internal states @@ -1344,8 +1353,8 @@ ExecutionBlockImpl::executeWithoutTrace(AqlCallStack const& callStack) TRI_ASSERT(ctx.clientCall.getSkipCount() == 0); switch (_execState) { case ExecState::CHECKCALL: { - LOG_QUERY("cfe46", DEBUG) - << printTypeInfo() << " determine next action on call " << ctx.clientCall; + LOG_QUERY("cfe46", DEBUG) << printTypeInfo() << " determine next action on call " + << ctx.clientCall; if constexpr (executorHasSideEffects) { // If the executor has sideEffects, and we need to skip the results we would @@ -1369,7 +1378,8 @@ ExecutionBlockImpl::executeWithoutTrace(AqlCallStack const& callStack) LOG_QUERY("1f786", DEBUG) << printTypeInfo() << " call skipRows " << ctx.clientCall; // Execute skipSome - auto [state, stats, skippedLocal, call] = executeSkipRowsRange(_lastRange, ctx.clientCall); + auto [state, stats, skippedLocal, call] = + executeSkipRowsRange(_lastRange, ctx.clientCall); #ifdef ARANGODB_ENABLE_MAINTAINER_MODE // Assertion: We did skip 'skippedLocal' documents here. @@ -1517,19 +1527,21 @@ ExecutionBlockImpl::executeWithoutTrace(AqlCallStack const& callStack) // executors. TRI_ASSERT(isMultiDepExecutor || !lastRangeHasDataRow()); TRI_ASSERT(!_lastRange.hasShadowRow()); - + #ifdef ARANGODB_ENABLE_MAINTAINER_MODE auto subqueryLevelBefore = ctx.stack.subqueryLevel(); #endif SkipResult skippedLocal; if (_callstackSplit) { - // we need to split the callstack to avoid stack overflows, so we move upstream - // execution into a separate thread - std::tie(_upstreamState, skippedLocal, _lastRange) = _callstackSplit->execute(ctx, _upstreamRequest); + // we need to split the callstack to avoid stack overflows, so we move + // upstream execution into a separate thread + std::tie(_upstreamState, skippedLocal, _lastRange) = + _callstackSplit->execute(ctx, _upstreamRequest); } else { - std::tie(_upstreamState, skippedLocal, _lastRange) = executeFetcher(ctx, _upstreamRequest); + std::tie(_upstreamState, skippedLocal, _lastRange) = + executeFetcher(ctx, _upstreamRequest); } - + #ifdef ARANGODB_ENABLE_MAINTAINER_MODE TRI_ASSERT(subqueryLevelBefore == ctx.stack.subqueryLevel()); #endif @@ -1891,8 +1903,8 @@ auto ExecutionBlockImpl::testInjectInputRange(DataRange range, SkipRes #endif template -ExecutionBlockImpl::ExecutionContext::ExecutionContext( - ExecutionBlockImpl& block, AqlCallStack const& callstack) +ExecutionBlockImpl::ExecutionContext::ExecutionContext(ExecutionBlockImpl& block, + AqlCallStack const& callstack) : stack(callstack), clientCallList(this->stack.popCall()) { if constexpr (std::is_same_v) { // In subqeryEndExecutor we actually manage two calls. @@ -1913,7 +1925,7 @@ ExecutionBlockImpl::ExecutionContext::ExecutionContext( // We got called with a skip count already set! // Caller is wrong fix it. TRI_ASSERT(clientCall.getSkipCount() == 0); - + TRI_ASSERT(!(clientCall.getOffset() == 0 && clientCall.softLimit == AqlCall::Limit{0u})); TRI_ASSERT(!(clientCall.hasSoftLimit() && clientCall.fullCount)); TRI_ASSERT(!(clientCall.hasSoftLimit() && clientCall.hasHardLimit())); @@ -1960,7 +1972,8 @@ auto ExecutionBlockImpl::PrefetchTask::stealResult() noexcept -> Prefe } template -void ExecutionBlockImpl::PrefetchTask::execute(ExecutionBlockImpl& block, AqlCallStack& stack) { +void ExecutionBlockImpl::PrefetchTask::execute(ExecutionBlockImpl& block, + AqlCallStack& stack) { if constexpr (std::is_same_v || executorHasSideEffects) { TRI_ASSERT(false); @@ -1981,12 +1994,12 @@ void ExecutionBlockImpl::PrefetchTask::execute(ExecutionBlockImpl& blo } template -ExecutionBlockImpl::CallstackSplit::CallstackSplit(ExecutionBlockImpl& block) : - _block(block), - _thread(&CallstackSplit::run, this, std::cref(ExecContext::current())) {} - +ExecutionBlockImpl::CallstackSplit::CallstackSplit(ExecutionBlockImpl& block) + : _block(block), + _thread(&CallstackSplit::run, this, std::cref(ExecContext::current())) {} + template -ExecutionBlockImpl::CallstackSplit::~CallstackSplit() { +ExecutionBlockImpl::CallstackSplit::~CallstackSplit() { _lock.lock(); _state.store(State::Stopped); _lock.unlock(); @@ -1996,7 +2009,8 @@ ExecutionBlockImpl::CallstackSplit::~CallstackSplit() { } template -auto ExecutionBlockImpl::CallstackSplit::execute(ExecutionContext& ctx, AqlCallType const& aqlCall) +auto ExecutionBlockImpl::CallstackSplit::execute(ExecutionContext& ctx, + AqlCallType const& aqlCall) -> UpstreamResult { std::variant result{std::nullopt}; Params params{result, ctx, aqlCall}; @@ -2008,7 +2022,7 @@ auto ExecutionBlockImpl::CallstackSplit::execute(ExecutionContext& ctx } _bell.notify_one(); - + std::unique_lock guard(_lock); _bell.wait(guard, [this]() { return _state.load(std::memory_order_acquire) != State::Executing; @@ -2021,7 +2035,7 @@ auto ExecutionBlockImpl::CallstackSplit::execute(ExecutionContext& ctx return std::get(std::move(result)); } - + template void ExecutionBlockImpl::CallstackSplit::run(ExecContext const& execContext) { ExecContextScope scope(&execContext); @@ -2038,7 +2052,7 @@ void ExecutionBlockImpl::CallstackSplit::run(ExecContext const& execCo try { _params->result = _block.executeFetcher(_params->ctx, _params->aqlCall); - } catch(...) { + } catch (...) { _params->result = std::current_exception(); } diff --git a/arangod/Aql/KShortestPathsExecutor.cpp b/arangod/Aql/KShortestPathsExecutor.cpp index 70a6d4cbac17..3ef29e1d48cc 100644 --- a/arangod/Aql/KShortestPathsExecutor.cpp +++ b/arangod/Aql/KShortestPathsExecutor.cpp @@ -40,6 +40,7 @@ #include "Graph/Queues/QueueTracer.h" #include "Graph/ShortestPathOptions.h" #include "Graph/ShortestPathResult.h" +#include "Graph/Steps/SingleServerProviderStep.h" #include "Transaction/Helpers.h" #include "Graph/algorithm-aliases.h" @@ -143,8 +144,8 @@ auto KShortestPathsExecutorInfos::getTargetVertex() const noexcept template auto KShortestPathsExecutorInfos::cache() const -> graph::TraverserCache* { - if constexpr (std::is_same_v> || - std::is_same_v> || + if constexpr (std::is_same_v>> || + std::is_same_v>> || std::is_same_v> || std::is_same_v> @@ -267,8 +268,8 @@ template auto KShortestPathsExecutor::doOutputPath(OutputAqlItemRow& output) -> void { transaction::BuilderLeaser tmp{&_trx}; tmp->clear(); - if constexpr (std::is_same_v> || - std::is_same_v> || + if constexpr (std::is_same_v>> || + std::is_same_v>> || std::is_same_v> || std::is_same_v>) { if (_finder.getNextPath(*tmp.builder())) { @@ -306,8 +307,8 @@ auto KShortestPathsExecutor::getVertexId(InputVertex const& vertex, try { std::string idString; // TODO: calculate expression once e.g. header constexpr bool and check then here - if constexpr (std::is_same_v> || - std::is_same_v> || + if constexpr (std::is_same_v>> || + std::is_same_v>> || std::is_same_v> || std::is_same_v>) { idString = _trx.extractIdString(in.slice()); @@ -378,11 +379,11 @@ template class ::arangodb::aql::KShortestPathsExecutorInfos>; -template class ::arangodb::aql::KShortestPathsExecutorInfos>; +template class ::arangodb::aql::KShortestPathsExecutorInfos>>; +template class ::arangodb::aql::KShortestPathsExecutorInfos>>; -template class ::arangodb::aql::KShortestPathsExecutor>; -template class ::arangodb::aql::KShortestPathsExecutor>; +template class ::arangodb::aql::KShortestPathsExecutor>>; +template class ::arangodb::aql::KShortestPathsExecutor>>; /* ClusterProvider Section */ diff --git a/arangod/Aql/KShortestPathsNode.cpp b/arangod/Aql/KShortestPathsNode.cpp index d3e1beb5cac2..e79ce98eefbc 100644 --- a/arangod/Aql/KShortestPathsNode.cpp +++ b/arangod/Aql/KShortestPathsNode.cpp @@ -48,6 +48,7 @@ #include "Graph/Queues/QueueTracer.h" #include "Graph/ShortestPathOptions.h" #include "Graph/ShortestPathResult.h" +#include "Graph/Steps/SingleServerProviderStep.h" #include "Indexes/Index.h" #include "OptimizerUtils.h" #include "Utils/CollectionNameResolver.h" @@ -360,13 +361,14 @@ std::unique_ptr KShortestPathsNode::createBlock( if (opts->query().queryOptions().getTraversalProfileLevel() == TraversalProfileLevel::None) { - using KPathRefactored = KPathEnumerator; + using KPathRefactored = + KPathEnumerator>; auto kPathUnique = std::make_unique( - SingleServerProvider{opts->query(), forwardProviderOptions, - opts->query().resourceMonitor()}, - SingleServerProvider{opts->query(), backwardProviderOptions, - opts->query().resourceMonitor()}, + SingleServerProvider{opts->query(), forwardProviderOptions, + opts->query().resourceMonitor()}, + SingleServerProvider{opts->query(), backwardProviderOptions, + opts->query().resourceMonitor()}, std::move(enumeratorOptions), std::move(validatorOptions), opts->query().resourceMonitor()); @@ -378,12 +380,13 @@ std::unique_ptr KShortestPathsNode::createBlock( &engine, this, std::move(registerInfos), std::move(executorInfos)); } else { // TODO: implement better initialization with less duplicate code - using TracedKPathRefactored = TracedKPathEnumerator; + using TracedKPathRefactored = + TracedKPathEnumerator>; auto kPathUnique = std::make_unique( - ProviderTracer{opts->query(), forwardProviderOptions, - opts->query().resourceMonitor()}, - ProviderTracer{opts->query(), backwardProviderOptions, - opts->query().resourceMonitor()}, + ProviderTracer>{ + opts->query(), forwardProviderOptions, opts->query().resourceMonitor()}, + ProviderTracer>{ + opts->query(), backwardProviderOptions, opts->query().resourceMonitor()}, std::move(enumeratorOptions), std::move(validatorOptions), opts->query().resourceMonitor()); diff --git a/arangod/CMakeLists.txt b/arangod/CMakeLists.txt index 90a43fb35a9c..68f7aa3a2a11 100644 --- a/arangod/CMakeLists.txt +++ b/arangod/CMakeLists.txt @@ -205,6 +205,7 @@ set(LIB_ARANGO_GRAPH_SOURCES Graph/Providers/SingleServerProvider.cpp Graph/Providers/BaseProviderOptions.cpp Graph/Providers/ProviderTracer.cpp + Graph/Steps/SingleServerProviderStep.cpp Graph/Types/UniquenessLevel.cpp Graph/Types/ValidationResult.cpp InternalRestHandler/InternalRestTraverserHandler.cpp diff --git a/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.cpp b/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.cpp index 1c8659abc16a..be69481f9e7a 100644 --- a/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.cpp +++ b/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.cpp @@ -27,6 +27,7 @@ #include "Graph/EdgeCursor.h" #include "Graph/EdgeDocumentToken.h" +#include "Graph/Steps/SingleServerProviderStep.h" #include "StorageEngine/PhysicalCollection.h" #include "Transaction/Helpers.h" #include "Transaction/Methods.h" @@ -35,14 +36,31 @@ // TODO: Needed for the IndexAccessor, should be modified #include "Graph/Providers/SingleServerProvider.h" +#ifdef USE_ENTERPRISE +#include "Enterprise/Graph/Steps/SmartGraphStep.h" +#endif + using namespace arangodb; using namespace arangodb::graph; namespace { IndexIteratorOptions defaultIndexIteratorOptions; + +#ifdef USE_ENTERPRISE +static bool CheckInaccessible(transaction::Methods* trx, VPackSlice const& edge) { + // for skipInaccessibleCollections we need to check the edge + // document, in that case nextWithExtra has no benefit + TRI_ASSERT(edge.isString()); + arangodb::velocypack::StringRef str(edge); + size_t pos = str.find('/'); + TRI_ASSERT(pos != std::string::npos); + return trx->isInaccessibleCollection(str.substr(0, pos).toString()); } +#endif +} // namespace -RefactoredSingleServerEdgeCursor::LookupInfo::LookupInfo( +template +RefactoredSingleServerEdgeCursor::LookupInfo::LookupInfo( transaction::Methods::IndexHandle idx, aql::AstNode* condition, std::optional memberToUpdate, aql::Expression* expression) : _idxHandle(std::move(idx)), @@ -51,19 +69,23 @@ RefactoredSingleServerEdgeCursor::LookupInfo::LookupInfo( _cursor(nullptr), _conditionMemberToUpdate(memberToUpdate) {} -RefactoredSingleServerEdgeCursor::LookupInfo::LookupInfo(LookupInfo&& other) noexcept +template +RefactoredSingleServerEdgeCursor::LookupInfo::LookupInfo(LookupInfo&& other) noexcept : _idxHandle(std::move(other._idxHandle)), _expression(std::move(other._expression)), _indexCondition(other._indexCondition), _cursor(std::move(other._cursor)){}; -RefactoredSingleServerEdgeCursor::LookupInfo::~LookupInfo() = default; +template +RefactoredSingleServerEdgeCursor::LookupInfo::~LookupInfo() = default; -aql::Expression* RefactoredSingleServerEdgeCursor::LookupInfo::getExpression() { +template +aql::Expression* RefactoredSingleServerEdgeCursor::LookupInfo::getExpression() { return _expression; } -void RefactoredSingleServerEdgeCursor::LookupInfo::rearmVertex( +template +void RefactoredSingleServerEdgeCursor::LookupInfo::rearmVertex( VertexType vertex, transaction::Methods* trx, arangodb::aql::Variable const* tmpVar) { auto& node = _indexCondition; // We need to rewire the search condition for the new vertex @@ -107,23 +129,25 @@ void RefactoredSingleServerEdgeCursor::LookupInfo::rearmVertex( // check if the underlying index iterator supports rearming if (_cursor != nullptr && _cursor->canRearm()) { // rearming supported - if (!_cursor->rearm(node, tmpVar, ::defaultIndexIteratorOptions)) { + if (!_cursor->rearm(node, tmpVar, defaultIndexIteratorOptions)) { _cursor = std::make_unique(_cursor->collection(), trx); } } else { // rearming not supported - we need to throw away the index iterator // and create a new one - _cursor = trx->indexScanForCondition(_idxHandle, node, tmpVar, ::defaultIndexIteratorOptions); + _cursor = trx->indexScanForCondition(_idxHandle, node, tmpVar, defaultIndexIteratorOptions); } } -IndexIterator& RefactoredSingleServerEdgeCursor::LookupInfo::cursor() { +template +IndexIterator& RefactoredSingleServerEdgeCursor::LookupInfo::cursor() { // If this kicks in, you forgot to call rearm with a specific vertex TRI_ASSERT(_cursor != nullptr); return *_cursor; } -RefactoredSingleServerEdgeCursor::RefactoredSingleServerEdgeCursor( +template +RefactoredSingleServerEdgeCursor::RefactoredSingleServerEdgeCursor( transaction::Methods* trx, arangodb::aql::Variable const* tmpVar, std::vector const& globalIndexConditions, std::unordered_map> const& depthBasedIndexConditions, @@ -149,30 +173,21 @@ RefactoredSingleServerEdgeCursor::RefactoredSingleServerEdgeCursor( } } -RefactoredSingleServerEdgeCursor::~RefactoredSingleServerEdgeCursor() {} - -#ifdef USE_ENTERPRISE -static bool CheckInaccessible(transaction::Methods* trx, VPackSlice const& edge) { - // for skipInaccessibleCollections we need to check the edge - // document, in that case nextWithExtra has no benefit - TRI_ASSERT(edge.isString()); - arangodb::velocypack::StringRef str(edge); - size_t pos = str.find('/'); - TRI_ASSERT(pos != std::string::npos); - return trx->isInaccessibleCollection(str.substr(0, pos).toString()); -} -#endif +template +RefactoredSingleServerEdgeCursor::~RefactoredSingleServerEdgeCursor() {} -void RefactoredSingleServerEdgeCursor::rearm(VertexType vertex, uint64_t /*depth*/) { +template +void RefactoredSingleServerEdgeCursor::rearm(VertexType vertex, uint64_t /*depth*/) { _currentCursor = 0; for (auto& info : _lookupInfo) { info.rearmVertex(vertex, _trx, _tmpVar); } } -void RefactoredSingleServerEdgeCursor::readAll(SingleServerProvider& provider, - aql::TraversalStats& stats, size_t depth, - Callback const& callback) { +template +void RefactoredSingleServerEdgeCursor::readAll(SingleServerProvider& provider, + aql::TraversalStats& stats, size_t depth, + Callback const& callback) { TRI_ASSERT(!_lookupInfo.empty()); VPackBuilder tmpBuilder; @@ -270,8 +285,9 @@ void RefactoredSingleServerEdgeCursor::readAll(SingleServerProvider& provider, } } -bool RefactoredSingleServerEdgeCursor::evaluateEdgeExpression(arangodb::aql::Expression* expression, - VPackSlice value) { +template +bool RefactoredSingleServerEdgeCursor::evaluateExpression(arangodb::aql::Expression* expression, + VPackSlice value) { if (expression == nullptr) { return true; } @@ -289,3 +305,9 @@ bool RefactoredSingleServerEdgeCursor::evaluateEdgeExpression(arangodb::aql::Exp return res.toBoolean(); } + +template class arangodb::graph::RefactoredSingleServerEdgeCursor; + +#ifdef USE_ENTERPRISE +template class arangodb::graph::RefactoredSingleServerEdgeCursor; +#endif diff --git a/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.h b/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.h index 646bd9f79541..9319c4a0ff9b 100644 --- a/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.h +++ b/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.h @@ -53,8 +53,10 @@ struct IndexAccessor; struct EdgeDocumentToken; -struct SingleServerProvider; +template +class SingleServerProvider; +template class RefactoredSingleServerEdgeCursor { public: struct LookupInfo { @@ -107,7 +109,7 @@ class RefactoredSingleServerEdgeCursor { arangodb::aql::FixedVarExpressionContext& _expressionCtx; public: - void readAll(SingleServerProvider& provider, aql::TraversalStats& stats, + void readAll(SingleServerProvider& provider, aql::TraversalStats& stats, size_t depth, Callback const& callback); void rearm(VertexType vertex, uint64_t depth); diff --git a/arangod/Graph/Enumerators/OneSidedEnumerator.cpp b/arangod/Graph/Enumerators/OneSidedEnumerator.cpp index 2b573453746f..70d640cc5511 100644 --- a/arangod/Graph/Enumerators/OneSidedEnumerator.cpp +++ b/arangod/Graph/Enumerators/OneSidedEnumerator.cpp @@ -29,21 +29,20 @@ #include "Futures/Future.h" #include "Graph/Options/OneSidedEnumeratorOptions.h" -#include "Graph/PathManagement/PathStore.h" -#include "Graph/PathManagement/PathStoreTracer.h" #include "Graph/PathManagement/PathValidator.h" #include "Graph/Providers/ClusterProvider.h" -#include "Graph/Providers/ProviderTracer.h" #include "Graph/Providers/SingleServerProvider.h" -#include "Graph/Queues/FifoQueue.h" -#include "Graph/Queues/LifoQueue.h" #include "Graph/Queues/QueueTracer.h" +#include "Graph/Steps/SingleServerProviderStep.h" +#include "Graph/Types/ValidationResult.h" #include "Graph/algorithm-aliases.h" -#include +#ifdef USE_ENTERPRISE +#include "Enterprise/Graph/algorithm-aliases-ee.h" +#endif + #include #include -#include using namespace arangodb; using namespace arangodb::graph; @@ -253,20 +252,50 @@ auto OneSidedEnumerator::stealStats() -> aql::TraversalStats { } /* SingleServerProvider Section */ +using SingleServerProviderStep = ::arangodb::graph::SingleServerProviderStep; // Breadth First Search -template class ::arangodb::graph::OneSidedEnumerator>; -template class ::arangodb::graph::OneSidedEnumerator>; -template class ::arangodb::graph::OneSidedEnumerator>; -template class ::arangodb::graph::OneSidedEnumerator>; -template class ::arangodb::graph::OneSidedEnumerator>; -template class ::arangodb::graph::OneSidedEnumerator>; +template class ::arangodb::graph::OneSidedEnumerator< + BFSConfiguration, VertexUniquenessLevel::PATH, false>>; +template class ::arangodb::graph::OneSidedEnumerator< + BFSConfiguration, VertexUniquenessLevel::PATH, true>>; +template class ::arangodb::graph::OneSidedEnumerator< + BFSConfiguration, VertexUniquenessLevel::NONE, false>>; +template class ::arangodb::graph::OneSidedEnumerator< + BFSConfiguration, VertexUniquenessLevel::NONE, true>>; +template class ::arangodb::graph::OneSidedEnumerator< + BFSConfiguration, VertexUniquenessLevel::GLOBAL, false>>; +template class ::arangodb::graph::OneSidedEnumerator< + BFSConfiguration, VertexUniquenessLevel::GLOBAL, true>>; // Depth First Search -template class ::arangodb::graph::OneSidedEnumerator>; -template class ::arangodb::graph::OneSidedEnumerator>; -template class ::arangodb::graph::OneSidedEnumerator>; -template class ::arangodb::graph::OneSidedEnumerator>; -template class ::arangodb::graph::OneSidedEnumerator>; -template class ::arangodb::graph::OneSidedEnumerator>; - +template class ::arangodb::graph::OneSidedEnumerator< + DFSConfiguration, VertexUniquenessLevel::PATH, false>>; +template class ::arangodb::graph::OneSidedEnumerator< + DFSConfiguration, VertexUniquenessLevel::PATH, true>>; +template class ::arangodb::graph::OneSidedEnumerator< + DFSConfiguration, VertexUniquenessLevel::NONE, false>>; +template class ::arangodb::graph::OneSidedEnumerator< + DFSConfiguration, VertexUniquenessLevel::NONE, true>>; +template class ::arangodb::graph::OneSidedEnumerator< + DFSConfiguration, VertexUniquenessLevel::GLOBAL, false>>; +template class ::arangodb::graph::OneSidedEnumerator< + DFSConfiguration, VertexUniquenessLevel::GLOBAL, true>>; + +#ifdef USE_ENTERPRISE +// Depth First Search +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::PATH, false>>; +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::NONE, false>>; +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::GLOBAL, false>>; + +// Tracing +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::PATH, true>>; +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::NONE, true>>; +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::GLOBAL, true>>; +#endif diff --git a/arangod/Graph/Enumerators/TwoSidedEnumerator.cpp b/arangod/Graph/Enumerators/TwoSidedEnumerator.cpp index 10df092a905d..4466219efb0f 100644 --- a/arangod/Graph/Enumerators/TwoSidedEnumerator.cpp +++ b/arangod/Graph/Enumerators/TwoSidedEnumerator.cpp @@ -40,9 +40,10 @@ #include "Graph/Providers/SingleServerProvider.h" #include "Graph/Queues/FifoQueue.h" #include "Graph/Queues/QueueTracer.h" +#include "Graph/Steps/SingleServerProviderStep.h" +#include "Graph/Types/ValidationResult.h" #include "Graph/algorithm-aliases.h" -#include #include #include #include @@ -428,18 +429,19 @@ auto TwoSidedEnumerator:: } /* SingleServerProvider Section */ +using SingleServerProviderStep = ::arangodb::graph::SingleServerProviderStep; template class ::arangodb::graph::TwoSidedEnumerator< - ::arangodb::graph::FifoQueue<::arangodb::graph::SingleServerProvider::Step>, - ::arangodb::graph::PathStore, SingleServerProvider, - ::arangodb::graph::PathValidator, VertexUniquenessLevel::PATH>>; + ::arangodb::graph::FifoQueue, + ::arangodb::graph::PathStore, SingleServerProvider, + ::arangodb::graph::PathValidator, PathStore, VertexUniquenessLevel::PATH>>; template class ::arangodb::graph::TwoSidedEnumerator< - ::arangodb::graph::QueueTracer<::arangodb::graph::FifoQueue<::arangodb::graph::SingleServerProvider::Step>>, - ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore>, - ::arangodb::graph::ProviderTracer, - ::arangodb::graph::PathValidator<::arangodb::graph::ProviderTracer, - ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore>, VertexUniquenessLevel::PATH>>; + ::arangodb::graph::QueueTracer<::arangodb::graph::FifoQueue>, + ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore>, + ::arangodb::graph::ProviderTracer>, + ::arangodb::graph::PathValidator<::arangodb::graph::ProviderTracer>, + ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore>, VertexUniquenessLevel::PATH>>; /* ClusterProvider Section */ diff --git a/arangod/Graph/PathManagement/PathResult.cpp b/arangod/Graph/PathManagement/PathResult.cpp index 275eeaf89a96..ff0fdb731feb 100644 --- a/arangod/Graph/PathManagement/PathResult.cpp +++ b/arangod/Graph/PathManagement/PathResult.cpp @@ -29,6 +29,12 @@ #include "Graph/Providers/ProviderTracer.h" #include "Graph/Providers/SingleServerProvider.h" +#include "Graph/Steps/SingleServerProviderStep.h" + +#ifdef USE_ENTERPRISE +#include "Enterprise/Graph/Steps/SmartGraphStep.h" +#endif + #include #include @@ -115,11 +121,16 @@ auto PathResult::isEmpty() const -> bool { /* SingleServerProvider Section */ -template class ::arangodb::graph::PathResult<::arangodb::graph::SingleServerProvider, - ::arangodb::graph::SingleServerProvider::Step>; +using SingleServerProviderStep = ::arangodb::graph::SingleServerProviderStep; + +template class ::arangodb::graph::PathResult<::arangodb::graph::SingleServerProvider, SingleServerProviderStep>; +template class ::arangodb::graph::PathResult<::arangodb::graph::ProviderTracer>, SingleServerProviderStep>; -template class ::arangodb::graph::PathResult<::arangodb::graph::ProviderTracer<::arangodb::graph::SingleServerProvider>, - ::arangodb::graph::SingleServerProvider::Step>; +#ifdef USE_ENTERPRISE +template class ::arangodb::graph::PathResult<::arangodb::graph::SingleServerProvider, enterprise::SmartGraphStep>; +template class ::arangodb::graph::PathResult< + ::arangodb::graph::ProviderTracer>, enterprise::SmartGraphStep>; +#endif /* ClusterProvider Section */ diff --git a/arangod/Graph/PathManagement/PathStore.cpp b/arangod/Graph/PathManagement/PathStore.cpp index e6402d410764..0ae2ece819f0 100644 --- a/arangod/Graph/PathManagement/PathStore.cpp +++ b/arangod/Graph/PathManagement/PathStore.cpp @@ -26,9 +26,13 @@ #include "Graph/Providers/ClusterProvider.h" #include "Graph/Providers/ProviderTracer.h" -#include "Graph/Providers/SingleServerProvider.h" +#include "Graph/Steps/SingleServerProviderStep.h" #include "Graph/Types/ValidationResult.h" +#ifdef USE_ENTERPRISE +#include "Enterprise/Graph/Steps/SmartGraphStep.h" +#endif + #include #include @@ -91,7 +95,7 @@ Step& PathStore::getStepReference(size_t position) { TRI_ASSERT(position <= size()); auto& step = _schreier.at(position); LOG_TOPIC("45bf6", TRACE, Logger::GRAPHS) - << " Get step: " << step.toString(); + << " Get step: " << step.toString(); return step; } @@ -169,10 +173,9 @@ auto PathStore::visitReversePath(Step const& step, } template -auto PathStore::modifyReversePath(Step& step, - std::function const& visitor) --> bool { - Step * walker = &step; +auto PathStore::modifyReversePath(Step& step, std::function const& visitor) + -> bool { + Step* walker = &step; // Guaranteed to make progress, as the schreier vector contains a loop-free tree. while (true) { bool cont = visitor(*walker); @@ -189,27 +192,60 @@ auto PathStore::modifyReversePath(Step& step, } /* SingleServerProvider Section */ +using SingleServerProviderStep = ::arangodb::graph::SingleServerProviderStep; -template class PathStore; +template class PathStore; -template void PathStore::buildPath>( - SingleServerProvider::Step const& vertex, - PathResult& path) const; +template void PathStore::buildPath, SingleServerProviderStep>>( + SingleServerProviderStep const& vertex, + PathResult, SingleServerProviderStep>& path) const; -template void PathStore::reverseBuildPath( - SingleServerProvider::Step const& vertex, - PathResult& path) const; +template void PathStore::reverseBuildPath>( + SingleServerProviderStep const& vertex, + PathResult, SingleServerProviderStep>& path) const; // Tracing -template void PathStore::buildPath< - PathResult, ProviderTracer::Step>>( - ProviderTracer::Step const& vertex, - PathResult, ProviderTracer::Step>& path) const; +template void PathStore::buildPath< + PathResult>, + ProviderTracer>::Step>>( + ProviderTracer>::Step const& vertex, + PathResult>, + ProviderTracer>::Step>& path) const; + +template void PathStore>::Step>::reverseBuildPath< + ProviderTracer>>( + ProviderTracer>::Step const& vertex, + PathResult>, + ProviderTracer>::Step>& path) const; + +#ifdef USE_ENTERPRISE +template class PathStore; + +template void PathStore::buildPath< + PathResult, enterprise::SmartGraphStep>>( + enterprise::SmartGraphStep const& vertex, + PathResult, enterprise::SmartGraphStep>& path) const; + +template void PathStore::reverseBuildPath>( + enterprise::SmartGraphStep const& vertex, + PathResult, enterprise::SmartGraphStep>& path) const; + +// Tracing -template void PathStore::Step>::reverseBuildPath>( - ProviderTracer::Step const& vertex, - PathResult, ProviderTracer::Step>& path) const; +template void PathStore::buildPath< + PathResult>, + ProviderTracer>::Step>>( + ProviderTracer>::Step const& vertex, + PathResult>, + ProviderTracer>::Step>& path) const; + +template void PathStore>::Step>::reverseBuildPath< + ProviderTracer>>( + ProviderTracer>::Step const& vertex, + PathResult>, + ProviderTracer>::Step>& path) const; +#endif /* ClusterProvider Section */ diff --git a/arangod/Graph/PathManagement/PathStoreTracer.cpp b/arangod/Graph/PathManagement/PathStoreTracer.cpp index 280bc7331e68..7e3783cdd1bf 100644 --- a/arangod/Graph/PathManagement/PathStoreTracer.cpp +++ b/arangod/Graph/PathManagement/PathStoreTracer.cpp @@ -32,8 +32,13 @@ #include "Graph/Providers/ClusterProvider.h" #include "Graph/Providers/ProviderTracer.h" #include "Graph/Providers/SingleServerProvider.h" +#include "Graph/Steps/SingleServerProviderStep.h" #include "Graph/Types/ValidationResult.h" +#ifdef USE_ENTERPRISE +#include "Enterprise/Graph/Steps/SmartGraphStep.h" +#endif + using namespace arangodb; using namespace arangodb::graph; @@ -71,7 +76,7 @@ typename PathStoreImpl::Step PathStoreTracer::getStep(size_t posi } template -typename PathStoreImpl::Step& PathStoreTracer::getStepReference(size_t position) { +typename PathStoreImpl::Step& PathStoreTracer::getStepReference(size_t position) { double start = TRI_microtime(); TRI_DEFER(_stats["getStepReference"].addTiming(TRI_microtime() - start)); return _impl.getStepReference(position); @@ -112,25 +117,49 @@ auto PathStoreTracer::visitReversePath( } template -auto PathStoreTracer::modifyReversePath(Step& step, const std::function& visitor) -> bool { +auto PathStoreTracer::modifyReversePath(Step& step, + const std::function& visitor) + -> bool { double start = TRI_microtime(); TRI_DEFER(_stats["modifyReversePath"].addTiming(TRI_microtime() - start)); return _impl.modifyReversePath(step, visitor); } /* SingleServerProvider Section */ +using SingleServerProviderStep = ::arangodb::graph::SingleServerProviderStep; -template class ::arangodb::graph::PathStoreTracer>; +template class ::arangodb::graph::PathStoreTracer>; // Tracing -template void ::arangodb::graph::PathStoreTracer>::buildPath< - PathResult, ProviderTracer::Step>>( - ProviderTracer::Step const& vertex, - PathResult, ProviderTracer::Step>& path) const; - -template void arangodb::graph::PathStoreTracer>::reverseBuildPath>( - ProviderTracer::Step const& vertex, - PathResult, ProviderTracer::Step>& path) const; +template void ::arangodb::graph::PathStoreTracer>::buildPath< + PathResult>, + ProviderTracer>::Step>>( + ProviderTracer>::Step const& vertex, + PathResult>, + ProviderTracer>::Step>& path) const; + +template void arangodb::graph::PathStoreTracer>::reverseBuildPath< + ProviderTracer>>( + ProviderTracer>::Step const& vertex, + PathResult>, + ProviderTracer>::Step>& path) const; + +#ifdef USE_ENTERPRISE +template class ::arangodb::graph::PathStoreTracer>; + +template void ::arangodb::graph::PathStoreTracer>::buildPath< + PathResult>, + ProviderTracer>::Step>>( + ProviderTracer>::Step const& vertex, + PathResult>, + ProviderTracer>::Step>& path) const; + +template void arangodb::graph::PathStoreTracer>::reverseBuildPath< + ProviderTracer>>( + ProviderTracer>::Step const& vertex, + PathResult>, + ProviderTracer>::Step>& path) const; +#endif /* ClusterProvider Section */ diff --git a/arangod/Graph/PathManagement/PathValidator.cpp b/arangod/Graph/PathManagement/PathValidator.cpp index 8bd7e15ce00b..356e2b242397 100644 --- a/arangod/Graph/PathManagement/PathValidator.cpp +++ b/arangod/Graph/PathManagement/PathValidator.cpp @@ -29,8 +29,13 @@ #include "Graph/Providers/ClusterProvider.h" #include "Graph/Providers/ProviderTracer.h" #include "Graph/Providers/SingleServerProvider.h" +#include "Graph/Steps/SingleServerProviderStep.h" #include "Graph/Types/ValidationResult.h" +#ifdef USE_ENTERPRISE +#include "Enterprise/Graph/Steps/SmartGraphStep.h" +#endif + #include "Basics/Exceptions.h" using namespace arangodb; @@ -217,17 +222,45 @@ namespace arangodb { namespace graph { /* SingleServerProvider Section */ -template class PathValidator, VertexUniquenessLevel::NONE>; -template class PathValidator, - PathStoreTracer::Step>>, VertexUniquenessLevel::NONE>; - -template class PathValidator, VertexUniquenessLevel::PATH>; -template class PathValidator, - PathStoreTracer::Step>>, VertexUniquenessLevel::PATH>; - -template class PathValidator, VertexUniquenessLevel::GLOBAL>; -template class PathValidator, - PathStoreTracer::Step>>, VertexUniquenessLevel::GLOBAL>; +using SingleServerProviderStep = ::arangodb::graph::SingleServerProviderStep; + +template class PathValidator, + PathStore, VertexUniquenessLevel::NONE>; +template class PathValidator>, + PathStoreTracer>::Step>>, + VertexUniquenessLevel::NONE>; + +template class PathValidator, + PathStore::Step>, VertexUniquenessLevel::PATH>; +template class PathValidator>, + PathStoreTracer>::Step>>, + VertexUniquenessLevel::PATH>; + +template class PathValidator, + PathStore::Step>, VertexUniquenessLevel::GLOBAL>; +template class PathValidator>, + PathStoreTracer>::Step>>, + VertexUniquenessLevel::GLOBAL>; + +#ifdef USE_ENTERPRISE +template class PathValidator, + PathStore, VertexUniquenessLevel::NONE>; +template class PathValidator>, + PathStoreTracer>::Step>>, + VertexUniquenessLevel::NONE>; + +template class PathValidator, + PathStore::Step>, VertexUniquenessLevel::PATH>; +template class PathValidator>, + PathStoreTracer>::Step>>, + VertexUniquenessLevel::PATH>; + +template class PathValidator, + PathStore::Step>, VertexUniquenessLevel::GLOBAL>; +template class PathValidator>, + PathStoreTracer>::Step>>, + VertexUniquenessLevel::GLOBAL>; +#endif /* ClusterProvider Section */ template class PathValidator, VertexUniquenessLevel::NONE>; diff --git a/arangod/Graph/PathManagement/SingleProviderPathResult.cpp b/arangod/Graph/PathManagement/SingleProviderPathResult.cpp index c1597ed4d74d..811a347aaeef 100644 --- a/arangod/Graph/PathManagement/SingleProviderPathResult.cpp +++ b/arangod/Graph/PathManagement/SingleProviderPathResult.cpp @@ -30,6 +30,11 @@ #include "Graph/Providers/ClusterProvider.h" #include "Graph/Providers/ProviderTracer.h" #include "Graph/Providers/SingleServerProvider.h" +#include "Graph/Steps/SingleServerProviderStep.h" + +#ifdef USE_ENTERPRISE +#include "Enterprise/Graph/Steps/SmartGraphStep.h" +#endif #include #include @@ -37,6 +42,10 @@ using namespace arangodb; using namespace arangodb::graph; +namespace arangodb::graph::enterprise { +class SmartGraphStep; +} // namespace arangodb::graph::enterprise + template SingleProviderPathResult::SingleProviderPathResult( Step* step, ProviderType& provider, PathStoreType& store) @@ -109,49 +118,54 @@ auto SingleProviderPathResult::toVelocyPack( template auto SingleProviderPathResult::toSchreierEntry( arangodb::velocypack::Builder& result, size_t& currentLength) -> void { - size_t prevIndex = 0; - - auto writeStepToBuilder = [&](Step& step) { - VPackArrayBuilder arrayGuard(&result); - // Create a VPackValue based on the char* inside the HashedStringRef, will save us a String copy each time. - result.add(VPackValue(step.getVertex().getID().begin())); - - // Index position of previous step - result.add(VPackValue(prevIndex)); - // The depth of the current step - result.add(VPackValue(step.getDepth())); - - bool isResponsible = step.isResponsible(_provider.trx()); - // Print if we have a loose end here (looseEnd := this server is NOT responsible) - result.add(VPackValue(!isResponsible)); - if (isResponsible) { - // This server needs to provide the data for the vertex - _provider.addVertexToBuilder(step.getVertex(), result); - } else { - result.add(VPackSlice::nullSlice()); - } + if constexpr (!std::is_same_v) { // TODO: eventually move to EE + TRI_ASSERT(false); + THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); + } else { + size_t prevIndex = 0; + + auto writeStepToBuilder = [&](Step& step) { + VPackArrayBuilder arrayGuard(&result); + // Create a VPackValue based on the char* inside the HashedStringRef, will save us a String copy each time. + result.add(VPackValue(step.getVertex().getID().begin())); + + // Index position of previous step + result.add(VPackValue(prevIndex)); + // The depth of the current step + result.add(VPackValue(step.getDepth())); + + bool isResponsible = step.isResponsible(_provider.trx()); + // Print if we have a loose end here (looseEnd := this server is NOT responsible) + result.add(VPackValue(!isResponsible)); + if (isResponsible) { + // This server needs to provide the data for the vertex + _provider.addVertexToBuilder(step.getVertex(), result); + } else { + result.add(VPackSlice::nullSlice()); + } - _provider.addEdgeToBuilder(step.getEdge(), result); - }; // TODO: Create method instead of lambda + _provider.addEdgeToBuilder(step.getEdge(), result); + }; // TODO: Create method instead of lambda - std::vector toWrite{}; + std::vector toWrite{}; - _store.modifyReversePath(*_step, [&](Step& step) -> bool { - if (!step.hasLocalSchreierIndex()) { - toWrite.emplace_back(&step); - return true; - } else { - prevIndex = step.getLocalSchreierIndex(); - return false; - } - }); + _store.modifyReversePath(*_step, [&](Step& step) -> bool { + if (!step.hasLocalSchreierIndex()) { + toWrite.emplace_back(&step); + return true; + } else { + prevIndex = step.getLocalSchreierIndex(); + return false; + } + }); - for (auto it = toWrite.rbegin(); it != toWrite.rend(); it++) { - TRI_ASSERT(!(*it)->hasLocalSchreierIndex()); - writeStepToBuilder(**it); + for (auto it = toWrite.rbegin(); it != toWrite.rend(); it++) { + TRI_ASSERT(!(*it)->hasLocalSchreierIndex()); + writeStepToBuilder(**it); - prevIndex = currentLength; - (*it)->setLocalSchreierIndex(currentLength++); + prevIndex = currentLength; + (*it)->setLocalSchreierIndex(currentLength++); + } } } @@ -162,15 +176,25 @@ auto SingleProviderPathResult::isEmpty() cons } /* SingleServerProvider Section */ +using SingleServerProviderStep = ::arangodb::graph::SingleServerProviderStep; + +template class ::arangodb::graph::SingleProviderPathResult< + ::arangodb::graph::SingleServerProvider, + ::arangodb::graph::PathStore, SingleServerProviderStep>; + +template class ::arangodb::graph::SingleProviderPathResult< + ::arangodb::graph::ProviderTracer<::arangodb::graph::SingleServerProvider>, + ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore>, SingleServerProviderStep>; +#ifdef USE_ENTERPRISE template class ::arangodb::graph::SingleProviderPathResult< - ::arangodb::graph::SingleServerProvider, ::arangodb::graph::PathStore<::arangodb::graph::SingleServerProvider::Step>, - ::arangodb::graph::SingleServerProvider::Step>; + ::arangodb::graph::SingleServerProvider, + ::arangodb::graph::PathStore, enterprise::SmartGraphStep>; template class ::arangodb::graph::SingleProviderPathResult< - ::arangodb::graph::ProviderTracer<::arangodb::graph::SingleServerProvider>, - ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::graph::SingleServerProvider::Step>>, - ::arangodb::graph::SingleServerProvider::Step>; + ::arangodb::graph::ProviderTracer<::arangodb::graph::SingleServerProvider>, + ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore>, enterprise::SmartGraphStep>; +#endif /* ClusterProvider Section */ template class ::arangodb::graph::SingleProviderPathResult< diff --git a/arangod/Graph/Providers/ClusterProvider.h b/arangod/Graph/Providers/ClusterProvider.h index e10dd2f1a9c5..6bc8ddcb6ed6 100644 --- a/arangod/Graph/Providers/ClusterProvider.h +++ b/arangod/Graph/Providers/ClusterProvider.h @@ -129,19 +129,6 @@ class ClusterProvider { return collectionNameResult.get().first; }; - /* to be moved-out later into seperate step */ - void setLocalSchreierIndex(size_t index) { - TRI_ASSERT(index != std::numeric_limits::max()); - TRI_ASSERT(!hasLocalSchreierIndex()); - _localSchreierIndex = index; - } - - bool hasLocalSchreierIndex() const { - return _localSchreierIndex != std::numeric_limits::max(); - } - - std::size_t getLocalSchreierIndex() const { return _localSchreierIndex; } - bool isResponsible(transaction::Methods* trx) const; friend auto operator<<(std::ostream& out, Step const& step) -> std::ostream&; diff --git a/arangod/Graph/Providers/ProviderTracer.cpp b/arangod/Graph/Providers/ProviderTracer.cpp index e78e41dfae7f..640f4684daa6 100644 --- a/arangod/Graph/Providers/ProviderTracer.cpp +++ b/arangod/Graph/Providers/ProviderTracer.cpp @@ -29,14 +29,18 @@ #include "Graph/Providers/ClusterProvider.h" #include "Graph/Providers/SingleServerProvider.h" +#include "Graph/Steps/SingleServerProviderStep.h" + +#ifdef USE_ENTERPRISE +#include "Enterprise/Graph/Steps/SmartGraphStep.h" +#endif using namespace arangodb; using namespace arangodb::graph; template ProviderTracer::ProviderTracer(arangodb::aql::QueryContext& queryContext, - Options opts, - arangodb::ResourceMonitor& resourceMonitor) + Options opts, arangodb::ResourceMonitor& resourceMonitor) : _impl{queryContext, std::move(opts), resourceMonitor} {} template @@ -48,7 +52,8 @@ ProviderTracer::~ProviderTracer() { } template -typename ProviderImpl::Step ProviderTracer::startVertex(VertexType vertex, size_t depth) { +typename ProviderImpl::Step ProviderTracer::startVertex(VertexType vertex, + size_t depth) { double start = TRI_microtime(); TRI_DEFER(_stats["startVertex"].addTiming(TRI_microtime() - start)); return _impl.startVertex(vertex, depth); @@ -107,5 +112,12 @@ transaction::Methods* ProviderTracer::trx() { return _impl.trx(); } -template class ::arangodb::graph::ProviderTracer; -template class ::arangodb::graph::ProviderTracer; \ No newline at end of file +using SingleServerProviderStep = ::arangodb::graph::SingleServerProviderStep; + +template class ::arangodb::graph::ProviderTracer>; + +#ifdef USE_ENTERPRISE +template class ::arangodb::graph::ProviderTracer>; +#endif + +template class ::arangodb::graph::ProviderTracer; diff --git a/arangod/Graph/Providers/SingleServerProvider.cpp b/arangod/Graph/Providers/SingleServerProvider.cpp index 46ac1b1bb44f..59039e8bd777 100644 --- a/arangod/Graph/Providers/SingleServerProvider.cpp +++ b/arangod/Graph/Providers/SingleServerProvider.cpp @@ -26,60 +26,33 @@ #include "Aql/QueryContext.h" #include "Graph/Cursors/RefactoredSingleServerEdgeCursor.h" +#include "Graph/Steps/SingleServerProviderStep.h" #include "Transaction/Helpers.h" #include "Futures/Future.h" #include "Futures/Utilities.h" +#ifdef USE_ENTERPRISE +#include "Enterprise/Graph/Steps/SmartGraphStep.h" +#endif + #include using namespace arangodb; using namespace arangodb::graph; -namespace arangodb { +// TODO: remove +/*namespace arangodb { namespace graph { -auto operator<<(std::ostream& out, SingleServerProvider::Step const& step) -> std::ostream& { - out << step._vertex.getID(); - return out; +auto operator<<(std::ostream& out, typename SingleServerProvider::Step const& +step) -> std::ostream& { out << step._vertex.getID(); return out; } } // namespace graph -} // namespace arangodb - -SingleServerProvider::Step::Step(VertexType v) : _vertex(v), _edge() {} - -SingleServerProvider::Step::Step(VertexType v, size_t depth) - : BaseStep(std::numeric_limits::max(), depth), _vertex(v), _edge() {} - -SingleServerProvider::Step::Step(VertexType v, EdgeDocumentToken edge, size_t prev) - : BaseStep(prev), _vertex(v), _edge(std::move(edge)) {} - -SingleServerProvider::Step::Step(VertexType v, EdgeDocumentToken edge, size_t prev, size_t depth) - : BaseStep(prev, depth), _vertex(v), _edge(std::move(edge)) {} +} // namespace arangodb*/ -SingleServerProvider::Step::~Step() = default; - -VertexType const& SingleServerProvider::Step::Vertex::getID() const { - return _vertex; -} - -EdgeDocumentToken const& SingleServerProvider::Step::Edge::getID() const { - return _token; -} - -bool SingleServerProvider::Step::Edge::isValid() const { - return getID().isValid(); -}; - -#ifndef USE_ENTERPRISE -bool SingleServerProvider::Step::isResponsible(transaction::Methods* trx) const { - return true; -}; -#endif - - - -void SingleServerProvider::addEdgeToBuilder(Step::Edge const& edge, - arangodb::velocypack::Builder& builder) { +template +void SingleServerProvider::addEdgeToBuilder(typename Step::Edge const& edge, + arangodb::velocypack::Builder& builder) { if (edge.isValid()) { insertEdgeIntoResult(edge.getID(), builder); } else { @@ -87,14 +60,10 @@ void SingleServerProvider::addEdgeToBuilder(Step::Edge const& edge, } }; -void SingleServerProvider::Step::Edge::addToBuilder(SingleServerProvider& provider, - arangodb::velocypack::Builder& builder) const { - provider.insertEdgeIntoResult(getID(), builder); -} - -SingleServerProvider::SingleServerProvider(arangodb::aql::QueryContext& queryContext, - BaseProviderOptions opts, - arangodb::ResourceMonitor& resourceMonitor) +template +SingleServerProvider::SingleServerProvider(arangodb::aql::QueryContext& queryContext, + BaseProviderOptions opts, + arangodb::ResourceMonitor& resourceMonitor) : _trx(std::make_unique(queryContext.newTrxContext())), _opts(std::move(opts)), _cache(_trx.get(), &queryContext, resourceMonitor, _stats, @@ -104,7 +73,8 @@ SingleServerProvider::SingleServerProvider(arangodb::aql::QueryContext& queryCon _cursor = buildCursor(opts.expressionContext()); } -void SingleServerProvider::activateCache(bool enableDocumentCache) { +template +void SingleServerProvider::activateCache(bool enableDocumentCache) { // Do not call this twice. // TRI_ASSERT(_cache == nullptr); // TODO: enableDocumentCache check + opts check + cacheManager check @@ -123,7 +93,8 @@ void SingleServerProvider::activateCache(bool enableDocumentCache) { // _cache = new RefactoredTraverserCache(query()); } -auto SingleServerProvider::startVertex(VertexType vertex, size_t depth) -> Step { +template +auto SingleServerProvider::startVertex(VertexType vertex, size_t depth) -> Step { LOG_TOPIC("78156", TRACE, Logger::GRAPHS) << " Start Vertex:" << vertex; @@ -132,7 +103,8 @@ auto SingleServerProvider::startVertex(VertexType vertex, size_t depth) -> Step return Step(_cache.persistString(vertex), depth); } -auto SingleServerProvider::fetch(std::vector const& looseEnds) +template +auto SingleServerProvider::fetch(std::vector const& looseEnds) -> futures::Future> { // Should never be called in SingleServer case TRI_ASSERT(false); @@ -144,8 +116,10 @@ auto SingleServerProvider::fetch(std::vector const& looseEnds) return futures::makeFuture(std::move(result)); } -auto SingleServerProvider::expand(Step const& step, size_t previous, - std::function const& callback) -> void { +template +auto SingleServerProvider::expand(Step const& step, size_t previous, + std::function const& callback) + -> void { TRI_ASSERT(!step.isLooseEnd()); auto const& vertex = step.getVertex(); TRI_ASSERT(_cursor != nullptr); @@ -173,34 +147,45 @@ auto SingleServerProvider::expand(Step const& step, size_t previous, }); } -void SingleServerProvider::addVertexToBuilder(Step::Vertex const& vertex, - arangodb::velocypack::Builder& builder) { +template +void SingleServerProvider::addVertexToBuilder(typename Step::Vertex const& vertex, + arangodb::velocypack::Builder& builder) { _cache.insertVertexIntoResult(_stats, vertex.getID(), builder); }; -void SingleServerProvider::insertEdgeIntoResult(EdgeDocumentToken edge, - arangodb::velocypack::Builder& builder) { +template +void SingleServerProvider::insertEdgeIntoResult(EdgeDocumentToken edge, + arangodb::velocypack::Builder& builder) { _cache.insertEdgeIntoResult(edge, builder); } -std::unique_ptr SingleServerProvider::buildCursor( +template +std::unique_ptr> SingleServerProvider::buildCursor( arangodb::aql::FixedVarExpressionContext& expressionContext) { - return std::make_unique( + return std::make_unique>( trx(), _opts.tmpVar(), _opts.indexInformations().first, _opts.indexInformations().second, expressionContext); } -arangodb::transaction::Methods* SingleServerProvider::trx() { +template +arangodb::transaction::Methods* SingleServerProvider::trx() { TRI_ASSERT(_trx != nullptr); TRI_ASSERT(_trx->state() != nullptr); TRI_ASSERT(_trx->transactionContextPtr() != nullptr); return _trx.get(); } -arangodb::aql::TraversalStats SingleServerProvider::stealStats() { +template +arangodb::aql::TraversalStats SingleServerProvider::stealStats() { auto t = _stats; // Placement new of stats, do not reallocate space. _stats.~TraversalStats(); new (&_stats) aql::TraversalStats{}; return t; } + +template class arangodb::graph::SingleServerProvider; + +#ifdef USE_ENTERPRISE +template class arangodb::graph::SingleServerProvider; +#endif diff --git a/arangod/Graph/Providers/SingleServerProvider.h b/arangod/Graph/Providers/SingleServerProvider.h index d35e28b8f7ab..1d7f9c0265ce 100644 --- a/arangod/Graph/Providers/SingleServerProvider.h +++ b/arangod/Graph/Providers/SingleServerProvider.h @@ -34,10 +34,6 @@ #include "Aql/TraversalStats.h" #include "Basics/ResourceUsage.h" -#include "Transaction/Methods.h" - -#include - namespace arangodb { namespace futures { @@ -59,94 +55,11 @@ namespace graph { // TODO: we need to control from the outside if and which parts of the vertex - (will be implemented in the future via template parameters) // data should be returned. This is most-likely done via a template parameter like // this: template -struct SingleServerProvider { +template +class SingleServerProvider { + public: using Options = BaseProviderOptions; - class Step : public arangodb::graph::BaseStep { - public: - class Vertex { - public: - explicit Vertex(VertexType v) : _vertex(v) {} - - VertexType const& getID() const; - - bool operator<(Vertex const& other) const noexcept { - return _vertex < other._vertex; - } - - bool operator>(Vertex const& other) const noexcept { - return _vertex > other._vertex; - } - - private: - VertexType _vertex; - }; - - class Edge { - public: - explicit Edge(EdgeDocumentToken tkn) noexcept : _token(std::move(tkn)) {} - Edge() noexcept : _token() {} - - void addToBuilder(SingleServerProvider& provider, - arangodb::velocypack::Builder& builder) const; - EdgeDocumentToken const& getID() const; - bool isValid() const; - - private: - EdgeDocumentToken _token; - }; - - Step(VertexType v); - Step(VertexType v, EdgeDocumentToken edge, size_t prev); - Step(VertexType v, EdgeDocumentToken edge, size_t prev, size_t depth); - Step(VertexType v, size_t depth); - ~Step(); - - bool operator<(Step const& other) const noexcept { - return _vertex < other._vertex; - } - - Vertex const& getVertex() const { return _vertex; } - Edge const& getEdge() const { return _edge; } - - std::string toString() const { - return ": " + _vertex.getID().toString(); - } - bool isProcessable() const { return !isLooseEnd(); } - bool isLooseEnd() const { return false; } - - VertexType getVertexIdentifier() const { return _vertex.getID(); } - - std::string getCollectionName() const { - auto collectionNameResult = extractCollectionName(_vertex.getID()); - if (collectionNameResult.fail()) { - THROW_ARANGO_EXCEPTION(collectionNameResult.result()); - } - return collectionNameResult.get().first; - }; - - void setLocalSchreierIndex(size_t index) { - TRI_ASSERT(index != std::numeric_limits::max()); - TRI_ASSERT(!hasLocalSchreierIndex()); - _localSchreierIndex = index; - } - - bool hasLocalSchreierIndex() const { - return _localSchreierIndex != std::numeric_limits::max(); - } - - std::size_t getLocalSchreierIndex() const { - return _localSchreierIndex; - } - - bool isResponsible(transaction::Methods* trx) const; - - friend auto operator<<(std::ostream& out, Step const& step) -> std::ostream&; - - private: - Vertex _vertex; - Edge _edge; - size_t _localSchreierIndex = std::numeric_limits::max(); - }; + using Step = StepType; public: SingleServerProvider(arangodb::aql::QueryContext& queryContext, Options opts, @@ -165,8 +78,9 @@ struct SingleServerProvider { void insertEdgeIntoResult(EdgeDocumentToken edge, arangodb::velocypack::Builder& builder); - void addVertexToBuilder(Step::Vertex const& vertex, arangodb::velocypack::Builder& builder); - void addEdgeToBuilder(Step::Edge const& edge, arangodb::velocypack::Builder& builder); + void addVertexToBuilder(typename Step::Vertex const& vertex, + arangodb::velocypack::Builder& builder); + void addEdgeToBuilder(typename Step::Edge const& edge, arangodb::velocypack::Builder& builder); void destroyEngines(){}; @@ -177,7 +91,7 @@ struct SingleServerProvider { private: void activateCache(bool enableDocumentCache); - std::unique_ptr buildCursor( + std::unique_ptr> buildCursor( arangodb::aql::FixedVarExpressionContext& expressionContext); private: @@ -185,7 +99,7 @@ struct SingleServerProvider { // alive - Note: _trx must be first here because it is used in _cursor std::unique_ptr _trx; - std::unique_ptr _cursor; + std::unique_ptr> _cursor; BaseProviderOptions _opts; diff --git a/arangod/Graph/Queues/QueueTracer.cpp b/arangod/Graph/Queues/QueueTracer.cpp index 0b3cfafad684..831d0091d6e4 100644 --- a/arangod/Graph/Queues/QueueTracer.cpp +++ b/arangod/Graph/Queues/QueueTracer.cpp @@ -22,13 +22,19 @@ //////////////////////////////////////////////////////////////////////////////// #include "QueueTracer.h" +#include #include "Basics/ScopeGuard.h" #include "Basics/system-functions.h" #include "Graph/Providers/ClusterProvider.h" -#include "Graph/Providers/SingleServerProvider.h" -#include "Graph/Queues/LifoQueue.h" #include "Graph/Queues/FifoQueue.h" +#include "Graph/Queues/LifoQueue.h" +#include "Graph/Steps/SingleServerProviderStep.h" + +#ifdef USE_ENTERPRISE +#include "Enterprise/Graph/Steps/SmartGraphStep.h" +#endif + #include "Logger/LogMacros.h" using namespace arangodb; @@ -103,8 +109,15 @@ auto QueueTracer::pop() -> typename QueueImpl::Step { } /* SingleServerProvider Section */ -template class ::arangodb::graph::QueueTracer>; -template class ::arangodb::graph::QueueTracer>; +using SingleServerProviderStep = ::arangodb::graph::SingleServerProviderStep; + +template class ::arangodb::graph::QueueTracer>; +template class ::arangodb::graph::QueueTracer>; + +#ifdef USE_ENTERPRISE +template class ::arangodb::graph::QueueTracer>; +template class ::arangodb::graph::QueueTracer>; +#endif /* ClusterServerProvider Section */ template class ::arangodb::graph::QueueTracer>; diff --git a/arangod/Graph/Steps/SingleServerProviderStep.cpp b/arangod/Graph/Steps/SingleServerProviderStep.cpp new file mode 100644 index 000000000000..6566c3e7c36d --- /dev/null +++ b/arangod/Graph/Steps/SingleServerProviderStep.cpp @@ -0,0 +1,76 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2021 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +/// @author Heiko Kernbach +//////////////////////////////////////////////////////////////////////////////// + +#include "./SingleServerProviderStep.h" + +#include "Graph/Providers/SingleServerProvider.h" + +using namespace arangodb; +using namespace arangodb::graph; + +namespace arangodb { +namespace graph { +auto operator<<(std::ostream& out, SingleServerProviderStep const& step) -> std::ostream& { + out << step._vertex.getID(); + return out; +} +} // namespace graph +} // namespace arangodb + +SingleServerProviderStep::SingleServerProviderStep(VertexType v) + : _vertex(v), _edge() {} + +SingleServerProviderStep::SingleServerProviderStep(VertexType v, size_t depth) + : BaseStep(std::numeric_limits::max(), depth), _vertex(v), _edge() {} + +SingleServerProviderStep::SingleServerProviderStep(VertexType v, + EdgeDocumentToken edge, size_t prev) + : BaseStep(prev), _vertex(v), _edge(std::move(edge)) {} + +SingleServerProviderStep::SingleServerProviderStep(VertexType v, EdgeDocumentToken edge, + size_t prev, size_t depth) + : BaseStep(prev, depth), _vertex(v), _edge(std::move(edge)) {} + +SingleServerProviderStep::~SingleServerProviderStep() = default; + +VertexType const& SingleServerProviderStep::Vertex::getID() const { + return _vertex; +} + +EdgeDocumentToken const& SingleServerProviderStep::Edge::getID() const { + return _token; +} + +bool SingleServerProviderStep::Edge::isValid() const { + return getID().isValid(); +}; + +void SingleServerProviderStep::Edge::addToBuilder(SingleServerProvider& provider, + arangodb::velocypack::Builder& builder) const { + provider.insertEdgeIntoResult(getID(), builder); +} + +#ifndef USE_ENTERPRISE +bool Step::isResponsible(transaction::Methods* trx) const { return true; }; +#endif diff --git a/arangod/Graph/Steps/SingleServerProviderStep.h b/arangod/Graph/Steps/SingleServerProviderStep.h new file mode 100644 index 000000000000..b341feaa231d --- /dev/null +++ b/arangod/Graph/Steps/SingleServerProviderStep.h @@ -0,0 +1,110 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2021 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +/// @author Heiko Kernbach +//////////////////////////////////////////////////////////////////////////////// + +#include +#include "Graph/EdgeDocumentToken.h" +#include "Graph/Providers/BaseStep.h" +#include "Graph/Providers/TypeAliases.h" + +#pragma once + +namespace arangodb { +namespace graph { + +template +class SingleServerProvider; + +class SingleServerProviderStep : public arangodb::graph::BaseStep { + public: + class Vertex { + public: + explicit Vertex(VertexType v) : _vertex(v) {} + + VertexType const& getID() const; + + bool operator<(Vertex const& other) const noexcept { + return _vertex < other._vertex; + } + + bool operator>(Vertex const& other) const noexcept { + return _vertex > other._vertex; + } + + private: + VertexType _vertex; + }; + + class Edge { + public: + explicit Edge(EdgeDocumentToken tkn) noexcept : _token(std::move(tkn)) {} + Edge() noexcept : _token() {} + + void addToBuilder(SingleServerProvider& provider, + arangodb::velocypack::Builder& builder) const; + EdgeDocumentToken const& getID() const; + bool isValid() const; + + private: + EdgeDocumentToken _token; + }; + + SingleServerProviderStep(VertexType v); + SingleServerProviderStep(VertexType v, EdgeDocumentToken edge, size_t prev); + SingleServerProviderStep(VertexType v, EdgeDocumentToken edge, size_t prev, size_t depth); + SingleServerProviderStep(VertexType v, size_t depth); + ~SingleServerProviderStep(); + + bool operator<(SingleServerProviderStep const& other) const noexcept { + return _vertex < other._vertex; + } + + Vertex const& getVertex() const { return _vertex; } + Edge const& getEdge() const { return _edge; } + + std::string toString() const { + return ": " + _vertex.getID().toString(); + } + bool isProcessable() const { return !isLooseEnd(); } + bool isLooseEnd() const { return false; } + + ::arangodb::graph::VertexType getVertexIdentifier() const { return _vertex.getID(); } + + std::string getCollectionName() const { + auto collectionNameResult = extractCollectionName(_vertex.getID()); + if (collectionNameResult.fail()) { + THROW_ARANGO_EXCEPTION(collectionNameResult.result()); + } + return collectionNameResult.get().first; + }; + + bool isResponsible(transaction::Methods* trx) const; + + friend auto operator<<(std::ostream& out, SingleServerProviderStep const& step) -> std::ostream&; + + private: + Vertex _vertex; + Edge _edge; +}; +} // namespace graph +} // namespace arangodb diff --git a/tests/Graph/GenericGraphProviderTest.cpp b/tests/Graph/GenericGraphProviderTest.cpp index 8a00d3e64e43..1f93bf29fc72 100644 --- a/tests/Graph/GenericGraphProviderTest.cpp +++ b/tests/Graph/GenericGraphProviderTest.cpp @@ -32,6 +32,7 @@ #include "Graph/Providers/ClusterProvider.h" #include "Graph/Providers/SingleServerProvider.h" +#include "Graph/Steps/SingleServerProviderStep.h" #include "Graph/TraverserOptions.h" #include @@ -50,7 +51,7 @@ static_assert(GTEST_HAS_TYPED_TEST, "We need typed tests for the following:"); // Add more providers here using TypesToTest = - ::testing::Types; + ::testing::Types, ClusterProvider>; template class GraphProviderTest : public ::testing::Test { @@ -93,7 +94,7 @@ class GraphProviderTest : public ::testing::Test { MockGraphProviderOptions{graph, MockGraphProvider::LooseEndBehaviour::NEVER}, resourceMonitor); } - if constexpr (std::is_same_v) { + if constexpr (std::is_same_v>) { s = std::make_unique(); singleServer = std::make_unique(s->server, "testVocbase"); @@ -119,7 +120,7 @@ class GraphProviderTest : public ::testing::Test { std::make_pair(std::move(usedIndexes), std::unordered_map>{}), *_expressionContext.get(), _emptyShardMap); - return SingleServerProvider(*query.get(), std::move(opts), resourceMonitor); + return SingleServerProvider(*query.get(), std::move(opts), resourceMonitor); } if constexpr (std::is_same_v) { // Prepare the DBServerResponses @@ -243,7 +244,7 @@ TYPED_TEST(GraphProviderTest, no_results_if_graph_is_empty) { TraversalStats stats = testee.stealStats(); EXPECT_EQ(stats.getFiltered(), 0); - if constexpr (std::is_same_v || + if constexpr (std::is_same_v> || std::is_same_v) { EXPECT_EQ(stats.getHttpRequests(), 0); } else if (std::is_same_v) { @@ -287,7 +288,7 @@ TYPED_TEST(GraphProviderTest, should_enumerate_a_single_edge) { { TraversalStats stats = testee.stealStats(); EXPECT_EQ(stats.getFiltered(), 0); - if constexpr (std::is_same_v || + if constexpr (std::is_same_v> || std::is_same_v) { EXPECT_EQ(stats.getHttpRequests(), 0); } else if (std::is_same_v) { @@ -349,7 +350,7 @@ TYPED_TEST(GraphProviderTest, should_enumerate_all_edges) { { TraversalStats stats = testee.stealStats(); EXPECT_EQ(stats.getFiltered(), 0); - if constexpr (std::is_same_v || + if constexpr (std::is_same_v> || std::is_same_v) { EXPECT_EQ(stats.getHttpRequests(), 0); } else if (std::is_same_v) { @@ -370,7 +371,7 @@ TYPED_TEST(GraphProviderTest, destroy_engines) { testee.destroyEngines(); TraversalStats statsAfterSteal = testee.stealStats(); - if constexpr (std::is_same_v || + if constexpr (std::is_same_v> || std::is_same_v) { EXPECT_EQ(statsAfterSteal.getHttpRequests(), 0); } else if (std::is_same_v) { diff --git a/tests/Graph/SingleServerProviderTest.cpp b/tests/Graph/SingleServerProviderTest.cpp index 95298433e0f2..61787cc79a60 100644 --- a/tests/Graph/SingleServerProviderTest.cpp +++ b/tests/Graph/SingleServerProviderTest.cpp @@ -29,6 +29,7 @@ #include "Basics/GlobalResourceMonitor.h" #include "Basics/ResourceUsage.h" #include "Graph/Providers/SingleServerProvider.h" +#include "Graph/Steps/SingleServerProviderStep.h" #include #include @@ -52,7 +53,7 @@ namespace arangodb { namespace tests { namespace single_server_provider_test { -using Step = SingleServerProvider::Step; +using Step = SingleServerProviderStep; class SingleServerProviderTest : public ::testing::Test { protected: @@ -77,7 +78,7 @@ class SingleServerProviderTest : public ::testing::Test { SingleServerProviderTest() {} ~SingleServerProviderTest() {} - auto makeProvider(MockGraph const& graph) -> arangodb::graph::SingleServerProvider { + auto makeProvider(MockGraph const& graph) -> arangodb::graph::SingleServerProvider { // Setup code for each provider type s = std::make_unique(); singleServer = From 1f6e7243c2335bfd68caaf0313769d9c6c1623eb Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Fri, 16 Jul 2021 15:38:21 +0200 Subject: [PATCH 14/72] Readded lost merge update --- arangod/Aql/EngineInfoContainerDBServerServerBased.cpp | 10 ++++++++-- .../Graph/Cursors/RefactoredSingleServerEdgeCursor.cpp | 4 ++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/arangod/Aql/EngineInfoContainerDBServerServerBased.cpp b/arangod/Aql/EngineInfoContainerDBServerServerBased.cpp index d58690823a7b..dc4c57b26059 100644 --- a/arangod/Aql/EngineInfoContainerDBServerServerBased.cpp +++ b/arangod/Aql/EngineInfoContainerDBServerServerBased.cpp @@ -508,6 +508,13 @@ Result EngineInfoContainerDBServerServerBased::buildEngines( continue; } + if (!trx.state()->knownServers().contains(server)) { + // we are about to add this server to the transaction. + // remember it, so we can roll the addition back for + // the second setup request if we need to + serversAdded.emplace(server); + } + networkCalls.emplace_back( buildSetupRequest(trx, server, infoSlice, didCreateEngine, snippetIds, serverToQueryId, serverToQueryIdLock, pool, options)); @@ -548,6 +555,7 @@ Result EngineInfoContainerDBServerServerBased::buildEngines( return fastPathResult.get(); } + // we got a lock timeout response for the fast path locking... { // in case of fast path failure, we need to cleanup engines auto requests = cleanupEngines(fastPathResult.get().errorNumber(), @@ -604,8 +612,6 @@ Result EngineInfoContainerDBServerServerBased::buildEngines( << "Potential deadlock detected, using slow path for locking. This " "is expected if exclusive locks are used."; - trx.state()->coordinatorRerollTransactionId(); - // Make sure we always use the same ordering on servers std::sort(engineInformation.begin(), engineInformation.end(), [](auto const& lhs, auto const& rhs) { diff --git a/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.cpp b/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.cpp index be69481f9e7a..d31a978a9253 100644 --- a/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.cpp +++ b/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.cpp @@ -286,8 +286,8 @@ void RefactoredSingleServerEdgeCursor::readAll(SingleServerProvider& } template -bool RefactoredSingleServerEdgeCursor::evaluateExpression(arangodb::aql::Expression* expression, - VPackSlice value) { +bool RefactoredSingleServerEdgeCursor::evaluateEdgeExpression(arangodb::aql::Expression* expression, + VPackSlice value) { if (expression == nullptr) { return true; } From 839cdc16e07eeea4c3c43f3f54aeb5a9d4a5cc06 Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Mon, 19 Jul 2021 09:58:50 +0200 Subject: [PATCH 15/72] Reduced total amount of Nodes per CallStack to 200. For some reason 250 as before do not fit into MacOs Stacksize in this branch. Needs to be investigated --- arangod/Aql/QueryOptions.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arangod/Aql/QueryOptions.cpp b/arangod/Aql/QueryOptions.cpp index ff74b4a824bb..77ce88fd8e8f 100644 --- a/arangod/Aql/QueryOptions.cpp +++ b/arangod/Aql/QueryOptions.cpp @@ -38,7 +38,7 @@ using namespace arangodb::aql; size_t QueryOptions::defaultMemoryLimit = 0; size_t QueryOptions::defaultMaxNumberOfPlans = 128; -size_t QueryOptions::defaultMaxNodesPerCallstack = 250; +size_t QueryOptions::defaultMaxNodesPerCallstack = 200; double QueryOptions::defaultMaxRuntime = 0.0; double QueryOptions::defaultTtl; bool QueryOptions::defaultFailOnWarning = false; From 5fe4b148a5f473f5144063d5ea136481bd255421 Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Mon, 19 Jul 2021 09:59:35 +0200 Subject: [PATCH 16/72] Make Globale variable overwrite less intrusive for tests --- tests/Mocks/Servers.cpp | 7 +++---- tests/Mocks/Servers.h | 1 + 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/Mocks/Servers.cpp b/tests/Mocks/Servers.cpp index 1754183ff374..c719e64d507c 100644 --- a/tests/Mocks/Servers.cpp +++ b/tests/Mocks/Servers.cpp @@ -188,7 +188,8 @@ MockServer::MockServer(arangodb::ServerState::RoleEnum myRole, bool injectCluste _started(false) { _oldRole = arangodb::ServerState::instance()->getRole(); arangodb::ServerState::instance()->setRole(myRole); - if (arangodb::ServerState::instance()->isCoordinator()) { + _originalMockingState = arangodb::ClusterEngine::Mocking; + if (injectClusterIndexes && arangodb::ServerState::instance()->isCoordinator()) { arangodb::ClusterEngine::Mocking = true; } init(); @@ -198,9 +199,7 @@ MockServer::~MockServer() { stopFeatures(); _server.setStateUnsafe(_oldApplicationServerState); - if (arangodb::ServerState::instance()->isCoordinator()) { - arangodb::ClusterEngine::Mocking = false; - } + arangodb::ClusterEngine::Mocking = _originalMockingState; arangodb::ServerState::instance()->setRole(_oldRole); arangodb::ServerState::instance()->setRebootId(_oldRebootId); } diff --git a/tests/Mocks/Servers.h b/tests/Mocks/Servers.h index 483014ef754d..bc6b7471e1ae 100644 --- a/tests/Mocks/Servers.h +++ b/tests/Mocks/Servers.h @@ -128,6 +128,7 @@ class MockServer { private: bool _started; arangodb::ServerState::RoleEnum _oldRole; + bool _originalMockingState; }; /// @brief a server with almost no features added (Metrics are available From d2b6e2159bd4e4f82b977b9e02b5733f239d5119 Mon Sep 17 00:00:00 2001 From: Heiko Date: Mon, 19 Jul 2021 17:00:57 +0200 Subject: [PATCH 17/72] Feature/hybrid smart graph enable bfs (#14494) * smartSearchNew does both, bfs and dfs * testing old vs new * preparations to handle bfs enumerator results * construct dfs result into builder * push - changing location of some methods - wip - does not compile * moved code, additional path interface interface methods * prep for cursorID in edgecursor * Added Parser for smartSearchInput, does not compile yet * added missing bfs templating * send vpackslice as it has been * write cursor id into step if necessary * compare slices old vs new * Use cursor ID in IndexAccess. And forward it to the RefactoredSingleServerEdgeCursors. * Call rearm with depth * Activate some 3.8 backwards compatibility methods * Do copy of steps inside the PathResults, as the pointers could be invalidated. * rm method toSchreier + interface, disable some logging * Do not implement move ctrs yourself... Also removed log output * remove auto-added duplicate and false import * make compiler happy * rm todo * include provider in pathstore * make comunity compiler hapy Co-authored-by: Michael Hackstein --- arangod/Aql/KShortestPathsNode.cpp | 8 +- .../RefactoredSingleServerEdgeCursor.cpp | 93 ++++++++++--------- .../RefactoredSingleServerEdgeCursor.h | 16 +++- .../Graph/Enumerators/OneSidedEnumerator.cpp | 40 +++++--- .../Graph/Enumerators/OneSidedEnumerator.h | 6 +- .../Enumerators/OneSidedEnumeratorInterface.h | 15 ++- arangod/Graph/PathManagement/PathStore.cpp | 1 + .../Graph/PathManagement/PathValidator.cpp | 6 ++ .../PathManagement/PathValidatorOptions.h | 13 +++ .../SingleProviderPathResult.cpp | 63 +------------ .../PathManagement/SingleProviderPathResult.h | 17 ++-- .../Graph/Providers/BaseProviderOptions.cpp | 10 +- arangod/Graph/Providers/BaseProviderOptions.h | 4 +- .../Graph/Providers/SingleServerProvider.cpp | 46 +++++---- arangod/Graph/Queues/QueueTracer.cpp | 1 - .../Graph/Steps/SingleServerProviderStep.cpp | 2 +- .../InternalRestTraverserHandler.cpp | 5 +- tests/Graph/GenericGraphProviderTest.cpp | 5 +- tests/Graph/SingleServerProviderTest.cpp | 5 +- 19 files changed, 183 insertions(+), 173 deletions(-) diff --git a/arangod/Aql/KShortestPathsNode.cpp b/arangod/Aql/KShortestPathsNode.cpp index e79ce98eefbc..fe5486d8809c 100644 --- a/arangod/Aql/KShortestPathsNode.cpp +++ b/arangod/Aql/KShortestPathsNode.cpp @@ -538,7 +538,7 @@ std::vector KShortestPathsNode::buildUsedIndexes indexAccessors.emplace_back(indexToUse, _toCondition->clone(options()->query().ast()), - 0, nullptr); + 0, nullptr, i); break; } case TRI_EDGE_OUT: { @@ -554,7 +554,7 @@ std::vector KShortestPathsNode::buildUsedIndexes indexAccessors.emplace_back(indexToUse, _fromCondition->clone(options()->query().ast()), - 0, nullptr); + 0, nullptr, i); break; } case TRI_EDGE_ANY: @@ -587,7 +587,7 @@ std::vector KShortestPathsNode::buildReverseUsed indexAccessors.emplace_back(indexToUse, _fromCondition->clone(options()->query().ast()), - 0, nullptr); + 0, nullptr, i); break; } case TRI_EDGE_OUT: { @@ -603,7 +603,7 @@ std::vector KShortestPathsNode::buildReverseUsed indexAccessors.emplace_back(indexToUse, _toCondition->clone(options()->query().ast()), - 0, nullptr); + 0, nullptr, i); break; } case TRI_EDGE_ANY: diff --git a/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.cpp b/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.cpp index d31a978a9253..d46fb59912a2 100644 --- a/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.cpp +++ b/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.cpp @@ -62,19 +62,16 @@ static bool CheckInaccessible(transaction::Methods* trx, VPackSlice const& edge) template RefactoredSingleServerEdgeCursor::LookupInfo::LookupInfo( transaction::Methods::IndexHandle idx, aql::AstNode* condition, - std::optional memberToUpdate, aql::Expression* expression) + std::optional memberToUpdate, aql::Expression* expression, size_t cursorID) : _idxHandle(std::move(idx)), _expression(expression), _indexCondition(condition), + _cursorID(cursorID), _cursor(nullptr), _conditionMemberToUpdate(memberToUpdate) {} template -RefactoredSingleServerEdgeCursor::LookupInfo::LookupInfo(LookupInfo&& other) noexcept - : _idxHandle(std::move(other._idxHandle)), - _expression(std::move(other._expression)), - _indexCondition(other._indexCondition), - _cursor(std::move(other._cursor)){}; +RefactoredSingleServerEdgeCursor::LookupInfo::LookupInfo(LookupInfo&& other) = default; template RefactoredSingleServerEdgeCursor::LookupInfo::~LookupInfo() = default; @@ -84,6 +81,11 @@ aql::Expression* RefactoredSingleServerEdgeCursor::LookupInfo::getExpressi return _expression; } +template +size_t RefactoredSingleServerEdgeCursor::LookupInfo::getCursorID() const { + return _cursorID; +} + template void RefactoredSingleServerEdgeCursor::LookupInfo::rearmVertex( VertexType vertex, transaction::Methods* trx, arangodb::aql::Variable const* tmpVar) { @@ -152,22 +154,28 @@ RefactoredSingleServerEdgeCursor::RefactoredSingleServerEdgeCursor( std::vector const& globalIndexConditions, std::unordered_map> const& depthBasedIndexConditions, arangodb::aql::FixedVarExpressionContext& expressionContext) - : _tmpVar(tmpVar), _currentCursor(0), _trx(trx), _expressionCtx(expressionContext) { + : _tmpVar(tmpVar), _trx(trx), _expressionCtx(expressionContext) { // We need at least one indexCondition, otherwise nothing to serve TRI_ASSERT(!globalIndexConditions.empty()); _lookupInfo.reserve(globalIndexConditions.size()); _depthLookupInfo.reserve(depthBasedIndexConditions.size()); + for (auto const& idxCond : globalIndexConditions) { _lookupInfo.emplace_back(idxCond.indexHandle(), idxCond.getCondition(), - idxCond.getMemberToUpdate(), idxCond.getExpression()); + idxCond.getMemberToUpdate(), + idxCond.getExpression(), idxCond.cursorId()); } for (auto const& obj : depthBasedIndexConditions) { + // Need to reset cursor ID. + // The cursorID relates to the used collection + // not the condition auto& [depth, idxCondArray] = obj; std::vector tmpLookupVec; for (auto const& idxCond : idxCondArray) { tmpLookupVec.emplace_back(idxCond.indexHandle(), idxCond.getCondition(), - idxCond.getMemberToUpdate(), idxCond.getExpression()); + idxCond.getMemberToUpdate(), + idxCond.getExpression(), idxCond.cursorId()); } _depthLookupInfo.try_emplace(depth, std::move(tmpLookupVec)); } @@ -177,9 +185,8 @@ template RefactoredSingleServerEdgeCursor::~RefactoredSingleServerEdgeCursor() {} template -void RefactoredSingleServerEdgeCursor::rearm(VertexType vertex, uint64_t /*depth*/) { - _currentCursor = 0; - for (auto& info : _lookupInfo) { +void RefactoredSingleServerEdgeCursor::rearm(VertexType vertex, uint64_t depth) { + for (auto& info : getLookupInfos(depth)) { info.rearmVertex(vertex, _trx, _tmpVar); } } @@ -188,7 +195,7 @@ template void RefactoredSingleServerEdgeCursor::readAll(SingleServerProvider& provider, aql::TraversalStats& stats, size_t depth, Callback const& callback) { - TRI_ASSERT(!_lookupInfo.empty()); + TRI_ASSERT(!getLookupInfos(depth).empty()); VPackBuilder tmpBuilder; auto evaluateEdgeExpressionHelper = [&](aql::Expression* expression, @@ -201,36 +208,16 @@ void RefactoredSingleServerEdgeCursor::readAll(SingleServerProvider& return evaluateEdgeExpression(expression, edge); }; - auto evaluateLookupInfos = [&](EdgeDocumentToken const& edgeToken, - VPackSlice const& edge) -> bool { - bool foundDepthInfo = - (_depthLookupInfo.find(depth) == _depthLookupInfo.end()) ? false : true; - if (foundDepthInfo && _depthLookupInfo.at(depth).size() > 0 && - _depthLookupInfo.at(depth)[_currentCursor].getExpression() != nullptr) { - if (!evaluateEdgeExpressionHelper( - _depthLookupInfo.at(depth)[_currentCursor].getExpression(), edgeToken, edge)) { - stats.incrFiltered(); - return false; - } - } else { - // eval global expression if available AND only if no depth specific expression needs to be handled before - if (_lookupInfo[_currentCursor].getExpression() != nullptr) { - if (!evaluateEdgeExpressionHelper(_lookupInfo[_currentCursor].getExpression(), - edgeToken, edge)) { - stats.incrFiltered(); - return false; - } - } - } - - return true; - }; + for (auto& lookupInfo : getLookupInfos(depth)) { + auto cursorID = lookupInfo.getCursorID(); + // we can only have a cursorID that is within the amount of collections in use. + TRI_ASSERT(cursorID < _lookupInfo.size()); - for (_currentCursor = 0; _currentCursor < _lookupInfo.size(); ++_currentCursor) { - auto& cursor = _lookupInfo[_currentCursor].cursor(); + auto& cursor = lookupInfo.cursor(); LogicalCollection* collection = cursor.collection(); auto cid = collection->id(); bool hasExtra = cursor.hasExtra(); + auto* expression = lookupInfo.getExpression(); if (hasExtra) { cursor.allExtra([&](LocalDocumentId const& token, VPackSlice edge) { @@ -242,13 +229,14 @@ void RefactoredSingleServerEdgeCursor::readAll(SingleServerProvider& #endif EdgeDocumentToken edgeToken(cid, token); - // eval depth-based expression first if available - bool needToRead = evaluateLookupInfos(edgeToken, edge); - if (!needToRead) { + // evaluate expression if available + if (expression != nullptr && + !evaluateEdgeExpressionHelper(expression, edgeToken, edge)) { + stats.incrFiltered(); return false; } - callback(std::move(edgeToken), edge, _currentCursor); + callback(std::move(edgeToken), edge, cursorID); return true; }); } else { @@ -271,12 +259,15 @@ void RefactoredSingleServerEdgeCursor::readAll(SingleServerProvider& #endif // eval depth-based expression first if available EdgeDocumentToken edgeToken(cid, token); - bool needToRead = evaluateLookupInfos(edgeToken, edgeDoc); - if (!needToRead) { + + // evaluate expression if available + if (expression != nullptr && + !evaluateEdgeExpressionHelper(expression, edgeToken, edgeDoc)) { + stats.incrFiltered(); return false; } - callback(std::move(edgeToken), edgeDoc, _currentCursor); + callback(std::move(edgeToken), edgeDoc, cursorID); return true; }) .ok(); @@ -306,6 +297,16 @@ bool RefactoredSingleServerEdgeCursor::evaluateEdgeExpression(arangodb::aq return res.toBoolean(); } +template +auto RefactoredSingleServerEdgeCursor::getLookupInfos(uint64_t depth) + -> std::vector& { + auto const& depthInfo = _depthLookupInfo.find(depth); + if (depthInfo == _depthLookupInfo.end()) { + return _lookupInfo; + } + return depthInfo->second; +} + template class arangodb::graph::RefactoredSingleServerEdgeCursor; #ifdef USE_ENTERPRISE diff --git a/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.h b/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.h index 9319c4a0ff9b..ae0995f00f0b 100644 --- a/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.h +++ b/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.h @@ -61,11 +61,12 @@ class RefactoredSingleServerEdgeCursor { public: struct LookupInfo { LookupInfo(transaction::Methods::IndexHandle idx, aql::AstNode* condition, - std::optional memberToUpdate, aql::Expression* expression); + std::optional memberToUpdate, + aql::Expression* expression, size_t cursorID); ~LookupInfo(); LookupInfo(LookupInfo const&) = delete; - LookupInfo(LookupInfo&&) noexcept; + LookupInfo(LookupInfo&&); LookupInfo& operator=(LookupInfo const&) = delete; void rearmVertex(VertexType vertex, transaction::Methods* trx, @@ -74,11 +75,14 @@ class RefactoredSingleServerEdgeCursor { IndexIterator& cursor(); aql::Expression* getExpression(); + size_t getCursorID() const; + private: // NOTE: The expression can be nullptr! transaction::Methods::IndexHandle _idxHandle; aql::Expression* _expression; aql::AstNode* _indexCondition; + size_t _cursorID; std::unique_ptr _cursor; @@ -101,7 +105,6 @@ class RefactoredSingleServerEdgeCursor { private: aql::Variable const* _tmpVar; - size_t _currentCursor; std::vector _lookupInfo; std::unordered_map> _depthLookupInfo; @@ -109,12 +112,15 @@ class RefactoredSingleServerEdgeCursor { arangodb::aql::FixedVarExpressionContext& _expressionCtx; public: - void readAll(SingleServerProvider& provider, aql::TraversalStats& stats, - size_t depth, Callback const& callback); + void readAll(SingleServerProvider& provider, + aql::TraversalStats& stats, size_t depth, Callback const& callback); void rearm(VertexType vertex, uint64_t depth); bool evaluateEdgeExpression(arangodb::aql::Expression* expression, VPackSlice value); + + private: + auto getLookupInfos(uint64_t depth) -> std::vector&; }; } // namespace graph } // namespace arangodb diff --git a/arangod/Graph/Enumerators/OneSidedEnumerator.cpp b/arangod/Graph/Enumerators/OneSidedEnumerator.cpp index 70d640cc5511..9491e91a4f6d 100644 --- a/arangod/Graph/Enumerators/OneSidedEnumerator.cpp +++ b/arangod/Graph/Enumerators/OneSidedEnumerator.cpp @@ -67,8 +67,10 @@ auto OneSidedEnumerator::destroyEngines() -> void { } template -void OneSidedEnumerator::clear() { - _interior.reset(); +void OneSidedEnumerator::clear(bool keepPathStore) { + if (!keepPathStore) { + _interior.reset(); + } _queue.clear(); _results.clear(); } @@ -98,7 +100,7 @@ auto OneSidedEnumerator::computeNeighbourhoodOfNextVertex() -> vo // This server cannot decide on this specific vertex. // Include it in results, to report back that we // found this undecided path - _results.push_back(&step); + _results.emplace_back(step); return; } ValidationResult res = _validator.validatePath(step); @@ -111,7 +113,7 @@ auto OneSidedEnumerator::computeNeighbourhoodOfNextVertex() -> vo << "<= " << _options.getMaxDepth(); if (step.getDepth() >= _options.getMinDepth() && !res.isFiltered()) { // Include it in results. - _results.push_back(&step); + _results.emplace_back(step); } else { _stats.incrFiltered(); } @@ -142,8 +144,8 @@ bool OneSidedEnumerator::isDone() const { * @param source The source vertex to start the paths */ template -void OneSidedEnumerator::reset(VertexRef source, size_t depth) { - clear(); +void OneSidedEnumerator::reset(VertexRef source, size_t depth, bool keepPathStore) { + clear(keepPathStore); auto firstStep = _provider.startVertex(source, depth); _queue.append(std::move(firstStep)); } @@ -168,7 +170,7 @@ auto OneSidedEnumerator::getNextPath() searchMoreResults(); while (!_results.empty()) { - auto* step = _results.back(); + auto step = std::move(_results.back()); _results.pop_back(); return std::make_unique(step, _provider, _interior); } @@ -217,9 +219,9 @@ auto OneSidedEnumerator::fetchResults() -> void { if (!_resultsFetched && !_results.empty()) { std::vector looseEnds{}; - for (auto* vertex : _results) { - if (!vertex->isProcessable()) { - looseEnds.emplace_back(vertex); + for (auto& vertex : _results) { + if (!vertex.isProcessable()) { + looseEnds.emplace_back(&vertex); } } @@ -291,11 +293,27 @@ template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::GLOBAL, false>>; -// Tracing +// DFS Tracing template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::PATH, true>>; template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::NONE, true>>; template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::GLOBAL, true>>; + +// Breath First Search +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::PATH, false>>; +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::NONE, false>>; +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::GLOBAL, false>>; + +// BFS Tracing +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::PATH, true>>; +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::NONE, true>>; +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::GLOBAL, true>>; #endif diff --git a/arangod/Graph/Enumerators/OneSidedEnumerator.h b/arangod/Graph/Enumerators/OneSidedEnumerator.h index 80e2ee7a898b..336d2733fe45 100644 --- a/arangod/Graph/Enumerators/OneSidedEnumerator.h +++ b/arangod/Graph/Enumerators/OneSidedEnumerator.h @@ -60,7 +60,7 @@ class OneSidedEnumerator : public TraversalEnumerator { private: using VertexRef = arangodb::velocypack::HashedStringRef; - using ResultList = std::vector; + using ResultList = std::vector; using GraphOptions = arangodb::graph::OneSidedEnumeratorOptions; public: @@ -72,7 +72,7 @@ class OneSidedEnumerator : public TraversalEnumerator { ~OneSidedEnumerator(); - void clear() override; + void clear(bool keepPathStore) override; /** * @brief Quick test if the finder can prove there is no more data available. @@ -92,7 +92,7 @@ class OneSidedEnumerator : public TraversalEnumerator { * @param source The source vertex to start the paths * @param depth The depth we're starting the search at */ - void reset(VertexRef source, size_t depth = 0) override; + void reset(VertexRef source, size_t depth = 0, bool keepPathStore = false) override; /** * @brief Get the next path, if available written into the result build. diff --git a/arangod/Graph/Enumerators/OneSidedEnumeratorInterface.h b/arangod/Graph/Enumerators/OneSidedEnumeratorInterface.h index 7cd217066d62..7e631986ea0b 100644 --- a/arangod/Graph/Enumerators/OneSidedEnumeratorInterface.h +++ b/arangod/Graph/Enumerators/OneSidedEnumeratorInterface.h @@ -24,6 +24,8 @@ #pragma once +#include "Graph/TraverserOptions.h" + #include #include #include @@ -42,27 +44,30 @@ class TraversalStats; namespace graph { // TODO Temporary, just Make everything compile +// IMPORTANT - DO NOT TEMPLATE THIS CLASS class PathResultInterface { public: PathResultInterface() {} virtual ~PathResultInterface() {} virtual auto toVelocyPack(arangodb::velocypack::Builder& builder) -> void = 0; - virtual auto toSchreierEntry(arangodb::velocypack::Builder& builder, - size_t& currentLength) - -> void = 0; }; +// IMPORTANT - DO NOT TEMPLATE THIS CLASS class TraversalEnumerator { public: using VertexRef = arangodb::velocypack::HashedStringRef; TraversalEnumerator(){}; virtual ~TraversalEnumerator() {} - virtual void clear() = 0; + // NOTE: keepPathStore is only required for 3.8 compatibility and + // can be removed in the version after 3.9 + virtual void clear(bool keepPathStore) = 0; [[nodiscard]] virtual bool isDone() const = 0; - virtual void reset(VertexRef source, size_t depth = 0) = 0; + // NOTE: keepPathStore is only required for 3.8 compatibility and + // can be removed in the version after 3.9 + virtual void reset(VertexRef source, size_t depth = 0, bool keepPathStore = false) = 0; virtual auto getNextPath() -> std::unique_ptr = 0; virtual bool skipPath() = 0; virtual auto destroyEngines() -> void = 0; diff --git a/arangod/Graph/PathManagement/PathStore.cpp b/arangod/Graph/PathManagement/PathStore.cpp index 0ae2ece819f0..c69d769a83ed 100644 --- a/arangod/Graph/PathManagement/PathStore.cpp +++ b/arangod/Graph/PathManagement/PathStore.cpp @@ -26,6 +26,7 @@ #include "Graph/Providers/ClusterProvider.h" #include "Graph/Providers/ProviderTracer.h" +#include "Graph/Providers/SingleServerProvider.h" #include "Graph/Steps/SingleServerProviderStep.h" #include "Graph/Types/ValidationResult.h" diff --git a/arangod/Graph/PathManagement/PathValidator.cpp b/arangod/Graph/PathManagement/PathValidator.cpp index 356e2b242397..eab704d47b1b 100644 --- a/arangod/Graph/PathManagement/PathValidator.cpp +++ b/arangod/Graph/PathManagement/PathValidator.cpp @@ -163,6 +163,9 @@ auto PathValidator::evaluateVertexCon // evaluate if vertex collection is allowed bool isAllowed = evaluateVertexRestriction(step); if (!isAllowed) { + if (_options.hasCompatibility38IncludeFirstVertex() && step.isFirst()) { + return ValidationResult{ValidationResult::Type::PRUNE}; + } return ValidationResult{ValidationResult::Type::FILTER}; } @@ -174,6 +177,9 @@ auto PathValidator::evaluateVertexCon // evaluate expression bool satifiesCondition = evaluateVertexExpression(expr, _tmpObjectBuilder.slice()); if (!satifiesCondition) { + if (_options.hasCompatibility38IncludeFirstVertex() && step.isFirst()) { + return ValidationResult{ValidationResult::Type::PRUNE}; + } return ValidationResult{ValidationResult::Type::FILTER}; } } diff --git a/arangod/Graph/PathManagement/PathValidatorOptions.h b/arangod/Graph/PathManagement/PathValidatorOptions.h index 14c5096146f4..1525a94793e3 100644 --- a/arangod/Graph/PathManagement/PathValidatorOptions.h +++ b/arangod/Graph/PathManagement/PathValidatorOptions.h @@ -76,6 +76,17 @@ class PathValidatorOptions { aql::FixedVarExpressionContext& getExpressionContext(); + // @brief Only required for rolling upgrades 3.8 => 3.9 + // If a graph is asked for the first vertex and that is filtered + // it can be removed for 3.9 => nextVersion. + void compatibility38IncludeFirstVertex() { + _compatibility38IncludeFirstVertex = true; + } + + bool hasCompatibility38IncludeFirstVertex() const { + return _compatibility38IncludeFirstVertex; + } + private: std::shared_ptr _allVerticesExpression; std::unordered_map> _vertexExpressionOnDepth; @@ -83,6 +94,8 @@ class PathValidatorOptions { arangodb::aql::FixedVarExpressionContext& _expressionCtx; std::vector _allowedVertexCollections; + + bool _compatibility38IncludeFirstVertex = false; }; } // namespace graph } // namespace arangodb diff --git a/arangod/Graph/PathManagement/SingleProviderPathResult.cpp b/arangod/Graph/PathManagement/SingleProviderPathResult.cpp index 811a347aaeef..e3963247318f 100644 --- a/arangod/Graph/PathManagement/SingleProviderPathResult.cpp +++ b/arangod/Graph/PathManagement/SingleProviderPathResult.cpp @@ -48,10 +48,8 @@ class SmartGraphStep; template SingleProviderPathResult::SingleProviderPathResult( - Step* step, ProviderType& provider, PathStoreType& store) - : _step(step), _provider(provider), _store(store) { - TRI_ASSERT(_step != nullptr); -} + Step step, ProviderType& provider, PathStoreType& store) + : _step(std::move(step)), _provider(provider), _store(store) {} template auto SingleProviderPathResult::clear() -> void { @@ -87,7 +85,7 @@ template auto SingleProviderPathResult::toVelocyPack( arangodb::velocypack::Builder& builder) -> void { if (_vertices.empty()) { - _store.visitReversePath(*_step, [&](Step const& s) -> bool { + _store.visitReversePath(_step, [&](Step const& s) -> bool { prependVertex(s.getVertex()); if (s.getEdge().isValid()) { prependEdge(s.getEdge()); @@ -115,60 +113,6 @@ auto SingleProviderPathResult::toVelocyPack( } } -template -auto SingleProviderPathResult::toSchreierEntry( - arangodb::velocypack::Builder& result, size_t& currentLength) -> void { - if constexpr (!std::is_same_v) { // TODO: eventually move to EE - TRI_ASSERT(false); - THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); - } else { - size_t prevIndex = 0; - - auto writeStepToBuilder = [&](Step& step) { - VPackArrayBuilder arrayGuard(&result); - // Create a VPackValue based on the char* inside the HashedStringRef, will save us a String copy each time. - result.add(VPackValue(step.getVertex().getID().begin())); - - // Index position of previous step - result.add(VPackValue(prevIndex)); - // The depth of the current step - result.add(VPackValue(step.getDepth())); - - bool isResponsible = step.isResponsible(_provider.trx()); - // Print if we have a loose end here (looseEnd := this server is NOT responsible) - result.add(VPackValue(!isResponsible)); - if (isResponsible) { - // This server needs to provide the data for the vertex - _provider.addVertexToBuilder(step.getVertex(), result); - } else { - result.add(VPackSlice::nullSlice()); - } - - _provider.addEdgeToBuilder(step.getEdge(), result); - }; // TODO: Create method instead of lambda - - std::vector toWrite{}; - - _store.modifyReversePath(*_step, [&](Step& step) -> bool { - if (!step.hasLocalSchreierIndex()) { - toWrite.emplace_back(&step); - return true; - } else { - prevIndex = step.getLocalSchreierIndex(); - return false; - } - }); - - for (auto it = toWrite.rbegin(); it != toWrite.rend(); it++) { - TRI_ASSERT(!(*it)->hasLocalSchreierIndex()); - writeStepToBuilder(**it); - - prevIndex = currentLength; - (*it)->setLocalSchreierIndex(currentLength++); - } - } -} - template auto SingleProviderPathResult::isEmpty() const -> bool { @@ -196,6 +140,7 @@ template class ::arangodb::graph::SingleProviderPathResult< ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore>, enterprise::SmartGraphStep>; #endif +// TODO: check if cluster is needed here /* ClusterProvider Section */ template class ::arangodb::graph::SingleProviderPathResult< ::arangodb::graph::ClusterProvider, ::arangodb::graph::PathStore<::arangodb::graph::ClusterProvider::Step>, diff --git a/arangod/Graph/PathManagement/SingleProviderPathResult.h b/arangod/Graph/PathManagement/SingleProviderPathResult.h index 13b7b3021b69..cf2ff82dd8f6 100644 --- a/arangod/Graph/PathManagement/SingleProviderPathResult.h +++ b/arangod/Graph/PathManagement/SingleProviderPathResult.h @@ -29,6 +29,8 @@ // TODO Temporary include #include "Graph/Enumerators/OneSidedEnumeratorInterface.h" +#include "Graph/Providers/TypeAliases.h" +#include "Graph/TraverserOptions.h" #include #include @@ -43,10 +45,10 @@ namespace graph { template class SingleProviderPathResult : public PathResultInterface { - using VertexRef = arangodb::velocypack::HashedStringRef; + using VertexRef = arangodb::velocypack::HashedStringRef; // TODO might remove public: - SingleProviderPathResult(Step* step, ProviderType& provider, PathStoreType& store); + SingleProviderPathResult(Step step, ProviderType& provider, PathStoreType& store); auto clear() -> void; auto appendVertex(typename Step::Vertex v) -> void; auto prependVertex(typename Step::Vertex v) -> void; @@ -55,16 +57,13 @@ class SingleProviderPathResult : public PathResultInterface { auto toVelocyPack(arangodb::velocypack::Builder& builder) -> void override; - /** - * @brief Appends this path as a SchreierVector entry into the given builder - */ - auto toSchreierEntry(arangodb::velocypack::Builder& builder, size_t& currentLength) - -> void override; - auto isEmpty() const -> bool; + ProviderType* getProvider() { return &_provider; } + PathStoreType* getStore() { return &_store; } + Step& getStep() { return _step; } private: - Step* _step; + Step _step; std::vector _vertices; std::vector _edges; diff --git a/arangod/Graph/Providers/BaseProviderOptions.cpp b/arangod/Graph/Providers/BaseProviderOptions.cpp index b23bef9404c1..d5febf54961d 100644 --- a/arangod/Graph/Providers/BaseProviderOptions.cpp +++ b/arangod/Graph/Providers/BaseProviderOptions.cpp @@ -28,8 +28,12 @@ using namespace arangodb::graph; IndexAccessor::IndexAccessor(transaction::Methods::IndexHandle idx, aql::AstNode* condition, std::optional memberToUpdate, - std::shared_ptr expression) - : _idx(idx), _indexCondition(condition), _memberToUpdate(memberToUpdate) { + std::shared_ptr expression, + size_t cursorId) + : _idx(idx), + _indexCondition(condition), + _memberToUpdate(memberToUpdate), + _cursorId(cursorId) { if (expression != nullptr) { _expression = std::move(expression); } @@ -49,6 +53,8 @@ std::optional IndexAccessor::getMemberToUpdate() const { return _memberToUpdate; } +size_t IndexAccessor::cursorId() const { return _cursorId; } + BaseProviderOptions::BaseProviderOptions( aql::Variable const* tmpVar, std::pair, std::unordered_map>> indexInfo, diff --git a/arangod/Graph/Providers/BaseProviderOptions.h b/arangod/Graph/Providers/BaseProviderOptions.h index ae84dc818a5b..709e9050e993 100644 --- a/arangod/Graph/Providers/BaseProviderOptions.h +++ b/arangod/Graph/Providers/BaseProviderOptions.h @@ -43,12 +43,13 @@ namespace graph { struct IndexAccessor { IndexAccessor(transaction::Methods::IndexHandle idx, aql::AstNode* condition, std::optional memberToUpdate, - std::shared_ptr expression); + std::shared_ptr expression, size_t cursorId); aql::AstNode* getCondition() const; aql::Expression* getExpression() const; transaction::Methods::IndexHandle indexHandle() const; std::optional getMemberToUpdate() const; + size_t cursorId() const; private: transaction::Methods::IndexHandle _idx; @@ -57,6 +58,7 @@ struct IndexAccessor { // TODO: This needs to be changed BEFORE merge std::shared_ptr _expression; + size_t _cursorId; }; struct BaseProviderOptions { diff --git a/arangod/Graph/Providers/SingleServerProvider.cpp b/arangod/Graph/Providers/SingleServerProvider.cpp index 59039e8bd777..a2bcc7491ac6 100644 --- a/arangod/Graph/Providers/SingleServerProvider.cpp +++ b/arangod/Graph/Providers/SingleServerProvider.cpp @@ -125,26 +125,32 @@ auto SingleServerProvider::expand(Step const& step, size_t previous, TRI_ASSERT(_cursor != nullptr); LOG_TOPIC("c9169", TRACE, Logger::GRAPHS) << " Expanding " << vertex.getID(); - _cursor->rearm(vertex.getID(), 0); - _cursor->readAll( - *this, _stats, step.getDepth(), - [&](EdgeDocumentToken&& eid, VPackSlice edge, size_t /*cursorIdx*/) -> void { - VertexType id = _cache.persistString(([&]() -> auto { - if (edge.isString()) { - return VertexType(edge); - } else { - VertexType other(transaction::helpers::extractFromFromDocument(edge)); - if (other == vertex.getID()) { // TODO: Check getId - discuss - other = VertexType(transaction::helpers::extractToFromDocument(edge)); - } - return other; - } - })()); - // TODO: Adjust log output - LOG_TOPIC("c9168", TRACE, Logger::GRAPHS) - << " Neighbor of " << vertex.getID() << " -> " << id; - callback(Step{id, std::move(eid), previous, step.getDepth() + 1}); - }); + _cursor->rearm(vertex.getID(), step.getDepth()); + _cursor->readAll(*this, _stats, step.getDepth(), [&](EdgeDocumentToken&& eid, VPackSlice edge, size_t cursorID) -> void { + VertexType id = _cache.persistString(([&]() -> auto { + if (edge.isString()) { + return VertexType(edge); + } else { + VertexType other(transaction::helpers::extractFromFromDocument(edge)); + if (other == vertex.getID()) { // TODO: Check getId - discuss + other = VertexType(transaction::helpers::extractToFromDocument(edge)); + } + return other; + } + })()); + // TODO: Adjust log output + LOG_TOPIC("c9168", TRACE, Logger::GRAPHS) + << " Neighbor of " << vertex.getID() << " -> " << id; +#ifdef USE_ENTERPRISE + if constexpr (std::is_same_v) { + callback(Step{id, std::move(eid), previous, step.getDepth() + 1, cursorID}); + } else { + callback(Step{id, std::move(eid), previous, step.getDepth() + 1}); + } +#else + callback(Step{id, std::move(eid), previous, step.getDepth() + 1}); +#endif + }); } template diff --git a/arangod/Graph/Queues/QueueTracer.cpp b/arangod/Graph/Queues/QueueTracer.cpp index 831d0091d6e4..5b0bc6f258b8 100644 --- a/arangod/Graph/Queues/QueueTracer.cpp +++ b/arangod/Graph/Queues/QueueTracer.cpp @@ -22,7 +22,6 @@ //////////////////////////////////////////////////////////////////////////////// #include "QueueTracer.h" -#include #include "Basics/ScopeGuard.h" #include "Basics/system-functions.h" diff --git a/arangod/Graph/Steps/SingleServerProviderStep.cpp b/arangod/Graph/Steps/SingleServerProviderStep.cpp index 6566c3e7c36d..d6d3dcf31438 100644 --- a/arangod/Graph/Steps/SingleServerProviderStep.cpp +++ b/arangod/Graph/Steps/SingleServerProviderStep.cpp @@ -72,5 +72,5 @@ void SingleServerProviderStep::Edge::addToBuilder(SingleServerProvider(engine); TRI_ASSERT(eng != nullptr); - eng->smartSearchNew(body, result); + eng->smartSearchNew(body, result); // TODO: Rename/Refactor. smartSearchNew does both (DFS & BFS) } else if (option == "smartSearchBFS") { if (engine->getType() != BaseEngine::EngineType::TRAVERSER) { generateError(ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER, @@ -246,7 +246,8 @@ void InternalRestTraverserHandler::queryEngine() { // Safe cast BaseTraverserEngines are all of type TRAVERSER auto eng = static_cast(engine); TRI_ASSERT(eng != nullptr); - eng->smartSearchBFS(body, result); + + eng->smartSearchNew(body, result); // TODO: Rename/Refactor. smartSearchNew does both (DFS & BFS) } else if (option == "smartSearchWeighted") { if (engine->getType() != BaseEngine::EngineType::TRAVERSER) { generateError(ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER, diff --git a/tests/Graph/GenericGraphProviderTest.cpp b/tests/Graph/GenericGraphProviderTest.cpp index 1f93bf29fc72..dc64f5384d5d 100644 --- a/tests/Graph/GenericGraphProviderTest.cpp +++ b/tests/Graph/GenericGraphProviderTest.cpp @@ -109,7 +109,7 @@ class GraphProviderTest : public ::testing::Test { auto indexCondition = singleServer->buildOutboundCondition(query.get(), tmpVar); std::vector usedIndexes{}; - usedIndexes.emplace_back(IndexAccessor{edgeIndexHandle, indexCondition, 0, nullptr}); + usedIndexes.emplace_back(IndexAccessor{edgeIndexHandle, indexCondition, 0, nullptr, 0}); _expressionContext = std::make_unique(*_trx.get(), *query, @@ -120,7 +120,8 @@ class GraphProviderTest : public ::testing::Test { std::make_pair(std::move(usedIndexes), std::unordered_map>{}), *_expressionContext.get(), _emptyShardMap); - return SingleServerProvider(*query.get(), std::move(opts), resourceMonitor); + return SingleServerProvider(*query.get(), std::move(opts), + resourceMonitor); } if constexpr (std::is_same_v) { // Prepare the DBServerResponses diff --git a/tests/Graph/SingleServerProviderTest.cpp b/tests/Graph/SingleServerProviderTest.cpp index 61787cc79a60..11357a998810 100644 --- a/tests/Graph/SingleServerProviderTest.cpp +++ b/tests/Graph/SingleServerProviderTest.cpp @@ -78,7 +78,8 @@ class SingleServerProviderTest : public ::testing::Test { SingleServerProviderTest() {} ~SingleServerProviderTest() {} - auto makeProvider(MockGraph const& graph) -> arangodb::graph::SingleServerProvider { + auto makeProvider(MockGraph const& graph) + -> arangodb::graph::SingleServerProvider { // Setup code for each provider type s = std::make_unique(); singleServer = @@ -97,7 +98,7 @@ class SingleServerProviderTest : public ::testing::Test { std::vector usedIndexes{}; auto expr = conditionKeyMatches(stringToMatch); - usedIndexes.emplace_back(IndexAccessor{edgeIndexHandle, indexCondition, 0, expr}); + usedIndexes.emplace_back(IndexAccessor{edgeIndexHandle, indexCondition, 0, expr, 0}); _expressionContext = std::make_unique(*_trx, *query, _functionsCache); From 35c947299ae50188b9a9f023560733c982718524 Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Tue, 3 Aug 2021 09:26:23 +0200 Subject: [PATCH 18/72] Feature/hybrid smart graph enable weighted (#14560) --- arangod/Cluster/TraverserEngine.cpp | 4 - arangod/Cluster/TraverserEngine.h | 8 +- .../Graph/Cache/RefactoredTraverserCache.cpp | 43 +++- .../Graph/Cache/RefactoredTraverserCache.h | 12 +- .../RefactoredSingleServerEdgeCursor.cpp | 9 +- .../RefactoredSingleServerEdgeCursor.h | 4 +- .../Graph/Enumerators/OneSidedEnumerator.cpp | 21 +- .../Graph/Enumerators/OneSidedEnumerator.h | 9 +- .../Enumerators/OneSidedEnumeratorInterface.h | 3 +- .../Graph/Providers/BaseProviderOptions.cpp | 20 +- arangod/Graph/Providers/BaseProviderOptions.h | 12 + arangod/Graph/Providers/BaseStep.h | 11 +- arangod/Graph/Providers/ClusterProvider.cpp | 3 +- arangod/Graph/Providers/ClusterProvider.h | 2 +- arangod/Graph/Providers/ProviderTracer.cpp | 5 +- arangod/Graph/Providers/ProviderTracer.h | 10 +- .../Graph/Providers/SingleServerProvider.cpp | 43 +++- .../Graph/Providers/SingleServerProvider.h | 10 +- arangod/Graph/Queues/FifoQueue.h | 3 +- arangod/Graph/Queues/LifoQueue.h | 3 +- arangod/Graph/Queues/QueueTracer.cpp | 3 + arangod/Graph/Queues/WeightedQueue.h | 135 +++++++++++ .../Graph/Steps/SingleServerProviderStep.cpp | 13 +- .../Graph/Steps/SingleServerProviderStep.h | 12 +- arangod/Graph/algorithm-aliases.h | 26 ++- .../InternalRestTraverserHandler.cpp | 2 +- tests/CMakeLists.txt | 1 + tests/Graph/TraverserCacheTest.cpp | 34 ++- tests/Graph/WeightedQueueTest.cpp | 213 ++++++++++++++++++ tests/Mocks/MockGraphProvider.cpp | 3 +- tests/Mocks/MockGraphProvider.h | 10 +- 31 files changed, 602 insertions(+), 85 deletions(-) create mode 100644 arangod/Graph/Queues/WeightedQueue.h create mode 100644 tests/Graph/WeightedQueueTest.cpp diff --git a/arangod/Cluster/TraverserEngine.cpp b/arangod/Cluster/TraverserEngine.cpp index 71f370a4bd33..09a72d06aae2 100644 --- a/arangod/Cluster/TraverserEngine.cpp +++ b/arangod/Cluster/TraverserEngine.cpp @@ -422,7 +422,3 @@ void TraverserEngine::smartSearchNew(VPackSlice, VPackBuilder&) { void TraverserEngine::smartSearchBFS(VPackSlice, VPackBuilder&) { THROW_ARANGO_EXCEPTION(TRI_ERROR_ONLY_ENTERPRISE); } - -void TraverserEngine::smartSearchWeighted(VPackSlice, VPackBuilder&) { - THROW_ARANGO_EXCEPTION(TRI_ERROR_ONLY_ENTERPRISE); -} diff --git a/arangod/Cluster/TraverserEngine.h b/arangod/Cluster/TraverserEngine.h index 2c084ceba92a..87b41a7a5248 100644 --- a/arangod/Cluster/TraverserEngine.h +++ b/arangod/Cluster/TraverserEngine.h @@ -117,14 +117,12 @@ class BaseTraverserEngine : public BaseEngine { virtual void smartSearch(arangodb::velocypack::Slice, arangodb::velocypack::Builder&) = 0; - virtual void smartSearchNew(arangodb::velocypack::Slice, arangodb::velocypack::Builder&) = 0; + virtual void smartSearchNew(arangodb::velocypack::Slice, + arangodb::velocypack::Builder&) = 0; virtual void smartSearchBFS(arangodb::velocypack::Slice, arangodb::velocypack::Builder&) = 0; - virtual void smartSearchWeighted(arangodb::velocypack::Slice, - arangodb::velocypack::Builder&) = 0; - EngineType getType() const override { return TRAVERSER; } bool produceVertices() const override; @@ -185,8 +183,6 @@ class TraverserEngine : public BaseTraverserEngine { void smartSearchNew(arangodb::velocypack::Slice, arangodb::velocypack::Builder&) override; void smartSearchBFS(arangodb::velocypack::Slice, arangodb::velocypack::Builder&) override; - - void smartSearchWeighted(arangodb::velocypack::Slice, arangodb::velocypack::Builder&) override; }; } // namespace traverser diff --git a/arangod/Graph/Cache/RefactoredTraverserCache.cpp b/arangod/Graph/Cache/RefactoredTraverserCache.cpp index 2a821a2f909b..8fb650870c1b 100644 --- a/arangod/Graph/Cache/RefactoredTraverserCache.cpp +++ b/arangod/Graph/Cache/RefactoredTraverserCache.cpp @@ -95,7 +95,7 @@ void RefactoredTraverserCache::clear() { template bool RefactoredTraverserCache::appendEdge(EdgeDocumentToken const& idToken, - ResultType& result) { + bool onlyId, ResultType& result) { auto col = _trx->vocbase().lookupCollection(idToken.cid()); if (ADB_UNLIKELY(col == nullptr)) { @@ -110,12 +110,22 @@ bool RefactoredTraverserCache::appendEdge(EdgeDocumentToken const& idToken, col->getPhysical() ->read(_trx, idToken.localDocumentId(), [&](LocalDocumentId const&, VPackSlice edge) -> bool { - // NOTE: Do not count this as Primary Index Scan, we counted - // it in the edge Index before copying... - if constexpr (std::is_same_v) { - result = aql::AqlValue(edge); - } else if constexpr (std::is_same_v) { - result.add(edge); + if (onlyId) { + // NOTE: Do not count this as Primary Index Scan, we + // counted it in the edge Index before copying... + if constexpr (std::is_same_v) { + result = aql::AqlValue(edge.get(StaticStrings::IdString).copyString()); + } else if constexpr (std::is_same_v) { + result.add(edge.get(StaticStrings::IdString)); + } + } else { + // NOTE: Do not count this as Primary Index Scan, we + // counted it in the edge Index before copying... + if constexpr (std::is_same_v) { + result = aql::AqlValue(edge); + } else if constexpr (std::is_same_v) { + result.add(edge); + } } return true; }) @@ -226,19 +236,30 @@ bool RefactoredTraverserCache::appendVertex(aql::TraversalStats& stats, void RefactoredTraverserCache::insertEdgeIntoResult(EdgeDocumentToken const& idToken, VPackBuilder& builder) { - if (!appendEdge(idToken, builder)) { + if (!appendEdge(idToken, false, builder)) { builder.add(VPackSlice::nullSlice()); } } -void RefactoredTraverserCache::insertVertexIntoResult(aql::TraversalStats& stats, - arangodb::velocypack::HashedStringRef const& idString, +void RefactoredTraverserCache::insertEdgeIdIntoResult(EdgeDocumentToken const& idToken, VPackBuilder& builder) { - if (!appendVertex(stats, idString, builder)) { + if (!appendEdge(idToken, true, builder)) { builder.add(VPackSlice::nullSlice()); } } +void RefactoredTraverserCache::insertVertexIntoResult( + aql::TraversalStats& stats, arangodb::velocypack::HashedStringRef const& idString, + VPackBuilder& builder, bool writeIdIfNotFound) { + if (!appendVertex(stats, idString, builder)) { + if (writeIdIfNotFound) { + builder.add(VPackValue(idString.toString())); + } else { + builder.add(VPackSlice::nullSlice()); + } + } +} + arangodb::velocypack::HashedStringRef RefactoredTraverserCache::persistString( arangodb::velocypack::HashedStringRef idString) { auto it = _persistedStrings.find(idString); diff --git a/arangod/Graph/Cache/RefactoredTraverserCache.h b/arangod/Graph/Cache/RefactoredTraverserCache.h index ca4a9f688a2d..77dcf44bbede 100644 --- a/arangod/Graph/Cache/RefactoredTraverserCache.h +++ b/arangod/Graph/Cache/RefactoredTraverserCache.h @@ -82,12 +82,18 @@ class RefactoredTraverserCache { ////////////////////////////////////////////////////////////////////////////// void insertEdgeIntoResult(graph::EdgeDocumentToken const& etkn, velocypack::Builder& builder); + ////////////////////////////////////////////////////////////////////////////// + /// @brief Inserts only the edges _id value into the given builder. + ////////////////////////////////////////////////////////////////////////////// + void insertEdgeIdIntoResult(graph::EdgeDocumentToken const& etkn, + velocypack::Builder& builder); + ////////////////////////////////////////////////////////////////////////////// /// @brief Inserts the real document identified by the _id string ////////////////////////////////////////////////////////////////////////////// void insertVertexIntoResult(aql::TraversalStats& stats, arangodb::velocypack::HashedStringRef const& idString, - velocypack::Builder& builder); + velocypack::Builder& builder, bool writeIdIfNotFound); ////////////////////////////////////////////////////////////////////////////// /// @brief Persist the given id string. The return value is guaranteed to @@ -109,10 +115,12 @@ class RefactoredTraverserCache { ////////////////////////////////////////////////////////////////////////////// /// @brief Lookup an edge document from the database. /// if this returns false the result is unmodified + /// if onlyId is set to true, the result will only contain the _Id + /// value not the document itself. ////////////////////////////////////////////////////////////////////////////// template - bool appendEdge(graph::EdgeDocumentToken const& etkn, ResultType& result); + bool appendEdge(graph::EdgeDocumentToken const& etkn, bool onlyId, ResultType& result); ////////////////////////////////////////////////////////////////////////////// /// @brief Helper Method to extract collection Name from given VertexIdentifier diff --git a/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.cpp b/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.cpp index d46fb59912a2..966fc25f07d7 100644 --- a/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.cpp +++ b/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.cpp @@ -153,8 +153,11 @@ RefactoredSingleServerEdgeCursor::RefactoredSingleServerEdgeCursor( transaction::Methods* trx, arangodb::aql::Variable const* tmpVar, std::vector const& globalIndexConditions, std::unordered_map> const& depthBasedIndexConditions, - arangodb::aql::FixedVarExpressionContext& expressionContext) - : _tmpVar(tmpVar), _trx(trx), _expressionCtx(expressionContext) { + arangodb::aql::FixedVarExpressionContext& expressionContext, bool requiresFullDocument) + : _tmpVar(tmpVar), + _trx(trx), + _expressionCtx(expressionContext), + _requiresFullDocument(requiresFullDocument) { // We need at least one indexCondition, otherwise nothing to serve TRI_ASSERT(!globalIndexConditions.empty()); _lookupInfo.reserve(globalIndexConditions.size()); @@ -216,7 +219,7 @@ void RefactoredSingleServerEdgeCursor::readAll(SingleServerProvider& auto& cursor = lookupInfo.cursor(); LogicalCollection* collection = cursor.collection(); auto cid = collection->id(); - bool hasExtra = cursor.hasExtra(); + bool hasExtra = !_requiresFullDocument && cursor.hasExtra(); auto* expression = lookupInfo.getExpression(); if (hasExtra) { diff --git a/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.h b/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.h index ae0995f00f0b..56a36e6e51d7 100644 --- a/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.h +++ b/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.h @@ -97,7 +97,8 @@ class RefactoredSingleServerEdgeCursor { transaction::Methods* trx, arangodb::aql::Variable const* tmpVar, std::vector const& globalIndexConditions, std::unordered_map> const& depthBasedIndexConditions, - arangodb::aql::FixedVarExpressionContext& expressionContext); + arangodb::aql::FixedVarExpressionContext& expressionContext, bool requiresFullDocument); + ~RefactoredSingleServerEdgeCursor(); using Callback = @@ -110,6 +111,7 @@ class RefactoredSingleServerEdgeCursor { transaction::Methods* _trx; arangodb::aql::FixedVarExpressionContext& _expressionCtx; + bool _requiresFullDocument; public: void readAll(SingleServerProvider& provider, diff --git a/arangod/Graph/Enumerators/OneSidedEnumerator.cpp b/arangod/Graph/Enumerators/OneSidedEnumerator.cpp index 9491e91a4f6d..60950d82f5ea 100644 --- a/arangod/Graph/Enumerators/OneSidedEnumerator.cpp +++ b/arangod/Graph/Enumerators/OneSidedEnumerator.cpp @@ -144,9 +144,10 @@ bool OneSidedEnumerator::isDone() const { * @param source The source vertex to start the paths */ template -void OneSidedEnumerator::reset(VertexRef source, size_t depth, bool keepPathStore) { +void OneSidedEnumerator::reset(VertexRef source, size_t depth, + double weight, bool keepPathStore) { clear(keepPathStore); - auto firstStep = _provider.startVertex(source, depth); + auto firstStep = _provider.startVertex(source, depth, weight); _queue.append(std::move(firstStep)); } @@ -316,4 +317,20 @@ template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::NONE, true>>; template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::GLOBAL, true>>; + +// Weighted Search +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::PATH, false>>; +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::NONE, false>>; +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::GLOBAL, false>>; + +// Weighted Tracing +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::PATH, true>>; +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::NONE, true>>; +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::GLOBAL, true>>; #endif diff --git a/arangod/Graph/Enumerators/OneSidedEnumerator.h b/arangod/Graph/Enumerators/OneSidedEnumerator.h index 336d2733fe45..61cc440e1b42 100644 --- a/arangod/Graph/Enumerators/OneSidedEnumerator.h +++ b/arangod/Graph/Enumerators/OneSidedEnumerator.h @@ -83,7 +83,7 @@ class OneSidedEnumerator : public TraversalEnumerator { [[nodiscard]] bool isDone() const override; /** - * @brief Reset to new source and target vertices. + * @brief Reset to new source vertex. * This API uses string references, this class will not take responsibility * for the referenced data. It is caller's responsibility to retain the * underlying data and make sure the StringRefs stay valid until next @@ -91,8 +91,13 @@ class OneSidedEnumerator : public TraversalEnumerator { * * @param source The source vertex to start the paths * @param depth The depth we're starting the search at + * @param weight The vertex ist starting to search at, only relevant for weighted searches + * @param keepPathStore flag to determine that we should keep internas of last + * run in memory. should be used if the last result is not processed yet, as + * we will create invalid memory access in the handed out Paths. */ - void reset(VertexRef source, size_t depth = 0, bool keepPathStore = false) override; + void reset(VertexRef source, size_t depth = 0, double weight = 0.0, + bool keepPathStore = false) override; /** * @brief Get the next path, if available written into the result build. diff --git a/arangod/Graph/Enumerators/OneSidedEnumeratorInterface.h b/arangod/Graph/Enumerators/OneSidedEnumeratorInterface.h index 7e631986ea0b..ef79aabfb903 100644 --- a/arangod/Graph/Enumerators/OneSidedEnumeratorInterface.h +++ b/arangod/Graph/Enumerators/OneSidedEnumeratorInterface.h @@ -67,7 +67,8 @@ class TraversalEnumerator { // NOTE: keepPathStore is only required for 3.8 compatibility and // can be removed in the version after 3.9 - virtual void reset(VertexRef source, size_t depth = 0, bool keepPathStore = false) = 0; + virtual void reset(VertexRef source, size_t depth = 0, double weight = 0.0, + bool keepPathStore = false) = 0; virtual auto getNextPath() -> std::unique_ptr = 0; virtual bool skipPath() = 0; virtual auto destroyEngines() -> void = 0; diff --git a/arangod/Graph/Providers/BaseProviderOptions.cpp b/arangod/Graph/Providers/BaseProviderOptions.cpp index d5febf54961d..a01ce0f59410 100644 --- a/arangod/Graph/Providers/BaseProviderOptions.cpp +++ b/arangod/Graph/Providers/BaseProviderOptions.cpp @@ -63,7 +63,8 @@ BaseProviderOptions::BaseProviderOptions( : _temporaryVariable(tmpVar), _indexInformation(std::move(indexInfo)), _expressionContext(expressionContext), - _collectionToShardMap(collectionToShardMap) {} + _collectionToShardMap(collectionToShardMap), + _weightCallback(std::nullopt) {} aql::Variable const* BaseProviderOptions::tmpVar() const { return _temporaryVariable; @@ -83,6 +84,23 @@ aql::FixedVarExpressionContext& BaseProviderOptions::expressionContext() const { return _expressionContext; } +bool BaseProviderOptions::hasWeightMethod() const { + return _weightCallback.has_value(); +} + +void BaseProviderOptions::setWeightEdgeCallback(WeightCallback callback) { + _weightCallback = std::move(callback); +} + +double BaseProviderOptions::weightEdge(double prefixWeight, + arangodb::velocypack::Slice edge) const { + if (!hasWeightMethod()) { + // We do not have a weight. Hardcode. + return 1.0; + } + return _weightCallback.value()(prefixWeight, edge); +} + ClusterBaseProviderOptions::ClusterBaseProviderOptions( std::shared_ptr cache, std::unordered_map const* engines, bool backward) diff --git a/arangod/Graph/Providers/BaseProviderOptions.h b/arangod/Graph/Providers/BaseProviderOptions.h index 709e9050e993..da9da66114b5 100644 --- a/arangod/Graph/Providers/BaseProviderOptions.h +++ b/arangod/Graph/Providers/BaseProviderOptions.h @@ -62,6 +62,9 @@ struct IndexAccessor { }; struct BaseProviderOptions { + using WeightCallback = + std::function; + public: BaseProviderOptions( aql::Variable const* tmpVar, @@ -76,6 +79,12 @@ struct BaseProviderOptions { aql::FixedVarExpressionContext& expressionContext() const; + bool hasWeightMethod() const; + + double weightEdge(double prefixWeight, arangodb::velocypack::Slice edge) const; + + void setWeightEdgeCallback(WeightCallback callback); + private: // The temporary Variable used in the Indexes aql::Variable const* _temporaryVariable; @@ -89,6 +98,9 @@ struct BaseProviderOptions { // CollectionName to ShardMap, used if the Traversal is pushed down to DBServer std::unordered_map> const& _collectionToShardMap; + + // Optional callback to compute the weight of an edge. + std::optional _weightCallback; }; struct ClusterBaseProviderOptions { diff --git a/arangod/Graph/Providers/BaseStep.h b/arangod/Graph/Providers/BaseStep.h index c9a49139d98c..56418d74359c 100644 --- a/arangod/Graph/Providers/BaseStep.h +++ b/arangod/Graph/Providers/BaseStep.h @@ -34,11 +34,9 @@ namespace graph { template class BaseStep { public: - BaseStep() : _previous{std::numeric_limits::max()}, _depth{0} {} - - BaseStep(size_t prev) : _previous{prev}, _depth{0} {} - - BaseStep(size_t prev, size_t depth) : _previous{prev}, _depth{depth} {} + BaseStep(size_t prev = std::numeric_limits::max(), size_t depth = 0, + double weight = 1.0) + : _previous{prev}, _depth{depth}, _weight(weight) {} size_t getPrevious() const { return _previous; } @@ -52,6 +50,8 @@ class BaseStep { size_t getDepth() const { return _depth; } + double getWeight() const { return _weight; } + ResultT> extractCollectionName( arangodb::velocypack::HashedStringRef const& idHashed) const { size_t pos = idHashed.find('/'); @@ -70,6 +70,7 @@ class BaseStep { private: size_t _previous; size_t _depth; + double _weight; }; } // namespace graph } // namespace arangodb diff --git a/arangod/Graph/Providers/ClusterProvider.cpp b/arangod/Graph/Providers/ClusterProvider.cpp index 8e4425c08742..4820190b807b 100644 --- a/arangod/Graph/Providers/ClusterProvider.cpp +++ b/arangod/Graph/Providers/ClusterProvider.cpp @@ -117,9 +117,10 @@ void ClusterProvider::clear() { } } -auto ClusterProvider::startVertex(VertexType vertex, size_t depth) -> Step { +auto ClusterProvider::startVertex(VertexType vertex, size_t depth, double weight) -> Step { LOG_TOPIC("da308", TRACE, Logger::GRAPHS) << " Start Vertex:" << vertex; // Create the default initial step. + TRI_ASSERT(weight == 0.0); // Not implemented yet return Step(_opts.getCache()->persistString(vertex)); } diff --git a/arangod/Graph/Providers/ClusterProvider.h b/arangod/Graph/Providers/ClusterProvider.h index 6bc8ddcb6ed6..d6efd0c53947 100644 --- a/arangod/Graph/Providers/ClusterProvider.h +++ b/arangod/Graph/Providers/ClusterProvider.h @@ -154,7 +154,7 @@ class ClusterProvider { void clear(); - auto startVertex(VertexType vertex, size_t depth = 0) -> Step; + auto startVertex(VertexType vertex, size_t depth = 0, double weight = 0.0) -> Step; auto fetch(std::vector const& looseEnds) -> futures::Future>; auto expand(Step const& from, size_t previous, std::function const& callback) -> void; diff --git a/arangod/Graph/Providers/ProviderTracer.cpp b/arangod/Graph/Providers/ProviderTracer.cpp index 640f4684daa6..058e84208e73 100644 --- a/arangod/Graph/Providers/ProviderTracer.cpp +++ b/arangod/Graph/Providers/ProviderTracer.cpp @@ -53,10 +53,11 @@ ProviderTracer::~ProviderTracer() { template typename ProviderImpl::Step ProviderTracer::startVertex(VertexType vertex, - size_t depth) { + size_t depth, + double weight) { double start = TRI_microtime(); TRI_DEFER(_stats["startVertex"].addTiming(TRI_microtime() - start)); - return _impl.startVertex(vertex, depth); + return _impl.startVertex(vertex, depth, weight); } template diff --git a/arangod/Graph/Providers/ProviderTracer.h b/arangod/Graph/Providers/ProviderTracer.h index 3c6f852ec37c..ca1de73def57 100644 --- a/arangod/Graph/Providers/ProviderTracer.h +++ b/arangod/Graph/Providers/ProviderTracer.h @@ -38,7 +38,7 @@ namespace arangodb { namespace aql { class QueryContext; class TraversalStats; -} +} // namespace aql namespace graph { @@ -59,10 +59,11 @@ class ProviderTracer { ProviderTracer& operator=(ProviderTracer const&) = delete; ProviderTracer& operator=(ProviderTracer&&) = default; - auto startVertex(VertexType vertex, size_t depth = 0) -> Step; + auto startVertex(VertexType vertex, size_t depth = 0, double weight = 0.0) -> Step; auto fetch(std::vector const& looseEnds) - -> futures::Future>; // rocks - auto expand(Step const& from, size_t previous, std::function callback) -> void; // index + -> futures::Future>; + auto expand(Step const& from, size_t previous, std::function callback) + -> void; void addVertexToBuilder(typename Step::Vertex const& vertex, arangodb::velocypack::Builder& builder); @@ -86,4 +87,3 @@ class ProviderTracer { } // namespace graph } // namespace arangodb - diff --git a/arangod/Graph/Providers/SingleServerProvider.cpp b/arangod/Graph/Providers/SingleServerProvider.cpp index bee63e4baec5..2c0aaf248cc6 100644 --- a/arangod/Graph/Providers/SingleServerProvider.cpp +++ b/arangod/Graph/Providers/SingleServerProvider.cpp @@ -56,6 +56,20 @@ void SingleServerProvider::addEdgeToBuilder(typename Step::Edge const& edg if (edge.isValid()) { insertEdgeIntoResult(edge.getID(), builder); } else { + // We can never hand out invalid ids. + // For production just be sure to add something sensible. + builder.add(VPackSlice::nullSlice()); + } +}; + +template +void SingleServerProvider::addEdgeIDToBuilder(typename Step::Edge const& edge, + arangodb::velocypack::Builder& builder) { + if (edge.isValid()) { + insertEdgeIntoResult(edge.getID(), builder); + } else { + // We can never hand out invalid ids. + // For production just be sure to add something sensible. builder.add(VPackSlice::nullSlice()); } }; @@ -94,13 +108,14 @@ void SingleServerProvider::activateCache(bool enableDocumentCache) { } template -auto SingleServerProvider::startVertex(VertexType vertex, size_t depth) -> Step { +auto SingleServerProvider::startVertex(VertexType vertex, size_t depth, double weight) + -> Step { LOG_TOPIC("78156", TRACE, Logger::GRAPHS) << " Start Vertex:" << vertex; // Create default initial step // Note: Refactor naming, Strings in our cache here are not allowed to be removed. - return Step(_cache.persistString(vertex), depth); + return Step(_cache.persistString(vertex), depth, weight); } template @@ -141,22 +156,20 @@ auto SingleServerProvider::expand(Step const& step, size_t previous, // TODO: Adjust log output LOG_TOPIC("c9168", TRACE, Logger::GRAPHS) << " Neighbor of " << vertex.getID() << " -> " << id; -#ifdef USE_ENTERPRISE - if constexpr (std::is_same_v) { - callback(Step{id, std::move(eid), previous, step.getDepth() + 1, cursorID}); + if (_opts.hasWeightMethod()) { + callback(Step{id, std::move(eid), previous, step.getDepth() + 1, + _opts.weightEdge(step.getWeight(), edge), cursorID}); } else { - callback(Step{id, std::move(eid), previous, step.getDepth() + 1}); + callback(Step{id, std::move(eid), previous, step.getDepth() + 1, 1.0, cursorID}); } -#else - callback(Step{id, std::move(eid), previous, step.getDepth() + 1}); -#endif }); } template void SingleServerProvider::addVertexToBuilder(typename Step::Vertex const& vertex, - arangodb::velocypack::Builder& builder) { - _cache.insertVertexIntoResult(_stats, vertex.getID(), builder); + arangodb::velocypack::Builder& builder, + bool writeIdIfNotFound) { + _cache.insertVertexIntoResult(_stats, vertex.getID(), builder, writeIdIfNotFound); } template @@ -165,12 +178,18 @@ void SingleServerProvider::insertEdgeIntoResult(EdgeDocumentToken edge, _cache.insertEdgeIntoResult(edge, builder); } +template +void SingleServerProvider::insertEdgeIdIntoResult(EdgeDocumentToken edge, + arangodb::velocypack::Builder& builder) { + _cache.insertEdgeIdIntoResult(edge, builder); +} + template std::unique_ptr> SingleServerProvider::buildCursor( arangodb::aql::FixedVarExpressionContext& expressionContext) { return std::make_unique>( trx(), _opts.tmpVar(), _opts.indexInformations().first, - _opts.indexInformations().second, expressionContext); + _opts.indexInformations().second, expressionContext, _opts.hasWeightMethod()); } template diff --git a/arangod/Graph/Providers/SingleServerProvider.h b/arangod/Graph/Providers/SingleServerProvider.h index 1d7f9c0265ce..e33fd76463c1 100644 --- a/arangod/Graph/Providers/SingleServerProvider.h +++ b/arangod/Graph/Providers/SingleServerProvider.h @@ -70,17 +70,23 @@ class SingleServerProvider { SingleServerProvider& operator=(SingleServerProvider const&) = delete; - auto startVertex(VertexType vertex, size_t depth = 0) -> Step; + auto startVertex(VertexType vertex, size_t depth = 0, double weight = 0.0) -> Step; auto fetch(std::vector const& looseEnds) -> futures::Future>; // rocks auto expand(Step const& from, size_t previous, std::function const& callback) -> void; // index void insertEdgeIntoResult(EdgeDocumentToken edge, arangodb::velocypack::Builder& builder); + void insertEdgeIdIntoResult(EdgeDocumentToken edge, arangodb::velocypack::Builder& builder); void addVertexToBuilder(typename Step::Vertex const& vertex, + arangodb::velocypack::Builder& builder, + bool writeIdIfNotFound = false); + void addEdgeToBuilder(typename Step::Edge const& edge, + arangodb::velocypack::Builder& builder); + + void addEdgeIDToBuilder(typename Step::Edge const& edge, arangodb::velocypack::Builder& builder); - void addEdgeToBuilder(typename Step::Edge const& edge, arangodb::velocypack::Builder& builder); void destroyEngines(){}; diff --git a/arangod/Graph/Queues/FifoQueue.h b/arangod/Graph/Queues/FifoQueue.h index 1d0408a8926a..9c9eaccd819c 100644 --- a/arangod/Graph/Queues/FifoQueue.h +++ b/arangod/Graph/Queues/FifoQueue.h @@ -35,6 +35,7 @@ namespace graph { template class FifoQueue { public: + static constexpr bool RequiresWeight = false; using Step = StepType; // TODO: Add Sorting (Performance - will be implemented in the future - cluster relevant) // -> loose ends to the front @@ -55,7 +56,7 @@ class FifoQueue { // if push_front() throws, no harm is done, and the memory usage increase // will be rolled back _queue.push_back(std::move(step)); - guard.steal(); // now we are responsible for tracking the memory + guard.steal(); // now we are responsible for tracking the memory } bool hasProcessableElement() const { diff --git a/arangod/Graph/Queues/LifoQueue.h b/arangod/Graph/Queues/LifoQueue.h index a691ffa12df8..4ab5e0b99d90 100644 --- a/arangod/Graph/Queues/LifoQueue.h +++ b/arangod/Graph/Queues/LifoQueue.h @@ -35,6 +35,7 @@ namespace graph { template class LifoQueue { public: + static constexpr bool RequiresWeight = false; using Step = StepType; // TODO: Add Sorting (Performance - will be implemented in the future - cluster relevant) // -> loose ends to the end @@ -55,7 +56,7 @@ class LifoQueue { // if push_front() throws, no harm is done, and the memory usage increase // will be rolled back _queue.push_front(std::move(step)); - guard.steal(); // now we are responsible for tracking the memory + guard.steal(); // now we are responsible for tracking the memory } bool hasProcessableElement() const { diff --git a/arangod/Graph/Queues/QueueTracer.cpp b/arangod/Graph/Queues/QueueTracer.cpp index 5b0bc6f258b8..815945cda71b 100644 --- a/arangod/Graph/Queues/QueueTracer.cpp +++ b/arangod/Graph/Queues/QueueTracer.cpp @@ -28,6 +28,7 @@ #include "Graph/Providers/ClusterProvider.h" #include "Graph/Queues/FifoQueue.h" #include "Graph/Queues/LifoQueue.h" +#include "Graph/Queues/WeightedQueue.h" #include "Graph/Steps/SingleServerProviderStep.h" #ifdef USE_ENTERPRISE @@ -112,10 +113,12 @@ using SingleServerProviderStep = ::arangodb::graph::SingleServerProviderStep; template class ::arangodb::graph::QueueTracer>; template class ::arangodb::graph::QueueTracer>; +template class ::arangodb::graph::QueueTracer>; #ifdef USE_ENTERPRISE template class ::arangodb::graph::QueueTracer>; template class ::arangodb::graph::QueueTracer>; +template class ::arangodb::graph::QueueTracer>; #endif /* ClusterServerProvider Section */ diff --git a/arangod/Graph/Queues/WeightedQueue.h b/arangod/Graph/Queues/WeightedQueue.h new file mode 100644 index 000000000000..18bbbe5ae1da --- /dev/null +++ b/arangod/Graph/Queues/WeightedQueue.h @@ -0,0 +1,135 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2021 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +//////////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include "Basics/ResourceUsage.h" +#include "Basics/debugging.h" +#include "Logger/LogMacros.h" + +#include +#include + +namespace arangodb { +namespace graph { + +template +class WeightedQueue { + public: + static constexpr bool RequiresWeight = true; + using Step = StepType; + // TODO: Add Sorting (Performance - will be implemented in the future - cluster relevant) + // -> loose ends to the end + + explicit WeightedQueue(arangodb::ResourceMonitor& resourceMonitor) + : _resourceMonitor{resourceMonitor} {} + ~WeightedQueue() { this->clear(); } + + void clear() { + if (!_queue.empty()) { + _resourceMonitor.decreaseMemoryUsage(_queue.size() * sizeof(Step)); + _queue.clear(); + } + } + + void append(Step step) { + arangodb::ResourceUsageScope guard(_resourceMonitor, sizeof(Step)); + // if emplace() throws, no harm is done, and the memory usage increase + // will be rolled back + _queue.emplace_back(std::move(step)); + guard.steal(); // now we are responsible for tracking the memory + // std::push_heap takes the last element in the queue, assumes that all + // other elements are in heap structure, and moves the last element into + // the correct position in the heap (incl. rebalancing of other elements) + // The heap structure guarantees that the first element in the queue + // is the "largest" element (in our case it is the smallest, as we inverted the comperator) + std::push_heap(_queue.begin(), _queue.end(), _cmpHeap); + } + + bool hasProcessableElement() const { + if (!isEmpty()) { + // The heap structure guarantees that the first element in the queue + // is the "largest" element (in our case it is the smallest, as we inverted the comperator) + auto const& first = _queue.front(); + return first.isProcessable(); + } + + return false; + } + + size_t size() const { return _queue.size(); } + + bool isEmpty() const { return _queue.empty(); } + + std::vector getLooseEnds() { + TRI_ASSERT(!hasProcessableElement()); + + std::vector steps; + for (auto& step : _queue) { + if (!step.isProcessable()) { + steps.emplace_back(&step); + } + } + + return steps; + } + + Step pop() { + TRI_ASSERT(!isEmpty()); + // std::pop_heap will move the front element (the one we would like to steal) + // to the back of the vector, keeping the tree intact otherwise. + // Now we steal the last element. + std::pop_heap(_queue.begin(), _queue.end(), _cmpHeap); + Step first = std::move(_queue.back()); + LOG_TOPIC("9cd66", TRACE, Logger::GRAPHS) + << " Pop: " << first.toString(); + _resourceMonitor.decreaseMemoryUsage(sizeof(Step)); + _queue.pop_back(); + return first; + } + + private: + struct WeightedComparator { + bool operator()(Step const& a, Step const& b) { + if (a.getWeight() == b.getWeight()) { + // Only false if A is not processable but B is. + return !a.isProcessable() || b.isProcessable(); + } + return a.getWeight() > b.getWeight(); + } + }; + + WeightedComparator _cmpHeap{}; + + /// @brief queue datastore + /// Note: Mutable is a required for hasProcessableElement right now which is const. We can easily make it non const here. + mutable std::vector _queue; + // std::priority_queue, WeightedComparator> _queue; + + /// @brief query context + arangodb::ResourceMonitor& _resourceMonitor; +}; + +} // namespace graph +} // namespace arangodb + diff --git a/arangod/Graph/Steps/SingleServerProviderStep.cpp b/arangod/Graph/Steps/SingleServerProviderStep.cpp index d6d3dcf31438..ca83e669bd85 100644 --- a/arangod/Graph/Steps/SingleServerProviderStep.cpp +++ b/arangod/Graph/Steps/SingleServerProviderStep.cpp @@ -41,16 +41,17 @@ auto operator<<(std::ostream& out, SingleServerProviderStep const& step) -> std: SingleServerProviderStep::SingleServerProviderStep(VertexType v) : _vertex(v), _edge() {} -SingleServerProviderStep::SingleServerProviderStep(VertexType v, size_t depth) - : BaseStep(std::numeric_limits::max(), depth), _vertex(v), _edge() {} +SingleServerProviderStep::SingleServerProviderStep(VertexType v, size_t depth, double weight) + : BaseStep(std::numeric_limits::max(), depth, weight), _vertex(v), _edge() {} SingleServerProviderStep::SingleServerProviderStep(VertexType v, EdgeDocumentToken edge, size_t prev) : BaseStep(prev), _vertex(v), _edge(std::move(edge)) {} SingleServerProviderStep::SingleServerProviderStep(VertexType v, EdgeDocumentToken edge, - size_t prev, size_t depth) - : BaseStep(prev, depth), _vertex(v), _edge(std::move(edge)) {} + size_t prev, size_t depth, + double weight, size_t) + : BaseStep(prev, depth, weight), _vertex(v), _edge(std::move(edge)) {} SingleServerProviderStep::~SingleServerProviderStep() = default; @@ -72,5 +73,7 @@ void SingleServerProviderStep::Edge::addToBuilder(SingleServerProvider std::ostream&; + friend auto operator<<(std::ostream& out, SingleServerProviderStep const& step) + -> std::ostream&; private: Vertex _vertex; diff --git a/arangod/Graph/algorithm-aliases.h b/arangod/Graph/algorithm-aliases.h index b76872a64475..113d9cb71eaa 100644 --- a/arangod/Graph/algorithm-aliases.h +++ b/arangod/Graph/algorithm-aliases.h @@ -29,6 +29,7 @@ #include "Graph/Queues/FifoQueue.h" #include "Graph/Queues/LifoQueue.h" #include "Graph/Queues/QueueTracer.h" +#include "Graph/Queues/WeightedQueue.h" #include "Graph/PathManagement/PathStore.h" #include "Graph/PathManagement/PathStoreTracer.h" @@ -76,6 +77,18 @@ struct DFSConfiguration { using Validator = PathValidator; }; +template +struct WeightedConfiguration { + using Provider = + typename std::conditional, ProviderType>::type; + using Step = typename Provider::Step; + using Queue = + typename std::conditional>, WeightedQueue>::type; + using Store = + typename std::conditional>, PathStore>::type; + using Validator = PathValidator; +}; + // BFS Traversal Enumerator implementation template using BFSEnumerator = @@ -86,8 +99,6 @@ template using TracedBFSEnumerator = OneSidedEnumerator>; -// BFS Traversal Enumerator implementation using Tracing (without provider tracing) - // DFS Traversal Enumerator implementation template using DFSEnumerator = @@ -98,5 +109,16 @@ template using TracedDFSEnumerator = OneSidedEnumerator>; +// Weighted Traversal Enumerator implementation +// TODO: Needs to be renamed as soon as we replace the existing variant, whic occupies this name +template +using WeightedEnumeratorRefactored = + OneSidedEnumerator>; + +// BFS Traversal Enumerator implementation using Tracing +template +using TracedWeightedEnumerator = + OneSidedEnumerator>; + } // namespace graph } // namespace arangodb diff --git a/arangod/InternalRestHandler/InternalRestTraverserHandler.cpp b/arangod/InternalRestHandler/InternalRestTraverserHandler.cpp index a0190b13ae30..82b5913177c0 100644 --- a/arangod/InternalRestHandler/InternalRestTraverserHandler.cpp +++ b/arangod/InternalRestHandler/InternalRestTraverserHandler.cpp @@ -257,7 +257,7 @@ void InternalRestTraverserHandler::queryEngine() { // Safe cast BaseTraverserEngines are all of type TRAVERSER auto eng = static_cast(engine); TRI_ASSERT(eng != nullptr); - eng->smartSearchWeighted(body, result); + eng->smartSearchNew(body, result); } else { // PATH Info wrong other error generateError(ResponseCode::NOT_FOUND, TRI_ERROR_HTTP_NOT_FOUND, ""); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index df9fc04c5945..604e4ae24bdb 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -239,6 +239,7 @@ set(ARANGODB_TESTS_SOURCES Graph/PathValidatorTest.cpp Graph/FifoQueueTest.cpp Graph/LifoQueueTest.cpp + Graph/WeightedQueueTest.cpp Graph/SingleServerProviderTest.cpp Greenspun/PrimitivesTest.cpp HotBackup/HotBackupCoordinatorTest.cpp diff --git a/tests/Graph/TraverserCacheTest.cpp b/tests/Graph/TraverserCacheTest.cpp index de9c24981706..e0d4b6136e82 100644 --- a/tests/Graph/TraverserCacheTest.cpp +++ b/tests/Graph/TraverserCacheTest.cpp @@ -87,7 +87,7 @@ TEST_F(TraverserCacheTest, it_should_return_a_null_aqlvalue_if_vertex_is_not_ava // NOTE: we do not have the data, so we get null for any vertex traverserCache->insertVertexIntoResult(stats, arangodb::velocypack::HashedStringRef(id), - builder); + builder, false); ASSERT_TRUE(builder.slice().isNull()); auto all = query->warnings().all(); ASSERT_EQ(all.size(), 1); @@ -100,6 +100,36 @@ TEST_F(TraverserCacheTest, it_should_return_a_null_aqlvalue_if_vertex_is_not_ava EXPECT_EQ(stats.getScannedIndex(), 0); } +TEST_F(TraverserCacheTest, it_should_on_request_return_the_id_aqlvalue_if_vertex_is_not_available) { + // prepare graph data - in this case, no data (no vertices and no edges, but collections v and e) + graph::MockGraph graph{}; + gdb.addGraph(graph); + + std::string vertexId = "v/Vertex"; + std::string expectedMessage = "vertex '" + vertexId + "' not found"; + + auto data = VPackParser::fromJson(R"({"_key":"Vertex", "_id": "v/Vertex"})"); + VPackSlice doc = data->slice(); + HashedStringRef id{doc.get("_id")}; + VPackBuilder builder; + + // NOTE: we do not have the data, so we get null for any vertex + traverserCache->insertVertexIntoResult(stats, arangodb::velocypack::HashedStringRef(id), + builder, true); + ASSERT_FALSE(builder.slice().isNull()); + ASSERT_TRUE(builder.slice().isString()); + ASSERT_EQ(builder.slice().copyString(), "v/Vertex"); + auto all = query->warnings().all(); + ASSERT_EQ(all.size(), 1); + ASSERT_TRUE(all[0].first == TRI_ERROR_ARANGO_DOCUMENT_NOT_FOUND); + ASSERT_TRUE(all[0].second == expectedMessage); + + // check stats + EXPECT_EQ(stats.getHttpRequests(), 0); + EXPECT_EQ(stats.getFiltered(), 0); + EXPECT_EQ(stats.getScannedIndex(), 0); +} + TEST_F(TraverserCacheTest, it_should_return_a_null_aqlvalue_if_edge_is_not_available) { // prepare graph data - in this case, no data (no vertices and no edges, but collections v and e) graph::MockGraph graph{}; @@ -225,7 +255,7 @@ TEST_F(TraverserCacheTest, it_should_insert_a_vertex_into_a_result_builder) { VPackBuilder builder; traverserCache->insertVertexIntoResult(stats, arangodb::velocypack::HashedStringRef(id), - builder); + builder, false); EXPECT_TRUE(builder.slice().get("_key").isString()); EXPECT_EQ(builder.slice().get("_key").toString(), "0"); diff --git a/tests/Graph/WeightedQueueTest.cpp b/tests/Graph/WeightedQueueTest.cpp new file mode 100644 index 000000000000..128bc1d68791 --- /dev/null +++ b/tests/Graph/WeightedQueueTest.cpp @@ -0,0 +1,213 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2020 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Heiko Kernbach +//////////////////////////////////////////////////////////////////////////////// + +#include "Basics/GlobalResourceMonitor.h" +#include "Basics/ResourceUsage.h" +#include "Basics/ResultT.h" +#include "Basics/StringUtils.h" +#include "Graph/Providers/BaseStep.h" +#include "Graph/Queues/WeightedQueue.h" + +#include "gtest/gtest.h" + +using namespace arangodb; +using namespace arangodb::graph; + +namespace arangodb { +namespace tests { +namespace queue_graph_cache_test { + +class Step : public arangodb::graph::BaseStep { + size_t _id; + bool _isLooseEnd; + + public: + Step(size_t id, double weight, bool isLooseEnd) + : arangodb::graph::BaseStep{0, 1, weight} { + _id = id; + _isLooseEnd = isLooseEnd; + }; + + ~Step() = default; + + Step(Step const& other) = default; + Step& operator=(Step const& other) = delete; + Step(Step&& other) = default; + Step& operator=(Step&& other) = default; + + bool operator==(Step const& other) { return _id == other._id; } + + bool isProcessable() const { return _isLooseEnd ? false : true; } + size_t id() const { return _id; } + std::string toString() { + return " _id: " + basics::StringUtils::itoa(static_cast(_id)) + + ", _weight: " + basics::StringUtils::ftoa(getWeight()); + } +}; + +class WeightedQueueTest : public ::testing::Test { + // protected: + public: + WeightedQueueTest() {} + ~WeightedQueueTest() {} + + public: + arangodb::GlobalResourceMonitor _global{}; + arangodb::ResourceMonitor _resourceMonitor{_global}; +}; + +TEST_F(WeightedQueueTest, it_should_be_empty_if_new_queue_initialized) { + auto queue = WeightedQueue(_resourceMonitor); + ASSERT_EQ(queue.size(), 0); + ASSERT_TRUE(queue.isEmpty()); +} + +TEST_F(WeightedQueueTest, it_should_contain_element_after_insertion) { + auto queue = WeightedQueue(_resourceMonitor); + auto step = Step{1, 1, false}; + queue.append(step); + ASSERT_EQ(queue.size(), 1); + ASSERT_FALSE(queue.isEmpty()); +} + +TEST_F(WeightedQueueTest, it_should_contain_zero_elements_after_clear) { + auto queue = WeightedQueue(_resourceMonitor); + queue.append(Step{1, 1, false}); + queue.append(Step{2, 4, false}); + queue.append(Step{3, 2, false}); + queue.append(Step{4, 3, true}); + ASSERT_EQ(queue.size(), 4); + queue.clear(); + ASSERT_TRUE(queue.isEmpty()); +} + +TEST_F(WeightedQueueTest, it_should_contain_processable_elements) { + auto queue = WeightedQueue(_resourceMonitor); + queue.append(Step{1, 5, false}); + queue.append(Step{2, 1, false}); + queue.append(Step{3, 2, true}); + queue.append(Step{4, 1.6, false}); + ASSERT_EQ(queue.size(), 4); + ASSERT_TRUE(queue.hasProcessableElement()); +} + +TEST_F(WeightedQueueTest, it_should_not_contain_processable_elements) { + auto queue = WeightedQueue(_resourceMonitor); + queue.append(Step{1, 4, true}); + queue.append(Step{2, 1.6, true}); + queue.append(Step{3, 1.2, true}); + queue.append(Step{4, 1.5, true}); + ASSERT_EQ(queue.size(), 4); + ASSERT_FALSE(queue.hasProcessableElement()); +} + +TEST_F(WeightedQueueTest, it_should_prioritize_processable_elements) { + // 2 and 3 have identical and smallest weight. + // 3 is processable, 2 not. + auto queue = WeightedQueue(_resourceMonitor); + queue.append(Step{1, 8, true}); + queue.append(Step{2, 2, true}); + queue.append(Step{3, 2, false}); + queue.append(Step{4, 6, false}); + EXPECT_EQ(queue.size(), 4); + EXPECT_TRUE(queue.hasProcessableElement()); + auto s = queue.pop(); + EXPECT_EQ(s.id(), 3); + EXPECT_FALSE(queue.hasProcessableElement()); + EXPECT_EQ(queue.size(), 3); +} + +TEST_F(WeightedQueueTest, it_should_order_by_asc_weight) { + // Random Input in random order. We shuffle it before each iteration, feel + // free to modify this in any way you like + std::vector input{{1, 1, false}, {2, 4, false}, {3, 6, false}, {4, 12, false}}; + // Some test orderings, feel free to add more orderings for tests. + std::vector>> orderings{ + {"DescWeight", + [](Step const& a, Step const& b) -> bool { + // DESC weight + return a.getWeight() > b.getWeight(); + }}, + {"AscWeight", + [](Step const& a, Step const& b) -> bool { + // ASC weight + return a.getWeight() < b.getWeight(); + }}, + {"RandomOrder", [](Step const& a, Step const& b) -> bool { + // RandomWeightOrder, first inject all uneven Steps, sort each "package" by ASC weight + // There is no specicial plan behind this, it is to stable "non"-sort by weight + auto modA = a.id() % 2; + auto modB = a.id() % 2; + if (modA != modB) { + return modA > modB; + } + return a.getWeight() < b.getWeight(); + }}}; + // No matter how the input is ordered, + // We need to get it back in exactly the same order, by asc weight + for (auto [name, o] : orderings) { + SCOPED_TRACE("Input ordered by " + name); + std::sort(input.begin(), input.end(), o); + auto queue = WeightedQueue(_resourceMonitor); + for (auto s : input) { + queue.append(s); + } + // We start with all inputs injected. + EXPECT_EQ(queue.size(), input.size()); + // Input is required + EXPECT_TRUE(queue.hasProcessableElement()); + // Smaller than anything + double weightBefore = -1.0; + + // Consume everything from the queue. + // It needs to be in increasing weight order. + while (queue.hasProcessableElement()) { + Step myStep = queue.pop(); + EXPECT_GE(myStep.getWeight(), weightBefore); + weightBefore = myStep.getWeight(); + } + // As all inputs are processable this queue shall be empty now. + ASSERT_EQ(queue.size(), 0); + ASSERT_FALSE(queue.hasProcessableElement()); + } +} + +TEST_F(WeightedQueueTest, it_should_pop_all_loose_ends) { + auto queue = WeightedQueue(_resourceMonitor); + queue.append(Step{2, 1.5, true}); + queue.append(Step{3, 5, true}); + queue.append(Step{1, 1, true}); + queue.append(Step{4, 6, true}); + EXPECT_EQ(queue.size(), 4); + EXPECT_FALSE(queue.hasProcessableElement()); + + std::vector myStepReferences = queue.getLooseEnds(); + EXPECT_EQ(myStepReferences.size(), 4); + + EXPECT_EQ(queue.size(), 4); + EXPECT_FALSE(queue.hasProcessableElement()); +} + +} // namespace queue_graph_cache_test +} // namespace tests +} // namespace arangodb diff --git a/tests/Mocks/MockGraphProvider.cpp b/tests/Mocks/MockGraphProvider.cpp index fc08796fcb5f..8ec9f1b7552f 100644 --- a/tests/Mocks/MockGraphProvider.cpp +++ b/tests/Mocks/MockGraphProvider.cpp @@ -95,9 +95,10 @@ auto MockGraphProvider::decideProcessable() const -> bool { } } -auto MockGraphProvider::startVertex(VertexType v, size_t depth) -> Step { +auto MockGraphProvider::startVertex(VertexType v, size_t depth, double weight) -> Step { LOG_TOPIC("78156", TRACE, Logger::GRAPHS) << " Start Vertex:" << v; + TRI_ASSERT(weight == 0.0); // Not handled yet return Step(v, decideProcessable()); } diff --git a/tests/Mocks/MockGraphProvider.h b/tests/Mocks/MockGraphProvider.h index 1fac5ecf3b31..4297fd713eb2 100644 --- a/tests/Mocks/MockGraphProvider.h +++ b/tests/Mocks/MockGraphProvider.h @@ -146,9 +146,7 @@ class MockGraphProvider { } } - bool isResponsible(transaction::Methods* trx) const { - return true; - } + bool isResponsible(transaction::Methods* trx) const { return true; } Vertex getVertex() const { /*if (!isProcessable()) { @@ -188,9 +186,7 @@ class MockGraphProvider { return _localSchreierIndex != std::numeric_limits::max(); } - std::size_t getLocalSchreierIndex() const { - return _localSchreierIndex; - } + std::size_t getLocalSchreierIndex() const { return _localSchreierIndex; } bool isProcessable() const { return _isProcessable; } @@ -222,7 +218,7 @@ class MockGraphProvider { MockGraphProvider& operator=(MockGraphProvider&&) = default; void destroyEngines(){}; - auto startVertex(VertexType vertex, size_t depth = 0) -> Step; + auto startVertex(VertexType vertex, size_t depth = 0, double weight = 0.0) -> Step; auto fetch(std::vector const& looseEnds) -> futures::Future>; auto expand(Step const& from, size_t previous) -> std::vector; auto expand(Step const& from, size_t previous, std::function callback) -> void; From a81e7737170bb974093769aaf2735a30d09929a0 Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Wed, 4 Aug 2021 08:26:13 +0200 Subject: [PATCH 19/72] Feature/hybrid smart graph include disjoint (#14578) --- arangod/Aql/ClusterNodes.cpp | 43 ++- arangod/Aql/ClusterNodes.h | 16 +- arangod/Aql/DistributeExecutor.cpp | 81 ++++- arangod/Aql/DistributeExecutor.h | 17 +- arangod/Aql/GraphNode.cpp | 12 + arangod/Aql/OptimizerRules.cpp | 357 ++++++++++--------- arangod/Aql/QuerySnippet.cpp | 48 ++- arangod/Aql/RestAqlHandler.cpp | 14 +- arangod/Aql/TraversalNode.cpp | 3 + arangod/Cluster/ClusterMethods.cpp | 2 +- arangod/Graph/BreadthFirstEnumerator.cpp | 21 +- arangod/Graph/BreadthFirstEnumerator.h | 19 +- arangod/Graph/Graph.cpp | 4 +- arangod/Graph/Graph.h | 4 +- arangod/Graph/GraphManager.cpp | 83 ++--- arangod/Graph/GraphManager.h | 13 +- arangod/Graph/PathEnumerator.cpp | 12 + arangod/Graph/PathEnumerator.h | 1 + arangod/Graph/TraverserOptions.cpp | 24 +- arangod/Graph/TraverserOptions.h | 7 + arangod/Graph/WeightedEnumerator.cpp | 11 + arangod/Graph/WeightedEnumerator.h | 4 +- arangod/StorageEngine/PhysicalCollection.cpp | 2 +- arangod/Transaction/Helpers.cpp | 9 + arangod/Transaction/Helpers.h | 3 + arangod/VocBase/LogicalCollection.cpp | 23 +- arangod/VocBase/LogicalCollection.h | 12 + 27 files changed, 558 insertions(+), 287 deletions(-) diff --git a/arangod/Aql/ClusterNodes.cpp b/arangod/Aql/ClusterNodes.cpp index 91eb7f96e494..fd6daf61a53f 100644 --- a/arangod/Aql/ClusterNodes.cpp +++ b/arangod/Aql/ClusterNodes.cpp @@ -263,8 +263,20 @@ DistributeNode::DistributeNode(ExecutionPlan* plan, ExecutionNodeId id, DistributeNode::DistributeNode(ExecutionPlan* plan, arangodb::velocypack::Slice const& base) : ScatterNode(plan, base), CollectionAccessingNode(plan, base), - _variable(Variable::varFromVPack(plan->getAst(), base, "variable")) {} - + _variable(Variable::varFromVPack(plan->getAst(), base, "variable")) { + auto sats = base.get("satelliteCollections"); + if (sats.isArray()) { + auto& queryCols = plan->getAst()->query().collections(); + _satellites.reserve(sats.length()); + + for (VPackSlice it : VPackArrayIterator(sats)) { + std::string v = arangodb::basics::VelocyPackHelper::getStringValue(it, ""); + auto c = queryCols.add(v, AccessMode::Type::READ, aql::Collection::Hint::Collection); + addSatellite(c); + } + } +} + /// @brief clone ExecutionNode recursively ExecutionNode* DistributeNode::clone(ExecutionPlan* plan, bool withDependencies, bool withProperties) const { @@ -272,6 +284,10 @@ ExecutionNode* DistributeNode::clone(ExecutionPlan* plan, bool withDependencies, collection(), _variable, _targetNodeId); c->copyClients(clients()); CollectionAccessingNode::cloneInto(*c); + c->_satellites.reserve(_satellites.size()); + for (auto& it : _satellites) { + c->_satellites.emplace_back(it); + } return cloneHelper(std::move(c), withDependencies, withProperties); } @@ -294,8 +310,8 @@ std::unique_ptr DistributeNode::createBlock( auto inRegs = RegIdSet{regId}; auto registerInfos = createRegisterInfos(inRegs, {}); - auto infos = DistributeExecutorInfos(clients(), collection(), - regId, getScatterType()); + auto infos = DistributeExecutorInfos(clients(), collection(), regId, + getScatterType(), getSatellites()); return std::make_unique>(&engine, this, std::move(registerInfos), @@ -311,6 +327,19 @@ void DistributeNode::doToVelocyPack(VPackBuilder& builder, unsigned flags) const builder.add(VPackValue("variable")); _variable->toVelocyPack(builder); + + if (!_satellites.empty()) { + builder.add(VPackValue("satelliteCollections")); + { + VPackArrayBuilder guard(&builder); + for (auto const& v : _satellites) { + // if the mapped shard for a collection is empty, it means that + // we have a vertex collection that is only relevant on some of the + // target servers + builder.add(VPackValue(v->name())); + } + } + } } void DistributeNode::replaceVariables(std::unordered_map const& replacements) { @@ -329,6 +358,12 @@ CostEstimate DistributeNode::estimateCost() const { return estimate; } +void DistributeNode::addSatellite(aql::Collection* satellite) { + // Only relevant for enterprise disjoint smart graphs + TRI_ASSERT(satellite->isSatellite()); + _satellites.emplace_back(satellite); +} + /*static*/ Collection const* GatherNode::findCollection(GatherNode const& root) noexcept { ExecutionNode const* node = root.getFirstDependency(); diff --git a/arangod/Aql/ClusterNodes.h b/arangod/Aql/ClusterNodes.h index 980fd7ddc4dc..c6ece35ce926 100644 --- a/arangod/Aql/ClusterNodes.h +++ b/arangod/Aql/ClusterNodes.h @@ -222,11 +222,17 @@ class DistributeNode final : public ScatterNode, public CollectionAccessingNode CostEstimate estimateCost() const override final; Variable const* getVariable() const noexcept { return _variable; } - + void setVariable(Variable const* var) noexcept { _variable = var; } - + ExecutionNodeId getTargetNodeId() const noexcept { return _targetNodeId; } + void addSatellite(aql::Collection*); + + std::vector const getSatellites() const noexcept { + return _satellites; + } + protected: /// @brief export to VelocyPack void doToVelocyPack(arangodb::velocypack::Builder&, unsigned flags) const override final; @@ -234,9 +240,13 @@ class DistributeNode final : public ScatterNode, public CollectionAccessingNode private: /// @brief the variable we must inspect to know where to distribute Variable const* _variable; - + /// @brief the id of the target ExecutionNode this DistributeNode belongs to. ExecutionNodeId _targetNodeId; + + /// @brief List of Satellite collections this node needs to distribute data to + /// in a satellite manner. + std::vector _satellites; }; /// @brief class GatherNode diff --git a/arangod/Aql/DistributeExecutor.cpp b/arangod/Aql/DistributeExecutor.cpp index ee8b145b88a6..ecff9b0b63bb 100644 --- a/arangod/Aql/DistributeExecutor.cpp +++ b/arangod/Aql/DistributeExecutor.cpp @@ -41,14 +41,16 @@ using namespace arangodb; using namespace arangodb::aql; -DistributeExecutorInfos::DistributeExecutorInfos( - std::vector clientIds, Collection const* collection, - RegisterId regId, ScatterNode::ScatterType type) +DistributeExecutorInfos::DistributeExecutorInfos(std::vector clientIds, + Collection const* collection, RegisterId regId, + ScatterNode::ScatterType type, + std::vector satellites) : ClientsExecutorInfos(std::move(clientIds)), _regId(regId), _collection(collection), _logCol(collection->getCollection()), - _type(type) {} + _type(type), + _satellites(std::move(satellites)) {} auto DistributeExecutorInfos::registerId() const noexcept -> RegisterId { TRI_ASSERT(_regId.isValid()); @@ -77,6 +79,35 @@ auto DistributeExecutorInfos::getResponsibleClient(arangodb::velocypack::Slice v return shardId; } +auto DistributeExecutorInfos::shouldDistributeToAll(arangodb::velocypack::Slice value) const + -> bool { + if (_satellites.empty()) { + // We can only distribute to all on Satellite Collections + return false; + } + auto id = value.get(StaticStrings::IdString); + if (!id.isString()) { + // We can only distribute to all if we can detect the collection name + return false; + } + + // NOTE: Copy Paste code, shall be unified + VPackStringRef vid(id); + size_t pos = vid.find('/'); + if (pos == std::string::npos) { + // Invalid input. Let the sharding take care of it, one server shall complain + return false; + } + vid = vid.substr(0, pos); + for (auto const& it : _satellites) { + if (vid.equals(it->name())) { + // This vertex is from a satellite collection start everywhere! + return true; + } + } + return false; +} + DistributeExecutor::DistributeExecutor(DistributeExecutorInfos const& infos) : _infos(infos) {} @@ -94,11 +125,25 @@ auto DistributeExecutor::distributeBlock(SharedAqlItemBlockPtr const& block, Ski choosenMap[key].emplace_back(i); } } else { - auto client = getClient(block, i); - if (!client.empty()) { - // We can only have clients we are prepared for - TRI_ASSERT(blockMap.find(client) != blockMap.end()); - choosenMap[client].emplace_back(i); + auto input = extractInput(block, i); + if (!input.isNone()) { + // NONE is ignored. + // Object is processd + // All others throw. + TRI_ASSERT(input.isObject()); + if (_infos.shouldDistributeToAll(input)) { + // This input should be added to all clients + for (auto const& [key, value] : blockMap) { + choosenMap[key].emplace_back(i); + } + } else { + auto client = getClient(input); + if (!client.empty()) { + // We can only have clients we are prepared for + TRI_ASSERT(blockMap.find(client) != blockMap.end()); + choosenMap[client].emplace_back(i); + } + } } } } @@ -123,19 +168,24 @@ auto DistributeExecutor::distributeBlock(SharedAqlItemBlockPtr const& block, Ski } } -auto DistributeExecutor::getClient(SharedAqlItemBlockPtr const& block, size_t rowIndex) const - -> std::string { +auto DistributeExecutor::extractInput(SharedAqlItemBlockPtr const& block, + size_t rowIndex) const -> VPackSlice { // check first input register AqlValue val = InputAqlItemRow{block, rowIndex}.getValue(_infos.registerId()); VPackSlice input = val.slice(); // will throw when wrong type if (input.isNone()) { - return {}; + return input; } - + if (!input.isObject()) { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "DistributeExecutor requires an object as input"); + THROW_ARANGO_EXCEPTION_MESSAGE( + TRI_ERROR_INTERNAL, "DistributeExecutor requires an object as input"); } + return input; +} + +auto DistributeExecutor::getClient(VPackSlice input) const -> std::string { auto res = _infos.getResponsibleClient(input); THROW_ARANGO_EXCEPTION_IF_FAIL(res.result()); return res.get(); @@ -145,4 +195,5 @@ ExecutionBlockImpl::ExecutionBlockImpl(ExecutionEngine* engi DistributeNode const* node, RegisterInfos registerInfos, DistributeExecutorInfos&& executorInfos) - : BlocksWithClientsImpl(engine, node, std::move(registerInfos), std::move(executorInfos)) {} + : BlocksWithClientsImpl(engine, node, std::move(registerInfos), + std::move(executorInfos)) {} diff --git a/arangod/Aql/DistributeExecutor.h b/arangod/Aql/DistributeExecutor.h index 9d13dca7efd3..bfbb91c20947 100644 --- a/arangod/Aql/DistributeExecutor.h +++ b/arangod/Aql/DistributeExecutor.h @@ -43,7 +43,8 @@ class DistributeNode; class DistributeExecutorInfos : public ClientsExecutorInfos { public: DistributeExecutorInfos(std::vector clientIds, Collection const* collection, - RegisterId regId, ScatterNode::ScatterType type); + RegisterId regId, ScatterNode::ScatterType type, + std::vector satellites); auto registerId() const noexcept -> RegisterId; auto scatterType() const noexcept -> ScatterNode::ScatterType; @@ -51,6 +52,8 @@ class DistributeExecutorInfos : public ClientsExecutorInfos { auto getResponsibleClient(arangodb::velocypack::Slice value) const -> ResultT; + auto shouldDistributeToAll(arangodb::velocypack::Slice value) const -> bool; + private: RegisterId _regId; @@ -63,6 +66,9 @@ class DistributeExecutorInfos : public ClientsExecutorInfos { /// @brief type of distribution that this nodes follows. ScatterNode::ScatterType _type; + + /// @brief list of collections that should be used + std::vector _satellites; }; // The DistributeBlock is actually implemented by specializing @@ -93,16 +99,13 @@ class DistributeExecutor { private: /** * @brief Compute which client needs to get this row - * NOTE: Has SideEffects - * If the input value does not contain an object, it is modified inplace with - * a new Object containing a key value! - * Hence this method is not const ;( - * * @param block The input block * @param rowIndex * @return std::string Identifier used by the client */ - auto getClient(SharedAqlItemBlockPtr const& block, size_t rowIndex) const -> std::string; + auto extractInput(SharedAqlItemBlockPtr const& block, size_t rowIndex) const + -> velocypack::Slice; + auto getClient(velocypack::Slice input) const -> std::string; private: DistributeExecutorInfos const& _infos; diff --git a/arangod/Aql/GraphNode.cpp b/arangod/Aql/GraphNode.cpp index 27bdf8e3e76d..229abab21b89 100644 --- a/arangod/Aql/GraphNode.cpp +++ b/arangod/Aql/GraphNode.cpp @@ -688,6 +688,18 @@ void GraphNode::getConditionVariables(std::vector& res) const { Collection const* GraphNode::collection() const { TRI_ASSERT(ServerState::instance()->isCoordinator()); TRI_ASSERT(!_edgeColls.empty()); + for (auto const* c : _edgeColls) { + // We are required to valuate non-satellites above + // satellites, as the collection is used as the protoype + // for this graphs sharding. + // The Satellite collection does not have sharding. + TRI_ASSERT(c != nullptr); + if (!c->isSatellite()) { + return c; + } + } + // We have not found any non-satellite Collection + // just return the first satellite then. TRI_ASSERT(_edgeColls.front() != nullptr); return _edgeColls.front(); } diff --git a/arangod/Aql/OptimizerRules.cpp b/arangod/Aql/OptimizerRules.cpp index b8cc85218451..e0cfb571f758 100644 --- a/arangod/Aql/OptimizerRules.cpp +++ b/arangod/Aql/OptimizerRules.cpp @@ -828,14 +828,14 @@ using EN = arangodb::aql::ExecutionNode; namespace arangodb { namespace aql { -// checks if the path variable (variable) can be optimized away, or restricted to some -// attributes (vertices, edges, weights) -bool optimizeTraversalPathVariable(Variable const* variable, TraversalNode* traversal, +// checks if the path variable (variable) can be optimized away, or restricted +// to some attributes (vertices, edges, weights) +bool optimizeTraversalPathVariable(Variable const* variable, TraversalNode* traversal, std::vector const& pruneVars) { if (variable == nullptr) { return false; } - + auto* options = static_cast(traversal->options()); @@ -866,10 +866,11 @@ bool optimizeTraversalPathVariable(Variable const* variable, TraversalNode* trav current->getVariablesUsedHere(vars); if (vars.find(variable) != vars.end()) { // path variable used here - Expression* exp = ExecutionNode::castTo(current)->expression(); + Expression* exp = + ExecutionNode::castTo(current)->expression(); AstNode const* node = exp->node(); if (!Ast::getReferencedAttributes(node, variable, attributes)) { - // full path variable is used, or accessed in a way that we don't + // full path variable is used, or accessed in a way that we don't // understand, e.g. "p" or "p[0]" or "p[*]..." canOptimize = false; } @@ -877,8 +878,8 @@ bool optimizeTraversalPathVariable(Variable const* variable, TraversalNode* trav break; } default: { - // if the path is used by any other node type, we don't know what to do - // and will not optimize parts of it away + // if the path is used by any other node type, we don't know what to + // do and will not optimize parts of it away vars.clear(); current->getVariablesUsedHere(vars); if (vars.find(variable) != vars.end()) { @@ -892,14 +893,17 @@ bool optimizeTraversalPathVariable(Variable const* variable, TraversalNode* trav if (canOptimize) { // check which attributes from the path are actually used - bool producePathsVertices = (attributes.find(StaticStrings::GraphQueryVertices) != attributes.end()); - bool producePathsEdges = (attributes.find(StaticStrings::GraphQueryEdges) != attributes.end()); - bool producePathsWeights = (attributes.find(StaticStrings::GraphQueryWeights) != attributes.end()) && - (options->mode == traverser::TraverserOptions::Order::WEIGHTED); + bool producePathsVertices = + (attributes.find(StaticStrings::GraphQueryVertices) != attributes.end()); + bool producePathsEdges = + (attributes.find(StaticStrings::GraphQueryEdges) != attributes.end()); + bool producePathsWeights = + (attributes.find(StaticStrings::GraphQueryWeights) != attributes.end()) && + (options->mode == traverser::TraverserOptions::Order::WEIGHTED); if (!producePathsVertices && !producePathsEdges && !producePathsWeights && !attributes.empty()) { - // none of the existing path attributes is actually accessed - but a different + // none of the existing path attributes is actually accessed - but a different // (non-existing) attribute is accessed, e.g. `p.whatever`. // in order to not optimize away our path variable, and then being unable to access // the non-existing attribute, we simply activate the production of vertices. @@ -918,11 +922,13 @@ bool optimizeTraversalPathVariable(Variable const* variable, TraversalNode* trav return false; /*modified*/ } -Collection* addCollectionToQuery(QueryContext& query, std::string const& cname, char const* context) { +Collection* addCollectionToQuery(QueryContext& query, std::string const& cname, + char const* context) { aql::Collection* coll = nullptr; if (!cname.empty()) { - coll = query.collections().add(cname, AccessMode::Type::READ, aql::Collection::Hint::Collection); + coll = query.collections().add(cname, AccessMode::Type::READ, + aql::Collection::Hint::Collection); // simon: code below is used for FULLTEXT(), WITHIN(), NEAR(), .. // could become unnecessary if the AST takes care of adding the collections if (!ServerState::instance()->isCoordinator()) { @@ -932,9 +938,9 @@ Collection* addCollectionToQuery(QueryContext& query, std::string const& cname, } if (coll == nullptr) { - THROW_ARANGO_EXCEPTION_MESSAGE( - TRI_ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH, - std::string("collection '") + cname + "' used in " + context + " not found"); + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH, + std::string("collection '") + cname + + "' used in " + context + " not found"); } return coll; @@ -1056,10 +1062,12 @@ void arangodb::aql::sortInValuesRule(Optimizer* opt, std::unique_ptrtype == NODE_TYPE_FCALL && - (static_cast(testNode->getData())->name == "SORTED_UNIQUE" || - static_cast(testNode->getData())->name == "SORTED")) { - // we don't need to sort results of a function that already returns sorted - // results + (static_cast(testNode->getData())->name == + "SORTED_UNIQUE" || + static_cast(testNode->getData())->name == + "SORTED")) { + // we don't need to sort results of a function that already returns + // sorted results AstNode* clone = ast->shallowCopyForModify(inNode); TRI_DEFER(FINALIZE_SUBTREE(clone)); @@ -1445,8 +1453,7 @@ void arangodb::aql::removeCollectVariablesRule(Optimizer* opt, } // end - if collectNode has outVariable collectNode->clearAggregates( - [&varsUsedLater, &modified]( - AggregateVarInfo const& aggregate) -> bool { + [&varsUsedLater, &modified](AggregateVarInfo const& aggregate) -> bool { if (varsUsedLater.find(aggregate.outVar) == varsUsedLater.end()) { // result of aggregate function not used later modified = true; @@ -1952,8 +1959,7 @@ void arangodb::aql::specializeCollectRule(Optimizer* opt, // test if we can use an alternative version of COLLECT with a hash table bool const canUseHashAggregation = - (!groupVariables.empty() && - !collectNode->hasOutVariable() && + (!groupVariables.empty() && !collectNode->hasOutVariable() && collectNode->getOptions().canUseMethod(CollectOptions::CollectMethod::HASH)); if (canUseHashAggregation && !opt->runOnlyRequiredRules(1)) { @@ -2029,8 +2035,7 @@ void arangodb::aql::specializeCollectRule(Optimizer* opt, // no need to run this specific rule again on the cloned plan opt->addPlan(std::move(newPlan), rule, true); } - } else if (groupVariables.empty() && - collectNode->hasOutVariable() == false && + } else if (groupVariables.empty() && collectNode->hasOutVariable() == false && collectNode->aggregateVariables().size() == 1 && collectNode->aggregateVariables()[0].type == "LENGTH") { // we have no groups and only a single aggregator of type LENGTH, so we @@ -2157,10 +2162,10 @@ void arangodb::aql::moveFiltersUpRule(Optimizer* opt, std::unique_ptraddPlan(std::move(plan), rule, modified); } -struct VariableReplacer final : public WalkerWorker { +struct VariableReplacer final + : public WalkerWorker { public: - explicit VariableReplacer( - std::unordered_map const& replacements) + explicit VariableReplacer(std::unordered_map const& replacements) : replacements(replacements) {} bool before(ExecutionNode* en) override final { @@ -2377,7 +2382,8 @@ void arangodb::aql::simplifyConditionsRule(Optimizer* opt, for (auto const& n : nodes) { auto nn = ExecutionNode::castTo(n); - if (!nn->expression()->isDeterministic() || nn->outVariable()->type() == Variable::Type::Const) { + if (!nn->expression()->isDeterministic() || + nn->outVariable()->type() == Variable::Type::Const) { // If this node is non-deterministic or has a constant expression, we must not touch it! continue; } @@ -2773,9 +2779,9 @@ void arangodb::aql::removeUnnecessaryCalculationsRule(Optimizer* opt, auto nLoop = n->getLoop(); if (nLoop != otherLoop) { - // original expression calls a function and is not contained in a loop. - // we're about to move this expression into a loop, but we don't want - // to move (expensive) function calls into loops + // original expression calls a function and is not contained in a + // loop. we're about to move this expression into a loop, but we + // don't want to move (expensive) function calls into loops continue; } VarSet outer = nLoop->getVarsValid(); @@ -2997,10 +3003,11 @@ struct SortToIndexNode final Collection const* coll = enumerateCollectionNode->collection(); TRI_ASSERT(coll != nullptr); - size_t numDocs = coll->count(&_plan->getAst()->query().trxForOptimization(), transaction::CountType::TryCache); + size_t numDocs = coll->count(&_plan->getAst()->query().trxForOptimization(), + transaction::CountType::TryCache); - bool canBeUsed = arangodb::aql::utils::getIndexForSortCondition(*coll, - &sortCondition, outVariable, numDocs, + bool canBeUsed = arangodb::aql::utils::getIndexForSortCondition( + *coll, &sortCondition, outVariable, numDocs, enumerateCollectionNode->hint(), usedIndexes, coveredAttributes); if (canBeUsed) { // If this bit is set, then usedIndexes has length exactly one @@ -3840,13 +3847,14 @@ void arangodb::aql::scatterInClusterRule(Optimizer* opt, std::unique_ptr DistributeNode* { - auto collection = static_cast(nullptr); - auto inputVariable = static_cast(nullptr); - + auto collection = static_cast(nullptr); + auto inputVariable = static_cast(nullptr); + + bool isGraphNode = false; // TODO: this seems a bit verbose, but is at least local & simple - // the modification nodes are all collectionaccessing, the graph nodes are - // currently assumed to be disjoint, and hence smart, so all collections - // are sharded the same way! + // the modification nodes are all collectionaccessing, the graph nodes + // are currently assumed to be disjoint, and hence smart, so all + // collections are sharded the same way! switch (node->getType()) { case ExecutionNode::INSERT: { auto const* insertNode = ExecutionNode::castTo(node); @@ -3879,6 +3887,7 @@ auto arangodb::aql::createDistributeNodeFor(ExecutionPlan& plan, ExecutionNode* TRI_ASSERT(traversalNode->isDisjoint()); collection = traversalNode->collection(); inputVariable = traversalNode->inVariable(); + isGraphNode = true; } break; case ExecutionNode::K_SHORTEST_PATHS: { auto kShortestPathsNode = ExecutionNode::castTo(node); @@ -3886,12 +3895,14 @@ auto arangodb::aql::createDistributeNodeFor(ExecutionPlan& plan, ExecutionNode* collection = kShortestPathsNode->collection(); // Subtle: KShortestPathsNode uses a reference when returning startInVariable inputVariable = &kShortestPathsNode->startInVariable(); + isGraphNode = true; } break; case ExecutionNode::SHORTEST_PATH: { auto shortestPathNode = ExecutionNode::castTo(node); TRI_ASSERT(shortestPathNode->isDisjoint()); collection = shortestPathNode->collection(); inputVariable = shortestPathNode->startInVariable(); + isGraphNode = true; } break; default: { TRI_ASSERT(false); @@ -3903,16 +3914,28 @@ auto arangodb::aql::createDistributeNodeFor(ExecutionPlan& plan, ExecutionNode* TRI_ASSERT(collection != nullptr); TRI_ASSERT(inputVariable != nullptr); - + // The DistributeNode needs specially prepared input, but we do not want to insert the // calculation for that just yet, because it would interfere with some optimizations, // in particular those that might completely remove the DistributeNode (which would) // also render the calculation pointless. So instead we insert this calculation in a // post-processing step when finalizing the plan in the Optimizer. - auto distNode = plan.createNode(&plan, plan.nextId(), - ScatterNode::ScatterType::SHARD, - collection, inputVariable, node->id()); + auto distNode = + plan.createNode(&plan, plan.nextId(), ScatterNode::ScatterType::SHARD, + collection, inputVariable, node->id()); + if (isGraphNode) { +#ifdef USE_ENTERPRISE + // Only relevant for Disjoint Smart Graphs that can only be part of the Enterprise version + auto graphNode = ExecutionNode::castTo(node); + auto vertices = graphNode->vertexColls(); + for (auto const& it : vertices) { + if (it->isSatellite()) { + distNode->addSatellite(it); + } + } +#endif + } TRI_ASSERT(distNode != nullptr); return distNode; } @@ -3949,9 +3972,8 @@ auto arangodb::aql::createGatherNodeFor(ExecutionPlan& plan, DistributeNode* nod // and we handle this case in here as well by resetting the root to the // inserted GATHER node. // -auto arangodb::aql::insertDistributeGatherSnippet(ExecutionPlan& plan, - ExecutionNode* at, SubqueryNode* snode) - -> DistributeNode* { +auto arangodb::aql::insertDistributeGatherSnippet(ExecutionPlan& plan, ExecutionNode* at, + SubqueryNode* snode) -> DistributeNode* { auto const parents = at->getParents(); auto const deps = at->getDependencies(); @@ -4011,7 +4033,7 @@ auto arangodb::aql::insertDistributeGatherSnippet(ExecutionPlan& plan, } auto extractSmartnessAndCollection(ExecutionNode* node) - -> std::tuple { + -> std::tuple { auto nodeType = node->getType(); auto collection = static_cast(nullptr); auto isSmart = bool{false}; @@ -4047,10 +4069,9 @@ auto extractSmartnessAndCollection(ExecutionNode* node) /// /// it will change plans in place - auto isGraphNode(ExecutionNode::NodeType nodeType) noexcept -> bool { return nodeType == ExecutionNode::TRAVERSAL || nodeType == ExecutionNode::SHORTEST_PATH || - nodeType == ExecutionNode::K_SHORTEST_PATHS; + nodeType == ExecutionNode::K_SHORTEST_PATHS; } auto isModificationNode(ExecutionNode::NodeType nodeType) noexcept -> bool { @@ -4230,40 +4251,36 @@ void arangodb::aql::collectInClusterRule(Optimizer* opt, std::unique_ptrgetType()) { + switch (p->getType()) { case ExecutionNode::REMOTE: hasFoundMultipleShards = true; break; case ExecutionNode::ENUMERATE_COLLECTION: - case ExecutionNode::INDEX: - { - auto col = getCollection(p); - if (col->numberOfShards() > 1) { - hasFoundMultipleShards = true; - } + case ExecutionNode::INDEX: { + auto col = getCollection(p); + if (col->numberOfShards() > 1) { + hasFoundMultipleShards = true; } - break; + } break; case ExecutionNode::TRAVERSAL: hasFoundMultipleShards = true; break; - case ExecutionNode::ENUMERATE_IRESEARCH_VIEW: - { - auto& viewNode = *ExecutionNode::castTo(p); - auto collections = viewNode.collections(); - auto const collCount = collections.size(); - TRI_ASSERT(collCount > 0); - if (collCount > 1) { - hasFoundMultipleShards = true; - } else if (1 == collCount) { - hasFoundMultipleShards = collections.front().get().numberOfShards() > 1; - } + case ExecutionNode::ENUMERATE_IRESEARCH_VIEW: { + auto& viewNode = *ExecutionNode::castTo(p); + auto collections = viewNode.collections(); + auto const collCount = collections.size(); + TRI_ASSERT(collCount > 0); + if (collCount > 1) { + hasFoundMultipleShards = true; + } else if (1 == collCount) { + hasFoundMultipleShards = + collections.front().get().numberOfShards() > 1; } - break; + } break; default: break; } - if (hasFoundMultipleShards) { break; } @@ -4305,19 +4322,21 @@ void arangodb::aql::collectInClusterRule(Optimizer* opt, std::unique_ptraggregationMethod() == CollectOptions::CollectMethod::COUNT) { TRI_ASSERT(collectNode->aggregateVariables().size() == 1); TRI_ASSERT(collectNode->hasOutVariable() == false); - // clone a COLLECT AGGREGATE var=LENGTH(_) operation from the coordinator to the - // DB server(s), and leave an aggregate COLLECT node on the - // coordinator for total aggregation + // clone a COLLECT AGGREGATE var=LENGTH(_) operation from the + // coordinator to the DB server(s), and leave an aggregate COLLECT + // node on the coordinator for total aggregation // add a new CollectNode on the DB server to do the actual counting auto outVariable = plan->getAst()->variables()->createTemporaryVariable(); std::vector aggregateVariables; - aggregateVariables.emplace_back(AggregateVarInfo{outVariable, collectNode->aggregateVariables()[0].inVar, "LENGTH"}); + aggregateVariables.emplace_back( + AggregateVarInfo{outVariable, + collectNode->aggregateVariables()[0].inVar, + "LENGTH"}); auto dbCollectNode = new CollectNode(plan.get(), plan->nextId(), collectNode->getOptions(), - collectNode->groupVariables(), - aggregateVariables, nullptr, - nullptr, std::vector(), + collectNode->groupVariables(), aggregateVariables, + nullptr, nullptr, std::vector(), collectNode->variableMap(), false); plan->registerNode(dbCollectNode); @@ -4347,7 +4366,7 @@ void arangodb::aql::collectInClusterRule(Optimizer* opt, std::unique_ptrgetAst()->variables()->createTemporaryVariable(); std::vector const groupVariables{ - GroupVarInfo{out, groupVars[0].inVar}}; + GroupVarInfo{out, groupVars[0].inVar}}; auto dbCollectNode = new CollectNode(plan.get(), plan->nextId(), collectNode->getOptions(), @@ -4409,8 +4428,9 @@ void arangodb::aql::collectInClusterRule(Optimizer* opt, std::unique_ptrnextId(), collectNode->getOptions(), - outVars, dbServerAggVars, nullptr, nullptr, std::vector(), + new CollectNode(plan.get(), plan->nextId(), + collectNode->getOptions(), outVars, dbServerAggVars, + nullptr, nullptr, std::vector(), collectNode->variableMap(), false); plan->registerNode(dbCollectNode); @@ -4425,7 +4445,8 @@ void arangodb::aql::collectInClusterRule(Optimizer* opt, std::unique_ptrgroupVariables()) { // replace input variables - copy.emplace_back(GroupVarInfo{/*outVar*/it.outVar, /*inVar*/outVars[i].outVar}); + copy.emplace_back(GroupVarInfo{/*outVar*/ it.outVar, + /*inVar*/ outVars[i].outVar}); ++i; } collectNode->groupVariables(copy); @@ -5953,16 +5974,21 @@ void arangodb::aql::optimizeTraversalsRule(Optimizer* opt, modified |= optimizeTraversalPathVariable(outVariable, traversal, pruneVars); // check if we can make use of the optimized neighbors enumerator - if (!ServerState::instance()->isCoordinator()) { - if (traversal->vertexOutVariable() != nullptr && traversal->edgeOutVariable() == nullptr && - traversal->pathOutVariable() == nullptr && options->isUseBreadthFirst() && - options->uniqueVertices == arangodb::traverser::TraverserOptions::GLOBAL && - !options->usesPrune() && !options->hasDepthLookupInfo()) { - // this is possible in case *only* vertices are produced (no edges, no path), - // the traversal is breadth-first, the vertex uniqueness level is set to "global", - // there is no pruning and there are no depth-specific filters - options->useNeighbors = true; - modified = true; + if (!options->isDisjoint()) { + // Use NeighborsEnumerator optimization only in case we have do not + // have a (Hybrid)Disjoint SmartGraph + if (!ServerState::instance()->isCoordinator()) { + if (traversal->vertexOutVariable() != nullptr && + traversal->edgeOutVariable() == nullptr && + traversal->pathOutVariable() == nullptr && options->isUseBreadthFirst() && + options->uniqueVertices == arangodb::traverser::TraverserOptions::GLOBAL && + !options->usesPrune() && !options->hasDepthLookupInfo()) { + // this is possible in case *only* vertices are produced (no edges, no path), + // the traversal is breadth-first, the vertex uniqueness level is set to "global", + // there is no pruning and there are no depth-specific filters + options->useNeighbors = true; + modified = true; + } } } } @@ -6165,8 +6191,7 @@ void arangodb::aql::inlineSubqueriesRule(Optimizer* opt, std::unique_ptrgetType() == EN::WINDOW && - subqueryNode->isInInnerLoop()) { + if (current->getType() == EN::WINDOW && subqueryNode->isInInnerLoop()) { // WINDOW captures all existing rows in the scope, moving WINDOW // ends up with different rows captured eligible = false; @@ -6178,9 +6203,9 @@ void arangodb::aql::inlineSubqueriesRule(Optimizer* opt, std::unique_ptr(current)->hasOutVariable()) { // COLLECT ... INTO captures all existing variables in the scope. - // if we move the subquery from one scope into another, we will end up with - // different variables captured, so we must not apply the optimization in - // this case. + // if we move the subquery from one scope into another, we will end up + // with different variables captured, so we must not apply the + // optimization in this case. eligible = false; break; } @@ -7428,8 +7453,7 @@ struct ParallelizableFinder final bool _isParallelizable; explicit ParallelizableFinder(bool parallelizeWrites) - : _parallelizeWrites(parallelizeWrites), - _isParallelizable(true) {} + : _parallelizeWrites(parallelizeWrites), _isParallelizable(true) {} ~ParallelizableFinder() = default; @@ -7438,8 +7462,7 @@ struct ParallelizableFinder final } bool before(ExecutionNode* node) override final { - if (node->getType() == ExecutionNode::SCATTER || - node->getType() == ExecutionNode::GATHER || + if (node->getType() == ExecutionNode::SCATTER || node->getType() == ExecutionNode::GATHER || node->getType() == ExecutionNode::DISTRIBUTE) { _isParallelizable = false; return true; // true to abort the whole walking process @@ -7459,10 +7482,9 @@ struct ParallelizableFinder final // can be parallelized, provided the rest of the plan // does not prohibit this if (node->isModificationNode() && - (!_parallelizeWrites || - (node->getType() != ExecutionNode::REMOVE && - node->getType() != ExecutionNode::REPLACE && - node->getType() != ExecutionNode::UPDATE))) { + (!_parallelizeWrites || (node->getType() != ExecutionNode::REMOVE && + node->getType() != ExecutionNode::REPLACE && + node->getType() != ExecutionNode::UPDATE))) { _isParallelizable = false; return true; // true to abort the whole walking process } @@ -7488,11 +7510,10 @@ bool isParallelizable(GatherNode* node, bool parallelizeWrites) { } return true; } -} +} // namespace /// @brief turn LENGTH(FOR doc IN ...) subqueries into an optimized count operation -void arangodb::aql::optimizeCountRule(Optimizer* opt, - std::unique_ptr plan, +void arangodb::aql::optimizeCountRule(Optimizer* opt, std::unique_ptr plan, OptimizerRule const& rule) { bool modified = false; @@ -7550,7 +7571,9 @@ void arangodb::aql::optimizeCountRule(Optimizer* opt, auto it = localCandidates.find(setter); if (it == localCandidates.end()) { - localCandidates.emplace(setter, std::make_pair(true, std::unordered_set({node}))); + localCandidates.emplace( + setter, + std::make_pair(true, std::unordered_set({node}))); } else { (*it).second.second.emplace(node); } @@ -7588,8 +7611,8 @@ void arangodb::aql::optimizeCountRule(Optimizer* opt, } bool valid = true; - // check if subquery result is used somewhere else before the current calculation - // we are looking at + // check if subquery result is used somewhere else before the current + // calculation we are looking at auto current = sn->getFirstParent(); while (current != nullptr && current != n) { vars.clear(); @@ -7643,8 +7666,8 @@ void arangodb::aql::optimizeCountRule(Optimizer* opt, } // from here we need to find the first FOR loop. - // if it is a full collection scan or an index scan, we note its out variable. - // if we find a nested loop, we abort searching + // if it is a full collection scan or an index scan, we note its out + // variable. if we find a nested loop, we abort searching bool valid = true; ExecutionNode* found = nullptr; Variable const* outVariable = nullptr; @@ -7667,13 +7690,14 @@ void arangodb::aql::optimizeCountRule(Optimizer* opt, } else { outVariable = dynamic_cast(current)->outVariable(); - if (type == EN::INDEX && ExecutionNode::castTo(current)->getIndexes().size() != 1) { + if (type == EN::INDEX && + ExecutionNode::castTo(current)->getIndexes().size() != 1) { // more than one index, so we would need to run uniqueness checks on the // results. this is currently unsupported, so don't apply the optimization valid = false; } else { - // a FOR loop without an early pruning filter. this is what we are - // looking for! + // a FOR loop without an early pruning filter. this is what we + // are looking for! found = current; } } @@ -7704,7 +7728,7 @@ void arangodb::aql::optimizeCountRule(Optimizer* opt, break; } - case EN::RETURN:{ + case EN::RETURN: { // we reached the end break; } @@ -7808,7 +7832,8 @@ void arangodb::aql::parallelizeGatherRule(Optimizer* opt, if (nodes.size() == 1 && !plan->contains(EN::DISTRIBUTE) && !plan->contains(EN::SCATTER)) { TRI_vocbase_t& vocbase = plan->getAst()->query().vocbase(); - bool parallelizeWrites = vocbase.server().getFeature().parallelizeGatherWrites(); + bool parallelizeWrites = + vocbase.server().getFeature().parallelizeGatherWrites(); GatherNode* gn = ExecutionNode::castTo(nodes[0]); if (!gn->isInSubquery() && isParallelizable(gn, parallelizeWrites)) { @@ -7840,7 +7865,7 @@ void arangodb::aql::asyncPrefetchRule(Optimizer* opt, std::unique_ptrisModificationNode()) { containsModificationNode = true; - return true; // found a modification node -> abort + return true; // found a modification node -> abort } return false; } @@ -7848,7 +7873,7 @@ void arangodb::aql::asyncPrefetchRule(Optimizer* opt, std::unique_ptrroot()->walk(checker); - + if (!checker.containsModificationNode) { // here we only set a flag that this plan should use async prefetching. // The actual prefetching is performed on node level and therefore also @@ -7875,18 +7900,19 @@ void arangodb::aql::enableAsyncPrefetching(ExecutionPlan& plan) { void arangodb::aql::activateCallstackSplit(ExecutionPlan& plan) { if (willUseV8(plan)) { - // V8 requires thread local context configuration, so we cannot + // V8 requires thread local context configuration, so we cannot // use our thread based split solution... return; } - + auto const& options = plan.getAst()->query().queryOptions(); struct CallstackSplitter : WalkerWorkerBase { - explicit CallstackSplitter(size_t maxNodes) : maxNodesPerCallstack(maxNodes) {} + explicit CallstackSplitter(size_t maxNodes) + : maxNodesPerCallstack(maxNodes) {} bool before(ExecutionNode* n) override { // This rule must be executed after subquery splicing, so we must not see any subqueries here! TRI_ASSERT(n->getType() != EN::SUBQUERY); - + if (n->getType() == EN::REMOTE) { // RemoteNodes provide a natural split in the callstack, so we can reset the counter here! count = 0; @@ -7899,7 +7925,7 @@ void arangodb::aql::activateCallstackSplit(ExecutionPlan& plan) { size_t maxNodesPerCallstack; size_t count = 0; }; - + CallstackSplitter walker(options.maxNodesPerCallstack); plan.root()->walk(walker); } @@ -7945,7 +7971,7 @@ void findSubqueriesSuitableForSplicing(ExecutionPlan const& plan, } bool before(ExecutionNode* node) override final { - TRI_ASSERT(node->getType() != EN::MUTEX); // should never appear here + TRI_ASSERT(node->getType() != EN::MUTEX); // should never appear here if (node->getType() == ExecutionNode::SUBQUERY) { _result.emplace_back(ExecutionNode::castTo(node)); @@ -8191,15 +8217,15 @@ void arangodb::aql::insertDistributeInputCalculation(ExecutionPlan& plan) { ::arangodb::containers::SmallVector::allocator_type::arena_type a; ::arangodb::containers::SmallVector nodes{a}; plan.findNodesOfType(nodes, ExecutionNode::DISTRIBUTE, true); - + for (auto const& n : nodes) { auto* distributeNode = ExecutionNode::castTo(n); auto* targetNode = plan.getNodesById().at(distributeNode->getTargetNodeId()); TRI_ASSERT(targetNode != nullptr); auto collection = static_cast(nullptr); - auto inputVariable = static_cast(nullptr); - auto alternativeVariable = static_cast(nullptr); + auto inputVariable = static_cast(nullptr); + auto alternativeVariable = static_cast(nullptr); auto createKeys = bool{false}; auto allowKeyConversionToObject = bool{false}; @@ -8207,13 +8233,13 @@ void arangodb::aql::insertDistributeInputCalculation(ExecutionPlan& plan) { auto fixupGraphInput = bool{false}; - std::function setInVariable; + std::function setInVariable; bool ignoreErrors = false; // TODO: this seems a bit verbose, but is at least local & simple - // the modification nodes are all collectionaccessing, the graph nodes are - // currently assumed to be disjoint, and hence smart, so all collections - // are sharded the same way! + // the modification nodes are all collectionaccessing, the graph nodes + // are currently assumed to be disjoint, and hence smart, so all + // collections are sharded the same way! switch (targetNode->getType()) { case ExecutionNode::INSERT: { auto* insertNode = ExecutionNode::castTo(targetNode); @@ -8312,8 +8338,8 @@ void arangodb::aql::insertDistributeInputCalculation(ExecutionPlan& plan) { default: { TRI_ASSERT(false); THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, - "Cannot distribute " + - targetNode->getTypeString() + "."); + "Cannot distribute " + + targetNode->getTypeString() + "."); } break; } TRI_ASSERT(inputVariable != nullptr); @@ -8327,20 +8353,19 @@ void arangodb::aql::insertDistributeInputCalculation(ExecutionPlan& plan) { CalculationNode* calcNode = nullptr; auto setter = plan.getVarSetBy(inputVariable->id); - if (setter == nullptr || // this can happen for $smartHandOver - setter->getType() == EN::ENUMERATE_COLLECTION || - setter->getType() == EN::INDEX) { + if (setter == nullptr || // this can happen for $smartHandOver + setter->getType() == EN::ENUMERATE_COLLECTION || setter->getType() == EN::INDEX) { // If our input variable is set by a collection/index enumeration, it is guaranteed to be an object // with a _key attribute, so we don't need to do anything. return; } - + // We insert an additional calculation node to create the input for our distribute node. Variable* variable = plan.getAst()->variables()->createTemporaryVariable(); - + // update the targetNode so that it uses the same input variable as our distribute node setInVariable(variable); - + auto* ast = plan.getAst(); auto args = ast->createNodeArray(); char const* function; @@ -8357,38 +8382,40 @@ void arangodb::aql::insertDistributeInputCalculation(ExecutionPlan& plan) { } auto flags = ast->createNodeObject(); flags->addMember(ast->createNodeObjectElement( - TRI_CHAR_LENGTH_PAIR("allowSpecifiedKeys"), - ast->createNodeValueBool(allowSpecifiedKeys))); - flags->addMember(ast->createNodeObjectElement( - TRI_CHAR_LENGTH_PAIR("ignoreErrors"), - ast->createNodeValueBool(ignoreErrors))); + TRI_CHAR_LENGTH_PAIR("allowSpecifiedKeys"), + ast->createNodeValueBool(allowSpecifiedKeys))); + flags->addMember( + ast->createNodeObjectElement(TRI_CHAR_LENGTH_PAIR("ignoreErrors"), + ast->createNodeValueBool(ignoreErrors))); auto const& collectionName = collection->name(); flags->addMember(ast->createNodeObjectElement( - TRI_CHAR_LENGTH_PAIR("collection"), - ast->createNodeValueString(collectionName.c_str(), collectionName.length()))); - //args->addMember(ast->createNodeValueString(collectionName.c_str(), collectionName.length())); - + TRI_CHAR_LENGTH_PAIR("collection"), + ast->createNodeValueString(collectionName.c_str(), collectionName.length()))); + // args->addMember(ast->createNodeValueString(collectionName.c_str(), collectionName.length())); + args->addMember(flags); } else { function = "MAKE_DISTRIBUTE_INPUT"; auto flags = ast->createNodeObject(); flags->addMember(ast->createNodeObjectElement( - TRI_CHAR_LENGTH_PAIR("allowKeyConversionToObject"), - ast->createNodeValueBool(allowKeyConversionToObject))); + TRI_CHAR_LENGTH_PAIR("allowKeyConversionToObject"), + ast->createNodeValueBool(allowKeyConversionToObject))); + flags->addMember( + ast->createNodeObjectElement(TRI_CHAR_LENGTH_PAIR("ignoreErrors"), + ast->createNodeValueBool(ignoreErrors))); + bool canUseCustomKey = collection->getCollection()->usesDefaultShardKeys() || + allowSpecifiedKeys; flags->addMember(ast->createNodeObjectElement( - TRI_CHAR_LENGTH_PAIR("ignoreErrors"), - ast->createNodeValueBool(ignoreErrors))); - bool canUseCustomKey = collection->getCollection()->usesDefaultShardKeys() || allowSpecifiedKeys; - flags->addMember(ast->createNodeObjectElement( - TRI_CHAR_LENGTH_PAIR("canUseCustomKey"), - ast->createNodeValueBool(canUseCustomKey))); - + TRI_CHAR_LENGTH_PAIR("canUseCustomKey"), ast->createNodeValueBool(canUseCustomKey))); + args->addMember(flags); } } - auto expr = std::make_unique(ast, ast->createNodeFunctionCall(function, args, true)); - calcNode = plan.createNode(&plan, plan.nextId(), std::move(expr), variable); + auto expr = + std::make_unique(ast, ast->createNodeFunctionCall(function, args, true)); + calcNode = plan.createNode(&plan, plan.nextId(), + std::move(expr), variable); distributeNode->setVariable(variable); plan.insertBefore(distributeNode, calcNode); plan.clearVarUsageComputed(); diff --git a/arangod/Aql/QuerySnippet.cpp b/arangod/Aql/QuerySnippet.cpp index 12961b2249e1..d26f5a395aba 100644 --- a/arangod/Aql/QuerySnippet.cpp +++ b/arangod/Aql/QuerySnippet.cpp @@ -604,7 +604,7 @@ auto QuerySnippet::prepareFirstBranch( // Check whether `servers` is the leader for any of the shards of the // prototype collection. - // We want to instantiate this snippet here exactly iff this is the case. + // We want to instantiate this snippet here exactly if this is the case. auto needInstanceHere = std::invoke([&]() { auto const* const protoCol = localGraphNode->isUsedAsSatellite() @@ -631,7 +631,7 @@ auto QuerySnippet::prepareFirstBranch( } // This is either one shard or a single SatelliteGraph which is not used - // as SatelliteGraph or a Disjoint SmartGraph. + // as SatelliteGraph or a (Hybrid-)Disjoint SmartGraph. uint64_t numShards = 0; for (auto* aqlCollection : localGraphNode->collections()) { // It is of utmost importance that this is an ordered set of Shards. @@ -654,18 +654,11 @@ auto QuerySnippet::prepareFirstBranch( // to be used in toVelocyPack methods of classes derived // from GraphNode if (localGraphNode->isDisjoint()) { - if (found->second == server) { + if (aqlCollection->isSatellite()) { + myExp.emplace(shard); + TRI_ASSERT(shards.size() == 1); + } else if (found->second == server) { myExp.emplace(shard); - } else { - // the target server does not have anything to do with the particular - // collection (e.g. because the collection's shards are all on other - // servers), but it may be asked for this collection, because vertex - // collections are registered _globally_ with the TraversalNode and - // not on a per-target server basis. - // so in order to serve later lookups for this collection, we insert - // an empty string into the collection->shard map. - // on lookup, we will react to this. - localGraphNode->addCollectionToShard(aqlCollection->name(), ""); } } else { localGraphNode->addCollectionToShard(aqlCollection->name(), shard); @@ -678,16 +671,41 @@ auto QuerySnippet::prepareFirstBranch( if (myExp.size() > 1) { myExpFinal.insert({aqlCollection->name(), std::move(myExp)}); } + } else { + if (localGraphNode->isDisjoint()) { + // the target server does not have anything to do with the particular + // collection (e.g. because the collection's shards are all on other + // servers), but it may be asked for this collection, because vertex + // collections are registered _globally_ with the TraversalNode and + // not on a per-target server basis. + // so in order to serve later lookups for this collection, we insert + // an empty string into the collection->shard map. + // on lookup, we will react to this. + localGraphNode->addCollectionToShard(aqlCollection->name(), ""); + } } } + // TODO: We need to exclude DBServers which only do have Satellite + // collections only This server is not allowed to receive a setup call + + // 1.) Satellites: 1x shard exists (681) -> not landing in myExpFinal + // => 2x shards vertex collections -> DB1 v: [s1, s2] , DB2 [] + // 2.) + #ifdef ARANGODB_ENABLE_MAINTAINER_MODE // additional verification checks for Disjoint SmartGraphs if (localGraphNode->isDisjoint()) { if (!myExpFinal.empty()) { size_t numberOfShards = myExpFinal.begin()->second.size(); - // We need one expansion for every collection in the Graph - TRI_ASSERT(myExpFinal.size() == localGraphNode->collections().size()); + // We need one expansion for every collection in the Graph (-1 per satellite) + size_t amountOfNonSatellites = 0; + for (auto const& col : localGraphNode->collections()) { + if (!col->isSatellite()) { + amountOfNonSatellites++; + } + } + TRI_ASSERT(myExpFinal.size() == amountOfNonSatellites); for (auto const& expDefinition : myExpFinal) { TRI_ASSERT(expDefinition.second.size() == numberOfShards); } diff --git a/arangod/Aql/RestAqlHandler.cpp b/arangod/Aql/RestAqlHandler.cpp index 99e2b7a1655a..1941aea7b914 100644 --- a/arangod/Aql/RestAqlHandler.cpp +++ b/arangod/Aql/RestAqlHandler.cpp @@ -109,12 +109,12 @@ void RestAqlHandler::setupClusterQuery() { generateError(rest::ResponseCode::METHOD_NOT_ALLOWED, TRI_ERROR_CLUSTER_ONLY_ON_DBSERVER); return; } - + TRI_IF_FAILURE("Query::setupTimeout") { // intentionally delay the request std::this_thread::sleep_for(std::chrono::milliseconds(RandomGenerator::interval(uint32_t(2000)))); } - + TRI_IF_FAILURE("Query::setupTimeoutFailSequence") { // simulate lock timeout during query setup uint32_t r = 100; @@ -125,7 +125,7 @@ void RestAqlHandler::setupClusterQuery() { std::this_thread::sleep_for(std::chrono::milliseconds(3000)); } } - + bool success = false; VPackSlice querySlice = this->parseVPackBody(success); if (!success) { @@ -287,10 +287,10 @@ void RestAqlHandler::setupClusterQuery() { double const ttl = options.ttl; // creates a StandaloneContext or a leased context - auto q = std::make_unique(clusterQueryId, + auto q = std::make_unique(clusterQueryId, createTransactionContext(access), std::move(options)); - + TRI_ASSERT(clusterQueryId == 0 || clusterQueryId == q->id()); VPackBufferUInt8 buffer; @@ -302,7 +302,7 @@ void RestAqlHandler::setupClusterQuery() { answerBuilder.add(StaticStrings::AqlRemoteResult, VPackValue(VPackValueType::Object)); if (clusterQueryId == 0) { // only return this attribute if we didn't get a query ID as input from - // the coordinator. this will be the case for setup requests from 3.7 + // the coordinator. this will be the case for setup requests from 3.7 // coordinators answerBuilder.add("queryId", VPackValue(q->id())); } @@ -348,7 +348,7 @@ void RestAqlHandler::setupClusterQuery() { } _queryRegistry->insertQuery(std::move(q), ttl, std::move(rGuard)); - + generateResult(rest::ResponseCode::OK, std::move(buffer)); } diff --git a/arangod/Aql/TraversalNode.cpp b/arangod/Aql/TraversalNode.cpp index 8b962cf8d737..494e6bbebcd1 100644 --- a/arangod/Aql/TraversalNode.cpp +++ b/arangod/Aql/TraversalNode.cpp @@ -656,6 +656,9 @@ std::unique_ptr TraversalNode::createBlock( } #endif } else { + if (isDisjoint()) { + opts->setDisjoint(); + } traverser = std::make_unique(opts); } diff --git a/arangod/Cluster/ClusterMethods.cpp b/arangod/Cluster/ClusterMethods.cpp index 653bbb1ad9e5..a09075befa85 100644 --- a/arangod/Cluster/ClusterMethods.cpp +++ b/arangod/Cluster/ClusterMethods.cpp @@ -526,7 +526,7 @@ ::ErrorCode distributeBabyOnShards(CreateOperationCtx& opCtx, LogicalCollection& VPackSlice keySlice = value.get(StaticStrings::KeyString); if (keySlice.isNone()) { // The user did not specify a key, let's create one: - _key = collinfo.keyGenerator()->generate(); + _key = collinfo.createKey(value); } else { userSpecifiedKey = true; if (keySlice.isString()) { diff --git a/arangod/Graph/BreadthFirstEnumerator.cpp b/arangod/Graph/BreadthFirstEnumerator.cpp index 007aa561c5bf..20d0f221d229 100644 --- a/arangod/Graph/BreadthFirstEnumerator.cpp +++ b/arangod/Graph/BreadthFirstEnumerator.cpp @@ -57,7 +57,7 @@ BreadthFirstEnumerator::~BreadthFirstEnumerator() { void BreadthFirstEnumerator::setStartVertex(arangodb::velocypack::StringRef startVertex) { PathEnumerator::setStartVertex(startVertex); - + _schreier.clear(); _schreierIndex = 0; _lastReturned = 0; @@ -161,6 +161,12 @@ bool BreadthFirstEnumerator::next() { } } +#ifdef USE_ENTERPRISE + if (!validDisjointPath(nextIdx, vId)) { + return; + } +#endif + growStorage(); TRI_ASSERT(_schreier.capacity() > _schreier.size()); _schreier.emplace_back(nextIdx, std::move(eid), vId); @@ -214,7 +220,8 @@ arangodb::aql::AqlValue BreadthFirstEnumerator::edgeToAqlValue(size_t index) { return _opts->cache()->fetchEdgeAqlResult(_schreier[index].edge); } -VPackSlice BreadthFirstEnumerator::pathToIndexToSlice(VPackBuilder& result, size_t index, bool fromPrune) { +VPackSlice BreadthFirstEnumerator::pathToIndexToSlice(VPackBuilder& result, + size_t index, bool fromPrune) { _tempPathHelper.clear(); while (index != 0) { // Walk backwards through the path and push everything found on the local @@ -352,7 +359,8 @@ void BreadthFirstEnumerator::growStorage() { TRI_ASSERT(capacity > _schreier.size()); if (capacity > _schreier.capacity()) { - arangodb::ResourceUsageScope guard(_opts->resourceMonitor(), (capacity - _schreier.capacity()) * pathStepSize()); + arangodb::ResourceUsageScope guard(_opts->resourceMonitor(), + (capacity - _schreier.capacity()) * pathStepSize()); _schreier.reserve(capacity); @@ -364,3 +372,10 @@ void BreadthFirstEnumerator::growStorage() { constexpr size_t BreadthFirstEnumerator::pathStepSize() const noexcept { return sizeof(void*) + sizeof(PathStep) + 2 * sizeof(NextStep); } + +#ifndef USE_ENTERPRISE +bool BreadthFirstEnumerator::validDisjointPath(size_t /*index*/, + arangodb::velocypack::StringRef const& /*vertex*/) const { + return true; +} +#endif diff --git a/arangod/Graph/BreadthFirstEnumerator.h b/arangod/Graph/BreadthFirstEnumerator.h index ce6bb1a7f31a..3e02cfcff4ae 100644 --- a/arangod/Graph/BreadthFirstEnumerator.h +++ b/arangod/Graph/BreadthFirstEnumerator.h @@ -71,9 +71,9 @@ class BreadthFirstEnumerator final : public arangodb::traverser::PathEnumerator explicit NextStep(size_t sourceIdx) : sourceIdx(sourceIdx) {} }; - /// @brief schreier vector to store the visited vertices. - /// note: for memory usage tracking, it is require to call growStorage() before - /// inserting into the schreier vector. + /// @brief schreier vector to store the visited vertices. + /// note: for memory usage tracking, it is require to call growStorage() + /// before inserting into the schreier vector. std::vector _schreier; /// @brief Next free index in schreier vector. @@ -104,12 +104,12 @@ class BreadthFirstEnumerator final : public arangodb::traverser::PathEnumerator arangodb::traverser::TraverserOptions* opts); ~BreadthFirstEnumerator(); - + void setStartVertex(arangodb::velocypack::StringRef startVertex) override; /// @brief Get the next Path element from the traversal. bool next() override; - + aql::AqlValue lastVertexToAqlValue() override; aql::AqlValue lastEdgeToAqlValue() override; @@ -152,14 +152,17 @@ class BreadthFirstEnumerator final : public arangodb::traverser::PathEnumerator aql::AqlValue pathToIndexToAqlValue(arangodb::velocypack::Builder& result, size_t index); - velocypack::Slice pathToIndexToSlice(arangodb::velocypack::Builder& result, size_t index, bool fromPrune); + velocypack::Slice pathToIndexToSlice(arangodb::velocypack::Builder& result, + size_t index, bool fromPrune); bool shouldPrune(); void growStorage(); - + constexpr size_t pathStepSize() const noexcept; + + bool validDisjointPath(size_t nextVertexIndex, + arangodb::velocypack::StringRef const& vertex) const; }; } // namespace graph } // namespace arangodb - diff --git a/arangod/Graph/Graph.cpp b/arangod/Graph/Graph.cpp index 60aab621acd1..a7c5aac1ab53 100644 --- a/arangod/Graph/Graph.cpp +++ b/arangod/Graph/Graph.cpp @@ -246,8 +246,8 @@ std::set const& Graph::edgeCollections() const { return _edgeColls; } -bool Graph::needsToBeSatellite(std::string const& edge) const { - return false; +EdgeDefinition::EdgeDefinitionType Graph::getEdgeDefinitionType(std::string const& edge) const { + return EdgeDefinition::EdgeDefinitionType::DEFAULT; } std::map const& Graph::edgeDefinitions() const { diff --git a/arangod/Graph/Graph.h b/arangod/Graph/Graph.h index 9d424ead1082..1ba75fbe0ebf 100644 --- a/arangod/Graph/Graph.h +++ b/arangod/Graph/Graph.h @@ -207,7 +207,7 @@ class Graph { virtual bool isSmart() const; virtual bool isDisjoint() const; virtual bool isSatellite() const; - virtual bool needsToBeSatellite(std::string const& edge) const; + virtual EdgeDefinition::EdgeDefinitionType getEdgeDefinitionType(std::string const& edge) const; uint64_t numberOfShards() const; uint64_t replicationFactor() const; @@ -321,7 +321,7 @@ class Graph { /// @brief the names of all orphanCollections std::set _orphanColls; - /// @brief the names of all orphanCollections + /// @brief the names of all satelliteCollections std::set _satelliteColls; /// @brief the names of all edgeCollections diff --git a/arangod/Graph/GraphManager.cpp b/arangod/Graph/GraphManager.cpp index 80a49c90cdba..747838fca463 100644 --- a/arangod/Graph/GraphManager.cpp +++ b/arangod/Graph/GraphManager.cpp @@ -504,9 +504,7 @@ Result GraphManager::applyOnAllGraphs(std::function documentCollectionsToCreate{}; - std::unordered_set satelliteDocumentCollectionsToCreate{}; std::unordered_set edgeCollectionsToCreate{}; - std::unordered_set satelliteEdgeCollectionsToCreate{}; std::unordered_set> existentDocumentCollections{}; std::unordered_set> existentEdgeCollections{}; @@ -533,12 +531,7 @@ Result GraphManager::ensureCollections(Graph* graph, bool waitForSync) const { } else if (!res.is(TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND)) { return res; } else { - // not found the collection, need to create it later - if (graph->needsToBeSatellite(edgeColl)) { // check for satellites - satelliteEdgeCollectionsToCreate.emplace(edgeColl); - } else { - edgeCollectionsToCreate.emplace(edgeColl); - } + edgeCollectionsToCreate.emplace(edgeColl); } } @@ -555,13 +548,7 @@ Result GraphManager::ensureCollections(Graph* graph, bool waitForSync) const { return res; } else { if (edgeCollectionsToCreate.find(vertexColl) == edgeCollectionsToCreate.end()) { - if (!graph->satelliteCollections().empty() && - graph->satelliteCollections().find(vertexColl) != - graph->satelliteCollections().end()) { - satelliteDocumentCollectionsToCreate.emplace(vertexColl); - } else { - documentCollectionsToCreate.emplace(vertexColl); - } + documentCollectionsToCreate.emplace(vertexColl); } } } @@ -601,52 +588,58 @@ Result GraphManager::ensureCollections(Graph* graph, bool waitForSync) const { } } + // Storage space for VPackSlices used in options + std::vector>> vpackLake{}; + + auto collectionsToCreate = + prepareCollectionsToCreate(graph, waitForSync, documentCollectionsToCreate, + edgeCollectionsToCreate, vpackLake); + if (!collectionsToCreate.ok()) { + return collectionsToCreate.result(); + } + + if (collectionsToCreate.get().empty()) { + // NOTE: Empty graph is allowed. + return TRI_ERROR_NO_ERROR; + } + + std::vector> created; + OperationOptions opOptions(ExecContext::current()); + return methods::Collections::create(vocbase, opOptions, collectionsToCreate.get(), + waitForSync, true, false, nullptr, created); +} + +#ifndef USE_ENTERPRISE +ResultT> GraphManager::prepareCollectionsToCreate( + Graph const* graph, bool waitForSync, + std::unordered_set const& documentsCollectionNames, + std::unordered_set const& edgeCollectionNames, + std::vector>>& vpackLake) const { + std::vector collectionsToCreate; + collectionsToCreate.reserve(documentsCollectionNames.size() + + edgeCollectionNames.size()); // IV. Create collections VPackBuilder optionsBuilder; optionsBuilder.openObject(); graph->createCollectionOptions(optionsBuilder, waitForSync); optionsBuilder.close(); VPackSlice options = optionsBuilder.slice(); - std::vector collectionsToCreate; - collectionsToCreate.reserve(documentCollectionsToCreate.size() + - edgeCollectionsToCreate.size()); + // Retain the options storage space + vpackLake.emplace_back(optionsBuilder.steal()); // Create Document Collections - for (auto const& vertexColl : documentCollectionsToCreate) { + for (auto const& vertexColl : documentsCollectionNames) { collectionsToCreate.emplace_back( CollectionCreationInfo{vertexColl, TRI_COL_TYPE_DOCUMENT, options}); } // Create Edge Collections - for (auto const& edgeColl : edgeCollectionsToCreate) { + for (auto const& edgeColl : edgeCollectionNames) { collectionsToCreate.emplace_back( CollectionCreationInfo{edgeColl, TRI_COL_TYPE_EDGE, options}); } - - // new builder - VPackBuilder satOptionsBuilder; - if (!satelliteDocumentCollectionsToCreate.empty() || !satelliteEdgeCollectionsToCreate.empty()) { - satOptionsBuilder.openObject(); - graph->createSatelliteCollectionOptions(satOptionsBuilder, waitForSync); - satOptionsBuilder.close(); - for (auto const& satColl : satelliteDocumentCollectionsToCreate) { - collectionsToCreate.emplace_back(CollectionCreationInfo{satColl, TRI_COL_TYPE_DOCUMENT, - satOptionsBuilder.slice()}); - } - for (auto const& satEdgeColl : satelliteEdgeCollectionsToCreate) { - collectionsToCreate.emplace_back(CollectionCreationInfo{satEdgeColl, TRI_COL_TYPE_EDGE, - satOptionsBuilder.slice()}); - } - } - if (collectionsToCreate.empty()) { - // NOTE: Empty graph is allowed. - return TRI_ERROR_NO_ERROR; - } - - std::vector> created; - OperationOptions opOptions(ExecContext::current()); - return methods::Collections::create(vocbase, opOptions, collectionsToCreate, - waitForSync, true, false, nullptr, created); + return collectionsToCreate; } +#endif bool GraphManager::onlySatellitesUsed(Graph const* graph) const { for (auto const& cname : graph->vertexCollections()) { diff --git a/arangod/Graph/GraphManager.h b/arangod/Graph/GraphManager.h index 967c5ed53866..9bfa91dee51a 100644 --- a/arangod/Graph/GraphManager.h +++ b/arangod/Graph/GraphManager.h @@ -30,20 +30,23 @@ #include "Aql/Query.h" #include "Aql/VariableGenerator.h" #include "Basics/ReadWriteLock.h" -#include "Cluster/ClusterInfo.h" #include "Basics/ResultT.h" +#include "Cluster/ClusterInfo.h" #include "Graph/Graph.h" #include "Transaction/Methods.h" #include "Transaction/StandaloneContext.h" #include "Utils/OperationResult.h" namespace arangodb { + +struct CollectionCreationInfo; + namespace graph { class GraphManager { private: TRI_vocbase_t& _vocbase; - + std::shared_ptr ctx() const; //////////////////////////////////////////////////////////////////////////////// @@ -199,6 +202,12 @@ class GraphManager { Result checkDropGraphPermissions(Graph const& graph, std::unordered_set const& followersToBeRemoved, std::unordered_set const& leadersToBeRemoved); + + ResultT> prepareCollectionsToCreate( + Graph const* graph, bool waitForSync, + std::unordered_set const& documentsCollectionNames, + std::unordered_set const& edgeCollectionNames, + std::vector>>& vpackLake) const; }; } // namespace graph } // namespace arangodb diff --git a/arangod/Graph/PathEnumerator.cpp b/arangod/Graph/PathEnumerator.cpp index 5fdd886dabd3..a08a60306401 100644 --- a/arangod/Graph/PathEnumerator.cpp +++ b/arangod/Graph/PathEnumerator.cpp @@ -296,6 +296,12 @@ bool DepthFirstEnumerator::next() { } } +#ifdef USE_ENTERPRISE + if (!validDisjointPath()) { + return; + } +#endif + _enumeratedPath.pushEdge(eid); foundPath = true; } @@ -413,3 +419,9 @@ bool DepthFirstEnumerator::shouldPrune() { } return evaluator->evaluate(); } + +#ifndef USE_ENTERPRISE +bool DepthFirstEnumerator::validDisjointPath() const { + return true; +} +#endif diff --git a/arangod/Graph/PathEnumerator.h b/arangod/Graph/PathEnumerator.h index f1ae77f93ff5..17cb870a92d9 100644 --- a/arangod/Graph/PathEnumerator.h +++ b/arangod/Graph/PathEnumerator.h @@ -187,6 +187,7 @@ class DepthFirstEnumerator final : public PathEnumerator { private: bool shouldPrune(); + bool validDisjointPath() const; velocypack::Slice pathToSlice(arangodb::velocypack::Builder& result, bool fromPrune); }; diff --git a/arangod/Graph/TraverserOptions.cpp b/arangod/Graph/TraverserOptions.cpp index 02d91a7f0013..0d9abd60a769 100644 --- a/arangod/Graph/TraverserOptions.cpp +++ b/arangod/Graph/TraverserOptions.cpp @@ -749,6 +749,16 @@ auto TraverserOptions::explicitDepthLookupAt() const -> std::unordered_set void { + return; +} + +auto TraverserOptions::isDisjoint() const -> bool { + return false; +} +#endif + bool TraverserOptions::evaluateVertexExpression(arangodb::velocypack::Slice vertex, uint64_t depth) { arangodb::aql::Expression* expression = nullptr; @@ -765,6 +775,12 @@ bool TraverserOptions::evaluateVertexExpression(arangodb::velocypack::Slice vert return evaluateExpression(expression, vertex); } +#ifndef USE_ENTERPRISE +bool TraverserOptions::checkSmartDestination(VPackSlice edge, velocypack::StringRef sourceVertex) { + return false; +} +#endif + bool TraverserOptions::destinationCollectionAllowed(VPackSlice edge, velocypack::StringRef sourceVertex) { if (hasVertexCollectionRestrictions()) { @@ -777,6 +793,11 @@ bool TraverserOptions::destinationCollectionAllowed(VPackSlice edge, return false; } } +#ifdef USE_ENTERPRISE + if (!checkSmartDestination(edge, sourceVertex)) { + return false; + } +#endif return true; } @@ -873,7 +894,8 @@ auto TraverserOptions::estimateDepth() const noexcept -> uint64_t { void TraverserOptions::readProduceInfo(VPackSlice obj) { _produceVertices = VPackHelper::getBooleanValue(obj, "produceVertices", true); - _producePathsVertices = VPackHelper::getBooleanValue(obj, "producePathsVertices", true); + _producePathsVertices = + VPackHelper::getBooleanValue(obj, "producePathsVertices", true); _producePathsEdges = VPackHelper::getBooleanValue(obj, "producePathsEdges", true); _producePathsWeights = VPackHelper::getBooleanValue(obj, "producePathsWeights", true); } diff --git a/arangod/Graph/TraverserOptions.h b/arangod/Graph/TraverserOptions.h index 403e82d2cbd5..0cc4ad8aa51e 100644 --- a/arangod/Graph/TraverserOptions.h +++ b/arangod/Graph/TraverserOptions.h @@ -107,6 +107,8 @@ struct TraverserOptions : public graph::BaseOptions { std::vector edgeCollections; + bool _isDisjoint = false; + explicit TraverserOptions(arangodb::aql::QueryContext& query); TraverserOptions(arangodb::aql::QueryContext& query, arangodb::velocypack::Slice definition); @@ -157,6 +159,8 @@ struct TraverserOptions : public graph::BaseOptions { bool evaluateVertexExpression(arangodb::velocypack::Slice, uint64_t); + bool checkSmartDestination(VPackSlice edge, velocypack::StringRef sourceVertex); + bool destinationCollectionAllowed(velocypack::Slice edge, velocypack::StringRef sourceVertex); void linkTraverser(arangodb::traverser::ClusterTraverser*); @@ -212,6 +216,9 @@ struct TraverserOptions : public graph::BaseOptions { auto explicitDepthLookupAt() const -> std::unordered_set; + auto setDisjoint() -> void; + auto isDisjoint() const -> bool; + private: void readProduceInfo(VPackSlice obj); }; diff --git a/arangod/Graph/WeightedEnumerator.cpp b/arangod/Graph/WeightedEnumerator.cpp index 18f88168cdf7..352010805cd7 100644 --- a/arangod/Graph/WeightedEnumerator.cpp +++ b/arangod/Graph/WeightedEnumerator.cpp @@ -75,6 +75,11 @@ bool WeightedEnumerator::expandEdge(NextEdge nextEdge) { // getSingleVertex does nothing but that and checking conditions // However, for global unique vertexes, we need the vertex getter. if (_traverser->getVertex(toVertex, nextEdge.depth)) { +#ifdef USE_ENTERPRISE + if (!validDisjointPath(nextEdge.fromIndex, toVertex)) { + return false; + } +#endif if (_opts->uniqueVertices == TraverserOptions::UniquenessLevel::PATH) { if (pathContainsVertex(nextEdge.fromIndex, toVertex)) { // This vertex is on the path. @@ -354,3 +359,9 @@ velocypack::StringRef WeightedEnumerator::getToVertex(velocypack::Slice edge, } return resSlice.stringRef(); } + +#ifndef USE_ENTERPRISE +bool WeightedEnumerator::validDisjointPath(size_t index, arangodb::velocypack::StringRef vertex) const { + return true; +} +#endif diff --git a/arangod/Graph/WeightedEnumerator.h b/arangod/Graph/WeightedEnumerator.h index f54c6a591c2c..434c8178be86 100644 --- a/arangod/Graph/WeightedEnumerator.h +++ b/arangod/Graph/WeightedEnumerator.h @@ -154,7 +154,7 @@ class WeightedEnumerator final : public arangodb::traverser::PathEnumerator { bool expandEdge(NextEdge edge); static velocypack::StringRef getToVertex(velocypack::Slice edge, velocypack::StringRef from); + bool validDisjointPath(size_t index, arangodb::velocypack::StringRef vertex) const; }; } // namespace graph -} // namespace arangodb - +} // namespace arangodb \ No newline at end of file diff --git a/arangod/StorageEngine/PhysicalCollection.cpp b/arangod/StorageEngine/PhysicalCollection.cpp index b5baecbc5862..f7c873058763 100644 --- a/arangod/StorageEngine/PhysicalCollection.cpp +++ b/arangod/StorageEngine/PhysicalCollection.cpp @@ -359,7 +359,7 @@ Result PhysicalCollection::newObjectForInsert(transaction::Methods*, VPackSlice s = value.get(StaticStrings::KeyString); if (s.isNone()) { TRI_ASSERT(!isRestore); // need key in case of restore - auto keyString = _logicalCollection.keyGenerator()->generate(); + auto keyString = _logicalCollection.createKey(value); if (keyString.empty()) { return Result(TRI_ERROR_ARANGO_OUT_OF_KEYS); diff --git a/arangod/Transaction/Helpers.cpp b/arangod/Transaction/Helpers.cpp index 91e8a4d6afb2..745ec5e26a29 100644 --- a/arangod/Transaction/Helpers.cpp +++ b/arangod/Transaction/Helpers.cpp @@ -86,6 +86,15 @@ arangodb::velocypack::StringRef transaction::helpers::extractKeyPart(VPackSlice return arangodb::velocypack::StringRef(); } +/// @brief extract the _key attribute from a StringRef +arangodb::velocypack::StringRef transaction::helpers::extractKeyPart(velocypack::StringRef key) { + size_t pos = key.find('/'); + if (pos == std::string::npos) { + return key; + } + return key.substr(pos + 1); +} + /// @brief extract the _id attribute from a slice, and convert it into a /// string, static method std::string transaction::helpers::extractIdString(CollectionNameResolver const* resolver, diff --git a/arangod/Transaction/Helpers.h b/arangod/Transaction/Helpers.h index 2e9fd1435a24..9e75bfedd2c9 100644 --- a/arangod/Transaction/Helpers.h +++ b/arangod/Transaction/Helpers.h @@ -53,6 +53,9 @@ namespace helpers { /// @brief extract the _key attribute from a slice arangodb::velocypack::StringRef extractKeyPart(VPackSlice); +/// @brief extract the _key attribute from a StringRef +arangodb::velocypack::StringRef extractKeyPart(velocypack::StringRef); + std::string extractIdString(CollectionNameResolver const*, VPackSlice, VPackSlice const&); /// @brief quick access to the _key attribute in a database document diff --git a/arangod/VocBase/LogicalCollection.cpp b/arangod/VocBase/LogicalCollection.cpp index 4f61864c29bf..d76ef15818b0 100644 --- a/arangod/VocBase/LogicalCollection.cpp +++ b/arangod/VocBase/LogicalCollection.cpp @@ -398,9 +398,18 @@ ErrorCode LogicalCollection::getResponsibleShard(arangodb::velocypack::Slice sli } /// @briefs creates a new document key, the input slice is ignored here -std::string LogicalCollection::createKey(VPackSlice) { +std::string LogicalCollection::createKey(VPackSlice input) { + if (isSatToSmartEdgeCollection() || isSmartToSatEdgeCollection()) { + return createSmartToSatKey(input); + } + return keyGenerator()->generate(); +} + +#ifndef USE_ENTERPRISE +std::string LogicalCollection::createSmartToSatKey(VPackSlice) { return keyGenerator()->generate(); } +#endif void LogicalCollection::prepareIndexes(VPackSlice indexesSlice) { TRI_ASSERT(_physical != nullptr); @@ -610,9 +619,7 @@ Result LogicalCollection::rename(std::string&& newName) { return TRI_ERROR_NO_ERROR; } -ErrorCode LogicalCollection::close() { - return getPhysical()->close(); -} +ErrorCode LogicalCollection::close() { return getPhysical()->close(); } arangodb::Result LogicalCollection::drop() { // make sure collection has been closed @@ -1256,6 +1263,14 @@ bool LogicalCollection::isSmartEdgeCollection() const noexcept { return (_internalValidatorTypes & InternalValidatorType::LogicalSmartEdge) != 0; } +bool LogicalCollection::isSatToSmartEdgeCollection() const noexcept { + return (_internalValidatorTypes & InternalValidatorType::SatToSmartEdge) != 0; +} + +bool LogicalCollection::isSmartToSatEdgeCollection() const noexcept { + return (_internalValidatorTypes & InternalValidatorType::SmartToSatEdge) != 0; +} + #ifndef USE_ENTERPRISE void LogicalCollection::decorateWithInternalEEValidators() { // Only available in Enterprise Mode diff --git a/arangod/VocBase/LogicalCollection.h b/arangod/VocBase/LogicalCollection.h index 3c5a110fafb0..30c4ee43e11c 100644 --- a/arangod/VocBase/LogicalCollection.h +++ b/arangod/VocBase/LogicalCollection.h @@ -103,6 +103,8 @@ class LogicalCollection : public LogicalDataSource { LogicalSmartEdge = 1, LocalSmartEdge = 2, RemoteSmartEdge = 4, + SmartToSatEdge = 8, + SatToSmartEdge = 16, }; ////////////////////////////////////////////////////////////////////////////// @@ -355,6 +357,10 @@ class LogicalCollection : public LogicalDataSource { bool isSmartEdgeCollection() const noexcept; + bool isSatToSmartEdgeCollection() const noexcept; + + bool isSmartToSatEdgeCollection() const noexcept; + protected: void addInternalValidator(std::unique_ptr); @@ -363,6 +369,12 @@ class LogicalCollection : public LogicalDataSource { Result updateSchema(VPackSlice schema); + /** + * Enterprise only method. See enterprise code for implementation + * Community has a dummy stub. + */ + std::string createSmartToSatKey(arangodb::velocypack::Slice input); + private: void prepareIndexes(velocypack::Slice indexesSlice); From 0f86657a63a27de2af091a07deda03271f899afd Mon Sep 17 00:00:00 2001 From: Heiko Date: Fri, 6 Aug 2021 16:35:34 +0200 Subject: [PATCH 20/72] format, later cursor init (#14588) --- arangod/Graph/WeightedEnumerator.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/arangod/Graph/WeightedEnumerator.cpp b/arangod/Graph/WeightedEnumerator.cpp index 352010805cd7..a455bd148d01 100644 --- a/arangod/Graph/WeightedEnumerator.cpp +++ b/arangod/Graph/WeightedEnumerator.cpp @@ -76,9 +76,9 @@ bool WeightedEnumerator::expandEdge(NextEdge nextEdge) { // However, for global unique vertexes, we need the vertex getter. if (_traverser->getVertex(toVertex, nextEdge.depth)) { #ifdef USE_ENTERPRISE - if (!validDisjointPath(nextEdge.fromIndex, toVertex)) { - return false; - } + if (!validDisjointPath(nextEdge.fromIndex, toVertex)) { + return false; + } #endif if (_opts->uniqueVertices == TraverserOptions::UniquenessLevel::PATH) { if (pathContainsVertex(nextEdge.fromIndex, toVertex)) { @@ -103,12 +103,12 @@ bool WeightedEnumerator::expandEdge(NextEdge nextEdge) { void WeightedEnumerator::expandVertex(size_t vertexIndex, size_t depth) { PathStep const& currentStep = _schreier[vertexIndex]; VPackStringRef vertex = currentStep.currentVertexId; - EdgeCursor* cursor = getCursor(vertex, depth); if (depth >= _opts->maxDepth) { return; } + EdgeCursor* cursor = getCursor(vertex, depth); cursor->readAll([&](graph::EdgeDocumentToken&& eid, VPackSlice e, size_t cursorIdx) -> void { // transform edge if required if (e.isString()) { @@ -241,7 +241,8 @@ arangodb::aql::AqlValue WeightedEnumerator::edgeToAqlValue(size_t index) { return _opts->cache()->fetchEdgeAqlResult(_schreier[index].fromEdgeToken); } -VPackSlice WeightedEnumerator::pathToIndexToSlice(VPackBuilder& result, size_t index, bool fromPrune) { +VPackSlice WeightedEnumerator::pathToIndexToSlice(VPackBuilder& result, + size_t index, bool fromPrune) { for (_tempPathHelper.clear(); index != 0; index = _schreier[index].fromIndex) { _tempPathHelper.emplace_back(index); } @@ -361,7 +362,8 @@ velocypack::StringRef WeightedEnumerator::getToVertex(velocypack::Slice edge, } #ifndef USE_ENTERPRISE -bool WeightedEnumerator::validDisjointPath(size_t index, arangodb::velocypack::StringRef vertex) const { +bool WeightedEnumerator::validDisjointPath(size_t index, + arangodb::velocypack::StringRef vertex) const { return true; } #endif From 6c16f497a86dfeb3d7b1e1c3c6369368087e7c84 Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Tue, 10 Aug 2021 09:20:43 +0200 Subject: [PATCH 21/72] Feature/hybrid smart graph modify graph (#14598) --- arangod/Graph/Graph.cpp | 28 +- arangod/Graph/Graph.h | 19 +- arangod/Graph/GraphManager.cpp | 236 +++++---- arangod/Graph/GraphManager.h | 46 +- arangod/Graph/GraphOperations.cpp | 49 +- arangod/Graph/GraphOperations.h | 4 +- arangod/RestHandler/RestGraphHandler.cpp | 12 +- arangod/V8Server/v8-general-graph.cpp | 74 +-- arangod/VocBase/Methods/Collections.cpp | 3 +- .../modules/@arangodb/arango-collection.js | 1 + js/client/modules/@arangodb/general-graph.js | 14 +- .../modules/@arangodb/general-graph-common.js | 459 ++---------------- js/server/modules/@arangodb/general-graph.js | 12 +- tests/js/common/shell/shell-general-graph.js | 8 +- 14 files changed, 319 insertions(+), 646 deletions(-) diff --git a/arangod/Graph/Graph.cpp b/arangod/Graph/Graph.cpp index a7c5aac1ab53..03252d83c794 100644 --- a/arangod/Graph/Graph.cpp +++ b/arangod/Graph/Graph.cpp @@ -238,7 +238,7 @@ std::set const& Graph::orphanCollections() const { return _orphanColls; } -std::set const& Graph::satelliteCollections() const { +std::unordered_set const& Graph::satelliteCollections() const { return _satelliteColls; } @@ -246,10 +246,6 @@ std::set const& Graph::edgeCollections() const { return _edgeColls; } -EdgeDefinition::EdgeDefinitionType Graph::getEdgeDefinitionType(std::string const& edge) const { - return EdgeDefinition::EdgeDefinitionType::DEFAULT; -} - std::map const& Graph::edgeDefinitions() const { return _edgeDefs; } @@ -453,7 +449,7 @@ void EdgeDefinition::toVelocyPack(VPackBuilder& builder) const { builder.close(); // array } -ResultT EdgeDefinition::createFromVelocypack(VPackSlice edgeDefinition, std::set const& satCollections) { +ResultT EdgeDefinition::createFromVelocypack(VPackSlice edgeDefinition) { Result res = EdgeDefinition::validateEdgeDefinition(edgeDefinition); if (res.fail()) { return res; @@ -520,19 +516,6 @@ bool EdgeDefinition::renameCollection(std::string const& oldName, std::string co return renamed; } -auto EdgeDefinition::getType() const -> EdgeDefinitionType { - return _type; -} - -auto EdgeDefinition::setType(EdgeDefinitionType type) -> bool { - TRI_ASSERT(type != EdgeDefinitionType::DEFAULT); - if (_type == EdgeDefinitionType::DEFAULT) { - _type = type; - return true; - } - return false; -} - bool EdgeDefinition::isFromVertexCollectionUsed(std::string const& collectionName) const { if (getFrom().find(collectionName) != getFrom().end()) { return true; @@ -639,7 +622,7 @@ ResultT Graph::addEdgeDefinition(EdgeDefinition const& ed } ResultT Graph::addEdgeDefinition(VPackSlice const& edgeDefinitionSlice) { - auto res = EdgeDefinition::createFromVelocypack(edgeDefinitionSlice, satelliteCollections()); + auto res = EdgeDefinition::createFromVelocypack(edgeDefinitionSlice); if (res.fail()) { return std::move(res).result(); @@ -807,3 +790,8 @@ std::optional> Graph::getEdgeDefini TRI_ASSERT(hasEdgeCollection(collectionName)); return {it->second}; } + +auto Graph::addSatellites(VPackSlice const&) -> Result { + // Enterprise only + return TRI_ERROR_NO_ERROR; +} diff --git a/arangod/Graph/Graph.h b/arangod/Graph/Graph.h index 1ba75fbe0ebf..7c66c8d7109e 100644 --- a/arangod/Graph/Graph.h +++ b/arangod/Graph/Graph.h @@ -56,12 +56,10 @@ class EdgeDefinition { public: EdgeDefinition(std::string edgeCollection_, std::set&& from_, - std::set&& to_, - EdgeDefinitionType type = EdgeDefinitionType::DEFAULT) + std::set&& to_) : _edgeCollection(std::move(edgeCollection_)), _from(std::move(from_)), - _to(std::move(to_)), - _type(type) {} + _to(std::move(to_)) {} std::string const& getName() const { return _edgeCollection; } void setName(std::string const& newName) { _edgeCollection = newName; } @@ -80,8 +78,7 @@ class EdgeDefinition { /// types of values. static Result validateEdgeDefinition(const velocypack::Slice& edgeDefinition); - static ResultT createFromVelocypack(velocypack::Slice edgeDefinition, - std::set const& satCollections); + static ResultT createFromVelocypack(velocypack::Slice edgeDefinition); void toVelocyPack(velocypack::Builder&) const; @@ -93,8 +90,6 @@ class EdgeDefinition { bool renameCollection(std::string const& oldName, std::string const& newName); - auto getType() const -> EdgeDefinitionType; - /* @brief * Set type of the EdgeDefinition. Only allowed to be called once and only if * type is DEFAULT. If type has been set, it is not changeable anymore. @@ -109,7 +104,6 @@ class EdgeDefinition { std::string _edgeCollection; std::set _from; std::set _to; - EdgeDefinitionType _type; }; class Graph { @@ -185,7 +179,7 @@ class Graph { std::set const& orphanCollections() const; /// @brief get the cids of all satelliteCollections - std::set const& satelliteCollections() const; + std::unordered_set const& satelliteCollections() const; /// @brief get the cids of all edgeCollections std::set const& edgeCollections() const; @@ -207,7 +201,6 @@ class Graph { virtual bool isSmart() const; virtual bool isDisjoint() const; virtual bool isSatellite() const; - virtual EdgeDefinition::EdgeDefinitionType getEdgeDefinitionType(std::string const& edge) const; uint64_t numberOfShards() const; uint64_t replicationFactor() const; @@ -281,6 +274,8 @@ class Graph { /// @brief Add an orphan vertex collection to this graphs definition Result addOrphanCollection(std::string&&); + virtual auto addSatellites(VPackSlice const& satellites) -> Result; + std::ostream& operator<<(std::ostream& ostream); private: @@ -322,7 +317,7 @@ class Graph { std::set _orphanColls; /// @brief the names of all satelliteCollections - std::set _satelliteColls; + std::unordered_set _satelliteColls; /// @brief the names of all edgeCollections std::set _edgeColls; diff --git a/arangod/Graph/GraphManager.cpp b/arangod/Graph/GraphManager.cpp index 747838fca463..78b2dd500608 100644 --- a/arangod/Graph/GraphManager.cpp +++ b/arangod/Graph/GraphManager.cpp @@ -96,58 +96,19 @@ Result GraphManager::createCollection(std::string const& name, TRI_col_type_e co auto& vocbase = ctx()->vocbase(); - VPackBuilder helper; - helper.openObject(); - - if (ServerState::instance()->isCoordinator()) { - Result res = - ShardingInfo::validateShardsAndReplicationFactor(options, vocbase.server(), true); - if (res.fail()) { - return res; - } - - bool const forceOneShard = - vocbase.server().getFeature().forceOneShard() || - vocbase.sharding() == "single"; - - if (forceOneShard) { - // force a single shard with shards distributed like "_graph" - helper.add(StaticStrings::NumberOfShards, VPackValue(1)); - helper.add(StaticStrings::DistributeShardsLike, - VPackValue(vocbase.shardingPrototypeName())); - } - } - - helper.close(); - - VPackBuilder mergedBuilder = - VPackCollection::merge(options, helper.slice(), false, true); - std::shared_ptr coll; OperationOptions opOptions(ExecContext::current()); auto res = arangodb::methods::Collections::create( // create collection vocbase, // collection vocbase opOptions, - name, // collection name - colType, // collection type - mergedBuilder.slice(), // collection properties + name, // collection name + colType, // collection type + options, // collection properties waitForSync, true, false, coll); return res; } -Result GraphManager::findOrCreateVertexCollectionByName(const std::string& name, bool waitForSync, - VPackSlice options) { - std::shared_ptr def; - - def = getCollectionByName(ctx()->vocbase(), name); - if (def == nullptr) { - return createVertexCollection(name, waitForSync, options); - } - - return Result(TRI_ERROR_NO_ERROR); -} - bool GraphManager::renameGraphCollection(std::string const& oldName, std::string const& newName) { // todo: return a result, by now just used in the graph modules @@ -227,62 +188,87 @@ Result GraphManager::checkForEdgeDefinitionConflicts(std::map const& edgeDefinitions, - bool waitForSync, VPackSlice options) { - for (auto const& it : edgeDefinitions) { - EdgeDefinition const& edgeDefinition = it.second; - Result res = findOrCreateCollectionsByEdgeDefinition(graph, edgeDefinition, - waitForSync, options); +Result GraphManager::findOrCreateCollectionsByEdgeDefinition(Graph& graph, + EdgeDefinition const& edgeDefinition, + bool waitForSync) { + std::unordered_set satellites = graph.satelliteCollections(); + // Validation Phase collect a list of collections to create + std::unordered_set documentCollectionsToCreate{}; + std::unordered_set edgeCollectionsToCreate{}; + std::unordered_set> existentDocumentCollections{}; + std::unordered_set> existentEdgeCollections{}; - if (res.fail()) { - return res; + auto& vocbase = ctx()->vocbase(); + std::string const& edgeCollName = edgeDefinition.getName(); + std::shared_ptr edgeColl; + Result res = methods::Collections::lookup(vocbase, edgeCollName, edgeColl); + if (res.ok()) { + TRI_ASSERT(edgeColl); + if (edgeColl->type() != TRI_COL_TYPE_EDGE) { + return Result(TRI_ERROR_GRAPH_EDGE_DEFINITION_IS_DOCUMENT, + "Collection: '" + edgeColl->name() + + "' is not an EdgeCollection"); + } else { + // found the collection + existentEdgeCollections.emplace(edgeColl); } + } else if (!res.is(TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND)) { + return res; + } else { + edgeCollectionsToCreate.emplace(edgeCollName); } - return Result{TRI_ERROR_NO_ERROR}; -} - -Result GraphManager::findOrCreateCollectionsByEdgeDefinition(Graph const& graph, - EdgeDefinition const& edgeDefinition, - bool waitForSync, - VPackSlice const options) { - std::string const& edgeCollection = edgeDefinition.getName(); - std::shared_ptr def = - getCollectionByName(ctx()->vocbase(), edgeCollection); - - if (def == nullptr) { - Result res = createEdgeCollection(edgeCollection, waitForSync, options); - if (res.fail()) { + for (auto const& vertexColl : edgeDefinition.getFrom()) { + std::shared_ptr col; + Result res = methods::Collections::lookup(vocbase, vertexColl, col); + if (res.ok()) { + TRI_ASSERT(col); + if (col->isSatellite()) { + satellites.emplace(col->name()); + } + existentDocumentCollections.emplace(col); + } else if (!res.is(TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND)) { return res; + } else { + if (edgeCollectionsToCreate.find(vertexColl) == edgeCollectionsToCreate.end()) { + auto res = ensureVertexShardingMatches(graph, *edgeColl, satellites, + vertexColl, true); + if (res.fail()) { + return res; + } + documentCollectionsToCreate.emplace(vertexColl); + } } } - std::unordered_set vertexCollections; - - // duplicates in from and to shouldn't occur, but are safely ignored here - for (auto const& colName : edgeDefinition.getFrom()) { - vertexCollections.emplace(colName); - } - for (auto const& colName : edgeDefinition.getTo()) { - vertexCollections.emplace(colName); - } - for (auto const& colName : vertexCollections) { - def = getCollectionByName(ctx()->vocbase(), colName); - if (def == nullptr) { - Result res = createVertexCollection(colName, waitForSync, options); - if (res.fail()) { - return res; + for (auto const& vertexColl : edgeDefinition.getTo()) { + std::shared_ptr col; + Result res = methods::Collections::lookup(vocbase, vertexColl, col); + if (res.ok()) { + TRI_ASSERT(col); + if (col->isSatellite()) { + satellites.emplace(col->name()); } + existentDocumentCollections.emplace(col); + } else if (!res.is(TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND)) { + return res; } else { - auto res = graph.validateCollection(*def.get()); - if (res.fail()) { - return res; + if (edgeCollectionsToCreate.find(vertexColl) == edgeCollectionsToCreate.end()) { + if (edgeColl) { + auto res = ensureVertexShardingMatches(graph, *edgeColl, satellites, + vertexColl, false); + if (res.fail()) { + return res; + } + } + + documentCollectionsToCreate.emplace(vertexColl); } } } - - return Result{TRI_ERROR_NO_ERROR}; + return ensureCollections(graph, documentCollectionsToCreate, + edgeCollectionsToCreate, existentDocumentCollections, + existentEdgeCollections, satellites, waitForSync); } /// @brief extract the collection by either id or name, may return nullptr! @@ -412,7 +398,7 @@ OperationResult GraphManager::createGraph(VPackSlice document, bool waitForSync) } // Make sure all collections exist and are created - res = ensureCollections(graph.get(), waitForSync); + res = ensureAllCollections(graph.get(), waitForSync); if (res.fail()) { return OperationResult{res, options}; } @@ -501,7 +487,9 @@ Result GraphManager::applyOnAllGraphs(std::function satellites = graph->satelliteCollections(); // Validation Phase collect a list of collections to create std::unordered_set documentCollectionsToCreate{}; std::unordered_set edgeCollectionsToCreate{}; @@ -543,6 +531,9 @@ Result GraphManager::ensureCollections(Graph* graph, bool waitForSync) const { Result res = methods::Collections::lookup(vocbase, vertexColl, col); if (res.ok()) { TRI_ASSERT(col); + if (col->isSatellite()) { + satellites.emplace(col->name()); + } existentDocumentCollections.emplace(col); } else if (!res.is(TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND)) { return res; @@ -552,37 +543,68 @@ Result GraphManager::ensureCollections(Graph* graph, bool waitForSync) const { } } } + return ensureCollections(*graph, documentCollectionsToCreate, + edgeCollectionsToCreate, existentDocumentCollections, + existentEdgeCollections, satellites, waitForSync); +} +Result GraphManager::ensureCollections( + Graph& graph, std::unordered_set& documentCollectionsToCreate, + std::unordered_set const& edgeCollectionsToCreate, + std::unordered_set> const& existentDocumentCollections, + std::unordered_set> const& existentEdgeCollections, + std::unordered_set const& satellites, bool waitForSync) const { // II. Validate graph // a) Initial Validation if (!existentDocumentCollections.empty()) { for (auto const& col : existentDocumentCollections) { - graph->ensureInitial(*col); + graph.ensureInitial(*col); } } // b) Enterprise Sharding #ifdef USE_ENTERPRISE + std::string createdInitialName; { - Result res = ensureEnterpriseCollectionSharding(graph, waitForSync, - documentCollectionsToCreate); + auto [res, createdCollectionName] = + ensureEnterpriseCollectionSharding(&graph, waitForSync, documentCollectionsToCreate); if (res.fail()) { return res; } + createdInitialName = createdCollectionName; } + + ScopeGuard guard([&]() { + // rollback initial collection, in case it got created + if (!createdInitialName.empty()) { + std::shared_ptr coll; + Result found = + methods::Collections::lookup(ctx()->vocbase(), createdInitialName, coll); + if (found.ok()) { + TRI_ASSERT(coll); + Result dropResult = arangodb::methods::Collections::drop(*coll, false, -1.0); + if (dropResult.fail()) { + LOG_TOPIC("04c89", WARN, Logger::GRAPHS) + << "While cleaning up graph `" << graph.name() << "`: " + << "Dropping collection `" << createdInitialName << "` failed with error " + << dropResult.errorNumber() << ": " << dropResult.errorMessage(); + } + } + } + }); #endif // III. Validate collections // document collections for (auto const& col : existentDocumentCollections) { - Result res = graph->validateCollection(*col); + Result res = graph.validateCollection(*col); if (res.fail()) { return res; } } // edge collections for (auto const& col : existentEdgeCollections) { - Result res = graph->validateCollection(*col); + Result res = graph.validateCollection(*col); if (res.fail()) { return res; } @@ -592,21 +614,33 @@ Result GraphManager::ensureCollections(Graph* graph, bool waitForSync) const { std::vector>> vpackLake{}; auto collectionsToCreate = - prepareCollectionsToCreate(graph, waitForSync, documentCollectionsToCreate, - edgeCollectionsToCreate, vpackLake); + prepareCollectionsToCreate(&graph, waitForSync, documentCollectionsToCreate, + edgeCollectionsToCreate, satellites, vpackLake); if (!collectionsToCreate.ok()) { return collectionsToCreate.result(); } if (collectionsToCreate.get().empty()) { // NOTE: Empty graph is allowed. +#ifdef USE_ENTERPRISE + guard.cancel(); +#endif return TRI_ERROR_NO_ERROR; } std::vector> created; OperationOptions opOptions(ExecContext::current()); - return methods::Collections::create(vocbase, opOptions, collectionsToCreate.get(), - waitForSync, true, false, nullptr, created); + + Result finalResult = methods::Collections::create(ctx()->vocbase(), opOptions, + collectionsToCreate.get(), waitForSync, + true, false, nullptr, created); +#ifdef USE_ENTERPRISE + if (finalResult.ok()) { + guard.cancel(); + } +#endif + + return finalResult; } #ifndef USE_ENTERPRISE @@ -614,6 +648,7 @@ ResultT> GraphManager::prepareCollectionsToC Graph const* graph, bool waitForSync, std::unordered_set const& documentsCollectionNames, std::unordered_set const& edgeCollectionNames, + std::unordered_set const& satellites, std::vector>>& vpackLake) const { std::vector collectionsToCreate; collectionsToCreate.reserve(documentsCollectionNames.size() + @@ -1079,3 +1114,12 @@ ResultT> GraphManager::buildGraphFromInput(std::string co return {TRI_ERROR_INTERNAL}; } } + +#ifndef USE_ENTERPRISE +Result GraphManager::ensureVertexShardingMatches(Graph const&, LogicalCollection&, + std::unordered_set const&, + std::string const&, bool) const { + // Only relevant for Enterprise graphs. + return TRI_ERROR_NO_ERROR; +} +#endif diff --git a/arangod/Graph/GraphManager.h b/arangod/Graph/GraphManager.h index 9bfa91dee51a..c539370808d8 100644 --- a/arangod/Graph/GraphManager.h +++ b/arangod/Graph/GraphManager.h @@ -49,12 +49,6 @@ class GraphManager { std::shared_ptr ctx() const; - //////////////////////////////////////////////////////////////////////////////// - /// @brief find or create vertex collection by name - //////////////////////////////////////////////////////////////////////////////// - Result findOrCreateVertexCollectionByName(const std::string& name, - bool waitForSync, VPackSlice options); - //////////////////////////////////////////////////////////////////////////////// /// @brief find or create collection by name and type //////////////////////////////////////////////////////////////////////////////// @@ -94,13 +88,8 @@ class GraphManager { //////////////////////////////////////////////////////////////////////////////// /// @brief find or create collections by EdgeDefinitions //////////////////////////////////////////////////////////////////////////////// - Result findOrCreateCollectionsByEdgeDefinitions(Graph const& graph, - std::map const& edgeDefinitions, - bool waitForSync, VPackSlice options); - - Result findOrCreateCollectionsByEdgeDefinition(Graph const& graph, - EdgeDefinition const& edgeDefinition, - bool waitForSync, VPackSlice options); + Result findOrCreateCollectionsByEdgeDefinition(Graph& graph, EdgeDefinition const& edgeDefinition, + bool waitForSync); //////////////////////////////////////////////////////////////////////////////// /// @brief create a vertex collection @@ -143,7 +132,7 @@ class GraphManager { * * @return Either OK or an error. */ - Result ensureCollections(Graph* graph, bool waitForSync) const; + Result ensureAllCollections(Graph* graph, bool waitForSync) const; /// @brief check if only SatelliteCollections are used bool onlySatellitesUsed(Graph const* graph) const; @@ -175,14 +164,24 @@ class GraphManager { private: #ifdef USE_ENTERPRISE - Result ensureEnterpriseCollectionSharding(Graph const* graph, bool waitForSync, - std::unordered_set& documentCollections) const; - Result ensureSmartCollectionSharding(Graph const* graph, bool waitForSync, - std::unordered_set& documentCollections) const; - Result ensureSatelliteCollectionSharding(Graph const* graph, bool waitForSync, - std::unordered_set& documentCollections) const; + std::pair ensureEnterpriseCollectionSharding( + Graph const* graph, bool waitForSync, + std::unordered_set& documentCollections) const; + std::pair ensureSmartCollectionSharding( + Graph const* graph, bool waitForSync, + std::unordered_set& documentCollections) const; + std::pair ensureSatelliteCollectionSharding( + Graph const* graph, bool waitForSync, + std::unordered_set& documentCollections) const; #endif + Result ensureCollections( + Graph& graph, std::unordered_set& documentCollectionsToCreate, + std::unordered_set const& edgeCollectionsToCreate, + std::unordered_set> const& existentDocumentCollections, + std::unordered_set> const& existentEdgeCollections, + std::unordered_set const& satellites, bool waitForSync) const; + /** * @brief Create a new in memory graph object from the given input. * This graph object does not create collections and does @@ -207,8 +206,13 @@ class GraphManager { Graph const* graph, bool waitForSync, std::unordered_set const& documentsCollectionNames, std::unordered_set const& edgeCollectionNames, + std::unordered_set const& satellites, std::vector>>& vpackLake) const; + + Result ensureVertexShardingMatches(Graph const& graph, LogicalCollection& edgeColl, + std::unordered_set const& satellites, + std::string const& vertexCollection, + bool fromSide) const; }; } // namespace graph } // namespace arangodb - diff --git a/arangod/Graph/GraphOperations.cpp b/arangod/Graph/GraphOperations.cpp index ab632adb8472..7768b0c4e6ef 100644 --- a/arangod/Graph/GraphOperations.cpp +++ b/arangod/Graph/GraphOperations.cpp @@ -223,10 +223,12 @@ Result GraphOperations::checkVertexCollectionAvailability(std::string const& ver } OperationResult GraphOperations::editEdgeDefinition(VPackSlice edgeDefinitionSlice, + VPackSlice definitionOptions, bool waitForSync, std::string const& edgeDefinitionName) { + TRI_ASSERT(definitionOptions.isObject()); OperationOptions options(ExecContext::current()); - auto maybeEdgeDef = EdgeDefinition::createFromVelocypack(edgeDefinitionSlice, graph().satelliteCollections()); + auto maybeEdgeDef = EdgeDefinition::createFromVelocypack(edgeDefinitionSlice); if (!maybeEdgeDef) { return OperationResult{std::move(maybeEdgeDef).result(), options}; } @@ -246,13 +248,17 @@ OperationResult GraphOperations::editEdgeDefinition(VPackSlice edgeDefinitionSli }; } + auto satData = definitionOptions.get(StaticStrings::GraphSatellites); + if (satData.isArray()) { + auto res = _graph.addSatellites(satData); + if (res.fail()) { + // Handles invalid Slice Content + return OperationResult{std::move(res), options}; + } + } + GraphManager gmngr{_vocbase}; - VPackBuilder collectionsOptions; - collectionsOptions.openObject(); - _graph.createCollectionOptions(collectionsOptions, waitForSync); - collectionsOptions.close(); - res = gmngr.findOrCreateCollectionsByEdgeDefinition(_graph, edgeDefinition, waitForSync, - collectionsOptions.slice()); + res = gmngr.findOrCreateCollectionsByEdgeDefinition(_graph, edgeDefinition, waitForSync); if (res.fail()) { return OperationResult(res, options); } @@ -300,12 +306,24 @@ OperationResult GraphOperations::addOrphanCollection(VPackSlice document, bool w bool createCollection) { GraphManager gmngr{_vocbase}; std::string collectionName = document.get("collection").copyString(); + std::shared_ptr def; OperationOptions options(ExecContext::current()); options.waitForSync = waitForSync; OperationResult result(Result(), options); + VPackSlice editOptions = document.get(StaticStrings::GraphOptions); + if (editOptions.isObject()) { + editOptions = editOptions.get(StaticStrings::GraphSatellites); + if (editOptions.isArray()) { + auto res = _graph.addSatellites(editOptions); + if (res.fail()) { + return OperationResult(std::move(res), options); + } + } + } + if (_graph.hasVertexCollection(collectionName)) { if (_graph.hasOrphanCollection(collectionName)) { return OperationResult(TRI_ERROR_GRAPH_COLLECTION_USED_IN_ORPHANS, options); @@ -325,7 +343,7 @@ OperationResult GraphOperations::addOrphanCollection(VPackSlice document, bool w if (def == nullptr) { if (createCollection) { // ensure that all collections are available - res = gmngr.ensureCollections(&_graph, waitForSync); + res = gmngr.ensureAllCollections(&_graph, waitForSync); if (res.fail()) { return OperationResult{std::move(res), options}; @@ -338,7 +356,7 @@ OperationResult GraphOperations::addOrphanCollection(VPackSlice document, bool w } } else { // Hint: Now needed because of the initial property - res = gmngr.ensureCollections(&_graph, waitForSync); + res = gmngr.ensureAllCollections(&_graph, waitForSync); if (res.fail()) { return OperationResult{std::move(res), options}; } @@ -471,7 +489,9 @@ OperationResult GraphOperations::eraseOrphanCollection(bool waitForSync, } OperationResult GraphOperations::addEdgeDefinition(VPackSlice edgeDefinitionSlice, + VPackSlice definitionOptions, bool waitForSync) { + TRI_ASSERT(definitionOptions.isObject()); OperationOptions options(ExecContext::current()); ResultT defRes = _graph.addEdgeDefinition(edgeDefinitionSlice); if (defRes.fail()) { @@ -489,7 +509,16 @@ OperationResult GraphOperations::addEdgeDefinition(VPackSlice edgeDefinitionSlic return OperationResult(res, options); } - res = gmngr.ensureCollections(&_graph, waitForSync); + auto satData = definitionOptions.get(StaticStrings::GraphSatellites); + if (satData.isArray()) { + auto res = _graph.addSatellites(satData); + if (res.fail()) { + // Handles invalid Slice Content + return OperationResult{std::move(res), options}; + } + } + + res = gmngr.ensureAllCollections(&_graph, waitForSync); if (res.fail()) { return OperationResult{std::move(res), options}; diff --git a/arangod/Graph/GraphOperations.h b/arangod/Graph/GraphOperations.h index f41b2417c554..ff2cdcb11555 100644 --- a/arangod/Graph/GraphOperations.h +++ b/arangod/Graph/GraphOperations.h @@ -154,7 +154,7 @@ class GraphOperations { //////////////////////////////////////////////////////////////////////////////// /// @brief create a new edge definition in an existing graph //////////////////////////////////////////////////////////////////////////////// - OperationResult addEdgeDefinition(VPackSlice edgeDefinition, bool waitForSync); + OperationResult addEdgeDefinition(VPackSlice edgeDefinition, VPackSlice options, bool waitForSync); //////////////////////////////////////////////////////////////////////////////// /// @brief remove an edge definition from an existing graph @@ -165,7 +165,7 @@ class GraphOperations { //////////////////////////////////////////////////////////////////////////////// /// @brief create edge definition in an existing graph //////////////////////////////////////////////////////////////////////////////// - OperationResult editEdgeDefinition(VPackSlice edgeDefinitionSlice, bool waitForSync, + OperationResult editEdgeDefinition(VPackSlice edgeDefinitionSlice, VPackSlice options, bool waitForSync, std::string const& edgeDefinitionName); //////////////////////////////////////////////////////////////////////////////// diff --git a/arangod/RestHandler/RestGraphHandler.cpp b/arangod/RestHandler/RestGraphHandler.cpp index dfe076b3bcb1..3affa1ac9a2e 100644 --- a/arangod/RestHandler/RestGraphHandler.cpp +++ b/arangod/RestHandler/RestGraphHandler.cpp @@ -682,9 +682,17 @@ Result RestGraphHandler::modifyEdgeDefinition(graph::Graph& graph, EdgeDefinitio OperationResult result(Result(), options); if (action == EdgeDefinitionAction::CREATE) { - result = gops.addEdgeDefinition(body, waitForSync); + VPackSlice editOptions = body.get(StaticStrings::GraphOptions); + if (!editOptions.isObject()) { + editOptions = VPackSlice::emptyObjectSlice(); + } + result = gops.addEdgeDefinition(body, editOptions, waitForSync); } else if (action == EdgeDefinitionAction::EDIT) { - result = gops.editEdgeDefinition(body, waitForSync, edgeDefinitionName); + VPackSlice editOptions = body.get(StaticStrings::GraphOptions); + if (!editOptions.isObject()) { + editOptions = VPackSlice::emptyObjectSlice(); + } + result = gops.editEdgeDefinition(body, editOptions, waitForSync, edgeDefinitionName); } else if (action == EdgeDefinitionAction::REMOVE) { // TODO Does this get waitForSync? Not according to the documentation. // if not, remove the parameter from eraseEdgeDefinition. What about diff --git a/arangod/V8Server/v8-general-graph.cpp b/arangod/V8Server/v8-general-graph.cpp index 90b65a09caa6..e0c58c8784d9 100644 --- a/arangod/V8Server/v8-general-graph.cpp +++ b/arangod/V8Server/v8-general-graph.cpp @@ -89,33 +89,6 @@ static void JS_DropGraph(v8::FunctionCallbackInfo const& args) { TRI_V8_TRY_CATCH_END } -static void JS_RenameGraphCollection(v8::FunctionCallbackInfo const& args) { - TRI_V8_TRY_CATCH_BEGIN(isolate); - v8::HandleScope scope(isolate); - - if (args.Length() < 2) { - TRI_V8_THROW_EXCEPTION_USAGE("_renameCollection(oldName, newName)"); - } else if (!args[0]->IsString()) { - TRI_V8_THROW_EXCEPTION(TRI_ERROR_GRAPH_CREATE_MISSING_NAME); - } else if (!args[1]->IsString()) { - TRI_V8_THROW_EXCEPTION(TRI_ERROR_GRAPH_CREATE_MISSING_NAME); - } - std::string oldName = TRI_ObjectToString(isolate, args[0]); - std::string newName = TRI_ObjectToString(isolate, args[1]); - if (oldName.empty() || newName.empty()) { - TRI_V8_THROW_EXCEPTION(TRI_ERROR_GRAPH_CREATE_MISSING_NAME); - } - - auto& vocbase = GetContextVocBase(isolate); - GraphManager gmngr{vocbase}; - bool r = gmngr.renameGraphCollection(oldName, newName); - - TRI_V8_RETURN(r); - - TRI_V8_RETURN_UNDEFINED(); - TRI_V8_TRY_CATCH_END -} - static void JS_GraphExists(v8::FunctionCallbackInfo const& args) { TRI_V8_TRY_CATCH_BEGIN(isolate); v8::HandleScope scope(isolate); @@ -284,8 +257,8 @@ static void JS_AddEdgeDefinitions(v8::FunctionCallbackInfo const& arg TRI_V8_TRY_CATCH_BEGIN(isolate); v8::HandleScope scope(isolate); - if (args.Length() < 2) { - TRI_V8_THROW_EXCEPTION_USAGE("_extendEdgeDefinitions(edgeDefinition)"); + if (args.Length() < 2 || args.Length() > 3) { + TRI_V8_THROW_EXCEPTION_USAGE("_extendEdgeDefinitions([, ])"); } if (!args[0]->IsString()) { TRI_V8_THROW_EXCEPTION(TRI_ERROR_GRAPH_CREATE_MISSING_NAME); @@ -298,6 +271,19 @@ static void JS_AddEdgeDefinitions(v8::FunctionCallbackInfo const& arg VPackBuilder edgeDefinition; TRI_V8ToVPack(isolate, edgeDefinition, args[1], false); + VPackBuilder options; + if (args.Length() == 3) { + // We have options + TRI_V8ToVPack(isolate, options, args[2], false); + } + if (!options.slice().isObject()) { + options.clear(); + // Fake empty options, silently ignore errors for now. + // Empty Options. + options.openObject(); + options.close(); + } + auto& vocbase = GetContextVocBase(isolate); GraphManager gmngr{vocbase}; auto graph = gmngr.lookupGraphByName(graphName); @@ -308,7 +294,7 @@ static void JS_AddEdgeDefinitions(v8::FunctionCallbackInfo const& arg auto ctx = transaction::V8Context::Create(vocbase, true); GraphOperations gops{*graph.get(), vocbase, ctx}; - OperationResult r = gops.addEdgeDefinition(edgeDefinition.slice(), false); + OperationResult r = gops.addEdgeDefinition(edgeDefinition.slice(), options.slice(), false); if (r.fail()) { TRI_V8_THROW_EXCEPTION_MESSAGE(r.errorNumber(), r.errorMessage()); @@ -334,7 +320,7 @@ static void JS_EditEdgeDefinitions(v8::FunctionCallbackInfo const& ar v8::HandleScope scope(isolate); if (args.Length() < 2) { - TRI_V8_THROW_EXCEPTION_USAGE("_editEdgeDefinitions(edgeDefinition)"); + TRI_V8_THROW_EXCEPTION_USAGE("_editEdgeDefinitions(, [])"); } if (!args[0]->IsString()) { TRI_V8_THROW_EXCEPTION(TRI_ERROR_GRAPH_CREATE_MISSING_NAME); @@ -347,6 +333,19 @@ static void JS_EditEdgeDefinitions(v8::FunctionCallbackInfo const& ar VPackBuilder edgeDefinition; TRI_V8ToVPack(isolate, edgeDefinition, args[1], false); + VPackBuilder options; + if (args.Length() == 3) { + // We have options + TRI_V8ToVPack(isolate, options, args[2], false); + } + if (!options.slice().isObject()) { + options.clear(); + // Fake empty options, silently ignore errors for now. + // Empty Options. + options.openObject(); + options.close(); + } + auto& vocbase = GetContextVocBase(isolate); GraphManager gmngr{vocbase}; auto graph = gmngr.lookupGraphByName(graphName); @@ -358,7 +357,7 @@ static void JS_EditEdgeDefinitions(v8::FunctionCallbackInfo const& ar auto ctx = transaction::V8Context::Create(vocbase, true); GraphOperations gops{*graph.get(), vocbase, ctx}; OperationResult r = - gops.editEdgeDefinition(edgeDefinition.slice(), false, + gops.editEdgeDefinition(edgeDefinition.slice(), options.slice(), false, edgeDefinition.slice().get("collection").copyString()); if (r.fail()) { @@ -489,6 +488,14 @@ static void JS_AddVertexCollection(v8::FunctionCallbackInfo const& ar VPackBuilder builder; builder.openObject(); builder.add("collection", VPackValue(vertexName)); + { + if (args.Length() >= 4) { + // We have options + builder.add(VPackValue("options")); + // Merge them into the builder, as Options entry + TRI_V8ToVPack(isolate, builder, args[3], false); + } + } builder.close(); OperationResult r = gops.addOrphanCollection(builder.slice(), false, createCollection); @@ -725,9 +732,6 @@ static void InitV8GeneralGraphModule(v8::Handle context, TRI_vocbas TRI_AddMethodVocbase(isolate, rt, TRI_V8_ASCII_STRING(isolate, "_graph"), JS_GetGraph); TRI_AddMethodVocbase(isolate, rt, TRI_V8_ASCII_STRING(isolate, "_list"), JS_GetGraphKeys); TRI_AddMethodVocbase(isolate, rt, TRI_V8_ASCII_STRING(isolate, "_listObjects"), JS_GetGraphs); - TRI_AddMethodVocbase(isolate, rt, - TRI_V8_ASCII_STRING(isolate, "_renameCollection"), - JS_RenameGraphCollection); v8g->GeneralGraphModuleTempl.Reset(isolate, rt); ft->SetClassName( diff --git a/arangod/VocBase/Methods/Collections.cpp b/arangod/VocBase/Methods/Collections.cpp index 7b63a990dc21..09b81e4d7b1d 100644 --- a/arangod/VocBase/Methods/Collections.cpp +++ b/arangod/VocBase/Methods/Collections.cpp @@ -613,7 +613,8 @@ Result Collections::properties(Context& ctxt, VPackBuilder& builder) { ignoreKeys.insert({StaticStrings::DistributeShardsLike, StaticStrings::IsSmart, StaticStrings::NumberOfShards, StaticStrings::ReplicationFactor, StaticStrings::MinReplicationFactor, - StaticStrings::ShardKeys, StaticStrings::ShardingStrategy}); + StaticStrings::ShardKeys, StaticStrings::ShardingStrategy, + StaticStrings::IsDisjoint}); // this transaction is held longer than the following if... auto trx = ctxt.trx(AccessMode::Type::READ, true, false); diff --git a/js/client/modules/@arangodb/arango-collection.js b/js/client/modules/@arangodb/arango-collection.js index 74873a1405bd..8e697adc6818 100644 --- a/js/client/modules/@arangodb/arango-collection.js +++ b/js/client/modules/@arangodb/arango-collection.js @@ -361,6 +361,7 @@ ArangoCollection.prototype.properties = function (properties) { 'cacheEnabled': true, 'syncByRevision': true, 'schema' : true, + 'isDisjoint': true, }; var a; diff --git a/js/client/modules/@arangodb/general-graph.js b/js/client/modules/@arangodb/general-graph.js index 369487e9326b..94212387fc7c 100644 --- a/js/client/modules/@arangodb/general-graph.js +++ b/js/client/modules/@arangodb/general-graph.js @@ -70,8 +70,9 @@ CommonGraph.prototype.__updateDefinitions = function (edgeDefs, orphans) { this.__orphanCollections = orphans; }; -CommonGraph.prototype._extendEdgeDefinitions = function (edgeDefinition) { - const data = edgeDefinition || {}; +CommonGraph.prototype._extendEdgeDefinitions = function (edgeDefinition, options) { + const data = _.clone(edgeDefinition) || {}; + data.options = options || {}; const uri = GRAPH_PREFIX + encodeURIComponent(this.__name) + "/edge"; const requestResult = arangosh.checkRequestResult(db._connection.POST(uri, data)); const graph = requestResult.graph; @@ -81,8 +82,9 @@ CommonGraph.prototype._extendEdgeDefinitions = function (edgeDefinition) { } }; -CommonGraph.prototype._editEdgeDefinitions = function (edgeDefinition) { - const data = edgeDefinition || {}; +CommonGraph.prototype._editEdgeDefinitions = function (edgeDefinition, options) { + const data = _.clone(edgeDefinition) || {}; + data.options = options || {}; const uri = GRAPH_PREFIX + encodeURIComponent(this.__name) + "/edge/" + edgeDefinition.collection; const requestResult = arangosh.checkRequestResult(db._connection.PUT(uri, data)); const graph = requestResult.graph; @@ -92,8 +94,8 @@ CommonGraph.prototype._editEdgeDefinitions = function (edgeDefinition) { } }; -CommonGraph.prototype._addVertexCollection = function (name, createCollection) { - const data = {}; +CommonGraph.prototype._addVertexCollection = function (name, createCollection, options = {}) { + const data = { options }; if (name) { data.collection = name; } diff --git a/js/common/modules/@arangodb/general-graph-common.js b/js/common/modules/@arangodb/general-graph-common.js index f3dd8bdba289..5d13b56b2d85 100644 --- a/js/common/modules/@arangodb/general-graph-common.js +++ b/js/common/modules/@arangodb/general-graph-common.js @@ -28,11 +28,9 @@ // ////////////////////////////////////////////////////////////////////////////// const arangodb = require('@arangodb'); -const internal = require('internal'); const ArangoCollection = arangodb.ArangoCollection; const ArangoError = arangodb.ArangoError; const db = arangodb.db; -const errors = arangodb.errors; const users = require('@arangodb/users'); const _ = require('lodash'); @@ -71,21 +69,31 @@ var isValidCollectionsParameter = function (x) { return true; }; +var checkROPermission = function(c) { + if (!users.isAuthActive()) { + return; + } + + let user = users.currentUser(); + if (user) { + let p = users.permission(user, db._name(), c); + var err = new ArangoError(); + if (p === 'none') { + err.errorNum = arangodb.errors.ERROR_FORBIDDEN.code; + err.errorMessage = arangodb.errors.ERROR_FORBIDDEN.message; + throw err; + } + } +}; + // ////////////////////////////////////////////////////////////////////////////// // / @brief find or create a collection by name // ////////////////////////////////////////////////////////////////////////////// -var findOrCreateCollectionByName = function (name, type, noCreate, options) { +var findCollectionByName = function (name, type) { let col = db._collection(name); let res = false; - if (col === null && !noCreate) { - if (type === ArangoCollection.TYPE_DOCUMENT) { - col = db._create(name, options); - } else { - col = db._createEdgeCollection(name, options); - } - res = true; - } else if (!(col instanceof ArangoCollection)) { + if (!(col instanceof ArangoCollection)) { var err = new ArangoError(); err.errorNum = arangodb.errors.ERROR_ARANGO_DATA_SOURCE_NOT_FOUND.code; err.errorMessage = name + arangodb.errors.ERROR_ARANGO_DATA_SOURCE_NOT_FOUND.message; @@ -104,7 +112,7 @@ var findOrCreateCollectionByName = function (name, type, noCreate, options) { // / @brief find or create a collection by name // ////////////////////////////////////////////////////////////////////////////// -var findOrCreateCollectionsByEdgeDefinitions = function (edgeDefinitions, noCreate, options) { +var findCollectionsByEdgeDefinitions = function (edgeDefinitions) { let vertexCollections = {}; let edgeCollections = {}; edgeDefinitions.forEach(function (e) { @@ -120,15 +128,15 @@ var findOrCreateCollectionsByEdgeDefinitions = function (edgeDefinitions, noCrea } e.from.forEach(function (v) { - findOrCreateCollectionByName(v, ArangoCollection.TYPE_DOCUMENT, noCreate, options); + findCollectionByName(v, ArangoCollection.TYPE_DOCUMENT); vertexCollections[v] = db[v]; }); e.to.forEach(function (v) { - findOrCreateCollectionByName(v, ArangoCollection.TYPE_DOCUMENT, noCreate, options); + findCollectionByName(v, ArangoCollection.TYPE_DOCUMENT); vertexCollections[v] = db[v]; }); - findOrCreateCollectionByName(e.collection, ArangoCollection.TYPE_EDGE, noCreate, options); + findCollectionByName(e.collection, ArangoCollection.TYPE_EDGE); edgeCollections[e.collection] = db[e.collection]; }); return [ @@ -317,14 +325,6 @@ var sortEdgeDefinitionInplace = function (edgeDefinition) { return edgeDefinition; }; -var sortEdgeDefinition = function (edgeDefinition) { - return { - collection: edgeDefinition.collection, - from: edgeDefinition.from.slice().sort(), - to: edgeDefinition.to.slice().sort(), - }; -}; - // ////////////////////////////////////////////////////////////////////////////// // / @brief create a new graph // ////////////////////////////////////////////////////////////////////////////// @@ -577,166 +577,6 @@ var updateBindCollections = function (graph) { bindVertexCollections(graph, graph.__orphanCollections); }; -var checkRWPermission = function (c) { - if (!users.isAuthActive()) { - return; - } - - let user = users.currentUser(); - if (user) { - let p = users.permission(user, db._name(), c); - if (p !== 'rw') { - var err = new ArangoError(); - err.errorNum = arangodb.errors.ERROR_FORBIDDEN.code; - err.errorMessage = arangodb.errors.ERROR_FORBIDDEN.message; - throw err; - } - } -}; - -var checkROPermission = function(c) { - if (!users.isAuthActive()) { - return; - } - - let user = users.currentUser(); - if (user) { - let p = users.permission(user, db._name(), c); - var err = new ArangoError(); - if (p === 'none') { - err.errorNum = arangodb.errors.ERROR_FORBIDDEN.code; - err.errorMessage = arangodb.errors.ERROR_FORBIDDEN.message; - throw err; - } - } -}; - -// ////////////////////////////////////////////////////////////////////////////// -// / @brief internal function for editing edge definitions -// ////////////////////////////////////////////////////////////////////////////// - -var changeEdgeDefinitionsForGraph = function (graph, edgeDefinition, newCollections, possibleOrphans, self) { - var graphCollections = []; - var graphObj = exports._graph(graph._key); - var eDs = graph.edgeDefinitions; - var gotAHit = false; - - // replace edgeDefintion - eDs.forEach( - function (eD, id) { - if (eD.collection === edgeDefinition.collection) { - gotAHit = true; - eDs[id].from = edgeDefinition.from; - eDs[id].to = edgeDefinition.to; - db._graphs.update(graph._key, {edgeDefinitions: eDs}); - if (graph._key === self.__name) { - self.__edgeDefinitions[id].from = edgeDefinition.from; - self.__edgeDefinitions[id].to = edgeDefinition.to; - } - } else { - // collect all used collections - graphCollections = _.union(graphCollections, eD.from); - graphCollections = _.union(graphCollections, eD.to); - } - } - ); - if (!gotAHit) { - return; - } - - // remove used collection from orphanage - if (graph._key === self.__name) { - newCollections.forEach( - function (nc) { - if (self.__vertexCollections[nc] === undefined) { - self.__vertexCollections[nc] = db[nc]; - } - try { - self._removeVertexCollection(nc, false); - } catch (ignore) {} - } - ); - possibleOrphans.forEach( - function (po) { - if (graphCollections.indexOf(po) === -1) { - delete self.__vertexCollections[po]; - self._addVertexCollection(po); - } - } - ); - } else { - newCollections.forEach( - function (nc) { - try { - graphObj._removeVertexCollection(nc, false); - } catch (ignore) {} - } - ); - possibleOrphans.forEach( - function (po) { - if (graphCollections.indexOf(po) === -1) { - delete graphObj.__vertexCollections[po]; - graphObj._addVertexCollection(po); - } - } - ); - } - -// move unused collections to orphanage -}; - -// ////////////////////////////////////////////////////////////////////////////// -// / @brief Helper for dropping collections of a graph. -// ////////////////////////////////////////////////////////////////////////////// - -var checkIfMayBeDropped = function (colName, graphName, graphs) { - var result = true; - - graphs.forEach( - function (graph) { - if (result === false) { - // Short circuit - return; - } - if (graph._key === graphName) { - return; - } - var edgeDefinitions = graph.edgeDefinitions; - if (edgeDefinitions) { - edgeDefinitions.forEach( - function (edgeDefinition) { - var from = edgeDefinition.from; - var to = edgeDefinition.to; - var collection = edgeDefinition.collection; - if (collection === colName || - from.indexOf(colName) !== -1 || - to.indexOf(colName) !== -1 - ) { - result = false; - } - } - ); - } - - var orphanCollections = graph.orphanCollections; - if (orphanCollections) { - if (orphanCollections.indexOf(colName) !== -1) { - result = false; - } - } - } - ); - - return result; -}; - -const edgeDefinitionsEqual = function (leftEdgeDef, rightEdgeDef) { - leftEdgeDef = sortEdgeDefinition(leftEdgeDef); - rightEdgeDef = sortEdgeDefinition(rightEdgeDef); - const stringify = obj => JSON.stringify(obj, Object.keys(obj).sort()); - return stringify(leftEdgeDef) === stringify(rightEdgeDef); -}; - // @brief Class Graph. Defines a graph in the Database. class Graph { constructor (info) { @@ -747,14 +587,14 @@ class Graph { info.edgeDefinitions.forEach((e) => { // Link the edge collection - edgeCollections[e.collection] = db[e.collection]; + edgeCollections[e.collection] = db._collection(e.collection); // Link from collections e.from.forEach((v) => { - vertexCollections[v] = db[v]; + vertexCollections[v] = db._collection(v); }); // Link to collections e.to.forEach((v) => { - vertexCollections[v] = db[v]; + vertexCollections[v] = db._collection(v); }); }); @@ -1672,10 +1512,6 @@ exports._relation = function (relationName, fromVertexCollections, toVertexColle }; }; -// ////////////////////////////////////////////////////////////////////////////// -// / @brief was docuBlock JSF_general_graph_graph -// ////////////////////////////////////////////////////////////////////////////// - exports._graph = function (graphName) { let gdb = getGraphCollection(); let g; @@ -1683,7 +1519,7 @@ exports._graph = function (graphName) { try { g = gdb.document(graphName); } catch (e) { - if (e.errorNum !== errors.ERROR_ARANGO_DOCUMENT_NOT_FOUND.code) { + if (e.errorNum !== arangodb.errors.ERROR_ARANGO_DOCUMENT_NOT_FOUND.code) { throw e; } let err = new ArangoError(); @@ -1704,7 +1540,7 @@ exports._graph = function (graphName) { throw err; } - findOrCreateCollectionsByEdgeDefinitions(g.edgeDefinitions, true); + findCollectionsByEdgeDefinitions(g.edgeDefinitions); return new Graph(g); }; @@ -1741,181 +1577,6 @@ exports._extendEdgeDefinitions = function (edgeDefinition) { ); }; -// ////////////////////////////////////////////////////////////////////////////// -// / @brief was docuBlock JSF_general_graph_create -// ////////////////////////////////////////////////////////////////////////////// - -exports._create = function (graphName, edgeDefinitions, orphanCollections, options) { - if (!Array.isArray(orphanCollections)) { - orphanCollections = []; - } - options = options || {}; - let gdb = getGraphCollection(); - let graphAlreadyExists = true; - if (!graphName) { - let err = new ArangoError(); - err.errorNum = arangodb.errors.ERROR_GRAPH_CREATE_MISSING_NAME.code; - err.errorMessage = arangodb.errors.ERROR_GRAPH_CREATE_MISSING_NAME.message; - throw err; - } - edgeDefinitions = edgeDefinitions || []; - if (!Array.isArray(edgeDefinitions)) { - let err = new ArangoError(); - err.errorNum = arangodb.errors.ERROR_GRAPH_CREATE_MALFORMED_EDGE_DEFINITION.code; - err.errorMessage = arangodb.errors.ERROR_GRAPH_CREATE_MALFORMED_EDGE_DEFINITION.message; - throw err; - } - - edgeDefinitions = _.cloneDeep(edgeDefinitions); - - // check, if a collection is already used in a different edgeDefinition - let tmpCollections = []; - let tmpEdgeDefinitions = {}; - edgeDefinitions.forEach( - (edgeDefinition) => { - let col = edgeDefinition.collection; - if (tmpCollections.indexOf(col) !== -1) { - let err = new ArangoError(); - err.errorNum = arangodb.errors.ERROR_GRAPH_COLLECTION_MULTI_USE.code; - err.errorMessage = arangodb.errors.ERROR_GRAPH_COLLECTION_MULTI_USE.message; - throw err; - } - tmpCollections.push(col); - tmpEdgeDefinitions[col] = edgeDefinition; - } - ); - gdb.toArray().forEach( - (singleGraph) => { - var sGEDs = singleGraph.edgeDefinitions; - if (!Array.isArray(sGEDs)) { - return; - } - sGEDs.forEach( - (sGED) => { - var col = sGED.collection; - if (tmpCollections.indexOf(col) !== -1) { - if (!edgeDefinitionsEqual(sGED, tmpEdgeDefinitions[col])) { - let err = new ArangoError(); - err.errorNum = arangodb.errors.ERROR_GRAPH_COLLECTION_USE_IN_MULTI_GRAPHS.code; - err.errorMessage = col + ' ' + - arangodb.errors.ERROR_GRAPH_COLLECTION_USE_IN_MULTI_GRAPHS.message; - throw err; - } - } - } - ); - } - ); - - try { - gdb.document(graphName); - } catch (e) { - if (e.errorNum !== errors.ERROR_ARANGO_DOCUMENT_NOT_FOUND.code) { - throw e; - } - graphAlreadyExists = false; - } - - if (graphAlreadyExists) { - let err = new ArangoError(); - err.errorNum = arangodb.errors.ERROR_GRAPH_DUPLICATE.code; - err.errorMessage = arangodb.errors.ERROR_GRAPH_DUPLICATE.message; - throw err; - } - - db._flushCache(); - let collections = findOrCreateCollectionsByEdgeDefinitions(edgeDefinitions, false, options); - orphanCollections.forEach( - (oC) => { - findOrCreateCollectionByName(oC, ArangoCollection.TYPE_DOCUMENT, false, options); - } - ); - - edgeDefinitions.forEach(sortEdgeDefinitionInplace); - orphanCollections = orphanCollections.sort(); - - var data = gdb.save({ - 'orphanCollections': orphanCollections, - 'edgeDefinitions': edgeDefinitions, - '_key': graphName, - 'numberOfShards': options.numberOfShards || undefined, - 'replicationFactor': options.replicationFactor || undefined, - }, options); - data.orphanCollections = orphanCollections; - data.edgeDefinitions = edgeDefinitions; - return new Graph(data); -}; - -// ////////////////////////////////////////////////////////////////////////////// -// / @brief was docuBlock JSF_general_graph_drop -// ////////////////////////////////////////////////////////////////////////////// - -exports._drop = function (graphId, dropCollections) { - let gdb = getGraphCollection(); - let graphs; - - if (!gdb.exists(graphId)) { - let err = new ArangoError(); - err.errorNum = arangodb.errors.ERROR_GRAPH_NOT_FOUND.code; - err.errorMessage = arangodb.errors.ERROR_GRAPH_NOT_FOUND.message; - throw err; - } - - checkRWPermission("_graphs"); - var graph = gdb.document(graphId); - if (dropCollections === true) { - graphs = exports._listObjects(); - // Here we collect all collections - // that are leading for distribution - let leaderCollections = new Set(); - let followerCollections = new Set(); - let dropColCB = (name) => { - if (checkIfMayBeDropped(name, graph._key, graphs)) { - checkRWPermission(name); - let colObj = db[name]; - if (colObj !== undefined) { - // If it is undefined the collection is gone already - if (colObj.properties().distributeShardsLike === undefined) { - leaderCollections.add(name); - } else { - followerCollections.add(name); - } - } - } - }; - // drop orphans - if (!graph.orphanCollections) { - graph.orphanCollections = []; - } - graph.orphanCollections.forEach(dropColCB); - var edgeDefinitions = graph.edgeDefinitions; - edgeDefinitions.forEach( - function (edgeDefinition) { - var from = edgeDefinition.from; - var to = edgeDefinition.to; - var collection = edgeDefinition.collection; - dropColCB(edgeDefinition.collection); - from.forEach(dropColCB); - to.forEach(dropColCB); - } - ); - let dropColl = (c) => { - try { - db._drop(c); - } catch (e) { - internal.print("Failed to Drop: '" + c + "' reason: " + e.message); - } - }; - followerCollections.forEach(dropColl); - leaderCollections.forEach(dropColl); - } - - gdb.remove(graphId); - db._flushCache(); - - return true; -}; - // ////////////////////////////////////////////////////////////////////////////// // / @brief check if a graph exists. // ////////////////////////////////////////////////////////////////////////////// @@ -1925,74 +1586,10 @@ exports._exists = function (graphId) { return gCol.exists(graphId); }; -// ////////////////////////////////////////////////////////////////////////////// -// / @brief rename a collection inside the _graphs collections -// ////////////////////////////////////////////////////////////////////////////// - -exports._renameCollection = function (oldName, newName) { - db._executeTransaction({ - collections: { - write: '_graphs' - }, - action: function (params) { - var gdb = getGraphCollection(); - if (!gdb) { - return; - } - gdb.toArray().forEach(function (doc) { - if (doc.isSmart) { - return; - } - let c = Object.assign({}, doc); - let changed = false; - if (c.edgeDefinitions) { - for (let i = 0; i < c.edgeDefinitions.length; ++i) { - var def = c.edgeDefinitions[i]; - if (def.collection === params.oldName) { - c.edgeDefinitions[i].collection = params.newName; - changed = true; - } - for (let j = 0; j < def.from.length; ++j) { - if (def.from[j] === params.oldName) { - c.edgeDefinitions[i].from[j] = params.newName; - changed = true; - } - } - for (let j = 0; j < def.to.length; ++j) { - if (def.to[j] === params.oldName) { - c.edgeDefinitions[i].to[j] = params.newName; - changed = true; - } - } - } - } - for (let i = 0; i < c.orphanCollections.length; ++i) { - if (c.orphanCollections[i] === params.oldName) { - c.orphanCollections[i] = params.newName; - changed = true; - } - } - - if (changed) { - gdb.update(doc._key, c); - } - }); - }, - params: { - oldName: oldName, - newName: newName - } - }); -}; - // ////////////////////////////////////////////////////////////////////////////// // / @brief was docuBlock JSF_general_graph_list // ////////////////////////////////////////////////////////////////////////////// -exports._list = function () { - return db._query(`FOR x IN _graphs RETURN x._key`).toArray(); -}; - exports._listObjects = function () { return db._query(`FOR x IN _graphs RETURN x`).toArray(); }; diff --git a/js/server/modules/@arangodb/general-graph.js b/js/server/modules/@arangodb/general-graph.js index 15cda4302a9e..debcf1a4e739 100644 --- a/js/server/modules/@arangodb/general-graph.js +++ b/js/server/modules/@arangodb/general-graph.js @@ -50,18 +50,18 @@ CommonGraph.prototype._deleteEdgeDefinition = function (edgeDefinition, dropColl this.__updateDefinitions(result.graph.edgeDefinitions, result.graph.orphanCollections); }; -CommonGraph.prototype._extendEdgeDefinitions = function (edgeDefinitions) { - let result = ArangoGraph._extendEdgeDefinitions(this.__name, edgeDefinitions); +CommonGraph.prototype._extendEdgeDefinitions = function (edgeDefinitions, options = {}) { + let result = ArangoGraph._extendEdgeDefinitions(this.__name, edgeDefinitions, options); this.__updateDefinitions(result.graph.edgeDefinitions, result.graph.orphanCollections); }; -CommonGraph.prototype._editEdgeDefinitions = function (edgeDefinitions) { - let result = ArangoGraph._editEdgeDefinitions(this.__name, edgeDefinitions); +CommonGraph.prototype._editEdgeDefinitions = function (edgeDefinitions, options = {}) { + let result = ArangoGraph._editEdgeDefinitions(this.__name, edgeDefinitions, options); this.__updateDefinitions(result.graph.edgeDefinitions, result.graph.orphanCollections); }; -CommonGraph.prototype._addVertexCollection = function (vertexName, createCollection = true) { - let result = ArangoGraph._addVertexCollection(this.__name, vertexName, createCollection); +CommonGraph.prototype._addVertexCollection = function (vertexName, createCollection = true, options = {}) { + let result = ArangoGraph._addVertexCollection(this.__name, vertexName, createCollection, options); this.__updateDefinitions(result.graph.edgeDefinitions, result.graph.orphanCollections); }; diff --git a/tests/js/common/shell/shell-general-graph.js b/tests/js/common/shell/shell-general-graph.js index 02adc9306518..5354d998ee9c 100644 --- a/tests/js/common/shell/shell-general-graph.js +++ b/tests/js/common/shell/shell-general-graph.js @@ -240,9 +240,9 @@ function GeneralGraphCreationSuite() { ) ); - graph._renameCollection("UnitTestRelationName1", "UnitTestsGraphRenamed1"); - graph._renameCollection("UnitTestVertices1", "UnitTestsGraphRenamed2"); - graph._renameCollection("UnitTestVertices4", "UnitTestsGraphRenamed3"); + db._collection("UnitTestRelationName1").rename("UnitTestsGraphRenamed1"); + db._collection("UnitTestVertices1").rename("UnitTestsGraphRenamed2"); + db._collection("UnitTestVertices4").rename("UnitTestsGraphRenamed3"); var doc = db._graphs.document(gN1); assertEqual(1, doc.edgeDefinitions.length); @@ -278,7 +278,7 @@ function GeneralGraphCreationSuite() { var g2 = graph._graph(gN2); g2._addVertexCollection("mj7"); - graph._renameCollection("mj7", "MarcelJansen"); + db._collection("mj7").rename("MarcelJansen"); var doc = db._graphs.document(gN1); assertEqual(1, doc.orphanCollections.length); From 0bf12eb5e96d44d433b4aea100d0929329e3c9a6 Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Tue, 10 Aug 2021 09:22:13 +0200 Subject: [PATCH 22/72] Feature/hybrid smart graph remove old code 2 (#14600) --- arangod/Cluster/TraverserEngine.cpp | 8 -------- arangod/Cluster/TraverserEngine.h | 9 --------- arangod/Graph/PathManagement/PathValidator.cpp | 2 +- 3 files changed, 1 insertion(+), 18 deletions(-) diff --git a/arangod/Cluster/TraverserEngine.cpp b/arangod/Cluster/TraverserEngine.cpp index 09a72d06aae2..04efe6701a8c 100644 --- a/arangod/Cluster/TraverserEngine.cpp +++ b/arangod/Cluster/TraverserEngine.cpp @@ -411,14 +411,6 @@ TraverserEngine::TraverserEngine(TRI_vocbase_t& vocbase, TraverserEngine::~TraverserEngine() = default; -void TraverserEngine::smartSearch(VPackSlice, VPackBuilder&) { - THROW_ARANGO_EXCEPTION(TRI_ERROR_ONLY_ENTERPRISE); -} - void TraverserEngine::smartSearchNew(VPackSlice, VPackBuilder&) { THROW_ARANGO_EXCEPTION(TRI_ERROR_ONLY_ENTERPRISE); } - -void TraverserEngine::smartSearchBFS(VPackSlice, VPackBuilder&) { - THROW_ARANGO_EXCEPTION(TRI_ERROR_ONLY_ENTERPRISE); -} diff --git a/arangod/Cluster/TraverserEngine.h b/arangod/Cluster/TraverserEngine.h index 87b41a7a5248..f4d0149f788b 100644 --- a/arangod/Cluster/TraverserEngine.h +++ b/arangod/Cluster/TraverserEngine.h @@ -115,14 +115,9 @@ class BaseTraverserEngine : public BaseEngine { graph::EdgeCursor* getCursor(arangodb::velocypack::StringRef nextVertex, uint64_t currentDepth); - virtual void smartSearch(arangodb::velocypack::Slice, arangodb::velocypack::Builder&) = 0; - virtual void smartSearchNew(arangodb::velocypack::Slice, arangodb::velocypack::Builder&) = 0; - virtual void smartSearchBFS(arangodb::velocypack::Slice, - arangodb::velocypack::Builder&) = 0; - EngineType getType() const override { return TRAVERSER; } bool produceVertices() const override; @@ -178,11 +173,7 @@ class TraverserEngine : public BaseTraverserEngine { ~TraverserEngine(); - void smartSearch(arangodb::velocypack::Slice, arangodb::velocypack::Builder&) override; - void smartSearchNew(arangodb::velocypack::Slice, arangodb::velocypack::Builder&) override; - - void smartSearchBFS(arangodb::velocypack::Slice, arangodb::velocypack::Builder&) override; }; } // namespace traverser diff --git a/arangod/Graph/PathManagement/PathValidator.cpp b/arangod/Graph/PathManagement/PathValidator.cpp index eab704d47b1b..6a3150f21f9b 100644 --- a/arangod/Graph/PathManagement/PathValidator.cpp +++ b/arangod/Graph/PathManagement/PathValidator.cpp @@ -140,7 +140,7 @@ auto PathValidator::evaluateVertexRes return true; } - auto allowedCollections = _options.getAllowedVertexCollections(); + auto const& allowedCollections = _options.getAllowedVertexCollections(); if (allowedCollections.empty()) { // all allowed return true; From 1ace9dc70e29921202085a5991b5f58b1e27a6c0 Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Wed, 11 Aug 2021 09:17:52 +0200 Subject: [PATCH 23/72] Added a test to validate if we are satellite leader (#14619) --- arangod/Graph/TraverserOptions.cpp | 7 +++++++ arangod/Graph/TraverserOptions.h | 2 ++ 2 files changed, 9 insertions(+) diff --git a/arangod/Graph/TraverserOptions.cpp b/arangod/Graph/TraverserOptions.cpp index 0d9abd60a769..620e0c8e449b 100644 --- a/arangod/Graph/TraverserOptions.cpp +++ b/arangod/Graph/TraverserOptions.cpp @@ -757,6 +757,13 @@ auto TraverserOptions::setDisjoint() -> void { auto TraverserOptions::isDisjoint() const -> bool { return false; } + +auto TraverserOptions::isSatelliteLeader() const -> bool { + // Can only be called in Enterprise code. + // Return false as security net. + TRI_ASSERT(false); + return false; +} #endif bool TraverserOptions::evaluateVertexExpression(arangodb::velocypack::Slice vertex, diff --git a/arangod/Graph/TraverserOptions.h b/arangod/Graph/TraverserOptions.h index 0cc4ad8aa51e..66ed0b21c2c0 100644 --- a/arangod/Graph/TraverserOptions.h +++ b/arangod/Graph/TraverserOptions.h @@ -219,6 +219,8 @@ struct TraverserOptions : public graph::BaseOptions { auto setDisjoint() -> void; auto isDisjoint() const -> bool; + auto isSatelliteLeader() const -> bool; + private: void readProduceInfo(VPackSlice obj); }; From fdf3fc29b6f08e0b765b3d3354415ac481ec2ceb Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Tue, 17 Aug 2021 12:25:54 +0200 Subject: [PATCH 24/72] Removed false comment --- arangod/Aql/ClusterNodes.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/arangod/Aql/ClusterNodes.cpp b/arangod/Aql/ClusterNodes.cpp index fd6daf61a53f..97fa2697b026 100644 --- a/arangod/Aql/ClusterNodes.cpp +++ b/arangod/Aql/ClusterNodes.cpp @@ -333,9 +333,6 @@ void DistributeNode::doToVelocyPack(VPackBuilder& builder, unsigned flags) const { VPackArrayBuilder guard(&builder); for (auto const& v : _satellites) { - // if the mapped shard for a collection is empty, it means that - // we have a vertex collection that is only relevant on some of the - // target servers builder.add(VPackValue(v->name())); } } From 6903a1673df236fd2057a5257254236195eb1151 Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Tue, 17 Aug 2021 12:27:36 +0200 Subject: [PATCH 25/72] Removed unused local struct --- arangod/Aql/ExecutionEngine.cpp | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/arangod/Aql/ExecutionEngine.cpp b/arangod/Aql/ExecutionEngine.cpp index 63a5a66802d8..ca4d867ad79a 100644 --- a/arangod/Aql/ExecutionEngine.cpp +++ b/arangod/Aql/ExecutionEngine.cpp @@ -52,32 +52,6 @@ using namespace arangodb; using namespace arangodb::aql; -// @brief Local struct to create the -// information required to build traverser engines -// on DB servers. -struct TraverserEngineShardLists { - explicit TraverserEngineShardLists(size_t length) { - // Make sure they all have a fixed size. - edgeCollections.resize(length); - } - - ~TraverserEngineShardLists() = default; - - // Mapping for edge collections to shardIds. - // We have to retain the ordering of edge collections, all - // vectors of these in one run need to have identical size. - // This is because the conditions to query those edges have the - // same ordering. - std::vector> edgeCollections; - - // Mapping for vertexCollections to shardIds. - std::unordered_map> vertexCollections; - -#ifdef USE_ENTERPRISE - std::set inaccessibleShards; -#endif -}; - /** * @brief Create AQL blocks from a list of ExectionNodes * Only works in cluster mode From 46df6e5aeba7eee668f28e1f359f6ba47b6768de Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Fri, 20 Aug 2021 10:16:55 +0200 Subject: [PATCH 26/72] Moved TraverserShardList to it's own file --- ...EngineInfoContainerDBServerServerBased.cpp | 119 +------------- .../EngineInfoContainerDBServerServerBased.h | 55 ------- arangod/Aql/TraverserEngineShardLists.cpp | 149 ++++++++++++++++++ arangod/Aql/TraverserEngineShardLists.h | 87 ++++++++++ arangod/CMakeLists.txt | 1 + 5 files changed, 238 insertions(+), 173 deletions(-) create mode 100644 arangod/Aql/TraverserEngineShardLists.cpp create mode 100644 arangod/Aql/TraverserEngineShardLists.h diff --git a/arangod/Aql/EngineInfoContainerDBServerServerBased.cpp b/arangod/Aql/EngineInfoContainerDBServerServerBased.cpp index c507aef8bb4f..a9c8628fc5c3 100644 --- a/arangod/Aql/EngineInfoContainerDBServerServerBased.cpp +++ b/arangod/Aql/EngineInfoContainerDBServerServerBased.cpp @@ -25,11 +25,11 @@ #include "Aql/Ast.h" #include "Aql/GraphNode.h" +#include "Aql/TraverserEngineShardLists.h" #include "Basics/StaticStrings.h" #include "Basics/StringUtils.h" #include "Cluster/ClusterFeature.h" #include "Cluster/ClusterTrxMethods.h" -#include "Graph/BaseOptions.h" #include "Network/Methods.h" #include "Network/NetworkFeature.h" #include "Network/Utils.h" @@ -78,123 +78,6 @@ Result ExtractRemoteAndShard(VPackSlice keySlice, ExecutionNodeId& remoteId, } // namespace -EngineInfoContainerDBServerServerBased::TraverserEngineShardLists::TraverserEngineShardLists( - GraphNode const* node, ServerID const& server, - std::unordered_map const& shardMapping, QueryContext& query) - : _node(node), _hasShard(false) { - auto const& edges = _node->edgeColls(); - TRI_ASSERT(!edges.empty()); - auto const& restrictToShards = query.queryOptions().restrictToShards; -#ifdef USE_ENTERPRISE - transaction::Methods trx{query.newTrxContext()}; -#endif - // Extract the local shards for edge collections. - for (auto const& col : edges) { -#ifdef USE_ENTERPRISE - if (trx.isInaccessibleCollection(col->id())) { - _inaccessible.insert(col->name()); - _inaccessible.insert(std::to_string(col->id().id())); - } -#endif - _edgeCollections.emplace_back( - getAllLocalShards(shardMapping, server, col->shardIds(restrictToShards), - col->isSatellite() && node->isSmart())); - } - // Extract vertices - auto const& vertices = _node->vertexColls(); - // Guaranteed by addGraphNode, this will inject vertex collections - // in anonymous graph case - // It might in fact be empty, if we only have edge collections in a graph. - // Or if we guarantee to never read vertex data. - for (auto const& col : vertices) { -#ifdef USE_ENTERPRISE - if (trx.isInaccessibleCollection(col->id())) { - _inaccessible.insert(col->name()); - _inaccessible.insert(std::to_string(col->id().id())); - } -#endif - auto shards = getAllLocalShards(shardMapping, server, col->shardIds(restrictToShards), - col->isSatellite() && node->isSmart()); - _vertexCollections.try_emplace(col->name(), std::move(shards)); - } -} - -std::vector EngineInfoContainerDBServerServerBased::TraverserEngineShardLists::getAllLocalShards( - std::unordered_map const& shardMapping, ServerID const& server, - std::shared_ptr> shardIds, bool colIsSatellite) { - std::vector localShards; - for (auto const& shard : *shardIds) { - auto const& it = shardMapping.find(shard); - TRI_ASSERT(it != shardMapping.end()); - if (it->second == server) { - localShards.emplace_back(shard); - // Guaranteed that the traversal will be executed on this server. - _hasShard = true; - } else if (colIsSatellite) { - // The satellite does not force run of a traversal here. - localShards.emplace_back(shard); - } - } - return localShards; -} - -void EngineInfoContainerDBServerServerBased::TraverserEngineShardLists::serializeIntoBuilder( - VPackBuilder& infoBuilder) const { - TRI_ASSERT(_hasShard); - TRI_ASSERT(infoBuilder.isOpenArray()); - infoBuilder.openObject(); - { - // Options - infoBuilder.add(VPackValue("options")); - graph::BaseOptions* opts = _node->options(); - opts->buildEngineInfo(infoBuilder); - } - { - // Variables - std::vector vars; - _node->getConditionVariables(vars); - if (!vars.empty()) { - infoBuilder.add(VPackValue("variables")); - infoBuilder.openArray(); - for (auto v : vars) { - v->toVelocyPack(infoBuilder); - } - infoBuilder.close(); - } - } - - infoBuilder.add(VPackValue("shards")); - infoBuilder.openObject(); - infoBuilder.add(VPackValue("vertices")); - infoBuilder.openObject(); - for (auto const& col : _vertexCollections) { - infoBuilder.add(VPackValue(col.first)); - infoBuilder.openArray(); - for (auto const& v : col.second) { - infoBuilder.add(VPackValue(v)); - } - infoBuilder.close(); // this collection - } - infoBuilder.close(); // vertices - - infoBuilder.add(VPackValue("edges")); - infoBuilder.openArray(); - for (auto const& edgeShards : _edgeCollections) { - infoBuilder.openArray(); - for (auto const& e : edgeShards) { - infoBuilder.add(VPackValue(e)); - } - infoBuilder.close(); - } - infoBuilder.close(); // edges - infoBuilder.close(); // shards - - _node->enhanceEngineInfo(infoBuilder); - - infoBuilder.close(); // base - TRI_ASSERT(infoBuilder.isOpenArray()); -} - EngineInfoContainerDBServerServerBased::EngineInfoContainerDBServerServerBased(QueryContext& query) noexcept : _query(query), _shardLocking(query), _lastSnippetId(1) { // NOTE: We need to start with _lastSnippetID > 0. 0 is reserved for GraphNodes diff --git a/arangod/Aql/EngineInfoContainerDBServerServerBased.h b/arangod/Aql/EngineInfoContainerDBServerServerBased.h index a65b544b7ddb..4322df03a72b 100644 --- a/arangod/Aql/EngineInfoContainerDBServerServerBased.h +++ b/arangod/Aql/EngineInfoContainerDBServerServerBased.h @@ -56,61 +56,6 @@ class QueryContext; class QuerySnippet; class EngineInfoContainerDBServerServerBased { - private: - // TODO Temporary. Do not check in!! - // We need to access the TraverserEngineShardList nothing else. - public: - // @brief Local struct to create the - // information required to build traverser engines - // on DB servers. - class TraverserEngineShardLists { - public: - TraverserEngineShardLists(GraphNode const*, ServerID const& server, - std::unordered_map const& shardMapping, - QueryContext& query); - - ~TraverserEngineShardLists() = default; - - void serializeIntoBuilder(arangodb::velocypack::Builder& infoBuilder) const; - - bool hasShard() const { return _hasShard; } - - /// inaccessible edge and verte collection names -#ifdef USE_ENTERPRISE - std::set inaccessibleCollNames() const { - return _inaccessible; - } -#endif - - private: - std::vector getAllLocalShards(std::unordered_map const& shardMapping, - ServerID const& server, - std::shared_ptr> shardIds, - bool colIsSatellite); - - private: - // The graph node we need to serialize - GraphNode const* _node; - - // Flag if we found any shard for the given server. - // If not serializeToBuilder will be a noop - bool _hasShard; - - // Mapping for edge collections to shardIds. - // We have to retain the ordering of edge collections, all - // vectors of these in one run need to have identical size. - // This is because the conditions to query those edges have the - // same ordering. - std::vector> _edgeCollections; - - // Mapping for vertexCollections to shardIds. - std::unordered_map> _vertexCollections; - -#ifdef USE_ENTERPRISE - std::set _inaccessible; -#endif - }; - public: explicit EngineInfoContainerDBServerServerBased(QueryContext& query) noexcept; diff --git a/arangod/Aql/TraverserEngineShardLists.cpp b/arangod/Aql/TraverserEngineShardLists.cpp new file mode 100644 index 000000000000..d488677a64fe --- /dev/null +++ b/arangod/Aql/TraverserEngineShardLists.cpp @@ -0,0 +1,149 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2021-2021 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +//////////////////////////////////////////////////////////////////////////////// + +#include "TraverserEngineShardLists.h" + +#include "Aql/GraphNode.h" +#include "Aql/QueryContext.h" +#include "Graph/BaseOptions.h" + +using namespace arangodb; +using namespace arangodb::aql; + +TraverserEngineShardLists::TraverserEngineShardLists( + GraphNode const* node, ServerID const& server, + std::unordered_map const& shardMapping, QueryContext& query) + : _node(node), _hasShard(false) { + auto const& edges = _node->edgeColls(); + TRI_ASSERT(!edges.empty()); + auto const& restrictToShards = query.queryOptions().restrictToShards; +#ifdef USE_ENTERPRISE + transaction::Methods trx{query.newTrxContext()}; +#endif + // Extract the local shards for edge collections. + for (auto const& col : edges) { +#ifdef USE_ENTERPRISE + if (trx.isInaccessibleCollection(col->id())) { + _inaccessible.insert(col->name()); + _inaccessible.insert(std::to_string(col->id().id())); + } +#endif + _edgeCollections.emplace_back( + getAllLocalShards(shardMapping, server, col->shardIds(restrictToShards), + col->isSatellite() && node->isSmart())); + } + // Extract vertices + auto const& vertices = _node->vertexColls(); + // Guaranteed by addGraphNode, this will inject vertex collections + // in anonymous graph case + // It might in fact be empty, if we only have edge collections in a graph. + // Or if we guarantee to never read vertex data. + for (auto const& col : vertices) { +#ifdef USE_ENTERPRISE + if (trx.isInaccessibleCollection(col->id())) { + _inaccessible.insert(col->name()); + _inaccessible.insert(std::to_string(col->id().id())); + } +#endif + auto shards = getAllLocalShards(shardMapping, server, col->shardIds(restrictToShards), + col->isSatellite() && node->isSmart()); + _vertexCollections.try_emplace(col->name(), std::move(shards)); + } +} + +std::vector TraverserEngineShardLists::getAllLocalShards( + std::unordered_map const& shardMapping, ServerID const& server, + std::shared_ptr> shardIds, bool colIsSatellite) { + std::vector localShards; + for (auto const& shard : *shardIds) { + auto const& it = shardMapping.find(shard); + TRI_ASSERT(it != shardMapping.end()); + if (it->second == server) { + localShards.emplace_back(shard); + // Guaranteed that the traversal will be executed on this server. + _hasShard = true; + } else if (colIsSatellite) { + // The satellite does not force run of a traversal here. + localShards.emplace_back(shard); + } + } + return localShards; +} + +void TraverserEngineShardLists::serializeIntoBuilder( + VPackBuilder& infoBuilder) const { + TRI_ASSERT(_hasShard); + TRI_ASSERT(infoBuilder.isOpenArray()); + infoBuilder.openObject(); + { + // Options + infoBuilder.add(VPackValue("options")); + graph::BaseOptions* opts = _node->options(); + opts->buildEngineInfo(infoBuilder); + } + { + // Variables + std::vector vars; + _node->getConditionVariables(vars); + if (!vars.empty()) { + infoBuilder.add(VPackValue("variables")); + infoBuilder.openArray(); + for (auto v : vars) { + v->toVelocyPack(infoBuilder); + } + infoBuilder.close(); + } + } + + infoBuilder.add(VPackValue("shards")); + infoBuilder.openObject(); + infoBuilder.add(VPackValue("vertices")); + infoBuilder.openObject(); + for (auto const& col : _vertexCollections) { + infoBuilder.add(VPackValue(col.first)); + infoBuilder.openArray(); + for (auto const& v : col.second) { + infoBuilder.add(VPackValue(v)); + } + infoBuilder.close(); // this collection + } + infoBuilder.close(); // vertices + + infoBuilder.add(VPackValue("edges")); + infoBuilder.openArray(); + for (auto const& edgeShards : _edgeCollections) { + infoBuilder.openArray(); + for (auto const& e : edgeShards) { + infoBuilder.add(VPackValue(e)); + } + infoBuilder.close(); + } + infoBuilder.close(); // edges + infoBuilder.close(); // shards + + _node->enhanceEngineInfo(infoBuilder); + + infoBuilder.close(); // base + TRI_ASSERT(infoBuilder.isOpenArray()); +} + + diff --git a/arangod/Aql/TraverserEngineShardLists.h b/arangod/Aql/TraverserEngineShardLists.h new file mode 100644 index 000000000000..86ee8f15c085 --- /dev/null +++ b/arangod/Aql/TraverserEngineShardLists.h @@ -0,0 +1,87 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2021 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +//////////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include "Cluster/ClusterTypes.h" + +#include +#include +#include + +namespace arangodb { +namespace aql { +class GraphNode; +class QueryContext; + +// @brief Struct to create the +// information required to build traverser engines +// on DB servers. +class TraverserEngineShardLists { + public: + TraverserEngineShardLists(GraphNode const*, ServerID const& server, + std::unordered_map const& shardMapping, + QueryContext& query); + + ~TraverserEngineShardLists() = default; + + void serializeIntoBuilder(arangodb::velocypack::Builder& infoBuilder) const; + + bool hasShard() const { return _hasShard; } + + /// inaccessible edge and verte collection names +#ifdef USE_ENTERPRISE + std::set inaccessibleCollNames() const { return _inaccessible; } +#endif + + private: + std::vector getAllLocalShards(std::unordered_map const& shardMapping, + ServerID const& server, + std::shared_ptr> shardIds, + bool colIsSatellite); + + private: + // The graph node we need to serialize + GraphNode const* _node; + + // Flag if we found any shard for the given server. + // If not serializeToBuilder will be a noop + bool _hasShard; + + // Mapping for edge collections to shardIds. + // We have to retain the ordering of edge collections, all + // vectors of these in one run need to have identical size. + // This is because the conditions to query those edges have the + // same ordering. + std::vector> _edgeCollections; + + // Mapping for vertexCollections to shardIds. + std::unordered_map> _vertexCollections; + +#ifdef USE_ENTERPRISE + std::set _inaccessible; +#endif +}; + +} // namespace aql +} // namespace arangodb diff --git a/arangod/CMakeLists.txt b/arangod/CMakeLists.txt index 411fb535f1e5..ad107f1c784b 100644 --- a/arangod/CMakeLists.txt +++ b/arangod/CMakeLists.txt @@ -421,6 +421,7 @@ set(LIB_ARANGO_AQL_SOURCES Aql/SubqueryStartExecutor.cpp Aql/Timing.cpp Aql/TraversalConditionFinder.cpp + Aql/TraverserEngineShardLists.cpp Aql/TraversalExecutor.cpp Aql/TraversalNode.cpp Aql/UnsortedGatherExecutor.cpp From 5da8cd6bdadc6329a2715ceeadb6c183eb4556fa Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Fri, 20 Aug 2021 10:46:39 +0200 Subject: [PATCH 27/72] Made Method call lower case --- arangod/ClusterEngine/ClusterIndexFactory.cpp | 4 ++-- arangod/ClusterEngine/ClusterIndexFactory.h | 2 +- tests/Mocks/StorageEngineMock.cpp | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/arangod/ClusterEngine/ClusterIndexFactory.cpp b/arangod/ClusterEngine/ClusterIndexFactory.cpp index bab36516d24c..db957efedd7f 100644 --- a/arangod/ClusterEngine/ClusterIndexFactory.cpp +++ b/arangod/ClusterEngine/ClusterIndexFactory.cpp @@ -154,7 +154,7 @@ struct PrimaryIndexFactory : public DefaultIndexFactory { namespace arangodb { -void ClusterIndexFactory::LinkIndexFactories(application_features::ApplicationServer& server, +void ClusterIndexFactory::linkIndexFactories(application_features::ApplicationServer& server, IndexFactory& factory) { static const EdgeIndexFactory edgeIndexFactory(server, "edge"); static const DefaultIndexFactory fulltextIndexFactory(server, "fulltext"); @@ -181,7 +181,7 @@ void ClusterIndexFactory::LinkIndexFactories(application_features::ApplicationSe ClusterIndexFactory::ClusterIndexFactory(application_features::ApplicationServer& server) : IndexFactory(server) { - LinkIndexFactories(server, *this); + linkIndexFactories(server, *this); } /// @brief index name aliases (e.g. "persistent" => "hash", "skiplist" => diff --git a/arangod/ClusterEngine/ClusterIndexFactory.h b/arangod/ClusterEngine/ClusterIndexFactory.h index ee4f683c762f..ab5d8178f673 100644 --- a/arangod/ClusterEngine/ClusterIndexFactory.h +++ b/arangod/ClusterEngine/ClusterIndexFactory.h @@ -29,7 +29,7 @@ namespace arangodb { class ClusterIndexFactory final : public IndexFactory { public: - static void LinkIndexFactories(application_features::ApplicationServer& server, + static void linkIndexFactories(application_features::ApplicationServer& server, IndexFactory& factory); explicit ClusterIndexFactory(application_features::ApplicationServer&); ~ClusterIndexFactory() = default; diff --git a/tests/Mocks/StorageEngineMock.cpp b/tests/Mocks/StorageEngineMock.cpp index 92b96fbcabd7..bbe8154ffca3 100644 --- a/tests/Mocks/StorageEngineMock.cpp +++ b/tests/Mocks/StorageEngineMock.cpp @@ -483,7 +483,7 @@ struct IndexFactoryMock : arangodb::IndexFactory { IndexFactoryMock(arangodb::application_features::ApplicationServer& server, bool injectClusterIndexes) : IndexFactory(server) { if (injectClusterIndexes) { - arangodb::ClusterIndexFactory::LinkIndexFactories(server, *this); + arangodb::ClusterIndexFactory::linkIndexFactories(server, *this); } } From 8d13644e7f484f5f5dedf5782b27c109fa65d35c Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Fri, 20 Aug 2021 10:51:15 +0200 Subject: [PATCH 28/72] Removed leftover comment --- arangod/Aql/ExecutionEngine.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/arangod/Aql/ExecutionEngine.cpp b/arangod/Aql/ExecutionEngine.cpp index ca4d867ad79a..08fdf58d845d 100644 --- a/arangod/Aql/ExecutionEngine.cpp +++ b/arangod/Aql/ExecutionEngine.cpp @@ -644,7 +644,6 @@ void ExecutionEngine::instantiateFromPlan(Query& query, ExecutionEngine* engine = nullptr; #ifdef USE_ENTERPRISE bool const pushToSingleServer = plan.hasAppliedRule( - // Feature HybridSmartGraphs: Check here. static_cast(OptimizerRule::RuleLevel::clusterOneShardRule)); #else bool const pushToSingleServer = false; From 5bdc99866c566ded1fe9ecc3a8a659056d304d15 Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Fri, 20 Aug 2021 14:49:16 +0200 Subject: [PATCH 29/72] Renamed smartSearchNew -> smartSearch. The new was only temporary. --- arangod/Cluster/TraverserEngine.cpp | 2 +- arangod/Cluster/TraverserEngine.h | 6 ++--- .../InternalRestTraverserHandler.cpp | 26 +++---------------- 3 files changed, 7 insertions(+), 27 deletions(-) diff --git a/arangod/Cluster/TraverserEngine.cpp b/arangod/Cluster/TraverserEngine.cpp index 04efe6701a8c..24d339db9903 100644 --- a/arangod/Cluster/TraverserEngine.cpp +++ b/arangod/Cluster/TraverserEngine.cpp @@ -411,6 +411,6 @@ TraverserEngine::TraverserEngine(TRI_vocbase_t& vocbase, TraverserEngine::~TraverserEngine() = default; -void TraverserEngine::smartSearchNew(VPackSlice, VPackBuilder&) { +void TraverserEngine::smartSearch(VPackSlice, VPackBuilder&) { THROW_ARANGO_EXCEPTION(TRI_ERROR_ONLY_ENTERPRISE); } diff --git a/arangod/Cluster/TraverserEngine.h b/arangod/Cluster/TraverserEngine.h index f4d0149f788b..f82e689f9c32 100644 --- a/arangod/Cluster/TraverserEngine.h +++ b/arangod/Cluster/TraverserEngine.h @@ -115,8 +115,8 @@ class BaseTraverserEngine : public BaseEngine { graph::EdgeCursor* getCursor(arangodb::velocypack::StringRef nextVertex, uint64_t currentDepth); - virtual void smartSearchNew(arangodb::velocypack::Slice, - arangodb::velocypack::Builder&) = 0; + virtual void smartSearch(arangodb::velocypack::Slice, + arangodb::velocypack::Builder&) = 0; EngineType getType() const override { return TRAVERSER; } @@ -173,7 +173,7 @@ class TraverserEngine : public BaseTraverserEngine { ~TraverserEngine(); - void smartSearchNew(arangodb::velocypack::Slice, arangodb::velocypack::Builder&) override; + void smartSearch(arangodb::velocypack::Slice, arangodb::velocypack::Builder&) override; }; } // namespace traverser diff --git a/arangod/InternalRestHandler/InternalRestTraverserHandler.cpp b/arangod/InternalRestHandler/InternalRestTraverserHandler.cpp index 82b5913177c0..ed2ff77f5cc7 100644 --- a/arangod/InternalRestHandler/InternalRestTraverserHandler.cpp +++ b/arangod/InternalRestHandler/InternalRestTraverserHandler.cpp @@ -227,7 +227,8 @@ void InternalRestTraverserHandler::queryEngine() { return; } engine->getVertexData(keysSlice, result, !depthSlice.isNone()); - } else if (option == "smartSearch") { + } else if (option == "smartSearch" || option == "smartSearchBFS" || + option == "smartSearchWeighted") { if (engine->getType() != BaseEngine::EngineType::TRAVERSER) { generateError(ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER, "this engine does not support the requested operation."); @@ -236,28 +237,7 @@ void InternalRestTraverserHandler::queryEngine() { // Safe cast BaseTraverserEngines are all of type TRAVERSER auto eng = static_cast(engine); TRI_ASSERT(eng != nullptr); - eng->smartSearchNew(body, result); // TODO: Rename/Refactor. smartSearchNew does both (DFS & BFS) - } else if (option == "smartSearchBFS") { - if (engine->getType() != BaseEngine::EngineType::TRAVERSER) { - generateError(ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER, - "this engine does not support the requested operation."); - return; - } - // Safe cast BaseTraverserEngines are all of type TRAVERSER - auto eng = static_cast(engine); - TRI_ASSERT(eng != nullptr); - - eng->smartSearchNew(body, result); // TODO: Rename/Refactor. smartSearchNew does both (DFS & BFS) - } else if (option == "smartSearchWeighted") { - if (engine->getType() != BaseEngine::EngineType::TRAVERSER) { - generateError(ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER, - "this engine does not support the requested operation."); - return; - } - // Safe cast BaseTraverserEngines are all of type TRAVERSER - auto eng = static_cast(engine); - TRI_ASSERT(eng != nullptr); - eng->smartSearchNew(body, result); + eng->smartSearch(body, result); } else { // PATH Info wrong other error generateError(ResponseCode::NOT_FOUND, TRI_ERROR_HTTP_NOT_FOUND, ""); From b09081545f56eeff99c7e021fe479e7a2800e766 Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Fri, 20 Aug 2021 14:51:39 +0200 Subject: [PATCH 30/72] Simplified assertion --- arangod/Aql/ShardLocking.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arangod/Aql/ShardLocking.cpp b/arangod/Aql/ShardLocking.cpp index db13a958695f..390c4235c578 100644 --- a/arangod/Aql/ShardLocking.cpp +++ b/arangod/Aql/ShardLocking.cpp @@ -63,7 +63,7 @@ void ShardLocking::addNode(ExecutionNode const* baseNode, size_t snippetId, } auto const graphIsUsedAsSatellite = graphNode->isUsedAsSatellite(); auto const isUsedAsSatellite = [&](auto const& col) { - return graphIsUsedAsSatellite || (pushToSingleServer && col->isSatellite()) || (col->isSatellite() && graphNode->isSmart()); + return graphIsUsedAsSatellite || (col->isSatellite() && (pushToSingleServer || graphNode->isSmart())); }; // Add all Edge Collections to the Transactions, Traversals do never write for (auto const& col : graphNode->edgeColls()) { From 7ae0ca15f8743e4c48f5afeac2b8578193dbaf58 Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Fri, 20 Aug 2021 14:58:07 +0200 Subject: [PATCH 31/72] Removed Development comments --- arangod/Aql/QuerySnippet.cpp | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/arangod/Aql/QuerySnippet.cpp b/arangod/Aql/QuerySnippet.cpp index d26f5a395aba..493845de2757 100644 --- a/arangod/Aql/QuerySnippet.cpp +++ b/arangod/Aql/QuerySnippet.cpp @@ -686,19 +686,12 @@ auto QuerySnippet::prepareFirstBranch( } } - // TODO: We need to exclude DBServers which only do have Satellite - // collections only This server is not allowed to receive a setup call - - // 1.) Satellites: 1x shard exists (681) -> not landing in myExpFinal - // => 2x shards vertex collections -> DB1 v: [s1, s2] , DB2 [] - // 2.) - #ifdef ARANGODB_ENABLE_MAINTAINER_MODE // additional verification checks for Disjoint SmartGraphs if (localGraphNode->isDisjoint()) { if (!myExpFinal.empty()) { size_t numberOfShards = myExpFinal.begin()->second.size(); - // We need one expansion for every collection in the Graph (-1 per satellite) + // We need one expansion for every non-satellite collection in the Graph size_t amountOfNonSatellites = 0; for (auto const& col : localGraphNode->collections()) { if (!col->isSatellite()) { From fdec8008e5d6fce48f7af96f57b68bc86c686ad2 Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Fri, 20 Aug 2021 15:06:16 +0200 Subject: [PATCH 32/72] Try to reduce deplicate code --- .../Graph/Cache/RefactoredTraverserCache.cpp | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/arangod/Graph/Cache/RefactoredTraverserCache.cpp b/arangod/Graph/Cache/RefactoredTraverserCache.cpp index 8fb650870c1b..7187c9990c9a 100644 --- a/arangod/Graph/Cache/RefactoredTraverserCache.cpp +++ b/arangod/Graph/Cache/RefactoredTraverserCache.cpp @@ -111,21 +111,14 @@ bool RefactoredTraverserCache::appendEdge(EdgeDocumentToken const& idToken, ->read(_trx, idToken.localDocumentId(), [&](LocalDocumentId const&, VPackSlice edge) -> bool { if (onlyId) { - // NOTE: Do not count this as Primary Index Scan, we - // counted it in the edge Index before copying... - if constexpr (std::is_same_v) { - result = aql::AqlValue(edge.get(StaticStrings::IdString).copyString()); - } else if constexpr (std::is_same_v) { - result.add(edge.get(StaticStrings::IdString)); - } - } else { - // NOTE: Do not count this as Primary Index Scan, we - // counted it in the edge Index before copying... - if constexpr (std::is_same_v) { - result = aql::AqlValue(edge); - } else if constexpr (std::is_same_v) { - result.add(edge); - } + edge = edge.get(StaticStrings::IdString); + } + // NOTE: Do not count this as Primary Index Scan, we + // counted it in the edge Index before copying... + if constexpr (std::is_same_v) { + result = aql::AqlValue(edge); + } else if constexpr (std::is_same_v) { + result.add(edge); } return true; }) From 159c5d1848e354e64f964fed5e99c854420bc1a1 Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Fri, 20 Aug 2021 15:08:07 +0200 Subject: [PATCH 33/72] Removed TODOs, we actually want to keep the version we have now --- arangod/Graph/Enumerators/OneSidedEnumeratorInterface.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/arangod/Graph/Enumerators/OneSidedEnumeratorInterface.h b/arangod/Graph/Enumerators/OneSidedEnumeratorInterface.h index ef79aabfb903..82c342dc009c 100644 --- a/arangod/Graph/Enumerators/OneSidedEnumeratorInterface.h +++ b/arangod/Graph/Enumerators/OneSidedEnumeratorInterface.h @@ -43,8 +43,6 @@ class TraversalStats; namespace graph { -// TODO Temporary, just Make everything compile -// IMPORTANT - DO NOT TEMPLATE THIS CLASS class PathResultInterface { public: PathResultInterface() {} @@ -53,7 +51,6 @@ class PathResultInterface { virtual auto toVelocyPack(arangodb::velocypack::Builder& builder) -> void = 0; }; -// IMPORTANT - DO NOT TEMPLATE THIS CLASS class TraversalEnumerator { public: using VertexRef = arangodb::velocypack::HashedStringRef; From d7b40bd43061af7ecd9886b265dea0cfdcbb1b43 Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Fri, 20 Aug 2021 15:18:41 +0200 Subject: [PATCH 34/72] Removed incorrect TODOs --- arangod/Graph/Graph.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/arangod/Graph/Graph.cpp b/arangod/Graph/Graph.cpp index 03252d83c794..9fe712aef3c5 100644 --- a/arangod/Graph/Graph.cpp +++ b/arangod/Graph/Graph.cpp @@ -116,7 +116,6 @@ Graph::Graph(velocypack::Slice const& slice, ServerDefaults const& serverDefault TRI_ASSERT(!_rev.empty()); if (slice.hasKey(StaticStrings::GraphEdgeDefinitions)) { - // TODO Feature HybridSmartGraphs: Check why we're landing here and not in SmartGraphEE - Cleanup! if (slice.isObject()) { if (slice.hasKey(StaticStrings::GraphSatellites) && slice.get(StaticStrings::GraphSatellites).isArray()) { @@ -159,7 +158,6 @@ Graph::Graph(TRI_vocbase_t& vocbase, std::string&& graphName, TRI_ASSERT(_rev.empty()); if (info.hasKey(StaticStrings::GraphEdgeDefinitions)) { - // TODO Feature HybridSmartGraphs: Check why we're landing here and not in SmartGraphEE - Cleanup! if (options.isObject()) { if (options.hasKey(StaticStrings::GraphSatellites) && options.get(StaticStrings::GraphSatellites).isArray()) { From cdd889999145e0fc27324623e4a721112ba1e0e2 Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Fri, 20 Aug 2021 15:44:22 +0200 Subject: [PATCH 35/72] Removed double exclude of enterprise version. --- arangod/Graph/BreadthFirstEnumerator.cpp | 2 -- arangod/Graph/PathEnumerator.cpp | 2 -- arangod/Graph/WeightedEnumerator.cpp | 2 -- 3 files changed, 6 deletions(-) diff --git a/arangod/Graph/BreadthFirstEnumerator.cpp b/arangod/Graph/BreadthFirstEnumerator.cpp index 20d0f221d229..e1cb0186a82a 100644 --- a/arangod/Graph/BreadthFirstEnumerator.cpp +++ b/arangod/Graph/BreadthFirstEnumerator.cpp @@ -161,11 +161,9 @@ bool BreadthFirstEnumerator::next() { } } -#ifdef USE_ENTERPRISE if (!validDisjointPath(nextIdx, vId)) { return; } -#endif growStorage(); TRI_ASSERT(_schreier.capacity() > _schreier.size()); diff --git a/arangod/Graph/PathEnumerator.cpp b/arangod/Graph/PathEnumerator.cpp index a08a60306401..32fff7d2fa9a 100644 --- a/arangod/Graph/PathEnumerator.cpp +++ b/arangod/Graph/PathEnumerator.cpp @@ -296,11 +296,9 @@ bool DepthFirstEnumerator::next() { } } -#ifdef USE_ENTERPRISE if (!validDisjointPath()) { return; } -#endif _enumeratedPath.pushEdge(eid); foundPath = true; diff --git a/arangod/Graph/WeightedEnumerator.cpp b/arangod/Graph/WeightedEnumerator.cpp index a455bd148d01..2334ef562fec 100644 --- a/arangod/Graph/WeightedEnumerator.cpp +++ b/arangod/Graph/WeightedEnumerator.cpp @@ -75,11 +75,9 @@ bool WeightedEnumerator::expandEdge(NextEdge nextEdge) { // getSingleVertex does nothing but that and checking conditions // However, for global unique vertexes, we need the vertex getter. if (_traverser->getVertex(toVertex, nextEdge.depth)) { -#ifdef USE_ENTERPRISE if (!validDisjointPath(nextEdge.fromIndex, toVertex)) { return false; } -#endif if (_opts->uniqueVertices == TraverserOptions::UniquenessLevel::PATH) { if (pathContainsVertex(nextEdge.fromIndex, toVertex)) { // This vertex is on the path. From 9240c2f51e9ad052fc319c14838af434cb14cd58 Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Fri, 20 Aug 2021 16:19:50 +0200 Subject: [PATCH 36/72] Worked on leftover todos --- arangod/Graph/PathManagement/SingleProviderPathResult.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/arangod/Graph/PathManagement/SingleProviderPathResult.h b/arangod/Graph/PathManagement/SingleProviderPathResult.h index cf2ff82dd8f6..eb03dbe5b620 100644 --- a/arangod/Graph/PathManagement/SingleProviderPathResult.h +++ b/arangod/Graph/PathManagement/SingleProviderPathResult.h @@ -27,7 +27,6 @@ #include #include "Containers/HashSet.h" -// TODO Temporary include #include "Graph/Enumerators/OneSidedEnumeratorInterface.h" #include "Graph/Providers/TypeAliases.h" #include "Graph/TraverserOptions.h" @@ -45,7 +44,6 @@ namespace graph { template class SingleProviderPathResult : public PathResultInterface { - using VertexRef = arangodb::velocypack::HashedStringRef; // TODO might remove public: SingleProviderPathResult(Step step, ProviderType& provider, PathStoreType& store); From f6a876e5bd24314dc1fc40f5a1ac100320a7d29d Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Fri, 20 Aug 2021 16:50:10 +0200 Subject: [PATCH 37/72] Improved TODO comment to why we cannot change it right now, but would like to. --- arangod/Graph/Providers/BaseProviderOptions.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arangod/Graph/Providers/BaseProviderOptions.h b/arangod/Graph/Providers/BaseProviderOptions.h index da9da66114b5..040745529d10 100644 --- a/arangod/Graph/Providers/BaseProviderOptions.h +++ b/arangod/Graph/Providers/BaseProviderOptions.h @@ -56,7 +56,9 @@ struct IndexAccessor { aql::AstNode* _indexCondition; std::optional _memberToUpdate; - // TODO: This needs to be changed BEFORE merge + // Note: We would prefer to have this a unique_ptr here. + // However the IndexAccessor is used in std::vector + // which then refuses to compile (deleted copy constructor) std::shared_ptr _expression; size_t _cursorId; }; From a89ee576ef48e61ebb1fc2c5a4898f13fafa03b7 Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Fri, 20 Aug 2021 17:05:49 +0200 Subject: [PATCH 38/72] Removed unused Member --- arangod/Graph/Providers/ClusterProvider.h | 1 - 1 file changed, 1 deletion(-) diff --git a/arangod/Graph/Providers/ClusterProvider.h b/arangod/Graph/Providers/ClusterProvider.h index d6efd0c53947..b4cb441f9b2a 100644 --- a/arangod/Graph/Providers/ClusterProvider.h +++ b/arangod/Graph/Providers/ClusterProvider.h @@ -140,7 +140,6 @@ class ClusterProvider { Vertex _vertex; Edge _edge; bool _fetched; - size_t _localSchreierIndex = std::numeric_limits::max(); // to be removed later }; public: From e1fe7bb81d10a90e558850918810bf7b48e0dc1e Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Fri, 20 Aug 2021 17:20:05 +0200 Subject: [PATCH 39/72] Removed dead code --- arangod/Graph/Providers/SingleServerProvider.cpp | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/arangod/Graph/Providers/SingleServerProvider.cpp b/arangod/Graph/Providers/SingleServerProvider.cpp index 2c0aaf248cc6..d7552f3eb0ea 100644 --- a/arangod/Graph/Providers/SingleServerProvider.cpp +++ b/arangod/Graph/Providers/SingleServerProvider.cpp @@ -41,15 +41,6 @@ using namespace arangodb; using namespace arangodb::graph; -// TODO: remove -/*namespace arangodb { -namespace graph { -auto operator<<(std::ostream& out, typename SingleServerProvider::Step const& -step) -> std::ostream& { out << step._vertex.getID(); return out; -} -} // namespace graph -} // namespace arangodb*/ - template void SingleServerProvider::addEdgeToBuilder(typename Step::Edge const& edge, arangodb::velocypack::Builder& builder) { @@ -83,7 +74,8 @@ SingleServerProvider::SingleServerProvider(arangodb::aql::QueryContext& qu _cache(_trx.get(), &queryContext, resourceMonitor, _stats, _opts.collectionToShardMap()), _stats{} { - // activateCache(false); // TODO CHECK RefactoredTraverserCache (will be discussed in the future, need to do benchmarks if affordable) + // TODO CHECK RefactoredTraverserCache (will be discussed in the future, need to do benchmarks if affordable) + // activateCache(false); _cursor = buildCursor(opts.expressionContext()); } From b74ca22b421aacd7a3c9e255e6536ce9ae47cd75 Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Fri, 20 Aug 2021 17:20:14 +0200 Subject: [PATCH 40/72] Removed dead code --- arangod/Graph/Queues/WeightedQueue.h | 1 - 1 file changed, 1 deletion(-) diff --git a/arangod/Graph/Queues/WeightedQueue.h b/arangod/Graph/Queues/WeightedQueue.h index 18bbbe5ae1da..3ba44b0fa22d 100644 --- a/arangod/Graph/Queues/WeightedQueue.h +++ b/arangod/Graph/Queues/WeightedQueue.h @@ -124,7 +124,6 @@ class WeightedQueue { /// @brief queue datastore /// Note: Mutable is a required for hasProcessableElement right now which is const. We can easily make it non const here. mutable std::vector _queue; - // std::priority_queue, WeightedComparator> _queue; /// @brief query context arangodb::ResourceMonitor& _resourceMonitor; From 30776f123195f4e178160cfa517779b335192f5f Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Fri, 20 Aug 2021 17:20:26 +0200 Subject: [PATCH 41/72] Fixed incude style --- arangod/Graph/Steps/SingleServerProviderStep.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arangod/Graph/Steps/SingleServerProviderStep.h b/arangod/Graph/Steps/SingleServerProviderStep.h index cb91d0f30f2a..d952e610706f 100644 --- a/arangod/Graph/Steps/SingleServerProviderStep.h +++ b/arangod/Graph/Steps/SingleServerProviderStep.h @@ -22,7 +22,7 @@ /// @author Heiko Kernbach //////////////////////////////////////////////////////////////////////////////// -#include +#include "Transaction/Methods.h" #include "Graph/EdgeDocumentToken.h" #include "Graph/Providers/BaseStep.h" #include "Graph/Providers/TypeAliases.h" From 1dbcc0e8313fb9d8f45070238729db1c33b55a29 Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Fri, 20 Aug 2021 17:29:14 +0200 Subject: [PATCH 42/72] Removed TODO, this is not important, we can replace pair, by struct. --- arangod/Aql/KShortestPathsNode.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/arangod/Aql/KShortestPathsNode.cpp b/arangod/Aql/KShortestPathsNode.cpp index fe5486d8809c..cccc9c963aa3 100644 --- a/arangod/Aql/KShortestPathsNode.cpp +++ b/arangod/Aql/KShortestPathsNode.cpp @@ -344,13 +344,11 @@ std::unique_ptr KShortestPathsNode::createBlock( // Create IndexAccessor for BaseProviderOptions (TODO: Location need to // be changed in the future) create BaseProviderOptions - // TODO FIXME START - only tmp workaround - we'll provide an empty depth based index info (also add an alias for large type) std::pair, std::unordered_map>> usedIndexes{}; usedIndexes.first = buildUsedIndexes(); std::pair, std::unordered_map>> reversedUsedIndexes{}; reversedUsedIndexes.first = buildReverseUsedIndexes(); - // TODO FIXME END BaseProviderOptions forwardProviderOptions(opts->tmpVar(), usedIndexes, opts->getExpressionCtx(), From 48e17918dceb08c50f76f70a16b4e4f9f3f6647c Mon Sep 17 00:00:00 2001 From: Heiko Kernbach Date: Fri, 3 Sep 2021 15:59:47 +0200 Subject: [PATCH 43/72] move method into class, make more methods const --- arangod/Graph/TraverserOptions.cpp | 51 ++++++++++++++---------------- arangod/Graph/TraverserOptions.h | 8 +++-- 2 files changed, 29 insertions(+), 30 deletions(-) diff --git a/arangod/Graph/TraverserOptions.cpp b/arangod/Graph/TraverserOptions.cpp index 620e0c8e449b..7ea4c05d0c32 100644 --- a/arangod/Graph/TraverserOptions.cpp +++ b/arangod/Graph/TraverserOptions.cpp @@ -45,25 +45,6 @@ using namespace arangodb::transaction; using namespace arangodb::traverser; using VPackHelper = arangodb::basics::VelocyPackHelper; -namespace { -arangodb::velocypack::StringRef getEdgeDestination(arangodb::velocypack::Slice edge, - arangodb::velocypack::StringRef origin) { - if (edge.isString()) { - return edge.stringRef(); - } - - TRI_ASSERT(edge.isObject()); - auto from = edge.get(arangodb::StaticStrings::FromString); - TRI_ASSERT(from.isString()); - if (from.stringRef() == origin) { - auto to = edge.get(arangodb::StaticStrings::ToString); - TRI_ASSERT(to.isString()); - return to.stringRef(); - } - return from.stringRef(); -} -} // namespace - TraverserOptions::TraverserOptions(arangodb::aql::QueryContext& query) : BaseOptions(query), _baseVertexExpression(nullptr), @@ -750,13 +731,9 @@ auto TraverserOptions::explicitDepthLookupAt() const -> std::unordered_set void { - return; -} +auto TraverserOptions::setDisjoint() -> void { return; } -auto TraverserOptions::isDisjoint() const -> bool { - return false; -} +auto TraverserOptions::isDisjoint() const -> bool { return false; } auto TraverserOptions::isSatelliteLeader() const -> bool { // Can only be called in Enterprise code. @@ -766,6 +743,24 @@ auto TraverserOptions::isSatelliteLeader() const -> bool { } #endif +auto TraverserOptions::getEdgeDestination(arangodb::velocypack::Slice edge, + arangodb::velocypack::StringRef origin) const + -> arangodb::velocypack::StringRef { + if (edge.isString()) { + return edge.stringRef(); + } + + TRI_ASSERT(edge.isObject()); + auto from = edge.get(arangodb::StaticStrings::FromString); + TRI_ASSERT(from.isString()); + if (from.stringRef() == origin) { + auto to = edge.get(arangodb::StaticStrings::ToString); + TRI_ASSERT(to.isString()); + return to.stringRef(); + } + return from.stringRef(); +} + bool TraverserOptions::evaluateVertexExpression(arangodb::velocypack::Slice vertex, uint64_t depth) { arangodb::aql::Expression* expression = nullptr; @@ -783,15 +778,15 @@ bool TraverserOptions::evaluateVertexExpression(arangodb::velocypack::Slice vert } #ifndef USE_ENTERPRISE -bool TraverserOptions::checkSmartDestination(VPackSlice edge, velocypack::StringRef sourceVertex) { +bool TraverserOptions::checkSmartDestination(VPackSlice edge, velocypack::StringRef sourceVertex) const { return false; } #endif bool TraverserOptions::destinationCollectionAllowed(VPackSlice edge, - velocypack::StringRef sourceVertex) { + velocypack::StringRef sourceVertex) const { if (hasVertexCollectionRestrictions()) { - auto destination = ::getEdgeDestination(edge, sourceVertex); + auto destination = getEdgeDestination(edge, sourceVertex); auto collection = transaction::helpers::extractCollectionFromId(destination); if (std::find(vertexCollections.begin(), vertexCollections.end(), std::string_view(collection.data(), collection.size())) == diff --git a/arangod/Graph/TraverserOptions.h b/arangod/Graph/TraverserOptions.h index 66ed0b21c2c0..73d9e5ce01c3 100644 --- a/arangod/Graph/TraverserOptions.h +++ b/arangod/Graph/TraverserOptions.h @@ -159,9 +159,9 @@ struct TraverserOptions : public graph::BaseOptions { bool evaluateVertexExpression(arangodb::velocypack::Slice, uint64_t); - bool checkSmartDestination(VPackSlice edge, velocypack::StringRef sourceVertex); + bool checkSmartDestination(VPackSlice edge, velocypack::StringRef sourceVertex) const; - bool destinationCollectionAllowed(velocypack::Slice edge, velocypack::StringRef sourceVertex); + bool destinationCollectionAllowed(velocypack::Slice edge, velocypack::StringRef sourceVertex) const; void linkTraverser(arangodb::traverser::ClusterTraverser*); @@ -221,6 +221,10 @@ struct TraverserOptions : public graph::BaseOptions { auto isSatelliteLeader() const -> bool; + auto getEdgeDestination(arangodb::velocypack::Slice edge, + arangodb::velocypack::StringRef origin) const + -> arangodb::velocypack::StringRef; + private: void readProduceInfo(VPackSlice obj); }; From 445b6cf9afaae0681beac1838df52d61125c124e Mon Sep 17 00:00:00 2001 From: Heiko Kernbach Date: Mon, 6 Sep 2021 17:40:39 +0200 Subject: [PATCH 44/72] added two new static strings: key post and prefix --- lib/Basics/StaticStrings.cpp | 2 ++ lib/Basics/StaticStrings.h | 2 ++ 2 files changed, 4 insertions(+) diff --git a/lib/Basics/StaticStrings.cpp b/lib/Basics/StaticStrings.cpp index 304270f067c1..0d535d3d4e45 100644 --- a/lib/Basics/StaticStrings.cpp +++ b/lib/Basics/StaticStrings.cpp @@ -43,6 +43,8 @@ std::string const StaticStrings::IndexGt("gt"); std::string const StaticStrings::AttachmentString("_attachment"); std::string const StaticStrings::IdString("_id"); std::string const StaticStrings::KeyString("_key"); +std::string const StaticStrings::PrefixOfKeyString("_key:"); +std::string const StaticStrings::PostfixOfKeyString(":_key"); std::string const StaticStrings::RevString("_rev"); std::string const StaticStrings::FromString("_from"); std::string const StaticStrings::ToString("_to"); diff --git a/lib/Basics/StaticStrings.h b/lib/Basics/StaticStrings.h index 578d5b3bf80f..c8a7d54ac055 100644 --- a/lib/Basics/StaticStrings.h +++ b/lib/Basics/StaticStrings.h @@ -48,6 +48,8 @@ class StaticStrings { static std::string const AttachmentString; static std::string const IdString; static std::string const KeyString; + static std::string const PrefixOfKeyString; + static std::string const PostfixOfKeyString; static std::string const RevString; static std::string const FromString; static std::string const ToString; From 51a1afe3f79014349b1a659d772c32d968e851fe Mon Sep 17 00:00:00 2001 From: Heiko Kernbach Date: Mon, 6 Sep 2021 17:42:24 +0200 Subject: [PATCH 45/72] more use of static strings --- arangod/Aql/Collection.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arangod/Aql/Collection.cpp b/arangod/Aql/Collection.cpp index e30d0bd39923..c110f38ecde0 100644 --- a/arangod/Aql/Collection.cpp +++ b/arangod/Aql/Collection.cpp @@ -186,7 +186,7 @@ std::vector Collection::shardKeys(bool normalize) const { if (normalize && coll->isSmart() && coll->type() == TRI_COL_TYPE_DOCUMENT) { // smart vertex collection always has ["_key:"] as shard keys TRI_ASSERT(originalKeys.size() == 1); - TRI_ASSERT(originalKeys[0] == "_key:"); + TRI_ASSERT(originalKeys[0] == StaticStrings::PrefixOfKeyString); // now normalize it this to _key return std::vector{StaticStrings::KeyString}; } From 88a731270f7c01a81d6b0f003928187dd35e9f44 Mon Sep 17 00:00:00 2001 From: Heiko Kernbach Date: Mon, 6 Sep 2021 18:04:15 +0200 Subject: [PATCH 46/72] throw in case we fail to inject variables --- arangod/Cluster/TraverserEngine.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arangod/Cluster/TraverserEngine.cpp b/arangod/Cluster/TraverserEngine.cpp index 24d339db9903..3a7a4347b7be 100644 --- a/arangod/Cluster/TraverserEngine.cpp +++ b/arangod/Cluster/TraverserEngine.cpp @@ -302,7 +302,9 @@ void BaseTraverserEngine::injectVariables(VPackSlice variableSlice) { if ((!pair.isArray()) || pair.length() != 2) { // Invalid communication. Skip TRI_ASSERT(false); - continue; + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, + "Internal Traverser communication " + "broken. Failed to inject variables."); } auto varId = arangodb::basics::VelocyPackHelper::getNumericValue(pair.at(0), From 77d1ec12bc8f4d0b05542589cfce46cb132dcbc0 Mon Sep 17 00:00:00 2001 From: Heiko Kernbach Date: Tue, 7 Sep 2021 11:45:55 +0200 Subject: [PATCH 47/72] changed name of a variable to a more descriptive one --- arangod/Aql/TraverserEngineShardLists.cpp | 4 ++-- arangod/Aql/TraverserEngineShardLists.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arangod/Aql/TraverserEngineShardLists.cpp b/arangod/Aql/TraverserEngineShardLists.cpp index d488677a64fe..15d9a2f403e5 100644 --- a/arangod/Aql/TraverserEngineShardLists.cpp +++ b/arangod/Aql/TraverserEngineShardLists.cpp @@ -72,7 +72,7 @@ TraverserEngineShardLists::TraverserEngineShardLists( std::vector TraverserEngineShardLists::getAllLocalShards( std::unordered_map const& shardMapping, ServerID const& server, - std::shared_ptr> shardIds, bool colIsSatellite) { + std::shared_ptr> shardIds, bool allowReadFromFollower) { std::vector localShards; for (auto const& shard : *shardIds) { auto const& it = shardMapping.find(shard); @@ -81,7 +81,7 @@ std::vector TraverserEngineShardLists::getAllLocalShards( localShards.emplace_back(shard); // Guaranteed that the traversal will be executed on this server. _hasShard = true; - } else if (colIsSatellite) { + } else if (allowReadFromFollower) { // The satellite does not force run of a traversal here. localShards.emplace_back(shard); } diff --git a/arangod/Aql/TraverserEngineShardLists.h b/arangod/Aql/TraverserEngineShardLists.h index 86ee8f15c085..1c8271be3ebe 100644 --- a/arangod/Aql/TraverserEngineShardLists.h +++ b/arangod/Aql/TraverserEngineShardLists.h @@ -58,7 +58,7 @@ class TraverserEngineShardLists { std::vector getAllLocalShards(std::unordered_map const& shardMapping, ServerID const& server, std::shared_ptr> shardIds, - bool colIsSatellite); + bool allowReadFromFollower); private: // The graph node we need to serialize From 38eaeb85d09b3da49adddb12a970811c3877311b Mon Sep 17 00:00:00 2001 From: Heiko Kernbach Date: Tue, 7 Sep 2021 12:35:30 +0200 Subject: [PATCH 48/72] refactored _collectionToShard --- arangod/Aql/GraphNode.cpp | 5 ++--- arangod/Aql/GraphNode.h | 8 ++++---- arangod/Aql/QuerySnippet.cpp | 2 +- arangod/Graph/BaseOptions.cpp | 8 ++++++-- arangod/Graph/BaseOptions.h | 2 +- 5 files changed, 14 insertions(+), 11 deletions(-) diff --git a/arangod/Aql/GraphNode.cpp b/arangod/Aql/GraphNode.cpp index 229abab21b89..170bd689ad64 100644 --- a/arangod/Aql/GraphNode.cpp +++ b/arangod/Aql/GraphNode.cpp @@ -398,8 +398,7 @@ GraphNode::GraphNode(ExecutionPlan* plan, arangodb::velocypack::Slice const& bas "graph needs a translation from collection to shard names"); } for (auto const& item : VPackObjectIterator(collectionToShard)) { - _collectionToShard.insert({item.key.copyString(), - std::vector{item.value.copyString()}}); + _collectionToShard.insert({item.key.copyString(), item.value.copyString()}); } // Out variables @@ -516,7 +515,7 @@ std::string const& GraphNode::collectionToShardName(std::string const& collName) "unable to find shard '" + collName + "' in query shard map"); } - return found->second.front(); + return found->second; } void GraphNode::doToVelocyPack(VPackBuilder& nodes, unsigned flags) const { diff --git a/arangod/Aql/GraphNode.h b/arangod/Aql/GraphNode.h index 1b79452bd187..669702402aaf 100644 --- a/arangod/Aql/GraphNode.h +++ b/arangod/Aql/GraphNode.h @@ -189,13 +189,13 @@ class GraphNode : public ExecutionNode { void injectVertexCollection(aql::Collection& other); std::vector collections() const; - void setCollectionToShard(std::unordered_map> const& map) { - _collectionToShard = map; + void resetCollectionToShard() { + _collectionToShard.clear(); } void addCollectionToShard(std::string const& coll, std::string const& shard) { // NOTE: Do not replace this by emplace or insert. // This is also used to overwrite the existing entry. - _collectionToShard[coll] = std::vector{shard}; + _collectionToShard[coll] = shard; } graph::Graph const* graph() const noexcept; @@ -268,7 +268,7 @@ class GraphNode : public ExecutionNode { std::unordered_map _engines; /// @brief list of shards involved, required for one-shard-databases - std::unordered_map> _collectionToShard; + std::unordered_map _collectionToShard; }; } // namespace aql diff --git a/arangod/Aql/QuerySnippet.cpp b/arangod/Aql/QuerySnippet.cpp index 493845de2757..cf2e40e7dc8c 100644 --- a/arangod/Aql/QuerySnippet.cpp +++ b/arangod/Aql/QuerySnippet.cpp @@ -598,7 +598,7 @@ auto QuerySnippet::prepareFirstBranch( // there are no local expansions auto* localGraphNode = ExecutionNode::castTo(exp.node); - localGraphNode->setCollectionToShard({}); // clear previous information + localGraphNode->resetCollectionToShard(); // clear previous information TRI_ASSERT(localGraphNode->isUsedAsSatellite() == exp.isSatellite); diff --git a/arangod/Graph/BaseOptions.cpp b/arangod/Graph/BaseOptions.cpp index d72cec2f3155..db15384856ac 100644 --- a/arangod/Graph/BaseOptions.cpp +++ b/arangod/Graph/BaseOptions.cpp @@ -366,8 +366,12 @@ void BaseOptions::serializeVariables(VPackBuilder& builder) const { } void BaseOptions::setCollectionToShard( - std::unordered_map> const& in) { - _collectionToShard = std::move(in); + std::unordered_map const& in) { + _collectionToShard.clear(); + _collectionToShard.reserve(in.size()); + for (auto const& [key, value] : in) { + _collectionToShard.emplace(key, std::vector{value}); + } } arangodb::transaction::Methods* BaseOptions::trx() const { return &_trx; } diff --git a/arangod/Graph/BaseOptions.h b/arangod/Graph/BaseOptions.h index 1150dbb40f2e..dad32f9f0585 100644 --- a/arangod/Graph/BaseOptions.h +++ b/arangod/Graph/BaseOptions.h @@ -117,7 +117,7 @@ struct BaseOptions { void serializeVariables(arangodb::velocypack::Builder&) const; - void setCollectionToShard(std::unordered_map> const&); + void setCollectionToShard(std::unordered_map const&); bool produceVertices() const { return _produceVertices; } From 1655054fd1f67a7aa11bbbeed93ffb090808ba0b Mon Sep 17 00:00:00 2001 From: Heiko Kernbach Date: Tue, 7 Sep 2021 12:52:46 +0200 Subject: [PATCH 49/72] moved declarations as not needed here --- arangod/Graph/GraphManager.h | 6 ------ 1 file changed, 6 deletions(-) diff --git a/arangod/Graph/GraphManager.h b/arangod/Graph/GraphManager.h index c539370808d8..8f6374a4e370 100644 --- a/arangod/Graph/GraphManager.h +++ b/arangod/Graph/GraphManager.h @@ -167,12 +167,6 @@ class GraphManager { std::pair ensureEnterpriseCollectionSharding( Graph const* graph, bool waitForSync, std::unordered_set& documentCollections) const; - std::pair ensureSmartCollectionSharding( - Graph const* graph, bool waitForSync, - std::unordered_set& documentCollections) const; - std::pair ensureSatelliteCollectionSharding( - Graph const* graph, bool waitForSync, - std::unordered_set& documentCollections) const; #endif Result ensureCollections( From 0bab44770931e2e29d86df1a08fe2b86e32abd55 Mon Sep 17 00:00:00 2001 From: Heiko Date: Tue, 7 Sep 2021 12:55:06 +0200 Subject: [PATCH 50/72] Update tests/Graph/DFSFinderTest.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Tobias Gödderz --- tests/Graph/DFSFinderTest.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/Graph/DFSFinderTest.cpp b/tests/Graph/DFSFinderTest.cpp index c3ad3e87a23c..0d869848ada8 100644 --- a/tests/Graph/DFSFinderTest.cpp +++ b/tests/Graph/DFSFinderTest.cpp @@ -276,7 +276,6 @@ TEST_P(DFSFinderTest, no_path_exists) { // Try again to make sure we stay at non-existing auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); - EXPECT_TRUE(result.isEmpty()); EXPECT_TRUE(finder.isDone()); } { From 8491fb1d97e508695c29622c89cf4baaf1d2a719 Mon Sep 17 00:00:00 2001 From: Heiko Date: Tue, 7 Sep 2021 12:56:12 +0200 Subject: [PATCH 51/72] Update tests/Graph/DFSFinderTest.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Tobias Gödderz --- tests/Graph/DFSFinderTest.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/Graph/DFSFinderTest.cpp b/tests/Graph/DFSFinderTest.cpp index 0d869848ada8..f758b03eced2 100644 --- a/tests/Graph/DFSFinderTest.cpp +++ b/tests/Graph/DFSFinderTest.cpp @@ -311,7 +311,6 @@ TEST_P(DFSFinderTest, path_depth_0) { // Try again to make sure we stay at non-existing auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); - EXPECT_TRUE(result.isEmpty()); EXPECT_TRUE(finder.isDone()); } { From 023882fd75d4ace6159f3a2d7b58d24f1bdaba3e Mon Sep 17 00:00:00 2001 From: Heiko Date: Tue, 7 Sep 2021 12:57:18 +0200 Subject: [PATCH 52/72] Update tests/Graph/DFSFinderTest.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Tobias Gödderz --- tests/Graph/DFSFinderTest.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/Graph/DFSFinderTest.cpp b/tests/Graph/DFSFinderTest.cpp index f758b03eced2..a4b8bbb2d5fb 100644 --- a/tests/Graph/DFSFinderTest.cpp +++ b/tests/Graph/DFSFinderTest.cpp @@ -599,7 +599,6 @@ TEST_P(DFSFinderTest, path_depth_1_to_2_skip) { } { - result.clear(); // Try again to make sure we stay at non-existing auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); From 6eabbb3c9605405f875796ee724bdbb4bec90a15 Mon Sep 17 00:00:00 2001 From: Heiko Date: Tue, 7 Sep 2021 12:57:34 +0200 Subject: [PATCH 53/72] Update tests/Graph/DFSFinderTest.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Tobias Gödderz --- tests/Graph/DFSFinderTest.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/Graph/DFSFinderTest.cpp b/tests/Graph/DFSFinderTest.cpp index a4b8bbb2d5fb..4fc84510cced 100644 --- a/tests/Graph/DFSFinderTest.cpp +++ b/tests/Graph/DFSFinderTest.cpp @@ -645,7 +645,6 @@ TEST_P(DFSFinderTest, path_loop) { result.clear(); auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); - EXPECT_TRUE(result.isEmpty()); EXPECT_TRUE(finder.isDone()); } } From 68c1f536311565107e244e11b965f91d31939163 Mon Sep 17 00:00:00 2001 From: Heiko Date: Tue, 7 Sep 2021 13:00:45 +0200 Subject: [PATCH 54/72] Update tests/Graph/DFSFinderTest.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Tobias Gödderz --- tests/Graph/DFSFinderTest.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/Graph/DFSFinderTest.cpp b/tests/Graph/DFSFinderTest.cpp index 4fc84510cced..76c53e35d1f6 100644 --- a/tests/Graph/DFSFinderTest.cpp +++ b/tests/Graph/DFSFinderTest.cpp @@ -346,7 +346,6 @@ TEST_P(DFSFinderTest, path_depth_1) { // Try again to make sure we stay at non-existing auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); - EXPECT_TRUE(result.isEmpty()); EXPECT_TRUE(finder.isDone()); } From c1a38d6a4de0b202ff18d4573aeed93e977a7e4a Mon Sep 17 00:00:00 2001 From: Heiko Date: Tue, 7 Sep 2021 13:01:49 +0200 Subject: [PATCH 55/72] Update tests/Graph/DFSFinderTest.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Tobias Gödderz --- tests/Graph/DFSFinderTest.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/Graph/DFSFinderTest.cpp b/tests/Graph/DFSFinderTest.cpp index 76c53e35d1f6..ddcc251e408d 100644 --- a/tests/Graph/DFSFinderTest.cpp +++ b/tests/Graph/DFSFinderTest.cpp @@ -382,7 +382,6 @@ TEST_P(DFSFinderTest, path_depth_2) { // Try again to make sure we stay at non-existing auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); - EXPECT_TRUE(result.isEmpty()); EXPECT_TRUE(finder.isDone()); } { From c9001a2d438ed78a88aa376b0c4ddb68ea5f89f7 Mon Sep 17 00:00:00 2001 From: Heiko Date: Tue, 7 Sep 2021 13:02:20 +0200 Subject: [PATCH 56/72] Update tests/Graph/DFSFinderTest.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Tobias Gödderz --- tests/Graph/DFSFinderTest.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/Graph/DFSFinderTest.cpp b/tests/Graph/DFSFinderTest.cpp index ddcc251e408d..d99681277f45 100644 --- a/tests/Graph/DFSFinderTest.cpp +++ b/tests/Graph/DFSFinderTest.cpp @@ -708,7 +708,6 @@ TEST_P(DFSFinderTest, triangle_loop) { // Try again to make sure we stay at non-existing auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); - EXPECT_TRUE(result.isEmpty()); EXPECT_TRUE(finder.isDone()); } } From 18f87ea188b3c6c5fcaf06dce8debac0f7794e9f Mon Sep 17 00:00:00 2001 From: Heiko Date: Tue, 7 Sep 2021 13:03:34 +0200 Subject: [PATCH 57/72] Update tests/Graph/DFSFinderTest.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Tobias Gödderz --- tests/Graph/DFSFinderTest.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/Graph/DFSFinderTest.cpp b/tests/Graph/DFSFinderTest.cpp index d99681277f45..de0495065ef8 100644 --- a/tests/Graph/DFSFinderTest.cpp +++ b/tests/Graph/DFSFinderTest.cpp @@ -704,7 +704,6 @@ TEST_P(DFSFinderTest, triangle_loop) { } { - result.clear(); // Try again to make sure we stay at non-existing auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); From d8e6cb70a6c74b9bb5e676a6afaac084e0645a19 Mon Sep 17 00:00:00 2001 From: Heiko Date: Tue, 7 Sep 2021 13:03:46 +0200 Subject: [PATCH 58/72] Update tests/Graph/DFSFinderTest.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Tobias Gödderz --- tests/Graph/DFSFinderTest.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/Graph/DFSFinderTest.cpp b/tests/Graph/DFSFinderTest.cpp index de0495065ef8..a9c85ae82088 100644 --- a/tests/Graph/DFSFinderTest.cpp +++ b/tests/Graph/DFSFinderTest.cpp @@ -640,7 +640,6 @@ TEST_P(DFSFinderTest, path_loop) { } { - result.clear(); auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); EXPECT_TRUE(finder.isDone()); From 23c11bf5a8618dec4cdeb61eb09e363f70188d34 Mon Sep 17 00:00:00 2001 From: Heiko Kernbach Date: Tue, 7 Sep 2021 13:04:11 +0200 Subject: [PATCH 59/72] removed unneeded clear --- tests/Graph/DFSFinderTest.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/Graph/DFSFinderTest.cpp b/tests/Graph/DFSFinderTest.cpp index d99681277f45..912963a736a1 100644 --- a/tests/Graph/DFSFinderTest.cpp +++ b/tests/Graph/DFSFinderTest.cpp @@ -538,7 +538,6 @@ TEST_P(DFSFinderTest, path_depth_1_to_2) { } { - result.clear(); // Try again to make sure we stay at non-existing auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); From ec68f06a44ce29f1133776ca5c48053ebb8eb57b Mon Sep 17 00:00:00 2001 From: Heiko Date: Tue, 7 Sep 2021 13:04:34 +0200 Subject: [PATCH 60/72] Update tests/Graph/DFSFinderTest.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Tobias Gödderz --- tests/Graph/DFSFinderTest.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/Graph/DFSFinderTest.cpp b/tests/Graph/DFSFinderTest.cpp index 7adeafb47d38..3b682eca48d2 100644 --- a/tests/Graph/DFSFinderTest.cpp +++ b/tests/Graph/DFSFinderTest.cpp @@ -417,7 +417,6 @@ TEST_P(DFSFinderTest, path_depth_3) { // Try again to make sure we stay at non-existing auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); - EXPECT_TRUE(result.isEmpty()); EXPECT_TRUE(finder.isDone()); } From f70445df0b897285a7e6820015ba812164b53d09 Mon Sep 17 00:00:00 2001 From: Heiko Date: Tue, 7 Sep 2021 13:05:28 +0200 Subject: [PATCH 61/72] Update tests/Graph/DFSFinderTest.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Tobias Gödderz --- tests/Graph/DFSFinderTest.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/Graph/DFSFinderTest.cpp b/tests/Graph/DFSFinderTest.cpp index 3b682eca48d2..163115e96db5 100644 --- a/tests/Graph/DFSFinderTest.cpp +++ b/tests/Graph/DFSFinderTest.cpp @@ -413,7 +413,6 @@ TEST_P(DFSFinderTest, path_depth_3) { } { - result.clear(); // Try again to make sure we stay at non-existing auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); From 1bf6c3cee819054e6d46dff6c4c66336109d3e31 Mon Sep 17 00:00:00 2001 From: Heiko Date: Tue, 7 Sep 2021 13:05:43 +0200 Subject: [PATCH 62/72] Update tests/Graph/DFSFinderTest.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Tobias Gödderz --- tests/Graph/DFSFinderTest.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/Graph/DFSFinderTest.cpp b/tests/Graph/DFSFinderTest.cpp index 163115e96db5..dfde4a65bed1 100644 --- a/tests/Graph/DFSFinderTest.cpp +++ b/tests/Graph/DFSFinderTest.cpp @@ -378,7 +378,6 @@ TEST_P(DFSFinderTest, path_depth_2) { } { - result.clear(); // Try again to make sure we stay at non-existing auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); From c0d3ebe62c8160786e829c03c0297463fa842888 Mon Sep 17 00:00:00 2001 From: Heiko Date: Tue, 7 Sep 2021 13:06:19 +0200 Subject: [PATCH 63/72] Update tests/Graph/DFSFinderTest.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Tobias Gödderz --- tests/Graph/DFSFinderTest.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/Graph/DFSFinderTest.cpp b/tests/Graph/DFSFinderTest.cpp index dfde4a65bed1..4d570b0507cf 100644 --- a/tests/Graph/DFSFinderTest.cpp +++ b/tests/Graph/DFSFinderTest.cpp @@ -470,7 +470,6 @@ TEST_P(DFSFinderTest, path_diamond) { // Try again to make sure we stay at non-existing auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); - EXPECT_TRUE(result.isEmpty()); EXPECT_TRUE(finder.isDone()); } { From d6f5296649daca886381f433d9966e4039cc7dd6 Mon Sep 17 00:00:00 2001 From: Heiko Date: Tue, 7 Sep 2021 13:06:30 +0200 Subject: [PATCH 64/72] Update tests/Graph/DFSFinderTest.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Tobias Gödderz --- tests/Graph/DFSFinderTest.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/Graph/DFSFinderTest.cpp b/tests/Graph/DFSFinderTest.cpp index 4d570b0507cf..f995e90be2b2 100644 --- a/tests/Graph/DFSFinderTest.cpp +++ b/tests/Graph/DFSFinderTest.cpp @@ -466,7 +466,6 @@ TEST_P(DFSFinderTest, path_diamond) { } { - result.clear(); // Try again to make sure we stay at non-existing auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); From cf32daa43aa81aeebb4e12fac915c8ed07878767 Mon Sep 17 00:00:00 2001 From: Heiko Date: Tue, 7 Sep 2021 13:06:42 +0200 Subject: [PATCH 65/72] Update tests/Graph/DFSFinderTest.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Tobias Gödderz --- tests/Graph/DFSFinderTest.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/Graph/DFSFinderTest.cpp b/tests/Graph/DFSFinderTest.cpp index f995e90be2b2..b60e2e18ce19 100644 --- a/tests/Graph/DFSFinderTest.cpp +++ b/tests/Graph/DFSFinderTest.cpp @@ -536,7 +536,6 @@ TEST_P(DFSFinderTest, path_depth_1_to_2) { // Try again to make sure we stay at non-existing auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); - EXPECT_TRUE(result.isEmpty()); EXPECT_TRUE(finder.isDone()); } } From 83714c7556c504f8e60a3b2881682db223ecf502 Mon Sep 17 00:00:00 2001 From: Heiko Date: Tue, 7 Sep 2021 13:07:06 +0200 Subject: [PATCH 66/72] Update tests/Graph/DFSFinderTest.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Tobias Gödderz --- tests/Graph/DFSFinderTest.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/Graph/DFSFinderTest.cpp b/tests/Graph/DFSFinderTest.cpp index b60e2e18ce19..1f23ca167c78 100644 --- a/tests/Graph/DFSFinderTest.cpp +++ b/tests/Graph/DFSFinderTest.cpp @@ -593,7 +593,6 @@ TEST_P(DFSFinderTest, path_depth_1_to_2_skip) { // Try again to make sure we stay at non-existing auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); - EXPECT_TRUE(result.isEmpty()); EXPECT_TRUE(finder.isDone()); } } From c98c94ccd9f040fc41d9c3ff0649f0f318af6c1e Mon Sep 17 00:00:00 2001 From: Heiko Kernbach Date: Tue, 7 Sep 2021 15:18:36 +0200 Subject: [PATCH 67/72] ssp test cleanup, disabled expr, added asserts --- tests/Graph/SingleServerProviderTest.cpp | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tests/Graph/SingleServerProviderTest.cpp b/tests/Graph/SingleServerProviderTest.cpp index 11357a998810..0cfbf12e1c6a 100644 --- a/tests/Graph/SingleServerProviderTest.cpp +++ b/tests/Graph/SingleServerProviderTest.cpp @@ -73,7 +73,8 @@ class SingleServerProviderTest : public ::testing::Test { std::unordered_map> _emptyShardMap{}; - std::string stringToMatch = "0-1"; + // can be used for further testing to generate a expression + // std::string stringToMatch = "0-1"; SingleServerProviderTest() {} ~SingleServerProviderTest() {} @@ -97,8 +98,10 @@ class SingleServerProviderTest : public ::testing::Test { _varNode = ::InitializeReference(*query->ast(), *_tmpVar); std::vector usedIndexes{}; - auto expr = conditionKeyMatches(stringToMatch); - usedIndexes.emplace_back(IndexAccessor{edgeIndexHandle, indexCondition, 0, expr, 0}); + + // can be used to create an expression, currently unused but may be helpful for additional tests + // auto expr = conditionKeyMatches(stringToMatch); + usedIndexes.emplace_back(IndexAccessor{edgeIndexHandle, indexCondition, 0, nullptr, 0}); _expressionContext = std::make_unique(*_trx, *query, _functionsCache); @@ -137,11 +140,16 @@ TEST_F(SingleServerProviderTest, it_can_provide_edges) { static_cast(startVertex.length())}; Step s = testee.startVertex(hashedStart); + std::vector results = {}; + VPackBuilder builder; + testee.expand(s, 0, [&](Step next) { - VPackBuilder hund; - testee.addEdgeToBuilder(next.getEdge(), hund); - LOG_DEVEL << next.getVertexIdentifier() << " e: " << hund.toJson(); + results.push_back(next.getVertex().getID().toString()); }); + + ASSERT_EQ(results.size(), 2); + ASSERT_EQ(results.at(0), "v/1"); + ASSERT_EQ(results.at(1), "v/2"); } } // namespace single_server_provider_test From 404440223472200788fb5762817fe1d06f36f07f Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Tue, 7 Sep 2021 15:48:36 +0200 Subject: [PATCH 68/72] Made Expression a unique_ptr, by explicitly deleting copy and allowing move on surrounding classes. This showed some stages where the expression was illegally copied, though it should be moved. C++ compiler warnings were not of much help though. --- arangod/Aql/KShortestPathsNode.cpp | 12 ++++++------ arangod/Graph/Providers/BaseProviderOptions.cpp | 4 ++-- arangod/Graph/Providers/BaseProviderOptions.h | 11 ++++++++--- tests/Graph/SingleServerProviderTest.cpp | 4 ++-- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/arangod/Aql/KShortestPathsNode.cpp b/arangod/Aql/KShortestPathsNode.cpp index cccc9c963aa3..a944eae758a6 100644 --- a/arangod/Aql/KShortestPathsNode.cpp +++ b/arangod/Aql/KShortestPathsNode.cpp @@ -350,10 +350,10 @@ std::unique_ptr KShortestPathsNode::createBlock( std::pair, std::unordered_map>> reversedUsedIndexes{}; reversedUsedIndexes.first = buildReverseUsedIndexes(); - BaseProviderOptions forwardProviderOptions(opts->tmpVar(), usedIndexes, + BaseProviderOptions forwardProviderOptions(opts->tmpVar(), std::move(usedIndexes), opts->getExpressionCtx(), opts->collectionToShard()); - BaseProviderOptions backwardProviderOptions(opts->tmpVar(), reversedUsedIndexes, + BaseProviderOptions backwardProviderOptions(opts->tmpVar(), std::move(reversedUsedIndexes), opts->getExpressionCtx(), opts->collectionToShard()); @@ -363,9 +363,9 @@ std::unique_ptr KShortestPathsNode::createBlock( KPathEnumerator>; auto kPathUnique = std::make_unique( - SingleServerProvider{opts->query(), forwardProviderOptions, + SingleServerProvider{opts->query(), std::move(forwardProviderOptions), opts->query().resourceMonitor()}, - SingleServerProvider{opts->query(), backwardProviderOptions, + SingleServerProvider{opts->query(), std::move(backwardProviderOptions), opts->query().resourceMonitor()}, std::move(enumeratorOptions), std::move(validatorOptions), opts->query().resourceMonitor()); @@ -382,9 +382,9 @@ std::unique_ptr KShortestPathsNode::createBlock( TracedKPathEnumerator>; auto kPathUnique = std::make_unique( ProviderTracer>{ - opts->query(), forwardProviderOptions, opts->query().resourceMonitor()}, + opts->query(), std::move(forwardProviderOptions), opts->query().resourceMonitor()}, ProviderTracer>{ - opts->query(), backwardProviderOptions, opts->query().resourceMonitor()}, + opts->query(), std::move(backwardProviderOptions), opts->query().resourceMonitor()}, std::move(enumeratorOptions), std::move(validatorOptions), opts->query().resourceMonitor()); diff --git a/arangod/Graph/Providers/BaseProviderOptions.cpp b/arangod/Graph/Providers/BaseProviderOptions.cpp index a01ce0f59410..860d5e104ae3 100644 --- a/arangod/Graph/Providers/BaseProviderOptions.cpp +++ b/arangod/Graph/Providers/BaseProviderOptions.cpp @@ -28,7 +28,7 @@ using namespace arangodb::graph; IndexAccessor::IndexAccessor(transaction::Methods::IndexHandle idx, aql::AstNode* condition, std::optional memberToUpdate, - std::shared_ptr expression, + std::unique_ptr expression, size_t cursorId) : _idx(idx), _indexCondition(condition), @@ -57,7 +57,7 @@ size_t IndexAccessor::cursorId() const { return _cursorId; } BaseProviderOptions::BaseProviderOptions( aql::Variable const* tmpVar, - std::pair, std::unordered_map>> indexInfo, + std::pair, std::unordered_map>>&& indexInfo, aql::FixedVarExpressionContext& expressionContext, std::unordered_map> const& collectionToShardMap) : _temporaryVariable(tmpVar), diff --git a/arangod/Graph/Providers/BaseProviderOptions.h b/arangod/Graph/Providers/BaseProviderOptions.h index 040745529d10..973b5f00a025 100644 --- a/arangod/Graph/Providers/BaseProviderOptions.h +++ b/arangod/Graph/Providers/BaseProviderOptions.h @@ -43,7 +43,9 @@ namespace graph { struct IndexAccessor { IndexAccessor(transaction::Methods::IndexHandle idx, aql::AstNode* condition, std::optional memberToUpdate, - std::shared_ptr expression, size_t cursorId); + std::unique_ptr expression, size_t cursorId); + IndexAccessor(IndexAccessor const&) = delete; + IndexAccessor(IndexAccessor&&) = default; aql::AstNode* getCondition() const; aql::Expression* getExpression() const; @@ -59,7 +61,7 @@ struct IndexAccessor { // Note: We would prefer to have this a unique_ptr here. // However the IndexAccessor is used in std::vector // which then refuses to compile (deleted copy constructor) - std::shared_ptr _expression; + std::unique_ptr _expression; size_t _cursorId; }; @@ -70,10 +72,13 @@ struct BaseProviderOptions { public: BaseProviderOptions( aql::Variable const* tmpVar, - std::pair, std::unordered_map>> indexInfo, + std::pair, std::unordered_map>>&& indexInfo, aql::FixedVarExpressionContext& expressionContext, std::unordered_map> const& collectionToShardMap); + BaseProviderOptions(BaseProviderOptions const&) = delete; + BaseProviderOptions(BaseProviderOptions&&) = default; + aql::Variable const* tmpVar() const; std::pair, std::unordered_map>> const& indexInformations() const; diff --git a/tests/Graph/SingleServerProviderTest.cpp b/tests/Graph/SingleServerProviderTest.cpp index 0cfbf12e1c6a..35f0d9842b1b 100644 --- a/tests/Graph/SingleServerProviderTest.cpp +++ b/tests/Graph/SingleServerProviderTest.cpp @@ -115,7 +115,7 @@ class SingleServerProviderTest : public ::testing::Test { /* * generates a condition #TMP._key == '' */ - std::shared_ptr conditionKeyMatches(std::string const& toMatch) { + std::unique_ptr conditionKeyMatches(std::string const& toMatch) { auto expectedKey = query->ast()->createNodeValueString(toMatch.c_str(), toMatch.length()); auto keyAccess = @@ -125,7 +125,7 @@ class SingleServerProviderTest : public ::testing::Test { // This condition cannot be fulfilled auto condition = query->ast()->createNodeBinaryOperator(aql::AstNodeType::NODE_TYPE_OPERATOR_BINARY_EQ, keyAccess, expectedKey); - return std::make_shared(query->ast(), condition); + return std::make_unique(query->ast(), condition); } }; From 535f7a60b50f0ac4ec644cbfb877880dc70f0194 Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Wed, 8 Sep 2021 11:03:33 +0200 Subject: [PATCH 69/72] Removed forgotten assert --- arangod/Aql/GraphNode.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/arangod/Aql/GraphNode.cpp b/arangod/Aql/GraphNode.cpp index 170bd689ad64..a4102ed38b2d 100644 --- a/arangod/Aql/GraphNode.cpp +++ b/arangod/Aql/GraphNode.cpp @@ -580,7 +580,6 @@ void GraphNode::doToVelocyPack(VPackBuilder& nodes, unsigned flags) const { { VPackObjectBuilder guard(&nodes); for (auto const& item : _collectionToShard) { - TRI_ASSERT(item.second.size() == 1); nodes.add(item.first, VPackValue(item.second.front())); } } From 90d995e8b874f0c7ccba40c238567867bb446c21 Mon Sep 17 00:00:00 2001 From: Heiko Kernbach Date: Wed, 8 Sep 2021 11:25:07 +0200 Subject: [PATCH 70/72] fix serializing of GraphNode --- arangod/Aql/GraphNode.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arangod/Aql/GraphNode.cpp b/arangod/Aql/GraphNode.cpp index a4102ed38b2d..9076c716f2a3 100644 --- a/arangod/Aql/GraphNode.cpp +++ b/arangod/Aql/GraphNode.cpp @@ -580,7 +580,7 @@ void GraphNode::doToVelocyPack(VPackBuilder& nodes, unsigned flags) const { { VPackObjectBuilder guard(&nodes); for (auto const& item : _collectionToShard) { - nodes.add(item.first, VPackValue(item.second.front())); + nodes.add(item.first, VPackValue(item.second)); } } From acf4c5989b3deff51be111da4d723e4b8ffdab2c Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Wed, 8 Sep 2021 15:30:53 +0200 Subject: [PATCH 71/72] Fixed compiler issue --- tests/Mocks/StorageEngineMock.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Mocks/StorageEngineMock.cpp b/tests/Mocks/StorageEngineMock.cpp index f648d51a82fa..4b8f8e3fad11 100644 --- a/tests/Mocks/StorageEngineMock.cpp +++ b/tests/Mocks/StorageEngineMock.cpp @@ -171,7 +171,7 @@ class EdgeIndexIteratorMock final : public arangodb::IndexIterator { } else { return cb(token, doc.get(arangodb::StaticStrings::FromString)); } - }); + }, arangodb::ReadOwnWrites::no); // We can only have good responses here. // Otherwise storage and Index do differ TRI_ASSERT(res.ok()); From 08545dfcd6ac2765ea9b44851c8d7c480a1ce1ff Mon Sep 17 00:00:00 2001 From: Michael Hackstein Date: Wed, 8 Sep 2021 19:00:28 +0200 Subject: [PATCH 72/72] Fixed UnitTest --- tests/Graph/SingleServerProviderTest.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/Graph/SingleServerProviderTest.cpp b/tests/Graph/SingleServerProviderTest.cpp index 35f0d9842b1b..37b13e2b20ed 100644 --- a/tests/Graph/SingleServerProviderTest.cpp +++ b/tests/Graph/SingleServerProviderTest.cpp @@ -147,9 +147,14 @@ TEST_F(SingleServerProviderTest, it_can_provide_edges) { results.push_back(next.getVertex().getID().toString()); }); + // Order is not guaranteed ASSERT_EQ(results.size(), 2); - ASSERT_EQ(results.at(0), "v/1"); - ASSERT_EQ(results.at(1), "v/2"); + if (results.at(0) == "v/1") { + ASSERT_EQ(results.at(1), "v/2"); + } else { + ASSERT_EQ(results.at(0), "v/2"); + ASSERT_EQ(results.at(1), "v/1"); + } } } // namespace single_server_provider_test