diff --git a/arangod/Aql/AqlFunctionsInternalCache.h b/arangod/Aql/AqlFunctionsInternalCache.h index 6262d436fdd4..f89b7e7c51e0 100644 --- a/arangod/Aql/AqlFunctionsInternalCache.h +++ b/arangod/Aql/AqlFunctionsInternalCache.h @@ -25,13 +25,11 @@ #include "Aql/AqlValue.h" #include "Basics/Common.h" -#include +#include "VocBase/Validators.h" #include #include - - namespace arangodb { namespace transaction { @@ -49,6 +47,8 @@ class AqlFunctionsInternalCache final { AqlFunctionsInternalCache() = default; ~AqlFunctionsInternalCache(); + AqlFunctionsInternalCache(AqlFunctionsInternalCache&&) = default; + void clear() noexcept; icu::RegexMatcher* buildRegexMatcher(char const* ptr, size_t length, bool caseInsensitive); @@ -97,4 +97,3 @@ class AqlFunctionsInternalCache final { } // namespace aql } // namespace arangodb - diff --git a/arangod/Aql/ClusterNodes.cpp b/arangod/Aql/ClusterNodes.cpp index a00486f0a338..9a0933a6812b 100644 --- a/arangod/Aql/ClusterNodes.cpp +++ b/arangod/Aql/ClusterNodes.cpp @@ -263,8 +263,20 @@ DistributeNode::DistributeNode(ExecutionPlan* plan, ExecutionNodeId id, DistributeNode::DistributeNode(ExecutionPlan* plan, arangodb::velocypack::Slice const& base) : ScatterNode(plan, base), CollectionAccessingNode(plan, base), - _variable(Variable::varFromVPack(plan->getAst(), base, "variable")) {} - + _variable(Variable::varFromVPack(plan->getAst(), base, "variable")) { + auto sats = base.get("satelliteCollections"); + if (sats.isArray()) { + auto& queryCols = plan->getAst()->query().collections(); + _satellites.reserve(sats.length()); + + for (VPackSlice it : VPackArrayIterator(sats)) { + std::string v = arangodb::basics::VelocyPackHelper::getStringValue(it, ""); + auto c = queryCols.add(v, AccessMode::Type::READ, aql::Collection::Hint::Collection); + addSatellite(c); + } + } +} + /// @brief clone ExecutionNode recursively ExecutionNode* DistributeNode::clone(ExecutionPlan* plan, bool withDependencies, bool withProperties) const { @@ -272,6 +284,10 @@ ExecutionNode* DistributeNode::clone(ExecutionPlan* plan, bool withDependencies, collection(), _variable, _targetNodeId); c->copyClients(clients()); CollectionAccessingNode::cloneInto(*c); + c->_satellites.reserve(_satellites.size()); + for (auto& it : _satellites) { + c->_satellites.emplace_back(it); + } return cloneHelper(std::move(c), withDependencies, withProperties); } @@ -294,8 +310,8 @@ std::unique_ptr DistributeNode::createBlock( auto inRegs = RegIdSet{regId}; auto registerInfos = createRegisterInfos(inRegs, {}); - auto infos = DistributeExecutorInfos(clients(), collection(), - regId, getScatterType()); + auto infos = DistributeExecutorInfos(clients(), collection(), regId, + getScatterType(), getSatellites()); return std::make_unique>(&engine, this, std::move(registerInfos), @@ -311,6 +327,16 @@ void DistributeNode::doToVelocyPack(VPackBuilder& builder, unsigned flags) const builder.add(VPackValue("variable")); _variable->toVelocyPack(builder); + + if (!_satellites.empty()) { + builder.add(VPackValue("satelliteCollections")); + { + VPackArrayBuilder guard(&builder); + for (auto const& v : _satellites) { + builder.add(VPackValue(v->name())); + } + } + } } void DistributeNode::replaceVariables(std::unordered_map const& replacements) { @@ -329,6 +355,12 @@ CostEstimate DistributeNode::estimateCost() const { return estimate; } +void DistributeNode::addSatellite(aql::Collection* satellite) { + // Only relevant for enterprise disjoint smart graphs + TRI_ASSERT(satellite->isSatellite()); + _satellites.emplace_back(satellite); +} + /*static*/ Collection const* GatherNode::findCollection(GatherNode const& root) noexcept { ExecutionNode const* node = root.getFirstDependency(); diff --git a/arangod/Aql/ClusterNodes.h b/arangod/Aql/ClusterNodes.h index 980fd7ddc4dc..c6ece35ce926 100644 --- a/arangod/Aql/ClusterNodes.h +++ b/arangod/Aql/ClusterNodes.h @@ -222,11 +222,17 @@ class DistributeNode final : public ScatterNode, public CollectionAccessingNode CostEstimate estimateCost() const override final; Variable const* getVariable() const noexcept { return _variable; } - + void setVariable(Variable const* var) noexcept { _variable = var; } - + ExecutionNodeId getTargetNodeId() const noexcept { return _targetNodeId; } + void addSatellite(aql::Collection*); + + std::vector const getSatellites() const noexcept { + return _satellites; + } + protected: /// @brief export to VelocyPack void doToVelocyPack(arangodb::velocypack::Builder&, unsigned flags) const override final; @@ -234,9 +240,13 @@ class DistributeNode final : public ScatterNode, public CollectionAccessingNode private: /// @brief the variable we must inspect to know where to distribute Variable const* _variable; - + /// @brief the id of the target ExecutionNode this DistributeNode belongs to. ExecutionNodeId _targetNodeId; + + /// @brief List of Satellite collections this node needs to distribute data to + /// in a satellite manner. + std::vector _satellites; }; /// @brief class GatherNode diff --git a/arangod/Aql/Collection.cpp b/arangod/Aql/Collection.cpp index e30d0bd39923..c110f38ecde0 100644 --- a/arangod/Aql/Collection.cpp +++ b/arangod/Aql/Collection.cpp @@ -186,7 +186,7 @@ std::vector Collection::shardKeys(bool normalize) const { if (normalize && coll->isSmart() && coll->type() == TRI_COL_TYPE_DOCUMENT) { // smart vertex collection always has ["_key:"] as shard keys TRI_ASSERT(originalKeys.size() == 1); - TRI_ASSERT(originalKeys[0] == "_key:"); + TRI_ASSERT(originalKeys[0] == StaticStrings::PrefixOfKeyString); // now normalize it this to _key return std::vector{StaticStrings::KeyString}; } diff --git a/arangod/Aql/DistributeExecutor.cpp b/arangod/Aql/DistributeExecutor.cpp index 2f349775b86b..75f04295d8fc 100644 --- a/arangod/Aql/DistributeExecutor.cpp +++ b/arangod/Aql/DistributeExecutor.cpp @@ -41,14 +41,16 @@ using namespace arangodb; using namespace arangodb::aql; -DistributeExecutorInfos::DistributeExecutorInfos( - std::vector clientIds, Collection const* collection, - RegisterId regId, ScatterNode::ScatterType type) +DistributeExecutorInfos::DistributeExecutorInfos(std::vector clientIds, + Collection const* collection, RegisterId regId, + ScatterNode::ScatterType type, + std::vector satellites) : ClientsExecutorInfos(std::move(clientIds)), _regId(regId), _collection(collection), _logCol(collection->getCollection()), - _type(type) {} + _type(type), + _satellites(std::move(satellites)) {} auto DistributeExecutorInfos::registerId() const noexcept -> RegisterId { TRI_ASSERT(_regId.isValid()); @@ -77,6 +79,35 @@ auto DistributeExecutorInfos::getResponsibleClient(arangodb::velocypack::Slice v return shardId; } +auto DistributeExecutorInfos::shouldDistributeToAll(arangodb::velocypack::Slice value) const + -> bool { + if (_satellites.empty()) { + // We can only distribute to all on Satellite Collections + return false; + } + auto id = value.get(StaticStrings::IdString); + if (!id.isString()) { + // We can only distribute to all if we can detect the collection name + return false; + } + + // NOTE: Copy Paste code, shall be unified + VPackStringRef vid(id); + size_t pos = vid.find('/'); + if (pos == std::string::npos) { + // Invalid input. Let the sharding take care of it, one server shall complain + return false; + } + vid = vid.substr(0, pos); + for (auto const& it : _satellites) { + if (vid.equals(it->name())) { + // This vertex is from a satellite collection start everywhere! + return true; + } + } + return false; +} + DistributeExecutor::DistributeExecutor(DistributeExecutorInfos const& infos) : _infos(infos) {} @@ -94,11 +125,25 @@ auto DistributeExecutor::distributeBlock(SharedAqlItemBlockPtr const& block, Ski choosenMap[key].emplace_back(i); } } else { - auto client = getClient(block, i); - if (!client.empty()) { - // We can only have clients we are prepared for - TRI_ASSERT(blockMap.find(client) != blockMap.end()); - choosenMap[client].emplace_back(i); + auto input = extractInput(block, i); + if (!input.isNone()) { + // NONE is ignored. + // Object is processd + // All others throw. + TRI_ASSERT(input.isObject()); + if (_infos.shouldDistributeToAll(input)) { + // This input should be added to all clients + for (auto const& [key, value] : blockMap) { + choosenMap[key].emplace_back(i); + } + } else { + auto client = getClient(input); + if (!client.empty()) { + // We can only have clients we are prepared for + TRI_ASSERT(blockMap.find(client) != blockMap.end()); + choosenMap[client].emplace_back(i); + } + } } } } @@ -123,19 +168,24 @@ auto DistributeExecutor::distributeBlock(SharedAqlItemBlockPtr const& block, Ski } } -auto DistributeExecutor::getClient(SharedAqlItemBlockPtr const& block, size_t rowIndex) const - -> std::string { +auto DistributeExecutor::extractInput(SharedAqlItemBlockPtr const& block, + size_t rowIndex) const -> VPackSlice { // check first input register AqlValue val = InputAqlItemRow{block, rowIndex}.getValue(_infos.registerId()); VPackSlice input = val.slice(); // will throw when wrong type if (input.isNone()) { - return {}; + return input; } - + if (!input.isObject()) { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "DistributeExecutor requires an object as input"); + THROW_ARANGO_EXCEPTION_MESSAGE( + TRI_ERROR_INTERNAL, "DistributeExecutor requires an object as input"); } + return input; +} + +auto DistributeExecutor::getClient(VPackSlice input) const -> std::string { auto res = _infos.getResponsibleClient(input); if (res.fail()) { THROW_ARANGO_EXCEPTION(std::move(res).result()); @@ -147,4 +197,5 @@ ExecutionBlockImpl::ExecutionBlockImpl(ExecutionEngine* engi DistributeNode const* node, RegisterInfos registerInfos, DistributeExecutorInfos&& executorInfos) - : BlocksWithClientsImpl(engine, node, std::move(registerInfos), std::move(executorInfos)) {} + : BlocksWithClientsImpl(engine, node, std::move(registerInfos), + std::move(executorInfos)) {} diff --git a/arangod/Aql/DistributeExecutor.h b/arangod/Aql/DistributeExecutor.h index 9d13dca7efd3..bfbb91c20947 100644 --- a/arangod/Aql/DistributeExecutor.h +++ b/arangod/Aql/DistributeExecutor.h @@ -43,7 +43,8 @@ class DistributeNode; class DistributeExecutorInfos : public ClientsExecutorInfos { public: DistributeExecutorInfos(std::vector clientIds, Collection const* collection, - RegisterId regId, ScatterNode::ScatterType type); + RegisterId regId, ScatterNode::ScatterType type, + std::vector satellites); auto registerId() const noexcept -> RegisterId; auto scatterType() const noexcept -> ScatterNode::ScatterType; @@ -51,6 +52,8 @@ class DistributeExecutorInfos : public ClientsExecutorInfos { auto getResponsibleClient(arangodb::velocypack::Slice value) const -> ResultT; + auto shouldDistributeToAll(arangodb::velocypack::Slice value) const -> bool; + private: RegisterId _regId; @@ -63,6 +66,9 @@ class DistributeExecutorInfos : public ClientsExecutorInfos { /// @brief type of distribution that this nodes follows. ScatterNode::ScatterType _type; + + /// @brief list of collections that should be used + std::vector _satellites; }; // The DistributeBlock is actually implemented by specializing @@ -93,16 +99,13 @@ class DistributeExecutor { private: /** * @brief Compute which client needs to get this row - * NOTE: Has SideEffects - * If the input value does not contain an object, it is modified inplace with - * a new Object containing a key value! - * Hence this method is not const ;( - * * @param block The input block * @param rowIndex * @return std::string Identifier used by the client */ - auto getClient(SharedAqlItemBlockPtr const& block, size_t rowIndex) const -> std::string; + auto extractInput(SharedAqlItemBlockPtr const& block, size_t rowIndex) const + -> velocypack::Slice; + auto getClient(velocypack::Slice input) const -> std::string; private: DistributeExecutorInfos const& _infos; diff --git a/arangod/Aql/EngineInfoContainerDBServerServerBased.cpp b/arangod/Aql/EngineInfoContainerDBServerServerBased.cpp index f053e9907ca7..9f7e43bbc3e6 100644 --- a/arangod/Aql/EngineInfoContainerDBServerServerBased.cpp +++ b/arangod/Aql/EngineInfoContainerDBServerServerBased.cpp @@ -25,11 +25,11 @@ #include "Aql/Ast.h" #include "Aql/GraphNode.h" +#include "Aql/TraverserEngineShardLists.h" #include "Basics/StaticStrings.h" #include "Basics/StringUtils.h" #include "Cluster/ClusterFeature.h" #include "Cluster/ClusterTrxMethods.h" -#include "Graph/BaseOptions.h" #include "Network/Methods.h" #include "Network/NetworkFeature.h" #include "Network/Utils.h" @@ -38,8 +38,8 @@ #include "Transaction/Manager.h" #include "Utils/CollectionNameResolver.h" -#include #include +#include using namespace arangodb; using namespace arangodb::aql; @@ -49,7 +49,7 @@ namespace { const double SETUP_TIMEOUT = 60.0; // Wait 2s to get the Lock in FastPath, otherwise assume dead-lock. const double FAST_PATH_LOCK_TIMEOUT = 2.0; - + std::string const finishUrl("/_api/aql/finish/"); std::string const traverserUrl("/_internal/traverser/"); @@ -78,120 +78,6 @@ Result ExtractRemoteAndShard(VPackSlice keySlice, ExecutionNodeId& remoteId, } // namespace -EngineInfoContainerDBServerServerBased::TraverserEngineShardLists::TraverserEngineShardLists( - GraphNode const* node, ServerID const& server, - std::unordered_map const& shardMapping, QueryContext& query) - : _node(node), _hasShard(false) { - auto const& edges = _node->edgeColls(); - TRI_ASSERT(!edges.empty()); - auto const& restrictToShards = query.queryOptions().restrictToShards; - // Extract the local shards for edge collections. - for (auto const& col : edges) { - TRI_ASSERT(col != nullptr); -#ifdef USE_ENTERPRISE - if (query.trxForOptimization().isInaccessibleCollection(col->id())) { - _inaccessible.insert(col->name()); - _inaccessible.insert(std::to_string(col->id().id())); - } -#endif - _edgeCollections.emplace_back( - getAllLocalShards(shardMapping, server, col->shardIds(restrictToShards))); - } - // Extract vertices - auto const& vertices = _node->vertexColls(); - // Guaranteed by addGraphNode, this will inject vertex collections - // in anonymous graph case - // It might in fact be empty, if we only have edge collections in a graph. - // Or if we guarantee to never read vertex data. - for (auto const& col : vertices) { - TRI_ASSERT(col != nullptr); -#ifdef USE_ENTERPRISE - if (query.trxForOptimization().isInaccessibleCollection(col->id())) { - _inaccessible.insert(col->name()); - _inaccessible.insert(std::to_string(col->id().id())); - } -#endif - auto shards = getAllLocalShards(shardMapping, server, col->shardIds(restrictToShards)); - _vertexCollections.try_emplace(col->name(), std::move(shards)); - } -} - -std::vector EngineInfoContainerDBServerServerBased::TraverserEngineShardLists::getAllLocalShards( - std::unordered_map const& shardMapping, - ServerID const& server, std::shared_ptr> shardIds) { - std::vector localShards; - for (auto const& shard : *shardIds) { - auto const& it = shardMapping.find(shard); - if (it == shardMapping.end()) { - THROW_ARANGO_EXCEPTION_MESSAGE( - TRI_ERROR_INTERNAL, - "no entry for shard '" + shard + "' in shard mapping table (" + std::to_string(shardMapping.size()) + " entries)"); - } - if (it->second == server) { - localShards.emplace_back(shard); - _hasShard = true; - } - } - return localShards; -} - -void EngineInfoContainerDBServerServerBased::TraverserEngineShardLists::serializeIntoBuilder( - VPackBuilder& infoBuilder) const { - TRI_ASSERT(_hasShard); - TRI_ASSERT(infoBuilder.isOpenArray()); - infoBuilder.openObject(); - { - // Options - infoBuilder.add(VPackValue("options")); - graph::BaseOptions* opts = _node->options(); - opts->buildEngineInfo(infoBuilder); - } - { - // Variables - std::vector vars; - _node->getConditionVariables(vars); - if (!vars.empty()) { - infoBuilder.add(VPackValue("variables")); - infoBuilder.openArray(); - for (auto v : vars) { - v->toVelocyPack(infoBuilder); - } - infoBuilder.close(); - } - } - - infoBuilder.add(VPackValue("shards")); - infoBuilder.openObject(); - infoBuilder.add(VPackValue("vertices")); - infoBuilder.openObject(); - for (auto const& col : _vertexCollections) { - infoBuilder.add(VPackValue(col.first)); - infoBuilder.openArray(); - for (auto const& v : col.second) { - infoBuilder.add(VPackValue(v)); - } - infoBuilder.close(); // this collection - } - infoBuilder.close(); // vertices - - infoBuilder.add(VPackValue("edges")); - infoBuilder.openArray(); - for (auto const& edgeShards : _edgeCollections) { - infoBuilder.openArray(); - for (auto const& e : edgeShards) { - infoBuilder.add(VPackValue(e)); - } - infoBuilder.close(); - } - infoBuilder.close(); // edges - infoBuilder.close(); // shards - - _node->enhanceEngineInfo(infoBuilder); - - infoBuilder.close(); // base - TRI_ASSERT(infoBuilder.isOpenArray()); -} - EngineInfoContainerDBServerServerBased::EngineInfoContainerDBServerServerBased(QueryContext& query) noexcept : _query(query), _shardLocking(query), _lastSnippetId(1) { // NOTE: We need to start with _lastSnippetID > 0. 0 is reserved for GraphNodes @@ -260,8 +146,7 @@ void EngineInfoContainerDBServerServerBased::closeSnippet(QueryId inputSnippet) } std::vector EngineInfoContainerDBServerServerBased::buildEngineInfo( - QueryId clusterQueryId, - VPackBuilder& infoBuilder, ServerID const& server, + QueryId clusterQueryId, VPackBuilder& infoBuilder, ServerID const& server, std::unordered_map const& nodesById, std::map& nodeAliases) { LOG_TOPIC("4bbe6", DEBUG, arangodb::Logger::AQL) @@ -313,7 +198,6 @@ arangodb::futures::Future EngineInfoContainerDBServerServerBased::buildS std::vector didCreateEngine, MapRemoteToSnippet& snippetIds, aql::ServerQueryIdList& serverToQueryId, std::mutex& serverToQueryIdLock, network::ConnectionPool* pool, network::RequestOptions const& options) const { - TRI_ASSERT(server.substr(0, 7) != "server:"); VPackBuffer buffer(infoSlice.byteSize()); @@ -328,14 +212,14 @@ arangodb::futures::Future EngineInfoContainerDBServerServerBased::buildS auto buildCallback = [this, server, didCreateEngine = std::move(didCreateEngine), - &serverToQueryId, &serverToQueryIdLock, &snippetIds, globalId]( - arangodb::futures::Try const& response) -> Result { + &serverToQueryId, &serverToQueryIdLock, &snippetIds, + globalId](arangodb::futures::Try const& response) -> Result { auto const& resolvedResponse = response.get(); auto queryId = globalId; RebootId rebootId{0}; - + TRI_ASSERT(server.substr(0, 7) != "server:"); - + std::unique_lock guard{serverToQueryIdLock}; if (resolvedResponse.fail()) { @@ -343,8 +227,8 @@ arangodb::futures::Future EngineInfoContainerDBServerServerBased::buildS LOG_TOPIC("f9a77", DEBUG, Logger::AQL) << server << " responded with " << res.errorNumber() << ": " << res.errorMessage(); - - serverToQueryId.emplace_back(ServerQueryIdEntry{ server, globalId, rebootId }); + + serverToQueryId.emplace_back(ServerQueryIdEntry{server, globalId, rebootId}); return res; } @@ -352,16 +236,16 @@ arangodb::futures::Future EngineInfoContainerDBServerServerBased::buildS if (responseSlice.isNone()) { return {TRI_ERROR_INTERNAL, "malformed response while building engines"}; } - auto result = parseResponse(responseSlice, snippetIds, server, + auto result = parseResponse(responseSlice, snippetIds, server, didCreateEngine, queryId, rebootId); - serverToQueryId.emplace_back(ServerQueryIdEntry{ server, queryId, rebootId }); + serverToQueryId.emplace_back(ServerQueryIdEntry{server, queryId, rebootId}); return result; }; - return network::sendRequestRetry(pool, "server:" + server, fuerte::RestVerb::Post, - "/_api/aql/setup", std::move(buffer), options, - std::move(headers)) + return network::sendRequestRetry(pool, "server:" + server, + fuerte::RestVerb::Post, "/_api/aql/setup", + std::move(buffer), options, std::move(headers)) .then([buildCallback = std::move(buildCallback)](futures::Try&& resp) mutable { return buildCallback(resp); }); @@ -382,7 +266,8 @@ bool EngineInfoContainerDBServerServerBased::isNotSatelliteLeader(VPackSlice inf return true; } - TRI_ASSERT((infoSlice.get("snippets").isObject() && !infoSlice.get("snippets").isEmptyObject()) || + TRI_ASSERT((infoSlice.get("snippets").isObject() && + !infoSlice.get("snippets").isEmptyObject()) || infoSlice.hasKey("traverserEngines")); return false; @@ -428,8 +313,8 @@ Result EngineInfoContainerDBServerServerBased::buildEngines( futures::collectAll(requests).wait(); } } catch (std::exception const& ex) { - LOG_TOPIC("2a9fe", WARN, Logger::AQL) - << "unable to clean up query snippets: " << ex.what(); + LOG_TOPIC("2a9fe", WARN, Logger::AQL) + << "unable to clean up query snippets: " << ex.what(); } }); @@ -439,22 +324,23 @@ Result EngineInfoContainerDBServerServerBased::buildEngines( // nullptr only happens on controlled shutdown return {TRI_ERROR_SHUTTING_DOWN}; } - + double oldTtl = _query.queryOptions().ttl; // we use a timeout of at least the trx timeout for DB server snippets. - // we assume this is safe because the RebootTracker on the coordinator + // we assume this is safe because the RebootTracker on the coordinator // will abort all snippets of failed coordinators eventually. // the ttl on the coordinator is not that high, i.e. if an AQL query // is abandoned by a client application or an end user, the coordinator // ttl should kick in a lot earlier and also terminate the query on the // DB server(s). - _query.queryOptions().ttl = std::max(oldTtl, transaction::Manager::idleTTLDBServer); + _query.queryOptions().ttl = + std::max(oldTtl, transaction::Manager::idleTTLDBServer); auto ttlGuard = scopeGuard([this, oldTtl]() noexcept { // restore previous TTL value _query.queryOptions().ttl = oldTtl; }); - + // remember which servers we add during our setup request ::arangodb::containers::HashSet serversAdded; @@ -465,11 +351,12 @@ Result EngineInfoContainerDBServerServerBased::buildEngines( options.database = _query.vocbase().name(); options.timeout = network::Timeout(SETUP_TIMEOUT); options.skipScheduler = true; // hack to speed up future.get() - + TRI_IF_FAILURE("Query::setupTimeout") { - options.timeout = network::Timeout(0.01 + (double) RandomGenerator::interval(uint32_t(10))); + options.timeout = + network::Timeout(0.01 + (double)RandomGenerator::interval(uint32_t(10))); } - + TRI_IF_FAILURE("Query::setupTimeoutFailSequence") { double t = 0.5; TRI_IF_FAILURE("Query::setupTimeoutFailSequenceRandom") { @@ -479,11 +366,12 @@ Result EngineInfoContainerDBServerServerBased::buildEngines( } options.timeout = network::Timeout(t); } - + /// cluster global query id, under which the query will be registered /// on DB servers from 3.8 onwards. - QueryId clusterQueryId = _query.vocbase().server().getFeature().clusterInfo().uniqid(); - + QueryId clusterQueryId = + _query.vocbase().server().getFeature().clusterInfo().uniqid(); + // decreases lock timeout manually for fast path auto oldLockTimeout = _query.getLockTimeout(); _query.setLockTimeout(FAST_PATH_LOCK_TIMEOUT); @@ -495,7 +383,8 @@ Result EngineInfoContainerDBServerServerBased::buildEngines( for (ServerID const& server : dbServers) { // Build Lookup Infos VPackBuilder infoBuilder; - auto didCreateEngine = buildEngineInfo(clusterQueryId, infoBuilder, server, nodesById, nodeAliases); + auto didCreateEngine = + buildEngineInfo(clusterQueryId, infoBuilder, server, nodesById, nodeAliases); VPackSlice infoSlice = infoBuilder.slice(); if (isNotSatelliteLeader(infoSlice)) { @@ -521,7 +410,7 @@ Result EngineInfoContainerDBServerServerBased::buildEngines( .thenValue([](std::vector>&& responses) -> Result { // We can directly report a non TRI_ERROR_LOCK_TIMEOUT // error as we need to abort after. - // Otherwise we need to report + // Otherwise we need to report Result res; for (auto const& tryRes : responses) { auto response = tryRes.get(); @@ -552,16 +441,17 @@ Result EngineInfoContainerDBServerServerBased::buildEngines( // we got a lock timeout response for the fast path locking... { // in case of fast path failure, we need to cleanup engines - auto requests = cleanupEngines(fastPathResult.get().errorNumber(), _query.vocbase().name(), serverToQueryId); + auto requests = cleanupEngines(fastPathResult.get().errorNumber(), + _query.vocbase().name(), serverToQueryId); // Wait for all cleanup requests to complete. // So we know that all Transactions are aborted. Result res; for (auto& tryRes : requests) { - network::Response const& response = tryRes.get(); + network::Response const& response = tryRes.get(); if (response.fail()) { // note first error, but continue iterating over all results LOG_TOPIC("2d319", DEBUG, Logger::AQL) - << "received error from server " << response.destination + << "received error from server " << response.destination << " during query cleanup: " << response.combinedResult().errorMessage(); res.reset(response.combinedResult()); } @@ -576,25 +466,26 @@ Result EngineInfoContainerDBServerServerBased::buildEngines( } snippetIds.clear(); - + // revert the addition of servers by us for (auto const& s : serversAdded) { trx.state()->removeKnownServer(s); } - + // fast path locking rolled back successfully! TRI_ASSERT(serverToQueryId.empty()); // we must generate a new query id, because the fast path setup has failed - clusterQueryId = _query.vocbase().server().getFeature().clusterInfo().uniqid(); + clusterQueryId = + _query.vocbase().server().getFeature().clusterInfo().uniqid(); if (trx.isMainTransaction() && !trx.state()->isReadOnlyTransaction()) { - // when we are not in a streaming transaction, it is ok to roll a new trx id. - // it is not ok to change the trx id inside a streaming transaction, + // when we are not in a streaming transaction, it is ok to roll a new trx + // id. it is not ok to change the trx id inside a streaming transaction, // because then the caller would not be able to "talk" to the transaction // any further. - // note: read-only transactions do not need to reroll their id, as there will - // be no locks taken. + // note: read-only transactions do not need to reroll their id, as there + // will be no locks taken. trx.state()->coordinatorRerollTransactionId(); } @@ -604,12 +495,12 @@ Result EngineInfoContainerDBServerServerBased::buildEngines( << "Potential deadlock detected, using slow path for locking. This " "is expected if exclusive locks are used."; - // Make sure we always use the same ordering on servers std::sort(engineInformation.begin(), engineInformation.end(), [](auto const& lhs, auto const& rhs) { // Entry <0> is the Server - return TransactionState::ServerIdLessThan(std::get<0>(lhs), std::get<0>(rhs)); + return TransactionState::ServerIdLessThan(std::get<0>(lhs), + std::get<0>(rhs)); }); #ifdef ARANGODB_ENABLE_MAINTAINER_MODE // Make sure we always maintain the correct ordering of servers @@ -635,7 +526,9 @@ Result EngineInfoContainerDBServerServerBased::buildEngines( overwrittenOptions.add("clusterQueryId", VPackValue(clusterQueryId)); addOptionsPart(overwrittenOptions, server); overwrittenOptions.close(); - auto newRequest = arangodb::velocypack::Collection::merge(infoSlice, overwrittenOptions.slice(), false); + auto newRequest = + arangodb::velocypack::Collection::merge(infoSlice, + overwrittenOptions.slice(), false); auto request = buildSetupRequest(trx, std::move(server), newRequest.slice(), std::move(didCreateEngine), snippetIds, serverToQueryId, @@ -656,8 +549,8 @@ Result EngineInfoContainerDBServerServerBased::buildEngines( Result EngineInfoContainerDBServerServerBased::parseResponse( VPackSlice response, MapRemoteToSnippet& queryIds, ServerID const& server, - std::vector const& didCreateEngine, - QueryId& globalQueryId, RebootId& rebootId) const { + std::vector const& didCreateEngine, QueryId& globalQueryId, + RebootId& rebootId) const { TRI_ASSERT(server.substr(0, 7) != "server:"); if (!response.isObject() || !response.get("result").isObject()) { @@ -690,7 +583,7 @@ Result EngineInfoContainerDBServerServerBased::parseResponse( // had prescribed globalQueryId = queryIdSlice.getNumber(); } - + VPackSlice rebootIdSlice = result.get(StaticStrings::RebootId); if (rebootIdSlice.isNumber()) { rebootId = RebootId(rebootIdSlice.getNumber()); @@ -775,7 +668,7 @@ std::vector EngineInfoContainerDBServerServerBased options.database = dbname; options.timeout = network::Timeout(10.0); // Picked arbitrarily options.skipScheduler = true; // hack to speed up future.get() - + // Shutdown query snippets VPackBuffer body; VPackBuilder builder(body); @@ -785,9 +678,10 @@ std::vector EngineInfoContainerDBServerServerBased requests.reserve(serverQueryIds.size()); for (auto const& [server, queryId, rebootId] : serverQueryIds) { TRI_ASSERT(server.substr(0, 7) != "server:"); - requests.emplace_back(network::sendRequestRetry(pool, "server:" + server, fuerte::RestVerb::Delete, - ::finishUrl + std::to_string(queryId), - /*copy*/ body, options)); + requests.emplace_back( + network::sendRequestRetry(pool, "server:" + server, fuerte::RestVerb::Delete, + ::finishUrl + std::to_string(queryId), + /*copy*/ body, options)); } _query.incHttpRequests(static_cast(serverQueryIds.size())); @@ -798,8 +692,9 @@ std::vector EngineInfoContainerDBServerServerBased auto allEngines = gn->engines(); for (auto const& engine : *allEngines) { TRI_ASSERT(engine.first.substr(0, 7) != "server:"); - requests.emplace_back(network::sendRequestRetry(pool, "server:" + engine.first, fuerte::RestVerb::Delete, - ::traverserUrl + basics::StringUtils::itoa(engine.second), noBody, options)); + requests.emplace_back(network::sendRequestRetry( + pool, "server:" + engine.first, fuerte::RestVerb::Delete, + ::traverserUrl + basics::StringUtils::itoa(engine.second), noBody, options)); } _query.incHttpRequests(static_cast(allEngines->size())); gn->clearEngines(); diff --git a/arangod/Aql/EngineInfoContainerDBServerServerBased.h b/arangod/Aql/EngineInfoContainerDBServerServerBased.h index 79dfae0626f7..4322df03a72b 100644 --- a/arangod/Aql/EngineInfoContainerDBServerServerBased.h +++ b/arangod/Aql/EngineInfoContainerDBServerServerBased.h @@ -41,7 +41,7 @@ namespace network { class ConnectionPool; struct RequestOptions; struct Response; -} +} // namespace network namespace velocypack { class Builder; @@ -56,57 +56,6 @@ class QueryContext; class QuerySnippet; class EngineInfoContainerDBServerServerBased { - private: - // @brief Local struct to create the - // information required to build traverser engines - // on DB servers. - class TraverserEngineShardLists { - public: - TraverserEngineShardLists(GraphNode const*, ServerID const& server, - std::unordered_map const& shardMapping, - QueryContext& query); - - ~TraverserEngineShardLists() = default; - - void serializeIntoBuilder(arangodb::velocypack::Builder& infoBuilder) const; - - bool hasShard() const { return _hasShard; } - - /// inaccessible edge and verte collection names -#ifdef USE_ENTERPRISE - std::set inaccessibleCollNames() const { - return _inaccessible; - } -#endif - - private: - std::vector getAllLocalShards(std::unordered_map const& shardMapping, - ServerID const& server, - std::shared_ptr> shardIds); - - private: - // The graph node we need to serialize - GraphNode const* _node; - - // Flag if we found any shard for the given server. - // If not serializeToBuilder will be a noop - bool _hasShard; - - // Mapping for edge collections to shardIds. - // We have to retain the ordering of edge collections, all - // vectors of these in one run need to have identical size. - // This is because the conditions to query those edges have the - // same ordering. - std::vector> _edgeCollections; - - // Mapping for vertexCollections to shardIds. - std::unordered_map> _vertexCollections; - -#ifdef USE_ENTERPRISE - std::set _inaccessible; -#endif - }; - public: explicit EngineInfoContainerDBServerServerBased(QueryContext& query) noexcept; @@ -123,7 +72,6 @@ class EngineInfoContainerDBServerServerBased { // the given queryid of the coordinator as data provider. void closeSnippet(QueryId inputSnippet); - // Build the Engines for the DBServer // * Creates one Query-Entry for each Snippet per Shard (multiple on the // same DB) @@ -140,28 +88,28 @@ class EngineInfoContainerDBServerServerBased { MapRemoteToSnippet& snippetIds, aql::ServerQueryIdList& serverQueryIds, std::map& nodeAliases); - // Insert a GraphNode that needs to generate TraverserEngines on // the DBServers. The GraphNode itself will retain on the coordinator. void addGraphNode(GraphNode* node, bool pushToSingleServer); private: /** - * @brief Helper method to generate the Request to be send to a specific database server. - * this request contains all the necessary information to create a transaction with correct shard - * locking, as well as QuerySnippets and GraphEngines on the receiver side. - * - * @param clusterQueryId cluster-wide query id (used from 3.8 onwards) - * @param infoBuilder (mutable) the request body will be written into this builder. - * @param server The DatabaseServer we suppose to send the request to, used to identify the shards on this server - * @param nodesById A vector to get Nodes by their id. - * @param nodeAliases (mutable) A map of node-aliases, if a server is responsible for more then one shard we need to duplicate some nodes in the query (e.g. an IndexNode can only access one shard at a time) this list can map cloned node -> original node ids. - * - * @return A vector with one entry per GraphNode in the query (in order) it indicates if this Server has created a GraphEngine for this Node and needs to participate in the GraphOperation or not. - */ - std::vector buildEngineInfo(QueryId clusterQueryId, VPackBuilder& infoBuilder, ServerID const& server, - std::unordered_map const& nodesById, - std::map& nodeAliases); + * @brief Helper method to generate the Request to be send to a specific database server. + * this request contains all the necessary information to create a transaction with correct shard + * locking, as well as QuerySnippets and GraphEngines on the receiver side. + * + * @param clusterQueryId cluster-wide query id (used from 3.8 onwards) + * @param infoBuilder (mutable) the request body will be written into this builder. + * @param server The DatabaseServer we suppose to send the request to, used to identify the shards on this server + * @param nodesById A vector to get Nodes by their id. + * @param nodeAliases (mutable) A map of node-aliases, if a server is responsible for more then one shard we need to duplicate some nodes in the query (e.g. an IndexNode can only access one shard at a time) this list can map cloned node -> original node ids. + * + * @return A vector with one entry per GraphNode in the query (in order) it indicates if this Server has created a GraphEngine for this Node and needs to participate in the GraphOperation or not. + */ + std::vector buildEngineInfo( + QueryId clusterQueryId, VPackBuilder& infoBuilder, ServerID const& server, + std::unordered_map const& nodesById, + std::map& nodeAliases); arangodb::futures::Future buildSetupRequest( transaction::Methods& trx, ServerID const& server, VPackSlice infoSlice, @@ -171,7 +119,6 @@ class EngineInfoContainerDBServerServerBased { [[nodiscard]] bool isNotSatelliteLeader(VPackSlice infoSlice) const; - /** * @brief Will send a shutdown to all engines registered in the list of * queryIds. @@ -186,8 +133,8 @@ class EngineInfoContainerDBServerServerBased { * -> queryid. */ std::vector> cleanupEngines( - ErrorCode errorCode, std::string const& dbname, aql::ServerQueryIdList& serverQueryIds) const; - + ErrorCode errorCode, std::string const& dbname, + aql::ServerQueryIdList& serverQueryIds) const; // Insert the Locking information into the message to be send to DBServers void addLockingPart(arangodb::velocypack::Builder& builder, ServerID const& server) const; diff --git a/arangod/Aql/ExecutionBlockImpl.cpp b/arangod/Aql/ExecutionBlockImpl.cpp index b67914fb98c8..2b112f65c59d 100644 --- a/arangod/Aql/ExecutionBlockImpl.cpp +++ b/arangod/Aql/ExecutionBlockImpl.cpp @@ -82,6 +82,7 @@ #include "Graph/Providers/SingleServerProvider.h" #include "Graph/Queues/FifoQueue.h" #include "Graph/Queues/QueueTracer.h" +#include "Graph/Steps/SingleServerProviderStep.h" #include "Graph/algorithm-aliases.h" #include @@ -92,11 +93,13 @@ #include /* SingleServerProvider Section */ +using SingleServerProviderStep = ::arangodb::graph::SingleServerProviderStep; + using KPathRefactored = - arangodb::graph::KPathEnumerator; + arangodb::graph::KPathEnumerator>; using KPathRefactoredTracer = - arangodb::graph::TracedKPathEnumerator; + arangodb::graph::TracedKPathEnumerator>; /* ClusterProvider Section */ using KPathRefactoredCluster = @@ -199,7 +202,7 @@ ExecutionBlockImpl::ExecutionBlockImpl(ExecutionEngine* engine, // Break the stack before waiting. // We should not use this here. _stackBeforeWaiting.popCall(); - + if (_exeNode->isCallstackSplitEnabled()) { _callstackSplit = std::make_unique(*this); } @@ -353,7 +356,7 @@ ExecutionBlockImpl::execute(AqlCallStack const& stack) { TRI_IF_FAILURE("ExecutionBlock::getOrSkipSome3") { THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); } - + // check if this block failed already. if (_firstFailure.fail()) { // if so, just return the stored error. @@ -391,13 +394,17 @@ ExecutionBlockImpl::execute(AqlCallStack const& stack) { TRI_ASSERT(_firstFailure.ok()); // store only the first failure we got _firstFailure = {ex.code(), ex.what()}; - LOG_QUERY("7289a", DEBUG) << printBlockInfo() << " local statemachine failed with exception: " << ex.what(); + LOG_QUERY("7289a", DEBUG) + << printBlockInfo() + << " local statemachine failed with exception: " << ex.what(); throw; } catch (std::exception const& ex) { TRI_ASSERT(_firstFailure.ok()); // store only the first failure we got _firstFailure = {TRI_ERROR_INTERNAL, ex.what()}; - LOG_QUERY("2bbd5", DEBUG) << printBlockInfo() << " local statemachine failed with exception: " << ex.what(); + LOG_QUERY("2bbd5", DEBUG) + << printBlockInfo() + << " local statemachine failed with exception: " << ex.what(); // Rewire the error, to be consistent with potentially next caller. THROW_ARANGO_EXCEPTION(_firstFailure); } @@ -538,7 +545,7 @@ auto ExecutionBlockImpl::allocateOutputBlock(AqlCall&& call) } template -void ExecutionBlockImpl::ensureOutputBlock(AqlCall&& call) { +void ExecutionBlockImpl::ensureOutputBlock(AqlCall&& call) { if (_outputItemRow == nullptr || !_outputItemRow->isInitialized()) { _outputItemRow = allocateOutputBlock(std::move(call)); } else { @@ -745,7 +752,8 @@ auto ExecutionBlockImpl::executeFetcher(ExecutionContext& ctx, // SubqueryStart and the partnered SubqueryEnd by *not* // pushing the upstream request. if constexpr (!std::is_same_v) { - ctx.stack.pushCall(createUpstreamCall(std::move(aqlCall), ctx.clientCallList.hasMoreCalls())); + ctx.stack.pushCall(createUpstreamCall(std::move(aqlCall), + ctx.clientCallList.hasMoreCalls())); } auto const result = std::invoke([&]() { @@ -770,23 +778,24 @@ auto ExecutionBlockImpl::executeFetcher(ExecutionContext& ctx, // TODO - we should avoid flooding the queue with too many tasks as that // can significantly delay processing of user REST requests. - + // we can safely ignore the result here, because we will try to // claim the task ourselves anyway. - std::ignore = SchedulerFeature::SCHEDULER->queue( - RequestLane::INTERNAL_LOW, - [block = this, task = _prefetchTask, stack = ctx.stack]() mutable { - if (!task->tryClaim()) { - return; - } - // task is a copy of the PrefetchTask shared_ptr, and we will only - // attempt to execute the task if we successfully claimed the task. - // i.e., it does not matter if this task lingers around in the - // scheduler queue even after the execution block has been destroyed, - // because in this case we will not be able to claim the task and - // simply return early without accessing the block. - task->execute(*block, stack); - }); + std::ignore = + SchedulerFeature::SCHEDULER->queue(RequestLane::INTERNAL_LOW, + [block = this, task = _prefetchTask, + stack = ctx.stack]() mutable { + if (!task->tryClaim()) { + return; + } + // task is a copy of the PrefetchTask shared_ptr, and we will only + // attempt to execute the task if we successfully claimed the task. + // i.e., it does not matter if this task lingers around in the + // scheduler queue even after the execution block has been destroyed, + // because in this case we will not be able to claim the task and + // simply return early without accessing the block. + task->execute(*block, stack); + }); } if constexpr (!std::is_same_v) { @@ -1248,10 +1257,10 @@ auto ExecutionBlockImpl::executeFastForward(typename Fetcher::DataRang template std::tuple ExecutionBlockImpl::executeWithoutTrace(AqlCallStack const& callStack) { - // We can only work on a Stack that has valid calls for all levels. + // We can only work on a Stack that has valid calls for all levels. TRI_ASSERT(callStack.hasAllValidCalls()); ExecutionContext ctx(*this, callStack); - + ExecutorState localExecutorState = ExecutorState::DONE; if constexpr (executorCanReturnWaiting) { @@ -1385,8 +1394,8 @@ ExecutionBlockImpl::executeWithoutTrace(AqlCallStack const& callStack) TRI_ASSERT(ctx.clientCall.getSkipCount() == 0); switch (_execState) { case ExecState::CHECKCALL: { - LOG_QUERY("cfe46", DEBUG) - << printTypeInfo() << " determine next action on call " << ctx.clientCall; + LOG_QUERY("cfe46", DEBUG) << printTypeInfo() << " determine next action on call " + << ctx.clientCall; if constexpr (executorHasSideEffects) { // If the executor has sideEffects, and we need to skip the results we would @@ -1639,19 +1648,21 @@ ExecutionBlockImpl::executeWithoutTrace(AqlCallStack const& callStack) // executors. TRI_ASSERT(isMultiDepExecutor || !lastRangeHasDataRow()); TRI_ASSERT(!_lastRange.hasShadowRow()); - + #ifdef ARANGODB_ENABLE_MAINTAINER_MODE auto subqueryLevelBefore = ctx.stack.subqueryLevel(); #endif SkipResult skippedLocal; if (_callstackSplit) { - // we need to split the callstack to avoid stack overflows, so we move upstream - // execution into a separate thread - std::tie(_upstreamState, skippedLocal, _lastRange) = _callstackSplit->execute(ctx, _upstreamRequest); + // we need to split the callstack to avoid stack overflows, so we move + // upstream execution into a separate thread + std::tie(_upstreamState, skippedLocal, _lastRange) = + _callstackSplit->execute(ctx, _upstreamRequest); } else { - std::tie(_upstreamState, skippedLocal, _lastRange) = executeFetcher(ctx, _upstreamRequest); + std::tie(_upstreamState, skippedLocal, _lastRange) = + executeFetcher(ctx, _upstreamRequest); } - + #ifdef ARANGODB_ENABLE_MAINTAINER_MODE TRI_ASSERT(subqueryLevelBefore == ctx.stack.subqueryLevel()); #endif @@ -2013,8 +2024,8 @@ auto ExecutionBlockImpl::testInjectInputRange(DataRange range, SkipRes #endif template -ExecutionBlockImpl::ExecutionContext::ExecutionContext( - ExecutionBlockImpl& block, AqlCallStack const& callstack) +ExecutionBlockImpl::ExecutionContext::ExecutionContext(ExecutionBlockImpl& block, + AqlCallStack const& callstack) : stack(callstack), clientCallList(this->stack.popCall()) { if constexpr (std::is_same_v) { // In subqeryEndExecutor we actually manage two calls. @@ -2035,7 +2046,7 @@ ExecutionBlockImpl::ExecutionContext::ExecutionContext( // We got called with a skip count already set! // Caller is wrong fix it. TRI_ASSERT(clientCall.getSkipCount() == 0); - + TRI_ASSERT(!(clientCall.getOffset() == 0 && clientCall.softLimit == AqlCall::Limit{0u})); TRI_ASSERT(!(clientCall.hasSoftLimit() && clientCall.fullCount)); TRI_ASSERT(!(clientCall.hasSoftLimit() && clientCall.hasHardLimit())); @@ -2082,7 +2093,8 @@ auto ExecutionBlockImpl::PrefetchTask::stealResult() noexcept -> Prefe } template -void ExecutionBlockImpl::PrefetchTask::execute(ExecutionBlockImpl& block, AqlCallStack& stack) { +void ExecutionBlockImpl::PrefetchTask::execute(ExecutionBlockImpl& block, + AqlCallStack& stack) { if constexpr (std::is_same_v || executorHasSideEffects) { TRI_ASSERT(false); @@ -2103,12 +2115,12 @@ void ExecutionBlockImpl::PrefetchTask::execute(ExecutionBlockImpl& blo } template -ExecutionBlockImpl::CallstackSplit::CallstackSplit(ExecutionBlockImpl& block) : - _block(block), - _thread(&CallstackSplit::run, this, std::cref(ExecContext::current())) {} - +ExecutionBlockImpl::CallstackSplit::CallstackSplit(ExecutionBlockImpl& block) + : _block(block), + _thread(&CallstackSplit::run, this, std::cref(ExecContext::current())) {} + template -ExecutionBlockImpl::CallstackSplit::~CallstackSplit() { +ExecutionBlockImpl::CallstackSplit::~CallstackSplit() { _lock.lock(); _state.store(State::Stopped); _lock.unlock(); @@ -2118,7 +2130,8 @@ ExecutionBlockImpl::CallstackSplit::~CallstackSplit() { } template -auto ExecutionBlockImpl::CallstackSplit::execute(ExecutionContext& ctx, AqlCallType const& aqlCall) +auto ExecutionBlockImpl::CallstackSplit::execute(ExecutionContext& ctx, + AqlCallType const& aqlCall) -> UpstreamResult { std::variant result{std::nullopt}; Params params{result, ctx, aqlCall}; @@ -2130,7 +2143,7 @@ auto ExecutionBlockImpl::CallstackSplit::execute(ExecutionContext& ctx } _bell.notify_one(); - + std::unique_lock guard(_lock); _bell.wait(guard, [this]() { return _state.load(std::memory_order_acquire) != State::Executing; @@ -2143,7 +2156,7 @@ auto ExecutionBlockImpl::CallstackSplit::execute(ExecutionContext& ctx return std::get(std::move(result)); } - + template void ExecutionBlockImpl::CallstackSplit::run(ExecContext const& execContext) { ExecContextScope scope(&execContext); @@ -2160,7 +2173,7 @@ void ExecutionBlockImpl::CallstackSplit::run(ExecContext const& execCo try { _params->result = _block.executeFetcher(_params->ctx, _params->aqlCall); - } catch(...) { + } catch (...) { _params->result = std::current_exception(); } diff --git a/arangod/Aql/ExecutionEngine.cpp b/arangod/Aql/ExecutionEngine.cpp index 736ec9b47b2d..08fdf58d845d 100644 --- a/arangod/Aql/ExecutionEngine.cpp +++ b/arangod/Aql/ExecutionEngine.cpp @@ -52,32 +52,6 @@ using namespace arangodb; using namespace arangodb::aql; -// @brief Local struct to create the -// information required to build traverser engines -// on DB servers. -struct TraverserEngineShardLists { - explicit TraverserEngineShardLists(size_t length) { - // Make sure they all have a fixed size. - edgeCollections.resize(length); - } - - ~TraverserEngineShardLists() = default; - - // Mapping for edge collections to shardIds. - // We have to retain the ordering of edge collections, all - // vectors of these in one run need to have identical size. - // This is because the conditions to query those edges have the - // same ordering. - std::vector> edgeCollections; - - // Mapping for vertexCollections to shardIds. - std::unordered_map> vertexCollections; - -#ifdef USE_ENTERPRISE - std::set inaccessibleShards; -#endif -}; - /** * @brief Create AQL blocks from a list of ExectionNodes * Only works in cluster mode diff --git a/arangod/Aql/FixedVarExpressionContext.cpp b/arangod/Aql/FixedVarExpressionContext.cpp index b264ec28a97b..cd074053faac 100644 --- a/arangod/Aql/FixedVarExpressionContext.cpp +++ b/arangod/Aql/FixedVarExpressionContext.cpp @@ -50,20 +50,24 @@ AqlValue FixedVarExpressionContext::getVariableValue(Variable const* variable, b return it->second; } -void FixedVarExpressionContext::clearVariableValues() { _vars.clear(); } +void FixedVarExpressionContext::clearVariableValues() noexcept { _vars.clear(); } void FixedVarExpressionContext::setVariableValue(Variable const* var, AqlValue const& value) { _vars.try_emplace(var, value); } +void FixedVarExpressionContext::clearVariableValue(Variable const* var) { + _vars.erase(var); +} + void FixedVarExpressionContext::serializeAllVariables(velocypack::Options const& opts, velocypack::Builder& builder) const { TRI_ASSERT(builder.isOpenArray()); for (auto const& it : _vars) { builder.openArray(); it.first->toVelocyPack(builder); - it.second.toVelocyPack(&opts, builder, /*resolveExternals*/true, - /*allowUnindexed*/false); + it.second.toVelocyPack(&opts, builder, /*resolveExternals*/ true, + /*allowUnindexed*/ false); builder.close(); } } diff --git a/arangod/Aql/FixedVarExpressionContext.h b/arangod/Aql/FixedVarExpressionContext.h index f63cdbc22b59..89c228d866cc 100644 --- a/arangod/Aql/FixedVarExpressionContext.h +++ b/arangod/Aql/FixedVarExpressionContext.h @@ -38,8 +38,7 @@ class AqlItemBlock; class FixedVarExpressionContext final : public QueryExpressionContext { public: - explicit FixedVarExpressionContext(transaction::Methods& trx, - QueryContext& query, + explicit FixedVarExpressionContext(transaction::Methods& trx, QueryContext& query, AqlFunctionsInternalCache& cache); ~FixedVarExpressionContext() override = default; @@ -49,10 +48,16 @@ class FixedVarExpressionContext final : public QueryExpressionContext { AqlValue getVariableValue(Variable const* variable, bool doCopy, bool& mustDestroy) const override; - void clearVariableValues(); + void clearVariableValues() noexcept; + // @brief This method will set the given variable to the given AQL value + // if the variable already holds a value, this method will keep the old value. void setVariableValue(Variable const*, AqlValue const&); + // @brief This method will only clear the given variable, and keep + // all others intact. If the variable does not exist, this is a noop. + void clearVariableValue(Variable const*); + void serializeAllVariables(velocypack::Options const& opts, arangodb::velocypack::Builder&) const; diff --git a/arangod/Aql/GraphNode.cpp b/arangod/Aql/GraphNode.cpp index f9c0d6f200ee..9cb1a77d0d87 100644 --- a/arangod/Aql/GraphNode.cpp +++ b/arangod/Aql/GraphNode.cpp @@ -90,7 +90,7 @@ TRI_edge_direction_e parseDirection(AstNode const* node) { return uint64ToDirection(dirNode->getIntValue()); } -} +} // namespace GraphNode::GraphNode(ExecutionPlan* plan, ExecutionNodeId id, TRI_vocbase_t* vocbase, AstNode const* direction, @@ -300,9 +300,10 @@ GraphNode::GraphNode(ExecutionPlan* plan, arangodb::velocypack::Slice const& bas _defaultDirection(uint64ToDirection(arangodb::basics::VelocyPackHelper::stringUInt64( base.get("defaultDirection")))), _optionsBuilt(false), - _isSmart(arangodb::basics::VelocyPackHelper::getBooleanValue(base, "isSmart", false)), - _isDisjoint(arangodb::basics::VelocyPackHelper::getBooleanValue(base, "isDisjoint", false)) { - + _isSmart(arangodb::basics::VelocyPackHelper::getBooleanValue(base, StaticStrings::IsSmart, + false)), + _isDisjoint(arangodb::basics::VelocyPackHelper::getBooleanValue(base, StaticStrings::IsDisjoint, + false)) { if (!ServerState::instance()->isDBServer()) { // Graph Information. Do we need to reload the graph here? std::string graphName; @@ -356,7 +357,8 @@ GraphNode::GraphNode(ExecutionPlan* plan, arangodb::velocypack::Slice const& bas auto getAqlCollectionFromName = [&](std::string const& name) -> aql::Collection& { // if the collection was already existent in the query, addCollection will // just return it. - return *query.collections().add(name, AccessMode::Type::READ, aql::Collection::Hint::Collection); + return *query.collections().add(name, AccessMode::Type::READ, + aql::Collection::Hint::Collection); }; auto vPackDirListIter = VPackArrayIterator(dirList); @@ -511,7 +513,9 @@ std::string const& GraphNode::collectionToShardName(std::string const& collName) auto found = _collectionToShard.find(collName); if (found == _collectionToShard.end()) { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL_AQL, "unable to find shard '" + collName + "' in query shard map"); + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL_AQL, + "unable to find shard '" + collName + + "' in query shard map"); } return found->second; } @@ -692,6 +696,18 @@ void GraphNode::getConditionVariables(std::vector& res) const { Collection const* GraphNode::collection() const { TRI_ASSERT(ServerState::instance()->isCoordinator()); TRI_ASSERT(!_edgeColls.empty()); + for (auto const* c : _edgeColls) { + // We are required to valuate non-satellites above + // satellites, as the collection is used as the protoype + // for this graphs sharding. + // The Satellite collection does not have sharding. + TRI_ASSERT(c != nullptr); + if (!c->isSatellite()) { + return c; + } + } + // We have not found any non-satellite Collection + // just return the first satellite then. TRI_ASSERT(_edgeColls.front() != nullptr); return _edgeColls.front(); } @@ -719,8 +735,8 @@ void GraphNode::enhanceEngineInfo(VPackBuilder& builder) const { } #endif -void GraphNode::addEdgeCollection(Collections const& collections, std::string const& name, - TRI_edge_direction_e dir) { +void GraphNode::addEdgeCollection(Collections const& collections, + std::string const& name, TRI_edge_direction_e dir) { auto aqlCollection = collections.get(name); if (aqlCollection != nullptr) { addEdgeCollection(*aqlCollection, dir); diff --git a/arangod/Aql/GraphNode.h b/arangod/Aql/GraphNode.h index 6febbc9824dc..7f53b4f13ad8 100644 --- a/arangod/Aql/GraphNode.h +++ b/arangod/Aql/GraphNode.h @@ -23,12 +23,12 @@ #pragma once -#include "Aql/types.h" #include "Aql/Condition.h" #include "Aql/ExecutionNode.h" #include "Aql/ExecutionNodeId.h" #include "Aql/GraphNode.h" #include "Aql/Graphs.h" +#include "Aql/types.h" #include "Cluster/ClusterTypes.h" #include "VocBase/LogicalCollection.h" #include "VocBase/voc-types.h" @@ -81,10 +81,13 @@ class GraphNode : public ExecutionNode { GraphNode(ExecutionPlan* plan, arangodb::velocypack::Slice const& base); public: + // QueryPlan decided that we use this graph as a satellite bool isUsedAsSatellite() const; + // Defines whether a GraphNode can fully be pushed down to a DBServer bool isLocalGraphNode() const; + // Will wait as soon as any of our collections is a satellite (in sync) void waitForSatelliteIfRequired(ExecutionEngine const* engine) const; - + // Can be fully pushed down to a DBServer and is available on all DBServers bool isEligibleAsSatelliteTraversal() const; protected: @@ -186,8 +189,8 @@ class GraphNode : public ExecutionNode { void injectVertexCollection(aql::Collection& other); std::vector collections() const; - void setCollectionToShard(std::map const& map) { - _collectionToShard = map; + void resetCollectionToShard() { + _collectionToShard.clear(); } void addCollectionToShard(std::string const& coll, std::string const& shard) { // NOTE: Do not replace this by emplace or insert. @@ -215,7 +218,8 @@ class GraphNode : public ExecutionNode { void graphCloneHelper(ExecutionPlan& plan, GraphNode& clone, bool withProperties) const; private: - void addEdgeCollection(aql::Collections const& collections, std::string const& name, TRI_edge_direction_e dir); + void addEdgeCollection(aql::Collections const& collections, + std::string const& name, TRI_edge_direction_e dir); void addEdgeCollection(aql::Collection& collection, TRI_edge_direction_e dir); void addVertexCollection(aql::Collections const& collections, std::string const& name); void addVertexCollection(aql::Collection& collection); @@ -278,7 +282,7 @@ class GraphNode : public ExecutionNode { std::unordered_map _engines; /// @brief list of shards involved, required for one-shard-databases - std::map _collectionToShard; + std::unordered_map _collectionToShard; }; } // namespace aql diff --git a/arangod/Aql/KShortestPathsExecutor.cpp b/arangod/Aql/KShortestPathsExecutor.cpp index 70a6d4cbac17..3ef29e1d48cc 100644 --- a/arangod/Aql/KShortestPathsExecutor.cpp +++ b/arangod/Aql/KShortestPathsExecutor.cpp @@ -40,6 +40,7 @@ #include "Graph/Queues/QueueTracer.h" #include "Graph/ShortestPathOptions.h" #include "Graph/ShortestPathResult.h" +#include "Graph/Steps/SingleServerProviderStep.h" #include "Transaction/Helpers.h" #include "Graph/algorithm-aliases.h" @@ -143,8 +144,8 @@ auto KShortestPathsExecutorInfos::getTargetVertex() const noexcept template auto KShortestPathsExecutorInfos::cache() const -> graph::TraverserCache* { - if constexpr (std::is_same_v> || - std::is_same_v> || + if constexpr (std::is_same_v>> || + std::is_same_v>> || std::is_same_v> || std::is_same_v> @@ -267,8 +268,8 @@ template auto KShortestPathsExecutor::doOutputPath(OutputAqlItemRow& output) -> void { transaction::BuilderLeaser tmp{&_trx}; tmp->clear(); - if constexpr (std::is_same_v> || - std::is_same_v> || + if constexpr (std::is_same_v>> || + std::is_same_v>> || std::is_same_v> || std::is_same_v>) { if (_finder.getNextPath(*tmp.builder())) { @@ -306,8 +307,8 @@ auto KShortestPathsExecutor::getVertexId(InputVertex const& vertex, try { std::string idString; // TODO: calculate expression once e.g. header constexpr bool and check then here - if constexpr (std::is_same_v> || - std::is_same_v> || + if constexpr (std::is_same_v>> || + std::is_same_v>> || std::is_same_v> || std::is_same_v>) { idString = _trx.extractIdString(in.slice()); @@ -378,11 +379,11 @@ template class ::arangodb::aql::KShortestPathsExecutorInfos>; -template class ::arangodb::aql::KShortestPathsExecutorInfos>; +template class ::arangodb::aql::KShortestPathsExecutorInfos>>; +template class ::arangodb::aql::KShortestPathsExecutorInfos>>; -template class ::arangodb::aql::KShortestPathsExecutor>; -template class ::arangodb::aql::KShortestPathsExecutor>; +template class ::arangodb::aql::KShortestPathsExecutor>>; +template class ::arangodb::aql::KShortestPathsExecutor>>; /* ClusterProvider Section */ diff --git a/arangod/Aql/KShortestPathsNode.cpp b/arangod/Aql/KShortestPathsNode.cpp index 732ad62d9da4..2be28755a2e8 100644 --- a/arangod/Aql/KShortestPathsNode.cpp +++ b/arangod/Aql/KShortestPathsNode.cpp @@ -48,6 +48,7 @@ #include "Graph/Queues/QueueTracer.h" #include "Graph/ShortestPathOptions.h" #include "Graph/ShortestPathResult.h" +#include "Graph/Steps/SingleServerProviderStep.h" #include "Indexes/Index.h" #include "OptimizerUtils.h" #include "Utils/CollectionNameResolver.h" @@ -337,26 +338,37 @@ std::unique_ptr KShortestPathsNode::createBlock( if (shortestPathType() == arangodb::graph::ShortestPathType::Type::KPaths) { arangodb::graph::TwoSidedEnumeratorOptions enumeratorOptions{opts->minDepth, opts->maxDepth}; + PathValidatorOptions validatorOptions(opts->tmpVar(), opts->getExpressionCtx()); if (!ServerState::instance()->isCoordinator()) { // Create IndexAccessor for BaseProviderOptions (TODO: Location need to // be changed in the future) create BaseProviderOptions - BaseProviderOptions forwardProviderOptions(opts->tmpVar(), buildUsedIndexes(), + + std::pair, std::unordered_map>> usedIndexes{}; + usedIndexes.first = buildUsedIndexes(); + + std::pair, std::unordered_map>> reversedUsedIndexes{}; + reversedUsedIndexes.first = buildReverseUsedIndexes(); + + BaseProviderOptions forwardProviderOptions(opts->tmpVar(), std::move(usedIndexes), + opts->getExpressionCtx(), opts->collectionToShard()); - BaseProviderOptions backwardProviderOptions(opts->tmpVar(), - buildReverseUsedIndexes(), + BaseProviderOptions backwardProviderOptions(opts->tmpVar(), std::move(reversedUsedIndexes), + opts->getExpressionCtx(), opts->collectionToShard()); if (opts->query().queryOptions().getTraversalProfileLevel() == TraversalProfileLevel::None) { - using KPathRefactored = KPathEnumerator; + using KPathRefactored = + KPathEnumerator>; auto kPathUnique = std::make_unique( - SingleServerProvider{opts->query(), forwardProviderOptions, - opts->query().resourceMonitor()}, - SingleServerProvider{opts->query(), backwardProviderOptions, - opts->query().resourceMonitor()}, - std::move(enumeratorOptions), opts->query().resourceMonitor()); + SingleServerProvider{opts->query(), std::move(forwardProviderOptions), + opts->query().resourceMonitor()}, + SingleServerProvider{opts->query(), std::move(backwardProviderOptions), + opts->query().resourceMonitor()}, + std::move(enumeratorOptions), std::move(validatorOptions), + opts->query().resourceMonitor()); auto executorInfos = KShortestPathsExecutorInfos(outputRegister, engine.getQuery(), @@ -366,13 +378,15 @@ std::unique_ptr KShortestPathsNode::createBlock( &engine, this, std::move(registerInfos), std::move(executorInfos)); } else { // TODO: implement better initialization with less duplicate code - using TracedKPathRefactored = TracedKPathEnumerator; + using TracedKPathRefactored = + TracedKPathEnumerator>; auto kPathUnique = std::make_unique( - ProviderTracer{opts->query(), forwardProviderOptions, - opts->query().resourceMonitor()}, - ProviderTracer{opts->query(), backwardProviderOptions, - opts->query().resourceMonitor()}, - std::move(enumeratorOptions), opts->query().resourceMonitor()); + ProviderTracer>{ + opts->query(), std::move(forwardProviderOptions), opts->query().resourceMonitor()}, + ProviderTracer>{ + opts->query(), std::move(backwardProviderOptions), opts->query().resourceMonitor()}, + std::move(enumeratorOptions), std::move(validatorOptions), + opts->query().resourceMonitor()); auto executorInfos = KShortestPathsExecutorInfos(outputRegister, engine.getQuery(), @@ -396,7 +410,8 @@ std::unique_ptr KShortestPathsNode::createBlock( opts->query().resourceMonitor()}, ClusterProvider{opts->query(), backwardProviderOptions, opts->query().resourceMonitor()}, - std::move(enumeratorOptions), opts->query().resourceMonitor()); + std::move(enumeratorOptions), std::move(validatorOptions), + opts->query().resourceMonitor()); auto executorInfos = KShortestPathsExecutorInfos(outputRegister, engine.getQuery(), @@ -412,7 +427,8 @@ std::unique_ptr KShortestPathsNode::createBlock( opts->query().resourceMonitor()}, ProviderTracer{opts->query(), backwardProviderOptions, opts->query().resourceMonitor()}, - std::move(enumeratorOptions), opts->query().resourceMonitor()); + std::move(enumeratorOptions), std::move(validatorOptions), + opts->query().resourceMonitor()); auto executorInfos = KShortestPathsExecutorInfos(outputRegister, engine.getQuery(), @@ -471,7 +487,7 @@ void KShortestPathsNode::kShortestPathsCloneHelper(ExecutionPlan& plan, c._fromCondition = _fromCondition->clone(_plan->getAst()); c._toCondition = _toCondition->clone(_plan->getAst()); } - + void KShortestPathsNode::replaceVariables(std::unordered_map const& replacements) { if (_inStartVariable != nullptr) { _inStartVariable = Variable::replace(_inStartVariable, replacements); @@ -520,7 +536,8 @@ std::vector KShortestPathsNode::buildUsedIndexes } indexAccessors.emplace_back(indexToUse, - _toCondition->clone(options()->query().ast()), 0); + _toCondition->clone(options()->query().ast()), + 0, nullptr, i); break; } case TRI_EDGE_OUT: { @@ -535,7 +552,8 @@ std::vector KShortestPathsNode::buildUsedIndexes } indexAccessors.emplace_back(indexToUse, - _fromCondition->clone(options()->query().ast()), 0); + _fromCondition->clone(options()->query().ast()), + 0, nullptr, i); break; } case TRI_EDGE_ANY: @@ -567,7 +585,8 @@ std::vector KShortestPathsNode::buildReverseUsed } indexAccessors.emplace_back(indexToUse, - _fromCondition->clone(options()->query().ast()), 0); + _fromCondition->clone(options()->query().ast()), + 0, nullptr, i); break; } case TRI_EDGE_OUT: { @@ -582,7 +601,8 @@ std::vector KShortestPathsNode::buildReverseUsed } indexAccessors.emplace_back(indexToUse, - _toCondition->clone(options()->query().ast()), 0); + _toCondition->clone(options()->query().ast()), + 0, nullptr, i); break; } case TRI_EDGE_ANY: diff --git a/arangod/Aql/OptimizerRules.cpp b/arangod/Aql/OptimizerRules.cpp index 6baeeae7361a..222ec8619269 100644 --- a/arangod/Aql/OptimizerRules.cpp +++ b/arangod/Aql/OptimizerRules.cpp @@ -828,14 +828,14 @@ using EN = arangodb::aql::ExecutionNode; namespace arangodb { namespace aql { -// checks if the path variable (variable) can be optimized away, or restricted to some -// attributes (vertices, edges, weights) -bool optimizeTraversalPathVariable(Variable const* variable, TraversalNode* traversal, +// checks if the path variable (variable) can be optimized away, or restricted +// to some attributes (vertices, edges, weights) +bool optimizeTraversalPathVariable(Variable const* variable, TraversalNode* traversal, std::vector const& pruneVars) { if (variable == nullptr) { return false; } - + auto* options = static_cast(traversal->options()); @@ -866,10 +866,11 @@ bool optimizeTraversalPathVariable(Variable const* variable, TraversalNode* trav current->getVariablesUsedHere(vars); if (vars.find(variable) != vars.end()) { // path variable used here - Expression* exp = ExecutionNode::castTo(current)->expression(); + Expression* exp = + ExecutionNode::castTo(current)->expression(); AstNode const* node = exp->node(); if (!Ast::getReferencedAttributes(node, variable, attributes)) { - // full path variable is used, or accessed in a way that we don't + // full path variable is used, or accessed in a way that we don't // understand, e.g. "p" or "p[0]" or "p[*]..." canOptimize = false; } @@ -877,8 +878,8 @@ bool optimizeTraversalPathVariable(Variable const* variable, TraversalNode* trav break; } default: { - // if the path is used by any other node type, we don't know what to do - // and will not optimize parts of it away + // if the path is used by any other node type, we don't know what to + // do and will not optimize parts of it away vars.clear(); current->getVariablesUsedHere(vars); if (vars.find(variable) != vars.end()) { @@ -892,14 +893,17 @@ bool optimizeTraversalPathVariable(Variable const* variable, TraversalNode* trav if (canOptimize) { // check which attributes from the path are actually used - bool producePathsVertices = (attributes.find(StaticStrings::GraphQueryVertices) != attributes.end()); - bool producePathsEdges = (attributes.find(StaticStrings::GraphQueryEdges) != attributes.end()); - bool producePathsWeights = (attributes.find(StaticStrings::GraphQueryWeights) != attributes.end()) && - (options->mode == traverser::TraverserOptions::Order::WEIGHTED); + bool producePathsVertices = + (attributes.find(StaticStrings::GraphQueryVertices) != attributes.end()); + bool producePathsEdges = + (attributes.find(StaticStrings::GraphQueryEdges) != attributes.end()); + bool producePathsWeights = + (attributes.find(StaticStrings::GraphQueryWeights) != attributes.end()) && + (options->mode == traverser::TraverserOptions::Order::WEIGHTED); if (!producePathsVertices && !producePathsEdges && !producePathsWeights && !attributes.empty()) { - // none of the existing path attributes is actually accessed - but a different + // none of the existing path attributes is actually accessed - but a different // (non-existing) attribute is accessed, e.g. `p.whatever`. // in order to not optimize away our path variable, and then being unable to access // the non-existing attribute, we simply activate the production of vertices. @@ -918,11 +922,13 @@ bool optimizeTraversalPathVariable(Variable const* variable, TraversalNode* trav return false; /*modified*/ } -Collection* addCollectionToQuery(QueryContext& query, std::string const& cname, char const* context) { +Collection* addCollectionToQuery(QueryContext& query, std::string const& cname, + char const* context) { aql::Collection* coll = nullptr; if (!cname.empty()) { - coll = query.collections().add(cname, AccessMode::Type::READ, aql::Collection::Hint::Collection); + coll = query.collections().add(cname, AccessMode::Type::READ, + aql::Collection::Hint::Collection); // simon: code below is used for FULLTEXT(), WITHIN(), NEAR(), .. // could become unnecessary if the AST takes care of adding the collections if (!ServerState::instance()->isCoordinator()) { @@ -932,9 +938,9 @@ Collection* addCollectionToQuery(QueryContext& query, std::string const& cname, } if (coll == nullptr) { - THROW_ARANGO_EXCEPTION_MESSAGE( - TRI_ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH, - std::string("collection '") + cname + "' used in " + context + " not found"); + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_FUNCTION_ARGUMENT_TYPE_MISMATCH, + std::string("collection '") + cname + + "' used in " + context + " not found"); } return coll; @@ -1056,10 +1062,12 @@ void arangodb::aql::sortInValuesRule(Optimizer* opt, std::unique_ptrtype == NODE_TYPE_FCALL && - (static_cast(testNode->getData())->name == "SORTED_UNIQUE" || - static_cast(testNode->getData())->name == "SORTED")) { - // we don't need to sort results of a function that already returns sorted - // results + (static_cast(testNode->getData())->name == + "SORTED_UNIQUE" || + static_cast(testNode->getData())->name == + "SORTED")) { + // we don't need to sort results of a function that already returns + // sorted results AstNode* clone = ast->shallowCopyForModify(inNode); auto sg = arangodb::scopeGuard([&]() noexcept { FINALIZE_SUBTREE(clone); }); @@ -1445,8 +1453,7 @@ void arangodb::aql::removeCollectVariablesRule(Optimizer* opt, } // end - if collectNode has outVariable collectNode->clearAggregates( - [&varsUsedLater, &modified]( - AggregateVarInfo const& aggregate) -> bool { + [&varsUsedLater, &modified](AggregateVarInfo const& aggregate) -> bool { if (varsUsedLater.find(aggregate.outVar) == varsUsedLater.end()) { // result of aggregate function not used later modified = true; @@ -1952,8 +1959,7 @@ void arangodb::aql::specializeCollectRule(Optimizer* opt, // test if we can use an alternative version of COLLECT with a hash table bool const canUseHashAggregation = - (!groupVariables.empty() && - !collectNode->hasOutVariable() && + (!groupVariables.empty() && !collectNode->hasOutVariable() && collectNode->getOptions().canUseMethod(CollectOptions::CollectMethod::HASH)); if (canUseHashAggregation && !opt->runOnlyRequiredRules(1)) { @@ -2029,8 +2035,7 @@ void arangodb::aql::specializeCollectRule(Optimizer* opt, // no need to run this specific rule again on the cloned plan opt->addPlan(std::move(newPlan), rule, true); } - } else if (groupVariables.empty() && - collectNode->hasOutVariable() == false && + } else if (groupVariables.empty() && collectNode->hasOutVariable() == false && collectNode->aggregateVariables().size() == 1 && collectNode->aggregateVariables()[0].type == "LENGTH") { // we have no groups and only a single aggregator of type LENGTH, so we @@ -2157,10 +2162,10 @@ void arangodb::aql::moveFiltersUpRule(Optimizer* opt, std::unique_ptraddPlan(std::move(plan), rule, modified); } -struct VariableReplacer final : public WalkerWorker { +struct VariableReplacer final + : public WalkerWorker { public: - explicit VariableReplacer( - std::unordered_map const& replacements) + explicit VariableReplacer(std::unordered_map const& replacements) : replacements(replacements) {} bool before(ExecutionNode* en) override final { @@ -2377,7 +2382,8 @@ void arangodb::aql::simplifyConditionsRule(Optimizer* opt, for (auto const& n : nodes) { auto nn = ExecutionNode::castTo(n); - if (!nn->expression()->isDeterministic() || nn->outVariable()->type() == Variable::Type::Const) { + if (!nn->expression()->isDeterministic() || + nn->outVariable()->type() == Variable::Type::Const) { // If this node is non-deterministic or has a constant expression, we must not touch it! continue; } @@ -2773,9 +2779,9 @@ void arangodb::aql::removeUnnecessaryCalculationsRule(Optimizer* opt, auto nLoop = n->getLoop(); if (nLoop != otherLoop) { - // original expression calls a function and is not contained in a loop. - // we're about to move this expression into a loop, but we don't want - // to move (expensive) function calls into loops + // original expression calls a function and is not contained in a + // loop. we're about to move this expression into a loop, but we + // don't want to move (expensive) function calls into loops continue; } VarSet outer = nLoop->getVarsValid(); @@ -2997,10 +3003,11 @@ struct SortToIndexNode final Collection const* coll = enumerateCollectionNode->collection(); TRI_ASSERT(coll != nullptr); - size_t numDocs = coll->count(&_plan->getAst()->query().trxForOptimization(), transaction::CountType::TryCache); + size_t numDocs = coll->count(&_plan->getAst()->query().trxForOptimization(), + transaction::CountType::TryCache); - bool canBeUsed = arangodb::aql::utils::getIndexForSortCondition(*coll, - &sortCondition, outVariable, numDocs, + bool canBeUsed = arangodb::aql::utils::getIndexForSortCondition( + *coll, &sortCondition, outVariable, numDocs, enumerateCollectionNode->hint(), usedIndexes, coveredAttributes); if (canBeUsed) { // If this bit is set, then usedIndexes has length exactly one @@ -3840,13 +3847,14 @@ void arangodb::aql::scatterInClusterRule(Optimizer* opt, std::unique_ptr DistributeNode* { - auto collection = static_cast(nullptr); - auto inputVariable = static_cast(nullptr); - + auto collection = static_cast(nullptr); + auto inputVariable = static_cast(nullptr); + + bool isGraphNode = false; // TODO: this seems a bit verbose, but is at least local & simple - // the modification nodes are all collectionaccessing, the graph nodes are - // currently assumed to be disjoint, and hence smart, so all collections - // are sharded the same way! + // the modification nodes are all collectionaccessing, the graph nodes + // are currently assumed to be disjoint, and hence smart, so all + // collections are sharded the same way! switch (node->getType()) { case ExecutionNode::INSERT: { auto const* insertNode = ExecutionNode::castTo(node); @@ -3879,6 +3887,7 @@ auto arangodb::aql::createDistributeNodeFor(ExecutionPlan& plan, ExecutionNode* TRI_ASSERT(traversalNode->isDisjoint()); collection = traversalNode->collection(); inputVariable = traversalNode->inVariable(); + isGraphNode = true; } break; case ExecutionNode::K_SHORTEST_PATHS: { auto kShortestPathsNode = ExecutionNode::castTo(node); @@ -3886,12 +3895,14 @@ auto arangodb::aql::createDistributeNodeFor(ExecutionPlan& plan, ExecutionNode* collection = kShortestPathsNode->collection(); // Subtle: KShortestPathsNode uses a reference when returning startInVariable inputVariable = &kShortestPathsNode->startInVariable(); + isGraphNode = true; } break; case ExecutionNode::SHORTEST_PATH: { auto shortestPathNode = ExecutionNode::castTo(node); TRI_ASSERT(shortestPathNode->isDisjoint()); collection = shortestPathNode->collection(); inputVariable = shortestPathNode->startInVariable(); + isGraphNode = true; } break; default: { TRI_ASSERT(false); @@ -3903,16 +3914,28 @@ auto arangodb::aql::createDistributeNodeFor(ExecutionPlan& plan, ExecutionNode* TRI_ASSERT(collection != nullptr); TRI_ASSERT(inputVariable != nullptr); - + // The DistributeNode needs specially prepared input, but we do not want to insert the // calculation for that just yet, because it would interfere with some optimizations, // in particular those that might completely remove the DistributeNode (which would) // also render the calculation pointless. So instead we insert this calculation in a // post-processing step when finalizing the plan in the Optimizer. - auto distNode = plan.createNode(&plan, plan.nextId(), - ScatterNode::ScatterType::SHARD, - collection, inputVariable, node->id()); + auto distNode = + plan.createNode(&plan, plan.nextId(), ScatterNode::ScatterType::SHARD, + collection, inputVariable, node->id()); + if (isGraphNode) { +#ifdef USE_ENTERPRISE + // Only relevant for Disjoint Smart Graphs that can only be part of the Enterprise version + auto graphNode = ExecutionNode::castTo(node); + auto vertices = graphNode->vertexColls(); + for (auto const& it : vertices) { + if (it->isSatellite()) { + distNode->addSatellite(it); + } + } +#endif + } TRI_ASSERT(distNode != nullptr); return distNode; } @@ -3949,9 +3972,8 @@ auto arangodb::aql::createGatherNodeFor(ExecutionPlan& plan, DistributeNode* nod // and we handle this case in here as well by resetting the root to the // inserted GATHER node. // -auto arangodb::aql::insertDistributeGatherSnippet(ExecutionPlan& plan, - ExecutionNode* at, SubqueryNode* snode) - -> DistributeNode* { +auto arangodb::aql::insertDistributeGatherSnippet(ExecutionPlan& plan, ExecutionNode* at, + SubqueryNode* snode) -> DistributeNode* { auto const parents = at->getParents(); auto const deps = at->getDependencies(); @@ -4011,7 +4033,7 @@ auto arangodb::aql::insertDistributeGatherSnippet(ExecutionPlan& plan, } auto extractSmartnessAndCollection(ExecutionNode* node) - -> std::tuple { + -> std::tuple { auto nodeType = node->getType(); auto collection = static_cast(nullptr); auto isSmart = bool{false}; @@ -4047,10 +4069,9 @@ auto extractSmartnessAndCollection(ExecutionNode* node) /// /// it will change plans in place - auto isGraphNode(ExecutionNode::NodeType nodeType) noexcept -> bool { return nodeType == ExecutionNode::TRAVERSAL || nodeType == ExecutionNode::SHORTEST_PATH || - nodeType == ExecutionNode::K_SHORTEST_PATHS; + nodeType == ExecutionNode::K_SHORTEST_PATHS; } auto isModificationNode(ExecutionNode::NodeType nodeType) noexcept -> bool { @@ -4230,40 +4251,36 @@ void arangodb::aql::collectInClusterRule(Optimizer* opt, std::unique_ptrgetType()) { + switch (p->getType()) { case ExecutionNode::REMOTE: hasFoundMultipleShards = true; break; case ExecutionNode::ENUMERATE_COLLECTION: - case ExecutionNode::INDEX: - { - auto col = getCollection(p); - if (col->numberOfShards() > 1) { - hasFoundMultipleShards = true; - } + case ExecutionNode::INDEX: { + auto col = getCollection(p); + if (col->numberOfShards() > 1) { + hasFoundMultipleShards = true; } - break; + } break; case ExecutionNode::TRAVERSAL: hasFoundMultipleShards = true; break; - case ExecutionNode::ENUMERATE_IRESEARCH_VIEW: - { - auto& viewNode = *ExecutionNode::castTo(p); - auto collections = viewNode.collections(); - auto const collCount = collections.size(); - TRI_ASSERT(collCount > 0); - if (collCount > 1) { - hasFoundMultipleShards = true; - } else if (1 == collCount) { - hasFoundMultipleShards = collections.front().get().numberOfShards() > 1; - } + case ExecutionNode::ENUMERATE_IRESEARCH_VIEW: { + auto& viewNode = *ExecutionNode::castTo(p); + auto collections = viewNode.collections(); + auto const collCount = collections.size(); + TRI_ASSERT(collCount > 0); + if (collCount > 1) { + hasFoundMultipleShards = true; + } else if (1 == collCount) { + hasFoundMultipleShards = + collections.front().get().numberOfShards() > 1; } - break; + } break; default: break; } - if (hasFoundMultipleShards) { break; } @@ -4305,19 +4322,21 @@ void arangodb::aql::collectInClusterRule(Optimizer* opt, std::unique_ptraggregationMethod() == CollectOptions::CollectMethod::COUNT) { TRI_ASSERT(collectNode->aggregateVariables().size() == 1); TRI_ASSERT(collectNode->hasOutVariable() == false); - // clone a COLLECT AGGREGATE var=LENGTH(_) operation from the coordinator to the - // DB server(s), and leave an aggregate COLLECT node on the - // coordinator for total aggregation + // clone a COLLECT AGGREGATE var=LENGTH(_) operation from the + // coordinator to the DB server(s), and leave an aggregate COLLECT + // node on the coordinator for total aggregation // add a new CollectNode on the DB server to do the actual counting auto outVariable = plan->getAst()->variables()->createTemporaryVariable(); std::vector aggregateVariables; - aggregateVariables.emplace_back(AggregateVarInfo{outVariable, collectNode->aggregateVariables()[0].inVar, "LENGTH"}); + aggregateVariables.emplace_back( + AggregateVarInfo{outVariable, + collectNode->aggregateVariables()[0].inVar, + "LENGTH"}); auto dbCollectNode = new CollectNode(plan.get(), plan->nextId(), collectNode->getOptions(), - collectNode->groupVariables(), - aggregateVariables, nullptr, - nullptr, std::vector(), + collectNode->groupVariables(), aggregateVariables, + nullptr, nullptr, std::vector(), collectNode->variableMap(), false); plan->registerNode(dbCollectNode); @@ -4347,7 +4366,7 @@ void arangodb::aql::collectInClusterRule(Optimizer* opt, std::unique_ptrgetAst()->variables()->createTemporaryVariable(); std::vector const groupVariables{ - GroupVarInfo{out, groupVars[0].inVar}}; + GroupVarInfo{out, groupVars[0].inVar}}; auto dbCollectNode = new CollectNode(plan.get(), plan->nextId(), collectNode->getOptions(), @@ -4409,8 +4428,9 @@ void arangodb::aql::collectInClusterRule(Optimizer* opt, std::unique_ptrnextId(), collectNode->getOptions(), - outVars, dbServerAggVars, nullptr, nullptr, std::vector(), + new CollectNode(plan.get(), plan->nextId(), + collectNode->getOptions(), outVars, dbServerAggVars, + nullptr, nullptr, std::vector(), collectNode->variableMap(), false); plan->registerNode(dbCollectNode); @@ -4425,7 +4445,8 @@ void arangodb::aql::collectInClusterRule(Optimizer* opt, std::unique_ptrgroupVariables()) { // replace input variables - copy.emplace_back(GroupVarInfo{/*outVar*/it.outVar, /*inVar*/outVars[i].outVar}); + copy.emplace_back(GroupVarInfo{/*outVar*/ it.outVar, + /*inVar*/ outVars[i].outVar}); ++i; } collectNode->groupVariables(copy); @@ -5977,16 +5998,21 @@ void arangodb::aql::optimizeTraversalsRule(Optimizer* opt, modified |= optimizeTraversalPathVariable(outVariable, traversal, pruneVars); // check if we can make use of the optimized neighbors enumerator - if (!ServerState::instance()->isCoordinator()) { - if (traversal->vertexOutVariable() != nullptr && traversal->edgeOutVariable() == nullptr && - traversal->pathOutVariable() == nullptr && options->isUseBreadthFirst() && - options->uniqueVertices == arangodb::traverser::TraverserOptions::GLOBAL && - !options->usesPrune() && !options->hasDepthLookupInfo()) { - // this is possible in case *only* vertices are produced (no edges, no path), - // the traversal is breadth-first, the vertex uniqueness level is set to "global", - // there is no pruning and there are no depth-specific filters - options->useNeighbors = true; - modified = true; + if (!options->isDisjoint()) { + // Use NeighborsEnumerator optimization only in case we have do not + // have a (Hybrid)Disjoint SmartGraph + if (!ServerState::instance()->isCoordinator()) { + if (traversal->vertexOutVariable() != nullptr && + traversal->edgeOutVariable() == nullptr && + traversal->pathOutVariable() == nullptr && options->isUseBreadthFirst() && + options->uniqueVertices == arangodb::traverser::TraverserOptions::GLOBAL && + !options->usesPrune() && !options->hasDepthLookupInfo()) { + // this is possible in case *only* vertices are produced (no edges, no path), + // the traversal is breadth-first, the vertex uniqueness level is set to "global", + // there is no pruning and there are no depth-specific filters + options->useNeighbors = true; + modified = true; + } } } } @@ -6189,8 +6215,7 @@ void arangodb::aql::inlineSubqueriesRule(Optimizer* opt, std::unique_ptrgetType() == EN::WINDOW && - subqueryNode->isInInnerLoop()) { + if (current->getType() == EN::WINDOW && subqueryNode->isInInnerLoop()) { // WINDOW captures all existing rows in the scope, moving WINDOW // ends up with different rows captured eligible = false; @@ -6202,9 +6227,9 @@ void arangodb::aql::inlineSubqueriesRule(Optimizer* opt, std::unique_ptr(current)->hasOutVariable()) { // COLLECT ... INTO captures all existing variables in the scope. - // if we move the subquery from one scope into another, we will end up with - // different variables captured, so we must not apply the optimization in - // this case. + // if we move the subquery from one scope into another, we will end up + // with different variables captured, so we must not apply the + // optimization in this case. eligible = false; break; } @@ -7452,8 +7477,7 @@ struct ParallelizableFinder final bool _isParallelizable; explicit ParallelizableFinder(bool parallelizeWrites) - : _parallelizeWrites(parallelizeWrites), - _isParallelizable(true) {} + : _parallelizeWrites(parallelizeWrites), _isParallelizable(true) {} ~ParallelizableFinder() = default; @@ -7462,8 +7486,7 @@ struct ParallelizableFinder final } bool before(ExecutionNode* node) override final { - if (node->getType() == ExecutionNode::SCATTER || - node->getType() == ExecutionNode::GATHER || + if (node->getType() == ExecutionNode::SCATTER || node->getType() == ExecutionNode::GATHER || node->getType() == ExecutionNode::DISTRIBUTE) { _isParallelizable = false; return true; // true to abort the whole walking process @@ -7483,10 +7506,9 @@ struct ParallelizableFinder final // can be parallelized, provided the rest of the plan // does not prohibit this if (node->isModificationNode() && - (!_parallelizeWrites || - (node->getType() != ExecutionNode::REMOVE && - node->getType() != ExecutionNode::REPLACE && - node->getType() != ExecutionNode::UPDATE))) { + (!_parallelizeWrites || (node->getType() != ExecutionNode::REMOVE && + node->getType() != ExecutionNode::REPLACE && + node->getType() != ExecutionNode::UPDATE))) { _isParallelizable = false; return true; // true to abort the whole walking process } @@ -7512,11 +7534,10 @@ bool isParallelizable(GatherNode* node, bool parallelizeWrites) { } return true; } -} +} // namespace /// @brief turn LENGTH(FOR doc IN ...) subqueries into an optimized count operation -void arangodb::aql::optimizeCountRule(Optimizer* opt, - std::unique_ptr plan, +void arangodb::aql::optimizeCountRule(Optimizer* opt, std::unique_ptr plan, OptimizerRule const& rule) { bool modified = false; @@ -7574,7 +7595,9 @@ void arangodb::aql::optimizeCountRule(Optimizer* opt, auto it = localCandidates.find(setter); if (it == localCandidates.end()) { - localCandidates.emplace(setter, std::make_pair(true, std::unordered_set({node}))); + localCandidates.emplace( + setter, + std::make_pair(true, std::unordered_set({node}))); } else { (*it).second.second.emplace(node); } @@ -7612,8 +7635,8 @@ void arangodb::aql::optimizeCountRule(Optimizer* opt, } bool valid = true; - // check if subquery result is used somewhere else before the current calculation - // we are looking at + // check if subquery result is used somewhere else before the current + // calculation we are looking at auto current = sn->getFirstParent(); while (current != nullptr && current != n) { vars.clear(); @@ -7667,8 +7690,8 @@ void arangodb::aql::optimizeCountRule(Optimizer* opt, } // from here we need to find the first FOR loop. - // if it is a full collection scan or an index scan, we note its out variable. - // if we find a nested loop, we abort searching + // if it is a full collection scan or an index scan, we note its out + // variable. if we find a nested loop, we abort searching bool valid = true; ExecutionNode* found = nullptr; Variable const* outVariable = nullptr; @@ -7691,13 +7714,14 @@ void arangodb::aql::optimizeCountRule(Optimizer* opt, } else { outVariable = dynamic_cast(current)->outVariable(); - if (type == EN::INDEX && ExecutionNode::castTo(current)->getIndexes().size() != 1) { + if (type == EN::INDEX && + ExecutionNode::castTo(current)->getIndexes().size() != 1) { // more than one index, so we would need to run uniqueness checks on the // results. this is currently unsupported, so don't apply the optimization valid = false; } else { - // a FOR loop without an early pruning filter. this is what we are - // looking for! + // a FOR loop without an early pruning filter. this is what we + // are looking for! found = current; } } @@ -7728,7 +7752,7 @@ void arangodb::aql::optimizeCountRule(Optimizer* opt, break; } - case EN::RETURN:{ + case EN::RETURN: { // we reached the end break; } @@ -7832,7 +7856,8 @@ void arangodb::aql::parallelizeGatherRule(Optimizer* opt, if (nodes.size() == 1 && !plan->contains(EN::DISTRIBUTE) && !plan->contains(EN::SCATTER)) { TRI_vocbase_t& vocbase = plan->getAst()->query().vocbase(); - bool parallelizeWrites = vocbase.server().getFeature().parallelizeGatherWrites(); + bool parallelizeWrites = + vocbase.server().getFeature().parallelizeGatherWrites(); GatherNode* gn = ExecutionNode::castTo(nodes[0]); if (!gn->isInSubquery() && isParallelizable(gn, parallelizeWrites)) { @@ -7864,7 +7889,7 @@ void arangodb::aql::asyncPrefetchRule(Optimizer* opt, std::unique_ptrisModificationNode()) { containsModificationNode = true; - return true; // found a modification node -> abort + return true; // found a modification node -> abort } return false; } @@ -7872,7 +7897,7 @@ void arangodb::aql::asyncPrefetchRule(Optimizer* opt, std::unique_ptrroot()->walk(checker); - + if (!checker.containsModificationNode) { // here we only set a flag that this plan should use async prefetching. // The actual prefetching is performed on node level and therefore also @@ -7950,18 +7975,19 @@ void arangodb::aql::enableReadOwnWritesForUpsertSubquery(ExecutionPlan& plan) { void arangodb::aql::activateCallstackSplit(ExecutionPlan& plan) { if (willUseV8(plan)) { - // V8 requires thread local context configuration, so we cannot + // V8 requires thread local context configuration, so we cannot // use our thread based split solution... return; } - + auto const& options = plan.getAst()->query().queryOptions(); struct CallstackSplitter : WalkerWorkerBase { - explicit CallstackSplitter(size_t maxNodes) : maxNodesPerCallstack(maxNodes) {} + explicit CallstackSplitter(size_t maxNodes) + : maxNodesPerCallstack(maxNodes) {} bool before(ExecutionNode* n) override { // This rule must be executed after subquery splicing, so we must not see any subqueries here! TRI_ASSERT(n->getType() != EN::SUBQUERY); - + if (n->getType() == EN::REMOTE) { // RemoteNodes provide a natural split in the callstack, so we can reset the counter here! count = 0; @@ -7974,7 +8000,7 @@ void arangodb::aql::activateCallstackSplit(ExecutionPlan& plan) { size_t maxNodesPerCallstack; size_t count = 0; }; - + CallstackSplitter walker(options.maxNodesPerCallstack); plan.root()->walk(walker); } @@ -8020,7 +8046,7 @@ void findSubqueriesSuitableForSplicing(ExecutionPlan const& plan, } bool before(ExecutionNode* node) override final { - TRI_ASSERT(node->getType() != EN::MUTEX); // should never appear here + TRI_ASSERT(node->getType() != EN::MUTEX); // should never appear here if (node->getType() == ExecutionNode::SUBQUERY) { _result.emplace_back(ExecutionNode::castTo(node)); @@ -8266,15 +8292,15 @@ void arangodb::aql::insertDistributeInputCalculation(ExecutionPlan& plan) { ::arangodb::containers::SmallVector::allocator_type::arena_type a; ::arangodb::containers::SmallVector nodes{a}; plan.findNodesOfType(nodes, ExecutionNode::DISTRIBUTE, true); - + for (auto const& n : nodes) { auto* distributeNode = ExecutionNode::castTo(n); auto* targetNode = plan.getNodesById().at(distributeNode->getTargetNodeId()); TRI_ASSERT(targetNode != nullptr); auto collection = static_cast(nullptr); - auto inputVariable = static_cast(nullptr); - auto alternativeVariable = static_cast(nullptr); + auto inputVariable = static_cast(nullptr); + auto alternativeVariable = static_cast(nullptr); auto createKeys = bool{false}; auto allowKeyConversionToObject = bool{false}; @@ -8282,13 +8308,13 @@ void arangodb::aql::insertDistributeInputCalculation(ExecutionPlan& plan) { auto fixupGraphInput = bool{false}; - std::function setInVariable; + std::function setInVariable; bool ignoreErrors = false; // TODO: this seems a bit verbose, but is at least local & simple - // the modification nodes are all collectionaccessing, the graph nodes are - // currently assumed to be disjoint, and hence smart, so all collections - // are sharded the same way! + // the modification nodes are all collectionaccessing, the graph nodes + // are currently assumed to be disjoint, and hence smart, so all + // collections are sharded the same way! switch (targetNode->getType()) { case ExecutionNode::INSERT: { auto* insertNode = ExecutionNode::castTo(targetNode); @@ -8387,8 +8413,8 @@ void arangodb::aql::insertDistributeInputCalculation(ExecutionPlan& plan) { default: { TRI_ASSERT(false); THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, - "Cannot distribute " + - targetNode->getTypeString() + "."); + "Cannot distribute " + + targetNode->getTypeString() + "."); } break; } TRI_ASSERT(inputVariable != nullptr); @@ -8402,20 +8428,19 @@ void arangodb::aql::insertDistributeInputCalculation(ExecutionPlan& plan) { CalculationNode* calcNode = nullptr; auto setter = plan.getVarSetBy(inputVariable->id); - if (setter == nullptr || // this can happen for $smartHandOver - setter->getType() == EN::ENUMERATE_COLLECTION || - setter->getType() == EN::INDEX) { + if (setter == nullptr || // this can happen for $smartHandOver + setter->getType() == EN::ENUMERATE_COLLECTION || setter->getType() == EN::INDEX) { // If our input variable is set by a collection/index enumeration, it is guaranteed to be an object // with a _key attribute, so we don't need to do anything. return; } - + // We insert an additional calculation node to create the input for our distribute node. Variable* variable = plan.getAst()->variables()->createTemporaryVariable(); - + // update the targetNode so that it uses the same input variable as our distribute node setInVariable(variable); - + auto* ast = plan.getAst(); auto args = ast->createNodeArray(); char const* function; @@ -8432,38 +8457,40 @@ void arangodb::aql::insertDistributeInputCalculation(ExecutionPlan& plan) { } auto flags = ast->createNodeObject(); flags->addMember(ast->createNodeObjectElement( - TRI_CHAR_LENGTH_PAIR("allowSpecifiedKeys"), - ast->createNodeValueBool(allowSpecifiedKeys))); - flags->addMember(ast->createNodeObjectElement( - TRI_CHAR_LENGTH_PAIR("ignoreErrors"), - ast->createNodeValueBool(ignoreErrors))); + TRI_CHAR_LENGTH_PAIR("allowSpecifiedKeys"), + ast->createNodeValueBool(allowSpecifiedKeys))); + flags->addMember( + ast->createNodeObjectElement(TRI_CHAR_LENGTH_PAIR("ignoreErrors"), + ast->createNodeValueBool(ignoreErrors))); auto const& collectionName = collection->name(); flags->addMember(ast->createNodeObjectElement( - TRI_CHAR_LENGTH_PAIR("collection"), - ast->createNodeValueString(collectionName.c_str(), collectionName.length()))); - //args->addMember(ast->createNodeValueString(collectionName.c_str(), collectionName.length())); - + TRI_CHAR_LENGTH_PAIR("collection"), + ast->createNodeValueString(collectionName.c_str(), collectionName.length()))); + // args->addMember(ast->createNodeValueString(collectionName.c_str(), collectionName.length())); + args->addMember(flags); } else { function = "MAKE_DISTRIBUTE_INPUT"; auto flags = ast->createNodeObject(); flags->addMember(ast->createNodeObjectElement( - TRI_CHAR_LENGTH_PAIR("allowKeyConversionToObject"), - ast->createNodeValueBool(allowKeyConversionToObject))); + TRI_CHAR_LENGTH_PAIR("allowKeyConversionToObject"), + ast->createNodeValueBool(allowKeyConversionToObject))); + flags->addMember( + ast->createNodeObjectElement(TRI_CHAR_LENGTH_PAIR("ignoreErrors"), + ast->createNodeValueBool(ignoreErrors))); + bool canUseCustomKey = collection->getCollection()->usesDefaultShardKeys() || + allowSpecifiedKeys; flags->addMember(ast->createNodeObjectElement( - TRI_CHAR_LENGTH_PAIR("ignoreErrors"), - ast->createNodeValueBool(ignoreErrors))); - bool canUseCustomKey = collection->getCollection()->usesDefaultShardKeys() || allowSpecifiedKeys; - flags->addMember(ast->createNodeObjectElement( - TRI_CHAR_LENGTH_PAIR("canUseCustomKey"), - ast->createNodeValueBool(canUseCustomKey))); - + TRI_CHAR_LENGTH_PAIR("canUseCustomKey"), ast->createNodeValueBool(canUseCustomKey))); + args->addMember(flags); } } - auto expr = std::make_unique(ast, ast->createNodeFunctionCall(function, args, true)); - calcNode = plan.createNode(&plan, plan.nextId(), std::move(expr), variable); + auto expr = + std::make_unique(ast, ast->createNodeFunctionCall(function, args, true)); + calcNode = plan.createNode(&plan, plan.nextId(), + std::move(expr), variable); distributeNode->setVariable(variable); plan.insertBefore(distributeNode, calcNode); plan.clearVarUsageComputed(); diff --git a/arangod/Aql/QueryExpressionContext.cpp b/arangod/Aql/QueryExpressionContext.cpp index 7f03b0ee4e7a..29b362eb6331 100644 --- a/arangod/Aql/QueryExpressionContext.cpp +++ b/arangod/Aql/QueryExpressionContext.cpp @@ -23,9 +23,9 @@ #include "QueryExpressionContext.h" +#include "Aql/AqlFunctionsInternalCache.h" #include "Aql/AqlValue.h" #include "Aql/QueryContext.h" -#include "Aql/AqlFunctionsInternalCache.h" #include "Transaction/Methods.h" using namespace arangodb; @@ -63,10 +63,8 @@ TRI_vocbase_t& QueryExpressionContext::vocbase() const { return _trx.vocbase(); } -transaction::Methods& QueryExpressionContext::trx() const { - return _trx; -} +transaction::Methods& QueryExpressionContext::trx() const { return _trx; } -bool QueryExpressionContext::killed() const { - return _query.killed(); -} +bool QueryExpressionContext::killed() const { return _query.killed(); } + +QueryContext& QueryExpressionContext::query() { return _query; } diff --git a/arangod/Aql/QueryExpressionContext.h b/arangod/Aql/QueryExpressionContext.h index a9b214c549bb..d4b06e7d8f14 100644 --- a/arangod/Aql/QueryExpressionContext.h +++ b/arangod/Aql/QueryExpressionContext.h @@ -34,11 +34,9 @@ class AqlFunctionsInternalCache; class QueryExpressionContext : public ExpressionContext { public: - explicit QueryExpressionContext(transaction::Methods& trx, - QueryContext& query, + explicit QueryExpressionContext(transaction::Methods& trx, QueryContext& query, AqlFunctionsInternalCache& cache) noexcept - : ExpressionContext(), - _trx(trx), _query(query), _aqlFunctionsInternalCache(cache) {} + : ExpressionContext(), _trx(trx), _query(query), _aqlFunctionsInternalCache(cache) {} void registerWarning(ErrorCode errorCode, char const* msg) override final; void registerError(ErrorCode errorCode, char const* msg) override final; @@ -47,7 +45,8 @@ class QueryExpressionContext : public ExpressionContext { bool caseInsensitive) override final; icu::RegexMatcher* buildLikeMatcher(char const* ptr, size_t length, bool caseInsensitive) override final; - icu::RegexMatcher* buildSplitMatcher(AqlValue splitExpression, velocypack::Options const* opts, + icu::RegexMatcher* buildSplitMatcher(AqlValue splitExpression, + velocypack::Options const* opts, bool& isEmptyExpression) override final; arangodb::ValidatorBase* buildValidator(arangodb::velocypack::Slice const&) override final; @@ -56,6 +55,7 @@ class QueryExpressionContext : public ExpressionContext { /// may be inaccessible on some platforms transaction::Methods& trx() const override final; bool killed() const override final; + QueryContext& query(); private: transaction::Methods& _trx; diff --git a/arangod/Aql/QueryOptions.cpp b/arangod/Aql/QueryOptions.cpp index 79146a4cf8c5..0be025e131a4 100644 --- a/arangod/Aql/QueryOptions.cpp +++ b/arangod/Aql/QueryOptions.cpp @@ -106,18 +106,17 @@ void QueryOptions::fromVelocyPack(VPackSlice slice) { } VPackSlice value; - + // use global memory limit value first if (QueryOptions::defaultMemoryLimit > 0) { memoryLimit = QueryOptions::defaultMemoryLimit; } - + // numeric options value = slice.get("memoryLimit"); if (value.isNumber()) { size_t v = value.getNumber(); - if (v > 0 && - (allowMemoryLimitOverride || v < memoryLimit)) { + if (v > 0 && (allowMemoryLimitOverride || v < memoryLimit)) { // only allow increasing the memory limit if the respective startup option // is set. and if it is set, only allow decreasing the memory limit memoryLimit = v; @@ -140,7 +139,7 @@ void QueryOptions::fromVelocyPack(VPackSlice slice) { if (value.isNumber()) { maxNodesPerCallstack = value.getNumber(); } - + value = slice.get("maxRuntime"); if (value.isNumber()) { maxRuntime = value.getNumber(); @@ -262,7 +261,8 @@ void QueryOptions::toVelocyPack(VPackBuilder& builder, bool disableOptimizerRule builder.add("satelliteSyncWait", VPackValue(satelliteSyncWait)); builder.add("ttl", VPackValue(ttl)); builder.add("profile", VPackValue(static_cast(profile))); - builder.add(StaticStrings::GraphTraversalProfileLevel, VPackValue(static_cast(traversalProfile))); + builder.add(StaticStrings::GraphTraversalProfileLevel, + VPackValue(static_cast(traversalProfile))); builder.add("allPlans", VPackValue(allPlans)); builder.add("verbosePlans", VPackValue(verbosePlans)); builder.add("stream", VPackValue(stream)); @@ -272,6 +272,7 @@ void QueryOptions::toVelocyPack(VPackBuilder& builder, bool disableOptimizerRule builder.add("fullCount", VPackValue(fullCount)); builder.add("count", VPackValue(count)); builder.add("verboseErrors", VPackValue(verboseErrors)); + if (!forceOneShardAttributeValue.empty()) { builder.add("forceOneShardAttributeValue", VPackValue(forceOneShardAttributeValue)); } diff --git a/arangod/Aql/QuerySnippet.cpp b/arangod/Aql/QuerySnippet.cpp index 12961b2249e1..cf2e40e7dc8c 100644 --- a/arangod/Aql/QuerySnippet.cpp +++ b/arangod/Aql/QuerySnippet.cpp @@ -598,13 +598,13 @@ auto QuerySnippet::prepareFirstBranch( // there are no local expansions auto* localGraphNode = ExecutionNode::castTo(exp.node); - localGraphNode->setCollectionToShard({}); // clear previous information + localGraphNode->resetCollectionToShard(); // clear previous information TRI_ASSERT(localGraphNode->isUsedAsSatellite() == exp.isSatellite); // Check whether `servers` is the leader for any of the shards of the // prototype collection. - // We want to instantiate this snippet here exactly iff this is the case. + // We want to instantiate this snippet here exactly if this is the case. auto needInstanceHere = std::invoke([&]() { auto const* const protoCol = localGraphNode->isUsedAsSatellite() @@ -631,7 +631,7 @@ auto QuerySnippet::prepareFirstBranch( } // This is either one shard or a single SatelliteGraph which is not used - // as SatelliteGraph or a Disjoint SmartGraph. + // as SatelliteGraph or a (Hybrid-)Disjoint SmartGraph. uint64_t numShards = 0; for (auto* aqlCollection : localGraphNode->collections()) { // It is of utmost importance that this is an ordered set of Shards. @@ -654,18 +654,11 @@ auto QuerySnippet::prepareFirstBranch( // to be used in toVelocyPack methods of classes derived // from GraphNode if (localGraphNode->isDisjoint()) { - if (found->second == server) { + if (aqlCollection->isSatellite()) { + myExp.emplace(shard); + TRI_ASSERT(shards.size() == 1); + } else if (found->second == server) { myExp.emplace(shard); - } else { - // the target server does not have anything to do with the particular - // collection (e.g. because the collection's shards are all on other - // servers), but it may be asked for this collection, because vertex - // collections are registered _globally_ with the TraversalNode and - // not on a per-target server basis. - // so in order to serve later lookups for this collection, we insert - // an empty string into the collection->shard map. - // on lookup, we will react to this. - localGraphNode->addCollectionToShard(aqlCollection->name(), ""); } } else { localGraphNode->addCollectionToShard(aqlCollection->name(), shard); @@ -678,6 +671,18 @@ auto QuerySnippet::prepareFirstBranch( if (myExp.size() > 1) { myExpFinal.insert({aqlCollection->name(), std::move(myExp)}); } + } else { + if (localGraphNode->isDisjoint()) { + // the target server does not have anything to do with the particular + // collection (e.g. because the collection's shards are all on other + // servers), but it may be asked for this collection, because vertex + // collections are registered _globally_ with the TraversalNode and + // not on a per-target server basis. + // so in order to serve later lookups for this collection, we insert + // an empty string into the collection->shard map. + // on lookup, we will react to this. + localGraphNode->addCollectionToShard(aqlCollection->name(), ""); + } } } @@ -686,8 +691,14 @@ auto QuerySnippet::prepareFirstBranch( if (localGraphNode->isDisjoint()) { if (!myExpFinal.empty()) { size_t numberOfShards = myExpFinal.begin()->second.size(); - // We need one expansion for every collection in the Graph - TRI_ASSERT(myExpFinal.size() == localGraphNode->collections().size()); + // We need one expansion for every non-satellite collection in the Graph + size_t amountOfNonSatellites = 0; + for (auto const& col : localGraphNode->collections()) { + if (!col->isSatellite()) { + amountOfNonSatellites++; + } + } + TRI_ASSERT(myExpFinal.size() == amountOfNonSatellites); for (auto const& expDefinition : myExpFinal) { TRI_ASSERT(expDefinition.second.size() == numberOfShards); } diff --git a/arangod/Aql/RestAqlHandler.cpp b/arangod/Aql/RestAqlHandler.cpp index 7426b631bb2a..7acc3fb617bb 100644 --- a/arangod/Aql/RestAqlHandler.cpp +++ b/arangod/Aql/RestAqlHandler.cpp @@ -109,12 +109,12 @@ void RestAqlHandler::setupClusterQuery() { generateError(rest::ResponseCode::METHOD_NOT_ALLOWED, TRI_ERROR_CLUSTER_ONLY_ON_DBSERVER); return; } - + TRI_IF_FAILURE("Query::setupTimeout") { // intentionally delay the request std::this_thread::sleep_for(std::chrono::milliseconds(RandomGenerator::interval(uint32_t(2000)))); } - + TRI_IF_FAILURE("Query::setupTimeoutFailSequence") { // simulate lock timeout during query setup uint32_t r = 100; @@ -125,7 +125,7 @@ void RestAqlHandler::setupClusterQuery() { std::this_thread::sleep_for(std::chrono::milliseconds(3000)); } } - + bool success = false; VPackSlice querySlice = this->parseVPackBody(success); if (!success) { @@ -287,10 +287,10 @@ void RestAqlHandler::setupClusterQuery() { double const ttl = options.ttl; // creates a StandaloneContext or a leased context - auto q = std::make_unique(clusterQueryId, + auto q = std::make_unique(clusterQueryId, createTransactionContext(access), std::move(options)); - + TRI_ASSERT(clusterQueryId == 0 || clusterQueryId == q->id()); VPackBufferUInt8 buffer; @@ -302,7 +302,7 @@ void RestAqlHandler::setupClusterQuery() { answerBuilder.add(StaticStrings::AqlRemoteResult, VPackValue(VPackValueType::Object)); if (clusterQueryId == 0) { // only return this attribute if we didn't get a query ID as input from - // the coordinator. this will be the case for setup requests from 3.7 + // the coordinator. this will be the case for setup requests from 3.7 // coordinators answerBuilder.add("queryId", VPackValue(q->id())); } @@ -348,7 +348,7 @@ void RestAqlHandler::setupClusterQuery() { } _queryRegistry->insertQuery(std::move(q), ttl, std::move(rGuard)); - + generateResult(rest::ResponseCode::OK, std::move(buffer)); } diff --git a/arangod/Aql/ShardLocking.cpp b/arangod/Aql/ShardLocking.cpp index a71ec6549c13..6bb965fe8f54 100644 --- a/arangod/Aql/ShardLocking.cpp +++ b/arangod/Aql/ShardLocking.cpp @@ -107,7 +107,7 @@ void ShardLocking::addNode(ExecutionNode const* baseNode, size_t snippetId, } auto const graphIsUsedAsSatellite = graphNode->isUsedAsSatellite(); auto const isUsedAsSatellite = [&](auto const& col) { - return graphIsUsedAsSatellite || (pushToSingleServer && col->isSatellite()); + return graphIsUsedAsSatellite || (col->isSatellite() && (pushToSingleServer || graphNode->isSmart())); }; // Add all Edge Collections to the Transactions, Traversals do never write for (auto const& col : graphNode->edgeColls()) { @@ -129,6 +129,7 @@ void ShardLocking::addNode(ExecutionNode const* baseNode, size_t snippetId, updateLocking(col, AccessMode::Type::READ, snippetId, restrictedShards, isUsedAsSatellite(col)); } + break; } case ExecutionNode::ENUMERATE_COLLECTION: diff --git a/arangod/Aql/TraversalNode.cpp b/arangod/Aql/TraversalNode.cpp index abc2629469fb..7de810efe67f 100644 --- a/arangod/Aql/TraversalNode.cpp +++ b/arangod/Aql/TraversalNode.cpp @@ -656,6 +656,9 @@ std::unique_ptr TraversalNode::createBlock( } #endif } else { + if (isDisjoint()) { + opts->setDisjoint(); + } traverser = std::make_unique(opts); } diff --git a/arangod/Aql/TraverserEngineShardLists.cpp b/arangod/Aql/TraverserEngineShardLists.cpp new file mode 100644 index 000000000000..be51fca39402 --- /dev/null +++ b/arangod/Aql/TraverserEngineShardLists.cpp @@ -0,0 +1,155 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2021-2021 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +//////////////////////////////////////////////////////////////////////////////// + +#include "TraverserEngineShardLists.h" + +#include "Aql/GraphNode.h" +#include "Aql/QueryContext.h" +#include "Graph/BaseOptions.h" + +using namespace arangodb; +using namespace arangodb::aql; + +TraverserEngineShardLists::TraverserEngineShardLists( + GraphNode const* node, ServerID const& server, + std::unordered_map const& shardMapping, QueryContext& query) + : _node(node), _hasShard(false) { + auto const& edges = _node->edgeColls(); + TRI_ASSERT(!edges.empty()); + auto const& restrictToShards = query.queryOptions().restrictToShards; +#ifdef USE_ENTERPRISE + transaction::Methods trx{query.newTrxContext()}; +#endif + // Extract the local shards for edge collections. + for (auto const& col : edges) { + TRI_ASSERT(col != nullptr); +#ifdef USE_ENTERPRISE + if (trx.isInaccessibleCollection(col->id())) { + _inaccessible.insert(col->name()); + _inaccessible.insert(std::to_string(col->id().id())); + } +#endif + _edgeCollections.emplace_back( + getAllLocalShards(shardMapping, server, col->shardIds(restrictToShards), + col->isSatellite() && node->isSmart())); + } + // Extract vertices + auto const& vertices = _node->vertexColls(); + // Guaranteed by addGraphNode, this will inject vertex collections + // in anonymous graph case + // It might in fact be empty, if we only have edge collections in a graph. + // Or if we guarantee to never read vertex data. + for (auto const& col : vertices) { + TRI_ASSERT(col != nullptr); +#ifdef USE_ENTERPRISE + if (trx.isInaccessibleCollection(col->id())) { + _inaccessible.insert(col->name()); + _inaccessible.insert(std::to_string(col->id().id())); + } +#endif + auto shards = getAllLocalShards(shardMapping, server, col->shardIds(restrictToShards), + col->isSatellite() && node->isSmart()); + _vertexCollections.try_emplace(col->name(), std::move(shards)); + } +} + +std::vector TraverserEngineShardLists::getAllLocalShards( + std::unordered_map const& shardMapping, ServerID const& server, + std::shared_ptr> shardIds, bool allowReadFromFollower) { + std::vector localShards; + for (auto const& shard : *shardIds) { + auto const& it = shardMapping.find(shard); + if (it == shardMapping.end()) { + THROW_ARANGO_EXCEPTION_MESSAGE( + TRI_ERROR_INTERNAL, + "no entry for shard '" + shard + "' in shard mapping table (" + std::to_string(shardMapping.size()) + " entries)"); + } + if (it->second == server) { + localShards.emplace_back(shard); + // Guaranteed that the traversal will be executed on this server. + _hasShard = true; + } else if (allowReadFromFollower) { + // The satellite does not force run of a traversal here. + localShards.emplace_back(shard); + } + } + return localShards; +} + +void TraverserEngineShardLists::serializeIntoBuilder( + VPackBuilder& infoBuilder) const { + TRI_ASSERT(_hasShard); + TRI_ASSERT(infoBuilder.isOpenArray()); + infoBuilder.openObject(); + { + // Options + infoBuilder.add(VPackValue("options")); + graph::BaseOptions* opts = _node->options(); + opts->buildEngineInfo(infoBuilder); + } + { + // Variables + std::vector vars; + _node->getConditionVariables(vars); + if (!vars.empty()) { + infoBuilder.add(VPackValue("variables")); + infoBuilder.openArray(); + for (auto v : vars) { + v->toVelocyPack(infoBuilder); + } + infoBuilder.close(); + } + } + + infoBuilder.add(VPackValue("shards")); + infoBuilder.openObject(); + infoBuilder.add(VPackValue("vertices")); + infoBuilder.openObject(); + for (auto const& col : _vertexCollections) { + infoBuilder.add(VPackValue(col.first)); + infoBuilder.openArray(); + for (auto const& v : col.second) { + infoBuilder.add(VPackValue(v)); + } + infoBuilder.close(); // this collection + } + infoBuilder.close(); // vertices + + infoBuilder.add(VPackValue("edges")); + infoBuilder.openArray(); + for (auto const& edgeShards : _edgeCollections) { + infoBuilder.openArray(); + for (auto const& e : edgeShards) { + infoBuilder.add(VPackValue(e)); + } + infoBuilder.close(); + } + infoBuilder.close(); // edges + infoBuilder.close(); // shards + + _node->enhanceEngineInfo(infoBuilder); + + infoBuilder.close(); // base + TRI_ASSERT(infoBuilder.isOpenArray()); +} + + diff --git a/arangod/Aql/TraverserEngineShardLists.h b/arangod/Aql/TraverserEngineShardLists.h new file mode 100644 index 000000000000..1c8271be3ebe --- /dev/null +++ b/arangod/Aql/TraverserEngineShardLists.h @@ -0,0 +1,87 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2021 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +//////////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include "Cluster/ClusterTypes.h" + +#include +#include +#include + +namespace arangodb { +namespace aql { +class GraphNode; +class QueryContext; + +// @brief Struct to create the +// information required to build traverser engines +// on DB servers. +class TraverserEngineShardLists { + public: + TraverserEngineShardLists(GraphNode const*, ServerID const& server, + std::unordered_map const& shardMapping, + QueryContext& query); + + ~TraverserEngineShardLists() = default; + + void serializeIntoBuilder(arangodb::velocypack::Builder& infoBuilder) const; + + bool hasShard() const { return _hasShard; } + + /// inaccessible edge and verte collection names +#ifdef USE_ENTERPRISE + std::set inaccessibleCollNames() const { return _inaccessible; } +#endif + + private: + std::vector getAllLocalShards(std::unordered_map const& shardMapping, + ServerID const& server, + std::shared_ptr> shardIds, + bool allowReadFromFollower); + + private: + // The graph node we need to serialize + GraphNode const* _node; + + // Flag if we found any shard for the given server. + // If not serializeToBuilder will be a noop + bool _hasShard; + + // Mapping for edge collections to shardIds. + // We have to retain the ordering of edge collections, all + // vectors of these in one run need to have identical size. + // This is because the conditions to query those edges have the + // same ordering. + std::vector> _edgeCollections; + + // Mapping for vertexCollections to shardIds. + std::unordered_map> _vertexCollections; + +#ifdef USE_ENTERPRISE + std::set _inaccessible; +#endif +}; + +} // namespace aql +} // namespace arangodb diff --git a/arangod/CMakeLists.txt b/arangod/CMakeLists.txt index 6c2b85ed96fe..be34d6478744 100644 --- a/arangod/CMakeLists.txt +++ b/arangod/CMakeLists.txt @@ -198,12 +198,14 @@ set(LIB_ARANGO_GRAPH_SOURCES Graph/PathManagement/PathStore.cpp Graph/PathManagement/PathStoreTracer.cpp Graph/PathManagement/PathValidator.cpp + Graph/PathManagement/PathValidatorOptions.cpp Graph/Options/OneSidedEnumeratorOptions.cpp Graph/Options/TwoSidedEnumeratorOptions.cpp Graph/Providers/ClusterProvider.cpp Graph/Providers/SingleServerProvider.cpp Graph/Providers/BaseProviderOptions.cpp Graph/Providers/ProviderTracer.cpp + Graph/Steps/SingleServerProviderStep.cpp Graph/Types/UniquenessLevel.cpp Graph/Types/ValidationResult.cpp InternalRestHandler/InternalRestTraverserHandler.cpp @@ -419,6 +421,7 @@ set(LIB_ARANGO_AQL_SOURCES Aql/SubqueryStartExecutor.cpp Aql/Timing.cpp Aql/TraversalConditionFinder.cpp + Aql/TraverserEngineShardLists.cpp Aql/TraversalExecutor.cpp Aql/TraversalNode.cpp Aql/UnsortedGatherExecutor.cpp diff --git a/arangod/Cluster/ClusterEdgeCursor.cpp b/arangod/Cluster/ClusterEdgeCursor.cpp index 4a34d7beaab7..a255cd1dfda6 100644 --- a/arangod/Cluster/ClusterEdgeCursor.cpp +++ b/arangod/Cluster/ClusterEdgeCursor.cpp @@ -38,7 +38,7 @@ using namespace arangodb; using namespace arangodb::graph; using namespace arangodb::traverser; -ClusterEdgeCursor::ClusterEdgeCursor(graph::BaseOptions const* opts) +ClusterEdgeCursor::ClusterEdgeCursor(graph::BaseOptions const* opts) : _position(0), _opts(opts), _cache(static_cast(opts->cache())), @@ -46,8 +46,8 @@ ClusterEdgeCursor::ClusterEdgeCursor(graph::BaseOptions const* opts) TRI_ASSERT(_cache != nullptr); if (_cache == nullptr) { - THROW_ARANGO_EXCEPTION_MESSAGE( - TRI_ERROR_INTERNAL, "no cache present for cluster edge cursor"); + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, + "no cache present for cluster edge cursor"); } } @@ -66,7 +66,7 @@ void ClusterEdgeCursor::readAll(EdgeCursor::Callback const& callback) { callback(EdgeDocumentToken(edge), edge, _position); } } - + ClusterTraverserEdgeCursor::ClusterTraverserEdgeCursor(traverser::TraverserOptions const* opts) : ClusterEdgeCursor(opts) {} @@ -75,7 +75,8 @@ traverser::TraverserOptions const* ClusterTraverserEdgeCursor::traverserOptions( return dynamic_cast(_opts); } -void ClusterTraverserEdgeCursor::rearm(arangodb::velocypack::StringRef vertexId, uint64_t depth) { +void ClusterTraverserEdgeCursor::rearm(arangodb::velocypack::StringRef vertexId, + uint64_t depth) { _edgeList.clear(); _position = 0; @@ -83,18 +84,20 @@ void ClusterTraverserEdgeCursor::rearm(arangodb::velocypack::StringRef vertexId, TRI_ASSERT(trx != nullptr); TRI_ASSERT(_cache != nullptr); - Result res = fetchEdgesFromEngines(*trx, *_cache, traverserOptions()->getExpressionCtx(), vertexId, depth, _edgeList); + Result res = fetchEdgesFromEngines(*trx, *_cache, traverserOptions()->getExpressionCtx(), + vertexId, depth, _edgeList); if (res.fail()) { THROW_ARANGO_EXCEPTION(res); } _httpRequests += _cache->engines()->size(); } -ClusterShortestPathEdgeCursor::ClusterShortestPathEdgeCursor(graph::BaseOptions const* opts, bool backward) - : ClusterEdgeCursor(opts), - _backward(backward) {} +ClusterShortestPathEdgeCursor::ClusterShortestPathEdgeCursor(graph::BaseOptions const* opts, + bool backward) + : ClusterEdgeCursor(opts), _backward(backward) {} -void ClusterShortestPathEdgeCursor::rearm(arangodb::velocypack::StringRef vertexId, uint64_t /*depth*/) { +void ClusterShortestPathEdgeCursor::rearm(arangodb::velocypack::StringRef vertexId, + uint64_t /*depth*/) { _edgeList.clear(); _position = 0; @@ -102,7 +105,8 @@ void ClusterShortestPathEdgeCursor::rearm(arangodb::velocypack::StringRef vertex transaction::BuilderLeaser b(trx); b->add(VPackValuePair(vertexId.data(), vertexId.length(), VPackValueType::String)); - Result res = fetchEdgesFromEngines(*trx, *_cache, b->slice(), _backward, _edgeList, _cache->insertedDocuments()); + Result res = fetchEdgesFromEngines(*trx, *_cache, b->slice(), _backward, + _edgeList, _cache->insertedDocuments()); if (res.fail()) { THROW_ARANGO_EXCEPTION(res); } diff --git a/arangod/Cluster/ClusterMethods.cpp b/arangod/Cluster/ClusterMethods.cpp index 41be023d93e7..83e5bb9f9326 100644 --- a/arangod/Cluster/ClusterMethods.cpp +++ b/arangod/Cluster/ClusterMethods.cpp @@ -526,7 +526,7 @@ ::ErrorCode distributeBabyOnShards(CreateOperationCtx& opCtx, LogicalCollection& VPackSlice keySlice = value.get(StaticStrings::KeyString); if (keySlice.isNone()) { // The user did not specify a key, let's create one: - _key = collinfo.keyGenerator()->generate(); + _key = collinfo.createKey(value); } else { userSpecifiedKey = true; if (keySlice.isString()) { diff --git a/arangod/Cluster/TraverserEngine.cpp b/arangod/Cluster/TraverserEngine.cpp index 30fcaebd70a8..3a7a4347b7be 100644 --- a/arangod/Cluster/TraverserEngine.cpp +++ b/arangod/Cluster/TraverserEngine.cpp @@ -117,7 +117,7 @@ BaseEngine::BaseEngine(TRI_vocbase_t& vocbase, TRI_ASSERT(shardList.isArray()); for (VPackSlice const shard : VPackArrayIterator(shardList)) { TRI_ASSERT(shard.isString()); - _query.collections().add(shard.copyString(), AccessMode::Type::READ, aql::Collection::Hint::Collection); + _query.collections().add(shard.copyString(), AccessMode::Type::READ, aql::Collection::Hint::Shard); } } @@ -302,7 +302,9 @@ void BaseTraverserEngine::injectVariables(VPackSlice variableSlice) { if ((!pair.isArray()) || pair.length() != 2) { // Invalid communication. Skip TRI_ASSERT(false); - continue; + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, + "Internal Traverser communication " + "broken. Failed to inject variables."); } auto varId = arangodb::basics::VelocyPackHelper::getNumericValue(pair.at(0), @@ -414,11 +416,3 @@ TraverserEngine::~TraverserEngine() = default; void TraverserEngine::smartSearch(VPackSlice, VPackBuilder&) { THROW_ARANGO_EXCEPTION(TRI_ERROR_ONLY_ENTERPRISE); } - -void TraverserEngine::smartSearchBFS(VPackSlice, VPackBuilder&) { - THROW_ARANGO_EXCEPTION(TRI_ERROR_ONLY_ENTERPRISE); -} - -void TraverserEngine::smartSearchWeighted(VPackSlice, VPackBuilder&) { - THROW_ARANGO_EXCEPTION(TRI_ERROR_ONLY_ENTERPRISE); -} diff --git a/arangod/Cluster/TraverserEngine.h b/arangod/Cluster/TraverserEngine.h index f5b04b9fe752..f82e689f9c32 100644 --- a/arangod/Cluster/TraverserEngine.h +++ b/arangod/Cluster/TraverserEngine.h @@ -115,13 +115,8 @@ class BaseTraverserEngine : public BaseEngine { graph::EdgeCursor* getCursor(arangodb::velocypack::StringRef nextVertex, uint64_t currentDepth); - virtual void smartSearch(arangodb::velocypack::Slice, arangodb::velocypack::Builder&) = 0; - - virtual void smartSearchBFS(arangodb::velocypack::Slice, - arangodb::velocypack::Builder&) = 0; - - virtual void smartSearchWeighted(arangodb::velocypack::Slice, - arangodb::velocypack::Builder&) = 0; + virtual void smartSearch(arangodb::velocypack::Slice, + arangodb::velocypack::Builder&) = 0; EngineType getType() const override { return TRAVERSER; } @@ -179,10 +174,6 @@ class TraverserEngine : public BaseTraverserEngine { ~TraverserEngine(); void smartSearch(arangodb::velocypack::Slice, arangodb::velocypack::Builder&) override; - - void smartSearchBFS(arangodb::velocypack::Slice, arangodb::velocypack::Builder&) override; - - void smartSearchWeighted(arangodb::velocypack::Slice, arangodb::velocypack::Builder&) override; }; } // namespace traverser diff --git a/arangod/ClusterEngine/ClusterIndexFactory.cpp b/arangod/ClusterEngine/ClusterIndexFactory.cpp index 95fb2d122194..f8c88a636a5c 100644 --- a/arangod/ClusterEngine/ClusterIndexFactory.cpp +++ b/arangod/ClusterEngine/ClusterIndexFactory.cpp @@ -113,7 +113,8 @@ struct EdgeIndexFactory : public DefaultIndexFactory { bool isClusterConstructor) const override { if (!isClusterConstructor) { // this index type cannot be created directly - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "cannot create edge index"); + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, + "cannot create edge index"); } auto& clusterEngine = _server.getFeature().engine(); @@ -136,7 +137,8 @@ struct PrimaryIndexFactory : public DefaultIndexFactory { bool isClusterConstructor) const override { if (!isClusterConstructor) { // this index type cannot be created directly - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "cannot create primary index"); + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, + "cannot create primary index"); } auto& clusterEngine = _server.getFeature().engine(); @@ -152,8 +154,8 @@ struct PrimaryIndexFactory : public DefaultIndexFactory { namespace arangodb { -ClusterIndexFactory::ClusterIndexFactory(application_features::ApplicationServer& server) - : IndexFactory(server) { +void ClusterIndexFactory::linkIndexFactories(application_features::ApplicationServer& server, + IndexFactory& factory) { static const EdgeIndexFactory edgeIndexFactory(server, "edge"); static const DefaultIndexFactory fulltextIndexFactory(server, "fulltext"); static const DefaultIndexFactory geoIndexFactory(server, "geo"); @@ -165,16 +167,21 @@ ClusterIndexFactory::ClusterIndexFactory(application_features::ApplicationServer static const DefaultIndexFactory skiplistIndexFactory(server, "skiplist"); static const DefaultIndexFactory ttlIndexFactory(server, "ttl"); - emplace(edgeIndexFactory._type, edgeIndexFactory); - emplace(fulltextIndexFactory._type, fulltextIndexFactory); - emplace(geoIndexFactory._type, geoIndexFactory); - emplace(geo1IndexFactory._type, geo1IndexFactory); - emplace(geo2IndexFactory._type, geo2IndexFactory); - emplace(hashIndexFactory._type, hashIndexFactory); - emplace(persistentIndexFactory._type, persistentIndexFactory); - emplace(primaryIndexFactory._type, primaryIndexFactory); - emplace(skiplistIndexFactory._type, skiplistIndexFactory); - emplace(ttlIndexFactory._type, ttlIndexFactory); + factory.emplace(edgeIndexFactory._type, edgeIndexFactory); + factory.emplace(fulltextIndexFactory._type, fulltextIndexFactory); + factory.emplace(geoIndexFactory._type, geoIndexFactory); + factory.emplace(geo1IndexFactory._type, geo1IndexFactory); + factory.emplace(geo2IndexFactory._type, geo2IndexFactory); + factory.emplace(hashIndexFactory._type, hashIndexFactory); + factory.emplace(persistentIndexFactory._type, persistentIndexFactory); + factory.emplace(primaryIndexFactory._type, primaryIndexFactory); + factory.emplace(skiplistIndexFactory._type, skiplistIndexFactory); + factory.emplace(ttlIndexFactory._type, ttlIndexFactory); +} + +ClusterIndexFactory::ClusterIndexFactory(application_features::ApplicationServer& server) + : IndexFactory(server) { + linkIndexFactories(server, *this); } /// @brief index name aliases (e.g. "persistent" => "hash", "skiplist" => @@ -189,12 +196,13 @@ std::unordered_map ClusterIndexFactory::indexAliases() return ae->indexFactory().indexAliases(); } -Result ClusterIndexFactory::enhanceIndexDefinition( - velocypack::Slice const definition, - velocypack::Builder& normalized, - bool isCreation, - TRI_vocbase_t const& vocbase) const { - auto& ce = _server.getFeature().engine(); +Result ClusterIndexFactory::enhanceIndexDefinition( // normalize definition + velocypack::Slice const definition, // source definition + velocypack::Builder& normalized, // normalized definition (out-param) + bool isCreation, // definition for index creation + TRI_vocbase_t const& vocbase // index vocbase +) const { + auto& ce = _server.getFeature().engine(); auto* ae = ce.actualEngine(); diff --git a/arangod/ClusterEngine/ClusterIndexFactory.h b/arangod/ClusterEngine/ClusterIndexFactory.h index 1804054e57b5..610713bf01b3 100644 --- a/arangod/ClusterEngine/ClusterIndexFactory.h +++ b/arangod/ClusterEngine/ClusterIndexFactory.h @@ -29,17 +29,20 @@ namespace arangodb { class ClusterIndexFactory final : public IndexFactory { public: + static void linkIndexFactories(application_features::ApplicationServer& server, + IndexFactory& factory); explicit ClusterIndexFactory(application_features::ApplicationServer&); ~ClusterIndexFactory() = default; - Result enhanceIndexDefinition( - velocypack::Slice const definition, - velocypack::Builder& normalized, - bool isCreation, - TRI_vocbase_t const& vocbase) const override; + Result enhanceIndexDefinition( // normalize definition + velocypack::Slice const definition, // source definition + velocypack::Builder& normalized, // normalized definition (out-param) + bool isCreation, // definition for index creation + TRI_vocbase_t const& vocbase // index vocbase + ) const override; - /// @brief index name aliases (e.g. "persistent" => "hash", "skiplist" => "hash") - /// used to display storage engine capabilities + /// @brief index name aliases (e.g. "persistent" => "hash", "skiplist" => + /// "hash") used to display storage engine capabilities std::unordered_map indexAliases() const override; void fillSystemIndexes(LogicalCollection& col, diff --git a/arangod/Graph/BaseOptions.cpp b/arangod/Graph/BaseOptions.cpp index e8d0a9330e99..db15384856ac 100644 --- a/arangod/Graph/BaseOptions.cpp +++ b/arangod/Graph/BaseOptions.cpp @@ -365,8 +365,13 @@ void BaseOptions::serializeVariables(VPackBuilder& builder) const { _expressionCtx.serializeAllVariables(_query.vpackOptions(), builder); } -void BaseOptions::setCollectionToShard(std::map const& in) { - _collectionToShard = std::move(in); +void BaseOptions::setCollectionToShard( + std::unordered_map const& in) { + _collectionToShard.clear(); + _collectionToShard.reserve(in.size()); + for (auto const& [key, value] : in) { + _collectionToShard.emplace(key, std::vector{value}); + } } arangodb::transaction::Methods* BaseOptions::trx() const { return &_trx; } @@ -462,6 +467,10 @@ void BaseOptions::injectTestCache(std::unique_ptr&& testCache) { _cache = std::move(testCache); } +arangodb::aql::FixedVarExpressionContext& BaseOptions::getExpressionCtx() { + return _expressionCtx; +} + arangodb::aql::FixedVarExpressionContext const& BaseOptions::getExpressionCtx() const { return _expressionCtx; } diff --git a/arangod/Graph/BaseOptions.h b/arangod/Graph/BaseOptions.h index 1ec81522948a..dad32f9f0585 100644 --- a/arangod/Graph/BaseOptions.h +++ b/arangod/Graph/BaseOptions.h @@ -117,7 +117,7 @@ struct BaseOptions { void serializeVariables(arangodb::velocypack::Builder&) const; - void setCollectionToShard(std::map const&); + void setCollectionToShard(std::unordered_map const&); bool produceVertices() const { return _produceVertices; } @@ -157,7 +157,7 @@ struct BaseOptions { void activateCache(bool enableDocumentCache, std::unordered_map const* engines); - std::map const& collectionToShard() const { + std::unordered_map> const& collectionToShard() const { return _collectionToShard; } @@ -178,6 +178,8 @@ struct BaseOptions { bool refactor() const { return _refactor; } aql::Variable const* tmpVar(); // TODO check public + arangodb::aql::FixedVarExpressionContext& getExpressionCtx(); + arangodb::aql::FixedVarExpressionContext const& getExpressionCtx() const; protected: @@ -215,7 +217,7 @@ struct BaseOptions { std::unique_ptr _cache; // @brief - translations for one-shard-databases - std::map _collectionToShard; + std::unordered_map> _collectionToShard; /// @brief a value of 1 (which is the default) means "no parallelism" size_t _parallelism; diff --git a/arangod/Graph/BreadthFirstEnumerator.cpp b/arangod/Graph/BreadthFirstEnumerator.cpp index 007aa561c5bf..e1cb0186a82a 100644 --- a/arangod/Graph/BreadthFirstEnumerator.cpp +++ b/arangod/Graph/BreadthFirstEnumerator.cpp @@ -57,7 +57,7 @@ BreadthFirstEnumerator::~BreadthFirstEnumerator() { void BreadthFirstEnumerator::setStartVertex(arangodb::velocypack::StringRef startVertex) { PathEnumerator::setStartVertex(startVertex); - + _schreier.clear(); _schreierIndex = 0; _lastReturned = 0; @@ -161,6 +161,10 @@ bool BreadthFirstEnumerator::next() { } } + if (!validDisjointPath(nextIdx, vId)) { + return; + } + growStorage(); TRI_ASSERT(_schreier.capacity() > _schreier.size()); _schreier.emplace_back(nextIdx, std::move(eid), vId); @@ -214,7 +218,8 @@ arangodb::aql::AqlValue BreadthFirstEnumerator::edgeToAqlValue(size_t index) { return _opts->cache()->fetchEdgeAqlResult(_schreier[index].edge); } -VPackSlice BreadthFirstEnumerator::pathToIndexToSlice(VPackBuilder& result, size_t index, bool fromPrune) { +VPackSlice BreadthFirstEnumerator::pathToIndexToSlice(VPackBuilder& result, + size_t index, bool fromPrune) { _tempPathHelper.clear(); while (index != 0) { // Walk backwards through the path and push everything found on the local @@ -352,7 +357,8 @@ void BreadthFirstEnumerator::growStorage() { TRI_ASSERT(capacity > _schreier.size()); if (capacity > _schreier.capacity()) { - arangodb::ResourceUsageScope guard(_opts->resourceMonitor(), (capacity - _schreier.capacity()) * pathStepSize()); + arangodb::ResourceUsageScope guard(_opts->resourceMonitor(), + (capacity - _schreier.capacity()) * pathStepSize()); _schreier.reserve(capacity); @@ -364,3 +370,10 @@ void BreadthFirstEnumerator::growStorage() { constexpr size_t BreadthFirstEnumerator::pathStepSize() const noexcept { return sizeof(void*) + sizeof(PathStep) + 2 * sizeof(NextStep); } + +#ifndef USE_ENTERPRISE +bool BreadthFirstEnumerator::validDisjointPath(size_t /*index*/, + arangodb::velocypack::StringRef const& /*vertex*/) const { + return true; +} +#endif diff --git a/arangod/Graph/BreadthFirstEnumerator.h b/arangod/Graph/BreadthFirstEnumerator.h index ce6bb1a7f31a..3e02cfcff4ae 100644 --- a/arangod/Graph/BreadthFirstEnumerator.h +++ b/arangod/Graph/BreadthFirstEnumerator.h @@ -71,9 +71,9 @@ class BreadthFirstEnumerator final : public arangodb::traverser::PathEnumerator explicit NextStep(size_t sourceIdx) : sourceIdx(sourceIdx) {} }; - /// @brief schreier vector to store the visited vertices. - /// note: for memory usage tracking, it is require to call growStorage() before - /// inserting into the schreier vector. + /// @brief schreier vector to store the visited vertices. + /// note: for memory usage tracking, it is require to call growStorage() + /// before inserting into the schreier vector. std::vector _schreier; /// @brief Next free index in schreier vector. @@ -104,12 +104,12 @@ class BreadthFirstEnumerator final : public arangodb::traverser::PathEnumerator arangodb::traverser::TraverserOptions* opts); ~BreadthFirstEnumerator(); - + void setStartVertex(arangodb::velocypack::StringRef startVertex) override; /// @brief Get the next Path element from the traversal. bool next() override; - + aql::AqlValue lastVertexToAqlValue() override; aql::AqlValue lastEdgeToAqlValue() override; @@ -152,14 +152,17 @@ class BreadthFirstEnumerator final : public arangodb::traverser::PathEnumerator aql::AqlValue pathToIndexToAqlValue(arangodb::velocypack::Builder& result, size_t index); - velocypack::Slice pathToIndexToSlice(arangodb::velocypack::Builder& result, size_t index, bool fromPrune); + velocypack::Slice pathToIndexToSlice(arangodb::velocypack::Builder& result, + size_t index, bool fromPrune); bool shouldPrune(); void growStorage(); - + constexpr size_t pathStepSize() const noexcept; + + bool validDisjointPath(size_t nextVertexIndex, + arangodb::velocypack::StringRef const& vertex) const; }; } // namespace graph } // namespace arangodb - diff --git a/arangod/Graph/Cache/RefactoredTraverserCache.cpp b/arangod/Graph/Cache/RefactoredTraverserCache.cpp index 04525f30fdac..e792ea322401 100644 --- a/arangod/Graph/Cache/RefactoredTraverserCache.cpp +++ b/arangod/Graph/Cache/RefactoredTraverserCache.cpp @@ -53,8 +53,7 @@ using namespace arangodb::graph; namespace { bool isWithClauseMissing(arangodb::basics::Exception const& ex) { - if (ServerState::instance()->isDBServer() && - ex.code() == TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND) { + if (ServerState::instance()->isDBServer() && ex.code() == TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND) { // on a DB server, we could have got here only in the OneShard case. // in this case turn the rather misleading "collection or view not found" // error into a nicer "collection not known to traversal, please add WITH" @@ -68,19 +67,20 @@ bool isWithClauseMissing(arangodb::basics::Exception const& ex) { return false; } -} // namespace +} // namespace RefactoredTraverserCache::RefactoredTraverserCache( arangodb::transaction::Methods* trx, aql::QueryContext* query, arangodb::ResourceMonitor& resourceMonitor, arangodb::aql::TraversalStats& stats, - std::map const& collectionToShardMap) + std::unordered_map> const& collectionToShardMap) : _query(query), _trx(trx), _stringHeap(resourceMonitor, 4096), /* arbitrary block-size may be adjusted for performance */ _collectionToShardMap(collectionToShardMap), _resourceMonitor(resourceMonitor), - _allowImplicitCollections(ServerState::instance()->isSingleServer() && - !_query->vocbase().server().getFeature().requireWith()) { + _allowImplicitCollections( + ServerState::instance()->isSingleServer() && + !_query->vocbase().server().getFeature().requireWith()) { TRI_ASSERT(!ServerState::instance()->isCoordinator()); } @@ -95,7 +95,7 @@ void RefactoredTraverserCache::clear() { template bool RefactoredTraverserCache::appendEdge(EdgeDocumentToken const& idToken, - ResultType& result) { + bool onlyId, ResultType& result) { auto col = _trx->vocbase().lookupCollection(idToken.cid()); if (ADB_UNLIKELY(col == nullptr)) { @@ -106,17 +106,24 @@ bool RefactoredTraverserCache::appendEdge(EdgeDocumentToken const& idToken, return false; } - auto res = col->getPhysical()->read( - _trx, idToken.localDocumentId(), [&](LocalDocumentId const&, VPackSlice edge) -> bool { - // NOTE: Do not count this as Primary Index Scan, we counted it in the - // edge Index before copying... - if constexpr (std::is_same_v) { - result = aql::AqlValue(edge); - } else if constexpr (std::is_same_v) { - result.add(edge); - } - return true; - }, ReadOwnWrites::no).ok(); + auto res = col->getPhysical() + ->read( + _trx, idToken.localDocumentId(), + [&](LocalDocumentId const&, VPackSlice edge) -> bool { + if (onlyId) { + edge = edge.get(StaticStrings::IdString); + } + // NOTE: Do not count this as Primary Index Scan, we + // counted it in the edge Index before copying... + if constexpr (std::is_same_v) { + result = aql::AqlValue(edge); + } else if constexpr (std::is_same_v) { + result.add(edge); + } + return true; + }, + ReadOwnWrites::no) + .ok(); if (ADB_UNLIKELY(!res)) { // We already had this token, inconsistent state. Return NULL in Production LOG_TOPIC("daac5", ERR, arangodb::Logger::GRAPHS) @@ -139,20 +146,7 @@ ResultT> RefactoredTraverserCache::extractCollect } std::string colName = idHashed.substr(0, pos).toString(); - if (_collectionToShardMap.empty()) { - TRI_ASSERT(!ServerState::instance()->isDBServer()); - return std::make_pair(colName, pos); - } - auto it = _collectionToShardMap.find(colName); - if (it == _collectionToShardMap.end()) { - // Connected to a vertex where we do not know the Shard to. - return Result{TRI_ERROR_QUERY_COLLECTION_LOCK_FAILED, - "collection not known to traversal: '" + colName + - "'. please add 'WITH " + colName + - "' as the first line in your AQL"}; - } - // We have translated to a Shard - return std::make_pair(it->second, pos); + return std::make_pair(colName, pos); } template @@ -163,43 +157,68 @@ bool RefactoredTraverserCache::appendVertex(aql::TraversalStats& stats, if (collectionNameResult.fail()) { THROW_ARANGO_EXCEPTION(collectionNameResult.result()); } - - std::string const& collectionName = collectionNameResult.get().first; - try { - transaction::AllowImplicitCollectionsSwitcher disallower(_trx->state()->options(), _allowImplicitCollections); - - Result res = _trx->documentFastPathLocal( - collectionName, - id.substr(collectionNameResult.get().second + 1).stringRef(), - [&](LocalDocumentId const&, VPackSlice doc) -> bool { - stats.addScannedIndex(1); - // copying... - if constexpr (std::is_same_v) { - result = aql::AqlValue(doc); - } else if constexpr (std::is_same_v) { - result.add(doc); - } - return true; - }); - if (res.ok()) { - return true; + auto findDocumentInShard = [&](std::string const& collectionName) -> bool { + try { + transaction::AllowImplicitCollectionsSwitcher disallower(_trx->state()->options(), + _allowImplicitCollections); + + Result res = _trx->documentFastPathLocal( + collectionName, id.substr(collectionNameResult.get().second + 1).stringRef(), + [&](LocalDocumentId const&, VPackSlice doc) -> bool { + stats.addScannedIndex(1); + // copying... + if constexpr (std::is_same_v) { + result = aql::AqlValue(doc); + } else if constexpr (std::is_same_v) { + result.add(doc); + } + return true; + }); + if (res.ok()) { + return true; + } + + if (!res.is(TRI_ERROR_ARANGO_DOCUMENT_NOT_FOUND)) { + // ok we are in a rather bad state. Better throw and abort. + THROW_ARANGO_EXCEPTION(res); + } + } catch (basics::Exception const& ex) { + if (isWithClauseMissing(ex)) { + // turn the error into a different error + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_COLLECTION_LOCK_FAILED, + "collection not known to traversal: '" + collectionName + + "'. please add 'WITH " + collectionName + + "' as the first line in your AQL"); + } + // rethrow original error + throw; } + return false; + }; - if (!res.is(TRI_ERROR_ARANGO_DOCUMENT_NOT_FOUND)) { - // ok we are in a rather bad state. Better throw and abort. - THROW_ARANGO_EXCEPTION(res); + std::string const& collectionName = collectionNameResult.get().first; + if (_collectionToShardMap.empty()) { + TRI_ASSERT(!ServerState::instance()->isDBServer()); + if (findDocumentInShard(collectionName)) { + return true; } - } catch (basics::Exception const& ex) { - if (isWithClauseMissing(ex)) { - // turn the error into a different error + } else { + auto it = _collectionToShardMap.find(collectionName); + if (it == _collectionToShardMap.end()) { + // Connected to a vertex where we do not know the Shard to. THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_COLLECTION_LOCK_FAILED, - "collection not known to traversal: '" + - collectionName + "'. please add 'WITH " + collectionName + - "' as the first line in your AQL"); + "collection not known to traversal: '" + collectionName + + "'. please add 'WITH " + collectionName + + "' as the first line in your AQL"); + } + for (auto const& shard : it->second) { + if (findDocumentInShard(shard)) { + // Short circuit, as soon as one shard contains this document + // we can return it. + return true; + } } - // rethrow original error - throw; } // Register a warning. It is okay though but helps the user @@ -211,19 +230,30 @@ bool RefactoredTraverserCache::appendVertex(aql::TraversalStats& stats, void RefactoredTraverserCache::insertEdgeIntoResult(EdgeDocumentToken const& idToken, VPackBuilder& builder) { - if (!appendEdge(idToken, builder)) { + if (!appendEdge(idToken, false, builder)) { builder.add(VPackSlice::nullSlice()); } } -void RefactoredTraverserCache::insertVertexIntoResult(aql::TraversalStats& stats, - arangodb::velocypack::HashedStringRef const& idString, +void RefactoredTraverserCache::insertEdgeIdIntoResult(EdgeDocumentToken const& idToken, VPackBuilder& builder) { - if (!appendVertex(stats, idString, builder)) { + if (!appendEdge(idToken, true, builder)) { builder.add(VPackSlice::nullSlice()); } } +void RefactoredTraverserCache::insertVertexIntoResult( + aql::TraversalStats& stats, arangodb::velocypack::HashedStringRef const& idString, + VPackBuilder& builder, bool writeIdIfNotFound) { + if (!appendVertex(stats, idString, builder)) { + if (writeIdIfNotFound) { + builder.add(VPackValue(idString.toString())); + } else { + builder.add(VPackSlice::nullSlice()); + } + } +} + arangodb::velocypack::HashedStringRef RefactoredTraverserCache::persistString( arangodb::velocypack::HashedStringRef idString) { auto it = _persistedStrings.find(idString); diff --git a/arangod/Graph/Cache/RefactoredTraverserCache.h b/arangod/Graph/Cache/RefactoredTraverserCache.h index 495fad9509da..77dcf44bbede 100644 --- a/arangod/Graph/Cache/RefactoredTraverserCache.h +++ b/arangod/Graph/Cache/RefactoredTraverserCache.h @@ -62,11 +62,10 @@ struct EdgeDocumentToken; class RefactoredTraverserCache { public: - explicit RefactoredTraverserCache(arangodb::transaction::Methods* trx, - aql::QueryContext* query, - arangodb::ResourceMonitor& resourceMonitor, - arangodb::aql::TraversalStats& stats, - std::map const& collectionToShardMap); + explicit RefactoredTraverserCache( + arangodb::transaction::Methods* trx, aql::QueryContext* query, + arangodb::ResourceMonitor& resourceMonitor, arangodb::aql::TraversalStats& stats, + std::unordered_map> const& collectionToShardMap); ~RefactoredTraverserCache(); RefactoredTraverserCache(RefactoredTraverserCache const&) = delete; @@ -81,15 +80,20 @@ class RefactoredTraverserCache { /// @brief Inserts the real document stored within the token /// into the given builder. ////////////////////////////////////////////////////////////////////////////// - void insertEdgeIntoResult(graph::EdgeDocumentToken const& etkn, - velocypack::Builder& builder); + void insertEdgeIntoResult(graph::EdgeDocumentToken const& etkn, velocypack::Builder& builder); + + ////////////////////////////////////////////////////////////////////////////// + /// @brief Inserts only the edges _id value into the given builder. + ////////////////////////////////////////////////////////////////////////////// + void insertEdgeIdIntoResult(graph::EdgeDocumentToken const& etkn, + velocypack::Builder& builder); ////////////////////////////////////////////////////////////////////////////// /// @brief Inserts the real document identified by the _id string ////////////////////////////////////////////////////////////////////////////// void insertVertexIntoResult(aql::TraversalStats& stats, arangodb::velocypack::HashedStringRef const& idString, - velocypack::Builder& builder); + velocypack::Builder& builder, bool writeIdIfNotFound); ////////////////////////////////////////////////////////////////////////////// /// @brief Persist the given id string. The return value is guaranteed to @@ -111,10 +115,12 @@ class RefactoredTraverserCache { ////////////////////////////////////////////////////////////////////////////// /// @brief Lookup an edge document from the database. /// if this returns false the result is unmodified + /// if onlyId is set to true, the result will only contain the _Id + /// value not the document itself. ////////////////////////////////////////////////////////////////////////////// template - bool appendEdge(graph::EdgeDocumentToken const& etkn, ResultType& result); + bool appendEdge(graph::EdgeDocumentToken const& etkn, bool onlyId, ResultType& result); ////////////////////////////////////////////////////////////////////////////// /// @brief Helper Method to extract collection Name from given VertexIdentifier @@ -148,9 +154,9 @@ class RefactoredTraverserCache { std::unordered_set _persistedStrings; private: - std::map const& _collectionToShardMap; + std::unordered_map> const& _collectionToShardMap; arangodb::ResourceMonitor& _resourceMonitor; - + /// @brief whether or not to allow adding of previously unknown collections /// during the traversal bool const _allowImplicitCollections; diff --git a/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.cpp b/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.cpp index 471042e80961..fee27b69626a 100644 --- a/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.cpp +++ b/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.cpp @@ -27,6 +27,7 @@ #include "Graph/EdgeCursor.h" #include "Graph/EdgeDocumentToken.h" +#include "Graph/Steps/SingleServerProviderStep.h" #include "StorageEngine/PhysicalCollection.h" #include "Transaction/Helpers.h" #include "Transaction/Methods.h" @@ -35,30 +36,58 @@ // TODO: Needed for the IndexAccessor, should be modified #include "Graph/Providers/SingleServerProvider.h" +#ifdef USE_ENTERPRISE +#include "Enterprise/Graph/Steps/SmartGraphStep.h" +#endif + using namespace arangodb; using namespace arangodb::graph; namespace { IndexIteratorOptions defaultIndexIteratorOptions; + +#ifdef USE_ENTERPRISE +static bool CheckInaccessible(transaction::Methods* trx, VPackSlice const& edge) { + // for skipInaccessibleCollections we need to check the edge + // document, in that case nextWithExtra has no benefit + TRI_ASSERT(edge.isString()); + arangodb::velocypack::StringRef str(edge); + size_t pos = str.find('/'); + TRI_ASSERT(pos != std::string::npos); + return trx->isInaccessibleCollection(str.substr(0, pos).toString()); } +#endif +} // namespace -RefactoredSingleServerEdgeCursor::LookupInfo::LookupInfo(transaction::Methods::IndexHandle idx, - aql::AstNode* condition, - std::optional memberToUpdate) +template +RefactoredSingleServerEdgeCursor::LookupInfo::LookupInfo( + transaction::Methods::IndexHandle idx, aql::AstNode* condition, + std::optional memberToUpdate, aql::Expression* expression, size_t cursorID) : _idxHandle(std::move(idx)), + _expression(expression), _indexCondition(condition), + _cursorID(cursorID), _cursor(nullptr), _conditionMemberToUpdate(memberToUpdate) {} -RefactoredSingleServerEdgeCursor::LookupInfo::LookupInfo(LookupInfo&& other) noexcept - : _idxHandle(std::move(other._idxHandle)), - _expression(std::move(other._expression)), - _indexCondition(other._indexCondition), - _cursor(std::move(other._cursor)){}; +template +RefactoredSingleServerEdgeCursor::LookupInfo::LookupInfo(LookupInfo&& other) = default; -RefactoredSingleServerEdgeCursor::LookupInfo::~LookupInfo() = default; +template +RefactoredSingleServerEdgeCursor::LookupInfo::~LookupInfo() = default; + +template +aql::Expression* RefactoredSingleServerEdgeCursor::LookupInfo::getExpression() { + return _expression; +} + +template +size_t RefactoredSingleServerEdgeCursor::LookupInfo::getCursorID() const { + return _cursorID; +} -void RefactoredSingleServerEdgeCursor::LookupInfo::rearmVertex( +template +void RefactoredSingleServerEdgeCursor::LookupInfo::rearmVertex( VertexType vertex, transaction::Methods* trx, arangodb::aql::Variable const* tmpVar) { auto& node = _indexCondition; // We need to rewire the search condition for the new vertex @@ -74,6 +103,25 @@ void RefactoredSingleServerEdgeCursor::LookupInfo::rearmVertex( TRI_ASSERT(idNode->isValueType(aql::VALUE_TYPE_STRING)); // must edit node in place; TODO replace node? // TODO i think there is now a mutable String node available + TEMPORARILY_UNLOCK_NODE(idNode); + idNode->setStringValue(vertex.data(), vertex.length()); + } else { + // If we have to inject the vertex value it has to be within + // the last member of the condition. + // We only get into this case iff the index used does + // not cover _from resp. _to. + // inject _from/_to value + auto expressionNode = _expression->nodeForModification(); + + TRI_ASSERT(expressionNode->numMembers() > 0); + auto dirCmp = expressionNode->getMemberUnchecked(expressionNode->numMembers() - 1); + TRI_ASSERT(dirCmp->type == aql::NODE_TYPE_OPERATOR_BINARY_EQ); + TRI_ASSERT(dirCmp->numMembers() == 2); + + auto idNode = dirCmp->getMemberUnchecked(1); + TRI_ASSERT(idNode->type == aql::NODE_TYPE_VALUE); + TRI_ASSERT(idNode->isValueType(aql::VALUE_TYPE_STRING)); + TEMPORARILY_UNLOCK_NODE(idNode); idNode->setStringValue(vertex.data(), vertex.length()); } @@ -83,7 +131,7 @@ void RefactoredSingleServerEdgeCursor::LookupInfo::rearmVertex( // check if the underlying index iterator supports rearming if (_cursor != nullptr && _cursor->canRearm()) { // rearming supported - if (!_cursor->rearm(node, tmpVar, ::defaultIndexIteratorOptions)) { + if (!_cursor->rearm(node, tmpVar, defaultIndexIteratorOptions)) { _cursor = std::make_unique(_cursor->collection(), trx); } } else { @@ -93,54 +141,88 @@ void RefactoredSingleServerEdgeCursor::LookupInfo::rearmVertex( } } -IndexIterator& RefactoredSingleServerEdgeCursor::LookupInfo::cursor() { +template +IndexIterator& RefactoredSingleServerEdgeCursor::LookupInfo::cursor() { // If this kicks in, you forgot to call rearm with a specific vertex TRI_ASSERT(_cursor != nullptr); return *_cursor; } -RefactoredSingleServerEdgeCursor::RefactoredSingleServerEdgeCursor( - arangodb::transaction::Methods* trx, - arangodb::aql::Variable const* tmpVar, std::vector const& indexConditions) - : _tmpVar(tmpVar), _currentCursor(0), _trx(trx) { +template +RefactoredSingleServerEdgeCursor::RefactoredSingleServerEdgeCursor( + transaction::Methods* trx, arangodb::aql::Variable const* tmpVar, + std::vector const& globalIndexConditions, + std::unordered_map> const& depthBasedIndexConditions, + arangodb::aql::FixedVarExpressionContext& expressionContext, bool requiresFullDocument) + : _tmpVar(tmpVar), + _trx(trx), + _expressionCtx(expressionContext), + _requiresFullDocument(requiresFullDocument) { // We need at least one indexCondition, otherwise nothing to serve - TRI_ASSERT(!indexConditions.empty()); - _lookupInfo.reserve(indexConditions.size()); - for (auto const& idxCond : indexConditions) { + TRI_ASSERT(!globalIndexConditions.empty()); + _lookupInfo.reserve(globalIndexConditions.size()); + _depthLookupInfo.reserve(depthBasedIndexConditions.size()); + + for (auto const& idxCond : globalIndexConditions) { _lookupInfo.emplace_back(idxCond.indexHandle(), idxCond.getCondition(), - idxCond.getMemberToUpdate()); + idxCond.getMemberToUpdate(), + idxCond.getExpression(), idxCond.cursorId()); } -} + for (auto const& obj : depthBasedIndexConditions) { + // Need to reset cursor ID. + // The cursorID relates to the used collection + // not the condition + auto& [depth, idxCondArray] = obj; -RefactoredSingleServerEdgeCursor::~RefactoredSingleServerEdgeCursor() {} - -#ifdef USE_ENTERPRISE -static bool CheckInaccessible(transaction::Methods* trx, VPackSlice const& edge) { - // for skipInaccessibleCollections we need to check the edge - // document, in that case nextWithExtra has no benefit - TRI_ASSERT(edge.isString()); - arangodb::velocypack::StringRef str(edge); - size_t pos = str.find('/'); - TRI_ASSERT(pos != std::string::npos); - return trx->isInaccessibleCollection(str.substr(0, pos).toString()); + std::vector tmpLookupVec; + for (auto const& idxCond : idxCondArray) { + tmpLookupVec.emplace_back(idxCond.indexHandle(), idxCond.getCondition(), + idxCond.getMemberToUpdate(), + idxCond.getExpression(), idxCond.cursorId()); + } + _depthLookupInfo.try_emplace(depth, std::move(tmpLookupVec)); + } } -#endif -void RefactoredSingleServerEdgeCursor::rearm(VertexType vertex, uint64_t /*depth*/) { - _currentCursor = 0; - for (auto& info : _lookupInfo) { +template +RefactoredSingleServerEdgeCursor::~RefactoredSingleServerEdgeCursor() {} + +template +void RefactoredSingleServerEdgeCursor::rearm(VertexType vertex, uint64_t depth) { + for (auto& info : getLookupInfos(depth)) { info.rearmVertex(vertex, _trx, _tmpVar); } } -void RefactoredSingleServerEdgeCursor::readAll(aql::TraversalStats& stats, - Callback const& callback) { - TRI_ASSERT(!_lookupInfo.empty()); - for (_currentCursor = 0; _currentCursor < _lookupInfo.size(); ++_currentCursor) { - auto& cursor = _lookupInfo[_currentCursor].cursor(); +template +void RefactoredSingleServerEdgeCursor::readAll(SingleServerProvider& provider, + aql::TraversalStats& stats, size_t depth, + Callback const& callback) { + TRI_ASSERT(!getLookupInfos(depth).empty()); + VPackBuilder tmpBuilder; + + auto evaluateEdgeExpressionHelper = [&](aql::Expression* expression, + EdgeDocumentToken edgeToken, VPackSlice edge) { + if (edge.isString()) { + tmpBuilder.clear(); + provider.insertEdgeIntoResult(edgeToken, tmpBuilder); + edge = tmpBuilder.slice(); + } + return evaluateEdgeExpression(expression, edge); + }; + + for (auto& lookupInfo : getLookupInfos(depth)) { + auto cursorID = lookupInfo.getCursorID(); + // we can only have a cursorID that is within the amount of collections in use. + TRI_ASSERT(cursorID < _lookupInfo.size()); + + auto& cursor = lookupInfo.cursor(); LogicalCollection* collection = cursor.collection(); auto cid = collection->id(); - if (cursor.hasExtra()) { + bool hasExtra = !_requiresFullDocument && cursor.hasExtra(); + auto* expression = lookupInfo.getExpression(); + + if (hasExtra) { cursor.allExtra([&](LocalDocumentId const& token, VPackSlice edge) { stats.addScannedIndex(1); #ifdef USE_ENTERPRISE @@ -148,31 +230,98 @@ void RefactoredSingleServerEdgeCursor::readAll(aql::TraversalStats& stats, return false; } #endif - callback(EdgeDocumentToken(cid, token), edge, _currentCursor); + + EdgeDocumentToken edgeToken(cid, token); + // evaluate expression if available + if (expression != nullptr && + !evaluateEdgeExpressionHelper(expression, edgeToken, edge)) { + stats.incrFiltered(); + return false; + } + + callback(std::move(edgeToken), edge, cursorID); return true; }); } else { cursor.all([&](LocalDocumentId const& token) { - return collection->getPhysical()->read(_trx, token, [&](LocalDocumentId const&, VPackSlice edgeDoc) { - stats.addScannedIndex(1); + return collection->getPhysical() + ->read( + _trx, token, + [&](LocalDocumentId const&, VPackSlice edgeDoc) { + stats.addScannedIndex(1); #ifdef USE_ENTERPRISE - if (_trx->skipInaccessible()) { - // TODO: we only need to check one of these - VPackSlice from = transaction::helpers::extractFromFromDocument(edgeDoc); - VPackSlice to = transaction::helpers::extractToFromDocument(edgeDoc); - if (CheckInaccessible(_trx, from) || CheckInaccessible(_trx, to)) { - return false; - } - } + if (_trx->skipInaccessible()) { + // TODO: we only need to check one of these + VPackSlice from = + transaction::helpers::extractFromFromDocument(edgeDoc); + VPackSlice to = transaction::helpers::extractToFromDocument(edgeDoc); + if (CheckInaccessible(_trx, from) || CheckInaccessible(_trx, to)) { + return false; + } + } #endif - callback(EdgeDocumentToken(cid, token), edgeDoc, _currentCursor); - return true; - }, ReadOwnWrites::no).ok(); + // eval depth-based expression first if available + EdgeDocumentToken edgeToken(cid, token); + + // evaluate expression if available + if (expression != nullptr && + !evaluateEdgeExpressionHelper(expression, edgeToken, edgeDoc)) { + stats.incrFiltered(); + return false; + } + + callback(std::move(edgeToken), edgeDoc, cursorID); + return true; + }, + ReadOwnWrites::no) + .ok(); }); } } } -arangodb::transaction::Methods* RefactoredSingleServerEdgeCursor::trx() const { - return _trx; +template +bool RefactoredSingleServerEdgeCursor::evaluateEdgeExpression(arangodb::aql::Expression* expression, + VPackSlice value) { + if (expression == nullptr) { + return true; + } + + TRI_ASSERT(value.isObject() || value.isNull()); + + aql::AqlValue edgeVal(aql::AqlValueHintDocumentNoCopy(value.begin())); + _expressionCtx.setVariableValue(_tmpVar, edgeVal); + ScopeGuard defer([&]() noexcept { + try { + _expressionCtx.clearVariableValue(_tmpVar); + } catch (...) { + // This method could in theory throw, if the + // _tmpVar is not in the list. However this is + // guaranteed by this code. If it would throw + // with not found, nothing bad has happened + } + }); + + bool mustDestroy = false; + aql::AqlValue res = expression->execute(&_expressionCtx, mustDestroy); + aql::AqlValueGuard guard(res, mustDestroy); + TRI_ASSERT(res.isBoolean()); + + return res.toBoolean(); +} + +template +auto RefactoredSingleServerEdgeCursor::getLookupInfos(uint64_t depth) + -> std::vector& { + auto const& depthInfo = _depthLookupInfo.find(depth); + if (depthInfo == _depthLookupInfo.end()) { + return _lookupInfo; + } + return depthInfo->second; } + +template class arangodb::graph::RefactoredSingleServerEdgeCursor; + +#ifdef USE_ENTERPRISE +template class arangodb::graph::RefactoredSingleServerEdgeCursor; +#endif diff --git a/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.h b/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.h index 6335bee3a6c6..56a36e6e51d7 100644 --- a/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.h +++ b/arangod/Graph/Cursors/RefactoredSingleServerEdgeCursor.h @@ -24,7 +24,9 @@ #pragma once +#include "Aql/AqlFunctionsInternalCache.h" #include "Aql/Expression.h" +#include "Aql/FixedVarExpressionContext.h" #include "Aql/QueryContext.h" #include "Indexes/IndexIterator.h" #include "Transaction/Methods.h" @@ -51,28 +53,36 @@ struct IndexAccessor; struct EdgeDocumentToken; +template +class SingleServerProvider; + +template class RefactoredSingleServerEdgeCursor { public: struct LookupInfo { LookupInfo(transaction::Methods::IndexHandle idx, aql::AstNode* condition, - std::optional memberToUpdate); + std::optional memberToUpdate, + aql::Expression* expression, size_t cursorID); ~LookupInfo(); LookupInfo(LookupInfo const&) = delete; - LookupInfo(LookupInfo&&) noexcept; + LookupInfo(LookupInfo&&); LookupInfo& operator=(LookupInfo const&) = delete; void rearmVertex(VertexType vertex, transaction::Methods* trx, arangodb::aql::Variable const* tmpVar); IndexIterator& cursor(); + aql::Expression* getExpression(); + + size_t getCursorID() const; private: - // This struct does only take responsibility for the expression // NOTE: The expression can be nullptr! transaction::Methods::IndexHandle _idxHandle; - std::unique_ptr _expression; + aql::Expression* _expression; aql::AstNode* _indexCondition; + size_t _cursorID; std::unique_ptr _cursor; @@ -83,9 +93,12 @@ class RefactoredSingleServerEdgeCursor { enum Direction { FORWARD, BACKWARD }; public: - RefactoredSingleServerEdgeCursor(arangodb::transaction::Methods* trx, - arangodb::aql::Variable const* tmpVar, - std::vector const& indexConditions); + RefactoredSingleServerEdgeCursor( + transaction::Methods* trx, arangodb::aql::Variable const* tmpVar, + std::vector const& globalIndexConditions, + std::unordered_map> const& depthBasedIndexConditions, + arangodb::aql::FixedVarExpressionContext& expressionContext, bool requiresFullDocument); + ~RefactoredSingleServerEdgeCursor(); using Callback = @@ -93,19 +106,23 @@ class RefactoredSingleServerEdgeCursor { private: aql::Variable const* _tmpVar; - size_t _currentCursor; std::vector _lookupInfo; + std::unordered_map> _depthLookupInfo; - arangodb::transaction::Methods* _trx; + transaction::Methods* _trx; + arangodb::aql::FixedVarExpressionContext& _expressionCtx; + bool _requiresFullDocument; public: - void readAll(aql::TraversalStats& stats, Callback const& callback); + void readAll(SingleServerProvider& provider, + aql::TraversalStats& stats, size_t depth, Callback const& callback); void rearm(VertexType vertex, uint64_t depth); + bool evaluateEdgeExpression(arangodb::aql::Expression* expression, VPackSlice value); + private: - [[nodiscard]] transaction::Methods* trx() const; + auto getLookupInfos(uint64_t depth) -> std::vector&; }; } // namespace graph } // namespace arangodb - diff --git a/arangod/Graph/EdgeDocumentToken.h b/arangod/Graph/EdgeDocumentToken.h index abeff524e397..8935cfdffc4c 100644 --- a/arangod/Graph/EdgeDocumentToken.h +++ b/arangod/Graph/EdgeDocumentToken.h @@ -118,12 +118,15 @@ struct EdgeDocumentToken { #ifdef ARANGODB_ENABLE_MAINTAINER_MODE TRI_ASSERT(_type == TokenType::COORDINATOR); #endif - return velocypack::Slice(_data.vpack).binaryEquals(velocypack::Slice(other._data.vpack)); + return velocypack::Slice(_data.vpack) + .binaryEquals(velocypack::Slice(other._data.vpack)); } bool equalsLocal(EdgeDocumentToken const& other) const { #ifdef ARANGODB_ENABLE_MAINTAINER_MODE - TRI_ASSERT(_type == TokenType::LOCAL); + // For local the cid and localDocumentId have illegal values on NONE + // and can be compared with real values + TRI_ASSERT(_type == TokenType::LOCAL || _type == TokenType::NONE); #endif return _data.document.cid == other.cid() && _data.document.localDocumentId == other.localDocumentId(); @@ -136,6 +139,14 @@ struct EdgeDocumentToken { return equalsLocal(other); } + bool isValid() const { + if (ServerState::instance()->isCoordinator()) { + return _data.vpack != nullptr; + } + return _data.document.cid != DataSourceId::none() && + _data.document.localDocumentId != LocalDocumentId::none(); + } + size_t hash() const { if (ServerState::instance()->isCoordinator()) { auto vslice = arangodb::velocypack::Slice(vpack()); diff --git a/arangod/Graph/Enumerators/OneSidedEnumerator.cpp b/arangod/Graph/Enumerators/OneSidedEnumerator.cpp index 7febe88af686..60950d82f5ea 100644 --- a/arangod/Graph/Enumerators/OneSidedEnumerator.cpp +++ b/arangod/Graph/Enumerators/OneSidedEnumerator.cpp @@ -29,54 +29,54 @@ #include "Futures/Future.h" #include "Graph/Options/OneSidedEnumeratorOptions.h" -#include "Graph/PathManagement/PathStore.h" -#include "Graph/PathManagement/PathStoreTracer.h" #include "Graph/PathManagement/PathValidator.h" #include "Graph/Providers/ClusterProvider.h" -#include "Graph/Providers/ProviderTracer.h" #include "Graph/Providers/SingleServerProvider.h" -#include "Graph/Queues/FifoQueue.h" #include "Graph/Queues/QueueTracer.h" +#include "Graph/Steps/SingleServerProviderStep.h" +#include "Graph/Types/ValidationResult.h" +#include "Graph/algorithm-aliases.h" + +#ifdef USE_ENTERPRISE +#include "Enterprise/Graph/algorithm-aliases-ee.h" +#endif -#include #include #include -#include using namespace arangodb; using namespace arangodb::graph; -template -OneSidedEnumerator::OneSidedEnumerator( - ProviderType&& forwardProvider, OneSidedEnumeratorOptions&& options, - arangodb::ResourceMonitor& resourceMonitor) +template +OneSidedEnumerator::OneSidedEnumerator(Provider&& forwardProvider, + OneSidedEnumeratorOptions&& options, + PathValidatorOptions validatorOptions, + arangodb::ResourceMonitor& resourceMonitor) : _options(std::move(options)), _queue(resourceMonitor), _provider(std::move(forwardProvider)), - _validator(_interior), _interior(resourceMonitor), - _resultPath{_provider} {} + _validator(_provider, _interior, std::move(validatorOptions)) {} -template -OneSidedEnumerator::~OneSidedEnumerator() { -} +template +OneSidedEnumerator::~OneSidedEnumerator() = default; -template -auto OneSidedEnumerator::destroyEngines() - -> void { +template +auto OneSidedEnumerator::destroyEngines() -> void { _provider.destroyEngines(); } -template -void OneSidedEnumerator::clear() { - _interior.reset(); +template +void OneSidedEnumerator::clear(bool keepPathStore) { + if (!keepPathStore) { + _interior.reset(); + } _queue.clear(); _results.clear(); } -template -auto OneSidedEnumerator::computeNeighbourhoodOfNextVertex() - -> void { +template +auto OneSidedEnumerator::computeNeighbourhoodOfNextVertex() -> void { // Pull next element from Queue // Do 1 step search TRI_ASSERT(!_queue.isEmpty()); @@ -91,13 +91,31 @@ auto OneSidedEnumerator:: TRI_ASSERT(_queue.hasProcessableElement()); } - auto step = _queue.pop(); - auto posPrevious = _interior.append(step); - + auto tmp = _queue.pop(); + auto posPrevious = _interior.append(std::move(tmp)); + auto& step = _interior.getStepReference(posPrevious); + + // only explore here if we're responsible + if (!step.isResponsible(_provider.trx())) { + // This server cannot decide on this specific vertex. + // Include it in results, to report back that we + // found this undecided path + _results.emplace_back(step); + return; + } ValidationResult res = _validator.validatePath(step); - if ((step.getDepth() >= _options.getMinDepth()) && !res.isFiltered()) { + + // TODO: Adjust log output + LOG_TOPIC("78155", TRACE, Logger::GRAPHS) + << std::boolalpha << " Validated Vertex: " << step.getVertex().getID() + << " filtered " << res.isFiltered() << " pruned " << res.isPruned() + << " depth " << _options.getMinDepth() << " <= " << step.getDepth() + << "<= " << _options.getMaxDepth(); + if (step.getDepth() >= _options.getMinDepth() && !res.isFiltered()) { // Include it in results. - _results.push_back(step); + _results.emplace_back(step); + } else { + _stats.incrFiltered(); } if (step.getDepth() < _options.getMaxDepth() && !res.isPruned()) { @@ -111,8 +129,8 @@ auto OneSidedEnumerator:: * @return true There will be no further path. * @return false There is a chance that there is more data available. */ -template -bool OneSidedEnumerator::isDone() const { +template +bool OneSidedEnumerator::isDone() const { return _results.empty() && searchDone(); } @@ -125,10 +143,11 @@ bool OneSidedEnumerator:: * * @param source The source vertex to start the paths */ -template -void OneSidedEnumerator::reset(VertexRef source) { - clear(); - auto firstStep = _provider.startVertex(source); +template +void OneSidedEnumerator::reset(VertexRef source, size_t depth, + double weight, bool keepPathStore) { + clear(keepPathStore); + auto firstStep = _provider.startVertex(source, depth, weight); _queue.append(std::move(firstStep)); } @@ -145,32 +164,23 @@ void OneSidedEnumerator:: * @return true Found and written a path, result is modified. * @return false No path found, result has not been changed. */ -template -bool OneSidedEnumerator::getNextPath(VPackBuilder& result) { +template +auto OneSidedEnumerator::getNextPath() + -> std::unique_ptr { while (!isDone()) { searchMoreResults(); while (!_results.empty()) { - auto const& vertex = _results.back(); - - // Performance Optimization: - // It seems to be pointless to first push - // everything in to the _resultPath object - // and then iterate again to return the path - // we should be able to return the path in the first go. - _resultPath.clear(); - _interior.buildPath(vertex, _resultPath); - TRI_ASSERT(!_resultPath.isEmpty()); + auto step = std::move(_results.back()); _results.pop_back(); - _resultPath.toVelocyPack(result); - return true; + return std::make_unique(step, _provider, _interior); } } - return false; + return nullptr; } -template -void OneSidedEnumerator::searchMoreResults() { +template +void OneSidedEnumerator::searchMoreResults() { while (_results.empty() && !searchDone()) { // TODO: check && !_queue.isEmpty() _resultsFetched = false; computeNeighbourhoodOfNextVertex(); @@ -186,8 +196,8 @@ void OneSidedEnumerator:: * @return false No path found. */ -template -bool OneSidedEnumerator::skipPath() { +template +bool OneSidedEnumerator::skipPath() { while (!isDone()) { searchMoreResults(); @@ -200,15 +210,13 @@ bool OneSidedEnumerator:: return false; } -template -auto OneSidedEnumerator::searchDone() const - -> bool { +template +auto OneSidedEnumerator::searchDone() const -> bool { return _queue.isEmpty(); } -template -auto OneSidedEnumerator::fetchResults() - -> void { +template +auto OneSidedEnumerator::fetchResults() -> void { if (!_resultsFetched && !_results.empty()) { std::vector looseEnds{}; @@ -235,22 +243,94 @@ auto OneSidedEnumerator:: _resultsFetched = true; } -template -auto OneSidedEnumerator::stealStats() - -> aql::TraversalStats { - aql::TraversalStats stats = _provider.stealStats(); - return stats; +template +auto OneSidedEnumerator::stealStats() -> aql::TraversalStats { + _stats += _provider.stealStats(); + + auto t = _stats; + // Placement new of stats, do not reallocate space. + _stats.~TraversalStats(); + new (&_stats) aql::TraversalStats{}; + return t; } /* SingleServerProvider Section */ +using SingleServerProviderStep = ::arangodb::graph::SingleServerProviderStep; +// Breadth First Search +template class ::arangodb::graph::OneSidedEnumerator< + BFSConfiguration, VertexUniquenessLevel::PATH, false>>; template class ::arangodb::graph::OneSidedEnumerator< - ::arangodb::graph::FifoQueue<::arangodb::graph::SingleServerProvider::Step>, - ::arangodb::graph::PathStore, SingleServerProvider, - ::arangodb::graph::PathValidator, VertexUniquenessLevel::PATH>>; + BFSConfiguration, VertexUniquenessLevel::PATH, true>>; +template class ::arangodb::graph::OneSidedEnumerator< + BFSConfiguration, VertexUniquenessLevel::NONE, false>>; +template class ::arangodb::graph::OneSidedEnumerator< + BFSConfiguration, VertexUniquenessLevel::NONE, true>>; +template class ::arangodb::graph::OneSidedEnumerator< + BFSConfiguration, VertexUniquenessLevel::GLOBAL, false>>; +template class ::arangodb::graph::OneSidedEnumerator< + BFSConfiguration, VertexUniquenessLevel::GLOBAL, true>>; +// Depth First Search +template class ::arangodb::graph::OneSidedEnumerator< + DFSConfiguration, VertexUniquenessLevel::PATH, false>>; +template class ::arangodb::graph::OneSidedEnumerator< + DFSConfiguration, VertexUniquenessLevel::PATH, true>>; +template class ::arangodb::graph::OneSidedEnumerator< + DFSConfiguration, VertexUniquenessLevel::NONE, false>>; +template class ::arangodb::graph::OneSidedEnumerator< + DFSConfiguration, VertexUniquenessLevel::NONE, true>>; +template class ::arangodb::graph::OneSidedEnumerator< + DFSConfiguration, VertexUniquenessLevel::GLOBAL, false>>; template class ::arangodb::graph::OneSidedEnumerator< - ::arangodb::graph::QueueTracer<::arangodb::graph::FifoQueue<::arangodb::graph::SingleServerProvider::Step>>, - ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore>, - ::arangodb::graph::ProviderTracer, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore>, VertexUniquenessLevel::PATH>>; + DFSConfiguration, VertexUniquenessLevel::GLOBAL, true>>; + +#ifdef USE_ENTERPRISE +// Depth First Search +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::PATH, false>>; +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::NONE, false>>; +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::GLOBAL, false>>; + +// DFS Tracing +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::PATH, true>>; +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::NONE, true>>; +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::GLOBAL, true>>; + +// Breath First Search +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::PATH, false>>; +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::NONE, false>>; +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::GLOBAL, false>>; + +// BFS Tracing +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::PATH, true>>; +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::NONE, true>>; +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::GLOBAL, true>>; + +// Weighted Search +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::PATH, false>>; +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::NONE, false>>; +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::GLOBAL, false>>; + +// Weighted Tracing +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::PATH, true>>; +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::NONE, true>>; +template class ::arangodb::graph::OneSidedEnumerator, arangodb::graph::VertexUniquenessLevel::GLOBAL, true>>; +#endif diff --git a/arangod/Graph/Enumerators/OneSidedEnumerator.h b/arangod/Graph/Enumerators/OneSidedEnumerator.h index 465dea72c907..61cc440e1b42 100644 --- a/arangod/Graph/Enumerators/OneSidedEnumerator.h +++ b/arangod/Graph/Enumerators/OneSidedEnumerator.h @@ -28,6 +28,8 @@ #include "Basics/ResourceUsage.h" +#include "Aql/TraversalStats.h" +#include "Graph/Enumerators/OneSidedEnumeratorInterface.h" #include "Graph/Options/OneSidedEnumeratorOptions.h" #include "Graph/PathManagement/SingleProviderPathResult.h" #include "Transaction/Methods.h" @@ -36,10 +38,6 @@ namespace arangodb { -namespace aql { -class TraversalStats; -} - namespace velocypack { class Builder; class HashedStringRef; @@ -48,31 +46,33 @@ class HashedStringRef; namespace graph { struct OneSidedEnumeratorOptions; +class PathValidatorOptions; -template -class PathResult; - -template -class OneSidedEnumerator { +template +class OneSidedEnumerator : public TraversalEnumerator { public: - using Step = typename ProviderType::Step; // public due to tracer access + using Step = typename Configuration::Step; // public due to tracer access + using Provider = typename Configuration::Provider; + using Store = typename Configuration::Store; + + using ResultPathType = SingleProviderPathResult; private: using VertexRef = arangodb::velocypack::HashedStringRef; - using Shell = std::multiset; using ResultList = std::vector; using GraphOptions = arangodb::graph::OneSidedEnumeratorOptions; public: - OneSidedEnumerator(ProviderType&& forwardProvider, OneSidedEnumeratorOptions&& options, + OneSidedEnumerator(Provider&& provider, OneSidedEnumeratorOptions&& options, + PathValidatorOptions pathValidatorOptions, arangodb::ResourceMonitor& resourceMonitor); OneSidedEnumerator(OneSidedEnumerator const& other) = delete; OneSidedEnumerator(OneSidedEnumerator&& other) noexcept = default; ~OneSidedEnumerator(); - void clear(); + void clear(bool keepPathStore) override; /** * @brief Quick test if the finder can prove there is no more data available. @@ -80,18 +80,24 @@ class OneSidedEnumerator { * @return true There will be no further path. * @return false There is a chance that there is more data available. */ - [[nodiscard]] bool isDone() const; + [[nodiscard]] bool isDone() const override; /** - * @brief Reset to new source and target vertices. + * @brief Reset to new source vertex. * This API uses string references, this class will not take responsibility * for the referenced data. It is caller's responsibility to retain the * underlying data and make sure the StringRefs stay valid until next * call of reset. * * @param source The source vertex to start the paths + * @param depth The depth we're starting the search at + * @param weight The vertex ist starting to search at, only relevant for weighted searches + * @param keepPathStore flag to determine that we should keep internas of last + * run in memory. should be used if the last result is not processed yet, as + * we will create invalid memory access in the handed out Paths. */ - void reset(VertexRef source); + void reset(VertexRef source, size_t depth = 0, double weight = 0.0, + bool keepPathStore = false) override; /** * @brief Get the next path, if available written into the result build. @@ -106,7 +112,7 @@ class OneSidedEnumerator { * @return true Found and written a path, result is modified. * @return false No path found, result has not been changed. */ - bool getNextPath(arangodb::velocypack::Builder& result); + auto getNextPath() -> std::unique_ptr override; /** * @brief Skip the next Path, like getNextPath, but does not return the path. @@ -115,14 +121,14 @@ class OneSidedEnumerator { * @return false No path found. */ - bool skipPath(); - auto destroyEngines() -> void; + bool skipPath() override; + auto destroyEngines() -> void override; /** * @brief Return statistics generated since * the last time this method was called. */ - auto stealStats() -> aql::TraversalStats; + auto stealStats() -> aql::TraversalStats override; private: [[nodiscard]] auto searchDone() const -> bool; @@ -143,16 +149,12 @@ class OneSidedEnumerator { GraphOptions _options; ResultList _results{}; bool _resultsFetched{false}; + aql::TraversalStats _stats{}; - // The next elements to process - QueueType _queue; - ProviderType _provider; - PathValidatorType _validator; - - // This stores all paths processed - PathStoreType _interior; - - SingleProviderPathResult _resultPath; + typename Configuration::Queue _queue; // The next elements to process + typename Configuration::Provider _provider; + typename Configuration::Store _interior; // This stores all paths processed + typename Configuration::Validator _validator; }; } // namespace graph } // namespace arangodb diff --git a/arangod/Graph/Enumerators/OneSidedEnumeratorInterface.h b/arangod/Graph/Enumerators/OneSidedEnumeratorInterface.h new file mode 100644 index 000000000000..82c342dc009c --- /dev/null +++ b/arangod/Graph/Enumerators/OneSidedEnumeratorInterface.h @@ -0,0 +1,77 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2021 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +/// @author Heiko Kernbach +//////////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include "Graph/TraverserOptions.h" + +#include +#include +#include + +namespace arangodb { + +namespace velocypack { +class HashedStringRef; +class Builder; +} // namespace velocypack + +namespace aql { +class TraversalStats; +} + +namespace graph { + +class PathResultInterface { + public: + PathResultInterface() {} + virtual ~PathResultInterface() {} + + virtual auto toVelocyPack(arangodb::velocypack::Builder& builder) -> void = 0; +}; + +class TraversalEnumerator { + public: + using VertexRef = arangodb::velocypack::HashedStringRef; + TraversalEnumerator(){}; + virtual ~TraversalEnumerator() {} + + // NOTE: keepPathStore is only required for 3.8 compatibility and + // can be removed in the version after 3.9 + virtual void clear(bool keepPathStore) = 0; + [[nodiscard]] virtual bool isDone() const = 0; + + // NOTE: keepPathStore is only required for 3.8 compatibility and + // can be removed in the version after 3.9 + virtual void reset(VertexRef source, size_t depth = 0, double weight = 0.0, + bool keepPathStore = false) = 0; + virtual auto getNextPath() -> std::unique_ptr = 0; + virtual bool skipPath() = 0; + virtual auto destroyEngines() -> void = 0; + + virtual auto stealStats() -> aql::TraversalStats = 0; +}; + +} // namespace graph +} // namespace arangodb diff --git a/arangod/Graph/Enumerators/TwoSidedEnumerator.cpp b/arangod/Graph/Enumerators/TwoSidedEnumerator.cpp index 51ea0ed1a842..4466219efb0f 100644 --- a/arangod/Graph/Enumerators/TwoSidedEnumerator.cpp +++ b/arangod/Graph/Enumerators/TwoSidedEnumerator.cpp @@ -40,9 +40,10 @@ #include "Graph/Providers/SingleServerProvider.h" #include "Graph/Queues/FifoQueue.h" #include "Graph/Queues/QueueTracer.h" +#include "Graph/Steps/SingleServerProviderStep.h" +#include "Graph/Types/ValidationResult.h" #include "Graph/algorithm-aliases.h" -#include #include #include #include @@ -54,12 +55,12 @@ using namespace arangodb::graph; template TwoSidedEnumerator::Ball::Ball( Direction dir, ProviderType&& provider, GraphOptions const& options, - arangodb::ResourceMonitor& resourceMonitor) + PathValidatorOptions validatorOptions, arangodb::ResourceMonitor& resourceMonitor) : _resourceMonitor(resourceMonitor), _interior(resourceMonitor), _queue(resourceMonitor), _provider(std::move(provider)), - _validator(_interior), + _validator(_provider, _interior, std::move(validatorOptions)), _direction(dir), _minDepth(options.getMinDepth()) {} @@ -67,9 +68,10 @@ template ::Ball::~Ball() = default; template -void TwoSidedEnumerator::Ball::reset(VertexRef center) { +void TwoSidedEnumerator::Ball::reset( + VertexRef center, size_t depth) { clear(); - auto firstStep = _provider.startVertex(center); + auto firstStep = _provider.startVertex(center, depth); _shell.emplace(std::move(firstStep)); } @@ -144,9 +146,9 @@ auto TwoSidedEnumerator:: // Notes for the future: // Vertices are now fetched. Thnink about other less-blocking and batch-wise // fetching (e.g. re-fetch at some later point). - // TODO: Discuss how to optimize here. Currently we'll mark looseEnds in fetch as fetched. - // This works, but we might create a batch limit here in the future. - // Also discuss: Do we want (re-)fetch logic here? + // TODO: Discuss how to optimize here. Currently we'll mark looseEnds in + // fetch as fetched. This works, but we might create a batch limit here in + // the future. Also discuss: Do we want (re-)fetch logic here? // TODO: maybe we can combine this with prefetching of paths } } @@ -245,10 +247,13 @@ auto TwoSidedEnumerator:: template TwoSidedEnumerator::TwoSidedEnumerator( ProviderType&& forwardProvider, ProviderType&& backwardProvider, - TwoSidedEnumeratorOptions&& options, arangodb::ResourceMonitor& resourceMonitor) + TwoSidedEnumeratorOptions&& options, PathValidatorOptions validatorOptions, + arangodb::ResourceMonitor& resourceMonitor) : _options(std::move(options)), - _left{Direction::FORWARD, std::move(forwardProvider), _options, resourceMonitor}, - _right{Direction::BACKWARD, std::move(backwardProvider), _options, resourceMonitor}, + _left{Direction::FORWARD, std::move(forwardProvider), _options, + validatorOptions, resourceMonitor}, + _right{Direction::BACKWARD, std::move(backwardProvider), _options, + std::move(validatorOptions), resourceMonitor}, _resultPath{_left.provider(), _right.provider()} {} template @@ -293,7 +298,7 @@ bool TwoSidedEnumerator:: */ template void TwoSidedEnumerator::reset( - VertexRef source, VertexRef target) { + VertexRef source, VertexRef target, size_t depth) { _results.clear(); _left.reset(source); _right.reset(target); @@ -424,27 +429,30 @@ auto TwoSidedEnumerator:: } /* SingleServerProvider Section */ +using SingleServerProviderStep = ::arangodb::graph::SingleServerProviderStep; template class ::arangodb::graph::TwoSidedEnumerator< - ::arangodb::graph::FifoQueue<::arangodb::graph::SingleServerProvider::Step>, - ::arangodb::graph::PathStore, SingleServerProvider, - ::arangodb::graph::PathValidator, VertexUniquenessLevel::PATH>>; + ::arangodb::graph::FifoQueue, + ::arangodb::graph::PathStore, SingleServerProvider, + ::arangodb::graph::PathValidator, PathStore, VertexUniquenessLevel::PATH>>; template class ::arangodb::graph::TwoSidedEnumerator< - ::arangodb::graph::QueueTracer<::arangodb::graph::FifoQueue<::arangodb::graph::SingleServerProvider::Step>>, - ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore>, - ::arangodb::graph::ProviderTracer, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore>, VertexUniquenessLevel::PATH>>; + ::arangodb::graph::QueueTracer<::arangodb::graph::FifoQueue>, + ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore>, + ::arangodb::graph::ProviderTracer>, + ::arangodb::graph::PathValidator<::arangodb::graph::ProviderTracer>, + ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore>, VertexUniquenessLevel::PATH>>; /* ClusterProvider Section */ template class ::arangodb::graph::TwoSidedEnumerator< ::arangodb::graph::FifoQueue<::arangodb::graph::ClusterProvider::Step>, ::arangodb::graph::PathStore, ClusterProvider, - ::arangodb::graph::PathValidator, VertexUniquenessLevel::PATH>>; + ::arangodb::graph::PathValidator, VertexUniquenessLevel::PATH>>; template class ::arangodb::graph::TwoSidedEnumerator< ::arangodb::graph::QueueTracer<::arangodb::graph::FifoQueue<::arangodb::graph::ClusterProvider::Step>>, ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore>, ::arangodb::graph::ProviderTracer, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore>, VertexUniquenessLevel::PATH>>; + ::arangodb::graph::PathValidator<::arangodb::graph::ProviderTracer, + ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore>, VertexUniquenessLevel::PATH>>; diff --git a/arangod/Graph/Enumerators/TwoSidedEnumerator.h b/arangod/Graph/Enumerators/TwoSidedEnumerator.h index c9df2adbdd2b..f1021287be41 100644 --- a/arangod/Graph/Enumerators/TwoSidedEnumerator.h +++ b/arangod/Graph/Enumerators/TwoSidedEnumerator.h @@ -47,6 +47,7 @@ class HashedStringRef; namespace graph { +class PathValidatorOptions; struct TwoSidedEnumeratorOptions; template @@ -69,10 +70,10 @@ class TwoSidedEnumerator { class Ball { public: Ball(Direction dir, ProviderType&& provider, GraphOptions const& options, - arangodb::ResourceMonitor& resourceMonitor); + PathValidatorOptions validatorOptions, arangodb::ResourceMonitor& resourceMonitor); ~Ball(); auto clear() -> void; - auto reset(VertexRef center) -> void; + auto reset(VertexRef center, size_t depth = 0) -> void; auto startNextDepth() -> void; [[nodiscard]] auto noPathLeft() const -> bool; [[nodiscard]] auto getDepth() const -> size_t; @@ -82,7 +83,8 @@ class TwoSidedEnumerator { auto buildPath(Step const& vertexInShell, PathResult& path) -> void; - auto matchResultsInShell(Step const& match, ResultList& results, PathValidatorType const& otherSideValidator) -> void; + auto matchResultsInShell(Step const& match, ResultList& results, + PathValidatorType const& otherSideValidator) -> void; auto computeNeighbourhoodOfNextVertex(Ball& other, ResultList& results) -> void; // Ensure that we have fetched all vertices @@ -118,7 +120,7 @@ class TwoSidedEnumerator { public: TwoSidedEnumerator(ProviderType&& forwardProvider, ProviderType&& backwardProvider, - TwoSidedEnumeratorOptions&& options, + TwoSidedEnumeratorOptions&& options, PathValidatorOptions validatorOptions, arangodb::ResourceMonitor& resourceMonitor); TwoSidedEnumerator(TwoSidedEnumerator const& other) = delete; TwoSidedEnumerator& operator=(TwoSidedEnumerator const& other) = delete; @@ -147,7 +149,7 @@ class TwoSidedEnumerator { * @param source The source vertex to start the paths * @param target The target vertex to end the paths */ - void reset(VertexRef source, VertexRef target); + void reset(VertexRef source, VertexRef target, size_t depth = 0); /** * @brief Get the next path, if available written into the result build. diff --git a/arangod/Graph/Graph.cpp b/arangod/Graph/Graph.cpp index 5d5f88174c8d..5c4d940549aa 100644 --- a/arangod/Graph/Graph.cpp +++ b/arangod/Graph/Graph.cpp @@ -116,6 +116,18 @@ Graph::Graph(velocypack::Slice const& slice, ServerDefaults const& serverDefault TRI_ASSERT(!_rev.empty()); if (slice.hasKey(StaticStrings::GraphEdgeDefinitions)) { + if (slice.isObject()) { + if (slice.hasKey(StaticStrings::GraphSatellites) && + slice.get(StaticStrings::GraphSatellites).isArray()) { + + for (VPackSlice it : VPackArrayIterator(slice.get(StaticStrings::GraphSatellites))) { + if (!it.isString()) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_GRAPH_INVALID_PARAMETER); + } + _satelliteColls.emplace(it.copyString()); + } + } + } parseEdgeDefinitions(slice.get(StaticStrings::GraphEdgeDefinitions)); } if (slice.hasKey(StaticStrings::GraphOrphans)) { @@ -146,6 +158,18 @@ Graph::Graph(TRI_vocbase_t& vocbase, std::string&& graphName, TRI_ASSERT(_rev.empty()); if (info.hasKey(StaticStrings::GraphEdgeDefinitions)) { + if (options.isObject()) { + if (options.hasKey(StaticStrings::GraphSatellites) && + options.get(StaticStrings::GraphSatellites).isArray()) { + + for (VPackSlice it : VPackArrayIterator(options.get(StaticStrings::GraphSatellites))) { + if (!it.isString()) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_GRAPH_INVALID_PARAMETER); + } + _satelliteColls.emplace(it.copyString()); + } + } + } parseEdgeDefinitions(info.get(StaticStrings::GraphEdgeDefinitions)); } if (info.hasKey(StaticStrings::GraphOrphans)) { @@ -215,6 +239,10 @@ std::set const& Graph::orphanCollections() const { return _orphanColls; } +std::unordered_set const& Graph::satelliteCollections() const { + return _satelliteColls; +} + std::set const& Graph::edgeCollections() const { return _edgeColls; } @@ -436,9 +464,11 @@ ResultT EdgeDefinition::createFromVelocypack(VPackSlice edgeDefi // duplicates in from and to shouldn't occur, but are safely ignored here for (VPackSlice it : VPackArrayIterator(from)) { + TRI_ASSERT(it.isString()); fromSet.emplace(it.copyString()); } for (VPackSlice it : VPackArrayIterator(to)) { + TRI_ASSERT(it.isString()); toSet.emplace(it.copyString()); } @@ -746,6 +776,10 @@ void Graph::createCollectionOptions(VPackBuilder& builder, bool waitForSync) con builder.add(StaticStrings::ReplicationFactor, VPackValue(replicationFactor())); } +void Graph::createSatelliteCollectionOptions(VPackBuilder& builder, bool waitForSync) const { + TRI_ASSERT(false); +} + std::optional> Graph::getEdgeDefinition( std::string const& collectionName) const { auto it = edgeDefinitions().find(collectionName); @@ -757,3 +791,8 @@ std::optional> Graph::getEdgeDefini TRI_ASSERT(hasEdgeCollection(collectionName)); return {it->second}; } + +auto Graph::addSatellites(VPackSlice const&) -> Result { + // Enterprise only + return TRI_ERROR_NO_ERROR; +} diff --git a/arangod/Graph/Graph.h b/arangod/Graph/Graph.h index 5c335aa85f64..7c66c8d7109e 100644 --- a/arangod/Graph/Graph.h +++ b/arangod/Graph/Graph.h @@ -45,9 +45,18 @@ struct ServerDefaults; namespace graph { class EdgeDefinition { + public: + enum EdgeDefinitionType { + DEFAULT, + SMART_TO_SMART, + SAT_TO_SAT, + SMART_TO_SAT, + SAT_TO_SMART + }; + public: EdgeDefinition(std::string edgeCollection_, std::set&& from_, - std::set&& to_) + std::set&& to_) : _edgeCollection(std::move(edgeCollection_)), _from(std::move(from_)), _to(std::move(to_)) {} @@ -81,6 +90,16 @@ class EdgeDefinition { bool renameCollection(std::string const& oldName, std::string const& newName); + /* @brief + * Set type of the EdgeDefinition. Only allowed to be called once and only if + * type is DEFAULT. If type has been set, it is not changeable anymore. + * + * @param type Type to be set + * + * @return True if type has been set, returns false in case type has not been set. + */ + auto setType(EdgeDefinitionType type) -> bool; + private: std::string _edgeCollection; std::set _from; @@ -150,6 +169,8 @@ class Graph { virtual void createCollectionOptions(VPackBuilder& builder, bool waitForSync) const; + virtual void createSatelliteCollectionOptions(VPackBuilder& builder, bool waitForSync) const; + public: /// @brief get the cids of all vertexCollections std::set const& vertexCollections() const; @@ -157,6 +178,9 @@ class Graph { /// @brief get the cids of all orphanCollections std::set const& orphanCollections() const; + /// @brief get the cids of all satelliteCollections + std::unordered_set const& satelliteCollections() const; + /// @brief get the cids of all edgeCollections std::set const& edgeCollections() const; @@ -250,6 +274,8 @@ class Graph { /// @brief Add an orphan vertex collection to this graphs definition Result addOrphanCollection(std::string&&); + virtual auto addSatellites(VPackSlice const& satellites) -> Result; + std::ostream& operator<<(std::ostream& ostream); private: @@ -290,6 +316,9 @@ class Graph { /// @brief the names of all orphanCollections std::set _orphanColls; + /// @brief the names of all satelliteCollections + std::unordered_set _satelliteColls; + /// @brief the names of all edgeCollections std::set _edgeColls; diff --git a/arangod/Graph/GraphManager.cpp b/arangod/Graph/GraphManager.cpp index e1cdeeefc80b..ae42abc54ac2 100644 --- a/arangod/Graph/GraphManager.cpp +++ b/arangod/Graph/GraphManager.cpp @@ -96,58 +96,19 @@ Result GraphManager::createCollection(std::string const& name, TRI_col_type_e co auto& vocbase = ctx()->vocbase(); - VPackBuilder helper; - helper.openObject(); - - if (ServerState::instance()->isCoordinator()) { - Result res = - ShardingInfo::validateShardsAndReplicationFactor(options, vocbase.server(), true); - if (res.fail()) { - return res; - } - - bool const forceOneShard = - vocbase.server().getFeature().forceOneShard() || - vocbase.sharding() == "single"; - - if (forceOneShard) { - // force a single shard with shards distributed like "_graph" - helper.add(StaticStrings::NumberOfShards, VPackValue(1)); - helper.add(StaticStrings::DistributeShardsLike, - VPackValue(vocbase.shardingPrototypeName())); - } - } - - helper.close(); - - VPackBuilder mergedBuilder = - VPackCollection::merge(options, helper.slice(), false, true); - std::shared_ptr coll; OperationOptions opOptions(ExecContext::current()); auto res = arangodb::methods::Collections::create( // create collection vocbase, // collection vocbase opOptions, - name, // collection name - colType, // collection type - mergedBuilder.slice(), // collection properties + name, // collection name + colType, // collection type + options, // collection properties waitForSync, true, false, coll); return res; } -Result GraphManager::findOrCreateVertexCollectionByName(const std::string& name, bool waitForSync, - VPackSlice options) { - std::shared_ptr def; - - def = getCollectionByName(ctx()->vocbase(), name); - if (def == nullptr) { - return createVertexCollection(name, waitForSync, options); - } - - return Result(TRI_ERROR_NO_ERROR); -} - bool GraphManager::renameGraphCollection(std::string const& oldName, std::string const& newName) { // todo: return a result, by now just used in the graph modules @@ -227,62 +188,87 @@ Result GraphManager::checkForEdgeDefinitionConflicts(std::map const& edgeDefinitions, - bool waitForSync, VPackSlice options) { - for (auto const& it : edgeDefinitions) { - EdgeDefinition const& edgeDefinition = it.second; - Result res = findOrCreateCollectionsByEdgeDefinition(graph, edgeDefinition, - waitForSync, options); +Result GraphManager::findOrCreateCollectionsByEdgeDefinition(Graph& graph, + EdgeDefinition const& edgeDefinition, + bool waitForSync) { + std::unordered_set satellites = graph.satelliteCollections(); + // Validation Phase collect a list of collections to create + std::unordered_set documentCollectionsToCreate{}; + std::unordered_set edgeCollectionsToCreate{}; + std::unordered_set> existentDocumentCollections{}; + std::unordered_set> existentEdgeCollections{}; - if (res.fail()) { - return res; + auto& vocbase = ctx()->vocbase(); + std::string const& edgeCollName = edgeDefinition.getName(); + std::shared_ptr edgeColl; + Result res = methods::Collections::lookup(vocbase, edgeCollName, edgeColl); + if (res.ok()) { + TRI_ASSERT(edgeColl); + if (edgeColl->type() != TRI_COL_TYPE_EDGE) { + return Result(TRI_ERROR_GRAPH_EDGE_DEFINITION_IS_DOCUMENT, + "Collection: '" + edgeColl->name() + + "' is not an EdgeCollection"); + } else { + // found the collection + existentEdgeCollections.emplace(edgeColl); } + } else if (!res.is(TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND)) { + return res; + } else { + edgeCollectionsToCreate.emplace(edgeCollName); } - return Result{TRI_ERROR_NO_ERROR}; -} - -Result GraphManager::findOrCreateCollectionsByEdgeDefinition(Graph const& graph, - EdgeDefinition const& edgeDefinition, - bool waitForSync, - VPackSlice const options) { - std::string const& edgeCollection = edgeDefinition.getName(); - std::shared_ptr def = - getCollectionByName(ctx()->vocbase(), edgeCollection); - - if (def == nullptr) { - Result res = createEdgeCollection(edgeCollection, waitForSync, options); - if (res.fail()) { + for (auto const& vertexColl : edgeDefinition.getFrom()) { + std::shared_ptr col; + Result res = methods::Collections::lookup(vocbase, vertexColl, col); + if (res.ok()) { + TRI_ASSERT(col); + if (col->isSatellite()) { + satellites.emplace(col->name()); + } + existentDocumentCollections.emplace(col); + } else if (!res.is(TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND)) { return res; + } else { + if (edgeCollectionsToCreate.find(vertexColl) == edgeCollectionsToCreate.end()) { + auto res = ensureVertexShardingMatches(graph, *edgeColl, satellites, + vertexColl, true); + if (res.fail()) { + return res; + } + documentCollectionsToCreate.emplace(vertexColl); + } } } - std::unordered_set vertexCollections; - - // duplicates in from and to shouldn't occur, but are safely ignored here - for (auto const& colName : edgeDefinition.getFrom()) { - vertexCollections.emplace(colName); - } - for (auto const& colName : edgeDefinition.getTo()) { - vertexCollections.emplace(colName); - } - for (auto const& colName : vertexCollections) { - def = getCollectionByName(ctx()->vocbase(), colName); - if (def == nullptr) { - Result res = createVertexCollection(colName, waitForSync, options); - if (res.fail()) { - return res; + for (auto const& vertexColl : edgeDefinition.getTo()) { + std::shared_ptr col; + Result res = methods::Collections::lookup(vocbase, vertexColl, col); + if (res.ok()) { + TRI_ASSERT(col); + if (col->isSatellite()) { + satellites.emplace(col->name()); } + existentDocumentCollections.emplace(col); + } else if (!res.is(TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND)) { + return res; } else { - auto res = graph.validateCollection(*def.get()); - if (res.fail()) { - return res; + if (edgeCollectionsToCreate.find(vertexColl) == edgeCollectionsToCreate.end()) { + if (edgeColl) { + auto res = ensureVertexShardingMatches(graph, *edgeColl, satellites, + vertexColl, false); + if (res.fail()) { + return res; + } + } + + documentCollectionsToCreate.emplace(vertexColl); } } } - - return Result{TRI_ERROR_NO_ERROR}; + return ensureCollections(graph, documentCollectionsToCreate, + edgeCollectionsToCreate, existentDocumentCollections, + existentEdgeCollections, satellites, waitForSync); } /// @brief extract the collection by either id or name, may return nullptr! @@ -412,7 +398,7 @@ OperationResult GraphManager::createGraph(VPackSlice document, bool waitForSync) } // Make sure all collections exist and are created - res = ensureCollections(graph.get(), waitForSync); + res = ensureAllCollections(graph.get(), waitForSync); if (res.fail()) { return OperationResult{res, options}; } @@ -459,8 +445,7 @@ OperationResult GraphManager::storeGraph(Graph const& graph, bool waitForSync, Result GraphManager::applyOnAllGraphs(std::function)> const& callback) const { std::string const queryStr{"FOR g IN _graphs RETURN g"}; arangodb::aql::Query query(transaction::StandaloneContext::Create(_vocbase), - arangodb::aql::QueryString{queryStr}, - nullptr); + arangodb::aql::QueryString{queryStr}, nullptr); query.queryOptions().skipAudit = true; aql::QueryResult queryResult = query.executeSync(); @@ -502,7 +487,9 @@ Result GraphManager::applyOnAllGraphs(std::function satellites = graph->satelliteCollections(); // Validation Phase collect a list of collections to create std::unordered_set documentCollectionsToCreate{}; std::unordered_set edgeCollectionsToCreate{}; @@ -532,7 +519,6 @@ Result GraphManager::ensureCollections(Graph* graph, bool waitForSync) const { } else if (!res.is(TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND)) { return res; } else { - // not found the collection, need to create it later edgeCollectionsToCreate.emplace(edgeColl); } } @@ -545,6 +531,9 @@ Result GraphManager::ensureCollections(Graph* graph, bool waitForSync) const { Result res = methods::Collections::lookup(vocbase, vertexColl, col); if (res.ok()) { TRI_ASSERT(col); + if (col->isSatellite()) { + satellites.emplace(col->name()); + } existentDocumentCollections.emplace(col); } else if (!res.is(TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND)) { return res; @@ -554,72 +543,138 @@ Result GraphManager::ensureCollections(Graph* graph, bool waitForSync) const { } } } + return ensureCollections(*graph, documentCollectionsToCreate, + edgeCollectionsToCreate, existentDocumentCollections, + existentEdgeCollections, satellites, waitForSync); +} +Result GraphManager::ensureCollections( + Graph& graph, std::unordered_set& documentCollectionsToCreate, + std::unordered_set const& edgeCollectionsToCreate, + std::unordered_set> const& existentDocumentCollections, + std::unordered_set> const& existentEdgeCollections, + std::unordered_set const& satellites, bool waitForSync) const { // II. Validate graph // a) Initial Validation if (!existentDocumentCollections.empty()) { for (auto const& col : existentDocumentCollections) { - graph->ensureInitial(*col); + graph.ensureInitial(*col); } } // b) Enterprise Sharding #ifdef USE_ENTERPRISE + std::string createdInitialName; { - Result res = ensureEnterpriseCollectionSharding(graph, waitForSync, - documentCollectionsToCreate); + auto [res, createdCollectionName] = + ensureEnterpriseCollectionSharding(&graph, waitForSync, documentCollectionsToCreate); if (res.fail()) { return res; } + createdInitialName = createdCollectionName; } + + ScopeGuard guard([&]() noexcept { + // rollback initial collection, in case it got created + if (!createdInitialName.empty()) { + std::shared_ptr coll; + Result found = + methods::Collections::lookup(ctx()->vocbase(), createdInitialName, coll); + if (found.ok()) { + TRI_ASSERT(coll); + Result dropResult = arangodb::methods::Collections::drop(*coll, false, -1.0); + if (dropResult.fail()) { + LOG_TOPIC("04c89", WARN, Logger::GRAPHS) + << "While cleaning up graph `" << graph.name() << "`: " + << "Dropping collection `" << createdInitialName << "` failed with error " + << dropResult.errorNumber() << ": " << dropResult.errorMessage(); + } + } + } + }); #endif // III. Validate collections // document collections for (auto const& col : existentDocumentCollections) { - Result res = graph->validateCollection(*col); + Result res = graph.validateCollection(*col); if (res.fail()) { return res; } } // edge collections for (auto const& col : existentEdgeCollections) { - Result res = graph->validateCollection(*col); + Result res = graph.validateCollection(*col); if (res.fail()) { return res; } } + // Storage space for VPackSlices used in options + std::vector>> vpackLake{}; + + auto collectionsToCreate = + prepareCollectionsToCreate(&graph, waitForSync, documentCollectionsToCreate, + edgeCollectionsToCreate, satellites, vpackLake); + if (!collectionsToCreate.ok()) { + return collectionsToCreate.result(); + } + + if (collectionsToCreate.get().empty()) { + // NOTE: Empty graph is allowed. +#ifdef USE_ENTERPRISE + guard.cancel(); +#endif + return TRI_ERROR_NO_ERROR; + } + + std::vector> created; + OperationOptions opOptions(ExecContext::current()); + + Result finalResult = methods::Collections::create(ctx()->vocbase(), opOptions, + collectionsToCreate.get(), waitForSync, + true, false, nullptr, created); +#ifdef USE_ENTERPRISE + if (finalResult.ok()) { + guard.cancel(); + } +#endif + + return finalResult; +} + +#ifndef USE_ENTERPRISE +ResultT> GraphManager::prepareCollectionsToCreate( + Graph const* graph, bool waitForSync, + std::unordered_set const& documentsCollectionNames, + std::unordered_set const& edgeCollectionNames, + std::unordered_set const& satellites, + std::vector>>& vpackLake) const { + std::vector collectionsToCreate; + collectionsToCreate.reserve(documentsCollectionNames.size() + + edgeCollectionNames.size()); // IV. Create collections VPackBuilder optionsBuilder; optionsBuilder.openObject(); graph->createCollectionOptions(optionsBuilder, waitForSync); optionsBuilder.close(); VPackSlice options = optionsBuilder.slice(); - std::vector collectionsToCreate; - collectionsToCreate.reserve(documentCollectionsToCreate.size() + - edgeCollectionsToCreate.size()); + // Retain the options storage space + vpackLake.emplace_back(optionsBuilder.steal()); // Create Document Collections - for (auto const& vertexColl : documentCollectionsToCreate) { + for (auto const& vertexColl : documentsCollectionNames) { collectionsToCreate.emplace_back( CollectionCreationInfo{vertexColl, TRI_COL_TYPE_DOCUMENT, options}); } // Create Edge Collections - for (auto const& edgeColl : edgeCollectionsToCreate) { + for (auto const& edgeColl : edgeCollectionNames) { collectionsToCreate.emplace_back( CollectionCreationInfo{edgeColl, TRI_COL_TYPE_EDGE, options}); } - if (collectionsToCreate.empty()) { - // NOTE: Empty graph is allowed. - return TRI_ERROR_NO_ERROR; - } - - std::vector> created; - OperationOptions opOptions(ExecContext::current()); - return methods::Collections::create(vocbase, opOptions, collectionsToCreate, - waitForSync, true, false, nullptr, created); + return collectionsToCreate; } +#endif bool GraphManager::onlySatellitesUsed(Graph const* graph) const { for (auto const& cname : graph->vertexCollections()) { @@ -650,8 +705,7 @@ Result GraphManager::readGraphKeys(velocypack::Builder& builder) const { Result GraphManager::readGraphByQuery(velocypack::Builder& builder, std::string const& queryStr) const { - arangodb::aql::Query query(ctx(), arangodb::aql::QueryString(queryStr), - nullptr); + arangodb::aql::Query query(ctx(), arangodb::aql::QueryString(queryStr), nullptr); query.queryOptions().skipAudit = true; LOG_TOPIC("f6782", DEBUG, arangodb::Logger::GRAPHS) @@ -670,7 +724,7 @@ Result GraphManager::readGraphByQuery(velocypack::Builder& builder, if (graphsSlice.isNone()) { return Result(TRI_ERROR_OUT_OF_MEMORY); - } + } if (!graphsSlice.isArray()) { LOG_TOPIC("338b7", ERR, arangodb::Logger::GRAPHS) << "cannot read graphs from _graphs collection"; @@ -1037,7 +1091,9 @@ ResultT> GraphManager::buildGraphFromInput(std::string co s = s.get(StaticStrings::ReplicationFactor); if ((s.isNumber() && s.getNumber() == 0) || (s.isString() && s.stringRef() == "satellite")) { - return Result{TRI_ERROR_BAD_PARAMETER, "invalid combination of 'isSmart' and 'satellite' replicationFactor"}; + return Result{TRI_ERROR_BAD_PARAMETER, + "invalid combination of 'isSmart' and 'satellite' " + "replicationFactor"}; } } } @@ -1058,3 +1114,12 @@ ResultT> GraphManager::buildGraphFromInput(std::string co return {TRI_ERROR_INTERNAL}; } } + +#ifndef USE_ENTERPRISE +Result GraphManager::ensureVertexShardingMatches(Graph const&, LogicalCollection&, + std::unordered_set const&, + std::string const&, bool) const { + // Only relevant for Enterprise graphs. + return TRI_ERROR_NO_ERROR; +} +#endif diff --git a/arangod/Graph/GraphManager.h b/arangod/Graph/GraphManager.h index 967c5ed53866..8f6374a4e370 100644 --- a/arangod/Graph/GraphManager.h +++ b/arangod/Graph/GraphManager.h @@ -30,27 +30,24 @@ #include "Aql/Query.h" #include "Aql/VariableGenerator.h" #include "Basics/ReadWriteLock.h" -#include "Cluster/ClusterInfo.h" #include "Basics/ResultT.h" +#include "Cluster/ClusterInfo.h" #include "Graph/Graph.h" #include "Transaction/Methods.h" #include "Transaction/StandaloneContext.h" #include "Utils/OperationResult.h" namespace arangodb { + +struct CollectionCreationInfo; + namespace graph { class GraphManager { private: TRI_vocbase_t& _vocbase; - - std::shared_ptr ctx() const; - //////////////////////////////////////////////////////////////////////////////// - /// @brief find or create vertex collection by name - //////////////////////////////////////////////////////////////////////////////// - Result findOrCreateVertexCollectionByName(const std::string& name, - bool waitForSync, VPackSlice options); + std::shared_ptr ctx() const; //////////////////////////////////////////////////////////////////////////////// /// @brief find or create collection by name and type @@ -91,13 +88,8 @@ class GraphManager { //////////////////////////////////////////////////////////////////////////////// /// @brief find or create collections by EdgeDefinitions //////////////////////////////////////////////////////////////////////////////// - Result findOrCreateCollectionsByEdgeDefinitions(Graph const& graph, - std::map const& edgeDefinitions, - bool waitForSync, VPackSlice options); - - Result findOrCreateCollectionsByEdgeDefinition(Graph const& graph, - EdgeDefinition const& edgeDefinition, - bool waitForSync, VPackSlice options); + Result findOrCreateCollectionsByEdgeDefinition(Graph& graph, EdgeDefinition const& edgeDefinition, + bool waitForSync); //////////////////////////////////////////////////////////////////////////////// /// @brief create a vertex collection @@ -140,7 +132,7 @@ class GraphManager { * * @return Either OK or an error. */ - Result ensureCollections(Graph* graph, bool waitForSync) const; + Result ensureAllCollections(Graph* graph, bool waitForSync) const; /// @brief check if only SatelliteCollections are used bool onlySatellitesUsed(Graph const* graph) const; @@ -172,14 +164,18 @@ class GraphManager { private: #ifdef USE_ENTERPRISE - Result ensureEnterpriseCollectionSharding(Graph const* graph, bool waitForSync, - std::unordered_set& documentCollections) const; - Result ensureSmartCollectionSharding(Graph const* graph, bool waitForSync, - std::unordered_set& documentCollections) const; - Result ensureSatelliteCollectionSharding(Graph const* graph, bool waitForSync, - std::unordered_set& documentCollections) const; + std::pair ensureEnterpriseCollectionSharding( + Graph const* graph, bool waitForSync, + std::unordered_set& documentCollections) const; #endif + Result ensureCollections( + Graph& graph, std::unordered_set& documentCollectionsToCreate, + std::unordered_set const& edgeCollectionsToCreate, + std::unordered_set> const& existentDocumentCollections, + std::unordered_set> const& existentEdgeCollections, + std::unordered_set const& satellites, bool waitForSync) const; + /** * @brief Create a new in memory graph object from the given input. * This graph object does not create collections and does @@ -199,7 +195,18 @@ class GraphManager { Result checkDropGraphPermissions(Graph const& graph, std::unordered_set const& followersToBeRemoved, std::unordered_set const& leadersToBeRemoved); + + ResultT> prepareCollectionsToCreate( + Graph const* graph, bool waitForSync, + std::unordered_set const& documentsCollectionNames, + std::unordered_set const& edgeCollectionNames, + std::unordered_set const& satellites, + std::vector>>& vpackLake) const; + + Result ensureVertexShardingMatches(Graph const& graph, LogicalCollection& edgeColl, + std::unordered_set const& satellites, + std::string const& vertexCollection, + bool fromSide) const; }; } // namespace graph } // namespace arangodb - diff --git a/arangod/Graph/GraphOperations.cpp b/arangod/Graph/GraphOperations.cpp index fa696e284186..7768b0c4e6ef 100644 --- a/arangod/Graph/GraphOperations.cpp +++ b/arangod/Graph/GraphOperations.cpp @@ -223,8 +223,10 @@ Result GraphOperations::checkVertexCollectionAvailability(std::string const& ver } OperationResult GraphOperations::editEdgeDefinition(VPackSlice edgeDefinitionSlice, + VPackSlice definitionOptions, bool waitForSync, std::string const& edgeDefinitionName) { + TRI_ASSERT(definitionOptions.isObject()); OperationOptions options(ExecContext::current()); auto maybeEdgeDef = EdgeDefinition::createFromVelocypack(edgeDefinitionSlice); if (!maybeEdgeDef) { @@ -246,13 +248,17 @@ OperationResult GraphOperations::editEdgeDefinition(VPackSlice edgeDefinitionSli }; } + auto satData = definitionOptions.get(StaticStrings::GraphSatellites); + if (satData.isArray()) { + auto res = _graph.addSatellites(satData); + if (res.fail()) { + // Handles invalid Slice Content + return OperationResult{std::move(res), options}; + } + } + GraphManager gmngr{_vocbase}; - VPackBuilder collectionsOptions; - collectionsOptions.openObject(); - _graph.createCollectionOptions(collectionsOptions, waitForSync); - collectionsOptions.close(); - res = gmngr.findOrCreateCollectionsByEdgeDefinition(_graph, edgeDefinition, waitForSync, - collectionsOptions.slice()); + res = gmngr.findOrCreateCollectionsByEdgeDefinition(_graph, edgeDefinition, waitForSync); if (res.fail()) { return OperationResult(res, options); } @@ -300,12 +306,24 @@ OperationResult GraphOperations::addOrphanCollection(VPackSlice document, bool w bool createCollection) { GraphManager gmngr{_vocbase}; std::string collectionName = document.get("collection").copyString(); + std::shared_ptr def; OperationOptions options(ExecContext::current()); options.waitForSync = waitForSync; OperationResult result(Result(), options); + VPackSlice editOptions = document.get(StaticStrings::GraphOptions); + if (editOptions.isObject()) { + editOptions = editOptions.get(StaticStrings::GraphSatellites); + if (editOptions.isArray()) { + auto res = _graph.addSatellites(editOptions); + if (res.fail()) { + return OperationResult(std::move(res), options); + } + } + } + if (_graph.hasVertexCollection(collectionName)) { if (_graph.hasOrphanCollection(collectionName)) { return OperationResult(TRI_ERROR_GRAPH_COLLECTION_USED_IN_ORPHANS, options); @@ -325,7 +343,7 @@ OperationResult GraphOperations::addOrphanCollection(VPackSlice document, bool w if (def == nullptr) { if (createCollection) { // ensure that all collections are available - res = gmngr.ensureCollections(&_graph, waitForSync); + res = gmngr.ensureAllCollections(&_graph, waitForSync); if (res.fail()) { return OperationResult{std::move(res), options}; @@ -338,7 +356,7 @@ OperationResult GraphOperations::addOrphanCollection(VPackSlice document, bool w } } else { // Hint: Now needed because of the initial property - res = gmngr.ensureCollections(&_graph, waitForSync); + res = gmngr.ensureAllCollections(&_graph, waitForSync); if (res.fail()) { return OperationResult{std::move(res), options}; } @@ -471,7 +489,9 @@ OperationResult GraphOperations::eraseOrphanCollection(bool waitForSync, } OperationResult GraphOperations::addEdgeDefinition(VPackSlice edgeDefinitionSlice, + VPackSlice definitionOptions, bool waitForSync) { + TRI_ASSERT(definitionOptions.isObject()); OperationOptions options(ExecContext::current()); ResultT defRes = _graph.addEdgeDefinition(edgeDefinitionSlice); if (defRes.fail()) { @@ -489,7 +509,16 @@ OperationResult GraphOperations::addEdgeDefinition(VPackSlice edgeDefinitionSlic return OperationResult(res, options); } - res = gmngr.ensureCollections(&_graph, waitForSync); + auto satData = definitionOptions.get(StaticStrings::GraphSatellites); + if (satData.isArray()) { + auto res = _graph.addSatellites(satData); + if (res.fail()) { + // Handles invalid Slice Content + return OperationResult{std::move(res), options}; + } + } + + res = gmngr.ensureAllCollections(&_graph, waitForSync); if (res.fail()) { return OperationResult{std::move(res), options}; diff --git a/arangod/Graph/GraphOperations.h b/arangod/Graph/GraphOperations.h index f41b2417c554..ff2cdcb11555 100644 --- a/arangod/Graph/GraphOperations.h +++ b/arangod/Graph/GraphOperations.h @@ -154,7 +154,7 @@ class GraphOperations { //////////////////////////////////////////////////////////////////////////////// /// @brief create a new edge definition in an existing graph //////////////////////////////////////////////////////////////////////////////// - OperationResult addEdgeDefinition(VPackSlice edgeDefinition, bool waitForSync); + OperationResult addEdgeDefinition(VPackSlice edgeDefinition, VPackSlice options, bool waitForSync); //////////////////////////////////////////////////////////////////////////////// /// @brief remove an edge definition from an existing graph @@ -165,7 +165,7 @@ class GraphOperations { //////////////////////////////////////////////////////////////////////////////// /// @brief create edge definition in an existing graph //////////////////////////////////////////////////////////////////////////////// - OperationResult editEdgeDefinition(VPackSlice edgeDefinitionSlice, bool waitForSync, + OperationResult editEdgeDefinition(VPackSlice edgeDefinitionSlice, VPackSlice options, bool waitForSync, std::string const& edgeDefinitionName); //////////////////////////////////////////////////////////////////////////////// diff --git a/arangod/Graph/PathEnumerator.cpp b/arangod/Graph/PathEnumerator.cpp index 5fdd886dabd3..32fff7d2fa9a 100644 --- a/arangod/Graph/PathEnumerator.cpp +++ b/arangod/Graph/PathEnumerator.cpp @@ -296,6 +296,10 @@ bool DepthFirstEnumerator::next() { } } + if (!validDisjointPath()) { + return; + } + _enumeratedPath.pushEdge(eid); foundPath = true; } @@ -413,3 +417,9 @@ bool DepthFirstEnumerator::shouldPrune() { } return evaluator->evaluate(); } + +#ifndef USE_ENTERPRISE +bool DepthFirstEnumerator::validDisjointPath() const { + return true; +} +#endif diff --git a/arangod/Graph/PathEnumerator.h b/arangod/Graph/PathEnumerator.h index f1ae77f93ff5..17cb870a92d9 100644 --- a/arangod/Graph/PathEnumerator.h +++ b/arangod/Graph/PathEnumerator.h @@ -187,6 +187,7 @@ class DepthFirstEnumerator final : public PathEnumerator { private: bool shouldPrune(); + bool validDisjointPath() const; velocypack::Slice pathToSlice(arangodb::velocypack::Builder& result, bool fromPrune); }; diff --git a/arangod/Graph/PathManagement/PathResult.cpp b/arangod/Graph/PathManagement/PathResult.cpp index 275eeaf89a96..ff0fdb731feb 100644 --- a/arangod/Graph/PathManagement/PathResult.cpp +++ b/arangod/Graph/PathManagement/PathResult.cpp @@ -29,6 +29,12 @@ #include "Graph/Providers/ProviderTracer.h" #include "Graph/Providers/SingleServerProvider.h" +#include "Graph/Steps/SingleServerProviderStep.h" + +#ifdef USE_ENTERPRISE +#include "Enterprise/Graph/Steps/SmartGraphStep.h" +#endif + #include #include @@ -115,11 +121,16 @@ auto PathResult::isEmpty() const -> bool { /* SingleServerProvider Section */ -template class ::arangodb::graph::PathResult<::arangodb::graph::SingleServerProvider, - ::arangodb::graph::SingleServerProvider::Step>; +using SingleServerProviderStep = ::arangodb::graph::SingleServerProviderStep; + +template class ::arangodb::graph::PathResult<::arangodb::graph::SingleServerProvider, SingleServerProviderStep>; +template class ::arangodb::graph::PathResult<::arangodb::graph::ProviderTracer>, SingleServerProviderStep>; -template class ::arangodb::graph::PathResult<::arangodb::graph::ProviderTracer<::arangodb::graph::SingleServerProvider>, - ::arangodb::graph::SingleServerProvider::Step>; +#ifdef USE_ENTERPRISE +template class ::arangodb::graph::PathResult<::arangodb::graph::SingleServerProvider, enterprise::SmartGraphStep>; +template class ::arangodb::graph::PathResult< + ::arangodb::graph::ProviderTracer>, enterprise::SmartGraphStep>; +#endif /* ClusterProvider Section */ diff --git a/arangod/Graph/PathManagement/PathStore.cpp b/arangod/Graph/PathManagement/PathStore.cpp index cf658defd7c3..f1a86da8cbaf 100644 --- a/arangod/Graph/PathManagement/PathStore.cpp +++ b/arangod/Graph/PathManagement/PathStore.cpp @@ -23,13 +23,17 @@ #include "PathStore.h" #include "Graph/PathManagement/PathResult.h" -#include "Graph/PathManagement/SingleProviderPathResult.h" #include "Graph/Providers/ClusterProvider.h" #include "Graph/Providers/ProviderTracer.h" #include "Graph/Providers/SingleServerProvider.h" +#include "Graph/Steps/SingleServerProviderStep.h" #include "Graph/Types/ValidationResult.h" +#ifdef USE_ENTERPRISE +#include "Enterprise/Graph/Steps/SmartGraphStep.h" +#endif + #include #include @@ -80,7 +84,7 @@ size_t PathStore::append(Step step) { } template -Step PathStore::get(size_t position) const { +Step PathStore::getStep(size_t position) const { TRI_ASSERT(position <= size()); Step step = _schreier.at(position); LOG_TOPIC("45bf5", TRACE, Logger::GRAPHS) @@ -89,6 +93,16 @@ Step PathStore::get(size_t position) const { return step; } +template +Step& PathStore::getStepReference(size_t position) { + TRI_ASSERT(position <= size()); + auto& step = _schreier.at(position); + LOG_TOPIC("45bf6", TRACE, Logger::GRAPHS) + << " Get step: " << step.toString(); + + return step; +} + template template auto PathStore::buildPath(Step const& vertex, PathResultType& path) const -> void { @@ -161,39 +175,80 @@ auto PathStore::visitReversePath(Step const& step, } } -/* SingleServerProvider Section */ +template +auto PathStore::modifyReversePath(Step& step, std::function const& visitor) + -> bool { + Step* walker = &step; + // Guaranteed to make progress, as the schreier vector contains a loop-free tree. + while (true) { + bool cont = visitor(*walker); + if (!cont) { + // Aborted + return false; + } + if (walker->isFirst()) { + // Visited the full path + return true; + } + walker = &_schreier.at(walker->getPrevious()); + } +} -template class PathStore; +/* SingleServerProvider Section */ +using SingleServerProviderStep = ::arangodb::graph::SingleServerProviderStep; -template void PathStore::buildPath>( - SingleServerProvider::Step const& vertex, - PathResult& path) const; +template class PathStore; -template void PathStore::buildPath< - SingleProviderPathResult>( - SingleServerProvider::Step const& vertex, - SingleProviderPathResult& path) const; +template void PathStore::buildPath, SingleServerProviderStep>>( + SingleServerProviderStep const& vertex, + PathResult, SingleServerProviderStep>& path) const; -template void PathStore::reverseBuildPath( - SingleServerProvider::Step const& vertex, - PathResult& path) const; +template void PathStore::reverseBuildPath>( + SingleServerProviderStep const& vertex, + PathResult, SingleServerProviderStep>& path) const; // Tracing -template void PathStore::buildPath< - PathResult, ProviderTracer::Step>>( - ProviderTracer::Step const& vertex, - PathResult, ProviderTracer::Step>& path) const; +template void PathStore::buildPath< + PathResult>, + ProviderTracer>::Step>>( + ProviderTracer>::Step const& vertex, + PathResult>, + ProviderTracer>::Step>& path) const; + +template void PathStore>::Step>::reverseBuildPath< + ProviderTracer>>( + ProviderTracer>::Step const& vertex, + PathResult>, + ProviderTracer>::Step>& path) const; + +#ifdef USE_ENTERPRISE +template class PathStore; -template void PathStore::buildPath< - SingleProviderPathResult, ProviderTracer::Step>>( - ProviderTracer::Step const& vertex, - SingleProviderPathResult, - ProviderTracer::Step>& path) const; +template void PathStore::buildPath< + PathResult, enterprise::SmartGraphStep>>( + enterprise::SmartGraphStep const& vertex, + PathResult, enterprise::SmartGraphStep>& path) const; + +template void PathStore::reverseBuildPath>( + enterprise::SmartGraphStep const& vertex, + PathResult, enterprise::SmartGraphStep>& path) const; + +// Tracing -template void PathStore::Step>::reverseBuildPath>( - ProviderTracer::Step const& vertex, - PathResult, ProviderTracer::Step>& path) const; +template void PathStore::buildPath< + PathResult>, + ProviderTracer>::Step>>( + ProviderTracer>::Step const& vertex, + PathResult>, + ProviderTracer>::Step>& path) const; + +template void PathStore>::Step>::reverseBuildPath< + ProviderTracer>>( + ProviderTracer>::Step const& vertex, + PathResult>, + ProviderTracer>::Step>& path) const; +#endif /* ClusterProvider Section */ diff --git a/arangod/Graph/PathManagement/PathStore.h b/arangod/Graph/PathManagement/PathStore.h index 8ec934790c33..a24eb1195f15 100644 --- a/arangod/Graph/PathManagement/PathStore.h +++ b/arangod/Graph/PathManagement/PathStore.h @@ -71,7 +71,10 @@ class PathStore { size_t append(Step step); // @briefs Method returns a step at given position - Step get(size_t position) const; + Step getStep(size_t position) const; + + // @briefs Method returns a step reference at given position + Step& getStepReference(size_t position); // @brief returns the current vector size size_t size() const { return _schreier.size(); } @@ -79,11 +82,14 @@ class PathStore { auto visitReversePath(Step const& step, std::function const& visitor) const -> bool; + auto modifyReversePath(Step& step, std::function const& visitor) -> bool; + template auto buildPath(Step const& vertex, PathResultType& path) const -> void; template - auto reverseBuildPath(Step const& vertex, PathResult& path) const -> void; + auto reverseBuildPath(Step const& vertex, PathResult& path) const + -> void; private: /// @brief schreier vector to store the visited vertices @@ -94,4 +100,3 @@ class PathStore { } // namespace graph } // namespace arangodb - diff --git a/arangod/Graph/PathManagement/PathStoreTracer.cpp b/arangod/Graph/PathManagement/PathStoreTracer.cpp index 3fe68aefe32b..d786a4961503 100644 --- a/arangod/Graph/PathManagement/PathStoreTracer.cpp +++ b/arangod/Graph/PathManagement/PathStoreTracer.cpp @@ -32,8 +32,13 @@ #include "Graph/Providers/ClusterProvider.h" #include "Graph/Providers/ProviderTracer.h" #include "Graph/Providers/SingleServerProvider.h" +#include "Graph/Steps/SingleServerProviderStep.h" #include "Graph/Types/ValidationResult.h" +#ifdef USE_ENTERPRISE +#include "Enterprise/Graph/Steps/SmartGraphStep.h" +#endif + using namespace arangodb; using namespace arangodb::graph; @@ -64,10 +69,17 @@ size_t PathStoreTracer::append(Step step) { } template -typename PathStoreImpl::Step PathStoreTracer::get(size_t position) const { +typename PathStoreImpl::Step PathStoreTracer::getStep(size_t position) const { + double start = TRI_microtime(); + auto sg = arangodb::scopeGuard([&]() noexcept { _stats["getStep"].addTiming(TRI_microtime() - start); }); + return _impl.getStep(position); +} + +template +typename PathStoreImpl::Step& PathStoreTracer::getStepReference(size_t position) { double start = TRI_microtime(); - auto sg = arangodb::scopeGuard([&]() noexcept { _stats["get"].addTiming(TRI_microtime() - start); }); - return _impl.get(position); + auto sg = arangodb::scopeGuard([&]() noexcept { _stats["getStepReference"].addTiming(TRI_microtime() - start); }); + return _impl.getStepReference(position); } template @@ -104,24 +116,50 @@ auto PathStoreTracer::visitReversePath( return _impl.visitReversePath(step, visitor); } +template +auto PathStoreTracer::modifyReversePath(Step& step, + const std::function& visitor) + -> bool { + double start = TRI_microtime(); + auto sg = arangodb::scopeGuard([&]() noexcept { _stats["modifyReversePath"].addTiming(TRI_microtime() - start); }); + return _impl.modifyReversePath(step, visitor); +} + /* SingleServerProvider Section */ +using SingleServerProviderStep = ::arangodb::graph::SingleServerProviderStep; -template class ::arangodb::graph::PathStoreTracer>; +template class ::arangodb::graph::PathStoreTracer>; // Tracing -template void ::arangodb::graph::PathStoreTracer>::buildPath< - PathResult, ProviderTracer::Step>>( - ProviderTracer::Step const& vertex, - PathResult, ProviderTracer::Step>& path) const; - -template void ::arangodb::graph::PathStoreTracer>::buildPath< - SingleProviderPathResult, ProviderTracer::Step>>( - ProviderTracer::Step const& vertex, - SingleProviderPathResult, ProviderTracer::Step>& path) const; - -template void arangodb::graph::PathStoreTracer>::reverseBuildPath>( - ProviderTracer::Step const& vertex, - PathResult, ProviderTracer::Step>& path) const; +template void ::arangodb::graph::PathStoreTracer>::buildPath< + PathResult>, + ProviderTracer>::Step>>( + ProviderTracer>::Step const& vertex, + PathResult>, + ProviderTracer>::Step>& path) const; + +template void arangodb::graph::PathStoreTracer>::reverseBuildPath< + ProviderTracer>>( + ProviderTracer>::Step const& vertex, + PathResult>, + ProviderTracer>::Step>& path) const; + +#ifdef USE_ENTERPRISE +template class ::arangodb::graph::PathStoreTracer>; + +template void ::arangodb::graph::PathStoreTracer>::buildPath< + PathResult>, + ProviderTracer>::Step>>( + ProviderTracer>::Step const& vertex, + PathResult>, + ProviderTracer>::Step>& path) const; + +template void arangodb::graph::PathStoreTracer>::reverseBuildPath< + ProviderTracer>>( + ProviderTracer>::Step const& vertex, + PathResult>, + ProviderTracer>::Step>& path) const; +#endif /* ClusterProvider Section */ diff --git a/arangod/Graph/PathManagement/PathStoreTracer.h b/arangod/Graph/PathManagement/PathStoreTracer.h index 288bd019f92a..a1895924b64a 100644 --- a/arangod/Graph/PathManagement/PathStoreTracer.h +++ b/arangod/Graph/PathManagement/PathStoreTracer.h @@ -60,7 +60,8 @@ class PathStoreTracer { // returns the index of inserted element size_t append(Step step); - auto get(size_t position) const -> Step; + auto getStep(size_t position) const -> Step; + auto getStepReference(size_t position) -> Step&; // @brief returns the current vector size size_t size() const; @@ -75,6 +76,8 @@ class PathStoreTracer { auto visitReversePath(Step const& step, std::function const& visitor) const -> bool; + auto modifyReversePath(Step& step, std::function const& visitor) -> bool; + private: PathStoreImpl _impl; diff --git a/arangod/Graph/PathManagement/PathValidator.cpp b/arangod/Graph/PathManagement/PathValidator.cpp index 28f0a8d4d3a2..6a3150f21f9b 100644 --- a/arangod/Graph/PathManagement/PathValidator.cpp +++ b/arangod/Graph/PathManagement/PathValidator.cpp @@ -22,24 +22,45 @@ //////////////////////////////////////////////////////////////////////////////// #include "PathValidator.h" +#include "Aql/AstNode.h" +#include "Aql/PruneExpressionEvaluator.h" #include "Graph/PathManagement/PathStore.h" #include "Graph/PathManagement/PathStoreTracer.h" #include "Graph/Providers/ClusterProvider.h" +#include "Graph/Providers/ProviderTracer.h" #include "Graph/Providers/SingleServerProvider.h" +#include "Graph/Steps/SingleServerProviderStep.h" #include "Graph/Types/ValidationResult.h" +#ifdef USE_ENTERPRISE +#include "Enterprise/Graph/Steps/SmartGraphStep.h" +#endif + #include "Basics/Exceptions.h" using namespace arangodb; using namespace arangodb::graph; -template -PathValidator::PathValidator(PathStore const& store) - : _store(store) {} +template +PathValidator::PathValidator( + ProviderType& provider, PathStore const& store, PathValidatorOptions opts) + : _store(store), _provider(provider), _options(std::move(opts)) {} + +template +PathValidator::~PathValidator() = default; + +template +auto PathValidator::validatePath( + typename PathStore::Step const& step) -> ValidationResult { + auto ctx = _options.getExpressionContext(); + // Reset variables + ctx.clearVariableValues(); + auto res = evaluateVertexCondition(step); + if (res.isFiltered() && res.isPruned()) { + // Can give up here. This Value is not used + return res; + } -template -auto PathValidator::validatePath(typename PathStore::Step const& step) - -> ValidationResult { if constexpr (vertexUniqueness == VertexUniquenessLevel::PATH) { _uniqueVertices.clear(); // Reserving here is pointless, we will test paths that increase by at most 1 entry. @@ -52,25 +73,24 @@ auto PathValidator::validatePath(typename PathStore return added; }); if (!success) { - return ValidationResult{ValidationResult::Type::FILTER}; + res.combine(ValidationResult::Type::FILTER); } - return ValidationResult{ValidationResult::Type::TAKE}; } if constexpr (vertexUniqueness == VertexUniquenessLevel::GLOBAL) { auto const& [unused, added] = _uniqueVertices.emplace(step.getVertexIdentifier()); // If this add fails, we need to exclude this path if (!added) { - return ValidationResult{ValidationResult::Type::FILTER}; + res.combine(ValidationResult::Type::FILTER); } - return ValidationResult{ValidationResult::Type::TAKE}; } - return ValidationResult{ValidationResult::Type::TAKE}; + return res; } -template -auto PathValidator::validatePath( +template +auto PathValidator::validatePath( typename PathStore::Step const& step, - PathValidator const& otherValidator) -> ValidationResult { + PathValidator const& otherValidator) + -> ValidationResult { if constexpr (vertexUniqueness == VertexUniquenessLevel::PATH) { // For PATH: take _uniqueVertices of otherValidator, and run Visitor of other side, check if one vertex is duplicate. auto const& otherUniqueVertices = otherValidator.exposeUniqueVertices(); @@ -106,22 +126,160 @@ auto PathValidator::validatePath( return ValidationResult{ValidationResult::Type::TAKE}; } -template -auto PathValidator::exposeUniqueVertices() const +template +auto PathValidator::exposeUniqueVertices() const -> ::arangodb::containers::HashSet, std::equal_to> const& { return _uniqueVertices; } +template +auto PathValidator::evaluateVertexRestriction( + typename PathStore::Step const& step) -> bool { + if (step.isFirst()) { + // first step => always allowed + return true; + } + + auto const& allowedCollections = _options.getAllowedVertexCollections(); + if (allowedCollections.empty()) { + // all allowed + return true; + } + + auto collectionName = step.getCollectionName(); + if (std::find(allowedCollections.begin(), allowedCollections.end(), + collectionName) != allowedCollections.end()) { + // found in allowed collections => allowed + return true; + } + + // not allowed + return false; +} + +template +auto PathValidator::evaluateVertexCondition( + typename PathStore::Step const& step) -> ValidationResult { + // evaluate if vertex collection is allowed + bool isAllowed = evaluateVertexRestriction(step); + if (!isAllowed) { + if (_options.hasCompatibility38IncludeFirstVertex() && step.isFirst()) { + return ValidationResult{ValidationResult::Type::PRUNE}; + } + return ValidationResult{ValidationResult::Type::FILTER}; + } + + auto expr = _options.getVertexExpression(step.getDepth()); + if (expr != nullptr) { + _tmpObjectBuilder.clear(); + _provider.addVertexToBuilder(step.getVertex(), _tmpObjectBuilder); + + // evaluate expression + bool satifiesCondition = evaluateVertexExpression(expr, _tmpObjectBuilder.slice()); + if (!satifiesCondition) { + if (_options.hasCompatibility38IncludeFirstVertex() && step.isFirst()) { + return ValidationResult{ValidationResult::Type::PRUNE}; + } + return ValidationResult{ValidationResult::Type::FILTER}; + } + } + return ValidationResult{ValidationResult::Type::TAKE}; +} + +template +auto PathValidator::evaluateVertexExpression( + arangodb::aql::Expression* expression, VPackSlice value) -> bool { + if (expression == nullptr) { + return true; + } + + TRI_ASSERT(value.isObject() || value.isNull()); + auto tmpVar = _options.getTempVar(); + bool mustDestroy = false; + auto ctx = _options.getExpressionContext(); + aql::AqlValue tmpVal{value}; + ctx.setVariableValue(tmpVar, tmpVal); + aql::AqlValue res = expression->execute(&ctx, mustDestroy); + aql::AqlValueGuard guard{res, mustDestroy}; + TRI_ASSERT(res.isBoolean()); + return res.toBoolean(); +} + +template +void PathValidator::setPruneEvaluator( + std::unique_ptr eval) { + _pruneEvaluator = std::move(eval); +} + +template +void PathValidator::setPostFilterEvaluator( + std::unique_ptr eval) { + _postFilterEvaluator = std::move(eval); +} + +template +void PathValidator::reset() { + if constexpr (vertexUniqueness != VertexUniquenessLevel::NONE) { + _uniqueVertices.clear(); + } +} + namespace arangodb { namespace graph { /* SingleServerProvider Section */ -template class PathValidator, VertexUniquenessLevel::PATH>; -template class PathValidator>, VertexUniquenessLevel::PATH>; +using SingleServerProviderStep = ::arangodb::graph::SingleServerProviderStep; + +template class PathValidator, + PathStore, VertexUniquenessLevel::NONE>; +template class PathValidator>, + PathStoreTracer>::Step>>, + VertexUniquenessLevel::NONE>; + +template class PathValidator, + PathStore::Step>, VertexUniquenessLevel::PATH>; +template class PathValidator>, + PathStoreTracer>::Step>>, + VertexUniquenessLevel::PATH>; + +template class PathValidator, + PathStore::Step>, VertexUniquenessLevel::GLOBAL>; +template class PathValidator>, + PathStoreTracer>::Step>>, + VertexUniquenessLevel::GLOBAL>; + +#ifdef USE_ENTERPRISE +template class PathValidator, + PathStore, VertexUniquenessLevel::NONE>; +template class PathValidator>, + PathStoreTracer>::Step>>, + VertexUniquenessLevel::NONE>; + +template class PathValidator, + PathStore::Step>, VertexUniquenessLevel::PATH>; +template class PathValidator>, + PathStoreTracer>::Step>>, + VertexUniquenessLevel::PATH>; + +template class PathValidator, + PathStore::Step>, VertexUniquenessLevel::GLOBAL>; +template class PathValidator>, + PathStoreTracer>::Step>>, + VertexUniquenessLevel::GLOBAL>; +#endif /* ClusterProvider Section */ -template class PathValidator, VertexUniquenessLevel::PATH>; -template class PathValidator>, VertexUniquenessLevel::PATH>; +template class PathValidator, VertexUniquenessLevel::NONE>; +template class PathValidator, + PathStoreTracer::Step>>, VertexUniquenessLevel::NONE>; + +template class PathValidator, VertexUniquenessLevel::PATH>; +template class PathValidator, + PathStoreTracer::Step>>, VertexUniquenessLevel::PATH>; + +template class PathValidator, VertexUniquenessLevel::GLOBAL>; +template class PathValidator, + PathStoreTracer::Step>>, VertexUniquenessLevel::GLOBAL>; } // namespace graph } // namespace arangodb diff --git a/arangod/Graph/PathManagement/PathValidator.h b/arangod/Graph/PathManagement/PathValidator.h index 0ca712af659e..297994eb0c1c 100644 --- a/arangod/Graph/PathManagement/PathValidator.h +++ b/arangod/Graph/PathManagement/PathValidator.h @@ -25,36 +25,75 @@ #include #include "Containers/HashSet.h" +#include "Graph/PathManagement/PathValidatorOptions.h" #include "Graph/Types/UniquenessLevel.h" +#include + namespace arangodb { +namespace aql { +class PruneExpressionEvaluator; +} namespace graph { class ValidationResult; -template +/* + * TODO: + * + * - EdgeUniqueness, like vertex Uniqueness, and sometimes Edge and Vertex Uniqueness enforce each other. + * (e.g. => VertexUniqueness == PATH => EdgeUniquess PATH || NONE. + * + * - Prune Condition. + * - PostFilter Condition. + * - Path Conditions. Vertex, Edge maybe in LookupInfo + * (e.g. p.vertices[*].name ALL == "HANS") + */ +template class PathValidator { using VertexRef = arangodb::velocypack::HashedStringRef; public: - PathValidator(PathStore const& store); + PathValidator(Provider& provider, PathStore const& store, PathValidatorOptions opts); + ~PathValidator(); auto validatePath(typename PathStore::Step const& step) -> ValidationResult; auto validatePath(typename PathStore::Step const& step, - PathValidator const& otherValidator) + PathValidator const& otherValidator) -> ValidationResult; + void setPruneEvaluator(std::unique_ptr eval); + + void setPostFilterEvaluator(std::unique_ptr eval); + + void reset(); + private: PathStore const& _store; + Provider& _provider; // Only for applied vertex uniqueness // TODO: Figure out if we can make this Member template dependend // e.g. std::enable_if ::arangodb::containers::HashSet, std::equal_to> _uniqueVertices; + PathValidatorOptions _options; + + std::unique_ptr _pruneEvaluator; + + std::unique_ptr _postFilterEvaluator; + + arangodb::velocypack::Builder _tmpObjectBuilder; + private: + auto evaluateVertexCondition(typename PathStore::Step const&) -> ValidationResult; + auto evaluateVertexRestriction(typename PathStore::Step const& step) -> bool; + auto exposeUniqueVertices() const -> ::arangodb::containers::HashSet, std::equal_to> const&; + + auto evaluateVertexExpression(arangodb::aql::Expression* expression, + arangodb::velocypack::Slice value) -> bool; }; } // namespace graph } // namespace arangodb diff --git a/arangod/Graph/PathManagement/PathValidatorOptions.cpp b/arangod/Graph/PathManagement/PathValidatorOptions.cpp new file mode 100644 index 000000000000..81935f1931a1 --- /dev/null +++ b/arangod/Graph/PathManagement/PathValidatorOptions.cpp @@ -0,0 +1,79 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2021-2021 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +//////////////////////////////////////////////////////////////////////////////// + +#include "PathValidatorOptions.h" + +#include "Aql/Expression.h" +#include "Aql/QueryContext.h" + +using namespace arangodb; +using namespace arangodb::graph; + +PathValidatorOptions::PathValidatorOptions(aql::Variable const* tmpVar, + arangodb::aql::FixedVarExpressionContext& expressionContext) + : _tmpVar(tmpVar), _expressionCtx(expressionContext) {} + +void PathValidatorOptions::setAllVerticesExpression(std::unique_ptr expression) { + // All edge expression should not be set before + TRI_ASSERT(_allVerticesExpression == nullptr); + _allVerticesExpression = std::move(expression); +} + +void PathValidatorOptions::setVertexExpression(uint64_t depth, + std::unique_ptr expression) { + // Should not respecifiy the condition on a certain depth + TRI_ASSERT(_vertexExpressionOnDepth.find(depth) == _vertexExpressionOnDepth.end()); + _vertexExpressionOnDepth.emplace(depth, std::move(expression)); +} + +aql::Expression* PathValidatorOptions::getVertexExpression(uint64_t depth) const { + auto const& it = _vertexExpressionOnDepth.find(depth); + if (it != _vertexExpressionOnDepth.end()) { + return it->second.get(); + } + return _allVerticesExpression.get(); +} + +void PathValidatorOptions::addAllowedVertexCollection(std::string const& collectionName) { + TRI_ASSERT(std::find(_allowedVertexCollections.begin(), + _allowedVertexCollections.end(), + collectionName) == _allowedVertexCollections.end()); + _allowedVertexCollections.emplace_back(collectionName); +} + +void PathValidatorOptions::addAllowedVertexCollections(std::vector const& collectionNames) { + for (auto& name : collectionNames) { + addAllowedVertexCollection(name); + } +} + +std::vector const& PathValidatorOptions::getAllowedVertexCollections() const { + return _allowedVertexCollections; +} + +aql::Variable const* PathValidatorOptions::getTempVar() const { + return _tmpVar; +} + +aql::FixedVarExpressionContext& PathValidatorOptions::getExpressionContext() { + return _expressionCtx; +} diff --git a/arangod/Graph/PathManagement/PathValidatorOptions.h b/arangod/Graph/PathManagement/PathValidatorOptions.h new file mode 100644 index 000000000000..1525a94793e3 --- /dev/null +++ b/arangod/Graph/PathManagement/PathValidatorOptions.h @@ -0,0 +1,101 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2021-2021 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +//////////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include +#include +#include + +#include "Aql/AqlFunctionsInternalCache.h" +#include "Aql/FixedVarExpressionContext.h" +#include "Transaction/Methods.h" + +namespace arangodb { +namespace aql { +class Expression; +class ExpressionContext; +class QueryContext; +struct Variable; +} // namespace aql + +namespace graph { +class PathValidatorOptions { + public: + PathValidatorOptions(aql::Variable const* tmpVar, + arangodb::aql::FixedVarExpressionContext& expressionContext); + ~PathValidatorOptions() = default; + PathValidatorOptions(PathValidatorOptions&&) = default; + PathValidatorOptions(PathValidatorOptions const&) = default; + + /** + * @brief Set the expression that needs to hold true for ALL vertices on the path. + */ + void setAllVerticesExpression(std::unique_ptr expression); + + /** + * @brief Set the expression that needs to hold true for the vertex on the + * given depth. NOTE: This will overrule the ALL vertex expression, so make + * sure this expression contains everything the ALL expression covers. + */ + void setVertexExpression(uint64_t depth, std::unique_ptr expression); + + /** + * @brief Get the Expression a vertex needs to hold if defined on the given + * depth. It may return a nullptr if all vertices are valid. + * Caller does NOT take responsibilty. Do not delete this pointer. + */ + aql::Expression* getVertexExpression(uint64_t depth) const; + + void addAllowedVertexCollection(std::string const& collectionName); + + void addAllowedVertexCollections(std::vector const& collectionNames); + + std::vector const& getAllowedVertexCollections() const; + + aql::Variable const* getTempVar() const; + + aql::FixedVarExpressionContext& getExpressionContext(); + + // @brief Only required for rolling upgrades 3.8 => 3.9 + // If a graph is asked for the first vertex and that is filtered + // it can be removed for 3.9 => nextVersion. + void compatibility38IncludeFirstVertex() { + _compatibility38IncludeFirstVertex = true; + } + + bool hasCompatibility38IncludeFirstVertex() const { + return _compatibility38IncludeFirstVertex; + } + + private: + std::shared_ptr _allVerticesExpression; + std::unordered_map> _vertexExpressionOnDepth; + aql::Variable const* _tmpVar; + arangodb::aql::FixedVarExpressionContext& _expressionCtx; + + std::vector _allowedVertexCollections; + + bool _compatibility38IncludeFirstVertex = false; +}; +} // namespace graph +} // namespace arangodb diff --git a/arangod/Graph/PathManagement/SingleProviderPathResult.cpp b/arangod/Graph/PathManagement/SingleProviderPathResult.cpp index 86732a8d9a8c..e3963247318f 100644 --- a/arangod/Graph/PathManagement/SingleProviderPathResult.cpp +++ b/arangod/Graph/PathManagement/SingleProviderPathResult.cpp @@ -25,9 +25,16 @@ #include "SingleProviderPathResult.h" #include "Basics/StaticStrings.h" +#include "Graph/PathManagement/PathStore.h" +#include "Graph/PathManagement/PathStoreTracer.h" #include "Graph/Providers/ClusterProvider.h" #include "Graph/Providers/ProviderTracer.h" #include "Graph/Providers/SingleServerProvider.h" +#include "Graph/Steps/SingleServerProviderStep.h" + +#ifdef USE_ENTERPRISE +#include "Enterprise/Graph/Steps/SmartGraphStep.h" +#endif #include #include @@ -35,43 +42,57 @@ using namespace arangodb; using namespace arangodb::graph; -template -SingleProviderPathResult::SingleProviderPathResult(ProviderType& provider) - : _provider(provider) {} +namespace arangodb::graph::enterprise { +class SmartGraphStep; +} // namespace arangodb::graph::enterprise + +template +SingleProviderPathResult::SingleProviderPathResult( + Step step, ProviderType& provider, PathStoreType& store) + : _step(std::move(step)), _provider(provider), _store(store) {} -template -auto SingleProviderPathResult::clear() -> void { +template +auto SingleProviderPathResult::clear() -> void { _vertices.clear(); _edges.clear(); } -template -auto SingleProviderPathResult::appendVertex(typename Step::Vertex v) +template +auto SingleProviderPathResult::appendVertex(typename Step::Vertex v) -> void { _vertices.push_back(std::move(v)); } -template -auto SingleProviderPathResult::prependVertex(typename Step::Vertex v) +template +auto SingleProviderPathResult::prependVertex(typename Step::Vertex v) -> void { _vertices.insert(_vertices.begin(), std::move(v)); } -template -auto SingleProviderPathResult::appendEdge(typename Step::Edge e) +template +auto SingleProviderPathResult::appendEdge(typename Step::Edge e) -> void { _edges.push_back(std::move(e)); } -template -auto SingleProviderPathResult::prependEdge(typename Step::Edge e) +template +auto SingleProviderPathResult::prependEdge(typename Step::Edge e) -> void { _edges.insert(_edges.begin(), std::move(e)); } -template -auto SingleProviderPathResult::toVelocyPack(arangodb::velocypack::Builder& builder) - -> void { +template +auto SingleProviderPathResult::toVelocyPack( + arangodb::velocypack::Builder& builder) -> void { + if (_vertices.empty()) { + _store.visitReversePath(_step, [&](Step const& s) -> bool { + prependVertex(s.getVertex()); + if (s.getEdge().isValid()) { + prependEdge(s.getEdge()); + } + return true; + }); + } VPackObjectBuilder path{&builder}; { builder.add(VPackValue(StaticStrings::GraphQueryVertices)); @@ -92,24 +113,40 @@ auto SingleProviderPathResult::toVelocyPack(arangodb::velocy } } -template -auto SingleProviderPathResult::isEmpty() const -> bool { - return _vertices.empty(); +template +auto SingleProviderPathResult::isEmpty() const + -> bool { + return false; } /* SingleServerProvider Section */ +using SingleServerProviderStep = ::arangodb::graph::SingleServerProviderStep; template class ::arangodb::graph::SingleProviderPathResult< - ::arangodb::graph::SingleServerProvider, ::arangodb::graph::SingleServerProvider::Step>; + ::arangodb::graph::SingleServerProvider, + ::arangodb::graph::PathStore, SingleServerProviderStep>; template class ::arangodb::graph::SingleProviderPathResult< - ::arangodb::graph::ProviderTracer<::arangodb::graph::SingleServerProvider>, - ::arangodb::graph::SingleServerProvider::Step>; + ::arangodb::graph::ProviderTracer<::arangodb::graph::SingleServerProvider>, + ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore>, SingleServerProviderStep>; -/* ClusterProvider Section */ +#ifdef USE_ENTERPRISE +template class ::arangodb::graph::SingleProviderPathResult< + ::arangodb::graph::SingleServerProvider, + ::arangodb::graph::PathStore, enterprise::SmartGraphStep>; + +template class ::arangodb::graph::SingleProviderPathResult< + ::arangodb::graph::ProviderTracer<::arangodb::graph::SingleServerProvider>, + ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore>, enterprise::SmartGraphStep>; +#endif +// TODO: check if cluster is needed here +/* ClusterProvider Section */ template class ::arangodb::graph::SingleProviderPathResult< - ::arangodb::graph::ClusterProvider, ::arangodb::graph::ClusterProvider::Step>; + ::arangodb::graph::ClusterProvider, ::arangodb::graph::PathStore<::arangodb::graph::ClusterProvider::Step>, + ::arangodb::graph::ClusterProvider::Step>; template class ::arangodb::graph::SingleProviderPathResult< - ::arangodb::graph::ProviderTracer<::arangodb::graph::ClusterProvider>, ::arangodb::graph::ClusterProvider::Step>; \ No newline at end of file + ::arangodb::graph::ProviderTracer<::arangodb::graph::ClusterProvider>, + ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::graph::ClusterProvider::Step>>, + ::arangodb::graph::ClusterProvider::Step>; diff --git a/arangod/Graph/PathManagement/SingleProviderPathResult.h b/arangod/Graph/PathManagement/SingleProviderPathResult.h index 9d0fd94b359f..eb03dbe5b620 100644 --- a/arangod/Graph/PathManagement/SingleProviderPathResult.h +++ b/arangod/Graph/PathManagement/SingleProviderPathResult.h @@ -27,7 +27,12 @@ #include #include "Containers/HashSet.h" +#include "Graph/Enumerators/OneSidedEnumeratorInterface.h" +#include "Graph/Providers/TypeAliases.h" +#include "Graph/TraverserOptions.h" + #include +#include namespace arangodb { @@ -37,27 +42,33 @@ class Builder; namespace graph { -template -class SingleProviderPathResult { - using VertexRef = arangodb::velocypack::HashedStringRef; +template +class SingleProviderPathResult : public PathResultInterface { public: - SingleProviderPathResult(ProviderType& provider); + SingleProviderPathResult(Step step, ProviderType& provider, PathStoreType& store); auto clear() -> void; auto appendVertex(typename Step::Vertex v) -> void; auto prependVertex(typename Step::Vertex v) -> void; auto appendEdge(typename Step::Edge e) -> void; auto prependEdge(typename Step::Edge e) -> void; - auto toVelocyPack(arangodb::velocypack::Builder& builder) -> void; - + + auto toVelocyPack(arangodb::velocypack::Builder& builder) -> void override; + auto isEmpty() const -> bool; + ProviderType* getProvider() { return &_provider; } + PathStoreType* getStore() { return &_store; } + Step& getStep() { return _step; } private: + Step _step; + std::vector _vertices; std::vector _edges; - + // Provider for the path ProviderType& _provider; + PathStoreType& _store; }; } // namespace graph } // namespace arangodb diff --git a/arangod/Graph/Providers/BaseProviderOptions.cpp b/arangod/Graph/Providers/BaseProviderOptions.cpp index 12e42e959f4a..860d5e104ae3 100644 --- a/arangod/Graph/Providers/BaseProviderOptions.cpp +++ b/arangod/Graph/Providers/BaseProviderOptions.cpp @@ -27,11 +27,24 @@ using namespace arangodb; using namespace arangodb::graph; IndexAccessor::IndexAccessor(transaction::Methods::IndexHandle idx, - aql::AstNode* condition, std::optional memberToUpdate) - : _idx(idx), _indexCondition(condition), _memberToUpdate(memberToUpdate) {} + aql::AstNode* condition, std::optional memberToUpdate, + std::unique_ptr expression, + size_t cursorId) + : _idx(idx), + _indexCondition(condition), + _memberToUpdate(memberToUpdate), + _cursorId(cursorId) { + if (expression != nullptr) { + _expression = std::move(expression); + } +} aql::AstNode* IndexAccessor::getCondition() const { return _indexCondition; } +aql::Expression* IndexAccessor::getExpression() const { + return _expression.get(); +} + transaction::Methods::IndexHandle IndexAccessor::indexHandle() const { return _idx; } @@ -40,25 +53,54 @@ std::optional IndexAccessor::getMemberToUpdate() const { return _memberToUpdate; } -BaseProviderOptions::BaseProviderOptions(aql::Variable const* tmpVar, - std::vector indexInfo, - std::map const& collectionToShardMap) +size_t IndexAccessor::cursorId() const { return _cursorId; } + +BaseProviderOptions::BaseProviderOptions( + aql::Variable const* tmpVar, + std::pair, std::unordered_map>>&& indexInfo, + aql::FixedVarExpressionContext& expressionContext, + std::unordered_map> const& collectionToShardMap) : _temporaryVariable(tmpVar), _indexInformation(std::move(indexInfo)), - _collectionToShardMap(collectionToShardMap) {} + _expressionContext(expressionContext), + _collectionToShardMap(collectionToShardMap), + _weightCallback(std::nullopt) {} aql::Variable const* BaseProviderOptions::tmpVar() const { return _temporaryVariable; } -std::vector const& BaseProviderOptions::indexInformations() const { +// first is global index information, second is depth-based index information. +std::pair, std::unordered_map>> const& +BaseProviderOptions::indexInformations() const { return _indexInformation; } -std::map const& BaseProviderOptions::collectionToShardMap() const { +std::unordered_map> const& BaseProviderOptions::collectionToShardMap() const { return _collectionToShardMap; } +aql::FixedVarExpressionContext& BaseProviderOptions::expressionContext() const { + return _expressionContext; +} + +bool BaseProviderOptions::hasWeightMethod() const { + return _weightCallback.has_value(); +} + +void BaseProviderOptions::setWeightEdgeCallback(WeightCallback callback) { + _weightCallback = std::move(callback); +} + +double BaseProviderOptions::weightEdge(double prefixWeight, + arangodb::velocypack::Slice edge) const { + if (!hasWeightMethod()) { + // We do not have a weight. Hardcode. + return 1.0; + } + return _weightCallback.value()(prefixWeight, edge); +} + ClusterBaseProviderOptions::ClusterBaseProviderOptions( std::shared_ptr cache, std::unordered_map const* engines, bool backward) @@ -77,4 +119,4 @@ bool ClusterBaseProviderOptions::isBackward() const { return _backward; } std::unordered_map const* ClusterBaseProviderOptions::engines() const { TRI_ASSERT(_engines != nullptr); return _engines; -} \ No newline at end of file +} diff --git a/arangod/Graph/Providers/BaseProviderOptions.h b/arangod/Graph/Providers/BaseProviderOptions.h index 6f2ce9bed657..973b5f00a025 100644 --- a/arangod/Graph/Providers/BaseProviderOptions.h +++ b/arangod/Graph/Providers/BaseProviderOptions.h @@ -23,6 +23,7 @@ #pragma once +#include "Aql/Expression.h" #include "Aql/FixedVarExpressionContext.h" #include "Cluster/ClusterInfo.h" #include "Graph/Cache/RefactoredClusterTraverserCache.h" @@ -41,37 +42,72 @@ namespace graph { struct IndexAccessor { IndexAccessor(transaction::Methods::IndexHandle idx, aql::AstNode* condition, - std::optional memberToUpdate); + std::optional memberToUpdate, + std::unique_ptr expression, size_t cursorId); + IndexAccessor(IndexAccessor const&) = delete; + IndexAccessor(IndexAccessor&&) = default; aql::AstNode* getCondition() const; + aql::Expression* getExpression() const; transaction::Methods::IndexHandle indexHandle() const; std::optional getMemberToUpdate() const; + size_t cursorId() const; private: transaction::Methods::IndexHandle _idx; aql::AstNode* _indexCondition; std::optional _memberToUpdate; + + // Note: We would prefer to have this a unique_ptr here. + // However the IndexAccessor is used in std::vector + // which then refuses to compile (deleted copy constructor) + std::unique_ptr _expression; + size_t _cursorId; }; struct BaseProviderOptions { + using WeightCallback = + std::function; + public: - BaseProviderOptions(aql::Variable const* tmpVar, std::vector indexInfo, - std::map const& collectionToShardMap); + BaseProviderOptions( + aql::Variable const* tmpVar, + std::pair, std::unordered_map>>&& indexInfo, + aql::FixedVarExpressionContext& expressionContext, + std::unordered_map> const& collectionToShardMap); + + BaseProviderOptions(BaseProviderOptions const&) = delete; + BaseProviderOptions(BaseProviderOptions&&) = default; aql::Variable const* tmpVar() const; - std::vector const& indexInformations() const; + std::pair, std::unordered_map>> const& indexInformations() const; + + std::unordered_map> const& collectionToShardMap() const; + + aql::FixedVarExpressionContext& expressionContext() const; - std::map const& collectionToShardMap() const; + bool hasWeightMethod() const; + + double weightEdge(double prefixWeight, arangodb::velocypack::Slice edge) const; + + void setWeightEdgeCallback(WeightCallback callback); private: // The temporary Variable used in the Indexes aql::Variable const* _temporaryVariable; // One entry per collection, ShardTranslation needs // to be done by Provider - std::vector _indexInformation; + std::pair, std::unordered_map>> _indexInformation; + + // The context of AQL variables. These variables are set from the outside. + // and the caller needs to make sure the reference stays valid + aql::FixedVarExpressionContext& _expressionContext; // CollectionName to ShardMap, used if the Traversal is pushed down to DBServer - std::map const& _collectionToShardMap; + std::unordered_map> const& _collectionToShardMap; + + // Optional callback to compute the weight of an edge. + std::optional _weightCallback; }; struct ClusterBaseProviderOptions { @@ -96,4 +132,3 @@ struct ClusterBaseProviderOptions { } // namespace graph } // namespace arangodb - diff --git a/arangod/Graph/Providers/BaseStep.h b/arangod/Graph/Providers/BaseStep.h index 81bc49531500..56418d74359c 100644 --- a/arangod/Graph/Providers/BaseStep.h +++ b/arangod/Graph/Providers/BaseStep.h @@ -23,18 +23,20 @@ #pragma once +#include + #include namespace arangodb { + namespace graph { template class BaseStep { public: - BaseStep() : _previous{std::numeric_limits::max()}, _depth{0} {} - - BaseStep(size_t prev) : _previous{prev}, _depth{0} {} - BaseStep(size_t prev, size_t depth) : _previous{prev}, _depth{depth} {} + BaseStep(size_t prev = std::numeric_limits::max(), size_t depth = 0, + double weight = 1.0) + : _previous{prev}, _depth{depth}, _weight(weight) {} size_t getPrevious() const { return _previous; } @@ -46,13 +48,29 @@ class BaseStep { return static_cast(this)->isLooseEnd(); } - size_t getDepth() const { - return _depth; + size_t getDepth() const { return _depth; } + + double getWeight() const { return _weight; } + + ResultT> extractCollectionName( + arangodb::velocypack::HashedStringRef const& idHashed) const { + size_t pos = idHashed.find('/'); + if (pos == std::string::npos) { + // Invalid input. If we get here somehow we managed to store invalid + // _from/_to values or the traverser did a let an illegal start through + TRI_ASSERT(false); + return Result{TRI_ERROR_GRAPH_INVALID_EDGE, + "edge contains invalid value " + idHashed.toString()}; + } + + std::string colName = idHashed.substr(0, pos).toString(); + return std::make_pair(colName, pos); } private: - size_t const _previous; - size_t const _depth; + size_t _previous; + size_t _depth; + double _weight; }; } // namespace graph } // namespace arangodb diff --git a/arangod/Graph/Providers/ClusterProvider.cpp b/arangod/Graph/Providers/ClusterProvider.cpp index 78eb2c6bce47..172e5d2c888c 100644 --- a/arangod/Graph/Providers/ClusterProvider.cpp +++ b/arangod/Graph/Providers/ClusterProvider.cpp @@ -68,7 +68,6 @@ VertexType getEdgeDestination(arangodb::velocypack::Slice edge, VertexType const } } // namespace - namespace arangodb { namespace graph { auto operator<<(std::ostream& out, ClusterProvider::Step const& step) -> std::ostream& { @@ -94,9 +93,11 @@ VertexType const& ClusterProvider::Step::Vertex::getID() const { } EdgeType const& ClusterProvider::Step::Edge::getID() const { return _edge; } -bool ClusterProvider::Step::Edge::isValid() const { - return !_edge.empty(); -}; +bool ClusterProvider::Step::Edge::isValid() const { return !_edge.empty(); }; + +bool ClusterProvider::Step::isResponsible(transaction::Methods* trx) const { + return true; +} ClusterProvider::ClusterProvider(arangodb::aql::QueryContext& queryContext, ClusterBaseProviderOptions opts, @@ -107,19 +108,19 @@ ClusterProvider::ClusterProvider(arangodb::aql::QueryContext& queryContext, _opts(std::move(opts)), _stats{} {} -ClusterProvider::~ClusterProvider() { - clear(); -} +ClusterProvider::~ClusterProvider() { clear(); } void ClusterProvider::clear() { for (auto const& entry : _vertexConnectedEdges) { - _resourceMonitor->decreaseMemoryUsage(costPerVertexOrEdgeType + (entry.second.size() * (costPerVertexOrEdgeType * 2))); + _resourceMonitor->decreaseMemoryUsage( + costPerVertexOrEdgeType + (entry.second.size() * (costPerVertexOrEdgeType * 2))); } } -auto ClusterProvider::startVertex(VertexType vertex) -> Step { +auto ClusterProvider::startVertex(VertexType vertex, size_t depth, double weight) -> Step { LOG_TOPIC("da308", TRACE, Logger::GRAPHS) << " Start Vertex:" << vertex; // Create the default initial step. + TRI_ASSERT(weight == 0.0); // Not implemented yet return Step(_opts.getCache()->persistString(vertex)); } @@ -203,9 +204,8 @@ void ClusterProvider::fetchVerticesFromEngines(std::vector const& looseEn _opts.getCache()->datalake().add(std::move(payload)); } - /* TODO: Needs to be taken care of as soon as we enable shortest paths for ClusterProvider - bool forShortestPath = true; - if (!forShortestPath) { + /* TODO: Needs to be taken care of as soon as we enable shortest paths for + ClusterProvider bool forShortestPath = true; if (!forShortestPath) { // Fill everything we did not find with NULL for (auto const& v : vertexIds) { result.try_emplace(v, VPackSlice::nullSlice()); @@ -215,15 +215,17 @@ void ClusterProvider::fetchVerticesFromEngines(std::vector const& looseEn */ } - // Note: This disables the TRI_DEFER + // Note: This disables the ScopeGuard futures.clear(); // put back all looseEnds we we're able to cache for (auto& lE : looseEnds) { if (!_opts.getCache()->isVertexCached(lE->getVertexIdentifier())) { // if we end up here, we we're not able to cache the requested vertex (e.g. it does not exist) - _query->warnings().registerWarning(TRI_ERROR_ARANGO_DOCUMENT_NOT_FOUND, lE->getVertexIdentifier().toString()); - _opts.getCache()->cacheVertex(std::move(lE->getVertexIdentifier()), VPackSlice::nullSlice()); + _query->warnings().registerWarning(TRI_ERROR_ARANGO_DOCUMENT_NOT_FOUND, + lE->getVertexIdentifier().toString()); + _opts.getCache()->cacheVertex(std::move(lE->getVertexIdentifier()), + VPackSlice::nullSlice()); } result.emplace_back(std::move(lE)); } @@ -249,18 +251,19 @@ void ClusterProvider::destroyEngines() { auto const* engines = _opts.engines(); for (auto const& engine : *engines) { _stats.addHttpRequests(1); - auto res = network::sendRequestRetry(pool, "server:" + engine.first, fuerte::RestVerb::Delete, - "/_internal/traverser/" + - arangodb::basics::StringUtils::itoa(engine.second), - VPackBuffer(), options) - .get(); + auto res = + network::sendRequestRetry(pool, "server:" + engine.first, fuerte::RestVerb::Delete, + "/_internal/traverser/" + + arangodb::basics::StringUtils::itoa(engine.second), + VPackBuffer(), options) + .get(); if (res.error != fuerte::Error::NoError) { // Note If there was an error on server side we do not have // CL_COMM_SENT LOG_TOPIC("d31a5", ERR, arangodb::Logger::GRAPHS) - << "Could not destroy all traversal engines: " - << TRI_errno_string(network::fuerteToArangoErrorCode(res)); + << "Could not destroy all traversal engines: " + << TRI_errno_string(network::fuerteToArangoErrorCode(res)); } } } @@ -339,7 +342,8 @@ Result ClusterProvider::fetchEdgesFromEngines(VertexType const& vertex) { arangodb::velocypack::HashedStringRef edgeIdRef(edge.get(StaticStrings::IdString)); - auto edgeToEmplace = std::make_pair(edgeIdRef, VertexType{getEdgeDestination(edge, vertex)}); + auto edgeToEmplace = + std::make_pair(edgeIdRef, VertexType{getEdgeDestination(edge, vertex)}); connectedEdges.emplace_back(edgeToEmplace); } @@ -348,10 +352,11 @@ Result ClusterProvider::fetchEdgesFromEngines(VertexType const& vertex) { _opts.getCache()->datalake().add(std::move(payload)); } } - // Note: This disables the TRI_DEFER + // Note: This disables the ScopeGuard futures.clear(); - _resourceMonitor->increaseMemoryUsage(costPerVertexOrEdgeType + (connectedEdges.size() * (costPerVertexOrEdgeType * 2))); + _resourceMonitor->increaseMemoryUsage( + costPerVertexOrEdgeType + (connectedEdges.size() * (costPerVertexOrEdgeType * 2))); _vertexConnectedEdges.emplace(vertex, std::move(connectedEdges)); return TRI_ERROR_NO_ERROR; @@ -398,12 +403,12 @@ auto ClusterProvider::expand(Step const& step, size_t previous, TRI_ASSERT(_opts.getCache()->isVertexCached(vertex.getID())); TRI_ASSERT(_vertexConnectedEdges.find(vertex.getID()) != _vertexConnectedEdges.end()); for (auto const& relation : _vertexConnectedEdges.at(vertex.getID())) { - bool fetched = _vertexConnectedEdges.find(relation.second) != _vertexConnectedEdges.end(); + bool fetched = + _vertexConnectedEdges.find(relation.second) != _vertexConnectedEdges.end(); callback(Step{relation.second, relation.first, previous, fetched}); } } - void ClusterProvider::addVertexToBuilder(Step::Vertex const& vertex, arangodb::velocypack::Builder& builder) { TRI_ASSERT(_opts.getCache()->isVertexCached(vertex.getID())); diff --git a/arangod/Graph/Providers/ClusterProvider.h b/arangod/Graph/Providers/ClusterProvider.h index 76ebb47fafdc..b4cb441f9b2a 100644 --- a/arangod/Graph/Providers/ClusterProvider.h +++ b/arangod/Graph/Providers/ClusterProvider.h @@ -121,6 +121,16 @@ class ClusterProvider { VertexType getVertexIdentifier() const { return _vertex.getID(); } + std::string getCollectionName() const { + auto collectionNameResult = extractCollectionName(_vertex.getID()); + if (collectionNameResult.fail()) { + THROW_ARANGO_EXCEPTION(collectionNameResult.result()); + } + return collectionNameResult.get().first; + }; + + bool isResponsible(transaction::Methods* trx) const; + friend auto operator<<(std::ostream& out, Step const& step) -> std::ostream&; private: @@ -143,7 +153,7 @@ class ClusterProvider { void clear(); - auto startVertex(VertexType vertex) -> Step; + auto startVertex(VertexType vertex, size_t depth = 0, double weight = 0.0) -> Step; auto fetch(std::vector const& looseEnds) -> futures::Future>; auto expand(Step const& from, size_t previous, std::function const& callback) -> void; diff --git a/arangod/Graph/Providers/ProviderTracer.cpp b/arangod/Graph/Providers/ProviderTracer.cpp index fe8d6a4122b4..2b4ad99e493e 100644 --- a/arangod/Graph/Providers/ProviderTracer.cpp +++ b/arangod/Graph/Providers/ProviderTracer.cpp @@ -29,15 +29,19 @@ #include "Graph/Providers/ClusterProvider.h" #include "Graph/Providers/SingleServerProvider.h" +#include "Graph/Steps/SingleServerProviderStep.h" + +#ifdef USE_ENTERPRISE +#include "Enterprise/Graph/Steps/SmartGraphStep.h" +#endif using namespace arangodb; using namespace arangodb::graph; template ProviderTracer::ProviderTracer(arangodb::aql::QueryContext& queryContext, - Options opts, - arangodb::ResourceMonitor& resourceMonitor) - : _impl{queryContext, opts, resourceMonitor} {} + Options opts, arangodb::ResourceMonitor& resourceMonitor) + : _impl{queryContext, std::move(opts), resourceMonitor} {} template ProviderTracer::~ProviderTracer() { @@ -48,10 +52,12 @@ ProviderTracer::~ProviderTracer() { } template -typename ProviderImpl::Step ProviderTracer::startVertex(VertexType vertex) { +typename ProviderImpl::Step ProviderTracer::startVertex(VertexType vertex, + size_t depth, + double weight) { double start = TRI_microtime(); auto sg = arangodb::scopeGuard([&]() noexcept { _stats["startVertex"].addTiming(TRI_microtime() - start); }); - return _impl.startVertex(vertex); + return _impl.startVertex(vertex, depth, weight); } template @@ -107,5 +113,12 @@ transaction::Methods* ProviderTracer::trx() { return _impl.trx(); } -template class ::arangodb::graph::ProviderTracer; +using SingleServerProviderStep = ::arangodb::graph::SingleServerProviderStep; + +template class ::arangodb::graph::ProviderTracer>; + +#ifdef USE_ENTERPRISE +template class ::arangodb::graph::ProviderTracer>; +#endif + template class ::arangodb::graph::ProviderTracer; diff --git a/arangod/Graph/Providers/ProviderTracer.h b/arangod/Graph/Providers/ProviderTracer.h index 13ed9cf33aaf..ca1de73def57 100644 --- a/arangod/Graph/Providers/ProviderTracer.h +++ b/arangod/Graph/Providers/ProviderTracer.h @@ -38,7 +38,7 @@ namespace arangodb { namespace aql { class QueryContext; class TraversalStats; -} +} // namespace aql namespace graph { @@ -59,10 +59,11 @@ class ProviderTracer { ProviderTracer& operator=(ProviderTracer const&) = delete; ProviderTracer& operator=(ProviderTracer&&) = default; - auto startVertex(VertexType vertex) -> Step; + auto startVertex(VertexType vertex, size_t depth = 0, double weight = 0.0) -> Step; auto fetch(std::vector const& looseEnds) - -> futures::Future>; // rocks - auto expand(Step const& from, size_t previous, std::function callback) -> void; // index + -> futures::Future>; + auto expand(Step const& from, size_t previous, std::function callback) + -> void; void addVertexToBuilder(typename Step::Vertex const& vertex, arangodb::velocypack::Builder& builder); @@ -86,4 +87,3 @@ class ProviderTracer { } // namespace graph } // namespace arangodb - diff --git a/arangod/Graph/Providers/SingleServerProvider.cpp b/arangod/Graph/Providers/SingleServerProvider.cpp index 398be4ff87f9..d7552f3eb0ea 100644 --- a/arangod/Graph/Providers/SingleServerProvider.cpp +++ b/arangod/Graph/Providers/SingleServerProvider.cpp @@ -26,70 +26,61 @@ #include "Aql/QueryContext.h" #include "Graph/Cursors/RefactoredSingleServerEdgeCursor.h" +#include "Graph/Steps/SingleServerProviderStep.h" #include "Transaction/Helpers.h" #include "Futures/Future.h" #include "Futures/Utilities.h" +#ifdef USE_ENTERPRISE +#include "Enterprise/Graph/Steps/SmartGraphStep.h" +#endif + #include using namespace arangodb; using namespace arangodb::graph; -namespace arangodb { -namespace graph { -auto operator<<(std::ostream& out, SingleServerProvider::Step const& step) -> std::ostream& { - out << step._vertex.getID(); - return out; -} -} // namespace graph -} // namespace arangodb - -SingleServerProvider::Step::Step(VertexType v) : _vertex(v), _edge() {} - -SingleServerProvider::Step::Step(VertexType v, EdgeDocumentToken edge, size_t prev) - : BaseStep(prev), _vertex(v), _edge(std::move(edge)) {} - -SingleServerProvider::Step::Step(VertexType v, EdgeDocumentToken edge, size_t prev, size_t depth) - : BaseStep(prev, depth), _vertex(v), _edge(std::move(edge)) {} - -SingleServerProvider::Step::~Step() = default; - -VertexType const& SingleServerProvider::Step::Vertex::getID() const { - return _vertex; -} - -EdgeDocumentToken const& SingleServerProvider::Step::Edge::getID() const { - return _token; -} - -bool SingleServerProvider::Step::Edge::isValid() const { - return getID().localDocumentId() != DataSourceId::none(); +template +void SingleServerProvider::addEdgeToBuilder(typename Step::Edge const& edge, + arangodb::velocypack::Builder& builder) { + if (edge.isValid()) { + insertEdgeIntoResult(edge.getID(), builder); + } else { + // We can never hand out invalid ids. + // For production just be sure to add something sensible. + builder.add(VPackSlice::nullSlice()); + } }; -void SingleServerProvider::addEdgeToBuilder(Step::Edge const& edge, - arangodb::velocypack::Builder& builder) { - insertEdgeIntoResult(edge.getID(), builder); +template +void SingleServerProvider::addEdgeIDToBuilder(typename Step::Edge const& edge, + arangodb::velocypack::Builder& builder) { + if (edge.isValid()) { + insertEdgeIntoResult(edge.getID(), builder); + } else { + // We can never hand out invalid ids. + // For production just be sure to add something sensible. + builder.add(VPackSlice::nullSlice()); + } }; -void SingleServerProvider::Step::Edge::addToBuilder(SingleServerProvider& provider, - arangodb::velocypack::Builder& builder) const { - provider.insertEdgeIntoResult(getID(), builder); -} - -SingleServerProvider::SingleServerProvider(arangodb::aql::QueryContext& queryContext, - BaseProviderOptions opts, - arangodb::ResourceMonitor& resourceMonitor) +template +SingleServerProvider::SingleServerProvider(arangodb::aql::QueryContext& queryContext, + BaseProviderOptions opts, + arangodb::ResourceMonitor& resourceMonitor) : _trx(std::make_unique(queryContext.newTrxContext())), _opts(std::move(opts)), _cache(_trx.get(), &queryContext, resourceMonitor, _stats, _opts.collectionToShardMap()), _stats{} { - // activateCache(false); // TODO CHECK RefactoredTraverserCache (will be discussed in the future, need to do benchmarks if affordable) - _cursor = buildCursor(); + // TODO CHECK RefactoredTraverserCache (will be discussed in the future, need to do benchmarks if affordable) + // activateCache(false); + _cursor = buildCursor(opts.expressionContext()); } -void SingleServerProvider::activateCache(bool enableDocumentCache) { +template +void SingleServerProvider::activateCache(bool enableDocumentCache) { // Do not call this twice. // TRI_ASSERT(_cache == nullptr); // TODO: enableDocumentCache check + opts check + cacheManager check @@ -108,16 +99,19 @@ void SingleServerProvider::activateCache(bool enableDocumentCache) { // _cache = new RefactoredTraverserCache(query()); } -auto SingleServerProvider::startVertex(VertexType vertex) -> Step { +template +auto SingleServerProvider::startVertex(VertexType vertex, size_t depth, double weight) + -> Step { LOG_TOPIC("78156", TRACE, Logger::GRAPHS) << " Start Vertex:" << vertex; // Create default initial step // Note: Refactor naming, Strings in our cache here are not allowed to be removed. - return Step(_cache.persistString(vertex)); + return Step(_cache.persistString(vertex), depth, weight); } -auto SingleServerProvider::fetch(std::vector const& looseEnds) +template +auto SingleServerProvider::fetch(std::vector const& looseEnds) -> futures::Future> { // Should never be called in SingleServer case TRI_ASSERT(false); @@ -129,13 +123,17 @@ auto SingleServerProvider::fetch(std::vector const& looseEnds) return futures::makeFuture(std::move(result)); } -auto SingleServerProvider::expand(Step const& step, size_t previous, - std::function const& callback) -> void { +template +auto SingleServerProvider::expand(Step const& step, size_t previous, + std::function const& callback) + -> void { TRI_ASSERT(!step.isLooseEnd()); auto const& vertex = step.getVertex(); TRI_ASSERT(_cursor != nullptr); - _cursor->rearm(vertex.getID(), 0); - _cursor->readAll(_stats, [&](EdgeDocumentToken&& eid, VPackSlice edge, size_t /*cursorIdx*/) -> void { + LOG_TOPIC("c9169", TRACE, Logger::GRAPHS) + << " Expanding " << vertex.getID(); + _cursor->rearm(vertex.getID(), step.getDepth()); + _cursor->readAll(*this, _stats, step.getDepth(), [&](EdgeDocumentToken&& eid, VPackSlice edge, size_t cursorID) -> void { VertexType id = _cache.persistString(([&]() -> auto { if (edge.isString()) { return VertexType(edge); @@ -147,33 +145,64 @@ auto SingleServerProvider::expand(Step const& step, size_t previous, return other; } })()); - callback(Step{id, std::move(eid), previous}); + // TODO: Adjust log output + LOG_TOPIC("c9168", TRACE, Logger::GRAPHS) + << " Neighbor of " << vertex.getID() << " -> " << id; + if (_opts.hasWeightMethod()) { + callback(Step{id, std::move(eid), previous, step.getDepth() + 1, + _opts.weightEdge(step.getWeight(), edge), cursorID}); + } else { + callback(Step{id, std::move(eid), previous, step.getDepth() + 1, 1.0, cursorID}); + } }); } -void SingleServerProvider::addVertexToBuilder(Step::Vertex const& vertex, - arangodb::velocypack::Builder& builder) { - _cache.insertVertexIntoResult(_stats, vertex.getID(), builder); +template +void SingleServerProvider::addVertexToBuilder(typename Step::Vertex const& vertex, + arangodb::velocypack::Builder& builder, + bool writeIdIfNotFound) { + _cache.insertVertexIntoResult(_stats, vertex.getID(), builder, writeIdIfNotFound); } -void SingleServerProvider::insertEdgeIntoResult(EdgeDocumentToken edge, - arangodb::velocypack::Builder& builder) { +template +void SingleServerProvider::insertEdgeIntoResult(EdgeDocumentToken edge, + arangodb::velocypack::Builder& builder) { _cache.insertEdgeIntoResult(edge, builder); } -std::unique_ptr SingleServerProvider::buildCursor() { - return std::make_unique(trx(), _opts.tmpVar(), - _opts.indexInformations()); +template +void SingleServerProvider::insertEdgeIdIntoResult(EdgeDocumentToken edge, + arangodb::velocypack::Builder& builder) { + _cache.insertEdgeIdIntoResult(edge, builder); } -arangodb::transaction::Methods* SingleServerProvider::trx() { +template +std::unique_ptr> SingleServerProvider::buildCursor( + arangodb::aql::FixedVarExpressionContext& expressionContext) { + return std::make_unique>( + trx(), _opts.tmpVar(), _opts.indexInformations().first, + _opts.indexInformations().second, expressionContext, _opts.hasWeightMethod()); +} + +template +arangodb::transaction::Methods* SingleServerProvider::trx() { + TRI_ASSERT(_trx != nullptr); + TRI_ASSERT(_trx->state() != nullptr); + TRI_ASSERT(_trx->transactionContextPtr() != nullptr); return _trx.get(); } -arangodb::aql::TraversalStats SingleServerProvider::stealStats() { +template +arangodb::aql::TraversalStats SingleServerProvider::stealStats() { auto t = _stats; // Placement new of stats, do not reallocate space. _stats.~TraversalStats(); new (&_stats) aql::TraversalStats{}; return t; } + +template class arangodb::graph::SingleServerProvider; + +#ifdef USE_ENTERPRISE +template class arangodb::graph::SingleServerProvider; +#endif diff --git a/arangod/Graph/Providers/SingleServerProvider.h b/arangod/Graph/Providers/SingleServerProvider.h index 3f2ee786362f..e33fd76463c1 100644 --- a/arangod/Graph/Providers/SingleServerProvider.h +++ b/arangod/Graph/Providers/SingleServerProvider.h @@ -34,10 +34,6 @@ #include "Aql/TraversalStats.h" #include "Basics/ResourceUsage.h" -#include "Transaction/Methods.h" - -#include - namespace arangodb { namespace futures { @@ -59,71 +55,14 @@ namespace graph { // TODO: we need to control from the outside if and which parts of the vertex - (will be implemented in the future via template parameters) // data should be returned. This is most-likely done via a template parameter like // this: template -struct SingleServerProvider { +template +class SingleServerProvider { + public: using Options = BaseProviderOptions; - class Step : public arangodb::graph::BaseStep { - public: - class Vertex { - public: - explicit Vertex(VertexType v) : _vertex(v) {} - - VertexType const& getID() const; - - bool operator<(Vertex const& other) const noexcept { - return _vertex < other._vertex; - } - - bool operator>(Vertex const& other) const noexcept { - return _vertex > other._vertex; - } - - private: - VertexType _vertex; - }; - - class Edge { - public: - explicit Edge(EdgeDocumentToken tkn) noexcept : _token(std::move(tkn)) {} - Edge() noexcept : _token() {} - - void addToBuilder(SingleServerProvider& provider, - arangodb::velocypack::Builder& builder) const; - EdgeDocumentToken const& getID() const; - bool isValid() const; - - private: - EdgeDocumentToken _token; - }; - - Step(VertexType v); - Step(VertexType v, EdgeDocumentToken edge, size_t prev); - Step(VertexType v, EdgeDocumentToken edge, size_t prev, size_t depth); - ~Step(); - - bool operator<(Step const& other) const noexcept { - return _vertex < other._vertex; - } - - Vertex const& getVertex() const { return _vertex; } - Edge const& getEdge() const { return _edge; } - - std::string toString() const { - return ": " + _vertex.getID().toString(); - } - bool isProcessable() const { return !isLooseEnd(); } - bool isLooseEnd() const { return false; } - - VertexType getVertexIdentifier() const { return _vertex.getID(); } - - friend auto operator<<(std::ostream& out, Step const& step) -> std::ostream&; - - private: - Vertex _vertex; - Edge _edge; - }; + using Step = StepType; public: - SingleServerProvider(arangodb::aql::QueryContext& queryContext, BaseProviderOptions opts, + SingleServerProvider(arangodb::aql::QueryContext& queryContext, Options opts, arangodb::ResourceMonitor& resourceMonitor); SingleServerProvider(SingleServerProvider const&) = delete; SingleServerProvider(SingleServerProvider&&) = default; @@ -131,16 +70,23 @@ struct SingleServerProvider { SingleServerProvider& operator=(SingleServerProvider const&) = delete; - auto startVertex(VertexType vertex) -> Step; + auto startVertex(VertexType vertex, size_t depth = 0, double weight = 0.0) -> Step; auto fetch(std::vector const& looseEnds) -> futures::Future>; // rocks auto expand(Step const& from, size_t previous, std::function const& callback) -> void; // index void insertEdgeIntoResult(EdgeDocumentToken edge, arangodb::velocypack::Builder& builder); + void insertEdgeIdIntoResult(EdgeDocumentToken edge, arangodb::velocypack::Builder& builder); - void addVertexToBuilder(Step::Vertex const& vertex, arangodb::velocypack::Builder& builder); - void addEdgeToBuilder(Step::Edge const& edge, arangodb::velocypack::Builder& builder); + void addVertexToBuilder(typename Step::Vertex const& vertex, + arangodb::velocypack::Builder& builder, + bool writeIdIfNotFound = false); + void addEdgeToBuilder(typename Step::Edge const& edge, + arangodb::velocypack::Builder& builder); + + void addEdgeIDToBuilder(typename Step::Edge const& edge, + arangodb::velocypack::Builder& builder); void destroyEngines(){}; @@ -151,21 +97,21 @@ struct SingleServerProvider { private: void activateCache(bool enableDocumentCache); - std::unique_ptr buildCursor(); + std::unique_ptr> buildCursor( + arangodb::aql::FixedVarExpressionContext& expressionContext); private: // Unique_ptr to have this class movable, and to keep reference of trx() // alive - Note: _trx must be first here because it is used in _cursor std::unique_ptr _trx; - std::unique_ptr _cursor; + std::unique_ptr> _cursor; BaseProviderOptions _opts; RefactoredTraverserCache _cache; - + arangodb::aql::TraversalStats _stats; }; } // namespace graph } // namespace arangodb - diff --git a/arangod/Graph/Queues/FifoQueue.h b/arangod/Graph/Queues/FifoQueue.h index 1d0408a8926a..9c9eaccd819c 100644 --- a/arangod/Graph/Queues/FifoQueue.h +++ b/arangod/Graph/Queues/FifoQueue.h @@ -35,6 +35,7 @@ namespace graph { template class FifoQueue { public: + static constexpr bool RequiresWeight = false; using Step = StepType; // TODO: Add Sorting (Performance - will be implemented in the future - cluster relevant) // -> loose ends to the front @@ -55,7 +56,7 @@ class FifoQueue { // if push_front() throws, no harm is done, and the memory usage increase // will be rolled back _queue.push_back(std::move(step)); - guard.steal(); // now we are responsible for tracking the memory + guard.steal(); // now we are responsible for tracking the memory } bool hasProcessableElement() const { diff --git a/arangod/Graph/Queues/LifoQueue.h b/arangod/Graph/Queues/LifoQueue.h index a691ffa12df8..4ab5e0b99d90 100644 --- a/arangod/Graph/Queues/LifoQueue.h +++ b/arangod/Graph/Queues/LifoQueue.h @@ -35,6 +35,7 @@ namespace graph { template class LifoQueue { public: + static constexpr bool RequiresWeight = false; using Step = StepType; // TODO: Add Sorting (Performance - will be implemented in the future - cluster relevant) // -> loose ends to the end @@ -55,7 +56,7 @@ class LifoQueue { // if push_front() throws, no harm is done, and the memory usage increase // will be rolled back _queue.push_front(std::move(step)); - guard.steal(); // now we are responsible for tracking the memory + guard.steal(); // now we are responsible for tracking the memory } bool hasProcessableElement() const { diff --git a/arangod/Graph/Queues/QueueTracer.cpp b/arangod/Graph/Queues/QueueTracer.cpp index ac43552dd635..e1eadb6b90fd 100644 --- a/arangod/Graph/Queues/QueueTracer.cpp +++ b/arangod/Graph/Queues/QueueTracer.cpp @@ -26,9 +26,15 @@ #include "Basics/ScopeGuard.h" #include "Basics/system-functions.h" #include "Graph/Providers/ClusterProvider.h" -#include "Graph/Providers/SingleServerProvider.h" -#include "Graph/Queues/LifoQueue.h" #include "Graph/Queues/FifoQueue.h" +#include "Graph/Queues/LifoQueue.h" +#include "Graph/Queues/WeightedQueue.h" +#include "Graph/Steps/SingleServerProviderStep.h" + +#ifdef USE_ENTERPRISE +#include "Enterprise/Graph/Steps/SmartGraphStep.h" +#endif + #include "Logger/LogMacros.h" using namespace arangodb; @@ -103,8 +109,17 @@ auto QueueTracer::pop() -> typename QueueImpl::Step { } /* SingleServerProvider Section */ -template class ::arangodb::graph::QueueTracer>; -template class ::arangodb::graph::QueueTracer>; +using SingleServerProviderStep = ::arangodb::graph::SingleServerProviderStep; + +template class ::arangodb::graph::QueueTracer>; +template class ::arangodb::graph::QueueTracer>; +template class ::arangodb::graph::QueueTracer>; + +#ifdef USE_ENTERPRISE +template class ::arangodb::graph::QueueTracer>; +template class ::arangodb::graph::QueueTracer>; +template class ::arangodb::graph::QueueTracer>; +#endif /* ClusterServerProvider Section */ template class ::arangodb::graph::QueueTracer>; diff --git a/arangod/Graph/Queues/WeightedQueue.h b/arangod/Graph/Queues/WeightedQueue.h new file mode 100644 index 000000000000..3ba44b0fa22d --- /dev/null +++ b/arangod/Graph/Queues/WeightedQueue.h @@ -0,0 +1,134 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2021 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +//////////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include "Basics/ResourceUsage.h" +#include "Basics/debugging.h" +#include "Logger/LogMacros.h" + +#include +#include + +namespace arangodb { +namespace graph { + +template +class WeightedQueue { + public: + static constexpr bool RequiresWeight = true; + using Step = StepType; + // TODO: Add Sorting (Performance - will be implemented in the future - cluster relevant) + // -> loose ends to the end + + explicit WeightedQueue(arangodb::ResourceMonitor& resourceMonitor) + : _resourceMonitor{resourceMonitor} {} + ~WeightedQueue() { this->clear(); } + + void clear() { + if (!_queue.empty()) { + _resourceMonitor.decreaseMemoryUsage(_queue.size() * sizeof(Step)); + _queue.clear(); + } + } + + void append(Step step) { + arangodb::ResourceUsageScope guard(_resourceMonitor, sizeof(Step)); + // if emplace() throws, no harm is done, and the memory usage increase + // will be rolled back + _queue.emplace_back(std::move(step)); + guard.steal(); // now we are responsible for tracking the memory + // std::push_heap takes the last element in the queue, assumes that all + // other elements are in heap structure, and moves the last element into + // the correct position in the heap (incl. rebalancing of other elements) + // The heap structure guarantees that the first element in the queue + // is the "largest" element (in our case it is the smallest, as we inverted the comperator) + std::push_heap(_queue.begin(), _queue.end(), _cmpHeap); + } + + bool hasProcessableElement() const { + if (!isEmpty()) { + // The heap structure guarantees that the first element in the queue + // is the "largest" element (in our case it is the smallest, as we inverted the comperator) + auto const& first = _queue.front(); + return first.isProcessable(); + } + + return false; + } + + size_t size() const { return _queue.size(); } + + bool isEmpty() const { return _queue.empty(); } + + std::vector getLooseEnds() { + TRI_ASSERT(!hasProcessableElement()); + + std::vector steps; + for (auto& step : _queue) { + if (!step.isProcessable()) { + steps.emplace_back(&step); + } + } + + return steps; + } + + Step pop() { + TRI_ASSERT(!isEmpty()); + // std::pop_heap will move the front element (the one we would like to steal) + // to the back of the vector, keeping the tree intact otherwise. + // Now we steal the last element. + std::pop_heap(_queue.begin(), _queue.end(), _cmpHeap); + Step first = std::move(_queue.back()); + LOG_TOPIC("9cd66", TRACE, Logger::GRAPHS) + << " Pop: " << first.toString(); + _resourceMonitor.decreaseMemoryUsage(sizeof(Step)); + _queue.pop_back(); + return first; + } + + private: + struct WeightedComparator { + bool operator()(Step const& a, Step const& b) { + if (a.getWeight() == b.getWeight()) { + // Only false if A is not processable but B is. + return !a.isProcessable() || b.isProcessable(); + } + return a.getWeight() > b.getWeight(); + } + }; + + WeightedComparator _cmpHeap{}; + + /// @brief queue datastore + /// Note: Mutable is a required for hasProcessableElement right now which is const. We can easily make it non const here. + mutable std::vector _queue; + + /// @brief query context + arangodb::ResourceMonitor& _resourceMonitor; +}; + +} // namespace graph +} // namespace arangodb + diff --git a/arangod/Graph/SingleServerEdgeCursor.cpp b/arangod/Graph/SingleServerEdgeCursor.cpp index 7772fdefe965..40b11f4f9583 100644 --- a/arangod/Graph/SingleServerEdgeCursor.cpp +++ b/arangod/Graph/SingleServerEdgeCursor.cpp @@ -127,7 +127,7 @@ bool SingleServerEdgeCursor::advanceCursor(IndexIterator*& cursor, bool SingleServerEdgeCursor::next(EdgeCursor::Callback const& callback) { // fills callback with next EdgeDocumentToken and Slice that contains the - // ohter side of the edge (we are standing on a node and want to iterate all + // other side of the edge (we are standing on a node and want to iterate all // connected edges TRI_ASSERT(!_cursors.empty()); @@ -225,6 +225,7 @@ void SingleServerEdgeCursor::readAll(EdgeCursor::Callback const& callback) { return false; } #endif + _opts->cache()->increaseCounter(); callback(EdgeDocumentToken(cid, token), edge, cursorId); return true; }); diff --git a/arangod/Graph/Steps/SingleServerProviderStep.cpp b/arangod/Graph/Steps/SingleServerProviderStep.cpp new file mode 100644 index 000000000000..ca83e669bd85 --- /dev/null +++ b/arangod/Graph/Steps/SingleServerProviderStep.cpp @@ -0,0 +1,79 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2021 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +/// @author Heiko Kernbach +//////////////////////////////////////////////////////////////////////////////// + +#include "./SingleServerProviderStep.h" + +#include "Graph/Providers/SingleServerProvider.h" + +using namespace arangodb; +using namespace arangodb::graph; + +namespace arangodb { +namespace graph { +auto operator<<(std::ostream& out, SingleServerProviderStep const& step) -> std::ostream& { + out << step._vertex.getID(); + return out; +} +} // namespace graph +} // namespace arangodb + +SingleServerProviderStep::SingleServerProviderStep(VertexType v) + : _vertex(v), _edge() {} + +SingleServerProviderStep::SingleServerProviderStep(VertexType v, size_t depth, double weight) + : BaseStep(std::numeric_limits::max(), depth, weight), _vertex(v), _edge() {} + +SingleServerProviderStep::SingleServerProviderStep(VertexType v, + EdgeDocumentToken edge, size_t prev) + : BaseStep(prev), _vertex(v), _edge(std::move(edge)) {} + +SingleServerProviderStep::SingleServerProviderStep(VertexType v, EdgeDocumentToken edge, + size_t prev, size_t depth, + double weight, size_t) + : BaseStep(prev, depth, weight), _vertex(v), _edge(std::move(edge)) {} + +SingleServerProviderStep::~SingleServerProviderStep() = default; + +VertexType const& SingleServerProviderStep::Vertex::getID() const { + return _vertex; +} + +EdgeDocumentToken const& SingleServerProviderStep::Edge::getID() const { + return _token; +} + +bool SingleServerProviderStep::Edge::isValid() const { + return getID().isValid(); +}; + +void SingleServerProviderStep::Edge::addToBuilder(SingleServerProvider& provider, + arangodb::velocypack::Builder& builder) const { + provider.insertEdgeIntoResult(getID(), builder); +} + +#ifndef USE_ENTERPRISE +bool SingleServerProviderStep::isResponsible(transaction::Methods* trx) const { + return true; +}; +#endif diff --git a/arangod/Graph/Steps/SingleServerProviderStep.h b/arangod/Graph/Steps/SingleServerProviderStep.h new file mode 100644 index 000000000000..d952e610706f --- /dev/null +++ b/arangod/Graph/Steps/SingleServerProviderStep.h @@ -0,0 +1,114 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2021 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +/// @author Heiko Kernbach +//////////////////////////////////////////////////////////////////////////////// + +#include "Transaction/Methods.h" +#include "Graph/EdgeDocumentToken.h" +#include "Graph/Providers/BaseStep.h" +#include "Graph/Providers/TypeAliases.h" + +#pragma once + +namespace arangodb { +namespace graph { + +template +class SingleServerProvider; + +class SingleServerProviderStep : public arangodb::graph::BaseStep { + public: + class Vertex { + public: + explicit Vertex(VertexType v) : _vertex(v) {} + + VertexType const& getID() const; + + bool operator<(Vertex const& other) const noexcept { + return _vertex < other._vertex; + } + + bool operator>(Vertex const& other) const noexcept { + return _vertex > other._vertex; + } + + private: + VertexType _vertex; + }; + + class Edge { + public: + explicit Edge(EdgeDocumentToken tkn) noexcept : _token(std::move(tkn)) {} + Edge() noexcept : _token() {} + + void addToBuilder(SingleServerProvider& provider, + arangodb::velocypack::Builder& builder) const; + EdgeDocumentToken const& getID() const; + bool isValid() const; + + private: + EdgeDocumentToken _token; + }; + + SingleServerProviderStep(VertexType v); + SingleServerProviderStep(VertexType v, EdgeDocumentToken edge, size_t prev); + SingleServerProviderStep(VertexType v, EdgeDocumentToken edge, size_t prev, + size_t depth, double weight, size_t); + SingleServerProviderStep(VertexType v, size_t depth, double weight = 0.0); + ~SingleServerProviderStep(); + + bool operator<(SingleServerProviderStep const& other) const noexcept { + return _vertex < other._vertex; + } + + Vertex const& getVertex() const { return _vertex; } + Edge const& getEdge() const { return _edge; } + + std::string toString() const { + return ": " + _vertex.getID().toString(); + } + bool isProcessable() const { return !isLooseEnd(); } + bool isLooseEnd() const { return false; } + + ::arangodb::graph::VertexType getVertexIdentifier() const { + return _vertex.getID(); + } + + std::string getCollectionName() const { + auto collectionNameResult = extractCollectionName(_vertex.getID()); + if (collectionNameResult.fail()) { + THROW_ARANGO_EXCEPTION(collectionNameResult.result()); + } + return collectionNameResult.get().first; + }; + + bool isResponsible(transaction::Methods* trx) const; + + friend auto operator<<(std::ostream& out, SingleServerProviderStep const& step) + -> std::ostream&; + + private: + Vertex _vertex; + Edge _edge; +}; +} // namespace graph +} // namespace arangodb diff --git a/arangod/Graph/TraverserCache.cpp b/arangod/Graph/TraverserCache.cpp index 2c719d2730b9..172eb166aba5 100644 --- a/arangod/Graph/TraverserCache.cpp +++ b/arangod/Graph/TraverserCache.cpp @@ -28,8 +28,8 @@ #include "Basics/StringHeap.h" #include "Basics/VelocyPackHelper.h" #include "Cluster/ServerState.h" -#include "Graph/EdgeDocumentToken.h" #include "Graph/BaseOptions.h" +#include "Graph/EdgeDocumentToken.h" #include "Logger/LogMacros.h" #include "Logger/Logger.h" #include "Logger/LoggerStream.h" @@ -42,8 +42,8 @@ #include "VocBase/ManagedDocumentResult.h" #include -#include #include +#include #include #include @@ -51,11 +51,11 @@ using namespace arangodb; using namespace arangodb::graph; namespace { -constexpr size_t costPerPersistedString = sizeof(void*) + sizeof(arangodb::velocypack::HashedStringRef); +constexpr size_t costPerPersistedString = + sizeof(void*) + sizeof(arangodb::velocypack::HashedStringRef); bool isWithClauseMissing(arangodb::basics::Exception const& ex) { - if (ServerState::instance()->isDBServer() && - ex.code() == TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND) { + if (ServerState::instance()->isDBServer() && ex.code() == TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND) { // on a DB server, we could have got here only in the OneShard case. // in this case turn the rather misleading "collection or view not found" // error into a nicer "collection not known to traversal, please add WITH" @@ -70,7 +70,7 @@ bool isWithClauseMissing(arangodb::basics::Exception const& ex) { return false; } -} // namespace +} // namespace TraverserCache::TraverserCache(aql::QueryContext& query, BaseOptions* opts) : _query(query), @@ -79,12 +79,11 @@ TraverserCache::TraverserCache(aql::QueryContext& query, BaseOptions* opts) _filteredDocuments(0), _stringHeap(query.resourceMonitor(), 4096), /* arbitrary block-size, may be adjusted for performance */ _baseOptions(opts), - _allowImplicitCollections(ServerState::instance()->isSingleServer() && - !_query.vocbase().server().getFeature().requireWith()) {} + _allowImplicitCollections( + ServerState::instance()->isSingleServer() && + !_query.vocbase().server().getFeature().requireWith()) {} -TraverserCache::~TraverserCache() { - clear(); -} +TraverserCache::~TraverserCache() { clear(); } void TraverserCache::clear() { _query.resourceMonitor().decreaseMemoryUsage(_persistedStrings.size() * ::costPerPersistedString); @@ -119,7 +118,8 @@ VPackSlice TraverserCache::lookupToken(EdgeDocumentToken const& idToken) { return VPackSlice(_mmdr.vpack()); } -bool TraverserCache::appendVertex(arangodb::velocypack::StringRef id, arangodb::velocypack::Builder& result) { +bool TraverserCache::appendVertex(arangodb::velocypack::StringRef id, + arangodb::velocypack::Builder& result) { if (!_baseOptions->produceVertices()) { // this traversal does not produce any vertices result.add(arangodb::velocypack::Slice::nullSlice()); @@ -141,20 +141,24 @@ bool TraverserCache::appendVertex(arangodb::velocypack::StringRef id, arangodb:: if (!map.empty()) { auto found = map.find(collectionName); if (found != map.end()) { - collectionName = found->second; + // Old API, could only convey exactly one Shard. + TRI_ASSERT(found->second.size() == 1); + collectionName = found->second.front(); } } try { - transaction::AllowImplicitCollectionsSwitcher disallower(_trx->state()->options(), _allowImplicitCollections); + transaction::AllowImplicitCollectionsSwitcher disallower(_trx->state()->options(), + _allowImplicitCollections); + + Result res = _trx->documentFastPathLocal(collectionName, id.substr(pos + 1), + [&](LocalDocumentId const&, VPackSlice doc) { + ++_insertedDocuments; + // copying... + result.add(doc); + return true; + }); - Result res = _trx->documentFastPathLocal(collectionName, id.substr(pos + 1), [&](LocalDocumentId const&, VPackSlice doc) { - ++_insertedDocuments; - // copying... - result.add(doc); - return true; - }); - if (res.ok()) { return true; } @@ -165,13 +169,13 @@ bool TraverserCache::appendVertex(arangodb::velocypack::StringRef id, arangodb:: } } catch (basics::Exception const& ex) { if (isWithClauseMissing(ex)) { - // turn the error into a different error + // turn the error into a different error THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_COLLECTION_LOCK_FAILED, - "collection not known to traversal: '" + - collectionName + "'. please add 'WITH " + collectionName + - "' as the first line in your AQL"); + "collection not known to traversal: '" + collectionName + + "'. please add 'WITH " + collectionName + + "' as the first line in your AQL"); } - // rethrow original error + // rethrow original error throw; } @@ -185,7 +189,8 @@ bool TraverserCache::appendVertex(arangodb::velocypack::StringRef id, arangodb:: return false; } -bool TraverserCache::appendVertex(arangodb::velocypack::StringRef id, arangodb::aql::AqlValue& result) { +bool TraverserCache::appendVertex(arangodb::velocypack::StringRef id, + arangodb::aql::AqlValue& result) { result = arangodb::aql::AqlValue(arangodb::aql::AqlValueHintNull()); if (!_baseOptions->produceVertices()) { @@ -207,20 +212,25 @@ bool TraverserCache::appendVertex(arangodb::velocypack::StringRef id, arangodb:: if (!map.empty()) { auto found = map.find(collectionName); if (found != map.end()) { - collectionName = found->second; + // Old API, could only convey exactly one Shard. + TRI_ASSERT(found->second.size() == 1); + collectionName = found->second.front(); } } try { - transaction::AllowImplicitCollectionsSwitcher disallower(_trx->state()->options(), _allowImplicitCollections); + transaction::AllowImplicitCollectionsSwitcher disallower(_trx->state()->options(), + _allowImplicitCollections); + + Result res = + _trx->documentFastPathLocal(collectionName, id.substr(pos + 1), + [&](LocalDocumentId const&, VPackSlice doc) { + ++_insertedDocuments; + // copying... + result = arangodb::aql::AqlValue(doc); + return true; + }); - Result res = _trx->documentFastPathLocal(collectionName, id.substr(pos + 1), [&](LocalDocumentId const&, VPackSlice doc) { - ++_insertedDocuments; - // copying... - result = arangodb::aql::AqlValue(doc); - return true; - }); - if (res.ok()) { return true; } @@ -231,11 +241,11 @@ bool TraverserCache::appendVertex(arangodb::velocypack::StringRef id, arangodb:: } } catch (basics::Exception const& ex) { if (isWithClauseMissing(ex)) { - // turn the error into a different error + // turn the error into a different error THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_COLLECTION_LOCK_FAILED, - "collection not known to traversal: '" + - collectionName + "'. please add 'WITH " + collectionName + - "' as the first line in your AQL"); + "collection not known to traversal: '" + collectionName + + "'. please add 'WITH " + collectionName + + "' as the first line in your AQL"); } // rethrow original error throw; @@ -262,7 +272,9 @@ aql::AqlValue TraverserCache::fetchEdgeAqlResult(EdgeDocumentToken const& idToke } arangodb::velocypack::StringRef TraverserCache::persistString(arangodb::velocypack::StringRef idString) { - return persistString(arangodb::velocypack::HashedStringRef(idString.data(), static_cast(idString.size()))).stringRef(); + return persistString(arangodb::velocypack::HashedStringRef( + idString.data(), static_cast(idString.size()))) + .stringRef(); } arangodb::velocypack::HashedStringRef TraverserCache::persistString(arangodb::velocypack::HashedStringRef idString) { @@ -273,9 +285,9 @@ arangodb::velocypack::HashedStringRef TraverserCache::persistString(arangodb::ve auto res = _stringHeap.registerString(idString); { ResourceUsageScope guard(_query.resourceMonitor(), ::costPerPersistedString); - + _persistedStrings.emplace(res); - + // now make the TraverserCache responsible for memory tracking guard.steal(); } diff --git a/arangod/Graph/TraverserOptions.cpp b/arangod/Graph/TraverserOptions.cpp index 02d91a7f0013..7ea4c05d0c32 100644 --- a/arangod/Graph/TraverserOptions.cpp +++ b/arangod/Graph/TraverserOptions.cpp @@ -45,25 +45,6 @@ using namespace arangodb::transaction; using namespace arangodb::traverser; using VPackHelper = arangodb::basics::VelocyPackHelper; -namespace { -arangodb::velocypack::StringRef getEdgeDestination(arangodb::velocypack::Slice edge, - arangodb::velocypack::StringRef origin) { - if (edge.isString()) { - return edge.stringRef(); - } - - TRI_ASSERT(edge.isObject()); - auto from = edge.get(arangodb::StaticStrings::FromString); - TRI_ASSERT(from.isString()); - if (from.stringRef() == origin) { - auto to = edge.get(arangodb::StaticStrings::ToString); - TRI_ASSERT(to.isString()); - return to.stringRef(); - } - return from.stringRef(); -} -} // namespace - TraverserOptions::TraverserOptions(arangodb::aql::QueryContext& query) : BaseOptions(query), _baseVertexExpression(nullptr), @@ -749,6 +730,37 @@ auto TraverserOptions::explicitDepthLookupAt() const -> std::unordered_set void { return; } + +auto TraverserOptions::isDisjoint() const -> bool { return false; } + +auto TraverserOptions::isSatelliteLeader() const -> bool { + // Can only be called in Enterprise code. + // Return false as security net. + TRI_ASSERT(false); + return false; +} +#endif + +auto TraverserOptions::getEdgeDestination(arangodb::velocypack::Slice edge, + arangodb::velocypack::StringRef origin) const + -> arangodb::velocypack::StringRef { + if (edge.isString()) { + return edge.stringRef(); + } + + TRI_ASSERT(edge.isObject()); + auto from = edge.get(arangodb::StaticStrings::FromString); + TRI_ASSERT(from.isString()); + if (from.stringRef() == origin) { + auto to = edge.get(arangodb::StaticStrings::ToString); + TRI_ASSERT(to.isString()); + return to.stringRef(); + } + return from.stringRef(); +} + bool TraverserOptions::evaluateVertexExpression(arangodb::velocypack::Slice vertex, uint64_t depth) { arangodb::aql::Expression* expression = nullptr; @@ -765,10 +777,16 @@ bool TraverserOptions::evaluateVertexExpression(arangodb::velocypack::Slice vert return evaluateExpression(expression, vertex); } +#ifndef USE_ENTERPRISE +bool TraverserOptions::checkSmartDestination(VPackSlice edge, velocypack::StringRef sourceVertex) const { + return false; +} +#endif + bool TraverserOptions::destinationCollectionAllowed(VPackSlice edge, - velocypack::StringRef sourceVertex) { + velocypack::StringRef sourceVertex) const { if (hasVertexCollectionRestrictions()) { - auto destination = ::getEdgeDestination(edge, sourceVertex); + auto destination = getEdgeDestination(edge, sourceVertex); auto collection = transaction::helpers::extractCollectionFromId(destination); if (std::find(vertexCollections.begin(), vertexCollections.end(), std::string_view(collection.data(), collection.size())) == @@ -777,6 +795,11 @@ bool TraverserOptions::destinationCollectionAllowed(VPackSlice edge, return false; } } +#ifdef USE_ENTERPRISE + if (!checkSmartDestination(edge, sourceVertex)) { + return false; + } +#endif return true; } @@ -873,7 +896,8 @@ auto TraverserOptions::estimateDepth() const noexcept -> uint64_t { void TraverserOptions::readProduceInfo(VPackSlice obj) { _produceVertices = VPackHelper::getBooleanValue(obj, "produceVertices", true); - _producePathsVertices = VPackHelper::getBooleanValue(obj, "producePathsVertices", true); + _producePathsVertices = + VPackHelper::getBooleanValue(obj, "producePathsVertices", true); _producePathsEdges = VPackHelper::getBooleanValue(obj, "producePathsEdges", true); _producePathsWeights = VPackHelper::getBooleanValue(obj, "producePathsWeights", true); } diff --git a/arangod/Graph/TraverserOptions.h b/arangod/Graph/TraverserOptions.h index 403e82d2cbd5..73d9e5ce01c3 100644 --- a/arangod/Graph/TraverserOptions.h +++ b/arangod/Graph/TraverserOptions.h @@ -107,6 +107,8 @@ struct TraverserOptions : public graph::BaseOptions { std::vector edgeCollections; + bool _isDisjoint = false; + explicit TraverserOptions(arangodb::aql::QueryContext& query); TraverserOptions(arangodb::aql::QueryContext& query, arangodb::velocypack::Slice definition); @@ -157,7 +159,9 @@ struct TraverserOptions : public graph::BaseOptions { bool evaluateVertexExpression(arangodb::velocypack::Slice, uint64_t); - bool destinationCollectionAllowed(velocypack::Slice edge, velocypack::StringRef sourceVertex); + bool checkSmartDestination(VPackSlice edge, velocypack::StringRef sourceVertex) const; + + bool destinationCollectionAllowed(velocypack::Slice edge, velocypack::StringRef sourceVertex) const; void linkTraverser(arangodb::traverser::ClusterTraverser*); @@ -212,6 +216,15 @@ struct TraverserOptions : public graph::BaseOptions { auto explicitDepthLookupAt() const -> std::unordered_set; + auto setDisjoint() -> void; + auto isDisjoint() const -> bool; + + auto isSatelliteLeader() const -> bool; + + auto getEdgeDestination(arangodb::velocypack::Slice edge, + arangodb::velocypack::StringRef origin) const + -> arangodb::velocypack::StringRef; + private: void readProduceInfo(VPackSlice obj); }; diff --git a/arangod/Graph/Types/ValidationResult.cpp b/arangod/Graph/Types/ValidationResult.cpp index ca269aaf610c..a23fe21fb18c 100644 --- a/arangod/Graph/Types/ValidationResult.cpp +++ b/arangod/Graph/Types/ValidationResult.cpp @@ -35,6 +35,8 @@ bool ValidationResult::isFiltered() const noexcept { return _type == Type::FILTER; } +void ValidationResult::combine(Type t) noexcept { _type = std::max(_type, t); } + std::ostream& arangodb::graph::operator<<(std::ostream& stream, ValidationResult const& res) { switch (res._type) { case ValidationResult::Type::TAKE: diff --git a/arangod/Graph/Types/ValidationResult.h b/arangod/Graph/Types/ValidationResult.h index 09ebef670492..41695b1ddc9c 100644 --- a/arangod/Graph/Types/ValidationResult.h +++ b/arangod/Graph/Types/ValidationResult.h @@ -39,6 +39,8 @@ class ValidationResult { bool isPruned() const noexcept; bool isFiltered() const noexcept; + void combine(Type t) noexcept; + private: Type _type; }; diff --git a/arangod/Graph/WeightedEnumerator.cpp b/arangod/Graph/WeightedEnumerator.cpp index 18f88168cdf7..2334ef562fec 100644 --- a/arangod/Graph/WeightedEnumerator.cpp +++ b/arangod/Graph/WeightedEnumerator.cpp @@ -75,6 +75,9 @@ bool WeightedEnumerator::expandEdge(NextEdge nextEdge) { // getSingleVertex does nothing but that and checking conditions // However, for global unique vertexes, we need the vertex getter. if (_traverser->getVertex(toVertex, nextEdge.depth)) { + if (!validDisjointPath(nextEdge.fromIndex, toVertex)) { + return false; + } if (_opts->uniqueVertices == TraverserOptions::UniquenessLevel::PATH) { if (pathContainsVertex(nextEdge.fromIndex, toVertex)) { // This vertex is on the path. @@ -98,12 +101,12 @@ bool WeightedEnumerator::expandEdge(NextEdge nextEdge) { void WeightedEnumerator::expandVertex(size_t vertexIndex, size_t depth) { PathStep const& currentStep = _schreier[vertexIndex]; VPackStringRef vertex = currentStep.currentVertexId; - EdgeCursor* cursor = getCursor(vertex, depth); if (depth >= _opts->maxDepth) { return; } + EdgeCursor* cursor = getCursor(vertex, depth); cursor->readAll([&](graph::EdgeDocumentToken&& eid, VPackSlice e, size_t cursorIdx) -> void { // transform edge if required if (e.isString()) { @@ -236,7 +239,8 @@ arangodb::aql::AqlValue WeightedEnumerator::edgeToAqlValue(size_t index) { return _opts->cache()->fetchEdgeAqlResult(_schreier[index].fromEdgeToken); } -VPackSlice WeightedEnumerator::pathToIndexToSlice(VPackBuilder& result, size_t index, bool fromPrune) { +VPackSlice WeightedEnumerator::pathToIndexToSlice(VPackBuilder& result, + size_t index, bool fromPrune) { for (_tempPathHelper.clear(); index != 0; index = _schreier[index].fromIndex) { _tempPathHelper.emplace_back(index); } @@ -354,3 +358,10 @@ velocypack::StringRef WeightedEnumerator::getToVertex(velocypack::Slice edge, } return resSlice.stringRef(); } + +#ifndef USE_ENTERPRISE +bool WeightedEnumerator::validDisjointPath(size_t index, + arangodb::velocypack::StringRef vertex) const { + return true; +} +#endif diff --git a/arangod/Graph/WeightedEnumerator.h b/arangod/Graph/WeightedEnumerator.h index f54c6a591c2c..434c8178be86 100644 --- a/arangod/Graph/WeightedEnumerator.h +++ b/arangod/Graph/WeightedEnumerator.h @@ -154,7 +154,7 @@ class WeightedEnumerator final : public arangodb::traverser::PathEnumerator { bool expandEdge(NextEdge edge); static velocypack::StringRef getToVertex(velocypack::Slice edge, velocypack::StringRef from); + bool validDisjointPath(size_t index, arangodb::velocypack::StringRef vertex) const; }; } // namespace graph -} // namespace arangodb - +} // namespace arangodb \ No newline at end of file diff --git a/arangod/Graph/algorithm-aliases.h b/arangod/Graph/algorithm-aliases.h index 2faa74fcdf19..113d9cb71eaa 100644 --- a/arangod/Graph/algorithm-aliases.h +++ b/arangod/Graph/algorithm-aliases.h @@ -28,7 +28,11 @@ #include "Graph/Queues/FifoQueue.h" #include "Graph/Queues/LifoQueue.h" +#include "Graph/Queues/QueueTracer.h" +#include "Graph/Queues/WeightedQueue.h" +#include "Graph/PathManagement/PathStore.h" +#include "Graph/PathManagement/PathStoreTracer.h" #include "Graph/PathManagement/PathValidator.h" #include "Graph/Providers/ProviderTracer.h" #include "Graph/Types/UniquenessLevel.h" @@ -40,54 +44,81 @@ namespace graph { template using KPathEnumerator = TwoSidedEnumerator, PathStore, Provider, - PathValidator, VertexUniquenessLevel::PATH>>; + PathValidator, VertexUniquenessLevel::PATH>>; // K_PATH implementation using Tracing template using TracedKPathEnumerator = TwoSidedEnumerator>, PathStoreTracer>, ProviderTracer, - PathValidator>, VertexUniquenessLevel::PATH>>; + PathValidator, PathStoreTracer>, VertexUniquenessLevel::PATH>>; + +template +struct BFSConfiguration { + using Provider = + typename std::conditional, ProviderType>::type; + using Step = typename Provider::Step; + using Queue = + typename std::conditional>, FifoQueue>::type; + using Store = + typename std::conditional>, PathStore>::type; + using Validator = PathValidator; +}; + +template +struct DFSConfiguration { + using Provider = + typename std::conditional, ProviderType>::type; + using Step = typename Provider::Step; + using Queue = + typename std::conditional>, LifoQueue>::type; + using Store = + typename std::conditional>, PathStore>::type; + using Validator = PathValidator; +}; + +template +struct WeightedConfiguration { + using Provider = + typename std::conditional, ProviderType>::type; + using Step = typename Provider::Step; + using Queue = + typename std::conditional>, WeightedQueue>::type; + using Store = + typename std::conditional>, PathStore>::type; + using Validator = PathValidator; +}; // BFS Traversal Enumerator implementation -template +template using BFSEnumerator = - OneSidedEnumerator, PathStore, Provider, - PathValidator, VertexUniquenessLevel::PATH>>; + OneSidedEnumerator>; // BFS Traversal Enumerator implementation using Tracing -template +template using TracedBFSEnumerator = - OneSidedEnumerator>, - PathStoreTracer>, ProviderTracer, - PathValidator>, VertexUniquenessLevel::PATH>>; - -// BFS Traversal Enumerator implementation using Tracing (without provider tracing) -template -using TracedBFSEnumeratorWOPT = - OneSidedEnumerator>, - PathStoreTracer>, Provider, - PathValidator>, VertexUniquenessLevel::PATH>>; + OneSidedEnumerator>; // DFS Traversal Enumerator implementation -template +template using DFSEnumerator = - OneSidedEnumerator, PathStore, Provider, - PathValidator, VertexUniquenessLevel::PATH>>; + OneSidedEnumerator>; // DFS Traversal Enumerator implementation using Tracing -template +template using TracedDFSEnumerator = - OneSidedEnumerator>, - PathStoreTracer>, ProviderTracer, - PathValidator>, VertexUniquenessLevel::PATH>>; + OneSidedEnumerator>; -// DFS Traversal Enumerator implementation using Tracing -template -using TracedDFSEnumeratorWOPT = - OneSidedEnumerator>, - PathStoreTracer>, Provider, - PathValidator>, VertexUniquenessLevel::PATH>>; +// Weighted Traversal Enumerator implementation +// TODO: Needs to be renamed as soon as we replace the existing variant, whic occupies this name +template +using WeightedEnumeratorRefactored = + OneSidedEnumerator>; + +// BFS Traversal Enumerator implementation using Tracing +template +using TracedWeightedEnumerator = + OneSidedEnumerator>; } // namespace graph } // namespace arangodb diff --git a/arangod/InternalRestHandler/InternalRestTraverserHandler.cpp b/arangod/InternalRestHandler/InternalRestTraverserHandler.cpp index d2c91f3f719d..e6d982dafd32 100644 --- a/arangod/InternalRestHandler/InternalRestTraverserHandler.cpp +++ b/arangod/InternalRestHandler/InternalRestTraverserHandler.cpp @@ -33,6 +33,7 @@ #include "Rest/GeneralResponse.h" #include "Transaction/StandaloneContext.h" +#include #include #include @@ -86,8 +87,9 @@ RestStatus InternalRestTraverserHandler::execute() { } void InternalRestTraverserHandler::createEngine() { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_NOT_IMPLEMENTED, - "API traversal engine creation no longer supported"); + THROW_ARANGO_EXCEPTION_MESSAGE( + TRI_ERROR_NOT_IMPLEMENTED, + "API traversal engine creation no longer supported"); } void InternalRestTraverserHandler::queryEngine() { @@ -118,7 +120,8 @@ void InternalRestTraverserHandler::queryEngine() { return; } - std::chrono::time_point start = std::chrono::steady_clock::now(); + std::chrono::time_point start = + std::chrono::steady_clock::now(); traverser::BaseEngine* engine = nullptr; while (true) { @@ -128,7 +131,8 @@ void InternalRestTraverserHandler::queryEngine() { break; } generateError(ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER, - "invalid TraverserEngine id - potentially the AQL query was already aborted or timed out"); + "invalid TraverserEngine id - potentially the AQL query " + "was already aborted or timed out"); return; } catch (basics::Exception const& ex) { // it is possible that the engine is already in use @@ -148,7 +152,7 @@ void InternalRestTraverserHandler::queryEngine() { generateError(ResponseCode::SERVER_ERROR, TRI_ERROR_LOCK_TIMEOUT); return; } - } + } TRI_ASSERT(engine != nullptr); @@ -162,8 +166,9 @@ void InternalRestTraverserHandler::queryEngine() { }); if (option == "lock") { - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_NOT_IMPLEMENTED, - "API for traversal engine locking no longer supported"); + THROW_ARANGO_EXCEPTION_MESSAGE( + TRI_ERROR_NOT_IMPLEMENTED, + "API for traversal engine locking no longer supported"); } VPackBuilder result; @@ -187,7 +192,7 @@ void InternalRestTraverserHandler::queryEngine() { // Safe cast BaseTraverserEngines are all of type TRAVERSER auto eng = static_cast(engine); TRI_ASSERT(eng != nullptr); - + VPackSlice variables = body.get("variables"); eng->injectVariables(variables); @@ -228,7 +233,8 @@ void InternalRestTraverserHandler::queryEngine() { return; } engine->getVertexData(keysSlice, result, !depthSlice.isNone()); - } else if (option == "smartSearch") { + } else if (option == "smartSearch" || option == "smartSearchBFS" || + option == "smartSearchWeighted") { if (engine->getType() != BaseEngine::EngineType::TRAVERSER) { generateError(ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER, "this engine does not support the requested operation."); @@ -238,26 +244,6 @@ void InternalRestTraverserHandler::queryEngine() { auto eng = static_cast(engine); TRI_ASSERT(eng != nullptr); eng->smartSearch(body, result); - } else if (option == "smartSearchBFS") { - if (engine->getType() != BaseEngine::EngineType::TRAVERSER) { - generateError(ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER, - "this engine does not support the requested operation."); - return; - } - // Safe cast BaseTraverserEngines are all of type TRAVERSER - auto eng = static_cast(engine); - TRI_ASSERT(eng != nullptr); - eng->smartSearchBFS(body, result); - } else if (option == "smartSearchWeighted") { - if (engine->getType() != BaseEngine::EngineType::TRAVERSER) { - generateError(ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER, - "this engine does not support the requested operation."); - return; - } - // Safe cast BaseTraverserEngines are all of type TRAVERSER - auto eng = static_cast(engine); - TRI_ASSERT(eng != nullptr); - eng->smartSearchWeighted(body, result); } else { // PATH Info wrong other error generateError(ResponseCode::NOT_FOUND, TRI_ERROR_HTTP_NOT_FOUND, ""); diff --git a/arangod/RestHandler/RestGraphHandler.cpp b/arangod/RestHandler/RestGraphHandler.cpp index dfe076b3bcb1..3affa1ac9a2e 100644 --- a/arangod/RestHandler/RestGraphHandler.cpp +++ b/arangod/RestHandler/RestGraphHandler.cpp @@ -682,9 +682,17 @@ Result RestGraphHandler::modifyEdgeDefinition(graph::Graph& graph, EdgeDefinitio OperationResult result(Result(), options); if (action == EdgeDefinitionAction::CREATE) { - result = gops.addEdgeDefinition(body, waitForSync); + VPackSlice editOptions = body.get(StaticStrings::GraphOptions); + if (!editOptions.isObject()) { + editOptions = VPackSlice::emptyObjectSlice(); + } + result = gops.addEdgeDefinition(body, editOptions, waitForSync); } else if (action == EdgeDefinitionAction::EDIT) { - result = gops.editEdgeDefinition(body, waitForSync, edgeDefinitionName); + VPackSlice editOptions = body.get(StaticStrings::GraphOptions); + if (!editOptions.isObject()) { + editOptions = VPackSlice::emptyObjectSlice(); + } + result = gops.editEdgeDefinition(body, editOptions, waitForSync, edgeDefinitionName); } else if (action == EdgeDefinitionAction::REMOVE) { // TODO Does this get waitForSync? Not according to the documentation. // if not, remove the parameter from eraseEdgeDefinition. What about diff --git a/arangod/RocksDBEngine/RocksDBEdgeIndex.cpp b/arangod/RocksDBEngine/RocksDBEdgeIndex.cpp index d492fb9d83ed..c361eedc72aa 100644 --- a/arangod/RocksDBEngine/RocksDBEdgeIndex.cpp +++ b/arangod/RocksDBEngine/RocksDBEdgeIndex.cpp @@ -112,7 +112,12 @@ class RocksDBEdgeIndexLookupIterator final : public IndexIterator { char const* typeName() const override { return "edge-index-iterator"; } - bool hasExtra() const override { return true; } + bool hasExtra() const override { + TRI_IF_FAILURE("RocksDBEdgeIndex::disableHasExtra") { + return false; + } + return true; + } /// @brief we provide a method to provide the index attribute values /// while scanning the index diff --git a/arangod/StorageEngine/EngineSelectorFeature.cpp b/arangod/StorageEngine/EngineSelectorFeature.cpp index ea0f4df8b923..77825a441b7a 100644 --- a/arangod/StorageEngine/EngineSelectorFeature.cpp +++ b/arangod/StorageEngine/EngineSelectorFeature.cpp @@ -55,10 +55,10 @@ std::unordered_map createEngineMap() { std::unordered_map map; // rocksdb is not deprecated and the engine of choice map.try_emplace(arangodb::RocksDBEngine::EngineName, - EngineInfo{ std::type_index(typeid(arangodb::RocksDBEngine)), false, true }); + EngineInfo{std::type_index(typeid(arangodb::RocksDBEngine)), false, true}); return map; } -} +} // namespace namespace arangodb { @@ -134,7 +134,8 @@ void EngineSelectorFeature::prepare() { if (selected == engines.end()) { if (_engineName == "mmfiles") { LOG_TOPIC("10eb6", FATAL, Logger::STARTUP) - << "the mmfiles storage engine is unavailable from version v3.7.0 onwards"; + << "the mmfiles storage engine is unavailable from version v3.7.0 " + "onwards"; } else { // should not happen LOG_TOPIC("3e975", FATAL, Logger::STARTUP) @@ -151,8 +152,7 @@ void EngineSelectorFeature::prepare() { "engine."; if (!ServerState::instance()->isCoordinator() && - !basics::FileUtils::isRegularFile(_engineFilePath) && - !_allowDeprecatedDeployments) { + !basics::FileUtils::isRegularFile(_engineFilePath) && !_allowDeprecatedDeployments) { LOG_TOPIC("ca0a7", FATAL, Logger::STARTUP) << "The " << _engineName << " storage engine cannot be used for new deployments."; @@ -173,10 +173,12 @@ void EngineSelectorFeature::prepare() { for (auto& engine : engines) { StorageEngine& e = server().getFeature(engine.second.type); // turn off all other storage engines - LOG_TOPIC("001b6", TRACE, Logger::STARTUP) << "disabling storage engine " << engine.first; + LOG_TOPIC("001b6", TRACE, Logger::STARTUP) + << "disabling storage engine " << engine.first; e.disable(); if (engine.first == _engineName) { - LOG_TOPIC("4a3fc", INFO, Logger::FIXME) << "using storage engine " << engine.first; + LOG_TOPIC("4a3fc", INFO, Logger::FIXME) + << "using storage engine " << engine.first; ce.setActualEngine(&e); } } @@ -239,8 +241,14 @@ void EngineSelectorFeature::unprepare() { _engine = nullptr; if (ServerState::instance()->isCoordinator()) { - ClusterEngine& ce = server().getFeature(); - ce.setActualEngine(nullptr); +#ifdef ARANGODB_USE_GOOGLE_TESTS + if (!arangodb::ClusterEngine::Mocking) { +#endif + ClusterEngine& ce = server().getFeature(); + ce.setActualEngine(nullptr); +#ifdef ARANGODB_USE_GOOGLE_TESTS + } +#endif } } diff --git a/arangod/StorageEngine/PhysicalCollection.cpp b/arangod/StorageEngine/PhysicalCollection.cpp index b1f854a3fff5..f1aace2fb1e8 100644 --- a/arangod/StorageEngine/PhysicalCollection.cpp +++ b/arangod/StorageEngine/PhysicalCollection.cpp @@ -359,7 +359,7 @@ Result PhysicalCollection::newObjectForInsert(transaction::Methods*, VPackSlice s = value.get(StaticStrings::KeyString); if (s.isNone()) { TRI_ASSERT(!isRestore); // need key in case of restore - auto keyString = _logicalCollection.keyGenerator()->generate(); + auto keyString = _logicalCollection.createKey(value); if (keyString.empty()) { return Result(TRI_ERROR_ARANGO_OUT_OF_KEYS); diff --git a/arangod/Transaction/Helpers.cpp b/arangod/Transaction/Helpers.cpp index 91e8a4d6afb2..745ec5e26a29 100644 --- a/arangod/Transaction/Helpers.cpp +++ b/arangod/Transaction/Helpers.cpp @@ -86,6 +86,15 @@ arangodb::velocypack::StringRef transaction::helpers::extractKeyPart(VPackSlice return arangodb::velocypack::StringRef(); } +/// @brief extract the _key attribute from a StringRef +arangodb::velocypack::StringRef transaction::helpers::extractKeyPart(velocypack::StringRef key) { + size_t pos = key.find('/'); + if (pos == std::string::npos) { + return key; + } + return key.substr(pos + 1); +} + /// @brief extract the _id attribute from a slice, and convert it into a /// string, static method std::string transaction::helpers::extractIdString(CollectionNameResolver const* resolver, diff --git a/arangod/Transaction/Helpers.h b/arangod/Transaction/Helpers.h index 2e9fd1435a24..9e75bfedd2c9 100644 --- a/arangod/Transaction/Helpers.h +++ b/arangod/Transaction/Helpers.h @@ -53,6 +53,9 @@ namespace helpers { /// @brief extract the _key attribute from a slice arangodb::velocypack::StringRef extractKeyPart(VPackSlice); +/// @brief extract the _key attribute from a StringRef +arangodb::velocypack::StringRef extractKeyPart(velocypack::StringRef); + std::string extractIdString(CollectionNameResolver const*, VPackSlice, VPackSlice const&); /// @brief quick access to the _key attribute in a database document diff --git a/arangod/Transaction/Methods.h b/arangod/Transaction/Methods.h index 867af033f26c..e3a1dcbbbf72 100644 --- a/arangod/Transaction/Methods.h +++ b/arangod/Transaction/Methods.h @@ -491,7 +491,7 @@ class Methods { /// @brief the transaction context std::shared_ptr _transactionContext; - + bool _mainTransaction; Future replicateOperations( diff --git a/arangod/V8Server/v8-general-graph.cpp b/arangod/V8Server/v8-general-graph.cpp index 90b65a09caa6..e0c58c8784d9 100644 --- a/arangod/V8Server/v8-general-graph.cpp +++ b/arangod/V8Server/v8-general-graph.cpp @@ -89,33 +89,6 @@ static void JS_DropGraph(v8::FunctionCallbackInfo const& args) { TRI_V8_TRY_CATCH_END } -static void JS_RenameGraphCollection(v8::FunctionCallbackInfo const& args) { - TRI_V8_TRY_CATCH_BEGIN(isolate); - v8::HandleScope scope(isolate); - - if (args.Length() < 2) { - TRI_V8_THROW_EXCEPTION_USAGE("_renameCollection(oldName, newName)"); - } else if (!args[0]->IsString()) { - TRI_V8_THROW_EXCEPTION(TRI_ERROR_GRAPH_CREATE_MISSING_NAME); - } else if (!args[1]->IsString()) { - TRI_V8_THROW_EXCEPTION(TRI_ERROR_GRAPH_CREATE_MISSING_NAME); - } - std::string oldName = TRI_ObjectToString(isolate, args[0]); - std::string newName = TRI_ObjectToString(isolate, args[1]); - if (oldName.empty() || newName.empty()) { - TRI_V8_THROW_EXCEPTION(TRI_ERROR_GRAPH_CREATE_MISSING_NAME); - } - - auto& vocbase = GetContextVocBase(isolate); - GraphManager gmngr{vocbase}; - bool r = gmngr.renameGraphCollection(oldName, newName); - - TRI_V8_RETURN(r); - - TRI_V8_RETURN_UNDEFINED(); - TRI_V8_TRY_CATCH_END -} - static void JS_GraphExists(v8::FunctionCallbackInfo const& args) { TRI_V8_TRY_CATCH_BEGIN(isolate); v8::HandleScope scope(isolate); @@ -284,8 +257,8 @@ static void JS_AddEdgeDefinitions(v8::FunctionCallbackInfo const& arg TRI_V8_TRY_CATCH_BEGIN(isolate); v8::HandleScope scope(isolate); - if (args.Length() < 2) { - TRI_V8_THROW_EXCEPTION_USAGE("_extendEdgeDefinitions(edgeDefinition)"); + if (args.Length() < 2 || args.Length() > 3) { + TRI_V8_THROW_EXCEPTION_USAGE("_extendEdgeDefinitions([, ])"); } if (!args[0]->IsString()) { TRI_V8_THROW_EXCEPTION(TRI_ERROR_GRAPH_CREATE_MISSING_NAME); @@ -298,6 +271,19 @@ static void JS_AddEdgeDefinitions(v8::FunctionCallbackInfo const& arg VPackBuilder edgeDefinition; TRI_V8ToVPack(isolate, edgeDefinition, args[1], false); + VPackBuilder options; + if (args.Length() == 3) { + // We have options + TRI_V8ToVPack(isolate, options, args[2], false); + } + if (!options.slice().isObject()) { + options.clear(); + // Fake empty options, silently ignore errors for now. + // Empty Options. + options.openObject(); + options.close(); + } + auto& vocbase = GetContextVocBase(isolate); GraphManager gmngr{vocbase}; auto graph = gmngr.lookupGraphByName(graphName); @@ -308,7 +294,7 @@ static void JS_AddEdgeDefinitions(v8::FunctionCallbackInfo const& arg auto ctx = transaction::V8Context::Create(vocbase, true); GraphOperations gops{*graph.get(), vocbase, ctx}; - OperationResult r = gops.addEdgeDefinition(edgeDefinition.slice(), false); + OperationResult r = gops.addEdgeDefinition(edgeDefinition.slice(), options.slice(), false); if (r.fail()) { TRI_V8_THROW_EXCEPTION_MESSAGE(r.errorNumber(), r.errorMessage()); @@ -334,7 +320,7 @@ static void JS_EditEdgeDefinitions(v8::FunctionCallbackInfo const& ar v8::HandleScope scope(isolate); if (args.Length() < 2) { - TRI_V8_THROW_EXCEPTION_USAGE("_editEdgeDefinitions(edgeDefinition)"); + TRI_V8_THROW_EXCEPTION_USAGE("_editEdgeDefinitions(, [])"); } if (!args[0]->IsString()) { TRI_V8_THROW_EXCEPTION(TRI_ERROR_GRAPH_CREATE_MISSING_NAME); @@ -347,6 +333,19 @@ static void JS_EditEdgeDefinitions(v8::FunctionCallbackInfo const& ar VPackBuilder edgeDefinition; TRI_V8ToVPack(isolate, edgeDefinition, args[1], false); + VPackBuilder options; + if (args.Length() == 3) { + // We have options + TRI_V8ToVPack(isolate, options, args[2], false); + } + if (!options.slice().isObject()) { + options.clear(); + // Fake empty options, silently ignore errors for now. + // Empty Options. + options.openObject(); + options.close(); + } + auto& vocbase = GetContextVocBase(isolate); GraphManager gmngr{vocbase}; auto graph = gmngr.lookupGraphByName(graphName); @@ -358,7 +357,7 @@ static void JS_EditEdgeDefinitions(v8::FunctionCallbackInfo const& ar auto ctx = transaction::V8Context::Create(vocbase, true); GraphOperations gops{*graph.get(), vocbase, ctx}; OperationResult r = - gops.editEdgeDefinition(edgeDefinition.slice(), false, + gops.editEdgeDefinition(edgeDefinition.slice(), options.slice(), false, edgeDefinition.slice().get("collection").copyString()); if (r.fail()) { @@ -489,6 +488,14 @@ static void JS_AddVertexCollection(v8::FunctionCallbackInfo const& ar VPackBuilder builder; builder.openObject(); builder.add("collection", VPackValue(vertexName)); + { + if (args.Length() >= 4) { + // We have options + builder.add(VPackValue("options")); + // Merge them into the builder, as Options entry + TRI_V8ToVPack(isolate, builder, args[3], false); + } + } builder.close(); OperationResult r = gops.addOrphanCollection(builder.slice(), false, createCollection); @@ -725,9 +732,6 @@ static void InitV8GeneralGraphModule(v8::Handle context, TRI_vocbas TRI_AddMethodVocbase(isolate, rt, TRI_V8_ASCII_STRING(isolate, "_graph"), JS_GetGraph); TRI_AddMethodVocbase(isolate, rt, TRI_V8_ASCII_STRING(isolate, "_list"), JS_GetGraphKeys); TRI_AddMethodVocbase(isolate, rt, TRI_V8_ASCII_STRING(isolate, "_listObjects"), JS_GetGraphs); - TRI_AddMethodVocbase(isolate, rt, - TRI_V8_ASCII_STRING(isolate, "_renameCollection"), - JS_RenameGraphCollection); v8g->GeneralGraphModuleTempl.Reset(isolate, rt); ft->SetClassName( diff --git a/arangod/VocBase/LogicalCollection.cpp b/arangod/VocBase/LogicalCollection.cpp index 5a2b89abb7b7..a6c750ef199b 100644 --- a/arangod/VocBase/LogicalCollection.cpp +++ b/arangod/VocBase/LogicalCollection.cpp @@ -398,10 +398,19 @@ ErrorCode LogicalCollection::getResponsibleShard(arangodb::velocypack::Slice sli } /// @briefs creates a new document key, the input slice is ignored here -std::string LogicalCollection::createKey(VPackSlice) { +std::string LogicalCollection::createKey(VPackSlice input) { + if (isSatToSmartEdgeCollection() || isSmartToSatEdgeCollection()) { + return createSmartToSatKey(input); + } return keyGenerator()->generate(); } +#ifndef USE_ENTERPRISE +std::string LogicalCollection::createSmartToSatKey(VPackSlice) { + return keyGenerator()->generate(); +} +#endif + void LogicalCollection::prepareIndexes(VPackSlice indexesSlice) { TRI_ASSERT(_physical != nullptr); @@ -610,9 +619,7 @@ Result LogicalCollection::rename(std::string&& newName) { return TRI_ERROR_NO_ERROR; } -ErrorCode LogicalCollection::close() { - return getPhysical()->close(); -} +ErrorCode LogicalCollection::close() { return getPhysical()->close(); } arangodb::Result LogicalCollection::drop() { // make sure collection has been closed @@ -1223,6 +1230,10 @@ void LogicalCollection::setInternalValidatorTypes(uint64_t type) { _internalValidatorTypes = type; } +uint64_t LogicalCollection::getInternalValidatorTypes() const { + return _internalValidatorTypes; +} + void LogicalCollection::addInternalValidator(std::unique_ptr validator) { // For the time beeing we only allow ONE internal validator. // This however is a non-necessary restriction and can be leveraged at any @@ -1252,6 +1263,14 @@ bool LogicalCollection::isSmartEdgeCollection() const noexcept { return (_internalValidatorTypes & InternalValidatorType::LogicalSmartEdge) != 0; } +bool LogicalCollection::isSatToSmartEdgeCollection() const noexcept { + return (_internalValidatorTypes & InternalValidatorType::SatToSmartEdge) != 0; +} + +bool LogicalCollection::isSmartToSatEdgeCollection() const noexcept { + return (_internalValidatorTypes & InternalValidatorType::SmartToSatEdge) != 0; +} + #ifndef USE_ENTERPRISE void LogicalCollection::decorateWithInternalEEValidators() { // Only available in Enterprise Mode diff --git a/arangod/VocBase/LogicalCollection.h b/arangod/VocBase/LogicalCollection.h index 33dbae7eea4f..5b4f9725893b 100644 --- a/arangod/VocBase/LogicalCollection.h +++ b/arangod/VocBase/LogicalCollection.h @@ -103,6 +103,8 @@ class LogicalCollection : public LogicalDataSource { LogicalSmartEdge = 1, LocalSmartEdge = 2, RemoteSmartEdge = 4, + SmartToSatEdge = 8, + SatToSmartEdge = 16, }; ////////////////////////////////////////////////////////////////////////////// @@ -349,6 +351,8 @@ class LogicalCollection : public LogicalDataSource { // (Technically no issue but will have side-effects on shards) void setInternalValidatorTypes(uint64_t type); + uint64_t getInternalValidatorTypes() const; + bool isShard() const noexcept; bool isLocalSmartEdgeCollection() const noexcept; @@ -357,6 +361,10 @@ class LogicalCollection : public LogicalDataSource { bool isSmartEdgeCollection() const noexcept; + bool isSatToSmartEdgeCollection() const noexcept; + + bool isSmartToSatEdgeCollection() const noexcept; + protected: void addInternalValidator(std::unique_ptr); @@ -365,6 +373,12 @@ class LogicalCollection : public LogicalDataSource { Result updateSchema(VPackSlice schema); + /** + * Enterprise only method. See enterprise code for implementation + * Community has a dummy stub. + */ + std::string createSmartToSatKey(arangodb::velocypack::Slice input); + private: void prepareIndexes(velocypack::Slice indexesSlice); diff --git a/arangod/VocBase/Methods/Collections.cpp b/arangod/VocBase/Methods/Collections.cpp index 0ed9d18573a7..d58f849341bf 100644 --- a/arangod/VocBase/Methods/Collections.cpp +++ b/arangod/VocBase/Methods/Collections.cpp @@ -613,7 +613,8 @@ Result Collections::properties(Context& ctxt, VPackBuilder& builder) { ignoreKeys.insert({StaticStrings::DistributeShardsLike, StaticStrings::IsSmart, StaticStrings::NumberOfShards, StaticStrings::ReplicationFactor, StaticStrings::MinReplicationFactor, - StaticStrings::ShardKeys, StaticStrings::ShardingStrategy}); + StaticStrings::ShardKeys, StaticStrings::ShardingStrategy, + StaticStrings::IsDisjoint}); // this transaction is held longer than the following if... auto trx = ctxt.trx(AccessMode::Type::READ, true, false); diff --git a/js/client/modules/@arangodb/arango-collection.js b/js/client/modules/@arangodb/arango-collection.js index 74873a1405bd..8e697adc6818 100644 --- a/js/client/modules/@arangodb/arango-collection.js +++ b/js/client/modules/@arangodb/arango-collection.js @@ -361,6 +361,7 @@ ArangoCollection.prototype.properties = function (properties) { 'cacheEnabled': true, 'syncByRevision': true, 'schema' : true, + 'isDisjoint': true, }; var a; diff --git a/js/client/modules/@arangodb/general-graph.js b/js/client/modules/@arangodb/general-graph.js index 369487e9326b..94212387fc7c 100644 --- a/js/client/modules/@arangodb/general-graph.js +++ b/js/client/modules/@arangodb/general-graph.js @@ -70,8 +70,9 @@ CommonGraph.prototype.__updateDefinitions = function (edgeDefs, orphans) { this.__orphanCollections = orphans; }; -CommonGraph.prototype._extendEdgeDefinitions = function (edgeDefinition) { - const data = edgeDefinition || {}; +CommonGraph.prototype._extendEdgeDefinitions = function (edgeDefinition, options) { + const data = _.clone(edgeDefinition) || {}; + data.options = options || {}; const uri = GRAPH_PREFIX + encodeURIComponent(this.__name) + "/edge"; const requestResult = arangosh.checkRequestResult(db._connection.POST(uri, data)); const graph = requestResult.graph; @@ -81,8 +82,9 @@ CommonGraph.prototype._extendEdgeDefinitions = function (edgeDefinition) { } }; -CommonGraph.prototype._editEdgeDefinitions = function (edgeDefinition) { - const data = edgeDefinition || {}; +CommonGraph.prototype._editEdgeDefinitions = function (edgeDefinition, options) { + const data = _.clone(edgeDefinition) || {}; + data.options = options || {}; const uri = GRAPH_PREFIX + encodeURIComponent(this.__name) + "/edge/" + edgeDefinition.collection; const requestResult = arangosh.checkRequestResult(db._connection.PUT(uri, data)); const graph = requestResult.graph; @@ -92,8 +94,8 @@ CommonGraph.prototype._editEdgeDefinitions = function (edgeDefinition) { } }; -CommonGraph.prototype._addVertexCollection = function (name, createCollection) { - const data = {}; +CommonGraph.prototype._addVertexCollection = function (name, createCollection, options = {}) { + const data = { options }; if (name) { data.collection = name; } diff --git a/js/common/modules/@arangodb/general-graph-common.js b/js/common/modules/@arangodb/general-graph-common.js index f3dd8bdba289..5d13b56b2d85 100644 --- a/js/common/modules/@arangodb/general-graph-common.js +++ b/js/common/modules/@arangodb/general-graph-common.js @@ -28,11 +28,9 @@ // ////////////////////////////////////////////////////////////////////////////// const arangodb = require('@arangodb'); -const internal = require('internal'); const ArangoCollection = arangodb.ArangoCollection; const ArangoError = arangodb.ArangoError; const db = arangodb.db; -const errors = arangodb.errors; const users = require('@arangodb/users'); const _ = require('lodash'); @@ -71,21 +69,31 @@ var isValidCollectionsParameter = function (x) { return true; }; +var checkROPermission = function(c) { + if (!users.isAuthActive()) { + return; + } + + let user = users.currentUser(); + if (user) { + let p = users.permission(user, db._name(), c); + var err = new ArangoError(); + if (p === 'none') { + err.errorNum = arangodb.errors.ERROR_FORBIDDEN.code; + err.errorMessage = arangodb.errors.ERROR_FORBIDDEN.message; + throw err; + } + } +}; + // ////////////////////////////////////////////////////////////////////////////// // / @brief find or create a collection by name // ////////////////////////////////////////////////////////////////////////////// -var findOrCreateCollectionByName = function (name, type, noCreate, options) { +var findCollectionByName = function (name, type) { let col = db._collection(name); let res = false; - if (col === null && !noCreate) { - if (type === ArangoCollection.TYPE_DOCUMENT) { - col = db._create(name, options); - } else { - col = db._createEdgeCollection(name, options); - } - res = true; - } else if (!(col instanceof ArangoCollection)) { + if (!(col instanceof ArangoCollection)) { var err = new ArangoError(); err.errorNum = arangodb.errors.ERROR_ARANGO_DATA_SOURCE_NOT_FOUND.code; err.errorMessage = name + arangodb.errors.ERROR_ARANGO_DATA_SOURCE_NOT_FOUND.message; @@ -104,7 +112,7 @@ var findOrCreateCollectionByName = function (name, type, noCreate, options) { // / @brief find or create a collection by name // ////////////////////////////////////////////////////////////////////////////// -var findOrCreateCollectionsByEdgeDefinitions = function (edgeDefinitions, noCreate, options) { +var findCollectionsByEdgeDefinitions = function (edgeDefinitions) { let vertexCollections = {}; let edgeCollections = {}; edgeDefinitions.forEach(function (e) { @@ -120,15 +128,15 @@ var findOrCreateCollectionsByEdgeDefinitions = function (edgeDefinitions, noCrea } e.from.forEach(function (v) { - findOrCreateCollectionByName(v, ArangoCollection.TYPE_DOCUMENT, noCreate, options); + findCollectionByName(v, ArangoCollection.TYPE_DOCUMENT); vertexCollections[v] = db[v]; }); e.to.forEach(function (v) { - findOrCreateCollectionByName(v, ArangoCollection.TYPE_DOCUMENT, noCreate, options); + findCollectionByName(v, ArangoCollection.TYPE_DOCUMENT); vertexCollections[v] = db[v]; }); - findOrCreateCollectionByName(e.collection, ArangoCollection.TYPE_EDGE, noCreate, options); + findCollectionByName(e.collection, ArangoCollection.TYPE_EDGE); edgeCollections[e.collection] = db[e.collection]; }); return [ @@ -317,14 +325,6 @@ var sortEdgeDefinitionInplace = function (edgeDefinition) { return edgeDefinition; }; -var sortEdgeDefinition = function (edgeDefinition) { - return { - collection: edgeDefinition.collection, - from: edgeDefinition.from.slice().sort(), - to: edgeDefinition.to.slice().sort(), - }; -}; - // ////////////////////////////////////////////////////////////////////////////// // / @brief create a new graph // ////////////////////////////////////////////////////////////////////////////// @@ -577,166 +577,6 @@ var updateBindCollections = function (graph) { bindVertexCollections(graph, graph.__orphanCollections); }; -var checkRWPermission = function (c) { - if (!users.isAuthActive()) { - return; - } - - let user = users.currentUser(); - if (user) { - let p = users.permission(user, db._name(), c); - if (p !== 'rw') { - var err = new ArangoError(); - err.errorNum = arangodb.errors.ERROR_FORBIDDEN.code; - err.errorMessage = arangodb.errors.ERROR_FORBIDDEN.message; - throw err; - } - } -}; - -var checkROPermission = function(c) { - if (!users.isAuthActive()) { - return; - } - - let user = users.currentUser(); - if (user) { - let p = users.permission(user, db._name(), c); - var err = new ArangoError(); - if (p === 'none') { - err.errorNum = arangodb.errors.ERROR_FORBIDDEN.code; - err.errorMessage = arangodb.errors.ERROR_FORBIDDEN.message; - throw err; - } - } -}; - -// ////////////////////////////////////////////////////////////////////////////// -// / @brief internal function for editing edge definitions -// ////////////////////////////////////////////////////////////////////////////// - -var changeEdgeDefinitionsForGraph = function (graph, edgeDefinition, newCollections, possibleOrphans, self) { - var graphCollections = []; - var graphObj = exports._graph(graph._key); - var eDs = graph.edgeDefinitions; - var gotAHit = false; - - // replace edgeDefintion - eDs.forEach( - function (eD, id) { - if (eD.collection === edgeDefinition.collection) { - gotAHit = true; - eDs[id].from = edgeDefinition.from; - eDs[id].to = edgeDefinition.to; - db._graphs.update(graph._key, {edgeDefinitions: eDs}); - if (graph._key === self.__name) { - self.__edgeDefinitions[id].from = edgeDefinition.from; - self.__edgeDefinitions[id].to = edgeDefinition.to; - } - } else { - // collect all used collections - graphCollections = _.union(graphCollections, eD.from); - graphCollections = _.union(graphCollections, eD.to); - } - } - ); - if (!gotAHit) { - return; - } - - // remove used collection from orphanage - if (graph._key === self.__name) { - newCollections.forEach( - function (nc) { - if (self.__vertexCollections[nc] === undefined) { - self.__vertexCollections[nc] = db[nc]; - } - try { - self._removeVertexCollection(nc, false); - } catch (ignore) {} - } - ); - possibleOrphans.forEach( - function (po) { - if (graphCollections.indexOf(po) === -1) { - delete self.__vertexCollections[po]; - self._addVertexCollection(po); - } - } - ); - } else { - newCollections.forEach( - function (nc) { - try { - graphObj._removeVertexCollection(nc, false); - } catch (ignore) {} - } - ); - possibleOrphans.forEach( - function (po) { - if (graphCollections.indexOf(po) === -1) { - delete graphObj.__vertexCollections[po]; - graphObj._addVertexCollection(po); - } - } - ); - } - -// move unused collections to orphanage -}; - -// ////////////////////////////////////////////////////////////////////////////// -// / @brief Helper for dropping collections of a graph. -// ////////////////////////////////////////////////////////////////////////////// - -var checkIfMayBeDropped = function (colName, graphName, graphs) { - var result = true; - - graphs.forEach( - function (graph) { - if (result === false) { - // Short circuit - return; - } - if (graph._key === graphName) { - return; - } - var edgeDefinitions = graph.edgeDefinitions; - if (edgeDefinitions) { - edgeDefinitions.forEach( - function (edgeDefinition) { - var from = edgeDefinition.from; - var to = edgeDefinition.to; - var collection = edgeDefinition.collection; - if (collection === colName || - from.indexOf(colName) !== -1 || - to.indexOf(colName) !== -1 - ) { - result = false; - } - } - ); - } - - var orphanCollections = graph.orphanCollections; - if (orphanCollections) { - if (orphanCollections.indexOf(colName) !== -1) { - result = false; - } - } - } - ); - - return result; -}; - -const edgeDefinitionsEqual = function (leftEdgeDef, rightEdgeDef) { - leftEdgeDef = sortEdgeDefinition(leftEdgeDef); - rightEdgeDef = sortEdgeDefinition(rightEdgeDef); - const stringify = obj => JSON.stringify(obj, Object.keys(obj).sort()); - return stringify(leftEdgeDef) === stringify(rightEdgeDef); -}; - // @brief Class Graph. Defines a graph in the Database. class Graph { constructor (info) { @@ -747,14 +587,14 @@ class Graph { info.edgeDefinitions.forEach((e) => { // Link the edge collection - edgeCollections[e.collection] = db[e.collection]; + edgeCollections[e.collection] = db._collection(e.collection); // Link from collections e.from.forEach((v) => { - vertexCollections[v] = db[v]; + vertexCollections[v] = db._collection(v); }); // Link to collections e.to.forEach((v) => { - vertexCollections[v] = db[v]; + vertexCollections[v] = db._collection(v); }); }); @@ -1672,10 +1512,6 @@ exports._relation = function (relationName, fromVertexCollections, toVertexColle }; }; -// ////////////////////////////////////////////////////////////////////////////// -// / @brief was docuBlock JSF_general_graph_graph -// ////////////////////////////////////////////////////////////////////////////// - exports._graph = function (graphName) { let gdb = getGraphCollection(); let g; @@ -1683,7 +1519,7 @@ exports._graph = function (graphName) { try { g = gdb.document(graphName); } catch (e) { - if (e.errorNum !== errors.ERROR_ARANGO_DOCUMENT_NOT_FOUND.code) { + if (e.errorNum !== arangodb.errors.ERROR_ARANGO_DOCUMENT_NOT_FOUND.code) { throw e; } let err = new ArangoError(); @@ -1704,7 +1540,7 @@ exports._graph = function (graphName) { throw err; } - findOrCreateCollectionsByEdgeDefinitions(g.edgeDefinitions, true); + findCollectionsByEdgeDefinitions(g.edgeDefinitions); return new Graph(g); }; @@ -1741,181 +1577,6 @@ exports._extendEdgeDefinitions = function (edgeDefinition) { ); }; -// ////////////////////////////////////////////////////////////////////////////// -// / @brief was docuBlock JSF_general_graph_create -// ////////////////////////////////////////////////////////////////////////////// - -exports._create = function (graphName, edgeDefinitions, orphanCollections, options) { - if (!Array.isArray(orphanCollections)) { - orphanCollections = []; - } - options = options || {}; - let gdb = getGraphCollection(); - let graphAlreadyExists = true; - if (!graphName) { - let err = new ArangoError(); - err.errorNum = arangodb.errors.ERROR_GRAPH_CREATE_MISSING_NAME.code; - err.errorMessage = arangodb.errors.ERROR_GRAPH_CREATE_MISSING_NAME.message; - throw err; - } - edgeDefinitions = edgeDefinitions || []; - if (!Array.isArray(edgeDefinitions)) { - let err = new ArangoError(); - err.errorNum = arangodb.errors.ERROR_GRAPH_CREATE_MALFORMED_EDGE_DEFINITION.code; - err.errorMessage = arangodb.errors.ERROR_GRAPH_CREATE_MALFORMED_EDGE_DEFINITION.message; - throw err; - } - - edgeDefinitions = _.cloneDeep(edgeDefinitions); - - // check, if a collection is already used in a different edgeDefinition - let tmpCollections = []; - let tmpEdgeDefinitions = {}; - edgeDefinitions.forEach( - (edgeDefinition) => { - let col = edgeDefinition.collection; - if (tmpCollections.indexOf(col) !== -1) { - let err = new ArangoError(); - err.errorNum = arangodb.errors.ERROR_GRAPH_COLLECTION_MULTI_USE.code; - err.errorMessage = arangodb.errors.ERROR_GRAPH_COLLECTION_MULTI_USE.message; - throw err; - } - tmpCollections.push(col); - tmpEdgeDefinitions[col] = edgeDefinition; - } - ); - gdb.toArray().forEach( - (singleGraph) => { - var sGEDs = singleGraph.edgeDefinitions; - if (!Array.isArray(sGEDs)) { - return; - } - sGEDs.forEach( - (sGED) => { - var col = sGED.collection; - if (tmpCollections.indexOf(col) !== -1) { - if (!edgeDefinitionsEqual(sGED, tmpEdgeDefinitions[col])) { - let err = new ArangoError(); - err.errorNum = arangodb.errors.ERROR_GRAPH_COLLECTION_USE_IN_MULTI_GRAPHS.code; - err.errorMessage = col + ' ' + - arangodb.errors.ERROR_GRAPH_COLLECTION_USE_IN_MULTI_GRAPHS.message; - throw err; - } - } - } - ); - } - ); - - try { - gdb.document(graphName); - } catch (e) { - if (e.errorNum !== errors.ERROR_ARANGO_DOCUMENT_NOT_FOUND.code) { - throw e; - } - graphAlreadyExists = false; - } - - if (graphAlreadyExists) { - let err = new ArangoError(); - err.errorNum = arangodb.errors.ERROR_GRAPH_DUPLICATE.code; - err.errorMessage = arangodb.errors.ERROR_GRAPH_DUPLICATE.message; - throw err; - } - - db._flushCache(); - let collections = findOrCreateCollectionsByEdgeDefinitions(edgeDefinitions, false, options); - orphanCollections.forEach( - (oC) => { - findOrCreateCollectionByName(oC, ArangoCollection.TYPE_DOCUMENT, false, options); - } - ); - - edgeDefinitions.forEach(sortEdgeDefinitionInplace); - orphanCollections = orphanCollections.sort(); - - var data = gdb.save({ - 'orphanCollections': orphanCollections, - 'edgeDefinitions': edgeDefinitions, - '_key': graphName, - 'numberOfShards': options.numberOfShards || undefined, - 'replicationFactor': options.replicationFactor || undefined, - }, options); - data.orphanCollections = orphanCollections; - data.edgeDefinitions = edgeDefinitions; - return new Graph(data); -}; - -// ////////////////////////////////////////////////////////////////////////////// -// / @brief was docuBlock JSF_general_graph_drop -// ////////////////////////////////////////////////////////////////////////////// - -exports._drop = function (graphId, dropCollections) { - let gdb = getGraphCollection(); - let graphs; - - if (!gdb.exists(graphId)) { - let err = new ArangoError(); - err.errorNum = arangodb.errors.ERROR_GRAPH_NOT_FOUND.code; - err.errorMessage = arangodb.errors.ERROR_GRAPH_NOT_FOUND.message; - throw err; - } - - checkRWPermission("_graphs"); - var graph = gdb.document(graphId); - if (dropCollections === true) { - graphs = exports._listObjects(); - // Here we collect all collections - // that are leading for distribution - let leaderCollections = new Set(); - let followerCollections = new Set(); - let dropColCB = (name) => { - if (checkIfMayBeDropped(name, graph._key, graphs)) { - checkRWPermission(name); - let colObj = db[name]; - if (colObj !== undefined) { - // If it is undefined the collection is gone already - if (colObj.properties().distributeShardsLike === undefined) { - leaderCollections.add(name); - } else { - followerCollections.add(name); - } - } - } - }; - // drop orphans - if (!graph.orphanCollections) { - graph.orphanCollections = []; - } - graph.orphanCollections.forEach(dropColCB); - var edgeDefinitions = graph.edgeDefinitions; - edgeDefinitions.forEach( - function (edgeDefinition) { - var from = edgeDefinition.from; - var to = edgeDefinition.to; - var collection = edgeDefinition.collection; - dropColCB(edgeDefinition.collection); - from.forEach(dropColCB); - to.forEach(dropColCB); - } - ); - let dropColl = (c) => { - try { - db._drop(c); - } catch (e) { - internal.print("Failed to Drop: '" + c + "' reason: " + e.message); - } - }; - followerCollections.forEach(dropColl); - leaderCollections.forEach(dropColl); - } - - gdb.remove(graphId); - db._flushCache(); - - return true; -}; - // ////////////////////////////////////////////////////////////////////////////// // / @brief check if a graph exists. // ////////////////////////////////////////////////////////////////////////////// @@ -1925,74 +1586,10 @@ exports._exists = function (graphId) { return gCol.exists(graphId); }; -// ////////////////////////////////////////////////////////////////////////////// -// / @brief rename a collection inside the _graphs collections -// ////////////////////////////////////////////////////////////////////////////// - -exports._renameCollection = function (oldName, newName) { - db._executeTransaction({ - collections: { - write: '_graphs' - }, - action: function (params) { - var gdb = getGraphCollection(); - if (!gdb) { - return; - } - gdb.toArray().forEach(function (doc) { - if (doc.isSmart) { - return; - } - let c = Object.assign({}, doc); - let changed = false; - if (c.edgeDefinitions) { - for (let i = 0; i < c.edgeDefinitions.length; ++i) { - var def = c.edgeDefinitions[i]; - if (def.collection === params.oldName) { - c.edgeDefinitions[i].collection = params.newName; - changed = true; - } - for (let j = 0; j < def.from.length; ++j) { - if (def.from[j] === params.oldName) { - c.edgeDefinitions[i].from[j] = params.newName; - changed = true; - } - } - for (let j = 0; j < def.to.length; ++j) { - if (def.to[j] === params.oldName) { - c.edgeDefinitions[i].to[j] = params.newName; - changed = true; - } - } - } - } - for (let i = 0; i < c.orphanCollections.length; ++i) { - if (c.orphanCollections[i] === params.oldName) { - c.orphanCollections[i] = params.newName; - changed = true; - } - } - - if (changed) { - gdb.update(doc._key, c); - } - }); - }, - params: { - oldName: oldName, - newName: newName - } - }); -}; - // ////////////////////////////////////////////////////////////////////////////// // / @brief was docuBlock JSF_general_graph_list // ////////////////////////////////////////////////////////////////////////////// -exports._list = function () { - return db._query(`FOR x IN _graphs RETURN x._key`).toArray(); -}; - exports._listObjects = function () { return db._query(`FOR x IN _graphs RETURN x`).toArray(); }; diff --git a/js/server/modules/@arangodb/general-graph.js b/js/server/modules/@arangodb/general-graph.js index 15cda4302a9e..debcf1a4e739 100644 --- a/js/server/modules/@arangodb/general-graph.js +++ b/js/server/modules/@arangodb/general-graph.js @@ -50,18 +50,18 @@ CommonGraph.prototype._deleteEdgeDefinition = function (edgeDefinition, dropColl this.__updateDefinitions(result.graph.edgeDefinitions, result.graph.orphanCollections); }; -CommonGraph.prototype._extendEdgeDefinitions = function (edgeDefinitions) { - let result = ArangoGraph._extendEdgeDefinitions(this.__name, edgeDefinitions); +CommonGraph.prototype._extendEdgeDefinitions = function (edgeDefinitions, options = {}) { + let result = ArangoGraph._extendEdgeDefinitions(this.__name, edgeDefinitions, options); this.__updateDefinitions(result.graph.edgeDefinitions, result.graph.orphanCollections); }; -CommonGraph.prototype._editEdgeDefinitions = function (edgeDefinitions) { - let result = ArangoGraph._editEdgeDefinitions(this.__name, edgeDefinitions); +CommonGraph.prototype._editEdgeDefinitions = function (edgeDefinitions, options = {}) { + let result = ArangoGraph._editEdgeDefinitions(this.__name, edgeDefinitions, options); this.__updateDefinitions(result.graph.edgeDefinitions, result.graph.orphanCollections); }; -CommonGraph.prototype._addVertexCollection = function (vertexName, createCollection = true) { - let result = ArangoGraph._addVertexCollection(this.__name, vertexName, createCollection); +CommonGraph.prototype._addVertexCollection = function (vertexName, createCollection = true, options = {}) { + let result = ArangoGraph._addVertexCollection(this.__name, vertexName, createCollection, options); this.__updateDefinitions(result.graph.edgeDefinitions, result.graph.orphanCollections); }; diff --git a/js/server/modules/@arangodb/testutils/aql-graph-traversal-generic-tests.js b/js/server/modules/@arangodb/testutils/aql-graph-traversal-generic-tests.js index ec4b88e58c43..bec37a370a03 100644 --- a/js/server/modules/@arangodb/testutils/aql-graph-traversal-generic-tests.js +++ b/js/server/modules/@arangodb/testutils/aql-graph-traversal-generic-tests.js @@ -2567,7 +2567,7 @@ function testCompleteGraphDfsUniqueEdgesPathD2(testGraph) { checkResIsValidDfsOf(expectedPathsAsTree, actualPaths); } -function testCompleteGraphDfsUniqueVerticesPathD3(testGraph) { +function completeGraphDfsUniqueVerticesPathD3Helper(testGraph) { assertTrue(testGraph.name().startsWith(protoGraphs.completeGraph.name())); const query = aql` FOR v, e, p IN 0..3 OUTBOUND ${testGraph.vertex('A')} GRAPH ${testGraph.name()} OPTIONS {uniqueVertices: "path"} @@ -2641,6 +2641,19 @@ function testCompleteGraphDfsUniqueVerticesPathD3(testGraph) { checkResIsValidDfsOf(expectedPathsAsTree, actualPaths); } +function testCompleteGraphDfsUniqueVerticesPathD3(testGraph) { + completeGraphDfsUniqueVerticesPathD3Helper(testGraph); +} + +function testCompleteGraphDfsUniqueVerticesPathD3NotHasExtra(testGraph) { + internal.debugSetFailAt("RocksDBEdgeIndex::disableHasExtra"); + try { + completeGraphDfsUniqueVerticesPathD3Helper(testGraph); + } finally { + internal.debugRemoveFailAt("RocksDBEdgeIndex::disableHasExtra"); + } +} + function testCompleteGraphDfsUniqueVerticesUniqueEdgesPathD2(testGraph) { assertTrue(testGraph.name().startsWith(protoGraphs.completeGraph.name())); const query = aql` @@ -5906,6 +5919,7 @@ const testsByGraph = { testCompleteGraphDfsUniqueVerticesPathD1, testCompleteGraphDfsUniqueVerticesPathD2, testCompleteGraphDfsUniqueVerticesPathD3, + testCompleteGraphDfsUniqueVerticesPathD3NotHasExtra, testCompleteGraphDfsUniqueEdgesPathD1, testCompleteGraphDfsUniqueEdgesPathD2, testCompleteGraphDfsUniqueVerticesUniqueEdgesPathD2, diff --git a/lib/Basics/StaticStrings.cpp b/lib/Basics/StaticStrings.cpp index eaaa75bcd251..d6e56e3ee510 100644 --- a/lib/Basics/StaticStrings.cpp +++ b/lib/Basics/StaticStrings.cpp @@ -43,6 +43,8 @@ std::string const StaticStrings::IndexGt("gt"); std::string const StaticStrings::AttachmentString("_attachment"); std::string const StaticStrings::IdString("_id"); std::string const StaticStrings::KeyString("_key"); +std::string const StaticStrings::PrefixOfKeyString("_key:"); +std::string const StaticStrings::PostfixOfKeyString(":_key"); std::string const StaticStrings::RevString("_rev"); std::string const StaticStrings::FromString("_from"); std::string const StaticStrings::ToString("_to"); @@ -266,14 +268,17 @@ std::string const StaticStrings::ReplicatedLogs("replicatedLogs"); // graph attribute names std::string const StaticStrings::GraphCollection("_graphs"); std::string const StaticStrings::IsDisjoint("isDisjoint"); +std::string const StaticStrings::IsHybrid("isHybrid"); std::string const StaticStrings::GraphIsSmart("isSmart"); std::string const StaticStrings::GraphIsSatellite("isSatellite"); +std::string const StaticStrings::GraphSatellites("satellites"); std::string const StaticStrings::GraphFrom("from"); std::string const StaticStrings::GraphTo("to"); std::string const StaticStrings::GraphOptions("options"); std::string const StaticStrings::GraphSmartGraphAttribute( "smartGraphAttribute"); std::string const StaticStrings::GraphEdgeDefinitions("edgeDefinitions"); +std::string const StaticStrings::GraphEdgeDefinitionType("type"); std::string const StaticStrings::GraphOrphans("orphanCollections"); std::string const StaticStrings::GraphInitial("initial"); std::string const StaticStrings::GraphInitialCid("initialCid"); @@ -322,7 +327,6 @@ std::string const StaticStrings::GraphQueryShortestPathType("shortestPathType"); // rest query parameter std::string const StaticStrings::GraphDropCollections("dropCollections"); std::string const StaticStrings::GraphDropCollection("dropCollection"); -std::string const StaticStrings::GraphCreateCollections("createCollections"); std::string const StaticStrings::GraphCreateCollection("createCollection"); // Replication diff --git a/lib/Basics/StaticStrings.h b/lib/Basics/StaticStrings.h index b2c453e4c244..bdd689281e07 100644 --- a/lib/Basics/StaticStrings.h +++ b/lib/Basics/StaticStrings.h @@ -48,6 +48,8 @@ class StaticStrings { static std::string const AttachmentString; static std::string const IdString; static std::string const KeyString; + static std::string const PrefixOfKeyString; + static std::string const PostfixOfKeyString; static std::string const RevString; static std::string const FromString; static std::string const ToString; @@ -246,7 +248,9 @@ class StaticStrings { // graph attribute names static std::string const GraphCollection; static std::string const IsDisjoint; + static std::string const IsHybrid; static std::string const GraphIsSatellite; + static std::string const GraphSatellites; static std::string const GraphIsSmart; static std::string const GraphFrom; static std::string const GraphTo; @@ -254,9 +258,9 @@ class StaticStrings { static std::string const GraphSmartGraphAttribute; static std::string const GraphDropCollections; static std::string const GraphDropCollection; - static std::string const GraphCreateCollections; static std::string const GraphCreateCollection; static std::string const GraphEdgeDefinitions; + static std::string const GraphEdgeDefinitionType; static std::string const GraphOrphans; static std::string const GraphInitial; static std::string const GraphInitialCid; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 64441a092482..28a641ab5fb9 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -62,6 +62,8 @@ set(ARANGODB_TESTS_COMMON_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Mocks/PreparedResponseConnectionPool.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Mocks/StorageEngineMock.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Mocks/Servers.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Mocks/MockGraph.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Mocks/MockGraphProvider.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Basics/icu-helper.cpp ${CMAKE_CURRENT_SOURCE_DIR}/IResearch/AgencyMock.cpp ${CMAKE_CURRENT_SOURCE_DIR}/IResearch/common.cpp @@ -237,12 +239,11 @@ set(ARANGODB_TESTS_SOURCES Graph/KPathFinderTest.cpp Graph/DFSFinderTest.cpp Graph/KShortestPathsFinder.cpp - Graph/MockGraph.cpp - Graph/MockGraphProvider.cpp Graph/PathStoreTest.cpp Graph/PathValidatorTest.cpp Graph/FifoQueueTest.cpp Graph/LifoQueueTest.cpp + Graph/WeightedQueueTest.cpp Graph/SingleServerProviderTest.cpp Greenspun/PrimitivesTest.cpp HotBackup/HotBackupCoordinatorTest.cpp @@ -378,7 +379,8 @@ endif () target_include_directories(arangodbtests PRIVATE ${INCLUDE_DIRECTORIES} - ) + ${CMAKE_CURRENT_SOURCE_DIR}/Mocks/ +) if (USE_SEPARATE_REPLICATION2_TESTS_BINARY) target_include_directories(arangodbtests_replication2 PRIVATE diff --git a/tests/Graph/DFSFinderTest.cpp b/tests/Graph/DFSFinderTest.cpp index fd1148652c4d..1f23ca167c78 100644 --- a/tests/Graph/DFSFinderTest.cpp +++ b/tests/Graph/DFSFinderTest.cpp @@ -57,8 +57,8 @@ namespace tests { namespace graph { class DFSFinderTest : public ::testing::TestWithParam { - // using DFSFinder = DFSEnumerator; - using DFSFinder = TracedDFSEnumeratorWOPT; + // using DFSFinder = DFSEnumerator; + using DFSFinder = TracedDFSEnumerator; protected: bool activateLogging{false}; @@ -68,11 +68,23 @@ class DFSFinderTest : public ::testing::TestWithParamnewTrxContext()}; + aql::Variable _tmpVar{"tmp", 0, false}; + arangodb::aql::AqlFunctionsInternalCache _functionsCache{}; + arangodb::aql::FixedVarExpressionContext _expressionContext{_trx, *_query.get(), + _functionsCache}; + DFSFinderTest() { if (activateLogging) { Logger::GRAPHS.setLogLevel(LogLevel::TRACE); } + // Important Note: + // Tests are using a LifoQueue. In those tests we do guarantee fetching in order + // e.g. (1) expands to (2), (3), (4) + // we will first traverse (4), then (3), then (2) + /* a chain 1->2->3->4 */ mockGraph.addEdge(1, 2); mockGraph.addEdge(2, 3); @@ -158,8 +170,11 @@ class DFSFinderTest : public ::testing::TestWithParam DFSFinder { arangodb::graph::OneSidedEnumeratorOptions options{minDepth, maxDepth}; - return DFSFinder{MockGraphProvider(mockGraph, *_query.get(), looseEndBehaviour(), false), - std::move(options), resourceMonitor}; + PathValidatorOptions validatorOpts{&_tmpVar, _expressionContext}; + return DFSFinder({*_query.get(), + MockGraphProviderOptions{mockGraph, looseEndBehaviour(), false}, + resourceMonitor}, + std::move(options), std::move(validatorOpts), resourceMonitor); } auto vId(size_t nr) -> std::string { @@ -248,8 +263,9 @@ TEST_P(DFSFinderTest, no_path_exists) { EXPECT_FALSE(finder.isDone()); { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); pathEquals(result.slice(), {91}); pathStructureValid(result.slice(), 0); EXPECT_TRUE(finder.isDone()); @@ -258,9 +274,8 @@ TEST_P(DFSFinderTest, no_path_exists) { { result.clear(); // Try again to make sure we stay at non-existing - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); - EXPECT_TRUE(result.isEmpty()); EXPECT_TRUE(finder.isDone()); } { @@ -282,8 +297,10 @@ TEST_P(DFSFinderTest, path_depth_0) { EXPECT_FALSE(finder.isDone()); { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathEquals(result.slice(), {1}); pathStructureValid(result.slice(), 0); EXPECT_TRUE(finder.isDone()); @@ -292,9 +309,8 @@ TEST_P(DFSFinderTest, path_depth_0) { { result.clear(); // Try again to make sure we stay at non-existing - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); - EXPECT_TRUE(result.isEmpty()); EXPECT_TRUE(finder.isDone()); } { @@ -315,8 +331,9 @@ TEST_P(DFSFinderTest, path_depth_1) { EXPECT_FALSE(finder.isDone()); { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); pathStructureValid(result.slice(), 1); pathEquals(result.slice(), {1, 2}); @@ -327,9 +344,8 @@ TEST_P(DFSFinderTest, path_depth_1) { { result.clear(); // Try again to make sure we stay at non-existing - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); - EXPECT_TRUE(result.isEmpty()); EXPECT_TRUE(finder.isDone()); } @@ -351,8 +367,10 @@ TEST_P(DFSFinderTest, path_depth_2) { EXPECT_FALSE(finder.isDone()); { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 2); pathEquals(result.slice(), {1, 2, 3}); @@ -360,11 +378,9 @@ TEST_P(DFSFinderTest, path_depth_2) { } { - result.clear(); // Try again to make sure we stay at non-existing - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); - EXPECT_TRUE(result.isEmpty()); EXPECT_TRUE(finder.isDone()); } { @@ -385,8 +401,10 @@ TEST_P(DFSFinderTest, path_depth_3) { EXPECT_FALSE(finder.isDone()); { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 3); pathEquals(result.slice(), {1, 2, 3, 4}); @@ -394,11 +412,9 @@ TEST_P(DFSFinderTest, path_depth_3) { } { - result.clear(); // Try again to make sure we stay at non-existing - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); - EXPECT_TRUE(result.isEmpty()); EXPECT_TRUE(finder.isDone()); } @@ -420,35 +436,39 @@ TEST_P(DFSFinderTest, path_diamond) { EXPECT_FALSE(finder.isDone()); { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 2); EXPECT_FALSE(finder.isDone()); } { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 2); EXPECT_FALSE(finder.isDone()); } { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 2); EXPECT_TRUE(finder.isDone()); } { - result.clear(); // Try again to make sure we stay at non-existing - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); - EXPECT_TRUE(result.isEmpty()); EXPECT_TRUE(finder.isDone()); } { @@ -469,8 +489,10 @@ TEST_P(DFSFinderTest, path_depth_1_to_2) { EXPECT_FALSE(finder.isDone()); { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 1); pathEquals(result.slice(), {10, 12}); @@ -479,8 +501,10 @@ TEST_P(DFSFinderTest, path_depth_1_to_2) { { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 2); pathEquals(result.slice(), {10, 12, 13}); EXPECT_FALSE(finder.isDone()); @@ -488,8 +512,10 @@ TEST_P(DFSFinderTest, path_depth_1_to_2) { { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 2); pathEquals(result.slice(), {10, 12, 11}); EXPECT_FALSE(finder.isDone()); @@ -497,19 +523,19 @@ TEST_P(DFSFinderTest, path_depth_1_to_2) { { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 1); pathEquals(result.slice(), {10, 11}); EXPECT_TRUE(finder.isDone()); } { - result.clear(); // Try again to make sure we stay at non-existing - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); - EXPECT_TRUE(result.isEmpty()); EXPECT_TRUE(finder.isDone()); } } @@ -524,8 +550,10 @@ TEST_P(DFSFinderTest, path_depth_1_to_2_skip) { EXPECT_FALSE(finder.isDone()); { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 1); pathEquals(result.slice(), {10, 12}); @@ -541,8 +569,10 @@ TEST_P(DFSFinderTest, path_depth_1_to_2_skip) { { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 2); pathEquals(result.slice(), {10, 12, 11}); EXPECT_FALSE(finder.isDone()); @@ -550,17 +580,183 @@ TEST_P(DFSFinderTest, path_depth_1_to_2_skip) { { result.clear(); - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + pathStructureValid(result.slice(), 1); pathEquals(result.slice(), {10, 11}); EXPECT_TRUE(finder.isDone()); } + { + // Try again to make sure we stay at non-existing + auto hasPath = finder.getNextPath(); + EXPECT_FALSE(hasPath); + EXPECT_TRUE(finder.isDone()); + } +} + +TEST_P(DFSFinderTest, path_loop) { + VPackBuilder result; + auto finder = pathFinder(1, 10); + + // Source and target are direct neighbors, there is only one path between them + auto source = vId(20); + + finder.reset(toHashedStringRef(source)); + + EXPECT_FALSE(finder.isDone()); + { + result.clear(); + auto hasPath = finder.getNextPath(); + EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + + pathStructureValid(result.slice(), 1); + pathEquals(result.slice(), {20, 21}); + + EXPECT_FALSE(finder.isDone()); + } + + { + result.clear(); + auto hasPath = finder.getNextPath(); + EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + + pathStructureValid(result.slice(), 2); + pathEquals(result.slice(), {20, 21, 22}); + + EXPECT_FALSE(finder.isDone()); + } + + { + auto hasPath = finder.getNextPath(); + EXPECT_FALSE(hasPath); + EXPECT_TRUE(finder.isDone()); + } +} + +TEST_P(DFSFinderTest, triangle_loop) { + VPackBuilder result; + auto finder = pathFinder(1, 10); + auto source = vId(30); + + finder.reset(toHashedStringRef(source)); + + EXPECT_FALSE(finder.isDone()); + { + result.clear(); + auto hasPath = finder.getNextPath(); + EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + + pathStructureValid(result.slice(), 1); + pathEquals(result.slice(), {30, 31}); + + EXPECT_FALSE(finder.isDone()); + } + + { + result.clear(); + auto hasPath = finder.getNextPath(); + EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + + pathStructureValid(result.slice(), 2); + pathEquals(result.slice(), {30, 31, 32}); + + EXPECT_FALSE(finder.isDone()); + } + + { + result.clear(); + auto hasPath = finder.getNextPath(); + EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + + pathStructureValid(result.slice(), 3); + pathEquals(result.slice(), {30, 31, 32, 34}); + + EXPECT_FALSE(finder.isDone()); + } + + { + result.clear(); + auto hasPath = finder.getNextPath(); + EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + + pathStructureValid(result.slice(), 3); + pathEquals(result.slice(), {30, 31, 32, 33}); + + EXPECT_FALSE(finder.isDone()); + } + + { + // Try again to make sure we stay at non-existing + auto hasPath = finder.getNextPath(); + EXPECT_FALSE(hasPath); + EXPECT_TRUE(finder.isDone()); + } +} + +TEST_P(DFSFinderTest, triangle_loop_skip) { + VPackBuilder result; + auto finder = pathFinder(1, 10); + auto source = vId(30); + + finder.reset(toHashedStringRef(source)); + + EXPECT_FALSE(finder.isDone()); + { + result.clear(); + auto hasPath = finder.getNextPath(); + EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + + pathStructureValid(result.slice(), 1); + pathEquals(result.slice(), {30, 31}); + + EXPECT_FALSE(finder.isDone()); + } + + { + result.clear(); + auto hasPath = finder.getNextPath(); + EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + + pathStructureValid(result.slice(), 2); + pathEquals(result.slice(), {30, 31, 32}); + + EXPECT_FALSE(finder.isDone()); + } + + { + result.clear(); + bool skipped = finder.skipPath(); + EXPECT_TRUE(skipped); + EXPECT_FALSE(finder.isDone()); + } + + { + result.clear(); + auto hasPath = finder.getNextPath(); + EXPECT_TRUE(hasPath); + hasPath->toVelocyPack(result); + + pathStructureValid(result.slice(), 3); + pathEquals(result.slice(), {30, 31, 32, 33}); + + EXPECT_FALSE(finder.isDone()); + } + { result.clear(); // Try again to make sure we stay at non-existing - auto hasPath = finder.getNextPath(result); + auto hasPath = finder.getNextPath(); EXPECT_FALSE(hasPath); EXPECT_TRUE(result.isEmpty()); EXPECT_TRUE(finder.isDone()); @@ -569,9 +765,6 @@ TEST_P(DFSFinderTest, path_depth_1_to_2_skip) { /* TODO: Add more tests * - path_depth_2_to_3 - * - path_loop - * - triangle_loop - * - triangle_loop_skip * - many_neighbours_source * - many_neighbours_target */ diff --git a/tests/Graph/FifoQueueTest.cpp b/tests/Graph/FifoQueueTest.cpp index 0eab67b09c03..5379d2c1d4e5 100644 --- a/tests/Graph/FifoQueueTest.cpp +++ b/tests/Graph/FifoQueueTest.cpp @@ -24,6 +24,7 @@ #include "Basics/GlobalResourceMonitor.h" #include "Basics/ResourceUsage.h" #include "Basics/StringUtils.h" +#include "Basics/ResultT.h" #include "Graph/Providers/BaseStep.h" #include "Graph/Queues/FifoQueue.h" diff --git a/tests/Graph/GenericGraphProviderTest.cpp b/tests/Graph/GenericGraphProviderTest.cpp index bb6e939ce3b9..dc64f5384d5d 100644 --- a/tests/Graph/GenericGraphProviderTest.cpp +++ b/tests/Graph/GenericGraphProviderTest.cpp @@ -32,6 +32,7 @@ #include "Graph/Providers/ClusterProvider.h" #include "Graph/Providers/SingleServerProvider.h" +#include "Graph/Steps/SingleServerProviderStep.h" #include "Graph/TraverserOptions.h" #include @@ -50,7 +51,7 @@ static_assert(GTEST_HAS_TYPED_TEST, "We need typed tests for the following:"); // Add more providers here using TypesToTest = - ::testing::Types; + ::testing::Types, ClusterProvider>; template class GraphProviderTest : public ::testing::Test { @@ -64,11 +65,14 @@ class GraphProviderTest : public ::testing::Test { std::unique_ptr server{nullptr}; std::unique_ptr query{nullptr}; std::unique_ptr> clusterEngines{nullptr}; + std::unique_ptr _trx{}; arangodb::GlobalResourceMonitor global{}; arangodb::ResourceMonitor resourceMonitor{global}; + arangodb::aql::AqlFunctionsInternalCache _functionsCache{}; + std::unique_ptr _expressionContext; - std::map _emptyShardMap{}; + std::unordered_map> _emptyShardMap{}; GraphProviderTest() {} ~GraphProviderTest() {} @@ -86,10 +90,11 @@ class GraphProviderTest : public ::testing::Test { // We now have collections "v" and "e" query = singleServer->getQuery("RETURN 1", {"v", "e"}); - return MockGraphProvider(graph, *query.get(), - MockGraphProvider::LooseEndBehaviour::NEVER); + return MockGraphProvider(*query.get(), + MockGraphProviderOptions{graph, MockGraphProvider::LooseEndBehaviour::NEVER}, + resourceMonitor); } - if constexpr (std::is_same_v) { + if constexpr (std::is_same_v>) { s = std::make_unique(); singleServer = std::make_unique(s->server, "testVocbase"); @@ -97,16 +102,26 @@ class GraphProviderTest : public ::testing::Test { // We now have collections "v" and "e" query = singleServer->getQuery("RETURN 1", {"v", "e"}); + _trx = std::make_unique(query->newTrxContext()); auto edgeIndexHandle = singleServer->getEdgeIndexHandle("e"); auto tmpVar = singleServer->generateTempVar(query.get()); auto indexCondition = singleServer->buildOutboundCondition(query.get(), tmpVar); - std::vector usedIndexes{ - IndexAccessor{edgeIndexHandle, indexCondition, 0}}; + std::vector usedIndexes{}; + usedIndexes.emplace_back(IndexAccessor{edgeIndexHandle, indexCondition, 0, nullptr, 0}); - BaseProviderOptions opts(tmpVar, std::move(usedIndexes), _emptyShardMap); - return SingleServerProvider(*query.get(), std::move(opts), resourceMonitor); + _expressionContext = + std::make_unique(*_trx.get(), *query, + _functionsCache); + + BaseProviderOptions opts( + tmpVar, + std::make_pair(std::move(usedIndexes), + std::unordered_map>{}), + *_expressionContext.get(), _emptyShardMap); + return SingleServerProvider(*query.get(), std::move(opts), + resourceMonitor); } if constexpr (std::is_same_v) { // Prepare the DBServerResponses @@ -122,10 +137,9 @@ class GraphProviderTest : public ::testing::Test { server.getSystemDatabase()); arangodb::aql::Query fakeQuery(ctx, queryString, nullptr); try { - fakeQuery.collections().add("s9880", AccessMode::Type::READ, - arangodb::aql::Collection::Hint::Shard); - } catch(...) { - + fakeQuery.collections().add("s9880", AccessMode::Type::READ, + arangodb::aql::Collection::Hint::Shard); + } catch (...) { } fakeQuery.prepareQuery(SerializationFormat::SHADOWROWS); auto ast = fakeQuery.ast(); @@ -137,10 +151,10 @@ class GraphProviderTest : public ::testing::Test { opts.setVariable(tmpVar); auto const* access = - ast->createNodeAttributeAccess(tmpVarRef, - StaticStrings::FromString.c_str(), - StaticStrings::FromString.length()); - auto const* cond = ast->createNodeBinaryOperator(NODE_TYPE_OPERATOR_BINARY_EQ, access, tmpIdNode); + ast->createNodeAttributeAccess(tmpVarRef, StaticStrings::FromString.c_str(), + StaticStrings::FromString.length()); + auto const* cond = + ast->createNodeBinaryOperator(NODE_TYPE_OPERATOR_BINARY_EQ, access, tmpIdNode); auto fromCondition = ast->createNodeNaryOperator(NODE_TYPE_OPERATOR_NARY_AND); fromCondition->addMember(cond); opts.addLookupInfo(fakeQuery.plan(), "s9880", StaticStrings::FromString, fromCondition); @@ -158,9 +172,13 @@ class GraphProviderTest : public ::testing::Test { std::tie(preparedResponses, engineId) = graph.simulateApi(server, expectedVerticesEdgesBundleToFetch, opts); // Note: Please don't remove for debugging purpose. - /*for (auto const& resp : preparedResponses) { - LOG_DEVEL << resp.generateResponse()->copyPayload().get()->toString(); - }*/ +#if 0 + for (auto const& resp : preparedResponses) { + auto payload = resp.generateResponse()->copyPayload(); + VPackSlice husti(payload->data()); + LOG_DEVEL << husti.toJson(); + } +#endif } server = std::make_unique(true, false); @@ -189,8 +207,7 @@ class GraphProviderTest : public ::testing::Test { clusterEngines = std::make_unique>(); clusterEngines->emplace("PRMR_0001", engineId); - auto clusterCache = - std::make_shared(resourceMonitor); + auto clusterCache = std::make_shared(resourceMonitor); ClusterBaseProviderOptions opts(clusterCache, clusterEngines.get(), false); return ClusterProvider(*query.get(), std::move(opts), resourceMonitor); @@ -228,7 +245,7 @@ TYPED_TEST(GraphProviderTest, no_results_if_graph_is_empty) { TraversalStats stats = testee.stealStats(); EXPECT_EQ(stats.getFiltered(), 0); - if constexpr (std::is_same_v || + if constexpr (std::is_same_v> || std::is_same_v) { EXPECT_EQ(stats.getHttpRequests(), 0); } else if (std::is_same_v) { @@ -272,7 +289,7 @@ TYPED_TEST(GraphProviderTest, should_enumerate_a_single_edge) { { TraversalStats stats = testee.stealStats(); EXPECT_EQ(stats.getFiltered(), 0); - if constexpr (std::is_same_v || + if constexpr (std::is_same_v> || std::is_same_v) { EXPECT_EQ(stats.getHttpRequests(), 0); } else if (std::is_same_v) { @@ -299,8 +316,7 @@ TYPED_TEST(GraphProviderTest, should_enumerate_all_edges) { std::unordered_set found{}; std::unordered_map>> const& expectedVerticesEdgesBundleToFetch = { - {0, {}} - }; + {0, {}}}; auto testee = this->makeProvider(g, expectedVerticesEdgesBundleToFetch); std::string startString = g.vertexToId(0); VPackHashedStringRef startH{startString.c_str(), @@ -335,7 +351,7 @@ TYPED_TEST(GraphProviderTest, should_enumerate_all_edges) { { TraversalStats stats = testee.stealStats(); EXPECT_EQ(stats.getFiltered(), 0); - if constexpr (std::is_same_v || + if constexpr (std::is_same_v> || std::is_same_v) { EXPECT_EQ(stats.getHttpRequests(), 0); } else if (std::is_same_v) { @@ -356,7 +372,7 @@ TYPED_TEST(GraphProviderTest, destroy_engines) { testee.destroyEngines(); TraversalStats statsAfterSteal = testee.stealStats(); - if constexpr (std::is_same_v || + if constexpr (std::is_same_v> || std::is_same_v) { EXPECT_EQ(statsAfterSteal.getHttpRequests(), 0); } else if (std::is_same_v) { diff --git a/tests/Graph/GraphMockProviderInstances.cpp b/tests/Graph/GraphMockProviderInstances.cpp index 617b3cd7163b..fbb6e3a8a9a4 100644 --- a/tests/Graph/GraphMockProviderInstances.cpp +++ b/tests/Graph/GraphMockProviderInstances.cpp @@ -25,47 +25,62 @@ #include "Graph/Enumerators/OneSidedEnumerator.cpp" #include "Graph/Enumerators/TwoSidedEnumerator.cpp" #include "Graph/PathManagement/PathResult.cpp" -#include "Graph/PathManagement/SingleProviderPathResult.cpp" #include "Graph/PathManagement/PathStore.cpp" +#include "Graph/PathManagement/SingleProviderPathResult.cpp" #include "Graph/Queues/FifoQueue.h" #include "Graph/Queues/LifoQueue.h" #include "Graph/PathManagement/PathStoreTracer.cpp" #include "Graph/PathManagement/PathStoreTracer.h" #include "Graph/PathManagement/PathValidator.cpp" +#include "Graph/Providers/ProviderTracer.cpp" #include "Graph/Queues/QueueTracer.cpp" template class ::arangodb::graph::PathResult<::arangodb::tests::graph::MockGraphProvider, ::arangodb::tests::graph::MockGraphProvider::Step>; template class ::arangodb::graph::SingleProviderPathResult< - ::arangodb::tests::graph::MockGraphProvider, ::arangodb::tests::graph::MockGraphProvider::Step>; + ::arangodb::tests::graph::MockGraphProvider, + ::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, + ::arangodb::tests::graph::MockGraphProvider::Step>; + +template class ::arangodb::graph::SingleProviderPathResult< + ::arangodb::tests::graph::MockGraphProvider, + ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>>, + ::arangodb::tests::graph::MockGraphProvider::Step>; + +template class ::arangodb::graph::ProviderTracer<::arangodb::tests::graph::MockGraphProvider>; template class ::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>; template class ::arangodb::graph::PathValidator< + ::arangodb::tests::graph::MockGraphProvider, ::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, VertexUniquenessLevel::PATH>; template class ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>>; template class ::arangodb::graph::PathValidator< + ::arangodb::tests::graph::MockGraphProvider, ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>>, VertexUniquenessLevel::PATH>; template class ::arangodb::graph::TwoSidedEnumerator< ::arangodb::graph::FifoQueue<::arangodb::tests::graph::MockGraphProvider::Step>, ::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, ::arangodb::tests::graph::MockGraphProvider, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, VertexUniquenessLevel::PATH>>; + ::arangodb::graph::PathValidator<::arangodb::tests::graph::MockGraphProvider, + ::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, VertexUniquenessLevel::PATH>>; -template class ::arangodb::graph::OneSidedEnumerator< - ::arangodb::graph::LifoQueue<::arangodb::tests::graph::MockGraphProvider::Step>, - ::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, ::arangodb::tests::graph::MockGraphProvider, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, VertexUniquenessLevel::PATH>>; +// BFS with PATH uniquness +template class ::arangodb::graph::OneSidedEnumerator<::arangodb::graph::BFSConfiguration< + ::arangodb::tests::graph::MockGraphProvider, ::arangodb::graph::VertexUniquenessLevel::PATH, false>>; +template class ::arangodb::graph::OneSidedEnumerator<::arangodb::graph::BFSConfiguration< + ::arangodb::tests::graph::MockGraphProvider, ::arangodb::graph::VertexUniquenessLevel::PATH, true>>; -template class ::arangodb::graph::OneSidedEnumerator< - ::arangodb::graph::FifoQueue<::arangodb::tests::graph::MockGraphProvider::Step>, - ::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, ::arangodb::tests::graph::MockGraphProvider, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, VertexUniquenessLevel::PATH>>; +// DFS with PATH uniquness +template class ::arangodb::graph::OneSidedEnumerator<::arangodb::graph::DFSConfiguration< + ::arangodb::tests::graph::MockGraphProvider, ::arangodb::graph::VertexUniquenessLevel::PATH, false>>; +template class ::arangodb::graph::OneSidedEnumerator<::arangodb::graph::DFSConfiguration< + ::arangodb::tests::graph::MockGraphProvider, ::arangodb::graph::VertexUniquenessLevel::PATH, true>>; template class ::arangodb::graph::QueueTracer<::arangodb::graph::FifoQueue<::arangodb::tests::graph::MockGraphProvider::Step>>; @@ -74,35 +89,13 @@ template class ::arangodb::graph::QueueTracer<::arangodb::graph::LifoQueue<::ara template class ::arangodb::graph::TwoSidedEnumerator< ::arangodb::graph::QueueTracer<::arangodb::graph::FifoQueue<::arangodb::tests::graph::MockGraphProvider::Step>>, ::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, ::arangodb::tests::graph::MockGraphProvider, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, VertexUniquenessLevel::PATH>>; - -template class ::arangodb::graph::OneSidedEnumerator< - ::arangodb::graph::QueueTracer<::arangodb::graph::LifoQueue<::arangodb::tests::graph::MockGraphProvider::Step>>, - ::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, ::arangodb::tests::graph::MockGraphProvider, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, VertexUniquenessLevel::PATH>>; - -template class ::arangodb::graph::OneSidedEnumerator< - ::arangodb::graph::QueueTracer<::arangodb::graph::FifoQueue<::arangodb::tests::graph::MockGraphProvider::Step>>, - ::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, ::arangodb::tests::graph::MockGraphProvider, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, VertexUniquenessLevel::PATH>>; + ::arangodb::graph::PathValidator<::arangodb::tests::graph::MockGraphProvider, + ::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>, VertexUniquenessLevel::PATH>>; template class ::arangodb::graph::TwoSidedEnumerator< ::arangodb::graph::QueueTracer<::arangodb::graph::FifoQueue<::arangodb::tests::graph::MockGraphProvider::Step>>, ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>>, ::arangodb::tests::graph::MockGraphProvider, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>>, + ::arangodb::graph::PathValidator<::arangodb::tests::graph::MockGraphProvider, + ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>>, VertexUniquenessLevel::PATH>>; - -template class ::arangodb::graph::OneSidedEnumerator< - ::arangodb::graph::QueueTracer<::arangodb::graph::LifoQueue<::arangodb::tests::graph::MockGraphProvider::Step>>, - ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>>, - ::arangodb::tests::graph::MockGraphProvider, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>>, - VertexUniquenessLevel::PATH>>; - -template class ::arangodb::graph::OneSidedEnumerator< - ::arangodb::graph::QueueTracer<::arangodb::graph::FifoQueue<::arangodb::tests::graph::MockGraphProvider::Step>>, - ::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>>, - ::arangodb::tests::graph::MockGraphProvider, - ::arangodb::graph::PathValidator<::arangodb::graph::PathStoreTracer<::arangodb::graph::PathStore<::arangodb::tests::graph::MockGraphProvider::Step>>, - VertexUniquenessLevel::PATH>>; \ No newline at end of file diff --git a/tests/Graph/GraphTestTools.h b/tests/Graph/GraphTestTools.h index fe1d517cc588..9a6ea406e2c9 100644 --- a/tests/Graph/GraphTestTools.h +++ b/tests/Graph/GraphTestTools.h @@ -83,6 +83,7 @@ struct GraphTestSetup // setup required application features features.emplace_back(server.addFeature(), false); features.emplace_back(server.addFeature(), false); + features.emplace_back(server.addFeature(), false); features.emplace_back(server.addFeature(), false); features.emplace_back(server.addFeature(), false); server.getFeature().setEngineTesting(&engine); diff --git a/tests/Graph/KPathFinderTest.cpp b/tests/Graph/KPathFinderTest.cpp index 68c92ddd7936..9340096c6704 100644 --- a/tests/Graph/KPathFinderTest.cpp +++ b/tests/Graph/KPathFinderTest.cpp @@ -69,6 +69,13 @@ class KPathFinderTest arangodb::GlobalResourceMonitor global{}; arangodb::ResourceMonitor resourceMonitor{global}; + // PathValidatorOptions parts (used for API not under test here) + aql::Variable _tmpVar{"tmp", 0, false}; + arangodb::aql::AqlFunctionsInternalCache _functionsCache{}; + + arangodb::transaction::Methods _trx{_query->newTrxContext()}; + arangodb::aql::FixedVarExpressionContext _expressionContext{_trx, *_query.get(), + _functionsCache}; KPathFinderTest() { if (activateLogging) { Logger::GRAPHS.setLogLevel(LogLevel::TRACE); @@ -159,9 +166,15 @@ class KPathFinderTest auto pathFinder(size_t minDepth, size_t maxDepth) -> KPathFinder { arangodb::graph::TwoSidedEnumeratorOptions options{minDepth, maxDepth}; - return KPathFinder{MockGraphProvider(mockGraph, *_query.get(), looseEndBehaviour(), false), - MockGraphProvider(mockGraph, *_query.get(), looseEndBehaviour(), true), - std::move(options), resourceMonitor}; + PathValidatorOptions validatorOpts{&_tmpVar, _expressionContext}; + return KPathFinder{ + MockGraphProvider(*_query.get(), + MockGraphProviderOptions{mockGraph, looseEndBehaviour(), false}, + resourceMonitor), + MockGraphProvider(*_query.get(), + MockGraphProviderOptions{mockGraph, looseEndBehaviour(), true}, + resourceMonitor), + std::move(options), std::move(validatorOpts), resourceMonitor}; } auto vId(size_t nr) -> std::string { diff --git a/tests/Graph/LifoQueueTest.cpp b/tests/Graph/LifoQueueTest.cpp index 33b116e952ab..7eb89ec19b31 100644 --- a/tests/Graph/LifoQueueTest.cpp +++ b/tests/Graph/LifoQueueTest.cpp @@ -24,6 +24,7 @@ #include "Basics/GlobalResourceMonitor.h" #include "Basics/ResourceUsage.h" #include "Basics/StringUtils.h" +#include "Basics/ResultT.h" #include "Graph/Providers/BaseStep.h" #include "Graph/Queues/LifoQueue.h" diff --git a/tests/Graph/PathValidatorTest.cpp b/tests/Graph/PathValidatorTest.cpp index fd4c37fe411c..12eb80bbd837 100644 --- a/tests/Graph/PathValidatorTest.cpp +++ b/tests/Graph/PathValidatorTest.cpp @@ -22,6 +22,10 @@ #include "gtest/gtest.h" +#include "../Mocks/Servers.h" + +#include "Aql/Ast.h" +#include "Aql/Query.h" #include "Basics/GlobalResourceMonitor.h" #include "Basics/ResourceUsage.h" #include "Basics/StringHeap.h" @@ -31,62 +35,68 @@ #include "Graph/Providers/BaseStep.h" #include "Graph/Types/UniquenessLevel.h" +#include "./MockGraph.h" +#include "./MockGraphProvider.h" + using namespace arangodb; using namespace arangodb::graph; using namespace arangodb::velocypack; +namespace { +aql::AstNode* InitializeReference(aql::Ast& ast, aql::Variable& var) { + ast.scopes()->start(aql::ScopeType::AQL_SCOPE_MAIN); + ast.scopes()->addVariable(&var); + aql::AstNode* a = ast.createNodeReference("tmp"); + ast.scopes()->endCurrent(); + return a; +} +} // namespace + namespace arangodb { namespace tests { namespace graph_path_validator_test { static_assert(GTEST_HAS_TYPED_TEST, "We need typed tests for the following:"); -class Step : public arangodb::graph::BaseStep { - public: - using Vertex = HashedStringRef; - using Edge = HashedStringRef; - - HashedStringRef _id; - - Step(HashedStringRef id, size_t previous) - : arangodb::graph::BaseStep{previous}, _id{id} {}; - - ~Step() = default; - - bool operator==(Step const& other) { return _id == other._id; } - - std::string toString() const { - return " _id: " + _id.toString() + - ", _previous: " + basics::StringUtils::itoa(getPrevious()); - } - - bool isProcessable() const { return true; } - HashedStringRef getVertex() const { return _id; } - HashedStringRef getEdge() const { return _id; } +using Step = typename graph::MockGraphProvider::Step; - arangodb::velocypack::HashedStringRef getVertexIdentifier() const { - return getVertex(); - } -}; - -using TypesToTest = - ::testing::Types, VertexUniquenessLevel::NONE>, - PathValidator, VertexUniquenessLevel::PATH>, - PathValidator, VertexUniquenessLevel::GLOBAL>>; +using TypesToTest = ::testing::Types< + PathValidator, VertexUniquenessLevel::NONE>, + PathValidator, VertexUniquenessLevel::PATH>, + PathValidator, VertexUniquenessLevel::GLOBAL>>; template class PathValidatorTest : public ::testing::Test { + protected: + graph::MockGraph mockGraph; + mocks::MockAqlServer _server{true}; + + std::unique_ptr _query{_server.createFakeQuery()}; + + private: arangodb::GlobalResourceMonitor _global{}; arangodb::ResourceMonitor _resourceMonitor{_global}; + std::unique_ptr _provider; + PathStore _pathStore{_resourceMonitor}; StringHeap _heap{_resourceMonitor, 4096}; + // Expression Parts + arangodb::transaction::Methods _trx{_query->newTrxContext()}; + aql::Ast* _ast{_query->ast()}; + aql::Variable _tmpVar{"tmp", 0, false}; + aql::AstNode* _varNode{::InitializeReference(*_ast, _tmpVar)}; + + arangodb::aql::AqlFunctionsInternalCache _functionsCache{}; + arangodb::aql::FixedVarExpressionContext _expressionContext{_trx, *_query, _functionsCache}; + PathValidatorOptions _opts{&_tmpVar, _expressionContext}; + protected: VertexUniquenessLevel getVertexUniquness() { - if constexpr (std::is_same_v, VertexUniquenessLevel::NONE>>) { + if constexpr (std::is_same_v, VertexUniquenessLevel::NONE>>) { return VertexUniquenessLevel::NONE; - } else if constexpr (std::is_same_v, VertexUniquenessLevel::PATH>>) { + } else if constexpr (std::is_same_v, VertexUniquenessLevel::PATH>>) { return VertexUniquenessLevel::PATH; } else { return VertexUniquenessLevel::GLOBAL; @@ -95,61 +105,104 @@ class PathValidatorTest : public ::testing::Test { PathStore& store() { return _pathStore; } - ValidatorType testee() { return ValidatorType{this->store()}; } - Step makeStep(size_t id, size_t previous) { - std::string idStr = basics::StringUtils::itoa(id); - HashedStringRef hStr(idStr.data(), static_cast(idStr.length())); - return Step(_heap.registerString(hStr), previous); + aql::Variable const* tmpVar() { return &_tmpVar; } + + aql::Query* query() { return _query.get(); } + + ValidatorType testee() { + ensureProvider(); + return ValidatorType{*_provider.get(), this->store(), _opts}; + } + + Step startPath(size_t id) { + ensureProvider(); + auto base = mockGraph.vertexToId(id); + HashedStringRef ref{base.c_str(), static_cast(base.length())}; + auto hStr = _heap.registerString(ref); + return _provider->startVertex(hStr); + } + + // Get and modify the options used in the Validator testee(). + // Make sure to Modify them before calling testee() method. + PathValidatorOptions& options() { return _opts; } + + std::vector expandPath(Step previous) { + // We at least have called startPath before, this ensured the provider + TRI_ASSERT(_provider != nullptr); + size_t prev = _pathStore.append(previous); + std::vector result; + _provider->expand(previous, prev, + [&result](Step s) { result.emplace_back(s); }); + return result; + } + + // Add a path defined by the given vector. We start a first() and end in last() + void addPath(std::vector path) { + // This function can only add paths of length 1 or more + TRI_ASSERT(path.size() >= 2); + for (size_t i = 0; i < path.size() - 1; ++i) { + mockGraph.addEdge(path.at(i), path.at(i + 1)); + } + } + + /* + * generates a condition #TMP._key == '' + */ + std::unique_ptr conditionKeyMatches(std::string const& toMatch) { + auto expectedKey = _ast->createNodeValueString(toMatch.c_str(), toMatch.length()); + auto keyAccess = + _ast->createNodeAttributeAccess(_varNode, StaticStrings::KeyString.c_str(), + StaticStrings::KeyString.length()); + // This condition cannot be fulfilled + auto condition = _ast->createNodeBinaryOperator(aql::AstNodeType::NODE_TYPE_OPERATOR_BINARY_EQ, + keyAccess, expectedKey); + return std::make_unique(_ast, condition); + } + + private: + void ensureProvider() { + if (_provider == nullptr) { + _provider = std::make_unique( + *_query.get(), + graph::MockGraphProviderOptions{mockGraph, graph::MockGraphProviderOptions::LooseEndBehaviour::NEVER, + false}, + _resourceMonitor); + } } }; TYPED_TEST_CASE(PathValidatorTest, TypesToTest); TYPED_TEST(PathValidatorTest, it_should_honor_uniqueness_on_single_path_first_duplicate) { - auto&& ps = this->store(); + // We add a loop that ends in the start vertex (0) again. + this->addPath({0, 1, 2, 3, 0}); auto validator = this->testee(); - size_t lastIndex = std::numeric_limits::max(); + Step s = this->startPath(0); { - Step s = this->makeStep(0, lastIndex); auto res = validator.validatePath(s); // The start vertex is always valid EXPECT_FALSE(res.isFiltered()); EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 0); - } - - // We add a loop that ends in the start vertex (0) again. - { - Step s = this->makeStep(1, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 1); - } - { - Step s = this->makeStep(2, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 2); } - { - Step s = this->makeStep(3, lastIndex); + // The next 3 steps are good to take. + for (size_t i = 0; i < 3; ++i) { + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1) + << "Not enough connections after step " << s.getVertexIdentifier(); + s = neighbors.at(0); auto res = validator.validatePath(s); EXPECT_FALSE(res.isFiltered()); EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 3); } - // Add duplicate vertex on Path + // Now we move to the duplicate vertex { - Step s = this->makeStep(0, lastIndex); + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1); + s = neighbors.at(0); auto res = validator.validatePath(s); + if (this->getVertexUniquness() == VertexUniquenessLevel::NONE) { // No uniqueness check, take the vertex EXPECT_FALSE(res.isFiltered()); @@ -163,49 +216,35 @@ TYPED_TEST(PathValidatorTest, it_should_honor_uniqueness_on_single_path_first_du } TYPED_TEST(PathValidatorTest, it_should_honor_uniqueness_on_single_path_last_duplicate) { - auto&& ps = this->store(); + // We add a loop that loops on the last vertex(3). + this->addPath({0, 1, 2, 3, 3}); auto validator = this->testee(); - size_t lastIndex = std::numeric_limits::max(); + Step s = this->startPath(0); { - Step s = this->makeStep(0, lastIndex); auto res = validator.validatePath(s); // The start vertex is always valid EXPECT_FALSE(res.isFiltered()); EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 0); - } - // We add a loop on the last vertex of the path (3) - { - Step s = this->makeStep(1, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 1); - } - { - Step s = this->makeStep(2, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 2); } - { - Step s = this->makeStep(3, lastIndex); + // The next 3 steps are good to take. + for (size_t i = 0; i < 3; ++i) { + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1) + << "Not enough connections after step " << s.getVertexIdentifier(); + s = neighbors.at(0); auto res = validator.validatePath(s); EXPECT_FALSE(res.isFiltered()); EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 3); } - // Add duplicate vertex on Path + // Now we move to the duplicate vertex { - Step s = this->makeStep(3, lastIndex); + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1); + s = neighbors.at(0); auto res = validator.validatePath(s); + if (this->getVertexUniquness() == VertexUniquenessLevel::NONE) { // No uniqueness check, take the vertex EXPECT_FALSE(res.isFiltered()); @@ -219,49 +258,35 @@ TYPED_TEST(PathValidatorTest, it_should_honor_uniqueness_on_single_path_last_dup } TYPED_TEST(PathValidatorTest, it_should_honor_uniqueness_on_single_path_interior_duplicate) { - auto&& ps = this->store(); + // We add a loop that loops on the last vertex(2). + this->addPath({0, 1, 2, 3, 2}); auto validator = this->testee(); - size_t lastIndex = std::numeric_limits::max(); + Step s = this->startPath(0); { - Step s = this->makeStep(0, lastIndex); auto res = validator.validatePath(s); // The start vertex is always valid EXPECT_FALSE(res.isFiltered()); EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 0); - } - // We add a loop that ends in one interior vertex (2) again. - { - Step s = this->makeStep(1, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 1); } - { - Step s = this->makeStep(2, lastIndex); + // The next 3 steps are good to take. + for (size_t i = 0; i < 3; ++i) { + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1) + << "Not enough connections after step " << s.getVertexIdentifier(); + s = neighbors.at(0); auto res = validator.validatePath(s); EXPECT_FALSE(res.isFiltered()); EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 2); - } - { - Step s = this->makeStep(3, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 3); } - // Add duplicate vertex on Path + // Now we move to the duplicate vertex { - Step s = this->makeStep(2, lastIndex); + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1); + s = neighbors.at(0); auto res = validator.validatePath(s); + if (this->getVertexUniquness() == VertexUniquenessLevel::NONE) { // No uniqueness check, take the vertex EXPECT_FALSE(res.isFiltered()); @@ -275,160 +300,201 @@ TYPED_TEST(PathValidatorTest, it_should_honor_uniqueness_on_single_path_interior } TYPED_TEST(PathValidatorTest, it_should_honor_uniqueness_on_global_paths_last_duplicate) { - auto&& ps = this->store(); + // We add a two paths, that share the same start and end vertex (3) + this->addPath({0, 1, 2, 3}); + this->addPath({0, 4, 5, 3}); + auto validator = this->testee(); - size_t lastIndex = std::numeric_limits::max(); + Step s = this->startPath(0); { - Step s = this->makeStep(0, lastIndex); auto res = validator.validatePath(s); // The start vertex is always valid EXPECT_FALSE(res.isFiltered()); EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 0); - } - // We add two paths, each without a loop. - // Both paths share a common vertex besides the start. - - // First path 0 -> 1 -> 2 -> 3 - { - Step s = this->makeStep(1, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 1); - } - { - Step s = this->makeStep(2, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 2); - } - { - Step s = this->makeStep(3, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 3); } - // First path 0 -> 4 -> 5 - lastIndex = 0; - { - Step s = this->makeStep(4, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 4); - } + auto branch = this->expandPath(s); + // 1 and 4, we do not care on the ordering. + ASSERT_EQ(branch.size(), 2); { - Step s = this->makeStep(5, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 5); + { + // Test the branch vertex itself + s = branch.at(0); + auto res = validator.validatePath(s); + EXPECT_FALSE(res.isFiltered()); + EXPECT_FALSE(res.isPruned()); + } + // The first branch is good until the end + for (size_t i = 0; i < 2; ++i) { + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1) + << "Not enough connections after step " << s.getVertexIdentifier(); + s = neighbors.at(0); + auto res = validator.validatePath(s); + EXPECT_FALSE(res.isFiltered()); + EXPECT_FALSE(res.isPruned()); + } } - - // Add duplicate vertex (3) which is the last on the first path { - Step s = this->makeStep(3, lastIndex); - auto res = validator.validatePath(s); - if (this->getVertexUniquness() != VertexUniquenessLevel::GLOBAL) { - // The vertex is visited twice, but not on same path. - // As long as we are not GLOBAL this is okay. + // The second branch is good but for the last vertex + { + // Test the branch vertex itself + s = branch.at(1); + auto res = validator.validatePath(s); EXPECT_FALSE(res.isFiltered()); EXPECT_FALSE(res.isPruned()); - } else { - // With GLOBAL uniqueness this vertex is illegal - EXPECT_TRUE(res.isFiltered()); - EXPECT_TRUE(res.isPruned()); + } + for (size_t i = 0; i < 1; ++i) { + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1) + << "Not enough connections after step " << s.getVertexIdentifier(); + s = neighbors.at(0); + auto res = validator.validatePath(s); + EXPECT_FALSE(res.isFiltered()); + EXPECT_FALSE(res.isPruned()); + } + + // Now we move to the duplicate vertex + { + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1); + s = neighbors.at(0); + auto res = validator.validatePath(s); + + if (this->getVertexUniquness() != VertexUniquenessLevel::GLOBAL) { + // The vertex is visited twice, but not on same path. + // As long as we are not GLOBAL this is okay. + // No uniqueness check, take the vertex + EXPECT_FALSE(res.isFiltered()); + EXPECT_FALSE(res.isPruned()); + } else { + // With GLOBAL uniqueness this vertex is illegal + EXPECT_TRUE(res.isFiltered()); + EXPECT_TRUE(res.isPruned()); + } } } } TYPED_TEST(PathValidatorTest, it_should_honor_uniqueness_on_global_paths_interior_duplicate) { - auto&& ps = this->store(); + // We add a two paths, that share the same start and end vertex (3) + this->addPath({0, 1, 2, 3}); + this->addPath({0, 4, 5, 1}); + auto validator = this->testee(); - size_t lastIndex = std::numeric_limits::max(); + Step s = this->startPath(0); { - Step s = this->makeStep(0, lastIndex); auto res = validator.validatePath(s); // The start vertex is always valid EXPECT_FALSE(res.isFiltered()); EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 0); } - // We add two paths, each without a loop. - // Both paths share a common vertex besides the start. - // First path 0 -> 1 -> 2 -> 3 - { - Step s = this->makeStep(1, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 1); - } + auto branch = this->expandPath(s); + // 1 and 4, we do need to care on the ordering, this is right now guaranteed. + // If this test fails at any point in time, we can add some code here that + // ensures that we first visit Vertex 1, then Vertex 4 + ASSERT_EQ(branch.size(), 2); { - Step s = this->makeStep(2, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 2); + // The first branch is good until the end + { + // Test the branch vertex itself + s = branch.at(0); + auto res = validator.validatePath(s); + EXPECT_FALSE(res.isFiltered()); + EXPECT_FALSE(res.isPruned()); + } + for (size_t i = 0; i < 2; ++i) { + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1) + << "Not enough connections after step " << s.getVertexIdentifier(); + s = neighbors.at(0); + auto res = validator.validatePath(s); + EXPECT_FALSE(res.isFiltered()); + EXPECT_FALSE(res.isPruned()); + } } { - Step s = this->makeStep(3, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 3); - } + // The second branch is good but for the last vertex + { + // Test the branch vertex itself + s = branch.at(1); + auto res = validator.validatePath(s); + EXPECT_FALSE(res.isFiltered()); + EXPECT_FALSE(res.isPruned()); + } + for (size_t i = 0; i < 1; ++i) { + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1) + << "Not enough connections after step " << s.getVertexIdentifier(); + s = neighbors.at(0); + auto res = validator.validatePath(s); + EXPECT_FALSE(res.isFiltered()); + EXPECT_FALSE(res.isPruned()); + } - // First path 0 -> 4 -> 5 - lastIndex = 0; - { - Step s = this->makeStep(4, lastIndex); - auto res = validator.validatePath(s); - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 4); + // Now we move to the duplicate vertex + { + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1); + s = neighbors.at(0); + auto res = validator.validatePath(s); + + if (this->getVertexUniquness() != VertexUniquenessLevel::GLOBAL) { + // The vertex is visited twice, but not on same path. + // As long as we are not GLOBAL this is okay. + // No uniqueness check, take the vertex + EXPECT_FALSE(res.isFiltered()); + EXPECT_FALSE(res.isPruned()); + } else { + // With GLOBAL uniqueness this vertex is illegal + EXPECT_TRUE(res.isFiltered()); + EXPECT_TRUE(res.isPruned()); + } + } } +} + +TYPED_TEST(PathValidatorTest, it_should_test_an_all_vertices_condition) { + this->addPath({0, 1}); + std::string keyToMatch = "1"; + + auto expression = this->conditionKeyMatches(keyToMatch); + auto& opts = this->options(); + opts.setAllVerticesExpression(std::move(expression)); + auto validator = this->testee(); { - Step s = this->makeStep(5, lastIndex); + // Testing x._key == "1" with `{_key: "1"} => Should succeed + Step s = this->startPath(1); auto res = validator.validatePath(s); EXPECT_FALSE(res.isFiltered()); EXPECT_FALSE(res.isPruned()); - lastIndex = ps.append(std::move(s)); - EXPECT_EQ(lastIndex, 5); } - // Add duplicate vertex (1) which is interior to the first path + // we start a new path, so reset the uniqueness checks + validator.reset(); + { - Step s = this->makeStep(1, lastIndex); - auto res = validator.validatePath(s); - if (this->getVertexUniquness() != VertexUniquenessLevel::GLOBAL) { - // The vertex is visited twice, but not on same path. - // As long as we are not GLOBAL this is okay. - EXPECT_FALSE(res.isFiltered()); - EXPECT_FALSE(res.isPruned()); - } else { - // With GLOBAL uniqueness this vertex is illegal + // Testing x._key == "1" with `{_key: "0"} => Should fail + Step s = this->startPath(0); + { + auto res = validator.validatePath(s); EXPECT_TRUE(res.isFiltered()); EXPECT_TRUE(res.isPruned()); } + + // Testing condition on level 1 (not start) + auto neighbors = this->expandPath(s); + ASSERT_EQ(neighbors.size(), 1); + s = neighbors.at(0); + { + // Testing x._key == "1" with `{_key: "1"} => Should succeed + auto res = validator.validatePath(s); + EXPECT_FALSE(res.isFiltered()); + EXPECT_FALSE(res.isPruned()); + } } } diff --git a/tests/Graph/SingleServerProviderTest.cpp b/tests/Graph/SingleServerProviderTest.cpp index ebe2c6811719..37b13e2b20ed 100644 --- a/tests/Graph/SingleServerProviderTest.cpp +++ b/tests/Graph/SingleServerProviderTest.cpp @@ -29,6 +29,7 @@ #include "Basics/GlobalResourceMonitor.h" #include "Basics/ResourceUsage.h" #include "Graph/Providers/SingleServerProvider.h" +#include "Graph/Steps/SingleServerProviderStep.h" #include #include @@ -38,10 +39,22 @@ using namespace arangodb::tests; using namespace arangodb::tests::graph; using namespace arangodb::graph; +namespace { +aql::AstNode* InitializeReference(aql::Ast& ast, aql::Variable& var) { + ast.scopes()->start(aql::ScopeType::AQL_SCOPE_MAIN); + ast.scopes()->addVariable(&var); + aql::AstNode* a = ast.createNodeReference(var.name); + ast.scopes()->endCurrent(); + return a; +} +} // namespace + namespace arangodb { namespace tests { namespace single_server_provider_test { +using Step = SingleServerProviderStep; + class SingleServerProviderTest : public ::testing::Test { protected: // Only used to mock a singleServer @@ -50,13 +63,24 @@ class SingleServerProviderTest : public ::testing::Test { std::unique_ptr query{nullptr}; arangodb::GlobalResourceMonitor _global{}; arangodb::ResourceMonitor _resourceMonitor{_global}; + arangodb::aql::AqlFunctionsInternalCache _functionsCache{}; + std::unique_ptr _expressionContext; + std::unique_ptr _trx{}; + + // Expression Parts + aql::Variable* _tmpVar{nullptr}; + aql::AstNode* _varNode{nullptr}; + + std::unordered_map> _emptyShardMap{}; - std::map _emptyShardMap{}; + // can be used for further testing to generate a expression + // std::string stringToMatch = "0-1"; SingleServerProviderTest() {} ~SingleServerProviderTest() {} - auto makeProvider(MockGraph const& graph) -> arangodb::graph::SingleServerProvider { + auto makeProvider(MockGraph const& graph) + -> arangodb::graph::SingleServerProvider { // Setup code for each provider type s = std::make_unique(); singleServer = @@ -65,20 +89,72 @@ class SingleServerProviderTest : public ::testing::Test { // We now have collections "v" and "e" query = singleServer->getQuery("RETURN 1", {"v", "e"}); + _trx = std::make_unique(query->newTrxContext()); auto edgeIndexHandle = singleServer->getEdgeIndexHandle("e"); - auto tmpVar = singleServer->generateTempVar(query.get()); - auto indexCondition = singleServer->buildOutboundCondition(query.get(), tmpVar); + _tmpVar = singleServer->generateTempVar(query.get()); - std::vector usedIndexes{IndexAccessor{edgeIndexHandle, indexCondition, 0}}; + auto indexCondition = singleServer->buildOutboundCondition(query.get(), _tmpVar); + _varNode = ::InitializeReference(*query->ast(), *_tmpVar); - BaseProviderOptions opts(tmpVar, std::move(usedIndexes), _emptyShardMap); + std::vector usedIndexes{}; + + // can be used to create an expression, currently unused but may be helpful for additional tests + // auto expr = conditionKeyMatches(stringToMatch); + usedIndexes.emplace_back(IndexAccessor{edgeIndexHandle, indexCondition, 0, nullptr, 0}); + + _expressionContext = + std::make_unique(*_trx, *query, _functionsCache); + BaseProviderOptions opts(_tmpVar, + std::make_pair(std::move(usedIndexes), + std::unordered_map>{}), + *_expressionContext.get(), _emptyShardMap); return {*query.get(), std::move(opts), _resourceMonitor}; } + + /* + * generates a condition #TMP._key == '' + */ + std::unique_ptr conditionKeyMatches(std::string const& toMatch) { + auto expectedKey = + query->ast()->createNodeValueString(toMatch.c_str(), toMatch.length()); + auto keyAccess = + query->ast()->createNodeAttributeAccess(_varNode, + StaticStrings::KeyString.c_str(), + StaticStrings::KeyString.length()); + // This condition cannot be fulfilled + auto condition = query->ast()->createNodeBinaryOperator(aql::AstNodeType::NODE_TYPE_OPERATOR_BINARY_EQ, + keyAccess, expectedKey); + return std::make_unique(query->ast(), condition); + } }; -TEST_F(SingleServerProviderTest, it_must_be_described) { - ASSERT_TRUE(true); +TEST_F(SingleServerProviderTest, it_can_provide_edges) { + MockGraph g; + g.addEdge(0, 1, 2); + g.addEdge(0, 2, 3); + g.addEdge(1, 2, 1); + auto testee = makeProvider(g); + auto startVertex = g.vertexToId(0); + HashedStringRef hashedStart{startVertex.c_str(), + static_cast(startVertex.length())}; + Step s = testee.startVertex(hashedStart); + + std::vector results = {}; + VPackBuilder builder; + + testee.expand(s, 0, [&](Step next) { + results.push_back(next.getVertex().getID().toString()); + }); + + // Order is not guaranteed + ASSERT_EQ(results.size(), 2); + if (results.at(0) == "v/1") { + ASSERT_EQ(results.at(1), "v/2"); + } else { + ASSERT_EQ(results.at(0), "v/2"); + ASSERT_EQ(results.at(1), "v/1"); + } } } // namespace single_server_provider_test diff --git a/tests/Graph/TraverserCacheTest.cpp b/tests/Graph/TraverserCacheTest.cpp index 65c7bea96fe6..153bf76b81ea 100644 --- a/tests/Graph/TraverserCacheTest.cpp +++ b/tests/Graph/TraverserCacheTest.cpp @@ -55,7 +55,7 @@ class TraverserCacheTest : public ::testing::Test { std::unique_ptr traverserCache{nullptr}; std::shared_ptr queryContext{nullptr}; std::unique_ptr trx{nullptr}; - std::map collectionToShardMap{}; // can be empty, only used in standalone mode + std::unordered_map> collectionToShardMap{}; // can be empty, only used in standalone mode arangodb::ResourceMonitor* _monitor; TraverserCacheTest() : gdb(s.server, "testVocbase") { @@ -65,7 +65,8 @@ class TraverserCacheTest : public ::testing::Test { _monitor = &query->resourceMonitor(); traverserCache = std::make_unique(trx.get(), query.get(), - query->resourceMonitor(), stats, collectionToShardMap); + query->resourceMonitor(), + stats, collectionToShardMap); } ~TraverserCacheTest() = default; @@ -85,7 +86,8 @@ TEST_F(TraverserCacheTest, it_should_return_a_null_aqlvalue_if_vertex_is_not_ava VPackBuilder builder; // NOTE: we do not have the data, so we get null for any vertex - traverserCache->insertVertexIntoResult(stats, arangodb::velocypack::HashedStringRef(id), builder); + traverserCache->insertVertexIntoResult(stats, arangodb::velocypack::HashedStringRef(id), + builder, false); ASSERT_TRUE(builder.slice().isNull()); auto all = query->warnings().all(); ASSERT_EQ(all.size(), 1); @@ -98,6 +100,36 @@ TEST_F(TraverserCacheTest, it_should_return_a_null_aqlvalue_if_vertex_is_not_ava EXPECT_EQ(stats.getScannedIndex(), 0); } +TEST_F(TraverserCacheTest, it_should_on_request_return_the_id_aqlvalue_if_vertex_is_not_available) { + // prepare graph data - in this case, no data (no vertices and no edges, but collections v and e) + graph::MockGraph graph{}; + gdb.addGraph(graph); + + std::string vertexId = "v/Vertex"; + std::string expectedMessage = "vertex '" + vertexId + "' not found"; + + auto data = VPackParser::fromJson(R"({"_key":"Vertex", "_id": "v/Vertex"})"); + VPackSlice doc = data->slice(); + HashedStringRef id{doc.get("_id")}; + VPackBuilder builder; + + // NOTE: we do not have the data, so we get null for any vertex + traverserCache->insertVertexIntoResult(stats, arangodb::velocypack::HashedStringRef(id), + builder, true); + ASSERT_FALSE(builder.slice().isNull()); + ASSERT_TRUE(builder.slice().isString()); + ASSERT_EQ(builder.slice().copyString(), "v/Vertex"); + auto all = query->warnings().all(); + ASSERT_EQ(all.size(), 1); + ASSERT_TRUE(all[0].first == TRI_ERROR_ARANGO_DOCUMENT_NOT_FOUND); + ASSERT_TRUE(all[0].second == expectedMessage); + + // check stats + EXPECT_EQ(stats.getHttpRequests(), 0); + EXPECT_EQ(stats.getFiltered(), 0); + EXPECT_EQ(stats.getScannedIndex(), 0); +} + TEST_F(TraverserCacheTest, it_should_return_a_null_aqlvalue_if_edge_is_not_available) { // prepare graph data - in this case, no data (no vertices and no edges, but collections v and e) graph::MockGraph graph{}; @@ -129,7 +161,8 @@ TEST_F(TraverserCacheTest, it_should_increase_memory_usage_when_persisting_a_str EXPECT_EQ(memoryUsageBefore, _monitor->current()); } -TEST_F(TraverserCacheTest, it_should_not_increase_memory_usage_twice_when_persisting_two_equal_strings) { +TEST_F(TraverserCacheTest, + it_should_not_increase_memory_usage_twice_when_persisting_two_equal_strings) { auto memoryUsageStart = _monitor->current(); auto data = VPackParser::fromJson(R"({"_key":"123", "value":123})"); @@ -168,7 +201,8 @@ TEST_F(TraverserCacheTest, it_should_increase_memory_usage_twice_when_persisting EXPECT_EQ(memoryUsageStart, _monitor->current()); } -TEST_F(TraverserCacheTest, it_should_increase_memory_usage_twice_when_persisting_a_string_clear_persist_again) { +TEST_F(TraverserCacheTest, + it_should_increase_memory_usage_twice_when_persisting_a_string_clear_persist_again) { auto memoryUsageBefore = _monitor->current(); auto data = VPackParser::fromJson(R"({"_key":"123", "value":123})"); @@ -220,7 +254,8 @@ TEST_F(TraverserCacheTest, it_should_insert_a_vertex_into_a_result_builder) { HashedStringRef id{doc.get("_id")}; VPackBuilder builder; - traverserCache->insertVertexIntoResult(stats, arangodb::velocypack::HashedStringRef(id), builder); + traverserCache->insertVertexIntoResult(stats, arangodb::velocypack::HashedStringRef(id), + builder, false); EXPECT_TRUE(builder.slice().get("_key").isString()); EXPECT_EQ(builder.slice().get("_key").toString(), "0"); @@ -235,7 +270,7 @@ TEST_F(TraverserCacheTest, it_should_insert_an_edge_into_a_result_builder) { graph.addEdge(0, 1); gdb.addGraph(graph); - std::string edgeKey = "0-1"; // (EdgeKey format: -) + std::string edgeKey = "0-1"; // (EdgeKey format: -) std::shared_ptr col = gdb.vocbase.lookupCollection("e"); @@ -244,7 +279,8 @@ TEST_F(TraverserCacheTest, it_should_insert_an_edge_into_a_result_builder) { bool called = false; auto result = col->getPhysical()->read(trx.get(), arangodb::velocypack::StringRef{edgeKey}, - [&fetchedDocumentId, &called, &edgeKey](LocalDocumentId const& ldid, VPackSlice edgeDocument) { + [&fetchedDocumentId, &called, + &edgeKey](LocalDocumentId const& ldid, VPackSlice edgeDocument) { fetchedDocumentId = ldid.id(); called = true; EXPECT_TRUE(edgeDocument.isObject()); @@ -256,8 +292,8 @@ TEST_F(TraverserCacheTest, it_should_insert_an_edge_into_a_result_builder) { ASSERT_TRUE(result.ok()); ASSERT_NE(fetchedDocumentId, 0); - DataSourceId dataSourceId{col->id()}; // valid - LocalDocumentId localDocumentId{fetchedDocumentId}; // valid + DataSourceId dataSourceId{col->id()}; // valid + LocalDocumentId localDocumentId{fetchedDocumentId}; // valid EdgeDocumentToken edt{dataSourceId, localDocumentId}; VPackBuilder builder; @@ -270,7 +306,7 @@ TEST_F(TraverserCacheTest, it_should_insert_an_edge_into_a_result_builder) { // check stats EXPECT_EQ(stats.getHttpRequests(), 0); EXPECT_EQ(stats.getFiltered(), 0); - EXPECT_EQ(stats.getScannedIndex(), 0); // <- see note in method: appendEdge + EXPECT_EQ(stats.getScannedIndex(), 0); // <- see note in method: appendEdge } } // namespace traverser_cache_test diff --git a/tests/Graph/WeightedQueueTest.cpp b/tests/Graph/WeightedQueueTest.cpp new file mode 100644 index 000000000000..128bc1d68791 --- /dev/null +++ b/tests/Graph/WeightedQueueTest.cpp @@ -0,0 +1,213 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2020 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Heiko Kernbach +//////////////////////////////////////////////////////////////////////////////// + +#include "Basics/GlobalResourceMonitor.h" +#include "Basics/ResourceUsage.h" +#include "Basics/ResultT.h" +#include "Basics/StringUtils.h" +#include "Graph/Providers/BaseStep.h" +#include "Graph/Queues/WeightedQueue.h" + +#include "gtest/gtest.h" + +using namespace arangodb; +using namespace arangodb::graph; + +namespace arangodb { +namespace tests { +namespace queue_graph_cache_test { + +class Step : public arangodb::graph::BaseStep { + size_t _id; + bool _isLooseEnd; + + public: + Step(size_t id, double weight, bool isLooseEnd) + : arangodb::graph::BaseStep{0, 1, weight} { + _id = id; + _isLooseEnd = isLooseEnd; + }; + + ~Step() = default; + + Step(Step const& other) = default; + Step& operator=(Step const& other) = delete; + Step(Step&& other) = default; + Step& operator=(Step&& other) = default; + + bool operator==(Step const& other) { return _id == other._id; } + + bool isProcessable() const { return _isLooseEnd ? false : true; } + size_t id() const { return _id; } + std::string toString() { + return " _id: " + basics::StringUtils::itoa(static_cast(_id)) + + ", _weight: " + basics::StringUtils::ftoa(getWeight()); + } +}; + +class WeightedQueueTest : public ::testing::Test { + // protected: + public: + WeightedQueueTest() {} + ~WeightedQueueTest() {} + + public: + arangodb::GlobalResourceMonitor _global{}; + arangodb::ResourceMonitor _resourceMonitor{_global}; +}; + +TEST_F(WeightedQueueTest, it_should_be_empty_if_new_queue_initialized) { + auto queue = WeightedQueue(_resourceMonitor); + ASSERT_EQ(queue.size(), 0); + ASSERT_TRUE(queue.isEmpty()); +} + +TEST_F(WeightedQueueTest, it_should_contain_element_after_insertion) { + auto queue = WeightedQueue(_resourceMonitor); + auto step = Step{1, 1, false}; + queue.append(step); + ASSERT_EQ(queue.size(), 1); + ASSERT_FALSE(queue.isEmpty()); +} + +TEST_F(WeightedQueueTest, it_should_contain_zero_elements_after_clear) { + auto queue = WeightedQueue(_resourceMonitor); + queue.append(Step{1, 1, false}); + queue.append(Step{2, 4, false}); + queue.append(Step{3, 2, false}); + queue.append(Step{4, 3, true}); + ASSERT_EQ(queue.size(), 4); + queue.clear(); + ASSERT_TRUE(queue.isEmpty()); +} + +TEST_F(WeightedQueueTest, it_should_contain_processable_elements) { + auto queue = WeightedQueue(_resourceMonitor); + queue.append(Step{1, 5, false}); + queue.append(Step{2, 1, false}); + queue.append(Step{3, 2, true}); + queue.append(Step{4, 1.6, false}); + ASSERT_EQ(queue.size(), 4); + ASSERT_TRUE(queue.hasProcessableElement()); +} + +TEST_F(WeightedQueueTest, it_should_not_contain_processable_elements) { + auto queue = WeightedQueue(_resourceMonitor); + queue.append(Step{1, 4, true}); + queue.append(Step{2, 1.6, true}); + queue.append(Step{3, 1.2, true}); + queue.append(Step{4, 1.5, true}); + ASSERT_EQ(queue.size(), 4); + ASSERT_FALSE(queue.hasProcessableElement()); +} + +TEST_F(WeightedQueueTest, it_should_prioritize_processable_elements) { + // 2 and 3 have identical and smallest weight. + // 3 is processable, 2 not. + auto queue = WeightedQueue(_resourceMonitor); + queue.append(Step{1, 8, true}); + queue.append(Step{2, 2, true}); + queue.append(Step{3, 2, false}); + queue.append(Step{4, 6, false}); + EXPECT_EQ(queue.size(), 4); + EXPECT_TRUE(queue.hasProcessableElement()); + auto s = queue.pop(); + EXPECT_EQ(s.id(), 3); + EXPECT_FALSE(queue.hasProcessableElement()); + EXPECT_EQ(queue.size(), 3); +} + +TEST_F(WeightedQueueTest, it_should_order_by_asc_weight) { + // Random Input in random order. We shuffle it before each iteration, feel + // free to modify this in any way you like + std::vector input{{1, 1, false}, {2, 4, false}, {3, 6, false}, {4, 12, false}}; + // Some test orderings, feel free to add more orderings for tests. + std::vector>> orderings{ + {"DescWeight", + [](Step const& a, Step const& b) -> bool { + // DESC weight + return a.getWeight() > b.getWeight(); + }}, + {"AscWeight", + [](Step const& a, Step const& b) -> bool { + // ASC weight + return a.getWeight() < b.getWeight(); + }}, + {"RandomOrder", [](Step const& a, Step const& b) -> bool { + // RandomWeightOrder, first inject all uneven Steps, sort each "package" by ASC weight + // There is no specicial plan behind this, it is to stable "non"-sort by weight + auto modA = a.id() % 2; + auto modB = a.id() % 2; + if (modA != modB) { + return modA > modB; + } + return a.getWeight() < b.getWeight(); + }}}; + // No matter how the input is ordered, + // We need to get it back in exactly the same order, by asc weight + for (auto [name, o] : orderings) { + SCOPED_TRACE("Input ordered by " + name); + std::sort(input.begin(), input.end(), o); + auto queue = WeightedQueue(_resourceMonitor); + for (auto s : input) { + queue.append(s); + } + // We start with all inputs injected. + EXPECT_EQ(queue.size(), input.size()); + // Input is required + EXPECT_TRUE(queue.hasProcessableElement()); + // Smaller than anything + double weightBefore = -1.0; + + // Consume everything from the queue. + // It needs to be in increasing weight order. + while (queue.hasProcessableElement()) { + Step myStep = queue.pop(); + EXPECT_GE(myStep.getWeight(), weightBefore); + weightBefore = myStep.getWeight(); + } + // As all inputs are processable this queue shall be empty now. + ASSERT_EQ(queue.size(), 0); + ASSERT_FALSE(queue.hasProcessableElement()); + } +} + +TEST_F(WeightedQueueTest, it_should_pop_all_loose_ends) { + auto queue = WeightedQueue(_resourceMonitor); + queue.append(Step{2, 1.5, true}); + queue.append(Step{3, 5, true}); + queue.append(Step{1, 1, true}); + queue.append(Step{4, 6, true}); + EXPECT_EQ(queue.size(), 4); + EXPECT_FALSE(queue.hasProcessableElement()); + + std::vector myStepReferences = queue.getLooseEnds(); + EXPECT_EQ(myStepReferences.size(), 4); + + EXPECT_EQ(queue.size(), 4); + EXPECT_FALSE(queue.hasProcessableElement()); +} + +} // namespace queue_graph_cache_test +} // namespace tests +} // namespace arangodb diff --git a/tests/Graph/MockGraph.cpp b/tests/Mocks/MockGraph.cpp similarity index 81% rename from tests/Graph/MockGraph.cpp rename to tests/Mocks/MockGraph.cpp index 7dba1cc3273e..b239a54f782a 100644 --- a/tests/Graph/MockGraph.cpp +++ b/tests/Mocks/MockGraph.cpp @@ -76,19 +76,24 @@ void MockGraph::VertexDef::addToBuilder(arangodb::velocypack::Builder& builder) builder.close(); } -void MockGraph::addEdge(std::string from, std::string to, double weight) { - _edges.emplace_back(EdgeDef{from, to, weight, _edgeCollectionName}); +MockGraph::EdgeDef MockGraph::addEdge(std::string from, std::string to, double weight) { + EdgeDef newEdge {from, to, weight, _edgeCollectionName}; + _edges.emplace_back(newEdge); _vertices.emplace(std::move(from)); _vertices.emplace(std::move(to)); + + return newEdge; } -void MockGraph::addEdge(size_t from, size_t to, double weight) { - addEdge(getVertexCollectionName() + "/" + basics::StringUtils::itoa(from), +MockGraph::EdgeDef MockGraph::addEdge(size_t from, size_t to, double weight) { + return addEdge(getVertexCollectionName() + "/" + basics::StringUtils::itoa(from), getVertexCollectionName() + "/" + basics::StringUtils::itoa(to), weight); } void MockGraph::storeData(TRI_vocbase_t& vocbase, std::string const& vertexCollectionName, - std::string const& edgeCollectionName) const { + std::string const& edgeCollectionName, + std::string const& edgeCollectionSecondName, + std::vector const& secondEdges) const { { // Insert vertices arangodb::OperationOptions options; @@ -111,28 +116,36 @@ void MockGraph::storeData(TRI_vocbase_t& vocbase, std::string const& vertexColle EXPECT_TRUE(added == vertices().size()); } - { - // Insert edges - arangodb::OperationOptions options; - arangodb::SingleCollectionTransaction trx(arangodb::transaction::StandaloneContext::Create(vocbase), - edgeCollectionName, - arangodb::AccessMode::Type::WRITE); - EXPECT_TRUE((trx.begin().ok())); - size_t added = 0; - velocypack::Builder b; - for (auto& edge : edges()) { - b.clear(); - edge.addToBuilder(b); - auto res = trx.insert(edgeCollectionName, b.slice(), options); - if (res.fail()) { - LOG_DEVEL << res.errorMessage() << " " << b.toJson(); + auto insertEdges = [&](std::string const& edgeCollectionToInsert, + std::vector const& edges) -> void { + { + // Insert edges + arangodb::OperationOptions options; + arangodb::SingleCollectionTransaction trx( + arangodb::transaction::StandaloneContext::Create(vocbase), + edgeCollectionToInsert, arangodb::AccessMode::Type::WRITE); + EXPECT_TRUE((trx.begin().ok())); + size_t added = 0; + velocypack::Builder b; + for (auto& edge : edges) { + b.clear(); + edge.addToBuilder(b); + auto res = trx.insert(edgeCollectionToInsert, b.slice(), options); + if (res.fail()) { + LOG_DEVEL << res.errorMessage() << " " << b.toJson(); + } + EXPECT_TRUE((res.ok())); + added++; } - EXPECT_TRUE((res.ok())); - added++; + + EXPECT_TRUE((trx.commit().ok())); + EXPECT_TRUE(added == edges.size()); } + }; - EXPECT_TRUE((trx.commit().ok())); - EXPECT_TRUE(added == edges().size()); + insertEdges(edgeCollectionName, edges()); + if (!edgeCollectionSecondName.empty()) { + insertEdges(edgeCollectionSecondName, secondEdges); } } @@ -166,8 +179,6 @@ void MockGraph::prepareServer(MockCoordinator& server) const { getEdgeShardNameServerPairs(), TRI_COL_TYPE_EDGE); } - - template <> // Future: Also engineID's need to be returned here. std::pair, uint64_t> MockGraph::simulateApi( @@ -184,15 +195,15 @@ std::pair, uint64_t> MockG { // init restaqlhandler arangodb::tests::PreparedRequestResponse prep{server.getSystemDatabase()}; - + // generate and add body here VPackBuilder builder; builder.openObject(); builder.add("lockInfo", VPackValue(VPackValueType::Object)); builder.add("read", VPackValue(VPackValueType::Array)); - // append here the collection names (?) <-- TODO: Check RestAqlHandler.cpp:230 - // builder.add(VPackValue(_vertexCollectionName)); + // append here the collection names (?) <-- TODO: Check + // RestAqlHandler.cpp:230 builder.add(VPackValue(_vertexCollectionName)); // builder.add(VPackValue(_edgeCollectionName)); // appending collection shard ids for (auto const& vShard : _vertexShards) { @@ -201,8 +212,8 @@ std::pair, uint64_t> MockG for (auto const& eShard : _edgeShards) { builder.add(VPackValue(eShard.first)); } - builder.close(); // array READ - builder.close(); // object lockInfo + builder.close(); // array READ + builder.close(); // object lockInfo builder.add("options", VPackValue(VPackValueType::Object)); builder.add("ttl", VPackValue(120)); @@ -216,7 +227,7 @@ std::pair, uint64_t> MockG builder.add("traverserEngines", VPackValue(VPackValueType::Array)); - builder.openObject(); // main container + builder.openObject(); // main container builder.add(VPackValue("options")); @@ -228,25 +239,25 @@ std::pair, uint64_t> MockG builder.add(VPackValue("vertices")); builder.openObject(); - for (auto const& vertexTuple: getVertexShardNameServerPairs()) { + for (auto const& vertexTuple : getVertexShardNameServerPairs()) { builder.add(_vertexCollectionName, VPackValue(VPackValueType::Array)); - builder.add(VPackValue(vertexTuple.first)); // shardID - builder.close(); // inner array + builder.add(VPackValue(vertexTuple.first)); // shardID + builder.close(); // inner array } - builder.close(); // vertices + builder.close(); // vertices builder.add(VPackValue("edges")); builder.openArray(); - for (auto const& edgeTuple: getEdgeShardNameServerPairs()) { + for (auto const& edgeTuple : getEdgeShardNameServerPairs()) { builder.openArray(); - builder.add(VPackValue(edgeTuple.first)); // shardID - builder.close(); // inner array + builder.add(VPackValue(edgeTuple.first)); // shardID + builder.close(); // inner array } - builder.close(); // edges - builder.close(); // shards - builder.close(); // main container - builder.close(); // array traverserEngines + builder.close(); // edges + builder.close(); // shards + builder.close(); // main container + builder.close(); // array traverserEngines builder.close(); // object (outer) prep.addBody(builder.slice()); diff --git a/tests/Graph/MockGraph.h b/tests/Mocks/MockGraph.h similarity index 92% rename from tests/Graph/MockGraph.h rename to tests/Mocks/MockGraph.h index 3ea12b4cdc9f..bc994d076dd1 100644 --- a/tests/Graph/MockGraph.h +++ b/tests/Mocks/MockGraph.h @@ -87,8 +87,8 @@ class MockGraph { MockGraph() {} ~MockGraph() {} - void addEdge(std::string from, std::string to, double weight = 1.0); - void addEdge(size_t from, size_t to, double weight = 1.0); + EdgeDef addEdge(std::string from, std::string to, double weight = 1.0); + EdgeDef addEdge(size_t from, size_t to, double weight = 1.0); auto edges() const -> std::vector const& { return _edges; } auto vertices() const -> std::unordered_set const& { @@ -121,9 +121,11 @@ class MockGraph { arangodb::graph::BaseOptions& opts) const; void storeData(TRI_vocbase_t& vocbase, std::string const& vertexCollectionName, - std::string const& edgeCollectionName) const; + std::string const& edgeCollectionName, + std::string const& edgeCollectionSecondName = "", + std::vector const& secondEdges = {}) const; - private: + protected: std::vector> const& getVertexShardNameServerPairs() const { return _vertexShards; } @@ -131,13 +133,13 @@ class MockGraph { return _edgeShards; } - private: + protected: std::vector _edges; std::unordered_set _vertices; - std::string _vertexCollectionName{"v"}; std::string _edgeCollectionName{"e"}; + private: std::vector> _vertexShards{ {"s9870", "PRMR_0001"}}; std::vector> _edgeShards{ diff --git a/tests/Graph/MockGraphProvider.cpp b/tests/Mocks/MockGraphProvider.cpp similarity index 90% rename from tests/Graph/MockGraphProvider.cpp rename to tests/Mocks/MockGraphProvider.cpp index 60d6bc3f3539..8ec9f1b7552f 100644 --- a/tests/Graph/MockGraphProvider.cpp +++ b/tests/Mocks/MockGraphProvider.cpp @@ -59,6 +59,12 @@ MockGraphProvider::Step::Step(size_t prev, VertexType v, EdgeType e, bool isProc _edge(e), _isProcessable(isProcessable) {} +MockGraphProvider::Step::Step(size_t prev, VertexType v, bool isProcessable, size_t depth) + : arangodb::graph::BaseStep{prev, depth}, + _vertex(v), + _edge({}), + _isProcessable(isProcessable) {} + MockGraphProvider::Step::Step(size_t prev, VertexType v, EdgeType e, bool isProcessable, size_t depth) : arangodb::graph::BaseStep{prev, depth}, @@ -66,13 +72,11 @@ MockGraphProvider::Step::Step(size_t prev, VertexType v, EdgeType e, _edge(e), _isProcessable(isProcessable) {} -MockGraphProvider::Step::~Step() {} - -MockGraphProvider::MockGraphProvider(MockGraph const& data, - arangodb::aql::QueryContext& queryContext, - LooseEndBehaviour looseEnds, bool reverse) - : _trx(queryContext.newTrxContext()), _reverse(reverse), _looseEnds(looseEnds), _stats{} { - for (auto const& it : data.edges()) { +MockGraphProvider::MockGraphProvider(arangodb::aql::QueryContext& queryContext, + MockGraphProviderOptions opts, + arangodb::ResourceMonitor&) + : _trx(queryContext.newTrxContext()), _reverse(opts.reverse()), _looseEnds(opts.looseEnds()), _stats{} { + for (auto const& it : opts.data().edges()) { _fromIndex[it._from].push_back(it); _toIndex[it._to].push_back(it); } @@ -91,9 +95,10 @@ auto MockGraphProvider::decideProcessable() const -> bool { } } -auto MockGraphProvider::startVertex(VertexType v) -> Step { +auto MockGraphProvider::startVertex(VertexType v, size_t depth, double weight) -> Step { LOG_TOPIC("78156", TRACE, Logger::GRAPHS) << " Start Vertex:" << v; + TRI_ASSERT(weight == 0.0); // Not handled yet return Step(v, decideProcessable()); } diff --git a/tests/Graph/MockGraphProvider.h b/tests/Mocks/MockGraphProvider.h similarity index 75% rename from tests/Graph/MockGraphProvider.h rename to tests/Mocks/MockGraphProvider.h index b6296cd12649..4297fd713eb2 100644 --- a/tests/Graph/MockGraphProvider.h +++ b/tests/Mocks/MockGraphProvider.h @@ -56,18 +56,39 @@ class HashedStringRef; namespace tests { namespace graph { + +class MockGraphProviderOptions { + public: + enum class LooseEndBehaviour { NEVER, ALWAYS }; + MockGraphProviderOptions(MockGraph const& data, LooseEndBehaviour looseEnds, + bool reverse = false) + : _data(data), _looseEnds(looseEnds), _reverse(reverse) {} + ~MockGraphProviderOptions() = default; + + LooseEndBehaviour looseEnds() const { return _looseEnds; } + MockGraph const& data() const { return _data; } + bool reverse() const { return _reverse; } + + private: + MockGraph const& _data; + LooseEndBehaviour _looseEnds; + bool _reverse; + ; +}; + class MockGraphProvider { using VertexType = arangodb::velocypack::HashedStringRef; using EdgeType = MockGraph::EdgeDef; public: - enum class LooseEndBehaviour { NEVER, ALWAYS }; + using Options = MockGraphProviderOptions; + using LooseEndBehaviour = typename MockGraphProviderOptions::LooseEndBehaviour; class Step : public arangodb::graph::BaseStep { public: class Vertex { public: - explicit Vertex(VertexType v) : _vertex(v) {}; + explicit Vertex(VertexType v) : _vertex(v){}; VertexType getID() const { return _vertex; } @@ -106,8 +127,9 @@ class MockGraphProvider { Step(VertexType v, bool isProcessable); Step(size_t prev, VertexType v, EdgeType e, bool isProcessable); + Step(size_t prev, VertexType v, bool isProcessable, size_t depth); Step(size_t prev, VertexType v, EdgeType e, bool isProcessable, size_t depth); - ~Step(); + ~Step() = default; bool operator<(Step const& other) const noexcept { return _vertex < other._vertex; @@ -124,6 +146,8 @@ class MockGraphProvider { } } + bool isResponsible(transaction::Methods* trx) const { return true; } + Vertex getVertex() const { /*if (!isProcessable()) { THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, @@ -144,6 +168,26 @@ class MockGraphProvider { VertexType getVertexIdentifier() const { return getVertex().getID(); } + std::string getCollectionName() const { + auto collectionNameResult = extractCollectionName(_vertex.getID()); + if (collectionNameResult.fail()) { + THROW_ARANGO_EXCEPTION(collectionNameResult.result()); + } + return collectionNameResult.get().first; + }; + + void setLocalSchreierIndex(size_t index) { + TRI_ASSERT(index != std::numeric_limits::max()); + TRI_ASSERT(!hasLocalSchreierIndex()); + _localSchreierIndex = index; + } + + bool hasLocalSchreierIndex() const { + return _localSchreierIndex != std::numeric_limits::max(); + } + + std::size_t getLocalSchreierIndex() const { return _localSchreierIndex; } + bool isProcessable() const { return _isProcessable; } bool isLooseEnd() const { return !isProcessable(); } @@ -159,11 +203,13 @@ class MockGraphProvider { Vertex _vertex; Edge _edge; bool _isProcessable; + size_t _localSchreierIndex; }; MockGraphProvider() = delete; - MockGraphProvider(MockGraph const& data, arangodb::aql::QueryContext& queryContext, - LooseEndBehaviour looseEnds, bool reverse = false); + MockGraphProvider(arangodb::aql::QueryContext& queryContext, Options opts, + arangodb::ResourceMonitor& resourceMonitor); + MockGraphProvider(MockGraphProvider const&) = delete; // TODO: check "Rule of 5" MockGraphProvider(MockGraphProvider&&) = default; ~MockGraphProvider(); @@ -172,7 +218,7 @@ class MockGraphProvider { MockGraphProvider& operator=(MockGraphProvider&&) = default; void destroyEngines(){}; - auto startVertex(VertexType vertex) -> Step; + auto startVertex(VertexType vertex, size_t depth = 0, double weight = 0.0) -> Step; auto fetch(std::vector const& looseEnds) -> futures::Future>; auto expand(Step const& from, size_t previous) -> std::vector; auto expand(Step const& from, size_t previous, std::function callback) -> void; diff --git a/tests/Mocks/Servers.cpp b/tests/Mocks/Servers.cpp index 632fc5dea1a1..c719e64d507c 100644 --- a/tests/Mocks/Servers.cpp +++ b/tests/Mocks/Servers.cpp @@ -35,8 +35,8 @@ #include "Aql/ExecutionEngine.h" #include "Aql/OptimizerRulesFeature.h" #include "Aql/Query.h" -#include "Basics/files.h" #include "Basics/StringUtils.h" +#include "Basics/files.h" #include "Cluster/ActionDescription.h" #include "Cluster/AgencyCache.h" #include "Cluster/ClusterFeature.h" @@ -81,7 +81,6 @@ #include "VocBase/vocbase.h" #include "utils/log.hpp" - #include "Servers.h" #include "TemplateSpecializer.h" @@ -124,6 +123,7 @@ static void SetupDatabaseFeaturePhase(MockServer& server) { SetupBasicFeaturePhase(server); server.addFeature(false); // true ?? server.addFeature(true); + server.addFeature(false); server.addFeature(false); server.addFeature(false); server.addFeature(false); @@ -141,7 +141,7 @@ static void SetupClusterFeaturePhase(MockServer& server) { SetupDatabaseFeaturePhase(server); server.addFeature(false); server.addFeature(false); - + // fake the exit code with which unresolved futures are returned on // shutdown. if we don't do this lots of places in ClusterInfo will // report failures during testing @@ -181,18 +181,26 @@ static void SetupAqlPhase(MockServer& server) { #endif } -MockServer::MockServer() +MockServer::MockServer(arangodb::ServerState::RoleEnum myRole, bool injectClusterIndexes) : _server(std::make_shared("", "", "", nullptr), nullptr), - _engine(_server), + _engine(_server, injectClusterIndexes), _oldRebootId(0), _started(false) { + _oldRole = arangodb::ServerState::instance()->getRole(); + arangodb::ServerState::instance()->setRole(myRole); + _originalMockingState = arangodb::ClusterEngine::Mocking; + if (injectClusterIndexes && arangodb::ServerState::instance()->isCoordinator()) { + arangodb::ClusterEngine::Mocking = true; + } init(); } MockServer::~MockServer() { stopFeatures(); _server.setStateUnsafe(_oldApplicationServerState); - + + arangodb::ClusterEngine::Mocking = _originalMockingState; + arangodb::ServerState::instance()->setRole(_oldRole); arangodb::ServerState::instance()->setRebootId(_oldRebootId); } @@ -207,9 +215,9 @@ void MockServer::init() { _server.setStateUnsafe(arangodb::application_features::ApplicationServer::State::IN_WAIT); arangodb::transaction::Methods::clearDataSourceRegistrationCallbacks(); - // many other places rely on the reboot id being initialized, + // many other places rely on the reboot id being initialized, // so we do it here in a central place - arangodb::ServerState::instance()->setRebootId(arangodb::RebootId{1}); + arangodb::ServerState::instance()->setRebootId(arangodb::RebootId{1}); } void MockServer::startFeatures() { @@ -258,7 +266,7 @@ void MockServer::startFeatures() { for (ApplicationFeature& f : orderedFeatures) { auto info = _features.find(&f); // We only start those features... - if(info != _features.end()) { + if (info != _features.end()) { if (info->second) { try { f.start(); @@ -297,7 +305,7 @@ void MockServer::stopFeatures() { for (ApplicationFeature& f : orderedFeatures) { auto info = _features.find(&f); // We only start those features... - if(info != _features.end()) { + if (info != _features.end()) { if (info->second) { try { f.stop(); @@ -312,7 +320,7 @@ void MockServer::stopFeatures() { for (ApplicationFeature& f : orderedFeatures) { auto info = _features.find(&f); // We only start those features... - if(info != _features.end()) { + if (info != _features.end()) { try { f.unprepare(); } catch (...) { @@ -454,10 +462,10 @@ std::pair, consensus::index_t> AgencyCache:: consensus::Store& AgencyCache::store() { return _readDB; } -MockClusterServer::MockClusterServer(bool useAgencyMockPool) - : MockServer(), _useAgencyMockPool(useAgencyMockPool) { - _oldRole = arangodb::ServerState::instance()->getRole(); - +MockClusterServer::MockClusterServer(bool useAgencyMockPool, + arangodb::ServerState::RoleEnum newRole, + bool injectClusterIndexes) + : MockServer(newRole, injectClusterIndexes), _useAgencyMockPool(useAgencyMockPool) { // Add features SetupAqlPhase(*this); @@ -478,7 +486,6 @@ MockClusterServer::~MockClusterServer() { ci.shutdownSyncers(); ci.waitForSyncersToStop(); _server.getFeature().shutdownAgencyCache(); - arangodb::ServerState::instance()->setRole(_oldRole); } void MockClusterServer::startFeatures() { @@ -518,6 +525,32 @@ void MockClusterServer::startFeatures() { _server.getFeature().clusterInfo().startSyncers(); } +std::unique_ptr MockClusterServer::createFakeQuery( + bool activateTracing, std::string queryString, + std::function callback) const { + auto bindParams = std::make_shared(); + bindParams->openObject(); + bindParams->close(); + VPackBuilder queryOptions; + queryOptions.openObject(); + if (activateTracing) { + queryOptions.add("profile", VPackValue(int(aql::ProfileLevel::TraceTwo))); + } + queryOptions.close(); + if (queryString.empty()) { + queryString = "RETURN 1"; + } + + aql::QueryString fakeQueryString(queryString); + auto query = std::make_unique( + arangodb::transaction::StandaloneContext::Create(getSystemDatabase()), + fakeQueryString, bindParams, queryOptions.slice()); + callback(*query); + query->prepareQuery(aql::SerializationFormat::SHADOWROWS); + + return query; +} + consensus::index_t MockClusterServer::agencyTrx(std::string const& key, std::string const& value) { // Build an agency transaction: @@ -545,10 +578,14 @@ void MockClusterServer::agencyCreateDatabase(std::string const& name) { st = ts.specialize(current_dbs_string); agencyTrx("/arango/Current/Databases/" + name, st); - _server.getFeature().clusterInfo().waitForPlan( - agencyTrx("/arango/Plan/Version", R"=({"op":"increment"})=")).wait(); - _server.getFeature().clusterInfo().waitForCurrent( - agencyTrx("/arango/Current/Version", R"=({"op":"increment"})=")).wait(); + _server.getFeature() + .clusterInfo() + .waitForPlan(agencyTrx("/arango/Plan/Version", R"=({"op":"increment"})=")) + .wait(); + _server.getFeature() + .clusterInfo() + .waitForCurrent(agencyTrx("/arango/Current/Version", R"=({"op":"increment"})=")) + .wait(); } void MockClusterServer::agencyCreateCollections(std::string const& name) { @@ -586,23 +623,10 @@ void MockClusterServer::agencyDropDatabase(std::string const& name) { .wait(); } -// Create a clusterWide Collection. -// This does NOT create Shards. -std::shared_ptr MockClusterServer::createCollection( - std::string const& dbName, std::string collectionName, - std::vector> shardNameToServerNamePairs, - TRI_col_type_e type) { - /* - std::string cID, uint64_t shards, - uint64_t replicationFactor, uint64_t writeConcern, - bool waitForRep, velocypack::Slice const& slice, - std::string coordinatorId, RebootId rebootId */ - // This is unsafe - std::string cid = "98765" + basics::StringUtils::itoa(type); - auto& databaseFeature = _server.getFeature(); - auto vocbase = databaseFeature.lookupDatabase(dbName); - - VPackBuilder props; +void MockClusterServer::buildCollectionProperties(VPackBuilder& props, + std::string const& collectionName, + std::string const& cid, TRI_col_type_e type, + VPackSlice additionalProperties) { { // This is hand-crafted unfortunately the code does not exist... VPackObjectBuilder guard(&props); @@ -614,43 +638,39 @@ std::shared_ptr MockClusterServer::createCollection( { VPackArrayBuilder guard2(&props); auto const primIndex = arangodb::velocypack::Parser::fromJson( - R"({"id":"0","type":"primary","name": + R"({"id":"0","type":"primary","name": "primary","fields":["_key"],"unique":true,"sparse":false })"); -props.add(primIndex->slice()); -if (type == TRI_COL_TYPE_EDGE) { - auto const fromIndex = arangodb::velocypack::Parser::fromJson( - R"({"id":"1","type":"edge","name": + props.add(primIndex->slice()); + if (type == TRI_COL_TYPE_EDGE) { + auto const fromIndex = arangodb::velocypack::Parser::fromJson( + R"({"id":"1","type":"edge","name": "edge_from","fields":["_from"],"unique":false,"sparse":false })"); -props.add(fromIndex->slice()); -auto const toIndex = arangodb::velocypack::Parser::fromJson( -R"({"id":"2","type":"edge","name": + props.add(fromIndex->slice()); + auto const toIndex = arangodb::velocypack::Parser::fromJson( + R"({"id":"2","type":"edge","name": "edge_to","fields":["_to"],"unique":false,"sparse":false})"); -props.add(toIndex->slice()); -} + props.add(toIndex->slice()); + } } - } - LogicalCollection dummy(*vocbase, props.slice(), true); - - auto shards = std::make_shared(); - for (auto const& [shard, server] : shardNameToServerNamePairs) { - shards->emplace(shard, std::vector{server}); - } - dummy.setShardMap(shards); - std::unordered_set const ignoreKeys{ - "allowUserKeys", "cid", "globallyUniqueId", "count", - "planId", "version", "objectId"}; - dummy.setStatus(TRI_VOC_COL_STATUS_LOADED); - VPackBuilder velocy = - dummy.toVelocyPackIgnore(ignoreKeys, LogicalDataSource::Serialization::List); + if (additionalProperties.isObject()) { + props.add(VPackObjectIterator(additionalProperties)); + } + } +} - agencyTrx("/arango/Plan/Collections/" + dbName + "/" + basics::StringUtils::itoa(dummy.planId().id()), velocy.toJson()); +void MockClusterServer::injectCollectionToAgency( + std::string const& dbName, VPackBuilder& velocy, DataSourceId const& planId, + std::vector> shardNameToServerNamePairs) { + agencyTrx("/arango/Plan/Collections/" + dbName + "/" + + basics::StringUtils::itoa(planId.id()), + velocy.toJson()); { - /* Hard-Coded section to inject the CURRENT counter part. - * We do not have a shard available here that could generate the values accordingly. - */ + /* Hard-Coded section to inject the CURRENT counter part. + * We do not have a shard available here that could generate the values accordingly. + */ VPackBuilder current; { VPackObjectBuilder report(¤t); @@ -674,7 +694,9 @@ props.add(toIndex->slice()); // NOTE: we omited Indexes } } - agencyTrx("/arango/Current/Collections/" + dbName + "/" + basics::StringUtils::itoa(dummy.planId().id()), current.toJson()); + agencyTrx("/arango/Current/Collections/" + dbName + "/" + + basics::StringUtils::itoa(planId.id()), + current.toJson()); } _server.getFeature() @@ -686,14 +708,43 @@ props.add(toIndex->slice()); .clusterInfo() .waitForCurrent(agencyTrx("/arango/Current/Version", R"=({"op":"increment"})=")) .wait(); +} + +// Create a clusterWide Collection. +// This does NOT create Shards. +std::shared_ptr MockClusterServer::createCollection( + std::string const& dbName, std::string collectionName, + std::vector> shardNameToServerNamePairs, + TRI_col_type_e type, VPackSlice additionalProperties) { + std::string cid = + std::to_string(_server.getFeature().clusterInfo().uniqid()); + auto& databaseFeature = _server.getFeature(); + auto vocbase = databaseFeature.lookupDatabase(dbName); + + VPackBuilder props; + buildCollectionProperties(props, collectionName, cid, type, additionalProperties); + LogicalCollection dummy(*vocbase, props.slice(), true); + + auto shards = std::make_shared(); + for (auto const& [shard, server] : shardNameToServerNamePairs) { + shards->emplace(shard, std::vector{server}); + } + dummy.setShardMap(shards); + + std::unordered_set const ignoreKeys{ + "allowUserKeys", "cid", "globallyUniqueId", "count", + "planId", "version", "objectId"}; + dummy.setStatus(TRI_VOC_COL_STATUS_LOADED); + VPackBuilder velocy = + dummy.toVelocyPackIgnore(ignoreKeys, LogicalDataSource::Serialization::List); + injectCollectionToAgency(dbName, velocy, dummy.planId(), shardNameToServerNamePairs); ClusterInfo& clusterInfo = server().getFeature().clusterInfo(); return clusterInfo.getCollection(dbName, collectionName); } MockDBServer::MockDBServer(bool start, bool useAgencyMock) - : MockClusterServer(useAgencyMock) { - arangodb::ServerState::instance()->setRole(arangodb::ServerState::RoleEnum::ROLE_DBSERVER); + : MockClusterServer(useAgencyMock, arangodb::ServerState::RoleEnum::ROLE_DBSERVER) { addFeature(false); // do not start the thread addFeature(false); // do not start the thread if (start) { @@ -754,11 +805,14 @@ void MockDBServer::createShard(std::string const& dbName, std::string shardName, #ifndef USE_ENTERPRISE props->add(StaticStrings::ReplicationFactor, VPackValue(1)); #endif + props->add(StaticStrings::InternalValidatorTypes, + VPackValue(clusterCollection.getInternalValidatorTypes())); } maintenance::ActionDescription ad( std::map{{maintenance::NAME, maintenance::CREATE_COLLECTION}, {maintenance::COLLECTION, - basics::StringUtils::itoa(clusterCollection.planId().id())}, + basics::StringUtils::itoa( + clusterCollection.planId().id())}, {maintenance::SHARD, shardName}, {maintenance::DATABASE, dbName}, {maintenance::SERVER_ID, "PRMR_0001"}, @@ -805,9 +859,9 @@ void MockDBServer::createShard(std::string const& dbName, std::string shardName, } } -MockCoordinator::MockCoordinator(bool start, bool useAgencyMock) - : MockClusterServer(useAgencyMock) { - arangodb::ServerState::instance()->setRole(arangodb::ServerState::RoleEnum::ROLE_COORDINATOR); +MockCoordinator::MockCoordinator(bool start, bool useAgencyMock, bool injectClusterIndexes) + : MockClusterServer(useAgencyMock, arangodb::ServerState::RoleEnum::ROLE_COORDINATOR, + injectClusterIndexes) { if (start) { MockCoordinator::startFeatures(); MockCoordinator::createDatabase("_system"); @@ -824,7 +878,7 @@ std::pair MockCoordinator::registerFakedDBServer(std:: { VPackObjectBuilder b(&builder); builder.add("endpoint", VPackValue(fakedEndpoint)); - builder.add("advertisedEndpoint",VPackValue(fakedEndpoint)); + builder.add("advertisedEndpoint", VPackValue(fakedEndpoint)); builder.add("host", VPackValue(fakedHost)); builder.add("version", VPackValue(rest::Version::getNumericServerVersion())); builder.add("versionString", VPackValue(rest::Version::getServerVersion())); diff --git a/tests/Mocks/Servers.h b/tests/Mocks/Servers.h index a5133c2a4ac6..bc6b7471e1ae 100644 --- a/tests/Mocks/Servers.h +++ b/tests/Mocks/Servers.h @@ -58,7 +58,13 @@ namespace mocks { /// @brief mock application server with no features added class MockServer { public: - MockServer(); + // Note, setting injectClusterIndexes causes the "create" methods to fail. + // this is all hardly worked around the Cluster engine and needs a proper + // clean up. It is highly recommended to not set injectClusterIndexes unless + // you want to specificly test something that selects an index, but cannot use + // it. Use with care for now. + MockServer(arangodb::ServerState::RoleEnum = arangodb::ServerState::RoleEnum::ROLE_SINGLE, + bool injectClusterIndexes = false); virtual ~MockServer(); application_features::ApplicationServer& server(); @@ -121,6 +127,8 @@ class MockServer { private: bool _started; + arangodb::ServerState::RoleEnum _oldRole; + bool _originalMockingState; }; /// @brief a server with almost no features added (Metrics are available @@ -202,30 +210,52 @@ class MockClusterServer : public MockServer, std::shared_ptr createCollection( std::string const& dbName, std::string collectionName, std::vector> shardNameToServerNamePairs, - TRI_col_type_e type); + TRI_col_type_e type, + VPackSlice additionalProperties = VPackSlice{VPackSlice::nullSlice()}); + +#ifdef USE_ENTERPRISE + std::shared_ptr createSmartCollection( + std::string const& dbName, std::string collectionName, + std::vector> shardNameToServerNamePairs, + TRI_col_type_e type, + VPackSlice additionalProperties = VPackSlice{VPackSlice::nullSlice()}); +#endif + void buildCollectionProperties(VPackBuilder& props, std::string const& collectionName, + std::string const& cid, TRI_col_type_e type, + VPackSlice additionalProperties); + + void injectCollectionToAgency(std::string const& dbName, VPackBuilder& velocy, + DataSourceId const& planId, + std::vector> shardNameToServerNamePairs); + + std::unique_ptr createFakeQuery( + bool activateTracing = false, std::string queryString = "", + std::function runBeforePrepare = + [](arangodb::aql::Query&) {}) const; // You can only create specialized types protected: - MockClusterServer(bool useAgencyMockConnection); + MockClusterServer(bool useAgencyMockConnection, arangodb::ServerState::RoleEnum role, + bool injectClusterIndexes = false); ~MockClusterServer(); protected: // Implementation knows the place when all features are included consensus::index_t agencyTrx(std::string const& key, std::string const& value); void agencyCreateDatabase(std::string const& name); - + // creation of collection is separated // as for DBerver at first maintenance should // create database and only after collections // will be populated in plan. void agencyCreateCollections(std::string const& name); - void agencyDropDatabase(std::string const& name); -protected: + + protected: std::unique_ptr _pool; + private: bool _useAgencyMockPool; - arangodb::ServerState::RoleEnum _oldRole; int _dummy; }; @@ -243,7 +273,8 @@ class MockDBServer : public MockClusterServer { class MockCoordinator : public MockClusterServer { public: - MockCoordinator(bool startFeatures = true, bool useAgencyMockConnection = true); + MockCoordinator(bool startFeatures = true, bool useAgencyMockConnection = true, + bool injectClusterIndexes = false); ~MockCoordinator(); TRI_vocbase_t* createDatabase(std::string const& name) override; diff --git a/tests/Mocks/StorageEngineMock.cpp b/tests/Mocks/StorageEngineMock.cpp index b3749d43c4a9..4b8f8e3fad11 100644 --- a/tests/Mocks/StorageEngineMock.cpp +++ b/tests/Mocks/StorageEngineMock.cpp @@ -33,6 +33,7 @@ #include "Cluster/ClusterFeature.h" #include "Cluster/ClusterInfo.h" #include "ClusterEngine/ClusterEngine.h" +#include "ClusterEngine/ClusterIndexFactory.h" #include "IResearch/IResearchCommon.h" #include "IResearch/IResearchFeature.h" #include "IResearch/IResearchLinkCoordinator.h" @@ -101,19 +102,19 @@ class EdgeIndexIteratorMock final : public arangodb::IndexIterator { typedef std::unordered_multimap Map; EdgeIndexIteratorMock(arangodb::LogicalCollection* collection, - arangodb::transaction::Methods* trx, arangodb::Index const* index, - Map const& map, std::unique_ptr&& keys) + arangodb::transaction::Methods* trx, + arangodb::Index const* index, Map const& map, + std::unique_ptr&& keys, bool isFrom) : IndexIterator(collection, trx, arangodb::ReadOwnWrites::no), _map(map), - _begin(_map.begin()), + _begin(_map.end()), _end(_map.end()), _keys(std::move(keys)), - _keysIt(_keys->slice()) {} - - char const* typeName() const override { return "edge-index-iterator-mock"; } + _keysIt(_keys->slice()), + _isFrom(isFrom) {} - bool nextImpl(LocalDocumentIdCallback const& cb, size_t limit) override { - while (limit && _begin != _end && _keysIt.valid()) { + bool prepareNextRange() { + if (_keysIt.valid()) { auto key = _keysIt.value(); if (key.isObject()) { @@ -121,16 +122,63 @@ class EdgeIndexIteratorMock final : public arangodb::IndexIterator { } std::tie(_begin, _end) = _map.equal_range(key.toString()); + _keysIt++; + return true; + } else { + // Just make sure begin and end are equal + _begin = _map.end(); + _end = _map.end(); + return false; + } + } - while (limit && _begin != _end) { - cb(_begin->second); - ++_begin; - --limit; - } + char const* typeName() const override { return "edge-index-iterator-mock"; } - ++_keysIt; + bool hasExtra() const override { return true; } + + bool nextImpl(LocalDocumentIdCallback const& cb, size_t limit) override { + // We can at most return limit + for (size_t l = 0; l < limit; ++l) { + while (_begin == _end) { + if (!prepareNextRange()) { + return false; + } + } + TRI_ASSERT(_begin != _end); + cb(_begin->second); + ++_begin; } - return _begin != _end && _keysIt.valid(); + // Returned due to limit. + if (_begin == _end) { + // Out limit hit the last index entry + // Return false if we do not have another range + return prepareNextRange(); + } + return true; + } + + bool nextExtraImpl(ExtraCallback const& cb, size_t limit) override { + auto res = nextImpl( + [&](arangodb::LocalDocumentId docId) -> bool { + auto res = _collection->getPhysical()->read( + _trx, docId, + [&](arangodb::LocalDocumentId const& token, arangodb::velocypack::Slice doc) -> bool { + // The nextExtra API in EdgeIndex will deliver the _id value of + // the oposite vertex. We simulate that here by reading the real + // document one more time and just extracting this value. + if (_isFrom) { + return cb(token, doc.get(arangodb::StaticStrings::ToString)); + } else { + return cb(token, doc.get(arangodb::StaticStrings::FromString)); + } + }, arangodb::ReadOwnWrites::no); + // We can only have good responses here. + // Otherwise storage and Index do differ + TRI_ASSERT(res.ok()); + return res.ok(); + }, + limit); + return res; } void resetImpl() override { @@ -145,6 +193,7 @@ class EdgeIndexIteratorMock final : public arangodb::IndexIterator { Map::const_iterator _end; std::unique_ptr _keys; arangodb::velocypack::ArrayIterator _keysIt; + bool _isFrom; }; // EdgeIndexIteratorMock class EdgeIndexMock final : public arangodb::Index { @@ -343,7 +392,7 @@ class EdgeIndexMock final : public arangodb::Index { return std::make_unique(&_collection, trx, this, isFrom ? _edgesFrom : _edgesTo, - std::move(keys)); + std::move(keys), isFrom); } /// @brief create the iterator @@ -373,7 +422,7 @@ class EdgeIndexMock final : public arangodb::Index { return std::make_unique(&_collection, trx, this, isFrom ? _edgesFrom : _edgesTo, - std::move(keys)); + std::move(keys), isFrom); } /// @brief the hash table for _from @@ -433,8 +482,13 @@ class AllIteratorMock final : public arangodb::IndexIterator { }; // AllIteratorMock struct IndexFactoryMock : arangodb::IndexFactory { - IndexFactoryMock(arangodb::application_features::ApplicationServer& server) - : IndexFactory(server) {} + IndexFactoryMock(arangodb::application_features::ApplicationServer& server, bool injectClusterIndexes) + : IndexFactory(server) { + if (injectClusterIndexes) { + arangodb::ClusterIndexFactory::linkIndexFactories(server, *this); + } + } + virtual void fillSystemIndexes(arangodb::LogicalCollection& col, std::vector>& systemIndexes) const override { // NOOP @@ -457,14 +511,17 @@ class HashIndexMap { struct VPackBuilderComparator { bool operator()(VPackBuilder const& builder1, VPackBuilder const& builder2) const { - return ::arangodb::basics::VelocyPackHelper::compare(builder1.slice(), builder2.slice(), true) == 0; + return ::arangodb::basics::VelocyPackHelper::compare(builder1.slice(), + builder2.slice(), true) == 0; } }; - using ValueMap = std::unordered_multimap; + using ValueMap = + std::unordered_multimap; using DocumentsIndexMap = std::unordered_map; - arangodb::velocypack::Slice getSliceByField(arangodb::velocypack::Slice const& doc, size_t i) { + arangodb::velocypack::Slice getSliceByField(arangodb::velocypack::Slice const& doc, + size_t i) { TRI_ASSERT(i < _fields.size()); TRI_ASSERT(!doc.isNone()); auto slice = doc; @@ -477,7 +534,8 @@ class HashIndexMap { return slice; } - void insertSlice(arangodb::LocalDocumentId const& documentId, arangodb::velocypack::Slice const& slice, size_t i) { + void insertSlice(arangodb::LocalDocumentId const& documentId, + arangodb::velocypack::Slice const& slice, size_t i) { VPackBuilder builder; if (slice.isNone() || slice.isNull()) { builder.add(VPackSlice::nullSlice()); @@ -488,11 +546,13 @@ class HashIndexMap { } public: - HashIndexMap(std::vector> const& fields) : _fields(fields), _valueMaps(fields.size()) { + HashIndexMap(std::vector> const& fields) + : _fields(fields), _valueMaps(fields.size()) { TRI_ASSERT(!_fields.empty()); } - void insert(arangodb::LocalDocumentId const& documentId, arangodb::velocypack::Slice const& doc) { + void insert(arangodb::LocalDocumentId const& documentId, + arangodb::velocypack::Slice const& doc) { VPackBuilder builder; builder.openArray(); auto toClose = true; @@ -512,14 +572,16 @@ class HashIndexMap { if (slice.isNone() || slice.isNull()) { break; } - } else { // expansion + } else { // expansion isExpansion = slice.isArray(); TRI_ASSERT(isExpansion); auto found = false; - for (auto sliceIt = arangodb::velocypack::ArrayIterator(slice); sliceIt != sliceIt.end(); ++sliceIt) { + for (auto sliceIt = arangodb::velocypack::ArrayIterator(slice); + sliceIt != sliceIt.end(); ++sliceIt) { auto subSlice = sliceIt.value(); if (!(subSlice.isNone() || subSlice.isNull())) { - for (auto fieldItForArray = fieldIt; fieldItForArray != _fields[i].end(); ++fieldItForArray) { + for (auto fieldItForArray = fieldIt; + fieldItForArray != _fields[i].end(); ++fieldItForArray) { TRI_ASSERT(subSlice.isObject()); subSlice = subSlice.get(fieldItForArray->name); if (subSlice.isNone() || subSlice.isNull()) { @@ -546,7 +608,8 @@ class HashIndexMap { if (slice.isArray() && i == _fields.size() - 1) { auto found = false; auto wasNull = false; - for (auto sliceIt = arangodb::velocypack::ArrayIterator(slice); sliceIt != sliceIt.end(); ++sliceIt) { + for (auto sliceIt = arangodb::velocypack::ArrayIterator(slice); + sliceIt != sliceIt.end(); ++sliceIt) { auto subSlice = sliceIt.value(); if (!(subSlice.isNone() || subSlice.isNull())) { insertSlice(documentId, subSlice, i); @@ -559,7 +622,7 @@ class HashIndexMap { insertSlice(documentId, VPackSlice::nullSlice(), i); } toClose = false; - } else { // object + } else { // object insertSlice(documentId, slice, i); builder.add(slice); } @@ -571,7 +634,8 @@ class HashIndexMap { _docIndexMap.try_emplace(documentId, std::move(builder)); } - bool remove(arangodb::LocalDocumentId const& documentId, arangodb::velocypack::Slice const& doc) { + bool remove(arangodb::LocalDocumentId const& documentId, + arangodb::velocypack::Slice const& doc) { size_t i = 0; auto documentRemoved = false; for (auto& map : _valueMaps) { @@ -594,7 +658,8 @@ class HashIndexMap { _docIndexMap.clear(); } - std::unordered_map find(std::unique_ptr&& keys) const { + std::unordered_map find( + std::unique_ptr&& keys) const { std::unordered_map found; TRI_ASSERT(keys->slice().isArray()); auto sliceIt = arangodb::velocypack::ArrayIterator(keys->slice()); @@ -607,7 +672,7 @@ class HashIndexMap { return std::unordered_map(); } if (found.empty()) { - std::transform(begin, end, std::inserter(found, found.end()), [] (auto const& item) { + std::transform(begin, end, std::inserter(found, found.end()), [](auto const& item) { return std::make_pair(item.second, &item.first); }); } else { @@ -686,9 +751,7 @@ class HashIndexIteratorMock final : public arangodb::IndexIterator { _end = _documents.end(); } - bool hasCovering() const override { - return true; - } + bool hasCovering() const override { return true; } private: HashIndexMap const& _map; @@ -708,8 +771,9 @@ class HashIndexMock final : public arangodb::Index { return nullptr; } - auto const type = arangodb::basics::VelocyPackHelper::getStringRef(typeSlice, - arangodb::velocypack::StringRef()); + auto const type = + arangodb::basics::VelocyPackHelper::getStringRef(typeSlice, + arangodb::velocypack::StringRef()); if (type.compare("hash") != 0) { return nullptr; @@ -783,17 +847,19 @@ class HashIndexMock final : public arangodb::Index { std::vector> const& allIndexes, arangodb::aql::AstNode const* node, arangodb::aql::Variable const* reference, size_t itemsInIndex) const override { - return arangodb::SortedIndexAttributeMatcher::supportsFilterCondition(allIndexes, this, node, reference, itemsInIndex); + return arangodb::SortedIndexAttributeMatcher::supportsFilterCondition( + allIndexes, this, node, reference, itemsInIndex); } Index::SortCosts supportsSortCondition(arangodb::aql::SortCondition const* sortCondition, - arangodb::aql::Variable const* reference, - size_t itemsInIndex) const override { - return arangodb::SortedIndexAttributeMatcher::supportsSortCondition(this, sortCondition, reference, itemsInIndex); + arangodb::aql::Variable const* reference, + size_t itemsInIndex) const override { + return arangodb::SortedIndexAttributeMatcher::supportsSortCondition(this, sortCondition, + reference, itemsInIndex); } - arangodb::aql::AstNode* specializeCondition( - arangodb::aql::AstNode* node, arangodb::aql::Variable const* reference) const override { + arangodb::aql::AstNode* specializeCondition(arangodb::aql::AstNode* node, + arangodb::aql::Variable const* reference) const override { return arangodb::SortedIndexAttributeMatcher::specializeCondition(this, node, reference); } @@ -807,8 +873,7 @@ class HashIndexMock final : public arangodb::Index { if (nullptr == node) { keys->close(); return std::make_unique(&_collection, trx, this, - _hashData, - std::move(keys)); + _hashData, std::move(keys)); } TRI_ASSERT(node->type == arangodb::aql::NODE_TYPE_OPERATOR_NARY_AND); @@ -827,7 +892,7 @@ class HashIndexMock final : public arangodb::Index { auto valNode = comp->getMember(1); if (!(attrNode->type == arangodb::aql::NODE_TYPE_ATTRIBUTE_ACCESS || - attrNode->type == arangodb::aql::NODE_TYPE_EXPANSION)) { + attrNode->type == arangodb::aql::NODE_TYPE_EXPANSION)) { // got value == a.b -> flip sides std::swap(attrNode, valNode); } @@ -837,11 +902,13 @@ class HashIndexMock final : public arangodb::Index { std::vector attributes; if (attrNode->type == arangodb::aql::NODE_TYPE_ATTRIBUTE_ACCESS) { do { - attributes.emplace_back(std::string(attrNode->getStringValue(), attrNode->getStringLength()), false); + attributes.emplace_back(std::string(attrNode->getStringValue(), + attrNode->getStringLength()), + false); attrNode = attrNode->getMember(0); } while (attrNode->type == arangodb::aql::NODE_TYPE_ATTRIBUTE_ACCESS); std::reverse(attributes.begin(), attributes.end()); - } else { // expansion + } else { // expansion TRI_ASSERT(attrNode->type == arangodb::aql::NODE_TYPE_EXPANSION); auto expNode = attrNode; TRI_ASSERT(expNode->numMembers() >= 2); @@ -850,7 +917,9 @@ class HashIndexMock final : public arangodb::Index { attrNode = left->getMember(1); TRI_ASSERT(attrNode->type == arangodb::aql::NODE_TYPE_ATTRIBUTE_ACCESS); do { - attributes.emplace_back(std::string(attrNode->getStringValue(), attrNode->getStringLength()), false); + attributes.emplace_back(std::string(attrNode->getStringValue(), + attrNode->getStringLength()), + false); attrNode = attrNode->getMember(0); } while (attrNode->type == arangodb::aql::NODE_TYPE_ATTRIBUTE_ACCESS); attributes.front().shouldExpand = true; @@ -861,16 +930,19 @@ class HashIndexMock final : public arangodb::Index { TRI_ASSERT(attrNode->type == arangodb::aql::NODE_TYPE_ATTRIBUTE_ACCESS || attrNode->type == arangodb::aql::NODE_TYPE_REFERENCE); while (attrNode->type == arangodb::aql::NODE_TYPE_ATTRIBUTE_ACCESS) { - attributesRight.emplace_back(std::string(attrNode->getStringValue(), attrNode->getStringLength()), false); + attributesRight.emplace_back(std::string(attrNode->getStringValue(), + attrNode->getStringLength()), + false); attrNode = attrNode->getMember(0); } - attributes.insert(attributes.end(), attributesRight.crbegin(), attributesRight.crend()); + attributes.insert(attributes.end(), attributesRight.crbegin(), + attributesRight.crend()); } allAttributes.emplace_back(std::move(attributes), valNode); } size_t nullsCount = 0; for (auto const& f : _fields) { - auto it = std::find_if(allAttributes.cbegin(), allAttributes.cend(), [&f] (auto const& attrs) { + auto it = std::find_if(allAttributes.cbegin(), allAttributes.cend(), [&f](auto const& attrs) { return arangodb::basics::AttributeName::isIdentical(attrs.first, f, true); }); if (it != allAttributes.cend()) { @@ -886,8 +958,7 @@ class HashIndexMock final : public arangodb::Index { keys->close(); return std::make_unique(&_collection, trx, this, - _hashData, - std::move(keys)); + _hashData, std::move(keys)); } HashIndexMock(arangodb::IndexId iid, arangodb::LogicalCollection& collection, @@ -1050,7 +1121,8 @@ bool PhysicalCollectionMock::dropIndex(arangodb::IndexId iid) { return false; } -void PhysicalCollectionMock::figuresSpecific(bool /*details*/, arangodb::velocypack::Builder&) { +void PhysicalCollectionMock::figuresSpecific(bool /*details*/, + arangodb::velocypack::Builder&) { before(); TRI_ASSERT(false); } @@ -1077,9 +1149,10 @@ void PhysicalCollectionMock::getPropertiesVPack(arangodb::velocypack::Builder&) before(); } -arangodb::Result PhysicalCollectionMock::insert( - arangodb::transaction::Methods* trx, arangodb::velocypack::Slice newSlice, - arangodb::ManagedDocumentResult& result, arangodb::OperationOptions& options) { +arangodb::Result PhysicalCollectionMock::insert(arangodb::transaction::Methods* trx, + arangodb::velocypack::Slice newSlice, + arangodb::ManagedDocumentResult& result, + arangodb::OperationOptions& options) { before(); TRI_ASSERT(newSlice.isObject()); @@ -1103,7 +1176,8 @@ arangodb::Result PhysicalCollectionMock::insert( TRI_ASSERT(builder.slice().get(arangodb::StaticStrings::KeyString).isString()); arangodb::velocypack::StringRef key{builder.slice().get(arangodb::StaticStrings::KeyString)}; - arangodb::LocalDocumentId id = ::generateDocumentId(_logicalCollection, revisionId, _lastDocumentId); + arangodb::LocalDocumentId id = + ::generateDocumentId(_logicalCollection, revisionId, _lastDocumentId); auto const& [ref, didInsert] = _documents.emplace(key, DocElement{builder.steal(), id.id()}); TRI_ASSERT(didInsert); @@ -1158,7 +1232,7 @@ arangodb::Result PhysicalCollectionMock::lookupKey( std::pair& result, arangodb::ReadOwnWrites) const { before(); - + auto it = _documents.find(arangodb::velocypack::StringRef{key}); if (it != _documents.end()) { if (_documents.find(arangodb::velocypack::StringRef{key}) != _documents.end()) { @@ -1237,6 +1311,24 @@ void PhysicalCollectionMock::prepareIndexes(arangodb::velocypack::Slice indexesS } } +arangodb::IndexEstMap PhysicalCollectionMock::clusterIndexEstimates(bool allowUpdating, + arangodb::TransactionId tid) { + TRI_ASSERT(arangodb::ServerState::instance()->isCoordinator()); + arangodb::IndexEstMap estimates; + for (auto const& it : _indexes) { + std::string id = std::to_string(it->id().id()); + if (it->hasSelectivityEstimate()) { + // Note: This may actually be bad, as this instance cannot + // have documents => The estimate is off. + estimates.emplace(std::move(id), it->selectivityEstimate()); + } else { + // Random hardcoded estimate. We do not actually know anything + estimates.emplace(std::move(id), 0.25); + } + } + return estimates; +} + arangodb::Result PhysicalCollectionMock::read(arangodb::transaction::Methods*, arangodb::velocypack::StringRef const& key, arangodb::IndexIterator::DocumentCallback const& cb, @@ -1280,9 +1372,10 @@ bool PhysicalCollectionMock::readDocument(arangodb::transaction::Methods* trx, return false; } -arangodb::Result PhysicalCollectionMock::remove( - arangodb::transaction::Methods& trx, arangodb::velocypack::Slice slice, - arangodb::ManagedDocumentResult& previous, arangodb::OperationOptions& options) { +arangodb::Result PhysicalCollectionMock::remove(arangodb::transaction::Methods& trx, + arangodb::velocypack::Slice slice, + arangodb::ManagedDocumentResult& previous, + arangodb::OperationOptions& options) { before(); auto key = slice.get(arangodb::StaticStrings::KeyString); @@ -1300,10 +1393,11 @@ arangodb::Result PhysicalCollectionMock::remove( return arangodb::Result(TRI_ERROR_ARANGO_DOCUMENT_NOT_FOUND); } -arangodb::Result PhysicalCollectionMock::update( - arangodb::transaction::Methods* trx, arangodb::velocypack::Slice newSlice, - arangodb::ManagedDocumentResult& result, arangodb::OperationOptions& options, - arangodb::ManagedDocumentResult& previous) { +arangodb::Result PhysicalCollectionMock::update(arangodb::transaction::Methods* trx, + arangodb::velocypack::Slice newSlice, + arangodb::ManagedDocumentResult& result, + arangodb::OperationOptions& options, + arangodb::ManagedDocumentResult& previous) { return updateInternal(trx, newSlice, result, options, previous, true); } @@ -1349,7 +1443,8 @@ arangodb::Result PhysicalCollectionMock::updateInternal( TRI_ASSERT(doc.isObject()); arangodb::RevisionId oldRev = arangodb::RevisionId::fromSlice(doc); if (!checkRevision(trx, expectedRev, oldRev)) { - return arangodb::Result(TRI_ERROR_ARANGO_CONFLICT, "_rev values mismatch"); + return arangodb::Result(TRI_ERROR_ARANGO_CONFLICT, + "_rev values mismatch"); } } arangodb::velocypack::Builder builder; @@ -1396,14 +1491,15 @@ arangodb::Result PhysicalCollectionMock::updateProperties(arangodb::velocypack:: } std::shared_ptr StorageEngineMock::buildLinkMock( - arangodb::IndexId id, arangodb::LogicalCollection& collection, VPackSlice const& info) { + arangodb::IndexId id, arangodb::LogicalCollection& collection, VPackSlice const& info) { auto index = std::shared_ptr( - new arangodb::iresearch::IResearchLinkMock(id, collection)); - auto res = static_cast(index.get())->init(info, [](irs::directory& dir) { - if (arangodb::iresearch::IResearchLinkMock::InitCallback != nullptr) { - arangodb::iresearch::IResearchLinkMock::InitCallback(dir); - } - }); + new arangodb::iresearch::IResearchLinkMock(id, collection)); + auto res = static_cast(index.get()) + ->init(info, [](irs::directory& dir) { + if (arangodb::iresearch::IResearchLinkMock::InitCallback != nullptr) { + arangodb::iresearch::IResearchLinkMock::InitCallback(dir); + } + }); if (!res.ok()) { THROW_ARANGO_EXCEPTION(res); @@ -1419,21 +1515,21 @@ std::function StorageEngineMock::recoveryTickCallback = []() -> void {}; /*static*/ std::string StorageEngineMock::versionFilenameResult; -StorageEngineMock::StorageEngineMock(arangodb::application_features::ApplicationServer& server) +StorageEngineMock::StorageEngineMock(arangodb::application_features::ApplicationServer& server, + bool injectClusterIndexes) : StorageEngine(server, "Mock", "", - std::unique_ptr(new IndexFactoryMock(server))), + std::unique_ptr( + new IndexFactoryMock(server, injectClusterIndexes))), vocbaseCount(1), _releasedTick(0) {} -arangodb::HealthData StorageEngineMock::healthCheck() { - return {}; -} +arangodb::HealthData StorageEngineMock::healthCheck() { return {}; } arangodb::WalAccess const* StorageEngineMock::walAccess() const { TRI_ASSERT(false); return nullptr; } - + void StorageEngineMock::addOptimizerRules(arangodb::aql::OptimizerRulesFeature& /*feature*/) { before(); // NOOP @@ -1443,9 +1539,7 @@ void StorageEngineMock::addRestHandlers(arangodb::rest::RestHandlerFactory& hand TRI_ASSERT(false); } -void StorageEngineMock::addV8Functions() { - TRI_ASSERT(false); -} +void StorageEngineMock::addV8Functions() { TRI_ASSERT(false); } void StorageEngineMock::changeCollection(TRI_vocbase_t& vocbase, arangodb::LogicalCollection const& collection, @@ -1535,7 +1629,7 @@ arangodb::Result StorageEngineMock::createView(TRI_vocbase_t& vocbase, return arangodb::Result(TRI_ERROR_NO_ERROR); // assume mock view persisted OK } - + arangodb::Result StorageEngineMock::compactAll(bool changeLevels, bool compactBottomMostLevel) { TRI_ASSERT(false); return arangodb::Result(); @@ -1595,8 +1689,8 @@ void StorageEngineMock::getCollectionInfo(TRI_vocbase_t& vocbase, arangodb::Data } ErrorCode StorageEngineMock::getCollectionsAndIndexes(TRI_vocbase_t& vocbase, - arangodb::velocypack::Builder& result, - bool wasCleanShutdown, bool isUpgrade) { + arangodb::velocypack::Builder& result, + bool wasCleanShutdown, bool isUpgrade) { TRI_ASSERT(false); return TRI_ERROR_INTERNAL; } @@ -1634,7 +1728,8 @@ arangodb::velocypack::Builder StorageEngineMock::getReplicationApplierConfigurat return arangodb::velocypack::Builder(); } -ErrorCode StorageEngineMock::getViews(TRI_vocbase_t& vocbase, arangodb::velocypack::Builder& result) { +ErrorCode StorageEngineMock::getViews(TRI_vocbase_t& vocbase, + arangodb::velocypack::Builder& result) { result.openArray(); for (auto& entry : views) { @@ -1663,9 +1758,9 @@ TRI_voc_tick_t StorageEngineMock::recoveryTick() { return recoveryTickResult; } -arangodb::Result StorageEngineMock::lastLogger( - TRI_vocbase_t& vocbase, - uint64_t tickStart, uint64_t tickEnd, arangodb::velocypack::Builder& builderSPtr) { +arangodb::Result StorageEngineMock::lastLogger(TRI_vocbase_t& vocbase, + uint64_t tickStart, uint64_t tickEnd, + arangodb::velocypack::Builder& builderSPtr) { TRI_ASSERT(false); return arangodb::Result(TRI_ERROR_NOT_IMPLEMENTED); } @@ -1708,14 +1803,14 @@ arangodb::Result StorageEngineMock::renameCollection(TRI_vocbase_t& vocbase, return arangodb::Result(TRI_ERROR_INTERNAL); } -ErrorCode StorageEngineMock::saveReplicationApplierConfiguration(TRI_vocbase_t& vocbase, - arangodb::velocypack::Slice slice, - bool doSync) { +ErrorCode StorageEngineMock::saveReplicationApplierConfiguration( + TRI_vocbase_t& vocbase, arangodb::velocypack::Slice slice, bool doSync) { TRI_ASSERT(false); return TRI_ERROR_NO_ERROR; } -ErrorCode StorageEngineMock::saveReplicationApplierConfiguration(arangodb::velocypack::Slice, bool) { +ErrorCode StorageEngineMock::saveReplicationApplierConfiguration(arangodb::velocypack::Slice, + bool) { TRI_ASSERT(false); return TRI_ERROR_NO_ERROR; } @@ -1804,7 +1899,8 @@ arangodb::Result TransactionCollectionMock::lockUsage() { } } - return arangodb::Result(_collection ? TRI_ERROR_NO_ERROR : TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND); + return arangodb::Result(_collection ? TRI_ERROR_NO_ERROR + : TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND); } arangodb::Result TransactionCollectionMock::doLock(arangodb::AccessMode::Type type) { @@ -1839,7 +1935,7 @@ TransactionStateMock::TransactionStateMock(TRI_vocbase_t& vocbase, arangodb::Tra arangodb::Result TransactionStateMock::abortTransaction(arangodb::transaction::Methods* trx) { ++abortTransactionCount; updateStatus(arangodb::transaction::Status::ABORTED); -// releaseUsage(); + // releaseUsage(); resetTransactionId(); return arangodb::Result(); @@ -1848,13 +1944,12 @@ arangodb::Result TransactionStateMock::abortTransaction(arangodb::transaction::M arangodb::Result TransactionStateMock::beginTransaction(arangodb::transaction::Hints hints) { ++beginTransactionCount; _hints = hints; - + arangodb::Result res = useCollections(); - if (res.fail()) { // something is wrong + if (res.fail()) { // something is wrong return res; } - if (!res.ok()) { updateStatus(arangodb::transaction::Status::ABORTED); resetTransactionId(); @@ -1872,7 +1967,7 @@ arangodb::Result TransactionStateMock::commitTransaction(arangodb::transaction:: return arangodb::Result(); } - + uint64_t TransactionStateMock::numCommits() const { return commitTransactionCount; } diff --git a/tests/Mocks/StorageEngineMock.h b/tests/Mocks/StorageEngineMock.h index ecb840dd564d..8ec1ace54623 100644 --- a/tests/Mocks/StorageEngineMock.h +++ b/tests/Mocks/StorageEngineMock.h @@ -28,6 +28,7 @@ #include #include "Basics/Result.h" +#include "IResearchLinkMock.h" #include "Indexes/IndexIterator.h" #include "Mocks/IResearchLinkMock.h" #include "Replication2/ReplicatedLog/PersistedLog.h" @@ -98,6 +99,10 @@ class PhysicalCollectionMock : public arangodb::PhysicalCollection { virtual uint64_t numberDocuments(arangodb::transaction::Methods* trx) const override; virtual std::string const& path() const override; virtual void prepareIndexes(arangodb::velocypack::Slice indexesSlice) override; + + arangodb::IndexEstMap clusterIndexEstimates(bool allowUpdating, + arangodb::TransactionId tid) override; + virtual arangodb::Result read(arangodb::transaction::Methods*, arangodb::velocypack::StringRef const& key, arangodb::IndexIterator::DocumentCallback const& cb, @@ -187,7 +192,8 @@ class StorageEngineMock : public arangodb::StorageEngine { std::map, arangodb::velocypack::Builder> views; std::atomic vocbaseCount; - explicit StorageEngineMock(arangodb::application_features::ApplicationServer& server); + explicit StorageEngineMock(arangodb::application_features::ApplicationServer& server, + bool injectClusterIndexes = false); arangodb::HealthData healthCheck() override; virtual void addOptimizerRules(arangodb::aql::OptimizerRulesFeature& feature) override; virtual void addRestHandlers(arangodb::rest::RestHandlerFactory& handlerFactory) override; diff --git a/tests/Transaction/ManagerSetup.h b/tests/Transaction/ManagerSetup.h index 924a3947ed76..6b4b13a4397a 100644 --- a/tests/Transaction/ManagerSetup.h +++ b/tests/Transaction/ManagerSetup.h @@ -44,23 +44,23 @@ namespace arangodb { namespace tests { namespace mocks { - - // ----------------------------------------------------------------------------- - // --SECTION-- setup / tear-down - // ----------------------------------------------------------------------------- - - struct TransactionManagerSetup { - arangodb::tests::mocks::MockAqlServer server; - TransactionManagerSetup() : server(false) { - server.addFeature(true); - server.startFeatures(); - } +// ----------------------------------------------------------------------------- +// --SECTION-- setup / tear-down +// ----------------------------------------------------------------------------- - ~TransactionManagerSetup() = default; - }; +struct TransactionManagerSetup { + arangodb::tests::mocks::MockAqlServer server; -} // namespace tests -} // namespace mocks -} // namespace arangodb + TransactionManagerSetup() : server(false) { + TRI_ASSERT(server.server().hasFeature()); + server.startFeatures(); + } + + ~TransactionManagerSetup() = default; +}; + +} // namespace mocks +} // namespace tests +} // namespace arangodb #endif diff --git a/tests/js/common/shell/shell-general-graph.js b/tests/js/common/shell/shell-general-graph.js index 02adc9306518..5354d998ee9c 100644 --- a/tests/js/common/shell/shell-general-graph.js +++ b/tests/js/common/shell/shell-general-graph.js @@ -240,9 +240,9 @@ function GeneralGraphCreationSuite() { ) ); - graph._renameCollection("UnitTestRelationName1", "UnitTestsGraphRenamed1"); - graph._renameCollection("UnitTestVertices1", "UnitTestsGraphRenamed2"); - graph._renameCollection("UnitTestVertices4", "UnitTestsGraphRenamed3"); + db._collection("UnitTestRelationName1").rename("UnitTestsGraphRenamed1"); + db._collection("UnitTestVertices1").rename("UnitTestsGraphRenamed2"); + db._collection("UnitTestVertices4").rename("UnitTestsGraphRenamed3"); var doc = db._graphs.document(gN1); assertEqual(1, doc.edgeDefinitions.length); @@ -278,7 +278,7 @@ function GeneralGraphCreationSuite() { var g2 = graph._graph(gN2); g2._addVertexCollection("mj7"); - graph._renameCollection("mj7", "MarcelJansen"); + db._collection("mj7").rename("MarcelJansen"); var doc = db._graphs.document(gN1); assertEqual(1, doc.orphanCollections.length); diff --git a/tests/js/server/aql/aql-graph-traverser.js b/tests/js/server/aql/aql-graph-traverser.js index f31ec90c3df4..b974af702fbf 100644 --- a/tests/js/server/aql/aql-graph-traverser.js +++ b/tests/js/server/aql/aql-graph-traverser.js @@ -2429,7 +2429,12 @@ function complexFilteringSuite() { // 1 Edge (Tri1->Tri2) // 1 Primary (Tri2) - assertEqual(stats.scannedIndex, 1); + if (isCluster) { + // one edge and one vertex lookup + assertEqual(stats.scannedIndex, 2); + } else { + assertEqual(stats.scannedIndex, 1); + } assertEqual(stats.filtered, 1); }, @@ -2513,7 +2518,7 @@ function complexFilteringSuite() { // 2 Primary lookup B,D // 2 Edge Lookups (2 B) (0 D) // 2 Primary Lookups (C, F) - assertTrue(stats.scannedIndex <= 5); + assertTrue(stats.scannedIndex <= 9); } else { // 2 Edge Lookups (A) // 2 Primary (B, D) for Filtering @@ -2556,7 +2561,7 @@ function complexFilteringSuite() { // 1 Primary lookup A // 2 Primary lookup B,D // 4 Primary Lookups (C, F, E, G) - assertTrue(stats.scannedIndex <= 7); + assertTrue(stats.scannedIndex <= 13); } else { // 2 Edge Lookups (A) // 4 Edge Lookups (2 B) (2 D) @@ -2600,7 +2605,7 @@ function complexFilteringSuite() { // 2 Primary lookup B,D // 2 Edge Lookups (0 B) (2 D) // 2 Primary Lookups (E, G) - assertTrue(stats.scannedIndex <= 5); + assertTrue(stats.scannedIndex <= 8); } else { // 2 Edge Lookups (A) // 2 Primary Lookups for Eval (B, D) @@ -2641,7 +2646,7 @@ function complexFilteringSuite() { // 1 Primary (B) // 2 Edge // 2 Primary (C,F) - assertTrue(stats.scannedIndex <= 4); + assertTrue(stats.scannedIndex <= 7); } else { // 2 Edge Lookups (A) // 2 Edge Lookups (B) @@ -2684,7 +2689,7 @@ function complexFilteringSuite() { // they may be inserted in the vertexToFetch list, which // lazy loads all vertices in it. if (stats.scannedIndex !== 8) { - assertTrue(stats.scannedIndex <= 5); + assertTrue(stats.scannedIndex <= 10); } } else { // 2 Edge Lookups (A) @@ -2739,7 +2744,7 @@ function complexFilteringSuite() { // 2 Primary lookup B,D // 2 Edge Lookups (2 B) (0 D) // 2 Primary Lookups (C, F) - assertTrue(stats.scannedIndex <= 5, stats.scannedIndex); + assertTrue(stats.scannedIndex <= 8, stats.scannedIndex); } else { // Cluster uses a lookup cache. // Pointless in single-server mode @@ -2797,7 +2802,7 @@ function complexFilteringSuite() { // 2 Primary lookup B,D // 2 Edge Lookups (2 B) (0 D) // 2 Primary Lookups (C, F) - assertTrue(stats.scannedIndex <= 5); + assertTrue(stats.scannedIndex <= 8); } else { // Cluster uses a lookup cache. // Pointless in single-server mode @@ -2847,7 +2852,7 @@ function complexFilteringSuite() { // 2 Primary lookup B,D // 2 Edge Lookups (2 B) (0 D) // 2 Primary Lookups (C, F) - assertTrue(stats.scannedIndex <= 7); + assertTrue(stats.scannedIndex <= 13, stats.scannedIndex); } else { // 2 Edge Lookups (A) // 2 Primary (B, D) for Filtering @@ -3657,7 +3662,7 @@ function optimizeQuantifierSuite() { let stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 5); + assertTrue(stats.scannedIndex <= 9, stats.scannedIndex); } else { // With traverser-read-cache // assertEqual(stats.scannedIndex, 9); @@ -3702,7 +3707,7 @@ function optimizeQuantifierSuite() { let stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 4); + assertTrue(stats.scannedIndex <= 8, stats.scannedIndex); } else { // With traverser-read-cache // assertEqual(stats.scannedIndex, 8); @@ -3731,7 +3736,7 @@ function optimizeQuantifierSuite() { stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 4); + assertTrue(stats.scannedIndex <= 8, stats.scannedIndex); } else { // With traverser-read-cache // assertEqual(stats.scannedIndex, 8); @@ -3761,7 +3766,7 @@ function optimizeQuantifierSuite() { let stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 5); + assertTrue(stats.scannedIndex <= 9, stats.scannedIndex); } else { // With traverser-read-cache // assertEqual(stats.scannedIndex, 9); @@ -3806,7 +3811,7 @@ function optimizeQuantifierSuite() { let stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 4); + assertTrue(stats.scannedIndex <= 8, stats.scannedIndex); } else { // With traverser-read-cache // assertEqual(stats.scannedIndex, 8); @@ -3834,7 +3839,7 @@ function optimizeQuantifierSuite() { stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 4); + assertTrue(stats.scannedIndex <= 8); } else { // With traverser-read-cache // assertEqual(stats.scannedIndex, 8); @@ -3865,7 +3870,7 @@ function optimizeQuantifierSuite() { let stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 5); + assertTrue(stats.scannedIndex <= 9, stats.scannedIndex); } else { // With traverser-read-cache // assertEqual(stats.scannedIndex, 9); @@ -3896,7 +3901,7 @@ function optimizeQuantifierSuite() { let stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 3); + assertTrue(stats.scannedIndex <= 7, stats.scannedIndex); } else { // With activated traverser-read-cache: // assertEqual(stats.scannedIndex, 7); @@ -3927,7 +3932,7 @@ function optimizeQuantifierSuite() { let stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 5); + assertTrue(stats.scannedIndex <= 9, stats.scannedIndex); } else { // With traverser-read-cache // assertEqual(stats.scannedIndex, 9); @@ -3959,7 +3964,7 @@ function optimizeQuantifierSuite() { let stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 3); + assertTrue(stats.scannedIndex <= 7, stats.scannedIndex); } else { // With activated traverser-read-cache: // assertEqual(stats.scannedIndex, 7); @@ -3991,7 +3996,7 @@ function optimizeQuantifierSuite() { let stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 5); + assertTrue(stats.scannedIndex <= 9, stats.scannedIndex); } else { // With activated traverser-read-cache: // assertEqual(stats.scannedIndex, 9); @@ -4023,7 +4028,7 @@ function optimizeQuantifierSuite() { let stats = cursor.getExtra().stats; assertEqual(stats.scannedFull, 0); if (isCluster) { - assertTrue(stats.scannedIndex <= 3); + assertTrue(stats.scannedIndex <= 6, stats.scannedIndex); } else { // With activated traverser-read-cache: // assertEqual(stats.scannedIndex, 7);