diff --git a/arangod/Aql/AllRowsFetcher.cpp b/arangod/Aql/AllRowsFetcher.cpp index 27702b4f5995..98096c152342 100644 --- a/arangod/Aql/AllRowsFetcher.cpp +++ b/arangod/Aql/AllRowsFetcher.cpp @@ -60,7 +60,7 @@ std::pair AllRowsFetcher::fetchAllRows() { return {ExecutionState::DONE, nullptr}; } -std::tuple AllRowsFetcher::execute(AqlCallStack& stack) { +std::tuple AllRowsFetcher::execute(AqlCallStack& stack) { if (!stack.isRelevant()) { auto [state, skipped, block] = _dependencyProxy->execute(stack); return {state, skipped, AqlItemBlockInputMatrix{block}}; @@ -79,13 +79,14 @@ std::tuple AllRowsFetcher::exec TRI_ASSERT(!_aqlItemMatrix->stoppedOnShadowRow()); while (true) { auto [state, skipped, block] = _dependencyProxy->execute(stack); - TRI_ASSERT(skipped == 0); + TRI_ASSERT(skipped.getSkipCount() == 0); // we will either build a complete fetched AqlItemBlockInputMatrix or return an empty one if (state == ExecutionState::WAITING) { + TRI_ASSERT(skipped.nothingSkipped()); TRI_ASSERT(block == nullptr); // On waiting we have nothing to return - return {state, 0, AqlItemBlockInputMatrix{ExecutorState::HASMORE}}; + return {state, SkipResult{}, AqlItemBlockInputMatrix{ExecutorState::HASMORE}}; } TRI_ASSERT(block != nullptr || state == ExecutionState::DONE); @@ -97,10 +98,10 @@ std::tuple AllRowsFetcher::exec // If we find a ShadowRow or ExecutionState == Done, we're done fetching. if (_aqlItemMatrix->stoppedOnShadowRow() || state == ExecutionState::DONE) { if (state == ExecutionState::HASMORE) { - return {state, 0, + return {state, skipped, AqlItemBlockInputMatrix{ExecutorState::HASMORE, _aqlItemMatrix.get()}}; } - return {state, 0, + return {state, skipped, AqlItemBlockInputMatrix{ExecutorState::DONE, _aqlItemMatrix.get()}}; } } diff --git a/arangod/Aql/AllRowsFetcher.h b/arangod/Aql/AllRowsFetcher.h index f285715c7e11..71c336ebbb88 100644 --- a/arangod/Aql/AllRowsFetcher.h +++ b/arangod/Aql/AllRowsFetcher.h @@ -44,6 +44,7 @@ enum class ExecutionState; template class DependencyProxy; class ShadowAqlItemRow; +class SkipResult; /** * @brief Interface for all AqlExecutors that do need all @@ -110,7 +111,7 @@ class AllRowsFetcher { * size_t => Amount of documents skipped * DataRange => Resulting data */ - std::tuple execute(AqlCallStack& stack); + std::tuple execute(AqlCallStack& stack); /** * @brief Fetch one new AqlItemRow from upstream. diff --git a/arangod/Aql/AqlCallStack.cpp b/arangod/Aql/AqlCallStack.cpp index 911ea0c86fd2..77ffed71ad4f 100644 --- a/arangod/Aql/AqlCallStack.cpp +++ b/arangod/Aql/AqlCallStack.cpp @@ -42,7 +42,7 @@ AqlCallStack::AqlCallStack(AqlCallStack const& other, AqlCall call) // We can only use this constructor on relevant levels // Alothers need to use passThrough constructor TRI_ASSERT(other._depth == 0); - _operations.push(std::move(call)); + _operations.emplace_back(std::move(call)); _compatibilityMode3_6 = other._compatibilityMode3_6; } @@ -51,7 +51,7 @@ AqlCallStack::AqlCallStack(AqlCallStack const& other) _depth(other._depth), _compatibilityMode3_6(other._compatibilityMode3_6) {} -AqlCallStack::AqlCallStack(std::stack&& operations) +AqlCallStack::AqlCallStack(std::vector&& operations) : _operations(std::move(operations)) {} bool AqlCallStack::isRelevant() const { return _depth == 0; } @@ -68,40 +68,41 @@ AqlCall AqlCallStack::popCall() { // to the upwards subquery. // => Simply put another fetchAll Call on the stack. // This code is to be removed in the next version after 3.7 - _operations.push(AqlCall{}); + _operations.emplace_back(AqlCall{}); } - auto call = _operations.top(); - _operations.pop(); + auto call = _operations.back(); + _operations.pop_back(); return call; } AqlCall const& AqlCallStack::peek() const { TRI_ASSERT(isRelevant()); + TRI_ASSERT(_compatibilityMode3_6 || !_operations.empty()); + if (is36Compatible() && _operations.empty()) { + // This is only for compatibility with 3.6 + // there we do not have the stack beeing passed-through + // in AQL, we only have a single call. + // We can only get into this state in the abscence of + // LIMIT => we always do an unlimted softLimit call + // to the upwards subquery. + // => Simply put another fetchAll Call on the stack. + // This code is to be removed in the next version after 3.7 + _operations.emplace_back(AqlCall{}); + } TRI_ASSERT(!_operations.empty()); - return _operations.top(); + return _operations.back(); } void AqlCallStack::pushCall(AqlCall&& call) { // TODO is this correct on subqueries? TRI_ASSERT(isRelevant()); - _operations.push(call); + _operations.emplace_back(std::move(call)); } void AqlCallStack::pushCall(AqlCall const& call) { // TODO is this correct on subqueries? TRI_ASSERT(isRelevant()); - _operations.push(call); -} - -void AqlCallStack::stackUpMissingCalls() { - while (!isRelevant()) { - // For every depth, we add an additional default call. - // The default is to produce unlimited many results, - // using DefaultBatchSize each. - _operations.emplace(AqlCall{}); - _depth--; - } - TRI_ASSERT(isRelevant()); + _operations.emplace_back(call); } void AqlCallStack::pop() { @@ -133,8 +134,9 @@ auto AqlCallStack::fromVelocyPack(velocypack::Slice const slice) -> ResultT{}; + auto stack = std::vector{}; auto i = std::size_t{0}; + stack.reserve(slice.length()); for (auto const entry : VPackArrayIterator(slice)) { auto maybeAqlCall = AqlCall::fromVelocyPack(entry); @@ -146,7 +148,7 @@ auto AqlCallStack::fromVelocyPack(velocypack::Slice const slice) -> ResultT ResultT{}; - reverseStack.reserve(_operations.size()); - { - auto ops = _operations; - while (!ops.empty()) { - reverseStack.emplace_back(ops.top()); - ops.pop(); - } - } - builder.openArray(); - for (auto it = reverseStack.rbegin(); it != reverseStack.rend(); ++it) { - auto const& call = *it; + for (auto const& call : _operations) { call.toVelocyPack(builder); } builder.close(); @@ -178,19 +169,50 @@ void AqlCallStack::toVelocyPack(velocypack::Builder& builder) const { auto AqlCallStack::toString() const -> std::string { auto result = std::string{}; result += "["; - auto ops = _operations; - if (!ops.empty()) { - auto op = ops.top(); - ops.pop(); + bool isFirst = true; + for (auto const& op : _operations) { + if (!isFirst) { + result += ","; + } + isFirst = false; result += " "; result += op.toString(); - while (!ops.empty()) { - op = ops.top(); - ops.pop(); - result += ", "; - result += op.toString(); - } } result += " ]"; return result; } + +auto AqlCallStack::createEquivalentFetchAllShadowRowsStack() const -> AqlCallStack { + AqlCallStack res{*this}; + std::replace_if( + res._operations.begin(), res._operations.end(), + [](auto const&) -> bool { return true; }, AqlCall{}); + return res; +} + +auto AqlCallStack::needToSkipSubquery() const noexcept -> bool { + return std::any_of(_operations.begin(), _operations.end(), [](AqlCall const& call) -> bool { + return call.needSkipMore() || call.hardLimit == 0; + }); +} + +auto AqlCallStack::shadowRowDepthToSkip() const -> size_t { + TRI_ASSERT(needToSkipSubquery()); + for (size_t i = 0; i < _operations.size(); ++i) { + auto& call = _operations.at(i); + if (call.needSkipMore() || call.hardLimit == 0) { + return _operations.size() - i - 1; + } + } + // unreachable + TRI_ASSERT(false); + THROW_ARANGO_EXCEPTION(TRI_ERROR_INTERNAL_AQL); +} + +auto AqlCallStack::modifyCallAtDepth(size_t depth) -> AqlCall& { + // depth 0 is back of vector + TRI_ASSERT(_operations.size() > depth); + // Take the depth-most from top of the vector. + auto& call = *(_operations.rbegin() + depth); + return call; +} diff --git a/arangod/Aql/AqlCallStack.h b/arangod/Aql/AqlCallStack.h index 4aa03285f5d8..a457578223bf 100644 --- a/arangod/Aql/AqlCallStack.h +++ b/arangod/Aql/AqlCallStack.h @@ -66,14 +66,6 @@ class AqlCallStack { // Put another call on top of the stack. void pushCall(AqlCall const& call); - // fill up all missing calls within this stack s.t. we reach depth == 0 - // This needs to be called if an executor requires to be fully executed, even if skipped, - // even if the subquery it is located in is skipped. - // The default operations added here will correspond to produce all Rows, unlimitted. - // e.g. every Modification Executor needs to call this functionality, as modifictions need to be - // performed even if skipped. - void stackUpMissingCalls(); - // Pops one subquery level. // if this isRelevent it pops the top-most call from the stack. // if this is not revelent it reduces the depth by 1. @@ -95,12 +87,58 @@ class AqlCallStack { void toVelocyPack(velocypack::Builder& builder) const; + auto is36Compatible() const noexcept -> bool { return _compatibilityMode3_6; } + + /** + * @brief Create an equivalent call stack that does a full-produce + * of all Subquery levels. This is required for blocks + * that are not allowed to be bpassed. + * The top-most call remains unmodified, as the Executor might + * require some soft limit on it. + * + * @return AqlCallStack a stack of equivalent size, that does not skip + * on any lower subquery. + */ + auto createEquivalentFetchAllShadowRowsStack() const -> AqlCallStack; + + /** + * @brief Check if we are in a subquery that is in-fact required to + * be skipped. This is relevant for executors that have created + * an equivalentFetchAllShadowRows stack, in order to decide if + * the need to produce output or if they are skipped. + * + * @return true + * @return false + */ + auto needToSkipSubquery() const noexcept -> bool; + + /** + * @brief This is only valid if needToSkipSubquery is true. + * It will resolve to the heighest subquery level + * (outermost) that needs to be skipped. + * + * + * @return size_t Depth of the subquery that asks to be skipped. + */ + auto shadowRowDepthToSkip() const -> size_t; + + /** + * @brief Get a reference to the call at the given shadowRowDepth + * + * @param depth ShadowRow depth we need to work on + * @return AqlCall& reference to the call, can be modified. + */ + auto modifyCallAtDepth(size_t depth) -> AqlCall&; + private: - explicit AqlCallStack(std::stack&& operations); + explicit AqlCallStack(std::vector&& operations); private: - // The list of operations, stacked by depth (e.g. bottom element is from main query) - std::stack _operations; + // The list of operations, stacked by depth (e.g. bottom element is from main + // query) NOTE: This is only mutable on 3.6 compatibility mode. We need to + // inject an additional call in any const operation here just to pretend we + // are not empty. Can be removed after 3.7. + mutable std::vector _operations; // The depth of subqueries that have not issued calls into operations, // as they have been skipped. diff --git a/arangod/Aql/AqlExecuteResult.cpp b/arangod/Aql/AqlExecuteResult.cpp index db10deeaab96..be7d49d1cb98 100644 --- a/arangod/Aql/AqlExecuteResult.cpp +++ b/arangod/Aql/AqlExecuteResult.cpp @@ -44,11 +44,25 @@ auto getStringView(velocypack::Slice slice) -> std::string_view { } } // namespace +AqlExecuteResult::AqlExecuteResult(ExecutionState state, SkipResult skipped, + SharedAqlItemBlockPtr&& block) + : _state(state), _skipped(skipped), _block(std::move(block)) { + // Make sure we only produce a valid response + // The block should have checked as well. + // We must return skipped and/or data when reporting HASMORE + + // noskip && no data => state != HASMORE + // <=> skipped || data || state != HASMORE + TRI_ASSERT(!_skipped.nothingSkipped() || + (_block != nullptr && _block->numEntries() > 0) || + _state != ExecutionState::HASMORE); +} + auto AqlExecuteResult::state() const noexcept -> ExecutionState { return _state; } -auto AqlExecuteResult::skipped() const noexcept -> std::size_t { +auto AqlExecuteResult::skipped() const noexcept -> SkipResult { return _skipped; } @@ -75,7 +89,8 @@ void AqlExecuteResult::toVelocyPack(velocypack::Builder& builder, builder.openObject(); builder.add(StaticStrings::AqlRemoteState, stateToValue(state())); - builder.add(StaticStrings::AqlRemoteSkipped, Value(skipped())); + builder.add(Value(StaticStrings::AqlRemoteSkipped)); + skipped().toVelocyPack(builder); if (block() != nullptr) { ObjectBuilder guard(&builder, StaticStrings::AqlRemoteBlock); block()->toVelocyPack(options, builder); @@ -101,7 +116,7 @@ auto AqlExecuteResult::fromVelocyPack(velocypack::Slice const slice, expectedPropertiesFound.emplace(StaticStrings::AqlRemoteBlock, false); auto state = ExecutionState::HASMORE; - auto skipped = std::size_t{}; + auto skipped = SkipResult{}; auto block = SharedAqlItemBlockPtr{}; auto const readState = [](velocypack::Slice slice) -> ResultT { @@ -127,24 +142,6 @@ auto AqlExecuteResult::fromVelocyPack(velocypack::Slice const slice, } }; - auto const readSkipped = [](velocypack::Slice slice) -> ResultT { - if (!slice.isInteger()) { - auto message = std::string{ - "When deserializating AqlExecuteResult: When reading skipped: " - "Unexpected type "}; - message += slice.typeName(); - return Result(TRI_ERROR_TYPE_ERROR, std::move(message)); - } - try { - return slice.getNumber(); - } catch (velocypack::Exception const& ex) { - auto message = std::string{ - "When deserializating AqlExecuteResult: When reading skipped: "}; - message += ex.what(); - return Result(TRI_ERROR_TYPE_ERROR, std::move(message)); - } - }; - auto const readBlock = [&itemBlockManager](velocypack::Slice slice) -> ResultT { if (slice.isNull()) { return SharedAqlItemBlockPtr{nullptr}; @@ -165,9 +162,9 @@ auto AqlExecuteResult::fromVelocyPack(velocypack::Slice const slice, if (auto propIt = expectedPropertiesFound.find(key); ADB_LIKELY(propIt != expectedPropertiesFound.end())) { if (ADB_UNLIKELY(propIt->second)) { - return Result( - TRI_ERROR_TYPE_ERROR, - "When deserializating AqlExecuteResult: Encountered duplicate key"); + return Result(TRI_ERROR_TYPE_ERROR, + "When deserializating AqlExecuteResult: " + "Encountered duplicate key"); } propIt->second = true; } @@ -179,7 +176,7 @@ auto AqlExecuteResult::fromVelocyPack(velocypack::Slice const slice, } state = maybeState.get(); } else if (key == StaticStrings::AqlRemoteSkipped) { - auto maybeSkipped = readSkipped(it.value); + auto maybeSkipped = SkipResult::fromVelocyPack(it.value); if (maybeSkipped.fail()) { return std::move(maybeSkipped).result(); } @@ -192,11 +189,12 @@ auto AqlExecuteResult::fromVelocyPack(velocypack::Slice const slice, block = maybeBlock.get(); } else { LOG_TOPIC("cc6f4", WARN, Logger::AQL) - << "When deserializating AqlExecuteResult: Encountered unexpected " + << "When deserializating AqlExecuteResult: Encountered " + "unexpected " "key " << keySlice.toJson(); - // If you run into this assertion during rolling upgrades after adding a - // new attribute, remove it in the older version. + // If you run into this assertion during rolling upgrades after + // adding a new attribute, remove it in the older version. TRI_ASSERT(false); } } @@ -214,6 +212,6 @@ auto AqlExecuteResult::fromVelocyPack(velocypack::Slice const slice, } auto AqlExecuteResult::asTuple() const noexcept - -> std::tuple { + -> std::tuple { return {state(), skipped(), block()}; } diff --git a/arangod/Aql/AqlExecuteResult.h b/arangod/Aql/AqlExecuteResult.h index 56eb448c83db..409e598f9835 100644 --- a/arangod/Aql/AqlExecuteResult.h +++ b/arangod/Aql/AqlExecuteResult.h @@ -25,6 +25,7 @@ #include "Aql/ExecutionState.h" #include "Aql/SharedAqlItemBlockPtr.h" +#include "Aql/SkipResult.h" #include @@ -42,23 +43,22 @@ namespace arangodb::aql { class AqlExecuteResult { public: - AqlExecuteResult(ExecutionState state, std::size_t skipped, SharedAqlItemBlockPtr&& block) - : _state(state), _skipped(skipped), _block(std::move(block)) {} + AqlExecuteResult(ExecutionState state, SkipResult skipped, SharedAqlItemBlockPtr&& block); void toVelocyPack(velocypack::Builder&, velocypack::Options const*); static auto fromVelocyPack(velocypack::Slice, AqlItemBlockManager&) -> ResultT; [[nodiscard]] auto state() const noexcept -> ExecutionState; - [[nodiscard]] auto skipped() const noexcept -> std::size_t; + [[nodiscard]] auto skipped() const noexcept -> SkipResult; [[nodiscard]] auto block() const noexcept -> SharedAqlItemBlockPtr const&; [[nodiscard]] auto asTuple() const noexcept - -> std::tuple; + -> std::tuple; private: ExecutionState _state = ExecutionState::HASMORE; - std::size_t _skipped = 0; + SkipResult _skipped{}; SharedAqlItemBlockPtr _block = nullptr; }; diff --git a/arangod/Aql/BlocksWithClients.cpp b/arangod/Aql/BlocksWithClients.cpp index b4a98714b920..89ebd3ca26cb 100644 --- a/arangod/Aql/BlocksWithClients.cpp +++ b/arangod/Aql/BlocksWithClients.cpp @@ -35,6 +35,7 @@ #include "Aql/InputAqlItemRow.h" #include "Aql/Query.h" #include "Aql/ScatterExecutor.h" +#include "Aql/SkipResult.h" #include "Basics/Exceptions.h" #include "Basics/StaticStrings.h" #include "Basics/StringBuffer.h" @@ -188,7 +189,8 @@ std::pair BlocksWithClientsImpl::skipSome(size } template -std::tuple BlocksWithClientsImpl::execute(AqlCallStack stack) { +std::tuple +BlocksWithClientsImpl::execute(AqlCallStack stack) { // This will not be implemented here! TRI_ASSERT(false); THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); @@ -197,17 +199,17 @@ std::tuple BlocksWithClientsImpl< template auto BlocksWithClientsImpl::executeForClient(AqlCallStack stack, std::string const& clientId) - -> std::tuple { - // traceExecuteBegin(stack); + -> std::tuple { + traceExecuteBegin(stack, clientId); auto res = executeWithoutTraceForClient(stack, clientId); - // traceExecuteEnd(res); + traceExecuteEnd(res, clientId); return res; } template auto BlocksWithClientsImpl::executeWithoutTraceForClient(AqlCallStack stack, std::string const& clientId) - -> std::tuple { + -> std::tuple { TRI_ASSERT(!clientId.empty()); if (ADB_UNLIKELY(clientId.empty())) { // Security bailout to avoid UB @@ -230,21 +232,32 @@ auto BlocksWithClientsImpl::executeWithoutTraceForClient(AqlCallStack // We do not have anymore data locally. // Need to fetch more from upstream auto& dataContainer = it->second; - - while (!dataContainer.hasDataFor(call)) { - if (_upstreamState == ExecutionState::DONE) { - // We are done, with everything, we will not be able to fetch any more rows - return {_upstreamState, 0, nullptr}; + while (true) { + while (!dataContainer.hasDataFor(call)) { + if (_upstreamState == ExecutionState::DONE) { + // We are done, with everything, we will not be able to fetch any more rows + return {_upstreamState, SkipResult{}, nullptr}; + } + + auto state = fetchMore(stack); + if (state == ExecutionState::WAITING) { + return {state, SkipResult{}, nullptr}; + } + _upstreamState = state; } - - auto state = fetchMore(stack); - if (state == ExecutionState::WAITING) { - return {state, 0, nullptr}; + { + // If we get here we have data and can return it. + // However the call might force us to drop everything (e.g. hardLimit == + // 0) So we need to refetch data eventually. + stack.pushCall(call); + auto [state, skipped, result] = dataContainer.execute(stack, _upstreamState); + if (state == ExecutionState::DONE || !skipped.nothingSkipped() || result != nullptr) { + // We have a valid result. + return {state, skipped, result}; + } + stack.popCall(); } - _upstreamState = state; } - // If we get here we have data and can return it. - return dataContainer.execute(call, _upstreamState); } template @@ -262,10 +275,9 @@ auto BlocksWithClientsImpl::fetchMore(AqlCallStack stack) -> Execution TRI_ASSERT(_dependencies.size() == 1); auto [state, skipped, block] = _dependencies[0]->execute(stack); - // We can never ever forward skip! // We could need the row in a different block, and once skipped // we cannot get it back. - TRI_ASSERT(skipped == 0); + TRI_ASSERT(skipped.getSkipCount() == 0); TRI_IF_FAILURE("ExecutionBlock::getBlock") { THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); @@ -274,7 +286,7 @@ auto BlocksWithClientsImpl::fetchMore(AqlCallStack stack) -> Execution // Waiting -> no block TRI_ASSERT(state != ExecutionState::WAITING || block == nullptr); if (block != nullptr) { - _executor.distributeBlock(block, _clientBlockData); + _executor.distributeBlock(block, skipped, _clientBlockData); } return state; @@ -287,7 +299,7 @@ std::pair BlocksWithClientsImpl size_t atMost, std::string const& shardId) { AqlCallStack stack(AqlCall::SimulateGetSome(atMost), true); auto [state, skipped, block] = executeForClient(stack, shardId); - TRI_ASSERT(skipped == 0); + TRI_ASSERT(skipped.nothingSkipped()); return {state, block}; } @@ -299,7 +311,7 @@ std::pair BlocksWithClientsImpl::skipSomeForSh AqlCallStack stack(AqlCall::SimulateSkipSome(atMost), true); auto [state, skipped, block] = executeForClient(stack, shardId); TRI_ASSERT(block == nullptr); - return {state, skipped}; + return {state, skipped.getSkipCount()}; } template class ::arangodb::aql::BlocksWithClientsImpl; diff --git a/arangod/Aql/BlocksWithClients.h b/arangod/Aql/BlocksWithClients.h index ead4589bf64f..75e68a37e306 100644 --- a/arangod/Aql/BlocksWithClients.h +++ b/arangod/Aql/BlocksWithClients.h @@ -48,6 +48,7 @@ class AqlItemBlock; struct Collection; class ExecutionEngine; class ExecutionNode; +class SkipResult; class ClientsExecutorInfos { public: @@ -87,7 +88,7 @@ class BlocksWithClients { * @return std::tuple */ virtual auto executeForClient(AqlCallStack stack, std::string const& clientId) - -> std::tuple = 0; + -> std::tuple = 0; }; /** @@ -130,7 +131,7 @@ class BlocksWithClientsImpl : public ExecutionBlock, public BlocksWithClients { std::pair skipSome(size_t atMost) final; /// @brief execute: shouldn't be used, use executeForClient - std::tuple execute(AqlCallStack stack) override; + std::tuple execute(AqlCallStack stack) override; /** * @brief Execute for client. @@ -141,7 +142,7 @@ class BlocksWithClientsImpl : public ExecutionBlock, public BlocksWithClients { * @return std::tuple */ auto executeForClient(AqlCallStack stack, std::string const& clientId) - -> std::tuple override; + -> std::tuple override; private: /** @@ -152,7 +153,7 @@ class BlocksWithClientsImpl : public ExecutionBlock, public BlocksWithClients { * @return std::tuple */ auto executeWithoutTraceForClient(AqlCallStack stack, std::string const& clientId) - -> std::tuple; + -> std::tuple; /** * @brief Load more data from upstream and distribute it into _clientBlockData diff --git a/arangod/Aql/ConstFetcher.cpp b/arangod/Aql/ConstFetcher.cpp index 14ba394b584e..42e858d1f9e8 100644 --- a/arangod/Aql/ConstFetcher.cpp +++ b/arangod/Aql/ConstFetcher.cpp @@ -25,6 +25,7 @@ #include "Aql/AqlCallStack.h" #include "Aql/DependencyProxy.h" #include "Aql/ShadowAqlItemRow.h" +#include "Aql/SkipResult.h" #include "Basics/Exceptions.h" #include "Basics/voc-errors.h" @@ -37,7 +38,7 @@ ConstFetcher::ConstFetcher(DependencyProxy& executionBlock) : _currentBlock{nullptr}, _rowIndex(0) {} auto ConstFetcher::execute(AqlCallStack& stack) - -> std::tuple { + -> std::tuple { // Note this fetcher can only be executed on top level (it is the singleton, or test) TRI_ASSERT(stack.isRelevant()); // We only peek the call here, as we do not take over ownership. @@ -45,7 +46,7 @@ auto ConstFetcher::execute(AqlCallStack& stack) auto call = stack.peek(); if (_blockForPassThrough == nullptr) { // we are done, nothing to move arround here. - return {ExecutionState::DONE, 0, AqlItemBlockInputRange{ExecutorState::DONE}}; + return {ExecutionState::DONE, SkipResult{}, AqlItemBlockInputRange{ExecutorState::DONE}}; } std::vector> sliceIndexes; sliceIndexes.emplace_back(_rowIndex, _blockForPassThrough->size()); @@ -152,7 +153,10 @@ auto ConstFetcher::execute(AqlCallStack& stack) SharedAqlItemBlockPtr resultBlock = _blockForPassThrough; _blockForPassThrough.reset(nullptr); _rowIndex = 0; - return {ExecutionState::DONE, call.getSkipCount(), + SkipResult skipped{}; + skipped.didSkip(call.getSkipCount()); + + return {ExecutionState::DONE, skipped, DataRange{ExecutorState::DONE, call.getSkipCount(), resultBlock, 0}}; } @@ -176,7 +180,9 @@ auto ConstFetcher::execute(AqlCallStack& stack) // No data to be returned // Block is dropped. resultBlock = nullptr; - return {ExecutionState::DONE, call.getSkipCount(), + SkipResult skipped{}; + skipped.didSkip(call.getSkipCount()); + return {ExecutionState::DONE, skipped, DataRange{ExecutorState::DONE, call.getSkipCount()}}; } @@ -187,8 +193,9 @@ auto ConstFetcher::execute(AqlCallStack& stack) _blockForPassThrough == nullptr ? ExecutorState::DONE : ExecutorState::HASMORE; resultBlock = resultBlock->slice(sliceIndexes); - return {resState, call.getSkipCount(), - DataRange{rangeState, call.getSkipCount(), resultBlock, 0}}; + SkipResult skipped{}; + skipped.didSkip(call.getSkipCount()); + return {resState, skipped, DataRange{rangeState, call.getSkipCount(), resultBlock, 0}}; } void ConstFetcher::injectBlock(SharedAqlItemBlockPtr block) { diff --git a/arangod/Aql/ConstFetcher.h b/arangod/Aql/ConstFetcher.h index 70fe73fa4c12..33e0215bee70 100644 --- a/arangod/Aql/ConstFetcher.h +++ b/arangod/Aql/ConstFetcher.h @@ -37,6 +37,7 @@ class AqlItemBlock; template class DependencyProxy; class ShadowAqlItemRow; +class SkipResult; /** * @brief Interface for all AqlExecutors that do only need one @@ -71,7 +72,7 @@ class ConstFetcher { * size_t => Amount of documents skipped * DataRange => Resulting data */ - auto execute(AqlCallStack& stack) -> std::tuple; + auto execute(AqlCallStack& stack) -> std::tuple; /** * @brief Fetch one new AqlItemRow from upstream. diff --git a/arangod/Aql/DependencyProxy.cpp b/arangod/Aql/DependencyProxy.cpp index cdbc955fac6c..0c9b8535987d 100644 --- a/arangod/Aql/DependencyProxy.cpp +++ b/arangod/Aql/DependencyProxy.cpp @@ -32,10 +32,10 @@ using namespace arangodb; using namespace arangodb::aql; template -std::tuple +std::tuple DependencyProxy::execute(AqlCallStack& stack) { ExecutionState state = ExecutionState::HASMORE; - size_t skipped = 0; + SkipResult skipped; SharedAqlItemBlockPtr block = nullptr; // Note: upstreamBlock will return next dependency // if we need to loop here @@ -47,16 +47,16 @@ DependencyProxy::execute(AqlCallStack& stack) { break; } } - } while (state != ExecutionState::WAITING && skipped == 0 && block == nullptr); + } while (state != ExecutionState::WAITING && skipped.nothingSkipped() && block == nullptr); return {state, skipped, block}; } template -std::tuple DependencyProxy::executeForDependency( - size_t dependency, AqlCallStack& stack) { - // TODO: assert dependency in range +std::tuple +DependencyProxy::executeForDependency(size_t dependency, + AqlCallStack& stack) { ExecutionState state = ExecutionState::HASMORE; - size_t skipped = 0; + SkipResult skipped; SharedAqlItemBlockPtr block = nullptr; if (!_distributeId.empty()) { @@ -81,7 +81,7 @@ std::tuple DependencyProxy::prefetchBlock(size_t atMost) { AqlCallStack stack = _injectedStack; stack.pushCall(AqlCall::SimulateGetSome(atMost)); // Also temporary, will not be used here. - size_t skipped = 0; + SkipResult skipped; do { // Note: upstreamBlock will return next dependency // if we need to loop here @@ -115,7 +115,7 @@ ExecutionState DependencyProxy::prefetchBlock(size_t atMost) { } // Cannot do skipping here // Temporary! - TRI_ASSERT(skipped == 0); + TRI_ASSERT(skipped.nothingSkipped()); if (state == ExecutionState::WAITING) { TRI_ASSERT(block == nullptr); @@ -190,7 +190,7 @@ DependencyProxy::fetchBlockForDependency(size_t dependency, si AqlCallStack stack = _injectedStack; stack.pushCall(AqlCall::SimulateGetSome(atMost)); // Also temporary, will not be used here. - size_t skipped = 0; + SkipResult skipped{}; if (_distributeId.empty()) { std::tie(state, skipped, block) = upstream.execute(stack); @@ -199,7 +199,7 @@ DependencyProxy::fetchBlockForDependency(size_t dependency, si std::tie(state, skipped, block) = upstreamWithClient->executeForClient(stack, _distributeId); } - TRI_ASSERT(skipped == 0); + TRI_ASSERT(skipped.nothingSkipped()); TRI_IF_FAILURE("ExecutionBlock::getBlock") { THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); } @@ -244,7 +244,7 @@ std::pair DependencyProxy::skipSomeFor SharedAqlItemBlockPtr block; while (state == ExecutionState::HASMORE && _skipped < atMost) { - size_t skippedNow; + SkipResult skippedNow; TRI_ASSERT(_skipped <= atMost); { // Make sure we call with the correct offset @@ -255,14 +255,14 @@ std::pair DependencyProxy::skipSomeFor } std::tie(state, skippedNow, block) = upstream.execute(stack); if (state == ExecutionState::WAITING) { - TRI_ASSERT(skippedNow == 0); + TRI_ASSERT(skippedNow.nothingSkipped()); return {state, 0}; } // Temporary. // If we return a block here it will be lost TRI_ASSERT(block == nullptr); - _skipped += skippedNow; + _skipped += skippedNow.getSkipCount(); TRI_ASSERT(_skipped <= atMost); } TRI_ASSERT(state != ExecutionState::WAITING); @@ -290,7 +290,7 @@ std::pair DependencyProxy::skipSome(si SharedAqlItemBlockPtr block; while (_skipped < toSkip) { - size_t skippedNow; + SkipResult skippedNow; // Note: upstreamBlock will return next dependency // if we need to loop here TRI_ASSERT(_skipped <= toSkip); @@ -309,10 +309,10 @@ std::pair DependencyProxy::skipSome(si upstreamWithClient->executeForClient(stack, _distributeId); } - TRI_ASSERT(skippedNow <= toSkip - _skipped); + TRI_ASSERT(skippedNow.getSkipCount() <= toSkip - _skipped); if (state == ExecutionState::WAITING) { - TRI_ASSERT(skippedNow == 0); + TRI_ASSERT(skippedNow.nothingSkipped()); return {state, 0}; } @@ -320,7 +320,7 @@ std::pair DependencyProxy::skipSome(si // If we return a block here it will be lost TRI_ASSERT(block == nullptr); - _skipped += skippedNow; + _skipped += skippedNow.getSkipCount(); // When the current dependency is done, advance. if (state == ExecutionState::DONE) { diff --git a/arangod/Aql/DependencyProxy.h b/arangod/Aql/DependencyProxy.h index 108e6bad1ac9..d28dda04a312 100644 --- a/arangod/Aql/DependencyProxy.h +++ b/arangod/Aql/DependencyProxy.h @@ -37,6 +37,7 @@ namespace arangodb::aql { class ExecutionBlock; class AqlItemBlockManager; +class SkipResult; /** * @brief Thin interface to access the methods of ExecutionBlock that are @@ -74,9 +75,9 @@ class DependencyProxy { TEST_VIRTUAL ~DependencyProxy() = default; // TODO Implement and document properly! - TEST_VIRTUAL std::tuple execute(AqlCallStack& stack); + TEST_VIRTUAL std::tuple execute(AqlCallStack& stack); - TEST_VIRTUAL std::tuple executeForDependency( + TEST_VIRTUAL std::tuple executeForDependency( size_t dependency, AqlCallStack& stack); // This is only TEST_VIRTUAL, so we ignore this lint warning: diff --git a/arangod/Aql/DistributeExecutor.cpp b/arangod/Aql/DistributeExecutor.cpp index 07cd89c9f4b5..a528e68c3fc6 100644 --- a/arangod/Aql/DistributeExecutor.cpp +++ b/arangod/Aql/DistributeExecutor.cpp @@ -31,6 +31,7 @@ #include "Aql/Query.h" #include "Aql/RegisterPlan.h" #include "Aql/ShadowAqlItemRow.h" +#include "Aql/SkipResult.h" #include "Basics/StaticStrings.h" #include "VocBase/LogicalCollection.h" @@ -152,6 +153,12 @@ auto DistributeExecutor::ClientBlockData::addBlock(SharedAqlItemBlockPtr block, _queue.emplace_back(block, std::move(usedIndexes)); } +auto DistributeExecutor::ClientBlockData::addSkipResult(SkipResult const& skipResult) -> void { + TRI_ASSERT(_skipped.subqueryDepth() == 1 || + _skipped.subqueryDepth() == skipResult.subqueryDepth()); + _skipped.merge(skipResult, false); +} + auto DistributeExecutor::ClientBlockData::hasDataFor(AqlCall const& call) -> bool { return _executorHasMore || !_queue.empty(); } @@ -166,7 +173,8 @@ auto DistributeExecutor::ClientBlockData::hasDataFor(AqlCall const& call) -> boo * * @return SharedAqlItemBlockPtr a joind block from the queue. */ -auto DistributeExecutor::ClientBlockData::popJoinedBlock() -> SharedAqlItemBlockPtr { +auto DistributeExecutor::ClientBlockData::popJoinedBlock() + -> std::tuple { // There are some optimizations available in this implementation. // Namely we could apply good logic to cut the blocks at shadow rows // in order to allow the IDexecutor to hand them out en-block. @@ -208,14 +216,16 @@ auto DistributeExecutor::ClientBlockData::popJoinedBlock() -> SharedAqlItemBlock // Drop block form queue. _queue.pop_front(); } - return newBlock; + SkipResult skip = _skipped; + _skipped.reset(); + return {newBlock, skip}; } -auto DistributeExecutor::ClientBlockData::execute(AqlCall call, ExecutionState upstreamState) - -> std::tuple { +auto DistributeExecutor::ClientBlockData::execute(AqlCallStack callStack, ExecutionState upstreamState) + -> std::tuple { TRI_ASSERT(_executor != nullptr); // Make sure we actually have data before you call execute - TRI_ASSERT(hasDataFor(call)); + TRI_ASSERT(hasDataFor(callStack.peek())); if (!_executorHasMore) { // This cast is guaranteed, we create this a couple lines above and only // this executor is used here. @@ -224,16 +234,15 @@ auto DistributeExecutor::ClientBlockData::execute(AqlCall call, ExecutionState u auto casted = static_cast>*>(_executor.get()); TRI_ASSERT(casted != nullptr); - auto block = popJoinedBlock(); + auto [block, skipped] = popJoinedBlock(); // We will at least get one block, otherwise the hasDataFor would // be required to return false! TRI_ASSERT(block != nullptr); - casted->injectConstantBlock(block); + casted->injectConstantBlock(block, skipped); _executorHasMore = true; } - AqlCallStack stack{call}; - auto [state, skipped, result] = _executor->execute(stack); + auto [state, skipped, result] = _executor->execute(callStack); // We have all data locally cannot wait here. TRI_ASSERT(state != ExecutionState::WAITING); @@ -257,7 +266,7 @@ auto DistributeExecutor::ClientBlockData::execute(AqlCall call, ExecutionState u DistributeExecutor::DistributeExecutor(DistributeExecutorInfos const& infos) : _infos(infos){}; -auto DistributeExecutor::distributeBlock(SharedAqlItemBlockPtr block, +auto DistributeExecutor::distributeBlock(SharedAqlItemBlockPtr block, SkipResult skipped, std::unordered_map& blockMap) -> void { std::unordered_map> choosenMap; @@ -289,6 +298,12 @@ auto DistributeExecutor::distributeBlock(SharedAqlItemBlockPtr block, } target->second.addBlock(block, std::move(value)); } + + // Add the skipResult to all clients. + // It needs to be fetched once for every client. + for (auto& [key, map] : blockMap) { + map.addSkipResult(skipped); + } } auto DistributeExecutor::getClient(SharedAqlItemBlockPtr block, size_t rowIndex) diff --git a/arangod/Aql/DistributeExecutor.h b/arangod/Aql/DistributeExecutor.h index b6954bc1ff3a..0e7bd1a618b6 100644 --- a/arangod/Aql/DistributeExecutor.h +++ b/arangod/Aql/DistributeExecutor.h @@ -92,29 +92,33 @@ class DistributeExecutor { auto clear() -> void; auto addBlock(SharedAqlItemBlockPtr block, std::vector usedIndexes) -> void; + + auto addSkipResult(SkipResult const& skipResult) -> void; auto hasDataFor(AqlCall const& call) -> bool; - auto execute(AqlCall call, ExecutionState upstreamState) - -> std::tuple; + auto execute(AqlCallStack callStack, ExecutionState upstreamState) + -> std::tuple; private: /** * @brief This call will join as many blocks as available from the queue * and return them in a SingleBlock. We then use the IdExecutor * to hand out the data contained in these blocks - * We do on purpose not give any kind of guarantees on the sizing of - * this block to be flexible with the implementation, and find a good + * We do on purpose not give any kind of guarantees on the sizing + * of this block to be flexible with the implementation, and find a good * trade-off between blocksize and block copy operations. * - * @return SharedAqlItemBlockPtr a joind block from the queue. + * @return SharedAqlItemBlockPtr a joined block from the queue. + * SkipResult the skip information matching to this block */ - auto popJoinedBlock() -> SharedAqlItemBlockPtr; + auto popJoinedBlock() -> std::tuple; private: AqlItemBlockManager& _blockManager; ExecutorInfos const& _infos; std::deque>> _queue; + SkipResult _skipped{}; // This is unique_ptr to get away with everything being forward declared... std::unique_ptr _executor; @@ -132,9 +136,10 @@ class DistributeExecutor { * Hence this method is not const ;( * * @param block The block to be distributed + * @param skipped The rows that have been skipped from upstream * @param blockMap Map client => Data. Will provide the required data to the correct client. */ - auto distributeBlock(SharedAqlItemBlockPtr block, + auto distributeBlock(SharedAqlItemBlockPtr block, SkipResult skipped, std::unordered_map& blockMap) -> void; private: diff --git a/arangod/Aql/ExecutionBlock.cpp b/arangod/Aql/ExecutionBlock.cpp index ec4ca75d3433..37538244f10f 100644 --- a/arangod/Aql/ExecutionBlock.cpp +++ b/arangod/Aql/ExecutionBlock.cpp @@ -298,7 +298,7 @@ bool ExecutionBlock::isInSplicedSubquery() const noexcept { return _isInSplicedSubquery; } -void ExecutionBlock::traceExecuteBegin(AqlCallStack const& stack) { +void ExecutionBlock::traceExecuteBegin(AqlCallStack const& stack, std::string const& clientId) { if (_profile >= PROFILE_LEVEL_BLOCKS) { if (_getSomeBegin <= 0.0) { _getSomeBegin = TRI_microtime(); @@ -311,20 +311,22 @@ void ExecutionBlock::traceExecuteBegin(AqlCallStack const& stack) { LOG_TOPIC("1e717", INFO, Logger::QUERIES) << "[query#" << queryId << "] " << "execute type=" << node->getTypeString() << " call= " << call - << " this=" << (uintptr_t)this << " id=" << node->id(); + << " this=" << (uintptr_t)this << " id=" << node->id() + << (clientId.empty() ? "" : " clientId=" + clientId); } } } -auto ExecutionBlock::traceExecuteEnd(std::tuple const& result) - -> std::tuple { +auto ExecutionBlock::traceExecuteEnd(std::tuple const& result, + std::string const& clientId) + -> std::tuple { if (_profile >= PROFILE_LEVEL_BLOCKS) { auto const& [state, skipped, block] = result; auto const items = block != nullptr ? block->size() : 0; ExecutionNode const* en = getPlanNode(); ExecutionStats::Node stats; stats.calls = 1; - stats.items = skipped + items; + stats.items = skipped.getSkipCount() + items; if (state != ExecutionState::WAITING) { stats.runtime = TRI_microtime() - _getSomeBegin; _getSomeBegin = 0.0; @@ -339,9 +341,11 @@ auto ExecutionBlock::traceExecuteEnd(std::tuple= PROFILE_LEVEL_TRACE_1) { ExecutionNode const* node = getPlanNode(); - LOG_QUERY("60bbc", INFO) << "execute done " << printBlockInfo() - << " state=" << stateToString(state) - << " skipped=" << skipped << " produced=" << items; + LOG_QUERY("60bbc", INFO) + << "execute done " << printBlockInfo() << " state=" << stateToString(state) + << " skipped=" << skipped.getSkipCount() << " produced=" << items + << (clientId.empty() ? "" : " clientId=" + clientId); + ; if (_profile >= PROFILE_LEVEL_TRACE_2) { if (block == nullptr) { diff --git a/arangod/Aql/ExecutionBlock.h b/arangod/Aql/ExecutionBlock.h index b3652bc5576c..c1544a123b45 100644 --- a/arangod/Aql/ExecutionBlock.h +++ b/arangod/Aql/ExecutionBlock.h @@ -26,6 +26,7 @@ #include "Aql/BlockCollector.h" #include "Aql/ExecutionState.h" +#include "Aql/SkipResult.h" #include "Basics/Result.h" #include @@ -144,19 +145,21 @@ class ExecutionBlock { /// * WAITING: We have async operation going on, nothing happend, please call again /// * HASMORE: Here is some data in the request range, there is still more, if required call again /// * DONE: Here is some data, and there will be no further data available. - /// 2. size_t: Amount of documents skipped. + /// 2. SkipResult: Amount of documents skipped. /// 3. SharedAqlItemBlockPtr: The next data block. - virtual std::tuple execute(AqlCallStack stack) = 0; + virtual std::tuple execute(AqlCallStack stack) = 0; [[nodiscard]] bool isInSplicedSubquery() const noexcept; protected: // Trace the start of a execute call - void traceExecuteBegin(AqlCallStack const& stack); + void traceExecuteBegin(AqlCallStack const& stack, + std::string const& clientId = ""); // Trace the end of a execute call, potentially with result - auto traceExecuteEnd(std::tuple const& result) - -> std::tuple; + auto traceExecuteEnd(std::tuple const& result, + std::string const& clientId = "") + -> std::tuple; [[nodiscard]] auto printBlockInfo() const -> std::string const; [[nodiscard]] auto printTypeInfo() const -> std::string const; diff --git a/arangod/Aql/ExecutionBlockImpl.cpp b/arangod/Aql/ExecutionBlockImpl.cpp index 8e71f0dc0192..133df32b4677 100644 --- a/arangod/Aql/ExecutionBlockImpl.cpp +++ b/arangod/Aql/ExecutionBlockImpl.cpp @@ -58,6 +58,7 @@ #include "Aql/ShortestPathExecutor.h" #include "Aql/SimpleModifier.h" #include "Aql/SingleRemoteModificationExecutor.h" +#include "Aql/SkipResult.h" #include "Aql/SortExecutor.h" #include "Aql/SortRegister.h" #include "Aql/SortedCollectExecutor.h" @@ -124,6 +125,23 @@ class TestLambdaSkipExecutor; } // namespace arangodb::aql #endif +/* + * Determine whether an executor cannot bypass subquery skips. + * This is if exection of this Executor does have side-effects + * other then it's own result. + */ + +template +constexpr bool executorHasSideEffects = + is_one_of_v, + ModificationExecutor, InsertModifier>, + ModificationExecutor, + ModificationExecutor, RemoveModifier>, + ModificationExecutor, + ModificationExecutor, UpdateReplaceModifier>, + ModificationExecutor, + ModificationExecutor, UpsertModifier>>; + /* * Determine whether we execute new style or old style skips, i.e. pre or post shadow row introduction * TODO: This should be removed once all executors and fetchers are ported to the new style. @@ -507,38 +525,17 @@ static SkipVariants constexpr skipType() { template std::pair ExecutionBlockImpl::skipSome(size_t const atMost) { - if constexpr (isNewStyleExecutor) { - AqlCallStack stack{AqlCall::SimulateSkipSome(atMost)}; - auto const [state, skipped, block] = execute(stack); - - // execute returns ExecutionState::DONE here, which stops execution after simulating a skip. - // If we indiscriminately return ExecutionState::HASMORE, then we end up in an infinite loop - // - // luckily we can dispose of this kludge once executors have been ported. - if (skipped < atMost && state == ExecutionState::DONE) { - return {ExecutionState::DONE, skipped}; - } else { - return {ExecutionState::HASMORE, skipped}; - } + AqlCallStack stack{AqlCall::SimulateSkipSome(atMost)}; + auto const [state, skipped, block] = execute(stack); + + // execute returns ExecutionState::DONE here, which stops execution after simulating a skip. + // If we indiscriminately return ExecutionState::HASMORE, then we end up in an infinite loop + // + // luckily we can dispose of this kludge once executors have been ported. + if (skipped.getSkipCount() < atMost && state == ExecutionState::DONE) { + return {ExecutionState::DONE, skipped.getSkipCount()}; } else { - traceSkipSomeBegin(atMost); - auto state = ExecutionState::HASMORE; - - while (state == ExecutionState::HASMORE && _skipped < atMost) { - auto res = skipSomeOnceWithoutTrace(atMost - _skipped); - TRI_ASSERT(state != ExecutionState::WAITING || res.second == 0); - state = res.first; - _skipped += res.second; - TRI_ASSERT(_skipped <= atMost); - } - - size_t skipped = 0; - if (state != ExecutionState::WAITING) { - std::swap(skipped, _skipped); - } - - TRI_ASSERT(skipped <= atMost); - return traceSkipSomeEnd(state, skipped); + return {ExecutionState::HASMORE, skipped.getSkipCount()}; } } @@ -616,8 +613,8 @@ std::pair ExecutionBlockImpl::initializeCursor _lastRange = DataRange(ExecutorState::HASMORE); } - TRI_ASSERT(_skipped == 0); - _skipped = 0; + TRI_ASSERT(_skipped.nothingSkipped()); + _skipped.reset(); TRI_ASSERT(_state == InternalState::DONE || _state == InternalState::FETCH_DATA); _state = InternalState::FETCH_DATA; @@ -640,7 +637,8 @@ std::pair ExecutionBlockImpl::shutdown(int err } template -std::tuple ExecutionBlockImpl::execute(AqlCallStack stack) { +std::tuple +ExecutionBlockImpl::execute(AqlCallStack stack) { // TODO remove this IF // These are new style executors if constexpr (isNewStyleExecutor) { @@ -674,7 +672,9 @@ std::tuple ExecutionBlockImpl ExecutionBlockImplsize()); if (myCall.getLimit() == 0) { - return {ExecutionState::DONE, 0, block}; + return {ExecutionState::DONE, SkipResult{}, block}; } } - return {state, 0, block}; + return {state, SkipResult{}, block}; } else if (AqlCall::IsFullCountCall(myCall)) { auto const [state, skipped] = skipSome(ExecutionBlock::SkipAllSize()); if (state != ExecutionState::WAITING) { myCall.didSkip(skipped); } - return {state, skipped, nullptr}; + SkipResult skipRes{}; + skipRes.didSkip(skipped); + return {state, skipRes, nullptr}; } else if (AqlCall::IsFastForwardCall(myCall)) { // No idea if DONE is correct here... - return {ExecutionState::DONE, 0, nullptr}; + return {ExecutionState::DONE, SkipResult{}, nullptr}; } // Should never get here! THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); @@ -709,8 +711,8 @@ namespace arangodb::aql { template <> template <> -auto ExecutionBlockImpl>::injectConstantBlock>(SharedAqlItemBlockPtr block) - -> void { +auto ExecutionBlockImpl>::injectConstantBlock>( + SharedAqlItemBlockPtr block, SkipResult skipped) -> void { // reinitialize the DependencyProxy _dependencyProxy.reset(); @@ -718,9 +720,16 @@ auto ExecutionBlockImpl>::injectConstantBlock ExecutionBlockImpl>:: SharedAqlItemBlockPtr block = input.cloneToBlock(_engine->itemBlockManager(), *(infos().registersToKeep()), infos().numberOfOutputRegisters()); - - injectConstantBlock(block); + TRI_ASSERT(_skipped.nothingSkipped()); + _skipped.reset(); + // We inject an empty copy of our skipped here, + // This is resettet, but will maintain the size + injectConstantBlock(block, _skipped); // end of default initializeCursor return ExecutionBlock::initializeCursor(input); @@ -1205,16 +1217,10 @@ static auto fastForwardType(AqlCall const& call, Executor const& e) -> FastForwa TRI_ASSERT(call.hasHardLimit()); return FastForwardVariant::FULLCOUNT; } - // TODO: We only need to do this is the executor actually require to call. - // e.g. Modifications will always need to be called. Limit only if it needs to report fullCount - if constexpr (is_one_of_v, - ModificationExecutor, InsertModifier>, - ModificationExecutor, - ModificationExecutor, RemoveModifier>, - ModificationExecutor, - ModificationExecutor, UpdateReplaceModifier>, - ModificationExecutor, - ModificationExecutor, UpsertModifier>>) { + // TODO: We only need to do this if the executor is required to call. + // e.g. Modifications and SubqueryStart will always need to be called. Limit only if it needs to report fullCount + if constexpr (is_one_of_v || + executorHasSideEffects) { return FastForwardVariant::EXECUTOR; } return FastForwardVariant::FETCHER; @@ -1222,13 +1228,19 @@ static auto fastForwardType(AqlCall const& call, Executor const& e) -> FastForwa template auto ExecutionBlockImpl::executeFetcher(AqlCallStack& stack, AqlCallType const& aqlCall) - -> std::tuple { + -> std::tuple { if constexpr (isNewStyleExecutor) { // TODO The logic in the MultiDependencySingleRowFetcher branch should be // moved into the MultiDependencySingleRowFetcher. static_assert(isMultiDepExecutor == std::is_same_v); if constexpr (std::is_same_v) { + static_assert( + !executorHasSideEffects, + "there is a special implementation for side-effect executors to " + "exchange the stack. For the MultiDependencyFetcher this special " + "case is not implemented. There is no reason to disallow this " + "case here however, it is just not needed thus far."); // Note the aqlCall is an AqlCallSet in this case: static_assert(std::is_same_v>); TRI_ASSERT(_lastRange.numberDependencies() == _dependencies.size()); @@ -1237,14 +1249,25 @@ auto ExecutionBlockImpl::executeFetcher(AqlCallStack& stack, AqlCallTy _lastRange.setDependency(dependency, range); } return {state, skipped, _lastRange}; + } else if constexpr (executorHasSideEffects) { + // If the executor has side effects, we cannot bypass any subqueries + // by skipping them. SO we need to fetch all shadow rows in order to + // trigger this Executor with everthing from above. + // NOTE: The Executor needs to discard shadowRows, and do the accouting. + static_assert(std::is_same_v>); + auto fetchAllStack = stack.createEquivalentFetchAllShadowRowsStack(); + fetchAllStack.pushCall(aqlCall); + auto res = _rowFetcher.execute(fetchAllStack); + // Just make sure we did not Skip anything + TRI_ASSERT(std::get(res).nothingSkipped()); + return res; } else { // If we are SubqueryStart, we remove the top element of the stack // which belongs to the subquery enclosed by this // SubqueryStart and the partnered SubqueryEnd by *not* // pushing the upstream request. if constexpr (!std::is_same_v) { - auto callCopy = _upstreamRequest; - stack.pushCall(std::move(callCopy)); + stack.pushCall(std::move(aqlCall)); } auto const result = _rowFetcher.execute(stack); @@ -1424,6 +1447,10 @@ auto ExecutionBlockImpl::shadowRowForwarding() -> ExecState // We still have shadowRows, we // need to forward them return ExecState::SHADOWROWS; + } else if (_outputItemRow->isFull()) { + // Fullfilled the call + // Need to return! + return ExecState::DONE; } else { if (didConsume) { // We did only consume the input @@ -1436,6 +1463,119 @@ auto ExecutionBlockImpl::shadowRowForwarding() -> ExecState } } +template +auto ExecutionBlockImpl::nextStateAfterShadowRows(ExecutorState const& state, + DataRange const& range) const + noexcept -> ExecState { + if (state == ExecutorState::DONE) { + // We have consumed everything, we are + // Done with this query + return ExecState::DONE; + } else if (range.hasDataRow()) { + // Multiple concatenated Subqueries + // This case is disallowed for now, as we do not know the + // look-ahead call + TRI_ASSERT(false); + // If we would know we could now go into a continue with next subquery + // state. + return ExecState::DONE; + } else if (range.hasShadowRow()) { + // We still have shadowRows, we + // need to forward them + return ExecState::SHADOWROWS; + } else { + // End of input, we are done for now + // Need to call again + return ExecState::DONE; + } +} + +template +auto ExecutionBlockImpl::sideEffectShadowRowForwarding(AqlCallStack& stack, + SkipResult& skipResult) + -> ExecState { + TRI_ASSERT(executorHasSideEffects); + if (!stack.needToSkipSubquery()) { + // We need to really produce things here + // fall back to original version as any other executor. + return shadowRowForwarding(); + } + TRI_ASSERT(_outputItemRow); + TRI_ASSERT(_outputItemRow->isInitialized()); + TRI_ASSERT(!_outputItemRow->allRowsUsed()); + if (!_lastRange.hasShadowRow()) { + // We got back without a ShadowRow in the LastRange + // Let client call again + return ExecState::DONE; + } + + auto const& [state, shadowRow] = _lastRange.nextShadowRow(); + TRI_ASSERT(shadowRow.isInitialized()); + uint64_t depthSkippingNow = static_cast(stack.shadowRowDepthToSkip()); + uint64_t shadowDepth = shadowRow.getDepth(); + + bool didWriteRow = false; + if (shadowRow.isRelevant()) { + LOG_QUERY("1b257", DEBUG) << printTypeInfo() << " init executor."; + // We found a relevant shadow Row. + // We need to reset the Executor + resetExecutor(); + } + + if (depthSkippingNow > shadowDepth) { + // We are skipping the outermost Subquery. + // Simply drop this ShadowRow + } else if (depthSkippingNow == shadowDepth) { + // We are skipping on this subquery level. + // Skip the row, but report skipped 1. + AqlCall& shadowCall = stack.modifyCallAtDepth(shadowDepth); + if (shadowCall.needSkipMore()) { + shadowCall.didSkip(1); + skipResult.didSkipSubquery(1, shadowDepth); + } else { + TRI_ASSERT(shadowCall.hardLimit == 0); + // Simply drop this shadowRow! + } + } else { + // We got a shadowRow of a subquery we are not skipping here. + // Do proper reporting on it's call. + AqlCall& shadowCall = stack.modifyCallAtDepth(shadowDepth); + TRI_ASSERT(!shadowCall.needSkipMore() && shadowCall.getLimit() > 0); + _outputItemRow->copyRow(shadowRow); + shadowCall.didProduce(1); + + TRI_ASSERT(_outputItemRow->produced()); + _outputItemRow->advanceRow(); + didWriteRow = true; + } + if (state == ExecutorState::DONE) { + // We have consumed everything, we are + // Done with this query + return ExecState::DONE; + } else if (_lastRange.hasDataRow()) { + // Multiple concatenated Subqueries + // This case is disallowed for now, as we do not know the + // look-ahead call + TRI_ASSERT(false); + // If we would know we could now go into a continue with next subquery + // state. + return ExecState::DONE; + } else if (_lastRange.hasShadowRow()) { + // We still have shadowRows, we + // need to forward them + return ExecState::SHADOWROWS; + } else if (didWriteRow) { + // End of input, we are done for now + // Need to call again + return ExecState::DONE; + } else { + // Done with this subquery. + // We did not write any output yet. + // So we can continue with upstream. + return ExecState::UPSTREAM; + } +} + template auto ExecutionBlockImpl::shadowRowForwarding() -> ExecState { TRI_ASSERT(_outputItemRow); @@ -1461,7 +1601,6 @@ auto ExecutionBlockImpl::shadowRowForwarding() -> ExecState { TRI_ASSERT(_outputItemRow->produced()); _outputItemRow->advanceRow(); - if (state == ExecutorState::DONE) { // We have consumed everything, we are // Done with this query @@ -1490,29 +1629,30 @@ auto ExecutionBlockImpl::executeFastForward(typename Fetcher::DataRang AqlCall& clientCall) -> std::tuple { TRI_ASSERT(isNewStyleExecutor); - if constexpr (std::is_same_v) { - if (clientCall.needsFullCount() && clientCall.getOffset() == 0 && - clientCall.getLimit() == 0) { - // We can savely call skipRows. - // It will not report anything if the row is already consumed - return executeSkipRowsRange(_lastRange, clientCall); - } - // Do not fastForward anything, the Subquery start will handle it by itself - return {ExecutorState::DONE, NoStats{}, 0, AqlCall{}}; - } auto type = fastForwardType(clientCall, _executor); switch (type) { - case FastForwardVariant::FULLCOUNT: - case FastForwardVariant::EXECUTOR: { + case FastForwardVariant::FULLCOUNT: { LOG_QUERY("cb135", DEBUG) << printTypeInfo() << " apply full count."; auto [state, stats, skippedLocal, call] = executeSkipRowsRange(_lastRange, clientCall); - if (type == FastForwardVariant::EXECUTOR) { - // We do not report the skip - skippedLocal = 0; + if constexpr (is_one_of_v) { + // The executor will have used all Rows. + // However we need to drop them from the input + // here. + inputRange.skipAllRemainingDataRows(); } + return {state, stats, skippedLocal, call}; + } + case FastForwardVariant::EXECUTOR: { + LOG_QUERY("2890e", DEBUG) << printTypeInfo() << " fast forward."; + // We use a DUMMY Call to simulate fullCount. + AqlCall dummy; + dummy.hardLimit = 0; + dummy.fullCount = true; + auto [state, stats, skippedLocal, call] = executeSkipRowsRange(_lastRange, dummy); + if constexpr (is_one_of_v) { // The executor will have used all Rows. // However we need to drop them from the input @@ -1520,7 +1660,7 @@ auto ExecutionBlockImpl::executeFastForward(typename Fetcher::DataRang inputRange.skipAllRemainingDataRows(); } - return {state, stats, skippedLocal, call}; + return {state, stats, 0, call}; } case FastForwardVariant::FETCHER: { LOG_QUERY("fa327", DEBUG) << printTypeInfo() << " bypass unused rows."; @@ -1531,8 +1671,8 @@ auto ExecutionBlockImpl::executeFastForward(typename Fetcher::DataRang return fastForwardCall; } else { #ifndef _WIN32 - // For some reason our Windows compiler complains about this static assert - // in the cases that should be in the above constexpr path. + // For some reason our Windows compiler complains about this static + // assert in the cases that should be in the above constexpr path. // So simply not compile it in. static_assert(std::is_same_v); #endif @@ -1604,7 +1744,7 @@ auto ExecutionBlockImpl::executeFastForward(typename Fetcher::DataRang * SharedAqlItemBlockPtr -> The resulting data */ template -std::tuple +std::tuple ExecutionBlockImpl::executeWithoutTrace(AqlCallStack stack) { if constexpr (isNewStyleExecutor) { if (!stack.isRelevant()) { @@ -1612,7 +1752,7 @@ ExecutionBlockImpl::executeWithoutTrace(AqlCallStack stack) { // We are bypassing subqueries. // This executor is not allowed to perform actions // However we need to maintain the upstream state. - size_t skippedLocal = 0; + SkipResult skippedLocal; typename Fetcher::DataRange bypassedRange{ExecutorState::HASMORE}; std::tie(_upstreamState, skippedLocal, bypassedRange) = executeFetcher(stack, _upstreamRequest); @@ -1638,25 +1778,44 @@ ExecutionBlockImpl::executeWithoutTrace(AqlCallStack stack) { _execState == ExecState::UPSTREAM); } - // Skip can only be > 0 if we are in upstream cases. - TRI_ASSERT(_skipped == 0 || _execState == ExecState::UPSTREAM); + // In some executors we may write something into the output, but then return waiting. + // In this case we are not allowed to lose the call we have been working on, we have + // noted down created or skipped rows in there. + // The client is disallowed to change her mind anyways + // so we simply continue to work on the call we already have + // The guarantee is, if we have returned the block, and modified + // our local call, then the outputItemRow is not initialized + if (_outputItemRow != nullptr && _outputItemRow->isInitialized()) { + clientCall = _outputItemRow->getClientCall(); + } - if constexpr (std::is_same_v) { - // TODO: implement forwarding of SKIP properly: - // We need to modify the execute API to instead return a vector of skipped - // values. - // Then we can simply push a skip on the Stack here and let it forward. - // In case of a modifaction we need to NOT forward a skip, but instead do - // a limit := limit + offset call and a hardLimit 0 call on top of the stack. - TRI_ASSERT(!clientCall.needSkipMore()); + // Skip can only be > 0 if we are in upstream cases, or if we got injected a block + TRI_ASSERT(_skipped.nothingSkipped() || _execState == ExecState::UPSTREAM || + (std::is_same_v>)); + + if constexpr (executorHasSideEffects) { + if (!_skipped.nothingSkipped()) { + // We get woken up on upstream, but we have not reported our + // local skip value to downstream + // In the sideEffect executor we need to apply the skip values on the + // incomming stack, which has not been modified yet. + // NOTE: We only apply the skipping on subquery level. + TRI_ASSERT(_skipped.subqueryDepth() == stack.subqueryLevel() + 1); + for (size_t i = 0; i < stack.subqueryLevel(); ++i) { + auto skippedSub = _skipped.getSkipOnSubqueryLevel(i); + if (skippedSub > 0) { + auto& call = stack.modifyCallAtDepth(i); + call.didSkip(skippedSub); + } + } + } + } + if constexpr (std::is_same_v) { // In subqeryEndExecutor we actually manage two calls. // The clientClient is defined of what will go into the Executor. // on SubqueryEnd this call is generated based on the call from downstream stack.pushCall(std::move(clientCall)); - // TODO: Implement different kind of calls we need to inject into Executor - // based on modification, or on forwarding. - // FOr now use a fetchUnlimited Call always clientCall = AqlCall{}; } if (_execState == ExecState::UPSTREAM) { @@ -1680,6 +1839,16 @@ ExecutionBlockImpl::executeWithoutTrace(AqlCallStack stack) { case ExecState::CHECKCALL: { LOG_QUERY("cfe46", DEBUG) << printTypeInfo() << " determine next action on call " << clientCall; + + if constexpr (executorHasSideEffects) { + // If the executor has sideEffects, and we need to skip the results we would + // produce here because we actually skip the subquery, we instead do a + // hardLimit 0 (aka FastForward) call instead to the local Executor + if (stack.needToSkipSubquery()) { + _execState = ExecState::FASTFORWARD; + break; + } + } _execState = nextState(clientCall); break; } @@ -1705,7 +1874,7 @@ ExecutionBlockImpl::executeWithoutTrace(AqlCallStack stack) { _executor.skipRowsRange(_lastRange, clientCall); if (subqueryState == ExecutionState::WAITING) { TRI_ASSERT(skippedLocal == 0); - return {subqueryState, 0, nullptr}; + return {subqueryState, SkipResult{}, nullptr}; } else if (subqueryState == ExecutionState::DONE) { state = ExecutorState::DONE; } else { @@ -1735,7 +1904,7 @@ ExecutionBlockImpl::executeWithoutTrace(AqlCallStack stack) { } #endif localExecutorState = state; - _skipped += skippedLocal; + _skipped.didSkip(skippedLocal); _engine->_stats += stats; // The execute might have modified the client call. if (state == ExecutorState::DONE) { @@ -1787,7 +1956,7 @@ ExecutionBlockImpl::executeWithoutTrace(AqlCallStack stack) { std::tie(subqueryState, stats, call) = _executor.produceRows(_lastRange, *_outputItemRow); if (subqueryState == ExecutionState::WAITING) { - return {subqueryState, 0, nullptr}; + return {subqueryState, SkipResult{}, nullptr}; } else if (subqueryState == ExecutionState::DONE) { state = ExecutorState::DONE; } else { @@ -1829,10 +1998,30 @@ ExecutionBlockImpl::executeWithoutTrace(AqlCallStack stack) { case ExecState::FASTFORWARD: { LOG_QUERY("96e2c", DEBUG) << printTypeInfo() << " all produced, fast forward to end up (sub-)query."; + + AqlCall callCopy = clientCall; + if constexpr (executorHasSideEffects) { + if (stack.needToSkipSubquery()) { + // Fast Forward call. + callCopy = AqlCall{0, false, 0, AqlCall::LimitType::HARD}; + } + } + auto [state, stats, skippedLocal, call] = - executeFastForward(_lastRange, clientCall); + executeFastForward(_lastRange, callCopy); - _skipped += skippedLocal; + if constexpr (executorHasSideEffects) { + if (!stack.needToSkipSubquery()) { + // We need to modify the original call. + clientCall = callCopy; + } + // else: We are bypassing the results. + // Do not count them here. + } else { + clientCall = callCopy; + } + + _skipped.didSkip(skippedLocal); _engine->_stats += stats; localExecutorState = state; @@ -1859,14 +2048,13 @@ ExecutionBlockImpl::executeWithoutTrace(AqlCallStack stack) { // executors. TRI_ASSERT(isMultiDepExecutor || !lastRangeHasDataRow()); TRI_ASSERT(!_lastRange.hasShadowRow()); - size_t skippedLocal = 0; + SkipResult skippedLocal; #ifdef ARANGODB_ENABLE_MAINTAINER_MODE auto subqueryLevelBefore = stack.subqueryLevel(); #endif std::tie(_upstreamState, skippedLocal, _lastRange) = executeFetcher(stack, _upstreamRequest); - #ifdef ARANGODB_ENABLE_MAINTAINER_MODE TRI_ASSERT(subqueryLevelBefore == stack.subqueryLevel()); #endif @@ -1876,18 +2064,45 @@ ExecutionBlockImpl::executeWithoutTrace(AqlCallStack stack) { // We might have some local accounting to this call. _clientRequest = clientCall; // We do not return anything in WAITING state, also NOT skipped. - return {_upstreamState, 0, nullptr}; + return {_upstreamState, SkipResult{}, nullptr}; } if constexpr (Executor::Properties::allowsBlockPassthrough == BlockPassthrough::Enable) { // We have a new range, passthrough can use this range. _hasUsedDataRangeBlock = false; } + + if constexpr (std::is_same_v) { + // We need to pop the last subquery from the returned skip + // We have not asked for a subquery skip. + TRI_ASSERT(skippedLocal.getSkipCount() == 0); + skippedLocal.decrementSubquery(); + } if constexpr (skipRowsType() == SkipRowsRangeVariant::FETCHER) { - _skipped += skippedLocal; // We skipped through passthrough, so count that a skip was solved. - clientCall.didSkip(skippedLocal); + _skipped.merge(skippedLocal, false); + clientCall.didSkip(skippedLocal.getSkipCount()); + } else if constexpr (is_one_of_v) { + // Subquery needs to include the topLevel Skip. + // But does not need to apply the count to clientCall. + _skipped.merge(skippedLocal, false); + // This is what has been asked for by the SubqueryEnd + auto subqueryCall = stack.popCall(); + subqueryCall.didSkip(skippedLocal.getSkipCount()); + stack.pushCall(std::move(subqueryCall)); + } else { + _skipped.merge(skippedLocal, true); } + if constexpr (std::is_same_v) { + // For the subqueryStart, we need to increment the SkipLevel by one + // as we may trigger this multiple times, check if we need to do it. + while (_skipped.subqueryDepth() < stack.subqueryLevel() + 1) { + // In fact, we only need to increase by 1 + TRI_ASSERT(_skipped.subqueryDepth() == stack.subqueryLevel()); + _skipped.incrementSubquery(); + } + } + if (_lastRange.hasShadowRow() && !_lastRange.peekShadowRow().isRelevant()) { _execState = ExecState::SHADOWROWS; } else { @@ -1924,9 +2139,19 @@ ExecutionBlockImpl::executeWithoutTrace(AqlCallStack stack) { } TRI_ASSERT(!_outputItemRow->allRowsUsed()); - - // This may write one or more rows. - _execState = shadowRowForwarding(); + if constexpr (executorHasSideEffects) { + _execState = sideEffectShadowRowForwarding(stack, _skipped); + } else { + // This may write one or more rows. + _execState = shadowRowForwarding(); + if constexpr (std::is_same_v) { + // we need to update the Top of the stack now + std::ignore = stack.popCall(); + // Copy the call + AqlCall modifiedCall = _outputItemRow->getClientCall(); + stack.pushCall(std::move(modifiedCall)); + } + } if constexpr (!std::is_same_v) { // Produce might have modified the clientCall // But only do this if we are not subquery. @@ -1952,17 +2177,29 @@ ExecutionBlockImpl::executeWithoutTrace(AqlCallStack stack) { _outputItemRow.reset(); // We return skipped here, reset member - size_t skipped = _skipped; - _skipped = 0; + SkipResult skipped = _skipped; +#ifdef ARANGODB_ENABLE_MAINTAINER_MODE + if (!stack.is36Compatible()) { + if constexpr (std::is_same_v) { + TRI_ASSERT(skipped.subqueryDepth() == stack.subqueryLevel() /*we inected a call*/); + } else { + TRI_ASSERT(skipped.subqueryDepth() == stack.subqueryLevel() + 1 /*we took our call*/); + } + } +#endif + + _skipped.reset(); if (localExecutorState == ExecutorState::HASMORE || _lastRange.hasDataRow() || _lastRange.hasShadowRow()) { // We have skipped or/and return data, otherwise we cannot return HASMORE - TRI_ASSERT(skipped > 0 || (outputBlock != nullptr && outputBlock->numEntries() > 0)); + TRI_ASSERT(!skipped.nothingSkipped() || + (outputBlock != nullptr && outputBlock->numEntries() > 0)); return {ExecutionState::HASMORE, skipped, std::move(outputBlock)}; } - // We must return skipped and/or data when reportingHASMORE + // We must return skipped and/or data when reporting HASMORE TRI_ASSERT(_upstreamState != ExecutionState::HASMORE || - (skipped > 0 || (outputBlock != nullptr && outputBlock->numEntries() > 0))); + (!skipped.nothingSkipped() || + (outputBlock != nullptr && outputBlock->numEntries() > 0))); return {_upstreamState, skipped, std::move(outputBlock)}; } else { // TODO this branch must never be taken with an executor that has not been diff --git a/arangod/Aql/ExecutionBlockImpl.h b/arangod/Aql/ExecutionBlockImpl.h index 8e1009c5b518..1b4365c78c40 100644 --- a/arangod/Aql/ExecutionBlockImpl.h +++ b/arangod/Aql/ExecutionBlockImpl.h @@ -52,6 +52,8 @@ class InputAqlItemRow; class OutputAqlItemRow; class Query; class ShadowAqlItemRow; +class SkipResult; +class ParallelUnsortedGatherExecutor; class MultiDependencySingleRowFetcher; template @@ -223,7 +225,7 @@ class ExecutionBlockImpl final : public ExecutionBlock { [[nodiscard]] std::pair initializeCursor(InputAqlItemRow const& input) override; template >>> - auto injectConstantBlock(SharedAqlItemBlockPtr block) -> void; + auto injectConstantBlock(SharedAqlItemBlockPtr block, SkipResult skipped) -> void; [[nodiscard]] Infos const& infos() const; @@ -242,9 +244,9 @@ class ExecutionBlockImpl final : public ExecutionBlock { /// * WAITING: We have async operation going on, nothing happend, please call again /// * HASMORE: Here is some data in the request range, there is still more, if required call again /// * DONE: Here is some data, and there will be no further data available. - /// 2. size_t: Amount of documents skipped. + /// 2. SkipResult: Amount of documents skipped. /// 3. SharedAqlItemBlockPtr: The next data block. - std::tuple execute(AqlCallStack stack) override; + std::tuple execute(AqlCallStack stack) override; template >>>> [[nodiscard]] RegisterId getOutputRegisterId() const noexcept; @@ -253,9 +255,9 @@ class ExecutionBlockImpl final : public ExecutionBlock { /** * @brief Inner execute() part, without the tracing calls. */ - std::tuple executeWithoutTrace(AqlCallStack stack); + std::tuple executeWithoutTrace(AqlCallStack stack); - std::tuple executeFetcher( + std::tuple executeFetcher( AqlCallStack& stack, AqlCallType const& aqlCall); std::tuple executeProduceRows( @@ -329,6 +331,26 @@ class ExecutionBlockImpl final : public ExecutionBlock { void resetExecutor(); + // Forwarding of ShadowRows if the executor has SideEffects. + // This skips over ShadowRows, and counts them in the correct + // position of the callStack as "skipped". + // as soon as we reach a place where there is no skip + // ordered in the outer shadow rows, this call + // will fall back to shadowRowForwardning. + [[nodiscard]] auto sideEffectShadowRowForwarding(AqlCallStack& stack, + SkipResult& skipResult) -> ExecState; + + /** + * @brief Transition to the next state after shadowRows + * + * @param state the state returned by the getShadowRowCall + * @param range the current data range + * @return ExecState The next state + */ + [[nodiscard]] auto nextStateAfterShadowRows(ExecutorState const& state, + DataRange const& range) const + noexcept -> ExecState; + void initOnce(); [[nodiscard]] auto executorNeedsCall(AqlCallType& call) const noexcept -> bool; @@ -361,7 +383,7 @@ class ExecutionBlockImpl final : public ExecutionBlock { InternalState _state; - size_t _skipped{}; + SkipResult _skipped{}; DataRange _lastRange; diff --git a/arangod/Aql/ExecutionEngine.cpp b/arangod/Aql/ExecutionEngine.cpp index 2fa51ea000df..a4e30799654e 100644 --- a/arangod/Aql/ExecutionEngine.cpp +++ b/arangod/Aql/ExecutionEngine.cpp @@ -39,6 +39,7 @@ #include "Aql/QueryRegistry.h" #include "Aql/RemoteExecutor.h" #include "Aql/ReturnExecutor.h" +#include "Aql/SkipResult.h" #include "Aql/WalkerWorker.h" #include "Basics/ScopeGuard.h" #include "Cluster/ServerState.h" @@ -564,16 +565,16 @@ std::pair ExecutionEngine::initializeCursor(SharedAqlIte } auto ExecutionEngine::execute(AqlCallStack const& stack) - -> std::tuple { + -> std::tuple { if (_query.killed()) { THROW_ARANGO_EXCEPTION(TRI_ERROR_QUERY_KILLED); } auto const res = _root->execute(stack); #ifdef ARANGODB_ENABLE_MAINTAINER_MODE if (std::get(res) == ExecutionState::WAITING) { - auto const skipped = std::get(res); + auto const skipped = std::get(res); auto const block = std::get(res); - TRI_ASSERT(skipped == 0); + TRI_ASSERT(skipped.nothingSkipped()); TRI_ASSERT(block == nullptr); } #endif @@ -581,7 +582,7 @@ auto ExecutionEngine::execute(AqlCallStack const& stack) } auto ExecutionEngine::executeForClient(AqlCallStack const& stack, std::string const& clientId) - -> std::tuple { + -> std::tuple { if (_query.killed()) { THROW_ARANGO_EXCEPTION(TRI_ERROR_QUERY_KILLED); } @@ -597,9 +598,9 @@ auto ExecutionEngine::executeForClient(AqlCallStack const& stack, std::string co auto const res = rootBlock->executeForClient(stack, clientId); #ifdef ARANGODB_ENABLE_MAINTAINER_MODE if (std::get(res) == ExecutionState::WAITING) { - auto const skipped = std::get(res); + auto const skipped = std::get(res); auto const& block = std::get(res); - TRI_ASSERT(skipped == 0); + TRI_ASSERT(skipped.nothingSkipped()); TRI_ASSERT(block == nullptr); } #endif @@ -621,7 +622,7 @@ std::pair ExecutionEngine::getSome(size_t AqlCallStack compatibilityStack{AqlCall::SimulateGetSome(atMost), true}; auto const [state, skipped, block] = _root->execute(std::move(compatibilityStack)); // We cannot trigger a skip operation from here - TRI_ASSERT(skipped == 0); + TRI_ASSERT(skipped.nothingSkipped()); return {state, block}; } @@ -643,7 +644,7 @@ std::pair ExecutionEngine::skipSome(size_t atMost) { // We cannot be triggered within a subquery from earlier versions. // Also we cannot produce anything ourselfes here. TRI_ASSERT(block == nullptr); - return {state, skipped}; + return {state, skipped.getSkipCount()}; } Result ExecutionEngine::shutdownSync(int errorCode) noexcept try { diff --git a/arangod/Aql/ExecutionEngine.h b/arangod/Aql/ExecutionEngine.h index f9af9ff3d2c2..2b51c217ff6f 100644 --- a/arangod/Aql/ExecutionEngine.h +++ b/arangod/Aql/ExecutionEngine.h @@ -47,6 +47,7 @@ class ExecutionNode; class ExecutionPlan; class QueryRegistry; class Query; +class SkipResult; enum class SerializationFormat; class ExecutionEngine { @@ -98,10 +99,10 @@ class ExecutionEngine { std::pair shutdown(int errorCode); auto execute(AqlCallStack const& stack) - -> std::tuple; + -> std::tuple; auto executeForClient(AqlCallStack const& stack, std::string const& clientId) - -> std::tuple; + -> std::tuple; /// @brief getSome std::pair getSome(size_t atMost); diff --git a/arangod/Aql/ModificationExecutor.cpp b/arangod/Aql/ModificationExecutor.cpp index 6554c6574fed..f3712fd09199 100644 --- a/arangod/Aql/ModificationExecutor.cpp +++ b/arangod/Aql/ModificationExecutor.cpp @@ -201,7 +201,6 @@ template doCollect(input, output.numRowsLeft()); upstreamState = input.upstreamState(); } - if (_modifier.nrOfOperations() > 0) { _modifier.transact(); @@ -266,7 +265,6 @@ template stats.addWritesExecuted(_modifier.nrOfWritesExecuted()); stats.addWritesIgnored(_modifier.nrOfWritesIgnored()); } - call.didSkip(_modifier.nrOfOperations()); } } diff --git a/arangod/Aql/MultiDependencySingleRowFetcher.cpp b/arangod/Aql/MultiDependencySingleRowFetcher.cpp index b275f0c19567..fba33607ad79 100644 --- a/arangod/Aql/MultiDependencySingleRowFetcher.cpp +++ b/arangod/Aql/MultiDependencySingleRowFetcher.cpp @@ -147,6 +147,8 @@ std::pair MultiDependencySingleRowFetcher::fet ++dep._rowIndex; } } + // We have delivered a shadowRow, we now may get additional subquery skip counters again. + _didReturnSubquerySkips = false; } ExecutionState const state = allDone ? ExecutionState::DONE : ExecutionState::HASMORE; @@ -370,11 +372,11 @@ auto MultiDependencySingleRowFetcher::useStack(AqlCallStack const& stack) -> voi auto MultiDependencySingleRowFetcher::executeForDependency(size_t const dependency, AqlCallStack& stack) - -> std::tuple { + -> std::tuple { auto [state, skipped, block] = _dependencyProxy->executeForDependency(dependency, stack); if (state == ExecutionState::WAITING) { - return {state, 0, AqlItemBlockInputRange{ExecutorState::HASMORE}}; + return {state, SkipResult{}, AqlItemBlockInputRange{ExecutorState::HASMORE}}; } ExecutorState execState = state == ExecutionState::DONE ? ExecutorState::DONE : ExecutorState::HASMORE; @@ -382,16 +384,17 @@ auto MultiDependencySingleRowFetcher::executeForDependency(size_t const dependen _dependencyStates.at(dependency) = state; if (block == nullptr) { - return {state, skipped, AqlItemBlockInputRange{execState, skipped}}; + return {state, skipped, AqlItemBlockInputRange{execState, skipped.getSkipCount()}}; } TRI_ASSERT(block != nullptr); auto [start, end] = block->getRelevantRange(); - return {state, skipped, AqlItemBlockInputRange{execState, skipped, block, start}}; + return {state, skipped, + AqlItemBlockInputRange{execState, skipped.getSkipCount(), block, start}}; } auto MultiDependencySingleRowFetcher::execute(AqlCallStack const& stack, AqlCallSet const& aqlCallSet) - -> std::tuple>> { + -> std::tuple>> { TRI_ASSERT(_callsInFlight.size() == numberDependencies()); auto ranges = std::vector>{}; @@ -400,7 +403,7 @@ auto MultiDependencySingleRowFetcher::execute(AqlCallStack const& stack, auto depCallIdx = size_t{0}; auto allAskedDepsAreWaiting = true; auto askedAtLeastOneDep = false; - auto skippedTotal = size_t{0}; + auto skippedTotal = SkipResult{}; // Iterate in parallel over `_callsInFlight` and `aqlCall.calls`. // _callsInFlight[i] corresponds to aqlCalls.calls[k] iff // aqlCalls.calls[k].dependency = i. @@ -435,16 +438,46 @@ auto MultiDependencySingleRowFetcher::execute(AqlCallStack const& stack, // Got a result, call is no longer in flight maybeCallInFlight = std::nullopt; allAskedDepsAreWaiting = false; + + // NOTE: + // in this fetcher case we do not have and do not want to have + // any control of the order the upstream responses are entering. + // Every of the upstream response will contain an identical skipped + // stack on the subqueries. + // We only need to forward the skipping of any one of those. + // So we implemented the following logic to return the skip + // information for the first on that arrives and all other + // subquery skip informations will be discarded. + if (!_didReturnSubquerySkips) { + // We have nothing skipped locally. + TRI_ASSERT(skippedTotal.subqueryDepth() == 1); + TRI_ASSERT(skippedTotal.getSkipCount() == 0); + + // We forward the skip block as is. + // This will also include the skips on subquery level + skippedTotal = skipped; + // Do this only once. + // The first response will contain the amount of rows skipped + // in subquery + _didReturnSubquerySkips = true; + } else { + // We only need the skip amount on the top level. + // Another dependency has forwarded the subquery level skips + // already + skippedTotal.mergeOnlyTopLevel(skipped); + } + } else { - TRI_ASSERT(skipped == 0); + TRI_ASSERT(skipped.nothingSkipped()); } - skippedTotal += skipped; + ranges.emplace_back(dependency, range); } } auto const state = std::invoke([&]() { if (askedAtLeastOneDep && allAskedDepsAreWaiting) { + TRI_ASSERT(skippedTotal.nothingSkipped()); return ExecutionState::WAITING; } else { return upstreamState(); diff --git a/arangod/Aql/MultiDependencySingleRowFetcher.h b/arangod/Aql/MultiDependencySingleRowFetcher.h index 277ebf628997..13358f5e95c4 100644 --- a/arangod/Aql/MultiDependencySingleRowFetcher.h +++ b/arangod/Aql/MultiDependencySingleRowFetcher.h @@ -39,6 +39,7 @@ class AqlItemBlock; template class DependencyProxy; class ShadowAqlItemRow; +class SkipResult; /** * @brief Interface for all AqlExecutors that do need one @@ -137,10 +138,10 @@ class MultiDependencySingleRowFetcher { auto useStack(AqlCallStack const& stack) -> void; [[nodiscard]] auto execute(AqlCallStack const&, AqlCallSet const&) - -> std::tuple>>; + -> std::tuple>>; [[nodiscard]] auto executeForDependency(size_t dependency, AqlCallStack& stack) - -> std::tuple; + -> std::tuple; [[nodiscard]] auto upstreamState() const -> ExecutionState; @@ -158,6 +159,8 @@ class MultiDependencySingleRowFetcher { /// in initOnce() to make sure that init() is called exactly once. std::vector> _callsInFlight; + bool _didReturnSubquerySkips{false}; + private: /** * @brief Delegates to ExecutionBlock::fetchBlock() diff --git a/arangod/Aql/OptimizerRules.cpp b/arangod/Aql/OptimizerRules.cpp index 0d344f22e0cb..9cd9e7b83bd2 100644 --- a/arangod/Aql/OptimizerRules.cpp +++ b/arangod/Aql/OptimizerRules.cpp @@ -74,9 +74,9 @@ namespace { -bool accessesCollectionVariable(arangodb::aql::ExecutionPlan const* plan, - arangodb::aql::ExecutionNode const* node, - ::arangodb::containers::HashSet& vars) { +bool accessesCollectionVariable( + arangodb::aql::ExecutionPlan const* plan, arangodb::aql::ExecutionNode const* node, + ::arangodb::containers::HashSet& vars) { using EN = arangodb::aql::ExecutionNode; if (node->getType() == EN::CALCULATION) { @@ -5796,9 +5796,9 @@ void arangodb::aql::optimizeTraversalsRule(Optimizer* opt, if (outVariable != nullptr && !n->isVarUsedLater(outVariable) && std::find(pruneVars.begin(), pruneVars.end(), outVariable) == pruneVars.end()) { outVariable = traversal->pathOutVariable(); - if (outVariable == nullptr || - (!n->isVarUsedLater(outVariable) && - std::find(pruneVars.begin(), pruneVars.end(), outVariable) == pruneVars.end())) { + if (outVariable == nullptr || (!n->isVarUsedLater(outVariable) && + std::find(pruneVars.begin(), pruneVars.end(), + outVariable) == pruneVars.end())) { // both traversal vertex and path outVariables not used later traversal->options()->setProduceVertices(false); modified = true; @@ -7275,12 +7275,12 @@ void arangodb::aql::moveFiltersIntoEnumerateRule(Optimizer* opt, ExecutionNode* filterParent = current->getFirstParent(); TRI_ASSERT(filterParent != nullptr); plan->unlinkNode(current); - + if (!current->isVarUsedLater(cn->outVariable())) { // also remove the calculation node plan->unlinkNode(cn); } - + current = filterParent; modified = true; } else if (current->getType() == EN::CALCULATION) { @@ -7403,19 +7403,11 @@ bool nodeMakesThisQueryLevelUnsuitableForSubquerySplicing(ExecutionNode const* n case ExecutionNode::DISTRIBUTE_CONSUMER: case ExecutionNode::SUBQUERY_START: case ExecutionNode::SUBQUERY_END: - // These nodes do not initiate a skip themselves, and thus are fine. - return false; case ExecutionNode::NORESULTS: - // no results currently cannot work, as they do not fetch from above. case ExecutionNode::LIMIT: - // limit blocks currently cannot work, both due to skipping and due to the - // limit and passthrough, which forbids passing shadow rows. - return true; - case ExecutionNode::COLLECT: { - auto const collectNode = ExecutionNode::castTo(node); - // Collect nodes skip iff using the COUNT method. - return collectNode->aggregationMethod() == CollectOptions::CollectMethod::COUNT; - } + case ExecutionNode::COLLECT: + // These nodes are fine + return false; case ExecutionNode::MAX_NODE_TYPE_VALUE: break; } @@ -7425,7 +7417,7 @@ bool nodeMakesThisQueryLevelUnsuitableForSubquerySplicing(ExecutionNode const* n "report this error. Try turning off the splice-subqueries rule to get " "your query working.", node->getTypeString().c_str()); -} +} // namespace void findSubqueriesSuitableForSplicing(ExecutionPlan const& plan, containers::SmallVector& result) { diff --git a/arangod/Aql/RemoteExecutor.cpp b/arangod/Aql/RemoteExecutor.cpp index c231fc4cb91e..f2f4afebd700 100644 --- a/arangod/Aql/RemoteExecutor.cpp +++ b/arangod/Aql/RemoteExecutor.cpp @@ -31,6 +31,7 @@ #include "Aql/InputAqlItemRow.h" #include "Aql/Query.h" #include "Aql/RestAqlHandler.h" +#include "Aql/SkipResult.h" #include "Basics/MutexLocker.h" #include "Basics/StringBuffer.h" #include "Basics/VelocyPackHelper.h" @@ -153,8 +154,7 @@ std::pair ExecutionBlockImpl ExecutionBlockImpl::shutdown(i } auto ExecutionBlockImpl::executeViaOldApi(AqlCallStack stack) - -> std::tuple { + -> std::tuple { // Use the old getSome/SkipSome API. auto myCall = stack.popCall(); @@ -444,7 +444,9 @@ auto ExecutionBlockImpl::executeViaOldApi(AqlCallStack stack) if (state != ExecutionState::WAITING) { myCall.didSkip(skipped); } - return {state, skipped, nullptr}; + SkipResult skipRes{}; + skipRes.didSkip(skipped); + return {state, skipRes, nullptr}; } else if (AqlCall::IsGetSomeCall(myCall)) { auto const [state, block] = getSomeWithoutTrace(myCall.getLimit()); // We do not need to count as softLimit will be overwritten, and hard cannot be set. @@ -452,20 +454,22 @@ auto ExecutionBlockImpl::executeViaOldApi(AqlCallStack stack) // However we can do a short-cut here to report DONE on hardLimit if we are on the top-level query. myCall.didProduce(block->size()); if (myCall.getLimit() == 0) { - return {ExecutionState::DONE, 0, block}; + return {ExecutionState::DONE, SkipResult{}, block}; } } - return {state, 0, block}; + return {state, SkipResult{}, block}; } else if (AqlCall::IsFullCountCall(myCall)) { auto const [state, skipped] = skipSome(ExecutionBlock::SkipAllSize()); if (state != ExecutionState::WAITING) { myCall.didSkip(skipped); } - return {state, skipped, nullptr}; + SkipResult skipRes{}; + skipRes.didSkip(skipped); + return {state, skipRes, nullptr}; } else if (AqlCall::IsFastForwardCall(myCall)) { // No idea if DONE is correct here... - return {ExecutionState::DONE, 0, nullptr}; + return {ExecutionState::DONE, SkipResult{}, nullptr}; } // Should never get here! @@ -473,14 +477,14 @@ auto ExecutionBlockImpl::executeViaOldApi(AqlCallStack stack) } auto ExecutionBlockImpl::execute(AqlCallStack stack) - -> std::tuple { + -> std::tuple { traceExecuteBegin(stack); auto res = executeWithoutTrace(stack); return traceExecuteEnd(res); } auto ExecutionBlockImpl::executeWithoutTrace(AqlCallStack stack) --> std::tuple { + -> std::tuple { if (ADB_UNLIKELY(api() == Api::GET_SOME)) { return executeViaOldApi(stack); } @@ -489,7 +493,7 @@ auto ExecutionBlockImpl::executeWithoutTrace(AqlCallStack stack) } auto ExecutionBlockImpl::executeViaNewApi(AqlCallStack callStack) - -> std::tuple { + -> std::tuple { // silence tests -- we need to introduce new failure tests for fetchers TRI_IF_FAILURE("ExecutionBlock::getOrSkipSome1") { THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); @@ -509,7 +513,7 @@ auto ExecutionBlockImpl::executeViaNewApi(AqlCallStack callStack if (_requestInFlight) { // Already sent a shutdown request, but haven't got an answer yet. - return {ExecutionState::WAITING, 0, nullptr}; + return {ExecutionState::WAITING, SkipResult{}, nullptr}; } // For every call we simply forward via HTTP @@ -553,7 +557,7 @@ auto ExecutionBlockImpl::executeViaNewApi(AqlCallStack callStack THROW_ARANGO_EXCEPTION(res); } - return {ExecutionState::WAITING, 0, nullptr}; + return {ExecutionState::WAITING, SkipResult{}, nullptr}; } auto ExecutionBlockImpl::deserializeExecuteCallResultBody(VPackSlice const slice) const @@ -564,14 +568,16 @@ auto ExecutionBlockImpl::deserializeExecuteCallResultBody(VPackS if (ADB_UNLIKELY(!slice.isObject())) { using namespace std::string_literals; - return Result{TRI_ERROR_TYPE_ERROR, "When parsing execute result: expected object, got "s + slice.typeName()}; + return Result{TRI_ERROR_TYPE_ERROR, + "When parsing execute result: expected object, got "s + slice.typeName()}; } if (auto value = slice.get(StaticStrings::AqlRemoteResult); !value.isNone()) { return AqlExecuteResult::fromVelocyPack(value, _engine->itemBlockManager()); } - return Result{TRI_ERROR_TYPE_ERROR, "When parsing execute result: field result missing"}; + return Result{TRI_ERROR_TYPE_ERROR, + "When parsing execute result: field result missing"}; } auto ExecutionBlockImpl::serializeExecuteCallBody(AqlCallStack const& callStack) const @@ -661,10 +667,10 @@ Result ExecutionBlockImpl::sendAsyncRequest(fuerte::RestVerb typ req->header.addMeta("x-shard-id", _ownName); req->header.addMeta("shard-id", _ownName); // deprecated in 3.7, remove later } - + LOG_TOPIC("2713c", DEBUG, Logger::COMMUNICATION) - << "request to '" << _server - << "' '" << fuerte::to_string(type) << " " << req->header.path << "'"; + << "request to '" << _server << "' '" << fuerte::to_string(type) << " " + << req->header.path << "'"; network::ConnectionPtr conn = pool->leaseConnection(spec.endpoint); diff --git a/arangod/Aql/RemoteExecutor.h b/arangod/Aql/RemoteExecutor.h index 49244489cf00..cd8da9a7afbb 100644 --- a/arangod/Aql/RemoteExecutor.h +++ b/arangod/Aql/RemoteExecutor.h @@ -32,12 +32,16 @@ #include -namespace arangodb::fuerte { inline namespace v1 { +namespace arangodb::fuerte { +inline namespace v1 { enum class RestVerb; -}} +} +} // namespace arangodb::fuerte namespace arangodb::aql { +class SkipResult; + // The RemoteBlock is actually implemented by specializing ExecutionBlockImpl, // so this class only exists to identify the specialization. class RemoteExecutor final {}; @@ -67,7 +71,7 @@ class ExecutionBlockImpl : public ExecutionBlock { std::pair shutdown(int errorCode) override; - std::tuple execute(AqlCallStack stack) override; + std::tuple execute(AqlCallStack stack) override; [[nodiscard]] auto api() const noexcept -> Api; @@ -85,13 +89,13 @@ class ExecutionBlockImpl : public ExecutionBlock { std::pair skipSomeWithoutTrace(size_t atMost); auto executeWithoutTrace(AqlCallStack stack) - -> std::tuple; + -> std::tuple; auto executeViaOldApi(AqlCallStack stack) - -> std::tuple; + -> std::tuple; auto executeViaNewApi(AqlCallStack stack) - -> std::tuple; + -> std::tuple; [[nodiscard]] auto deserializeExecuteCallResultBody(velocypack::Slice) const -> ResultT; @@ -166,6 +170,6 @@ class ExecutionBlockImpl : public ExecutionBlock { Api _apiToUse = Api::EXECUTE; }; -} // namespace arangodb +} // namespace arangodb::aql #endif // ARANGOD_AQL_REMOTE_EXECUTOR_H diff --git a/arangod/Aql/RestAqlHandler.cpp b/arangod/Aql/RestAqlHandler.cpp index fa57a645b8b2..725eac165320 100644 --- a/arangod/Aql/RestAqlHandler.cpp +++ b/arangod/Aql/RestAqlHandler.cpp @@ -740,7 +740,7 @@ RestStatus RestAqlHandler::handleUseQuery(std::string const& operation, auto& executeCall = maybeExecuteCall.get(); auto items = SharedAqlItemBlockPtr{}; - auto skipped = size_t{}; + auto skipped = SkipResult{}; auto state = ExecutionState::HASMORE; // shardId is set IFF the root node is scatter or distribute diff --git a/arangod/Aql/ScatterExecutor.cpp b/arangod/Aql/ScatterExecutor.cpp index bdf56efdb391..b81449906d9a 100644 --- a/arangod/Aql/ScatterExecutor.cpp +++ b/arangod/Aql/ScatterExecutor.cpp @@ -64,10 +64,11 @@ auto ScatterExecutor::ClientBlockData::clear() -> void { _executorHasMore = false; } -auto ScatterExecutor::ClientBlockData::addBlock(SharedAqlItemBlockPtr block) -> void { +auto ScatterExecutor::ClientBlockData::addBlock(SharedAqlItemBlockPtr block, + SkipResult skipped) -> void { // NOTE: - // There given ItemBlock will be reused in all requesting blocks. - // However, the next followwing block could be passthrough. + // The given ItemBlock will be reused in all requesting blocks. + // However, the next following block could be passthrough. // If it is, it will modify that data stored in block. // If now anther client requests the same block, it is not // the original any more, but a modified version. @@ -75,20 +76,20 @@ auto ScatterExecutor::ClientBlockData::addBlock(SharedAqlItemBlockPtr block) -> // is empty. If another peer-calculation has written to this value // this assertion does not hold true anymore. // Hence we are required to do an indepth cloning here. - _queue.emplace_back(block->slice(0, block->size())); + _queue.emplace_back(block->slice(0, block->size()), skipped); } auto ScatterExecutor::ClientBlockData::hasDataFor(AqlCall const& call) -> bool { return _executorHasMore || !_queue.empty(); } -auto ScatterExecutor::ClientBlockData::execute(AqlCall call, ExecutionState upstreamState) - -> std::tuple { +auto ScatterExecutor::ClientBlockData::execute(AqlCallStack callStack, ExecutionState upstreamState) + -> std::tuple { TRI_ASSERT(_executor != nullptr); // Make sure we actually have data before you call execute - TRI_ASSERT(hasDataFor(call)); + TRI_ASSERT(hasDataFor(callStack.peek())); if (!_executorHasMore) { - auto const& block = _queue.front(); + auto const& [block, skipResult] = _queue.front(); // This cast is guaranteed, we create this a couple lines above and only // this executor is used here. // Unfortunately i did not get a version compiled were i could only forward @@ -96,12 +97,11 @@ auto ScatterExecutor::ClientBlockData::execute(AqlCall call, ExecutionState upst auto casted = static_cast>*>(_executor.get()); TRI_ASSERT(casted != nullptr); - casted->injectConstantBlock(block); + casted->injectConstantBlock(block, skipResult); _executorHasMore = true; _queue.pop_front(); } - AqlCallStack stack{call}; - auto [state, skipped, result] = _executor->execute(stack); + auto [state, skipped, result] = _executor->execute(callStack); // We have all data locally cannot wait here. TRI_ASSERT(state != ExecutionState::WAITING); @@ -124,12 +124,12 @@ auto ScatterExecutor::ClientBlockData::execute(AqlCall call, ExecutionState upst ScatterExecutor::ScatterExecutor(ExecutorInfos const&){}; -auto ScatterExecutor::distributeBlock(SharedAqlItemBlockPtr block, +auto ScatterExecutor::distributeBlock(SharedAqlItemBlockPtr block, SkipResult skipped, std::unordered_map& blockMap) const -> void { // Scatter returns every block on every client as is. for (auto& [id, list] : blockMap) { - list.addBlock(block); + list.addBlock(block, skipped); } } diff --git a/arangod/Aql/ScatterExecutor.h b/arangod/Aql/ScatterExecutor.h index 53bd1bb1b5c5..10a1316cec02 100644 --- a/arangod/Aql/ScatterExecutor.h +++ b/arangod/Aql/ScatterExecutor.h @@ -31,6 +31,7 @@ namespace arangodb { namespace aql { +class SkipResult; class ExecutionEngine; class ScatterNode; @@ -56,14 +57,14 @@ class ScatterExecutor { ExecutorInfos const& scatterInfos); auto clear() -> void; - auto addBlock(SharedAqlItemBlockPtr block) -> void; + auto addBlock(SharedAqlItemBlockPtr block, SkipResult skipped) -> void; auto hasDataFor(AqlCall const& call) -> bool; - auto execute(AqlCall call, ExecutionState upstreamState) - -> std::tuple; + auto execute(AqlCallStack callStack, ExecutionState upstreamState) + -> std::tuple; private: - std::deque _queue; + std::deque> _queue; // This is unique_ptr to get away with everything beeing forward declared... std::unique_ptr _executor; bool _executorHasMore; @@ -72,7 +73,7 @@ class ScatterExecutor { ScatterExecutor(ExecutorInfos const&); ~ScatterExecutor() = default; - auto distributeBlock(SharedAqlItemBlockPtr block, + auto distributeBlock(SharedAqlItemBlockPtr block, SkipResult skipped, std::unordered_map& blockMap) const -> void; }; diff --git a/arangod/Aql/SingleRowFetcher.cpp b/arangod/Aql/SingleRowFetcher.cpp index 62e054c5b875..44acf7bdc00a 100644 --- a/arangod/Aql/SingleRowFetcher.cpp +++ b/arangod/Aql/SingleRowFetcher.cpp @@ -30,6 +30,7 @@ #include "Aql/ExecutionBlock.h" #include "Aql/ExecutionState.h" #include "Aql/InputAqlItemRow.h" +#include "Aql/SkipResult.h" using namespace arangodb; using namespace arangodb::aql; @@ -77,28 +78,31 @@ SingleRowFetcher::fetchBlockForPassthrough(size_t atMost) { } template -std::tuple +std::tuple SingleRowFetcher::execute(AqlCallStack& stack) { auto [state, skipped, block] = _dependencyProxy->execute(stack); if (state == ExecutionState::WAITING) { // On waiting we have nothing to return - return {state, 0, AqlItemBlockInputRange{ExecutorState::HASMORE}}; + return {state, SkipResult{}, AqlItemBlockInputRange{ExecutorState::HASMORE}}; } if (block == nullptr) { if (state == ExecutionState::HASMORE) { - return {state, skipped, AqlItemBlockInputRange{ExecutorState::HASMORE, skipped}}; + return {state, skipped, + AqlItemBlockInputRange{ExecutorState::HASMORE, skipped.getSkipCount()}}; } - return {state, skipped, AqlItemBlockInputRange{ExecutorState::DONE, skipped}}; + return {state, skipped, + AqlItemBlockInputRange{ExecutorState::DONE, skipped.getSkipCount()}}; } auto [start, end] = block->getRelevantRange(); if (state == ExecutionState::HASMORE) { TRI_ASSERT(block != nullptr); return {state, skipped, - AqlItemBlockInputRange{ExecutorState::HASMORE, skipped, block, start}}; + AqlItemBlockInputRange{ExecutorState::HASMORE, + skipped.getSkipCount(), block, start}}; } return {state, skipped, - AqlItemBlockInputRange{ExecutorState::DONE, skipped, block, start}}; + AqlItemBlockInputRange{ExecutorState::DONE, skipped.getSkipCount(), block, start}}; } template diff --git a/arangod/Aql/SingleRowFetcher.h b/arangod/Aql/SingleRowFetcher.h index e33717eff395..f33447576c01 100644 --- a/arangod/Aql/SingleRowFetcher.h +++ b/arangod/Aql/SingleRowFetcher.h @@ -40,6 +40,7 @@ namespace arangodb::aql { class AqlItemBlock; template class DependencyProxy; +class SkipResult; /** * @brief Interface for all AqlExecutors that do only need one @@ -74,7 +75,7 @@ class SingleRowFetcher { * size_t => Amount of documents skipped * DataRange => Resulting data */ - std::tuple execute(AqlCallStack& stack); + std::tuple execute(AqlCallStack& stack); /** * @brief Fetch one new AqlItemRow from upstream. diff --git a/arangod/Aql/SkipResult.cpp b/arangod/Aql/SkipResult.cpp new file mode 100644 index 000000000000..094c13bb99fa --- /dev/null +++ b/arangod/Aql/SkipResult.cpp @@ -0,0 +1,184 @@ + +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2018 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +//////////////////////////////////////////////////////////////////////////////// + +#include "SkipResult.h" + +#include "Cluster/ResultT.h" + +#include +#include +#include + +using namespace arangodb::aql; + +SkipResult::SkipResult() {} + +SkipResult::SkipResult(SkipResult const& other) : _skipped{other._skipped} {} + +auto SkipResult::getSkipCount() const noexcept -> size_t { + TRI_ASSERT(!_skipped.empty()); + return _skipped.back(); +} + +auto SkipResult::didSkip(size_t skipped) -> void { + TRI_ASSERT(!_skipped.empty()); + _skipped.back() += skipped; +} + +auto SkipResult::didSkipSubquery(size_t skipped, size_t depth) -> void { + TRI_ASSERT(!_skipped.empty()); + TRI_ASSERT(_skipped.size() > depth + 1); + size_t index = _skipped.size() - depth - 2; + size_t& localSkip = _skipped.at(index); + localSkip += skipped; +} + +auto SkipResult::getSkipOnSubqueryLevel(size_t depth) -> size_t { + TRI_ASSERT(!_skipped.empty()); + TRI_ASSERT(_skipped.size() > depth); + return _skipped.at(depth); +} + +auto SkipResult::nothingSkipped() const noexcept -> bool { + TRI_ASSERT(!_skipped.empty()); + return std::all_of(_skipped.begin(), _skipped.end(), + [](size_t const& e) -> bool { return e == 0; }); +} + +auto SkipResult::toVelocyPack(VPackBuilder& builder) const noexcept -> void { + VPackArrayBuilder guard(&builder); + TRI_ASSERT(!_skipped.empty()); + for (auto const& s : _skipped) { + builder.add(VPackValue(s)); + } +} + +auto SkipResult::fromVelocyPack(VPackSlice slice) -> arangodb::ResultT { + if (!slice.isArray()) { + auto message = std::string{ + "When deserializating AqlExecuteResult: When reading skipped: " + "Unexpected type "}; + message += slice.typeName(); + return Result(TRI_ERROR_TYPE_ERROR, std::move(message)); + } + if (slice.isEmptyArray()) { + auto message = std::string{ + "When deserializating AqlExecuteResult: When reading skipped: " + "Got an empty list of skipped values."}; + return Result(TRI_ERROR_TYPE_ERROR, std::move(message)); + } + try { + SkipResult res; + auto it = VPackArrayIterator(slice); + while (it.valid()) { + auto val = it.value(); + if (!val.isInteger()) { + auto message = std::string{ + "When deserializating AqlExecuteResult: When reading skipped: " + "Unexpected type "}; + message += slice.typeName(); + return Result(TRI_ERROR_TYPE_ERROR, std::move(message)); + } + if (!it.isFirst()) { + res.incrementSubquery(); + } + res.didSkip(val.getNumber()); + ++it; + } + return {res}; + } catch (velocypack::Exception const& ex) { + auto message = std::string{ + "When deserializating AqlExecuteResult: When reading skipped: "}; + message += ex.what(); + return Result(TRI_ERROR_TYPE_ERROR, std::move(message)); + } +} + +auto SkipResult::incrementSubquery() -> void { _skipped.emplace_back(0); } +auto SkipResult::decrementSubquery() -> void { + TRI_ASSERT(!_skipped.empty()); + _skipped.pop_back(); + TRI_ASSERT(!_skipped.empty()); +} +auto SkipResult::subqueryDepth() const noexcept -> size_t { + TRI_ASSERT(!_skipped.empty()); + return _skipped.size(); +} + +auto SkipResult::reset() -> void { + for (size_t i = 0; i < _skipped.size(); ++i) { + _skipped[i] = 0; + } +} + +auto SkipResult::merge(SkipResult const& other, bool excludeTopLevel) noexcept -> void { + _skipped.reserve(other.subqueryDepth()); + while (other.subqueryDepth() > subqueryDepth()) { + incrementSubquery(); + } + TRI_ASSERT(other._skipped.size() <= _skipped.size()); + for (size_t i = 0; i < other._skipped.size(); ++i) { + if (excludeTopLevel && i + 1 == other._skipped.size()) { + // Do not copy top level + continue; + } + _skipped[i] += other._skipped[i]; + } +} + +auto SkipResult::mergeOnlyTopLevel(SkipResult const& other) noexcept -> void { + _skipped.reserve(other.subqueryDepth()); + while (other.subqueryDepth() > subqueryDepth()) { + incrementSubquery(); + } + _skipped.back() += other._skipped.back(); +} + +auto SkipResult::operator+=(SkipResult const& b) noexcept -> SkipResult& { + didSkip(b.getSkipCount()); + return *this; +} + +auto SkipResult::operator==(SkipResult const& b) const noexcept -> bool { + if (_skipped.size() != b._skipped.size()) { + return false; + } + for (size_t i = 0; i < _skipped.size(); ++i) { + if (_skipped[i] != b._skipped[i]) { + return false; + } + } + return true; +} + +auto SkipResult::operator!=(SkipResult const& b) const noexcept -> bool { + return !(*this == b); +} +namespace arangodb::aql { +std::ostream& operator<<(std::ostream& stream, arangodb::aql::SkipResult const& result) { + VPackBuilder temp; + result.toVelocyPack(temp); + stream << temp.toJson(); + return stream; +} +} // namespace arangodb::aql diff --git a/arangod/Aql/SkipResult.h b/arangod/Aql/SkipResult.h new file mode 100644 index 000000000000..6850f26acda7 --- /dev/null +++ b/arangod/Aql/SkipResult.h @@ -0,0 +1,89 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2018 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGOD_AQL_SKIP_RESULT_H +#define ARANGOD_AQL_SKIP_RESULT_H + +// for size_t +#include +#include +#include + +namespace arangodb { +template +class ResultT; +} +namespace arangodb::velocypack { +class Builder; +class Slice; +} // namespace arangodb::velocypack + +namespace arangodb::aql { + +class SkipResult { + public: + static auto fromVelocyPack(velocypack::Slice) -> arangodb::ResultT; + + SkipResult(); + + ~SkipResult() = default; + + SkipResult(SkipResult const& other); + SkipResult& operator=(const SkipResult&) = default; + + auto getSkipCount() const noexcept -> size_t; + + auto didSkip(size_t skipped) -> void; + + auto didSkipSubquery(size_t skipped, size_t depth) -> void; + + auto getSkipOnSubqueryLevel(size_t depth) -> size_t; + + auto nothingSkipped() const noexcept -> bool; + + auto toVelocyPack(arangodb::velocypack::Builder& builder) const noexcept -> void; + + auto incrementSubquery() -> void; + + auto decrementSubquery() -> void; + + auto subqueryDepth() const noexcept -> size_t; + + auto reset() -> void; + + auto merge(SkipResult const& other, bool excludeTopLevel) noexcept -> void; + auto mergeOnlyTopLevel(SkipResult const& other) noexcept -> void; + + auto operator+=(SkipResult const& b) noexcept -> SkipResult&; + + auto operator==(SkipResult const& b) const noexcept -> bool; + auto operator!=(SkipResult const& b) const noexcept -> bool; + + private: + std::vector _skipped{0}; +}; + +std::ostream& operator<<(std::ostream&, arangodb::aql::SkipResult const&); + +} // namespace arangodb::aql + +#endif diff --git a/arangod/Aql/SortingGatherExecutor.h b/arangod/Aql/SortingGatherExecutor.h index 96f179bee753..35bc09dcb961 100644 --- a/arangod/Aql/SortingGatherExecutor.h +++ b/arangod/Aql/SortingGatherExecutor.h @@ -29,6 +29,8 @@ #include "Aql/ExecutorInfos.h" #include "Aql/InputAqlItemRow.h" +#include + namespace arangodb { namespace transaction { diff --git a/arangod/Aql/SubqueryExecutor.cpp b/arangod/Aql/SubqueryExecutor.cpp index c84b47f0346f..76be9ef6c6cc 100644 --- a/arangod/Aql/SubqueryExecutor.cpp +++ b/arangod/Aql/SubqueryExecutor.cpp @@ -77,88 +77,51 @@ SubqueryExecutor::~SubqueryExecutor() = default; template std::pair SubqueryExecutor::produceRows(OutputAqlItemRow& output) { -#if 0 - if (_state == ExecutorState::DONE && !_input.isInitialized()) { - // We have seen DONE upstream, and we have discarded our local reference - // to the last input, we will not be able to produce results anymore. - return {_state, NoStats{}}; - } - while (true) { - if (_subqueryInitialized) { - // Continue in subquery - - // Const case - if (_infos.isConst() && !_input.isFirstDataRowInBlock()) { - // Simply write - writeOutput(output); - return {_state, NoStats{}}; - } - - // Non const case, or first run in const - auto res = _subquery.getSome(ExecutionBlock::DefaultBatchSize); - if (res.first == ExecutionState::WAITING) { - TRI_ASSERT(res.second == nullptr); - return {res.first, NoStats{}}; - } - // We get a result - if (res.second != nullptr) { - TRI_IF_FAILURE("SubqueryBlock::executeSubquery") { - THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); - } - - if (_infos.returnsData()) { - TRI_ASSERT(_subqueryResults != nullptr); - _subqueryResults->emplace_back(std::move(res.second)); - } - } - - // Subquery DONE - if (res.first == ExecutionState::DONE) { - writeOutput(output); - return {_state, NoStats{}}; - } + TRI_ASSERT(false); + THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); +} - } else { - // init new subquery - if (!_input) { - std::tie(_state, _input) = _fetcher.fetchRow(); - if (_state == ExecutionState::WAITING) { - TRI_ASSERT(!_input); - return {_state, NoStats{}}; - } - if (!_input) { - TRI_ASSERT(_state == ExecutionState::DONE); +template +auto SubqueryExecutor::initializeSubquery(AqlItemBlockInputRange& input) + -> std::tuple { + // init new subquery + if (!_input) { + std::tie(_state, _input) = input.nextDataRow(); + LOG_DEVEL_SQ << uint64_t(this) << " nextDataRow: " << _state << " " + << _input.isInitialized(); + if (!_input) { + LOG_DEVEL_SQ << uint64_t(this) << "exit, no more input" << _state; + return {translatedReturnType(), false}; + } + } - // We are done! - return {_state, NoStats{}}; - } - } + TRI_ASSERT(_input); + if (!_infos.isConst() || _input.isFirstDataRowInBlock()) { + LOG_DEVEL_SQ << "Subquery: Initialize cursor"; + auto [state, result] = _subquery.initializeCursor(_input); + if (state == ExecutionState::WAITING) { + LOG_DEVEL_SQ << "Waiting on initialize cursor"; + return {state, false}; + } - TRI_ASSERT(_input); - if (!_infos.isConst() || _input.isFirstDataRowInBlock()) { - auto initRes = _subquery.initializeCursor(_input); - if (initRes.first == ExecutionState::WAITING) { - return {ExecutionState::WAITING, NoStats{}}; - } - if (initRes.second.fail()) { - // Error during initialize cursor - THROW_ARANGO_EXCEPTION(initRes.second); - } - _subqueryResults = std::make_unique>(); - } - // on const subquery we can retoggle init as soon as we have new input. - _subqueryInitialized = true; + if (result.fail()) { + // Error during initialize cursor + THROW_ARANGO_EXCEPTION(result); } + _subqueryResults = std::make_unique>(); } -#endif - TRI_ASSERT(false); - THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); + // on const subquery we can retoggle init as soon as we have new input. + _subqueryInitialized = true; + return {translatedReturnType(), true}; } template auto SubqueryExecutor::produceRows(AqlItemBlockInputRange& input, OutputAqlItemRow& output) -> std::tuple { + // We need to return skip in skipRows before + TRI_ASSERT(_skipped == 0); + auto getUpstreamCall = [&]() { AqlCall upstreamCall = output.getClientCall(); if constexpr (isModificationSubquery) { @@ -175,7 +138,12 @@ auto SubqueryExecutor::produceRows(AqlItemBlockInputRang // to the last input, we will not be able to produce results anymore. return {translatedReturnType(), NoStats{}, getUpstreamCall()}; } - while (true) { + if (output.isFull()) { + // This can happen if there is no upstream + _state = input.upstreamState(); + } + + while (!output.isFull()) { if (_subqueryInitialized) { // Continue in subquery @@ -185,12 +153,12 @@ auto SubqueryExecutor::produceRows(AqlItemBlockInputRang writeOutput(output); LOG_DEVEL_SQ << uint64_t(this) << "wrote output is const " << _state << " " << getUpstreamCall(); - return {translatedReturnType(), NoStats{}, getUpstreamCall()}; + continue; } // Non const case, or first run in const auto [state, skipped, block] = _subquery.execute(AqlCallStack(AqlCall{})); - TRI_ASSERT(skipped == 0); + TRI_ASSERT(skipped.nothingSkipped()); if (state == ExecutionState::WAITING) { return {state, NoStats{}, getUpstreamCall()}; } @@ -214,40 +182,22 @@ auto SubqueryExecutor::produceRows(AqlItemBlockInputRang writeOutput(output); LOG_DEVEL_SQ << uint64_t(this) << "wrote output subquery done " << _state << " " << getUpstreamCall(); - return {translatedReturnType(), NoStats{}, getUpstreamCall()}; } - } else { - // init new subquery - if (!_input) { - std::tie(_state, _input) = input.nextDataRow(); - LOG_DEVEL_SQ << uint64_t(this) << " nextDataRow: " << _state << " " - << _input.isInitialized(); - if (!_input) { - LOG_DEVEL_SQ << uint64_t(this) << "exit produce, no more input" << _state; - return {translatedReturnType(), NoStats{}, getUpstreamCall()}; - } + auto const [state, initialized] = initializeSubquery(input); + if (state == ExecutionState::WAITING) { + LOG_DEVEL_SQ << "Waiting on initialize cursor"; + return {state, NoStats{}, AqlCall{}}; } - - TRI_ASSERT(_input); - if (!_infos.isConst() || _input.isFirstDataRowInBlock()) { - LOG_DEVEL_SQ << "Subquery: Initialize cursor"; - auto [state, result] = _subquery.initializeCursor(_input); - if (state == ExecutionState::WAITING) { - LOG_DEVEL_SQ << "Waiting on initialize cursor"; - return {state, NoStats{}, AqlCall{}}; - } - - if (result.fail()) { - // Error during initialize cursor - THROW_ARANGO_EXCEPTION(result); - } - _subqueryResults = std::make_unique>(); + if (!initialized) { + TRI_ASSERT(!_input); + return {state, NoStats{}, getUpstreamCall()}; } - // on const subquery we can retoggle init as soon as we have new input. - _subqueryInitialized = true; + TRI_ASSERT(_subqueryInitialized); } } + + return {translatedReturnType(), NoStats{}, getUpstreamCall()}; } template @@ -256,6 +206,7 @@ void SubqueryExecutor::writeOutput(OutputAqlItemRow& out TRI_IF_FAILURE("SubqueryBlock::getSome") { THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); } + TRI_ASSERT(!output.isFull()); if (!_infos.isConst() || _input.isFirstDataRowInBlock()) { // In the non const case we need to move the data into the output for every // row. @@ -326,8 +277,6 @@ auto SubqueryExecutor::skipRowsRange<>(AqlItemBlockInputRange& inputRange, return upstreamCall; }; - size_t skipped = 0; - LOG_DEVEL_SQ << uint64_t(this) << "skipRowsRange " << call; if (_state == ExecutorState::DONE && !_input.isInitialized()) { @@ -335,78 +284,62 @@ auto SubqueryExecutor::skipRowsRange<>(AqlItemBlockInputRange& inputRange, // to the last input, we will not be able to produce results anymore. return {translatedReturnType(), NoStats{}, 0, getUpstreamCall()}; } - while (true) { + TRI_ASSERT(call.needSkipMore()); + // We cannot have a modifying subquery considered const + TRI_ASSERT(!_infos.isConst()); + bool isFullCount = call.getLimit() == 0 && call.getOffset() == 0; + while (isFullCount || _skipped < call.getOffset()) { if (_subqueryInitialized) { // Continue in subquery - // Const case - if (_infos.isConst() && !_input.isFirstDataRowInBlock()) { - // Simply write - _subqueryInitialized = false; - _input = InputAqlItemRow(CreateInvalidInputRowHint{}); - skipped += 1; - call.didSkip(1); - LOG_DEVEL_SQ << uint64_t(this) << "did skip one"; - return {translatedReturnType(), NoStats{}, skipped, getUpstreamCall()}; - } - - // Non const case, or first run in const - auto [state, skipped, block] = _subquery.execute(AqlCallStack(AqlCall{})); - TRI_ASSERT(skipped == 0); + // While skipping we do not care for the result. + // Simply jump over it. + AqlCall subqueryCall{}; + subqueryCall.hardLimit = 0; + auto [state, skipRes, block] = _subquery.execute(AqlCallStack(subqueryCall)); + TRI_ASSERT(skipRes.nothingSkipped()); if (state == ExecutionState::WAITING) { return {state, NoStats{}, 0, getUpstreamCall()}; } - - // We get a result - if (block != nullptr) { - TRI_IF_FAILURE("SubqueryBlock::executeSubquery") { - THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); - } - - if (_infos.returnsData()) { - TRI_ASSERT(_subqueryResults != nullptr); - _subqueryResults->emplace_back(std::move(block)); - } + // We get a result, but we asked for no rows. + // so please give us no rows. + TRI_ASSERT(block == nullptr); + TRI_IF_FAILURE("SubqueryBlock::executeSubquery") { + THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); } // Subquery DONE if (state == ExecutionState::DONE) { _subqueryInitialized = false; _input = InputAqlItemRow(CreateInvalidInputRowHint{}); - skipped += 1; - call.didSkip(1); + _skipped += 1; LOG_DEVEL_SQ << uint64_t(this) << "did skip one"; - return {translatedReturnType(), NoStats{}, skipped, getUpstreamCall()}; } } else { - // init new subquery - if (!_input) { - std::tie(_state, _input) = inputRange.nextDataRow(); - - if (!_input) { - LOG_DEVEL_SQ << uint64_t(this) << "skipped nothing waiting for input " << _state; - return {translatedReturnType(), NoStats{}, skipped, getUpstreamCall()}; - } + auto const [state, initialized] = initializeSubquery(inputRange); + if (state == ExecutionState::WAITING) { + LOG_DEVEL_SQ << "Waiting on initialize cursor"; + return {state, NoStats{}, 0, AqlCall{}}; } - - TRI_ASSERT(_input); - if (!_infos.isConst() || _input.isFirstDataRowInBlock()) { - auto [state, result] = _subquery.initializeCursor(_input); - if (state == ExecutionState::WAITING) { - return {state, NoStats{}, 0, getUpstreamCall()}; + if (!initialized) { + TRI_ASSERT(!_input); + if (state == ExecutionState::DONE) { + // We are done, we will not get any more input. + break; } - - if (result.fail()) { - // Error during initialize cursor - THROW_ARANGO_EXCEPTION(result); - } - _subqueryResults = std::make_unique>(); + return {state, NoStats{}, 0, getUpstreamCall()}; } - // on const subquery we can retoggle init as soon as we have new input. - _subqueryInitialized = true; + TRI_ASSERT(_subqueryInitialized); } } + // If we get here, we are done with one set of skipping. + // We either skipped the offset + // or the fullCount + // or both if limit == 0. + call.didSkip(_skipped); + _skipped = 0; + return {translatedReturnType(), NoStats{}, call.getSkipCount(), getUpstreamCall()}; } template class ::arangodb::aql::SubqueryExecutor; diff --git a/arangod/Aql/SubqueryExecutor.h b/arangod/Aql/SubqueryExecutor.h index b9e5c299827b..b28823180849 100644 --- a/arangod/Aql/SubqueryExecutor.h +++ b/arangod/Aql/SubqueryExecutor.h @@ -120,6 +120,19 @@ class SubqueryExecutor { */ auto translatedReturnType() const noexcept -> ExecutionState; + /** + * @brief Initiliaze the subquery with next input row + * Throws if there was an error during initialize cursor + * + * + * @param input Container for more data + * @return std::tuple Result state (WAITING or + * translatedReturnType()) + * bool flag if we have initialized the query, if not, we require more data. + */ + auto initializeSubquery(AqlItemBlockInputRange& input) + -> std::tuple; + private: Fetcher& _fetcher; SubqueryExecutorInfos& _infos; @@ -144,6 +157,8 @@ class SubqueryExecutor { // Cache for the input row we are currently working on InputAqlItemRow _input; + + size_t _skipped = 0; }; } // namespace aql } // namespace arangodb diff --git a/arangod/CMakeLists.txt b/arangod/CMakeLists.txt index c5ae8e10dc53..69ab26c764ef 100644 --- a/arangod/CMakeLists.txt +++ b/arangod/CMakeLists.txt @@ -349,6 +349,7 @@ set(LIB_ARANGO_AQL_SOURCES Aql/SimpleModifier.cpp Aql/SingleRemoteModificationExecutor.cpp Aql/SingleRowFetcher.cpp + Aql/SkipResult.cpp Aql/SortCondition.cpp Aql/SortExecutor.cpp Aql/SortNode.cpp diff --git a/tests/Aql/DependencyProxyMock.cpp b/tests/Aql/DependencyProxyMock.cpp index 620038a51a0c..05a4dc621b0c 100644 --- a/tests/Aql/DependencyProxyMock.cpp +++ b/tests/Aql/DependencyProxyMock.cpp @@ -25,6 +25,8 @@ #include "gtest/gtest.h" +#include "Aql/SkipResult.h" + #include namespace arangodb::tests::aql { @@ -130,10 +132,11 @@ DependencyProxyMock& DependencyProxyMock:: } template -std::tuple +std::tuple DependencyProxyMock::execute(AqlCallStack& stack) { TRI_ASSERT(_block != nullptr); - return {arangodb::aql::ExecutionState::DONE, 0, _block}; + SkipResult res{}; + return {arangodb::aql::ExecutionState::DONE, res, _block}; } template diff --git a/tests/Aql/DependencyProxyMock.h b/tests/Aql/DependencyProxyMock.h index 2d7c6d8b89af..5d873b84ea47 100644 --- a/tests/Aql/DependencyProxyMock.h +++ b/tests/Aql/DependencyProxyMock.h @@ -33,6 +33,9 @@ #include namespace arangodb { +namespace aql { +class SkipResult; +} namespace tests { namespace aql { @@ -51,7 +54,7 @@ class DependencyProxyMock : public ::arangodb::aql::DependencyProxy skipSome(size_t atMost) override; - std::tuple execute( + std::tuple execute( arangodb::aql::AqlCallStack& stack) override; private: @@ -101,8 +104,7 @@ class MultiDependencyProxyMock // NOLINTNEXTLINE google-default-arguments std::pair fetchBlockForDependency( - size_t dependency, - size_t atMost = arangodb::aql::ExecutionBlock::DefaultBatchSize) override; + size_t dependency, size_t atMost = arangodb::aql::ExecutionBlock::DefaultBatchSize) override; std::pair skipSomeForDependency(size_t dependency, size_t atMost) override; diff --git a/tests/Aql/ExecutionBlockImplTest.cpp b/tests/Aql/ExecutionBlockImplTest.cpp index 5199c766d248..97bf70ad6ec8 100644 --- a/tests/Aql/ExecutionBlockImplTest.cpp +++ b/tests/Aql/ExecutionBlockImplTest.cpp @@ -58,7 +58,7 @@ using LambdaExe = TestLambdaSkipExecutor; // This test is supposed to only test getSome return values, // it is not supposed to test the fetch logic! - +#if 0 class ExecutionBlockImplTest : public ::testing::Test { protected: // ExecutionState state @@ -400,6 +400,7 @@ TEST_F(ExecutionBlockImplTest, ASSERT_EQ(state, ExecutionState::DONE); ASSERT_EQ(block, nullptr); } +#endif /** * @brief Shared Test case initializer to test the execute API @@ -691,7 +692,7 @@ class ExecutionBlockImplExecuteSpecificTest : public SharedExecutionBlockImplTes * @return std::tuple Response of execute(call); */ auto runTest(ProduceCall& prod, SkipCall& skip, AqlCall call) - -> std::tuple { + -> std::tuple { AqlCallStack stack{std::move(call)}; auto singleton = createSingleton(); if (GetParam()) { @@ -775,7 +776,7 @@ TEST_P(ExecutionBlockImplExecuteSpecificTest, test_toplevel_unlimited_call) { auto [state, skipped, block] = runTest(execImpl, skipCall, fullCall); EXPECT_EQ(state, ExecutionState::DONE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); EXPECT_EQ(block, nullptr); // Once with empty, once with the line by Singleton EXPECT_EQ(nrCalls, 2); @@ -797,7 +798,7 @@ TEST_P(ExecutionBlockImplExecuteSpecificTest, test_toplevel_softlimit_call) { auto [state, skipped, block] = runTest(execImpl, skipCall, fullCall); EXPECT_EQ(state, ExecutionState::DONE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); EXPECT_EQ(block, nullptr); // Once with empty, once with the line by Singleton EXPECT_EQ(nrCalls, 2); @@ -819,7 +820,7 @@ TEST_P(ExecutionBlockImplExecuteSpecificTest, test_toplevel_hardlimit_call) { auto [state, skipped, block] = runTest(execImpl, skipCall, fullCall); EXPECT_EQ(state, ExecutionState::DONE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); EXPECT_EQ(block, nullptr); // Once with empty, once with the line by Singleton EXPECT_EQ(nrCalls, 2); @@ -838,7 +839,7 @@ TEST_P(ExecutionBlockImplExecuteSpecificTest, test_toplevel_offset_call) { auto [state, skipped, block] = runTest(execImpl, skipCall, fullCall); EXPECT_EQ(state, ExecutionState::DONE); - EXPECT_EQ(skipped, 1); + EXPECT_EQ(skipped.getSkipCount(), 1); if (GetParam()) { // Do never call skip, pass through EXPECT_EQ(nrCalls, 0); @@ -866,7 +867,7 @@ TEST_P(ExecutionBlockImplExecuteSpecificTest, test_toplevel_offset_only_call) { auto [state, skipped, block] = runTest(execImpl, skipCall, fullCall); EXPECT_EQ(state, ExecutionState::DONE); - EXPECT_EQ(skipped, 1); + EXPECT_EQ(skipped.getSkipCount(), 1); if (GetParam()) { // Do never call skip, pass through EXPECT_EQ(nrCalls, 0); @@ -902,7 +903,7 @@ TEST_P(ExecutionBlockImplExecuteSpecificTest, test_relevant_shadowrow_does_not_f // First call. Fetch all rows (data only) auto const& [state, skipped, block] = testee->execute(stack); EXPECT_EQ(state, ExecutionState::HASMORE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); ASSERT_NE(block, nullptr); EXPECT_EQ(block->size(), ExecutionBlock::DefaultBatchSize); EXPECT_FALSE(block->hasShadowRows()); @@ -911,7 +912,7 @@ TEST_P(ExecutionBlockImplExecuteSpecificTest, test_relevant_shadowrow_does_not_f // Second call. only a single shadowRow left auto const& [state, skipped, block] = testee->execute(stack); EXPECT_EQ(state, ExecutionState::DONE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); ASSERT_NE(block, nullptr); EXPECT_EQ(block->size(), 1); EXPECT_TRUE(block->hasShadowRows()); @@ -945,7 +946,7 @@ TEST_P(ExecutionBlockImplExecuteSpecificTest, set_of_shadowrows_does_not_fit_in_ // First call. Fetch all rows (data only) auto const& [state, skipped, block] = testee->execute(stack); EXPECT_EQ(state, ExecutionState::HASMORE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); ASSERT_NE(block, nullptr); EXPECT_EQ(block->size(), ExecutionBlock::DefaultBatchSize); EXPECT_FALSE(block->hasShadowRows()); @@ -954,7 +955,7 @@ TEST_P(ExecutionBlockImplExecuteSpecificTest, set_of_shadowrows_does_not_fit_in_ // Second call. only the shadowRows are left auto const& [state, skipped, block] = testee->execute(stack); EXPECT_EQ(state, ExecutionState::DONE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); ASSERT_NE(block, nullptr); ASSERT_EQ(block->size(), 2); EXPECT_TRUE(block->hasShadowRows()); @@ -995,7 +996,7 @@ TEST_P(ExecutionBlockImplExecuteSpecificTest, set_of_shadowrows_does_not_fit_ful // First call. Fetch all rows (data + relevant shadow row) auto const& [state, skipped, block] = testee->execute(stack); EXPECT_EQ(state, ExecutionState::HASMORE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); ASSERT_NE(block, nullptr); EXPECT_EQ(block->size(), ExecutionBlock::DefaultBatchSize); EXPECT_TRUE(block->hasShadowRows()); @@ -1007,7 +1008,7 @@ TEST_P(ExecutionBlockImplExecuteSpecificTest, set_of_shadowrows_does_not_fit_ful // Second call. only the shadowRows are left auto const& [state, skipped, block] = testee->execute(stack); EXPECT_EQ(state, ExecutionState::DONE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); ASSERT_NE(block, nullptr); EXPECT_EQ(block->size(), 1); EXPECT_TRUE(block->hasShadowRows()); @@ -1602,7 +1603,7 @@ class ExecutionBlockImplExecuteIntegrationTest * @param testReg The register to evaluate * @param numShadowRows Number of preceeding shadowRows in result. */ - void ValidateResult(std::shared_ptr data, size_t skipped, + void ValidateResult(std::shared_ptr data, SkipResult skipped, SharedAqlItemBlockPtr result, RegisterId testReg, size_t numShadowRows = 0) { auto const& call = getCall(); @@ -1611,7 +1612,8 @@ class ExecutionBlockImplExecuteIntegrationTest TRI_ASSERT(data->slice().isArray()); VPackSlice expected = data->slice(); - ValidateSkipMatches(call, static_cast(expected.length()), skipped); + ValidateSkipMatches(call, static_cast(expected.length()), + skipped.getSkipCount()); VPackArrayIterator expectedIt{expected}; // Skip Part @@ -1687,7 +1689,7 @@ TEST_P(ExecutionBlockImplExecuteIntegrationTest, test_waiting_block_mock) { auto [state, skipped, block] = testee.execute(stack); if (doesWaiting()) { EXPECT_EQ(state, ExecutionState::WAITING); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); EXPECT_EQ(block, nullptr); std::tie(state, skipped, block) = testee.execute(stack); } @@ -1721,7 +1723,7 @@ TEST_P(ExecutionBlockImplExecuteIntegrationTest, test_produce_only) { if (doesWaiting()) { auto const [state, skipped, block] = producer->execute(stack); EXPECT_EQ(state, ExecutionState::WAITING); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); EXPECT_EQ(block, nullptr); } auto const [state, skipped, block] = producer->execute(stack); @@ -1755,7 +1757,7 @@ TEST_P(ExecutionBlockImplExecuteIntegrationTest, test_produce_using_two) { if (doesWaiting()) { auto const [state, skipped, block] = producer->execute(stack); EXPECT_EQ(state, ExecutionState::WAITING); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); EXPECT_EQ(block, nullptr); } auto const [state, skipped, block] = producer->execute(stack); @@ -1816,7 +1818,7 @@ TEST_P(ExecutionBlockImplExecuteIntegrationTest, DISABLED_test_call_forwarding_p if (doesWaiting()) { auto const [state, skipped, block] = lower->execute(stack); EXPECT_EQ(state, ExecutionState::WAITING); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); EXPECT_EQ(block, nullptr); // Reset call counters upperState.reset(); @@ -1900,7 +1902,7 @@ TEST_P(ExecutionBlockImplExecuteIntegrationTest, DISABLED_test_call_forwarding_i if (doesWaiting()) { auto const [state, skipped, block] = lower->execute(stack); EXPECT_EQ(state, ExecutionState::WAITING); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); EXPECT_EQ(block, nullptr); } auto const [state, skipped, block] = lower->execute(stack); @@ -1945,7 +1947,7 @@ TEST_P(ExecutionBlockImplExecuteIntegrationTest, test_multiple_upstream_calls) { size_t killSwitch = 0; while (state == ExecutionState::WAITING) { EXPECT_TRUE(doesWaiting()); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); EXPECT_EQ(block, nullptr); std::tie(state, skipped, block) = testee->execute(stack); // Kill switch to avoid endless loop in case of error. @@ -2004,7 +2006,7 @@ TEST_P(ExecutionBlockImplExecuteIntegrationTest, test_multiple_upstream_calls_pa size_t waited = 0; while (state == ExecutionState::WAITING && waited < 2 /* avoid endless waiting*/) { EXPECT_EQ(state, ExecutionState::WAITING); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); EXPECT_EQ(block, nullptr); waited++; std::tie(state, skipped, block) = testee->execute(stack); @@ -2014,10 +2016,10 @@ TEST_P(ExecutionBlockImplExecuteIntegrationTest, test_multiple_upstream_calls_pa EXPECT_EQ(block, nullptr); if (fullCount) { // We skipped everything - EXPECT_EQ(skipped, 1000); + EXPECT_EQ(skipped.getSkipCount(), 1000); EXPECT_EQ(state, ExecutionState::DONE); } else { - EXPECT_EQ(skipped, offset); + EXPECT_EQ(skipped.getSkipCount(), offset); EXPECT_EQ(state, ExecutionState::HASMORE); } } else { @@ -2033,7 +2035,7 @@ TEST_P(ExecutionBlockImplExecuteIntegrationTest, test_multiple_upstream_calls_pa size_t waited = 0; while (state == ExecutionState::WAITING && waited < 3 /* avoid endless waiting*/) { EXPECT_EQ(state, ExecutionState::WAITING); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); EXPECT_EQ(block, nullptr); waited++; std::tie(state, skipped, block) = testee->execute(stack); @@ -2051,8 +2053,8 @@ TEST_P(ExecutionBlockImplExecuteIntegrationTest, test_multiple_upstream_calls_pa ASSERT_EQ(block->size(), 1); // Book-keeping for call. // We need to request data from above with the correct call. - if (skipped > 0) { - call.didSkip(skipped); + if (!skipped.nothingSkipped()) { + call.didSkip(skipped.getSkipCount()); } call.didProduce(1); auto got = block->getValueReference(0, outReg).slice(); @@ -2061,15 +2063,15 @@ TEST_P(ExecutionBlockImplExecuteIntegrationTest, test_multiple_upstream_calls_pa << " in row " << i << " and register " << outReg; if (i == 0) { // The first data row includes skip - EXPECT_EQ(skipped, offset); + EXPECT_EQ(skipped.getSkipCount(), offset); } else { if (call.getLimit() == 0 && call.hasHardLimit() && call.needsFullCount()) { // The last row, with fullCount needs to contain data. - EXPECT_EQ(skipped, 1000 - limit - offset); + EXPECT_EQ(skipped.getSkipCount(), 1000 - limit - offset); } else { // Do not skip on later data rows // Except the last one on fullcount - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); } } // NOTE: We might want to get into this situation. @@ -2136,7 +2138,7 @@ TEST_P(ExecutionBlockImplExecuteIntegrationTest, only_relevant_shadowRows) { if (doesWaiting()) { // We wait between lines EXPECT_EQ(state, ExecutionState::WAITING); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); EXPECT_EQ(block, nullptr); std::tie(state, skipped, block) = testee->execute(stack); } @@ -2147,7 +2149,7 @@ TEST_P(ExecutionBlockImplExecuteIntegrationTest, only_relevant_shadowRows) { EXPECT_EQ(state, ExecutionState::HASMORE); } // Cannot skip a shadowRow - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); ASSERT_NE(block, nullptr); ASSERT_EQ(block->size(), 1); EXPECT_TRUE(block->hasShadowRows()); @@ -2194,7 +2196,7 @@ TEST_P(ExecutionBlockImplExecuteIntegrationTest, input_and_relevant_shadowRow) { if (doesWaiting()) { auto const [state, skipped, block] = testee->execute(stack); EXPECT_EQ(state, ExecutionState::WAITING); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); EXPECT_EQ(block, nullptr); } auto const [state, skipped, block] = testee->execute(stack); @@ -2246,7 +2248,7 @@ TEST_P(ExecutionBlockImplExecuteIntegrationTest, input_and_non_relevant_shadowRo if (doesWaiting()) { auto const [state, skipped, block] = testee->execute(stack); EXPECT_EQ(state, ExecutionState::WAITING); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); EXPECT_EQ(block, nullptr); } auto const [state, skipped, block] = testee->execute(stack); @@ -2315,7 +2317,7 @@ TEST_P(ExecutionBlockImplExecuteIntegrationTest, multiple_subqueries) { if (doesWaiting()) { auto const [state, skipped, block] = testee->execute(stack); EXPECT_EQ(state, ExecutionState::WAITING); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); EXPECT_EQ(block, nullptr); } auto const [state, skipped, block] = testee->execute(stack); @@ -2337,7 +2339,7 @@ TEST_P(ExecutionBlockImplExecuteIntegrationTest, multiple_subqueries) { testee->execute(forwardStack); // We do not care for any data left EXPECT_EQ(forwardState, ExecutionState::HASMORE); - EXPECT_EQ(forwardSkipped, 0); + EXPECT_EQ(forwardSkipped.getSkipCount(), 0); // However there need to be two shadow rows ASSERT_NE(forwardBlock, nullptr); ASSERT_EQ(forwardBlock->size(), 2); @@ -2396,7 +2398,7 @@ TEST_P(ExecutionBlockImplExecuteIntegrationTest, empty_subquery) { // we only wait exactly once, only one block upstream that is not sliced. auto const& [state, skipped, block] = testee->execute(stack); EXPECT_EQ(state, ExecutionState::WAITING); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); EXPECT_EQ(block, nullptr); } auto call = getCall(); @@ -2408,10 +2410,10 @@ TEST_P(ExecutionBlockImplExecuteIntegrationTest, empty_subquery) { EXPECT_EQ(state, ExecutionState::HASMORE); ASSERT_NE(block, nullptr); if (skip) { - EXPECT_EQ(skipped, 1); + EXPECT_EQ(skipped.getSkipCount(), 1); EXPECT_EQ(block->size(), 2); } else { - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); EXPECT_EQ(block->size(), 3); } size_t row = 0; @@ -2446,7 +2448,7 @@ TEST_P(ExecutionBlockImplExecuteIntegrationTest, empty_subquery) { auto const& [state, skipped, block] = testee->execute(stack); EXPECT_EQ(state, ExecutionState::HASMORE); ASSERT_NE(block, nullptr); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); EXPECT_EQ(block->size(), 1); size_t row = 0; AssertIsShadowRowOfDepth(block, row, 0); @@ -2472,7 +2474,7 @@ TEST_P(ExecutionBlockImplExecuteIntegrationTest, empty_subquery) { auto const& [state, skipped, block] = testee->execute(stack); EXPECT_EQ(state, ExecutionState::DONE); ASSERT_NE(block, nullptr); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); EXPECT_EQ(block->size(), 2); size_t row = 0; AssertIsShadowRowOfDepth(block, row, 0); @@ -2542,7 +2544,7 @@ TEST_P(ExecutionBlockImplExecuteIntegrationTest, auto [state, skipped, block] = testee.execute(stack); if (doesWaiting()) { EXPECT_EQ(state, ExecutionState::WAITING); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); EXPECT_EQ(block, nullptr); std::tie(state, skipped, block) = testee.execute(stack); } @@ -2601,7 +2603,7 @@ TEST_P(ExecutionBlockImplExecuteIntegrationTest, DISABLED_test_outer_subquery_fo auto [state, skipped, block] = testee.execute(stack); if (doesWaiting()) { EXPECT_EQ(state, ExecutionState::WAITING); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); EXPECT_EQ(block, nullptr); std::tie(state, skipped, block) = testee.execute(stack); } diff --git a/tests/Aql/ExecutionBlockImplTestInstances.cpp b/tests/Aql/ExecutionBlockImplTestInstances.cpp index b5bfc68a589b..a7af31f4c8f6 100644 --- a/tests/Aql/ExecutionBlockImplTestInstances.cpp +++ b/tests/Aql/ExecutionBlockImplTestInstances.cpp @@ -3,7 +3,7 @@ #include "TestExecutorHelper.h" #include "TestLambdaExecutor.h" -template class ::arangodb::aql::ExecutionBlockImpl; -template class ::arangodb::aql::ExecutionBlockImpl; +// template class ::arangodb::aql::ExecutionBlockImpl; +// template class ::arangodb::aql::ExecutionBlockImpl; template class ::arangodb::aql::ExecutionBlockImpl; template class ::arangodb::aql::ExecutionBlockImpl; diff --git a/tests/Aql/ExecutorTestHelper.h b/tests/Aql/ExecutorTestHelper.h index c3b2686246f1..1a007f01782f 100644 --- a/tests/Aql/ExecutorTestHelper.h +++ b/tests/Aql/ExecutorTestHelper.h @@ -210,7 +210,7 @@ struct ExecutorTestHelper { ExecutorTestHelper(ExecutorTestHelper const&) = delete; ExecutorTestHelper(ExecutorTestHelper&&) = delete; explicit ExecutorTestHelper(arangodb::aql::Query& query) - : _expectedSkip{0}, + : _expectedSkip{}, _expectedState{ExecutionState::HASMORE}, _testStats{false}, _unorderedOutput{false}, @@ -300,8 +300,24 @@ struct ExecutorTestHelper { return *this; } - auto expectSkipped(std::size_t skip) -> ExecutorTestHelper& { - _expectedSkip = skip; + /** + * @brief + * + * @tparam Ts numeric type, can actually only be size_t + * @param skipOnLevel List of skip counters returned per level. subquery skips first, the last entry is the skip on the executor + * @return ExecutorTestHelper& chaining! + */ + template + auto expectSkipped(T skipFirst, Ts const... skipOnHigherLevel) -> ExecutorTestHelper& { + _expectedSkip = SkipResult{}; + // This is obvious, proof: Homework. + (_expectedSkip.didSkip(static_cast(skipFirst)), ..., + (_expectedSkip.incrementSubquery(), + _expectedSkip.didSkip(static_cast(skipOnHigherLevel)))); + + // NOTE: the above will increment didSkip by the first entry. + // For all following entries it will first increment the subquery depth + // and then add the didSkip on them. return *this; } @@ -377,7 +393,7 @@ struct ExecutorTestHelper { auto inputBlock = generateInputRanges(itemBlockManager); - auto skippedTotal = size_t{0}; + auto skippedTotal = SkipResult{}; auto finalState = ExecutionState::HASMORE; TRI_ASSERT(!_pipeline.empty()); @@ -387,7 +403,7 @@ struct ExecutorTestHelper { if (!loop) { auto const [state, skipped, result] = _pipeline.get().front()->execute(_callStack); - skippedTotal = skipped; + skippedTotal.merge(skipped, false); finalState = state; if (result != nullptr) { allResults.add(result); @@ -397,8 +413,8 @@ struct ExecutorTestHelper { auto const [state, skipped, result] = _pipeline.get().front()->execute(_callStack); finalState = state; auto call = _callStack.popCall(); - skippedTotal += skipped; - call.didSkip(skipped); + skippedTotal.merge(skipped, false); + call.didSkip(skipped.getSkipCount()); if (result != nullptr) { call.didProduce(result->size()); allResults.add(result); @@ -409,7 +425,6 @@ struct ExecutorTestHelper { (!_callStack.peek().hasSoftLimit() || (_callStack.peek().getLimit() + _callStack.peek().getOffset()) > 0)); } - EXPECT_EQ(skippedTotal, _expectedSkip); EXPECT_EQ(finalState, _expectedState); SharedAqlItemBlockPtr result = allResults.steal(); @@ -500,7 +515,7 @@ struct ExecutorTestHelper { MatrixBuilder _output; std::vector> _outputShadowRows{}; std::array _outputRegisters; - std::size_t _expectedSkip; + SkipResult _expectedSkip; ExecutionState _expectedState; ExecutionStats _expectedStats; bool _testStats; diff --git a/tests/Aql/HashedCollectExecutorTest.cpp b/tests/Aql/HashedCollectExecutorTest.cpp index 6aca1b82d975..63dffa7cd6d9 100644 --- a/tests/Aql/HashedCollectExecutorTest.cpp +++ b/tests/Aql/HashedCollectExecutorTest.cpp @@ -257,7 +257,7 @@ TEST_P(HashedCollectExecutorTest, collect_only_soft_less_second_call) { AqlCallStack stack{call}; auto const [state, skipped, result] = testee.execute(stack); EXPECT_EQ(state, ExecutionState::HASMORE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); ASSERT_NE(result, nullptr); asserthelper::ValidateBlocksAreEqualUnordered(result, buildExpectedOutput(), matchedRows, 2, registersToTest); @@ -270,7 +270,7 @@ TEST_P(HashedCollectExecutorTest, collect_only_soft_less_second_call) { AqlCallStack stack{call}; auto const [state, skipped, result] = testee.execute(stack); EXPECT_EQ(state, ExecutionState::DONE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); ASSERT_NE(result, nullptr); asserthelper::ValidateBlocksAreEqualUnordered(result, buildExpectedOutput(), matchedRows, 0, registersToTest); diff --git a/tests/Aql/IdExecutorTest.cpp b/tests/Aql/IdExecutorTest.cpp index ac203cf39da3..4ec6f1b94df8 100644 --- a/tests/Aql/IdExecutorTest.cpp +++ b/tests/Aql/IdExecutorTest.cpp @@ -297,7 +297,7 @@ TEST_F(IdExecutionBlockTest, test_initialize_cursor_get) { AqlCallStack stack(std::move(call)); auto const& [state, skipped, block] = testee.execute(stack); EXPECT_EQ(state, ExecutionState::DONE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); EXPECT_EQ(block, nullptr); } { @@ -312,7 +312,7 @@ TEST_F(IdExecutionBlockTest, test_initialize_cursor_get) { AqlCallStack stack(std::move(call)); auto const& [state, skipped, block] = testee.execute(stack); EXPECT_EQ(state, ExecutionState::DONE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); ASSERT_NE(block, nullptr); EXPECT_EQ(block->size(), 1); auto const& val = block->getValueReference(0, 0); @@ -340,7 +340,7 @@ TEST_F(IdExecutionBlockTest, test_initialize_cursor_skip) { AqlCallStack stack(std::move(call)); auto const& [state, skipped, block] = testee.execute(stack); EXPECT_EQ(state, ExecutionState::DONE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); EXPECT_EQ(block, nullptr); } { @@ -356,7 +356,7 @@ TEST_F(IdExecutionBlockTest, test_initialize_cursor_skip) { AqlCallStack stack(std::move(call)); auto const& [state, skipped, block] = testee.execute(stack); EXPECT_EQ(state, ExecutionState::DONE); - EXPECT_EQ(skipped, 1); + EXPECT_EQ(skipped.getSkipCount(), 1); ASSERT_EQ(block, nullptr); } } @@ -381,7 +381,7 @@ TEST_F(IdExecutionBlockTest, test_initialize_cursor_fullCount) { AqlCallStack stack(std::move(call)); auto const& [state, skipped, block] = testee.execute(stack); EXPECT_EQ(state, ExecutionState::DONE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); EXPECT_EQ(block, nullptr); } { @@ -398,7 +398,7 @@ TEST_F(IdExecutionBlockTest, test_initialize_cursor_fullCount) { AqlCallStack stack(std::move(call)); auto const& [state, skipped, block] = testee.execute(stack); EXPECT_EQ(state, ExecutionState::DONE); - EXPECT_EQ(skipped, 1); + EXPECT_EQ(skipped.getSkipCount(), 1); ASSERT_EQ(block, nullptr); } } @@ -443,7 +443,8 @@ TEST_P(BlockOverloadTest, test_hardlimit_const_fetcher) { // Inject block auto inputBlock = buildBlock<1>(itemBlockManager, {{0}, {1}, {2}, {3}, {4}, {5}, {6}}); - testee.injectConstantBlock(inputBlock); + + testee.injectConstantBlock(inputBlock, SkipResult{}); } { // Now call with too small hardLimit @@ -455,9 +456,9 @@ TEST_P(BlockOverloadTest, test_hardlimit_const_fetcher) { auto const& [state, skipped, block] = testee.execute(stack); EXPECT_EQ(state, ExecutionState::DONE); if (useFullCount()) { - EXPECT_EQ(skipped, 4); + EXPECT_EQ(skipped.getSkipCount(), 4); } else { - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); } asserthelper::ValidateBlocksAreEqual(block, expectedOutputBlock); @@ -468,7 +469,7 @@ TEST_P(BlockOverloadTest, test_hardlimit_const_fetcher) { AqlCallStack stack(std::move(call)); auto const& [state, skipped, block] = testee.execute(stack); EXPECT_EQ(state, ExecutionState::DONE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); EXPECT_EQ(block, nullptr); } } @@ -480,7 +481,7 @@ TEST_P(BlockOverloadTest, test_hardlimit_const_fetcher_shadow_rows_at_end) { auto inputBlock = buildBlock<1>(itemBlockManager, {{0}, {1}, {2}, {3}, {4}, {5}, {6}}, {{5, 0}, {6, 1}}); - testee.injectConstantBlock(inputBlock); + testee.injectConstantBlock(inputBlock, SkipResult{}); } { // Now call with too small hardLimit @@ -493,9 +494,9 @@ TEST_P(BlockOverloadTest, test_hardlimit_const_fetcher_shadow_rows_at_end) { auto const& [state, skipped, block] = testee.execute(stack); EXPECT_EQ(state, ExecutionState::DONE); if (useFullCount()) { - EXPECT_EQ(skipped, 2); + EXPECT_EQ(skipped.getSkipCount(), 2); } else { - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); } asserthelper::ValidateBlocksAreEqual(block, expectedOutputBlock); } @@ -505,7 +506,7 @@ TEST_P(BlockOverloadTest, test_hardlimit_const_fetcher_shadow_rows_at_end) { AqlCallStack stack(std::move(call)); auto const& [state, skipped, block] = testee.execute(stack); EXPECT_EQ(state, ExecutionState::DONE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); EXPECT_EQ(block, nullptr); } } @@ -517,7 +518,7 @@ TEST_P(BlockOverloadTest, test_hardlimit_const_fetcher_shadow_rows_in_between) { auto inputBlock = buildBlock<1>(itemBlockManager, {{0}, {1}, {2}, {3}, {4}, {5}, {6}}, {{3, 0}, {4, 1}, {6, 0}}); - testee.injectConstantBlock(inputBlock); + testee.injectConstantBlock(inputBlock, SkipResult{}); } { // Now call with too small hardLimit @@ -530,9 +531,9 @@ TEST_P(BlockOverloadTest, test_hardlimit_const_fetcher_shadow_rows_in_between) { auto const& [state, skipped, block] = testee.execute(stack); EXPECT_EQ(state, ExecutionState::HASMORE); if (useFullCount()) { - EXPECT_EQ(skipped, 1); + EXPECT_EQ(skipped.getSkipCount(), 1); } else { - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); } asserthelper::ValidateBlocksAreEqual(block, expectedOutputBlock); } @@ -544,7 +545,7 @@ TEST_P(BlockOverloadTest, test_hardlimit_const_fetcher_shadow_rows_in_between) { AqlCallStack stack(std::move(call)); auto const& [state, skipped, block] = testee.execute(stack); EXPECT_EQ(state, ExecutionState::DONE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); asserthelper::ValidateBlocksAreEqual(block, expectedOutputBlock); } } @@ -557,7 +558,7 @@ TEST_P(BlockOverloadTest, test_hardlimit_const_fetcher_consecutive_shadow_rows) auto inputBlock = buildBlock<1>(itemBlockManager, {{0}, {1}, {2}, {3}, {4}, {5}, {6}}, {{3, 0}, {4, 1}, {5, 0}, {6, 0}}); - testee.injectConstantBlock(inputBlock); + testee.injectConstantBlock(inputBlock, SkipResult{}); } // We can only return until the next top-level shadow row is reached. { @@ -571,9 +572,9 @@ TEST_P(BlockOverloadTest, test_hardlimit_const_fetcher_consecutive_shadow_rows) auto const& [state, skipped, block] = testee.execute(stack); EXPECT_EQ(state, ExecutionState::HASMORE); if (useFullCount()) { - EXPECT_EQ(skipped, 1); + EXPECT_EQ(skipped.getSkipCount(), 1); } else { - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); } asserthelper::ValidateBlocksAreEqual(block, expectedOutputBlock); } @@ -586,7 +587,7 @@ TEST_P(BlockOverloadTest, test_hardlimit_const_fetcher_consecutive_shadow_rows) AqlCallStack stack(std::move(call)); auto const& [state, skipped, block] = testee.execute(stack); EXPECT_EQ(state, ExecutionState::HASMORE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); asserthelper::ValidateBlocksAreEqual(block, expectedOutputBlock); } { @@ -598,7 +599,7 @@ TEST_P(BlockOverloadTest, test_hardlimit_const_fetcher_consecutive_shadow_rows) AqlCallStack stack(std::move(call)); auto const& [state, skipped, block] = testee.execute(stack); EXPECT_EQ(state, ExecutionState::DONE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); asserthelper::ValidateBlocksAreEqual(block, expectedOutputBlock); } { @@ -607,7 +608,7 @@ TEST_P(BlockOverloadTest, test_hardlimit_const_fetcher_consecutive_shadow_rows) AqlCallStack stack(std::move(call)); auto const& [state, skipped, block] = testee.execute(stack); EXPECT_EQ(state, ExecutionState::DONE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); EXPECT_EQ(block, nullptr); } } diff --git a/tests/Aql/RemoteExecutorTest.cpp b/tests/Aql/RemoteExecutorTest.cpp index c5e4bef76825..8a82034d4b60 100644 --- a/tests/Aql/RemoteExecutorTest.cpp +++ b/tests/Aql/RemoteExecutorTest.cpp @@ -71,9 +71,7 @@ class DeSerializeAqlCallTest : public ::testing::TestWithParam { public: DeSerializeAqlCallTest() = default; - void SetUp() override { - aqlCall = GetParam(); - } + void SetUp() override { aqlCall = GetParam(); } protected: AqlCall aqlCall{}; @@ -118,20 +116,18 @@ class DeSerializeAqlCallStackTest : public ::testing::TestWithParam::error(-1); }); - ASSERT_TRUE(maybeDeSerializedCallStack.ok()) << maybeDeSerializedCallStack.errorMessage(); + ASSERT_TRUE(maybeDeSerializedCallStack.ok()) + << maybeDeSerializedCallStack.errorMessage(); auto const deSerializedCallStack = *maybeDeSerializedCallStack; ASSERT_EQ(aqlCallStack, deSerializedCallStack); } -INSTANTIATE_TEST_CASE_P(DeSerializeAqlCallStackTestVariations, DeSerializeAqlCallStackTest, testingAqlCallStacks); - +INSTANTIATE_TEST_CASE_P(DeSerializeAqlCallStackTestVariations, + DeSerializeAqlCallStackTest, testingAqlCallStacks); class DeSerializeAqlExecuteResultTest : public ::testing::TestWithParam { public: DeSerializeAqlExecuteResultTest() = default; - void SetUp() override { - aqlExecuteResult = GetParam(); - } + void SetUp() override { aqlExecuteResult = GetParam(); } protected: - AqlExecuteResult aqlExecuteResult{ExecutionState::DONE, 0, nullptr}; + AqlExecuteResult aqlExecuteResult{ExecutionState::DONE, SkipResult{}, nullptr}; }; ResourceMonitor resourceMonitor{}; AqlItemBlockManager manager{&resourceMonitor, SerializationFormat::SHADOWROWS}; +auto MakeSkipResult(size_t const i) -> SkipResult { + SkipResult res{}; + res.didSkip(i); + return res; +} + auto const testingAqlExecuteResults = ::testing::ValuesIn(std::array{ - AqlExecuteResult{ExecutionState::DONE, 0, nullptr}, - AqlExecuteResult{ExecutionState::HASMORE, 0, nullptr}, - AqlExecuteResult{ExecutionState::HASMORE, 4, nullptr}, - AqlExecuteResult{ExecutionState::DONE, 0, buildBlock<1>(manager, {{42}})}, - AqlExecuteResult{ExecutionState::HASMORE, 3, buildBlock<2>(manager, {{3, 42}, {4, 41}})}, + AqlExecuteResult{ExecutionState::DONE, MakeSkipResult(0), nullptr}, + AqlExecuteResult{ExecutionState::HASMORE, MakeSkipResult(4), nullptr}, + AqlExecuteResult{ExecutionState::DONE, MakeSkipResult(0), buildBlock<1>(manager, {{42}})}, + AqlExecuteResult{ExecutionState::HASMORE, MakeSkipResult(3), + buildBlock<2>(manager, {{3, 42}, {4, 41}})}, }); TEST_P(DeSerializeAqlExecuteResultTest, testSuite) { @@ -203,7 +204,8 @@ TEST_P(DeSerializeAqlExecuteResultTest, testSuite) { ASSERT_EQ(aqlExecuteResult.state(), deSerializedAqlExecuteResult.state()); ASSERT_EQ(aqlExecuteResult.skipped(), deSerializedAqlExecuteResult.skipped()); - ASSERT_EQ(aqlExecuteResult.block() == nullptr, deSerializedAqlExecuteResult.block() == nullptr); + ASSERT_EQ(aqlExecuteResult.block() == nullptr, + deSerializedAqlExecuteResult.block() == nullptr); if (aqlExecuteResult.block() != nullptr) { ASSERT_EQ(*aqlExecuteResult.block(), *deSerializedAqlExecuteResult.block()) << "left: " << blockToString(aqlExecuteResult.block()) @@ -212,6 +214,7 @@ TEST_P(DeSerializeAqlExecuteResultTest, testSuite) { ASSERT_EQ(aqlExecuteResult, deSerializedAqlExecuteResult); } -INSTANTIATE_TEST_CASE_P(DeSerializeAqlExecuteResultTestVariations, DeSerializeAqlExecuteResultTest, testingAqlExecuteResults); +INSTANTIATE_TEST_CASE_P(DeSerializeAqlExecuteResultTestVariations, + DeSerializeAqlExecuteResultTest, testingAqlExecuteResults); } // namespace arangodb::tests::aql diff --git a/tests/Aql/ScatterExecutorTest.cpp b/tests/Aql/ScatterExecutorTest.cpp index 883a8b8498c0..d5cd0968fb98 100644 --- a/tests/Aql/ScatterExecutorTest.cpp +++ b/tests/Aql/ScatterExecutorTest.cpp @@ -159,7 +159,7 @@ TEST_P(RandomOrderTest, all_clients_should_get_the_block) { AqlCallStack stack{call}; auto const [state, skipped, block] = testee.executeForClient(stack, client); EXPECT_EQ(state, ExecutionState::DONE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); ValidateBlocksAreEqual(block, inputBlock); } } @@ -179,7 +179,7 @@ TEST_P(RandomOrderTest, all_clients_can_skip_the_block) { AqlCallStack stack{call}; auto const [state, skipped, block] = testee.executeForClient(stack, client); EXPECT_EQ(state, ExecutionState::DONE); - EXPECT_EQ(skipped, 3); + EXPECT_EQ(skipped.getSkipCount(), 3); EXPECT_EQ(block, nullptr); } } @@ -201,7 +201,7 @@ TEST_P(RandomOrderTest, all_clients_can_fullcount_the_block) { AqlCallStack stack{call}; auto const [state, skipped, block] = testee.executeForClient(stack, client); EXPECT_EQ(state, ExecutionState::DONE); - EXPECT_EQ(skipped, 2); + EXPECT_EQ(skipped.getSkipCount(), 2); ValidateBlocksAreEqual(block, expectedBlock); } } @@ -223,7 +223,7 @@ TEST_P(RandomOrderTest, all_clients_can_have_different_calls) { AqlCallStack stack{call}; auto const [state, skipped, block] = testee.executeForClient(stack, client); EXPECT_EQ(state, ExecutionState::DONE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); ValidateBlocksAreEqual(block, inputBlock); } else if (client == "b") { AqlCall call{}; @@ -232,7 +232,7 @@ TEST_P(RandomOrderTest, all_clients_can_have_different_calls) { AqlCallStack stack{call}; auto const [state, skipped, block] = testee.executeForClient(stack, client); EXPECT_EQ(state, ExecutionState::DONE); - EXPECT_EQ(skipped, 2); + EXPECT_EQ(skipped.getSkipCount(), 2); auto expectedBlock = buildBlock<1>(itemBlockManager, {{2}, {3}}); ValidateBlocksAreEqual(block, expectedBlock); } else if (client == "c") { @@ -242,7 +242,7 @@ TEST_P(RandomOrderTest, all_clients_can_have_different_calls) { AqlCallStack stack{call}; auto const [state, skipped, block] = testee.executeForClient(stack, client); EXPECT_EQ(state, ExecutionState::HASMORE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); auto expectedBlock = buildBlock<1>(itemBlockManager, {{0}, {1}}); ValidateBlocksAreEqual(block, expectedBlock); } @@ -254,7 +254,7 @@ TEST_P(RandomOrderTest, all_clients_can_have_different_calls) { AqlCallStack stack{call}; auto const [state, skipped, block] = testee.executeForClient(stack, client); EXPECT_EQ(state, ExecutionState::HASMORE); - EXPECT_EQ(skipped, 1); + EXPECT_EQ(skipped.getSkipCount(), 1); auto expectedBlock = buildBlock<1>(itemBlockManager, {{3}, {4}}); ValidateBlocksAreEqual(block, expectedBlock); } @@ -283,7 +283,7 @@ TEST_P(RandomOrderTest, get_does_not_jump_over_shadowrows) { AqlCallStack stack{call}; auto const [state, skipped, block] = testee.executeForClient(stack, client); EXPECT_EQ(state, ExecutionState::HASMORE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); ValidateBlocksAreEqual(block, firstExpectedBlock); } @@ -295,7 +295,7 @@ TEST_P(RandomOrderTest, get_does_not_jump_over_shadowrows) { AqlCallStack stack{call}; auto const [state, skipped, block] = testee.executeForClient(stack, client); EXPECT_EQ(state, ExecutionState::DONE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); ValidateBlocksAreEqual(block, secondExpectedBlock); } } @@ -321,7 +321,7 @@ TEST_P(RandomOrderTest, handling_of_higher_depth_shadowrows_produce) { AqlCallStack stack{call}; auto const [state, skipped, block] = testee.executeForClient(stack, client); EXPECT_EQ(state, ExecutionState::HASMORE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); ValidateBlocksAreEqual(block, firstExpectedBlock); } @@ -333,7 +333,7 @@ TEST_P(RandomOrderTest, handling_of_higher_depth_shadowrows_produce) { AqlCallStack stack{call}; auto const [state, skipped, block] = testee.executeForClient(stack, client); EXPECT_EQ(state, ExecutionState::DONE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); ValidateBlocksAreEqual(block, secondExpectedBlock); } } @@ -360,7 +360,7 @@ TEST_P(RandomOrderTest, handling_of_higher_depth_shadowrows_skip) { AqlCallStack stack{call}; auto const [state, skipped, block] = testee.executeForClient(stack, client); EXPECT_EQ(state, ExecutionState::HASMORE); - EXPECT_EQ(skipped, 2); + EXPECT_EQ(skipped.getSkipCount(), 2); ValidateBlocksAreEqual(block, firstExpectedBlock); } @@ -372,7 +372,7 @@ TEST_P(RandomOrderTest, handling_of_higher_depth_shadowrows_skip) { AqlCallStack stack{call}; auto const [state, skipped, block] = testee.executeForClient(stack, client); EXPECT_EQ(state, ExecutionState::DONE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); ValidateBlocksAreEqual(block, secondExpectedBlock); } } @@ -398,7 +398,7 @@ TEST_P(RandomOrderTest, handling_of_consecutive_shadow_rows) { AqlCallStack stack{call}; auto const [state, skipped, block] = testee.executeForClient(stack, client); EXPECT_EQ(state, ExecutionState::HASMORE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); auto expected = buildBlock<1>(itemBlockManager, {{0}, {1}, {2}, {3}}, {{2, 0}, {3, 1}}); ValidateBlocksAreEqual(block, expected); @@ -409,7 +409,7 @@ TEST_P(RandomOrderTest, handling_of_consecutive_shadow_rows) { AqlCallStack stack{call}; auto const [state, skipped, block] = testee.executeForClient(stack, client); EXPECT_EQ(state, ExecutionState::DONE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); auto expected = buildBlock<1>(itemBlockManager, {{4}, {5}}, {{0, 0}, {1, 1}}); ValidateBlocksAreEqual(block, expected); } @@ -434,7 +434,7 @@ TEST_P(RandomOrderTest, shadowrows_with_different_call_types) { AqlCallStack stack{call}; auto const [state, skipped, block] = testee.executeForClient(stack, client); EXPECT_EQ(state, ExecutionState::HASMORE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); auto expectedBlock = buildBlock<1>(itemBlockManager, {{0}, {1}, {2}, {3}}, {{3, 0}}); ValidateBlocksAreEqual(block, expectedBlock); @@ -445,7 +445,7 @@ TEST_P(RandomOrderTest, shadowrows_with_different_call_types) { AqlCallStack stack{call}; auto const [state, skipped, block] = testee.executeForClient(stack, client); EXPECT_EQ(state, ExecutionState::HASMORE); - EXPECT_EQ(skipped, 2); + EXPECT_EQ(skipped.getSkipCount(), 2); auto expectedBlock = buildBlock<1>(itemBlockManager, {{2}, {3}}, {{1, 0}}); ValidateBlocksAreEqual(block, expectedBlock); } else if (client == "c") { @@ -455,7 +455,7 @@ TEST_P(RandomOrderTest, shadowrows_with_different_call_types) { AqlCallStack stack{call}; auto const [state, skipped, block] = testee.executeForClient(stack, client); EXPECT_EQ(state, ExecutionState::HASMORE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); auto expectedBlock = buildBlock<1>(itemBlockManager, {{0}, {1}}); ValidateBlocksAreEqual(block, expectedBlock); } @@ -467,7 +467,7 @@ TEST_P(RandomOrderTest, shadowrows_with_different_call_types) { AqlCallStack stack{call}; auto const [state, skipped, block] = testee.executeForClient(stack, client); EXPECT_EQ(state, ExecutionState::HASMORE); - EXPECT_EQ(skipped, 1); + EXPECT_EQ(skipped.getSkipCount(), 1); auto expectedBlock = buildBlock<1>(itemBlockManager, {{3}}, {{0, 0}}); ValidateBlocksAreEqual(block, expectedBlock); } @@ -484,7 +484,7 @@ TEST_P(RandomOrderTest, shadowrows_with_different_call_types) { AqlCallStack stack{call}; auto const [state, skipped, block] = testee.executeForClient(stack, client); EXPECT_EQ(state, ExecutionState::DONE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); auto expectedBlock = buildBlock<1>(itemBlockManager, {{4}, {5}}, {{1, 0}}); ValidateBlocksAreEqual(block, expectedBlock); } else if (client == "b") { @@ -493,7 +493,7 @@ TEST_P(RandomOrderTest, shadowrows_with_different_call_types) { AqlCallStack stack{call}; auto const [state, skipped, block] = testee.executeForClient(stack, client); EXPECT_EQ(state, ExecutionState::DONE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); auto expectedBlock = buildBlock<1>(itemBlockManager, {{4}, {5}}, {{1, 0}}); ValidateBlocksAreEqual(block, expectedBlock); } else if (client == "c") { @@ -503,7 +503,7 @@ TEST_P(RandomOrderTest, shadowrows_with_different_call_types) { AqlCallStack stack{call}; auto const [state, skipped, block] = testee.executeForClient(stack, client); EXPECT_EQ(state, ExecutionState::DONE); - EXPECT_EQ(skipped, 1); + EXPECT_EQ(skipped.getSkipCount(), 1); auto expectedBlock = buildBlock<1>(itemBlockManager, {{5}}, {{0, 0}}); ValidateBlocksAreEqual(block, expectedBlock); } @@ -579,7 +579,7 @@ TEST_F(ScatterExecutionBlockTest, any_ordering_of_calls_is_fine) { } else { EXPECT_EQ(state, ExecutionState::HASMORE); } - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); ASSERT_TRUE(callNr < blocks.size()); ValidateBlocksAreEqual(block, blocks[callNr]); callNr++; diff --git a/tests/Aql/SingleRowFetcherTest.cpp b/tests/Aql/SingleRowFetcherTest.cpp index f0ed040e2783..7ea984f8b9e6 100644 --- a/tests/Aql/SingleRowFetcherTest.cpp +++ b/tests/Aql/SingleRowFetcherTest.cpp @@ -1228,7 +1228,7 @@ TEST_F(SingleRowFetcherTestPassBlocks, handling_shadowrows_in_execute_oneAndDone // First no data row auto [state, skipped, input] = testee.execute(stack); EXPECT_EQ(input.getRowIndex(), 0); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); EXPECT_EQ(state, ExecutionState::DONE); } // testee is destroyed here } @@ -1263,7 +1263,7 @@ TEST_F(SingleRowFetcherTestPassBlocks, handling_shadowrows_in_execute_twoAndHasM auto [state, skipped, input] = testee.execute(stack); // We only have one block, no more calls to execute necessary EXPECT_EQ(state, ExecutionState::DONE); - EXPECT_EQ(skipped, 0); + EXPECT_EQ(skipped.getSkipCount(), 0); EXPECT_EQ(input.getRowIndex(), 0); // Now validate the input range diff --git a/tests/Aql/SkipResultTest.cpp b/tests/Aql/SkipResultTest.cpp new file mode 100644 index 000000000000..958ab0ad5563 --- /dev/null +++ b/tests/Aql/SkipResultTest.cpp @@ -0,0 +1,267 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2018 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Michael Hackstein +//////////////////////////////////////////////////////////////////////////////// + +#include "gtest/gtest.h" + +#include "Aql/SkipResult.h" +#include "Cluster/ResultT.h" + +#include +#include + +using namespace arangodb; +using namespace arangodb::aql; + +namespace arangodb { +namespace tests { +namespace aql { + +class SkipResultTest : public ::testing::Test { + protected: + SkipResultTest() {} +}; + +TEST_F(SkipResultTest, defaults_to_0_skip) { + SkipResult testee{}; + EXPECT_EQ(testee.getSkipCount(), 0); +} + +TEST_F(SkipResultTest, counts_skip) { + SkipResult testee{}; + testee.didSkip(5); + EXPECT_EQ(testee.getSkipCount(), 5); +} + +TEST_F(SkipResultTest, accumulates_skips) { + SkipResult testee{}; + testee.didSkip(3); + testee.didSkip(6); + testee.didSkip(8); + EXPECT_EQ(testee.getSkipCount(), 17); +} + +TEST_F(SkipResultTest, is_copyable) { + SkipResult original{}; + original.didSkip(6); + SkipResult testee{original}; + + EXPECT_EQ(testee.getSkipCount(), original.getSkipCount()); + + original.didSkip(7); + EXPECT_NE(testee.getSkipCount(), original.getSkipCount()); +} + +TEST_F(SkipResultTest, can_report_if_we_skip) { + SkipResult testee{}; + EXPECT_TRUE(testee.nothingSkipped()); + testee.didSkip(3); + EXPECT_FALSE(testee.nothingSkipped()); + testee.didSkip(6); + EXPECT_FALSE(testee.nothingSkipped()); +} + +TEST_F(SkipResultTest, serialize_deserialize_empty) { + SkipResult original{}; + VPackBuilder builder; + original.toVelocyPack(builder); + auto maybeTestee = SkipResult::fromVelocyPack(builder.slice()); + ASSERT_FALSE(maybeTestee.fail()); + auto testee = maybeTestee.get(); + EXPECT_EQ(testee.nothingSkipped(), original.nothingSkipped()); + EXPECT_EQ(testee.getSkipCount(), original.getSkipCount()); + EXPECT_EQ(testee, original); +} + +TEST_F(SkipResultTest, serialize_deserialize_with_count) { + SkipResult original{}; + original.didSkip(6); + VPackBuilder builder; + original.toVelocyPack(builder); + auto maybeTestee = SkipResult::fromVelocyPack(builder.slice()); + ASSERT_FALSE(maybeTestee.fail()); + auto testee = maybeTestee.get(); + EXPECT_EQ(testee.nothingSkipped(), original.nothingSkipped()); + EXPECT_EQ(testee.getSkipCount(), original.getSkipCount()); + EXPECT_EQ(testee, original); +} + +TEST_F(SkipResultTest, can_be_added) { + SkipResult a{}; + a.didSkip(6); + SkipResult b{}; + b.didSkip(7); + a += b; + EXPECT_EQ(a.getSkipCount(), 13); +} + +TEST_F(SkipResultTest, can_add_a_subquery_depth) { + SkipResult a{}; + a.didSkip(5); + EXPECT_EQ(a.getSkipCount(), 5); + a.incrementSubquery(); + EXPECT_EQ(a.getSkipCount(), 0); + a.didSkip(7); + EXPECT_EQ(a.getSkipCount(), 7); + a.decrementSubquery(); + EXPECT_EQ(a.getSkipCount(), 5); +} + +TEST_F(SkipResultTest, nothing_skip_on_subquery) { + SkipResult a{}; + EXPECT_TRUE(a.nothingSkipped()); + a.didSkip(6); + EXPECT_FALSE(a.nothingSkipped()); + a.incrementSubquery(); + EXPECT_EQ(a.getSkipCount(), 0); + EXPECT_FALSE(a.nothingSkipped()); +} + +TEST_F(SkipResultTest, serialize_deserialize_with_a_subquery) { + SkipResult original{}; + original.didSkip(6); + original.incrementSubquery(); + original.didSkip(2); + + VPackBuilder builder; + original.toVelocyPack(builder); + auto maybeTestee = SkipResult::fromVelocyPack(builder.slice()); + ASSERT_FALSE(maybeTestee.fail()); + auto testee = maybeTestee.get(); + // Use built_in eq + EXPECT_EQ(testee, original); + // Manual test + EXPECT_EQ(testee.nothingSkipped(), original.nothingSkipped()); + EXPECT_EQ(testee.getSkipCount(), original.getSkipCount()); + EXPECT_EQ(testee.subqueryDepth(), original.subqueryDepth()); + original.decrementSubquery(); + testee.decrementSubquery(); + EXPECT_EQ(testee.nothingSkipped(), original.nothingSkipped()); + EXPECT_EQ(testee.getSkipCount(), original.getSkipCount()); + EXPECT_EQ(testee.subqueryDepth(), original.subqueryDepth()); +} + +TEST_F(SkipResultTest, equality) { + auto buildTestSet = []() -> std::vector { + SkipResult empty{}; + SkipResult skip1{}; + skip1.didSkip(6); + + SkipResult skip2{}; + skip2.didSkip(8); + + SkipResult subQuery1{}; + subQuery1.incrementSubquery(); + subQuery1.didSkip(4); + + SkipResult subQuery2{}; + subQuery2.didSkip(8); + subQuery2.incrementSubquery(); + subQuery2.didSkip(4); + + SkipResult subQuery3{}; + subQuery3.didSkip(8); + subQuery3.incrementSubquery(); + return {empty, skip1, skip2, subQuery1, subQuery2, subQuery3}; + }; + + // We create two identical sets with different entries + auto set1 = buildTestSet(); + auto set2 = buildTestSet(); + for (size_t i = 0; i < set1.size(); ++i) { + for (size_t j = 0; j < set2.size(); ++j) { + // Addresses are different + EXPECT_NE(&set1.at(i), &set2.at(j)); + // Identical index => Equal object + if (i == j) { + EXPECT_EQ(set1.at(i), set2.at(j)); + } else { + EXPECT_NE(set1.at(i), set2.at(j)); + } + } + } +} + +TEST_F(SkipResultTest, merge_with_toplevel) { + SkipResult a{}; + a.didSkip(12); + a.incrementSubquery(); + a.didSkip(8); + + SkipResult b{}; + b.didSkip(9); + b.incrementSubquery(); + b.didSkip(2); + + a.merge(b, false); + + SkipResult expected{}; + expected.didSkip(12); + expected.didSkip(9); + expected.incrementSubquery(); + expected.didSkip(8); + expected.didSkip(2); + EXPECT_EQ(a, expected); +} + +TEST_F(SkipResultTest, merge_without_toplevel) { + SkipResult a{}; + a.didSkip(12); + a.incrementSubquery(); + a.didSkip(8); + + SkipResult b{}; + b.didSkip(9); + b.incrementSubquery(); + b.didSkip(2); + + a.merge(b, true); + + SkipResult expected{}; + expected.didSkip(12); + expected.didSkip(9); + expected.incrementSubquery(); + expected.didSkip(8); + EXPECT_EQ(a, expected); +} + +TEST_F(SkipResultTest, reset) { + SkipResult a{}; + a.didSkip(12); + a.incrementSubquery(); + a.didSkip(8); + + EXPECT_EQ(a.getSkipCount(), 8); + EXPECT_EQ(a.subqueryDepth(), 2); + EXPECT_FALSE(a.nothingSkipped()); + a.reset(); + + EXPECT_EQ(a.getSkipCount(), 0); + EXPECT_EQ(a.subqueryDepth(), 2); + EXPECT_TRUE(a.nothingSkipped()); + + a.decrementSubquery(); + EXPECT_EQ(a.getSkipCount(), 0); +} + +} // namespace aql +} // namespace tests +} // namespace arangodb diff --git a/tests/Aql/SpliceSubqueryOptimizerRuleTest.cpp b/tests/Aql/SpliceSubqueryOptimizerRuleTest.cpp index 94e9940f4e42..19b3d524716f 100644 --- a/tests/Aql/SpliceSubqueryOptimizerRuleTest.cpp +++ b/tests/Aql/SpliceSubqueryOptimizerRuleTest.cpp @@ -313,49 +313,41 @@ TEST_F(SpliceSubqueryNodeOptimizerRuleTest, splice_subquery_with_sort) { verifyQueryResult(query, expected->slice()); } -// Must be changed as soon as the subquery implementation with shadow rows handle skipping, -// and the splice-subqueries optimizer rule is changed to allow it. -TEST_F(SpliceSubqueryNodeOptimizerRuleTest, dont_splice_subquery_with_skip__inner_limit_offset) { +TEST_F(SpliceSubqueryNodeOptimizerRuleTest, splice_subquery_with_skip__inner_limit_offset) { auto const queryString = R"aql(FOR i IN 0..2 LET a = (FOR j IN 0..2 LIMIT 1, 1 RETURN 3*i + j) RETURN FIRST(a))aql"; auto const expectedString = R"res([1, 4, 7])res"; - verifySubquerySplicing(queryString, 0, 1); + verifySubquerySplicing(queryString, 1); auto expected = arangodb::velocypack::Parser::fromJson(expectedString); verifyQueryResult(queryString, expected->slice()); } -// Must be changed as soon as the subquery implementation with shadow rows handle skipping, -// and the splice-subqueries optimizer rule is changed to allow it. -TEST_F(SpliceSubqueryNodeOptimizerRuleTest, dont_splice_subquery_with_skip__outer_limit_offset) { +TEST_F(SpliceSubqueryNodeOptimizerRuleTest, splice_subquery_with_skip__outer_limit_offset) { auto const queryString = R"aql(FOR i IN 0..2 LET a = (FOR j IN 0..2 RETURN 3*i + j) LIMIT 1, 1 RETURN FIRST(a))aql"; auto const expectedString = R"res([3])res"; - verifySubquerySplicing(queryString, 0, 1); + verifySubquerySplicing(queryString, 1); auto expected = arangodb::velocypack::Parser::fromJson(expectedString); verifyQueryResult(queryString, expected->slice()); } -// Must be changed as soon as the subquery implementation with shadow rows handle skipping, -// and the splice-subqueries optimizer rule is changed to allow it. -TEST_F(SpliceSubqueryNodeOptimizerRuleTest, dont_splice_subquery_with_skip__inner_collect_count) { +TEST_F(SpliceSubqueryNodeOptimizerRuleTest, splice_subquery_with_skip__inner_collect_count) { auto const queryString = R"aql(FOR i IN 0..2 LET a = (FOR j IN 0..i COLLECT WITH COUNT INTO n RETURN n) RETURN FIRST(a))aql"; auto const expectedString = R"res([1, 2, 3])res"; - verifySubquerySplicing(queryString, 0, 1); + verifySubquerySplicing(queryString, 1); auto expected = arangodb::velocypack::Parser::fromJson(expectedString); verifyQueryResult(queryString, expected->slice()); } -// Must be changed as soon as the subquery implementation with shadow rows handle skipping, -// and the splice-subqueries optimizer rule is changed to allow it. -TEST_F(SpliceSubqueryNodeOptimizerRuleTest, dont_splice_subquery_with_skip__outer_collect_count) { +TEST_F(SpliceSubqueryNodeOptimizerRuleTest, splice_subquery_with_skip__outer_collect_count) { // the RAND() is there to avoid the subquery being removed auto const queryString = R"aql(FOR i IN 0..2 LET a = (FOR j IN 0..FLOOR(2*RAND()) RETURN 1) @@ -363,14 +355,14 @@ TEST_F(SpliceSubqueryNodeOptimizerRuleTest, dont_splice_subquery_with_skip__oute RETURN n)aql"; auto const expectedString = R"res([3])res"; - verifySubquerySplicing(queryString, 0, 1); + verifySubquerySplicing(queryString, 1); auto expected = arangodb::velocypack::Parser::fromJson(expectedString); verifyQueryResult(queryString, expected->slice()); } // Must be changed as soon as the subquery implementation with shadow rows handle skipping, // and the splice-subqueries optimizer rule is changed to allow it. -TEST_F(SpliceSubqueryNodeOptimizerRuleTest, dont_splice_subquery_with_skip__full_count) { +TEST_F(SpliceSubqueryNodeOptimizerRuleTest, splice_subquery_with_skip__full_count) { // the RAND() is there to avoid the subquery being removed auto const queryString = R"aql(FOR i IN 0..2 LET a = (FOR j IN 0..FLOOR(2*RAND()) RETURN 1) @@ -378,7 +370,7 @@ TEST_F(SpliceSubqueryNodeOptimizerRuleTest, dont_splice_subquery_with_skip__full RETURN i)aql"; auto const expectedString = R"res([0])res"; - verifySubquerySplicing(queryString, 0, 1, "{}", R"opts({"fullCount": true})opts"); + verifySubquerySplicing(queryString, 1, 0, "{}", R"opts({"fullCount": true})opts"); auto expected = arangodb::velocypack::Parser::fromJson(expectedString); verifyQueryResult(queryString, expected->slice()); } @@ -412,7 +404,7 @@ TEST_F(SpliceSubqueryNodeOptimizerRuleTest, splice_nested_subquery_with_innermos FOR i IN 0..1 LET js = ( // this subquery should be spliced FOR j IN 0..1 + FLOOR(RAND()) - LET ks = ( // this subquery should not be spliced + LET ks = ( // this subquery should be spliced FOR k IN 0..2 + FLOOR(RAND()) LIMIT 1, 2 RETURN 6*i + 3*j + k @@ -423,7 +415,7 @@ TEST_F(SpliceSubqueryNodeOptimizerRuleTest, splice_nested_subquery_with_innermos )aql"; auto const expectedString = R"res([[[1, 2], [4, 5]], [[7, 8], [10, 11]]])res"; - verifySubquerySplicing(queryString, 1, 1); + verifySubquerySplicing(queryString, 2); auto expected = arangodb::velocypack::Parser::fromJson(expectedString); verifyQueryResult(queryString, expected->slice()); } @@ -440,7 +432,7 @@ TEST_F(SpliceSubqueryNodeOptimizerRuleTest, splice_nested_subquery_with_innermos )aql"; auto const expectedString = R"res([{"a": 1, "b": [[3, 4]]}, {"a": 2, "b": [[3, 4]]}])res"; - verifySubquerySplicing(queryString, 1, 1); + verifySubquerySplicing(queryString, 2); auto expected = arangodb::velocypack::Parser::fromJson(expectedString); verifyQueryResult(queryString, expected->slice()); } @@ -450,7 +442,7 @@ TEST_F(SpliceSubqueryNodeOptimizerRuleTest, splice_nested_subquery_with_innermos TEST_F(SpliceSubqueryNodeOptimizerRuleTest, splice_nested_subquery_with_outermost_skip) { auto const queryString = R"aql( FOR i IN 0..2 - LET js = ( // this subquery should not be spliced + LET js = ( // this subquery should be spliced FOR j IN 0..1 + FLOOR(RAND()) LET ks = ( // this subquery should be spliced FOR k IN 0..1 + FLOOR(RAND()) @@ -463,19 +455,19 @@ TEST_F(SpliceSubqueryNodeOptimizerRuleTest, splice_nested_subquery_with_outermos )aql"; auto const expectedString = R"res([[[4, 5], [6, 7]], [[8, 9], [10, 11]]])res"; - verifySubquerySplicing(queryString, 1, 1); + verifySubquerySplicing(queryString, 2); auto expected = arangodb::velocypack::Parser::fromJson(expectedString); verifyQueryResult(queryString, expected->slice()); } // Must be changed as soon as the subquery implementation with shadow rows handle skipping, // and the splice-subqueries optimizer rule is changed to allow it. -TEST_F(SpliceSubqueryNodeOptimizerRuleTest, dont_splice_subquery_with_limit_and_no_offset) { +TEST_F(SpliceSubqueryNodeOptimizerRuleTest, splice_subquery_with_limit_and_no_offset) { auto query = R"aql( FOR i IN 2..4 LET a = (FOR j IN [i, i+10, i+20] LIMIT 0, 1 RETURN j) RETURN FIRST(a))aql"; - verifySubquerySplicing(query, 0, 1); + verifySubquerySplicing(query, 1); auto expected = arangodb::velocypack::Parser::fromJson(R"([2, 3, 4])"); verifyQueryResult(query, expected->slice()); @@ -496,7 +488,6 @@ TEST_F(SpliceSubqueryNodeOptimizerRuleTest, splice_nested_empty_subqueries) { RETURN [results] )aql"; auto const expectedString = R"res([[[]]])res"; - verifySubquerySplicing(queryString, 2); auto expected = arangodb::velocypack::Parser::fromJson(expectedString); verifyQueryResult(queryString, expected->slice()); @@ -521,7 +512,7 @@ TEST_F(SpliceSubqueryNodeOptimizerRuleTest, splice_subquery_with_upsert) { auto const bindString = R"bind({"key": "myKey"})bind"; auto const expectedString = R"res([["UnitTestCollection/myKey"]])res"; - verifySubquerySplicing(queryString, 1, 1, bindString); + verifySubquerySplicing(queryString, 2, 0, bindString); auto expected = arangodb::velocypack::Parser::fromJson(expectedString); verifyQueryResult(queryString, expected->slice(), bindString); @@ -781,8 +772,7 @@ TEST_F(SpliceSubqueryNodeOptimizerRuleTest, splice_subquery_with_collect_in_subq verifyQueryResult(queryString, expected->slice()); } -// Disabled as long as the subquery implementation with shadow rows cannot yet handle skipping. -TEST_F(SpliceSubqueryNodeOptimizerRuleTest, DISABLED_splice_subquery_with_limit_and_offset) { +TEST_F(SpliceSubqueryNodeOptimizerRuleTest, splice_subquery_with_limit_and_offset) { auto query = R"aql( FOR i IN 2..4 LET a = (FOR j IN [0, i, i+10] LIMIT 1, 1 RETURN j) @@ -793,9 +783,7 @@ TEST_F(SpliceSubqueryNodeOptimizerRuleTest, DISABLED_splice_subquery_with_limit_ verifyQueryResult(query, expected->slice()); } -// Disabled as long as the subquery implementation with shadow rows cannot yet handle skipping. -TEST_F(SpliceSubqueryNodeOptimizerRuleTest, - DISABLED_splice_subquery_collect_within_empty_nested_subquery) { +TEST_F(SpliceSubqueryNodeOptimizerRuleTest, splice_subquery_collect_within_empty_nested_subquery) { auto query = R"aql( FOR k IN 1..2 LET sub1 = ( @@ -813,6 +801,46 @@ TEST_F(SpliceSubqueryNodeOptimizerRuleTest, verifyQueryResult(query, expected->slice()); } +TEST_F(SpliceSubqueryNodeOptimizerRuleTest, splice_subquery_skip_nodes) { + auto query = R"aql( + FOR k IN 1..10 + LET sub1 = ( + FOR j IN 1..10 + LET sub2 = ( + FOR i IN 1..4 + LIMIT 2,10 + RETURN i + ) + LIMIT 2,10 + RETURN [j, sub2] + ) + LIMIT 3, 10 + RETURN [k, sub1])aql"; + verifySubquerySplicing(query, 2); + + VPackBuilder builder; + builder.openArray(); + for (size_t k = 4; k <= 10; ++k) { + builder.openArray(); + builder.add(VPackValue(k)); + builder.openArray(); + for (size_t j = 3; j <= 10; ++j) { + builder.openArray(); + builder.add(VPackValue(j)); + builder.openArray(); + for (size_t i = 3; i <= 4; ++i) { + builder.add(VPackValue(i)); + } + builder.close(); + builder.close(); + } + builder.close(); + builder.close(); + } + builder.close(); + verifyQueryResult(query, builder.slice()); +} + // TODO Check isInSplicedSubquery // TODO Test cluster rules diff --git a/tests/Aql/SplicedSubqueryIntegrationTest.cpp b/tests/Aql/SplicedSubqueryIntegrationTest.cpp index f6f37d99f4b1..943afe458c2a 100644 --- a/tests/Aql/SplicedSubqueryIntegrationTest.cpp +++ b/tests/Aql/SplicedSubqueryIntegrationTest.cpp @@ -236,24 +236,34 @@ class SplicedSubqueryIntegrationTest auto createSkipCall() -> SkipCall { return [](AqlItemBlockInputRange& input, AqlCall& call) -> std::tuple { - auto skipped = size_t{0}; + while (call.shouldSkip() && input.skippedInFlight() > 0) { + if (call.getOffset() > 0) { + call.didSkip(input.skip(call.getOffset())); + } else { + EXPECT_TRUE(call.needsFullCount()); + EXPECT_EQ(call.getLimit(), 0); + EXPECT_TRUE(call.hasHardLimit()); + call.didSkip(input.skipAll()); + } + } + // If we overfetched and have data, throw it away while (input.hasDataRow() && call.shouldSkip()) { auto const& [state, inputRow] = input.nextDataRow(); EXPECT_TRUE(inputRow.isInitialized()); call.didSkip(1); - skipped++; } auto upstreamCall = AqlCall{call}; - return {input.upstreamState(), NoStats{}, skipped, upstreamCall}; + return {input.upstreamState(), NoStats{}, call.getSkipCount(), upstreamCall}; }; }; // Asserts if called. This is to check that when we use skip to // skip over a subquery, the subquery's produce is not invoked + // with data auto createAssertCall() -> ProduceCall { return [](AqlItemBlockInputRange& input, OutputAqlItemRow& output) -> std::tuple { - EXPECT_TRUE(false); + EXPECT_FALSE(input.hasDataRow()); NoStats stats{}; AqlCall call{}; @@ -333,7 +343,7 @@ TEST_P(SplicedSubqueryIntegrationTest, single_subquery) { .run(); }; -TEST_P(SplicedSubqueryIntegrationTest, DISABLED_single_subquery_skip_and_produce) { +TEST_P(SplicedSubqueryIntegrationTest, single_subquery_skip_and_produce) { auto call = AqlCall{5}; auto pipeline = createSubquery(); ExecutorTestHelper<1, 2>{*fakedQuery} @@ -347,7 +357,7 @@ TEST_P(SplicedSubqueryIntegrationTest, DISABLED_single_subquery_skip_and_produce .run(); }; -TEST_P(SplicedSubqueryIntegrationTest, DISABLED_single_subquery_skip_all) { +TEST_P(SplicedSubqueryIntegrationTest, single_subquery_skip_all) { auto call = AqlCall{20}; auto pipeline = createSubquery(); ExecutorTestHelper<1, 2>{*fakedQuery} @@ -361,7 +371,7 @@ TEST_P(SplicedSubqueryIntegrationTest, DISABLED_single_subquery_skip_all) { .run(); }; -TEST_P(SplicedSubqueryIntegrationTest, DISABLED_single_subquery_fullcount) { +TEST_P(SplicedSubqueryIntegrationTest, single_subquery_fullcount) { auto call = AqlCall{0, true, 0, AqlCall::LimitType::HARD}; auto pipeline = createSubquery(); ExecutorTestHelper<1, 2>{*fakedQuery} @@ -375,6 +385,8 @@ TEST_P(SplicedSubqueryIntegrationTest, DISABLED_single_subquery_fullcount) { .run(); }; +// NOTE: This test can be enabled if we can continue +// working on the second subquery without returning to consumer TEST_P(SplicedSubqueryIntegrationTest, DISABLED_single_subquery_skip_produce_count) { auto call = AqlCall{2, true, 2, AqlCall::LimitType::HARD}; auto pipeline = createSubquery(); @@ -442,7 +454,7 @@ TEST_P(SplicedSubqueryIntegrationTest, do_nothing_in_subquery) { .run(); }; -TEST_P(SplicedSubqueryIntegrationTest, DISABLED_check_call_passes_subquery) { +TEST_P(SplicedSubqueryIntegrationTest, check_call_passes_subquery) { auto call = AqlCall{10}; auto pipeline = concatPipelines(createCallAssertPipeline(call), createSubquery()); @@ -456,8 +468,9 @@ TEST_P(SplicedSubqueryIntegrationTest, DISABLED_check_call_passes_subquery) { .run(); }; -TEST_P(SplicedSubqueryIntegrationTest, DISABLED_check_skipping_subquery) { +TEST_P(SplicedSubqueryIntegrationTest, check_skipping_subquery) { auto call = AqlCall{10}; + LOG_DEVEL << call; auto pipeline = createSubquery(createAssertPipeline()); executorTestHelper.setPipeline(std::move(pipeline)) @@ -465,7 +478,23 @@ TEST_P(SplicedSubqueryIntegrationTest, DISABLED_check_skipping_subquery) { .setInputSplitType(getSplit()) .setCall(call) .expectOutput({0}, {}) - .expectSkipped(0) + .expectSkipped(8) .expectedState(ExecutionState::DONE) .run(); }; + +TEST_P(SplicedSubqueryIntegrationTest, check_soft_limit_subquery) { + auto call = AqlCall{0, false, 4, AqlCall::LimitType::SOFT}; + LOG_DEVEL << call; + auto pipeline = createSubquery(createAssertPipeline()); + + ExecutorTestHelper<1, 2>{*fakedQuery} + .setPipeline(std::move(pipeline)) + .setInputValueList(1, 2, 5, 2, 1, 5, 7, 1) + .setInputSplitType(getSplit()) + .setCall(call) + .expectOutput({0, 1}, {{1, R"([])"}, {2, R"([])"}, {5, R"([])"}, {2, R"([])"}}) + .expectSkipped(0) + .expectedState(ExecutionState::HASMORE) + .run(); +}; \ No newline at end of file diff --git a/tests/Aql/SubqueryStartExecutorTest.cpp b/tests/Aql/SubqueryStartExecutorTest.cpp index dc02b404e2d0..d73fbc9cc5e8 100644 --- a/tests/Aql/SubqueryStartExecutorTest.cpp +++ b/tests/Aql/SubqueryStartExecutorTest.cpp @@ -101,7 +101,7 @@ TEST_P(SubqueryStartExecutorTest, empty_input_does_not_add_shadow_rows) { .expectedStats(ExecutionStats{}) .expectedState(ExecutionState::DONE) .expectOutput({0}, {}) - .expectSkipped(0) + .expectSkipped(0, 0) .setCallStack(queryStack(AqlCall{}, AqlCall{})) .run(); } @@ -112,7 +112,7 @@ TEST_P(SubqueryStartExecutorTest, adds_a_shadowrow_after_single_input) { .setInputValue({{R"("a")"}}) .expectedStats(ExecutionStats{}) .expectedState(ExecutionState::DONE) - .expectSkipped(0) + .expectSkipped(0, 0) .expectOutput({0}, {{R"("a")"}, {R"("a")"}}, {{1, 0}}) .setCallStack(queryStack(AqlCall{}, AqlCall{})) .run(); @@ -128,7 +128,7 @@ TEST_P(SubqueryStartExecutorTest, .setInputValue({{{R"("a")"}}, {{R"("b")"}}, {{R"("c")"}}}) .expectedStats(ExecutionStats{}) .expectedState(ExecutionState::HASMORE) - .expectSkipped(0) + .expectSkipped(0, 0) .expectOutput({0}, {{R"("a")"}, {R"("a")"}}, {{1, 0}}) .setCallStack(queryStack(AqlCall{}, AqlCall{})) .run(); @@ -144,7 +144,7 @@ TEST_P(SubqueryStartExecutorTest, DISABLED_adds_a_shadowrow_after_every_input_li .setInputValue({{{R"("a")"}}, {{R"("b")"}}, {{R"("c")"}}}) .expectedStats(ExecutionStats{}) .expectedState(ExecutionState::DONE) - .expectSkipped(0) + .expectSkipped(0, 0) .expectOutput({0}, {{R"("a")"}, {R"("a")"}, {R"("b")"}, {R"("b")"}, {R"("c")"}, {R"("c")"}}, {{1, 0}, {3, 0}, {5, 0}}) .setCallStack(queryStack(AqlCall{}, AqlCall{})) @@ -159,7 +159,7 @@ TEST_P(SubqueryStartExecutorTest, adds_a_shadowrow_after_every_input_line) { .setInputValue({{{R"("a")"}}, {{R"("b")"}}, {{R"("c")"}}}) .expectedStats(ExecutionStats{}) .expectedState(ExecutionState::DONE) - .expectSkipped(0) + .expectSkipped(0, 0) .expectOutput({0}, {{R"("a")"}, {R"("a")"}, {R"("b")"}, {R"("b")"}, {R"("c")"}, {R"("c")"}}, {{1, 0}, {3, 0}, {5, 0}}) .setCallStack(queryStack(AqlCall{}, AqlCall{})) @@ -181,7 +181,7 @@ TEST_P(SubqueryStartExecutorTest, shadow_row_does_not_fit_in_current_block) { .setInputValue({{R"("a")"}}) .expectedStats(ExecutionStats{}) .expectedState(ExecutionState::HASMORE) - .expectSkipped(0) + .expectSkipped(0, 0) .expectOutput({0}, {{R"("a")"}}, {}) .setCallStack(queryStack(AqlCall{}, AqlCall{})) .run(); @@ -194,7 +194,7 @@ TEST_P(SubqueryStartExecutorTest, shadow_row_does_not_fit_in_current_block) { .setInputValue({{R"("a")"}}) .expectedStats(ExecutionStats{}) .expectedState(ExecutionState::DONE) - .expectSkipped(0) + .expectSkipped(0, 0) .expectOutput({0}, {{R"("a")"}, {R"("a")"}}, {{1, 0}}) .setCallStack(queryStack(AqlCall{}, AqlCall{})) .run(true); @@ -208,7 +208,7 @@ TEST_P(SubqueryStartExecutorTest, skip_in_subquery) { .expectedStats(ExecutionStats{}) .expectedState(ExecutionState::DONE) .expectOutput({0}, {{R"("a")"}}, {{0, 0}}) - .expectSkipped(1) + .expectSkipped(0, 1) .setCallStack(queryStack(AqlCall{}, AqlCall{10, false})) .run(); } @@ -220,7 +220,7 @@ TEST_P(SubqueryStartExecutorTest, fullCount_in_subquery) { .expectedStats(ExecutionStats{}) .expectedState(ExecutionState::DONE) .expectOutput({0}, {{R"("a")"}}, {{0, 0}}) - .expectSkipped(1) + .expectSkipped(0, 1) .setCallStack(queryStack(AqlCall{}, AqlCall{0, true, 0, AqlCall::LimitType::HARD})) .run(); } @@ -234,12 +234,20 @@ TEST_P(SubqueryStartExecutorTest, shadow_row_forwarding) { ExecutionNode::SUBQUERY_START)) .addConsumer(helper.createExecBlock(MakeBaseInfos(1), ExecutionNode::SUBQUERY_START)); + + if (GetCompatMode() == CompatibilityMode::VERSION36) { + // We will not get this infromation because the + // query stack is too small on purpose + helper.expectSkipped(0, 0); + } else { + helper.expectSkipped(0, 0, 0); + } + helper.setPipeline(std::move(pipe)) .setInputValue({{R"("a")"}}) .expectedStats(ExecutionStats{}) .expectedState(ExecutionState::DONE) .expectOutput({0}, {{R"("a")"}, {R"("a")"}, {R"("a")"}}, {{1, 0}, {2, 1}}) - .expectSkipped(0) .setCallStack(stack) .run(); } @@ -253,12 +261,20 @@ TEST_P(SubqueryStartExecutorTest, shadow_row_forwarding_many_inputs_single_call) ExecutionNode::SUBQUERY_START)) .addConsumer(helper.createExecBlock(MakeBaseInfos(1), ExecutionNode::SUBQUERY_START)); + + if (GetCompatMode() == CompatibilityMode::VERSION36) { + // We will not get this infromation because the + // query stack is too small on purpose + helper.expectSkipped(0, 0); + } else { + helper.expectSkipped(0, 0, 0); + } + helper.setPipeline(std::move(pipe)) .setInputValue({{R"("a")"}, {R"("b")"}, {R"("c")"}}) .expectedStats(ExecutionStats{}) .expectedState(ExecutionState::HASMORE) .expectOutput({0}, {{R"("a")"}, {R"("a")"}, {R"("a")"}}, {{1, 0}, {2, 1}}) - .expectSkipped(0) .setCallStack(stack) .run(); } @@ -272,6 +288,13 @@ TEST_P(SubqueryStartExecutorTest, shadow_row_forwarding_many_inputs_many_request ExecutionNode::SUBQUERY_START)) .addConsumer(helper.createExecBlock(MakeBaseInfos(1), ExecutionNode::SUBQUERY_START)); + if (GetCompatMode() == CompatibilityMode::VERSION36) { + // We will not get this infromation because the + // query stack is too small on purpose + helper.expectSkipped(0, 0); + } else { + helper.expectSkipped(0, 0, 0); + } helper.setPipeline(std::move(pipe)) .setInputValue({{R"("a")"}, {R"("b")"}, {R"("c")"}}) .expectedStats(ExecutionStats{}) @@ -280,7 +303,6 @@ TEST_P(SubqueryStartExecutorTest, shadow_row_forwarding_many_inputs_many_request {0}, {{R"("a")"}, {R"("a")"}, {R"("a")"}, {R"("b")"}, {R"("b")"}, {R"("b")"}, {R"("c")"}, {R"("c")"}, {R"("c")"}}, {{1, 0}, {2, 1}, {4, 0}, {5, 1}, {7, 0}, {8, 1}}) - .expectSkipped(0) .setCallStack(stack) .run(true); } @@ -303,12 +325,19 @@ TEST_P(SubqueryStartExecutorTest, shadow_row_forwarding_many_inputs_not_enough_s ExecutionNode::SUBQUERY_START)) .addConsumer(helper.createExecBlock(MakeBaseInfos(1), ExecutionNode::SUBQUERY_START)); + + if (GetCompatMode() == CompatibilityMode::VERSION36) { + // We will not get this infromation because the + // query stack is too small on purpose + helper.expectSkipped(0, 0); + } else { + helper.expectSkipped(0, 0, 0); + } helper.setPipeline(std::move(pipe)) .setInputValue({{R"("a")"}, {R"("b")"}, {R"("c")"}}) .expectedStats(ExecutionStats{}) .expectedState(ExecutionState::HASMORE) .expectOutput({0}, {{R"("a")"}, {R"("a")"}}, {{1, 0}}) - .expectSkipped(0) .setCallStack(stack) .run(); } @@ -323,6 +352,15 @@ TEST_P(SubqueryStartExecutorTest, shadow_row_forwarding_many_inputs_not_enough_s ExecutionNode::SUBQUERY_START)) .addConsumer(helper.createExecBlock(MakeBaseInfos(1), ExecutionNode::SUBQUERY_START)); + + if (GetCompatMode() == CompatibilityMode::VERSION36) { + // We will not get this infromation because the + // query stack is too small on purpose + helper.expectSkipped(0, 0); + } else { + helper.expectSkipped(0, 0, 0); + } + helper.setPipeline(std::move(pipe)) .setInputValue({{R"("a")"}, {R"("b")"}, {R"("c")"}}) .expectedStats(ExecutionStats{}) @@ -331,12 +369,105 @@ TEST_P(SubqueryStartExecutorTest, shadow_row_forwarding_many_inputs_not_enough_s {0}, {{R"("a")"}, {R"("a")"}, {R"("a")"}, {R"("b")"}, {R"("b")"}, {R"("b")"}, {R"("c")"}, {R"("c")"}, {R"("c")"}}, {{1, 0}, {2, 1}, {4, 0}, {5, 1}, {7, 0}, {8, 1}}) - .expectSkipped(0) .setCallStack(stack) .run(true); } } -// TODO: -// * Add tests for Skipping -// - on Higher level subquery +TEST_P(SubqueryStartExecutorTest, skip_in_outer_subquery) { + if (GetCompatMode() == CompatibilityMode::VERSION37) { + ExecutorTestHelper<1, 1>(*fakedQuery) + .setExecBlock(MakeBaseInfos(1), ExecutionNode::SUBQUERY_START) + .setInputValue({{R"("a")"}, {R"("b")"}}) + .expectedStats(ExecutionStats{}) + .expectedState(ExecutionState::DONE) + .expectOutput({0}, {{R"("b")"}, {R"("b")"}}, {{1, 0}}) + .expectSkipped(1, 0) + .setCallStack(queryStack(AqlCall{1, false, AqlCall::Infinity{}}, AqlCall{})) + .run(); + } else { + // The feature is not available in 3.6 or earlier. + } +} + +TEST_P(SubqueryStartExecutorTest, DISABLED_skip_only_in_outer_subquery) { + if (GetCompatMode() == CompatibilityMode::VERSION37) { + ExecutorTestHelper<1, 1>(*fakedQuery) + .setExecBlock(MakeBaseInfos(1), ExecutionNode::SUBQUERY_START) + .setInputValue({{R"("a")"}, {R"("b")"}}) + .expectedStats(ExecutionStats{}) + .expectedState(ExecutionState::DONE) + .expectOutput({0}, {}) + .expectSkipped(1, 0) + .setCallStack(queryStack(AqlCall{1, false}, AqlCall{})) + .run(); + } else { + // The feature is not available in 3.7 or earlier. + } +} + +TEST_P(SubqueryStartExecutorTest, fullCount_in_outer_subquery) { + if (GetCompatMode() == CompatibilityMode::VERSION37) { + ExecutorTestHelper<1, 1>(*fakedQuery) + .setExecBlock(MakeBaseInfos(1), ExecutionNode::SUBQUERY_START) + .setInputValue({{R"("a")"}, {R"("b")"}, {R"("c")"}, {R"("d")"}, {R"("e")"}, {R"("f")"}}) + .expectedStats(ExecutionStats{}) + .expectedState(ExecutionState::DONE) + .expectOutput({0}, {}) + .expectSkipped(6, 0) + .setCallStack(queryStack(AqlCall{0, true, 0, AqlCall::LimitType::HARD}, AqlCall{})) + .run(); + } else { + // The feature is not available in 3.7 or earlier. + } +} + +TEST_P(SubqueryStartExecutorTest, fastForward_in_inner_subquery) { + if (GetCompatMode() == CompatibilityMode::VERSION37) { + ExecutorTestHelper<1, 1>(*fakedQuery) + .setExecBlock(MakeBaseInfos(1), ExecutionNode::SUBQUERY_START) + .setInputValue({{R"("a")"}, {R"("b")"}, {R"("c")"}, {R"("d")"}, {R"("e")"}, {R"("f")"}}) + .expectedStats(ExecutionStats{}) + .expectedState(ExecutionState::HASMORE) + .expectOutput({0}, {{R"("a")"}}, {{0, 0}}) + .expectSkipped(0, 0) + .setCallStack(queryStack(AqlCall{0, false, AqlCall::Infinity{}}, + AqlCall{0, false, 0, AqlCall::LimitType::HARD})) + .run(); + } else { + // The feature is not available in 3.7 or earlier. + } +} + +TEST_P(SubqueryStartExecutorTest, skip_out_skip_in) { + if (GetCompatMode() == CompatibilityMode::VERSION37) { + ExecutorTestHelper<1, 1>(*fakedQuery) + .setExecBlock(MakeBaseInfos(1), ExecutionNode::SUBQUERY_START) + .setInputValue({{R"("a")"}, {R"("b")"}, {R"("c")"}, {R"("d")"}, {R"("e")"}, {R"("f")"}}) + .expectedStats(ExecutionStats{}) + .expectedState(ExecutionState::HASMORE) + .expectOutput({0}, {{R"("c")"}}, {{0, 0}}) + .expectSkipped(2, 1) + .setCallStack(queryStack(AqlCall{2, false, AqlCall::Infinity{}}, + AqlCall{10, false, AqlCall::Infinity{}})) + .run(); + } else { + // The feature is not available in 3.7 or earlier. + } +} + +TEST_P(SubqueryStartExecutorTest, fullbypass_in_outer_subquery) { + if (GetCompatMode() == CompatibilityMode::VERSION37) { + ExecutorTestHelper<1, 1>(*fakedQuery) + .setExecBlock(MakeBaseInfos(1), ExecutionNode::SUBQUERY_START) + .setInputValue({{R"("a")"}, {R"("b")"}, {R"("c")"}, {R"("d")"}, {R"("e")"}, {R"("f")"}}) + .expectedStats(ExecutionStats{}) + .expectedState(ExecutionState::DONE) + .expectOutput({0}, {}) + .expectSkipped(0, 0) + .setCallStack(queryStack(AqlCall{0, false, 0, AqlCall::LimitType::HARD}, AqlCall{})) + .run(); + } else { + // The feature is not available in 3.7 or earlier. + } +} diff --git a/tests/Aql/WaitingExecutionBlockMock.cpp b/tests/Aql/WaitingExecutionBlockMock.cpp index c48de11e02e4..5ed988f8478e 100644 --- a/tests/Aql/WaitingExecutionBlockMock.cpp +++ b/tests/Aql/WaitingExecutionBlockMock.cpp @@ -27,6 +27,7 @@ #include "Aql/ExecutionEngine.h" #include "Aql/ExecutionState.h" #include "Aql/QueryOptions.h" +#include "Aql/SkipResult.h" #include "Logger/LogMacros.h" @@ -113,7 +114,7 @@ std::pair WaitingExecutionBlockMock::skip } } -std::tuple WaitingExecutionBlockMock::execute(AqlCallStack stack) { +std::tuple WaitingExecutionBlockMock::execute(AqlCallStack stack) { traceExecuteBegin(stack); auto res = executeWithoutTrace(stack); traceExecuteEnd(res); @@ -121,7 +122,7 @@ std::tuple WaitingExecutionBlockM } // NOTE: Does not care for shadowrows! -std::tuple WaitingExecutionBlockMock::executeWithoutTrace( +std::tuple WaitingExecutionBlockMock::executeWithoutTrace( AqlCallStack stack) { while (!stack.isRelevant()) { stack.pop(); @@ -135,7 +136,7 @@ std::tuple WaitingExecutionBlockM if (_variant != WaitingBehaviour::NEVER && !_hasWaited) { // If we ordered waiting check on _hasWaited and wait if not _hasWaited = true; - return {ExecutionState::WAITING, 0, nullptr}; + return {ExecutionState::WAITING, SkipResult{}, nullptr}; } if (_variant == WaitingBehaviour::ALWAYS) { // If we always wait, reset. @@ -154,7 +155,9 @@ std::tuple WaitingExecutionBlockM // Sorry we can only return one block. // This means we have prepared the first block. // But still need more data. - return {ExecutionState::HASMORE, skipped, result}; + SkipResult skipRes{}; + skipRes.didSkip(skipped); + return {ExecutionState::HASMORE, skipRes, result}; } else { dropBlock(); continue; @@ -177,7 +180,9 @@ std::tuple WaitingExecutionBlockM // Sorry we can only return one block. // This means we have prepared the first block. // But still need more data. - return {ExecutionState::HASMORE, skipped, result}; + SkipResult skipRes{}; + skipRes.didSkip(skipped); + return {ExecutionState::HASMORE, skipRes, result}; } size_t canReturn = _data.front()->size() - _inflight; @@ -212,16 +217,20 @@ std::tuple WaitingExecutionBlockM dropBlock(); } } + SkipResult skipRes{}; + skipRes.didSkip(skipped); if (!_data.empty()) { - return {ExecutionState::HASMORE, skipped, result}; + return {ExecutionState::HASMORE, skipRes, result}; } else if (result != nullptr && result->size() < myCall.hardLimit) { - return {ExecutionState::HASMORE, skipped, result}; + return {ExecutionState::HASMORE, skipRes, result}; } else { - return {ExecutionState::DONE, skipped, result}; + return {ExecutionState::DONE, skipRes, result}; } } } - return {ExecutionState::DONE, skipped, result}; + SkipResult skipRes{}; + skipRes.didSkip(skipped); + return {ExecutionState::DONE, skipRes, result}; } void WaitingExecutionBlockMock::dropBlock() { diff --git a/tests/Aql/WaitingExecutionBlockMock.h b/tests/Aql/WaitingExecutionBlockMock.h index 2c147ba08e91..b1ad73e01bd8 100644 --- a/tests/Aql/WaitingExecutionBlockMock.h +++ b/tests/Aql/WaitingExecutionBlockMock.h @@ -35,6 +35,7 @@ class AqlItemBlock; class ExecutionEngine; class ExecutionNode; struct ResourceMonitor; +class SkipResult; } // namespace aql namespace tests { @@ -106,15 +107,15 @@ class WaitingExecutionBlockMock final : public arangodb::aql::ExecutionBlock { */ std::pair skipSome(size_t atMost) override; - std::tuple execute( + std::tuple execute( arangodb::aql::AqlCallStack stack) override; private: void dropBlock(); // Implementation of execute - std::tuple executeWithoutTrace( - arangodb::aql::AqlCallStack stack); + std::tuple + executeWithoutTrace(arangodb::aql::AqlCallStack stack); private: std::deque _data; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9ed9e68f7a40..4c8d6934d0dd 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -64,6 +64,7 @@ set(ARANGODB_TESTS_SOURCES Aql/RowFetcherHelper.cpp Aql/ScatterExecutorTest.cpp Aql/ShortestPathExecutorTest.cpp + Aql/SkipResultTest.cpp Aql/SingleRowFetcherTest.cpp Aql/SortedCollectExecutorTest.cpp Aql/SortExecutorTest.cpp diff --git a/tests/js/server/aql/aql-optimizer-indexes.js b/tests/js/server/aql/aql-optimizer-indexes.js index a08553b36d69..b42fd4955654 100644 --- a/tests/js/server/aql/aql-optimizer-indexes.js +++ b/tests/js/server/aql/aql-optimizer-indexes.js @@ -862,13 +862,9 @@ function optimizerIndexesTestSuite () { }); assertEqual("SingletonNode", nodeTypes[0], query); - assertEqual("SubqueryNode", nodeTypes[1], query); - - var subNodeTypes = plan.nodes[1].subquery.nodes.map(function(node) { - return node.type; - }); - assertNotEqual(-1, subNodeTypes.indexOf("IndexNode"), query); - assertEqual(-1, subNodeTypes.indexOf("SortNode"), query); + assertEqual("SubqueryStartNode", nodeTypes[1], query); + assertNotEqual(-1, nodeTypes.indexOf("IndexNode"), query); + assertEqual(-1, nodeTypes.indexOf("SortNode"), query); assertEqual("ReturnNode", nodeTypes[nodeTypes.length - 1], query); var results = AQL_EXECUTE(query, {}, opt); @@ -3776,26 +3772,39 @@ function optimizerIndexesMultiCollectionTestSuite () { var query = "FOR i IN " + c1.name() + " LET res = (FOR j IN " + c2.name() + " FILTER j.value == i.value SORT j.ref LIMIT 1 RETURN j) SORT res[0] RETURN i"; var plan = AQL_EXPLAIN(query, {}, opt).plan; - var nodeTypes = plan.nodes.map(function(node) { - return node.type; - }); + let idx = -1; + const nodeTypes = []; + const subqueryTypes = []; + { + let inSubquery = false; + for (const node of plan.nodes) { + const n = node.type; + if (n === "SubqueryStartNode" ) { + nodeTypes.push(n); + inSubquery = true; + } else if (n === "SubqueryEndNode" ) { + nodeTypes.push(n); + inSubquery = false; + } else if (inSubquery) { + if (n === "IndexNode") { + idx = node; + } + subqueryTypes.push(n); + } else { + nodeTypes.push(n); + } + } + } assertEqual("SingletonNode", nodeTypes[0], query); assertEqual(-1, nodeTypes.indexOf("IndexNode"), query); // no index for outer query assertNotEqual(-1, nodeTypes.indexOf("SortNode"), query); // sort node for outer query - var sub = nodeTypes.indexOf("SubqueryNode"); + var sub = nodeTypes.indexOf("SubqueryStartNode"); assertNotEqual(-1, sub); - - var subNodeTypes = plan.nodes[sub].subquery.nodes.map(function(node) { - return node.type; - }); - - assertEqual("SingletonNode", subNodeTypes[0], query); - var idx = subNodeTypes.indexOf("IndexNode"); assertNotEqual(-1, idx, query); // index used for inner query - assertEqual("hash", plan.nodes[sub].subquery.nodes[idx].indexes[0].type); - assertNotEqual(-1, subNodeTypes.indexOf("SortNode"), query); // must have sort node for inner query + assertEqual("hash", idx.indexes[0].type); + assertNotEqual(-1, subqueryTypes.indexOf("SortNode"), query); // must have sort node for inner query }, //////////////////////////////////////////////////////////////////////////////// @@ -3807,26 +3816,39 @@ function optimizerIndexesMultiCollectionTestSuite () { var query = "FOR i IN " + c1.name() + " LET res = (FOR j IN " + c2.name() + " FILTER j.value == i.value SORT j.value LIMIT 1 RETURN j) SORT res[0] RETURN i"; var plan = AQL_EXPLAIN(query, {}, opt).plan; - var nodeTypes = plan.nodes.map(function(node) { - return node.type; - }); - + const nodeTypes = []; + const subqueryTypes = []; + let idx = -1; + { + let inSubquery = false; + for (const node of plan.nodes) { + const n = node.type; + if (n === "SubqueryStartNode" ) { + nodeTypes.push(n); + inSubquery = true; + } else if (n === "SubqueryEndNode" ) { + nodeTypes.push(n); + inSubquery = false; + } else if (inSubquery) { + if (n === "IndexNode") { + idx = node; + } + subqueryTypes.push(n); + } else { + nodeTypes.push(n); + } + } + } assertEqual("SingletonNode", nodeTypes[0], query); assertEqual(-1, nodeTypes.indexOf("IndexNode"), query); // no index for outer query assertNotEqual(-1, nodeTypes.indexOf("SortNode"), query); // sort node for outer query - var sub = nodeTypes.indexOf("SubqueryNode"); + var sub = nodeTypes.indexOf("SubqueryStartNode"); assertNotEqual(-1, sub); - var subNodeTypes = plan.nodes[sub].subquery.nodes.map(function(node) { - return node.type; - }); - - assertEqual("SingletonNode", subNodeTypes[0], query); - var idx = subNodeTypes.indexOf("IndexNode"); - assertNotEqual(-1, idx, query); // index used for inner query - assertEqual("hash", plan.nodes[sub].subquery.nodes[idx].indexes[0].type); - assertEqual(-1, subNodeTypes.indexOf("SortNode"), query); // must not have sort node for inner query + assertNotEqual(-1, subqueryTypes.indexOf("IndexNode"), query); // index used for inner query + assertEqual("hash", idx.indexes[0].type); + assertEqual(-1, subqueryTypes.indexOf("SortNode"), query); // must not have sort node for inner query }, //////////////////////////////////////////////////////////////////////////////// @@ -3839,26 +3861,40 @@ function optimizerIndexesMultiCollectionTestSuite () { var query = "FOR i IN " + c1.name() + " LET res = (FOR j IN " + c2.name() + " FILTER j.value == i.value SORT j.ref LIMIT 1 RETURN j) SORT res[0] RETURN i"; var plan = AQL_EXPLAIN(query, {}, opt).plan; - var nodeTypes = plan.nodes.map(function(node) { - return node.type; - }); + const nodeTypes = []; + const subqueryTypes = []; + let idx = -1; + { + let inSubquery = false; + for (const node of plan.nodes) { + const n = node.type; + if (n === "SubqueryStartNode" ) { + nodeTypes.push(n); + inSubquery = true; + } else if (n === "SubqueryEndNode" ) { + nodeTypes.push(n); + inSubquery = false; + } else if (inSubquery) { + if (n === "IndexNode") { + idx = node; + } + subqueryTypes.push(n); + } else { + nodeTypes.push(n); + } + } + } assertEqual("SingletonNode", nodeTypes[0], query); assertEqual(-1, nodeTypes.indexOf("IndexNode"), query); // no index for outer query assertNotEqual(-1, nodeTypes.indexOf("SortNode"), query); // sort node for outer query - var sub = nodeTypes.indexOf("SubqueryNode"); + var sub = nodeTypes.indexOf("SubqueryStartNode"); assertNotEqual(-1, sub); - - var subNodeTypes = plan.nodes[sub].subquery.nodes.map(function(node) { - return node.type; - }); - - assertEqual("SingletonNode", subNodeTypes[0], query); - var idx = subNodeTypes.indexOf("IndexNode"); - assertNotEqual(-1, idx, query); // index used for inner query - assertEqual("hash", plan.nodes[sub].subquery.nodes[idx].indexes[0].type); - assertNotEqual(-1, subNodeTypes.indexOf("SortNode"), query); // must have sort node for inner query + assertNotEqual(-1, idx); + assertNotEqual(-1, subqueryTypes.indexOf("IndexNode"), query); // index used for inner query + assertEqual("hash", idx.indexes[0].type); + assertNotEqual(-1, subqueryTypes.indexOf("SortNode"), query); // must have sort node for inner query }, //////////////////////////////////////////////////////////////////////////////// @@ -3871,26 +3907,40 @@ function optimizerIndexesMultiCollectionTestSuite () { var query = "FOR i IN " + c1.name() + " LET res = (FOR j IN " + c2.name() + " FILTER j.value == i.value SORT j.value LIMIT 1 RETURN j) SORT res[0] RETURN i"; var plan = AQL_EXPLAIN(query, {}, opt).plan; - var nodeTypes = plan.nodes.map(function(node) { - return node.type; - }); + const nodeTypes = []; + const subqueryTypes = []; + let idx = -1; + { + let inSubquery = false; + for (const node of plan.nodes) { + const n = node.type; + if (n === "SubqueryStartNode" ) { + nodeTypes.push(n); + inSubquery = true; + } else if (n === "SubqueryEndNode" ) { + nodeTypes.push(n); + inSubquery = false; + } else if (inSubquery) { + if (n === "IndexNode") { + idx = node; + } + subqueryTypes.push(n); + } else { + nodeTypes.push(n); + } + } + } assertEqual("SingletonNode", nodeTypes[0], query); assertEqual(-1, nodeTypes.indexOf("IndexNode"), query); // no index for outer query assertNotEqual(-1, nodeTypes.indexOf("SortNode"), query); // sort node for outer query - var sub = nodeTypes.indexOf("SubqueryNode"); - assertNotEqual(-1, sub); + assertNotEqual(-1, nodeTypes.indexOf("SubqueryStartNode"), query); - var subNodeTypes = plan.nodes[sub].subquery.nodes.map(function(node) { - return node.type; - }); - - assertEqual("SingletonNode", subNodeTypes[0], query); - var idx = subNodeTypes.indexOf("IndexNode"); + assertNotEqual(-1, subqueryTypes.indexOf("IndexNode"), query); assertNotEqual(-1, idx, query); // index used for inner query - assertEqual("hash", plan.nodes[sub].subquery.nodes[idx].indexes[0].type); - assertEqual(-1, subNodeTypes.indexOf("SortNode"), query); // we're filtering on a constant, must not have sort node for inner query + assertEqual("hash", idx.indexes[0].type); + assertEqual(-1, subqueryTypes.indexOf("SortNode"), query); // we're filtering on a constant, must not have sort node for inner query }, //////////////////////////////////////////////////////////////////////////////// @@ -3903,26 +3953,40 @@ function optimizerIndexesMultiCollectionTestSuite () { var query = "FOR i IN " + c1.name() + " LET res = (FOR z IN 1..2 FOR j IN " + c2.name() + " FILTER j.value == i.value SORT j.value LIMIT 1 RETURN j) SORT res[0] RETURN i"; var plan = AQL_EXPLAIN(query, {}, opt).plan; - var nodeTypes = plan.nodes.map(function(node) { - return node.type; - }); + const nodeTypes = []; + const subqueryTypes = []; + let idx = -1; + { + let inSubquery = false; + for (const node of plan.nodes) { + const n = node.type; + if (n === "SubqueryStartNode" ) { + nodeTypes.push(n); + inSubquery = true; + } else if (n === "SubqueryEndNode" ) { + nodeTypes.push(n); + inSubquery = false; + } else if (inSubquery) { + if (n === "IndexNode") { + idx = node; + } + subqueryTypes.push(n); + } else { + nodeTypes.push(n); + } + } + } assertEqual("SingletonNode", nodeTypes[0], query); assertEqual(-1, nodeTypes.indexOf("IndexNode"), query); // no index for outer query assertNotEqual(-1, nodeTypes.indexOf("SortNode"), query); // sort node for outer query - var sub = nodeTypes.indexOf("SubqueryNode"); - assertNotEqual(-1, sub); - - var subNodeTypes = plan.nodes[sub].subquery.nodes.map(function(node) { - return node.type; - }); - - assertEqual("SingletonNode", subNodeTypes[0], query); - var idx = subNodeTypes.indexOf("IndexNode"); + var sub = nodeTypes.indexOf("SubqueryStartNode"); + assertNotEqual(-1, sub, query); + assertNotEqual(-1, subqueryTypes.indexOf("IndexNode"), query); assertNotEqual(-1, idx, query); // index used for inner query - assertEqual("hash", plan.nodes[sub].subquery.nodes[idx].indexes[0].type); - assertNotEqual(-1, subNodeTypes.indexOf("SortNode"), query); // we're filtering on a constant, but we're in an inner loop + assertEqual("hash", idx.indexes[0].type); + assertNotEqual(-1, subqueryTypes.indexOf("SortNode"), query); // we're filtering on a constant, but we're in an inner loop }, //////////////////////////////////////////////////////////////////////////////// @@ -3935,26 +3999,39 @@ function optimizerIndexesMultiCollectionTestSuite () { var query = "FOR i IN " + c1.name() + " LET res = (FOR j IN " + c2.name() + " FILTER j.ref == i.ref SORT j.ref LIMIT 1 RETURN j) SORT res[0] RETURN i"; var plan = AQL_EXPLAIN(query, {}, opt).plan; - var nodeTypes = plan.nodes.map(function(node) { - return node.type; - }); + const nodeTypes = []; + const subqueryTypes = []; + let idx = -1; + { + let inSubquery = false; + for (const node of plan.nodes) { + const n = node.type; + if (n === "SubqueryStartNode" ) { + nodeTypes.push(n); + inSubquery = true; + } else if (n === "SubqueryEndNode" ) { + nodeTypes.push(n); + inSubquery = false; + } else if (inSubquery) { + if (n === "IndexNode") { + idx = node; + } + subqueryTypes.push(n); + } else { + nodeTypes.push(n); + } + } + } assertEqual("SingletonNode", nodeTypes[0], query); assertEqual(-1, nodeTypes.indexOf("IndexNode"), query); // no index for outer query assertNotEqual(-1, nodeTypes.indexOf("SortNode"), query); // sort node for outer query + assertNotEqual(-1, nodeTypes.indexOf("SubqueryStartNode"), query); - var sub = nodeTypes.indexOf("SubqueryNode"); - assertNotEqual(-1, sub); - - var subNodeTypes = plan.nodes[sub].subquery.nodes.map(function(node) { - return node.type; - }); - - assertEqual("SingletonNode", subNodeTypes[0], query); - var idx = subNodeTypes.indexOf("IndexNode"); + assertNotEqual(-1, subqueryTypes.indexOf("IndexNode"), query); assertNotEqual(-1, idx, query); // index used for inner query - assertEqual("skiplist", plan.nodes[sub].subquery.nodes[idx].indexes[0].type); - assertEqual(-1, subNodeTypes.indexOf("SortNode"), query); // must not have sort node for inner query + assertEqual("skiplist", idx.indexes[0].type); + assertEqual(-1, subqueryTypes.indexOf("SortNode"), query); // must not have sort node for inner query }, //////////////////////////////////////////////////////////////////////////////// diff --git a/tests/js/server/aql/aql-optimizer-rule-move-calculations-down.js b/tests/js/server/aql/aql-optimizer-rule-move-calculations-down.js index 54e7c8d0bd7d..37023b6157d1 100644 --- a/tests/js/server/aql/aql-optimizer-rule-move-calculations-down.js +++ b/tests/js/server/aql/aql-optimizer-rule-move-calculations-down.js @@ -371,7 +371,7 @@ function optimizerRuleTestSuite () { expected.push("test" + i + "-" + i); } - var query = "FOR i IN 0..100 LET result = (UPDATE {_key: CONCAT('test', TO_STRING(i))} WITH {updated: true} IN " + cn + " RETURN CONCAT(NEW._key, '-', NEW.value)) LIMIT 10 RETURN result[0]"; + var query = "FOR i IN 0..99 LET result = (UPDATE {_key: CONCAT('test', TO_STRING(i))} WITH {updated: true} IN " + cn + " RETURN CONCAT(NEW._key, '-', NEW.value)) LIMIT 10 RETURN result[0]"; var planDisabled = AQL_EXPLAIN(query, {}, paramDisabled); var planEnabled = AQL_EXPLAIN(query, {}, paramEnabled); diff --git a/tests/js/server/aql/aql-optimizer-rule-no-document-materialization-arangosearch.js b/tests/js/server/aql/aql-optimizer-rule-no-document-materialization-arangosearch.js index d6fb5cfe0102..59626c692b9a 100644 --- a/tests/js/server/aql/aql-optimizer-rule-no-document-materialization-arangosearch.js +++ b/tests/js/server/aql/aql-optimizer-rule-no-document-materialization-arangosearch.js @@ -145,10 +145,8 @@ function noDocumentMaterializationArangoSearchRuleTestSuite () { "SORT CONCAT(a, e) LIMIT 10 RETURN d.obj.e.e1"; let plan = AQL_EXPLAIN(query).plan; assertTrue(plan.nodes.filter(obj => { - return obj.type === "SubqueryNode"; - })[0].subquery.nodes.filter(obj => { return obj.type === "EnumerateViewNode"; - })[0].noMaterialization); + })[1].noMaterialization); let result = AQL_EXECUTE(query); assertEqual(2, result.json.length); let expectedKeys = new Set([14, 4]); diff --git a/tests/js/server/aql/aql-subquery.js b/tests/js/server/aql/aql-subquery.js index a2cc8ded83b7..d3002e80137f 100644 --- a/tests/js/server/aql/aql-subquery.js +++ b/tests/js/server/aql/aql-subquery.js @@ -343,8 +343,8 @@ function ahuacatlSubqueryTestSuite () { /// A count collect block will produce an output even if it does not get an input /// specifically it will rightfully count 0. /// The insert block will write into the collection if it gets an input. -/// So the assertion here is, that if a subquery has no input, than all it's -/// Parts do not have side-effects, but the subquery still prduces valid results +/// Even if the outer subquery is skipped. Henve we require to have documents +/// inserted here. //////////////////////////////////////////////////////////////////////////////// testCollectWithinEmptyNestedSubquery: function () { const colName = "UnitTestSubqueryCollection"; @@ -367,7 +367,7 @@ function ahuacatlSubqueryTestSuite () { var actual = getQueryResults(query); assertEqual(expected, actual); - assertEqual(db[colName].count(), 0); + assertEqual(db[colName].count(), 1); } finally { db._drop(colName); }