diff --git a/arangod/Aql/AqlItemBlockInputRange.h b/arangod/Aql/AqlItemBlockInputRange.h index 6a03bb1e6942..f70b1501214d 100644 --- a/arangod/Aql/AqlItemBlockInputRange.h +++ b/arangod/Aql/AqlItemBlockInputRange.h @@ -58,7 +58,6 @@ class AqlItemBlockInputRange { bool hasShadowRow() const noexcept; arangodb::aql::ShadowAqlItemRow peekShadowRow() const; - std::pair peekShadowRowAndState() const; std::pair nextShadowRow(); diff --git a/arangod/Aql/BlocksWithClients.cpp b/arangod/Aql/BlocksWithClients.cpp index a3eed57cd616..021d64116bf4 100644 --- a/arangod/Aql/BlocksWithClients.cpp +++ b/arangod/Aql/BlocksWithClients.cpp @@ -118,32 +118,6 @@ auto BlocksWithClientsImpl::initializeCursor(InputAqlItemRow const& in return ExecutionBlock::initializeCursor(input); } -template -auto BlocksWithClientsImpl::getBlock(size_t atMost) - -> std::pair { - if (_engine->getQuery()->killed()) { - THROW_ARANGO_EXCEPTION(TRI_ERROR_QUERY_KILLED); - } - - auto res = _dependencies[0]->getSome(atMost); - if (res.first == ExecutionState::WAITING) { - return {res.first, false}; - } - - TRI_IF_FAILURE("ExecutionBlock::getBlock") { - THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); - } - - _upstreamState = res.first; - - if (res.second != nullptr) { - _buffer.emplace_back(std::move(res.second)); - return {res.first, true}; - } - - return {res.first, false}; -} - /// @brief shutdown template auto BlocksWithClientsImpl::shutdown(int errorCode) @@ -176,18 +150,6 @@ size_t BlocksWithClientsImpl::getClientId(std::string const& shardId) return it->second; } -template -std::pair BlocksWithClientsImpl::getSome(size_t) { - TRI_ASSERT(false); - THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); -} - -template -std::pair BlocksWithClientsImpl::skipSome(size_t) { - TRI_ASSERT(false); - THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); -} - template std::tuple BlocksWithClientsImpl::execute(AqlCallStack stack) { diff --git a/arangod/Aql/BlocksWithClients.h b/arangod/Aql/BlocksWithClients.h index 75e68a37e306..aba33a50be2b 100644 --- a/arangod/Aql/BlocksWithClients.h +++ b/arangod/Aql/BlocksWithClients.h @@ -122,14 +122,6 @@ class BlocksWithClientsImpl : public ExecutionBlock, public BlocksWithClients { /// @brief shutdown std::pair shutdown(int) override; - std::pair getBlock(size_t atMost); - - /// @brief getSome: shouldn't be used, use skipSomeForShard - std::pair getSome(size_t atMost) final; - - /// @brief skipSome: shouldn't be used, use skipSomeForShard - std::pair skipSome(size_t atMost) final; - /// @brief execute: shouldn't be used, use executeForClient std::tuple execute(AqlCallStack stack) override; diff --git a/arangod/Aql/CalculationExecutor.cpp b/arangod/Aql/CalculationExecutor.cpp index 191324660f17..f0723de46e75 100644 --- a/arangod/Aql/CalculationExecutor.cpp +++ b/arangod/Aql/CalculationExecutor.cpp @@ -86,50 +86,6 @@ std::vector const& CalculationExecutorInfos::getExpInRegs() const no return _expInRegs; } -template -std::pair::Stats> -CalculationExecutor::produceRows(OutputAqlItemRow& output) { - // TRI_ASSERT(false); - // THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); - ExecutionState state; - InputAqlItemRow row = InputAqlItemRow{CreateInvalidInputRowHint{}}; - std::tie(state, row) = _fetcher.fetchRow(); - - if (state == ExecutionState::WAITING) { - TRI_ASSERT(!row); - TRI_ASSERT(!_infos.getQuery().hasEnteredContext()); - return {state, NoStats{}}; - } - - if (!row) { - TRI_ASSERT(state == ExecutionState::DONE); -#ifdef ARANGODB_ENABLE_MAINTAINER_MODE - TRI_ASSERT(_fetcher.isAtShadowRow() || (!_fetcher.hasRowsLeftInBlock() && - !_infos.getQuery().hasEnteredContext())); -#endif - return {state, NoStats{}}; - } - - doEvaluation(row, output); - - // _hasEnteredContext implies the query has entered the context, but not - // the other way round because it may be owned by exterior. - TRI_ASSERT(!_hasEnteredContext || _infos.getQuery().hasEnteredContext()); - - // The following only affects V8Conditions. If we should exit the V8 context - // between blocks, because we might have to wait for client or upstream, then - // hasEnteredContext => state == HASMORE, - // as we only leave the context open when there are rows left in the current - // block. - // Note that _infos.getQuery().hasEnteredContext() may be true, even if - // _hasEnteredContext is false, if (and only if) the query context is owned - // by exterior. - TRI_ASSERT(!shouldExitContextBetweenBlocks() || !_hasEnteredContext || - state == ExecutionState::HASMORE); - - return {state, NoStats{}}; -} - template std::tuple::Stats, AqlCall> CalculationExecutor::produceRows(AqlItemBlockInputRange& inputRange, @@ -143,7 +99,7 @@ CalculationExecutor::produceRows(AqlItemBlockInputRange& inputR while (inputRange.hasDataRow()) { // This executor is passthrough. it has enough place to write. TRI_ASSERT(!output.isFull()); - std::tie(state, input) = inputRange.nextDataRow(); + std::tie(state, input) = inputRange.nextDataRow(); TRI_ASSERT(input.isInitialized()); doEvaluation(input, output); @@ -168,13 +124,6 @@ CalculationExecutor::produceRows(AqlItemBlockInputRange& inputR return {inputRange.upstreamState(), NoStats{}, output.getClientCall()}; } -template -std::tuple::Stats, SharedAqlItemBlockPtr> -CalculationExecutor::fetchBlockForPassthrough(size_t atMost) { - auto rv = _fetcher.fetchBlockForPassthrough(atMost); - return {rv.first, {}, std::move(rv.second)}; -} - template template void CalculationExecutor::enterContext() { diff --git a/arangod/Aql/CalculationExecutor.h b/arangod/Aql/CalculationExecutor.h index 8ab0b812721b..3fc09d231d0c 100644 --- a/arangod/Aql/CalculationExecutor.h +++ b/arangod/Aql/CalculationExecutor.h @@ -99,13 +99,6 @@ class CalculationExecutor { CalculationExecutor(Fetcher& fetcher, CalculationExecutorInfos&); ~CalculationExecutor(); - /** - * @brief produce the next Row of Aql Values. - * - * @return ExecutionState, and if successful exactly one new Row of AqlItems. - */ - std::pair produceRows(OutputAqlItemRow& output); - /** * @brief produce the next Row of Aql Values. * @@ -114,8 +107,6 @@ class CalculationExecutor { [[nodiscard]] std::tuple produceRows( AqlItemBlockInputRange& inputRange, OutputAqlItemRow& output); - std::tuple fetchBlockForPassthrough(size_t atMost); - private: // specialized implementations void doEvaluation(InputAqlItemRow& input, OutputAqlItemRow& output); diff --git a/arangod/Aql/ConstFetcher.cpp b/arangod/Aql/ConstFetcher.cpp index f4182e144be4..13019b399faa 100644 --- a/arangod/Aql/ConstFetcher.cpp +++ b/arangod/Aql/ConstFetcher.cpp @@ -153,7 +153,6 @@ auto ConstFetcher::execute(AqlCallStack& stack) _rowIndex = 0; SkipResult skipped{}; skipped.didSkip(call.getSkipCount()); - return {ExecutionState::DONE, skipped, DataRange{ExecutorState::DONE, call.getSkipCount(), resultBlock, 0}}; } @@ -171,28 +170,23 @@ auto ConstFetcher::execute(AqlCallStack& stack) // Remove the indexes from the slice list sliceIndexes.erase(sliceIndexes.begin()); } - // NOTE: The above if may have invalidated from and to memory. - // Do not use them below this point! - if (sliceIndexes.empty()) { - // No data to be returned - // Block is dropped. - resultBlock = nullptr; - SkipResult skipped{}; - skipped.didSkip(call.getSkipCount()); - return {ExecutionState::DONE, skipped, - DataRange{ExecutorState::DONE, call.getSkipCount()}}; - } - - // Slowest path need to slice, this unfortunately requires copy of data ExecutionState resState = _blockForPassThrough == nullptr ? ExecutionState::DONE : ExecutionState::HASMORE; ExecutorState rangeState = _blockForPassThrough == nullptr ? ExecutorState::DONE : ExecutorState::HASMORE; - resultBlock = resultBlock->slice(sliceIndexes); SkipResult skipped{}; skipped.didSkip(call.getSkipCount()); + + if (sliceIndexes.empty()) { + // No data to be returned + resultBlock = nullptr; + return {resState, skipped, DataRange{rangeState, call.getSkipCount()}}; + } + + // Slowest path need to slice, this unfortunately requires copy of data + resultBlock = resultBlock->slice(sliceIndexes); return {resState, skipped, DataRange{rangeState, call.getSkipCount(), resultBlock, 0}}; } @@ -278,15 +272,6 @@ auto ConstFetcher::canUseFullBlock(std::vector> const& return true; } -std::pair ConstFetcher::fetchBlockForPassthrough(size_t) { - // Should only be called once, and then _blockForPassThrough should be - // initialized. However, there are still some blocks left that ask their - // parent even after they got DONE the last time, and I don't currently have - // time to track them down. Thus the following assert is commented out. - // TRI_ASSERT(_blockForPassThrough != nullptr); - return {ExecutionState::DONE, std::move(_blockForPassThrough)}; -} - std::pair ConstFetcher::fetchShadowRow(size_t) const { return {ExecutionState::DONE, ShadowAqlItemRow{CreateInvalidShadowRowHint{}}}; } diff --git a/arangod/Aql/ConstFetcher.h b/arangod/Aql/ConstFetcher.h index 33e0215bee70..1bb07fea582d 100644 --- a/arangod/Aql/ConstFetcher.h +++ b/arangod/Aql/ConstFetcher.h @@ -95,9 +95,6 @@ class ConstFetcher { TEST_VIRTUAL std::pair skipRows(size_t); void injectBlock(SharedAqlItemBlockPtr block); - // Argument will be ignored! - std::pair fetchBlockForPassthrough(size_t); - void setDistributeId(std::string const&) { // This is not implemented for this fetcher TRI_ASSERT(false); diff --git a/arangod/Aql/ConstrainedSortExecutor.cpp b/arangod/Aql/ConstrainedSortExecutor.cpp index 54aa1ce7248a..ef77a8d5f3ab 100644 --- a/arangod/Aql/ConstrainedSortExecutor.cpp +++ b/arangod/Aql/ConstrainedSortExecutor.cpp @@ -134,8 +134,6 @@ bool ConstrainedSortExecutor::compareInput(size_t const& rowPos, ConstrainedSortExecutor::ConstrainedSortExecutor(Fetcher& fetcher, SortExecutorInfos& infos) : _infos(infos), - _fetcher(fetcher), - _state(ExecutionState::HASMORE), _returnNext(0), _rowsPushed(0), _rowsRead(0), @@ -271,33 +269,26 @@ auto ConstrainedSortExecutor::skipRowsRange(AqlItemBlockInputRange& inputRange, return {state, NoStats{}, call.getSkipCount(), AqlCall{}}; } -std::pair ConstrainedSortExecutor::expectedNumberOfRows(size_t) const { - // This block cannot support atMost - size_t rowsLeft = 0; - if (_state != ExecutionState::DONE) { - ExecutionState state; - size_t expectedRows; - std::tie(state, expectedRows) = - _fetcher.preFetchNumberOfRows(ExecutionBlock::DefaultBatchSize); - if (state == ExecutionState::WAITING) { - TRI_ASSERT(expectedRows == 0); - return {state, 0}; - } - // Return the minimum of upstream + limit - rowsLeft = (std::min)(expectedRows, _infos.limit()); - } else { - // We have exactly the following rows available: - rowsLeft = _rows.size() - _returnNext; +[[nodiscard]] auto ConstrainedSortExecutor::expectedNumberOfRowsNew( + AqlItemBlockInputRange const& input, AqlCall const& call) const noexcept -> size_t { + size_t rowsPerBlock = _infos.limit(); + size_t subqueries = input.countShadowRows(); + if (subqueries == 0) { + // we are a top level block, just pretend we run in the only subquery. + subqueries = 1; } - - if (rowsLeft == 0) { - if (doneSkipping()) { - return {ExecutionState::DONE, rowsLeft}; - } - // We always report at least 1 row here, for a possible LIMIT block with fullCount to work. - // However, we should never have to do this if the LIMIT block doesn't overfetch with getSome. - rowsLeft = 1; + // we return rowsPerBlock for every subquery. + size_t totalRows = subqueries * rowsPerBlock; + // We can only have returnNext reach the total amount of Rows in a block + // We have at least 1 block, hence totalRows needs to be higher + TRI_ASSERT(_returnNext <= totalRows); + // We have totalRows at most available. + // adn we have _returnNext many of them already returned + // from the first data-rows block. + // In unlucky case we overestumate here, if we get called + // while operating on any other data-row block then the first. + if (input.countShadowRows() == 0) { + return std::min(call.getLimit(), totalRows - _returnNext); } - - return {ExecutionState::HASMORE, rowsLeft}; + return totalRows - _returnNext; } diff --git a/arangod/Aql/ConstrainedSortExecutor.h b/arangod/Aql/ConstrainedSortExecutor.h index 9541dd0cbde4..a0b7e9ea4a6c 100644 --- a/arangod/Aql/ConstrainedSortExecutor.h +++ b/arangod/Aql/ConstrainedSortExecutor.h @@ -90,7 +90,8 @@ class ConstrainedSortExecutor { * @brief This Executor knows how many rows it will produce and most by itself * It also knows that it could produce less if the upstream only has fewer rows. */ - std::pair expectedNumberOfRows(size_t atMost) const; + [[nodiscard]] auto expectedNumberOfRowsNew(AqlItemBlockInputRange const& input, + AqlCall const& call) const noexcept -> size_t; private: bool compareInput(size_t const& rosPos, InputAqlItemRow const& row) const; @@ -108,8 +109,6 @@ class ConstrainedSortExecutor { private: Infos& _infos; - Fetcher& _fetcher; - ExecutionState _state; size_t _returnNext; std::vector _rows; size_t _rowsPushed; diff --git a/arangod/Aql/CountCollectExecutor.cpp b/arangod/Aql/CountCollectExecutor.cpp index b66ce1e61662..232a98665964 100644 --- a/arangod/Aql/CountCollectExecutor.cpp +++ b/arangod/Aql/CountCollectExecutor.cpp @@ -103,14 +103,17 @@ auto CountCollectExecutor::skipRowsRange(AqlItemBlockInputRange& inputRange, Aql AqlCall{0, false, 0, AqlCall::LimitType::HARD}}; } -std::pair CountCollectExecutor::expectedNumberOfRows(size_t) const { - TRI_ASSERT(false); - return {ExecutionState::HASMORE, 1}; -} - auto CountCollectExecutor::expectedNumberOfRowsNew(AqlItemBlockInputRange const& input, AqlCall const& call) const noexcept -> size_t { + auto subqueries = input.countShadowRows(); + if (subqueries > 0) { + // We will return 1 row for every subquery execution. + // We do overallocate here if we have nested subqueries + return subqueries; + } + // No subqueries, we are at the end of the execution. + // We either return 1, or the callLimit. return std::min(1, call.getLimit()); } diff --git a/arangod/Aql/CountCollectExecutor.h b/arangod/Aql/CountCollectExecutor.h index 21e51a9e29f8..8ff1b1e3d89c 100644 --- a/arangod/Aql/CountCollectExecutor.h +++ b/arangod/Aql/CountCollectExecutor.h @@ -103,8 +103,6 @@ class CountCollectExecutor { [[nodiscard]] auto skipRowsRange(AqlItemBlockInputRange& inputRange, AqlCall& call) -> std::tuple; - std::pair expectedNumberOfRows(size_t atMost) const; - [[nodiscard]] auto expectedNumberOfRowsNew(AqlItemBlockInputRange const& input, AqlCall const& call) const noexcept -> size_t; diff --git a/arangod/Aql/DependencyProxy.cpp b/arangod/Aql/DependencyProxy.cpp index 4c7d28f05476..eb6a7700ff7e 100644 --- a/arangod/Aql/DependencyProxy.cpp +++ b/arangod/Aql/DependencyProxy.cpp @@ -221,147 +221,6 @@ DependencyProxy::fetchBlockForDependency(size_t dependency, si return {state, block}; } -template -std::pair DependencyProxy::skipSomeForDependency( - size_t const dependency, size_t const atMost) { - TRI_ASSERT(blockPassthrough == BlockPassthrough::Disable); - - TRI_ASSERT(_blockPassThroughQueue.empty()); - TRI_ASSERT(_blockQueue.empty()); - - TRI_ASSERT(atMost > 0); - TRI_ASSERT(_skipped <= atMost); - - ExecutionBlock& upstream = upstreamBlockForDependency(dependency); - - ExecutionState state = ExecutionState::HASMORE; - - // Temporary. - // Just do a copy of the stack here to not mess with it. - AqlCallStack stack = _injectedStack; - stack.pushCall(AqlCallList{AqlCall::SimulateSkipSome(atMost)}); - // Also temporary, will not be used here. - SharedAqlItemBlockPtr block; - - while (state == ExecutionState::HASMORE && _skipped < atMost) { - SkipResult skippedNow; - TRI_ASSERT(_skipped <= atMost); - { - // Make sure we call with the correct offset - // This is just a temporary dance until execute is implemented everywhere. - auto& tmpCall = stack.modifyTopCall(); - tmpCall.offset = atMost - _skipped; - } - std::tie(state, skippedNow, block) = upstream.execute(stack); - if (state == ExecutionState::WAITING) { - TRI_ASSERT(skippedNow.nothingSkipped()); - return {state, 0}; - } - // Temporary. - // If we return a block here it will be lost - TRI_ASSERT(block == nullptr); - - _skipped += skippedNow.getSkipCount(); - TRI_ASSERT(_skipped <= atMost); - } - TRI_ASSERT(state != ExecutionState::WAITING); - - size_t skipped = _skipped; - _skipped = 0; - TRI_ASSERT(skipped <= atMost); - return {state, skipped}; -} - -template -std::pair DependencyProxy::skipSome(size_t const toSkip) { - TRI_ASSERT(_blockPassThroughQueue.empty()); - TRI_ASSERT(_blockQueue.empty()); - - TRI_ASSERT(toSkip > 0); - TRI_ASSERT(_skipped <= toSkip); - ExecutionState state = ExecutionState::HASMORE; - - // Temporary. - // Just do a copy of the stack here to not mess with it. - AqlCallStack stack = _injectedStack; - stack.pushCall(AqlCallList{AqlCall::SimulateSkipSome(toSkip)}); - // Also temporary, will not be used here. - SharedAqlItemBlockPtr block; - - while (_skipped < toSkip) { - SkipResult skippedNow; - // Note: upstreamBlock will return next dependency - // if we need to loop here - TRI_ASSERT(_skipped <= toSkip); - { - // Make sure we call with the correct offset - // This is just a temporary dance until execute is implemented everywhere. - auto& tmpCall = stack.modifyTopCall(); - tmpCall.offset = toSkip - _skipped; - } - if (_distributeId.empty()) { - std::tie(state, skippedNow, block) = upstreamBlock().execute(stack); - } else { - auto upstreamWithClient = dynamic_cast(&upstreamBlock()); - std::tie(state, skippedNow, block) = - upstreamWithClient->executeForClient(stack, _distributeId); - } - - TRI_ASSERT(skippedNow.getSkipCount() <= toSkip - _skipped); - - if (state == ExecutionState::WAITING) { - TRI_ASSERT(skippedNow.nothingSkipped()); - return {state, 0}; - } - - // Temporary. - // If we return a block here it will be lost - TRI_ASSERT(block == nullptr); - - _skipped += skippedNow.getSkipCount(); - - // When the current dependency is done, advance. - if (state == ExecutionState::DONE) { - if (!advanceDependency()) { - break; - } else { - state = ExecutionState::HASMORE; - } - } - } - - size_t skipped = _skipped; - _skipped = 0; - - TRI_ASSERT(skipped <= toSkip); - return {state, skipped}; -} - -template -std::pair DependencyProxy::fetchBlockForPassthrough( - size_t atMost) { - TRI_ASSERT(blockPassthrough == BlockPassthrough::Enable); // TODO check this with enable_if in the header already - - if (_blockPassThroughQueue.empty()) { - ExecutionState state = prefetchBlock(atMost); - // prefetchBlock returns HASMORE iff it pushed a block onto _blockPassThroughQueue. - // If it didn't, it got either WAITING from upstream, or DONE + nullptr. - if (state == ExecutionState::WAITING || state == ExecutionState::DONE) { - return {state, nullptr}; - } - TRI_ASSERT(state == ExecutionState::HASMORE); - } - - TRI_ASSERT(!_blockPassThroughQueue.empty()); - - ExecutionState state; - SharedAqlItemBlockPtr block; - std::tie(state, block) = _blockPassThroughQueue.front(); - _blockPassThroughQueue.pop_front(); - - return {state, std::move(block)}; -} - template DependencyProxy::DependencyProxy( std::vector const& dependencies, AqlItemBlockManager& itemBlockManager, diff --git a/arangod/Aql/DependencyProxy.h b/arangod/Aql/DependencyProxy.h index d28dda04a312..dc87a880b901 100644 --- a/arangod/Aql/DependencyProxy.h +++ b/arangod/Aql/DependencyProxy.h @@ -93,15 +93,6 @@ class DependencyProxy { [[nodiscard]] TEST_VIRTUAL std::pair fetchBlockForDependency( size_t dependency, size_t atMost = ExecutionBlock::DefaultBatchSize); - // See comment on fetchBlockForDependency(). - [[nodiscard]] TEST_VIRTUAL std::pair skipSomeForDependency( - size_t dependency, size_t atMost); - - // TODO enable_if - [[nodiscard]] std::pair fetchBlockForPassthrough(size_t atMost); - - [[nodiscard]] TEST_VIRTUAL std::pair skipSome(size_t atMost); - [[nodiscard]] TEST_VIRTUAL RegisterId getNrInputRegisters() const; // Tries to fetch a block from upstream and push it, wrapped, onto diff --git a/arangod/Aql/DistinctCollectExecutor.cpp b/arangod/Aql/DistinctCollectExecutor.cpp index 30cffcd2e3f6..e5b15d74f663 100644 --- a/arangod/Aql/DistinctCollectExecutor.cpp +++ b/arangod/Aql/DistinctCollectExecutor.cpp @@ -64,9 +64,8 @@ transaction::Methods* DistinctCollectExecutorInfos::getTransaction() const { return _trxPtr; } -DistinctCollectExecutor::DistinctCollectExecutor(Fetcher& fetcher, Infos& infos) +DistinctCollectExecutor::DistinctCollectExecutor(Fetcher&, Infos& infos) : _infos(infos), - _fetcher(fetcher), _seen(1024, AqlValueGroupHash(_infos.getTransaction(), 1), AqlValueGroupEqual(_infos.getTransaction())) {} @@ -74,17 +73,6 @@ DistinctCollectExecutor::~DistinctCollectExecutor() { destroyValues(); } void DistinctCollectExecutor::initializeCursor() { destroyValues(); } -std::pair DistinctCollectExecutor::produceRows(OutputAqlItemRow& output) { - TRI_ASSERT(false); - THROW_ARANGO_EXCEPTION(TRI_ERROR_INTERNAL_AQL); -} - -std::pair DistinctCollectExecutor::expectedNumberOfRows(size_t atMost) const { - // This block cannot know how many elements will be returned exactly. - // but it is upper bounded by the input. - return _fetcher.preFetchNumberOfRows(atMost); -} - [[nodiscard]] auto DistinctCollectExecutor::expectedNumberOfRowsNew( AqlItemBlockInputRange const& input, AqlCall const& call) const noexcept -> size_t { if (input.finalState() == ExecutorState::DONE) { diff --git a/arangod/Aql/DistinctCollectExecutor.h b/arangod/Aql/DistinctCollectExecutor.h index 00197b40d4b6..ef657eaf6a26 100644 --- a/arangod/Aql/DistinctCollectExecutor.h +++ b/arangod/Aql/DistinctCollectExecutor.h @@ -100,13 +100,6 @@ class DistinctCollectExecutor { void initializeCursor(); - /** - * @brief produce the next Row of Aql Values. - * - * @return ExecutionState, and if successful exactly one new Row of AqlItems. - */ - std::pair produceRows(OutputAqlItemRow& output); - /** * @brief produce the next Rows of Aql Values. * @@ -118,8 +111,6 @@ class DistinctCollectExecutor { [[nodiscard]] auto skipRowsRange(AqlItemBlockInputRange& inputRange, AqlCall& call) -> std::tuple; - std::pair expectedNumberOfRows(size_t atMost) const; - [[nodiscard]] auto expectedNumberOfRowsNew(AqlItemBlockInputRange const& input, AqlCall const& call) const noexcept -> size_t; @@ -129,7 +120,6 @@ class DistinctCollectExecutor { private: Infos const& _infos; - Fetcher& _fetcher; std::unordered_set _seen; }; diff --git a/arangod/Aql/EnumerateCollectionExecutor.cpp b/arangod/Aql/EnumerateCollectionExecutor.cpp index 179f39b504b8..dfe26b980c4f 100644 --- a/arangod/Aql/EnumerateCollectionExecutor.cpp +++ b/arangod/Aql/EnumerateCollectionExecutor.cpp @@ -117,9 +117,8 @@ RegisterId EnumerateCollectionExecutorInfos::getOutputRegisterId() const { return _outputRegisterId; } -EnumerateCollectionExecutor::EnumerateCollectionExecutor(Fetcher& fetcher, Infos& infos) +EnumerateCollectionExecutor::EnumerateCollectionExecutor(Fetcher&, Infos& infos) : _infos(infos), - _fetcher(fetcher), _documentProducer(nullptr), _documentProducingFunctionContext(_currentRow, nullptr, _infos.getOutputRegisterId(), _infos.getProduceResult(), _infos.getQuery(), @@ -152,18 +151,6 @@ EnumerateCollectionExecutor::EnumerateCollectionExecutor(Fetcher& fetcher, Infos EnumerateCollectionExecutor::~EnumerateCollectionExecutor() = default; -std::pair EnumerateCollectionExecutor::produceRows( - OutputAqlItemRow& output) { - TRI_ASSERT(false); - THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); -} - -std::tuple EnumerateCollectionExecutor::skipRows( - size_t const toSkip) { - TRI_ASSERT(false); - THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); -} - uint64_t EnumerateCollectionExecutor::skipEntries(size_t toSkip, EnumerateCollectionStats& stats) { uint64_t actuallySkipped = 0; diff --git a/arangod/Aql/EnumerateCollectionExecutor.h b/arangod/Aql/EnumerateCollectionExecutor.h index fed25b83300c..a14ab3f5570a 100644 --- a/arangod/Aql/EnumerateCollectionExecutor.h +++ b/arangod/Aql/EnumerateCollectionExecutor.h @@ -125,15 +125,6 @@ class EnumerateCollectionExecutor { */ void initializeNewRow(AqlItemBlockInputRange& inputRange); - /** - * @brief produce the next Row of Aql Values. - * - * @return ExecutionState, and if successful exactly one new Row of AqlItems. - */ - - std::pair produceRows(OutputAqlItemRow& output); - std::tuple skipRows(size_t atMost); - /** * @brief produce the next Rows of Aql Values. * @@ -160,7 +151,6 @@ class EnumerateCollectionExecutor { private: Infos& _infos; - Fetcher& _fetcher; IndexIterator::DocumentCallback _documentProducer; IndexIterator::DocumentCallback _documentSkipper; DocumentProducingFunctionContext _documentProducingFunctionContext; diff --git a/arangod/Aql/EnumerateListExecutor.cpp b/arangod/Aql/EnumerateListExecutor.cpp index 03a5932f8d1a..555bef6db878 100644 --- a/arangod/Aql/EnumerateListExecutor.cpp +++ b/arangod/Aql/EnumerateListExecutor.cpp @@ -74,11 +74,6 @@ RegisterId EnumerateListExecutorInfos::getOutputRegister() const noexcept { EnumerateListExecutor::EnumerateListExecutor(Fetcher& fetcher, EnumerateListExecutorInfos& infos) : _infos(infos), _currentRow{CreateInvalidInputRowHint{}}, _inputArrayPosition(0), _inputArrayLength(0) {} -std::pair EnumerateListExecutor::produceRows(OutputAqlItemRow& output) { - TRI_ASSERT(false); - THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); -} - void EnumerateListExecutor::initializeNewRow(AqlItemBlockInputRange& inputRange) { if (_currentRow) { std::ignore = inputRange.nextDataRow(); diff --git a/arangod/Aql/EnumerateListExecutor.h b/arangod/Aql/EnumerateListExecutor.h index 70916e2a7464..8d08ac114205 100644 --- a/arangod/Aql/EnumerateListExecutor.h +++ b/arangod/Aql/EnumerateListExecutor.h @@ -90,13 +90,6 @@ class EnumerateListExecutor { EnumerateListExecutor(Fetcher&, EnumerateListExecutorInfos&); ~EnumerateListExecutor() = default; - /** - * @brief produce the next Row of Aql Values. - * - * @return ExecutionState, and if successful exactly one new Row of AqlItems. - */ - std::pair produceRows(OutputAqlItemRow& output); - /** * @brief Will fetch a new InputRow if necessary and store their local state * diff --git a/arangod/Aql/ExecutionBlock.cpp b/arangod/Aql/ExecutionBlock.cpp index 6458047c5958..a3b2bf4f1aac 100644 --- a/arangod/Aql/ExecutionBlock.cpp +++ b/arangod/Aql/ExecutionBlock.cpp @@ -146,128 +146,6 @@ std::pair ExecutionBlock::shutdown(int errorCode) { return {ExecutionState::DONE, _shutdownResult}; } -void ExecutionBlock::traceGetSomeBegin(size_t atMost) { - if (_profile >= PROFILE_LEVEL_BLOCKS) { - if (_getSomeBegin <= 0.0) { - _getSomeBegin = TRI_microtime(); - } - if (_profile >= PROFILE_LEVEL_TRACE_1) { - auto const node = getPlanNode(); - auto const queryId = this->_engine->getQuery()->id(); - LOG_TOPIC("ca7db", INFO, Logger::QUERIES) - << "[query#" << queryId << "] " - << "getSome type=" << node->getTypeString() << " atMost=" << atMost - << " this=" << (uintptr_t)this << " id=" << node->id(); - } - } -} - -std::pair ExecutionBlock::traceGetSomeEnd( - ExecutionState state, SharedAqlItemBlockPtr result) { - TRI_ASSERT(result != nullptr || state != ExecutionState::HASMORE); - if (_profile >= PROFILE_LEVEL_BLOCKS) { - ExecutionNode const* en = getPlanNode(); - ExecutionStats::Node stats; - auto const items = result != nullptr ? result->size() : 0; - stats.calls = 1; - stats.items = items; - if (state != ExecutionState::WAITING) { - stats.runtime = TRI_microtime() - _getSomeBegin; - _getSomeBegin = 0.0; - } - - auto it = _engine->_stats.nodes.find(en->id()); - if (it != _engine->_stats.nodes.end()) { - it->second += stats; - } else { - _engine->_stats.nodes.try_emplace(en->id(), stats); - } - - if (_profile >= PROFILE_LEVEL_TRACE_1) { - ExecutionNode const* node = getPlanNode(); - auto const queryId = this->_engine->getQuery()->id(); - LOG_TOPIC("07a60", INFO, Logger::QUERIES) - << "[query#" << queryId << "] " - << "getSome done type=" << node->getTypeString() - << " this=" << (uintptr_t)this << " id=" << node->id() - << " state=" << stateToString(state) << " items=" << items; - - if (_profile >= PROFILE_LEVEL_TRACE_2) { - if (result == nullptr) { - LOG_TOPIC("daa64", INFO, Logger::QUERIES) - << "[query#" << queryId << "] " - << "getSome type=" << node->getTypeString() << " result: nullptr"; - } else { - VPackBuilder builder; - auto const options = trxVpackOptions(); - result->toSimpleVPack(options, builder); - LOG_TOPIC("fcd9c", INFO, Logger::QUERIES) - << "[query#" << queryId << "] " - << "getSome type=" << node->getTypeString() - << " result: " << VPackDumper::toString(builder.slice(), options); - } - } - } - } - return {state, std::move(result)}; -} - -void ExecutionBlock::traceSkipSomeBegin(size_t atMost) { - if (_profile >= PROFILE_LEVEL_BLOCKS) { - if (_getSomeBegin <= 0.0) { - _getSomeBegin = TRI_microtime(); - } - if (_profile >= PROFILE_LEVEL_TRACE_1) { - auto node = getPlanNode(); - auto const queryId = this->_engine->getQuery()->id(); - LOG_TOPIC("dba8a", INFO, Logger::QUERIES) - << "[query#" << queryId << "] " - << "skipSome type=" << node->getTypeString() << " atMost=" << atMost - << " this=" << (uintptr_t)this << " id=" << node->id(); - } - } -} - -std::pair ExecutionBlock::traceSkipSomeEnd( - std::pair const res) { - ExecutionState const state = res.first; - size_t const skipped = res.second; - - if (_profile >= PROFILE_LEVEL_BLOCKS) { - ExecutionNode const* en = getPlanNode(); - ExecutionStats::Node stats; - stats.calls = 1; - stats.items = skipped; - if (state != ExecutionState::WAITING) { - stats.runtime = TRI_microtime() - _getSomeBegin; - _getSomeBegin = 0.0; - } - - auto it = _engine->_stats.nodes.find(en->id()); - if (it != _engine->_stats.nodes.end()) { - it->second += stats; - } else { - _engine->_stats.nodes.try_emplace(en->id(), stats); - } - - if (_profile >= PROFILE_LEVEL_TRACE_1) { - ExecutionNode const* node = getPlanNode(); - auto const queryId = this->_engine->getQuery()->id(); - LOG_TOPIC("d1950", INFO, Logger::QUERIES) - << "[query#" << queryId << "] " - << "skipSome done type=" << node->getTypeString() - << " this=" << (uintptr_t)this << " id=" << node->id() - << " state=" << stateToString(state) << " skipped=" << skipped; - } - } - return res; -} - -std::pair ExecutionBlock::traceSkipSomeEnd(ExecutionState state, - size_t skipped) { - return traceSkipSomeEnd({state, skipped}); -} - ExecutionState ExecutionBlock::getHasMoreState() { if (_done) { return ExecutionState::DONE; diff --git a/arangod/Aql/ExecutionBlock.h b/arangod/Aql/ExecutionBlock.h index c1544a123b45..3913af1eb115 100644 --- a/arangod/Aql/ExecutionBlock.h +++ b/arangod/Aql/ExecutionBlock.h @@ -71,9 +71,9 @@ class ExecutionBlock { /// @brief Number to use when we skip all. Should really be inf, but don't /// use something near std::numeric_limits::max() to avoid overflows /// in calculations. - /// This is used as an argument for skipSome(), e.g. when counting everything. + /// This is used as an argument for skipRowsRange(), e.g. when counting everything. /// Setting this to any other value >0 does not (and must not) affect the - /// results. It's only to reduce the number of necessary skipSome calls. + /// results. It's only to reduce the number of necessary skipRowsRange calls. [[nodiscard]] static constexpr inline size_t SkipAllSize() { return 1000000000; } @@ -96,36 +96,6 @@ class ExecutionBlock { /// @brief shutdown, will be called exactly once for the whole query [[nodiscard]] virtual std::pair shutdown(int errorCode); - /// @brief getSome, gets some more items, semantic is as follows: not - /// more than atMost items may be delivered. The method tries to - /// return a block of at most atMost items, however, it may return - /// less (for example if there are not enough items to come). However, - /// if it returns an actual block, it must contain at least one item. - /// getSome() also takes care of tracing and clearing registers; don't do it - /// in getOrSkipSome() implementations. - [[nodiscard]] virtual std::pair getSome(size_t atMost) = 0; - - // Trace the start of a getSome call - void traceGetSomeBegin(size_t atMost); - - // Trace the end of a getSome call, potentially with result - [[nodiscard]] std::pair traceGetSomeEnd( - ExecutionState state, SharedAqlItemBlockPtr result); - - void traceSkipSomeBegin(size_t atMost); - - [[nodiscard]] std::pair traceSkipSomeEnd(std::pair res); - - [[nodiscard]] std::pair traceSkipSomeEnd(ExecutionState state, - size_t skipped); - - /// @brief skipSome, skips some more items, semantic is as follows: not - /// more than atMost items may be skipped. The method tries to - /// skip a block of at most atMost items, however, it may skip - /// less (for example if there are not enough items to come). The number of - /// elements skipped is returned. - [[nodiscard]] virtual std::pair skipSome(size_t atMost) = 0; - [[nodiscard]] ExecutionState getHasMoreState(); // TODO: Can we get rid of this? Problem: Subquery Executor is using it. diff --git a/arangod/Aql/ExecutionBlockImpl.cpp b/arangod/Aql/ExecutionBlockImpl.cpp index b3867cd544d4..205f0130f887 100644 --- a/arangod/Aql/ExecutionBlockImpl.cpp +++ b/arangod/Aql/ExecutionBlockImpl.cpp @@ -110,8 +110,6 @@ using namespace arangodb::aql; } CREATE_HAS_MEMBER_CHECK(initializeCursor, hasInitializeCursor); -CREATE_HAS_MEMBER_CHECK(skipRows, hasSkipRows); -CREATE_HAS_MEMBER_CHECK(fetchBlockForPassthrough, hasFetchBlockForPassthrough); CREATE_HAS_MEMBER_CHECK(expectedNumberOfRows, hasExpectedNumberOfRows); CREATE_HAS_MEMBER_CHECK(skipRowsRange, hasSkipRowsRange); CREATE_HAS_MEMBER_CHECK(expectedNumberOfRowsNew, hasExpectedNumberOfRowsNew); @@ -223,154 +221,6 @@ ExecutionBlockImpl::ExecutionBlockImpl(ExecutionEngine* engine, template ExecutionBlockImpl::~ExecutionBlockImpl() = default; -template -std::pair ExecutionBlockImpl::getSome(size_t atMost) { - if constexpr (isNewStyleExecutor) { - AqlCallStack stack{AqlCallList{AqlCall::SimulateGetSome(atMost)}}; - auto const [state, skipped, block] = execute(stack); - return {state, block}; - } else { - traceGetSomeBegin(atMost); - auto result = getSomeWithoutTrace(atMost); - return traceGetSomeEnd(result.first, std::move(result.second)); - } -} - -template -std::pair ExecutionBlockImpl::getSomeWithoutTrace(size_t atMost) { - if constexpr (isNewStyleExecutor) { - TRI_ASSERT(false); - THROW_ARANGO_EXCEPTION(TRI_ERROR_INTERNAL_AQL); - } else { - TRI_ASSERT(atMost <= ExecutionBlock::DefaultBatchSize); - // silence tests -- we need to introduce new failure tests for fetchers - TRI_IF_FAILURE("ExecutionBlock::getOrSkipSome1") { - THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); - } - TRI_IF_FAILURE("ExecutionBlock::getOrSkipSome2") { - THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); - } - TRI_IF_FAILURE("ExecutionBlock::getOrSkipSome3") { - THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); - } - - if (getQuery().killed()) { - THROW_ARANGO_EXCEPTION(TRI_ERROR_QUERY_KILLED); - } - - if (_state == InternalState::DONE) { - // We are done, so we stay done - return {ExecutionState::DONE, nullptr}; - } - - if (!_outputItemRow) { - ExecutionState state; - SharedAqlItemBlockPtr newBlock; - std::tie(state, newBlock) = - requestWrappedBlock(atMost, _infos.numberOfOutputRegisters()); - if (state == ExecutionState::WAITING) { - TRI_ASSERT(newBlock == nullptr); - return {state, nullptr}; - } - if (newBlock == nullptr) { - TRI_ASSERT(state == ExecutionState::DONE); - _state = InternalState::DONE; - // _rowFetcher must be DONE now already - return {state, nullptr}; - } - TRI_ASSERT(newBlock != nullptr); - TRI_ASSERT(newBlock->size() > 0); - // We cannot hold this assertion, if we are on a pass-through - // block and the upstream uses execute already. - // TRI_ASSERT(newBlock->size() <= atMost); - _outputItemRow = createOutputRow(newBlock, AqlCall{}); - } - - ExecutionState state = ExecutionState::HASMORE; - ExecutorStats executorStats{}; - - TRI_ASSERT(atMost > 0); - - if (isInSplicedSubquery()) { - // The loop has to be entered at least once! - TRI_ASSERT(!_outputItemRow->isFull()); - while (!_outputItemRow->isFull() && _state != InternalState::DONE) { - // Assert that write-head is always pointing to a free row - TRI_ASSERT(!_outputItemRow->produced()); - switch (_state) { - case InternalState::FETCH_DATA: { - std::tie(state, executorStats) = _executor.produceRows(*_outputItemRow); - // Count global but executor-specific statistics, like number of - // filtered rows. - _engine->_stats += executorStats; - if (_outputItemRow->produced()) { - _outputItemRow->advanceRow(); - } - - if (state == ExecutionState::WAITING) { - return {state, nullptr}; - } - - if (state == ExecutionState::DONE) { - _state = InternalState::FETCH_SHADOWROWS; - } - break; - } - case InternalState::FETCH_SHADOWROWS: { - state = fetchShadowRowInternal(); - if (state == ExecutionState::WAITING) { - return {state, nullptr}; - } - break; - } - case InternalState::DONE: { - TRI_ASSERT(false); // Invalid state - } - } - } - // Modify the return state. - // As long as we do still have ShadowRows - // We need to return HASMORE! - if (_state == InternalState::DONE) { - state = ExecutionState::DONE; - } else { - state = ExecutionState::HASMORE; - } - } else { - // The loop has to be entered at least once! - TRI_ASSERT(!_outputItemRow->isFull()); - while (!_outputItemRow->isFull()) { - std::tie(state, executorStats) = _executor.produceRows(*_outputItemRow); - // Count global but executor-specific statistics, like number of filtered rows. - _engine->_stats += executorStats; - if (_outputItemRow->produced()) { - _outputItemRow->advanceRow(); - } - - if (state == ExecutionState::WAITING) { - return {state, nullptr}; - } - - if (state == ExecutionState::DONE) { - auto outputBlock = _outputItemRow->stealBlock(); - // This is not strictly necessary here, as we shouldn't be called again after DONE. - _outputItemRow.reset(); - return {state, std::move(outputBlock)}; - } - } - - TRI_ASSERT(state == ExecutionState::HASMORE); - TRI_ASSERT(_outputItemRow->isFull()); - } - - auto outputBlock = _outputItemRow->stealBlock(); - // we guarantee that we do return a valid pointer in the HASMORE case. - TRI_ASSERT(outputBlock != nullptr || _state == InternalState::DONE); - _outputItemRow.reset(); - return {state, std::move(outputBlock)}; - } -} - template std::unique_ptr ExecutionBlockImpl::createOutputRow( SharedAqlItemBlockPtr& newBlock, AqlCall&& call) { @@ -418,161 +268,6 @@ typename ExecutionBlockImpl::Infos const& ExecutionBlockImpl namespace arangodb::aql { -enum class SkipVariants { FETCHER, EXECUTOR, GET_SOME }; - -// Specifying the namespace here is important to MSVC. -template -struct ExecuteSkipVariant {}; - -template <> -struct ExecuteSkipVariant { - template - static std::tuple executeSkip( - Executor& executor, typename Executor::Fetcher& fetcher, size_t toSkip) { - auto res = fetcher.skipRows(toSkip); - return std::make_tuple(res.first, typename Executor::Stats{}, res.second); // tuple, cannot use initializer list due to build failure - } -}; - -template <> -struct ExecuteSkipVariant { - template - static std::tuple executeSkip( - Executor& executor, typename Executor::Fetcher& fetcher, size_t toSkip) { - return executor.skipRows(toSkip); - } -}; - -template <> -struct ExecuteSkipVariant { - template - static std::tuple executeSkip( - Executor& executor, typename Executor::Fetcher& fetcher, size_t toSkip) { - // this function should never be executed - TRI_ASSERT(false); - // Make MSVC happy: - return std::make_tuple(ExecutionState::DONE, typename Executor::Stats{}, 0); // tuple, cannot use initializer list due to build failure - } -}; - -template -static SkipVariants constexpr skipType() { - static_assert(!isNewStyleExecutor); - bool constexpr useFetcher = - Executor::Properties::allowsBlockPassthrough == BlockPassthrough::Enable && - !std::is_same>::value; - - bool constexpr useExecutor = hasSkipRows::value; - - // ConstFetcher and SingleRowFetcher can skip, but - // it may not be done for modification subqueries. - static_assert(useFetcher == - (std::is_same::value || - (std::is_same>::value && - !std::is_same>::value)), - "Unexpected fetcher for SkipVariants::FETCHER"); - - static_assert(!useFetcher || hasSkipRows::value, - "Fetcher is chosen for skipping, but has not skipRows method!"); - - static_assert( - useExecutor == - (std::is_same::value || - std::is_same>::value || - std::is_same>::value || - std::is_same>::value || - std::is_same>::value || - std::is_same>::value || - std::is_same>::value || - std::is_same>::value || - std::is_same>::value || - std::is_same>::value || - std::is_same>::value || - std::is_same>::value || - std::is_same>::value || - std::is_same>::value || - std::is_same>::value || - std::is_same>::value || - std::is_same>::value || - std::is_same>::value || - std::is_same>::value || - std::is_same>::value || - std::is_same>::value || - std::is_same::value || - std::is_same::value || - std::is_same::value || - std::is_same::value || - std::is_same::value || - std::is_same::value || - std::is_same>::value || - std::is_same>::value), - "Unexpected executor for SkipVariants::EXECUTOR"); - - // The LimitExecutor will not work correctly with SkipVariants::FETCHER! - static_assert( - !std::is_same::value || useFetcher, - "LimitExecutor needs to implement skipRows() to work correctly"); - - if (useExecutor) { - return SkipVariants::EXECUTOR; - } else if (useFetcher) { - return SkipVariants::FETCHER; - } else { - return SkipVariants::GET_SOME; - } -} - -} // namespace arangodb::aql - -template -std::pair ExecutionBlockImpl::skipSome(size_t const atMost) { - AqlCallStack stack{AqlCallList{AqlCall::SimulateSkipSome(atMost)}}; - auto const [state, skipped, block] = execute(stack); - - // execute returns ExecutionState::DONE here, which stops execution after simulating a skip. - // If we indiscriminately return ExecutionState::HASMORE, then we end up in an infinite loop - // - // luckily we can dispose of this kludge once executors have been ported. - if (skipped.getSkipCount() < atMost && state == ExecutionState::DONE) { - return {ExecutionState::DONE, skipped.getSkipCount()}; - } else { - return {ExecutionState::HASMORE, skipped.getSkipCount()}; - } -} - -template -std::pair ExecutionBlockImpl::skipSomeOnceWithoutTrace(size_t atMost) { - if constexpr (isNewStyleExecutor) { - TRI_ASSERT(false); - THROW_ARANGO_EXCEPTION(TRI_ERROR_INTERNAL_AQL); - } else { - constexpr SkipVariants customSkipType = skipType(); - - if constexpr (customSkipType == SkipVariants::GET_SOME) { - atMost = std::min(atMost, DefaultBatchSize); - auto res = getSomeWithoutTrace(atMost); - - size_t skipped = 0; - if (res.second != nullptr) { - skipped = res.second->size(); - } - TRI_ASSERT(skipped <= atMost); - - return {res.first, skipped}; - } - - ExecutionState state; - typename Executor::Stats stats; - size_t skipped; - std::tie(state, stats, skipped) = - ExecuteSkipVariant::executeSkip(_executor, _rowFetcher, atMost); - _engine->_stats += stats; - TRI_ASSERT(skipped <= atMost); - - return {state, skipped}; - } -} - template struct InitializeCursor {}; @@ -596,6 +291,7 @@ struct InitializeCursor { executor.initializeCursor(); } }; +} // namespace arangodb::aql template std::pair ExecutionBlockImpl::initializeCursor(InputAqlItemRow const& input) { @@ -640,69 +336,21 @@ std::pair ExecutionBlockImpl::shutdown(int err template std::tuple ExecutionBlockImpl::execute(AqlCallStack stack) { - // TODO remove this IF - // These are new style executors - if constexpr (isNewStyleExecutor) { - // Only this executor is fully implemented - traceExecuteBegin(stack); - // silence tests -- we need to introduce new failure tests for fetchers - TRI_IF_FAILURE("ExecutionBlock::getOrSkipSome1") { - THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); - } - TRI_IF_FAILURE("ExecutionBlock::getOrSkipSome2") { - THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); - } - TRI_IF_FAILURE("ExecutionBlock::getOrSkipSome3") { - THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); - } - initOnce(); - - auto res = executeWithoutTrace(stack); - return traceExecuteEnd(res); + // Only this executor is fully implemented + traceExecuteBegin(stack); + // silence tests -- we need to introduce new failure tests for fetchers + TRI_IF_FAILURE("ExecutionBlock::getOrSkipSome1") { + THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); } - - // Fall back to getSome/skipSome - auto myCallList = stack.popCall(); - auto myCall = myCallList.popNextCall(); - - TRI_ASSERT(AqlCall::IsSkipSomeCall(myCall) || AqlCall::IsGetSomeCall(myCall) || - AqlCall::IsFullCountCall(myCall) || AqlCall::IsFastForwardCall(myCall)); - _rowFetcher.useStack(stack); - - if (AqlCall::IsSkipSomeCall(myCall)) { - auto const [state, skipped] = skipSome(myCall.getOffset()); - if (state != ExecutionState::WAITING) { - myCall.didSkip(skipped); - } - SkipResult skipRes{}; - skipRes.didSkip(skipped); - return {state, skipRes, nullptr}; - } else if (AqlCall::IsGetSomeCall(myCall)) { - auto const [state, block] = getSome(myCall.getLimit()); - // We do not need to count as softLimit will be overwritten, and hard cannot be set. - if (stack.empty() && myCall.hasHardLimit() && !myCall.needsFullCount() && block != nullptr) { - // However we can do a short-cut here to report DONE on hardLimit if we are on the top-level query. - myCall.didProduce(block->size()); - if (myCall.getLimit() == 0) { - return {ExecutionState::DONE, SkipResult{}, block}; - } - } - - return {state, SkipResult{}, block}; - } else if (AqlCall::IsFullCountCall(myCall)) { - auto const [state, skipped] = skipSome(ExecutionBlock::SkipAllSize()); - if (state != ExecutionState::WAITING) { - myCall.didSkip(skipped); - } - SkipResult skipRes{}; - skipRes.didSkip(skipped); - return {state, skipRes, nullptr}; - } else if (AqlCall::IsFastForwardCall(myCall)) { - // No idea if DONE is correct here... - return {ExecutionState::DONE, SkipResult{}, nullptr}; + TRI_IF_FAILURE("ExecutionBlock::getOrSkipSome2") { + THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); } - // Should never get here! - THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); + TRI_IF_FAILURE("ExecutionBlock::getOrSkipSome3") { + THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); + } + initOnce(); + auto res = executeWithoutTrace(stack); + return traceExecuteEnd(res); } // Work around GCC bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=56480 @@ -853,185 +501,6 @@ ExecutionBlockImpl>>::shut } // namespace arangodb::aql -namespace arangodb::aql { - -// The constant "PASSTHROUGH" is somehow reserved with MSVC. -enum class RequestWrappedBlockVariant { - DEFAULT, - PASS_THROUGH, - INPUTRESTRICTED -}; - -// Specifying the namespace here is important to MSVC. -template -struct RequestWrappedBlock {}; - -template <> -struct RequestWrappedBlock { - /** - * @brief Default requestWrappedBlock() implementation. Just get a new block - * from the AqlItemBlockManager. - */ - template - static std::pair run( -#ifdef ARANGODB_ENABLE_MAINTAINER_MODE - typename Executor::Infos const&, -#endif - Executor& executor, ExecutionEngine& engine, size_t nrItems, RegisterCount nrRegs) { - return {ExecutionState::HASMORE, - engine.itemBlockManager().requestBlock(nrItems, nrRegs)}; - } -}; - -template <> -struct RequestWrappedBlock { - /** - * @brief If blocks can be passed through, we do not create new blocks. - * Instead, we take the input blocks and reuse them. - */ - template - static std::pair run( -#ifdef ARANGODB_ENABLE_MAINTAINER_MODE - typename Executor::Infos const& infos, -#endif - Executor& executor, ExecutionEngine& engine, size_t nrItems, RegisterCount nrRegs) { - static_assert(Executor::Properties::allowsBlockPassthrough == BlockPassthrough::Enable, - "This function can only be used with executors supporting " - "`allowsBlockPassthrough`"); - static_assert(hasFetchBlockForPassthrough::value, - "An Executor with allowsBlockPassthrough must implement " - "fetchBlockForPassthrough"); - - SharedAqlItemBlockPtr block; - - ExecutionState state; - typename Executor::Stats executorStats; - std::tie(state, executorStats, block) = executor.fetchBlockForPassthrough(nrItems); - engine._stats += executorStats; - - if (state == ExecutionState::WAITING) { - TRI_ASSERT(block == nullptr); - return {state, nullptr}; - } - if (block == nullptr) { - TRI_ASSERT(state == ExecutionState::DONE); - return {state, nullptr}; - } - - // Now we must have a block. - TRI_ASSERT(block != nullptr); - // Assert that the block has enough registers. This must be guaranteed by - // the register planning. - TRI_ASSERT(block->getNrRegs() == nrRegs); -#ifdef ARANGODB_ENABLE_MAINTAINER_MODE - // Check that all output registers are empty. - for (auto const& reg : *infos.getOutputRegisters()) { - for (size_t row = 0; row < block->size(); row++) { - AqlValue const& val = block->getValueReference(row, reg); - TRI_ASSERT(val.isEmpty()); - } - } -#endif - - return {ExecutionState::HASMORE, block}; - } -}; - -template <> -struct RequestWrappedBlock { - /** - * @brief If the executor can set an upper bound on the output size knowing - * the input size, usually because size(input) >= size(output), let it - * prefetch an input block to give us this upper bound. - * Only then we allocate a new block with at most this upper bound. - */ - template - static std::pair run( -#ifdef ARANGODB_ENABLE_MAINTAINER_MODE - typename Executor::Infos const&, -#endif - Executor& executor, ExecutionEngine& engine, size_t nrItems, RegisterCount nrRegs) { - static_assert(Executor::Properties::inputSizeRestrictsOutputSize, - "This function can only be used with executors supporting " - "`inputSizeRestrictsOutputSize`"); - static_assert(hasExpectedNumberOfRows::value, - "An Executor with inputSizeRestrictsOutputSize must " - "implement expectedNumberOfRows"); - - SharedAqlItemBlockPtr block; - - ExecutionState state; - size_t expectedRows = 0; - // Note: this might trigger a prefetch on the rowFetcher! - std::tie(state, expectedRows) = executor.expectedNumberOfRows(nrItems); - if (state == ExecutionState::WAITING) { - return {state, nullptr}; - } - nrItems = (std::min)(expectedRows, nrItems); - if (nrItems == 0) { - TRI_ASSERT(state == ExecutionState::DONE); - if (state != ExecutionState::DONE) { - auto const executorName = boost::core::demangle(typeid(Executor).name()); - THROW_ARANGO_EXCEPTION_FORMAT( - TRI_ERROR_INTERNAL_AQL, - "Unexpected result of expectedNumberOfRows in %s", executorName.c_str()); - } - return {state, nullptr}; - } - block = engine.itemBlockManager().requestBlock(nrItems, nrRegs); - - return {ExecutionState::HASMORE, block}; - } -}; - -} // namespace arangodb::aql - -template -std::pair ExecutionBlockImpl::requestWrappedBlock( - size_t nrItems, RegisterCount nrRegs) { - if constexpr (!isNewStyleExecutor) { - static_assert(Executor::Properties::allowsBlockPassthrough == BlockPassthrough::Disable || - !Executor::Properties::inputSizeRestrictsOutputSize, - "At most one of Properties::allowsBlockPassthrough or " - "Properties::inputSizeRestrictsOutputSize should be true for " - "each Executor"); - static_assert((Executor::Properties::allowsBlockPassthrough == BlockPassthrough::Enable) == - hasFetchBlockForPassthrough::value, - "Executors should implement the method " - "fetchBlockForPassthrough() iff " - "Properties::allowsBlockPassthrough is true"); - static_assert( - Executor::Properties::inputSizeRestrictsOutputSize == - hasExpectedNumberOfRows::value, - "Executors should implement the method expectedNumberOfRows() iff " - "Properties::inputSizeRestrictsOutputSize is true"); - } - - constexpr RequestWrappedBlockVariant variant = - isNewStyleExecutor - ? RequestWrappedBlockVariant::DEFAULT - : Executor::Properties::allowsBlockPassthrough == BlockPassthrough::Enable - ? RequestWrappedBlockVariant::PASS_THROUGH - : Executor::Properties::inputSizeRestrictsOutputSize - ? RequestWrappedBlockVariant::INPUTRESTRICTED - : RequestWrappedBlockVariant::DEFAULT; - - // Override for spliced subqueries, this optimization does not work there. - if (isInSplicedSubquery() && variant == RequestWrappedBlockVariant::INPUTRESTRICTED) { - return RequestWrappedBlock::run( -#ifdef ARANGODB_ENABLE_MAINTAINER_MODE - infos(), -#endif - executor(), *_engine, nrItems, nrRegs); - } - - return RequestWrappedBlock::run( -#ifdef ARANGODB_ENABLE_MAINTAINER_MODE - infos(), -#endif - executor(), *_engine, nrItems, nrRegs); -} - // TODO: We need to define the size of this block based on Input / Executor / Subquery depth template auto ExecutionBlockImpl::allocateOutputBlock(AqlCall&& call, DataRange const& inputRange) @@ -1078,12 +547,18 @@ auto ExecutionBlockImpl::allocateOutputBlock(AqlCall&& call, DataRange // that the upstream is no block using less than batchSize many rows, but returns HASMORE. if (inputRange.upstreamState() == ExecutorState::DONE || call.hasSoftLimit()) { blockSize = _executor.expectedNumberOfRowsNew(inputRange, call); + +#ifdef ARANGODB_ENABLE_MAINTAINER_MODE // The executor cannot expect to produce more then the limit! if constexpr (!std::is_same_v) { // Except the subqueryStartExecutor, it's limit differs // from it's output (it needs to count the new ShadowRows in addition) - TRI_ASSERT(blockSize <= call.getLimit()); + // This however is only correct, as long as we are in no subquery context + if (inputRange.countShadowRows() == 0) { + TRI_ASSERT(blockSize <= call.getLimit()); + } } +#endif blockSize += inputRange.countShadowRows(); // We have an upper bound by DefaultBatchSize; @@ -1180,9 +655,6 @@ static SkipRowsRangeVariant constexpr skipRowsType() { !std::is_same>::value)), "Unexpected fetcher for SkipVariants::FETCHER"); - static_assert(!useFetcher || hasSkipRows::value, - "Fetcher is chosen for skipping, but has not skipRows method!"); - static_assert( useExecutor == (is_one_of_v< diff --git a/arangod/Aql/ExecutionBlockImpl.h b/arangod/Aql/ExecutionBlockImpl.h index 9502e595bc1d..f22522da20db 100644 --- a/arangod/Aql/ExecutionBlockImpl.h +++ b/arangod/Aql/ExecutionBlockImpl.h @@ -176,54 +176,6 @@ class ExecutionBlockImpl final : public ExecutionBlock { /// better be called in instantiateFromPlan and similar methods. void init(); - /** - * @brief Produce atMost many output rows, or less. - * May return waiting if I/O has to be performed - * so we can release this thread. - * Is required to return DONE if it is guaranteed - * that this block does not produce more rows, - * Returns HASMORE if the DONE guarantee cannot be given. - * HASMORE does not give strict guarantee, it maybe that - * HASMORE is returned but no more rows can be produced. - * - * @param atMost Upper bound of AqlItemRows to be returned. - * Target is to get as close to this upper bound - * as possible. - * - * @return A pair with the following properties: - * ExecutionState: - * WAITING => IO going on, immediately return to caller. - * DONE => No more to expect from Upstream, if you are done with - * this row return DONE to caller. - * HASMORE => There is potentially more from above, call again if - * you need more input. - * AqlItemBlock: - * A matrix of result rows. - * Guaranteed to be non nullptr in the HASMORE case, maybe a nullptr - * in DONE. Is a nullptr in WAITING. - */ - [[nodiscard]] std::pair getSome(size_t atMost) override; - - /** - * @brief Like get some, but lines are skipped and not returned. - * This can use optimizations to not actually create the data. - * - * @param atMost Upper bound of AqlItemRows to be skipped. - * Target is to get as close to this upper bound - * as possible. - * - * @return A pair with the following properties: - * ExecutionState: - * WAITING => IO going on, immediatly return to caller. - * DONE => No more to expect from Upstream, if you are done with - * this row return DONE to caller. - * HASMORE => There is potentially more from above, call again if - * you need more input. size_t: Number of rows effectively - * skipped. On WAITING this is always 0. On any other state - * this is between 0 and atMost. - */ - [[nodiscard]] std::pair skipSome(size_t atMost) override; - [[nodiscard]] std::pair initializeCursor(InputAqlItemRow const& input) override; template >>> @@ -272,32 +224,6 @@ class ExecutionBlockImpl final : public ExecutionBlock { auto executeFastForward(typename Fetcher::DataRange& inputRange, AqlCall& clientCall) -> std::tuple; - /** - * @brief Inner getSome() part, without the tracing calls. - */ - [[nodiscard]] std::pair getSomeWithoutTrace(size_t atMost); - - /** - * @brief Inner skipSome() part, without the tracing calls. - */ - [[nodiscard]] std::pair skipSomeOnceWithoutTrace(size_t atMost); - - /** - * @brief Allocates a new AqlItemBlock and returns it, with the specified - * number of rows (nrItems) and columns (nrRegs). - * In case the Executor supports pass-through of blocks (i.e. reuse the - * input blocks as output blocks), it returns such an input block. In - * this case, the number of columns must still match - this has to be - * guaranteed by register planning. - * The state will be HASMORE if and only if it returns an actual block, - * which it always will in the non-pass-through case (modulo - * exceptions). If it is WAITING or DONE, the returned block is always - * a nullptr. This happens only if upstream is WAITING, or - * respectively, if it is DONE and did not return a new block. - */ - [[nodiscard]] std::pair requestWrappedBlock( - size_t nrItems, RegisterId nrRegs); - [[nodiscard]] std::unique_ptr createOutputRow(SharedAqlItemBlockPtr& newBlock, AqlCall&& call); diff --git a/arangod/Aql/ExecutionNode.cpp b/arangod/Aql/ExecutionNode.cpp index 3640e291f1b2..44189786c004 100644 --- a/arangod/Aql/ExecutionNode.cpp +++ b/arangod/Aql/ExecutionNode.cpp @@ -502,8 +502,9 @@ ExecutionNode* ExecutionNode::cloneHelper(std::unique_ptr other, return registeredNode; } -void ExecutionNode::cloneWithoutRegisteringAndDependencies(ExecutionPlan& plan, ExecutionNode& other, bool withProperties) const { - +void ExecutionNode::cloneWithoutRegisteringAndDependencies(ExecutionPlan& plan, + ExecutionNode& other, + bool withProperties) const { if (&plan == _plan) { // same execution plan for source and target // now assign a new id to the cloned node, otherwise it will fail diff --git a/arangod/Aql/FilterExecutor.cpp b/arangod/Aql/FilterExecutor.cpp index f151119acfde..4fcaad364267 100644 --- a/arangod/Aql/FilterExecutor.cpp +++ b/arangod/Aql/FilterExecutor.cpp @@ -55,51 +55,10 @@ RegisterId FilterExecutorInfos::getInputRegister() const noexcept { } FilterExecutor::FilterExecutor(Fetcher& fetcher, Infos& infos) - : _infos(infos), _fetcher(fetcher) {} + : _infos(infos) {} FilterExecutor::~FilterExecutor() = default; -std::pair FilterExecutor::produceRows(OutputAqlItemRow& output) { - TRI_IF_FAILURE("FilterExecutor::produceRows") { - THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG); - } - ExecutionState state; - FilterStats stats{}; - InputAqlItemRow input{CreateInvalidInputRowHint{}}; - - while (true) { - std::tie(state, input) = _fetcher.fetchRow(); - - if (state == ExecutionState::WAITING) { - return {state, stats}; - } - - if (!input) { - TRI_ASSERT(state == ExecutionState::DONE); - return {state, stats}; - } - TRI_ASSERT(input.isInitialized()); - - if (input.getValue(_infos.getInputRegister()).toBoolean()) { - output.copyRow(input); - return {state, stats}; - } else { - stats.incrFiltered(); - } - - if (state == ExecutionState::DONE) { - return {state, stats}; - } - TRI_ASSERT(state == ExecutionState::HASMORE); - } -} - -std::pair FilterExecutor::expectedNumberOfRows(size_t atMost) const { - // This block cannot know how many elements will be returned exactly. - // but it is upper bounded by the input. - return _fetcher.preFetchNumberOfRows(atMost); -} - auto FilterExecutor::skipRowsRange(AqlItemBlockInputRange& inputRange, AqlCall& call) -> std::tuple { FilterStats stats{}; diff --git a/arangod/Aql/FilterExecutor.h b/arangod/Aql/FilterExecutor.h index 8c443b69c5c3..5b7455e5186b 100644 --- a/arangod/Aql/FilterExecutor.h +++ b/arangod/Aql/FilterExecutor.h @@ -83,13 +83,6 @@ class FilterExecutor { FilterExecutor(Fetcher& fetcher, Infos&); ~FilterExecutor(); - /** - * @brief produce the next Row of Aql Values. - * - * @return ExecutionState, and if successful exactly one new Row of AqlItems. - */ - [[nodiscard]] std::pair produceRows(OutputAqlItemRow& output); - /** * @brief produce the next Rows of Aql Values. * @@ -106,14 +99,11 @@ class FilterExecutor { [[nodiscard]] std::tuple skipRowsRange( AqlItemBlockInputRange& inputRange, AqlCall& call); - [[nodiscard]] std::pair expectedNumberOfRows(size_t atMost) const; - [[nodiscard]] auto expectedNumberOfRowsNew(AqlItemBlockInputRange const& input, AqlCall const& call) const noexcept -> size_t; private: Infos& _infos; - Fetcher& _fetcher; }; } // namespace arangodb::aql diff --git a/arangod/Aql/HashedCollectExecutor.cpp b/arangod/Aql/HashedCollectExecutor.cpp index 4efd9f0f50f1..ef64bb96a211 100644 --- a/arangod/Aql/HashedCollectExecutor.cpp +++ b/arangod/Aql/HashedCollectExecutor.cpp @@ -203,11 +203,6 @@ void HashedCollectExecutor::writeCurrentGroupToOutput(OutputAqlItemRow& output) } } -std::pair HashedCollectExecutor::produceRows(OutputAqlItemRow& output) { - TRI_ASSERT(false); - THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); -} - auto HashedCollectExecutor::consumeInputRange(AqlItemBlockInputRange& inputRange) -> bool { TRI_ASSERT(!_isInitialized); do { @@ -355,11 +350,6 @@ decltype(HashedCollectExecutor::_allGroups)::iterator HashedCollectExecutor::fin return result; }; -std::pair HashedCollectExecutor::expectedNumberOfRows(size_t atMost) const { - TRI_ASSERT(false); - THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); -} - [[nodiscard]] auto HashedCollectExecutor::expectedNumberOfRowsNew( AqlItemBlockInputRange const& input, AqlCall const& call) const noexcept -> size_t { if (!_isInitialized) { diff --git a/arangod/Aql/HashedCollectExecutor.h b/arangod/Aql/HashedCollectExecutor.h index 49cd7a047910..2d219c415b72 100644 --- a/arangod/Aql/HashedCollectExecutor.h +++ b/arangod/Aql/HashedCollectExecutor.h @@ -136,14 +136,6 @@ class HashedCollectExecutor { HashedCollectExecutor(Fetcher& fetcher, Infos&); ~HashedCollectExecutor(); - /** - * @brief produce the next Row of Aql Values. - * - * @return ExecutionState, and if successful exactly one new Row of AqlItems. - * @deprecated - */ - std::pair produceRows(OutputAqlItemRow& output); - /** * @brief produce the next Row of Aql Values. * @@ -160,14 +152,6 @@ class HashedCollectExecutor { [[nodiscard]] auto skipRowsRange(AqlItemBlockInputRange& inputRange, AqlCall& call) -> std::tuple; - /** - * @brief This Executor does not know how many distinct rows will be fetched - * from upstream, it can only report how many it has found by itself, plus - * it knows that it can only create as many new rows as pulled from upstream. - * So it will overestimate. - */ - std::pair expectedNumberOfRows(size_t atMost) const; - /** * @brief This Executor does not know how many distinct rows will be fetched * from upstream, it can only report how many it has found by itself, plus diff --git a/arangod/Aql/IResearchViewExecutor.cpp b/arangod/Aql/IResearchViewExecutor.cpp index 7f1a63a144d6..be000a2f31bb 100644 --- a/arangod/Aql/IResearchViewExecutor.cpp +++ b/arangod/Aql/IResearchViewExecutor.cpp @@ -405,13 +405,6 @@ IResearchViewExecutorBase::IResearchViewExecutorBase( _filterCtx.emplace(_execCtx); } -template -std::pair::Stats> -IResearchViewExecutorBase::produceRows(OutputAqlItemRow& output) { - TRI_ASSERT(false); - THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); -} - template std::tuple::Stats, AqlCall> IResearchViewExecutorBase::produceRows(AqlItemBlockInputRange& inputRange, @@ -452,13 +445,6 @@ IResearchViewExecutorBase::produceRows(AqlItemBlockInputRange& inp return {inputRange.upstreamState(), stats, upstreamCall}; } -template -std::tuple::Stats, size_t> -IResearchViewExecutorBase::skipRows(size_t toSkip) { - TRI_ASSERT(false); - THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); -} - template std::tuple::Stats, size_t, AqlCall> IResearchViewExecutorBase::skipRowsRange(AqlItemBlockInputRange& inputRange, diff --git a/arangod/Aql/IResearchViewExecutor.h b/arangod/Aql/IResearchViewExecutor.h index 80daa1fe3b7f..86f9db433f7e 100644 --- a/arangod/Aql/IResearchViewExecutor.h +++ b/arangod/Aql/IResearchViewExecutor.h @@ -150,14 +150,6 @@ class IResearchViewExecutorBase { using Infos = IResearchViewExecutorInfos; using Stats = IResearchViewStats; - /** - * @brief produce the next Row of Aql Values. - * - * @return ExecutionState, and if successful exactly one new Row of AqlItems. - */ - std::tuple skipRows(size_t toSkip); - std::pair produceRows(OutputAqlItemRow& output); - /** * @brief produce the next Rows of Aql Values. * diff --git a/arangod/Aql/IdExecutor.cpp b/arangod/Aql/IdExecutor.cpp index 0fd9bd1aaafa..70953692b53a 100644 --- a/arangod/Aql/IdExecutor.cpp +++ b/arangod/Aql/IdExecutor.cpp @@ -124,14 +124,7 @@ auto IdExecutor::skipRowsRange(AqlItemBlockInputRange& inputRange, } call.didSkip(skipped); // TODO: Do we need to do counting here? - return {inputRange.upstreamState(), stats, skipped, call}; -} - -template -std::tuple::Stats, SharedAqlItemBlockPtr> -IdExecutor::fetchBlockForPassthrough(size_t atMost) { - TRI_ASSERT(false); - THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); + return {inputRange.upstreamState(), stats, call.getSkipCount(), call}; } template class ::arangodb::aql::IdExecutor; diff --git a/arangod/Aql/IdExecutor.h b/arangod/Aql/IdExecutor.h index 2b45ac5430e5..a72f1701a6ce 100644 --- a/arangod/Aql/IdExecutor.h +++ b/arangod/Aql/IdExecutor.h @@ -116,9 +116,6 @@ class IdExecutor { auto skipRowsRange(AqlItemBlockInputRange& inputRange, AqlCall& call) -> std::tuple; - // Deprecated remove me - std::tuple fetchBlockForPassthrough(size_t atMost); - private: Fetcher& _fetcher; Infos& _infos; diff --git a/arangod/Aql/IndexExecutor.cpp b/arangod/Aql/IndexExecutor.cpp index fcd1e2eb0b7b..90607b344d32 100644 --- a/arangod/Aql/IndexExecutor.cpp +++ b/arangod/Aql/IndexExecutor.cpp @@ -646,16 +646,6 @@ bool IndexExecutor::advanceCursor() { return false; } -std::pair IndexExecutor::produceRows(OutputAqlItemRow& output) { - TRI_ASSERT(false); - THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); -} - -std::tuple IndexExecutor::skipRows(size_t toSkip) { - TRI_ASSERT(false); - THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); -} - IndexExecutor::CursorReader& IndexExecutor::getCursor() { TRI_ASSERT(_currentIndex < _cursors.size()); return _cursors[_currentIndex]; diff --git a/arangod/Aql/IndexExecutor.h b/arangod/Aql/IndexExecutor.h index 0cb203fae272..a4b0f1761305 100644 --- a/arangod/Aql/IndexExecutor.h +++ b/arangod/Aql/IndexExecutor.h @@ -220,21 +220,12 @@ class IndexExecutor { IndexExecutor(Fetcher& fetcher, Infos&); ~IndexExecutor(); - /** - * @brief produce the next Row of Aql Values. - * - * @return ExecutionState, and if successful exactly one new Row of AqlItems. - */ - std::pair produceRows(OutputAqlItemRow& output); - std::tuple skipRows(size_t toSkip); - auto produceRows(AqlItemBlockInputRange& inputRange, OutputAqlItemRow& output) -> std::tuple; auto skipRowsRange(AqlItemBlockInputRange& inputRange, AqlCall& clientCall) -> std::tuple; - public: void initializeCursor(); private: diff --git a/arangod/Aql/KShortestPathsExecutor.cpp b/arangod/Aql/KShortestPathsExecutor.cpp index 051df62c8787..735c636e979d 100644 --- a/arangod/Aql/KShortestPathsExecutor.cpp +++ b/arangod/Aql/KShortestPathsExecutor.cpp @@ -158,12 +158,6 @@ auto KShortestPathsExecutor::shutdown(int errorCode) -> std::pair std::pair { - TRI_ASSERT(false); - THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); -} - auto KShortestPathsExecutor::produceRows(AqlItemBlockInputRange& input, OutputAqlItemRow& output) -> std::tuple { while (!output.isFull()) { diff --git a/arangod/Aql/KShortestPathsExecutor.h b/arangod/Aql/KShortestPathsExecutor.h index e06ee743d3ce..05eeae502225 100644 --- a/arangod/Aql/KShortestPathsExecutor.h +++ b/arangod/Aql/KShortestPathsExecutor.h @@ -166,8 +166,6 @@ class KShortestPathsExecutor { * * @return ExecutionState, and if successful exactly one new Row of AqlItems. */ - [[nodiscard]] auto produceRows(OutputAqlItemRow& output) - -> std::pair; [[nodiscard]] auto produceRows(AqlItemBlockInputRange& input, OutputAqlItemRow& output) -> std::tuple; [[nodiscard]] auto skipRowsRange(AqlItemBlockInputRange& input, AqlCall& call) diff --git a/arangod/Aql/MaterializeExecutor.cpp b/arangod/Aql/MaterializeExecutor.cpp index 895e35f6e81f..7d19562f84f9 100644 --- a/arangod/Aql/MaterializeExecutor.cpp +++ b/arangod/Aql/MaterializeExecutor.cpp @@ -72,12 +72,6 @@ arangodb::aql::MaterializerExecutorInfos::MaterializerExecutorInfos( _outMaterializedDocumentRegId(outDocRegId), _trx(trx) {} -template -std::pair arangodb::aql::MaterializeExecutor::produceRows(OutputAqlItemRow& output) { - TRI_ASSERT(false); - THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); -} - template std::tuple arangodb::aql::MaterializeExecutor::produceRows( AqlItemBlockInputRange& inputRange, OutputAqlItemRow& output) { @@ -133,12 +127,6 @@ std::tuple arangodb::aql::MaterializeEx return {inputRange.upstreamState(), NoStats{}, skipped, call}; } -template -std::tuple arangodb::aql::MaterializeExecutor::skipRows(size_t toSkipRequested) { - TRI_ASSERT(false); - THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); -} - template class ::arangodb::aql::MaterializeExecutor; template class ::arangodb::aql::MaterializeExecutor; diff --git a/arangod/Aql/MaterializeExecutor.h b/arangod/Aql/MaterializeExecutor.h index 32911e7eee66..d81f302877d1 100644 --- a/arangod/Aql/MaterializeExecutor.h +++ b/arangod/Aql/MaterializeExecutor.h @@ -108,11 +108,8 @@ class MaterializeExecutor { MaterializeExecutor(MaterializeExecutor&&) = default; MaterializeExecutor(MaterializeExecutor const&) = delete; - MaterializeExecutor(Fetcher& fetcher, Infos& infos) - : _readDocumentContext(infos), _infos(infos), _fetcher(fetcher) {} - - std::pair produceRows(OutputAqlItemRow& output); - std::tuple skipRows(size_t toSkipRequested); + MaterializeExecutor(Fetcher&, Infos& infos) + : _readDocumentContext(infos), _infos(infos) {} /** * @brief produce the next Row of Aql Values. @@ -151,7 +148,6 @@ class MaterializeExecutor { }; ReadContext _readDocumentContext; Infos const& _infos; - Fetcher& _fetcher; // for single collection case LogicalCollection const* _collection = nullptr; diff --git a/arangod/Aql/ModificationExecutor.cpp b/arangod/Aql/ModificationExecutor.cpp index 8591b32e7690..ad093fd744d2 100644 --- a/arangod/Aql/ModificationExecutor.cpp +++ b/arangod/Aql/ModificationExecutor.cpp @@ -150,14 +150,6 @@ void ModificationExecutor::doOutput(OutputAqlItemRow& } } -template -std::pair::Stats> -ModificationExecutor::produceRows(OutputAqlItemRow& output) { - TRI_ASSERT(false); - - return {ExecutionState::DONE, ModificationStats{}}; -} - template [[nodiscard]] auto ModificationExecutor::produceRows( typename FetcherType::DataRange& input, OutputAqlItemRow& output) diff --git a/arangod/Aql/ModificationExecutor.h b/arangod/Aql/ModificationExecutor.h index deec6db66e29..64b1dc620df2 100644 --- a/arangod/Aql/ModificationExecutor.h +++ b/arangod/Aql/ModificationExecutor.h @@ -171,8 +171,6 @@ class ModificationExecutor { ModificationExecutor(FetcherType&, Infos&); ~ModificationExecutor() = default; - std::pair produceRows(OutputAqlItemRow& output); - [[nodiscard]] auto produceRows(typename FetcherType::DataRange& input, OutputAqlItemRow& output) -> std::tuple; @@ -189,7 +187,6 @@ class ModificationExecutor { // WAITING ExecutionState _lastState; ModificationExecutorInfos& _infos; - FetcherType& _fetcher; ModifierType _modifier; }; diff --git a/arangod/Aql/MultiDependencySingleRowFetcher.cpp b/arangod/Aql/MultiDependencySingleRowFetcher.cpp index fba33607ad79..18f5a0926eb4 100644 --- a/arangod/Aql/MultiDependencySingleRowFetcher.cpp +++ b/arangod/Aql/MultiDependencySingleRowFetcher.cpp @@ -61,23 +61,6 @@ std::pair MultiDependencySingleRowFetcher return res; } -std::pair MultiDependencySingleRowFetcher::skipSomeForDependency( - size_t const dependency, size_t const atMost) { - TRI_ASSERT(!_dependencyInfos.empty()); - TRI_ASSERT(dependency < _dependencyInfos.size()); - auto& depInfo = _dependencyInfos[dependency]; - TRI_ASSERT(depInfo._upstreamState != ExecutionState::DONE); - - // There are still some blocks left that ask their parent even after they got - // DONE the last time, and I don't currently have time to track them down. - // Thus the following assert is commented out. - // TRI_ASSERT(_upstreamState != ExecutionState::DONE); - auto res = _dependencyProxy->skipSomeForDependency(dependency, atMost); - depInfo._upstreamState = res.first; - - return res; -} - std::pair MultiDependencySingleRowFetcher::fetchShadowRow(size_t const atMost) { // If any dependency is in an invalid state, but not done, refetch it for (size_t dependency = 0; dependency < _dependencyInfos.size(); ++dependency) { @@ -244,43 +227,6 @@ std::pair MultiDependencySingleRowFetcher::fetc return {rowState, row}; } -std::pair MultiDependencySingleRowFetcher::skipRowsForDependency( - size_t const dependency, size_t const atMost) { - TRI_ASSERT(dependency < _dependencyInfos.size()); - auto& depInfo = _dependencyInfos[dependency]; - - TRI_ASSERT((!indexIsValid(depInfo) || !isAtShadowRow(depInfo) || - ShadowAqlItemRow{depInfo._currentBlock, depInfo._rowIndex}.isRelevant())); - - size_t skip = 0; - while (indexIsValid(depInfo) && !isAtShadowRow(depInfo) && skip < atMost) { - ++skip; - ++depInfo._rowIndex; - } - - if (skip > 0) { - ExecutionState const state = - noMoreDataRows(depInfo) ? ExecutionState::DONE : ExecutionState::HASMORE; - return {state, skip}; - } - - TRI_ASSERT(!indexIsValid(depInfo) || isAtShadowRow(depInfo)); - TRI_ASSERT(skip == 0); - - if (noMoreDataRows(depInfo) || isDone(depInfo)) { - return {ExecutionState::DONE, 0}; - } - - TRI_ASSERT(!indexIsValid(depInfo)); - ExecutionState state; - size_t skipped; - std::tie(state, skipped) = skipSomeForDependency(dependency, atMost); - if (state == ExecutionState::HASMORE && skipped < atMost) { - state = ExecutionState::DONE; - } - return {state, skipped}; -} - bool MultiDependencySingleRowFetcher::indexIsValid( const MultiDependencySingleRowFetcher::DependencyInfo& info) const { return info._currentBlock != nullptr && info._rowIndex < info._currentBlock->size(); diff --git a/arangod/Aql/MultiDependencySingleRowFetcher.h b/arangod/Aql/MultiDependencySingleRowFetcher.h index 13358f5e95c4..9e7ce61c59bd 100644 --- a/arangod/Aql/MultiDependencySingleRowFetcher.h +++ b/arangod/Aql/MultiDependencySingleRowFetcher.h @@ -130,8 +130,6 @@ class MultiDependencySingleRowFetcher { TEST_VIRTUAL std::pair fetchRowForDependency( size_t dependency, size_t atMost = ExecutionBlock::DefaultBatchSize); - std::pair skipRowsForDependency(size_t dependency, size_t atMost); - std::pair fetchShadowRow(size_t atMost = ExecutionBlock::DefaultBatchSize); //@deprecated @@ -140,9 +138,6 @@ class MultiDependencySingleRowFetcher { [[nodiscard]] auto execute(AqlCallStack const&, AqlCallSet const&) -> std::tuple>>; - [[nodiscard]] auto executeForDependency(size_t dependency, AqlCallStack& stack) - -> std::tuple; - [[nodiscard]] auto upstreamState() const -> ExecutionState; private: @@ -162,6 +157,9 @@ class MultiDependencySingleRowFetcher { bool _didReturnSubquerySkips{false}; private: + [[nodiscard]] auto executeForDependency(size_t dependency, AqlCallStack& stack) + -> std::tuple; + /** * @brief Delegates to ExecutionBlock::fetchBlock() */ diff --git a/arangod/Aql/NoResultsExecutor.cpp b/arangod/Aql/NoResultsExecutor.cpp index a7620d3d843f..fbd947a087a0 100644 --- a/arangod/Aql/NoResultsExecutor.cpp +++ b/arangod/Aql/NoResultsExecutor.cpp @@ -49,4 +49,4 @@ auto NoResultsExecutor::skipRowsRange(AqlItemBlockInputRange& inputRange, AqlCal noexcept -> size_t { // Well nevermind the input, but we will always return 0 rows here. return 0; -} \ No newline at end of file +} diff --git a/arangod/Aql/NoResultsExecutor.h b/arangod/Aql/NoResultsExecutor.h index c12bd06842a2..b7726033dc1f 100644 --- a/arangod/Aql/NoResultsExecutor.h +++ b/arangod/Aql/NoResultsExecutor.h @@ -72,11 +72,6 @@ class NoResultsExecutor { [[nodiscard]] auto skipRowsRange(AqlItemBlockInputRange& inputRange, AqlCall& call) const noexcept -> std::tuple; - inline std::pair expectedNumberOfRows(size_t) const { - // Well nevermind the input, but we will always return 0 rows here. - return {ExecutionState::DONE, 0}; - } - [[nodiscard]] auto expectedNumberOfRowsNew(AqlItemBlockInputRange const& input, AqlCall const& call) const noexcept -> size_t; }; diff --git a/arangod/Aql/OptimizerRules.cpp b/arangod/Aql/OptimizerRules.cpp index 941e991e6fa2..0ded669c1f9c 100644 --- a/arangod/Aql/OptimizerRules.cpp +++ b/arangod/Aql/OptimizerRules.cpp @@ -2032,13 +2032,15 @@ class arangodb::aql::RedundantCalculationsReplacer final template void replaceStartTargetVariables(ExecutionNode* en) { - auto node = std::invoke([en](auto) { - if constexpr (std::is_base_of_v) { - return dynamic_cast(en); - } else { - return static_cast(en); - } - }, 0); + auto node = std::invoke( + [en](auto) { + if constexpr (std::is_base_of_v) { + return dynamic_cast(en); + } else { + return static_cast(en); + } + }, + 0); if (node->_inStartVariable != nullptr) { node->_inStartVariable = Variable::replace(node->_inStartVariable, _replacements); } @@ -5738,7 +5740,8 @@ void arangodb::aql::optimizeTraversalsRule(Optimizer* opt, // variables from them for (auto const& n : tNodes) { TraversalNode* traversal = ExecutionNode::castTo(n); - auto* options = static_cast(traversal->options()); + auto* options = + static_cast(traversal->options()); std::vector pruneVars; traversal->getPruneVariables(pruneVars); @@ -5780,18 +5783,15 @@ void arangodb::aql::optimizeTraversalsRule(Optimizer* opt, traversal->setPathOutput(nullptr); modified = true; } - + // check if we can make use of the optimized neighbors enumerator if (!ServerState::instance()->isCoordinator()) { - if (traversal->vertexOutVariable() != nullptr && - traversal->edgeOutVariable() == nullptr && - traversal->pathOutVariable() == nullptr && - options->useBreadthFirst && + if (traversal->vertexOutVariable() != nullptr && traversal->edgeOutVariable() == nullptr && + traversal->pathOutVariable() == nullptr && options->useBreadthFirst && options->uniqueVertices == arangodb::traverser::TraverserOptions::GLOBAL && - !options->usesPrune() && - !options->hasDepthLookupInfo()) { + !options->usesPrune() && !options->hasDepthLookupInfo()) { // this is possible in case *only* vertices are produced (no edges, no path), - // the traversal is breadth-first, the vertex uniqueness level is set to "global", + // the traversal is breadth-first, the vertex uniqueness level is set to "global", // there is no pruning and there are no depth-specific filters options->useNeighbors = true; modified = true; diff --git a/arangod/Aql/RemoteExecutor.cpp b/arangod/Aql/RemoteExecutor.cpp index 03783c0ce1c2..88788f1edca4 100644 --- a/arangod/Aql/RemoteExecutor.cpp +++ b/arangod/Aql/RemoteExecutor.cpp @@ -81,12 +81,6 @@ ExecutionBlockImpl::ExecutionBlockImpl( (!arangodb::ServerState::instance()->isCoordinator() && !ownName.empty())); } -std::pair ExecutionBlockImpl::getSome(size_t atMost) { - traceGetSomeBegin(atMost); - auto result = getSomeWithoutTrace(atMost); - return traceGetSomeEnd(result.first, std::move(result.second)); -} - std::pair ExecutionBlockImpl::getSomeWithoutTrace(size_t atMost) { // silence tests -- we need to introduce new failure tests for fetchers TRI_IF_FAILURE("ExecutionBlock::getOrSkipSome1") { @@ -163,12 +157,6 @@ std::pair ExecutionBlockImpl ExecutionBlockImpl::skipSome(size_t atMost) { - traceSkipSomeBegin(atMost); - auto result = skipSomeWithoutTrace(atMost); - return traceSkipSomeEnd(result.first, result.second); -} - std::pair ExecutionBlockImpl::skipSomeWithoutTrace(size_t atMost) { std::unique_lock guard(_communicationMutex); @@ -461,7 +449,7 @@ auto ExecutionBlockImpl::executeViaOldApi(AqlCallStack stack) return {state, SkipResult{}, block}; } else if (AqlCall::IsFullCountCall(myCall)) { - auto const [state, skipped] = skipSome(ExecutionBlock::SkipAllSize()); + auto const [state, skipped] = skipSomeWithoutTrace(ExecutionBlock::SkipAllSize()); if (state != ExecutionState::WAITING) { myCall.didSkip(skipped); } diff --git a/arangod/Aql/RemoteExecutor.h b/arangod/Aql/RemoteExecutor.h index cd8da9a7afbb..7a019c82061c 100644 --- a/arangod/Aql/RemoteExecutor.h +++ b/arangod/Aql/RemoteExecutor.h @@ -63,10 +63,6 @@ class ExecutionBlockImpl : public ExecutionBlock { ~ExecutionBlockImpl() override = default; - std::pair getSome(size_t atMost) override; - - std::pair skipSome(size_t atMost) override; - std::pair initializeCursor(InputAqlItemRow const& input) override; std::pair shutdown(int errorCode) override; diff --git a/arangod/Aql/ReturnExecutor.cpp b/arangod/Aql/ReturnExecutor.cpp index 7a5fc4344cde..2b3080ed51c7 100644 --- a/arangod/Aql/ReturnExecutor.cpp +++ b/arangod/Aql/ReturnExecutor.cpp @@ -49,22 +49,10 @@ ReturnExecutorInfos::ReturnExecutorInfos(RegisterId inputRegister, RegisterId nr } ReturnExecutor::ReturnExecutor(Fetcher& fetcher, ReturnExecutorInfos& infos) - : _infos(infos), _fetcher(fetcher) {} + : _infos(infos) {} ReturnExecutor::~ReturnExecutor() = default; -// TODO: @deprecated remove -std::pair ReturnExecutor::expectedNumberOfRows(size_t atMost) const { - return _fetcher.preFetchNumberOfRows(atMost); -} - -// TODO: @deprecated remove -auto ReturnExecutor::produceRows(OutputAqlItemRow& output) - -> std::pair { - TRI_ASSERT(false); - THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); -} - auto ReturnExecutor::skipRowsRange(AqlItemBlockInputRange& inputRange, AqlCall& call) -> std::tuple { TRI_IF_FAILURE("ReturnExecutor::produceRows") { diff --git a/arangod/Aql/ReturnExecutor.h b/arangod/Aql/ReturnExecutor.h index 69d03fa7accd..819500caef06 100644 --- a/arangod/Aql/ReturnExecutor.h +++ b/arangod/Aql/ReturnExecutor.h @@ -88,14 +88,6 @@ class ReturnExecutor { ReturnExecutor(Fetcher& fetcher, ReturnExecutorInfos&); ~ReturnExecutor(); - /** - * @brief produce the next Row of Aql Values. - * - * @return ExecutionState, - * if something was written output.hasValue() == true - */ - auto produceRows(OutputAqlItemRow& output) -> std::pair; - /** * @brief skip the next Rows of Aql Values. * @@ -112,15 +104,11 @@ class ReturnExecutor { [[nodiscard]] auto produceRows(AqlItemBlockInputRange& input, OutputAqlItemRow& output) -> std::tuple; - [[nodiscard]] auto expectedNumberOfRows(size_t atMost) const - -> std::pair; - [[nodiscard]] auto expectedNumberOfRowsNew(AqlItemBlockInputRange const& input, AqlCall const& call) const noexcept -> size_t; private: ReturnExecutorInfos& _infos; - Fetcher& _fetcher; }; } // namespace aql } // namespace arangodb diff --git a/arangod/Aql/ScatterExecutor.cpp b/arangod/Aql/ScatterExecutor.cpp index b81449906d9a..41835411f89f 100644 --- a/arangod/Aql/ScatterExecutor.cpp +++ b/arangod/Aql/ScatterExecutor.cpp @@ -42,7 +42,7 @@ ScatterExecutorInfos::ScatterExecutorInfos( ClientsExecutorInfos(std::move(clientIds)) {} ScatterExecutor::ClientBlockData::ClientBlockData(ExecutionEngine& engine, - ScatterNode const* node, + ExecutionNode const* node, ExecutorInfos const& scatterInfos) : _queue{}, _executor(nullptr), _executorHasMore{false} { // We only get shared ptrs to const data. so we need to copy here... @@ -86,14 +86,15 @@ auto ScatterExecutor::ClientBlockData::hasDataFor(AqlCall const& call) -> bool { auto ScatterExecutor::ClientBlockData::execute(AqlCallStack callStack, ExecutionState upstreamState) -> std::tuple { TRI_ASSERT(_executor != nullptr); + // Make sure we actually have data before you call execute TRI_ASSERT(hasDataFor(callStack.peek())); if (!_executorHasMore) { auto const& [block, skipResult] = _queue.front(); // This cast is guaranteed, we create this a couple lines above and only // this executor is used here. - // Unfortunately i did not get a version compiled were i could only forward - // declare the teplates in header. + // Unfortunately i did not get a version compiled were i could only + // forward declare the templates in header. auto casted = static_cast>*>(_executor.get()); TRI_ASSERT(casted != nullptr); @@ -102,7 +103,6 @@ auto ScatterExecutor::ClientBlockData::execute(AqlCallStack callStack, Execution _queue.pop_front(); } auto [state, skipped, result] = _executor->execute(callStack); - // We have all data locally cannot wait here. TRI_ASSERT(state != ExecutionState::WAITING); diff --git a/arangod/Aql/ScatterExecutor.h b/arangod/Aql/ScatterExecutor.h index 10a1316cec02..86b5c26171ef 100644 --- a/arangod/Aql/ScatterExecutor.h +++ b/arangod/Aql/ScatterExecutor.h @@ -53,7 +53,7 @@ class ScatterExecutor { class ClientBlockData { public: - ClientBlockData(ExecutionEngine& engine, ScatterNode const* node, + ClientBlockData(ExecutionEngine& engine, ExecutionNode const* node, ExecutorInfos const& scatterInfos); auto clear() -> void; diff --git a/arangod/Aql/ShortestPathExecutor.cpp b/arangod/Aql/ShortestPathExecutor.cpp index a263121ee432..16b2af7ce0b2 100644 --- a/arangod/Aql/ShortestPathExecutor.cpp +++ b/arangod/Aql/ShortestPathExecutor.cpp @@ -241,11 +241,6 @@ auto ShortestPathExecutor::pathLengthAvailable() -> size_t { return _path->length() - _posInPath; } -std::pair ShortestPathExecutor::produceRows(OutputAqlItemRow& output) { - TRI_ASSERT(false); - THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); -} - auto ShortestPathExecutor::produceRows(AqlItemBlockInputRange& input, OutputAqlItemRow& output) -> std::tuple { while (true) { diff --git a/arangod/Aql/ShortestPathExecutor.h b/arangod/Aql/ShortestPathExecutor.h index 280e4fba7a4f..98cd5c5cc8fc 100644 --- a/arangod/Aql/ShortestPathExecutor.h +++ b/arangod/Aql/ShortestPathExecutor.h @@ -182,8 +182,6 @@ class ShortestPathExecutor { /** * @brief produce the next Row of Aql Values. */ - [[nodiscard]] auto produceRows(OutputAqlItemRow& output) - -> std::pair; [[nodiscard]] auto produceRows(AqlItemBlockInputRange& input, OutputAqlItemRow& output) -> std::tuple; [[nodiscard]] auto skipRowsRange(AqlItemBlockInputRange& input, AqlCall& call) diff --git a/arangod/Aql/SingleRemoteModificationExecutor.cpp b/arangod/Aql/SingleRemoteModificationExecutor.cpp index 228742ca79a1..16bb9f760bf1 100644 --- a/arangod/Aql/SingleRemoteModificationExecutor.cpp +++ b/arangod/Aql/SingleRemoteModificationExecutor.cpp @@ -64,17 +64,10 @@ std::unique_ptr merge(VPackSlice document, std::string const& key, template SingleRemoteModificationExecutor::SingleRemoteModificationExecutor(Fetcher& fetcher, Infos& info) - : _info(info), _fetcher(fetcher), _upstreamState(ExecutionState::HASMORE) { + : _info(info), _upstreamState(ExecutionState::HASMORE) { TRI_ASSERT(arangodb::ServerState::instance()->isCoordinator()); }; -template -std::pair::Stats> -SingleRemoteModificationExecutor::produceRows(OutputAqlItemRow& output) { - TRI_ASSERT(false); - return {ExecutionState::DONE, Stats{}}; -} - template [[nodiscard]] auto SingleRemoteModificationExecutor::produceRows( AqlItemBlockInputRange& input, OutputAqlItemRow& output) diff --git a/arangod/Aql/SingleRemoteModificationExecutor.h b/arangod/Aql/SingleRemoteModificationExecutor.h index dfbc3c6c4a6d..2cca89725800 100644 --- a/arangod/Aql/SingleRemoteModificationExecutor.h +++ b/arangod/Aql/SingleRemoteModificationExecutor.h @@ -97,8 +97,6 @@ struct SingleRemoteModificationExecutor { * @return ExecutionState, * if something was written output.hasValue() == true */ - std::pair produceRows(OutputAqlItemRow& output); - [[nodiscard]] auto produceRows(AqlItemBlockInputRange& input, OutputAqlItemRow& output) -> std::tuple; [[nodiscard]] auto skipRowsRange(AqlItemBlockInputRange& input, AqlCall& call) @@ -110,7 +108,6 @@ struct SingleRemoteModificationExecutor { OperationResult&) -> void; Infos& _info; - Fetcher& _fetcher; ExecutionState _upstreamState; }; diff --git a/arangod/Aql/SingleRowFetcher.cpp b/arangod/Aql/SingleRowFetcher.cpp index 44acf7bdc00a..72be13e99d3a 100644 --- a/arangod/Aql/SingleRowFetcher.cpp +++ b/arangod/Aql/SingleRowFetcher.cpp @@ -68,15 +68,6 @@ SingleRowFetcher::SingleRowFetcher() _currentRow{CreateInvalidInputRowHint{}}, _currentShadowRow{CreateInvalidShadowRowHint{}} {} -template -#ifndef ARANGODB_USE_GOOGLE_TESTS -template -#endif -std::pair -SingleRowFetcher::fetchBlockForPassthrough(size_t atMost) { - return _dependencyProxy->fetchBlockForPassthrough(atMost); -} - template std::tuple SingleRowFetcher::execute(AqlCallStack& stack) { @@ -105,19 +96,6 @@ SingleRowFetcher::execute(AqlCallStack& stack) { AqlItemBlockInputRange{ExecutorState::DONE, skipped.getSkipCount(), block, start}}; } -template -std::pair SingleRowFetcher::skipRows(size_t atMost) { - TRI_ASSERT(!_currentRow.isInitialized() || _currentRow.isLastRowInBlock()); - TRI_ASSERT(!indexIsValid()); - - auto res = _dependencyProxy->skipSome(atMost); - _upstreamState = res.first; - - TRI_ASSERT(res.second <= atMost); - - return res; -} - template bool SingleRowFetcher::fetchBlockIfNecessary(size_t atMost) { // Fetch a new block iff necessary @@ -281,11 +259,6 @@ bool SingleRowFetcher::isAtShadowRow() const { } #endif -#ifndef ARANGODB_USE_GOOGLE_TESTS -template std::pair -SingleRowFetcher::fetchBlockForPassthrough(size_t atMost); -#endif - //@deprecated template auto SingleRowFetcher::useStack(AqlCallStack const& stack) -> void { diff --git a/arangod/Aql/SingleRowFetcher.h b/arangod/Aql/SingleRowFetcher.h index f33447576c01..31a4358b1060 100644 --- a/arangod/Aql/SingleRowFetcher.h +++ b/arangod/Aql/SingleRowFetcher.h @@ -118,18 +118,6 @@ class SingleRowFetcher { [[nodiscard]] TEST_VIRTUAL std::pair fetchShadowRow( size_t atMost = ExecutionBlock::DefaultBatchSize); - [[nodiscard]] TEST_VIRTUAL std::pair skipRows(size_t atMost); - - // template methods may not be virtual, thus this #ifdef. -#ifndef ARANGODB_USE_GOOGLE_TESTS - template > - [[nodiscard]] -#else - [[nodiscard]] TEST_VIRTUAL -#endif - std::pair - fetchBlockForPassthrough(size_t atMost); - [[nodiscard]] std::pair preFetchNumberOfRows(size_t atMost); void setDistributeId(std::string const& id); diff --git a/arangod/Aql/SortExecutor.cpp b/arangod/Aql/SortExecutor.cpp index 6cf17cffc829..cb6dca681958 100644 --- a/arangod/Aql/SortExecutor.cpp +++ b/arangod/Aql/SortExecutor.cpp @@ -113,19 +113,13 @@ AqlItemBlockManager& SortExecutorInfos::itemBlockManager() noexcept { return _manager; } -SortExecutor::SortExecutor(Fetcher& fetcher, SortExecutorInfos& infos) +SortExecutor::SortExecutor(Fetcher&, SortExecutorInfos& infos) : _infos(infos), - _fetcher(fetcher), _input(nullptr), _currentRow(CreateInvalidInputRowHint{}), _returnNext(0) {} SortExecutor::~SortExecutor() = default; -std::pair SortExecutor::produceRows(OutputAqlItemRow& output) { - TRI_ASSERT(false); - THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); -} - void SortExecutor::initializeInputMatrix(AqlItemBlockInputMatrix& inputMatrix) { TRI_ASSERT(_input == nullptr); ExecutorState state; @@ -230,17 +224,31 @@ std::tuple SortExecutor::skipRowsRange( return {ExecutorState::HASMORE, NoStats{}, call.getSkipCount(), upstreamCall}; } -std::pair SortExecutor::expectedNumberOfRows(size_t atMost) const { - if (_input == nullptr) { - // This executor does not know anything yet. - // Just take whatever is presented from upstream. - // This will return WAITING a couple of times - return _fetcher.preFetchNumberOfRows(atMost); +[[nodiscard]] auto SortExecutor::expectedNumberOfRowsNew(AqlItemBlockInputMatrix const& input, + AqlCall const& call) const + noexcept -> size_t { + size_t rowsAvailable = input.countDataRows(); + if (_input != nullptr) { + if (_returnNext < _sortedIndexes.size()) { + TRI_ASSERT(_returnNext <= rowsAvailable); + // if we have input, we are enumerating rows + // In a block within the given matrix. + // Unfortunately there could be more than + // one full block in the matrix and we do not know + // in which block we are. + // So if we are in the first block this will be accurate + rowsAvailable -= _returnNext; + // If we are in a later block, we will allocate space + // again for the first block. + // Nevertheless this is highly unlikely and + // only is bad if we sort few elements within highly nested + // subqueries. + } + // else we are in DONE state and not yet reset. + // We do not exactly now how many rows will be there } - TRI_ASSERT(_returnNext <= _sortedIndexes.size()); - size_t rowsLeft = _sortedIndexes.size() - _returnNext; - if (rowsLeft > 0) { - return {ExecutionState::HASMORE, rowsLeft}; + if (input.countShadowRows() == 0) { + return std::min(call.getLimit(), rowsAvailable); } - return {ExecutionState::DONE, rowsLeft}; + return rowsAvailable; } diff --git a/arangod/Aql/SortExecutor.h b/arangod/Aql/SortExecutor.h index 973cd20393a6..3714065c164c 100644 --- a/arangod/Aql/SortExecutor.h +++ b/arangod/Aql/SortExecutor.h @@ -96,18 +96,9 @@ class SortExecutor { using Infos = SortExecutorInfos; using Stats = NoStats; - SortExecutor(Fetcher& fetcher, Infos&); + SortExecutor(Fetcher&, Infos& infos); ~SortExecutor(); - /** - * @brief produce the next Row of Aql Values. - * - * @return ExecutionState, - * if something was written output.hasValue() == true - */ - std::pair produceRows(OutputAqlItemRow& output); - - std::pair expectedNumberOfRows(size_t) const; void initializeInputMatrix(AqlItemBlockInputMatrix& inputMatrix); /** @@ -126,14 +117,15 @@ class SortExecutor { [[nodiscard]] std::tuple skipRowsRange( AqlItemBlockInputMatrix& inputMatrix, AqlCall& call); + [[nodiscard]] auto expectedNumberOfRowsNew(AqlItemBlockInputMatrix const& input, + AqlCall const& call) const noexcept -> size_t; + private: void doSorting(); private: Infos& _infos; - Fetcher& _fetcher; - AqlItemMatrix const* _input; InputAqlItemRow _currentRow; diff --git a/arangod/Aql/SortedCollectExecutor.cpp b/arangod/Aql/SortedCollectExecutor.cpp index cc12029222a0..a0728cfdcba1 100644 --- a/arangod/Aql/SortedCollectExecutor.cpp +++ b/arangod/Aql/SortedCollectExecutor.cpp @@ -291,14 +291,6 @@ void SortedCollectExecutor::CollectGroup::writeToOutput(OutputAqlItemRow& output output.advanceRow(); } -std::pair SortedCollectExecutor::produceRows(OutputAqlItemRow& output) { - THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); -} - -std::pair SortedCollectExecutor::expectedNumberOfRows(size_t atMost) const { - THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); -} - [[nodiscard]] auto SortedCollectExecutor::expectedNumberOfRowsNew( AqlItemBlockInputRange const& input, AqlCall const& call) const noexcept -> size_t { if (input.finalState() == ExecutorState::DONE) { diff --git a/arangod/Aql/SortedCollectExecutor.h b/arangod/Aql/SortedCollectExecutor.h index 21c633176d55..9b5006235d0e 100644 --- a/arangod/Aql/SortedCollectExecutor.h +++ b/arangod/Aql/SortedCollectExecutor.h @@ -178,13 +178,6 @@ class SortedCollectExecutor { SortedCollectExecutor(SortedCollectExecutor const&) = delete; SortedCollectExecutor(Fetcher& fetcher, Infos&); - /** - * @brief produce the next Row of Aql Values. - * - * @return ExecutionState, and if successful exactly one new Row of AqlItems. - */ - auto produceRows(OutputAqlItemRow& output) -> std::pair; - /** * @brief produce the next Rows of Aql Values. * @@ -206,9 +199,6 @@ class SortedCollectExecutor { * it will produce exactly. It can however only * overestimate never underestimate. */ - [[nodiscard]] auto expectedNumberOfRows(size_t atMost) const - -> std::pair; - [[nodiscard]] auto expectedNumberOfRowsNew(AqlItemBlockInputRange const& input, AqlCall const& call) const noexcept -> size_t; diff --git a/arangod/Aql/SortingGatherExecutor.h b/arangod/Aql/SortingGatherExecutor.h index 777b5394a46d..46fed455a66f 100644 --- a/arangod/Aql/SortingGatherExecutor.h +++ b/arangod/Aql/SortingGatherExecutor.h @@ -129,8 +129,7 @@ class SortingGatherExecutor { * @return std::tuple * ExecutorState: DONE or HASMORE (only within a subquery) * Stats: Stats gerenated here - * AqlCall: Request to upstream - * size:t: Dependency to request + * AqlCallSet: Request to specific upstream dependency */ [[nodiscard]] auto produceRows(MultiAqlItemBlockInputRange& input, OutputAqlItemRow& output) -> std::tuple; @@ -140,12 +139,11 @@ class SortingGatherExecutor { * * @param input DataRange delivered by the fetcher * @param call skip request form consumer - * @return std::tuple + * @return std::tuple * ExecutorState: DONE or HASMORE (only within a subquery) * Stats: Stats gerenated here * size_t: Number of rows skipped - * AqlCall: Request to upstream - * size:t: Dependency to request + * AqlCallSet: Request to specific upstream dependency */ [[nodiscard]] auto skipRowsRange(MultiAqlItemBlockInputRange& input, AqlCall& call) -> std::tuple; diff --git a/arangod/Aql/SubqueryEndExecutor.cpp b/arangod/Aql/SubqueryEndExecutor.cpp index 6973ac216de4..828addf5c28e 100644 --- a/arangod/Aql/SubqueryEndExecutor.cpp +++ b/arangod/Aql/SubqueryEndExecutor.cpp @@ -76,16 +76,11 @@ bool SubqueryEndExecutorInfos::isModificationSubquery() const noexcept { return _isModificationSubquery; } -SubqueryEndExecutor::SubqueryEndExecutor(Fetcher& fetcher, SubqueryEndExecutorInfos& infos) - : _fetcher(fetcher), _infos(infos), _accumulator(_infos.vpackOptions()) {} +SubqueryEndExecutor::SubqueryEndExecutor(Fetcher&, SubqueryEndExecutorInfos& infos) + : _infos(infos), _accumulator(_infos.vpackOptions()) {} SubqueryEndExecutor::~SubqueryEndExecutor() = default; -std::pair SubqueryEndExecutor::produceRows(OutputAqlItemRow& output) { - TRI_ASSERT(false); - THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); -} - auto SubqueryEndExecutor::produceRows(AqlItemBlockInputRange& input, OutputAqlItemRow& output) -> std::tuple { // We can not account for skipped rows here. @@ -192,10 +187,10 @@ size_t SubqueryEndExecutor::Accumulator::numValues() const noexcept { return _numValues; } -// We write at most the number of rows that we get from above (potentially much less if we accumulate a lot) -std::pair SubqueryEndExecutor::expectedNumberOfRows(size_t atMost) const { - ExecutionState state{ExecutionState::HASMORE}; - size_t expected = 0; - std::tie(state, expected) = _fetcher.preFetchNumberOfRows(atMost); - return {state, expected}; +// We do not write any output for inbound dataRows +// We will only write output for shadowRows. This is accounted for in ExecutionBlockImpl +[[nodiscard]] auto SubqueryEndExecutor::expectedNumberOfRowsNew(AqlItemBlockInputRange const&, + AqlCall const&) const + noexcept -> size_t { + return 0; } diff --git a/arangod/Aql/SubqueryEndExecutor.h b/arangod/Aql/SubqueryEndExecutor.h index 19b684f03b9e..00d437c91599 100644 --- a/arangod/Aql/SubqueryEndExecutor.h +++ b/arangod/Aql/SubqueryEndExecutor.h @@ -85,9 +85,6 @@ class SubqueryEndExecutor { SubqueryEndExecutor(Fetcher& fetcher, SubqueryEndExecutorInfos& infos); ~SubqueryEndExecutor(); - [[deprecated]] std::pair produceRows(OutputAqlItemRow& output); - std::pair expectedNumberOfRows(size_t atMost) const; - // produceRows accumulates all input rows it can get into _accumulator, which // will then be read out by ExecutionBlockImpl // TODO: can the production of output be moved to produceRows again? @@ -100,6 +97,9 @@ class SubqueryEndExecutor { [[nodiscard]] auto skipRowsRange(AqlItemBlockInputRange& input, AqlCall& call) -> std::tuple; + [[nodiscard]] auto expectedNumberOfRowsNew(AqlItemBlockInputRange const& input, + AqlCall const& call) const noexcept -> size_t; + /** * @brief Consume the given shadow row and write the aggregated value to it * @@ -138,7 +138,6 @@ class SubqueryEndExecutor { }; private: - Fetcher& _fetcher; SubqueryEndExecutorInfos& _infos; Accumulator _accumulator; diff --git a/arangod/Aql/SubqueryExecutor.cpp b/arangod/Aql/SubqueryExecutor.cpp index fd07ce296c75..41b2c47fd606 100644 --- a/arangod/Aql/SubqueryExecutor.cpp +++ b/arangod/Aql/SubqueryExecutor.cpp @@ -68,19 +68,6 @@ SubqueryExecutor::SubqueryExecutor(Fetcher& fetcher, template SubqueryExecutor::~SubqueryExecutor() = default; -/** - * This follows the following state machine: - * If we have a subquery ongoing we need to ask it for hasMore, until it is DONE. - * In the case of DONE we write the result, and remove it from ongoing. - * If we do not have a subquery ongoing, we fetch a row and we start a new Subquery and ask it for hasMore. - */ - -template -std::pair SubqueryExecutor::produceRows(OutputAqlItemRow& output) { - TRI_ASSERT(false); - THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); -} - template auto SubqueryExecutor::initializeSubquery(AqlItemBlockInputRange& input) -> std::tuple { @@ -115,6 +102,12 @@ auto SubqueryExecutor::initializeSubquery(AqlItemBlockIn return {translatedReturnType(), true}; } +/** + * This follows the following state machine: + * If we have a subquery ongoing we need to ask it for hasMore, until it is DONE. + * In the case of DONE we write the result, and remove it from ongoing. + * If we do not have a subquery ongoing, we fetch a row and we start a new Subquery and ask it for hasMore. + */ template auto SubqueryExecutor::produceRows(AqlItemBlockInputRange& input, OutputAqlItemRow& output) @@ -253,13 +246,6 @@ std::pair SubqueryExecutor::shut return {state, _shutdownResult}; } -template -std::tuple::Stats, SharedAqlItemBlockPtr> -SubqueryExecutor::fetchBlockForPassthrough(size_t atMost) { - auto rv = _fetcher.fetchBlockForPassthrough(atMost); - return {rv.first, {}, std::move(rv.second)}; -} - template auto SubqueryExecutor::translatedReturnType() const noexcept -> ExecutionState { @@ -343,6 +329,7 @@ auto SubqueryExecutor::skipRowsRange<>(AqlItemBlockInputRange& inputRange, _skipped = 0; return {translatedReturnType(), NoStats{}, call.getSkipCount(), getUpstreamCall()}; } + template [[nodiscard]] auto SubqueryExecutor::expectedNumberOfRowsNew( AqlItemBlockInputRange const& input, AqlCall const& call) const noexcept -> size_t { diff --git a/arangod/Aql/SubqueryExecutor.h b/arangod/Aql/SubqueryExecutor.h index b9659b5a2178..6920343f4965 100644 --- a/arangod/Aql/SubqueryExecutor.h +++ b/arangod/Aql/SubqueryExecutor.h @@ -94,8 +94,6 @@ class SubqueryExecutor { * @return ExecutionState, * if something was written output.hasValue() == true */ - std::pair produceRows(OutputAqlItemRow& output); - [[nodiscard]] auto produceRows(AqlItemBlockInputRange& input, OutputAqlItemRow& output) -> std::tuple; @@ -105,8 +103,6 @@ class SubqueryExecutor { auto skipRowsRange(AqlItemBlockInputRange& inputRange, AqlCall& call) -> std::tuple; - std::tuple fetchBlockForPassthrough(size_t atMost); - private: /** * actually write the subquery output to the line diff --git a/arangod/Aql/SubqueryStartExecutor.cpp b/arangod/Aql/SubqueryStartExecutor.cpp index 0d1252870239..be69c7f6e07e 100644 --- a/arangod/Aql/SubqueryStartExecutor.cpp +++ b/arangod/Aql/SubqueryStartExecutor.cpp @@ -32,12 +32,7 @@ using namespace arangodb; using namespace arangodb::aql; -SubqueryStartExecutor::SubqueryStartExecutor(Fetcher& fetcher, Infos& infos) {} - -std::pair SubqueryStartExecutor::produceRows(OutputAqlItemRow& output) { - TRI_ASSERT(false); - THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); -} +SubqueryStartExecutor::SubqueryStartExecutor(Fetcher&, Infos&) {} auto SubqueryStartExecutor::produceRows(AqlItemBlockInputRange& input, OutputAqlItemRow& output) -> std::tuple { @@ -94,13 +89,6 @@ auto SubqueryStartExecutor::produceShadowRow(AqlItemBlockInputRange& input, return false; } -// TODO: remove me -auto SubqueryStartExecutor::expectedNumberOfRows(size_t atMost) const - -> std::pair { - TRI_ASSERT(false); - return {ExecutionState::DONE, 0}; -} - [[nodiscard]] auto SubqueryStartExecutor::expectedNumberOfRowsNew( AqlItemBlockInputRange const& input, AqlCall const& call) const noexcept -> size_t { // The DataRow is consumed after a shadowRow is produced. diff --git a/arangod/Aql/SubqueryStartExecutor.h b/arangod/Aql/SubqueryStartExecutor.h index 0182e3153514..e0e714f6dde0 100644 --- a/arangod/Aql/SubqueryStartExecutor.h +++ b/arangod/Aql/SubqueryStartExecutor.h @@ -50,11 +50,9 @@ class SubqueryStartExecutor { using Fetcher = SingleRowFetcher; using Infos = ExecutorInfos; using Stats = NoStats; - SubqueryStartExecutor(Fetcher& fetcher, Infos& infos); + SubqueryStartExecutor(Fetcher&, Infos& infos); ~SubqueryStartExecutor() = default; - [[deprecated]] std::pair produceRows(OutputAqlItemRow& output); - // produceRows for SubqueryStart reads a data row from its input and produces // a copy of that row and a shadow row. This requires some amount of internal // state as it can happen that after producing the copied data row the output @@ -71,8 +69,6 @@ class SubqueryStartExecutor { // previously auto produceShadowRow(AqlItemBlockInputRange& input, OutputAqlItemRow& output) -> bool; - std::pair expectedNumberOfRows(size_t atMost) const; - [[nodiscard]] auto expectedNumberOfRowsNew(AqlItemBlockInputRange const& input, AqlCall const& call) const noexcept -> size_t; diff --git a/arangod/Aql/TraversalExecutor.cpp b/arangod/Aql/TraversalExecutor.cpp index ae3948715130..617eeaa2d29f 100644 --- a/arangod/Aql/TraversalExecutor.cpp +++ b/arangod/Aql/TraversalExecutor.cpp @@ -146,11 +146,10 @@ std::vector> const& TraversalExecutorInfo TraversalExecutor::TraversalExecutor(Fetcher& fetcher, Infos& infos) : _infos(infos), _inputRow{CreateInvalidInputRowHint{}}, _traverser(infos.traverser()) { - // reset the traverser, so that no residual state is left in it. This is - // important because the TraversalExecutor is sometimes reconstructed (in place) - // with the same TraversalExecutorInfos as before. Those infos contain the traverser - // which might contain state from a previous run. + // important because the TraversalExecutor is sometimes reconstructed (in + // place) with the same TraversalExecutorInfos as before. Those infos contain + // the traverser which might contain state from a previous run. _traverser.done(); } diff --git a/arangod/Aql/UnsortedGatherExecutor.cpp b/arangod/Aql/UnsortedGatherExecutor.cpp index 041405187bf5..8777b450b1bc 100644 --- a/arangod/Aql/UnsortedGatherExecutor.cpp +++ b/arangod/Aql/UnsortedGatherExecutor.cpp @@ -40,14 +40,14 @@ struct Dependency { size_t _number; }; -UnsortedGatherExecutor::UnsortedGatherExecutor(Fetcher& fetcher, Infos& infos) - : _fetcher{fetcher} {} +UnsortedGatherExecutor::UnsortedGatherExecutor(Fetcher&, Infos&) {} UnsortedGatherExecutor::~UnsortedGatherExecutor() = default; auto UnsortedGatherExecutor::produceRows(typename Fetcher::DataRange& input, OutputAqlItemRow& output) -> std::tuple { + initialize(input); while (!output.isFull() && !done()) { if (input.hasDataRow(currentDependency())) { auto [state, inputRow] = input.nextDataRow(currentDependency()); @@ -92,6 +92,7 @@ auto UnsortedGatherExecutor::produceRows(typename Fetcher::DataRange& input, auto UnsortedGatherExecutor::skipRowsRange(typename Fetcher::DataRange& input, AqlCall& call) -> std::tuple { + initialize(input); auto skipped = size_t{0}; if (done()) { @@ -124,9 +125,15 @@ auto UnsortedGatherExecutor::skipRowsRange(typename Fetcher::DataRange& input, A return {ExecutorState::HASMORE, Stats{}, skipped, callSet}; } -auto UnsortedGatherExecutor::numDependencies() const - noexcept(noexcept(_fetcher.numberDependencies())) -> size_t { - return _fetcher.numberDependencies(); +auto UnsortedGatherExecutor::initialize(typename Fetcher::DataRange const& input) -> void { + // Dependencies can never change + TRI_ASSERT(_numDependencies == 0 || _numDependencies == input.numberDependencies()); + _numDependencies = input.numberDependencies(); +} + +auto UnsortedGatherExecutor::numDependencies() const noexcept -> size_t { + TRI_ASSERT(_numDependencies != 0); + return _numDependencies; } auto UnsortedGatherExecutor::done() const noexcept -> bool { diff --git a/arangod/Aql/UnsortedGatherExecutor.h b/arangod/Aql/UnsortedGatherExecutor.h index 86e28b70272c..bd908df8ae3f 100644 --- a/arangod/Aql/UnsortedGatherExecutor.h +++ b/arangod/Aql/UnsortedGatherExecutor.h @@ -101,15 +101,15 @@ class UnsortedGatherExecutor { -> std::tuple; private: - [[nodiscard]] auto numDependencies() const - noexcept(noexcept(static_cast(nullptr)->numberDependencies())) -> size_t; + auto initialize(typename Fetcher::DataRange const& input) -> void; + [[nodiscard]] auto numDependencies() const noexcept -> size_t; [[nodiscard]] auto done() const noexcept -> bool; [[nodiscard]] auto currentDependency() const noexcept -> size_t; auto advanceDependency() noexcept -> void; private: - Fetcher& _fetcher; size_t _currentDependency{0}; + size_t _numDependencies{0}; }; } // namespace arangodb::aql diff --git a/arangod/GeneralServer/SslServerFeature.cpp b/arangod/GeneralServer/SslServerFeature.cpp index 2a39c61e1d37..d5d337391029 100644 --- a/arangod/GeneralServer/SslServerFeature.cpp +++ b/arangod/GeneralServer/SslServerFeature.cpp @@ -119,7 +119,8 @@ void SslServerFeature::collectOptions(std::shared_ptr options) { new StringParameter(&_ecdhCurve)); options->addOption("--ssl.prefer-http1-in-alpn", - "Allows to let the server prefer HTTP/1.1 over HTTP/2 in ALPN protocol negotiations", + "Allows to let the server prefer HTTP/1.1 over HTTP/2 in " + "ALPN protocol negotiations", new BooleanParameter(&_preferHttp11InAlpn)); } @@ -206,14 +207,14 @@ class BIOGuard { }; } // namespace -static inline bool searchForProtocol(const unsigned char** out, unsigned char* outlen, - const unsigned char* in, unsigned int inlen, - const char* proto) { +static inline bool searchForProtocol(const unsigned char** out, + unsigned char* outlen, const unsigned char* in, + unsigned int inlen, const char* proto) { size_t len = strlen(proto); size_t i = 0; while (i + len <= inlen) { if (memcmp(in + i, proto, len) == 0) { - *out = (const unsigned char *)(in + i + 1); + *out = (const unsigned char*)(in + i + 1); *outlen = proto[0]; return true; } @@ -226,7 +227,7 @@ static int alpn_select_proto_cb(SSL* ssl, const unsigned char** out, unsigned char* outlen, const unsigned char* in, unsigned int inlen, void* arg) { int rv = 0; - bool const* preferHttp11InAlpn = (bool*) arg; + bool const* preferHttp11InAlpn = (bool*)arg; if (*preferHttp11InAlpn) { if (!searchForProtocol(out, outlen, in, inlen, "\x8http/1.1")) { if (!searchForProtocol(out, outlen, in, inlen, "\x2h2")) { @@ -234,7 +235,7 @@ static int alpn_select_proto_cb(SSL* ssl, const unsigned char** out, } } } else { - rv = nghttp2_select_next_protocol((unsigned char **)out, outlen, in, inlen); + rv = nghttp2_select_next_protocol((unsigned char**)out, outlen, in, inlen); } if (rv < 0) { @@ -375,7 +376,8 @@ asio_ns::ssl::context SslServerFeature::createSslContextInternal(std::string key sslContext.set_verify_mode(SSL_VERIFY_NONE); - SSL_CTX_set_alpn_select_cb(sslContext.native_handle(), alpn_select_proto_cb, (void*) (&_preferHttp11InAlpn)); + SSL_CTX_set_alpn_select_cb(sslContext.native_handle(), alpn_select_proto_cb, + (void*)(&_preferHttp11InAlpn)); return sslContext; } catch (std::exception const& ex) { diff --git a/tests/Aql/DependencyProxyMock.cpp b/tests/Aql/DependencyProxyMock.cpp index 05a4dc621b0c..874e98ebe6e0 100644 --- a/tests/Aql/DependencyProxyMock.cpp +++ b/tests/Aql/DependencyProxyMock.cpp @@ -149,39 +149,6 @@ size_t DependencyProxyMock::numFetchBlockCalls() const { return _numFetchBlockCalls; } -template -std::pair DependencyProxyMock::skipSome(size_t atMost) { - ExecutionState state; - SharedAqlItemBlockPtr block; - - std::tie(state, block) = _itemsToReturn.front(); - - if (block == nullptr) { - return {ExecutionState::DONE, 0}; - } - - size_t const firstShadowRow = [&]() { - size_t row; - for (row = 0; row < block->size(); ++row) { - if (block->isShadowRow(row)) break; - } - return row; - }(); - atMost = (std::min)(firstShadowRow, atMost); - - if (block->size() <= atMost) { - // Return the whole block - std::tie(state, block) = fetchBlock(atMost); - return {state, block->size()}; - } - TRI_ASSERT(block != nullptr); - TRI_ASSERT(block->size() > atMost); - SharedAqlItemBlockPtr rest = block->slice(atMost, block->size()); - _itemsToReturn.front().second = rest; - - return {ExecutionState::HASMORE, atMost}; -} - template MultiDependencyProxyMock::MultiDependencyProxyMock( arangodb::aql::ResourceMonitor& monitor, @@ -203,12 +170,6 @@ MultiDependencyProxyMock::fetchBlockForDependency(size_t depe return getDependencyMock(dependency).fetchBlock(atMost); } -template -std::pair MultiDependencyProxyMock::skipSomeForDependency( - size_t dependency, size_t atMost) { - return getDependencyMock(dependency).skipSome(atMost); -} - template bool MultiDependencyProxyMock::allBlocksFetched() const { for (auto& dep : _dependencyMocks) { diff --git a/tests/Aql/DependencyProxyMock.h b/tests/Aql/DependencyProxyMock.h index 5d873b84ea47..2a5cd9cf92f0 100644 --- a/tests/Aql/DependencyProxyMock.h +++ b/tests/Aql/DependencyProxyMock.h @@ -52,8 +52,6 @@ class DependencyProxyMock : public ::arangodb::aql::DependencyProxy skipSome(size_t atMost) override; - std::tuple execute( arangodb::aql::AqlCallStack& stack) override; @@ -106,9 +104,6 @@ class MultiDependencyProxyMock std::pair fetchBlockForDependency( size_t dependency, size_t atMost = arangodb::aql::ExecutionBlock::DefaultBatchSize) override; - std::pair skipSomeForDependency(size_t dependency, - size_t atMost) override; - inline size_t numberDependencies() const override { return _dependencyMocks.size(); } diff --git a/tests/Aql/DistinctCollectExecutorTest.cpp b/tests/Aql/DistinctCollectExecutorTest.cpp index cb89c2f844fc..af71b0f247b3 100644 --- a/tests/Aql/DistinctCollectExecutorTest.cpp +++ b/tests/Aql/DistinctCollectExecutorTest.cpp @@ -36,6 +36,7 @@ #include "Aql/Query.h" #include "Aql/SingleRowFetcher.h" #include "Aql/Stats.h" +#include "ExecutorTestHelper.h" #include "Mocks/Servers.h" #include "Transaction/Context.h" #include "Transaction/Methods.h" diff --git a/tests/Aql/EnumerateCollectionExecutorTest.cpp b/tests/Aql/EnumerateCollectionExecutorTest.cpp index 7f0074de3351..122f054d3a2d 100644 --- a/tests/Aql/EnumerateCollectionExecutorTest.cpp +++ b/tests/Aql/EnumerateCollectionExecutorTest.cpp @@ -344,7 +344,7 @@ TEST_P(EnumerateCollectionExecutorTestProduce, DISABLED_produce_all_documents) { uint64_t numberOfDocumentsToInsert = 10; std::vector queryResults; - // auto vpackOptions = insertDocuments(numberOfDocumentsToInsert, queryResults); + std::ignore = insertDocuments(numberOfDocumentsToInsert, queryResults); EXPECT_EQ(vocbase.lookupCollection("UnitTestCollection") ->numberDocuments(fakedQuery->trx(), transaction::CountType::Normal), numberOfDocumentsToInsert); // validate that our document inserts worked diff --git a/tests/Aql/ExecutionBlockImplTest.cpp b/tests/Aql/ExecutionBlockImplTest.cpp index de300b14f54c..44676224f896 100644 --- a/tests/Aql/ExecutionBlockImplTest.cpp +++ b/tests/Aql/ExecutionBlockImplTest.cpp @@ -31,7 +31,6 @@ #include "TestExecutorHelper.h" #include "TestLambdaExecutor.h" #include "WaitingExecutionBlockMock.h" -#include "fakeit.hpp" #include "Aql/AqlCallStack.h" #include "Aql/AqlItemBlock.h" @@ -58,349 +57,6 @@ using LambdaExe = TestLambdaSkipExecutor; // This test is supposed to only test getSome return values, // it is not supposed to test the fetch logic! -#if 0 -class ExecutionBlockImplTest : public ::testing::Test { - protected: - // ExecutionState state - SharedAqlItemBlockPtr result; - - // Mock of the ExecutionEngine - fakeit::Mock mockEngine; - ExecutionEngine& engine; - - // Mock of the AqlItemBlockManager - fakeit::Mock mockBlockManager; - AqlItemBlockManager& itemBlockManager; - - // Mock of the transaction - fakeit::Mock mockTrx; - transaction::Methods& trx; - - // Mock of the transaction context - fakeit::Mock mockContext; - transaction::Context& context; - - // Mock of the Query - fakeit::Mock mockQuery; - Query& query; - - ExecutionState state; - ResourceMonitor monitor; - - // Mock of the QueryOptions - fakeit::Mock mockQueryOptions; - QueryOptions& lqueryOptions; - ProfileLevel profile; - - // This is not used thus far in Base-Clase - ExecutionNode const* node = nullptr; - - // Executor Infos - TestExecutorHelperInfos infos; - TestEmptyExecutorHelperInfos emptyInfos; - - SharedAqlItemBlockPtr block; - - ExecutionBlockImplTest() - : engine(mockEngine.get()), - itemBlockManager(mockBlockManager.get()), - trx(mockTrx.get()), - context(mockContext.get()), - query(mockQuery.get()), - lqueryOptions(mockQueryOptions.get()), - profile(ProfileLevel(PROFILE_LEVEL_NONE)), - node(nullptr), - infos(0, 1, 1, {}, {0}), - emptyInfos(0, 1, 1, {}, {0}), - block(nullptr) { - fakeit::When(Method(mockBlockManager, requestBlock)).AlwaysDo([&](size_t nrItems, RegisterId nrRegs) -> SharedAqlItemBlockPtr { - return SharedAqlItemBlockPtr{new AqlItemBlock(itemBlockManager, nrItems, nrRegs)}; - }); - - fakeit::When(Method(mockEngine, itemBlockManager)).AlwaysReturn(itemBlockManager); - fakeit::When(Method(mockEngine, getQuery)).AlwaysReturn(&query); - fakeit::When(OverloadedMethod(mockBlockManager, returnBlock, void(AqlItemBlock*&))) - .AlwaysDo([&](AqlItemBlock*& block) -> void { - AqlItemBlockManager::deleteBlock(block); - block = nullptr; - }); - fakeit::When(Method(mockBlockManager, resourceMonitor)).AlwaysReturn(&monitor); - fakeit::When(ConstOverloadedMethod(mockQuery, queryOptions, QueryOptions const&())) - .AlwaysDo([&]() -> QueryOptions const& { return lqueryOptions; }); - fakeit::When(OverloadedMethod(mockQuery, queryOptions, QueryOptions & ())) - .AlwaysDo([&]() -> QueryOptions& { return lqueryOptions; }); - fakeit::When(Method(mockQuery, trx)).AlwaysReturn(&trx); - - fakeit::When(Method(mockQueryOptions, getProfileLevel)).AlwaysReturn(profile); - - fakeit::When(Method(mockTrx, transactionContextPtr)).AlwaysReturn(&context); - fakeit::When(Method(mockContext, getVPackOptions)).AlwaysReturn(&velocypack::Options::Defaults); - } -}; - -TEST_F(ExecutionBlockImplTest, - there_is_a_block_in_the_upstream_with_no_rows_inside_the_executor_waits_using_getsome) { - std::deque blockDeque; - SharedAqlItemBlockPtr block = buildBlock<1>(itemBlockManager, {{42}}); - blockDeque.push_back(std::move(block)); - - WaitingExecutionBlockMock dependency{&engine, node, std::move(blockDeque)}; - - ExecutionBlockImpl testee(&engine, node, std::move(infos)); - testee.addDependency(&dependency); - - size_t atMost = 1000; - std::tie(state, block) = testee.getSome(atMost); - ASSERT_EQ(state, ExecutionState::WAITING); - std::tie(state, block) = testee.getSome(atMost); - ASSERT_NE(block, nullptr); - ASSERT_EQ(block->size(), 1); - ASSERT_EQ(state, ExecutionState::DONE); - - // done should stay done! - std::tie(state, block) = testee.getSome(atMost); - ASSERT_EQ(block, nullptr); - ASSERT_EQ(state, ExecutionState::DONE); -} - -TEST_F(ExecutionBlockImplTest, - there_is_a_block_in_the_upstream_with_no_rows_inside_the_executor_waits_using_skipsome) { - std::deque blockDeque; - SharedAqlItemBlockPtr block = buildBlock<1>(itemBlockManager, {{42}}); - blockDeque.push_back(std::move(block)); - - WaitingExecutionBlockMock dependency{&engine, node, std::move(blockDeque)}; - - ExecutionBlockImpl testee(&engine, node, std::move(infos)); - testee.addDependency(&dependency); - - size_t atMost = 1; - size_t skipped = 0; - - std::tie(state, skipped) = testee.skipSome(atMost); - ASSERT_EQ(state, ExecutionState::WAITING); - ASSERT_EQ(skipped, 0); - - std::tie(state, skipped) = testee.skipSome(atMost); - ASSERT_EQ(state, ExecutionState::DONE); - ASSERT_EQ(skipped, 1); - - // done should stay done! - std::tie(state, skipped) = testee.skipSome(atMost); - ASSERT_EQ(state, ExecutionState::DONE); - ASSERT_EQ(skipped, 0); -} - -TEST_F(ExecutionBlockImplTest, - there_are_multiple_blocks_in_the_upstream_with_no_rows_inside_the_executor_waits_using_getsome_one_block) { - // we are checking multiple input blocks - // we are only fetching 1 row each (atMost = 1) - // after a DONE is returned, it must stay done! - - std::deque blockDeque; - SharedAqlItemBlockPtr blocka = buildBlock<1>(itemBlockManager, {{42}}); - SharedAqlItemBlockPtr blockb = buildBlock<1>(itemBlockManager, {{42}}); - SharedAqlItemBlockPtr blockc = buildBlock<1>(itemBlockManager, {{42}}); - SharedAqlItemBlockPtr blockd = buildBlock<1>(itemBlockManager, {{42}}); - SharedAqlItemBlockPtr blocke = buildBlock<1>(itemBlockManager, {{42}}); - blockDeque.push_back(std::move(blocka)); - blockDeque.push_back(std::move(blockb)); - blockDeque.push_back(std::move(blockc)); - blockDeque.push_back(std::move(blockd)); - blockDeque.push_back(std::move(blocke)); - - WaitingExecutionBlockMock dependency{&engine, node, std::move(blockDeque)}; - - ExecutionBlockImpl testee(&engine, node, std::move(infos)); - testee.addDependency(&dependency); - size_t atMost = 1; - size_t total = 0; - - std::tie(state, block) = testee.getSome(atMost); - ASSERT_EQ(state, ExecutionState::WAITING); - - std::tie(state, block) = testee.getSome(atMost); - ASSERT_EQ(state, ExecutionState::HASMORE); - total = total + block->size(); - - std::tie(state, block) = testee.getSome(atMost); - ASSERT_EQ(state, ExecutionState::WAITING); - - std::tie(state, block) = testee.getSome(atMost); - ASSERT_EQ(state, ExecutionState::HASMORE); - total = total + block->size(); - - std::tie(state, block) = testee.getSome(atMost); - ASSERT_EQ(state, ExecutionState::WAITING); - - std::tie(state, block) = testee.getSome(atMost); - ASSERT_EQ(state, ExecutionState::HASMORE); - total = total + block->size(); - - std::tie(state, block) = testee.getSome(atMost); - ASSERT_EQ(state, ExecutionState::WAITING); - - std::tie(state, block) = testee.getSome(atMost); - ASSERT_EQ(state, ExecutionState::HASMORE); - total = total + block->size(); - - std::tie(state, block) = testee.getSome(atMost); - ASSERT_EQ(state, ExecutionState::WAITING); - - std::tie(state, block) = testee.getSome(atMost); - ASSERT_EQ(state, ExecutionState::DONE); - total = total + block->size(); - - std::tie(state, block) = testee.getSome(atMost); - ASSERT_EQ(state, ExecutionState::DONE); - - ASSERT_EQ(total, 5); -} - -TEST_F(ExecutionBlockImplTest, - there_are_multiple_blocks_in_the_upstream_with_no_rows_inside_the_executor_waits_using_getsome_multiple_blocks) { - // as test above, BUT with a higher atMost value. - - std::deque blockDeque; - SharedAqlItemBlockPtr blocka = buildBlock<1>(itemBlockManager, {{42}}); - SharedAqlItemBlockPtr blockb = buildBlock<1>(itemBlockManager, {{42}}); - SharedAqlItemBlockPtr blockc = buildBlock<1>(itemBlockManager, {{42}}); - SharedAqlItemBlockPtr blockd = buildBlock<1>(itemBlockManager, {{42}}); - SharedAqlItemBlockPtr blocke = buildBlock<1>(itemBlockManager, {{42}}); - blockDeque.push_back(std::move(blocka)); - blockDeque.push_back(std::move(blockb)); - blockDeque.push_back(std::move(blockc)); - blockDeque.push_back(std::move(blockd)); - blockDeque.push_back(std::move(blocke)); - - WaitingExecutionBlockMock dependency{&engine, node, std::move(blockDeque)}; - - ExecutionBlockImpl testee(&engine, node, std::move(infos)); - testee.addDependency(&dependency); - size_t atMost = 2; - size_t total = 0; - - std::tie(state, block) = testee.getSome(atMost); - ASSERT_EQ(state, ExecutionState::WAITING); - - std::tie(state, block) = testee.getSome(atMost); - ASSERT_EQ(state, ExecutionState::WAITING); - - std::tie(state, block) = testee.getSome(atMost); - ASSERT_EQ(state, ExecutionState::HASMORE); - total = total + block->size(); - - std::tie(state, block) = testee.getSome(atMost); - ASSERT_EQ(state, ExecutionState::WAITING); - - std::tie(state, block) = testee.getSome(atMost); - ASSERT_EQ(state, ExecutionState::WAITING); - - std::tie(state, block) = testee.getSome(atMost); - ASSERT_EQ(state, ExecutionState::HASMORE); - total = total + block->size(); - - std::tie(state, block) = testee.getSome(atMost); - ASSERT_EQ(state, ExecutionState::WAITING); - - std::tie(state, block) = testee.getSome(atMost); - ASSERT_EQ(state, ExecutionState::DONE); - total = total + block->size(); - - std::tie(state, block) = testee.getSome(atMost); - ASSERT_EQ(state, ExecutionState::DONE); - ASSERT_EQ(block, nullptr); - - ASSERT_EQ(total, 5); -} - -TEST_F(ExecutionBlockImplTest, - there_are_multiple_blocks_in_the_upstream_with_no_rows_inside_the_executor_waits_using_skipsome) { - // we are checking multiple input blocks - // we are only fetching 1 row each (atMost = 1) - // after a DONE is returned, it must stay done! - - std::deque blockDeque; - SharedAqlItemBlockPtr blocka = buildBlock<1>(itemBlockManager, {{42}}); - SharedAqlItemBlockPtr blockb = buildBlock<1>(itemBlockManager, {{42}}); - SharedAqlItemBlockPtr blockc = buildBlock<1>(itemBlockManager, {{42}}); - SharedAqlItemBlockPtr blockd = buildBlock<1>(itemBlockManager, {{42}}); - SharedAqlItemBlockPtr blocke = buildBlock<1>(itemBlockManager, {{42}}); - blockDeque.push_back(std::move(blocka)); - blockDeque.push_back(std::move(blockb)); - blockDeque.push_back(std::move(blockc)); - blockDeque.push_back(std::move(blockd)); - blockDeque.push_back(std::move(blocke)); - - WaitingExecutionBlockMock dependency{&engine, node, std::move(blockDeque)}; - - ExecutionBlockImpl testee(&engine, node, std::move(infos)); - testee.addDependency(&dependency); - size_t atMost = 1; - size_t skipped = 0; - - std::tie(state, skipped) = testee.skipSome(atMost); - ASSERT_EQ(state, ExecutionState::WAITING); - ASSERT_EQ(skipped, 0); - - std::tie(state, skipped) = testee.skipSome(atMost); - ASSERT_EQ(state, ExecutionState::HASMORE); - ASSERT_EQ(skipped, 1); - - std::tie(state, skipped) = testee.skipSome(atMost); - ASSERT_EQ(state, ExecutionState::WAITING); - ASSERT_EQ(skipped, 0); - - std::tie(state, skipped) = testee.skipSome(atMost); - ASSERT_EQ(state, ExecutionState::HASMORE); - ASSERT_EQ(skipped, 1); - - std::tie(state, skipped) = testee.skipSome(atMost); - ASSERT_EQ(state, ExecutionState::WAITING); - ASSERT_EQ(skipped, 0); - - std::tie(state, skipped) = testee.skipSome(atMost); - ASSERT_EQ(state, ExecutionState::HASMORE); - ASSERT_EQ(skipped, 1); - - std::tie(state, skipped) = testee.skipSome(atMost); - ASSERT_EQ(state, ExecutionState::WAITING); - ASSERT_EQ(skipped, 0); - - std::tie(state, skipped) = testee.skipSome(atMost); - ASSERT_EQ(state, ExecutionState::HASMORE); - ASSERT_EQ(skipped, 1); - - std::tie(state, skipped) = testee.skipSome(atMost); - ASSERT_EQ(state, ExecutionState::WAITING); - ASSERT_EQ(skipped, 0); - - std::tie(state, skipped) = testee.skipSome(atMost); - ASSERT_EQ(state, ExecutionState::DONE); - ASSERT_EQ(skipped, 1); - - std::tie(state, skipped) = testee.skipSome(atMost); - ASSERT_EQ(state, ExecutionState::DONE); - ASSERT_EQ(skipped, 0); -} - -TEST_F(ExecutionBlockImplTest, - there_is_an_invalid_empty_block_in_the_upstream_the_executor_waits_using_getsome) { - std::deque blockDeque; - SharedAqlItemBlockPtr block = buildBlock<1>(itemBlockManager, {{42}}); - blockDeque.push_back(std::move(block)); - - WaitingExecutionBlockMock dependency{&engine, node, std::move(blockDeque)}; - - ExecutionBlockImpl testee(&engine, node, std::move(emptyInfos)); - testee.addDependency(&dependency); - - size_t atMost = 1000; - std::tie(state, block) = testee.getSome(atMost); - ASSERT_EQ(state, ExecutionState::DONE); - ASSERT_EQ(block, nullptr); -} -#endif /** * @brief Shared Test case initializer to test the execute API @@ -2033,6 +1689,7 @@ TEST_P(ExecutionBlockImplExecuteIntegrationTest, test_multiple_upstream_calls_pa ++it; } for (size_t i = 0; i < limit && it.valid(); ++i) { + call.skippedRows = 0; auto stack = buildStack(call); auto [state, skipped, block] = testee->execute(stack); if (doesWaiting()) { @@ -2459,14 +2116,14 @@ TEST_P(ExecutionBlockImplExecuteIntegrationTest, empty_subquery) { AssertValueEquals(block, row, outReg, 4); if (skip) { // wo do not have empty input, we can skip - EXPECT_EQ(skipAsserter.getNumberCalls(), 1); + EXPECT_EQ(skipAsserter.getNumberCalls(), 2); // we need to call getSome never EXPECT_EQ(getAsserter.getNumberCalls(), 0); } else { // we do not skip EXPECT_EQ(skipAsserter.getNumberCalls(), 0); // wo do not have empty input, we can produce - EXPECT_EQ(getAsserter.getNumberCalls(), 1); + EXPECT_EQ(getAsserter.getNumberCalls(), 2); } getAsserter.reset(); skipAsserter.reset(); @@ -2487,15 +2144,15 @@ TEST_P(ExecutionBlockImplExecuteIntegrationTest, empty_subquery) { AssertIsShadowRowOfDepth(block, row, 1); AssertValueEquals(block, row, outReg, 6); if (skip) { - // wo do not have empty input, we can skip - EXPECT_EQ(skipAsserter.getNumberCalls(), 1); + // wo do have empty input, we can skip + EXPECT_EQ(skipAsserter.getNumberCalls(), 2); // we need to call getSome never EXPECT_EQ(getAsserter.getNumberCalls(), 0); } else { // we do not skip EXPECT_EQ(skipAsserter.getNumberCalls(), 0); - // wo do not have empty input, we can produce - EXPECT_EQ(getAsserter.getNumberCalls(), 1); + // wo do have empty input, we can produce + EXPECT_EQ(getAsserter.getNumberCalls(), 2); } getAsserter.reset(); @@ -2572,7 +2229,6 @@ INSTANTIATE_TEST_CASE_P( skipAndHardLimit(), skipAndHardLimitAndFullCount(), onlyFullCount(), onlySkipAndCount()), ::testing::Bool())); - } // namespace aql } // namespace tests } // namespace arangodb diff --git a/tests/Aql/ExecutionBlockImplTestInstances.cpp b/tests/Aql/ExecutionBlockImplTestInstances.cpp index a7af31f4c8f6..8e2496e598cf 100644 --- a/tests/Aql/ExecutionBlockImplTestInstances.cpp +++ b/tests/Aql/ExecutionBlockImplTestInstances.cpp @@ -3,7 +3,6 @@ #include "TestExecutorHelper.h" #include "TestLambdaExecutor.h" -// template class ::arangodb::aql::ExecutionBlockImpl; -// template class ::arangodb::aql::ExecutionBlockImpl; +template class ::arangodb::aql::ExecutionBlockImpl; template class ::arangodb::aql::ExecutionBlockImpl; template class ::arangodb::aql::ExecutionBlockImpl; diff --git a/tests/Aql/ExecutorTestHelper.cpp b/tests/Aql/ExecutorTestHelper.cpp index 1ce468851eab..3f3f82f2919e 100644 --- a/tests/Aql/ExecutorTestHelper.cpp +++ b/tests/Aql/ExecutorTestHelper.cpp @@ -169,23 +169,3 @@ auto asserthelper::ValidateBlocksAreEqualUnordered( } } } - -std::ostream& arangodb::tests::aql::operator<<(std::ostream& stream, - arangodb::tests::aql::ExecutorCall call) { - return stream << [call]() { - switch (call) { - case ExecutorCall::SKIP_ROWS: - return "SKIP_ROWS"; - case ExecutorCall::PRODUCE_ROWS: - return "PRODUCE_ROWS"; - case ExecutorCall::FETCH_FOR_PASSTHROUGH: - return "FETCH_FOR_PASSTHROUGH"; - case ExecutorCall::EXPECTED_NR_ROWS: - return "EXPECTED_NR_ROWS"; - } - // The control flow cannot reach this. It is only here to make MSVC happy, - // which is unable to figure out that the switch above is complete. - TRI_ASSERT(false); - THROW_ARANGO_EXCEPTION(TRI_ERROR_INTERNAL_AQL); - }(); -} diff --git a/tests/Aql/ExecutorTestHelper.h b/tests/Aql/ExecutorTestHelper.h index 3596877162f2..298b421a3189 100644 --- a/tests/Aql/ExecutorTestHelper.h +++ b/tests/Aql/ExecutorTestHelper.h @@ -440,136 +440,6 @@ struct ExecutorTestHelper { std::vector> _execNodes; }; -enum class ExecutorCall { - SKIP_ROWS, - PRODUCE_ROWS, - FETCH_FOR_PASSTHROUGH, - EXPECTED_NR_ROWS, -}; - -std::ostream& operator<<(std::ostream& stream, ExecutorCall call); - -using ExecutorStepResult = std::tuple; - -// TODO Add skipRows by passing 3 additional integers i, j, k, saying we should -// - skip i rows -// - produce j rows -// - skip k rows -// TODO Make the calls to skipRows, fetchBlockForPassthrough and (later) expectedNumberOfRows -// somehow optional. e.g. call a templated function or so. -// TODO Add calls to expectedNumberOfRows - -template -std::tuple, arangodb::aql::ExecutionStats> -runExecutor(arangodb::aql::AqlItemBlockManager& manager, Executor& executor, - arangodb::aql::OutputAqlItemRow& outputRow, size_t const numSkip, - size_t const numProduce, bool const skipRest) { - using namespace arangodb::aql; - ExecutionState state = ExecutionState::HASMORE; - std::vector results{}; - ExecutionStats stats{}; - - uint64_t rowsLeft = 0; - size_t skippedTotal = 0; - size_t producedTotal = 0; - - enum class RunState { - SKIP_OFFSET, - FETCH_FOR_PASSTHROUGH, - PRODUCE, - SKIP_REST, - BREAK - }; - - while (state != ExecutionState::DONE) { - RunState const runState = [&]() { - if (skippedTotal < numSkip) { - return RunState::SKIP_OFFSET; - } - if (rowsLeft == 0 && (producedTotal < numProduce || numProduce == 0)) { - return RunState::FETCH_FOR_PASSTHROUGH; - } - if (producedTotal < numProduce || !skipRest) { - return RunState::PRODUCE; - } - if (skipRest) { - return RunState::SKIP_REST; - } - return RunState::BREAK; - }(); - - switch (runState) { - // Skip first - // TODO don't do this for executors which don't have skipRows - case RunState::SKIP_OFFSET: { - size_t skipped; - typename Executor::Stats executorStats{}; - std::tie(state, executorStats, skipped) = executor.skipRows(numSkip); - results.emplace_back(std::make_tuple(ExecutorCall::SKIP_ROWS, state, skipped)); - stats += executorStats; - skippedTotal += skipped; - } break; - // Get a new block for pass-through if we still need to produce rows and - // the current (imagined, via rowsLeft) block is "empty". - // TODO: Don't do this at all for non-passThrough blocks - case RunState::FETCH_FOR_PASSTHROUGH: { - ExecutionState fetchBlockState; - typename Executor::Stats executorStats{}; - SharedAqlItemBlockPtr block{}; - std::tie(fetchBlockState, executorStats, block) = - executor.fetchBlockForPassthrough(1000); - size_t const blockSize = block != nullptr ? block->size() : 0; - results.emplace_back(std::make_tuple(ExecutorCall::FETCH_FOR_PASSTHROUGH, - fetchBlockState, blockSize)); - stats += executorStats; - rowsLeft = blockSize; - if (fetchBlockState != ExecutionState::WAITING && - fetchBlockState != ExecutionState::DONE) { - EXPECT_GT(rowsLeft, 0); - } - if (fetchBlockState != ExecutionState::WAITING && block == nullptr) { - EXPECT_EQ(ExecutionState::DONE, fetchBlockState); - // Abort - state = ExecutionState::DONE; - } - } break; - // Produce rows - case RunState::PRODUCE: { - EXPECT_GT(rowsLeft, 0); - typename Executor::Stats executorStats{}; - size_t const rowsBefore = outputRow.numRowsWritten(); - std::tie(state, executorStats) = executor.produceRows(outputRow); - size_t const rowsAfter = outputRow.numRowsWritten(); - size_t const rowsProduced = rowsAfter - rowsBefore; - results.emplace_back(std::make_tuple(ExecutorCall::PRODUCE_ROWS, state, rowsProduced)); - stats += executorStats; - EXPECT_LE(rowsProduced, rowsLeft); - rowsLeft -= rowsProduced; - producedTotal += rowsProduced; - - if (outputRow.produced()) { - outputRow.advanceRow(); - } - } break; - // TODO don't do this for executors which don't have skipRows - case RunState::SKIP_REST: { - size_t skipped; - typename Executor::Stats executorStats{}; - std::tie(state, executorStats, skipped) = - executor.skipRows(ExecutionBlock::SkipAllSize()); - results.emplace_back(std::make_tuple(ExecutorCall::SKIP_ROWS, state, skipped)); - stats += executorStats; - } break; - // We're done - case RunState::BREAK: { - state = ExecutionState::DONE; - } break; - } - } - - return {outputRow.stealBlock(), results, stats}; -} - } // namespace aql } // namespace tests } // namespace arangodb diff --git a/tests/Aql/LimitExecutorTest.cpp b/tests/Aql/LimitExecutorTest.cpp index 0a51f1b6ce49..512a900841fd 100644 --- a/tests/Aql/LimitExecutorTest.cpp +++ b/tests/Aql/LimitExecutorTest.cpp @@ -323,7 +323,6 @@ TEST_P(LimitExecutorTest, testSuite) { auto expectedStats = ExecutionStats{}; expectedStats += expectedLimitStats; - // fakedQuery->queryOptions().profile = PROFILE_LEVEL_TRACE_2; makeExecutorTestHelper<1, 1>() .addConsumer(std::move(infos), ExecutionNode::LIMIT) .setInputFromRowNum(numInputRows) diff --git a/tests/Aql/MultiDepFetcherHelper.cpp b/tests/Aql/MultiDepFetcherHelper.cpp deleted file mode 100644 index c8c57e639dac..000000000000 --- a/tests/Aql/MultiDepFetcherHelper.cpp +++ /dev/null @@ -1,93 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -/// DISCLAIMER -/// -/// Copyright 2019 ArangoDB GmbH, Cologne, Germany -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// -/// Copyright holder is ArangoDB GmbH, Cologne, Germany -/// -/// @author Tobias Gödderz -//////////////////////////////////////////////////////////////////////////////// - -#include "MultiDepFetcherHelper.h" -#include "gtest/gtest.h" -#include "Aql/AqlItemRowPrinter.h" - -#include "Aql/AqlItemBlockManager.h" -#include "Aql/ShadowAqlItemRow.h" - -#include - - -using namespace arangodb; -using namespace arangodb::aql; -using namespace arangodb::tests; -using namespace arangodb::tests::aql; - -constexpr auto options = &velocypack::Options::Defaults; - -void arangodb::tests::aql::runFetcher(arangodb::aql::MultiDependencySingleRowFetcher& testee, - std::vector const& inputOutputPairs) { - size_t i = 0; - for (auto const& iop : inputOutputPairs) { - struct Visitor : public boost::static_visitor<> { - Visitor(MultiDependencySingleRowFetcher& testee, size_t i) - : testee(testee), i(i) {} - void operator()(ConcreteFetcherIOPair const& iop) { - auto const& args = iop.first; - auto const& expected = iop.second; - auto actual = testee.preFetchNumberOfRows(args.atMost); - EXPECT_EQ(expected, actual) << "during step " << i; - } - void operator()(ConcreteFetcherIOPair const& iop) { - auto const& args = iop.first; - auto const& expected = iop.second; - auto const& expectedState = expected.first; - auto const& expectedRow = expected.second; - auto actual = testee.fetchRowForDependency(args.dependency, args.atMost); - auto const& actualState = actual.first; - auto const& actualRow = actual.second; - EXPECT_EQ(expectedState, actualState) << "during step " << i; - EXPECT_TRUE(expectedRow.equates(actualRow, options)) - << " expected: " << expectedRow << "\n actual: " << actualRow - << "\n during step " << i; - } - void operator()(ConcreteFetcherIOPair const& iop) { - auto const& args = iop.first; - auto const& expected = iop.second; - auto actual = testee.skipRowsForDependency(args.dependency, args.atMost); - EXPECT_EQ(expected, actual) << "during step " << i; - } - void operator()(ConcreteFetcherIOPair const& iop) { - auto const& args = iop.first; - auto const& expected = iop.second; - auto const& expectedState = expected.first; - auto const& expectedRow = expected.second; - auto actual = testee.fetchShadowRow(args.atMost); - auto const& actualState = actual.first; - auto const& actualRow = actual.second; - EXPECT_EQ(expectedState, actualState) << "during step " << i; - EXPECT_TRUE(expectedRow.equates(actualRow, options)) - << " expected: " << expectedRow << "\n actual: " << actualRow - << "\n during step " << i; - } - - private: - MultiDependencySingleRowFetcher& testee; - size_t const i; - } visitor{testee, i}; - boost::apply_visitor(visitor, iop); - ++i; - } -} diff --git a/tests/Aql/MultiDepFetcherHelper.h b/tests/Aql/MultiDepFetcherHelper.h deleted file mode 100644 index 0f123240b332..000000000000 --- a/tests/Aql/MultiDepFetcherHelper.h +++ /dev/null @@ -1,81 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -/// DISCLAIMER -/// -/// Copyright 2019 ArangoDB GmbH, Cologne, Germany -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// -/// Copyright holder is ArangoDB GmbH, Cologne, Germany -/// -/// @author Tobias Gödderz -//////////////////////////////////////////////////////////////////////////////// - -#ifndef TESTS_AQL_MULTIDEPFETCHERHELPER_H -#define TESTS_AQL_MULTIDEPFETCHERHELPER_H - -#include "Aql/ExecutionBlock.h" -#include "Aql/ExecutionState.h" -#include "Aql/ExecutionStats.h" -#include "Aql/MultiDependencySingleRowFetcher.h" -#include "Aql/OutputAqlItemRow.h" -#include "Aql/SharedAqlItemBlockPtr.h" - -#include - -#include - -namespace arangodb { -namespace tests { -namespace aql { - -// std::pair preFetchNumberOfRows(size_t atMost) -struct PrefetchNumberOfRows { - using Result = std::pair; - size_t atMost; -}; - -// std::pair fetchRowForDependency(size_t dependency, size_t atMost) -struct FetchRowForDependency { - using Result = std::pair; - size_t dependency; - size_t atMost; -}; - -// std::pair skipRowsForDependency(size_t dependency, size_t atMost); -struct SkipRowsForDependency { - using Result = std::pair; - size_t dependency; - size_t atMost; -}; - -// std::pair fetchShadowRow(size_t atMost); -struct FetchShadowRow { - using Result = std::pair; - size_t atMost; -}; - -template -using ConcreteFetcherIOPair = std::pair; - -using FetcherIOPair = - boost::variant, ConcreteFetcherIOPair, - ConcreteFetcherIOPair, ConcreteFetcherIOPair>; - -void runFetcher(arangodb::aql::MultiDependencySingleRowFetcher& testee, - std::vector const& inputOutputPairs); - -} // namespace aql -} // namespace tests -} // namespace arangodb - -#endif // TESTS_AQL_MULTIDEPFETCHERHELPER_H diff --git a/tests/Aql/MultiDependencySingleRowFetcherTest.cpp b/tests/Aql/MultiDependencySingleRowFetcherTest.cpp index ac28824a9623..94402b0a7cae 100644 --- a/tests/Aql/MultiDependencySingleRowFetcherTest.cpp +++ b/tests/Aql/MultiDependencySingleRowFetcherTest.cpp @@ -23,13 +23,15 @@ /// @author Jan Christoph Uhde //////////////////////////////////////////////////////////////////////////////// +#include "AqlExecutorTestCase.h" #include "AqlItemBlockHelper.h" #include "AqlItemRowPrinter.h" #include "DependencyProxyMock.h" -#include "MultiDepFetcherHelper.h" +#include "ExecutorTestHelper.h" #include "gtest/gtest.h" #include "Aql/ExecutionBlock.h" +#include "Aql/ExecutorInfos.h" #include "Aql/InputAqlItemRow.h" #include "Aql/MultiDependencySingleRowFetcher.h" #include "Aql/ResourceUsage.h" @@ -45,1455 +47,735 @@ namespace arangodb { namespace tests { namespace aql { -class MultiDependencySingleRowFetcherTest : public ::testing::Test { +class MultiDependencySingleRowFetcherTest + : public AqlExecutorTestCaseWithParam> { protected: - ResourceMonitor monitor; - AqlItemBlockManager itemBlockManager; - ExecutionState state; + auto doesWait() -> bool { + auto const [wait, deps] = GetParam(); + return wait; + } - MultiDependencySingleRowFetcherTest() - : itemBlockManager(&monitor, SerializationFormat::SHADOWROWS) {} -}; + auto numberDependencies() -> size_t { + auto const [wait, deps] = GetParam(); + return deps; + } -TEST_F(MultiDependencySingleRowFetcherTest, - no_blocks_upstream_single_dependency_the_producer_doesnt_wait) { - VPackBuilder input; - MultiDependencyProxyMock<::arangodb::aql::BlockPassthrough::Disable> dependencyProxyMock{monitor, 0, 1}; - InputAqlItemRow row{CreateInvalidInputRowHint{}}; - dependencyProxyMock.getDependencyMock(0).shouldReturn(ExecutionState::DONE, nullptr); + // This will create inputData.size() many dependecies. + // Each will be initialized with the given deque of blocks + // Note: Caller needs to make sure that ShadowRows are present in correct + // order and correct amount in all deques + auto buildFetcher(std::vector> inputData) + -> MultiDependencySingleRowFetcher { + // We need at least 1 dependency! + TRI_ASSERT(!inputData.empty()); + WaitingExecutionBlockMock::WaitingBehaviour waiting = + doesWait() ? WaitingExecutionBlockMock::WaitingBehaviour::ONCE + : WaitingExecutionBlockMock::WaitingBehaviour::NEVER; + for (auto blockDeque : inputData) { + auto dep = std::make_unique(fakedQuery->engine(), + generateNodeDummy(), + std::move(blockDeque), waiting); + _dependencies.emplace_back(dep.get()); + _blocks.emplace_back(std::move(dep)); + } - { - MultiDependencySingleRowFetcher testee(dependencyProxyMock); - - testee.initDependencies(); - - std::tie(state, row) = testee.fetchRowForDependency(0); - ASSERT_EQ(state, ExecutionState::DONE); - ASSERT_FALSE(row); - } // testee is destroyed here - // testee must be destroyed before verify, because it may call returnBlock - // in the destructor - ASSERT_TRUE(dependencyProxyMock.allBlocksFetched()); - ASSERT_EQ(dependencyProxyMock.numFetchBlockCalls(), 1); -} + MultiDependencySingleRowFetcher testee{_proxy}; + testee.init(); + return testee; + } -TEST_F(MultiDependencySingleRowFetcherTest, - no_blocks_upstream_single_dependency_the_producer_waits) { - VPackBuilder input; - MultiDependencyProxyMock<::arangodb::aql::BlockPassthrough::Disable> dependencyProxyMock{monitor, 0, 1}; - InputAqlItemRow row{CreateInvalidInputRowHint{}}; - dependencyProxyMock.getDependencyMock(0) - .shouldReturn(ExecutionState::WAITING, nullptr) - .andThenReturn(ExecutionState::DONE, nullptr); + auto makeStack() -> AqlCallStack { + // We need a stack for the API, an empty one will do. + // We are not testing subqueries here + AqlCallStack stack{AqlCallList{AqlCall{}}}; + stack.popCall(); + return stack; + } - { - MultiDependencySingleRowFetcher testee(dependencyProxyMock); - - testee.initDependencies(); - - std::tie(state, row) = testee.fetchRowForDependency(0); - ASSERT_EQ(state, ExecutionState::WAITING); - ASSERT_FALSE(row); - - std::tie(state, row) = testee.fetchRowForDependency(0); - ASSERT_EQ(state, ExecutionState::DONE); - ASSERT_FALSE(row); - } // testee is destroyed here - // testee must be destroyed before verify, because it may call returnBlock - // in the destructor - ASSERT_TRUE(dependencyProxyMock.allBlocksFetched()); - ASSERT_EQ(dependencyProxyMock.numFetchBlockCalls(), 2); -} + auto makeSameCallToAllDependencies(AqlCall call) -> AqlCallSet { + AqlCallSet set{}; + for (size_t i = 0; i < _dependencies.size(); ++i) { + set.calls.emplace_back(AqlCallSet::DepCallPair{i, AqlCallList{call}}); + } + return set; + } -TEST_F(MultiDependencySingleRowFetcherTest, - single_upstream_block_with_a_single_row_single_dependency_the_producer_returns_done_immediately) { - VPackBuilder input; - MultiDependencyProxyMock<::arangodb::aql::BlockPassthrough::Disable> dependencyProxyMock{monitor, 1, 1}; - InputAqlItemRow row{CreateInvalidInputRowHint{}}; - SharedAqlItemBlockPtr block = buildBlock<1>(itemBlockManager, {{42}}); - dependencyProxyMock.getDependencyMock(0).shouldReturn(ExecutionState::DONE, - std::move(block)); + auto testWaiting(MultiDependencySingleRowFetcher& testee, AqlCallSet const& set) { + if (doesWait()) { + auto stack = makeStack(); + auto [state, skipped, ranges] = testee.execute(stack, set); + EXPECT_EQ(state, ExecutionState::WAITING); + EXPECT_TRUE(skipped.nothingSkipped()); + EXPECT_EQ(ranges.size(), set.size()); + for (auto const& [dep, range] : ranges) { + EXPECT_FALSE(range.hasDataRow()); + EXPECT_FALSE(range.hasShadowRow()); + EXPECT_EQ(range.upstreamState(), ExecutorState::HASMORE); + } + } + } - { - MultiDependencySingleRowFetcher testee(dependencyProxyMock); - - testee.initDependencies(); - - std::tie(state, row) = testee.fetchRowForDependency(0); - ASSERT_EQ(state, ExecutionState::DONE); - ASSERT_TRUE(row); - ASSERT_EQ(row.getNrRegisters(), 1); - ASSERT_EQ(row.getValue(0).slice().getInt(), 42); - } // testee is destroyed here - // testee must be destroyed before verify, because it may call returnBlock - // in the destructor - ASSERT_TRUE(dependencyProxyMock.allBlocksFetched()); - ASSERT_EQ(dependencyProxyMock.numFetchBlockCalls(), 1); -} + void validateNextIsDataRow(AqlItemBlockInputRange& testee, + ExecutorState expectedState, int64_t value) { + EXPECT_TRUE(testee.hasDataRow()); + EXPECT_FALSE(testee.hasShadowRow()); + // We have the next row + EXPECT_EQ(testee.upstreamState(), ExecutorState::HASMORE); + auto rowIndexBefore = testee.getRowIndex(); + // Validate that shadowRowAPI does not move on + { + auto row = testee.peekShadowRow(); + EXPECT_FALSE(row.isInitialized()); + ASSERT_EQ(rowIndexBefore, testee.getRowIndex()) + << "Skipped a non processed row."; + } + { + auto [state, row] = testee.nextShadowRow(); + EXPECT_FALSE(row.isInitialized()); + ASSERT_EQ(rowIndexBefore, testee.getRowIndex()) + << "Skipped a non processed row."; + } + // Validate Data Row API + { + auto [state, row] = testee.peekDataRow(); + EXPECT_EQ(state, expectedState); + EXPECT_TRUE(row.isInitialized()); + auto val = row.getValue(0); + ASSERT_TRUE(val.isNumber()); + EXPECT_EQ(val.toInt64(), value); + ASSERT_EQ(rowIndexBefore, testee.getRowIndex()) + << "Skipped a non processed row."; + } -TEST_F(MultiDependencySingleRowFetcherTest, - single_upstream_block_with_a_single_row_single_dependency_the_producer_returns_hasmore_then_done_with_a_nullptr) { - VPackBuilder input; - MultiDependencyProxyMock<::arangodb::aql::BlockPassthrough::Disable> dependencyProxyMock{monitor, 1, 1}; - InputAqlItemRow row{CreateInvalidInputRowHint{}}; - SharedAqlItemBlockPtr block = buildBlock<1>(itemBlockManager, {{42}}); - dependencyProxyMock.getDependencyMock(0) - .shouldReturn(ExecutionState::HASMORE, std::move(block)) - .andThenReturn(ExecutionState::DONE, nullptr); + { + auto [state, row] = testee.nextDataRow(); + EXPECT_EQ(state, expectedState); + EXPECT_TRUE(row.isInitialized()); + auto val = row.getValue(0); + ASSERT_TRUE(val.isNumber()); + EXPECT_EQ(val.toInt64(), value); + ASSERT_NE(rowIndexBefore, testee.getRowIndex()) + << "Did not go to next row."; + } + EXPECT_EQ(expectedState, testee.upstreamState()); + } - { - MultiDependencySingleRowFetcher testee(dependencyProxyMock); - - testee.initDependencies(); - - std::tie(state, row) = testee.fetchRowForDependency(0); - ASSERT_EQ(state, ExecutionState::HASMORE); - ASSERT_TRUE(row); - ASSERT_EQ(row.getNrRegisters(), 1); - ASSERT_EQ(row.getValue(0).slice().getInt(), 42); - - std::tie(state, row) = testee.fetchRowForDependency(0); - ASSERT_EQ(state, ExecutionState::DONE); - ASSERT_FALSE(row); - } // testee is destroyed here - // testee must be destroyed before verify, because it may call returnBlock - // in the destructor - ASSERT_TRUE(dependencyProxyMock.allBlocksFetched()); - ASSERT_EQ(dependencyProxyMock.numFetchBlockCalls(), 2); -} + void validateNextIsShadowRow(AqlItemBlockInputRange& testee, ExecutorState expectedState, + int64_t value, uint64_t depth) { + EXPECT_TRUE(testee.hasShadowRow()); + // The next is a ShadowRow, the state shall be done + EXPECT_EQ(testee.upstreamState(), ExecutorState::DONE); + + auto rowIndexBefore = testee.getRowIndex(); + // Validate that inputRowAPI does not move on + { + auto [state, row] = testee.peekDataRow(); + EXPECT_EQ(state, ExecutorState::DONE); + EXPECT_FALSE(row.isInitialized()); + ASSERT_EQ(rowIndexBefore, testee.getRowIndex()) + << "Skipped a non processed row."; + } + { + auto [state, row] = testee.nextDataRow(); + EXPECT_EQ(state, ExecutorState::DONE); + EXPECT_FALSE(row.isInitialized()); + ASSERT_EQ(rowIndexBefore, testee.getRowIndex()) + << "Skipped a non processed row."; + } + // Validate ShadowRow API + { + auto row = testee.peekShadowRow(); + EXPECT_TRUE(row.isInitialized()); + auto val = row.getValue(0); + ASSERT_TRUE(val.isNumber()); + EXPECT_EQ(val.toInt64(), value); + EXPECT_EQ(row.getDepth(), depth); + ASSERT_EQ(rowIndexBefore, testee.getRowIndex()) + << "Skipped a non processed row."; + } + { + auto [state, row] = testee.nextShadowRow(); + EXPECT_EQ(state, expectedState); + EXPECT_TRUE(row.isInitialized()); + auto val = row.getValue(0); + ASSERT_TRUE(val.isNumber()); + EXPECT_EQ(val.toInt64(), value); + EXPECT_EQ(row.getDepth(), depth); + ASSERT_NE(rowIndexBefore, testee.getRowIndex()) + << "Did not go to next row."; + } + } -TEST_F(MultiDependencySingleRowFetcherTest, - single_upstream_block_with_a_single_row_single_dependency_the_producer_waits_then_returns_done) { - VPackBuilder input; - MultiDependencyProxyMock<::arangodb::aql::BlockPassthrough::Disable> dependencyProxyMock{monitor, 1, 1}; - InputAqlItemRow row{CreateInvalidInputRowHint{}}; - SharedAqlItemBlockPtr block = buildBlock<1>(itemBlockManager, {{42}}); - dependencyProxyMock.getDependencyMock(0) - .shouldReturn(ExecutionState::WAITING, nullptr) - .andThenReturn(ExecutionState::DONE, std::move(block)); + private: + // This vector is not read anywhere, it just serves as a data lake for the ExecutionBlocks generated by the tests + // s.t. they are garbage collected after the test execution is done. + std::vector> _blocks; + // The dependencies, they are referenced by _proxy, modifing this will modify the proxy + std::vector _dependencies{}; + DependencyProxy _proxy{_dependencies, itemBlockManager, + make_shared_unordered_set({0}), + 1, nullptr}; +}; - { - MultiDependencySingleRowFetcher testee(dependencyProxyMock); - - testee.initDependencies(); - - std::tie(state, row) = testee.fetchRowForDependency(0); - ASSERT_EQ(state, ExecutionState::WAITING); - ASSERT_FALSE(row); - - std::tie(state, row) = testee.fetchRowForDependency(0); - ASSERT_EQ(state, ExecutionState::DONE); - ASSERT_TRUE(row); - ASSERT_EQ(row.getNrRegisters(), 1); - ASSERT_EQ(row.getValue(0).slice().getInt(), 42); - } // testee is destroyed here - // testee must be destroyed before verify, because it may call returnBlock - // in the destructor - ASSERT_TRUE(dependencyProxyMock.allBlocksFetched()); - ASSERT_EQ(dependencyProxyMock.numFetchBlockCalls(), 2); -} +INSTANTIATE_TEST_CASE_P(MultiDependencySingleRowFetcherTest, MultiDependencySingleRowFetcherTest, + ::testing::Combine(::testing::Bool(), + ::testing::Range(static_cast(1), + static_cast(4)))); -TEST_F(MultiDependencySingleRowFetcherTest, - single_upstream_block_with_a_single_row_single_dependency_the_producer_waits_returns_hasmore_then_done) { - VPackBuilder input; - MultiDependencyProxyMock<::arangodb::aql::BlockPassthrough::Disable> dependencyProxyMock{monitor, 1, 1}; - InputAqlItemRow row{CreateInvalidInputRowHint{}}; - SharedAqlItemBlockPtr block = buildBlock<1>(itemBlockManager, {{42}}); - dependencyProxyMock.getDependencyMock(0) - .shouldReturn(ExecutionState::WAITING, nullptr) - .andThenReturn(ExecutionState::HASMORE, std::move(block)) - .andThenReturn(ExecutionState::DONE, nullptr); +TEST_P(MultiDependencySingleRowFetcherTest, no_blocks_upstream) { + std::vector> data; + for (size_t i = 0; i < numberDependencies(); ++i) { + data.emplace_back(std::deque{}); + } - { - MultiDependencySingleRowFetcher testee(dependencyProxyMock); - - testee.initDependencies(); - - std::tie(state, row) = testee.fetchRowForDependency(0); - ASSERT_EQ(state, ExecutionState::WAITING); - ASSERT_FALSE(row); - - std::tie(state, row) = testee.fetchRowForDependency(0); - ASSERT_EQ(state, ExecutionState::HASMORE); - ASSERT_TRUE(row); - ASSERT_EQ(row.getNrRegisters(), 1); - ASSERT_EQ(row.getValue(0).slice().getInt(), 42); - - std::tie(state, row) = testee.fetchRowForDependency(0); - ASSERT_EQ(state, ExecutionState::DONE); - ASSERT_FALSE(row); - } // testee is destroyed here - // testee must be destroyed before verify, because it may call returnBlock - // in the destructor - ASSERT_TRUE(dependencyProxyMock.allBlocksFetched()); - ASSERT_EQ(dependencyProxyMock.numFetchBlockCalls(), 3); + auto testee = buildFetcher(data); + + auto set = makeSameCallToAllDependencies(AqlCall{}); + testWaiting(testee, set); + + auto stack = makeStack(); + auto [state, skipped, ranges] = testee.execute(stack, set); + EXPECT_EQ(state, ExecutionState::DONE); + EXPECT_TRUE(skipped.nothingSkipped()); + EXPECT_EQ(ranges.size(), set.size()); + for (auto const& [dep, range] : ranges) { + // All Ranges are empty + EXPECT_FALSE(range.hasDataRow()); + EXPECT_FALSE(range.hasShadowRow()); + EXPECT_EQ(range.upstreamState(), ExecutorState::DONE); + } } -// TODO the following tests should be simplified, a simple output -// specification should be compared with the actual output. - -TEST_F(MultiDependencySingleRowFetcherTest, - multiple_blocks_upstream_single_dependency_the_producer_doesnt_wait) { - MultiDependencyProxyMock<::arangodb::aql::BlockPassthrough::Disable> dependencyProxyMock{monitor, 1, 1}; - InputAqlItemRow row{CreateInvalidInputRowHint{}}; - // three 1-column matrices with 3, 2 and 1 rows, respectively - SharedAqlItemBlockPtr block1 = buildBlock<1>(itemBlockManager, {{{1}}, {{2}}, {{3}}}), - block2 = buildBlock<1>(itemBlockManager, {{{4}}, {{5}}}), - block3 = buildBlock<1>(itemBlockManager, {{{6}}}); - dependencyProxyMock.getDependencyMock(0) - .shouldReturn(ExecutionState::HASMORE, std::move(block1)) - .andThenReturn(ExecutionState::HASMORE, std::move(block2)) - .andThenReturn(ExecutionState::DONE, std::move(block3)); +TEST_P(MultiDependencySingleRowFetcherTest, one_block_upstream_all_deps_equal) { + std::vector> data; + for (size_t i = 0; i < numberDependencies(); ++i) { + std::deque blockDeque{}; + blockDeque.emplace_back(buildBlock<1>(itemBlockManager, {{1}, {2}, {3}})); + data.emplace_back(std::move(blockDeque)); + } - { - MultiDependencySingleRowFetcher testee(dependencyProxyMock); + auto testee = buildFetcher(data); + + auto set = makeSameCallToAllDependencies(AqlCall{}); + testWaiting(testee, set); + + auto stack = makeStack(); + auto [state, skipped, ranges] = testee.execute(stack, set); + EXPECT_EQ(state, ExecutionState::DONE); + EXPECT_TRUE(skipped.nothingSkipped()); + EXPECT_EQ(ranges.size(), set.size()); + for (auto [dep, range] : ranges) { + // All Ranges are non empty + validateNextIsDataRow(range, ExecutorState::HASMORE, 1); + validateNextIsDataRow(range, ExecutorState::HASMORE, 2); + validateNextIsDataRow(range, ExecutorState::DONE, 3); + } +} - testee.initDependencies(); +TEST_P(MultiDependencySingleRowFetcherTest, one_block_upstream_all_deps_differ) { + std::vector> data; + for (size_t i = 0; i < numberDependencies(); ++i) { + std::deque blockDeque{}; + blockDeque.emplace_back( + buildBlock<1>(itemBlockManager, {{1 * (i + 1)}, {2 * (i + 1)}, {3 * (i + 1)}})); + data.emplace_back(std::move(blockDeque)); + } - int64_t rowIdxAndValue; - for (rowIdxAndValue = 1; rowIdxAndValue <= 5; rowIdxAndValue++) { - std::tie(state, row) = testee.fetchRowForDependency(0); - ASSERT_EQ(state, ExecutionState::HASMORE); - ASSERT_TRUE(row); - ASSERT_EQ(row.getNrRegisters(), 1); - ASSERT_EQ(row.getValue(0).slice().getInt(), rowIdxAndValue); - } - rowIdxAndValue = 6; - std::tie(state, row) = testee.fetchRowForDependency(0); - ASSERT_EQ(state, ExecutionState::DONE); - ASSERT_TRUE(row); - ASSERT_EQ(row.getNrRegisters(), 1); - ASSERT_EQ(row.getValue(0).slice().getInt(), rowIdxAndValue); - } // testee is destroyed here - // testee must be destroyed before verify, because it may call returnBlock - // in the destructor - ASSERT_TRUE(dependencyProxyMock.allBlocksFetched()); - ASSERT_EQ(dependencyProxyMock.numFetchBlockCalls(), 3); + auto testee = buildFetcher(data); + + auto set = makeSameCallToAllDependencies(AqlCall{}); + testWaiting(testee, set); + + auto stack = makeStack(); + auto [state, skipped, ranges] = testee.execute(stack, set); + EXPECT_EQ(state, ExecutionState::DONE); + EXPECT_TRUE(skipped.nothingSkipped()); + EXPECT_EQ(ranges.size(), set.size()); + for (auto [dep, range] : ranges) { + // All Ranges are non empty + validateNextIsDataRow(range, ExecutorState::HASMORE, 1 * (dep + 1)); + validateNextIsDataRow(range, ExecutorState::HASMORE, 2 * (dep + 1)); + validateNextIsDataRow(range, ExecutorState::DONE, 3 * (dep + 1)); + } } -TEST_F(MultiDependencySingleRowFetcherTest, - multiple_blocks_upstream_single_dependency_the_producer_waits) { - MultiDependencyProxyMock<::arangodb::aql::BlockPassthrough::Disable> dependencyProxyMock{monitor, 1, 1}; - InputAqlItemRow row{CreateInvalidInputRowHint{}}; - // three 1-column matrices with 3, 2 and 1 rows, respectively - SharedAqlItemBlockPtr block1 = buildBlock<1>(itemBlockManager, {{{1}}, {{2}}, {{3}}}), - block2 = buildBlock<1>(itemBlockManager, {{{4}}, {{5}}}), - block3 = buildBlock<1>(itemBlockManager, {{{6}}}); - dependencyProxyMock.getDependencyMock(0) - .shouldReturn(ExecutionState::WAITING, nullptr) - .andThenReturn(ExecutionState::HASMORE, std::move(block1)) - .andThenReturn(ExecutionState::WAITING, nullptr) - .andThenReturn(ExecutionState::HASMORE, std::move(block2)) - .andThenReturn(ExecutionState::WAITING, nullptr) - .andThenReturn(ExecutionState::DONE, std::move(block3)); +TEST_P(MultiDependencySingleRowFetcherTest, many_blocks_upstream_all_deps_equal) { + std::vector> data; + for (size_t i = 0; i < numberDependencies(); ++i) { + std::deque blockDeque{}; + blockDeque.emplace_back(buildBlock<1>(itemBlockManager, {{1}, {2}, {3}})); + blockDeque.emplace_back(buildBlock<1>(itemBlockManager, {{4}, {5}, {6}})); + blockDeque.emplace_back(buildBlock<1>(itemBlockManager, {{7}, {8}, {9}})); + data.emplace_back(std::move(blockDeque)); + } - { - MultiDependencySingleRowFetcher testee(dependencyProxyMock); + auto testee = buildFetcher(data); - testee.initDependencies(); + auto set = makeSameCallToAllDependencies(AqlCall{}); + testWaiting(testee, set); - int64_t rowIdxAndValue; - for (rowIdxAndValue = 1; rowIdxAndValue <= 5; rowIdxAndValue++) { - if (rowIdxAndValue == 1 || rowIdxAndValue == 4) { - // wait at the beginning of the 1st and 2nd block - std::tie(state, row) = testee.fetchRowForDependency(0); - ASSERT_EQ(state, ExecutionState::WAITING); - ASSERT_FALSE(row); - } - std::tie(state, row) = testee.fetchRowForDependency(0); - ASSERT_EQ(state, ExecutionState::HASMORE); - ASSERT_TRUE(row); - ASSERT_EQ(row.getNrRegisters(), 1); - ASSERT_EQ(row.getValue(0).slice().getInt(), rowIdxAndValue); + auto stack = makeStack(); + { + // First Block + auto [state, skipped, ranges] = testee.execute(stack, set); + EXPECT_EQ(state, ExecutionState::HASMORE); + EXPECT_TRUE(skipped.nothingSkipped()); + EXPECT_EQ(ranges.size(), set.size()); + for (auto [dep, range] : ranges) { + // All Ranges are non empty + validateNextIsDataRow(range, ExecutorState::HASMORE, 1); + validateNextIsDataRow(range, ExecutorState::HASMORE, 2); + validateNextIsDataRow(range, ExecutorState::HASMORE, 3); } - rowIdxAndValue = 6; - // wait at the beginning of the 3rd block - std::tie(state, row) = testee.fetchRowForDependency(0); - ASSERT_EQ(state, ExecutionState::WAITING); - ASSERT_FALSE(row); - // last row and DONE - std::tie(state, row) = testee.fetchRowForDependency(0); - ASSERT_EQ(state, ExecutionState::DONE); - ASSERT_TRUE(row); - ASSERT_EQ(row.getNrRegisters(), 1); - ASSERT_EQ(row.getValue(0).slice().getInt(), rowIdxAndValue); - } // testee is destroyed here - // testee must be destroyed before verify, because it may call returnBlock - // in the destructor - ASSERT_TRUE(dependencyProxyMock.allBlocksFetched()); - ASSERT_EQ(dependencyProxyMock.numFetchBlockCalls(), 6); -} - -TEST_F(MultiDependencySingleRowFetcherTest, - multiple_blocks_upstream_single_dependency_the_producer_the_producer_waits_and_doesnt_return_done_asap) { - MultiDependencyProxyMock<::arangodb::aql::BlockPassthrough::Disable> dependencyProxyMock{monitor, 1, 1}; - InputAqlItemRow row{CreateInvalidInputRowHint{}}; - // three 1-column matrices with 3, 2 and 1 rows, respectively - SharedAqlItemBlockPtr block1 = buildBlock<1>(itemBlockManager, {{{1}}, {{2}}, {{3}}}), - block2 = buildBlock<1>(itemBlockManager, {{{4}}, {{5}}}), - block3 = buildBlock<1>(itemBlockManager, {{{6}}}); - dependencyProxyMock.getDependencyMock(0) - .shouldReturn(ExecutionState::WAITING, nullptr) - .andThenReturn(ExecutionState::HASMORE, std::move(block1)) - .andThenReturn(ExecutionState::WAITING, nullptr) - .andThenReturn(ExecutionState::HASMORE, std::move(block2)) - .andThenReturn(ExecutionState::WAITING, nullptr) - .andThenReturn(ExecutionState::HASMORE, std::move(block3)) - .andThenReturn(ExecutionState::DONE, nullptr); + } { - MultiDependencySingleRowFetcher testee(dependencyProxyMock); - - testee.initDependencies(); - - for (int64_t rowIdxAndValue = 1; rowIdxAndValue <= 6; rowIdxAndValue++) { - if (rowIdxAndValue == 1 || rowIdxAndValue == 4 || rowIdxAndValue == 6) { - // wait at the beginning of the 1st, 2nd and 3rd block - std::tie(state, row) = testee.fetchRowForDependency(0); - ASSERT_EQ(state, ExecutionState::WAITING); - ASSERT_FALSE(row); - } - std::tie(state, row) = testee.fetchRowForDependency(0); - ASSERT_EQ(state, ExecutionState::HASMORE); - ASSERT_TRUE(row); - ASSERT_EQ(row.getNrRegisters(), 1); - ASSERT_EQ(row.getValue(0).slice().getInt(), rowIdxAndValue); + // Second Block + auto [state, skipped, ranges] = testee.execute(stack, set); + EXPECT_EQ(state, ExecutionState::HASMORE); + EXPECT_TRUE(skipped.nothingSkipped()); + EXPECT_EQ(ranges.size(), set.size()); + for (auto [dep, range] : ranges) { + // All Ranges are non empty + validateNextIsDataRow(range, ExecutorState::HASMORE, 4); + validateNextIsDataRow(range, ExecutorState::HASMORE, 5); + validateNextIsDataRow(range, ExecutorState::HASMORE, 6); } - std::tie(state, row) = testee.fetchRowForDependency(0); - ASSERT_EQ(state, ExecutionState::DONE); - ASSERT_FALSE(row); - } // testee is destroyed here - // testee must be destroyed before verify, because it may call returnBlock - // in the destructor - ASSERT_TRUE(dependencyProxyMock.allBlocksFetched()); - ASSERT_EQ(dependencyProxyMock.numFetchBlockCalls(), 7); -} - -/********************* - * Multi Dependencies - *********************/ - -TEST_F(MultiDependencySingleRowFetcherTest, - no_blocks_upstream_multiple_dependencies_the_producers_dont_wait) { - VPackBuilder input; - size_t numDeps = 3; - MultiDependencyProxyMock<::arangodb::aql::BlockPassthrough::Disable> dependencyProxyMock{monitor, 0, numDeps}; - InputAqlItemRow row{CreateInvalidInputRowHint{}}; - for (size_t i = 0; i < numDeps; ++i) { - dependencyProxyMock.getDependencyMock(i).shouldReturn(ExecutionState::DONE, nullptr); } { - MultiDependencySingleRowFetcher testee(dependencyProxyMock); - - testee.initDependencies(); - - for (size_t i = 0; i < numDeps; ++i) { - std::tie(state, row) = testee.fetchRowForDependency(i); - ASSERT_EQ(state, ExecutionState::DONE); - ASSERT_FALSE(row); + // Third Block + auto [state, skipped, ranges] = testee.execute(stack, set); + EXPECT_EQ(state, ExecutionState::DONE); + EXPECT_TRUE(skipped.nothingSkipped()); + EXPECT_EQ(ranges.size(), set.size()); + for (auto [dep, range] : ranges) { + // All Ranges are non empty + validateNextIsDataRow(range, ExecutorState::HASMORE, 7); + validateNextIsDataRow(range, ExecutorState::HASMORE, 8); + validateNextIsDataRow(range, ExecutorState::DONE, 9); } - } // testee is destroyed here - // testee must be destroyed before verify, because it may call returnBlock - // in the destructor - ASSERT_TRUE(dependencyProxyMock.allBlocksFetched()); - ASSERT_EQ(dependencyProxyMock.numFetchBlockCalls(), numDeps); + } } -TEST_F(MultiDependencySingleRowFetcherTest, - no_blocks_upstream_multiple_dependencies_the_producers_wait) { - VPackBuilder input; - size_t numDeps = 3; - MultiDependencyProxyMock<::arangodb::aql::BlockPassthrough::Disable> dependencyProxyMock{monitor, 0, numDeps}; - InputAqlItemRow row{CreateInvalidInputRowHint{}}; - for (size_t i = 0; i < numDeps; ++i) { - dependencyProxyMock.getDependencyMock(i) - .shouldReturn(ExecutionState::WAITING, nullptr) - .andThenReturn(ExecutionState::DONE, nullptr); +TEST_P(MultiDependencySingleRowFetcherTest, many_blocks_upstream_all_deps_differ) { + std::vector> data; + for (size_t i = 0; i < numberDependencies(); ++i) { + std::deque blockDeque{}; + blockDeque.emplace_back( + buildBlock<1>(itemBlockManager, {{1 * (i + 1)}, {2 * (i + 1)}, {3 * (i + 1)}})); + blockDeque.emplace_back( + buildBlock<1>(itemBlockManager, {{4 * (i + 1)}, {5 * (i + 1)}, {6 * (i + 1)}})); + blockDeque.emplace_back( + buildBlock<1>(itemBlockManager, {{7 * (i + 1)}, {8 * (i + 1)}, {9 * (i + 1)}})); + data.emplace_back(std::move(blockDeque)); } - { - MultiDependencySingleRowFetcher testee(dependencyProxyMock); - - testee.initDependencies(); - - for (size_t i = 0; i < numDeps; ++i) { - std::tie(state, row) = testee.fetchRowForDependency(i); - ASSERT_EQ(state, ExecutionState::WAITING); - ASSERT_FALSE(row); - } - - for (size_t i = 0; i < numDeps; ++i) { - std::tie(state, row) = testee.fetchRowForDependency(i); - ASSERT_EQ(state, ExecutionState::DONE); - ASSERT_FALSE(row); - } - } // testee is destroyed here - // testee must be destroyed before verify, because it may call returnBlock - // in the destructor - ASSERT_TRUE(dependencyProxyMock.allBlocksFetched()); - ASSERT_EQ(dependencyProxyMock.numFetchBlockCalls(), 2 * numDeps); -} + auto testee = buildFetcher(data); -TEST_F(MultiDependencySingleRowFetcherTest, - single_upstream_block_with_a_single_row_multi_dependency_the_producer_returns_done_immediately) { - VPackBuilder input; - size_t numDeps = 3; - MultiDependencyProxyMock<::arangodb::aql::BlockPassthrough::Disable> dependencyProxyMock{monitor, 1, numDeps}; - InputAqlItemRow row{CreateInvalidInputRowHint{}}; - SharedAqlItemBlockPtr blockDep1 = buildBlock<1>(itemBlockManager, {{42}}); - SharedAqlItemBlockPtr blockDep2 = buildBlock<1>(itemBlockManager, {{23}}); - SharedAqlItemBlockPtr blockDep3 = buildBlock<1>(itemBlockManager, {{1337}}); - dependencyProxyMock.getDependencyMock(0).shouldReturn(ExecutionState::DONE, - std::move(blockDep1)); - dependencyProxyMock.getDependencyMock(1).shouldReturn(ExecutionState::DONE, - std::move(blockDep2)); - dependencyProxyMock.getDependencyMock(2).shouldReturn(ExecutionState::DONE, - std::move(blockDep3)); + auto set = makeSameCallToAllDependencies(AqlCall{}); + testWaiting(testee, set); + auto stack = makeStack(); { - MultiDependencySingleRowFetcher testee(dependencyProxyMock); - - testee.initDependencies(); - - for (size_t i = 0; i < numDeps; ++i) { - std::tie(state, row) = testee.fetchRowForDependency(i); - ASSERT_EQ(state, ExecutionState::DONE); - ASSERT_TRUE(row); - ASSERT_EQ(row.getNrRegisters(), 1); - if (i == 0) { - ASSERT_EQ(row.getValue(0).slice().getInt(), 42); - } else if (i == 1) { - ASSERT_EQ(row.getValue(0).slice().getInt(), 23); - } else { - ASSERT_EQ(row.getValue(0).slice().getInt(), 1337); - } + // First Block + auto [state, skipped, ranges] = testee.execute(stack, set); + EXPECT_EQ(state, ExecutionState::HASMORE); + EXPECT_TRUE(skipped.nothingSkipped()); + EXPECT_EQ(ranges.size(), set.size()); + for (auto [dep, range] : ranges) { + // All Ranges are non empty + validateNextIsDataRow(range, ExecutorState::HASMORE, 1 * (dep + 1)); + validateNextIsDataRow(range, ExecutorState::HASMORE, 2 * (dep + 1)); + validateNextIsDataRow(range, ExecutorState::HASMORE, 3 * (dep + 1)); } - } // testee is destroyed here - // testee must be destroyed before verify, because it may call returnBlock - // in the destructor - ASSERT_TRUE(dependencyProxyMock.allBlocksFetched()); - ASSERT_EQ(dependencyProxyMock.numFetchBlockCalls(), numDeps); -} - -TEST_F(MultiDependencySingleRowFetcherTest, - single_upstream_block_with_a_single_row_multi_dependency_the_producer_returns_hasmore_then_done_with_a_nullptr) { - VPackBuilder input; - size_t numDeps = 3; - MultiDependencyProxyMock<::arangodb::aql::BlockPassthrough::Disable> dependencyProxyMock{monitor, 1, numDeps}; - InputAqlItemRow row{CreateInvalidInputRowHint{}}; - SharedAqlItemBlockPtr blockDep1 = buildBlock<1>(itemBlockManager, {{42}}); - SharedAqlItemBlockPtr blockDep2 = buildBlock<1>(itemBlockManager, {{23}}); - SharedAqlItemBlockPtr blockDep3 = buildBlock<1>(itemBlockManager, {{1337}}); - dependencyProxyMock.getDependencyMock(0) - .shouldReturn(ExecutionState::HASMORE, std::move(blockDep1)) - .andThenReturn(ExecutionState::DONE, nullptr); - dependencyProxyMock.getDependencyMock(1) - .shouldReturn(ExecutionState::HASMORE, std::move(blockDep2)) - .andThenReturn(ExecutionState::DONE, nullptr); - dependencyProxyMock.getDependencyMock(2) - .shouldReturn(ExecutionState::HASMORE, std::move(blockDep3)) - .andThenReturn(ExecutionState::DONE, nullptr); + } { - MultiDependencySingleRowFetcher testee(dependencyProxyMock); - - testee.initDependencies(); - - for (size_t i = 0; i < numDeps; ++i) { - std::tie(state, row) = testee.fetchRowForDependency(i); - ASSERT_EQ(state, ExecutionState::HASMORE); - ASSERT_TRUE(row); - ASSERT_EQ(row.getNrRegisters(), 1); - if (i == 0) { - ASSERT_EQ(row.getValue(0).slice().getInt(), 42); - } else if (i == 1) { - ASSERT_EQ(row.getValue(0).slice().getInt(), 23); - } else { - ASSERT_EQ(row.getValue(0).slice().getInt(), 1337); - } + // Second Block + auto [state, skipped, ranges] = testee.execute(stack, set); + EXPECT_EQ(state, ExecutionState::HASMORE); + EXPECT_TRUE(skipped.nothingSkipped()); + EXPECT_EQ(ranges.size(), set.size()); + for (auto [dep, range] : ranges) { + // All Ranges are non empty + validateNextIsDataRow(range, ExecutorState::HASMORE, 4 * (dep + 1)); + validateNextIsDataRow(range, ExecutorState::HASMORE, 5 * (dep + 1)); + validateNextIsDataRow(range, ExecutorState::HASMORE, 6 * (dep + 1)); } + } - for (size_t i = 0; i < numDeps; ++i) { - std::tie(state, row) = testee.fetchRowForDependency(i); - ASSERT_EQ(state, ExecutionState::DONE); - ASSERT_FALSE(row); + { + // Third Block + auto [state, skipped, ranges] = testee.execute(stack, set); + EXPECT_EQ(state, ExecutionState::DONE); + EXPECT_TRUE(skipped.nothingSkipped()); + EXPECT_EQ(ranges.size(), set.size()); + for (auto [dep, range] : ranges) { + // All Ranges are non empty + validateNextIsDataRow(range, ExecutorState::HASMORE, 7 * (dep + 1)); + validateNextIsDataRow(range, ExecutorState::HASMORE, 8 * (dep + 1)); + validateNextIsDataRow(range, ExecutorState::DONE, 9 * (dep + 1)); } - } // testee is destroyed here - // testee must be destroyed before verify, because it may call returnBlock - // in the destructor - ASSERT_TRUE(dependencyProxyMock.allBlocksFetched()); - ASSERT_EQ(dependencyProxyMock.numFetchBlockCalls(), 2 * numDeps); + } } -TEST_F(MultiDependencySingleRowFetcherTest, - single_upstream_block_with_a_single_row_multi_dependency_the_producer_waits_then_returns_done) { - VPackBuilder input; - size_t numDeps = 3; - MultiDependencyProxyMock<::arangodb::aql::BlockPassthrough::Disable> dependencyProxyMock{monitor, 1, numDeps}; - InputAqlItemRow row{CreateInvalidInputRowHint{}}; - SharedAqlItemBlockPtr blockDep1 = buildBlock<1>(itemBlockManager, {{42}}); - SharedAqlItemBlockPtr blockDep2 = buildBlock<1>(itemBlockManager, {{23}}); - SharedAqlItemBlockPtr blockDep3 = buildBlock<1>(itemBlockManager, {{1337}}); - dependencyProxyMock.getDependencyMock(0) - .shouldReturn(ExecutionState::WAITING, nullptr) - .andThenReturn(ExecutionState::DONE, std::move(blockDep1)); - dependencyProxyMock.getDependencyMock(1) - .shouldReturn(ExecutionState::WAITING, nullptr) - .andThenReturn(ExecutionState::DONE, std::move(blockDep2)); - dependencyProxyMock.getDependencyMock(2) - .shouldReturn(ExecutionState::WAITING, nullptr) - .andThenReturn(ExecutionState::DONE, std::move(blockDep3)); - - { - MultiDependencySingleRowFetcher testee(dependencyProxyMock); - - testee.initDependencies(); - - for (size_t i = 0; i < numDeps; ++i) { - std::tie(state, row) = testee.fetchRowForDependency(i); - ASSERT_EQ(state, ExecutionState::WAITING); - ASSERT_FALSE(row); - } +TEST_P(MultiDependencySingleRowFetcherTest, many_blocks_upstream_all_deps_differ_sequentially) { + // Difference to the test above is that we first fetch all from Dep1, then Dep2 then Dep3 + std::vector> data; + for (size_t i = 0; i < numberDependencies(); ++i) { + std::deque blockDeque{}; + blockDeque.emplace_back( + buildBlock<1>(itemBlockManager, {{1 * (i + 1)}, {2 * (i + 1)}, {3 * (i + 1)}})); + blockDeque.emplace_back( + buildBlock<1>(itemBlockManager, {{4 * (i + 1)}, {5 * (i + 1)}, {6 * (i + 1)}})); + blockDeque.emplace_back( + buildBlock<1>(itemBlockManager, {{7 * (i + 1)}, {8 * (i + 1)}, {9 * (i + 1)}})); + data.emplace_back(std::move(blockDeque)); + } - for (size_t i = 0; i < numDeps; ++i) { - std::tie(state, row) = testee.fetchRowForDependency(i); - ASSERT_EQ(state, ExecutionState::DONE); - ASSERT_TRUE(row); - ASSERT_EQ(row.getNrRegisters(), 1); - if (i == 0) { - ASSERT_EQ(row.getValue(0).slice().getInt(), 42); - } else if (i == 1) { - ASSERT_EQ(row.getValue(0).slice().getInt(), 23); - } else { - ASSERT_EQ(row.getValue(0).slice().getInt(), 1337); + auto testee = buildFetcher(data); + auto stack = makeStack(); + for (size_t dep = 0; dep < numberDependencies(); ++dep) { + AqlCallSet set{}; + set.calls.emplace_back(AqlCallSet::DepCallPair{dep, AqlCallList{AqlCall{}}}); + testWaiting(testee, set); + + { + // First Block + auto [state, skipped, ranges] = testee.execute(stack, set); + EXPECT_EQ(state, ExecutionState::HASMORE); + EXPECT_TRUE(skipped.nothingSkipped()); + EXPECT_EQ(ranges.size(), set.size()); + for (auto [dep, range] : ranges) { + // All Ranges are non empty + validateNextIsDataRow(range, ExecutorState::HASMORE, 1 * (dep + 1)); + validateNextIsDataRow(range, ExecutorState::HASMORE, 2 * (dep + 1)); + validateNextIsDataRow(range, ExecutorState::HASMORE, 3 * (dep + 1)); } } - } // testee is destroyed here - // testee must be destroyed before verify, because it may call returnBlock - // in the destructor - ASSERT_TRUE(dependencyProxyMock.allBlocksFetched()); - ASSERT_EQ(dependencyProxyMock.numFetchBlockCalls(), 2 * numDeps); -} -TEST_F(MultiDependencySingleRowFetcherTest, - single_upstream_block_with_a_single_row_multi_dependency_the_producer_waits_returns_more_then_done) { - VPackBuilder input; - size_t numDeps = 3; - MultiDependencyProxyMock<::arangodb::aql::BlockPassthrough::Disable> dependencyProxyMock{monitor, 1, numDeps}; - InputAqlItemRow row{CreateInvalidInputRowHint{}}; - SharedAqlItemBlockPtr blockDep1 = buildBlock<1>(itemBlockManager, {{42}}); - SharedAqlItemBlockPtr blockDep2 = buildBlock<1>(itemBlockManager, {{23}}); - SharedAqlItemBlockPtr blockDep3 = buildBlock<1>(itemBlockManager, {{1337}}); - dependencyProxyMock.getDependencyMock(0) - .shouldReturn(ExecutionState::WAITING, nullptr) - .andThenReturn(ExecutionState::HASMORE, std::move(blockDep1)) - .andThenReturn(ExecutionState::DONE, nullptr); - dependencyProxyMock.getDependencyMock(1) - .shouldReturn(ExecutionState::WAITING, nullptr) - .andThenReturn(ExecutionState::HASMORE, std::move(blockDep2)) - .andThenReturn(ExecutionState::DONE, nullptr); - dependencyProxyMock.getDependencyMock(2) - .shouldReturn(ExecutionState::WAITING, nullptr) - .andThenReturn(ExecutionState::HASMORE, std::move(blockDep3)) - .andThenReturn(ExecutionState::DONE, nullptr); - - { - MultiDependencySingleRowFetcher testee(dependencyProxyMock); - - testee.initDependencies(); - - for (size_t i = 0; i < numDeps; ++i) { - std::tie(state, row) = testee.fetchRowForDependency(i); - ASSERT_EQ(state, ExecutionState::WAITING); - ASSERT_FALSE(row); + { + // Second Block + auto [state, skipped, ranges] = testee.execute(stack, set); + EXPECT_EQ(state, ExecutionState::HASMORE); + EXPECT_TRUE(skipped.nothingSkipped()); + EXPECT_EQ(ranges.size(), set.size()); + for (auto [dep, range] : ranges) { + // All Ranges are non empty + validateNextIsDataRow(range, ExecutorState::HASMORE, 4 * (dep + 1)); + validateNextIsDataRow(range, ExecutorState::HASMORE, 5 * (dep + 1)); + validateNextIsDataRow(range, ExecutorState::HASMORE, 6 * (dep + 1)); + } } - for (size_t i = 0; i < numDeps; ++i) { - std::tie(state, row) = testee.fetchRowForDependency(i); - ASSERT_EQ(state, ExecutionState::HASMORE); - ASSERT_TRUE(row); - ASSERT_EQ(row.getNrRegisters(), 1); - if (i == 0) { - ASSERT_EQ(row.getValue(0).slice().getInt(), 42); - } else if (i == 1) { - ASSERT_EQ(row.getValue(0).slice().getInt(), 23); + { + // Third Block + auto [state, skipped, ranges] = testee.execute(stack, set); + if (dep + 1 == numberDependencies()) { + // Only the last dependency reports a global DONE + EXPECT_EQ(state, ExecutionState::DONE); } else { - ASSERT_EQ(row.getValue(0).slice().getInt(), 1337); + // All others still report HASMORE on the other parts + EXPECT_EQ(state, ExecutionState::HASMORE); } - } - - for (size_t i = 0; i < numDeps; ++i) { - std::tie(state, row) = testee.fetchRowForDependency(i); - ASSERT_EQ(state, ExecutionState::DONE); - ASSERT_FALSE(row); - } - } // testee is destroyed here - // testee must be destroyed before verify, because it may call returnBlock - // in the destructor - ASSERT_TRUE(dependencyProxyMock.allBlocksFetched()); - ASSERT_EQ(dependencyProxyMock.numFetchBlockCalls(), 3 * numDeps); -} - -// TODO the following tests should be simplified, a simple output -// specification should be compared with the actual output. - -TEST_F(MultiDependencySingleRowFetcherTest, - multiple_blocks_upstream_multiple_dependencies_the_producer_does_not_wait) { - size_t numDeps = 3; - MultiDependencyProxyMock<::arangodb::aql::BlockPassthrough::Disable> dependencyProxyMock{monitor, 1, numDeps}; - InputAqlItemRow row{CreateInvalidInputRowHint{}}; - // three 1-column matrices with 3, 2 and 1 rows, respectively - SharedAqlItemBlockPtr block1Dep1 = - buildBlock<1>(itemBlockManager, {{{1}}, {{2}}, {{3}}}), - block2Dep1 = buildBlock<1>(itemBlockManager, {{{4}}, {{5}}}), - block3Dep1 = buildBlock<1>(itemBlockManager, {{{6}}}); - // two 1-column matrices with 1 and 2 rows, respectively - SharedAqlItemBlockPtr block1Dep2 = buildBlock<1>(itemBlockManager, {{{7}}}), - block2Dep2 = buildBlock<1>(itemBlockManager, {{{8}}, {{9}}}); - // single 1-column matrices with 2 rows - SharedAqlItemBlockPtr block1Dep3 = buildBlock<1>(itemBlockManager, {{{10}}, {{11}}}); - dependencyProxyMock.getDependencyMock(0) - .shouldReturn(ExecutionState::HASMORE, std::move(block1Dep1)) - .andThenReturn(ExecutionState::HASMORE, std::move(block2Dep1)) - .andThenReturn(ExecutionState::DONE, std::move(block3Dep1)); - dependencyProxyMock.getDependencyMock(1) - .shouldReturn(ExecutionState::HASMORE, std::move(block1Dep2)) - .andThenReturn(ExecutionState::DONE, std::move(block2Dep2)); - dependencyProxyMock.getDependencyMock(2).shouldReturn(ExecutionState::DONE, - std::move(block1Dep3)); - - { - MultiDependencySingleRowFetcher testee(dependencyProxyMock); - - testee.initDependencies(); - int64_t rowIdxAndValue; - for (rowIdxAndValue = 1; rowIdxAndValue <= 5; rowIdxAndValue++) { - std::tie(state, row) = testee.fetchRowForDependency(0); - ASSERT_EQ(state, ExecutionState::HASMORE); - ASSERT_TRUE(row); - ASSERT_EQ(row.getNrRegisters(), 1); - ASSERT_EQ(row.getValue(0).slice().getInt(), rowIdxAndValue); - } - rowIdxAndValue = 6; - std::tie(state, row) = testee.fetchRowForDependency(0); - ASSERT_EQ(state, ExecutionState::DONE); - ASSERT_TRUE(row); - ASSERT_EQ(row.getNrRegisters(), 1); - ASSERT_EQ(row.getValue(0).slice().getInt(), rowIdxAndValue); - - for (rowIdxAndValue = 7; rowIdxAndValue <= 8; rowIdxAndValue++) { - std::tie(state, row) = testee.fetchRowForDependency(1); - ASSERT_EQ(state, ExecutionState::HASMORE); - ASSERT_TRUE(row); - ASSERT_EQ(row.getNrRegisters(), 1); - ASSERT_EQ(row.getValue(0).slice().getInt(), rowIdxAndValue); - } - rowIdxAndValue = 9; - std::tie(state, row) = testee.fetchRowForDependency(1); - ASSERT_EQ(state, ExecutionState::DONE); - ASSERT_TRUE(row); - ASSERT_EQ(row.getNrRegisters(), 1); - ASSERT_EQ(row.getValue(0).slice().getInt(), rowIdxAndValue); - - for (rowIdxAndValue = 10; rowIdxAndValue <= 10; rowIdxAndValue++) { - std::tie(state, row) = testee.fetchRowForDependency(2); - ASSERT_EQ(state, ExecutionState::HASMORE); - ASSERT_TRUE(row); - ASSERT_EQ(row.getNrRegisters(), 1); - ASSERT_EQ(row.getValue(0).slice().getInt(), rowIdxAndValue); + EXPECT_TRUE(skipped.nothingSkipped()); + EXPECT_EQ(ranges.size(), set.size()); + for (auto [dep, range] : ranges) { + // All Ranges are non empty + validateNextIsDataRow(range, ExecutorState::HASMORE, 7 * (dep + 1)); + validateNextIsDataRow(range, ExecutorState::HASMORE, 8 * (dep + 1)); + validateNextIsDataRow(range, ExecutorState::DONE, 9 * (dep + 1)); + } } - rowIdxAndValue = 11; - std::tie(state, row) = testee.fetchRowForDependency(2); - ASSERT_EQ(state, ExecutionState::DONE); - ASSERT_TRUE(row); - ASSERT_EQ(row.getNrRegisters(), 1); - ASSERT_EQ(row.getValue(0).slice().getInt(), rowIdxAndValue); - } // testee is destroyed here - // testee must be destroyed before verify, because it may call returnBlock - // in the destructor - ASSERT_TRUE(dependencyProxyMock.allBlocksFetched()); - ASSERT_EQ(dependencyProxyMock.numFetchBlockCalls(), 3 + 2 + 1); + } } -TEST_F(MultiDependencySingleRowFetcherTest, - multiple_blocks_upstream_multiple_dependencies_the_producer_waits) { - size_t numDeps = 3; - MultiDependencyProxyMock<::arangodb::aql::BlockPassthrough::Disable> dependencyProxyMock{monitor, 1, numDeps}; - InputAqlItemRow row{CreateInvalidInputRowHint{}}; - // three 1-column matrices with 3, 2 and 1 rows, respectively - SharedAqlItemBlockPtr block1Dep1 = - buildBlock<1>(itemBlockManager, {{{1}}, {{2}}, {{3}}}), - block2Dep1 = buildBlock<1>(itemBlockManager, {{{4}}, {{5}}}), - block3Dep1 = buildBlock<1>(itemBlockManager, {{{6}}}); - // two 1-column matrices with 1 and 2 rows, respectively - SharedAqlItemBlockPtr block1Dep2 = buildBlock<1>(itemBlockManager, {{{7}}}), - block2Dep2 = buildBlock<1>(itemBlockManager, {{{8}}, {{9}}}); - // single 1-column matrices with 2 rows - SharedAqlItemBlockPtr block1Dep3 = buildBlock<1>(itemBlockManager, {{{10}}, {{11}}}); - dependencyProxyMock.getDependencyMock(0) - .shouldReturn(ExecutionState::WAITING, nullptr) - .andThenReturn(ExecutionState::HASMORE, std::move(block1Dep1)) - .andThenReturn(ExecutionState::WAITING, nullptr) - .andThenReturn(ExecutionState::HASMORE, std::move(block2Dep1)) - .andThenReturn(ExecutionState::WAITING, nullptr) - .andThenReturn(ExecutionState::DONE, std::move(block3Dep1)); - dependencyProxyMock.getDependencyMock(1) - .shouldReturn(ExecutionState::WAITING, nullptr) - .andThenReturn(ExecutionState::HASMORE, std::move(block1Dep2)) - .andThenReturn(ExecutionState::WAITING, nullptr) - .andThenReturn(ExecutionState::DONE, std::move(block2Dep2)); - dependencyProxyMock.getDependencyMock(2) - .shouldReturn(ExecutionState::WAITING, nullptr) - .andThenReturn(ExecutionState::DONE, std::move(block1Dep3)); - - { - MultiDependencySingleRowFetcher testee(dependencyProxyMock); - - testee.initDependencies(); +TEST_P(MultiDependencySingleRowFetcherTest, + many_blocks_upstream_all_deps_differ_sequentially_using_shadowRows_no_callList) { + // NOTE: The fetcher does NOT care to synchronize the shadowRows between blocks. + // This has to be done by the InputRange hold externally. + // It has seperate tests + std::vector> data; + for (size_t i = 0; i < numberDependencies(); ++i) { + std::deque blockDeque{}; + blockDeque.emplace_back( + buildBlock<1>(itemBlockManager, + {{1 * (i + 1)}, {2 * (i + 1)}, {0}, {3 * (i + 1)}}, {{2, 0}})); + blockDeque.emplace_back( + buildBlock<1>(itemBlockManager, + {{4 * (i + 1)}, {5 * (i + 1)}, {1}, {2}, {6 * (i + 1)}}, + {{2, 0}, {3, 1}})); + blockDeque.emplace_back( + buildBlock<1>(itemBlockManager, {{7 * (i + 1)}, {8 * (i + 1)}, {9 * (i + 1)}})); + data.emplace_back(std::move(blockDeque)); + } - int64_t rowIdxAndValue; - for (rowIdxAndValue = 1; rowIdxAndValue <= 5; rowIdxAndValue++) { - if (rowIdxAndValue == 1 || rowIdxAndValue == 4) { - // wait at the beginning of the 1st and 2nd block - std::tie(state, row) = testee.fetchRowForDependency(0); - ASSERT_EQ(state, ExecutionState::WAITING); - ASSERT_FALSE(row); + auto testee = buildFetcher(data); + auto stack = makeStack(); + for (size_t dep = 0; dep < numberDependencies(); ++dep) { + AqlCallSet set{}; + set.calls.emplace_back(AqlCallSet::DepCallPair{dep, AqlCallList{AqlCall{}}}); + testWaiting(testee, set); + + { + // First Block, split after shadowRow, we cannot overfetch + auto [state, skipped, ranges] = testee.execute(stack, set); + EXPECT_EQ(state, ExecutionState::HASMORE); + EXPECT_TRUE(skipped.nothingSkipped()); + EXPECT_EQ(ranges.size(), set.size()); + for (auto [dep, range] : ranges) { + // All Ranges are non empty + validateNextIsDataRow(range, ExecutorState::HASMORE, 1 * (dep + 1)); + validateNextIsDataRow(range, ExecutorState::DONE, 2 * (dep + 1)); + validateNextIsShadowRow(range, ExecutorState::HASMORE, 0, 0); } - std::tie(state, row) = testee.fetchRowForDependency(0); - ASSERT_EQ(state, ExecutionState::HASMORE); - ASSERT_TRUE(row); - ASSERT_EQ(row.getNrRegisters(), 1); - ASSERT_EQ(row.getValue(0).slice().getInt(), rowIdxAndValue); } - rowIdxAndValue = 6; - // wait at the beginning of the 3rd block - std::tie(state, row) = testee.fetchRowForDependency(0); - ASSERT_EQ(state, ExecutionState::WAITING); - ASSERT_FALSE(row); - // last row and DONE - std::tie(state, row) = testee.fetchRowForDependency(0); - ASSERT_EQ(state, ExecutionState::DONE); - ASSERT_TRUE(row); - ASSERT_EQ(row.getNrRegisters(), 1); - ASSERT_EQ(row.getValue(0).slice().getInt(), rowIdxAndValue); - - for (rowIdxAndValue = 7; rowIdxAndValue <= 8; rowIdxAndValue++) { - if (rowIdxAndValue == 7 || rowIdxAndValue == 8) { - // wait at the beginning of the 1st and 2nd block - std::tie(state, row) = testee.fetchRowForDependency(1); - ASSERT_EQ(state, ExecutionState::WAITING); - ASSERT_FALSE(row); + + { + // First Block, part 2 + auto [state, skipped, ranges] = testee.execute(stack, set); + EXPECT_EQ(state, ExecutionState::HASMORE); + EXPECT_TRUE(skipped.nothingSkipped()); + EXPECT_EQ(ranges.size(), set.size()); + for (auto [dep, range] : ranges) { + validateNextIsDataRow(range, ExecutorState::HASMORE, 3 * (dep + 1)); } - std::tie(state, row) = testee.fetchRowForDependency(1); - ASSERT_EQ(state, ExecutionState::HASMORE); - ASSERT_TRUE(row); - ASSERT_EQ(row.getNrRegisters(), 1); - ASSERT_EQ(row.getValue(0).slice().getInt(), rowIdxAndValue); } - rowIdxAndValue = 9; - std::tie(state, row) = testee.fetchRowForDependency(1); - ASSERT_EQ(state, ExecutionState::DONE); - ASSERT_TRUE(row); - ASSERT_EQ(row.getNrRegisters(), 1); - ASSERT_EQ(row.getValue(0).slice().getInt(), rowIdxAndValue); - - for (rowIdxAndValue = 10; rowIdxAndValue <= 10; rowIdxAndValue++) { - if (rowIdxAndValue == 10) { - // wait at the beginning of the 1st and 2nd block - std::tie(state, row) = testee.fetchRowForDependency(2); - ASSERT_EQ(state, ExecutionState::WAITING); - ASSERT_FALSE(row); + + { + // Second Block, split after the higher depth shadow row + auto [state, skipped, ranges] = testee.execute(stack, set); + EXPECT_EQ(state, ExecutionState::HASMORE); + EXPECT_TRUE(skipped.nothingSkipped()); + EXPECT_EQ(ranges.size(), set.size()); + for (auto [dep, range] : ranges) { + // All Ranges are non empty + validateNextIsDataRow(range, ExecutorState::HASMORE, 4 * (dep + 1)); + validateNextIsDataRow(range, ExecutorState::DONE, 5 * (dep + 1)); + validateNextIsShadowRow(range, ExecutorState::HASMORE, 1, 0); + validateNextIsShadowRow(range, ExecutorState::HASMORE, 2, 1); } - std::tie(state, row) = testee.fetchRowForDependency(2); - ASSERT_EQ(state, ExecutionState::HASMORE); - ASSERT_TRUE(row); - ASSERT_EQ(row.getNrRegisters(), 1); - ASSERT_EQ(row.getValue(0).slice().getInt(), rowIdxAndValue); } - rowIdxAndValue = 11; - std::tie(state, row) = testee.fetchRowForDependency(2); - ASSERT_EQ(state, ExecutionState::DONE); - ASSERT_TRUE(row); - ASSERT_EQ(row.getNrRegisters(), 1); - ASSERT_EQ(row.getValue(0).slice().getInt(), rowIdxAndValue); - } // testee is destroyed here - // testee must be destroyed before verify, because it may call returnBlock - // in the destructor - ASSERT_TRUE(dependencyProxyMock.allBlocksFetched()); - ASSERT_EQ(dependencyProxyMock.numFetchBlockCalls(), 12); -} - -TEST_F(MultiDependencySingleRowFetcherTest, - multiple_blocks_upstream_multiple_dependencies_the_producer_waits_and_doesnt_return_done_asap) { - size_t numDeps = 3; - MultiDependencyProxyMock<::arangodb::aql::BlockPassthrough::Disable> dependencyProxyMock{monitor, 1, numDeps}; - InputAqlItemRow row{CreateInvalidInputRowHint{}}; - // three 1-column matrices with 3, 2 and 1 rows, respectively - SharedAqlItemBlockPtr block1Dep1 = - buildBlock<1>(itemBlockManager, {{{1}}, {{2}}, {{3}}}), - block2Dep1 = buildBlock<1>(itemBlockManager, {{{4}}, {{5}}}), - block3Dep1 = buildBlock<1>(itemBlockManager, {{{6}}}); - // two 1-column matrices with 1 and 2 rows, respectively - SharedAqlItemBlockPtr block1Dep2 = buildBlock<1>(itemBlockManager, {{{7}}}), - block2Dep2 = buildBlock<1>(itemBlockManager, {{{8}}, {{9}}}); - // single 1-column matrices with 2 rows - SharedAqlItemBlockPtr block1Dep3 = buildBlock<1>(itemBlockManager, {{{10}}, {{11}}}); - dependencyProxyMock.getDependencyMock(0) - .shouldReturn(ExecutionState::WAITING, nullptr) - .andThenReturn(ExecutionState::HASMORE, std::move(block1Dep1)) - .andThenReturn(ExecutionState::WAITING, nullptr) - .andThenReturn(ExecutionState::HASMORE, std::move(block2Dep1)) - .andThenReturn(ExecutionState::WAITING, nullptr) - .andThenReturn(ExecutionState::HASMORE, std::move(block3Dep1)) - .andThenReturn(ExecutionState::DONE, nullptr); - dependencyProxyMock.getDependencyMock(1) - .shouldReturn(ExecutionState::WAITING, nullptr) - .andThenReturn(ExecutionState::HASMORE, std::move(block1Dep2)) - .andThenReturn(ExecutionState::WAITING, nullptr) - .andThenReturn(ExecutionState::HASMORE, std::move(block2Dep2)) - .andThenReturn(ExecutionState::DONE, nullptr); - dependencyProxyMock.getDependencyMock(2) - .shouldReturn(ExecutionState::WAITING, nullptr) - .andThenReturn(ExecutionState::HASMORE, std::move(block1Dep3)) - .andThenReturn(ExecutionState::DONE, nullptr); - - { - MultiDependencySingleRowFetcher testee(dependencyProxyMock); - - testee.initDependencies(); - int64_t rowIdxAndValue; - for (rowIdxAndValue = 1; rowIdxAndValue <= 5; rowIdxAndValue++) { - if (rowIdxAndValue == 1 || rowIdxAndValue == 4) { - // wait at the beginning of the 1st and 2nd block - std::tie(state, row) = testee.fetchRowForDependency(0); - ASSERT_EQ(state, ExecutionState::WAITING); - ASSERT_FALSE(row); + { + // Second Block, part 2 + auto [state, skipped, ranges] = testee.execute(stack, set); + EXPECT_EQ(state, ExecutionState::HASMORE); + EXPECT_TRUE(skipped.nothingSkipped()); + EXPECT_EQ(ranges.size(), set.size()); + for (auto [dep, range] : ranges) { + // All Ranges are non empty + validateNextIsDataRow(range, ExecutorState::HASMORE, 6 * (dep + 1)); } - std::tie(state, row) = testee.fetchRowForDependency(0); - ASSERT_EQ(state, ExecutionState::HASMORE); - ASSERT_TRUE(row); - ASSERT_EQ(row.getNrRegisters(), 1); - ASSERT_EQ(row.getValue(0).slice().getInt(), rowIdxAndValue); } - rowIdxAndValue = 6; - // wait at the beginning of the 3rd block - std::tie(state, row) = testee.fetchRowForDependency(0); - ASSERT_EQ(state, ExecutionState::WAITING); - ASSERT_FALSE(row); - // last row and DONE - std::tie(state, row) = testee.fetchRowForDependency(0); - ASSERT_EQ(state, ExecutionState::HASMORE); - ASSERT_TRUE(row); - ASSERT_EQ(row.getNrRegisters(), 1); - ASSERT_EQ(row.getValue(0).slice().getInt(), rowIdxAndValue); - std::tie(state, row) = testee.fetchRowForDependency(0); - ASSERT_EQ(state, ExecutionState::DONE); - ASSERT_FALSE(row); - - for (rowIdxAndValue = 7; rowIdxAndValue <= 8; rowIdxAndValue++) { - if (rowIdxAndValue == 7 || rowIdxAndValue == 8) { - // wait at the beginning of the 1st and 2nd block - std::tie(state, row) = testee.fetchRowForDependency(1); - ASSERT_EQ(state, ExecutionState::WAITING); - ASSERT_FALSE(row); + + { + // Third Block + auto [state, skipped, ranges] = testee.execute(stack, set); + if (dep + 1 == numberDependencies()) { + // Only the last dependency reports a global DONE + EXPECT_EQ(state, ExecutionState::DONE); + } else { + // All others still report HASMORE on the other parts + EXPECT_EQ(state, ExecutionState::HASMORE); } - std::tie(state, row) = testee.fetchRowForDependency(1); - ASSERT_EQ(state, ExecutionState::HASMORE); - ASSERT_TRUE(row); - ASSERT_EQ(row.getNrRegisters(), 1); - ASSERT_EQ(row.getValue(0).slice().getInt(), rowIdxAndValue); - } - rowIdxAndValue = 9; - std::tie(state, row) = testee.fetchRowForDependency(1); - ASSERT_EQ(state, ExecutionState::HASMORE); - ASSERT_TRUE(row); - ASSERT_EQ(row.getNrRegisters(), 1); - ASSERT_EQ(row.getValue(0).slice().getInt(), rowIdxAndValue); - std::tie(state, row) = testee.fetchRowForDependency(1); - ASSERT_EQ(state, ExecutionState::DONE); - ASSERT_FALSE(row); - - for (rowIdxAndValue = 10; rowIdxAndValue <= 10; rowIdxAndValue++) { - if (rowIdxAndValue == 10) { - // wait at the beginning of the 1st and 2nd block - std::tie(state, row) = testee.fetchRowForDependency(2); - ASSERT_EQ(state, ExecutionState::WAITING); - ASSERT_FALSE(row); + + EXPECT_TRUE(skipped.nothingSkipped()); + EXPECT_EQ(ranges.size(), set.size()); + for (auto [dep, range] : ranges) { + // All Ranges are non empty + validateNextIsDataRow(range, ExecutorState::HASMORE, 7 * (dep + 1)); + validateNextIsDataRow(range, ExecutorState::HASMORE, 8 * (dep + 1)); + validateNextIsDataRow(range, ExecutorState::DONE, 9 * (dep + 1)); } - std::tie(state, row) = testee.fetchRowForDependency(2); - ASSERT_EQ(state, ExecutionState::HASMORE); - ASSERT_TRUE(row); - ASSERT_EQ(row.getNrRegisters(), 1); - ASSERT_EQ(row.getValue(0).slice().getInt(), rowIdxAndValue); } - rowIdxAndValue = 11; - std::tie(state, row) = testee.fetchRowForDependency(2); - ASSERT_EQ(state, ExecutionState::HASMORE); - ASSERT_TRUE(row); - ASSERT_EQ(row.getNrRegisters(), 1); - ASSERT_EQ(row.getValue(0).slice().getInt(), rowIdxAndValue); - std::tie(state, row) = testee.fetchRowForDependency(2); - ASSERT_EQ(state, ExecutionState::DONE); - ASSERT_FALSE(row); - } // testee is destroyed here - // testee must be destroyed before verify, because it may call returnBlock - // in the destructor - ASSERT_TRUE(dependencyProxyMock.allBlocksFetched()); - ASSERT_EQ(dependencyProxyMock.numFetchBlockCalls(), 15); + } } -using CutAt = uint64_t; - -class MultiDependencySingleRowFetcherShadowRowTest : public testing::TestWithParam { - protected: - ResourceMonitor monitor{}; - AqlItemBlockManager itemBlockManager; - - MultiDependencySingleRowFetcherShadowRowTest() - : itemBlockManager(&monitor, SerializationFormat::SHADOWROWS) {} - - uint64_t cutAt() const { return GetParam(); } - - std::vector> alternatingDataAndShadowRows( - std::vector const& values) { - MatrixBuilder<1> matrixBuilder; - for (auto const& val : values) { - matrixBuilder.emplace_back(RowBuilder<1>{{val}}); - } - SharedAqlItemBlockPtr block = - buildBlock<1>(itemBlockManager, std::move(matrixBuilder)); +TEST_P(MultiDependencySingleRowFetcherTest, + many_blocks_upstream_all_deps_differ_sequentially_using_shadowRows_no_callList_reverse_order) { + // NOTE: The fetcher does NOT care to synchronize the shadowRows between blocks. + // This has to be done by the InputRange hold externally. + // It has seperate tests + std::vector> data; + for (size_t i = 0; i < numberDependencies(); ++i) { + std::deque blockDeque{}; + blockDeque.emplace_back( + buildBlock<1>(itemBlockManager, + {{1 * (i + 1)}, {2 * (i + 1)}, {0}, {3 * (i + 1)}}, {{2, 0}})); + blockDeque.emplace_back( + buildBlock<1>(itemBlockManager, + {{4 * (i + 1)}, {5 * (i + 1)}, {1}, {2}, {6 * (i + 1)}}, + {{2, 0}, {3, 1}})); + blockDeque.emplace_back( + buildBlock<1>(itemBlockManager, {{7 * (i + 1)}, {8 * (i + 1)}, {9 * (i + 1)}})); + data.emplace_back(std::move(blockDeque)); + } - for (size_t row = 0; row < block->size(); ++row) { - if (row % 2 == 1) { - block->setShadowRowDepth(row, AqlValue{AqlValueHintUInt{0}}); + auto testee = buildFetcher(data); + auto stack = makeStack(); + for (size_t depCounter = numberDependencies(); depCounter > 0; --depCounter) { + TRI_ASSERT(depCounter > 0); + auto dep = depCounter - 1; + AqlCallSet set{}; + set.calls.emplace_back(AqlCallSet::DepCallPair{dep, AqlCallList{AqlCall{}}}); + testWaiting(testee, set); + + { + // First Block, split after shadowRow, we cannot overfetch + auto [state, skipped, ranges] = testee.execute(stack, set); + EXPECT_EQ(state, ExecutionState::HASMORE); + EXPECT_TRUE(skipped.nothingSkipped()); + EXPECT_EQ(ranges.size(), set.size()); + for (auto [dep, range] : ranges) { + // All Ranges are non empty + validateNextIsDataRow(range, ExecutorState::HASMORE, 1 * (dep + 1)); + validateNextIsDataRow(range, ExecutorState::DONE, 2 * (dep + 1)); + validateNextIsShadowRow(range, ExecutorState::HASMORE, 0, 0); } } - std::vector> result; - - if (cutAt() != 0 && cutAt() < block->size()) { - SharedAqlItemBlockPtr block1 = block->slice(0, cutAt()); - SharedAqlItemBlockPtr block2 = block->slice(cutAt(), block->size()); - result.emplace_back(ExecutionState::HASMORE, block1); - result.emplace_back(ExecutionState::DONE, block2); - } else { - result.emplace_back(ExecutionState::DONE, block); + { + // First Block, part 2 + auto [state, skipped, ranges] = testee.execute(stack, set); + EXPECT_EQ(state, ExecutionState::HASMORE); + EXPECT_TRUE(skipped.nothingSkipped()); + EXPECT_EQ(ranges.size(), set.size()); + for (auto [dep, range] : ranges) { + validateNextIsDataRow(range, ExecutorState::HASMORE, 3 * (dep + 1)); + } } - return result; - } - - std::vector> onlyShadowRows( - std::vector const& values) { - MatrixBuilder<1> matrixBuilder; - for (auto const& val : values) { - matrixBuilder.emplace_back(RowBuilder<1>{{val}}); + { + // Second Block, split after the higher depth shadow row + auto [state, skipped, ranges] = testee.execute(stack, set); + EXPECT_EQ(state, ExecutionState::HASMORE); + EXPECT_TRUE(skipped.nothingSkipped()); + EXPECT_EQ(ranges.size(), set.size()); + for (auto [dep, range] : ranges) { + // All Ranges are non empty + validateNextIsDataRow(range, ExecutorState::HASMORE, 4 * (dep + 1)); + validateNextIsDataRow(range, ExecutorState::DONE, 5 * (dep + 1)); + validateNextIsShadowRow(range, ExecutorState::HASMORE, 1, 0); + validateNextIsShadowRow(range, ExecutorState::HASMORE, 2, 1); + } } - SharedAqlItemBlockPtr block = - buildBlock<1>(itemBlockManager, std::move(matrixBuilder)); - for (size_t row = 0; row < block->size(); ++row) { - block->setShadowRowDepth(1, AqlValue{AqlValueHintUInt{0}}); + { + // Second Block, part 2 + auto [state, skipped, ranges] = testee.execute(stack, set); + EXPECT_EQ(state, ExecutionState::HASMORE); + EXPECT_TRUE(skipped.nothingSkipped()); + EXPECT_EQ(ranges.size(), set.size()); + for (auto [dep, range] : ranges) { + // All Ranges are non empty + validateNextIsDataRow(range, ExecutorState::HASMORE, 6 * (dep + 1)); + } } - // TODO cut block into pieces - return {{ExecutionState::DONE, std::move(block)}}; - } - - // Get a row pointing to a row with the specified value in the only register - // in an anonymous block. - InputAqlItemRow inputRow(int value) { - SharedAqlItemBlockPtr block = buildBlock<1>(itemBlockManager, {{{value}}}); - return InputAqlItemRow{block, 0}; - } - // Get a shadow row pointing to a row with the specified value, and specified - // shadow row depth, in the only register in an anonymous block. - ShadowAqlItemRow shadowRow(int value, uint64_t depth = 0) { - SharedAqlItemBlockPtr block = buildBlock<1>(itemBlockManager, {{{value}}}); - block->setShadowRowDepth(0, AqlValue{AqlValueHintUInt{depth}}); - return ShadowAqlItemRow{block, 0}; - } - - InputAqlItemRow invalidInputRow() const { - return InputAqlItemRow{CreateInvalidInputRowHint{}}; - } - - ShadowAqlItemRow invalidShadowRow() const { - return ShadowAqlItemRow{CreateInvalidShadowRowHint{}}; - } -}; + { + // Third Block + auto [state, skipped, ranges] = testee.execute(stack, set); + if (dep == 0) { + // Only the last dependency reports a global DONE + EXPECT_EQ(state, ExecutionState::DONE); + } else { + // All others still report HASMORE on the other parts + EXPECT_EQ(state, ExecutionState::HASMORE); + } -INSTANTIATE_TEST_CASE_P(MultiDependencySingleRowFetcherShadowRowTestInstance, - MultiDependencySingleRowFetcherShadowRowTest, - testing::Range(static_cast(0), static_cast(4))); - -TEST_P(MultiDependencySingleRowFetcherShadowRowTest, simple_fetch_shadow_row_test) { - constexpr size_t numDeps = 1; - MultiDependencyProxyMock dependencyProxyMock{monitor, 1, numDeps}; - - dependencyProxyMock.getDependencyMock(0).shouldReturn( - alternatingDataAndShadowRows({0, 1, 2, 3})); - - MultiDependencySingleRowFetcher testee{dependencyProxyMock}; - testee.initDependencies(); - - auto ioPairs = std::vector{}; - auto add = [&ioPairs](auto call, auto result) { - ioPairs.emplace_back(std::make_pair(call, result)); - }; - - if (cutAt() == 1) { - add(FetchRowForDependency{0, 1000}, - FetchRowForDependency::Result{ExecutionState::HASMORE, inputRow(0)}); - add(FetchRowForDependency{0, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, invalidInputRow()}); - } else { - add(FetchRowForDependency{0, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, inputRow(0)}); - } - add(FetchRowForDependency{0, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, invalidInputRow()}); - add(FetchShadowRow{1000}, FetchShadowRow::Result{ExecutionState::HASMORE, shadowRow(1)}); - if (cutAt() == 3) { - add(FetchRowForDependency{0, 1000}, - FetchRowForDependency::Result{ExecutionState::HASMORE, inputRow(2)}); - add(FetchRowForDependency{0, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, invalidInputRow()}); - } else { - add(FetchRowForDependency{0, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, inputRow(2)}); + EXPECT_TRUE(skipped.nothingSkipped()); + EXPECT_EQ(ranges.size(), set.size()); + for (auto [dep, range] : ranges) { + // All Ranges are non empty + validateNextIsDataRow(range, ExecutorState::HASMORE, 7 * (dep + 1)); + validateNextIsDataRow(range, ExecutorState::HASMORE, 8 * (dep + 1)); + validateNextIsDataRow(range, ExecutorState::DONE, 9 * (dep + 1)); + } + } } - add(FetchRowForDependency{0, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, invalidInputRow()}); - add(FetchShadowRow{1000}, FetchShadowRow::Result{ExecutionState::HASMORE, shadowRow(3)}); - add(FetchRowForDependency{0, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, invalidInputRow()}); - add(FetchShadowRow{1000}, - FetchShadowRow::Result{ExecutionState::DONE, invalidShadowRow()}); - - runFetcher(testee, ioPairs); } -TEST_P(MultiDependencySingleRowFetcherShadowRowTest, fetch_shadow_rows_2_deps) { - constexpr size_t numDeps = 2; - MultiDependencyProxyMock dependencyProxyMock{monitor, 1, numDeps}; - - dependencyProxyMock.getDependencyMock(0).shouldReturn( - alternatingDataAndShadowRows({0, 1, 2, 3})); - dependencyProxyMock.getDependencyMock(1).shouldReturn( - alternatingDataAndShadowRows({4, 1, 6, 3})); - - MultiDependencySingleRowFetcher testee{dependencyProxyMock}; - testee.initDependencies(); - - auto ioPairs = std::vector{}; - auto add = [&ioPairs](auto call, auto result) { - ioPairs.emplace_back(std::make_pair(call, result)); - }; - - // fetch dep 1 - if (cutAt() == 1) { - add(FetchRowForDependency{0, 1000}, - FetchRowForDependency::Result{ExecutionState::HASMORE, inputRow(0)}); - add(FetchRowForDependency{0, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, invalidInputRow()}); - } else { - add(FetchRowForDependency{0, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, inputRow(0)}); +TEST_P(MultiDependencySingleRowFetcherTest, + many_blocks_upstream_all_deps_differ_sequentially_using_shadowRows_no_callList_offset) { + // NOTE: The fetcher does NOT care to synchronize the shadowRows between blocks. + // This has to be done by the InputRange hold externally. + // It has seperate tests + std::vector> data; + for (size_t i = 0; i < numberDependencies(); ++i) { + std::deque blockDeque{}; + blockDeque.emplace_back( + buildBlock<1>(itemBlockManager, + {{1 * (i + 1)}, {2 * (i + 1)}, {0}, {3 * (i + 1)}}, {{2, 0}})); + blockDeque.emplace_back( + buildBlock<1>(itemBlockManager, + {{4 * (i + 1)}, {5 * (i + 1)}, {1}, {2}, {6 * (i + 1)}}, + {{2, 0}, {3, 1}})); + blockDeque.emplace_back( + buildBlock<1>(itemBlockManager, {{7 * (i + 1)}, {8 * (i + 1)}, {9 * (i + 1)}})); + data.emplace_back(std::move(blockDeque)); } - // dep 1 should stay done - add(FetchRowForDependency{0, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, invalidInputRow()}); - // fetching the shadow row should not yet be possible - add(FetchShadowRow{1000}, - FetchShadowRow::Result{ExecutionState::HASMORE, invalidShadowRow()}); - // dep 1 should stay done - add(FetchRowForDependency{0, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, invalidInputRow()}); - // fetch dep 2 - if (cutAt() == 1) { - add(FetchRowForDependency{1, 1000}, - FetchRowForDependency::Result{ExecutionState::HASMORE, inputRow(4)}); - add(FetchRowForDependency{1, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, invalidInputRow()}); - } else { - add(FetchRowForDependency{1, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, inputRow(4)}); - } - // dep 2 should stay done - add(FetchRowForDependency{1, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, invalidInputRow()}); - // dep 1 should stay done - add(FetchRowForDependency{0, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, invalidInputRow()}); - // Fetch the first shadow row. - add(FetchShadowRow{1000}, FetchShadowRow::Result{ExecutionState::HASMORE, shadowRow(1)}); - // fetch dep 1 again - if (cutAt() == 3) { - add(FetchRowForDependency{0, 1000}, - FetchRowForDependency::Result{ExecutionState::HASMORE, inputRow(2)}); - add(FetchRowForDependency{0, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, invalidInputRow()}); - } else { - add(FetchRowForDependency{0, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, inputRow(2)}); - } - // dep 1 should stay done - add(FetchRowForDependency{0, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, invalidInputRow()}); - // fetching the shadow row should not yet be possible - add(FetchShadowRow{1000}, - FetchShadowRow::Result{ExecutionState::HASMORE, invalidShadowRow()}); - // dep 1 should stay done - add(FetchRowForDependency{0, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, invalidInputRow()}); - // fetch dep 2 again - if (cutAt() == 3) { - add(FetchRowForDependency{1, 1000}, - FetchRowForDependency::Result{ExecutionState::HASMORE, inputRow(6)}); - add(FetchRowForDependency{1, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, invalidInputRow()}); - } else { - add(FetchRowForDependency{1, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, inputRow(6)}); - } - // dep 2 should stay done - add(FetchRowForDependency{1, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, invalidInputRow()}); - // dep 1 should stay done - add(FetchRowForDependency{0, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, invalidInputRow()}); - // Fetch the second shadow row. - add(FetchShadowRow{1000}, FetchShadowRow::Result{ExecutionState::HASMORE, shadowRow(3)}); - // We're now done. - add(FetchShadowRow{1000}, - FetchShadowRow::Result{ExecutionState::DONE, invalidShadowRow()}); - - runFetcher(testee, ioPairs); -} -TEST_P(MultiDependencySingleRowFetcherShadowRowTest, fetch_shadow_rows_2_deps_reverse_pull) { - constexpr size_t numDeps = 2; - MultiDependencyProxyMock dependencyProxyMock{monitor, 1, numDeps}; - - dependencyProxyMock.getDependencyMock(0).shouldReturn( - alternatingDataAndShadowRows({0, 1, 2, 3})); - dependencyProxyMock.getDependencyMock(1).shouldReturn( - alternatingDataAndShadowRows({4, 1, 6, 3})); - - MultiDependencySingleRowFetcher testee{dependencyProxyMock}; - testee.initDependencies(); - - auto ioPairs = std::vector{}; - auto add = [&ioPairs](auto call, auto result) { - ioPairs.emplace_back(std::make_pair(call, result)); - }; - - // fetch dep 2 - if (cutAt() == 1) { - add(FetchRowForDependency{1, 1000}, - FetchRowForDependency::Result{ExecutionState::HASMORE, inputRow(4)}); - add(FetchRowForDependency{1, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, invalidInputRow()}); - } else { - add(FetchRowForDependency{1, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, inputRow(4)}); - } - // dep 2 should stay done - add(FetchRowForDependency{1, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, invalidInputRow()}); - // fetching the shadow row should not yet be possible - add(FetchShadowRow{1000}, - FetchShadowRow::Result{ExecutionState::HASMORE, invalidShadowRow()}); - // dep 2 should stay done - add(FetchRowForDependency{1, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, invalidInputRow()}); - // fetch dep 1 - if (cutAt() == 1) { - add(FetchRowForDependency{0, 1000}, - FetchRowForDependency::Result{ExecutionState::HASMORE, inputRow(0)}); - add(FetchRowForDependency{0, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, invalidInputRow()}); - } else { - add(FetchRowForDependency{0, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, inputRow(0)}); - } - // dep 1 should stay done - add(FetchRowForDependency{0, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, invalidInputRow()}); - // dep 2 should stay done - add(FetchRowForDependency{1, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, invalidInputRow()}); - // Fetch the first shadow row. - add(FetchShadowRow{1000}, FetchShadowRow::Result{ExecutionState::HASMORE, shadowRow(1)}); - // fetch dep 2 again - if (cutAt() == 3) { - add(FetchRowForDependency{1, 1000}, - FetchRowForDependency::Result{ExecutionState::HASMORE, inputRow(6)}); - add(FetchRowForDependency{1, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, invalidInputRow()}); - } else { - add(FetchRowForDependency{1, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, inputRow(6)}); - } - // dep 2 should stay done - add(FetchRowForDependency{1, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, invalidInputRow()}); - // fetching the shadow row should not yet be possible - add(FetchShadowRow{1000}, - FetchShadowRow::Result{ExecutionState::HASMORE, invalidShadowRow()}); - // dep 2 should stay done - add(FetchRowForDependency{1, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, invalidInputRow()}); - // fetch dep 1 again - if (cutAt() == 3) { - add(FetchRowForDependency{0, 1000}, - FetchRowForDependency::Result{ExecutionState::HASMORE, inputRow(2)}); - add(FetchRowForDependency{0, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, invalidInputRow()}); - } else { - add(FetchRowForDependency{0, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, inputRow(2)}); - } - // dep 2 should stay done - add(FetchRowForDependency{1, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, invalidInputRow()}); - // dep 1 should stay done - add(FetchRowForDependency{0, 1000}, - FetchRowForDependency::Result{ExecutionState::DONE, invalidInputRow()}); - // Fetch the second shadow row. - add(FetchShadowRow{1000}, FetchShadowRow::Result{ExecutionState::HASMORE, shadowRow(3)}); - // We're now done. - add(FetchShadowRow{1000}, - FetchShadowRow::Result{ExecutionState::DONE, invalidShadowRow()}); - - runFetcher(testee, ioPairs); -} + auto testee = buildFetcher(data); + auto stack = makeStack(); + for (size_t dep = 0; dep < numberDependencies(); ++dep) { + AqlCallSet set{}; + // offset 10 + set.calls.emplace_back(AqlCallSet::DepCallPair{dep, AqlCallList{AqlCall{10}}}); + testWaiting(testee, set); + + { + // First Block, split after shadowRow, we cannot overfetch + auto [state, skipped, ranges] = testee.execute(stack, set); + EXPECT_EQ(state, ExecutionState::HASMORE); + EXPECT_FALSE(skipped.nothingSkipped()); + EXPECT_EQ(skipped.getSkipCount(), 2); + EXPECT_EQ(ranges.size(), set.size()); + for (auto [dep, range] : ranges) { + // All Ranges are non empty + validateNextIsShadowRow(range, ExecutorState::HASMORE, 0, 0); + } + } -TEST_P(MultiDependencySingleRowFetcherShadowRowTest, simple_skip_shadow_row_test) { - constexpr size_t numDeps = 1; - MultiDependencyProxyMock dependencyProxyMock{monitor, 1, numDeps}; - - dependencyProxyMock.getDependencyMock(0).shouldReturn( - alternatingDataAndShadowRows({0, 1, 2, 3})); - - MultiDependencySingleRowFetcher testee{dependencyProxyMock}; - testee.initDependencies(); - - auto ioPairs = std::vector{}; - auto add = [&ioPairs](auto call, auto result) { - ioPairs.emplace_back(std::make_pair(call, result)); - }; - - - add(SkipRowsForDependency{0, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 1}); - add(SkipRowsForDependency{0, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 0}); - add(FetchShadowRow{1000}, FetchShadowRow::Result{ExecutionState::HASMORE, shadowRow(1)}); - if (cutAt() == 3) { - add(SkipRowsForDependency{0, 1000}, - SkipRowsForDependency::Result{ExecutionState::HASMORE, 1}); - add(SkipRowsForDependency{0, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 0}); - } else { - add(SkipRowsForDependency{0, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 1}); - } - add(SkipRowsForDependency{0, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 0}); - add(FetchShadowRow{1000}, FetchShadowRow::Result{ExecutionState::HASMORE, shadowRow(3)}); - add(SkipRowsForDependency{0, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 0}); - add(FetchShadowRow{1000}, - FetchShadowRow::Result{ExecutionState::DONE, invalidShadowRow()}); - - runFetcher(testee, ioPairs); -} + { + // Second Block, split after the higher depth shadow row + auto [state, skipped, ranges] = testee.execute(stack, set); + EXPECT_EQ(state, ExecutionState::HASMORE); + EXPECT_FALSE(skipped.nothingSkipped()); + // We skip 1 from firstBlock part 2 and 2 in this block + EXPECT_EQ(skipped.getSkipCount(), 3); + EXPECT_EQ(ranges.size(), set.size()); + for (auto [dep, range] : ranges) { + // All Ranges are non empty + validateNextIsShadowRow(range, ExecutorState::HASMORE, 1, 0); + validateNextIsShadowRow(range, ExecutorState::HASMORE, 2, 1); + } + } -TEST_P(MultiDependencySingleRowFetcherShadowRowTest, skip_shadow_rows_2_deps) { - constexpr size_t numDeps = 2; - MultiDependencyProxyMock dependencyProxyMock{monitor, 1, numDeps}; - - dependencyProxyMock.getDependencyMock(0).shouldReturn( - alternatingDataAndShadowRows({0, 1, 2, 3})); - dependencyProxyMock.getDependencyMock(1).shouldReturn( - alternatingDataAndShadowRows({4, 1, 6, 3})); - - MultiDependencySingleRowFetcher testee{dependencyProxyMock}; - testee.initDependencies(); - - auto ioPairs = std::vector{}; - auto add = [&ioPairs](auto call, auto result) { - ioPairs.emplace_back(std::make_pair(call, result)); - }; - - // fetch dep 1 - add(SkipRowsForDependency{0, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 1}); - // dep 1 should stay done - add(SkipRowsForDependency{0, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 0}); - // fetching the shadow row should not yet be possible - add(FetchShadowRow{1000}, - FetchShadowRow::Result{ExecutionState::HASMORE, invalidShadowRow()}); - // dep 1 should stay done - add(SkipRowsForDependency{0, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 0}); - // fetch dep 2 - if (cutAt() == 1) { - add(SkipRowsForDependency{1, 1000}, - SkipRowsForDependency::Result{ExecutionState::HASMORE, 1}); - add(SkipRowsForDependency{1, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 0}); - } else { - add(SkipRowsForDependency{1, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 1}); - } - // dep 2 should stay done - add(SkipRowsForDependency{1, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 0}); - // dep 1 should stay done - add(SkipRowsForDependency{0, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 0}); - // Fetch the first shadow row. - add(FetchShadowRow{1000}, FetchShadowRow::Result{ExecutionState::HASMORE, shadowRow(1)}); - // fetch dep 1 again - if (cutAt() == 3) { - add(SkipRowsForDependency{0, 1000}, - SkipRowsForDependency::Result{ExecutionState::HASMORE, 1}); - add(SkipRowsForDependency{0, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 0}); - } else { - add(SkipRowsForDependency{0, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 1}); - } - // dep 1 should stay done - add(SkipRowsForDependency{0, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 0}); - // fetching the shadow row should not yet be possible - add(FetchShadowRow{1000}, - FetchShadowRow::Result{ExecutionState::HASMORE, invalidShadowRow()}); - // dep 1 should stay done - add(SkipRowsForDependency{0, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 0}); - // fetch dep 2 again - if (cutAt() == 3) { - add(SkipRowsForDependency{1, 1000}, - SkipRowsForDependency::Result{ExecutionState::HASMORE, 1}); - add(SkipRowsForDependency{1, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 0}); - } else { - add(SkipRowsForDependency{1, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 1}); - } - // dep 2 should stay done - add(SkipRowsForDependency{1, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 0}); - // dep 1 should stay done - add(SkipRowsForDependency{0, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 0}); - // Fetch the second shadow row. - add(FetchShadowRow{1000}, FetchShadowRow::Result{ExecutionState::HASMORE, shadowRow(3)}); - // We're now done. - add(FetchShadowRow{1000}, - FetchShadowRow::Result{ExecutionState::DONE, invalidShadowRow()}); - - runFetcher(testee, ioPairs); -} + { + // Third Block + auto [state, skipped, ranges] = testee.execute(stack, set); + if (dep + 1 == numberDependencies()) { + // Only the last dependency reports a global DONE + EXPECT_EQ(state, ExecutionState::DONE); + } else { + // All others still report HASMORE on the other parts + EXPECT_EQ(state, ExecutionState::HASMORE); + } -TEST_P(MultiDependencySingleRowFetcherShadowRowTest, skip_shadow_rows_2_deps_reverse_pull) { - constexpr size_t numDeps = 2; - MultiDependencyProxyMock dependencyProxyMock{monitor, 1, numDeps}; - - dependencyProxyMock.getDependencyMock(0).shouldReturn( - alternatingDataAndShadowRows({0, 1, 2, 3})); - dependencyProxyMock.getDependencyMock(1).shouldReturn( - alternatingDataAndShadowRows({4, 1, 6, 3})); - - MultiDependencySingleRowFetcher testee{dependencyProxyMock}; - testee.initDependencies(); - - auto ioPairs = std::vector{}; - auto add = [&ioPairs](auto call, auto result) { - ioPairs.emplace_back(std::make_pair(call, result)); - }; - - // fetch dep 2 - add(SkipRowsForDependency{1, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 1}); - // dep 2 should stay done - add(SkipRowsForDependency{1, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 0}); - // fetching the shadow row should not yet be possible - add(FetchShadowRow{1000}, - FetchShadowRow::Result{ExecutionState::HASMORE, invalidShadowRow()}); - // dep 2 should stay done - add(SkipRowsForDependency{1, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 0}); - // fetch dep 1 - if (cutAt() == 1) { - add(SkipRowsForDependency{0, 1000}, - SkipRowsForDependency::Result{ExecutionState::HASMORE, 1}); - add(SkipRowsForDependency{0, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 0}); - } else { - add(SkipRowsForDependency{0, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 1}); - } - // dep 1 should stay done - add(SkipRowsForDependency{0, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 0}); - // dep 2 should stay done - add(SkipRowsForDependency{1, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 0}); - // Fetch the first shadow row. - add(FetchShadowRow{1000}, FetchShadowRow::Result{ExecutionState::HASMORE, shadowRow(1)}); - // fetch dep 2 again - if (cutAt() == 3) { - add(SkipRowsForDependency{1, 1000}, - SkipRowsForDependency::Result{ExecutionState::HASMORE, 1}); - add(SkipRowsForDependency{1, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 0}); - } else { - add(SkipRowsForDependency{1, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 1}); - } - // dep 2 should stay done - add(SkipRowsForDependency{1, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 0}); - // fetching the shadow row should not yet be possible - add(FetchShadowRow{1000}, - FetchShadowRow::Result{ExecutionState::HASMORE, invalidShadowRow()}); - // dep 2 should stay done - add(SkipRowsForDependency{1, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 0}); - // fetch dep 1 again - if (cutAt() == 3) { - add(SkipRowsForDependency{0, 1000}, - SkipRowsForDependency::Result{ExecutionState::HASMORE, 1}); - add(SkipRowsForDependency{0, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 0}); - } else { - add(SkipRowsForDependency{0, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 1}); + // We skip 1 from secondBlock part 2 and 3 in this block + EXPECT_EQ(skipped.getSkipCount(), 4); + EXPECT_EQ(ranges.size(), set.size()); + for (auto [dep, range] : ranges) { + // All Ranges are empty + EXPECT_FALSE(range.hasShadowRow()); + EXPECT_FALSE(range.hasDataRow()); + EXPECT_EQ(range.upstreamState(), ExecutorState::DONE); + } + } } - // dep 2 should stay done - add(SkipRowsForDependency{1, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 0}); - // dep 1 should stay done - add(SkipRowsForDependency{0, 1000}, - SkipRowsForDependency::Result{ExecutionState::DONE, 0}); - // Fetch the second shadow row. - add(FetchShadowRow{1000}, FetchShadowRow::Result{ExecutionState::HASMORE, shadowRow(3)}); - // We're now done. - add(FetchShadowRow{1000}, - FetchShadowRow::Result{ExecutionState::DONE, invalidShadowRow()}); - - runFetcher(testee, ioPairs); } } // namespace aql diff --git a/tests/Aql/RowFetcherHelper.cpp b/tests/Aql/RowFetcherHelper.cpp index 4af2bcd7a6da..170be7729659 100644 --- a/tests/Aql/RowFetcherHelper.cpp +++ b/tests/Aql/RowFetcherHelper.cpp @@ -52,14 +52,14 @@ namespace {} // namespace // - SECTION SINGLEROWFETCHER - // ----------------------------------------- -template<::arangodb::aql::BlockPassthrough passBlocksThrough> +template <::arangodb::aql::BlockPassthrough passBlocksThrough> SingleRowFetcherHelper::SingleRowFetcherHelper( AqlItemBlockManager& manager, std::shared_ptr> const& vPackBuffer, bool returnsWaiting) : SingleRowFetcherHelper(manager, 1, returnsWaiting, vPackBufferToAqlItemBlock(manager, vPackBuffer)) {} -template<::arangodb::aql::BlockPassthrough passBlocksThrough> +template <::arangodb::aql::BlockPassthrough passBlocksThrough> SingleRowFetcherHelper::SingleRowFetcherHelper( ::arangodb::aql::AqlItemBlockManager& manager, size_t const blockSize, bool const returnsWaiting, ::arangodb::aql::SharedAqlItemBlockPtr input) @@ -73,10 +73,10 @@ SingleRowFetcherHelper::SingleRowFetcherHelper( TRI_ASSERT(_blockSize > 0); } -template<::arangodb::aql::BlockPassthrough passBlocksThrough> +template <::arangodb::aql::BlockPassthrough passBlocksThrough> SingleRowFetcherHelper::~SingleRowFetcherHelper() = default; -template<::arangodb::aql::BlockPassthrough passBlocksThrough> +template <::arangodb::aql::BlockPassthrough passBlocksThrough> // NOLINTNEXTLINE google-default-arguments std::pair SingleRowFetcherHelper::fetchRow(size_t) { // If this assertion fails, the Executor has fetched more rows after DONE. @@ -104,7 +104,7 @@ std::pair SingleRowFetcherHelper +template <::arangodb::aql::BlockPassthrough passBlocksThrough> // NOLINTNEXTLINE google-default-arguments std::pair SingleRowFetcherHelper::fetchShadowRow(size_t) { // If this assertion fails, the Executor has fetched more rows after DONE. @@ -119,8 +119,8 @@ std::pair SingleRowFetcherHelper= _nrItems) { - _returnedDoneOnFetchShadowRow = true; - return {ExecutionState::DONE, ShadowAqlItemRow{CreateInvalidShadowRowHint{}}}; + _returnedDoneOnFetchShadowRow = true; + return {ExecutionState::DONE, ShadowAqlItemRow{CreateInvalidShadowRowHint{}}}; } auto res = SingleRowFetcher::fetchShadowRow(); if (res.second.isInitialized()) { @@ -133,54 +133,7 @@ std::pair SingleRowFetcherHelper -std::pair SingleRowFetcherHelper::skipRows(size_t const atMost) { - ExecutionState state = ExecutionState::HASMORE; - - while (atMost > _skipped) { - InputAqlItemRow row{CreateInvalidInputRowHint{}}; - std::tie(state, row) = fetchRow(); - if (state == ExecutionState::WAITING) { - return {state, 0}; - } - if (row.isInitialized()) { - ++_skipped; - } - if (state == ExecutionState::DONE) { - size_t skipped = _skipped; - _skipped = 0; - _totalSkipped += skipped; - return {state, skipped}; - } - } - - size_t skipped = _skipped; - _skipped = 0; - _totalSkipped += skipped; - return {state, skipped}; -} - -template<::arangodb::aql::BlockPassthrough passBlocksThrough> -std::pair -SingleRowFetcherHelper::fetchBlockForPassthrough(size_t const atMost) { - if (wait()) { - return {ExecutionState::WAITING, nullptr}; - } - - size_t const remainingRows = _blockSize - _curIndexInBlock; - size_t const from = _curRowIndex; - size_t const to = _curRowIndex + remainingRows; - - bool const isLastBlock = _curRowIndex + _blockSize >= _nrItems; - bool const askingForMore = _curRowIndex + atMost > _nrItems; - - bool const done = isLastBlock && askingForMore; - - ExecutionState const state = done ? ExecutionState::DONE : ExecutionState::HASMORE; - return {state, _itemBlock->slice(from, to)}; -} - -template<::arangodb::aql::BlockPassthrough passBlocksThrough> +template <::arangodb::aql::BlockPassthrough passBlocksThrough> std::pair SingleRowFetcherHelper::fetchBlock(size_t const atMost) { size_t const remainingRows = _blockSize - _curIndexInBlock; diff --git a/tests/Aql/RowFetcherHelper.h b/tests/Aql/RowFetcherHelper.h index 670a13123f9c..4f4a27e10394 100644 --- a/tests/Aql/RowFetcherHelper.h +++ b/tests/Aql/RowFetcherHelper.h @@ -56,7 +56,7 @@ namespace aql { /** * @brief Mock for SingleRowFetcher */ -template<::arangodb::aql::BlockPassthrough passBlocksThrough> +template <::arangodb::aql::BlockPassthrough passBlocksThrough> class SingleRowFetcherHelper : public arangodb::aql::SingleRowFetcher { public: @@ -85,11 +85,6 @@ class SingleRowFetcherHelper size_t totalSkipped() const { return _totalSkipped; } - std::pair skipRows(size_t atMost) override; - - std::pair fetchBlockForPassthrough( - size_t atMost) override; - std::pair fetchBlock(size_t atMost) override; arangodb::aql::AqlItemBlockManager& itemBlockManager() { diff --git a/tests/Aql/TestLambdaExecutor.cpp b/tests/Aql/TestLambdaExecutor.cpp index facdfe75c65d..025b369a09bf 100644 --- a/tests/Aql/TestLambdaExecutor.cpp +++ b/tests/Aql/TestLambdaExecutor.cpp @@ -75,18 +75,6 @@ TestLambdaExecutor::TestLambdaExecutor(Fetcher&, Infos& infos) : _infos(infos) { TestLambdaExecutor::~TestLambdaExecutor() {} -auto TestLambdaExecutor::fetchBlockForPassthrough(size_t atMost) - -> std::tuple { - TRI_ASSERT(false); - THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); -} - -auto TestLambdaExecutor::produceRows(OutputAqlItemRow& output) - -> std::tuple { - TRI_ASSERT(false); - THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); -} - auto TestLambdaExecutor::produceRows(AqlItemBlockInputRange& input, OutputAqlItemRow& output) -> std::tuple { return _infos.getProduceLambda()(input, output); @@ -99,18 +87,6 @@ TestLambdaSkipExecutor::TestLambdaSkipExecutor(Fetcher&, Infos& infos) TestLambdaSkipExecutor::~TestLambdaSkipExecutor() {} -auto TestLambdaSkipExecutor::fetchBlockForPassthrough(size_t atMost) - -> std::tuple { - TRI_ASSERT(false); - THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); -} - -auto TestLambdaSkipExecutor::produceRows(OutputAqlItemRow& output) - -> std::tuple { - TRI_ASSERT(false); - THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); -} - auto TestLambdaSkipExecutor::produceRows(AqlItemBlockInputRange& input, OutputAqlItemRow& output) -> std::tuple { return _infos.getProduceLambda()(input, output); diff --git a/tests/Aql/TestLambdaExecutor.h b/tests/Aql/TestLambdaExecutor.h index 1843b7f1a24d..0813d4747495 100644 --- a/tests/Aql/TestLambdaExecutor.h +++ b/tests/Aql/TestLambdaExecutor.h @@ -139,24 +139,6 @@ class TestLambdaExecutor { TestLambdaExecutor(Fetcher&, Infos&); ~TestLambdaExecutor(); - /** - * @brief NOT IMPLEMENTED. JUST FOR COMPILER - * TODO: REMOVE ME after we have switch everything over to produceRow. - * - * @param atMost - * @return std::tuple - */ - auto fetchBlockForPassthrough(size_t atMost) - -> std::tuple; - /** - * @brief NOT IMPLEMENTED. JUST FOR COMPILER - * TODO: REMOVE ME after we have switch everything over to produceRow. - * - * @param output - * @return std::tuple - */ - auto produceRows(OutputAqlItemRow& output) -> std::tuple; - /** * @brief produceRows API. Just calls the ProduceCall in the Infos. * @@ -196,25 +178,6 @@ class TestLambdaSkipExecutor { TestLambdaSkipExecutor(Fetcher&, Infos&); ~TestLambdaSkipExecutor(); - /** - * @brief NOT IMPLEMENTED. JUST FOR COMPILER - * TODO: REMOVE ME after we have switch everything over to produceRow. - * - * @param atMost - * @return std::tuple - */ - auto fetchBlockForPassthrough(size_t atMost) - -> std::tuple; - - /** - * @brief NOT IMPLEMENTED. JUST FOR COMPILER - * TODO: REMOVE ME after we have switch everything over to produceRow. - * - * @param output - * @return std::tuple - */ - auto produceRows(OutputAqlItemRow& output) -> std::tuple; - /** * @brief skipRows API. Just calls the SkipCall in the infos * diff --git a/tests/Aql/WaitingExecutionBlockMock.cpp b/tests/Aql/WaitingExecutionBlockMock.cpp index 25493224ef67..be1ea4428c23 100644 --- a/tests/Aql/WaitingExecutionBlockMock.cpp +++ b/tests/Aql/WaitingExecutionBlockMock.cpp @@ -38,16 +38,59 @@ using namespace arangodb::aql; using namespace arangodb::tests; using namespace arangodb::tests::aql; +namespace { +static auto blocksToInfos(std::deque const& blocks) -> ExecutorInfos { + auto readInput = make_shared_unordered_set(); + auto writeOutput = make_shared_unordered_set(); + std::unordered_set toClear{}; + std::unordered_set toKeep{}; + RegisterId regs = 1; + for (auto const& b : blocks) { + if (b != nullptr) { + // Find the first non-nullptr block + regs = b->getNrRegs(); + + break; + } + } + // if non found sorry blind guess the number of registers here. + // This can happen if you insert the data later into this Mock. + // If you do so this register planning is of + // for the rime being no test is showing this behavior. + // Consider adding data first if the test fails + + for (RegisterId r = 0; r < regs; ++r) { + toKeep.emplace(r); + } + return {readInput, writeOutput, regs, regs, toClear, toKeep}; +} +} // namespace WaitingExecutionBlockMock::WaitingExecutionBlockMock(ExecutionEngine* engine, ExecutionNode const* node, std::deque&& data, WaitingBehaviour variant) : ExecutionBlock(engine, node), - _data(std::move(data)), - _resourceMonitor(), - _inflight(0), _hasWaited(false), - _variant{variant} {} + _variant{variant}, + _infos{::blocksToInfos(data)}, + _blockData{*engine, node, _infos} { + SkipResult s; + for (auto const& b : data) { + if (b != nullptr) { + TRI_ASSERT(s.nothingSkipped()); + _blockData.addBlock(b, s); + if (b->hasShadowRows()) { + _doesContainShadowRows = true; + } + } + } + + if (!data.empty() && data.back() == nullptr) { + // If the last block in _data is explicitly a nullptr + // we will lie on the last row + _shouldLieOnLastRow = true; + } +} std::pair WaitingExecutionBlockMock::initializeCursor( arangodb::aql::InputAqlItemRow const& input) { @@ -56,7 +99,6 @@ std::pair WaitingExecutionBlock return {ExecutionState::WAITING, TRI_ERROR_NO_ERROR}; } _hasWaited = false; - _inflight = 0; return {ExecutionState::DONE, TRI_ERROR_NO_ERROR}; } @@ -66,54 +108,6 @@ std::pair WaitingExecutionBlockMock::shut return std::make_pair(state, res); } -std::pair WaitingExecutionBlockMock::getSome(size_t atMost) { - if (_variant != WaitingBehaviour::NEVER && !_hasWaited) { - _hasWaited = true; - if (_returnedDone) { - return {ExecutionState::DONE, nullptr}; - } - return {ExecutionState::WAITING, nullptr}; - } - _hasWaited = false; - - if (_data.empty()) { - _returnedDone = true; - return {ExecutionState::DONE, nullptr}; - } - - auto result = std::move(_data.front()); - _data.pop_front(); - - if (_data.empty()) { - _returnedDone = true; - return {ExecutionState::DONE, std::move(result)}; - } else { - return {ExecutionState::HASMORE, std::move(result)}; - } -} - -std::pair WaitingExecutionBlockMock::skipSome(size_t atMost) { - traceSkipSomeBegin(atMost); - if (_variant != WaitingBehaviour::NEVER && !_hasWaited) { - _hasWaited = true; - return traceSkipSomeEnd(ExecutionState::WAITING, 0); - } - _hasWaited = false; - - if (_data.empty()) { - return traceSkipSomeEnd(ExecutionState::DONE, 0); - } - - size_t skipped = _data.front()->size(); - _data.pop_front(); - - if (_data.empty()) { - return traceSkipSomeEnd(ExecutionState::DONE, skipped); - } else { - return traceSkipSomeEnd(ExecutionState::HASMORE, skipped); - } -} - std::tuple WaitingExecutionBlockMock::execute(AqlCallStack stack) { traceExecuteBegin(stack); auto res = executeWithoutTrace(stack); @@ -121,16 +115,13 @@ std::tuple WaitingExecutionBl return res; } -// NOTE: Does not care for shadowrows! std::tuple WaitingExecutionBlockMock::executeWithoutTrace( AqlCallStack stack) { - auto myCallList = stack.popCall(); - auto myCall = myCallList.popNextCall(); + auto myCall = stack.peek(); TRI_ASSERT(!(myCall.getOffset() == 0 && myCall.softLimit == AqlCall::Limit{0})); TRI_ASSERT(!(myCall.hasSoftLimit() && myCall.fullCount)); TRI_ASSERT(!(myCall.hasSoftLimit() && myCall.hasHardLimit())); - if (_variant != WaitingBehaviour::NEVER && !_hasWaited) { // If we ordered waiting check on _hasWaited and wait if not _hasWaited = true; @@ -140,99 +131,63 @@ std::tuple WaitingExecutionBl // If we always wait, reset. _hasWaited = false; } - size_t skipped = 0; - SharedAqlItemBlockPtr result = nullptr; - if (!_data.empty() && _data.front() == nullptr) { - dropBlock(); + if (!_blockData.hasDataFor(myCall)) { + return {ExecutionState::DONE, {}, nullptr}; } - while (!_data.empty()) { - if (_data.front() == nullptr) { - if ((skipped > 0 || result != nullptr) && - !(myCall.hasHardLimit() && myCall.getLimit() == 0)) { - // This is a specific break point return now. - // Sorry we can only return one block. - // This means we have prepared the first block. - // But still need more data. - SkipResult skipRes{}; - skipRes.didSkip(skipped); - return {ExecutionState::HASMORE, skipRes, result}; - } else { - dropBlock(); - continue; - } - } - if (_data.front()->size() <= _inflight) { - dropBlock(); - continue; + SkipResult localSkipped; + while (true) { + auto [state, skipped, result] = _blockData.execute(stack, ExecutionState::DONE); + // We loop here if we only skip + localSkipped.merge(skipped, false); + bool shouldReturn = state == ExecutionState::DONE || result != nullptr; + + if (result != nullptr && !result->hasShadowRows()) { + // Count produced rows + auto& modCall = stack.modifyTopCall(); + modCall.didProduce(result->size()); } - TRI_ASSERT(_data.front()->size() > _inflight); - // Drop while skip - if (myCall.getOffset() > 0) { - size_t canSkip = (std::min)(_data.front()->size() - _inflight, myCall.getOffset()); - _inflight += canSkip; - myCall.didSkip(canSkip); - skipped += canSkip; - continue; - } else if (myCall.getLimit() > 0) { - if (result != nullptr) { - // Sorry we can only return one block. - // This means we have prepared the first block. - // But still need more data. - SkipResult skipRes{}; - skipRes.didSkip(skipped); - return {ExecutionState::HASMORE, skipRes, result}; - } - - size_t canReturn = _data.front()->size() - _inflight; - if (canReturn <= myCall.getLimit()) { - // We can return the remainder of this block - if (_inflight == 0) { - // use full block - result = std::move(_data.front()); - } else { - // Slice out the last part - result = _data.front()->slice(_inflight, _data.front()->size()); - } - dropBlock(); - } else { - // Slice out limit many rows starting at _inflight - result = _data.front()->slice(_inflight, _inflight + myCall.getLimit()); - // adjust _inflight to the fist non-returned row. - _inflight += myCall.getLimit(); + if (!skipped.nothingSkipped()) { + auto& modCall = stack.modifyTopCall(); + modCall.didSkip(skipped.getSkipCount()); + // Reset the internal counter. + // We reuse the call to upstream + // this inturn uses this counter to report nrRowsSkipped + modCall.skippedRows = 0; + if (!modCall.needSkipMore() && modCall.getLimit() == 0) { + // We do not have anything to do for this call + shouldReturn = true; } - TRI_ASSERT(result != nullptr); - myCall.didProduce(result->size()); - } else if (myCall.needsFullCount()) { - size_t counts = _data.front()->size() - _inflight; - dropBlock(); - myCall.didSkip(counts); - skipped += counts; - } else { - if (myCall.getLimit() == 0 && !myCall.needsFullCount() && myCall.hasHardLimit()) { - while (!_data.empty()) { - // Drop data we are in fastForward phase - dropBlock(); + } + if (shouldReturn) { + if (!_doesContainShadowRows && state == ExecutionState::HASMORE) { + // FullCount phase, let us loop until we are done + // We do not have anything to do for this call + // But let us only do this on top-level queries + + auto call = stack.peek(); + if (call.hasHardLimit() && call.getLimit() == 0) { + // We are in fullCount/fastForward phase now. + while (state == ExecutionState::HASMORE) { + auto [nextState, nextSkipped, nextResult] = + _blockData.execute(stack, ExecutionState::DONE); + state = nextState; + // We are disallowed to have any result here. + TRI_ASSERT(nextResult == nullptr); + localSkipped.merge(nextSkipped, false); + } } } - SkipResult skipRes{}; - skipRes.didSkip(skipped); - if (!_data.empty()) { - return {ExecutionState::HASMORE, skipRes, result}; - } else if (result != nullptr && result->size() < myCall.hardLimit) { - return {ExecutionState::HASMORE, skipRes, result}; - } else { - return {ExecutionState::DONE, skipRes, result}; + // We want to "lie" on upstream if we have hit a softLimit exactly on the last row + if (state == ExecutionState::DONE && _shouldLieOnLastRow) { + auto const& call = stack.peek(); + if (call.hasSoftLimit() && call.getLimit() == 0 && call.getOffset() == 0) { + state = ExecutionState::HASMORE; + } } + + // We have a valid result. + return {state, localSkipped, result}; } } - SkipResult skipRes{}; - skipRes.didSkip(skipped); - return {ExecutionState::DONE, skipRes, result}; -} - -void WaitingExecutionBlockMock::dropBlock() { - TRI_ASSERT(!_data.empty()); - _data.pop_front(); - _inflight = 0; } diff --git a/tests/Aql/WaitingExecutionBlockMock.h b/tests/Aql/WaitingExecutionBlockMock.h index b1ad73e01bd8..e33a421b8fd3 100644 --- a/tests/Aql/WaitingExecutionBlockMock.h +++ b/tests/Aql/WaitingExecutionBlockMock.h @@ -26,6 +26,7 @@ #include "Aql/ExecutionBlock.h" #include "Aql/ExecutionState.h" #include "Aql/ResourceUsage.h" +#include "Aql/ScatterExecutor.h" #include @@ -82,48 +83,21 @@ class WaitingExecutionBlockMock final : public arangodb::aql::ExecutionBlock { std::pair initializeCursor( arangodb::aql::InputAqlItemRow const& input) override; - /** - * @brief The return values are alternating. On non-WAITING case - * it will return atMost many elements from _data. - * - * - * @param atMost This many elements will be returned at Most - * - * @return First: - * Second: - */ - std::pair getSome(size_t atMost) override; - - /** - * @brief The return values are alternating. On non-WAITING case - * it will return atMost, or whatever is not skipped over on data, - * whichever number is lower. - * - * - * @param atMost This many elements will be skipped at most - * - * @return First: - * Second: - */ - std::pair skipSome(size_t atMost) override; - std::tuple execute( arangodb::aql::AqlCallStack stack) override; private: - void dropBlock(); - // Implementation of execute std::tuple executeWithoutTrace(arangodb::aql::AqlCallStack stack); private: - std::deque _data; - arangodb::aql::ResourceMonitor _resourceMonitor; - size_t _inflight; - bool _returnedDone = false; bool _hasWaited; WaitingBehaviour _variant; + bool _doesContainShadowRows{false}; + bool _shouldLieOnLastRow{false}; + arangodb::aql::ExecutorInfos _infos; + typename arangodb::aql::ScatterExecutor::ClientBlockData _blockData; }; } // namespace aql diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c34cca74d301..5583a3d29fe0 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -57,7 +57,6 @@ set(ARANGODB_TESTS_SOURCES Aql/KShortestPathsExecutorTest.cpp Aql/LimitExecutorTest.cpp Aql/MockTypedNode.cpp - Aql/MultiDepFetcherHelper.cpp Aql/MultiDependencySingleRowFetcherTest.cpp Aql/NgramMatchFunctionTest.cpp Aql/NgramSimilarityFunctionTest.cpp