diff --git a/arangod/Aql/Aggregator.cpp b/arangod/Aql/Aggregator.cpp index adcd44e837f5..f940c927ca46 100644 --- a/arangod/Aql/Aggregator.cpp +++ b/arangod/Aql/Aggregator.cpp @@ -24,6 +24,7 @@ #include "Aggregator.h" #include "Aql/AqlValue.h" +#include "Aql/AqlValueMaterializer.h" #include "Basics/VelocyPackHelper.h" #include "Transaction/Context.h" #include "Transaction/Helpers.h" diff --git a/arangod/Aql/AqlItemBlock.cpp b/arangod/Aql/AqlItemBlock.cpp index aae7db4a31f7..fa4329230f7a 100644 --- a/arangod/Aql/AqlItemBlock.cpp +++ b/arangod/Aql/AqlItemBlock.cpp @@ -33,6 +33,8 @@ #include "Aql/SharedAqlItemBlockPtr.h" #include "Basics/StaticStrings.h" #include "Basics/VelocyPackHelper.h" +#include "Transaction/Context.h" +#include "Transaction/Methods.h" #include #include @@ -566,7 +568,7 @@ SharedAqlItemBlockPtr AqlItemBlock::steal(std::vector const& chosen, /// corresponding position /// "raw": List of actual values, positions 0 and 1 are always null /// such that actual indices start at 2 -void AqlItemBlock::toVelocyPack(transaction::Methods* trx, VPackBuilder& result) const { +void AqlItemBlock::toVelocyPack(velocypack::Options const* const trxOptions, VPackBuilder& result) const { TRI_ASSERT(result.isOpenObject()); VPackOptions options(VPackOptions::Defaults); options.buildUnindexedArrays = true; @@ -650,7 +652,7 @@ void AqlItemBlock::toVelocyPack(transaction::Methods* trx, VPackBuilder& result) if (it == table.end()) { currentState = Next; - a.toVelocyPack(trx, raw, false); + a.toVelocyPack(trxOptions, raw, false); table.try_emplace(a, pos++); } else { currentState = Positional; @@ -699,20 +701,21 @@ void AqlItemBlock::toVelocyPack(transaction::Methods* trx, VPackBuilder& result) result.add("raw", raw.slice()); } -void AqlItemBlock::rowToSimpleVPack(size_t const row, transaction::Methods* trx, arangodb::velocypack::Builder& builder) const { +void AqlItemBlock::rowToSimpleVPack(size_t const row, velocypack::Options const* options, arangodb::velocypack::Builder& builder) const { VPackArrayBuilder rowBuilder{&builder}; if (isShadowRow(row)) { - getShadowRowDepth(row).toVelocyPack(trx, *rowBuilder, false); + getShadowRowDepth(row).toVelocyPack(options, *rowBuilder, false); } else { - AqlValue{AqlValueHintNull{}}.toVelocyPack(trx, *rowBuilder, false); + AqlValue{AqlValueHintNull{}}.toVelocyPack(options, *rowBuilder, false); } for (RegisterId reg = 0; reg < getNrRegs(); ++reg) { - getValueReference(row, reg).toVelocyPack(trx, *rowBuilder, false); + getValueReference(row, reg).toVelocyPack(options, *rowBuilder, false); } } -void AqlItemBlock::toSimpleVPack(transaction::Methods* trx, arangodb::velocypack::Builder& builder) const { +void AqlItemBlock::toSimpleVPack(velocypack::Options const* options, + arangodb::velocypack::Builder& builder) const { VPackObjectBuilder block{&builder}; block->add("nrItems", VPackValue(size())); block->add("nrRegs", VPackValue(getNrRegs())); @@ -720,7 +723,7 @@ void AqlItemBlock::toSimpleVPack(transaction::Methods* trx, arangodb::velocypack { VPackArrayBuilder matrixBuilder{block.builder}; for (size_t row = 0; row < size(); ++row) { - rowToSimpleVPack(row, trx, *matrixBuilder.builder); + rowToSimpleVPack(row, options, *matrixBuilder.builder); } } } diff --git a/arangod/Aql/AqlItemBlock.h b/arangod/Aql/AqlItemBlock.h index 182c25e692b7..611b79bc659c 100644 --- a/arangod/Aql/AqlItemBlock.h +++ b/arangod/Aql/AqlItemBlock.h @@ -208,7 +208,7 @@ class AqlItemBlock { /// @brief toJson, transfer a whole AqlItemBlock to Json, the result can /// be used to recreate the AqlItemBlock via the Json constructor - void toVelocyPack(transaction::Methods* trx, arangodb::velocypack::Builder&) const; + void toVelocyPack(velocypack::Options const*, arangodb::velocypack::Builder&) const; /// @brief Creates a human-readable velocypack of the block. Adds an object /// `{nrItems, nrRegs, matrix}` to the builder. @@ -217,9 +217,9 @@ class AqlItemBlock { // (of length nrRegs+1 (sic)). The first entry contains the shadow row depth, // or `null` for data rows. The entries with indexes 1..nrRegs contain the // registers 0..nrRegs-1, respectively. - void toSimpleVPack(transaction::Methods* trx, arangodb::velocypack::Builder&) const; + void toSimpleVPack(velocypack::Options const*, arangodb::velocypack::Builder&) const; - void rowToSimpleVPack(size_t row, transaction::Methods* trx, + void rowToSimpleVPack(size_t row, velocypack::Options const*, velocypack::Builder& builder) const; /// @brief test if the given row is a shadow row and conveys subquery diff --git a/arangod/Aql/AqlValue.cpp b/arangod/Aql/AqlValue.cpp index fb08cd1a3430..11258ff954b7 100644 --- a/arangod/Aql/AqlValue.cpp +++ b/arangod/Aql/AqlValue.cpp @@ -40,8 +40,6 @@ #include #include -#include - using namespace arangodb; using namespace arangodb::aql; @@ -933,7 +931,7 @@ v8::Handle AqlValue::toV8(v8::Isolate* isolate, transaction::Methods* } /// @brief materializes a value into the builder -void AqlValue::toVelocyPack(transaction::Methods* trx, arangodb::velocypack::Builder& builder, +void AqlValue::toVelocyPack(VPackOptions const* options, arangodb::velocypack::Builder& builder, bool resolveExternals) const { switch (type()) { case VPACK_SLICE_POINTER: @@ -949,7 +947,7 @@ void AqlValue::toVelocyPack(transaction::Methods* trx, arangodb::velocypack::Bui bool const sanitizeCustom = true; arangodb::basics::VelocyPackHelper::sanitizeNonClientTypes( slice(), VPackSlice::noneSlice(), builder, - trx->transactionContextPtr()->getVPackOptions(), sanitizeExternals, + options, sanitizeExternals, sanitizeCustom); } else { builder.add(slice()); @@ -961,7 +959,7 @@ void AqlValue::toVelocyPack(transaction::Methods* trx, arangodb::velocypack::Bui for (auto const& it : *_data.docvec) { size_t const n = it->size(); for (size_t i = 0; i < n; ++i) { - it->getValueReference(i, 0).toVelocyPack(trx, builder, resolveExternals); + it->getValueReference(i, 0).toVelocyPack(options, builder, resolveExternals); } } builder.close(); @@ -980,8 +978,13 @@ void AqlValue::toVelocyPack(transaction::Methods* trx, arangodb::velocypack::Bui } } +void AqlValue::toVelocyPack(transaction::Methods* trx, arangodb::velocypack::Builder& builder, + bool resolveExternals) const { + toVelocyPack(trx->transactionContextPtr()->getVPackOptions(), builder, resolveExternals); +} + /// @brief materializes a value into the builder -AqlValue AqlValue::materialize(transaction::Methods* trx, bool& hasCopied, +AqlValue AqlValue::materialize(VPackOptions const* options, bool& hasCopied, bool resolveExternals) const { switch (type()) { case VPACK_INLINE: @@ -997,7 +1000,7 @@ AqlValue AqlValue::materialize(transaction::Methods* trx, bool& hasCopied, ConditionalDeleter> deleter(shouldDelete); std::shared_ptr> buffer(new VPackBuffer, deleter); VPackBuilder builder(buffer); - toVelocyPack(trx, builder, resolveExternals); + toVelocyPack(options, builder, resolveExternals); hasCopied = true; return AqlValue(buffer.get(), shouldDelete); } @@ -1008,6 +1011,11 @@ AqlValue AqlValue::materialize(transaction::Methods* trx, bool& hasCopied, return AqlValue(); } +AqlValue AqlValue::materialize(transaction::Methods* trx, bool& hasCopied, + bool resolveExternals) const { + return materialize(trx->transactionContextPtr()->getVPackOptions(), hasCopied, resolveExternals); +} + /// @brief clone a value AqlValue AqlValue::clone() const { switch (type()) { @@ -1173,23 +1181,24 @@ AqlValue AqlValue::CreateFromBlocks(transaction::Methods* trx, } /// @brief comparison for AqlValue objects -int AqlValue::Compare(transaction::Methods* trx, AqlValue const& left, +int AqlValue::Compare(velocypack::Options const* options, AqlValue const& left, AqlValue const& right, bool compareUtf8) { AqlValue::AqlValueType const leftType = left.type(); AqlValue::AqlValueType const rightType = right.type(); if (leftType != rightType) { + // TODO implement this case more efficiently if (leftType == RANGE || rightType == RANGE || leftType == DOCVEC || rightType == DOCVEC) { // range|docvec against x - transaction::BuilderLeaser leftBuilder(trx); - left.toVelocyPack(trx, *leftBuilder.get(), false); + VPackBuilder leftBuilder; + left.toVelocyPack(options, leftBuilder, false); - transaction::BuilderLeaser rightBuilder(trx); - right.toVelocyPack(trx, *rightBuilder.get(), false); + VPackBuilder rightBuilder; + right.toVelocyPack(options, rightBuilder, false); - return arangodb::basics::VelocyPackHelper::compare( - leftBuilder->slice(), rightBuilder->slice(), compareUtf8, - trx->transactionContextPtr()->getVPackOptions()); + return arangodb::basics::VelocyPackHelper::compare(leftBuilder.slice(), + rightBuilder.slice(), + compareUtf8, options); } // fall-through to other types intentional } @@ -1201,9 +1210,8 @@ int AqlValue::Compare(transaction::Methods* trx, AqlValue const& left, case VPACK_SLICE_POINTER: case VPACK_MANAGED_SLICE: case VPACK_MANAGED_BUFFER: { - return arangodb::basics::VelocyPackHelper::compare( - left.slice(), right.slice(), compareUtf8, - trx->transactionContextPtr()->getVPackOptions()); + return arangodb::basics::VelocyPackHelper::compare(left.slice(), right.slice(), + compareUtf8, options); } case DOCVEC: { // use lexicographic ordering of AqlValues regardless of block, @@ -1232,7 +1240,7 @@ int AqlValue::Compare(transaction::Methods* trx, AqlValue const& left, AqlValue const& rval = right._data.docvec->at(rblock)->getValueReference(ritem, 0); - int cmp = Compare(trx, lval, rval, compareUtf8); + int cmp = Compare(options, lval, rval, compareUtf8); if (cmp != 0) { return cmp; @@ -1280,6 +1288,11 @@ int AqlValue::Compare(transaction::Methods* trx, AqlValue const& left, return 0; } +int AqlValue::Compare(transaction::Methods* trx, AqlValue const& left, + AqlValue const& right, bool compareUtf8) { + return Compare(trx->transactionContextPtr()->getVPackOptions(), left, right, compareUtf8); +} + AqlValue::AqlValue(std::vector* docvec) noexcept { TRI_ASSERT(docvec != nullptr); _data.docvec = docvec; @@ -1610,68 +1623,8 @@ AqlValueGuard::~AqlValueGuard() { } void AqlValueGuard::steal() { _destroy = false; } -AqlValue& AqlValueGuard::value() { return _value; } -AqlValueMaterializer::AqlValueMaterializer(transaction::Methods* trx) - : trx(trx), materialized(), hasCopied(false) {} -AqlValueMaterializer::AqlValueMaterializer(AqlValueMaterializer const& other) - : trx(other.trx), materialized(other.materialized), hasCopied(other.hasCopied) { - if (other.hasCopied) { - // copy other's slice - materialized = other.materialized.clone(); - } -} -AqlValueMaterializer& AqlValueMaterializer::operator=(AqlValueMaterializer const& other) { - if (this != &other) { - TRI_ASSERT(trx == other.trx); // must be from same transaction - trx = other.trx; // to shut up cppcheck - if (hasCopied) { - // destroy our own slice - materialized.destroy(); - hasCopied = false; - } - // copy other's slice - materialized = other.materialized.clone(); - hasCopied = other.hasCopied; - } - return *this; -} - -AqlValueMaterializer::AqlValueMaterializer(AqlValueMaterializer&& other) noexcept - : trx(other.trx), materialized(other.materialized), hasCopied(other.hasCopied) { - // reset other - other.hasCopied = false; - // cppcheck-suppress * - other.materialized = AqlValue(); -} - -AqlValueMaterializer& AqlValueMaterializer::operator=(AqlValueMaterializer&& other) noexcept { - if (this != &other) { - TRI_ASSERT(trx == other.trx); // must be from same transaction - trx = other.trx; // to shut up cppcheck - if (hasCopied) { - // destroy our own slice - materialized.destroy(); - } - // reset other - materialized = other.materialized; - hasCopied = other.hasCopied; - other.materialized = AqlValue(); - } - return *this; -} - -AqlValueMaterializer::~AqlValueMaterializer() { - if (hasCopied) { - materialized.destroy(); - } -} - -arangodb::velocypack::Slice AqlValueMaterializer::slice(AqlValue const& value, - bool resolveExternals) { - materialized = value.materialize(trx, hasCopied, resolveExternals); - return materialized.slice(); -} +AqlValue& AqlValueGuard::value() { return _value; } size_t std::hash::operator()(arangodb::aql::AqlValue const& x) const noexcept { diff --git a/arangod/Aql/AqlValue.h b/arangod/Aql/AqlValue.h index 50600bd9701b..e85da13300bc 100644 --- a/arangod/Aql/AqlValue.h +++ b/arangod/Aql/AqlValue.h @@ -47,6 +47,7 @@ namespace velocypack { template class Buffer; class Builder; +struct Options; class Slice; class StringRef; } @@ -329,11 +330,12 @@ struct AqlValue final { v8::Handle toV8(v8::Isolate* isolate, transaction::Methods*) const; /// @brief materializes a value into the builder - void toVelocyPack(transaction::Methods*, arangodb::velocypack::Builder& builder, - bool resolveExternals) const; + void toVelocyPack(velocypack::Options const*, arangodb::velocypack::Builder&, bool resolveExternals) const; + void toVelocyPack(transaction::Methods*, arangodb::velocypack::Builder&, bool resolveExternals) const; /// @brief materialize a value into a new one. this expands docvecs and /// ranges + AqlValue materialize(velocypack::Options const*, bool& hasCopied, bool resolveExternals) const; AqlValue materialize(transaction::Methods*, bool& hasCopied, bool resolveExternals) const; /// @brief return the slice for the value @@ -364,6 +366,8 @@ struct AqlValue final { arangodb::aql::RegisterId); /// @brief compare function for two values + static int Compare(velocypack::Options const*, AqlValue const& left, + AqlValue const& right, bool useUtf8); static int Compare(transaction::Methods*, AqlValue const& left, AqlValue const& right, bool useUtf8); @@ -399,28 +403,6 @@ class AqlValueGuard { bool _destroy; }; -struct AqlValueMaterializer { - explicit AqlValueMaterializer(transaction::Methods* trx); - - AqlValueMaterializer(AqlValueMaterializer const& other); - - // cppcheck-suppress operatorEqVarError - AqlValueMaterializer& operator=(AqlValueMaterializer const& other); - - AqlValueMaterializer(AqlValueMaterializer&& other) noexcept; - - // cppcheck-suppress operatorEqVarError - AqlValueMaterializer& operator=(AqlValueMaterializer&& other) noexcept; - - ~AqlValueMaterializer(); - - arangodb::velocypack::Slice slice(AqlValue const& value, bool resolveExternals); - - transaction::Methods* trx; - AqlValue materialized; - bool hasCopied; -}; - static_assert(sizeof(AqlValue) == 16, "invalid AqlValue size"); } // namespace aql diff --git a/arangod/Aql/AqlValueMaterializer.cpp b/arangod/Aql/AqlValueMaterializer.cpp new file mode 100644 index 000000000000..1caf6c1461d8 --- /dev/null +++ b/arangod/Aql/AqlValueMaterializer.cpp @@ -0,0 +1,107 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2019 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Max Neunhoeffer +/// @author Jan Steemann +/// @author Tobias Gödderz +//////////////////////////////////////////////////////////////////////////////// + +#include "AqlValueMaterializer.h" + +#include "Basics/debugging.h" +#include "Transaction/Context.h" +#include "Transaction/Methods.h" + +#include + +using namespace arangodb; +using namespace arangodb::aql; + +AqlValueMaterializer::AqlValueMaterializer(velocypack::Options const* options) + : options(options), + materialized(), + hasCopied(false) {} +AqlValueMaterializer::AqlValueMaterializer(transaction::Methods* trx) + : options(trx->transactionContextPtr()->getVPackOptions()), + materialized(), + hasCopied(false) {} + +AqlValueMaterializer::AqlValueMaterializer(AqlValueMaterializer const& other) + : options(other.options), + materialized(other.materialized), + hasCopied(other.hasCopied) { + if (other.hasCopied) { + // copy other's slice + materialized = other.materialized.clone(); + } +} + +AqlValueMaterializer& AqlValueMaterializer::operator=(AqlValueMaterializer const& other) { + if (this != &other) { + TRI_ASSERT(options == other.options); // must be from same transaction + options = other.options; + if (hasCopied) { + // destroy our own slice + materialized.destroy(); + hasCopied = false; + } + // copy other's slice + materialized = other.materialized.clone(); + hasCopied = other.hasCopied; + } + return *this; +} + +AqlValueMaterializer::AqlValueMaterializer(AqlValueMaterializer&& other) noexcept + : options(other.options), + materialized(other.materialized), + hasCopied(other.hasCopied) { + // reset other + other.hasCopied = false; + // cppcheck-suppress * + other.materialized = AqlValue(); +} + +AqlValueMaterializer& AqlValueMaterializer::operator=(AqlValueMaterializer&& other) noexcept { + if (this != &other) { + TRI_ASSERT(options == other.options); // must be from same transaction + options = other.options; + if (hasCopied) { + // destroy our own slice + materialized.destroy(); + } + // reset other + materialized = other.materialized; + hasCopied = other.hasCopied; + other.materialized = AqlValue(); + } + return *this; +} + +AqlValueMaterializer::~AqlValueMaterializer() { + if (hasCopied) { + materialized.destroy(); + } +} + +arangodb::velocypack::Slice AqlValueMaterializer::slice(AqlValue const& value, + bool resolveExternals) { + materialized = value.materialize(options, hasCopied, resolveExternals); + return materialized.slice(); +} diff --git a/arangod/Aql/AqlValueMaterializer.h b/arangod/Aql/AqlValueMaterializer.h new file mode 100644 index 000000000000..bef63f78abed --- /dev/null +++ b/arangod/Aql/AqlValueMaterializer.h @@ -0,0 +1,70 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2019 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Max Neunhoeffer +/// @author Jan Steemann +/// @author Tobias Gödderz +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGOD_AQL_AQLVALUEMATERIALIZER_H +#define ARANGOD_AQL_AQLVALUEMATERIALIZER_H + +#include "Aql/AqlValue.h" + +namespace arangodb { + +namespace transaction { +class Methods; +} + +namespace velocypack { +class Slice; +struct Options; +} // namespace velocypack + +namespace aql { + +/// @brief Helper class to materialize AqlValues (see AqlValue::materialize). +struct AqlValueMaterializer { + explicit AqlValueMaterializer(velocypack::Options const* options); + explicit AqlValueMaterializer(arangodb::transaction::Methods* trx); + + AqlValueMaterializer(AqlValueMaterializer const& other); + + // cppcheck-suppress operatorEqVarError + AqlValueMaterializer& operator=(AqlValueMaterializer const& other); + + AqlValueMaterializer(AqlValueMaterializer&& other) noexcept; + + // cppcheck-suppress operatorEqVarError + AqlValueMaterializer& operator=(AqlValueMaterializer&& other) noexcept; + + ~AqlValueMaterializer(); + + arangodb::velocypack::Slice slice(arangodb::aql::AqlValue const& value, bool resolveExternals); + + arangodb::velocypack::Options const* options; + arangodb::aql::AqlValue materialized; + bool hasCopied; +}; + +} // namespace aql +} // namespace arangodb + +#endif // ARANGOD_AQL_AQLVALUEMATERIALIZER_H diff --git a/arangod/Aql/Ast.cpp b/arangod/Aql/Ast.cpp index 5a7b54afd0b8..7e59a6a3c0e4 100644 --- a/arangod/Aql/Ast.cpp +++ b/arangod/Aql/Ast.cpp @@ -37,6 +37,7 @@ #include "Aql/Graphs.h" #include "Aql/ModificationOptions.h" #include "Aql/Query.h" +#include "Aql/AqlValueMaterializer.h" #include "Basics/Exceptions.h" #include "Basics/StringUtils.h" #include "Basics/tri-strings.h" diff --git a/arangod/Aql/ClusterNodes.cpp b/arangod/Aql/ClusterNodes.cpp index f2863c2817f0..8a0821a276c8 100644 --- a/arangod/Aql/ClusterNodes.cpp +++ b/arangod/Aql/ClusterNodes.cpp @@ -49,6 +49,7 @@ #include "Aql/SortRegister.h" #include "Aql/SortingGatherExecutor.h" #include "Aql/UnsortedGatherExecutor.h" +#include "Aql/UnsortingGatherExecutor.h" #include "Aql/types.h" #include "Basics/VelocyPackHelper.h" #include "Cluster/ServerState.h" @@ -492,27 +493,16 @@ std::unique_ptr GatherNode::createBlock( if (_elements.empty()) { TRI_ASSERT(getRegisterPlan()->nrRegs[previousNode->getDepth()] == getRegisterPlan()->nrRegs[getDepth()]); - - if (ServerState::instance()->isCoordinator()) { - // In the coordinator case the GatherBlock will fetch from RemoteBlocks. - if (_parallelism == Parallelism::Parallel) { - UnsortedGatherExecutorInfos infos(getRegisterPlan()->nrRegs[getDepth()], - calcRegsToKeep(), getRegsToClear()); - return std::make_unique>(&engine, this, std::move(infos)); - } - IdExecutorInfos infos(getRegisterPlan()->nrRegs[getDepth()], - calcRegsToKeep(), getRegsToClear()); - // We want to immediately move the block on and not wait for additional requests here (hence passthrough) - return std::make_unique>>>( - &engine, this, std::move(infos)); + if (ServerState::instance()->isCoordinator() && _parallelism == Parallelism::Parallel) { + UnsortedGatherExecutorInfos infos(getRegisterPlan()->nrRegs[getDepth()], + calcRegsToKeep(), getRegsToClear()); + return std::make_unique>(&engine, this, std::move(infos)); } else { IdExecutorInfos infos(getRegisterPlan()->nrRegs[getDepth()], calcRegsToKeep(), getRegsToClear()); - // In the DBServer case the GatherBlock will merge local results and then expose them (directly or indirectly) - // to the RemoteBlock on coordinator. We want to trigger as few requests as possible, so we invest the little - // memory inefficiency that we have here in favor of a better grouping of requests. - return std::make_unique>>>( - &engine, this, std::move(infos)); + + return std::make_unique>(&engine, this, + std::move(infos)); } } diff --git a/arangod/Aql/ConstrainedSortExecutor.cpp b/arangod/Aql/ConstrainedSortExecutor.cpp index 66d72456faa9..4e66a630a851 100644 --- a/arangod/Aql/ConstrainedSortExecutor.cpp +++ b/arangod/Aql/ConstrainedSortExecutor.cpp @@ -54,9 +54,9 @@ void eraseRow(SharedAqlItemBlockPtr& block, size_t row) { /// @brief OurLessThan class arangodb::aql::ConstrainedLessThan { public: - ConstrainedLessThan(arangodb::transaction::Methods* trx, + ConstrainedLessThan(velocypack::Options const* options, std::vector& sortRegisters) noexcept - : _trx(trx), _heapBuffer(nullptr), _sortRegisters(sortRegisters) {} + : _vpackOptions(options), _heapBuffer(nullptr), _sortRegisters(sortRegisters) {} void setBuffer(arangodb::aql::AqlItemBlock* heap) { _heapBuffer = heap; } @@ -67,7 +67,7 @@ class arangodb::aql::ConstrainedLessThan { auto const& lhs = _heapBuffer->getValueReference(a, sortReg.reg); auto const& rhs = _heapBuffer->getValueReference(b, sortReg.reg); - int const cmp = arangodb::aql::AqlValue::Compare(_trx, lhs, rhs, true); + int const cmp = arangodb::aql::AqlValue::Compare(_vpackOptions, lhs, rhs, true); if (cmp < 0) { return sortReg.asc; @@ -80,7 +80,7 @@ class arangodb::aql::ConstrainedLessThan { } private: - arangodb::transaction::Methods* _trx; + velocypack::Options const* const _vpackOptions; arangodb::aql::AqlItemBlock* _heapBuffer; std::vector& _sortRegisters; }; // ConstrainedLessThan @@ -123,7 +123,7 @@ bool ConstrainedSortExecutor::compareInput(size_t const& rowPos, InputAqlItemRow auto const& lhs = _heapBuffer->getValueReference(rowPos, reg.reg); auto const& rhs = row.getValue(reg.reg); - int const cmp = arangodb::aql::AqlValue::Compare(_infos.trx(), lhs, rhs, true); + int const cmp = arangodb::aql::AqlValue::Compare(_infos.vpackOptions(), lhs, rhs, true); if (cmp < 0) { return reg.asc; @@ -144,7 +144,8 @@ ConstrainedSortExecutor::ConstrainedSortExecutor(Fetcher& fetcher, SortExecutorI _skippedAfter(0), _heapBuffer(_infos._manager.requestBlock(_infos._limit, _infos.numberOfOutputRegisters())), - _cmpHeap(std::make_unique(_infos.trx(), _infos.sortRegisters())), + _cmpHeap(std::make_unique(_infos.vpackOptions(), + _infos.sortRegisters())), _heapOutputRow{_heapBuffer, make_shared_unordered_set(), make_shared_unordered_set(_infos.numberOfOutputRegisters()), _infos.registersToClear()} { diff --git a/arangod/Aql/DependencyProxy.cpp b/arangod/Aql/DependencyProxy.cpp index 03a893f165c0..3c833850b55e 100644 --- a/arangod/Aql/DependencyProxy.cpp +++ b/arangod/Aql/DependencyProxy.cpp @@ -255,7 +255,8 @@ template DependencyProxy::DependencyProxy( std::vector const& dependencies, AqlItemBlockManager& itemBlockManager, std::shared_ptr const> inputRegisters, - RegisterId nrInputRegisters) + RegisterId nrInputRegisters, + velocypack::Options const* const options) : _dependencies(dependencies), _itemBlockManager(itemBlockManager), _inputRegisters(std::move(inputRegisters)), @@ -264,7 +265,8 @@ DependencyProxy::DependencyProxy( _blockQueue(), _blockPassThroughQueue(), _currentDependency(0), - _skipped(0) {} + _skipped(0), + _vpackOptions(options) {} template RegisterId DependencyProxy::getNrInputRegisters() const { @@ -316,5 +318,11 @@ bool DependencyProxy::advanceDependency() { return true; } +template +velocypack::Options const* DependencyProxy::velocypackOptions() const + noexcept { + return _vpackOptions; +} + template class ::arangodb::aql::DependencyProxy; template class ::arangodb::aql::DependencyProxy; diff --git a/arangod/Aql/DependencyProxy.h b/arangod/Aql/DependencyProxy.h index 4eb238adba43..e5a55d2095b8 100644 --- a/arangod/Aql/DependencyProxy.h +++ b/arangod/Aql/DependencyProxy.h @@ -68,7 +68,8 @@ class DependencyProxy { DependencyProxy(std::vector const& dependencies, AqlItemBlockManager& itemBlockManager, std::shared_ptr const> inputRegisters, - RegisterId nrInputRegisters); + RegisterId nrInputRegisters, + velocypack::Options const*); TEST_VIRTUAL ~DependencyProxy() = default; @@ -109,6 +110,8 @@ class DependencyProxy { void setDistributeId(std::string const& distId) { _distributeId = distId; } + [[nodiscard]] velocypack::Options const* velocypackOptions() const noexcept; + protected: [[nodiscard]] AqlItemBlockManager& itemBlockManager(); [[nodiscard]] AqlItemBlockManager const& itemBlockManager() const; @@ -134,6 +137,7 @@ class DependencyProxy { // only modified in case of multiple dependencies + Passthrough otherwise always 0 size_t _currentDependency; size_t _skipped; + velocypack::Options const* const _vpackOptions; }; } // namespace arangodb::aql diff --git a/arangod/Aql/DistributeExecutor.cpp b/arangod/Aql/DistributeExecutor.cpp index f2011af5676b..1042214c0710 100644 --- a/arangod/Aql/DistributeExecutor.cpp +++ b/arangod/Aql/DistributeExecutor.cpp @@ -251,11 +251,18 @@ std::pair ExecutionBlockImpl::getBlock SharedAqlItemBlockPtr cur = _buffer[_index]; - while (_pos < cur->size()) { - // this may modify the input item buffer in place - size_t const id = sendToClient(cur); + for (; _pos < cur->size(); ++_pos) { + if (!cur->isShadowRow(_pos)) { + // this may modify the input item buffer in place + size_t const id = sendToClient(cur); - _distBuffer[id].emplace_back(_index, _pos++); + _distBuffer[id].emplace_back(_index, _pos); + } else { + // A shadow row must always be distributed to all clients. + for (auto& dist : _distBuffer) { + dist.emplace_back(_index, _pos); + } + } } if (_pos == cur->size()) { diff --git a/arangod/Aql/ExecutionBlock.cpp b/arangod/Aql/ExecutionBlock.cpp index 6fae6e23107f..7b8294f9de03 100644 --- a/arangod/Aql/ExecutionBlock.cpp +++ b/arangod/Aql/ExecutionBlock.cpp @@ -70,7 +70,7 @@ std::string const& stateToString(aql::ExecutionState state) { ExecutionBlock::ExecutionBlock(ExecutionEngine* engine, ExecutionNode const* ep) : _engine(engine), - _trx(engine->getQuery()->trx()), + _trxVpackOptions(engine->getQuery()->trx()->transactionContextPtr()->getVPackOptions()), _shutdownResult(TRI_ERROR_NO_ERROR), _done(false), _isInSplicedSubquery(ep != nullptr ? ep->isInSplicedSubquery() : false), @@ -190,8 +190,8 @@ std::pair ExecutionBlock::traceGetSomeEnd << "getSome type=" << node->getTypeString() << " result: nullptr"; } else { VPackBuilder builder; - result->toSimpleVPack(transaction(), builder); - auto options = transaction()->transactionContextPtr()->getVPackOptions(); + auto const options = trxVpackOptions(); + result->toSimpleVPack(options, builder); LOG_TOPIC("fcd9c", INFO, Logger::QUERIES) << "[query#" << queryId << "] " << "getSome type=" << node->getTypeString() @@ -272,7 +272,9 @@ ExecutionState ExecutionBlock::getHasMoreState() { ExecutionNode const* ExecutionBlock::getPlanNode() const { return _exeNode; } -transaction::Methods* ExecutionBlock::transaction() const { return _trx; } +velocypack::Options const* ExecutionBlock::trxVpackOptions() const noexcept { + return _trxVpackOptions; +} void ExecutionBlock::addDependency(ExecutionBlock* ep) { TRI_ASSERT(ep != nullptr); diff --git a/arangod/Aql/ExecutionBlock.h b/arangod/Aql/ExecutionBlock.h index 98d22c35c8b1..713b337649f5 100644 --- a/arangod/Aql/ExecutionBlock.h +++ b/arangod/Aql/ExecutionBlock.h @@ -117,7 +117,7 @@ class ExecutionBlock { // TODO: Can we get rid of this? Problem: Subquery Executor is using it. ExecutionNode const* getPlanNode() const; - transaction::Methods* transaction() const; + [[nodiscard]] velocypack::Options const* trxVpackOptions() const noexcept; /// @brief add a dependency void addDependency(ExecutionBlock* ep); @@ -128,8 +128,7 @@ class ExecutionBlock { /// @brief the execution engine ExecutionEngine* _engine; - /// @brief the transaction for this query - transaction::Methods* _trx; + velocypack::Options const* _trxVpackOptions; /// @brief the Result returned during the shutdown phase. Is kept for multiple /// waiting phases. diff --git a/arangod/Aql/ExecutionBlockImpl.cpp b/arangod/Aql/ExecutionBlockImpl.cpp index 4da26b85d191..f0613b1c52a8 100644 --- a/arangod/Aql/ExecutionBlockImpl.cpp +++ b/arangod/Aql/ExecutionBlockImpl.cpp @@ -42,7 +42,6 @@ #include "Aql/IResearchViewExecutor.h" #include "Aql/IdExecutor.h" #include "Aql/IndexExecutor.h" -#include "Aql/IndexNode.h" #include "Aql/InputAqlItemRow.h" #include "Aql/KShortestPathsExecutor.h" #include "Aql/LimitExecutor.h" @@ -65,6 +64,7 @@ #include "Aql/SubqueryStartExecutor.h" #include "Aql/TraversalExecutor.h" #include "Aql/UnsortedGatherExecutor.h" +#include "Aql/UnsortingGatherExecutor.h" #include "Aql/SimpleModifier.h" #include "Aql/UpsertModifier.h" @@ -108,7 +108,8 @@ ExecutionBlockImpl::ExecutionBlockImpl(ExecutionEngine* engine, typename Executor::Infos&& infos) : ExecutionBlock(engine, node), _dependencyProxy(_dependencies, engine->itemBlockManager(), - infos.getInputRegisters(), infos.numberOfInputRegisters()), + infos.getInputRegisters(), infos.numberOfInputRegisters(), + trxVpackOptions()), _rowFetcher(_dependencyProxy), _infos(std::move(infos)), _executor(_rowFetcher, _infos), @@ -370,9 +371,9 @@ static SkipVariants constexpr skipType() { std::is_same>::value || std::is_same::value || std::is_same::value || - std::is_same>>::value || std::is_same::value || std::is_same::value || + std::is_same::value || std::is_same::value || std::is_same>::value || std::is_same>::value), @@ -871,8 +872,6 @@ template class ::arangodb::aql::ExecutionBlockImpl>; template class ::arangodb::aql::ExecutionBlockImpl< IdExecutor>>; -template class ::arangodb::aql::ExecutionBlockImpl< - IdExecutor>>; template class ::arangodb::aql::ExecutionBlockImpl; template class ::arangodb::aql::ExecutionBlockImpl; @@ -897,6 +896,7 @@ template class ::arangodb::aql::ExecutionBlockImpl; template class ::arangodb::aql::ExecutionBlockImpl; template class ::arangodb::aql::ExecutionBlockImpl; template class ::arangodb::aql::ExecutionBlockImpl; +template class ::arangodb::aql::ExecutionBlockImpl; template class ::arangodb::aql::ExecutionBlockImpl>; template class ::arangodb::aql::ExecutionBlockImpl>; diff --git a/arangod/Aql/ExecutionNode.cpp b/arangod/Aql/ExecutionNode.cpp index 6c77ce57276a..1e1ab721fba3 100644 --- a/arangod/Aql/ExecutionNode.cpp +++ b/arangod/Aql/ExecutionNode.cpp @@ -567,6 +567,7 @@ bool ExecutionNode::isEqualTo(ExecutionNode const& other) const { return ((this->getType() == other.getType()) && (_id == other._id) && (_depth == other._depth) && + (isInSplicedSubquery() == other.isInSplicedSubquery()) && (std::equal(_dependencies.begin(), _dependencies.end(), other._dependencies.begin(), comparator))); } diff --git a/arangod/Aql/Expression.cpp b/arangod/Aql/Expression.cpp index eaf6752c7769..58156e36a32f 100644 --- a/arangod/Aql/Expression.cpp +++ b/arangod/Aql/Expression.cpp @@ -37,6 +37,7 @@ #include "Aql/Range.h" #include "Aql/V8Executor.h" #include "Aql/Variable.h" +#include "Aql/AqlValueMaterializer.h" #include "Basics/Exceptions.h" #include "Basics/NumberUtils.h" #include "Basics/StringBuffer.h" diff --git a/arangod/Aql/Functions.cpp b/arangod/Aql/Functions.cpp index 243a11689e8c..570049662540 100644 --- a/arangod/Aql/Functions.cpp +++ b/arangod/Aql/Functions.cpp @@ -32,6 +32,7 @@ #include "Aql/Query.h" #include "Aql/Range.h" #include "Aql/V8Executor.h" +#include "Aql/AqlValueMaterializer.h" #include "Basics/Exceptions.h" #include "Basics/HybridLogicalClock.h" #include "Basics/Mutex.h" diff --git a/arangod/Aql/IdExecutor.h b/arangod/Aql/IdExecutor.h index df550ab6efc3..78390f3c113e 100644 --- a/arangod/Aql/IdExecutor.h +++ b/arangod/Aql/IdExecutor.h @@ -31,6 +31,8 @@ #include #include +// TODO Clean up unused variants of the IdExecutor - some of them aren't in use anymore. + namespace arangodb { namespace transaction { class Methods; @@ -57,6 +59,7 @@ class IdExecutorInfos : public ExecutorInfos { std::string const& distributeId(); + // TODO This is probably needed only for UnsortingGather now, so can be removed here. [[nodiscard]] bool isResponsibleForInitializeCursor() const; private: diff --git a/arangod/Aql/IndexExecutor.cpp b/arangod/Aql/IndexExecutor.cpp index 838b8c3f0545..3b4abb029598 100644 --- a/arangod/Aql/IndexExecutor.cpp +++ b/arangod/Aql/IndexExecutor.cpp @@ -38,6 +38,7 @@ #include "Aql/OutputAqlItemRow.h" #include "Aql/Query.h" #include "Aql/SingleRowFetcher.h" +#include "Aql/AqlValueMaterializer.h" #include "Basics/ScopeGuard.h" #include "Cluster/ServerState.h" #include "ExecutorExpressionContext.h" diff --git a/arangod/Aql/InputAqlItemRow.cpp b/arangod/Aql/InputAqlItemRow.cpp index 2126d2b93f1b..3ea9bb944d99 100644 --- a/arangod/Aql/InputAqlItemRow.cpp +++ b/arangod/Aql/InputAqlItemRow.cpp @@ -119,7 +119,7 @@ SharedAqlItemBlockPtr InputAqlItemRow::cloneToBlock(AqlItemBlockManager& manager /// corresponding position /// "raw": List of actual values, positions 0 and 1 are always null /// such that actual indices start at 2 -void InputAqlItemRow::toVelocyPack(transaction::Methods* trx, VPackBuilder& result) const { +void InputAqlItemRow::toVelocyPack(velocypack::Options const* const trxOptions, VPackBuilder& result) const { TRI_ASSERT(isInitialized()); TRI_ASSERT(result.isOpenObject()); VPackOptions options(VPackOptions::Defaults); @@ -203,7 +203,7 @@ void InputAqlItemRow::toVelocyPack(transaction::Methods* trx, VPackBuilder& resu if (it == table.end()) { currentState = Next; - a.toVelocyPack(trx, raw, false); + a.toVelocyPack(trxOptions, raw, false); table.try_emplace(a, pos++); } else { currentState = Positional; @@ -290,7 +290,8 @@ bool InputAqlItemRow::operator!=(InputAqlItemRow const& other) const noexcept { return !(*this == other); } -bool InputAqlItemRow::equates(InputAqlItemRow const& other) const noexcept { +bool InputAqlItemRow::equates(InputAqlItemRow const& other, + velocypack::Options const* const options) const noexcept { if (!isInitialized() || !other.isInitialized()) { return isInitialized() == other.isInitialized(); } @@ -298,8 +299,9 @@ bool InputAqlItemRow::equates(InputAqlItemRow const& other) const noexcept { if (getNrRegisters() != other.getNrRegisters()) { return false; } - // NOLINTNEXTLINE(modernize-use-transparent-functors) - auto const eq = std::equal_to{}; + auto const eq = [options](auto left, auto right) { + return 0 == AqlValue::Compare(options, left, right, false); + }; for (RegisterId i = 0; i < getNrRegisters(); ++i) { if (!eq(getValue(i), other.getValue(i))) { return false; diff --git a/arangod/Aql/InputAqlItemRow.h b/arangod/Aql/InputAqlItemRow.h index c02ead2b44d3..7a808d82b872 100644 --- a/arangod/Aql/InputAqlItemRow.h +++ b/arangod/Aql/InputAqlItemRow.h @@ -33,11 +33,9 @@ #include namespace arangodb { -namespace transaction { -class Methods; -} namespace velocypack { class Builder; +struct Options; } namespace aql { @@ -103,7 +101,8 @@ class InputAqlItemRow { // blocks are equal, because comparing rows of blocks with different layouts // does not make sense. // Invalid rows are considered equivalent. - bool equates(InputAqlItemRow const& other) const noexcept; + [[nodiscard]] bool equates(InputAqlItemRow const& other, + velocypack::Options const* options) const noexcept; bool isInitialized() const noexcept; @@ -132,7 +131,7 @@ class InputAqlItemRow { /// @brief toVelocyPack, transfer a single AqlItemRow to Json, the result can /// be used to recreate the AqlItemBlock via the Json constructor /// Uses the same API as an AqlItemBlock with only a single row - void toVelocyPack(transaction::Methods* trx, arangodb::velocypack::Builder&) const; + void toVelocyPack(velocypack::Options const*, arangodb::velocypack::Builder&) const; private: AqlItemBlock& block() noexcept; diff --git a/arangod/Aql/ModificationExecutor.cpp b/arangod/Aql/ModificationExecutor.cpp index bcfaf80d25aa..188c246b9a56 100644 --- a/arangod/Aql/ModificationExecutor.cpp +++ b/arangod/Aql/ModificationExecutor.cpp @@ -47,6 +47,7 @@ using namespace arangodb; using namespace arangodb::aql; using namespace arangodb::basics; + namespace arangodb { namespace aql { diff --git a/arangod/Aql/MultiDependencySingleRowFetcher.cpp b/arangod/Aql/MultiDependencySingleRowFetcher.cpp index e91e95194d7c..01ddc11d5e91 100644 --- a/arangod/Aql/MultiDependencySingleRowFetcher.cpp +++ b/arangod/Aql/MultiDependencySingleRowFetcher.cpp @@ -25,6 +25,12 @@ #include "Aql/AqlItemBlock.h" #include "Aql/DependencyProxy.h" #include "Aql/ShadowAqlItemRow.h" +#include "Logger/LogMacros.h" +#include "Transaction/Context.h" +#include "Transaction/Methods.h" + +#include +#include using namespace arangodb; using namespace arangodb::aql; @@ -119,7 +125,8 @@ std::pair MultiDependencySingleRowFetcher::fet } else { TRI_ASSERT(row.isInitialized()); // All shadow rows must be equal! - TRI_ASSERT(row.equates(ShadowAqlItemRow{dep._currentBlock, dep._rowIndex})); + auto const options = _dependencyProxy->velocypackOptions(); + TRI_ASSERT(row.equates(ShadowAqlItemRow{dep._currentBlock, dep._rowIndex}, options)); } } } diff --git a/arangod/Aql/OptimizerRules.cpp b/arangod/Aql/OptimizerRules.cpp index f7af804c4fd7..5c5260aacb7d 100644 --- a/arangod/Aql/OptimizerRules.cpp +++ b/arangod/Aql/OptimizerRules.cpp @@ -7325,6 +7325,8 @@ bool nodeMakesThisQueryLevelUnsuitableForSubquerySplicing(ExecutionNode const* c case ExecutionNode::RETURN: case ExecutionNode::DISTRIBUTE: case ExecutionNode::SCATTER: + case ExecutionNode::GATHER: + case ExecutionNode::REMOTE: case ExecutionNode::REMOTESINGLE: case ExecutionNode::MATERIALIZE: case ExecutionNode::DISTRIBUTE_CONSUMER: @@ -7332,10 +7334,6 @@ bool nodeMakesThisQueryLevelUnsuitableForSubquerySplicing(ExecutionNode const* c case ExecutionNode::SUBQUERY_END: // These nodes do not initiate a skip themselves, and thus are fine. return false; - // UnsortingGather currently does not work. Also, as we would possibly add - // them with remote nodes in the query, we exclude these here, too. - case ExecutionNode::REMOTE: - case ExecutionNode::GATHER: case ExecutionNode::NORESULTS: // no results currently cannot work, as they do not fetch from above. case ExecutionNode::LIMIT: @@ -7564,6 +7562,8 @@ void arangodb::aql::spliceSubqueriesRule(Optimizer* opt, std::unique_ptrcreateNode(plan.get(), plan->nextId(), &plan->getAst()->query()->vocbase(), "", "", ""); + scatterNode->setIsInSplicedSubquery(true); + remoteNode->setIsInSplicedSubquery(true); plan->insertAfter(start, scatterNode); plan->insertAfter(scatterNode, remoteNode); diff --git a/arangod/Aql/OutputAqlItemRow.cpp b/arangod/Aql/OutputAqlItemRow.cpp index dbcaa6912ed7..079653a618a8 100644 --- a/arangod/Aql/OutputAqlItemRow.cpp +++ b/arangod/Aql/OutputAqlItemRow.cpp @@ -192,9 +192,9 @@ void OutputAqlItemRow::advanceRow() { _numValuesWritten = 0; } -void OutputAqlItemRow::toVelocyPack(transaction::Methods& trx, VPackBuilder& builder) { +void OutputAqlItemRow::toVelocyPack(velocypack::Options const* options, VPackBuilder& builder) { TRI_ASSERT(produced()); - block().rowToSimpleVPack(_baseIndex, &trx, builder); + block().rowToSimpleVPack(_baseIndex, options, builder); } SharedAqlItemBlockPtr OutputAqlItemRow::stealBlock() { diff --git a/arangod/Aql/OutputAqlItemRow.h b/arangod/Aql/OutputAqlItemRow.h index 65edfe7fe170..969b06e2e895 100644 --- a/arangod/Aql/OutputAqlItemRow.h +++ b/arangod/Aql/OutputAqlItemRow.h @@ -159,7 +159,7 @@ class OutputAqlItemRow { // The data of this row will be copied. void decreaseShadowRowDepth(ShadowAqlItemRow const& sourceRow); - void toVelocyPack(transaction::Methods& trx, velocypack::Builder& builder); + void toVelocyPack(velocypack::Options const* options, velocypack::Builder& builder); private: [[nodiscard]] std::unordered_set const& outputRegisters() const { diff --git a/arangod/Aql/QuerySnippet.cpp b/arangod/Aql/QuerySnippet.cpp index 9f257a3e3968..3038df860c58 100644 --- a/arangod/Aql/QuerySnippet.cpp +++ b/arangod/Aql/QuerySnippet.cpp @@ -425,6 +425,7 @@ DistributeConsumerNode* QuerySnippet::createConsumerNode(ExecutionPlan* plan, TRI_ASSERT(consumer != nullptr); // Hand over responsibility to plan, s.t. it can clean up if one of the below fails plan->registerNode(uniq_consumer.release()); + consumer->setIsInSplicedSubquery(internalScatter->isInSplicedSubquery()); consumer->addDependency(internalScatter); consumer->cloneRegisterPlan(internalScatter); internalScatter->addClient(consumer); diff --git a/arangod/Aql/RegexCache.cpp b/arangod/Aql/RegexCache.cpp index 929a994513eb..e15909d04180 100644 --- a/arangod/Aql/RegexCache.cpp +++ b/arangod/Aql/RegexCache.cpp @@ -22,9 +22,11 @@ //////////////////////////////////////////////////////////////////////////////// #include "RegexCache.h" + +#include "Aql/AqlValueMaterializer.h" +#include "Basics/StringUtils.h" #include "Basics/Utf8Helper.h" -#include -#include +#include "Basics/tryEmplaceHelper.h" #include #include diff --git a/arangod/Aql/RemoteExecutor.cpp b/arangod/Aql/RemoteExecutor.cpp index aab865c1ef0a..15d5f91bcf06 100644 --- a/arangod/Aql/RemoteExecutor.cpp +++ b/arangod/Aql/RemoteExecutor.cpp @@ -37,6 +37,8 @@ #include "Network/NetworkFeature.h" #include "Network/Utils.h" #include "Rest/CommonDefines.h" +#include "Transaction/Context.h" +#include "Transaction/Methods.h" #include "VocBase/vocbase.h" #include @@ -304,7 +306,7 @@ std::pair ExecutionBlockImpl::initialize builder.add("pos", VPackValue(0)); builder.add(VPackValue("items")); builder.openObject(/*unindexed*/ true); - input.toVelocyPack(_engine->getQuery()->trx(), builder); + input.toVelocyPack(_engine->getQuery()->trx()->transactionContextPtr()->getVPackOptions(), builder); builder.close(); builder.close(); diff --git a/arangod/Aql/RestAqlHandler.cpp b/arangod/Aql/RestAqlHandler.cpp index 485b7f97119a..9882ddca99ed 100644 --- a/arangod/Aql/RestAqlHandler.cpp +++ b/arangod/Aql/RestAqlHandler.cpp @@ -692,7 +692,8 @@ RestStatus RestAqlHandler::handleUseQuery(std::string const& operation, // Backwards Compatibility answerBuilder.add(StaticStrings::Error, VPackValue(false)); } else { - items->toVelocyPack(_query->trx(), answerBuilder); + items->toVelocyPack(_query->trx()->transactionContextPtr()->getVPackOptions(), + answerBuilder); } } else if (operation == "skipSome") { auto atMost = diff --git a/arangod/Aql/ShadowAqlItemRow.cpp b/arangod/Aql/ShadowAqlItemRow.cpp index b6d78b750708..c7a77595ba04 100644 --- a/arangod/Aql/ShadowAqlItemRow.cpp +++ b/arangod/Aql/ShadowAqlItemRow.cpp @@ -22,6 +22,10 @@ #include "ShadowAqlItemRow.h" +#include "Basics/VelocyPackHelper.h" +#include "Transaction/Methods.h" +#include "Transaction/Context.h" + using namespace arangodb; using namespace arangodb::aql; @@ -92,7 +96,8 @@ bool ShadowAqlItemRow::operator!=(ShadowAqlItemRow const& other) const noexcept return !(*this == other); } -bool ShadowAqlItemRow::equates(ShadowAqlItemRow const& other) const noexcept { +bool ShadowAqlItemRow::equates(ShadowAqlItemRow const& other, + velocypack::Options const* options) const noexcept { if (!isInitialized() || !other.isInitialized()) { return isInitialized() == other.isInitialized(); } @@ -103,8 +108,9 @@ bool ShadowAqlItemRow::equates(ShadowAqlItemRow const& other) const noexcept { if (getDepth() != other.getDepth()) { return false; } - // NOLINTNEXTLINE(modernize-use-transparent-functors) - auto const eq = std::equal_to{}; + auto const eq = [options](auto left, auto right) { + return 0 == AqlValue::Compare(options, left, right, false); + }; for (RegisterId i = 0; i < getNrRegisters(); ++i) { if (!eq(getValue(i), other.getValue(i))) { return false; diff --git a/arangod/Aql/ShadowAqlItemRow.h b/arangod/Aql/ShadowAqlItemRow.h index 7cd53726933a..187ccdaa4f22 100644 --- a/arangod/Aql/ShadowAqlItemRow.h +++ b/arangod/Aql/ShadowAqlItemRow.h @@ -28,6 +28,9 @@ #include namespace arangodb { +namespace velocypack { +struct Options; +} namespace aql { struct CreateInvalidShadowRowHint { @@ -113,7 +116,8 @@ class ShadowAqlItemRow { // blocks are equal, because comparing rows of blocks with different layouts // does not make sense. // Invalid rows are considered equivalent. - bool equates(ShadowAqlItemRow const& other) const noexcept; + [[nodiscard]] bool equates(ShadowAqlItemRow const& other, + velocypack::Options const* option) const noexcept; private: AqlItemBlock& block() noexcept; diff --git a/arangod/Aql/SortExecutor.cpp b/arangod/Aql/SortExecutor.cpp index 5c9e6ccb7c37..c5121dc8e09c 100644 --- a/arangod/Aql/SortExecutor.cpp +++ b/arangod/Aql/SortExecutor.cpp @@ -43,9 +43,9 @@ namespace { /// @brief OurLessThan class OurLessThan { public: - OurLessThan(arangodb::transaction::Methods* trx, AqlItemMatrix const& input, + OurLessThan(velocypack::Options const* options, AqlItemMatrix const& input, std::vector const& sortRegisters) noexcept - : _trx(trx), _input(input), _sortRegisters(sortRegisters) {} + : _vpackOptions(options), _input(input), _sortRegisters(sortRegisters) {} bool operator()(AqlItemMatrix::RowIndex const& a, AqlItemMatrix::RowIndex const& b) const { InputAqlItemRow left = _input.getRow(a); @@ -54,7 +54,7 @@ class OurLessThan { AqlValue const& lhs = left.getValue(reg.reg); AqlValue const& rhs = right.getValue(reg.reg); - int const cmp = AqlValue::Compare(_trx, lhs, rhs, true); + int const cmp = AqlValue::Compare(_vpackOptions, lhs, rhs, true); if (cmp < 0) { return reg.asc; @@ -67,7 +67,7 @@ class OurLessThan { } private: - arangodb::transaction::Methods* _trx; + velocypack::Options const* _vpackOptions; AqlItemMatrix const& _input; std::vector const& _sortRegisters; }; // OurLessThan @@ -90,27 +90,29 @@ SortExecutorInfos::SortExecutorInfos( // cppcheck-suppress passedByValue std::unordered_set registersToClear, // cppcheck-suppress passedByValue - std::unordered_set registersToKeep, transaction::Methods* trx, bool stable) + std::unordered_set registersToKeep, + velocypack::Options const* options, bool stable) : ExecutorInfos(mapSortRegistersToRegisterIds(sortRegisters), nullptr, nrInputRegisters, nrOutputRegisters, std::move(registersToClear), std::move(registersToKeep)), _limit(limit), _manager(manager), - _trx(trx), + _vpackOptions(options), _sortRegisters(std::move(sortRegisters)), _stable(stable) { - TRI_ASSERT(trx != nullptr); TRI_ASSERT(!_sortRegisters.empty()); } -transaction::Methods* SortExecutorInfos::trx() const { return _trx; } - std::vector& SortExecutorInfos::sortRegisters() { return _sortRegisters; } bool SortExecutorInfos::stable() const { return _stable; } +velocypack::Options const* SortExecutorInfos::vpackOptions() const noexcept { + return _vpackOptions; +} + SortExecutor::SortExecutor(Fetcher& fetcher, SortExecutorInfos& infos) : _infos(infos), _fetcher(fetcher), _input(nullptr), _returnNext(0) {} SortExecutor::~SortExecutor() = default; @@ -160,7 +162,7 @@ void SortExecutor::doSorting() { TRI_ASSERT(_input != nullptr); _sortedIndexes = _input->produceRowIndexes(); // comparison function - OurLessThan ourLessThan(_infos.trx(), *_input, _infos.sortRegisters()); + OurLessThan ourLessThan(_infos.vpackOptions(), *_input, _infos.sortRegisters()); if (_infos.stable()) { std::stable_sort(_sortedIndexes.begin(), _sortedIndexes.end(), ourLessThan); } else { diff --git a/arangod/Aql/SortExecutor.h b/arangod/Aql/SortExecutor.h index 6094dbc36649..7420611808bd 100644 --- a/arangod/Aql/SortExecutor.h +++ b/arangod/Aql/SortExecutor.h @@ -54,14 +54,14 @@ class SortExecutorInfos : public ExecutorInfos { AqlItemBlockManager& manager, RegisterId nrInputRegisters, RegisterId nrOutputRegisters, std::unordered_set registersToClear, std::unordered_set registersToKeep, - transaction::Methods* trx, bool stable); + velocypack::Options const*, bool stable); SortExecutorInfos() = delete; SortExecutorInfos(SortExecutorInfos&&) = default; SortExecutorInfos(SortExecutorInfos const&) = delete; ~SortExecutorInfos() = default; - arangodb::transaction::Methods* trx() const; + [[nodiscard]] velocypack::Options const* vpackOptions() const noexcept; std::vector& sortRegisters(); @@ -69,9 +69,7 @@ class SortExecutorInfos : public ExecutorInfos { std::size_t _limit; AqlItemBlockManager& _manager; - - private: - arangodb::transaction::Methods* _trx; + velocypack::Options const* _vpackOptions; std::vector _sortRegisters; bool _stable; }; diff --git a/arangod/Aql/SortNode.cpp b/arangod/Aql/SortNode.cpp index 64cc9967fd79..c70ba96722bd 100644 --- a/arangod/Aql/SortNode.cpp +++ b/arangod/Aql/SortNode.cpp @@ -37,6 +37,8 @@ #include "Aql/WalkerWorker.h" #include "Basics/StringBuffer.h" #include "Basics/VelocyPackHelper.h" +#include "Transaction/Context.h" +#include "Transaction/Methods.h" namespace { std::string const ConstrainedHeap = "constrained-heap"; @@ -237,8 +239,10 @@ std::unique_ptr SortNode::createBlock( } SortExecutorInfos infos(std::move(sortRegs), _limit, engine.itemBlockManager(), getRegisterPlan()->nrRegs[previousNode->getDepth()], - getRegisterPlan()->nrRegs[getDepth()], getRegsToClear(), - calcRegsToKeep(), engine.getQuery()->trx(), _stable); + getRegisterPlan()->nrRegs[getDepth()], + getRegsToClear(), calcRegsToKeep(), + engine.getQuery()->trx()->transactionContextPtr()->getVPackOptions(), + _stable); if (sorterType() == SorterType::Standard) { return std::make_unique>(&engine, this, std::move(infos)); diff --git a/arangod/Aql/SubqueryEndExecutionNode.cpp b/arangod/Aql/SubqueryEndExecutionNode.cpp index 413add93be0f..44a2f2fb0153 100644 --- a/arangod/Aql/SubqueryEndExecutionNode.cpp +++ b/arangod/Aql/SubqueryEndExecutionNode.cpp @@ -30,6 +30,8 @@ #include "Aql/RegisterPlan.h" #include "Aql/SubqueryEndExecutor.h" #include "Meta/static_assert_size.h" +#include "Transaction/Context.h" +#include "Transaction/Methods.h" #include #include @@ -85,10 +87,11 @@ std::unique_ptr SubqueryEndNode::createBlock( auto outReg = variableToRegisterId(_outVariable); outputRegisters->emplace(outReg); + auto const vpackOptions = trx->transactionContextPtr()->getVPackOptions(); SubqueryEndExecutorInfos infos(inputRegisters, outputRegisters, getRegisterPlan()->nrRegs[previousNode->getDepth()], - getRegisterPlan()->nrRegs[getDepth()], - getRegsToClear(), calcRegsToKeep(), trx, inReg, outReg); + getRegisterPlan()->nrRegs[getDepth()], getRegsToClear(), + calcRegsToKeep(), vpackOptions, inReg, outReg); return std::make_unique>(&engine, this, std::move(infos)); diff --git a/arangod/Aql/SubqueryEndExecutor.cpp b/arangod/Aql/SubqueryEndExecutor.cpp index 417958597c5f..21dd54e4cda7 100644 --- a/arangod/Aql/SubqueryEndExecutor.cpp +++ b/arangod/Aql/SubqueryEndExecutor.cpp @@ -45,10 +45,10 @@ SubqueryEndExecutorInfos::SubqueryEndExecutorInfos( RegisterId nrInputRegisters, RegisterId nrOutputRegisters, std::unordered_set const& registersToClear, std::unordered_set registersToKeep, - transaction::Methods* trxPtr, RegisterId inReg, RegisterId outReg) + velocypack::Options const* const options, RegisterId inReg, RegisterId outReg) : ExecutorInfos(std::move(readableInputRegisters), std::move(writeableOutputRegisters), nrInputRegisters, nrOutputRegisters, registersToClear, std::move(registersToKeep)), - _trxPtr(trxPtr), + _vpackOptions(options), _outReg(outReg), _inReg(inReg) {} @@ -60,6 +60,10 @@ bool SubqueryEndExecutorInfos::usesInputRegister() const { return _inReg != RegisterPlan::MaxRegisterId; } +velocypack::Options const* SubqueryEndExecutorInfos::vpackOptions() const noexcept { + return _vpackOptions; +} + SubqueryEndExecutor::SubqueryEndExecutor(Fetcher& fetcher, SubqueryEndExecutorInfos& infos) : _fetcher(fetcher), _infos(infos), _accumulator(nullptr), _state(ACCUMULATE) { resetAccumulator(); @@ -87,7 +91,7 @@ std::pair SubqueryEndExecutor::produceRows(OutputAqlIte if (inputRow.isInitialized() && _infos.usesInputRegister()) { TRI_ASSERT(_accumulator->isOpenArray()); AqlValue value = inputRow.getValue(_infos.getInputRegister()); - value.toVelocyPack(_infos.getTrxPtr(), *_accumulator, false); + value.toVelocyPack(_infos.vpackOptions(), *_accumulator, false); } // We have received DONE on data rows, so now diff --git a/arangod/Aql/SubqueryEndExecutor.h b/arangod/Aql/SubqueryEndExecutor.h index 18113302c2da..882c614f7d98 100644 --- a/arangod/Aql/SubqueryEndExecutor.h +++ b/arangod/Aql/SubqueryEndExecutor.h @@ -48,7 +48,7 @@ class SubqueryEndExecutorInfos : public ExecutorInfos { RegisterId nrInputRegisters, RegisterId nrOutputRegisters, std::unordered_set const& registersToClear, std::unordered_set registersToKeep, - transaction::Methods* trxPtr, RegisterId inReg, + velocypack::Options const* options, RegisterId inReg, RegisterId outReg); SubqueryEndExecutorInfos() = delete; @@ -56,13 +56,13 @@ class SubqueryEndExecutorInfos : public ExecutorInfos { SubqueryEndExecutorInfos(SubqueryEndExecutorInfos const&) = delete; ~SubqueryEndExecutorInfos(); - transaction::Methods* getTrxPtr() const noexcept { return _trxPtr; } + [[nodiscard]] velocypack::Options const* vpackOptions() const noexcept; inline RegisterId getOutputRegister() const { return _outReg; } bool usesInputRegister() const; inline RegisterId getInputRegister() const { return _inReg; } private: - transaction::Methods* _trxPtr; + velocypack::Options const* _vpackOptions; RegisterId const _outReg; RegisterId const _inReg; }; diff --git a/arangod/Aql/UnsortingGatherExecutor.cpp b/arangod/Aql/UnsortingGatherExecutor.cpp new file mode 100644 index 000000000000..9c4a70a9bf31 --- /dev/null +++ b/arangod/Aql/UnsortingGatherExecutor.cpp @@ -0,0 +1,130 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2019 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Tobias Gödderz +//////////////////////////////////////////////////////////////////////////////// + +#include "UnsortingGatherExecutor.h" +#include + +#include "Aql/IdExecutor.h" // for IdExecutorInfos +#include "Aql/MultiDependencySingleRowFetcher.h" +#include "Aql/OutputAqlItemRow.h" +#include "Aql/Stats.h" +#include "Basics/Exceptions.h" +#include "Basics/debugging.h" +#include "Basics/voc-errors.h" + +using namespace arangodb; +using namespace arangodb::aql; + +UnsortingGatherExecutor::UnsortingGatherExecutor(Fetcher& fetcher, Infos& infos) + : _fetcher(fetcher) {} + +UnsortingGatherExecutor::~UnsortingGatherExecutor() = default; + +auto UnsortingGatherExecutor::produceRows(OutputAqlItemRow& output) + -> std::pair { + while (!output.isFull() && !done()) { + // Note that fetchNextRow may return DONE (because the current dependency is + // DONE), and also return an unitialized row in that case, but we are not + // DONE completely - that's what `done()` is for. + auto [state, inputRow] = fetchNextRow(output.numRowsLeft()); + if (state == ExecutionState::WAITING) { + return {state, {}}; + } + // HASMORE => inputRow.isInitialized() + TRI_ASSERT(state == ExecutionState::DONE || inputRow.isInitialized()); + if (inputRow.isInitialized()) { + output.copyRow(inputRow); + TRI_ASSERT(output.produced()); + output.advanceRow(); + } + } + + auto state = done() ? ExecutionState::DONE : ExecutionState::HASMORE; + return {state, {}}; +} + +auto UnsortingGatherExecutor::fetcher() const noexcept -> const Fetcher& { + return _fetcher; +} + +auto UnsortingGatherExecutor::fetcher() noexcept -> Fetcher& { + return _fetcher; +} + +auto UnsortingGatherExecutor::numDependencies() const + noexcept(noexcept(_fetcher.numberDependencies())) -> size_t { + return _fetcher.numberDependencies(); +} + +auto UnsortingGatherExecutor::fetchNextRow(size_t atMost) + -> std::pair { + auto res = fetcher().fetchRowForDependency(currentDependency(), atMost); + if (res.first == ExecutionState::DONE) { + advanceDependency(); + } + return res; +} + +auto UnsortingGatherExecutor::skipNextRows(size_t atMost) + -> std::pair { + auto res = fetcher().skipRowsForDependency(currentDependency(), atMost); + if (res.first == ExecutionState::DONE) { + advanceDependency(); + } + return res; +} + +auto UnsortingGatherExecutor::done() const noexcept -> bool { + return _currentDependency >= numDependencies(); +} + +auto UnsortingGatherExecutor::currentDependency() const noexcept -> size_t { + return _currentDependency; +} + +auto UnsortingGatherExecutor::advanceDependency() noexcept -> void { + TRI_ASSERT(_currentDependency < numDependencies()); + ++_currentDependency; +} + +auto UnsortingGatherExecutor::skipRows(size_t const atMost) + -> std::tuple { + auto const rowsLeftToSkip = [&atMost, &skipped = this->_skipped]() { + TRI_ASSERT(atMost >= skipped); + return atMost - skipped; + }; + while (rowsLeftToSkip() > 0 && !done()) { + // Note that skipNextRow may return DONE (because the current dependency is + // DONE), and also return an unitialized row in that case, but we are not + // DONE completely - that's what `done()` is for. + auto [state, skipped] = skipNextRows(rowsLeftToSkip()); + _skipped += skipped; + if (state == ExecutionState::WAITING) { + return {state, {}, 0}; + } + } + + auto state = done() ? ExecutionState::DONE : ExecutionState::HASMORE; + auto skipped = size_t{0}; + std::swap(skipped, _skipped); + return {state, {}, skipped}; +} diff --git a/arangod/Aql/UnsortingGatherExecutor.h b/arangod/Aql/UnsortingGatherExecutor.h new file mode 100644 index 000000000000..9e17f660cf06 --- /dev/null +++ b/arangod/Aql/UnsortingGatherExecutor.h @@ -0,0 +1,98 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2019 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Tobias Gödderz +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGOD_AQL_UNSORTINGGATHEREXECUTOR_H +#define ARANGOD_AQL_UNSORTINGGATHEREXECUTOR_H + +#include "Aql/ExecutionState.h" +#include "Aql/ExecutorInfos.h" +#include "Aql/MultiDependencySingleRowFetcher.h" +#include "Aql/types.h" + +#include +#include + +namespace arangodb::aql { + +class NoStats; +class InputAqlItemRow; +class OutputAqlItemRow; +class IdExecutorInfos; +class SharedAqlItemBlockPtr; + +/** +* @brief Produces all rows from its dependencies, which may be more than one, +* in some unspecified order. It is, purposefully, strictly synchronous, and +* always waits for an answer before requesting the next row(s). +* +* The actual implementation fetches all available rows from the first +* dependency, then from the second, and so forth. But that is not guaranteed. +*/ +class UnsortingGatherExecutor { + public: + struct Properties { + static constexpr bool preservesOrder = false; + static constexpr BlockPassthrough allowsBlockPassthrough = BlockPassthrough::Disable; + // TODO I think we can set this to true (but needs to implement + // hasExpectedNumberOfRows for that) + static constexpr bool inputSizeRestrictsOutputSize = false; + }; + using Fetcher = MultiDependencySingleRowFetcher; + // TODO I should probably implement custom Infos, we don't need distributeId(). + using Infos = IdExecutorInfos; + using Stats = NoStats; + + UnsortingGatherExecutor(Fetcher& fetcher, Infos&); + ~UnsortingGatherExecutor(); + + /** + * @brief produce the next Row of Aql Values. + * + * @return ExecutionState, + * if something was written output.hasValue() == true + */ + [[nodiscard]] auto produceRows(OutputAqlItemRow& output) + -> std::pair; + + [[nodiscard]] auto skipRows(size_t atMost) -> std::tuple; + + private: + [[nodiscard]] auto numDependencies() const + noexcept(noexcept(static_cast(nullptr)->numberDependencies())) -> size_t; + [[nodiscard]] auto fetcher() const noexcept -> Fetcher const&; + [[nodiscard]] auto fetcher() noexcept -> Fetcher&; + [[nodiscard]] auto done() const noexcept -> bool; + [[nodiscard]] auto currentDependency() const noexcept -> size_t; + [[nodiscard]] auto fetchNextRow(size_t atMost) + -> std::pair; + [[nodiscard]] auto skipNextRows(size_t atMost) -> std::pair; + auto advanceDependency() noexcept -> void; + + private: + Fetcher& _fetcher; + size_t _currentDependency{0}; + size_t _skipped{0}; +}; + +} // namespace arangodb::aql + +#endif // ARANGOD_AQL_UNSORTINGGATHEREXECUTOR_H diff --git a/arangod/CMakeLists.txt b/arangod/CMakeLists.txt index 0aa6b7c446db..15dec25015e2 100644 --- a/arangod/CMakeLists.txt +++ b/arangod/CMakeLists.txt @@ -223,6 +223,7 @@ set(LIB_ARANGO_AQL_SOURCES Aql/AqlTransaction.cpp Aql/AqlValue.cpp Aql/AqlValueGroup.cpp + Aql/AqlValueMaterializer.cpp Aql/Arithmetic.cpp Aql/Ast.cpp Aql/AstHelper.cpp @@ -349,6 +350,7 @@ set(LIB_ARANGO_AQL_SOURCES Aql/TraversalExecutor.cpp Aql/TraversalNode.cpp Aql/UnsortedGatherExecutor.cpp + Aql/UnsortingGatherExecutor.cpp Aql/UpdateReplaceModifier.cpp Aql/UpsertModifier.cpp Aql/V8Executor.cpp diff --git a/arangod/IResearch/AqlHelper.h b/arangod/IResearch/AqlHelper.h index bf86f9f0448a..8319f773c2ea 100644 --- a/arangod/IResearch/AqlHelper.h +++ b/arangod/IResearch/AqlHelper.h @@ -326,8 +326,10 @@ class ScopedAqlValue : private irs::util::noncopyable { } void toVelocyPack(velocypack::Builder& builder) const { - _node->isConstant() ? _node->toVelocyPackValue(builder) - : _value.toVelocyPack(nullptr, builder, false); + _node->isConstant() + ? _node->toVelocyPackValue(builder) + : _value.toVelocyPack(static_cast(nullptr), + builder, false); } private: diff --git a/tests/Aql/AqlItemRowTest.cpp b/tests/Aql/AqlItemRowTest.cpp index 1272de49eb70..15de8dab1930 100644 --- a/tests/Aql/AqlItemRowTest.cpp +++ b/tests/Aql/AqlItemRowTest.cpp @@ -47,6 +47,7 @@ class AqlItemRowsTest : public ::testing::Test { protected: ResourceMonitor monitor; AqlItemBlockManager itemBlockManager{&monitor, SerializationFormat::SHADOWROWS}; + velocypack::Options const* const options{&velocypack::Options::Defaults}; void AssertResultMatrix(AqlItemBlock* in, VPackSlice result, std::unordered_set const& regsToKeep, @@ -320,6 +321,7 @@ class AqlItemRowsCommonEqTest : public ::testing::Test { protected: ResourceMonitor monitor; AqlItemBlockManager itemBlockManager{&monitor, SerializationFormat::SHADOWROWS}; + velocypack::Options const* const options{&velocypack::Options::Defaults}; }; using RowTypes = ::testing::Types; @@ -404,6 +406,7 @@ TYPED_TEST(AqlItemRowsCommonEqTest, row_eq_operators) { TYPED_TEST(AqlItemRowsCommonEqTest, row_equivalence) { using RowType = TypeParam; + auto const options = this->options; SharedAqlItemBlockPtr block = buildBlock<1>(this->itemBlockManager, {{{0}}, {{1}}}); SharedAqlItemBlockPtr otherBlock = @@ -418,33 +421,34 @@ TYPED_TEST(AqlItemRowsCommonEqTest, row_equivalence) { RowType const otherInvalidRow = createInvalidRow(); // same rows must be considered equivalent - EXPECT_TRUE((RowType{block, 0}.equates(RowType{block, 0}))); - EXPECT_TRUE((RowType{block, 1}.equates(RowType{block, 1}))); + EXPECT_TRUE((RowType{block, 0}.equates(RowType{block, 0}, options))); + EXPECT_TRUE((RowType{block, 1}.equates(RowType{block, 1}, options))); // different rows must be non-equivalent - EXPECT_FALSE((RowType{block, 0}.equates(RowType{block, 1}))); - EXPECT_FALSE((RowType{block, 1}.equates(RowType{block, 0}))); + EXPECT_FALSE((RowType{block, 0}.equates(RowType{block, 1}, options))); + EXPECT_FALSE((RowType{block, 1}.equates(RowType{block, 0}, options))); // different row in different block must be non-equivalent, even with the same index - EXPECT_FALSE((RowType{block, 0}.equates(RowType{otherBlock, 0}))); - EXPECT_FALSE((RowType{otherBlock, 0}.equates(RowType{block, 0}))); + EXPECT_FALSE((RowType{block, 0}.equates(RowType{otherBlock, 0}, options))); + EXPECT_FALSE((RowType{otherBlock, 0}.equates(RowType{block, 0}, options))); // an equivalent row in a different block must be considered equivalent, even with a different index - EXPECT_TRUE((RowType{block, 1}.equates(RowType{otherBlock, 0}))); - EXPECT_TRUE((RowType{otherBlock, 0}.equates(RowType{block, 1}))); + EXPECT_TRUE((RowType{block, 1}.equates(RowType{otherBlock, 0}, options))); + EXPECT_TRUE((RowType{otherBlock, 0}.equates(RowType{block, 1}, options))); // comparisons with an invalid row must be false - EXPECT_FALSE((RowType{block, 0}.equates(invalidRow))); - EXPECT_FALSE((invalidRow.equates(RowType{block, 0}))); + EXPECT_FALSE((RowType{block, 0}.equates(invalidRow, options))); + EXPECT_FALSE((invalidRow.equates(RowType{block, 0}, options))); // two invalid rows must be equal - EXPECT_TRUE((invalidRow.equates(otherInvalidRow))); + EXPECT_TRUE((invalidRow.equates(otherInvalidRow, options))); } class AqlShadowRowsEqTest : public ::testing::Test { protected: ResourceMonitor monitor; AqlItemBlockManager itemBlockManager{&monitor, SerializationFormat::SHADOWROWS}; + velocypack::Options const* const options{&velocypack::Options::Defaults}; }; TEST_F(AqlShadowRowsEqTest, shadow_row_depth_equivalence) { @@ -460,20 +464,20 @@ TEST_F(AqlShadowRowsEqTest, shadow_row_depth_equivalence) { otherBlock->setShadowRowDepth(0, AqlValue{AqlValueHintUInt{1}}); // same rows must be considered equivalent - EXPECT_TRUE((ShadowAqlItemRow{block, 0}.equates(ShadowAqlItemRow{block, 0}))); - EXPECT_TRUE((ShadowAqlItemRow{block, 1}.equates(ShadowAqlItemRow{block, 1}))); + EXPECT_TRUE((ShadowAqlItemRow{block, 0}.equates(ShadowAqlItemRow{block, 0}, options))); + EXPECT_TRUE((ShadowAqlItemRow{block, 1}.equates(ShadowAqlItemRow{block, 1}, options))); // different rows must be non-equivalent - EXPECT_FALSE((ShadowAqlItemRow{block, 0}.equates(ShadowAqlItemRow{block, 1}))); - EXPECT_FALSE((ShadowAqlItemRow{block, 1}.equates(ShadowAqlItemRow{block, 0}))); + EXPECT_FALSE((ShadowAqlItemRow{block, 0}.equates(ShadowAqlItemRow{block, 1}, options))); + EXPECT_FALSE((ShadowAqlItemRow{block, 1}.equates(ShadowAqlItemRow{block, 0}, options))); // different row in different block must be non-equivalent, even with the same index - EXPECT_FALSE((ShadowAqlItemRow{block, 0}.equates(ShadowAqlItemRow{otherBlock, 0}))); - EXPECT_FALSE((ShadowAqlItemRow{otherBlock, 0}.equates(ShadowAqlItemRow{block, 0}))); + EXPECT_FALSE((ShadowAqlItemRow{block, 0}.equates(ShadowAqlItemRow{otherBlock, 0}, options))); + EXPECT_FALSE((ShadowAqlItemRow{otherBlock, 0}.equates(ShadowAqlItemRow{block, 0}, options))); // an equivalent row in a different block must be considered equivalent, even with a different index - EXPECT_TRUE((ShadowAqlItemRow{block, 1}.equates(ShadowAqlItemRow{otherBlock, 0}))); - EXPECT_TRUE((ShadowAqlItemRow{otherBlock, 0}.equates(ShadowAqlItemRow{block, 1}))); + EXPECT_TRUE((ShadowAqlItemRow{block, 1}.equates(ShadowAqlItemRow{otherBlock, 0}, options))); + EXPECT_TRUE((ShadowAqlItemRow{otherBlock, 0}.equates(ShadowAqlItemRow{block, 1}, options))); } } // namespace aql diff --git a/tests/Aql/DependencyProxyMock.cpp b/tests/Aql/DependencyProxyMock.cpp index f17bcd9098be..d118b1786e9e 100644 --- a/tests/Aql/DependencyProxyMock.cpp +++ b/tests/Aql/DependencyProxyMock.cpp @@ -24,6 +24,8 @@ #include "gtest/gtest.h" +#include + namespace arangodb { namespace tests { namespace aql { @@ -42,7 +44,7 @@ DependencyProxyMock::DependencyProxyMock(arangodb::aql::Resou ::arangodb::aql::RegisterId nrRegisters) : DependencyProxy({}, _itemBlockManager, std::shared_ptr>(), - nrRegisters), + nrRegisters, &velocypack::Options::Defaults), _itemsToReturn(), _numFetchBlockCalls(0), _monitor(monitor), @@ -174,7 +176,7 @@ MultiDependencyProxyMock::MultiDependencyProxyMock( ::arangodb::aql::RegisterId nrRegisters, size_t nrDeps) : DependencyProxy({}, _itemBlockManager, std::shared_ptr>(), - nrRegisters), + nrRegisters, &velocypack::Options::Defaults), _itemBlockManager(&monitor, SerializationFormat::SHADOWROWS) { _dependencyMocks.reserve(nrDeps); for (size_t i = 0; i < nrDeps; ++i) { diff --git a/tests/Aql/ExecutionBlockImplTest.cpp b/tests/Aql/ExecutionBlockImplTest.cpp index edb4733c3b49..9cc5da41d437 100644 --- a/tests/Aql/ExecutionBlockImplTest.cpp +++ b/tests/Aql/ExecutionBlockImplTest.cpp @@ -36,6 +36,7 @@ #include "Aql/ExecutionEngine.h" #include "Aql/Query.h" #include "Aql/SingleRowFetcher.h" +#include "Transaction/Context.h" #include "Transaction/Methods.h" using namespace arangodb; @@ -65,6 +66,10 @@ class ExecutionBlockImplTest : public ::testing::Test { fakeit::Mock mockTrx; transaction::Methods& trx; + // Mock of the transaction context + fakeit::Mock mockContext; + transaction::Context& context; + // Mock of the Query fakeit::Mock mockQuery; Query& query; @@ -90,6 +95,7 @@ class ExecutionBlockImplTest : public ::testing::Test { : engine(mockEngine.get()), itemBlockManager(mockBlockManager.get()), trx(mockTrx.get()), + context(mockContext.get()), query(mockQuery.get()), lqueryOptions(mockQueryOptions.get()), profile(ProfileLevel(PROFILE_LEVEL_NONE)), @@ -116,6 +122,9 @@ class ExecutionBlockImplTest : public ::testing::Test { fakeit::When(Method(mockQuery, trx)).AlwaysReturn(&trx); fakeit::When(Method(mockQueryOptions, getProfileLevel)).AlwaysReturn(profile); + + fakeit::When(Method(mockTrx, transactionContextPtr)).AlwaysReturn(&context); + fakeit::When(Method(mockContext, getVPackOptions)).AlwaysReturn(&velocypack::Options::Defaults); } }; diff --git a/tests/Aql/MultiDepFetcherHelper.cpp b/tests/Aql/MultiDepFetcherHelper.cpp index 815d7078ea6a..c8c57e639dac 100644 --- a/tests/Aql/MultiDepFetcherHelper.cpp +++ b/tests/Aql/MultiDepFetcherHelper.cpp @@ -35,6 +35,7 @@ using namespace arangodb::aql; using namespace arangodb::tests; using namespace arangodb::tests::aql; +constexpr auto options = &velocypack::Options::Defaults; void arangodb::tests::aql::runFetcher(arangodb::aql::MultiDependencySingleRowFetcher& testee, std::vector const& inputOutputPairs) { @@ -58,7 +59,9 @@ void arangodb::tests::aql::runFetcher(arangodb::aql::MultiDependencySingleRowFet auto const& actualState = actual.first; auto const& actualRow = actual.second; EXPECT_EQ(expectedState, actualState) << "during step " << i; - EXPECT_TRUE(expectedRow.equates(actualRow)) << " expected: " << expectedRow << "\n actual: " << actualRow << "\n during step " << i; + EXPECT_TRUE(expectedRow.equates(actualRow, options)) + << " expected: " << expectedRow << "\n actual: " << actualRow + << "\n during step " << i; } void operator()(ConcreteFetcherIOPair const& iop) { auto const& args = iop.first; @@ -75,7 +78,9 @@ void arangodb::tests::aql::runFetcher(arangodb::aql::MultiDependencySingleRowFet auto const& actualState = actual.first; auto const& actualRow = actual.second; EXPECT_EQ(expectedState, actualState) << "during step " << i; - EXPECT_TRUE(expectedRow.equates(actualRow)) << " expected: " << expectedRow << "\n actual: " << actualRow << "\n during step " << i; + EXPECT_TRUE(expectedRow.equates(actualRow, options)) + << " expected: " << expectedRow << "\n actual: " << actualRow + << "\n during step " << i; } private: diff --git a/tests/Aql/SortExecutorTest.cpp b/tests/Aql/SortExecutorTest.cpp index 568092d6d23b..a29ea647bb52 100644 --- a/tests/Aql/SortExecutorTest.cpp +++ b/tests/Aql/SortExecutorTest.cpp @@ -54,12 +54,6 @@ namespace arangodb { namespace tests { namespace aql { -int compareAqlValues(irs::sort::prepared const*, arangodb::transaction::Methods* trx, - arangodb::aql::AqlValue const& lhs, - arangodb::aql::AqlValue const& rhs) { - return arangodb::aql::AqlValue::Compare(trx, lhs, rhs, true); -} - class SortExecutorTest : public ::testing::Test { protected: ExecutionState state; @@ -67,14 +61,7 @@ class SortExecutorTest : public ::testing::Test { AqlItemBlockManager itemBlockManager; SharedAqlItemBlockPtr block; - // Mock of the Transaction - // Enough for this test, will only be passed through and accessed - // on documents alone. - fakeit::Mock mockTrx; - transaction::Methods& trx; - - fakeit::Mock mockContext; - transaction::Context& ctxt; + velocypack::Options const* vpackOptions{&velocypack::Options::Defaults}; Variable sortVar; SortElement sl; @@ -84,13 +71,9 @@ class SortExecutorTest : public ::testing::Test { SortExecutorTest() : itemBlockManager(&monitor, SerializationFormat::SHADOWROWS), block(new AqlItemBlock(itemBlockManager, 1000, 1)), - trx(mockTrx.get()), - ctxt(mockContext.get()), sortVar("mySortVar", 0), sl(&sortVar, true), sortReg(0, sl) { - fakeit::When(Method(mockTrx, transactionContextPtr)).AlwaysReturn(&ctxt); - fakeit::When(Method(mockContext, getVPackOptions)).AlwaysReturn(&arangodb::velocypack::Options::Defaults); sortRegisters.emplace_back(std::move(sortReg)); } }; @@ -98,7 +81,7 @@ class SortExecutorTest : public ::testing::Test { TEST_F(SortExecutorTest, no_rows_upstream_producer_doesnt_wait) { SortExecutorInfos infos(std::move(sortRegisters), /*limit (ignored for default sort)*/ 0, - itemBlockManager, 1, 1, {}, {0}, &trx, false); + itemBlockManager, 1, 1, {}, {0}, vpackOptions, false); VPackBuilder input; AllRowsFetcherHelper fetcher(input.steal(), false); SortExecutor testee(fetcher, infos); @@ -117,7 +100,7 @@ TEST_F(SortExecutorTest, no_rows_upstream_producer_doesnt_wait) { TEST_F(SortExecutorTest, no_rows_upstream_producer_waits) { SortExecutorInfos infos(std::move(sortRegisters), /*limit (ignored for default sort)*/ 0, - itemBlockManager, 1, 1, {}, {0}, &trx, false); + itemBlockManager, 1, 1, {}, {0}, vpackOptions, false); VPackBuilder input; AllRowsFetcherHelper fetcher(input.steal(), true); SortExecutor testee(fetcher, infos); @@ -141,7 +124,7 @@ TEST_F(SortExecutorTest, no_rows_upstream_producer_waits) { TEST_F(SortExecutorTest, rows_upstream_we_are_waiting_for_list_of_numbers) { SortExecutorInfos infos(std::move(sortRegisters), /*limit (ignored for default sort)*/ 0, - itemBlockManager, 1, 1, {}, {0}, &trx, false); + itemBlockManager, 1, 1, {}, {0}, vpackOptions, false); std::shared_ptr input = VPackParser::fromJson("[[5],[3],[1],[2],[4]]"); AllRowsFetcherHelper fetcher(input->steal(), true); diff --git a/tests/Geo/GeoConstructorTest.cpp b/tests/Geo/GeoConstructorTest.cpp index e4aedfb7684b..74d7147888e6 100644 --- a/tests/Geo/GeoConstructorTest.cpp +++ b/tests/Geo/GeoConstructorTest.cpp @@ -34,6 +34,7 @@ #include "Aql/Functions.h" #include "Aql/Query.h" #include "Containers/SmallVector.h" +#include "Transaction/Context.h" #include "Transaction/Methods.h" #include @@ -57,12 +58,20 @@ class GeoConstructorTest : public ::testing::Test { fakeit::Mock trxMock; transaction::Methods& trx; + fakeit::Mock contextMock; + transaction::Context& context; SmallVector::allocator_type::arena_type arena; SmallVector params; GeoConstructorTest() - : expressionContext(expressionContextMock.get()), trx(trxMock.get()), params{arena} {} + : expressionContext(expressionContextMock.get()), + trx(trxMock.get()), + context(contextMock.get()), + params{arena} { + fakeit::When(Method(trxMock, transactionContextPtr)).AlwaysReturn(&context); + fakeit::When(Method(contextMock, getVPackOptions)).AlwaysReturn(&velocypack::Options::Defaults); + } }; namespace geo_point { diff --git a/tests/Geo/GeoFunctionsTest.cpp b/tests/Geo/GeoFunctionsTest.cpp index 67ba6a74970e..fb327fbd0b15 100644 --- a/tests/Geo/GeoFunctionsTest.cpp +++ b/tests/Geo/GeoFunctionsTest.cpp @@ -34,6 +34,7 @@ #include "Aql/Functions.h" #include "Aql/Query.h" #include "Containers/SmallVector.h" +#include "Transaction/Context.h" #include "Transaction/Methods.h" #include @@ -64,16 +65,25 @@ class GeoEqualsTest : public ::testing::Test { fakeit::Mock trxMock; transaction::Methods& trx; + fakeit::Mock contextMock; + transaction::Context& context; SmallVector::allocator_type::arena_type arena; SmallVector paramsA; SmallVector paramsB; SmallVector paramsC; - - GeoEqualsTest() : expressionContext(expressionContextMock.get()), - trx(trxMock.get()), paramsA{arena}, paramsB{arena}, paramsC{arena} { + + GeoEqualsTest() + : expressionContext(expressionContextMock.get()), + trx(trxMock.get()), + context(contextMock.get()), + paramsA{arena}, + paramsB{arena}, + paramsC{arena} { + fakeit::When(Method(trxMock, transactionContextPtr)).AlwaysReturn(&context); + fakeit::When(Method(contextMock, getVPackOptions)).AlwaysReturn(&velocypack::Options::Defaults); } - + ~GeoEqualsTest() { clearVector(paramsA); clearVector(paramsB); diff --git a/tests/js/server/aql/aql-gather-block-cluster.js b/tests/js/server/aql/aql-gather-block-cluster.js index a53c62939fbb..dbccb6c278c7 100644 --- a/tests/js/server/aql/aql-gather-block-cluster.js +++ b/tests/js/server/aql/aql-gather-block-cluster.js @@ -498,14 +498,43 @@ function gatherBlockTestSuite () { testSubqueryValuePropagation : function () { c4 = db._create(cn4, {numberOfShards:3}); c4.insert({Hallo:1}); - var query = "FOR i IN 1..1 LET s = (FOR j IN 1..i FOR k IN " + cn4 + " RETURN j) RETURN s"; - // check the return value - var expected = [ [ 1 ] ]; - var actual = AQL_EXECUTE(query).json; + const query = `FOR i IN 1..1 LET s = (FOR j IN 1..i FOR k IN ${cn4} RETURN j) RETURN s`; + // check the return value + const expected = [ [ 1 ] ]; + const rules = ['-splice-subqueries']; + const opts = {optimizer:{rules}}; + const plan = AQL_EXPLAIN(query, {}, opts).plan; + const nodeTypes = plan.nodes.map(function(node) { + return node.type; + }); + assertNotEqual(0, nodeTypes.filter(type => type === 'SubqueryNode').length); + assertEqual(0, nodeTypes.filter(type => type === 'SubqueryStartNode').length); + assertEqual(0, nodeTypes.filter(type => type === 'SubqueryEndNode').length); + const actual = AQL_EXECUTE(query, {}, opts).json; assertEqual(expected, actual, query); }, - + + testSplicedSubqueryValuePropagation : function () { + c4 = db._create(cn4, {numberOfShards:3}); + c4.insert({Hallo:1}); + const query = `FOR i IN 1..1 LET s = (FOR j IN 1..i FOR k IN ${cn4} RETURN j) RETURN s`; + // check the return value + const expected = [ [ 1 ] ]; + const rules = ['+splice-subqueries']; + const opts = {optimizer:{rules}}; + const plan = AQL_EXPLAIN(query, {}, opts).plan; + const nodeTypes = plan.nodes.map(function(node) { + return node.type; + }); + assertEqual(0, nodeTypes.filter(type => type === 'SubqueryNode').length); + assertNotEqual(0, nodeTypes.filter(type => type === 'SubqueryStartNode').length); + assertNotEqual(0, nodeTypes.filter(type => type === 'SubqueryEndNode').length); + const actual = AQL_EXECUTE(query, {}, opts).json; + + assertEqual(expected, actual, query); + }, + testCalculationNotMovedOverBoundary : function () { c4 = db._create(cn4, {numberOfShards:3}); c4.insert({Hallo:1}); diff --git a/tests/js/server/aql/aql-optimizer-indexes.js b/tests/js/server/aql/aql-optimizer-indexes.js index f8a13608a282..eeb9211d6c6b 100644 --- a/tests/js/server/aql/aql-optimizer-indexes.js +++ b/tests/js/server/aql/aql-optimizer-indexes.js @@ -904,15 +904,6 @@ function optimizerIndexesTestSuite () { const subqueryStartIdx = nodeTypes.indexOf('SubqueryStartNode'); const subqueryEndIdx = nodeTypes.indexOf('SubqueryEndNode'); const hasSplicedSubquery = subqueryStartIdx !== -1 && subqueryEndIdx !== -1; - { // TODO Remove this block as soon as subquery splicing is enabled in the cluster again. - // It's here so the test will fail as soon as that happens, so the actual test will not be forgotten - // to be re-enabled. - const isCluster = require("@arangodb/cluster").isCluster(); - if (isCluster) { - assertFalse(hasSplicedSubquery); - return; - } - } assertTrue(hasSplicedSubquery, JSON.stringify({ subqueryStartIdx, subqueryEndIdx, diff --git a/tests/js/server/aql/aql-optimizer-rule-use-index-for-sort.js b/tests/js/server/aql/aql-optimizer-rule-use-index-for-sort.js index 90e63faaa4b1..e63d346bb663 100644 --- a/tests/js/server/aql/aql-optimizer-rule-use-index-for-sort.js +++ b/tests/js/server/aql/aql-optimizer-rule-use-index-for-sort.js @@ -1075,15 +1075,6 @@ function optimizerRuleTestSuite() { const query = `FOR i IN [123] RETURN (FOR v IN ${colName} FILTER v.a == i SORT v.b ASC RETURN v)`; const plan = AQL_EXPLAIN(query, {}, {optimizer: {rules: ['+splice-subqueries']}}).plan; const rules = plan.rules; - { // TODO Remove this block as soon as subquery splicing is enabled in the cluster again. - // It's here so the test will fail as soon as that happens, so the actual test will not be forgotten - // to be re-enabled. - const isCluster = require("@arangodb/cluster").isCluster(); - if (isCluster) { - assertEqual(-1, rules.indexOf("splice-subqueries")); - return; - } - } assertNotEqual(-1, rules.indexOf(ruleName)); assertNotEqual(-1, rules.indexOf(secondRuleName)); assertNotEqual(-1, rules.indexOf("remove-filter-covered-by-index")); @@ -1125,15 +1116,6 @@ function optimizerRuleTestSuite() { const query = `FOR i IN [123] RETURN (FOR v IN ${colName} FILTER v.a == i SORT v.b DESC RETURN v)`; const plan = AQL_EXPLAIN(query, {}, {optimizer: {rules: ['+splice-subqueries']}}).plan; const rules = plan.rules; - { // TODO Remove this block as soon as subquery splicing is enabled in the cluster again. - // It's here so the test will fail as soon as that happens, so the actual test will not be forgotten - // to be re-enabled. - const isCluster = require("@arangodb/cluster").isCluster(); - if (isCluster) { - assertEqual(-1, rules.indexOf("splice-subqueries")); - return; - } - } assertNotEqual(-1, rules.indexOf(ruleName)); assertNotEqual(-1, rules.indexOf(secondRuleName)); assertNotEqual(-1, rules.indexOf("remove-filter-covered-by-index")); diff --git a/tests/js/server/aql/aql-profiler-cluster.js b/tests/js/server/aql/aql-profiler-cluster.js index 5e9669ca5249..762cd436bcd6 100644 --- a/tests/js/server/aql/aql-profiler-cluster.js +++ b/tests/js/server/aql/aql-profiler-cluster.js @@ -86,32 +86,27 @@ function ahuacatlProfilerTestSuite () { fuzzy = fuzzy || db._engine().name === 'mmfiles'; return _.sum( _.values(rowsPerClient) - .map(fuzzy ? mmfilesBatches: optimalBatches) + .map(fuzzy ? mmfilesBatches : optimalBatches) ); }; + const dbServerBatch = (rows, fuzzy = false) => { + fuzzy = fuzzy || db._engine().name === 'mmfiles'; + return (fuzzy ? mmfilesBatches : optimalBatches)(rows); + }; const dbServerOptimalBatches = (rowsPerClient) => _.sum( _.values(rowsPerClient) .map(optimalBatches) ); - const groupedDBServerBatches = (rowsPerShard) => { - const shardIds = Object.keys(rowsPerShard); - const shardToServerMapping = getResponsibleServers(shardIds); - const callsPerServer = {}; - - for (const [shard, rows] of Object.entries(rowsPerShard)) { - const server = shardToServerMapping[shard]; - const callInfo = callsPerServer[server] || {calls: 0, overhead: 0}; + const groupedBatches = (rowsPerClient, fuzzy) => { + const callInfo = {calls: 0, overhead: 0}; + + for (const [shard, rows] of Object.entries(rowsPerClient)) { const testHere = rows + callInfo.overhead; - if (db._engine().name === 'mmfiles') { - callInfo.calls += mmfilesBatches(testHere); - } else { - callInfo.calls += optimalBatches(testHere); - } + callInfo.calls += dbServerBatch(testHere, fuzzy); callInfo.overhead = testHere % defaultBatchSize; - callsPerServer[server] = callInfo; } - return _.sum(_.values(callsPerServer).map(c => c.calls)); + return callInfo.calls; }; return { @@ -178,32 +173,34 @@ function ahuacatlProfilerTestSuite () { // Number of local getSome calls that do not return WAITING. // This is at least 1. - // Batches are just passed through, but empty ones are skipped. // DONE can only be returned when the last shard is asked, so iff the last // asked shard is empty, there is one more call (the last call returns // DONE without any results). // As there is no guaranteed order in which the shards are processed, we // have to allow a range. const localCalls = (rowsPerShard) => { - const batches = _.sum( - _.values(rowsPerShard) - .map(optimalNonEmptyBatches) - ); - return [Math.max(1, batches), Math.max(1, batches+1)]; + const batches = optimalNonEmptyBatches(_.sum(_.values(rowsPerShard))); + return [ + Math.max(1, batches), + Math.max(1, batches+1) + ]; }; // If we figure out that we are done depends on randomness. // In some cases we get the full batch on last shard, in this case the DBServer knows it is done. - // In other cases we get the full batch on an early shard, but 0 documents later, in chis case the DBServer does not know it is done - // in advance. - const fuzzyDBServerBatches = rowsPerServer => [dbServerBatches(rowsPerServer, false), dbServerBatches(rowsPerServer, true)]; + // In other cases we get the full batch on an early shard, but 0 documents later, in this case the DBServer does + // not know it is done in advance. + const fuzzyDBServerBatches = rowsPerServer => [ + groupedBatches(rowsPerServer, false), + groupedBatches(rowsPerServer, true) + ]; const coordinatorBatches = (rowsPerShard) => addIntervals(fuzzyDBServerBatches(rowsPerShard), localCalls(rowsPerShard)); const genNodeList = (rowsPerShard, rowsPerServer) => [ { type : SingletonBlock, calls : numberOfShards, items : numberOfShards }, - { type : EnumerateCollectionBlock, calls : groupedDBServerBatches(rowsPerShard), items : totalItems(rowsPerShard) }, + { type : EnumerateCollectionBlock, calls : groupedBatches(rowsPerShard), items : totalItems(rowsPerShard) }, // Twice the number due to WAITING, fuzzy, because the Gather does not know { type : RemoteBlock, calls : fuzzyDBServerBatches(rowsPerServer).map(i => i * 2), items : totalItems(rowsPerShard) }, // We get dbServerBatches(rowsPerShard) times WAITING, plus the non-waiting getSome calls. diff --git a/tests/js/server/aql/aql-subquery.js b/tests/js/server/aql/aql-subquery.js index 092c15ead066..034f9cf223ad 100644 --- a/tests/js/server/aql/aql-subquery.js +++ b/tests/js/server/aql/aql-subquery.js @@ -196,18 +196,6 @@ function ahuacatlSubqueryTestSuite () { testSpliceSubqueryOutVariableName : function () { const explainResult = AQL_EXPLAIN("FOR u IN _users LET theLetVariable = (FOR j IN _users RETURN j) RETURN theLetVariable"); - { // TODO Remove this block as soon as subquery splicing is enabled in the cluster again. - // It's here so the test will fail as soon as that happens, so the actual test will not be forgotten - // to be re-enabled. - const isCluster = require("@arangodb/cluster").isCluster(); - if (isCluster) { - const numSubqueryEndNode = findExecutionNodes(explainResult, "SubqueryEndNode").length; - - assertEqual(0, numSubqueryEndNode); - return; - } - } - const subqueryEndNode = findExecutionNodes(explainResult, "SubqueryEndNode")[0]; assertEqual(subqueryEndNode.outVariable.name, "theLetVariable");