8000 [CINFRA] Merging devel (#18152) · arangodb/arangodb@bb52831 · GitHub
[go: up one dir, main page]
Skip to content

Commit bb52831

Browse files
apetencheaDronplaneKVS85markuspf
authored
[CINFRA] Merging devel (#18152)
* SEARCH-279 snapshot for inverted index (#18128) * make inverted index read docs from custom snapshot * wip - adding late materialization snapshot * wip * wip * switch inverted index to viewsnapshot * fix * wip * wip * try to fix mac build * fix tests build * add explicit instantiations * add test * fix issue * add js tests * clang-format * cleanup * adress review comments * fix for cluster * review comments * jslint * cleanup * Update OpenSSL to 1.1.1t and OpenLDAP to 2.6.4 (#18148) * Update ArangoDB Starter to 0.15.7 * [GORDO-1518] Fix memory leak in tests (#18127) * Free memory held by unique_ptr * Shutdown runtimes after tests; fixes memory leak --------- Co-authored-by: Andrei Lobov <andrei.lobov@arangodb.com> Co-authored-by: Vadim <vadim@arangodb.com> Co-authored-by: Markus Pfeiffer <markuspf@users.noreply.github.com>
1 parent 2328398 commit bb52831

24 files changed

+623
-322
lines changed

‎CHANGELOG

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
devel
22
-----
33

4+
* Updated ArangoDB Starter to 0.15.7.
5+
6+
* Updated OpenSSL to 1.1.1t and OpenLDAP to 2.6.4.
7+
48
* Made all transactions used by the gharial API on coordinators and a few
59
others marked "globally managed". This fixes an issue where
610
transaction conflicts could lead to a silent out of sync situation

‎VERSIONS

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
CXX_STANDARD "20"
2-
STARTER_REV "0.15.7-preview-1"
2+
STARTER_REV "0.15.7"
33
SYNCER_REV "v2.14.0"
44
GCC_LINUX "11.2.1_git20220219-r2"
55
MSVC_WINDOWS "2022"
66
LLVM_CLANG_MACOS "14"
77
MACOS_MIN "10.15"
8-
OPENSSL_LINUX "1.1.1s"
9-
OPENSSL_MACOS "1.1.1s"
10-
OPENSSL_WINDOWS "1.1.1s"
8+
OPENSSL_LINUX "1.1.1t"
9+
OPENSSL_MACOS "1.1.1t"
10+
OPENSSL_WINDOWS "1.1.1t"
1111
USE_RCLONE "true"
1212
RCLONE_VERSION "1.59.0"
1313
MINIMAL_DEBUG_INFO "On"

‎arangod/Aql/DocumentProducingHelper.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ IndexIterator::DocumentCallback aql::buildDocumentCallback(
153153
}
154154

155155
template<bool checkUniqueness>
156-
std::function<bool(LocalDocumentId const& token)> aql::getNullCallback(
156+
IndexIterator::LocalDocumentIdCallback aql::getNullCallback(
157157
DocumentProducingFunctionContext& context) {
158158
TRI_ASSERT(!context.hasFilter());
159159

@@ -435,6 +435,7 @@ IndexIterator::CoveringCallback aql::getCallback(
435435
IndexIteratorCoveringData& covering) {
436436
TRI_ASSERT(context.getAllowCoveringIndexOptimization());
437437
if constexpr (checkUniqueness) {
438+
TRI_ASSERT(token.isSet());
438439
if (!context.checkUniqueness(token)) {
439440
// Document already found, skip it
440441
return false;

‎arangod/Aql/ExecutionBlockImpl.tpp

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -690,8 +690,9 @@ static SkipRowsRangeVariant constexpr skipRowsType() {
690690
#ifdef USE_ENTERPRISE
691691
arangodb::iresearch::OffsetMaterializeExecutor,
692692
#endif
693-
MaterializeExecutor<void>,
694-
MaterializeExecutor<std::string const&>>) ||
693+
MaterializeExecutor<void, false>,
694+
MaterializeExecutor<std::string const&, true>,
695+
MaterializeExecutor<std::string const&, false>>) ||
695696
IsSearchExecutor<Executor>::value,
696697
"Unexpected executor for SkipVariants::EXECUTOR");
697698

‎arangod/Aql/ExecutionNode.cpp

Lines changed: 48 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -2827,18 +2827,23 @@ ExecutionNode* AsyncNode::clone(ExecutionPlan* plan, bool withDependencies,
28272827
}
28282828

28292829
namespace {
2830-
constexpr std::string_view MATERIALIZE_NODE_IN_NM_DOC_PARAM = "inNmDocId";
2831-
constexpr std::string_view MATERIALIZE_NODE_OUT_VARIABLE_PARAM = "outVariable";
2832-
constexpr std::string_view MATERIALIZE_NODE_MULTI_NODE_PARAM = "multiNode";
2830+
constexpr std::string_view kMaterializeNodeInNmDocParam = "inNmDocId";
2831+
constexpr std::string_view kMaterializeNodeOutVariableParam = "outVariable";
2832+
constexpr std::string_view kMaterializeNodeMultiNodeParam = "multiNode";
2833+
constexpr std::string_view kMaterializeNodeInLocalDocIdParam = "inLocalDocId";
28332834
} // namespace
28342835

28352836
MaterializeNode* materialize::createMaterializeNode(
28362837
ExecutionPlan* plan, arangodb::velocypack::Slice const base) {
2837-
auto isMulti = base.get(MATERIALIZE_NODE_MULTI_NODE_PARAM);
2838+
auto isMulti = base.get(kMaterializeNodeMultiNodeParam);
28382839
if (isMulti.isBoolean() && isMulti.getBoolean()) {
28392840
return new MaterializeMultiNode(plan, base);
28402841
}
2841-
return new MaterializeSingleNode(plan, base);
2842+
auto inLocalDocId = base.get(kMaterializeNodeInLocalDocIdParam);
2843+
if (inLocalDocId.isBoolean() && inLocalDocId.getBoolean()) {
2844+
return new MaterializeSingleNode<true>(plan, base);
2845+
}
2846+
return new MaterializeSingleNode<false>(plan, base);
28422847
}
28432848

28442849
MaterializeNode::MaterializeNode(ExecutionPlan* plan, ExecutionNodeId id,
@@ -2852,27 +2857,19 @@ MaterializeNode::MaterializeNode(ExecutionPlan* plan,
28522857
arangodb::velocypack::Slice const& base)
28532858
: ExecutionNode(plan, base),
28542859
_inNonMaterializedDocId(aql::Variable::varFromVPack(
2855-
plan->getAst(), base, MATERIALIZE_NODE_IN_NM_DOC_PARAM, true)),
2860+
plan->getAst(), base, kMaterializeNodeInNmDocParam, true)),
28562861
_outVariable(aql::Variable::varFromVPack(
2857-
plan->getAst(), base, MATERIALIZE_NODE_OUT_VARIABLE_PARAM)) {}
2862+
plan->getAst(), base, kMaterializeNodeOutVariableParam)) {}
28582863

28592864
void MaterializeNode::doToVelocyPack(velocypack::Builder& nodes,
28602865
unsigned /*flags*/) const {
2861-
nodes.add(VPackValue(MATERIALIZE_NODE_IN_NM_DOC_PARAM));
2866+
nodes.add(VPackValue(kMaterializeNodeInNmDocParam));
28622867
_inNonMaterializedDocId->toVelocyPack(nodes);
28632868

2864-
nodes.add(VPackValue(MATERIALIZE_NODE_OUT_VARIABLE_PARAM));
2869+
nodes.add(VPackValue(kMaterializeNodeOutVariableParam));
28652870
_outVariable->toVelocyPack(nodes);
28662871
}
28672872

2868-
auto MaterializeNode::getReadableInputRegisters(
2869-
RegisterId const inNmDocId) const -> RegIdSet {
2870-
// TODO (Dronplane) for index exectutor here we possibly will
2871-
// return additional SearchDoc register. So will keep this function for time
2872-
// being.
2873-
return RegIdSet{inNmDocId};
2874-
}
2875-
28762873
CostEstimate MaterializeNode::estimateCost() const {
28772874
if (_dependencies.empty()) {
28782875
// we should always have dependency as we need input for materializing
@@ -2907,7 +2904,7 @@ void MaterializeMultiNode::doToVelocyPack(velocypack::Builder& nodes,
29072904
unsigned flags) const {
29082905
// call base class method
29092906
MaterializeNode::doToVelocyPack(nodes, flags);
2910-
nodes.add(MATERIALIZE_NODE_MULTI_NODE_PARAM, velocypack::Value(true));
2907+
nodes.add(kMaterializeNodeMultiNodeParam, velocypack::Value(true));
29112908
}
29122909

29132910
std::unique_ptr<ExecutionBlock> MaterializeMultiNode::createBlock(
@@ -2929,7 +2926,7 @@ std::unique_ptr<ExecutionBlock> MaterializeMultiNode::createBlock(
29292926
outDocumentRegId = it->second.registerId;
29302927
}
29312928

2932-
auto readableInputRegisters = getReadableInputRegisters(inNmDocIdRegId);
2929+
RegIdSet readableInputRegisters{inNmDocIdRegId};
29332930
auto writableOutputRegisters = RegIdSet{outDocumentRegId};
29342931

29352932
auto registerInfos = createRegisterInfos(std::move(readableInputRegisters),
@@ -2938,7 +2935,7 @@ std::unique_ptr<ExecutionBlock> MaterializeMultiNode::createBlock(
29382935
auto executorInfos = MaterializerExecutorInfos<void>(
29392936
inNmDocIdRegId, outDocumentRegId, engine.getQuery());
29402937

2941-
return std::make_unique<ExecutionBlockImpl<MaterializeExecutor<void>>>(
2938+
return std::make_unique<ExecutionBlockImpl<MaterializeExecutor<void, false>>>(
29422939
&engine, this, std::move(registerInfos), std::move(executorInfos));
29432940
}
29442941

@@ -2961,47 +2958,53 @@ ExecutionNode* MaterializeMultiNode::clone(ExecutionPlan* plan,
29612958
return cloneHelper(std::move(c), withDependencies, withProperties);
29622959
}
29632960

2964-
MaterializeSingleNode::MaterializeSingleNode(ExecutionPlan* plan,
2965-
ExecutionNodeId id,
2966-
aql::Collection const* collection,
2967-
aql::Variable const& inDocId,
2968-
aql::Variable const& outVariable)
2961+
template<bool localDocumentId>
2962+
MaterializeSingleNode<localDocumentId>::MaterializeSingleNode(
2963+
ExecutionPlan* plan, ExecutionNodeId id, aql::Collection const* collection,
2964+
aql::Variable const& inDocId, aql::Variable const& outVariable)
29692965
: MaterializeNode(plan, id, inDocId, outVariable),
29702966
CollectionAccessingNode(collection) {}
29712967

2972-
MaterializeSingleNode::MaterializeSingleNode(
2968+
template<bool localDocumentId>
2969+
MaterializeSingleNode<localDocumentId>::MaterializeSingleNode(
29732970
ExecutionPlan* plan, arangodb::velocypack::Slice const& base)
29742971
: MaterializeNode(plan, base), CollectionAccessingNode(plan, base) {}
29752972

2976-
void MaterializeSingleNode::doToVelocyPack(velocypack::Builder& nodes,
2977-
unsigned flags) const {
2973+
template<bool localDocumentId>
2974+
void MaterializeSingleNode<localDocumentId>::doToVelocyPack(
2975+
velocypack::Builder& nodes, unsigned flags) const {
29782976
// call base class method
29792977
MaterializeNode::doToVelocyPack(nodes, flags);
29802978

29812979
// add collection information
29822980
CollectionAccessingNode::toVelocyPack(nodes, flags);
2981+
nodes.add(kMaterializeNodeInLocalDocIdParam, localDocumentId);
29832982
}
29842983

2985-
std::unique_ptr<ExecutionBlock> MaterializeSingleNode::createBlock(
2984+
template<bool localDocumentId>
2985+
std::unique_ptr<ExecutionBlock>
2986+
MaterializeSingleNode<localDocumentId>::createBlock(
29862987
ExecutionEngine& engine,
29872988
std::unordered_map<ExecutionNode*, ExecutionBlock*> const&) const {
29882989
ExecutionNode const* previousNode = getFirstDependency();
29892990
TRI_ASSERT(previousNode != nullptr);
2990-
RegisterId inNmDocIdRegId;
2991-
{
2992-
auto it = getRegisterPlan()->varInfo.find(_inNonMaterializedDocId->id);
2993-
TRI_ASSERT(it != getRegisterPlan()->varInfo.end());
2994-
inNmDocIdRegId = it->second.registerId;
2995-
}
29962991
RegisterId outDocumentRegId;
29972992
{
29982993
auto it = getRegisterPlan()->varInfo.find(_outVariable->id);
29992994
TRI_ASSERT(it != getRegisterPlan()->varInfo.end());
30002995
outDocumentRegId = it->second.registerId;
30012996
}
30022997
auto const& name = collection()->name();
3003-
3004-
auto readableInputRegisters = getReadableInputRegisters(inNmDocIdRegId);
2998+
RegisterId inNmDocIdRegId;
2999+
{
3000+
auto it = getRegisterPlan()->varInfo.find(_inNonMaterializedDocId->id);
3001+
TRI_ASSERT(it != getRegisterPlan()->varInfo.end());
3002+
inNmDocIdRegId = it->second.registerId;
3003+
}
3004+
RegIdSet readableInputRegisters;
3005+
if (inNmDocIdRegId.isValid()) {
3006+
readableInputRegisters.emplace(inNmDocIdRegId);
3007+
}
30053008
auto writableOutputRegisters = RegIdSet{outDocumentRegId};
30063009

30073010
auto registerInfos = createRegisterInfos(std::move(readableInputRegisters),
@@ -3010,13 +3013,13 @@ std::unique_ptr<ExecutionBlock> MaterializeSingleNode::createBlock(
30103013
auto executorInfos = MaterializerExecutorInfos<decltype(name)>(
30113014
inNmDocIdRegId, outDocumentRegId, engine.getQuery(), name);
30123015
return std::make_unique<
3013-
ExecutionBlockImpl<MaterializeExecutor<decltype(name)>>>(
3016+
ExecutionBlockImpl<MaterializeExecutor<decltype(name), localDocumentId>>>(
30143017
&engine, this, std::move(registerInfos), std::move(executorInfos));
30153018
}
30163019

3017-
ExecutionNode* MaterializeSingleNode::clone(ExecutionPlan* plan,
3018-
bool withDependencies,
3019-
bool withProperties) const {
3020+
template<bool localDocumentId>
3021+
ExecutionNode* MaterializeSingleNode<localDocumentId>::clone(
3022+
ExecutionPlan* plan, bool withDependencies, bool withProperties) const {
30203023
TRI_ASSERT(plan);
30213024

30223025
auto* outVariable = _outVariable;
@@ -3028,8 +3031,11 @@ ExecutionNode* MaterializeSingleNode::clone(ExecutionPlan* plan,
30283031
plan->getAst()->variables()->createVariable(inNonMaterializedDocId);
30293032
}
30303033

3031-
auto c = std::make_unique<MaterializeSingleNode>(
3034+
auto c = std::make_unique<MaterializeSingleNode<localDocumentId>>(
30323035
plan, _id, collection(), *inNonMaterializedDocId, *outVariable);
30333036
CollectionAccessingNode::cloneInto(*c);
30343037
return cloneHelper(std::move(c), withDependencies, withProperties);
30353038
}
3039+
3040+
template class arangodb::aql::materialize::MaterializeSingleNode<false>;
3041+
template class arangodb::aql::materialize::MaterializeSingleNode<true>;

‎arangod/Aql/ExecutionNode.h

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1170,8 +1170,6 @@ class MaterializeNode : public ExecutionNode {
11701170
void doToVelocyPack(arangodb::velocypack::Builder& nodes,
11711171
unsigned flags) const override;
11721172

1173-
auto getReadableInputRegisters(RegisterId inNmDocId) const -> RegIdSet;
1174-
11751173
protected:
11761174
/// @brief input variable non-materialized document ids
11771175
aql::Variable const* _inNonMaterializedDocId;
@@ -1205,6 +1203,7 @@ class MaterializeMultiNode : public MaterializeNode {
12051203
unsigned flags) const override final;
12061204
};
12071205

1206+
template<bool localDocumentId>
12081207
class MaterializeSingleNode : public MaterializeNode,
12091208
public CollectionAccessingNode {
12101209
public:

‎arangod/Aql/IndexExecutor.cpp

Lines changed: 26 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -106,10 +106,21 @@ IndexIterator::CoveringCallback getCallback(
106106
transaction::Methods::IndexHandle const& index,
107107
IndexNode::IndexValuesVars const& outNonMaterializedIndVars,
108108
IndexNode::IndexValuesRegisters const& outNonMaterializedIndRegs) {
109-
return [&context, &index, &outNonMaterializedIndVars,
110-
&outNonMaterializedIndRegs](LocalDocumentId const& token,
111-
IndexIteratorCoveringData& covering) {
112-
if constexpr (checkUniqueness) {
109+
auto impl = [&context, &index, &outNonMaterializedIndVars,
110+
&outNonMaterializedIndRegs]<typename TokenType>(
111+
TokenType&& token, IndexIteratorCoveringData& covering) {
112+
constexpr bool isLocalDocumentId =
113+
std::is_same_v<LocalDocumentId, std::decay_t<TokenType>>;
114+
// can't be a static_assert as this implementation is still possible
115+
// just can't be used right now as for restriction in late materialization
116+
// rule
117+
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
118+
if constexpr (checkUniqueness && !isLocalDocumentId) {
119+
TRI_ASSERT(false);
120+
}
121+
#endif
122+
if constexpr (checkUniqueness && isLocalDocumentId) {
123+
TRI_ASSERT(token.isSet());
113124
if (!context.checkUniqueness(token)) {
114125
// Document already found, skip it
115126
return false;
@@ -135,11 +146,17 @@ IndexIterator::CoveringCallback getCallback(
135146
OutputAqlItemRow& output = context.getOutputRow();
136147
RegisterId registerId = context.getOutputRegister();
137148

138-
// move a document id
139-
AqlValue v(AqlValueHintUInt(token.id()));
140-
AqlValueGuard guard{v, true};
141149
TRI_ASSERT(!output.isFull());
142-
output.moveValueInto(registerId, input, guard);
150+
// move a document id
151+
if constexpr (isLocalDocumentId) {
152+
TRI_ASSERT(token.isSet());
153+
AqlValue v(AqlValueHintUInt(token.id()));
154+
AqlValueGuard guard{v, false};
155+
output.moveValueInto(registerId, input, guard);
156+
} else {
157+
AqlValueGuard guard{token, true};
158+
output.moveValueInto(registerId, input, guard);
159+
}
143160

144161
// hash/skiplist/persistent
145162
if (covering.isArray()) {
@@ -170,6 +187,7 @@ IndexIterator::CoveringCallback getCallback(
170187
output.advanceRow();
171188
return true;
172189
};
190+
return impl;
173191
}
174192

175193
} // namespace

‎arangod/Aql/IndexNode.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -192,7 +192,7 @@ IndexNode::IndexNode(ExecutionPlan* plan,
192192
_outNonMaterializedIndVars.second.try_emplace(var, fieldNumber);
193193
}
194194
}
195-
195+
_options.forLateMaterialization = isLateMaterialized();
196196
prepareProjections();
197197
}
198198

@@ -555,6 +555,7 @@ void IndexNode::setLateMaterialized(aql::Variable const* docIdVariable,
555555
_outNonMaterializedIndVars.second.try_emplace(indVars.second.var,
556556
indVars.second.indexFieldNum);
557557
}
558+
_options.forLateMaterialization = true;
558559
}
559560

560561
transaction::Methods::IndexHandle IndexNode::getSingleIndex() const {

‎arangod/Aql/IndexNodeOptimizerRules.cpp

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,10 @@ void arangodb::aql::lateDocumentMaterializationRule(
122122
auto& indexes = indexNode->getIndexes();
123123
TRI_ASSERT(!indexes.empty());
124124
if (indexes.size() != 1) {
125+
// When enabling this please consider how inverted index would
126+
// operate together with persistent as first produces
127+
// SearchDocs but latter LocalDocumentIds. Usage of
128+
// two separate variables might be the simplest solution
125129
continue; // several indexes are not supported
126130
}
127131
auto& index = indexes.front();
@@ -326,15 +330,23 @@ void arangodb::aql::lateDocumentMaterializationRule(
326330

327331
// we could apply late materialization
328332
// 1. We need to notify index node - it should not materialize
329-
// documents, but produce only localDocIds
333+
// documents, but produce only localDocIds or SearchDocs
334+
// in case of inverted index
330335
indexNode->setLateMaterialized(localDocIdTmp, index->id(),
331336
uniqueVariables);
332337
// 2. We need to add materializer after limit node to do materialization
333338
// insert a materialize node
334-
auto* materializeNode = plan->registerNode(
335-
std::make_unique<materialize::MaterializeSingleNode>(
339+
auto makeMaterializer = [&]() -> std::unique_ptr<ExecutionNode> {
340+
if (index->type() == Index::TRI_IDX_TYPE_INVERTED_INDEX) {
341+
return std::make_unique<materialize::MaterializeSingleNode<false>>(
336342
plan.get(), plan->nextId(), indexNode->collection(),
337-
*localDocIdTmp, *var));
343+
*localDocIdTmp, *var);
344+
}
345+
return std::make_unique<materialize::MaterializeSingleNode<true>>(
346+
plan.get(), plan->nextId(), indexNode->collection(),
347+
*localDocIdTmp, *var);
348+
};
349+
auto* materializeNode = plan->registerNode(makeMaterializer());
338350
TRI_ASSERT(materializeNode);
339351

340352
// on cluster we need to materialize node stay close to sort node on db

0 commit comments

Comments
 (0)
0