8000 SEARCH-323 SEARCH-279 Have a dedicated storage engine snapshot for Ar… · cloudhub-js/arangodb@7a9e43e · GitHub
[go: up one dir, main page]
Skip to content

Commit 7a9e43e

Browse files
DronplaneDronplane
authored
SEARCH-323 SEARCH-279 Have a dedicated storage engine snapshot for ArangoSearch index and aggressive LM (arangodb#17895)
* store db snapshot in datastore * wip * wip * wip * naive impl * no-buffer implementation * wip * make materializer buffered * fix merge artifact * get rid of collection pointer register * fix tests * clang format * clang-format * make collection resolving cacheable + memory tracking * make tick conversion safe * single record buffer implementation * cleanup * fix for merge executor * optimize buffer * wip * refactor key buffer structure * apply review comments * make load/store explicit * move common reading utils to separate file * clang-format * fix build * wip * review comments * remove redundant template instantiation * move to string_view collection resolving * fix by tests * fix explainer * fix snapshotview issue * fix after merge * clang-format * adress review comments * get rid of explicit template instantiations * try to fix clang build * another try
1 parent 29f2e0e commit 7a9e43e

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

56 files changed

+1392
-1713
lines changed

‎arangod/Aql/CMakeLists.txt

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,6 @@ add_library(arango_aql STATIC
8080
IndexNodeOptimizerRules.cpp
8181
InputAqlItemRow.cpp
8282
InsertModifier.cpp
83-
IResearchViewExecutor.cpp
8483
IResearchViewNode.cpp
8584
IResearchViewOptimizerRules.cpp
8685
LateMaterializedExpressionContext.cpp

‎arangod/Aql/ExecutionBlockImpl.tpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -690,7 +690,7 @@ static SkipRowsRangeVariant constexpr skipRowsType() {
690690
#ifdef USE_ENTERPRISE
691691
arangodb::iresearch::OffsetMaterializeExecutor,
692692
#endif
693-
MaterializeExecutor<RegisterId>,
693+
MaterializeExecutor<void>,
694694
MaterializeExecutor<std::string const&>>) ||
695695
IsSearchExecutor<Executor>::value,
696696
"Unexpected executor for SkipVariants::EXECUTOR");

‎arangod/Aql/ExecutionNode.cpp

Lines changed: 24 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -2824,14 +2824,15 @@ ExecutionNode* AsyncNode::clone(ExecutionPlan* plan, bool withDependencies,
28242824
}
28252825

28262826
namespace {
2827-
const char* MATERIALIZE_NODE_IN_NM_COL_PARAM = "inNmColPtr";
2828-
const char* MATERIALIZE_NODE_IN_NM_DOC_PARAM = "inNmDocId";
2829-
const char* MATERIALIZE_NODE_OUT_VARIABLE_PARAM = "outVariable";
2827+
constexpr std::string_view MATERIALIZE_NODE_IN_NM_DOC_PARAM = "inNmDocId";
2828+
constexpr std::string_view MATERIALIZE_NODE_OUT_VARIABLE_PARAM = "outVariable";
2829+
constexpr std::string_view MATERIALIZE_NODE_MULTI_NODE_PARAM = "multiNode";
28302830
} // namespace
28312831

28322832
MaterializeNode* materialize::createMaterializeNode(
2833-
ExecutionPlan* plan, arangodb::velocypack::Slice const& base) {
2834-
if (base.hasKey(MATERIALIZE_NODE_IN_NM_COL_PARAM)) {
2833+
ExecutionPlan* plan, arangodb::velocypack::Slice const base) {
2834+
auto isMulti = base.get(MATERIALIZE_NODE_MULTI_NODE_PARAM);
2835+
if (isMulti.isBoolean() && isMulti.getBoolean()) {
28352836
return new MaterializeMultiNode(plan, base);
28362837
}
28372838
return new MaterializeSingleNode(plan, base);
@@ -2861,6 +2862,14 @@ void MaterializeNode::doToVelocyPack(velocypack::Builder& nodes,
28612862
_outVariable->toVelocyPack(nodes);
28622863
}
28632864

2865+
auto MaterializeNode::getReadableInputRegisters(
2866+
RegisterId const inNmDocId) const -> RegIdSet {
2867+
// TODO (Dronplane) for index exectutor here we possibly will
2868+
// return additional SearchDoc register. So will keep this function for time
2869+
// being.
2870+
return RegIdSet{inNmDocId};
2871+
}
2872+
28642873
CostEstimate MaterializeNode::estimateCost() const {
28652874
if (_dependencies.empty()) {
28662875
// we should always have dependency as we need input for materializing
@@ -2883,25 +2892,19 @@ std::vector<Variable const*> MaterializeNode::getVariablesSetHere() const {
28832892

28842893
MaterializeMultiNode::MaterializeMultiNode(ExecutionPlan* plan,
28852894
ExecutionNodeId id,
2886-
aql::Variable const& inColPtr,
28872895
aql::Variable const& inDocId,
28882896
aql::Variable const& outVariable)
2889-
: MaterializeNode(plan, id, inDocId, outVariable),
2890-
_inNonMaterializedColPtr(&inColPtr) {}
2897+
: MaterializeNode(plan, id, inDocId, outVariable) {}
28912898

28922899
MaterializeMultiNode::MaterializeMultiNode(
28932900
ExecutionPlan* plan, arangodb::velocypack::Slice const& base)
2894-
: MaterializeNode(plan, base),
2895-
_inNonMaterializedColPtr(aql::Variable::varFromVPack(
2896-
plan->getAst(), base, MATERIALIZE_NODE_IN_NM_COL_PARAM, true)) {}
2901+
: MaterializeNode(plan, base) {}
28972902

28982903
void MaterializeMultiNode::doToVelocyPack(velocypack::Builder& nodes,
28992904
unsigned flags) const {
29002905
// call base class method
29012906
MaterializeNode::doToVelocyPack(nodes, flags);
2902-
2903-
nodes.add(VPackValue(MATERIALIZE_NODE_IN_NM_COL_PARAM));
2904-
_inNonMaterializedColPtr->toVelocyPack(nodes);
2907+
nodes.add(MATERIALIZE_NODE_MULTI_NODE_PARAM, velocypack::Value(true));
29052908
}
29062909

29072910
std::unique_ptr<ExecutionBlock> MaterializeMultiNode::createBlock(
@@ -2910,12 +2913,6 @@ std::unique_ptr<ExecutionBlock> MaterializeMultiNode::createBlock(
29102913
ExecutionNode const* previousNode = getFirstDependency();
29112914
TRI_ASSERT(previousNode != nullptr);
29122915

2913-
RegisterId inNmColPtrRegId;
2914-
{
2915-
auto it = getRegisterPlan()->varInfo.find(_inNonMaterializedColPtr->id);
2916-
TRI_ASSERT(it != getRegisterPlan()->varInfo.end());
2917-
inNmColPtrRegId = it->second.registerId;
2918-
}
29192916
RegisterId inNmDocIdRegId;
29202917
{
29212918
auto it = getRegisterPlan()->varInfo.find(_inNonMaterializedDocId->id);
@@ -2929,18 +2926,16 @@ std::unique_ptr<ExecutionBlock> MaterializeMultiNode::createBlock(
29292926
outDocumentRegId = it->second.registerId;
29302927
}
29312928

2932-
auto readableInputRegisters =
2933-
getReadableInputRegisters(inNmColPtrRegId, inNmDocIdRegId);
2929+
auto readableInputRegisters = getReadableInputRegisters(inNmDocIdRegId);
29342930
auto writableOutputRegisters = RegIdSet{outDocumentRegId};
29352931

29362932
auto registerInfos = createRegisterInfos(std::move(readableInputRegisters),
29372933
std::move(writableOutputRegisters));
29382934

2939-
auto executorInfos = MaterializerExecutorInfos(
2940-
inNmColPtrRegId, inNmDocIdRegId, outDocumentRegId, engine.getQuery());
2935+
auto executorInfos = MaterializerExecutorInfos<void>(
2936+
inNmDocIdRegId, outDocumentRegId, engine.getQuery());
29412937

2942-
return std::make_unique<
2943-
ExecutionBlockImpl<MaterializeExecutor<decltype(inNmColPtrRegId)>>>(
2938+
return std::make_unique<ExecutionBlockImpl<MaterializeExecutor<void>>>(
29442939
&engine, this, std::move(registerInfos), std::move(executorInfos));
29452940
}
29462941

@@ -2951,29 +2946,18 @@ ExecutionNode* MaterializeMultiNode::clone(ExecutionPlan* plan,
29512946

29522947
auto* outVariable = _outVariable;
29532948
auto* inNonMaterializedDocId = _inNonMaterializedDocId;
2954-
auto* inNonMaterializedColId = _inNonMaterializedColPtr;
29552949

29562950
if (withProperties) {
29572951
outVariable = plan->getAst()->variables()->createVariable(outVariable);
29582952
inNonMaterializedDocId =
29592953
plan->getAst()->variables()->createVariable(inNonMaterializedDocId);
2960-
inNonMaterializedColId =
2961-
plan->getAst()->variables()->createVariable(inNonMaterializedColId);
29622954
}
29632955

29642956
auto c = std::make_unique<MaterializeMultiNode>(
2965-
plan, _id, *inNonMaterializedColId, *inNonMaterializedDocId,
2966-
*outVariable);
2957+
plan, _id, *inNonMaterializedDocId, *outVariable);
29672958
return cloneHelper(std::move(c), withDependencies, withProperties);
29682959
}
29692960

2970-
void MaterializeMultiNode::getVariablesUsedHere(VarSet& vars) const {
2971-
// call base class method
2972-
MaterializeNode::getVariablesUsedHere(vars);
2973-
2974-
vars.emplace(_inNonMaterializedColPtr);
2975-
}
2976-
29772961
MaterializeSingleNode::MaterializeSingleNode(ExecutionPlan* plan,
29782962
ExecutionNodeId id,
29792963
aql::Collection const* collection,
@@ -3014,14 +2998,14 @@ std::unique_ptr<ExecutionBlock> MaterializeSingleNode::createBlock(
30142998
}
30152999
auto const& name = collection()->name();
30163000

3017-
auto readableInputRegisters = getReadableInputRegisters(name, inNmDocIdRegId);
3001+
auto readableInputRegisters = getReadableInputRegisters(inNmDocIdRegId);
30183002
auto writableOutputRegisters = RegIdSet{outDocumentRegId};
30193003

30203004
auto registerInfos = createRegisterInfos(std::move(readableInputRegisters),
30213005
std::move(writableOutputRegisters));
30223006

30233007
auto executorInfos = MaterializerExecutorInfos<decltype(name)>(
3024-
name, inNmDocIdRegId, outDocumentRegId, engine.getQuery());
3008+
inNmDocIdRegId, outDocumentRegId, engine.getQuery(), name);
30253009
return std::make_unique<
30263010
ExecutionBlockImpl<MaterializeExecutor<decltype(name)>>>(
30273011
&engine, this, std::move(registerInfos), std::move(executorInfos));

‎arangod/Aql/ExecutionNode.h

Lines changed: 2 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1170,9 +1170,7 @@ class MaterializeNode : public ExecutionNode {
11701170
void doToVelocyPack(arangodb::velocypack::Builder& nodes,
11711171
unsigned flags) const override;
11721172

1173-
template<typename T>
1174-
auto getReadableInputRegisters(T collectionSource, RegisterId inNmDocId) const
1175-
-> RegIdSet;
1173+
auto getReadableInputRegisters(RegisterId inNmDocId) const -> RegIdSet;
11761174

11771175
protected:
11781176
/// @brief input variable non-materialized document ids
@@ -1182,20 +1180,9 @@ class MaterializeNode : public ExecutionNode {
11821180
Variable const* _outVariable;
11831181
};
11841182

1185-
template<typename T>
1186-
auto MaterializeNode::getReadableInputRegisters(
1187-
T const collectionSource, RegisterId const inNmDocId) const -> RegIdSet {
1188-
if constexpr (std::is_same_v<T, RegisterId>) {
1189-
return RegIdSet{collectionSource, inNmDocId};
1190-
} else {
1191-
return RegIdSet{inNmDocId};
1192-
}
1193-
}
1194-
11951183
class MaterializeMultiNode : public MaterializeNode {
11961184
public:
11971185
MaterializeMultiNode(ExecutionPlan* plan, ExecutionNodeId id,
1198-
aql::Variable const& inColPtr,
11991186
aql::Variable const& inDocId,
12001187
aql::Variable const& outVariable);
12011188

@@ -1212,17 +1199,10 @@ class MaterializeMultiNode : public MaterializeNode {
12121199
ExecutionNode* clone(ExecutionPlan* plan, bool withDependencies,
12131200
bool withProperties) const override final;
12141201

1215-
/// @brief getVariablesUsedHere, modifying the set in-place
1216-
void getVariablesUsedHere(VarSet& vars) const override final;
1217-
12181202
protected:
12191203
/// @brief export to VelocyPack
12201204
void doToVelocyPack(arangodb::velocypack::Builder& nodes,
12211205
unsigned flags) const override final;
1222-
1223-
private:
1224-
/// @brief input variable non-materialized collection ids
1225-
aql::Variable const* _inNonMaterializedColPtr;
12261206
};
12271207

12281208
class MaterializeSingleNode : public MaterializeNode,
@@ -1253,7 +1233,7 @@ class MaterializeSingleNode : public MaterializeNode,
12531233
};
12541234

12551235
MaterializeNode* createMaterializeNode(ExecutionPlan* plan,
1256-
arangodb::velocypack::Slice const& base);
1236+
arangodb::velocypack::Slice const base);
12571237

12581238
} // namespace materialize
12591239
} // namespace aql

0 commit comments

Comments
 (0)
0