8000 prototype for forceOneShardAttributeValue (#14701) · arangodb/arangodb@a2eba02 · GitHub
[go: up one dir, main page]
Skip to content

Commit a2eba02

Browse files
jsteemannmchackihkernbach
authored
prototype for forceOneShardAttributeValue (#14701)
Co-authored-by: Michael Hackstein <michael@arangodb.com> Co-authored-by: Heiko Kernbach <heiko@arangodb.com>
1 parent 128d570 commit a2eba02

File tree

6 files changed

+159
-40
lines changed

6 files changed

+159
-40
lines changed

‎CHANGELOG

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,46 @@
11
devel
22
-----
33

4+
* (EE only) Bug-fix: If you created a ArangoSearch view on Satellite-
5+
Collections only and then join with a collection only having a single
6+
shard the cluster-one-shard-rule was falsely applied and could lead to
7+
empty view results. The Rule will now detect the situation properly,
8+
and not trigger.
9+
10+
* (EE only) If you have a query using only satellite collections,
11+
now the cluster-one-shard-rule can be applied to improve
12+
query performance.
13+
14+
* (Enterprise Edition only): added query option `forceOneShardAttributeValue`
15+
to explicitly set a shard key value that will be used during query snippet
16+
distribution to limit the query to a specific server in the cluster.
17+
18+
This query option can be used in complex queries in case the query optimizer
19+
cannot automatically detect that the query can be limited to only a single
20+
server (e.g. in a disjoint smart graph case).
21+
When the option is set to the correct shard key value, the query will be
22+
limited to the target server determined by the shard key value. It thus
23+
requires that all collections in the query use the same distribution
24+
(i.e. `distributeShardsLike` attribute via disjoint SmartGraphs).
25+
26+
Limiting the query to a single DB server is a performance optimization
27+
and may make complex queries run a lot faster because of the reduced
28+
setup and teardown costs and the reduced cluster-internal traffic during
29+
query execution.
30+
31+
If the option is set incorrectly, i.e. to a wrong shard key value, then
32+
the query may be shipped to a wrong DB server and may not return results
33+
(i.e. empty result set). It is thus the caller's responsibility to set
34+
the `forceOneShardAttributeValue` correctly or not use it.
35+
36+
The `forceOneShardAttributeValue` option will only honor string values.
37+
All other values as well as the empty string will be ignored and treated
38+
as if the option is not set.
39+
40+
If the option is set and the query satisfies the requirements for using
41+
the option, the query's execution plan will contain the "cluster-one-shard"
42+
optimizer rule.
43+
444
* Updated ArangoDB Starter to 0.15.2.
545

646
* SEARCH-238: Improved SortNodes placement optimization in cluster so

‎arangod/Aql/EngineInfoContainerDBServerServerBased.cpp

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ EngineInfoContainerDBServerServerBased::TraverserEngineShardLists::TraverserEngi
8787
auto const& restrictToShards = query.queryOptions().restrictToShards;
8888
// Extract the local shards for edge collections.
8989
for (auto const& col : edges) {
90+
TRI_ASSERT(col != nullptr);
9091
#ifdef USE_ENTERPRISE
9192
if (query.trxForOptimization().isInaccessibleCollection(col->id())) {
9293
_inaccessible.insert(col->name());
@@ -103,6 +104,7 @@ EngineInfoContainerDBServerServerBased::TraverserEngineShardLists::TraverserEngi
103104
// It might in fact be empty, if we only have edge collections in a graph.
104105
// Or if we guarantee to never read vertex data.
105106
for (auto const& col : vertices) {
107+
TRI_ASSERT(col != nullptr);
106108
#ifdef USE_ENTERPRISE
107109
if (query.trxForOptimization().isInaccessibleCollection(col->id())) {
108110
_inaccessible.insert(col->name());
@@ -120,7 +122,11 @@ std::vector<ShardID> EngineInfoContainerDBServerServerBased::TraverserEngineShar
120122
std::vector<ShardID> localShards;
121123
for (auto const& shard : *shardIds) {
122124
auto const& it = shardMapping.find(shard);
123-
TRI_ASSERT(it != shardMapping.end());
125+
if (it == shardMapping.end()) {
126+
THROW_ARANGO_EXCEPTION_MESSAGE(
127+
TRI_ERROR_INTERNAL,
128+
"no entry for shard '" + shard + "' in shard mapping table (" + std::to_string(shardMapping.size()) + " entries)");
129+
}
124130
if (it->second == server) {
125131
localShards.emplace_back(shard);
126132
_hasShard = true;
@@ -853,15 +859,15 @@ void EngineInfoContainerDBServerServerBased::addOptionsPart(arangodb::velocypack
853859
#endif
854860
}
855861

856-
// Insert the Variables information into the message to be send to DBServers
862+
// Insert the Variables information into the message to be sent to DBServers
857863
void EngineInfoContainerDBServerServerBased::addVariablesPart(arangodb::velocypack::Builder& builder) const {
858864
TRI_ASSERT(builder.isOpenObject());
859865
builder.add(VPackValue("variables"));
860866
// This will open and close an Object.
861867
_query.ast()->variables()->toVelocyPack(builder);
862868
}
863869

864-
// Insert the Snippets information into the message to be send to DBServers
870+
// Insert the Snippets information into the message to be sent to DBServers
865871
void EngineInfoContainerDBServerServerBased::addSnippetPart(
866872
std::unordered_map<ExecutionNodeId, ExecutionNode*> const& nodesById,
867873
arangodb::velocypack::Builder& builder, ShardLocking& shardLocking,
@@ -875,7 +881,7 @@ void EngineInfoContainerDBServerServerBased::addSnippetPart(
875881
builder.close(); // snippets
876882
}
877883

878-
// Insert the TraversalEngine information into the message to be send to DBServers
884+
// Insert the TraversalEngine information into the message to be sent to DBServers
879885
std::vector<bool> EngineInfoContainerDBServerServerBased::addTraversalEnginesPart(
880886
arangodb::velocypack::Builder& infoBuilder,
881887
std::unordered_map<ShardID, ServerID> const& shardMapping, ServerID const& server) const {

‎arangod/Aql/GraphNode.cpp

Lines changed: 7 a F438 dditions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -466,12 +466,14 @@ void GraphNode::setGraphInfoAndCopyColls(std::vector<Collection*> const& edgeCol
466466
std::vector<Collection*> const& vertexColls) {
467467
_graphInfo.openArray();
468468
for (auto& it : edgeColls) {
469+
TRI_ASSERT(it != nullptr);
469470
_edgeColls.emplace_back(it);
470471
_graphInfo.add(VPackValue(it->name()));
471472
}
472473
_graphInfo.close();
473474

474475
for (auto& it : vertexColls) {
476+
TRI_ASSERT(it != nullptr);
475477
addVertexCollection(*it);
476478
}
477479
}
@@ -547,6 +549,7 @@ void GraphNode::doToVelocyPack(VPackBuilder& nodes, unsigned flags) const {
547549
{
548550
VPackArrayBuilder guard(&nodes);
549551
for (auto const& e : _edgeColls) {
552+
TRI_ASSERT(e != nullptr);
550553
auto const& shard = collectionToShardName(e->name());
551554
// if the mapped shard for a collection is empty, it means that
552555
// we have an edge collection that is only relevant on some of the
@@ -561,6 +564,7 @@ void GraphNode::doToVelocyPack(VPackBuilder& nodes, unsigned flags) const {
561564
{
562565
VPackArrayBuilder guard(&nodes);
563566
for (auto const& v : _vertexColls) {
567+
TRI_ASSERT(v != nullptr);
564568
// if the mapped shard for a collection is empty, it means that
565569
// we have a vertex collection that is only relevant on some of the
566570
// target servers
@@ -631,6 +635,7 @@ CostEstimate GraphNode::estimateCost() const {
631635
double baseCost = 1;
632636
size_t baseNumItems = 0;
633637
for (auto& e : _edgeColls) {
638+
TRI_ASSERT(e != nullptr);
634639
auto count = e->count(_options->trx(), transaction::CountType::TryCache);
635640
// Assume an estimate if 10% hit rate
636641
baseCost *= count / 10;
@@ -794,9 +799,11 @@ std::vector<aql::Collection const*> GraphNode::collections() const {
794799
set.reserve(_edgeColls.size() + _vertexColls.size());
795800

796801
for (auto const& collPointer : _edgeColls) {
802+
TRI_ASSERT(collPointer != nullptr);
797803
set.emplace(collPointer);
798804
}
799805
for (auto const& collPointer : _vertexColls) {
806+
TRI_ASSERT(collPointer != nullptr);
800807
set.emplace(collPointer);
801808
}
802809

‎arangod/Aql/QueryOptions.cpp

Lines changed: 19 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -172,50 +172,43 @@ void QueryOptions::fromVelocyPack(VPackSlice slice) {
172172
traversalProfile = static_cast<TraversalProfileLevel>(value.getNumber<uint16_t>());
173173
}
174174

175-
value = slice.get("allPlans");
176-
if (value.isBool()) {
175+
if (value = slice.get("allPlans"); value.isBool()) {
177176
allPlans = value.getBool();
178177
}
179-
value = slice.get("verbosePlans");
180-
if (value.isBool()) {
178+
if (value = slice.get("verbosePlans"); value.isBool()) {
181179
verbosePlans = value.getBool();
182180
}
183-
value = slice.get("stream");
184-
if (value.isBool()) {
181+
if (value = slice.get("stream"); value.isBool()) {
185182
stream = value.getBool();
186183
}
187-
value = slice.get("silent");
188-
if (value.isBool()) {
184+
if (value = slice.get("silent"); value.isBool()) {
189185
silent = value.getBool();
190186
}
191-
value = slice.get("failOnWarning");
192-
if (value.isBool()) {
187+
if (value = slice.get("failOnWarning"); value.isBool()) {
193188
failOnWarning = value.getBool();
194189
}
195-
value = slice.get("cache");
196-
if (value.isBool()) {
190+
if (value = slice.get("cache"); value.isBool()) {
197191
cache = value.getBool();
198192
}
199-
value = slice.get("fullCount");
200-
if (value.isBool()) {
193+
if (value = slice.get("fullCount"); value.isBool()) {
201194
fullCount = value.getBool();
202195
}
203-
value = slice.get("count");
204-
if (value.isBool()) {
196+
if (value = slice.get("count"); value.isBool()) {
205197
count = value.getBool();
206198
}
207-
value = slice.get("verboseErrors");
208-
if (value.isBool()) {
199+
if (value = slice.get("verboseErrors"); value.isBool()) {
209200
verboseErrors = value.getBool();
210201
}
211-
value = slice.get("explainRegisters");
212-
if (value.isBool()) {
213-
explainRegisters =
214-
value.getBool() ? ExplainRegisterPlan::Yes : ExplainRegisterPlan::No;
202+
if (value = slice.get("explainRegisters"); value.isBool()) {
203+
explainRegisters = value.getBool() ? ExplainRegisterPlan::Yes : ExplainRegisterPlan::No;
215204
}
216-
205+
217206
// note: skipAudit is intentionally not read here.
218207
// the end user cannot override this setting
208+
209+
if (value = slice.get("forceOneShardAttributeValue"); value.isString()) {
210+
forceOneShardAttributeValue = value.copyString();
211+
}
219212

220213
VPackSlice optimizer = slice.get("optimizer");
221214
if (optimizer.isObject()) {
@@ -279,6 +272,9 @@ void QueryOptions::toVelocyPack(VPackBuilder& builder, bool disableOptimizerRule
279272
builder.add("fullCount", VPackValue(fullCount));
280273
builder.add("count", VPackValue(count));
281274
builder.add("verboseErrors", VPackValue(verboseErrors));
275+
if (!forceOneShardAttributeValue.empty()) {
276+
builder.add("forceOneShardAttributeValue", VPackValue(forceOneShardAttributeValue));
277+
}
282278

283279
// note: skipAudit is intentionally not serialized here.
284280
// the end user cannot override this setting anyway.

‎arangod/Aql/QueryOptions.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,10 @@ struct QueryOptions {
9292
bool skipAudit; // skips audit logging - used only internally
9393
ExplainRegisterPlan explainRegisters;
9494

95+
/// @brief shard key attribute value used to push a query down
96+
/// to a single server
97+
std::string forceOneShardAttributeValue;
98+
9599
/// @brief optimizer rules to turn off/on manually
96100
std::vector<std::string> optimizerRules;
97101

0 commit comments

Comments
 (0)
0