8000 Feature/hybrid smart graph testsuite (#14056) · arangodb/arangodb@0392473 · GitHub
[go: up one dir, main page]
Skip to content

Commit 0392473

Browse files
hkernbachhkernbach
authored
Feature/hybrid smart graph testsuite (#14056)
1 parent 517067a commit 0392473

File tree

75 files changed

+2459
-1077
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

75 files changed

+2459
-1077
lines changed

‎arangod/Aql/AqlFunctionsInternalCache.h

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,11 @@
2525

2626
#include "Aql/AqlValue.h"
2727
#include "Basics/Common.h"
28-
#include <VocBase/Validators.h>
28+
#include "VocBase/Validators.h"
2929

3030
#include <unicode/regex.h>
3131
#include <memory>
3232

33-
34-
3533
namespace arangodb {
3634

3735
namespace transaction {
@@ -49,6 +47,8 @@ class AqlFunctionsInternalCache final {
4947
AqlFunctionsInternalCache() = default;
5048
~AqlFunctionsInternalCache();
5149

50+
AqlFunctionsInternalCache(AqlFunctionsInternalCache&&) = default;
51+
5252
void clear() noexcept;
5353

5454
icu::RegexMatcher* buildRegexMatcher(char const* ptr, size_t length, bool caseInsensitive);
@@ -97,4 +97,3 @@ class AqlFunctionsInternalCache final {
9797

9898
} // namespace aql
9999
} // namespace arangodb
100-

‎arangod/Aql/EngineInfoContainerDBServerServerBased.cpp

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,8 @@ EngineInfoContainerDBServerServerBased::TraverserEngineShardLists::TraverserEngi
8787
}
8888
#endif
8989
_edgeCollections.emplace_back(
90-
getAllLocalShards(shardMapping, server, col->shardIds(restrictToShards)));
90+
getAllLocalShards(shardMapping, server, col->shardIds(restrictToShards),
91+
col->isSatellite() && node->isSmart()));
9192
}
9293
// Extract vertices
9394
auto const& vertices = _node->vertexColls();
@@ -102,21 +103,26 @@ EngineInfoContainerDBServerServerBased::TraverserEngineShardLists::TraverserEngi
102103
_inaccessible.insert(std::to_string(col->id().id()));
103104
}
104105
#endif
105-
auto shards = getAllLocalShards(shardMapping, server, col->shardIds(restrictToShards));
106+
auto shards = getAllLocalShards(shardMapping, server, col->shardIds(restrictToShards),
107+
col->isSatellite() && node->isSmart());
106108
_vertexCollections.try_emplace(col->name(), std::move(shards));
107109
}
108110
}
109111

110112
std::vector<ShardID> EngineInfoContainerDBServerServerBased::TraverserEngineShardLists::getAllLocalShards(
111113
std::unordered_map<ShardID, ServerID> const& shardMapping,
112-
ServerID const& server, std::shared_ptr<std::vector<std::string>> shardIds) {
114+
ServerID const& server, std::shared_ptr<std::vector<std::string>> shardIds, bool colIsSatellite) {
113115
std::vector<ShardID> localShards;
114116
for (auto const& shard : *shardIds) {
115117
auto const& it = shardMapping.find(shard);
116118
TRI_ASSERT(it != shardMapping.end());
117119
if (it->second == server) {
118120
localShards.emplace_back(shard);
121+
// Guaranteed that the traversal will be executed on this server.
119122
_hasShard = true;
123+
} else if (colIsSatellite) {
124+
// The satellite does not force run of a traversal here.
125+
localShards.emplace_back(shard);
120126
}
121127
}
122128
return localShards;

‎arangod/Aql/EngineInfoContainerDBServerServerBased.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,8 @@ class EngineInfoContainerDBServerServerBased {
8181
private:
8282
std::vector<ShardID> getAllLocalShards(std::unordered_map<ShardID, ServerID> const& shardMapping,
8383
ServerID const& server,
84-
std::shared_ptr<std::vector<std::string>> shardIds);
84+
std::shared_ptr<std::vector<std::string>> shardIds,
85+
bool colIsSatellite);
8586

8687
private:
8788
// The graph node we need to serialize

‎arangod/Aql/FixedVarExpressionContext.cpp

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,14 +56,18 @@ void FixedVarExpressionContext::setVariableValue(Variable const* var, AqlValue c
5656
_vars.try_emplace(var, value);
5757
}
5858

59+
void FixedVarExpressionContext::clearVariableValue(Variable const* var) {
60+
_vars.erase(var);
61+
}
62+
5963
void FixedVarExpressionContext::serializeAllVariables(velocypack::Options const& opts,
6064
velocypack::Builder& builder) const {
6165
TRI_ASSERT(builder.isOpenArray());
6266
for (auto const& it : _vars) {
6367
builder.openArray();
6468
it.first->toVelocyPack(builder);
65-
it.second.toVelocyPack(&opts, builder, /*resolveExternals*/true,
66-
/*allowUnindexed*/false);
69+
it.second.toVelocyPack(&opts, builder, /*resolveExternals*/ true,
70+
/*allowUnindexed*/ false);
6771
builder.close();
6872
}
6973
}

‎arangod/Aql/FixedVarExpressionContext.h

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,7 @@ class AqlItemBlock;
3838

3939
class FixedVarExpressionContext final : public QueryExpressionContext {
4040
public:
41-
explicit FixedVarExpressionContext(transaction::Methods& trx,
42-
QueryContext& query,
41+
explicit FixedVarExpressionContext(transaction::Methods& trx, QueryContext& query,
4342
AqlFunctionsInternalCache& cache);
4443

4544
~FixedVarExpressionContext() override = default;
@@ -51,8 +50,14 @@ class FixedVarExpressionContext final : public QueryExpressionContext {
5150

5251
void clearVariableValues();
5352

53+
// @brief This method will set the given variable to the given AQL value
54+
// if the variable already holds a value, this method will keep the old value.
5455
void setVariableValue(Variable const*, AqlValue const&);
5556

57+
// @brief This method will only clear the given variable, and keep
58+
// all others intact. If the variable does not exist, this is a noop.
59+
void clearVariableValue(Variable const*);
60+
5661
void serializeAllVariables(velocypack::Options const& opts,
5762
arangodb::velocypack::Builder&) const;
5863

‎arangod/Aql/GraphNode.cpp

Lines changed: 17 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ TRI_edge_direction_e parseDirection(AstNode const* node) {
9090
return uint64ToDirection(dirNode->getIntValue());
9191
}
9292

93-
}
93+
} // namespace
9494

9595
GraphNode::GraphNode(ExecutionPlan* plan, ExecutionNodeId id,
9696
TRI_vocbase_t* vocbase, AstNode const* direction,
@@ -107,7 +107,6 @@ GraphNode::GraphNode(ExecutionPlan* plan, ExecutionNodeId id,
107107
_optionsBuilt(false),
108108
_isSmart(false),
109109
_isDisjoint(false),
110-
_isHybrid(false),
111110
_options(std::move(options)) {
112111
// Direction is already the correct Integer.
113112
// Is not inserted by user but by enum.
@@ -301,9 +300,10 @@ GraphNode::GraphNode(ExecutionPlan* plan, arangodb::velocypack::Slice const& bas
301300
_defaultDirection(uint64ToDirection(arangodb::basics::VelocyPackHelper::stringUInt64(
302301
base.get("defaultDirection")))),
303302
_optionsBuilt(false),
304-
_isSmart(arangodb::basics::VelocyPackHelper::getBooleanValue(base, "isSmart", false)),
305-
_isDisjoint(arangodb::basics::VelocyPackHelper::getBooleanValue(base, "isDisjoint", false)) {
306-
303+
_isSmart(arangodb::basics::VelocyPackHelper::getBooleanValue(base, StaticStrings::IsSmart,
304+
false)),
305+
_isDisjoint(arangodb::basics::VelocyPackHelper::getBooleanValue(base, StaticStrings::IsDisjoint,
306+
false)) {
307307
if (!ServerState::instance()->isDBServer()) {
308308
// Graph Information. Do we need to reload the graph here?
309309
std::string graphName;
@@ -357,7 +357,8 @@ GraphNode::GraphNode(ExecutionPlan* plan, arangodb::velocypack::Slice const& bas
357357
auto getAqlCollectionFromName = [&](std::string const& name) -> aql::Collection& {
358358
// if the collection was already existent in the query, addCollection will
359359
// just return it.
360-
return *query.collections().add(name, AccessMode::Type::READ, aql::Collection::Hint::Collection);
360+
return *query.collections().add(name, AccessMode::Type::READ,
361+
aql::Collection::Hint::Collection);
361362
};
362363

363364
auto vPackDirListIter = VPackArrayIterator(dirList);
@@ -397,7 +398,8 @@ GraphNode::GraphNode(ExecutionPlan* plan, arangodb::velocypack::Slice const& bas
397398
"graph needs a translation from collection to shard names");
398399
}
399400
for (auto const& item : VPackObjectIterator(collectionToShard)) {
400-
_collectionToShard.insert({item.key.copyString(), item.value.copyString()});
401+
_collectionToShard.insert({item.key.copyString(),
402+
std::vector<std::string>{item.value.copyString()}});
401403
}
402404

403405
// Out variables
@@ -510,9 +512,11 @@ std::string const& GraphNode::collectionToShardName(std::string const& collName)
510512

511513
auto found = _collectionToShard.find(collName);
512514
if (found == _collectionToShard.end()) {
513-
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL_AQL, "unable to find shard '" + collName + "' in query shard map");
515+
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL_AQL,
516+
"unable to find shard '" + collName +
517+
"' in query shard map");
514518
}
515-
return found->second;
519+
return found->second.front();
516520
}
517521

518522
void GraphNode::toVelocyPackHelper(VPackBuilder& nodes, unsigned flags,
@@ -581,7 +585,8 @@ void GraphNode::toVelocyPackHelper(VPackBuilder& nodes, unsigned flags,
581585
{
582586
VPackObjectBuilder guard(&nodes);
583587
for (auto const& item : _collectionToShard) {
584-
nodes.add(item.first, VPackValue(item.second));
588+
TRI_ASSERT(item.second.size() == 1);
589+
nodes.add(item.first, VPackValue(item.second.front()));
585590
}
586591
}
587592

@@ -714,8 +719,8 @@ void GraphNode::enhanceEngineInfo(VPackBuilder& builder) const {
714719
}
715720
#endif
716721

717-
void GraphNode::addEdgeCollection(Collections const& collections, std::string const& name,
718-
TRI_edge_direction_e dir) {
722+
void GraphNode::addEdgeCollection(Collections const& collections,
723+
std::string const& name, TRI_edge_direction_e dir) {
719724
auto aqlCollection = collections.get(name);
720725
if (aqlCollection != nullptr) {
721726
addEdgeCollection(*aqlCollection, dir);
@@ -811,8 +816,6 @@ bool GraphNode::isSmart() const { return _isSmart; }
811816

812817
bool GraphNode::isDisjoint() const { return _isDisjoint; }
813818

814-
bool GraphNode::isHybrid() const { return _isHybrid; }
815-
816819
TRI_vocbase_t* GraphNode::vocbase() const { return _vocbase; }
817820

818821
Variable const* GraphNode::vertexOutVariable() const {

‎arangod/Aql/GraphNode.h

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,12 @@
2323

2424
#pragma once
2525

26-
#include "Aql/types.h"
2726
#include "Aql/Condition.h"
2827
#include "Aql/ExecutionNode.h"
2928
#include "Aql/ExecutionNodeId.h"
3029
#include "Aql/GraphNode.h"
3130
#include "Aql/Graphs.h"
31+
#include "Aql/types.h"
3232
#include "Cluster/ClusterTypes.h"
3333
#include "VocBase/LogicalCollection.h"
3434
#include "VocBase/voc-types.h"
@@ -81,10 +81,13 @@ class GraphNode : public ExecutionNode {
8181
GraphNode(ExecutionPlan* plan, arangodb::velocypack::Slice const& base);
8282

8383
public:
84+
// QueryPlan decided that we use this graph as a satellite
8485
bool isUsedAsSatellite() const;
86+
// Defines whether a GraphNode can fully be pushed down to a DBServer
8587
bool isLocalGraphNode() const;
88+
// Will wait as soon as any of our collections is a satellite (in sync)
8689
void waitForSatelliteIfRequired(ExecutionEngine const* engine) const;
87-
90+
// Can be fully pushed down to a DBServer and is available on all DBServers
8891
bool isEligibleAsSatelliteTraversal() const;
8992

9093
protected:
@@ -122,9 +125,6 @@ class GraphNode : public ExecutionNode {
122125
/// @brief flag, if the graph is a Disjoint SmartGraph (Enterprise Edition only!)
123126
bool isDisjoint() const;
124127

125-
/// @brief flag, if the graph is a Hybrid SmartGraph (Enterprise Edition only!)
126-
bool isHybrid() const;
127-
128128
/// @brief return the database
129129
TRI_vocbase_t* vocbase() const;
130130

@@ -192,20 +192,21 @@ class GraphNode : public ExecutionNode {
192192
void injectVertexCollection(aql::Collection& other);
193193

194194
std::vector<aql::Collection const*> collections() const;
195-
void setCollectionToShard(std::map<std::string, std::string> const& map) {
195+
void setCollectionToShard(std::unordered_map<std::string, std::vector<std::string>> const& map) {
196196
_collectionToShard = map;
197197
}
198198
void addCollectionToShard(std::string const& coll, std::string const& shard) {
199199
// NOTE: Do not replace this by emplace or insert.
200200
// This is also used to overwrite the existing entry.
201-
_collectionToShard[coll] = shard;
201+
_collectionToShard[coll] = std::vector<std::string>{shard};
202202
}
203203

204204
public:
205205
graph::Graph const* graph() const noexcept;
206206

207207
private:
208-
void addEdgeCollection(aql::Collections const& collections, std::string const& name, TRI_edge_direction_e dir);
208+
void addEdgeCollection(aql::Collections const& collections,
209+
std::string const& name, TRI_edge_direction_e dir);
209210
void addEdgeCollection(aql::Collection& collection, TRI_edge_direction_e dir);
210211
void addVertexCollection(aql::Collections const& collections, std::string const& name);
211212
void addVertexCollection(aql::Collection& collection);
@@ -258,9 +259,6 @@ class GraphNode : public ExecutionNode {
258259
/// @brief flag, if graph is smart *and* disjoint (Enterprise Edition only!)
259260
bool _isDisjoint;
260261

261-
/// @brief flag, if graph is smart *and* hybrid (Enterprise Edition only!)
262-
bool _isHybrid;
263-
264262
/// @brief The directions edges are followed
265263
std::vector<TRI_edge_direction_e> _directions;
266264

@@ -271,7 +269,7 @@ class GraphNode : public ExecutionNode {
271269
std::unordered_map<ServerID, aql::EngineId> _engines;
272270

273271
/// @brief list of shards involved, required for one-shard-databases
274-
std::map<std::string, std::string> _collectionToShard;
272+
std::unordered_map<std::string, std::vector<std::string>> _collectionToShard;
275273
};
276274

277275
} // namespace aql

0 commit comments

Comments
 (0)
0