8000 Feature/hybrid smart graph api (#14037) · arangodb/arangodb@e951cba · GitHub
[go: up one dir, main page]
Skip to content

Commit e951cba

Browse files
hkernbachhkernbach
authored
Feature/hybrid smart graph api (#14037)
Added First Draft of Hybrid Smart Graph API
1 parent d396a1f commit e951cba

File tree

8 files changed

+150
-16
lines changed

8 files changed

+150
-16
lines changed

‎arangod/Aql/GraphNode.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,7 @@ GraphNode::GraphNode(ExecutionPlan* plan, ExecutionNodeId id,
107107
_optionsBuilt(false),
108108
_isSmart(false),
109109
_isDisjoint(false),
110+
_isHybrid(false),
110111
_options(std::move(options)) {
111112
// Direction is already the correct Integer.
112113
// Is not inserted by user but by enum.
@@ -810,6 +811,8 @@ bool GraphNode::isSmart() const { return _isSmart; }
810811

811812
bool GraphNode::isDisjoint() const { return _isDisjoint; }
812813

814+
bool GraphNode::isHybrid() const { return _isHybrid; }
815+
813816
TRI_vocbase_t* GraphNode::vocbase() const { return _vocbase; }
814817

815818
Variable const* GraphNode::vertexOutVariable() const {

‎arangod/Aql/GraphNode.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,9 @@ class GraphNode : public ExecutionNode {
123123
/// @brief flag, if the graph is a Disjoint SmartGraph (Enterprise Edition only!)
124124
bool isDisjoint() const;
125125

126+
/// @brief flag, if the graph is a Hybrid SmartGraph (Enterprise Edition only!)
127+
bool isHybrid() const;
128+
126129
/// @brief return the database
127130
TRI_vocbase_t* vocbase() const;
128131

@@ -256,6 +259,9 @@ class GraphNode : public ExecutionNode {
256259
/// @brief flag, if graph is smart *and* disjoint (Enterprise Edition only!)
257260
bool _isDisjoint;
258261

262+
/// @brief flag, if graph is smart *and* hybrid (Enterprise Edition only!)
263+
bool _isHybrid;
264+
259265
/// @brief The directions edges are followed
260266
std::vector<TRI_edge_direction_e> _directions;
261267

‎arangod/Graph/Graph.cpp

Lines changed: 62 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,19 @@ Graph::Graph(velocypack::Slice const& slice, ServerDefaults const& serverDefault
116116
TRI_ASSERT(!_rev.empty());
117117

118118
if (slice.hasKey(StaticStrings::GraphEdgeDefinitions)) {
119+
// TODO Feature HybridSmartGraphs: Check why we're landing here and not in SmartGraphEE - Cleanup!
120+
if (slice.isObject()) {
121+
if (slice.hasKey(StaticStrings::GraphSatellites) &&
122+
slice.get(StaticStrings::GraphSatellites).isArray()) {
123+
124+
for (VPackSlice it : VPackArrayIterator(slice.get(StaticStrings::GraphSatellites))) {
125+
if (!it.isString()) {
126+
THROW_ARANGO_EXCEPTION(TRI_ERROR_GRAPH_INVALID_PARAMETER);
127+
}
128+
_satelliteColls.emplace(it.copyString());
129+
}
130+
}
131+
}
119132
parseEdgeDefinitions(slice.get(StaticStrings::GraphEdgeDefinitions));
120133
}
121134
if (slice.hasKey(StaticStrings::GraphOrphans)) {
@@ -146,6 +159,19 @@ Graph::Graph(TRI_vocbase_t& vocbase, std::string&& graphName,
146159
TRI_ASSERT(_rev.empty());
147160

148161
if (info.hasKey(StaticStrings::GraphEdgeDefinitions)) {
162+
// TODO Feature HybridSmartGraphs: Check why we're landing here and not in SmartGraphEE - Cleanup!
163+
if (options.isObject()) {
164+
if (options.hasKey(StaticStrings::GraphSatellites) &&
165+
options.get(StaticStrings::GraphSatellites).isArray()) {
166+
167+
for (VPackSlice it : VPackArrayIterator(options.get(StaticStrings::GraphSatellites))) {
168+
if (!it.isString()) {
169+
THROW_ARANGO_EXCEPTION(TRI_ERROR_GRAPH_INVALID_PARAMETER);
170+
}
171+
_satelliteColls.emplace(it.copyString());
172+
}
173+
}
174+
}
149175
parseEdgeDefinitions(info.get(StaticStrings::GraphEdgeDefinitions));
150176
}
151177
if (info.hasKey(StaticStrings::GraphOrphans)) {
@@ -212,10 +238,18 @@ std::set<std::string> const& Graph::orphanCollections() const {
212238
return _orphanColls;
213239
}
214240

241+
std::set<std::string> const& Graph::satelliteCollections() const {
242+
return _satelliteColls;
243+
}
244+
215245
std::set<std::string> const& Graph::edgeCollections() const {
216246
return _edgeColls;
217247
}
218248

249+
bool Graph::needsToBeSatellite(std::string const& edge) const {
250+
return false;
251+
}
252+
219253
std::map<std::string, EdgeDefinition> const& Graph::edgeDefinitions() const {
220254
return _edgeDefs;
221255
}
@@ -417,9 +451,10 @@ void EdgeDefinition::toVelocyPack(VPackBuilder& builder) const {
417451
builder.add(VPackValue(to));
418452
}
419453
builder.close(); // array
454+
builder.add("type", VPackValue(getType()));
420455
}
421456

422-
ResultT<EdgeDefinition> EdgeDefinition::createFromVelocypack(VPackSlice edgeDefinition) {
457+
ResultT<EdgeDefinition> EdgeDefinition::createFromVelocypack(VPackSlice edgeDefinition, std::set<std::string> const& satCollections) {
423458
Result res = EdgeDefinition::validateEdgeDefini 10000 tion(edgeDefinition);
424459
if (res.fail()) {
425460
return res;
@@ -433,9 +468,11 @@ ResultT<EdgeDefinition> EdgeDefinition::createFromVelocypack(VPackSlice edgeDefi
433468

434469
// duplicates in from and to shouldn't occur, but are safely ignored here
435470
for (VPackSlice it : VPackArrayIterator(from)) {
471+
TRI_ASSERT(it.isString());
436472
fromSet.emplace(it.copyString());
437473
}
438474
for (VPackSlice it : VPackArrayIterator(to)) {
475+
TRI_ASSERT(it.isString());
439476
toSet.emplace(it.copyString());
440477
}
441478

@@ -484,6 +521,19 @@ bool EdgeDefinition::renameCollection(std::string const& oldName, std::string co
484521
return renamed;
485522
}
486523

524+
auto EdgeDefinition::getType() const -> EdgeDefinitionType {
525+
return _type;
526+
}
527+
528+
auto EdgeDefinition::setType(EdgeDefinitionType type) -> bool {
529+
TRI_ASSERT(type != EdgeDefinitionType::DEFAULT);
530+
if (_type == EdgeDefinitionType::DEFAULT) {
531+
_type = type;
532+
return true;
533+
}
534+
return false;
535+
}
536+
487537
bool EdgeDefinition::isFromVertexCollectionUsed(std::string const& collectionName) const {
488538
if (getFrom().find(collectionName) != getFrom().end()) {
489539
return true;
@@ -522,6 +572,7 @@ void EdgeDefinition::addToBuilder(VPackBuilder& builder) const {
522572
builder.add(VPackValue(to));
523573
}
524574
builder.close(); // to
575+
builder.add(StaticStrings::GraphEdgeDefinitionType, VPackValue(getType()));
525576

526577
builder.close(); // obj
527578
}
@@ -590,7 +641,7 @@ ResultT<EdgeDefinition const*> Graph::addEdgeDefinition(EdgeDefinition const& ed
590641
}
591642

592643
ResultT<EdgeDefinition const*> Graph::addEdgeDefinition(VPackSlice const& edgeDefinitionSlice) {
593-
auto res = EdgeDefinition::createFromVelocypack(edgeDefinitionSlice);
644+
auto res = EdgeDefinition::createFromVelocypack(edgeDefinitionSlice, satelliteCollections());
594645

595646
if (res.fail()) {
596647
return std::move(res).result();
@@ -724,6 +775,11 @@ bool Graph::isDisjoint() const {
724775
return false;
725776
}
726777

778+
bool Graph::isHybrid() const {
779+
TRI_ASSERT(_satelliteColls.empty());
780+
return false;
781+
}
782+
727783
bool Graph::isSatellite() const { return _isSatellite; }
728784

729785
void Graph::createCollectionOptions(VPackBuilder& builder, bool waitForSync) const {
@@ -743,6 +799,10 @@ void Graph::createCollectionOptions(VPackBuilder& builder, bool waitForSync) con
743799
builder.add(StaticStrings::ReplicationFactor, VPackValue(replicationFactor()));
744800
}
745801

802+
void Graph::createSatelliteCollectionOptions(VPackBuilder& builder, bool waitForSync) const {
803+
TRI_ASSERT(false);
804+
}
805+
746806
std::optional<std::reference_wrapper<const EdgeDefinition>> Graph::getEdgeDefinition(
747807
std::string const& collectionName) const {
748808
auto it = edgeDefinitions().find(collectionName);

‎arangod/Graph/Graph.h

Lines changed: 38 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,12 +46,23 @@ struct ServerDefaults;
4646
namespace graph {
4747

4848
class EdgeDefinition {
49+
public:
50+
enum EdgeDefinitionType {
51+
DEFAULT,
52+
SMART_TO_SMART,
53+
SAT_TO_SAT,
54+
SMART_TO_SAT,
55+
SAT_TO_SMART
56+
};
57+
4958
public:
5059
EdgeDefinition(std::string edgeCollection_, std::set<std::string>&& from_,
51-
std::set<std::string>&& to_)
60+
std::set<std::string>&& to_,
61+
EdgeDefinitionType type = EdgeDefinitionType::DEFAULT)
5262
: _edgeCollection(std::move(edgeCollection_)),
5363
_from(std::move(from_)),
54-
_to(std::move(to_)) {}
64+
_to(std::move(to_)),
65+
_type(type) {}
5566

5667
std::string const& getName() const { return _edgeCollection; }
5768
void setName(std::string const& newName) { _edgeCollection = newName; }
@@ -70,7 +81,8 @@ class EdgeDefinition {
7081
/// types of values.
7182
static Result validateEdgeDefinition(const velocypack::Slice& edgeDefinition);
7283

73-
static ResultT<EdgeDefinition> createFromVelocypack(velocypack::Slice edgeDefinition);
84+
static ResultT<EdgeDefinition> createFromVelocypack(velocypack::Slice edgeDefinition,
85+
std::set<std::string> const& satCollections);
7486

7587
void toVelocyPack(velocypack::Builder&) const;
7688

@@ -82,10 +94,23 @@ class EdgeDefinition {
8294

8395
bool renameCollection(std::string const& oldName, std::string const& newName);
8496

97+
auto getType() const -> EdgeDefinitionType;
98+
99+
/* @brief
100+
* Set type of the EdgeDefinition. Only allowed to be called once and only if
101+
* type is DEFAULT. If type has been set, it is not changeable anymore.
102+
*
103+
* @param type Type to be set
104+
*
105+
* @return True if type has been set, returns false in case type has not been set.
106+
*/
107+
auto setType(EdgeDefinitionType type) -> bool;
108+
85109
private:
86110
std::string _edgeCollection;
87111
std::set<std::string> _from;
88112
std::set<std::string> _to;
113+
EdgeDefinitionType _type;
89114
};
90115

91116
class Graph {
@@ -151,13 +176,18 @@ class Graph {
151176

152177
virtual void createCollectionOptions(VPackBuilder& builder, bool waitForSync) const;
153178

179+
virtual void createSatelliteCollectionOptions(VPackBuilder& builder, bool waitForSync) const;
180+
154181
public:
155182
/// @brief get the cids of all vertexCollections
156183
std::set<std::string> const& vertexCollections() const;
157184

158185
/// @brief get the cids of all orphanCollections
159186
std::set<std::string> const& orphanCollections() const;
160187

188+
/// @brief get the cids of all satelliteCollections
189+
std::set<std::string> const& satelliteCollections() const;
190+
161191
/// @brief get the cids of all edgeCollections
162192
std::set<std::string> const& edgeCollections() const;
163193

@@ -178,6 +208,8 @@ class Graph {
178208
virtual bool isSmart() const;
179209
virtual bool isDisjoint() const;
180210
virtual bool isSatellite() const;
211+
virtual bool isHybrid() const;
212+
virtual bool needsToBeSatellite(std::string const& edge) const;
181213

182214
uint64_t numberOfShards() const;
183215
uint64_t replicationFactor() const;
@@ -291,6 +323,9 @@ class Graph {
291323
/// @brief the names of all orphanCollections
292324
std::set<std::string> _orphanColls;
293325

326+
/// @brief the names of all orphanCollections
327+
std::set<std::string> _satelliteColls;
328+
294329
/// @brief the names of all edgeCollections
295330
std::set<std::string> _edgeColls;
296331

‎arangod/Graph/GraphManager.cpp

Lines changed: 34 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -459,8 +459,7 @@ OperationResult GraphManager::storeGraph(Graph const& graph, bool waitForSync,
459459
Result GraphManager::applyOnAllGraphs(std::function<Result(std::unique_ptr<Graph>)> const& callback) const {
460460
std::string const queryStr{"FOR g IN _graphs RETURN g"};
461461
arangodb::aql::Query query(transaction::StandaloneContext::Create(_vocbase),
462-
arangodb::aql::QueryString{queryStr},
463-
nullptr);
462+
arangodb::aql::QueryString{queryStr}, nullptr);
464463
query.queryOptions().skipAudit = true;
465464
aql::QueryResult queryResult = query.executeSync();
466465

@@ -505,7 +504,9 @@ Result GraphManager::applyOnAllGraphs(std::function<Result(std::unique_ptr<Graph
505504
Result GraphManager::ensureCollections(Graph* graph, bool waitForSync) const {
506505
// Validation Phase collect a list of collections to create
507506
std::unordered_set<std::string> documentCollectionsToCreate{};
507+
std::unordered_set<std::string> satelliteDocumentCollectionsToCreate{};
508508
std::unordered_set<std::string> edgeCollectionsToCreate{};
509+
std::unordered_set<std::string> satelliteEdgeCollectionsToCreate{};
509510
std::unordered_set<std::shared_ptr<LogicalCollection>> existentDocumentCollections{};
510511
std::unordered_set<std::shared_ptr<LogicalCollection>> existentEdgeCollections{};
511512

@@ -533,7 +534,11 @@ Result GraphManager::ensureCollections(Graph* graph, bool waitForSync) const {
533534
return res;
534535
} else {
535536
// not found the collection, need to create it later
536-
edgeCollectionsToCreate.emplace(edgeColl);
537+
if (graph->isHybrid() && graph->needsToBeSatellite(edgeColl)) { // check for satellites
538+
satelliteEdgeCollectionsToCreate.emplace(edgeColl);
539+
} else {
540+
edgeCollectionsToCreate.emplace(edgeColl);
541+
}
537542
}
538543
}
539544

@@ -550,7 +555,11 @@ Result GraphManager::ensureCollections(Graph* graph, bool waitForSync) const {
550555
return res;
551556
} else {
552557
if (edgeCollectionsToCreate.find(vertexColl) == edgeCollectionsToCreate.end()) {
553-
documentCollectionsToCreate.emplace(vertexColl);
558+
if (graph->isHybrid() && graph->satelliteCollections().find(vertexColl) != graph->satelliteCollections().end()) {
559+
satelliteDocumentCollectionsToCreate.emplace(vertexColl);
560+
} else {
561+
documentCollectionsToCreate.emplace(vertexColl);
562+
}
554563
}
555564
}
556565
}
@@ -610,6 +619,22 @@ Result GraphManager::ensureCollections(Graph* graph, bool waitForSync) const {
610619
collectionsToCreate.emplace_back(
611620
CollectionCreationInfo{edgeColl, TRI_COL_TYPE_EDGE, options});
612621
}
622+
623+
// new builder
624+
VPackBuilder satOptionsBuilder;
625+
if (graph->isHybrid()) {
626+
satOptionsBuilder.openObject();
627+
graph->createSatelliteCollectionOptions(satOptionsBuilder, waitForSync);
628+
satOptionsBuilder.close();
629+
for (auto const& satColl : satelliteDocumentCollectionsToCreate) {
630+
collectionsToCreate.emplace_back(CollectionCreationInfo{satColl, TRI_COL_TYPE_DOCUMENT,
631+
satOptionsBuilder.slice()});
632+
}
633+
for (auto const& satEdgeColl : satelliteEdgeCollectionsToCreate) {
634+
collectionsToCreate.emplace_back(CollectionCreationInfo{satEdgeColl, TRI_COL_TYPE_EDGE,
635+
satOptionsBuilder.slice()});
636+
}
637+
}
613638
if (collectionsToCreate.empty()) {
614639
// NOTE: Empty graph is allowed.
615640
return TRI_ERROR_NO_ERROR;
@@ -650,8 +675,7 @@ Result GraphManager::readGraphKeys(velocypack::Builder& builder) const {
650675

651676
Result GraphManager::readGraphByQuery(velocypack::Builder& builder,
652677
std::string const& queryStr) const {
653-
arangodb::aql::Query query(ctx(), arangodb::aql::QueryString(queryStr),
654-
nullptr);
678+
arangodb::aql::Query query(ctx(), arangodb::aql::QueryString(queryStr), nullptr);
655679
query.queryOptions().skipAudit = true;
656680

657681
LOG_TOPIC("f6782", DEBUG, arangodb::Logger::GRAPHS)
@@ -670,7 +694,7 @@ Result GraphManager::readGraphByQuery(velocypack::Builder& builder,
670694

671695
if (graphsSlice.isNone()) {
672696
return Result(TRI_ERROR_OUT_OF_MEMORY);
673-
}
697+
}
674698
if (!graphsSlice.isArray()) {
675699
LOG_TOPIC("338b7", ERR, arangodb::Logger::GRAPHS)
676700
<< "cannot read graphs from _graphs collection";
@@ -1037,7 +1061,9 @@ ResultT<std::unique_ptr<Graph>> GraphManager::buildGraphFromInput(std::string co
10371061
s = s.get(StaticStrings::ReplicationFactor);
10381062
if ((s.isNumber() && s.getNumber<int>() == 0) ||
10391063
(s.isString() && s.stringRef() == "satellite")) {
1040-
return Result{TRI_ERROR_BAD_PARAMETER, "invalid combination of 'isSmart' and 'satellite' replicationFactor"};
1064+
return Result{TRI_ERROR_BAD_PARAMETER,
1065+
"invalid combination of 'isSmart' and 'satellite' "
1066+
"replicationFactor"};
10411067
}
10421068
}
10431069
}

‎arangod/Graph/GraphOperations.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -226,7 +226,7 @@ OperationResult GraphOperations::editEdgeDefinition(VPackSlice edgeDefinitionSli
226226
bool waitForSync,
227227
std::string const& edgeDefinitionName) {
228228
OperationOptions options(ExecContext::current());
229-
auto maybeEdgeDef = EdgeDefinition::createFromVelocypack(edgeDefinitionSlice);
229+
auto maybeEdgeDef = EdgeDefinition::createFromVelocypack(edgeDefinitionSlice, graph().satelliteCollections());
230230
if (!maybeEdgeDef) {
231231
return OperationResult{std::move(maybeEdgeDef).result(), options};
232232
}

0 commit comments

Comments
 (0)
0