8000 [PREG-207] Use struct for pregel options (#17925) · rohan-flutterint/arangodb@16903f3 · GitHub
[go: up one dir, main page]
Skip to content

Commit 16903f3

Browse files
jvolmerjvolmer
authored
[PREG-207] Use struct for pregel options (arangodb#17925)
* Use struct for pregel options * Fix error in js test
1 parent 5a54112 commit 16903f3

File tree

9 files changed

+291
-94
lines changed

9 files changed

+291
-94
lines changed

‎arangod/Pregel/PregelFeature.cpp

Lines changed: 65 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -38,13 +38,15 @@
3838
#include "Cluster/ClusterInfo.h"
3939
#include "Cluster/ServerState.h"
4040
#include "GeneralServer/AuthenticationFeature.h"
41+
#include "Graph/GraphManager.h"
4142
#include "Metrics/CounterBuilder.h"
4243
#include "Metrics/GaugeBuilder.h"
4344
#include "Network/Methods.h"
4445
#include "Network/NetworkFeature.h"
4546
#include "Pregel/AlgoRegistry.h"
4647
#include "Pregel/Conductor.h"
4748
#include "Pregel/ExecutionNumber.h"
49+
#include "Pregel/PregelOptions.h"
4850
#include "Pregel/Utils.h"
4951
#include "Pregel/Worker.h"
5052
#include "RestServer/DatabasePathFeature.h"
@@ -96,25 +98,71 @@ network::Headers buildHeaders() {
9698

9799
} // namespace
98100

99-
ResultT<ExecutionNumber> PregelFeature::startExecution(
100-
TRI_vocbase_t& vocbase, std::string algorithm,
101-
std::vector<std::string> const& vertexCollections,
102-
std::vector<std::string> const& edgeCollections,
103-
std::unordered_map<std::string, std::vector<std::string>> const&
104-
edgeCollectionRestrictions,
105-
VPackSlice const& params) {
101+
ResultT<ExecutionNumber> PregelFeature::startExecution(TRI_vocbase_t& vocbase,
102+
PregelOptions options) {
106103
if (isStopping() || _softShutdownOngoing.load(std::memory_order_relaxed)) {
107104
return Result{TRI_ERROR_SHUTTING_DOWN, "pregel system not available"};
108105
}
109106

107+
// // extract the collections
108+
std::vector<std::string> vertexCollections;
109+
std::vector<std::string> edgeCollections;
110+
std::unordered_map<std::string, std::vector<std::string>>
111+
edgeCollectionRestrictions;
112+
113+
if (std::holds_alternative<GraphCollectionNames>(
114+
options.graphSource.graphOrCollections)) {
115+
auto collectionNames =
116+
std::get<GraphCollectionNames>(options.graphSource.graphOrCollections);
117+
vertexCollections = collectionNames.vertexCollections;
118+
edgeCollections = collectionNames.edgeCollections;
119+
edgeCollectionRestrictions =
120+
options.graphSource.edgeCollectionRestrictions.items;
121+
} else {
122+
auto graphName =
123+
std::get<GraphName>(options.graphSource.graphOrCollections);
124+
if (graphName.graph == "") {
125+
return Result{TRI_ERROR_BAD_PARAMETER, "expecting graphName as string"};
126+
}
127+
128+
graph::GraphManager gmngr{vocbase};
129+
auto graphRes = gmngr.lookupGraphByName(graphName.graph);
130+
if (graphRes.fail()) {
131+
return std::move(graphRes).result();
132+
}
133+
std::unique_ptr<graph::Graph> graph = std::move(graphRes.get());
134+
135+
auto const& gv = graph->vertexCollections();
136+
for (auto const& v : gv) {
137+
vertexCollections.push_back(v);
138+
}
139+
140+
auto const& ge = graph->edgeCollections();
141+
for (auto const& e : ge) {
142+
edgeCollections.push_back(e);
143+
}
144+
145+
auto const& ed = graph->edgeDefinitions();
146+
for (auto const& e : ed) {
147+
auto const& from = e.second.getFrom();
148+
// intentionally create map entry
149+
for (auto const& f : from) {
150+
auto& restrictions = edgeCollectionRestrictions[f];
151+
restrictions.push_back(e.second.getName());
152+
}
153+
}
154+
}
155+
110156
ServerState* ss = ServerState::instance();
111157

112158
// check the access rights to collections
113159
ExecContext const& exec = ExecContext::current();
114160
if (!exec.isSuperuser()) {
115-
TRI_ASSERT(params.isObject());
116-
VPackSlice storeSlice = params.get("store");
161+
// TODO get rid of that when we have a pregel parameter struct
162+
TRI_ASSERT(options.userParameters.slice().isObject());
163+
VPackSlice storeSlice = options.userParameters.slice().get("store");
117164
bool storeResults = !storeSlice.isBool() || storeSlice.getBool();
165+
118166
for (std::string const& vc : vertexCollections) {
119167
bool canWrite = exec.canUseCollection(vc, auth::Level::RW);
120168
bool canRead = exec.canUseCollection(vc, auth::Level::RO);
@@ -176,10 +224,13 @@ ResultT<ExecutionNumber> PregelFeature::startExecution(
176224
if (!coll->isSmart()) {
177225
std::vector<std::string> eKeys = coll->shardKeys();
178226

179-
std::string shardKeyAttribute = "vertex";
180-
if (params.hasKey("shardKeyAttribute")) {
181-
shardKeyAttribute = params.get("shardKeyAttribute").copyString();
182-
}
227+
// TODO get rid of that when we have a pregel parameter struct
228+
std::string shardKeyAttribute =
229+
options.userParameters.slice().hasKey("shardKeyAttribute")
230+
? options.userParameters.slice()
231+
.get("shardKeyAttribute")
232+
.copyString()
233+
: "vertex";
183234

184235
if (eKeys.size() != 1 || eKeys[0] != shardKeyAttribute) {
185236
return Result{
@@ -221,7 +272,7 @@ ResultT<ExecutionNumber> PregelFeature::startExecution(
221272
auto en = createExecutionNumber();
222273
auto c = std::make_shared<pregel::Conductor>(
223274
en, vocbase, vertexCollections, edgeColls, edgeCollectionRestrictions,
224-
algorithm, params, *this);
275+
options.algorithm, options.userParameters.slice(), *this);
225276
addConductor(std::move(c), en);
226277
TRI_ASSERT(conductor(en));
227278
conductor(en)->start();

‎arangod/Pregel/PregelFeature.h

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
#include "Basics/Common.h"
3838
#include "Basics/Mutex.h"
3939
#include "Pregel/ExecutionNumber.h"
40+
#include "Pregel/PregelOptions.h"
4041
#include "ProgramOptions/ProgramOptions.h"
4142
#include "RestServer/arangod.h"
4243
#include "Scheduler/Scheduler.h"
@@ -56,13 +57,8 @@ class PregelFeature final : public ArangodFeature {
5657
explicit PregelFeature(Server& server);
5758
~PregelFeature();
5859

59-
ResultT<ExecutionNumber> startExecution(
60-
TRI_vocbase_t& vocbase, std::string algorithm,
61-
std::vector<std::string> const& vertexCollections,
62-
std::vector<std::string> const& edgeCollections,
63-
std::unordered_map<std::string, std::vector<std::string>> const&
64-
edgeCollectionRestrictions,
65-
VPackSlice const& params);
60+
ResultT<ExecutionNumber> startExecution(TRI_vocbase_t& vocbase,
61+
PregelOptions options);
6662

6763
void collectOptions(std::shared_ptr<arangodb::options::ProgramOptions>
6864
options) override final;

‎arangod/Pregel/PregelOptions.h

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
////////////////////////////////////////////////////////////////////////////////
2+
/// DISCLAIMER
3+
///
4+
/// Copyright 2014-2022 ArangoDB GmbH, Cologne, Germany
5+
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
6+
///
7+
/// Licensed under the Apache License, Version 2.0 (the "License");
8+
/// you may not use this file except in compliance with the License.
9+
/// You may obtain a copy of the License at
10+
///
11+
/// http://www.apache.org/licenses/LICENSE-2.0
12+
///
13+
/// Unless required by applicable law or agreed to in writing, software
14+
/// distributed under the License is distributed on an "AS IS" BASIS,
15+
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
/// See the License for the specific language governing permissions and
17+
/// limitations under the License.
18+
///
19+
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
20+
///
21+
/// @author Julia Volmer
22+
////////////////////////////////////////////////////////////////////////////////
23+
#pragma once
24+
25+
#include <string>
26+
#include <variant>
27+
#include "Cluster/ClusterTypes.h"
28+
#include "Inspection/Types.h"
29+
#include "velocypack/Builder.h"
30+
31+
namespace arangodb::pregel {
32+
33+
using VertexCollectionID = CollectionID;
34+
using EdgeCollectionID = CollectionID;
35+
using VertexShardID = ShardID;
36+
using EdgeShardID = ShardID;
37+
38+
struct GraphCollectionNames {
39+
std::vector<std::string> vertexCollections;
40+
std::vector<std::string> edgeCollections;
41+
};
42+
template<typename Inspector>
43+
auto inspect(Inspector& f, GraphCollectionNames& x) {
44+
return f.object(x).fields(f.field("vertexCollections", x.vertexCollections),
45+
f.field("edgeCollections", x.edgeCollections));
46+
}
47+
48+
struct GraphName {
49+
std::string graph;
50+
};
51+
template<typename Inspector>
52+
auto inspect(Inspector& f, GraphName& x) {
53+
return f.object(x).fields(f.field("graph", x.graph));
54+
}
55+
56+
/**
57+
Maps from vertex collection name to a list of edge collections that this
58+
vertex collection is restricted to.
59+
It is only used for a collection if there is at least one entry for the
60+
collection!
61+
**/
62+
struct EdgeCollectionRestrictions {
63+
std::unordered_map<VertexCollectionID, std::vector<EdgeCollectionID>> items;
64+
auto add(EdgeCollectionRestrictions others) const
65+
-> EdgeCollectionRestrictions;
66+
};
67+
template<typename Inspector>
68+
auto inspect(Inspector& f, EdgeCollectionRestrictions& x) {
69+
return f.object(x).fields(f.field("items", x.items));
70+
}
71+
72+
struct GraphOrCollection : std::variant<GraphCollectionNames, GraphName> {};
73+
template<class Inspector>
74+
auto inspect(Inspector& f, GraphOrCollection& x) {
75+
return f.variant(x).unqualified().alternatives(
76+
arangodb::inspection::type<GraphCollectionNames>("collectionNames"),
77+
arangodb::inspection::type<GraphName>("graphName"));
78+
}
79+
80+
struct GraphSource {
81+
GraphOrCollection graphOrCollections;
82+
EdgeCollectionRestrictions edgeCollectionRestrictions;
83+
};
84+
template<typename Inspector>
85+
auto inspect(Inspector& f, GraphSource& x) {
86+
return f.object(x).fields(
87+
f.field("graphOrCollection", x.graphOrCollections),
88+
f.field("edgeCollectionRestrictions", x.edgeCollectionRestrictions));
89+
}
90+
91+
struct PregelOptions {
92+
std::string algorithm;
93+
VPackBuilder userParameters;
94+
GraphSource graphSource;
95+
};
96+
97+
} // namespace arangodb::pregel

‎arangod/Pregel/REST/CMakeLists.txt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
target_sources(arango_pregel PRIVATE
22
RestControlPregelHandler.cpp
3-
RestPregelHandler.cpp)
3+
RestPregelHandler.cpp
4+
RestOptions.cpp)

‎arangod/Pregel/REST/RestControlPregelHandler.cpp

Lines changed: 9 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -28,15 +28,16 @@
2828
#include "Cluster/ClusterFeature.h"
2929
#include "Cluster/ClusterInfo.h"
3030
#include "Cluster/ServerState.h"
31-
#include "Graph/Graph.h"
32-
#include "Graph/GraphManager.h"
31+
#include "Inspection/VPackWithErrorT.h"
3332
#include "Pregel/Conductor.h"
3433
#include "Pregel/ExecutionNumber.h"
3534
#include "Pregel/PregelFeature.h"
35+
#include "Pregel/REST/RestOptions.h"
3636
#include "Transaction/StandaloneContext.h"
3737

3838
#include <velocypack/Builder.h>
3939
#include <velocypack/Iterator.h>
40+
#include <velocypack/SharedSlice.h>
4041

4142
using namespace arangodb::basics;
4243
using namespace arangodb::rest;
@@ -118,75 +119,15 @@ void RestControlPregelHandler::startExecution() {
118119
return;
119120
}
120121

121-
// algorithm
122-
std::string algorithm =
123-
VelocyPackHelper::getStringValue(body, "algorithm", StaticStrings::Empty);
124-
if ("" == algorithm) {
122+
auto restOptions = inspection::deserializeWithErrorT<pregel::RestOptions>(
123+
velocypack::SharedSlice(velocypack::SharedSlice{}, body));
124+
if (!restOptions.ok()) {
125125
generateError(rest::ResponseCode::NOT_FOUND, TRI_ERROR_HTTP_NOT_FOUND,
126-
"invalid algorithm");
127-
return;
128-
}
129-
130-
// extract the parameters
131-
auto parameters = body.get("params");
132-
if (!parameters.isObject()) {
133-
parameters = VPackSlice::emptyObjectSlice();
134-
}
135-
136-
// extract the collections
137-
std::vector<std::string> vertexCollections;
138-
std::vector<std::string> edgeCollections;
139-
std::unordered_map<std::string, std::vector<std::string>>
140-
edgeCollectionRestrictions;
141-
auto vc = body.get("vertexCollections");
142-
auto ec = body.get("edgeCollections");
143-
if (vc.isArray() && ec.isArray()) {
144-
for (auto v : VPackArrayIterator(vc)) {
145-
vertexCollections.push_back(v.copyString());
146-
}
147-
for (auto e : VPackArrayIterator(ec)) {
148-
edgeCollections.push_back(e.copyString());
149-
}
150-
} else {
151-
auto gs = VelocyPackHelper::getStringValue(body, "graphName", "");
152-
if ("" == gs) {
153-
generateError(rest::ResponseCode::BAD, TRI_ERROR_BAD_PARAMETER,
154-
"expecting graphName as string");
155-
return;
156-
}
157-
158-
graph::GraphManager gmngr{_vocbase};
159-
auto graphRes = gmngr.lookupGraphByName(gs);
160-
if (graphRes.fail()) {
161-
generateError(std::move(graphRes).result());
162-
return;
163-
}
164-
std::unique_ptr<graph::Graph> graph = std::move(graphRes.get());
165-
166-
auto const& gv = graph->vertexCollections();
167-
for (auto const& v : gv) {
168-
vertexCollections.push_back(v);
169-
}
170-
171-
auto const& ge = graph->edgeCollections();
172-
for (auto const& e : ge) {
173-
edgeCollections.push_back(e);
174-
}
175-
176-
auto const& ed = graph->edgeDefinitions();
177-
for (auto const& e : ed) {
178-
auto const& from = e.second.getFrom();
179-
// intentionally create map entry
180-
for (auto const& f : from) {
181-
auto& restrictions = edgeCollectionRestrictions[f];
182-
restrictions.push_back(e.second.getName());
183-
}
184-
}
126+
restOptions.error().error());
185127
}
128+
auto options = std::move(restOptions).get().options();
186129

187-
auto res = _pregel.startExecution(_vocbase, algorithm, vertexCollections,
188-
edgeCollections, edgeCollectionRestrictions,
189-
parameters);
130+
auto res = _pregel.startExecution(_vocbase, options);
190131
if (res.fail()) {
191132
generateError(res.result());
192133
return;

‎arangod/Pregel/REST/RestOptions.cpp

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#include "RestOptions.h"
2+
#include <variant>
3+
4+
using namespace arangodb::pregel;
5+
6+
auto RestOptions::options() -> PregelOptions {
7+
if (std::holds_alternative<pregel::RestGraphSettings>(*this)) {
8+
auto x = std::get<pregel::RestGraphSettings>(*this);
9+
return PregelOptions{
10+
.algorithm = x.options.algorithm,
11+
.userParameters = x.options.userParameters,
12+
.graphSource = {{GraphName{.graph = x.graph}},
13+
{x.options.edgeCollectionRestrictions}}};
14+
}
15+
auto x = std::get<pregel::RestCollectionSettings>(*this);
16+
return PregelOptions{
17+
.algorithm = x.options.algorithm,
18+
.userParameters = x.options.userParameters,
19+
.graphSource = {
20+
{GraphCollectionNames{.vertexCollections = x.vertexCollections,
21+
.edgeCollections = x.edgeCollections}},
22+
{x.options.edgeCollectionRestrictions}}};
23+
}

0 commit comments

Comments
 (0)
0