8000 mop is fixing the cluster load · lethalbrains/arangodb@d247e7a · GitHub
[go: up one dir, main page]
10000
Skip to content

Commit d247e7a

Browse files
kvahedkvahed
committed
mop is fixing the cluster load
2 parents 4c27442 + f00e419 commit d247e7a

37 files changed

+607
-410
lines changed

‎3rdParty/velocypack/include/velocypack/Parser.h

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,14 @@ class Parser {
132132
parser.parse(json);
133133
return parser.steal();
134134
}
135+
136+
static std::shared_ptr<Builder> fromJson(
137+
char const* start, size_t size,
138+
Options const* options = &Options::Defaults) {
139+
Parser parser(options);
140+
parser.parse(start, size);
141+
return parser.steal();
142+
}
135143

136144
static std::shared_ptr<Builder> fromJson(
137145
uint8_t const* start, size_t size,

‎CHANGELOG

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,12 @@
11
v3.0.0 (XXXX-XX-XX)
22
-------------------
33

4+
* renamed AQL optimizer rule "merge-traversal-filter" to "optimize-traversals"
5+
Additionally, the optimizer rule will remove unused edge and paths result variables
6+
from the traversal in case they are specified in the `FOR` section of the traversal,
7+
but are not referenced later in the query. This saves constructing edges and paths
8+
results.
9+
410
* added new endpoint "srv://" for DNS service records
511

612
* The result order of the AQL functions VALUES and KEYS has never been guaranteed

‎arangod/Agency/Store.cpp

Lines changed: 12 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -145,14 +145,6 @@ Node& Node::operator= (VPackSlice const& slice) {
145145
_children.clear();
146146
_value.reset();
147147
_value.append(reinterpret_cast<char const*>(slice.begin()), slice.byteSize());
148-
/*
149-
notifyObservers(uri());
150-
Node *par = _parent;
151-
while (par != 0) {
152 10000 -
_parent->notifyObservers(uri());
153-
par = par->_parent;
154-
}
155-
*/
156148
return *this;
157149
}
158150

@@ -167,14 +159,6 @@ Node& Node::operator= (Node const& rhs) {
167159
_node_name = rhs._node_name;
168160
_value = rhs._value;
169161
_children = rhs._children;
170-
/*
171-
notifyObservers(uri());
172-
Node *par = _parent;
173-
while (par != 0) {
174-
_parent->notifyObservers(uri());
175-
par = par->_parent;
176-
}
177-
*/
178162
return *this;
179163
}
180164

@@ -208,7 +192,7 @@ NodeType Node::type() const {
208192
// lh-value at path vector
209193
Node& Node::operator ()(std::vector<std::string> const& pv) {
210194
if (pv.size()) {
211-
std::string const key = pv.at(0);
195+
std::string const& key = pv.at(0);
212196
if (_children.find(key) == _children.end()) {
213197
_children[key] = std::make_shared<Node>(key, this);
214198
}
@@ -223,7 +207,7 @@ Node& Node::operator ()(std::vector<std::string> const& pv) {
223207
// rh-value at path vector
224208
Node const& Node::operator ()(std::vector<std::string> const& pv) const {
225209
if (pv.size()) {
226-
std::string const key = pv.at(0);
210+
std::string const& key = pv.at(0);
227211
if (_children.find(key) == _children.end()) {
228212
throw StoreException(
229213
std::string("Node ") + key + std::string(" not found"));
@@ -300,61 +284,6 @@ bool Node::removeTimeToLive () {
300284
return true;
301285
}
302286

303-
// Add observing url for this node
304-
/*bool Node::addObserver (std::string const& uri) {
305-
auto it = std::find(_observers.begin(), _observers.end(), uri);
306-
if (it==_observers.end()) {
307-
_observers.emplace(uri);
308-
return true;
309-
}
310-
return false;
311-
}*/
312-
313-
/*void Node::notifyObservers (std::string const& origin) const {
314-
315-
for (auto const& i : _observers) {
316-
317-
Builder body;
318-
body.openObject();
319-
body.add(uri(), VPackValue(VPackValueType::Object));
320-
body.add("op",VPackValue("modified"));
321-
body.close();
322-
body.close();
323-
324-
std::stringstream endpoint;
325-
std::string path = "/";
326-
size_t pos = 7;
327-
if (i.find("http://")==0) {
328-
endpoint << "tcp://";
329-
} else if (i.find("https://")==0) {
330-
endpoint << "ssl://";
331-
++pos;
332-
} else {
333-
LOG_TOPIC(WARN,Logger::AGENCY) << "Malformed notification URL " << i;
334-
return;
335-
}
336-
337-
size_t slash_p = i.find("/",pos);
338-
if ((slash_p==std::string::npos)) {
339-
endpoint << i.substr(pos);
340-
} else {
341-
endpoint << i.substr(pos,slash_p-pos);
342-
path = i.substr(slash_p);
343-
}
344-
345-
std::unique_ptr<std::map<std::string, std::string>> headerFields =
346-
std::make_unique<std::map<std::string, std::string> >();
347-
348-
ClusterCommResult res =
349-
arangodb::ClusterComm::instance()->asyncRequest(
350-
"1", 1, endpoint.str(), GeneralRequest::RequestType::POST, path,
351-
std::make_shared<std::string>(body.toString()), headerFields, nullptr,
352-
0.0, true);
353-
354-
}
355-
356-
}*/
357-
358287
inline bool Node::observedBy (std::string const& url) const {
359288
auto ret = root()._observer_table.equal_range(url);
360289
for (auto it = ret.first; it!=ret.second; ++it) {
@@ -504,10 +433,9 @@ template<> bool Node::handle<OBSERVE> (VPackSlice const& slice) {
504433
if (!observedBy(url)) {
505434
root()._observer_table.emplace(std::pair<std::string,std::string>(url,uri));
506435
root()._observed_table.emplace(std::pair<std::string,std::string>(uri,url));
507-
// _observers.emplace(url);
508436
return true;
509437
}
510-
438+
511439
return false;
512440

513441
}
@@ -520,7 +448,9 @@ template<> bool Node::handle<UNOBSERVE> (VPackSlice const& slice) {
520448
return false;
521449
std::string url (slice.get("url").copyString()),
522450
uri (this->uri());
523-
451+
452+
// delete in both cases a single entry (ensured above)
453+
// breaking the iterators is fine then
524454
auto ret = root()._observer_table.equal_range(url);
525455
for (auto it = ret.first; it!=ret.second; ++it) {
526456
if (it->second == uri) {
@@ -750,8 +680,10 @@ std::vector<bool> Store::apply (
750680

751681
for (auto it = ret.first; it!=ret.second; ++it) {
752682
body.add(it->second->key,VPackValue(VPackValueType::Object));
683+
body.add(it->second->modified,VPackValue(VPackValueType::Object));
753684
body.add("op",VPackValue(it->second->oper));
754685
body.close();
686+
body.close();
755687
}
756688

757689
body.close();
@@ -934,14 +866,16 @@ void Store::dumpToBuilder (Builder& builder) const {
934866
}
935867
}
936868
{
937-
VPackObjectBuilder guard(&builder);
869+
VPackArrayBuilder garray(&builder);
938870
for (auto const& i : _observer_table) {
871+
VPackObjectBuilder guard(&builder);
939872
builder.add(i.first, VPackValue(i.second));
940873
}
941874
}
942875
{
943-
VPackObjectBuilder guard(&builder);
876+
VPackArrayBuilder garray(&builder);
944877
for (auto const& i : _observed_table) {
878+
VPackObjectBuilder guard(&builder);
945879
builder.add(i.first, VPackValue(i.second));
946880
}
947881
}

‎arangod/Agency/Store.h

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -174,8 +174,6 @@ class Node {
174174
TimePoint _ttl; /**< @brief my expiry */
175175
Buffer<uint8_t> _value; /**< @brief my value */
176176

177-
// std::unordered_set<std::string> _observers; /**< @brief my observers */
178-
179177
/// @brief Table of expiries in tree (only used in root node)
180178
std::multimap<TimePoint, std::shared_ptr<Node>> _time_table;
181179

‎arangod/Aql/AstNode.h

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -383,24 +383,26 @@ struct AstNode {
383383

384384
/// @brief whether or not a value node is of type attribute access that
385385
/// refers to a variable reference
386-
AstNode const* getAttributeAccessForVariable() const {
387-
if (type != NODE_TYPE_ATTRIBUTE_ACCESS && type != NODE_TYPE_EXPANSION) {
386+
AstNode const* getAttributeAccessForVariable(bool allowIndexedAccess) const {
387+
if (type != NODE_TYPE_ATTRIBUTE_ACCESS && type != NODE_TYPE_EXPANSION
388+
&& !(allowIndexedAccess && type == NODE_TYPE_INDEXED_ACCESS)) {
388389
return nullptr;
389390
}
390391

391392
auto node = this;
392393

393394
while (node->type == NODE_TYPE_ATTRIBUTE_ACCESS ||
395+
(allowIndexedAccess && node->type == NODE_TYPE_INDEXED_ACCESS) ||
394396
node->type == NODE_TYPE_EXPANSION) {
395-
if (node->type == NODE_TYPE_ATTRIBUTE_ACCESS) {
397+
if (node->type == NODE_TYPE_ATTRIBUTE_ACCESS || node->type == NODE_TYPE_INDEXED_ACCESS) {
396398
node = node->getMember(0);
397399
} else {
398400
// expansion, i.e. [*]
399401
TRI_ASSERT(node->type == NODE_TYPE_EXPANSION);
400402
TRI_ASSERT(node->numMembers() >= 2);
401403

402404
if (node->getMember(1)->type != NODE_TYPE_REFERENCE) {
403-
if (node->getMember(1)->getAttributeAccessForVariable() == nullptr) {
405+
if (node->getMember(1)->getAttributeAccessForVariable(allowIndexedAccess) == nullptr) {
404406
return nullptr;
405407
}
406408
}
@@ -421,13 +423,14 @@ struct AstNode {
421423
/// @brief whether or not a value node is of type attribute access that
422424
/// refers to any variable reference
423425
bool isAttributeAccessForVariable() const {
424-
return (getAttributeAccessForVariable() != nullptr);
426+
return (getAttributeAccessForVariable(false) != nullptr);
425427
}
426428

427429
/// @brief whether or not a value node is of type attribute access that
428430
/// refers to the specified variable reference
429-
bool isAttributeAccessForVariable(Variable const* variable) const {
430-
auto node = getAttributeAccessForVariable();
431+
bool isAttributeAccessForVariable(Variable const* variable, bool allowIndexedAccess) const {
432+
auto node = getAttributeAccessForVariable(allowIndexedAccess);
433+
431434

432435
if (node == nullptr) {
433436
return false;

‎arangod/Aql/IndexBlock.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,7 @@ int IndexBlock::initialize() {
179179
auto lhs = leaf->getMember(0);
180180
auto rhs = leaf->getMember(1);
181181

182-
if (lhs->isAttributeAccessForVariable(outVariable)) {
182+
if (lhs->isAttributeAccessForVariable(outVariable, false)) {
183183
// Index is responsible for the left side, check if right side has to be
184184
// evaluated
185185
if (!rhs->isConstant()) {

‎arangod/Aql/Optimizer.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -410,8 +410,8 @@ void Optimizer::setupRules() {
410410
true);
411411

412412
// merge filters into traversals
413-
registerRule("merge-traversal-filter", mergeFilterIntoTraversalRule,
414-
mergeFilterIntoTraversalRule_pass6, true);
413+
registerRule("optimize-traversals", optimizeTraversalsRule,
414+
optimizeTraversalsRule_pass6, true);
415415

416416
/// "Pass 5": try to remove redundant or unnecessary nodes (second try)
417417
// remove filters from the query that are not necessary at all

‎arangod/Aql/Optimizer.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,7 @@ class Optimizer {
162162
removeUnnecessaryCalculationsRule_pass6 = 870,
163163

164164
// merge filters into graph traversals
165-
mergeFilterIntoTraversalRule_pass6 = 880,
165+
optimizeTraversalsRule_pass6 = 880,
166166

167167
/// Pass 9: push down calculations beyond FILTERs and LIMITs
168168
moveCalculationsDownRule_pass9 = 900,

‎arangod/Aql/OptimizerRules.cpp

Lines changed: 40 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
#include "Aql/SortCondition.h"
3636
#include "Aql/SortNode.h"
3737
#include "Aql/TraversalConditionFinder.h"
38+
#include "Aql/TraversalNode.h"
3839
#include "Aql/Variable.h"
3940
#include "Aql/types.h"
4041
#include "Basics/AttributeNameParser.h"
@@ -1544,7 +1545,8 @@ void arangodb::aql::removeUnnecessaryCalculationsRule(
15441545
// If this node can throw, we must not optimize it away!
15451546
continue;
15461547
}
1547-
} else {
1548+
// will remove calculation when we get here
1549+
} else if (n->getType() == EN::SUBQUERY) {
15481550
auto nn = static_cast<SubqueryNode*>(n);
15491551

15501552
if (nn->canThrow()) {
@@ -1556,7 +1558,8 @@ void arangodb::aql::removeUnnecessaryCalculationsRule(
15561558
// subqueries that modify data must not be optimized away
15571559
continue;
15581560
}
1559-
}
1561+
// will remove calculation when we get here
1562+
}
15601563

15611564
auto outvar = n->getVariablesSetHere();
15621565
TRI_ASSERT(outvar.size() == 1);
@@ -3465,22 +3468,52 @@ void arangodb::aql::patchUpdateStatementsRule(Optimizer* opt,
34653468
opt->addPlan(plan, rule, modified);
34663469
}
34673470

3468-
/// @brief merges filter nodes into graph traversal nodes
3469-
void arangodb::aql::mergeFilterIntoTraversalRule(Optimizer* opt,
3470-
ExecutionPlan* plan,
3471-
Optimizer::Rule const* rule) {
3471+
/// @brief optimizes away unused traversal output variables and
3472+
/// merges filter nodes into graph traversal nodes
3473+
void arangodb::aql::optimizeTraversalsRule(Optimizer* opt,
3474+
ExecutionPlan* plan,
3475+
Optimizer::Rule const* rule) {
34723476
std::vector<ExecutionNode*> tNodes(
34733477
plan->findNodesOfType(EN::TRAVERSAL, true));
34743478

34753479
if (tNodes.empty()) {
3480+
// no traversals present
34763481
opt->addPlan(plan, rule, false);
34773482
return;
34783483
}
3484+
3485+
bool modified = false;
3486+
3487+
// first make a pass over all traversal nodes and remove unused
3488+
// variables from them
3489+
for (auto const& n : tNodes) {
3490+
TraversalNode* traversal = static_cast<TraversalNode*>(n);
3491+
3492+
auto varsUsedLater = n->getVarsUsedLater();
3493+
3494+
// note that we can NOT optimize away the vertex output variable
3495+
// yet, as many traversal internals depend on the number of vertices
3496+
// found/built
3497+
auto outVariable = traversal->edgeOutVariable();
3498+
if (outVariable != nullptr &&
3499+
varsUsedLater.find(outVariable) == varsUsedLater.end()) {
3500+
// traversal vertex outVariable not used later
3501+
traversal->setEdgeOutput(nullptr);
3502+
modified = true;
3503+
}
3504+
3505+
outVariable = traversal->pathOutVariable();
3506+
if (outVariable != nullptr &&
3507+
varsUsedLater.find(outVariable) == varsUsedLater.end()) {
3508+
// traversal vertex outVariable not used later
3509+
traversal->setPathOutput(nullptr);
3510+
modified = true;
3511+
}
3512+
}
34793513

34803514
// These are all the end nodes where we start
34813515
std::vector<ExecutionNode*> nodes(plan->findEndNodes(true));
34823516

3483-
bool modified = false;
34843517
for (auto const& n : nodes) {
34853518
TraversalConditionFinder finder(plan, &modified);
34863519
n->walk(&finder);

‎arangod/Aql/OptimizerRules.h

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -185,9 +185,10 @@ void removeDataModificationOutVariablesRule(Optimizer*, ExecutionPlan*,
185185
void patchUpdateStatementsRule(Optimizer*, ExecutionPlan*,
186186
Optimizer::Rule const*);
187187

188-
/// @brief merges filter nodes into graph traversal nodes
189-
void mergeFilterIntoTraversalRule(Optimizer* opt, ExecutionPlan* plan,
190-
Optimizer::Rule const* rule);
188+
/// @brief optimizes away unused traversal output variables and
189+
/// merges filter nodes into graph traversal nodes
190+
void optimizeTraversalsRule(Optimizer* opt, ExecutionPlan* plan,
191+
Optimizer::Rule const* rule);
191192

192193
/// @brief moves simple subqueries one level higher
193194
void inlineSubqueriesRule(Optimizer*, ExecutionPlan*, Optimizer::Rule const*);

0 commit comments

Comments
 (0)
0