8000 Merge remote-tracking branch 'origin/devel' into feature/timout-run-i… · arangodb/arangodb@6257f6f · GitHub
[go: up one dir, main page]
Skip to content

Commit 6257f6f

Browse files
ObiWahnObiWahn
committed
Merge remote-tracking branch 'origin/devel' into feature/timout-run-in-local-arangosh
* origin/devel: Move subqueries down (#11135) Update OskarTestSuitesBlackList (#11132) fix shrink method in aqlitemblock (#11122)
2 parents a4f12fe + c9777b4 commit 6257f6f

File tree

4 files changed

+169
-25
lines changed

4 files changed

+169
-25
lines changed

‎UnitTests/OskarTestSuitesBlackList

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
upgrade_data_3.2.*
22
upgrade_data_3.3.*
3+
upgrade_data_3.4.*
4+
upgrade_data_3.5.*
35
audit
46
server_secrets

‎arangod/Aql/AqlItemBlock.cpp

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -347,6 +347,13 @@ void AqlItemBlock::shrink(size_t nrItems) {
347347

348348
decreaseMemoryUsage(sizeof(AqlValue) * (_nrItems - nrItems) * _nrRegs);
349349

350+
// remove the shadow row indices pointing to now invalid rows.
351+
_shadowRowIndexes.erase(_shadowRowIndexes.lower_bound(nrItems),
352+
_shadowRowIndexes.end());
353+
354+
// adjust the size of the block
355+
_nrItems = nrItems;
356+
350357
for (size_t i = numEntries(); i < _data.size(); ++i) {
351358
AqlValue& a = _data[i];
352359
if (a.requiresDestruction()) {
@@ -368,13 +375,6 @@ void AqlItemBlock::shrink(size_t nrItems) {
368375
}
369376
a.erase();
370377
}
371-
372-
// remove the shadow row indices pointing to now invalid rows.
373-
_shadowRowIndexes.erase(_shadowRowIndexes.lower_bound(nrItems),
374-
_shadowRowIndexes.end());
375-
376-
// adjust the size of the block
377-
_nrItems = nrItems;
378378
}
379379

380380
void AqlItemBlock::rescale(size_t nrItems, RegisterId nrRegs) {

‎arangod/Aql/OptimizerRules.cpp

Lines changed: 32 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -75,12 +75,20 @@
7575
namespace {
7676

7777
bool accessesCollectionVariable(arangodb::aql::ExecutionPlan const* plan,
78-
arangodb::aql::CalculationNode const* node,
78+
arangodb::aql::ExecutionNode const* node,
7979
::arangodb::containers::HashSet<arangodb::aql::Variable const*>& vars) {
8080
using EN = arangodb::aql::ExecutionNode;
8181

82-
vars.clear();
83-
arangodb::aql::Ast::getReferencedVariables(node->expression()->node(), vars);
82+
if (node->getType() == EN::CALCULATION) {
83+
auto nn = EN::castTo<arangodb::aql::CalculationNode const*>(node);
84+
vars.clear();
85+
arangodb::aql::Ast::getReferencedVariables(nn->expression()->node(), vars);
86+
} else if (node->getType() == EN::SUBQUERY) {
87+
auto nn = EN::castTo<arangodb::aql::SubqueryNode const*>(node);
88+
vars.clear();
89+
nn->getVariablesUsedHere(vars);
90+
}
91+
8492
for (auto const& it : vars) {
8593
auto setter = plan->getVarSetBy(it->id);
8694
if (setter == nullptr) {
@@ -1686,23 +1694,34 @@ void arangodb::aql::moveCalculationsDownRule(Optimizer* opt,
16861694
OptimizerRule const& rule) {
16871695
::arangodb::containers::SmallVector<ExecutionNode*>::allocator_type::arena_type a;
16881696
::arangodb::containers::SmallVector<ExecutionNode*> nodes{a};
1689-
plan->findNodesOfType(nodes, EN::CALCULATION, true);
1697+
plan->findNodesOfType(nodes, {EN::CALCULATION, EN::SUBQUERY}, true);
16901698

16911699
std::vector<ExecutionNode*> stack;
16921700
::arangodb::containers::HashSet<Variable const*> vars;
16931701
::arangodb::containers::HashSet<Variable const*> usedHere;
16941702
bool modified = false;
16951703

16961704
for (auto const& n : nodes) {
1697-
auto nn = ExecutionNode::castTo<CalculationNode*>(n);
1698-
if (!nn->expression()->isDeterministic()) {
1699-
// we will only move expressions down that cannot throw and that are
1700-
// deterministic
1701-
continue;
1702-
}
1703-
17041705
// this is the variable that the calculation will set
1705-
auto variable = nn->outVariable();
1706+
Variable const* variable = nullptr;
1707+
1708+
if (n->getType() == EN::CALCULATION) {
1709+
auto nn = ExecutionNode::castTo<CalculationNode*>(n);
1710+
if (!nn->expression()->isDeterministic()) {
1711+
// we will only move expressions down that cannot throw and that are
1712+
// deterministic
1713+
continue;
1714+
}
1715+
variable = nn->outVariable();
1716+
} else if (n->getType() == EN::SUBQUERY) {
1717+
auto nn = ExecutionNode::castTo<SubqueryNode*>(n);
1718+
if (!nn->isDeterministic() || nn->isModificationNode()) {
1719+
// we will only move subqueries down that are deterministic and are not
1720+
// modification subqueries
1721+
continue;
1722+
}
1723+
variable = nn->outVariable();
1724+
}
17061725

17071726
stack.clear();
17081727
n->parents(stack);
@@ -1746,7 +1765,7 @@ void arangodb::aql::moveCalculationsDownRule(Optimizer* opt,
17461765
// however, if our calculation uses any data from a
17471766
// collection/index/view, it probably makes sense to not move it,
17481767
// because the result set may be huge
1749-
if (::accessesCollectionVariable(plan.get(), nn, vars)) {
1768+
if (::accessesCollectionVariable(plan.get(), n, vars)) {
17501769
break;
17511770
}
17521771
}

‎tests/js/server/aql/aql-optimizer-rule-move-calculations-down.js

Lines changed: 128 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,12 @@ function optimizerRuleTestSuite () {
8383
"FOR i IN 1..10 LET result = IS_STRING(i) LIMIT 1 RETURN result",
8484
"FOR i IN 1..10 LET result = IS_STRING(i) SORT i RETURN result",
8585
"FOR i IN 1..10 LET result = PUSH(i, i + 1) LET x = (FOR j IN result RETURN j) RETURN x",
86+
"FOR i IN 1..10 LET result = (RETURN i + 1) FILTER i < 5 RETURN result",
87+
"FOR i IN 1..10 LET a = (RETURN i + 1) LET b = (RETURN i + 2) FILTER i < 2 LET c = b FILTER i < 10 RETURN c",
88+
"FOR i IN 1..10 LET result = (RETURN IS_STRING(i)) FILTER i < 2 RETURN result",
89+
"FOR i IN 1..10 LET result = (RETURN IS_STRING(i)) LIMIT 1 RETURN result",
90+
"FOR i IN 1..10 LET result = (RETURN IS_STRING(i)) SORT i RETURN result",
91+
"FOR i IN 1..10 LET result = (RETURN PUSH(i, i + 1)) LET x = (FOR j IN result RETURN j) RETURN x",
8692
];
8793

8894
queries.forEach(function(query) {
@@ -110,7 +116,17 @@ function optimizerRuleTestSuite () {
110116
"FOR i IN 1..10 FILTER i < 10 LET result = IS_STRING(i) FOR j IN 1..2 RETURN j",
111117
"FOR i IN 1..10 LET result = IS_STRING(i) COLLECT r = result RETURN r",
112118
"FOR i IN 1..10 LET result = MAX(i) FILTER result < 3 RETURN result",
113-
"FOR i IN 1..10 LET result = RAND() FILTER i < 10 RETURN result"
119+
"FOR i IN 1..10 LET result = RAND() FILTER i < 10 RETURN result",
120+
"LET a = (RETURN PASSTHRU(39)) FOR i IN 1..10 RETURN a",
121+
"FOR i IN 1..10 LET result = (RETURN i < 5) FILTER result RETURN i",
122+
"FOR i IN 1..10 LET result = (RETURN i + 1) SORT result RETURN i",
123+
"FOR i IN 1..10 LET result = (RETURN i + 1) LET test = (FOR j IN 1..result RETURN j) RETURN test",
124+
"FOR i IN 1..10 LIMIT 2 LET result = (RETURN i < 5) RETURN i",
125+
"FOR i IN 1..10 LET result = (RETURN IS_STRING(i)) FOR j IN 1..2 RETURN j",
126+
"FOR i IN 1..10 FILTER i < 10 LET result = (RETURN IS_STRING(i)) FOR j IN 1..2 RETURN j",
127+
"FOR i IN 1..10 LET result = (RETURN IS_STRING(i)) COLLECT r = result RETURN r",
128+
"FOR i IN 1..10 LET result = (RETURN MAX(i)) FILTER result < 3 RETURN result",
129+
"FOR i IN 1..10 LET result = (RETURN RAND()) FILTER i < 10 RETURN result"
114130
];
115131

116132
queries.forEach(function(query) {
@@ -132,7 +148,15 @@ function optimizerRuleTestSuite () {
132148
"FOR i IN 1..10 LET result = IS_STRING(i) LET test = result + 1 LIMIT 1 RETURN test",
133149
"FOR i IN 1..10 LET result = IS_STRING(i) SORT i RETURN result",
134150
"FOR i IN 1..10 LET result = i + 1 LET test = (FOR j IN 1..2 RETURN j) RETURN result IN test",
135-
"FOR i IN 1..10 LET v = i * 2 LIMIT 2 LET result = v < 5 RETURN v"
151+
"FOR i IN 1..10 LET v = i * 2 LIMIT 2 LET result = v < 5 RETURN v",
152+
"FOR i IN 1..10 LET result = (RETURN IS_STRING(i)) FILTER i < 10 RETURN result",
153+
"FOR i IN 1..10 LET result = (RETURN IS_STRING(i)) FILTER i < 10 FILTER i > 2 RETURN result",
154+
"FOR i IN 1..10 LET result = (RETURN IS_STRING(i)) FILTER i < 2 LIMIT 1 RETURN result",
155+
"FOR i IN 1..10 LET result = (RETURN IS_STRING(i)) LIMIT 1 RETURN result",
156+
"FOR i IN 1..10 LET result = IS_STRING(i) LET test = (RETURN result + 1) LIMIT 1 RETURN test",
157+
"FOR i IN 1..10 LET result = (RETURN IS_STRING(i)) SORT i RETURN result",
158+
"FOR i IN 1..10 LET result = (RETURN i + 1) LET test = (FOR j IN 1..2 RETURN j) RETURN result IN test",
159+
"FOR i IN 1..10 LET v = (RETURN i * 2) LIMIT 2 LET result = (RETURN v < 5) RETURN v"
136160
];
137161

138162
queries.forEach(function(query) {
@@ -153,7 +177,14 @@ function optimizerRuleTestSuite () {
153177
[ "FOR i IN 1..10 LET result = IS_STRING(i) LIMIT 1 RETURN result", [ "SingletonNode","CalculationNode","EnumerateListNode","LimitNode","CalculationNode","ReturnNode" ] ],
154178
[ "FOR i IN 1..10 LET result = IS_STRING(i) LET test = result + 1 LIMIT 1 RETURN test", [ "SingletonNode","CalculationNode","EnumerateListNode","LimitNode","CalculationNode","CalculationNode","ReturnNode" ] ] ,
155179
[ "FOR i IN 1..10 LET result = IS_STRING(i) SORT i RETURN result", [ "SingletonNode","CalculationNode","EnumerateListNode","SortNode","CalculationNode","ReturnNode" ] ],
156-
[ "FOR i IN 1..10 LET v = i * 2 LIMIT 2 LET result = v < 5 RETURN v", [ "SingletonNode","CalculationNode","EnumerateListNode","LimitNode","CalculationNode","CalculationNode","ReturnNode" ] ]
180+
[ "FOR i IN 1..10 LET v = i * 2 LIMIT 2 LET result = v < 5 RETURN v", [ "SingletonNode","CalculationNode","EnumerateListNode","LimitNode","CalculationNode","CalculationNode","ReturnNode" ] ],
181+
[ "FOR i IN 1..10 LET result = (RETURN IS_STRING(i)) FILTER i < 10 RETURN result", [ "SingletonNode", "CalculationNode", "EnumerateListNode", "CalculationNode", "FilterNode", "SubqueryNode", "ReturnNode" ] ],
182+
[ "FOR i IN 1..10 LET result = (RETURN IS_STRING(i)) FILTER i < 10 FILTER i > 2 RETURN result", [ "SingletonNode","CalculationNode","EnumerateListNode","CalculationNode","FilterNode","CalculationNode","FilterNode","SubqueryNode","ReturnNode" ] ],
183+
[ "FOR i IN 1..10 LET result = (RETURN IS_STRING(i)) FILTER i < 2 LIMIT 1 RETURN result", [ "SingletonNode","CalculationNode","EnumerateListNode","CalculationNode","FilterNode","LimitNode","SubqueryNode","ReturnNode" ] ],
184+
[ "FOR i IN 1..10 LET result = (RETURN IS_STRING(i)) LIMIT 1 RETURN result", [ "SingletonNode","CalculationNode","EnumerateListNode","LimitNode","SubqueryNode","ReturnNode" ] ],
185+
[ "FOR i IN 1..10 LET result = IS_STRING(i) LET test = (RETURN result + 1) LIMIT 1 RETURN test", [ "SingletonNode","CalculationNode","EnumerateListNode","LimitNode","CalculationNode","SubqueryNode","ReturnNode" ] ] ,
186+
[ "FOR i IN 1..10 LET result = (RETURN IS_STRING(i)) SORT i RETURN result", [ "SingletonNode","CalculationNode","EnumerateListNode","SortNode","SubqueryNode","ReturnNode" ] ],
187+
[ "FOR i IN 1..10 LET v = (RETURN i * 2) LIMIT 2 LET result = (RETURN v < 5) RETURN v", [ "SingletonNode","CalculationNode","EnumerateListNode","LimitNode","SubqueryNode","SubqueryNode","ReturnNode" ] ]
157188
];
158189

159190
plans.forEach(function(plan) {
@@ -182,7 +213,21 @@ function optimizerRuleTestSuite () {
182213
[ "FOR i IN 1..10 LET a = SUM(1..i) LET b = MIN(1..i) FILTER i == 3 RETURN [ a, b ]", [ [ 6, 1 ] ] ],
183214
[ "FOR i IN 1..10 LET a = SUM(1..i) LIMIT 3 RETURN a", [ 1, 3, 6 ] ],
184215
[ "FOR i IN 1..10 LET a = SUM(1..i) LIMIT 3 FILTER a > 2 RETURN a", [ 3, 6 ] ],
185-
[ "FOR i IN 1..10 LET a = SUM(1..i) LIMIT 5 FILTER i > 2 RETURN a", [ 6, 10, 15 ] ]
216+
[ "FOR i IN 1..10 LET a = SUM(1..i) LIMIT 5 FILTER i > 2 RETURN a", [ 6, 10, 15 ] ],
217+
[ "FOR i IN 1..10 LET a = (RETURN i + 1) FILTER i < 4 RETURN a[0]", [ 2, 3, 4 ] ],
218+
[ "FOR i IN 1..10 LET a = (RETURN i + 1) FILTER i < 7 FILTER i > 1 RETURN a[0]", [ 3, 4, 5, 6, 7 ] ],
219+
[ "FOR i IN 1..10 LET a = (RETURN i + 1) LIMIT 4 RETURN a[0]", [ 2, 3, 4, 5 ] ],
220+
[ "FOR i IN 1..10 LET a = (RETURN i + 1) LET b = (RETURN a[0] + 1) FILTER i < 3 RETURN b[0]", [ 3, 4 ] ],
221+
[ "FOR i IN 1..10 LET a = i + 1 LET b = (RETURN a + 1) LIMIT 4 RETURN b[0]", [ 3, 4, 5, 6 ] ],
222+
[ "FOR i IN 1..10 LET a = (RETURN i + 1) LET b = (RETURN a[0] + 1) FILTER i < 5 LIMIT 4 RETURN b[0]", [ 3, 4, 5, 6 ] ],
223+
[ "FOR i IN 1..10 LET a = (RETURN i + 1) LET x = (FOR j IN 1..i RETURN j) RETURN a[0] - 1 IN x ? 1 : 0", [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ] ],
224+
[ "FOR i IN 1..10 LET a = (RETURN i + 1) LET x = (FOR j IN 1..i RETURN j) RETURN 10000 a[0] IN x ? 1 : 0", [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ] ],
225+
[ "FOR i IN 1..10 LET a = (RETURN MAX(1..i)) FILTER i > 4 RETURN a[0]", [ 5, 6, 7, 8, 9, 10 ] ],
226+
[ "FOR i IN 1..10 LET a = (RETURN SUM(1..i)) FILTER i == 3 RETURN a[0]", [ 6 ] ],
227+
[ "FOR i IN 1..10 LET a = (RETURN SUM(1..i)) LET b = (RETURN MIN(1..i)) FILTER i == 3 RETURN [ a[0], b[0] ]", [ [ 6, 1 ] ] ],
228+
[ "FOR i IN 1..10 LET a = (RETURN SUM(1..i)) LIMIT 3 RETURN a[0]", [ 1, 3, 6 ] ],
229+
[ "FOR i IN 1..10 LET a = (RETURN SUM(1..i)) LIMIT 3 FILTER a[0] > 2 RETURN a[0]", [ 3, 6 ] ],
230+
[ "FOR i IN 1..10 LET a = (RETURN SUM(1..i)) LIMIT 5 FILTER i > 2 RETURN a[0]", [ 6, 10, 15 ] ]
186231
];
187232

188233
queries.forEach(function(query) {
@@ -260,7 +305,85 @@ function optimizerRuleTestSuite () {
260305

261306
assertEqual(resultDisabled.json, expected, query);
262307
assertEqual(resultEnabled.json, expected, query);
263-
}
308+
},
309+
310+
testCollection4 : function () {
311+
var expected = [ ];
312+
for (var i = 0; i < 100; ++i) {
313+
expected.push("test" + i + "-" + i);
314+
}
315+
316+
var query = "FOR i IN " + cn + " LET result = (RETURN CONCAT(i._key, '-', i.value)) SORT i.value RETURN result[0]";
317+
var planDisabled = AQL_EXPLAIN(query, { }, paramDisabled);
318+
var planEnabled = AQL_EXPLAIN(query, { }, paramEnabled);
319+
320+
var resultDisabled = AQL_EXECUTE(query, { }, paramDisabled);
321+
var resultEnabled = AQL_EXECUTE(query, { }, paramEnabled);
322+
323+
assertEqual(-1, planDisabled.plan.rules.indexOf(ruleName), query[0]);
324+
assertNotEqual(-1, planEnabled.plan.rules.indexOf(ruleName), query[0]);
325+
326+
assertEqual(resultDisabled.json, expected, query[0]);
327+
assertEqual(resultEnabled.json, expected, query[0]);
328+
},
329+
330+
////////////////////////////////////////////////////////////////////////////////
331+
/// @brief test results
332+
////////////////////////////////////////////////////////////////////////////////
333+
334+
testCollection5 : function () {
335+
var expected = [ "test43-43", "test44-44" ];
336+
337+
var query = "FOR i IN " + cn + " LET result = (RETURN CONCAT(i._key, '-', i.value)) FILTER i.value > 42 SORT i.value LIMIT 2 RETURN result[0]";
338+
var planDisabled = AQL_EXPLAIN(query, { }, paramDisabled);
339+
var planEnabled = AQL_EXPLAIN(query, { }, paramEnabled);
340+
341+
var resultDisabled = AQL_EXECUTE(query, { }, paramDisabled);
342+
var resultEnabled = AQL_EXECUTE(query, { }, paramEnabled);
343+
344+
assertEqual(-1, planDisabled.plan.rules.indexOf(ruleName), query);
345+
assertNotEqual(-1, planEnabled.plan.rules.indexOf(ruleName), query);
346+
347+
assertEqual(resultDisabled.json, expected, query);
348+
assertEqual(resultEnabled.json, expected, query);
349+
},
350+
351+
testCollection6 : function () {
352+
var expected = [ "test0-0", "test1-1" ];
353+
354+
var query = "FOR i IN " + cn + " LET result = (RETURN CONCAT(i._key, '-', i.value)) SORT i.value LIMIT 2 RETURN result[0]";
355+
var planDisabled = AQL_EXPLAIN(query, { }, paramDisabled);
356+
var planEnabled = AQL_EXPLAIN(query, { }, paramEnabled);
357+
358+
var resultDisabled = AQL_EXECUTE(query, { }, paramDisabled);
359+
var resultEnabled = AQL_EXECUTE(query, { }, paramEnabled);
360+
361+
assertEqual(-1, planDisabled.plan.rules.indexOf(ruleName), query);
362+
assertNotEqual(-1, planEnabled.plan.rules.indexOf(ruleName), query);
363+
364+
assertEqual(resultDisabled.json, expected, query);
365+
assertEqual(resultEnabled.json, expected, query);
366+
},
367+
368+
testModify: function () {
369+
var expected = [];
370+
for (var i = 0; i < 10; ++i) {
371+
expected.push("test" + i + "-" + i);
372+
}
373+
374+
var query = "FOR i IN 0..100 LET result = (UPDATE {_key: CONCAT('test', TO_STRING(i))} WITH {updated: true} IN " + cn + " RETURN CONCAT(NEW._key, '-', NEW.value)) LIMIT 10 RETURN result[0]";
375+
var planDisabled = AQL_EXPLAIN(query, {}, paramDisabled);
376+
var planEnabled = AQL_EXPLAIN(query, {}, paramEnabled);
377+
378+
var resultDisabled = AQL_EXECUTE(query, {}, paramDisabled);
379+
var resultEnabled = AQL_EXECUTE(query, {}, paramEnabled);
380+
381+
assertEqual(-1, planDisabled.plan.rules.indexOf(ruleName), query[0]);
382+
assertEqual(-1, planEnabled.plan.rules.indexOf(ruleName), query[0]);
383+
384+
assertEqual(resultDisabled.json, expected, query[0]);
385+
assertEqual(resultEnabled.json, expected, query[0]);
386+
},
264387

265388
};
266389
}

0 commit comments

Comments
 (0)
0