8000 improve query plan serialization time (#17897) · rohan-flutterint/arangodb@114b453 · GitHub
[go: up one dir, main page]
Skip to content

Commit 114b453

Browse files
jsteemannjsteemann
authored
improve query plan serialization time (arangodb#17897)
1 parent f0031e9 commit 114b453

17 files changed

+143
-128
lines changed

‎CHANGELOG

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,22 @@
11
devel
22
-----
33

4+
* Added new stage "instantiating executors" to the query profiling output.
5+
The time spent in "instantiating executors" is the time needed to create
6+
the query executors from the final query execution time. In cluster mode,
7+
this stage also includes the time needed for physically distributing the
8+
query snippets to the participating database servers.
9+
Previously, the time spent for instantiating executors and the physical
10+
distribution was contained in the "optimizing plan" stage, which was
11+
misleading.
12+
13+
* Remove constant values for query variables from query plan serialization
14+
in cases they were not needed. Previously, constant values of query variables
15+
were always serialized for all occurrences of a variable in a query plan.
16+
If the constant values were large, this contributed to higher serialization
17+
and thus query setup times. Now the constant values are only serialized
18+
for relevant parts of query execution plans.
19+
420
* BTS-199: remove check for environment variable `GLIBCXX_FORCE_NEW` from
521
server start, and remove setting this variable from startup scripts.
622
The reason is that the environment variable only controls the behavior of

‎arangod/Aql/Ast.cpp

Lines changed: 7 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1346,10 +1346,7 @@ AstNode* Ast::createNodeValueInt(int64_t value) {
13461346
AstNode* node = createNode(NODE_TYPE_VALUE);
13471347
node->setValueType(VALUE_TYPE_INT);
13481348
node->setIntValue(value);
1349-
node->setFlag(DETERMINED_CONSTANT, VALUE_CONSTANT);
1350-
node->setFlag(DETERMINED_SIMPLE, VALUE_SIMPLE);
1351-
node->setFlag(DETERMINED_RUNONDBSERVER, VALUE_RUNONDBSERVER);
1352-
1349+
node->setConstantFlags();
13531350
return node;
13541351
}
13551352

@@ -1367,10 +1364,7 @@ AstNode* Ast::createNodeValueDouble(double value) {
13671364
AstNode* node = createNode(NODE_TYPE_VALUE);
13681365
node->setValueType(VALUE_TYPE_DOUBLE);
13691366
node->setDoubleValue(value);
1370-
node->setFlag(DETERMINED_CONSTANT, VALUE_CONSTANT);
1371-
node->setFlag(DETERMINED_SIMPLE, VALUE_SIMPLE);
1372-
node->setFlag(DETERMINED_RUNONDBSERVER, VALUE_RUNONDBSERVER);
1373-
1367+
node->setConstantFlags();
13741368
return node;
13751369
}
13761370

@@ -1403,10 +1397,7 @@ AstNode* Ast::createNodeValueString(char const* value, size_t length) {
14031397
AstNode* node = createNode(NODE_TYPE_VALUE);
14041398
node->setValueType(VALUE_TYPE_STRING);
14051399
node->setStringValue(value, length);
1406-
node->setFlag(DETERMINED_CONSTANT, VALUE_CONSTANT);
1407-
node->setFlag(DETERMINED_SIMPLE, VALUE_SIMPLE);
1408-
node->setFlag(DETERMINED_RUNONDBSERVER, VALUE_RUNONDBSERVER);
1409-
1400+
node->setConstantFlags();
14101401
return node;
14111402
}
14121403

@@ -3966,7 +3957,7 @@ AstNode* Ast::optimizeObject(AstNode* node) {
39663957
/// sure then that string values are valid through the query lifetime.
39673958
AstNode* Ast::nodeFromVPack(VPackSlice slice, bool copyStringValues) {
39683959
if (slice.isBoolean()) {
3969-
return createNodeValueBool(slice.getBoolean());
3960+
return createNodeValueBool(slice.isTrue());
39703961
}
39713962

39723963
if (slice.isNumber()) {
@@ -4008,16 +3999,12 @@ AstNode* Ast::nodeFromVPack(VPackSlice slice, bool copyStringValues) {
40083999
it.next();
40094000
}
40104001

4011-
node->setFlag(DETERMINED_CONSTANT, VALUE_CONSTANT);
4012-
node->setFlag(DETERMINED_SIMPLE, VALUE_SIMPLE);
4013-
node->setFlag(DETERMINED_RUNONDBSERVER, VALUE_RUNONDBSERVER);
4014-
4002+
node->setConstantFlags();
40154003
return node;
40164004
}
40174005

40184006
if (slice.isObject()) {
40194007
VPackObjectIterator it(slice, true);
4020-
40214008
auto node = createNodeObject();
40224009
node->members.reserve(static_cast<size_t>(it.size()));
40234010

@@ -4037,10 +4024,7 @@ AstNode* Ast::nodeFromVPack(VPackSlice slice, bool copyStringValues) {
40374024
it.next();
40384025
}
40394026

4040-
node->setFlag(DETERMINED_CONSTANT, VALUE_CONSTANT);
4041-
node->setFlag(DETERMINED_SIMPLE, VALUE_SIMPLE);
4042-
node->setFlag(DETERMINED_RUNONDBSERVER, VALUE_RUNONDBSERVER);
4043-
4027+
node->setConstantFlags();
40444028
return node;
40454029
}
40464030

@@ -4369,6 +4353,7 @@ AstNode* Ast::endSubQuery() {
43694353
}
43704354

43714355
bool Ast::isInSubQuery() const { return (_queries.size() > 1); }
4356+
43724357
std::unordered_set<std::string> Ast::bindParameters() const {
43734358
return std::unordered_set<std::string>(_bindParameters);
43744359
}

‎arangod/Aql/AstNode.cpp

Lines changed: 31 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -36,19 +36,21 @@
3636
#include "Basics/Utf8Helper.h"
3737
#include "Basics/VelocyPackHelper.h"
3838
#include "Basics/fasthash.h"
39+
#include "Containers/FlatHashSet.h"
3940
#include "Transaction/Methods.h"
4041

42+
#include <array>
4143
#ifdef ARANGODB_ENABLE_MAINTAINER_MODE
4244
#include <iostream>
4345
#endif
46+
#include <unordered_map>
4447

4548
#include <velocypack/Builder.h>
4649
#include <velocypack/Dumper.h>
4750
#include <velocypack/Iterator.h>
4851
#include <velocypack/Sink.h>
4952
#include <velocypack/Slice.h>
5053
#include <velocypack/ValueType.h>
51-
#include <array>
5254

5355
using namespace arangodb;
5456
using namespace arangodb::aql;
@@ -521,17 +523,11 @@ AstNode::AstNode(Ast* ast, arangodb::velocypack::Slice slice)
521523
case NODE_TYPE_OPERATOR_BINARY_NIN:
522524
case NODE_TYPE_OPERATOR_BINARY_ARRAY_IN:
523525
case NODE_TYPE_OPERATOR_BINARY_ARRAY_NIN: {
524-
bool sorted = false;
525-
VPackSlice v = slice.get("sorted");
526-
if (v.isBoolean()) {
527-
sorted = v.getBoolean();
528-
}
529-
setBoolValue(sorted);
526+
setBoolValue(slice.get("sorted").isTrue());
530527
break;
531528
}
532529
case NODE_TYPE_ARRAY: {
533-
VPackSlice v = slice.get("sorted");
534-
if (v.isBoolean() && v.getBoolean()) {
530+
if (VPackSlice v = slice.get("sorted"); v.isTrue()) {
535531
setFlag(DETERMINED_SORTED, VALUE_SORTED);
536532
}
537533
break;
@@ -544,12 +540,7 @@ AstNode::AstNode(Ast* ast, arangodb::velocypack::Slice slice)
544540
case NODE_TYPE_OPERATOR_BINARY_EQ:
545541
case NODE_TYPE_OPERATOR_BINARY_LT:
546542
case NODE_TYPE_OPERATOR_BINARY_LE: {
547-
bool excludesNull = false;
548-
VPackSlice v = slice.get("excludesNull");
549-
if (v.isBoolean()) {
550-
excludesNull = v.getBoolean();
551-
}
552-
setExcludesNull(excludesNull);
543+
setExcludesNull(slice.get("excludesNull").isTrue());
553544
break;
554545
}
555546
case NODE_TYPE_OBJECT:
@@ -614,9 +605,7 @@ AstNode::AstNode(Ast* ast, arangodb::velocypack::Slice slice)
614605
break;
615606
}
616607

617-
VPackSlice subNodes = slice.get("subNodes");
618-
619-
if (subNodes.isArray()) {
608+
if (VPackSlice subNodes = slice.get("subNodes"); subNodes.isArray()) {
620609
members.reserve(subNodes.length());
621610

622611
try {
@@ -893,13 +882,18 @@ void AstNode::validateValueType(int type) {
893882
}
894883

895884
/// @brief fetch a node's type from VPack
896-
AstNodeType AstNode::getNodeTypeFromVPack(
897-
arangodb::velocypack::Slice const& slice) {
885+
AstNodeType AstNode::getNodeTypeFromVPack(arangodb::velocypack::Slice slice) {
898886
int type = slice.get("typeID").getNumericValue<int>();
899887
validateType(type);
900888
return static_cast<AstNodeType>(type);
901889
}
902890

891+
void AstNode::setConstantFlags() noexcept {
892+
setFlag(DETERMINED_CONSTANT, VALUE_CONSTANT);
893+
setFlag(DETERMINED_SIMPLE, VALUE_SIMPLE);
894+
setFlag(DETERMINED_RUNONDBSERVER, VALUE_RUNONDBSERVER);
895+
}
896+
903897
bool AstNode::valueHasVelocyPackRepresentation() const {
904898
switch (type) {
905899
case NODE_TYPE_VALUE:
@@ -973,7 +967,7 @@ void AstNode::toVelocyPackValue(VPackBuilder& builder) const {
973967
return;
974968
}
975969
if (type == NODE_TYPE_ARRAY) {
976-
builder.openArray(false);
970+
builder.openArray(/*allowUnindexed*/ false);
977971
size_t const n = numMembers();
978972
for (size_t i = 0; i < n; ++i) {
979973
auto member = getMemberUnchecked(i);
@@ -988,15 +982,22 @@ void AstNode::toVelocyPackValue(VPackBuilder& builder) const {
988982
if (type == NODE_TYPE_OBJECT) {
989983
builder.openObject();
990984

991-
std::unordered_set<std::string_view> keys;
985+
containers::FlatHashSet<std::string_view> keys;
992986
size_t const n = numMembers();
993987

988+
// only check for duplicate keys if we have more than a single attribute
989+
bool checkUniqueness = (n > 1);
990+
if (checkUniqueness && hasFlag(DETERMINED_CHECKUNIQUENESS)) {
991+
// turn off duplicate keys checking if everything was already checked
992+
checkUniqueness = hasFlag(VALUE_CHECKUNIQUENESS);
993+
}
994+
994995
for (size_t i = 0; i < n; ++i) {
995996
auto member = getMemberUnchecked(i);
996997
if (member != nullptr) {
997998
std::string_view key(member->getStringView());
998999

999-
if (n > 1 && !keys.emplace(key).second) {
1000+
if (checkUniqueness && !keys.emplace(key).second) {
10001001
// duplicate key, skip it
10011002
continue;
10021003
}
@@ -1016,7 +1017,7 @@ void AstNode::toVelocyPackValue(VPackBuilder& builder) const {
10161017

10171018
VPackSlice slice = tmp.slice();
10181019
if (slice.isObject()) {
1019-
slice = slice.get(getString());
1020+
slice = slice.get(getStringView());
10201021
if (!slice.isNone()) {
10211022
builder.add(slice);
10221023
return;
@@ -1037,6 +1038,7 @@ void AstNode::toVelocyPack(VPackBuilder& builder, bool verbose) const {
10371038
if (verbose) {
10381039
builder.add("typeID", VPackValue(static_cast<int>(type)));
10391040
}
1041+
10401042
if (type == NODE_TYPE_COLLECTION || type == NODE_TYPE_VIEW ||
10411043
type == NODE_TYPE_PARAMETER || type == NODE_TYPE_PARAMETER_DATASOURCE ||
10421044
type == NODE_TYPE_ATTRIBUTE_ACCESS || type == NODE_TYPE_OBJECT_ELEMENT ||
@@ -1100,10 +1102,9 @@ void AstNode::toVelocyPack(VPackBuilder& builder, bool verbose) const {
11001102
}
11011103

11021104
// dump sub-nodes
1103-
size_t const n = members.size();
1104-
if (n > 0) {
1105+
if (size_t const n = members.size(); n > 0) {
11051106
builder.add(VPackValue("subNodes"));
1106-
builder.openArray(false);
1107+
builder.openArray(/*allowUnindexed*/ true);
11071108
for (size_t i = 0; i < n; ++i) {
11081109
AstNode* member = getMemberUnchecked(i);
11091110
if (member != nullptr) {
@@ -1790,18 +1791,16 @@ bool AstNode::mustCheckUniqueness() const {
17901791
bool mustCheck = false;
17911792

17921793
// check the actual key members now
1793-
size_t const n = numMembers();
1794-
1795-
if (n >= 2) {
1796-
std::unordered_set<std::string> keys;
1794+
if (size_t const n = numMembers(); n >= 2) {
1795+
containers::FlatHashSet<std::string_view> keys;
17971796

17981797
// only useful to check when there are 2 or more keys
17991798
for (size_t i = 0; i < n; ++i) {
18001799
auto member = getMemberUnchecked(i);
18011800

18021801
if (member->type == NODE_TYPE_OBJECT_ELEMENT) {
18031802
// constant key
1804-
if (!keys.emplace(member->getString()).second) {
1803+
if (!keys.emplace(member->getStringView()).second) {
18051804
// duplicate key
18061805
mustCheck = true;
18071806
break;

‎arangod/Aql/AstNode.h

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -297,15 +297,11 @@ struct AstNode {
297297
static void validateValueType(int type);
298298

299299
/// @brief fetch a node's type from VPack
300-
static AstNodeType getNodeTypeFromVPack(
301-
arangodb::velocypack::Slice const& slice);
302-
303-
/**
304-
* @brief Helper class to check if this node can be represented as VelocyPack
305-
* If this method returns FALSE a call to "toVelocyPackValue" will yield
306-
* no change in the handed in builder.
307-
* On TRUE it is guaranteed that the handed in Builder was modified.
308-
*/
300+
static AstNodeType getNodeTypeFromVPack(arangodb::velocypack::Slice slice);
301+
302+
void setConstantFlags() noexcept;
303+
304+
/// @brief function to check if this node can be represented as VelocyPack.
309305
bool valueHasVelocyPackRepresentation() const;
310306

311307
/// @brief build a VelocyPack representation of the node value

‎arangod/Aql/EngineInfoContainerDBServerServerBased.cpp

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -192,8 +192,9 @@ EngineInfoContainerDBServerServerBased::buildSetupRequest(
192192
network::RequestOptions const& options) const {
193193
TRI_ASSERT(!server.starts_with("server:"));
194194

195-
VPackBuffer<uint8_t> buffer(infoSlice.byteSize());
196-
buffer.append(infoSlice.begin(), infoSlice.byteSize());
195+
auto byteSize = infoSlice.byteSize();
196+
VPackBuffer<uint8_t> buffer(byteSize);
197+
buffer.append(infoSlice.begin(), byteSize);
197198

198199
// add the transaction ID header
199200
network::Headers headers;

‎arangod/Aql/Query.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -347,6 +347,8 @@ void Query::prepareQuery(SerializationFormat format) {
347347
}
348348
}
349349

350+
enterState(QueryExecutionState::ValueType::PHYSICAL_INSTANTIATION);
351+
350352
// simon: assumption is _queryString is empty for DBServer snippets
351353
bool const planRegisters = !_queryString.empty();
352354
ExecutionEngine::instantiateFromPlan(*this, *plan, planRegisters, format);

‎arangod/Aql/QueryExecutionState.cpp

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -31,16 +31,17 @@ using namespace arangodb::aql;
3131

3232
/// @brief names of query phases / states
3333
static std::string const StateNames[] = {
34-
"initializing", // INITIALIZATION
35-
"parsing", // PARSING
36-
"optimizing ast", // AST_OPTIMIZATION
37-
"loading collections", // LOADING_COLLECTIONS
38-
"instantiating plan", // PLAN_INSTANTIATION
39-
"optimizing plan", // PLAN_OPTIMIZATION
40-
"executing", // EXECUTION
41-
"finalizing", // FINALIZATION
42-
"finished", // FINISHED
43-
"killed", // KILLED
34+
"initializing", // INITIALIZATION
35+
"parsing", // PARSING
36+
"optimizing ast", // AST_OPTIMIZATION
37+
"loading collections", // LOADING_COLLECTIONS
38+
"instantiating plan", // PLAN_INSTANTIATION
39+
"optimizing plan", // PLAN_OPTIMIZATION
40+
"instantiating executors", // PHYSICAL_INSTANTIATION
41+
"executing", // EXECUTION
42+
"finalizing", // FINALIZATION
43+
"finished", // FINISHED
44+
"killed", // KILLED
4445

4546
"invalid" // INVALID
4647
};

‎arangod/Aql/QueryExecutionState.h

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,7 @@
2828
#include <iosfwd>
2929
#include <string>
3030

31-
namespace arangodb {
32-
namespace aql {
33-
namespace QueryExecutionState {
31+
namespace arangodb::aql::QueryExecutionState {
3432

3533
/// @brief execution states
3634
enum class ValueType {
@@ -40,6 +38,7 @@ enum class ValueType {
4038
LOADING_COLLECTIONS,
4139
PLAN_INSTANTIATION,
4240
PLAN_OPTIMIZATION,
41+
PHYSICAL_INSTANTIATION,
4342
EXECUTION,
4443
FINALIZATION,
4544
FINISHED,
@@ -52,9 +51,7 @@ size_t toNumber(QueryExecutionState::ValueType value);
5251
std::string const& toString(QueryExecutionState::ValueType state);
5352
std::string toStringWithPrefix(QueryExecutionState::ValueType state);
5453

55-
} // namespace QueryExecutionState
56-
} // namespace aql
57-
} // namespace arangodb
54+
} // namespace arangodb::aql::QueryExecutionState
5855

5956
std::ostream& operator<<(std::ostream&,
6057
arangodb::aql::QueryExecutionState::ValueType);

‎arangod/Aql/QueryProfile.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ static_assert(
7878
static_cast<int>(QueryExecutionState::ValueType::INITIALIZATION) == 0,
7979
"unexpected min QueryExecutionState enum value");
8080
static_assert(static_cast<int>(QueryExecutionState::ValueType::INVALID_STATE) <
81-
11,
81+
12,
8282
"unexpected max QueryExecutionState enum value");
8383

8484
} // namespace aql

‎arangod/Aql/TraversalNode.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -513,7 +513,7 @@ void TraversalNode::doToVelocyPack(VPackBuilder& nodes, unsigned flags) const {
513513
nodes.add(VPackValue("variables"));
514514
VPackArrayBuilder postFilterVariablesGuard(&nodes);
515515
for (auto const& var : _postFilterVariables) {
516-
var->toVelocyPack(nodes);
516+
var->toVelocyPack(nodes, Variable::WithConstantValue{});
517517
}
518518
}
519519
}

0 commit comments

Comments
 (0)
0