8000 Merge remote-tracking branch 'origin/devel' into feature/validation-p… · arangodb/arangodb@11e90b2 · GitHub
[go: up one dir, main page]
Skip to content

Commit 11e90b2

Browse files
ObiWahnObiWahn
committed
Merge remote-tracking branch 'origin/devel' into feature/validation-phase-2
* origin/devel: fix python invocation (#11142) increase timeout to 15 minutes (#11141) Update CHANGELOG Docs: Remove misleading statement ...about the non-existing API /_api/database/properties Update CHANGELOG Bug fix/fixes 19022020 (#11136) Move subqueries down (#11135) Update OskarTestSuitesBlackList (#11132) fix shrink method in aqlitemblock (#11122) Documentation/rotate jwt docublocks (#11120) remove non-deterministic test assumptions fix test timeout handling (#11118)
2 parents 5aa13ae + e3e9513 commit 11e90b2

31 files changed

+403
-202
lines changed

‎3rdParty/V8/gypfiles/shim_headers.gypi

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@
6262
'outputs': [
6363
'<!@pymod_do_main(generate_shim_headers <@(generator_args) --outputs)',
6464
],
65-
'action': ['python',
65+
'action': ['<(PYTHON_EXECUTABLE)',
6666
'<(generator_path)',
6767
'<@(generator_args)',
6868
'--generate',

‎3rdParty/V8/gypfiles/v8.gyp

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1408,7 +1408,7 @@
14081408
'<(SHARED_INTERMEDIATE_DIR)/src/regexp/special-case.cc',
14091409
],
14101410
'action': [
1411-
'python',
1411+
'<(PYTHON_EXECUTABLE)',
14121412
'<(V8_ROOT)/tools/run.py',
14131413
'<@(_inputs)',
14141414
'<@(_outputs)',
@@ -1510,7 +1510,8 @@
15101510
}],
15111511
],
15121512
'action': [
1513-
'python', '<(V8_ROOT)/tools/testrunner/utils/dump_build_config_gyp.py',
1513+
'<(PYTHON_EXECUTABLE)',
1514+
'<(V8_ROOT)/tools/testrunner/utils/dump_build_config_gyp.py',
15141515
'<@(v8_dump_build_config_args)',
15151516
],
15161517
},
@@ -1546,7 +1547,7 @@
15461547
'<(SHARED_INTERMEDIATE_DIR)/debug-support.cc',
15471548
],
15481549
'action': [
1549-
'python',
1550+
'<(PYTHON_EXECUTABLE)',
15501551
'<(V8_ROOT)/tools/gen-postmortem-metadata.py',
15511552
'<@(_outputs)',
15521553
'<@(heapobject_files)'

‎3rdParty/fuerte/include/fuerte/loop.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,8 @@ typedef asio_ns::executor_work_guard<asio_ns::io_context::executor_type>
4949
class EventLoopService {
5050
public:
5151
// Initialize an EventLoopService with a given number of threads
52-
// and a given number of io_context
53-
explicit EventLoopService(unsigned int threadCount = 1);
52+
// and a given number of io_contexts
53+
explicit EventLoopService(unsigned int threadCount = 1, char const* name = "");
5454
virtual ~EventLoopService();
5555

5656
// Prevent copying

‎3rdParty/fuerte/src/loop.cpp

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,15 +26,28 @@
2626
#include <fuerte/loop.h>
2727
#include <fuerte/types.h>
2828

29+
#ifdef __linux__
30+
#include <sys/prctl.h>
31+
#endif
32+
2933
namespace arangodb { namespace fuerte { inline namespace v1 {
3034

31-
EventLoopService::EventLoopService(unsigned int threadCount)
35+
EventLoopService::EventLoopService(unsigned int threadCount, char const* name)
3236
: _lastUsed(0), _sslContext(nullptr) {
3337
for (unsigned i = 0; i < threadCount; i++) {
3438
_ioContexts.emplace_back(std::make_shared<asio_ns::io_context>(1));
3539
_guards.emplace_back(asio_ns::make_work_guard(*_ioContexts.back()));
3640
asio_ns::io_context* ctx = _ioContexts.back().get();
37-
_threads.emplace_back([ctx]() { ctx->run(); });
41+
_threads.emplace_back([ctx, name]() {
42+
#ifdef __linux__
43+
// set name of threadpool thread, so threads can be distinguished from each other
44+
if (name != nullptr && *name != '\0') {
45+
prctl(PR_SET_NAME, name, 0, 0, 0);
46+
}
47+
#endif
48+
ctx->run();
49+
});
50+
3851
}
3952
}
4053

@@ -61,7 +74,7 @@ void EventLoopService::stop() {
6174
std::this_thread::sleep_for(std::chrono::milliseconds(50));
6275
std::for_each(_ioContexts.begin(), _ioContexts.end(), [](auto& c) { c->stop(); });
6376
std::this_thread::sleep_for(std::chrono::milliseconds(50));
64-
std::for_each(_threads.begin(), _threads.end(), [](auto& t) { if(t.joinable()) { t.join(); } });
77+
std::for_each(_threads.begin(), _threads.end(), [](auto& t) { if (t.joinable()) { t.join(); } });
6578
}
6679

6780
}}} // namespace arangodb::fuerte::v1

‎CHANGELOG

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,12 @@
11
devel
22
-----
33

4+
* Fixed issue #11137: http-proxy can't pass delete method req.body to foxx.
5+
6+
* Disable "collect-in-cluster" AQL optimizer rule in case a LIMIT node is
7+
between the COLLECT and the source data. In this case it is not safe to apply
8+
the distributed collect, as it may alter the results.
9+
410
* Rebuild UI.
511

612
* Fix arangorestore:

‎Documentation/DocuBlocks/Rest/Administration/get_admin_server_jwt.md

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
2+
@startDocuBlock get_admin_server_jwt
3+
@brief Retrieve JWT secrets info
4+
5+
@RESTHEADER{GET /_admin/server/jwt, Fetch information about the currently loaded secrets, handleJWT:get}
6+
7+
@RESTDESCRIPTION
8+
Get information about the currently loaded secrets.
9+
10+
To utilize the API a superuser JWT token is necessary, otherwise the response
11+
will be _HTTP 403 Forbidden_.
12+
13+
@RESTRETURNCODES
14+
15+
@RESTRETURNCODE{200}
16+
17+
@RESTREPLYBODY{error,boolean,required,}
18+
boolean flag to indicate whether an error occurred (*false* in this case)
19+
20+
@RESTREPLYBODY{code,integer,required,int64}
21+
the HTTP status code - 200 in this case
22+
23+
@RESTREPLYBODY{result,object,required,jwt_secret_struct}
24+
The result object.
25+
26+
@RESTSTRUCT{active,jwt_secret_struct,object,required,}
27+
An object with the SHA-256 hash of the active secret.
28+
29+
@RESTSTRUCT{passive,jwt_secret_struct,array,required,object}
30+
An array of objects with the SHA-256 hashes of the passive secrets.
31+
Can be empty.
32+
33+
@RESTRETURNCODE{403}
34+
if the request was not authenticated as a user with sufficient rights
35+
36+
@endDocuBlock

‎Documentation/DocuBlocks/Rest/Administration/post_admin_server_jwt.md

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
2+
@startDocuBlock post_admin_server_jwt
3+
@brief Hot-reload JWT secrets
4+
5+
@RESTHEADER{POST /_admin/server/jwt, Hot-reload the JWT secret(s) from disk, handleJWT:post}
6+
7+
@RESTDESCRIPTION
8+
Sending a request without payload to this endpoint reloads the JWT secret(s)
9+
from disk. Only the files specified via the arangod startup option
10+
`--server.jwt-secret-keyfile` or `--server.jwt-secret-folder` are used.
11+
It is not possible to change the locations where files are loaded from
12+
without restarting the process.
13+
14+
To utilize the API a superuser JWT token is necessary, otherwise the response
15+
will be _HTTP 403 Forbidden_.
16+
17+
@RESTRETURNCODES
18+
19+
@RESTRETURNCODE{200}
20+
21+
@RESTREPLYBODY{error,boolean,required,}
22+
boolean flag to indicate whether an error occurred (*false* in this case)
23+
24+
@RESTREPLYBODY{code,integer,required,int64}
25+
the HTTP status code - 200 in this case
26+
27+
@RESTREPLYBODY{result,object,required,jwt_secret_struct}
28+
The result object.
29+
30+
@RESTSTRUCT{active,jwt_secret_struct,object,required,}
31+
An object with the SHA-256 hash of the active secret.
32+
33+
@RESTSTRUCT{passive,jwt_secret_struct,array,required,object}
34+
An array of objects with the SHA-256 hashes of the passive secrets.
35+
Can be empty.
36+
37+
@RESTRETURNCODE{403}
38+
if the request was not authenticated as a user with sufficient rights
39+
40+
@endDocuBlock

‎Documentation/DocuBlocks/Rest/Administration/put_admin_server_mode.md

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,11 @@ The mode of the server `readonly` or `default`.
1111
Update mode information about a server. The json response will contain
1212
a field `mode` with the value `readonly` or `default`. In a read-only server
1313
all write operations will fail with an error code of `1004` (_ERROR_READ_ONLY_).
14-
Creating or dropping of databases and collections will also fail with error code `11` (_ERROR_FORBIDDEN_).
14+
Creating or dropping of databases and collections will also fail with error
15+
code `11` (_ERROR_FORBIDDEN_).
1516

16-
This API so it *does require* authentication and administrative server rights.
17+
This is a protected API. It requires authentication and administrative
18+
server rights.
1719

1820
@RESTRETURNCODES
1921

‎Documentation/DocuBlocks/Rest/Database/get_api_database_current.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11

22
@startDocuBlock get_api_database_current
3-
@brief retrieves information about the current database (alias /_api/database/properties)
3+
@brief retrieves information about the current database
44

55
@RESTHEADER{GET /_api/database/current, Information of the database, getDatabases:current}
66

77
@RESTDESCRIPTION
8-
Retrieves information about the current database
8+
Retrieves the properties of the current database
99

1010
The response is a JSON object with the following attributes:
1111

‎UnitTests/OskarTestSuitesBlackList

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
upgrade_data_3.2.*
22
upgrade_data_3.3.*
3+
upgrade_data_3.4.*
4+
upgrade_data_3.5.*
35
audit
46
server_secrets

‎arangod/Agency/Store.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -280,7 +280,8 @@ std::vector<bool> Store::applyLogEntries(arangodb::velocypack::Builder const& qu
280280
if (pos == std::string::npos || pos == 0) {
281281
break;
282282
} else {
283-
uri = uri.substr(0, pos);
283+
// this is superior to uri = uri.substr(0, pos);
284+
uri.resize(pos);
284285
}
285286
}
286287
}

‎arangod/Aql/AqlItemBlock.cpp

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -347,6 +347,13 @@ void AqlItemBlock::shrink(size_t nrItems) {
347347

348348
decreaseMemoryUsage(sizeof(AqlValue) * (_nrItems - nrItems) * _nrRegs);
349349

350+
// remove the shadow row indices pointing to now invalid rows.
351+
_shadowRowIndexes.erase(_shadowRowIndexes.lower_bound(nrItems),
352+
_shadowRowIndexes.end());
353+
354+
// adjust the size of the block
355+
_nrItems = nrItems;
356+
350357
for (size_t i = numEntries(); i < _data.size(); ++i) {
351358
AqlValue& a = _data[i];
352359
if (a.requiresDestruction()) {
@@ -368,13 +375,6 @@ void AqlItemBlock::shrink(size_t nrItems) {
368375
}
369376
a.erase();
370377
}
371-
372-
// remove the shadow row indices pointing to now invalid rows.
373-
_shadowRowIndexes.erase(_shadowRowIndexes.lower_bound(nrItems),
374-
_shadowRowIndexes.end());
375-
376-
// adjust the size of the block
377-
_nrItems = nrItems;
378378
}
379379

380380
void AqlItemBlock::rescale(size_t nrItems, RegisterId nrRegs) {

‎arangod/Aql/Condition.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -618,7 +618,7 @@ std::pair<bool, bool> Condition::findIndexes(EnumerateCollectionNode const* node
618618

619619
size_t itemsInIndex;
620620
if (!collectionName.empty() && collectionName[0] == '_' &&
621-
collectionName.substr(0, 11) == "_statistics") {
621+
collectionName.compare(0, 11, "_statistics", 11) == 0) {
622622
// use hard-coded number of items in index, because we are dealing with
623623
// the statistics collection here. this saves a roundtrip to the DB servers
624624
// for statistics queries that do not need a fully accurate collection count

‎arangod/Aql/Functions.cpp

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -669,7 +669,7 @@ std::string extractCollectionName(transaction::Methods* trx,
669669
VPackSlice slice = materializer.slice(value, true);
670670
VPackSlice id = slice;
671671

672-
if (slice.isObject() && slice.hasKey(StaticStrings::IdString)) {
672+
if (slice.isObject()) {
673673
id = slice.get(StaticStrings::IdString);
674674
}
675675
if (id.isString()) {
@@ -683,7 +683,8 @@ std::string extractCollectionName(transaction::Methods* trx,
683683
size_t pos = identifier.find('/');
684684

685685
if (pos != std::string::npos) {
686-
return identifier.substr(0, pos);
686+
// this is superior to identifier.substr(0, pos)
687+
identifier.resize(pos);
687688
}
688689

689690
return identifier;
@@ -875,16 +876,20 @@ void getDocumentByIdentifier(transaction::Methods* trx, std::string& collectionN
875876
searchBuilder->add(VPackValue(identifier));
876877
} else {
877878
if (collectionName.empty()) {
878-
searchBuilder->add(VPackValue(identifier.substr(pos + 1)));
879+
char const* p = identifier.data() + pos + 1;
880+
size_t l = identifier.size() - pos - 1;
881+
searchBuilder->add(VPackValuePair(p, l, VPackValueType::String));
879882
collectionName = identifier.substr(0, pos);
880-
} else if (identifier.substr(0, pos) != collectionName) {
883+
} else if (identifier.compare(0, pos, collectionName) != 0) {
881884
// Requesting an _id that cannot be stored in this collection
882885
if (ignoreError) {
883886
return;
884887
}
885888
THROW_ARANGO_EXCEPTION(TRI_ERROR_ARANGO_CROSS_COLLECTION_REQUEST);
886889
} else {
887-
searchBuilder->add(VPackValue(identifier.substr(pos + 1)));
890+
char const* p = identifier.data() + pos + 1;
891+
size_t l = identifier.size() - pos - 1;
892+
searchBuilder->add(VPackValuePair(p, l, VPackValueType::String));
888893
}
889894
}
890895

‎arangod/Aql/OptimizerRules.cpp

Lines changed: 32 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -75,12 +75,20 @@
7575
namespace {
7676

7777
bool accessesCollectionVariable(arangodb::aql::ExecutionPlan const* plan,
78-
arangodb::aql::CalculationNode const* node,
78+
arangodb::aql::ExecutionNode const* node,
7979
::arangodb::containers::HashSet<arangodb::aql::Variable const*>& vars) {
8080
using EN = arangodb::aql::ExecutionNode;
8181

82-
vars.clear();
83-
arangodb::aql::Ast::getReferencedVariables(node->expression()->node(), vars);
82+
if (node->getType() == EN::CALCULATION) {
83+
auto nn = EN::castTo<arangodb::aql::CalculationNode const*>(node);
84+
vars.clear();
85+
arangodb::aql::Ast::getReferencedVariables(nn->expression()->node(), vars);
86+
} else if (node->getType() == EN::SUBQUERY) {
87+
auto nn = EN::castTo<arangodb::aql::SubqueryNode const*>(node);
88+
vars.clear();
89+
nn->getVariablesUsedHere(vars);
90+
}
91+
8492
for (auto const& it : vars) {
8593
auto setter = plan->getVarSetBy(it->id);
8694
if (setter == nullptr) {
@@ -1686,23 +1694,34 @@ void arangodb::aql::moveCalculationsDownRule(Optimizer* opt,
16861694
OptimizerRule const& rule) {
16871695
::arangodb::containers::SmallVector<ExecutionNode*>::allocator_type::arena_type a;
16881696
::arangodb::containers::SmallVector<ExecutionNode*> nodes{a};
1689-
plan->findNodesOfType(nodes, EN::CALCULATION, true);
1697+
plan->findNodesOfType(nodes, {EN::CALCULATION, EN::SUBQUERY}, true);
16901698

16911699
std::vector<ExecutionNode*> stack;
16921700
::arangodb::containers::HashSet<Variable const*> vars;
16931701
::arangodb::containers::HashSet<Variable const*> usedHere;
16941702
bool modified = false;
16951703

16961704
for (auto const& n : nodes) {
1697-
auto nn = ExecutionNode::castTo<CalculationNode*>(n);
1698-
if (!nn->expression()->isDeterministic()) {
1699-
// we will only move expressions down that cannot throw and that are
1700-
// deterministic
1701-
continue;
1702-
}
1703-
17041705
// this is the variable that the calculation will set
1705-
auto variable = nn->outVariable();
1706+
Variable const* variable = nullptr;
1707+
1708+
if (n->getType() == EN::CALCULATION) {
1709+
auto nn = ExecutionNode::castTo<CalculationNode*>(n);
1710+
if (!nn->expression()->isDeterministic()) {
1711+
// we will only move expressions down that cannot throw and that are
1712+
// deterministic
1713+
continue;
1714+
}
1715+
variable = nn->outVariable();
1716+
} else if (n->getType() == EN::SUBQUERY) {
1717+
auto nn = ExecutionNode::castTo<SubqueryNode*>(n);
1718+
if (!nn->isDeterministic() || nn->isModificationNode()) {
1719+
// we will only move subqueries down that are deterministic and are not
1720+
// modification subqueries
1721+
continue;
1722+
}
1723+
variable = nn->outVariable();
1724+
}
17061725

17071726
stack.clear();
17081727
n->parents(stack);
@@ -1746,7 +1765,7 @@ void arangodb::aql::moveCalculationsDownRule(Optimizer* opt,
17461765
// however, if our calculation uses any data from a
17471766
// collection/index/view, it probably makes sense to not move it,
17481767
// because the result set may be huge
1749-
if (::accessesCollectionVariable(plan.get(), nn, vars)) {
1768+
if (::accessesCollectionVariable(plan.get(), n, vars)) {
17501769
break;
17511770
}
17521771
}

‎arangod/Cluster/ClusterFeature.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -416,6 +416,7 @@ void ClusterFeature::prepare() {
416416
config.idleConnectionMilli = 1000;
417417
config.verifyHosts = false;
418418
config.clusterInfo = &clusterInfo();
419+
config.name = "AgencyComm";
419420

420421
_pool = std::make_unique<network::ConnectionPool>(config);
421422

0 commit comments

Comments
 (0)
0