8000 Merge branch 'devel' of https://github.com/arangodb/arangodb into bug… · arangodb/arangodb@0172e43 · GitHub
[go: up one dir, main page]
Skip to content

Commit 0172e43

Browse files
ObiWahnObiWahn
committed
Merge branch 'devel' of https://github.com/arangodb/arangodb into bug-fix/oasis-statistics
* 'devel' of https://github.com/arangodb/arangodb: Don't use non-existent variable (#9407) relax test condition for windows (#9393) improve handling when procdump detects the process is dead (#9381) Bug fix/internal issue #586 (#9401) fix tests that didn't properly use env variable to look for test (#9399) Pregel additional test & TSan error fix (#9357) use a lock when calling unload (#9375) @maierlars 😍 (#9394) fix typo (#9400) add initializeCursor back to DistinctCollectExecutor (#9386) Feature/add tcpdump support (#9396)
2 parents 4db348f + 9668561 commit 0172e43

39 files changed

+449
-1427
lines changed

‎3rdParty/velocypack/include/velocypack/Builder.h‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -799,7 +799,9 @@ struct ObjectBuilder final : public BuilderContainer,
799799
try {
800800
builder->close();
801801
} catch (...) {
802-
// destructors must not throw
802+
// destructors must not throw. however, we can at least
803+
// signal something is very wrong in debug mode
804+
VELOCYPACK_ASSERT(false);
803805
}
804806
}
805807
};
@@ -826,7 +828,9 @@ struct ArrayBuilder final : public BuilderContainer,
826828
try {
827829
builder->close();
828830
} catch (...) {
829-
// destructors must not throw
831+
// destructors must not throw. however, we can at least
832+
// signal something is very wrong in debug mode
833+
VELOCYPACK_ASSERT(false);
830834
}
831835
}
832836
};

‎3rdParty/velocypack/include/velocypack/Slice.h‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -534,10 +534,14 @@ class Slice {
534534
}
535535

536536
// tests whether the Slice is an empty array
537-
constexpr bool isEmptyArray() const noexcept { return head() == 0x01; }
537+
bool isEmptyArray() const {
538+
return isArray() && length() == 0;
539+
}
538540

539541
// tests whether the Slice is an empty object
540-
constexpr bool isEmptyObject() const noexcept { return head() == 0x0a; }
542+
bool isEmptyObject() const {
543+
return isObject() && length() == 0;
544+
}
541545

542546
// translates an integer key into a string
543547
Slice translate() const;

‎README_maintainers.md‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -374,6 +374,35 @@ syntax --option value --sub:option value. Using Valgrind could look like this:
374374
- we force the logging not to happen asynchronous
375375
- eventually you may still add temporary `console.log()` statements to tests you debug.
376376
377+
Running tcpdump / windump for the SUT
378+
-------------------------------------
379+
Don't want to miss a beat of your test? If you want to invoke tcpdump with sudo, make sure
380+
that your current shell has sudo enabled. Try like this:
381+
382+
sudo /bin/true; ./scripts/unittest http_server \
383+
--sniff sudo --cleanup false
384+
385+
The pcap file will end up in your tests temporary directory.
386+
You may need to press an additional `ctrl+c` to force stop the sudo'ed tcpdump.
387+
388+
On windows you can use TShark, you need a npcap enabled installation. List your devices
389+
to sniff on using the -D option:
390+
391+
c:/Program\ Files/wireshark/tshark.exe -D
392+
1. \Device\NPF_{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX} (Npcap Loopback Adapter)
393+
2. \Device\NPF_{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX} (Ethernet)
394+
3. \\.\USBPcap1 (USBPcap1)
395+
396+
choose the `Npcap Loopback Adapter` number - 1:
397+
398+
./scripts/unittest http_server \
399+
--sniff true \
400+
--cleanup false \
401+
--sniffDevice 1\
402+
--sniffProgram c:/Programm Files/wireshark/tshark.exe
403+
404+
you can later on use wireshark to inpsect the capture files.
405+
377406
Debugging AQL execution blocks
378407
------------------------------
379408
To debug AQL execution blocks, two steps are required:

‎arangod/Aql/DistinctCollectExecutor.cpp‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -64,12 +64,11 @@ DistinctCollectExecutor::DistinctCollectExecutor(Fetcher& fetcher, Infos& infos)
6464
AqlValueGroupEqual(_infos.getTransaction())) {}
6565

6666
DistinctCollectExecutor::~DistinctCollectExecutor() {
67-
// destroy all AqlValues captured
68-
for (auto& it : _seen) {
69-
for (auto& it2 : it) {
70-
const_cast<AqlValue*>(&it2)->destroy();
71-
}
72-
}
67+
destroyValues();
68+
}
69+
70+
void DistinctCollectExecutor::initializeCursor() {
71+
destroyValues();
7372
}
7473

7574
std::pair<ExecutionState, NoStats> DistinctCollectExecutor::produceRows(OutputAqlItemRow& output) {
@@ -138,3 +137,13 @@ std::pair<ExecutionState, size_t> DistinctCollectExecutor::expectedNumberOfRows(
138137
// but it is upper bounded by the input.
139138
return _fetcher.preFetchNumberOfRows(atMost);
140139
}
140+
141+
void DistinctCollectExecutor::destroyValues() {
142+
// destroy all AqlValues captured
143+
for (auto& it : _seen) {
144+
for (auto& it2 : it) {
145+
const_cast<AqlValue*>(&it2)->destroy();
146+
}
147+
}
148+
_seen.clear();
149+
}

‎arangod/Aql/DistinctCollectExecutor.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,8 @@ class DistinctCollectExecutor {
9595
DistinctCollectExecutor(DistinctCollectExecutor const&) = delete;
9696
DistinctCollectExecutor(Fetcher& fetcher, Infos&);
9797
~DistinctCollectExecutor();
98+
99+
void initializeCursor();
98100

99101
/**
100102
* @brief produce the next Row of Aql Values.
@@ -107,6 +109,7 @@ class DistinctCollectExecutor {
107109

108110
private:
109111
Infos const& infos() const noexcept { return _infos; };
112+
void destroyValues();
110113

111114
private:
112115
Infos const& _infos;

‎arangod/Aql/ExecutionBlockImpl.cpp‎

< CDA3 div class="d-flex flex-justify-end flex-items-center">
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -372,6 +372,9 @@ std::pair<ExecutionState, Result> ExecutionBlockImpl<Executor>::initializeCursor
372372
static_assert(!std::is_same<Executor, IndexExecutor>::value || customInit,
373373
"IndexExecutor is expected to implement a custom "
374374
"initializeCursor method!");
375+
static_assert(!std::is_same<Executor, DistinctCollectExecutor>::value || customInit,
376+
"DistinctCollectExecutor is expected to implement a custom "
377+
"initializeCursor method!");
375378
InitializeCursor<customInit>::init(_executor, _rowFetcher, _infos);
376379

377380
// // use this with c++17 instead of specialisation below

‎arangod/IResearch/IResearchAnalyzerFeature.cpp‎

Lines changed: 30 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -303,7 +303,8 @@ irs::analysis::analyzer::ptr text_vpack_builder(irs::string_ref const& args) noe
303303
bool text_vpack_normalizer(const irs::string_ref& args, std::string& out) noexcept {
304304
std::string tmp;
305305
if (irs::analysis::analyzers::normalize(tmp, "text", irs::text_format::json,
306-
arangodb::iresearch::slice<char>(args).toString())) {
306+
arangodb::iresearch::slice<char>(args).toString(),
307+
false)) {
307308
auto vpack = VPackParser::fromJson(tmp);
308309
out.resize(vpack->slice().byteSize());
309310
std::memcpy(&out[0], vpack->slice().begin(), out.size());
@@ -327,7 +328,7 @@ namespace stem_vpack {
327328
bool stem_vpack_normalizer(const irs::string_ref& args, std::string& out) noexcept {
328329
std::string tmp;
329330
if (irs::analysis::analyzers::normalize(tmp, "stem", irs::text_format::json,
330-
arangodb::iresearch::slice<char>(args).toString())) {
331+
arangodb::iresearch::slice<char>(args).toString(), false)) {
331332
auto vpack = VPackParser::fromJson(tmp);
332333
out.resize(vpack->slice().byteSize());
333334
std::memcpy(&out[0], vpack->slice().begin(), out.size());
@@ -351,7 +352,7 @@ namespace norm_vpack {
351352
bool norm_vpack_normalizer(const irs::string_ref& args, std::string& out) noexcept {
352353
std::string tmp;
353354
if (irs::analysis::analyzers::normalize(tmp, "norm", irs::text_format::json,
354-
arangodb::iresearch::slice<char>(args).toString())) {
355+
arangodb::iresearch::slice<char>(args).toString(), false)) {
355356
auto vpack = VPackParser::fromJson(tmp);
356357
out.resize(vpack->slice().byteSize());
357358
std::memcpy(&out[0], vpack->slice().begin(), out.size());
@@ -500,6 +501,9 @@ bool equalAnalyzer(
500501

501502
if (!::normalize(normalizedProperties, type, properties)) {
502503
// failed to normalize definition
504+
LOG_TOPIC("dfac1", WARN, arangodb::iresearch::TOPIC)
505+
<< "failed to normalize properties for analyzer type '" << type << "' properties '"
506+
<< properties.toString() << "'";
503507
return false;
504508
}
505509

@@ -1157,12 +1161,22 @@ arangodb::Result IResearchAnalyzerFeature::emplaceAnalyzer( // emplace
11571161

11581162
erase = false;
11591163
} else if (!equalAnalyzer(*analyzer, type, properties, features)) { // duplicate analyzer with different configuration
1160-
return arangodb::Result(
1161-
TRI_ERROR_BAD_PARAMETER,
1162-
"name collision detected while registering an arangosearch analizer name '" + std::string(name) +
1163-
"' type '" + std::string(type) + "' properties '" + properties.toString() +
1164-
"', previous registration type '" + std::string(analyzer->type()) +
1165-
"' properties '" + analyzer->properties().toString() + "'");
1164+
std::ostringstream errorText;
1165+
errorText << "name collision detected while registering an arangosearch analyzer name '" << name
1166+
<< "' type '" << type << "' properties '" << properties.toString()
1167+
<< "' features '";
1168+
for(auto feature : features) {
1169+
errorText << feature->name() << " ";
1170+
}
1171+
errorText << "', previous registration type '" << analyzer->type()
1172+
<< "' properties '" << analyzer->properties().toString() << "'"
1173+
<< " features '";
1174+
for(auto feature : analyzer->features()) {
1175+
errorText << feature->name() << " ";
1176+
}
1177+
errorText << "'";
1178+
return arangodb::Result(TRI_ERROR_BAD_PARAMETER, errorText.str());
1179+
11661180
}
11671181

11681182
result = itr;
@@ -2227,35 +2241,14 @@ arangodb::Result IResearchAnalyzerFeature::storeAnalyzer(AnalyzerPool& pool) {
22272241

22282242
try {
22292243
auto collection = getAnalyzerCollection(*vocbase);
2230-
22312244
if (!collection) {
2232-
auto collectionCallback = [&collection]( // store collection
2233-
std::shared_ptr<arangodb::LogicalCollection> const& col // args
2234-
)->void {
2235-
collection = col;
2236-
};
2237-
static auto const properties = // analyzer collection properties
2238-
arangodb::velocypack::Parser::fromJson("{ \"isSystem\": true }");
2239-
auto res = arangodb::methods::Collections::create( // create collection
2240-
*vocbase, // collection vocbase
2241-
ANALYZER_COLLECTION_NAME, // collection name
2242-
TRI_col_type_e::TRI_COL_TYPE_DOCUMENT, // collection type
2243-
properties->slice(), // collection properties
2244-
true, // waitsForSyncReplication same as UpgradeTasks::createSystemCollection(...)
2245-
true, // enforceReplicationFactor same as UpgradeTasks::createSystemCollection(...)
2246-
collectionCallback // callback if created
2247-
);
2248-
2249-
if (!res.ok()) {
2250-
return res;
2251-
}
2252-
2253-
if (!collection) {
2254-
return arangodb::Result( // result
2255-
TRI_ERROR_INTERNAL, // code
2256-
std::string("failure to create collection '") + ANALYZER_COLLECTION_NAME + "' in vocbase '" + vocbase->name() + "' vocbase while persising arangosearch analyzer '" + pool.name()+ "'"
2257-
);
2258-
}
2245+
return arangodb::Result( // result
2246+
TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND, // code
2247+
std::string("failure to find collection '") +
2248+
ANALYZER_COLLECTION_NAME +
2249+
"' in vocbase '" + vocbase->name() +
2250+
"' vocbase while persising arangosearch analyzer '" + pool.name()+ "'"
2251+
);
22592252
}
22602253

22612254
arangodb::SingleCollectionTransaction trx( // transaction

‎arangod/Pregel/PregelFeature.cpp‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -316,12 +316,16 @@ void PregelFeature::cleanupWorker(uint64_t executionNumber) {
316316

317317
void PregelFeature::cleanupAll() {
318318
MUTEX_LOCKER(guard, _mutex);
319-
_conductors.clear();
320-
for (auto it : _workers) {
319+
decltype(_conductors) cs = std::move(_conductors);
320+
decltype(_workers) ws = std::move(_workers);
321+
guard.unlock();
322+
323+
// cleanup all workers & conductors without holding the lock
324+
cs.clear();
325+
for (auto it : ws) {
321326
it.second.second->cancelGlobalStep(VPackSlice());
322327
}
323328
std::this_thread::sleep_for(std::chrono::microseconds(1000 * 100)); // 100ms to send out cancel calls
324-
_workers.clear();
325329
}
326330

327331
void PregelFeature::handleConductorRequest(std::string const& path, VPackSlice const& body,

‎arangod/RocksDBEngine/RocksDBCollection.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -202,11 +202,12 @@ void RocksDBCollection::load() {
202202
}
203203

204204
void RocksDBCollection::unload() {
205+
WRITE_LOCKER(guard, _exclusiveLock);
205206
if (useCache()) {
206207
destroyCache();
207208
TRI_ASSERT(_cache.get() == nullptr);
208209
}
209-
READ_LOCKER(guard, _indexesLock);
210+
READ_LOCKER(indexGuard, _indexesLock);
210211
for (auto it : _indexes) {
211212
it->unload();
212213
}

‎js/client/modules/@arangodb/crash-utils.js‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -185,30 +185,30 @@ Crash analysis of: ` + JSON.stringify(instanceInfo) + '\n';
185185
return 'cdb ' + args.join(' ');
186186
}
187187

188-
function checkMonitorAlive (binary, arangod, options, res) {
189-
if (arangod.hasOwnProperty('monitor') ) {
188+
function checkMonitorAlive (binary, instanceInfo, options, res) {
189+
if (instanceInfo.hasOwnProperty('monitor') ) {
190190
// Windows: wait for procdump to do its job...
191-
if (!arangod.monitor.hasOwnProperty('status')) {
192-
let rc = statusExternal(arangod.monitor.pid, false);
191+
if (!instanceInfo.monitor.hasOwnProperty('status')) {
192+
let rc = statusExternal(instanceInfo.monitor.pid, false);
193193
if (rc.status !== 'RUNNING') {
194-
arangod.monitor = rc;
194+
instanceInfo.monitor = rc;
195195
// procdump doesn't set propper exit codes, check for
196196
// dumps that may exist:
197-
if (fs.exists(arangod.coreFilePattern)) {
197+
if (fs.exists(instanceInfo.coreFilePattern)) {
198198
print("checkMonitorAlive: marking crashy");
199-
arangod.monitor.monitorExited = true;
200-
arangod.monitor.pid = null;
199+
instanceInfo.monitor.monitorExited = true;
200+
instanceInfo.monitor.pid = null;
201201
pu.serverCrashed = true;
202202
options.cleanup = false;
203-
arangod['exitStatus'] = {};
204-
analyzeCrash(binary, arangod, options, "the process monitor commanded error");
205-
Object.assign(arangod.exitStatus,
206-
killExternal(arangod.pid, abortSignal));
203+
instanceInfo['exitStatus'] = {};
204+
analyzeCrash(binary, instanceInfo, options, "the process monitor commanded error");
205+
Object.assign(instanceInfo.exitStatus,
206+
killExternal(instanceInfo.pid, abortSignal));
207207
return false;
208208
}
209209
}
210210
}
211-
else return arangod.monitor.exitStatus;
211+
else return instanceInfo.monitor.exitStatus;
212212
}
213213
return true;
214214
}

0 commit comments

Comments
 (0)
0