8000 APM-209 (#15067) · open-bigdata/arangodb@3128a9c · GitHub
[go: up one dir, main page]
Skip to content

Commit 3128a9c

Browse files
cpjuliaKVS85jsteemann
authored
APM-209 (arangodb#15067)
Co-authored-by: Vadim <vadim@arangodb.com> Co-authored-by: Jan <jsteemann@users.noreply.github.com>
1 parent 45c28d1 commit 3128a9c

File tree

5 files changed

+165
-73
lines changed

5 files changed

+165
-73
lines changed

‎CHANGELOG

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,16 @@
11
devel
22
-----
33

4+
* APM-209: Histogram displaying is now switched off by default. For displaying
5+
it, the new flag `histogram.generate` must be set to true. Its default value
6+
is false for compatibility with other versions and also for complying with the
7+
histogram not being displayed by default. If this flag is not set to true, but
8+
other histogram flags are addressed, e.g. `--histogram.interval-size 500`,
9+
everything will still run normally, but a warning message will be displayed
10+
saying that the histogram is switched off and setting this flag would not be
11+
of use. When the flag is set to true, the histogram is displayed before the
12+
summary in the output.
13+
414
* Extend Windows minidumps with memory regions referenced from CPU registers or
515
the stack to provide more contextual information in case of crashes.
616

‎arangosh/Benchmark/BenchFeature.cpp

Lines changed: 108 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -41,15 +41,13 @@
4141
#include "Basics/FileUtils.h"
4242
#include "Basics/NumberOfCores.h"
4343
#include "Basics/StaticStrings.h"
44-
#include "Basics/StringUtils.h"
4544
#include "Basics/Utf8Helper.h"
4645
#include "Basics/application-exit.h"
4746
#include "Basics/files.h"
4847
#include "Basics/system-functions.h"
4948
#include "Benchmark/BenchmarkCounter.h"
5049
#include "Benchmark/BenchmarkOperation.h"
5150
#include "Benchmark/BenchmarkStats.h"
52-
#include "Benchmark/BenchmarkThread.h"
5351
#include "FeaturePhases/BasicFeaturePhaseClient.h"
5452
#include "Logger/LogMacros.h"
5553
#include "Logger/Logger.h"
@@ -92,6 +90,7 @@ BenchFeature::BenchFeature(application_features::ApplicationServer& server, int*
9290
_progress(true),
9391
_quiet(false),
9492
_waitForSync(false),
93+
_generateHistogram(false),
9594
_runs(1),
9695
_junitReportFile(""),
9796
_jsonReportFile(""),
@@ -144,6 +143,10 @@ void BenchFeature::collectOptions(std::shared_ptr<ProgramOptions> options) {
144143
options->addOption("--histogram.percentiles", "which percentiles to calculate",
145144
new VectorParameter<DoubleParameter>(&_percentiles),
146145
arangodb::options::makeDefaultFlags(options::Flags::FlushOnFirst));
146+
options->addOption("--histogram.generate", "display histogram",
147+
new BooleanParameter(&_generateHistogram),
148+
arangodb::options::makeDefaultFlags(options::Flags::FlushOnFirst))
149+
.setIntroducedIn(31000);
147150

148151
options->addOption("--async", "send asynchronous requests", new BooleanParameter(&_async));
149152

@@ -239,6 +242,26 @@ void BenchFeature::collectOptions(std::shared_ptr<ProgramOptions> options) {
239242
"--verbose", "print out replies if the HTTP header indicates DB errors", false);
240243
}
241244

245+
void BenchFeature::validateOptions(std::shared_ptr<ProgramOptions> options) {
246+
if (!_generateHistogram) {
247+
if (options->processingResult().touched("--histogram.interval-size")) {
248+
LOG_TOPIC("8b53b", WARN, arangodb::Logger::BENCH)
249+
<< "For flag '--histogram.interval-size "
250+
<< _histogramIntervalSize << "': histogram is disabled by default. Enable it with flag '--histogram.generate = true'.";
251+
}
252+
if (options->processingResult().touched("--histogram.num-intervals")) {
253+
LOG_TOPIC("02916", WARN, arangodb::Logger::BENCH)
254+
<< "For flag '--histogram.num-intervals "
255+
<< _histogramNumIntervals << "': histogram is disabled by default. Enable it with flag '--histogram.generate = true'.";
256+
}
257+
if (options->processingResult().touched("--histogram.percentiles")) {
258+
LOG_TOPIC("ad47b", WARN, arangodb::Logger::BENCH)
259+
<< "For flag '--histogram.percentiles "
260+
<< _percentiles << "': histogram is disabled by default. Enable it with flag '--histogram.generate = true'.";
261+
}
262+
}
263+
}
264+
242265
void BenchFeature::status(std::string const& value) {
243266
if (!_quiet) {
244267
LOG_TOPIC("a6905", INFO, arangodb::Logger::BENCH) << value;
@@ -251,6 +274,49 @@ void BenchFeature::updateStartCounter() { ++_started; }
251274

252275
int BenchFeature::getStartCounter() { return _started; }
253276

277+
void BenchFeature::setupHistogram(std::stringstream& pp) {
278+
pp << "Interval/Percentile:";
279+
for (auto percentile : _percentiles) {
280+
pp << std::fixed << std::right << std::setw(13) << std::setprecision(2)
281+
<< percentile << "%";
282+
}
283+
pp << '\n';
284+
285+
}
286+
void BenchFeature::updateStatsValues(std::stringstream& pp, VPackBuilder& builder,
287+
const std::vector<std::unique_ptr<BenchmarkThread>>& threads,
288+
BenchmarkStats& totalStats) {
289+
for (size_t i = 0; i < static_cast<size_t>(_concurrency); ++i) {
290+
if (_duration != 0) {
291+
_realOperations += threads[i]->_counter;
292+
}
293+
294+
totalStats.add(threads[i]->stats());
295+
if (_generateHistogram) {
296+
double scope;
297+
auto res = threads[i]->getPercentiles(_percentiles, scope);
298+
299+
builder.add(std::to_string(i), VPackValue(VPackValueType::Object));
300+
size_t j = 0;
301+
302+
pp << " " << std::right << std::fixed << std::setw(17)
303+
<< std::setprecision(4) << (threads[i]->_histogramIntervalSize * 1000)
304+
<< std::setw(0) << "ms";
305+
306+
builder.add("IntervalSize", VPackValue(threads[i]->_histogramIntervalSize));
307+
308+
for (auto time : res) {
309+
builder.add(std::to_string(_percentiles[j]), VPackValue(time));
310+
pp << " " << std::right << std::fixed << std::setw(9)
311+
<< std::setprecision(4) << (time * 1000) << std::setw(0) << "ms";
312+
j++;
313+
}
314+
pp << '\n';
315+
builder.close();
316+
}
317+
}
318+
}
319+
254320
void BenchFeature::start() {
255321
std::sort(_percentiles.begin(), _percentiles.end());
256322

@@ -289,6 +355,7 @@ void BenchFeature::start() {
289355

290356
client.setDatabaseName(connectDB);
291357
}
358+
292359
int ret = EXIT_SUCCESS;
293360

294361
*_result = ret;
@@ -333,19 +400,16 @@ void BenchFeature::start() {
333400
// aggregated stats for all runs
334401
BenchmarkStats totalStats;
335402

336-
auto builder = std::make_shared<VPackBuilder>();
337-
builder->openObject();
338-
builder->add("histogram", VPackValue(VPackValueType::Object));
403+
VPackBuilder builder;
404+
builder.openObject();
339405
std::vector<std::unique_ptr<BenchmarkThread>> threads;
340406
bool ok = true;
341407
std::vector<BenchRunResult> results;
342408
std::stringstream pp;
343-
pp << "Interval/Percentile:";
344-
for (auto percentile : _percentiles) {
345-
pp << std::fixed << std::right << std::setw(12) << std::setprecision(2)
346-
<< percentile << "%";
409+
if (_generateHistogram) {
410+
builder.add("histogram", VPackValue(VPackValueType::Object));
411+
setupHistogram(pp);
347412
}
348-
pp << std::endl;
349413

350414
for (uint64_t j = 0; j < _runs; j++) {
351415
status("starting threads...");
@@ -365,7 +429,7 @@ void BenchFeature::start() {
365429
auto thread = std::make_unique<BenchmarkThread>(
366430
server(), benchmark.get(), &startCondition, &BenchFeature::updateStartCounter,
367431
static_cast<int>(i), _batchSize, &operationsCounter, client,
368-
_keepAlive, _async, _histogramIntervalSize, _histogramNumIntervals);
432+
_keepAlive, _async, _histogramIntervalSize, _histogramNumIntervals, _generateHistogram);
369433
thread->setOffset(i * realStep);
370434
thread->start();
371435
threads.push_back(std::move(thread));
@@ -432,45 +496,31 @@ void BenchFeature::start() {
432496
requestTime,
433497
});
434498

435-
for (size_t i = 0; i < static_cast<size_t>(_concurrency); ++i) {
436-
if (_duration != 0) {
437-
_realOperations += threads[i]->_counter;
438-
}
439-
440-
totalStats.add(threads[i]->stats());
499+
updateStatsValues(pp, builder, threads, totalStats);
441500

442-
double scope;
443-
auto res = threads[i]->getPercentiles(_percentiles, scope);
501+
threads.clear();
502+
}
503+
if (_generateHistogram) {
504+
builder.close();
505+
}
444506

445-
builder->add(std::to_string(i), VPackValue(VPackValueType::Object));
446-
size_t j = 0;
507+
std::cout << '\n';
447508

448-
pp << " " << std::left << std::setfill('0') << std::fixed << std::setw(8)
449-
<< std::setprecision(6) << (threads[i]->_histogramIntervalSize * 1000)
450-
<< std::setw(0) << "ms ";
509+
std::sort(std::begin(results), std::end(results),
510+
[](BenchRunResult const& a, BenchRunResult const& b)
511+
{ return a._time < b._time; });
451512

452-
builder->add("IntervalSize", VPackValue(threads[i]->_histogramIntervalSize));
513+
report(client, results, totalStats, pp.str(), builder);
453514

454-
for (auto time : res) {
455-
builder->add(std::to_string(_percentiles[j]), VPackValue(time));
456-
pp << " " << std::left << std::setfill('0') << std::fixed << std::setw(8)
457-
<< std::setprecision(6) << (time * 1000) << std::setw(0) << "ms";
458-
j++;
459-
}
515+
builder.close();
460516

461-
builder->close();
462-
pp << std::endl;
463-
}
464-
threads.clear();
517+
if (!_jsonReportFile.empty()) {
518+
auto json = builder.toJson();
519+
TRI_WriteFile(_jsonReportFile.c_str(), json.c_str(), json.length());
465520
}
466521

467-
std::cout << std::endl;
468-
builder->close();
469-
470-
report(client, results, totalStats, pp.str(), *builder);
471-
472522
if (!ok) {
473-
std::cout << "At least one of the runs produced failures!" << std::endl;
523+
std::cout << "At least one of the runs produced failures!" << '\n';
474524
}
475525

476526
benchmark->tearDown();
@@ -482,10 +532,13 @@ void BenchFeature::start() {
482532
*_result = ret;
483533
}
484534

485-
bool BenchFeature::report(ClientFeature& client, std::vector<BenchRunResult> results,
535+
void BenchFeature::report(ClientFeature& client, std::vector<BenchRunResult> const& results,
486536
BenchmarkStats const& stats,
487537
std::string const& histogram, VPackBuilder& builder) {
488-
std::cout << std::endl;
538+
539+
if (_generateHistogram) {
540+
std::cout << histogram << '\n';
541+
}
489542

490543
std::cout << "Total number of operations: " << _realOperations << ", runs: " << _runs
491544
<< ", keep alive: " << (_keepAlive ? "yes" : "no")
@@ -513,10 +566,13 @@ bool BenchFeature::report(ClientFeature& client, std::vector<BenchRunResult> res
513566
builder.add("database", VPackValue(client.databaseName()));
514567
builder.add("collection", VPackValue(_collection));
515568

516-
std::sort(results.begin(), results.end(),
517-
[](BenchRunResult a, BenchRunResult b) { return a._time < b._time; });
569+
TRI_ASSERT(std::is_sorted(std::begin(results), std::end(results),
570+
[](BenchRunResult const& a, BenchRunResult const& b)
571+
{ return a._time < b._time; }));
572+
518573

519574
BenchRunResult output{0, 0, 0, 0};
575+
520576
if (_runs > 1) {
521577
size_t size = results.size();
522578

@@ -556,31 +612,24 @@ bool BenchFeature::report(ClientFeature& client, std::vector<BenchRunResult> res
556612
printResult(output, builder);
557613
builder.close();
558614

559-
std::cout << "Min request time: " << std::setprecision(6)
615+
std::cout << "Min request time: " << std::setprecision(4)
560616
<< (stats.min * 1000) << "ms" << std::endl
561-
<< "Avg request time: " << std::setprecision(6)
617+
<< "Avg request time: " << std::setprecision(4)
562618
<< (stats.avg() * 1000) << "ms" << std::endl
563-
<< "Max request time: " << std::setprecision(6)
619+
<< "Max request time: " << std::setprecision(4)
564620
<< (stats.max * 1000) << "ms" << std::endl
565-
<< std::endl;
566-
567-
std::cout << histogram;
621+
<< '\n';
568622

569623
builder.add("min", VPackValue(stats.min));
570624
builder.add("avg", VPackValue(stats.avg()));
571625
builder.add("max", VPackValue(stats.max));
572-
builder.close();
573626

574-
if (!_jsonReportFile.empty()) {
575-
auto json = builder.toJson();
576-
TRI_WriteFile(_jsonReportFile.c_str(), json.c_str(), json.length());
577-
}
578627

579-
if (_junitReportFile.empty()) {
580-
return true;
628+
629+
if (!_junitReportFile.empty()) {
630+
writeJunitReport(output);
581631
}
582632

583-
return writeJunitReport(output);
584633
}
585634

586635
bool BenchFeature::writeJunitReport(BenchRunResult const& result) {

‎arangosh/Benchmark/BenchFeature.h

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,8 @@
2626
#include <atomic>
2727

2828
#include "ApplicationFeatures/ApplicationFeature.h"
29-
29+
#include < F1D9 span class="pl-pds">"Benchmark/BenchmarkThread.h"
30+
#include "Benchmark/BenchmarkStats.h"
3031
#include <velocypack/velocypack-aliases.h>
3132

3233
namespace arangodb {
@@ -74,15 +75,19 @@ class BenchFeature final : public application_features::ApplicationFeature {
7475
uint64_t replicationFactor() const { return _replicationFactor; }
7576
uint64_t numberOfShards() const { return _numberOfShards; }
7677
bool waitForSync() const { return _waitForSync; }
78+
void validateOptions(std::shared_ptr<options::ProgramOptions>) override final;
79+
7780

7881
std::string const& customQuery() const { return _customQuery; }
7982
std::string const& customQueryFile() const { return _customQueryFile; }
8083

8184
private:
8285
void status(std::string const& value);
83-
bool report(ClientFeature&, std::vector<BenchRunResult>, arangobench::BenchmarkStats const& stats, std::string const& histogram, VPackBuilder& builder);
86+
void report(ClientFeature& client, std::vector<BenchRunResult> const& results, arangobench::BenchmarkStats const& stats, std::string const& histogram, VPackBuilder& builder);
8487
void printResult(BenchRunResult const& result, VPackBuilder& builder);
8588
bool writeJunitReport(BenchRunResult const& result);
89+
void setupHistogram(std::stringstream& pp);
90+
void updateStatsValues(std::stringstream& pp, VPackBuilder& builder, std::vector<std::unique_ptr<arangodb::arangobench::BenchmarkThread>> const& threads, arangodb::arangobench::BenchmarkStats& totalStats);
8691

8792
uint64_t _concurrency;
8893
uint64_t _operations;
@@ -100,6 +105,7 @@ class BenchFeature final : public application_features::ApplicationFeature {
100105
bool _verbose;
101106
bool _quiet;
102107
bool _waitForSync;
108+
bool _generateHistogram; // don't generate histogram by default
103109
uint64_t _runs;
104110
std::string _junitReportFile;
105111
std::string _jsonReportFile;

0 commit comments

Comments
 (0)
0