8000 Feature/sockstat table formatting (#13636) · arangodb/arangodb@76dbd16 · GitHub
[go: up one dir, main page]
Skip to content

Commit 76dbd16

Browse files
dothebartjsteemann
dothebart
and
jsteemann
authored
Feature/sockstat table formatting (#13636)
Co-authored-by: Jan <jsteemann@users.noreply.github.com>
1 parent 1d94b53 commit 76dbd16

File tree

6 files changed

+937
-24
lines changed

6 files changed

+937
-24
lines changed

‎LICENSES-OTHER-COMPONENTS.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -599,6 +599,15 @@ License Id: ICU
599599

600600
### Backend libraries
601601

602+
#### AsciiTable
603+
604+
* Name ascii-table
605+
* Version: 0.0.8
606+
* GitHub: https://github.com/sorensen/ascii-table
607+
* License: https://github.com/sorensen/ascii-table/blob/master/license
608+
* License Name: MIT License
609+
* License Id: MIT
610+
602611
#### JSUnity
603612

604613
* Name: jsunity

‎README_maintainers.md‎

Lines changed: 80 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -852,9 +852,9 @@ Once this is completed, you may run it like this:
852852
--cluster true
853853
854854
For possible `javaOptions` see
855-
[arangodb-java-driver/dev-README.md#test-provided-deployment](https://github.com/arangodb/arangodb-java-driver/blob/next/arangodb-java-driver/dev-README.md)
855+
[arangodb-java-driver/dev-README.md#test-provided-deployment](https://github.com/arangodb/arangodb-java-driver/blob/master/dev-README.md)
856856
in the java source, or the
857-
[surefire documentation](https://maven.apache.org/surefire/maven-surefire-plugin/examples/single-test.html]
857+
[surefire documentation](https://maven.apache.org/surefire/maven-surefire-plugin/examples/single-test.html)
858858
859859
#### ArangoJS
860860
@@ -931,13 +931,24 @@ Debugging rspec with gdb:
931931

932932
server> ./scripts/unittest http_server --test api-import-spec.rb --server tcp://127.0.0.1:7777
933933
- or -
934-
server> ARANGO_SERVER="127.0.0.1:6666" rspec -Itests/rb/HttpInterface --format d --color tests/rb/HttpInterface/api-import-spec.rb
934+
server> ARANGO_SERVER="127.0.0.1:6666" \
935+
rspec -Itests/rb/HttpInterface --format d \
936+
--color tests/rb/HttpInterface/api-import-spec.rb
935937

936-
client> gdb --args ./build/bin/arangod --server.endpoint http+tcp://127.0.0.1:6666 --server.authentication false --log.level communication=trace ../arangodb-data-test-mmfiles
938+
client> gdb --args ./build/bin/arangod --server.endpoint http+tcp://127.0.0.1:6666 \
939+
--server.authentication false \
940+
--log.level communication=trace \
941+
../arangodb-data-test
937942

938943
Debugging a storage engine:
939944

940-
host> rm -fr ../arangodb-data-rocksdb/; gdb --args ./build/bin/arangod --console --server.storage-engine rocksdb --foxx.queues false --server.statistics false --server.endpoint http+tcp://0.0.0.0:7777 ../arangodb-data-rocksdb
945+
host> rm -fr ../arangodb-data-rocksdb/; \
946+
gdb --args ./build/bin/arangod \
947+
--console \
948+
--foxx.queues false \
949+
--server.statistics false \
950+
--server.endpoint http+tcp://0.0.0.0:7777 \
951+
../arangodb-data-rocksdb
941952
(gdb) catch throw
942953
(gdb) r
943954
arangod> require("jsunity").runTest("tests/js/client/shell/shell-client.js");
@@ -994,6 +1005,69 @@ Currently available Analyzers are:
9941005
- locateShortServerLife - whether the servers lifetime for the tests isn't at least 10 times as long as startup/shutdown
9951006
- locateLongSetupTeardown - locate tests that may use a lot of time in setup/teardown
9961007
- yaml - dumps the json file as a yaml file
997-
1008+
- unitTestTabularPrintResults - prints a table, add one (or more) of the following columns to print by adding it to `--tableColumns`:
1009+
- `duration` - the time spent in the complete testfile
1010+
- `status` - sucess/fail
1011+
- `failed` - fail?
1012+
- `total` - the time spent in the testcase
1013+
- `totalSetUp` - the time spent in setup summarized
1014+
- `totalTearDown` - the time spent in teardown summarized
1015+
- `processStats.sum_servers.minorPageFaults` - Delta run values from `/proc/<pid>/io` summarized over all instances
1016+
- `processStats.sum_servers.majorPageFaults` -
1017+
- `processStats.sum_servers.userTime` -
1018+
- `processStats.sum_servers.systemTime` -
1019+
- `processStats.sum_servers.numberOfThreads` -
1020+
- `processStats.sum_servers.residentSize` -
1021+
- `processStats.sum_servers.residentSizePercent` -
1022+
- `processStats.sum_servers.virtualSize` -
1023+
- `processStats.sum_servers.rchar` -
1024+
- `processStats.sum_servers.wchar` -
1025+
- `processStats.sum_servers.syscr` -
1026+
- `processStats.sum_servers.syscw` -
1027+
- `processStats.sum_servers.read_bytes` -
1028+
- `processStats.sum_servers.write_bytes` -
1029+
- `processStats.sum_servers.cancelled_write_bytes` -
1030+
- `processStats.sum_servers.sockstat_sockets_used` - Absolute values from `/proc/<pid>/net/sockstat` summarized over all instances
1031+
- `processStats.sum_servers.sockstat_TCP_inuse` -
1032+
- `processStats.sum_servers.sockstat_TCP_orphan` -
1033+
- `processStats.sum_servers.sockstat_TCP_tw` -
1034+
- `processStats.sum_servers.sockstat_TCP_alloc` -
1035+
- `processStats.sum_servers.sockstat_TCP_mem` -
1036+
- `processStats.sum_servers.sockstat_UDP_inuse` -
1037+
- `processStats.sum_servers.sockstat_UDP_mem` -
1038+
- `processStats.sum_servers.sockstat_UDPLITE_inuse` -
1039+
- `processStats.sum_servers.sockstat_RAW_inuse` -
1040+
- `processStats.sum_servers.sockstat_FRAG_inuse` -
1041+
- `processStats.sum_servers.sockstat_FRAG_memory` -
1042+
1043+
Process stats are kept by process.
1044+
So if your DB-Server had the PID `1721882`, you can dial its values by specifying
1045+
`processStats.1721882_dbserver.sockstat_TCP_tw`
1046+
into the generated table.
1047+
1048+
i.e.
9981049
9991050
./scripts/examine_results.js -- 'yaml,locateLongRunning' --readFile out/UNITTEST_RESULT.json
1051+
1052+
or:
1053+
1054+
./scripts/examine_results.js -- 'unitTestTabularPrintResults' \
1055+
--readFile out/UNITTEST_RESULT.json \
1056+
--tableColumns 'duration,processStats.sum_servers.sockstat_TCP_orphan,processStats.sum_servers.sockstat_TCP_tw
1057+
1058+
revalidating one testcase using jq:
1059+
1060+
jq '.shell_client."enterprise/tests/js/common/shell/smart-graph-enterprise-cluster.js"' < \
1061+
out/UNITTEST_RESULT.json |grep sockstat_TCP_tw
1062+
1063+
getting the PIDs of the server in the testrun using jq:
1064+
1065+
jq '.shell_client."enterprise/tests/js/common/shell/smart-graph-enterprise-cluster.js"' < \
1066+
out/UNITTEST_RESULT.json |grep '"[0-9]*_[agent|dbserver|coordinator]'
1067+
"1721674_agent": {
1068+
"1721675_agent": {
1069+
"1721676_agent": {
1070+
"1721882_dbserver": {
1071+
"1721883_dbserver": {
1072+
"1721884_dbserver": {
1073+
"1721885_coordinator": {

‎js/client/modules/@arangodb/testutils/process-utils.js‎

Lines changed: 55 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,12 @@ const createBaseConfigBuilder = function (type, options, instanceInfo, database
200200
if (!options.jwtSecret) {
201201
cfg.setAuth(options.username, options.password);
202202
}
203+
if (options.hasOwnProperty('logForceDirect')) {
204+
cfg.config['log.force-direct'] = true;
205+
}
206+
if (options.hasOwnProperty('serverRequestTimeout')) {
207+
cfg.config['server.request-timeout'] = options.serverRequestTimeout;
208+
}
203209
cfg.setDatabase(database);
204210
cfg.setEndpoint(instanceInfo.endpoint);
205211
cfg.setRootDir(instanceInfo.rootDir);
@@ -612,6 +618,15 @@ function getProcessStats(pid) {
612618
print(x.stack);
613619
throw x;
614620
}
621+
/*
622+
* rchar: 1409391
623+
* wchar: 681539
624+
* syscr: 3303
625+
* syscw: 2969
626+
* read_bytes: 0
627+
* write_bytes: 0
628+
* cancelled_write_bytes: 0
629+
*/
615630
let lineStart = 0;
616631
let maxBuffer = ioraw.length;
617632
for (let j = 0; j < maxBuffer; j++) {
@@ -622,6 +637,25 @@ function getProcessStats(pid) {
622637
processStats[x[0]] = parseInt(x[1]);
623638
}
624639
}
640+
/*
641+
* sockets: used 1272
642+
* TCP: inuse 27 orphan 0 tw 117 alloc 382 mem 25
643+
* UDP: inuse 19 mem 17
644+
* UDPLITE: inuse 0
645+
* RAW: inuse 0
646+
* FRAG: inuse 0 memory 0
647+
*/
648+
ioraw = getSockStatFile(pid);
649+
ioraw.split('\n').forEach(line => {
650+
if (line.length > 0) {
651+
let x = line.split(":");
652+
let values = x[1].split(" ");
653+
for (let k = 1; k < values.length; k+= 2) {
654+
processStats['sockstat_' + x[0] + '_' + values[k]]
655+
= parseInt(values[k + 1]);
656+
}
657+
}
658+
});
625659
}
626660
return processStats;
627661
}
@@ -635,15 +669,28 @@ function initProcessStats(instanceInfo) {
635669
function getDeltaProcessStats(instanceInfo) {
636670
try {
637671
let deltaStats = {};
672+
let deltaSum = {};
638673
instanceInfo.arangods.forEach((arangod) => {
639674
let newStats = getProcessStats(arangod.pid);
640675
let myDeltaStats = {};
641676
for (let key in arangod.stats) {
642-
myDeltaStats[key] = newStats[key] - arangod.stats[key];
677+
if (key.startsWith('sockstat_')) {
678+
myDeltaStats[key] = newStats[key];
679+
} else {
680+
myDeltaStats[key] = newStats[key] - arangod.stats[key];
681+
}
643682
}
644683
deltaStats[arangod.pid + '_' + arangod.role] = myDeltaStats;
645684
arangod.stats = newStats;
685+
for (let key in myDeltaStats) {
686+
if (deltaSum.hasOwnProperty(key)) {
687+
deltaSum[key] += myDeltaStats[key];
688+
} else {
689+
deltaSum[key] = myDeltaStats[key];
690+
}
691+
}
646692
});
693+
deltaStats['sum_servers'] = deltaSum;
647694
return deltaStats;
648695
}
649696
catch (x) {
@@ -1350,12 +1397,14 @@ function executeArangod (cmd, args, options) {
13501397
function getSockStat(arangod, options, preamble) {
13511398
if (options.getSockStat && (platform === 'linux')) {
13521399
let sockStat = preamble + arangod.pid + "\n";
1353-
try {
1354-
sockStat += fs.read("/proc/" + arangod.pid + "/net/sockstat");
1355-
return sockStat;
1356-
}
1357-
catch (e) {/* oops, process already gone? don't care. */ }
1400+
sockStat += getSockStatFile(arangod.pid);
1401+
return sockStat;
13581402
}
1403+
}
1404+
function getSockStatFile(pid) {
1405+
try {
1406+
return fs.read("/proc/" + pid + "/net/sockstat");
1407+
} catch (e) {/* oops, process already gone? don't care. */ }
13591408
return "";
13601409
}
13611410

0 commit comments

Comments
 (0)
0