8000 Cleaner test output, enhanced filters. by adityamukho · Pull Request #16021 · arangodb/arangodb · GitHub
[go: up one dir, main page]
Skip to content

Cleaner test output, enhanced filters. #16021

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 14 commits into from
Mar 31, 2022
Merged

Cleaner test output, enhanced filters. #16021

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
8071a91
Enhanced testCase filter to match suite and test case glob patterns.
Mar 18, 2022
abbcb27
Enhanced testCase filter for arrays as well.
Mar 18, 2022
bd976cc
Merge remote-tracking branch 'origin/devel' into refactor/test-framework
Mar 21, 2022
d5022f1
Added option to suppress server start/stop messages printed by proces…
Mar 21, 2022
615c01b
Merge remote-tracking branch 'origin/devel' into refactor/test-framework
Mar 24, 2022
626f6fd
More noise suppression.
Mar 24, 2022
d18a455
Set global log.level to error when noStartStopLogs is set.
Mar 24, 2022
56ff539
Merge remote-tracking branch 'origin/devel' into refactor/test-framework
Mar 29, 2022
4d75a84
Merge remote-tracking branch 'origin/devel' into refactor/test-framework
Mar 29, 2022
99d348e
Fixes PR review comments.
Mar 29, 2022
3e7eafd
Sniff only when not forbidden.
Mar 29, 2022
c85b181
Removed sniffing template.
Mar 30, 2022
c49af37
Merge remote-tracking branch 'origin/devel' into refactor/test-framework
Mar 30, 2022
c8a1df5
Inverted default option value.
Mar 30, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 48 additions & 16 deletions js/client/modules/@arangodb/testutils/process-utils.js
View file
Open in desktop
10000
Original file line number Diff line number Diff line change
Expand Up @@ -815,7 +815,7 @@ function killRemainingProcesses(results) {
print(killExternal(running[i].pid, abortSignal));
}
results.crashed = true;
};
}
}

// //////////////////////////////////////////////////////////////////////////////
Expand Down Expand Up @@ -1364,7 +1364,9 @@ function checkInstanceAlive(instanceInfo, options, {skipHealthCheck = false} = {
) {
rc = checkServersGOOD(instanceInfo);
if (first) {
print(RESET + "Waiting for all servers to go GOOD...");
if (options.extremeVerbosity || !options.noStartStopLogs) {
print(RESET + "Waiting for all servers to go GOOD...");
}
first = false;
}
}
Expand Down Expand Up @@ -1532,7 +1534,9 @@ function shutdownArangod (arangod, options, forceTerminate) {
const requestOptions = makeAuthorizationHeaders(options);
requestOptions.method = 'DELETE';
requestOptions.timeout = 60; // 60 seconds hopefully are enough for getting a response
print(Date() + ' ' + arangod.url + '/_admin/shutdown');
if (options.extremeVerbosity || !options.noStartStopLogs) {
print(Date() + ' ' + arangod.url + '/_admin/shutdown');
}
let sockStat = getSockStat(arangod, options, "Sock stat for: ");
const reply = download(arangod.url + '/_admin/shutdown', '', requestOptions);
if ((reply.code !== 200) && // if the server should reply, we expect 200 - if not:
Expand All @@ -1545,7 +1549,7 @@ function shutdownArangod (arangod, options, forceTerminate) {
print(Date() + ' Wrong shutdown response: ' + JSON.stringify(reply) + "' " + sockStat + " continuing with hard kill!");
shutdownArangod(arangod, options, true);
}
else {
else if (options.extremeVerbosity || !options.noStartStopLogs) {
print(sockStat);
}
if (options.extremeVerbosity) {
Expand Down Expand Up @@ -1622,7 +1626,9 @@ function shutDownOneInstance(options, arangod, fullInstance, counters, forceTerm
if (arangod.role !== 'agent') {
counters.nonAgenciesCount--;
}
print(Date() + ' Server "' + arangod.role + '" shutdown: Success: pid', arangod.pid);
if (options.extremeVerbosity || !options.noStartStopLogs) {
print(Date() + ' Server "' + arangod.role + '" shutdown: Success: pid', arangod.pid);
}
crashUtils.stopProcdump(options, arangod);
return false;
}
Expand Down Expand Up @@ -1664,7 +1670,9 @@ function shutdownInstance (instanceInfo, options, forceTerminate) {
let requestOptions = makeAuthorizationHeaders(options);
requestOptions.method = 'PUT';

print(coords[0].url + "/_admin/cluster/maintenance");
if (options.extremeVerbosity || !options.noStartStopLogs) {
print(coords[0].url + "/_admin/cluster/maintenance");
}
download(coords[0].url + "/_admin/cluster/maintenance", JSON.stringify("on"), requestOptions);
}
} catch (err) {
Expand Down Expand Up @@ -1697,7 +1705,10 @@ function shutdownInstance (instanceInfo, options, forceTerminate) {
if (b.role === 'agent') return -1;
return 0;
});
print(Date() + ' Shutdown order ' + JSON.stringify(toShutdown));

if (options.extremeVerbosity || !options.noStartStopLogs) {
print(Date() + ' Shutdown order ' + JSON.stringify(toShutdown));
}

let nonAgenciesCount = instanceInfo.arangods
.filter(arangod => {
Expand Down Expand Up @@ -1733,7 +1744,10 @@ function shutdownInstance (instanceInfo, options, forceTerminate) {
arangod.exitStatus = {
status: 'RUNNING'
};
print(Date() + " Commanded shut down: " + JSON.stringify(arangod));

if (options.extremeVerbosity || !options.noStartStopLogs) {
print(Date() + " Commanded shut down: " + JSON.stringify(arangod));
}
}
return true;
}
Expand Down Expand Up @@ -1787,7 +1801,9 @@ function shutdownInstance (instanceInfo, options, forceTerminate) {
if (arangod.role !== 'agent') {
nonAgenciesCount--;
}
print(Date() + ' Server "' + arangod.role + '" shutdown: Success: pid', arangod.pid);
if (options.extremeVerbosity || !options.noStartStopLogs) {
print(Date() + ' Server "' + arangod.role + '" shutdown: Success: pid', arangod.pid);
}
crashUtils.stopProcdump(options, arangod);
return false;
}
Expand All @@ -1805,7 +1821,9 @@ function shutdownInstance (instanceInfo, options, forceTerminate) {
// e.g. 2 + coordinator + (s)
roleNames.push(roles[r] + ' ' + r + '(s)');
}
print(roleNames.join(', ') + ' are still running...');
if (options.extremeVerbosity || !options.noStartStopLogs) {
print(roleNames.join(', ') + ' are still running...');
}
require('internal').wait(1, false);
}
}
Expand Down Expand Up @@ -1895,7 +1913,9 @@ function checkClusterAlive(options, instanceInfo, addArgs) {
arangod.upAndRunning = true;
return;
}
print(Date() + " tickeling cluster node " + arangod.url + " - " + arangod.role);
if (options.extremeVerbosity || !options.noStartStopLogs) {
print(Date() + " tickeling cluster node " + arangod.url + " - " + arangod.role);
}
let url = arangod.url;
if (arangod.role === "coordinator" && arangod.args["javascript.enabled"] !== "false") {
url += '/_admin/aardvark/index.html';
Expand Down Expand Up @@ -1944,7 +1964,9 @@ function checkClusterAlive(options, instanceInfo, addArgs) {
}
}

print("Determining server IDs");
if (options.extremeVerbosity || !options.noStartStopLogs) {
print("Determining server IDs");
}
instanceInfo.arangods.forEach(arangod => {
if (arangod.suspended) {
return;
Expand Down Expand Up @@ -2151,7 +2173,9 @@ function launchFinalize(options, instanceInfo, startTime) {
}
});
}
print(CYAN + Date() + ' up and running in ' + (time() - startTime) + ' seconds' + RESET);
if (options.extremeVerbosity || !options.noStartStopLogs) {
print(CYAN + Date() + ' up and running in ' + (time() - startTime) + ' seconds' + RESET);
}
var matchPort = /.*:.*:([0-9]*)/;
var ports = [];
var processInfo = [];
Expand All @@ -2178,7 +2202,9 @@ function launchFinalize(options, instanceInfo, startTime) {
' - ' + arangod.args['database.directory']);
});

print(Date() + ' sniffing template:\n tcpdump -ni lo -s0 -w /tmp/out.pcap ' + ports.join(' or ') + '\n');
if (options.extremeVerbosity || !options.noStartStopLogs) {
print(Date() + ' sniffing template:\n tcpdump -ni lo -s0 -w /tmp/out.pcap ' + ports.join(' or ') + '\n');
}
if (options.sniff !== undefined && options.sniff !== false) {
options.cleanup = false;
let device = 'lo';
Expand Down Expand Up @@ -2218,7 +2244,9 @@ function launchFinalize(options, instanceInfo, startTime) {
print(CYAN + 'launching ' + prog + ' ' + JSON.stringify(args) + RESET);
tcpdump = executeExternal(prog, args);
}
print(processInfo.join('\n') + '\n');
if (options.extremeVerbosity || !options.noStartStopLogs) {
print(processInfo.join('\n') + '\n');
}
internal.sleep(options.sleepBeforeStart);
initProcessStats(instanceInfo);
}
Expand Down Expand Up @@ -2282,6 +2310,8 @@ function startArango (protocol, options, addArgs, rootDir, role) {

if (options.verbose) {
args['log.level'] = 'debug';
} else if (!options.extremeVerbosity && options.noStartStopLogs) {
args['log.level'] = 'all=error';
}

instanceInfo.url = endpointToURL(instanceInfo.endpoint);
Expand Down Expand Up @@ -2367,7 +2397,9 @@ function startInstanceAgency (instanceInfo, protocol, options, addArgs, rootDir)
instanceInfo.endpoint = instanceInfo.arangods[instanceInfo.arangods.length - 1].endpoint;
instanceInfo.url = instanceInfo.arangods[instanceInfo.arangods.length - 1].url;
instanceInfo.role = 'agent';
print(Date() + ' Agency Endpoint: ' + instanceInfo.endpoint);
if (options.extremeVerbosity || !options.noStartStopLogs) {
print(Date() + ' Agency Endpoint: ' + instanceInfo.endpoint);
}
}

checkClusterAlive(options, instanceInfo, addArgs);
Expand Down
8 changes: 6 additions & 2 deletions js/client/modules/@arangodb/testutils/test-utils.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -466,7 +466,9 @@ function performTests (options, testList, testname, runFn, serverOptions, startS

let shutDownStart = time();
results['testDuration'] = shutDownStart - testrunStart;
print(Date() + ' Shutting down...');
if (options.extremeVerbosity || !options.noStartStopLogs) {
print(Date() + ' Shutting down...');
}
if (startStopHandlers !== undefined && startStopHandlers.hasOwnProperty('preStop')) {
customInstanceInfos['preStop'] = startStopHandlers.preStop(options,
serverOptions,
Expand Down Expand Up @@ -507,7 +509,9 @@ function performTests (options, testList, testname, runFn, serverOptions, startS
}
results['shutdownTime'] = time() - shutDownStart;

print('done.');
if (options.extremeVerbosity || !options.noStartStopLogs) {
print('done.');
}

return results;
}
Expand Down
2 changes: 2 additions & 0 deletions js/client/modules/@arangodb/testutils/testing.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,7 @@ let optionsDocumentation = [
' - `crashAnalysisText`: output of debugger in case of crash',
' - `getSockStat`: on linux collect socket stats before shutdown',
' - `verbose`: if set to true, be more verbose',
' - `noStartStopLogs`: if set to true, suppress startup and shutdown messages printed by process manager. Overridden by `extremeVerbosity`',
' - `extremeVerbosity`: if set to true, then there will be more test run',
' output, especially for cluster tests.',
' - `testCase`: filter a jsunity testsuite for one special test case',
Expand Down Expand Up @@ -229,6 +230,7 @@ const optionsDefaults = {
'valgrindArgs': {},
'valgrindHosts': false,
'verbose': false,
'noStartStopLogs': false,
'vst': false,
'http2': false,
'walFlushTimeout': 30000,
Expand Down
28 changes: 16 additions & 12 deletions js/common/modules/jsunity.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ var internal = require('internal');
var print = internal.print;
var fs = require('fs');
var console = require('console');
var minimatch = require('minimatch');

var TOTAL = 0;
var PASSED = 0;
Expand All @@ -50,6 +51,7 @@ var ENDTEST = 0.0;
var STARTSUITE = 0.0;
var ENDTEARDOWN = 0.0;
var testFilter = "undefined";
var suiteName = "undefined";
var currentSuiteName = "undefined";
var testCount = 0;
var startMessage = "";
Expand Down Expand Up @@ -193,15 +195,22 @@ jsUnity.results.endTeardownAll = function(index) {
TOTALTEARDOWNS += RESULTS.teardownAllDuration;
};

function MatchesTestFilter(key) {
function matchesTestFilter(suiteName, key) {
if (testFilter === "undefined" || testFilter === undefined || testFilter === null) {
return true;
}
if (typeof testFilter === 'string') {
return key === testFilter;
var suiteMatched = true;
if (testFilter.includes('::')) {
var [suiteFilter, testCaseFilter] = testFilter.split('::');
suiteMatched = minimatch(suiteName, suiteFilter);
testFilter = testCaseFilter;
}

return suiteMatched && minimatch(key, testFilter);
}
if (Array.isArray(testFilter)) {
return testFilter.includes(key);
return testFilter.some(key => matchesTestFilter(suiteName, key));
}
return false;
}
Expand Down Expand Up @@ -244,7 +253,7 @@ function Run (testsuite) {

for (var key in definition) {
if (key.indexOf('test') === 0) {
if (!MatchesTestFilter(key)) {
if (!matchesTestFilter(suite.suiteName, key)) {
// print(`test "${key}" doesn't match "${testFilter}", skipping`);
nonMatchedTests.push(key);
continue;
Expand All @@ -258,21 +267,16 @@ function Run (testsuite) {
}
}
if (tests.length === 0) {
let err = `There is no test in testsuite "${suite.suiteName}" or your filter "${testFilter}" didn't match on anything. We have: [${nonMatchedTests}]`;
print(`${internal.COLORS.COLOR_RED}${err}${internal.COLORS.COLOR_RESET}`);
let res = {
suiteName: suite.suiteName,
message: err,
duration: 0,
setUpDuration: 0,
tearDownDuration: 0,
passed: 0,
status: false,
failed: 1,
total: 1
status: true,
failed: 0,
total: 0
};
TOTAL += 1;
FAILED += 2;
COMPLETE[suite.suiteName] = res;
return res;
}
Expand Down
0