8000 Add new chaos test suite. (#14406) · arangodb/arangodb@ffd528a · GitHub
[go: up one dir, main page]
Skip to content

Commit ffd528a

Browse files
mpoetermaierlars
mpoeter
authored and
maierlars
committed
Add new chaos test suite. (#14406)
1 parent 28930dd commit ffd528a

File tree

7 files changed

+252
-68
lines changed

7 files changed

+252
-68
lines changed

‎js/client/modules/@arangodb/testsuites/chaos.js

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
/* jshint strict: false, sub: true */
2+
/* global */
3+
'use strict';
4+
5+
////////////////////////////////////////////////////////////////////////////////
6+
/// DISCLAIMER
7+
///
8+
/// Copyright 2014-2021 ArangoDB GmbH, Cologne, Germany
9+
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
10+
///
11+
/// Licensed under the Apache License, Version 2.0 (the "License")
12+
/// you may not use this file except in compliance with the License.
13+
/// You may obtain a copy of the License at
14+
///
15+
/// http://www.apache.org/licenses/LICENSE-2.0
16+
///
17+
/// Unless required by applicable law or agreed to in writing, software
18+
/// distributed under the License is distributed on an "AS IS" BASIS,
19+
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20+
/// See the License for the specific language governing permissions and
21+
/// limitations under the License.
22+
///
23+
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
24+
///
25+
/// @author Manuel Pöter
26+
////////////////////////////////////////////////////////////////////////////////
27+
28+
const functionsDocumentation = {
29+
'chaos': 'chaos tests'
30+
};
31+
const optionsDocumentation = [];
32+
33+
const _ = require('lodash');
34+
const tu = require('@arangodb/testutils/test-utils');
35+
36+
const testPaths = {
37+
'chaos': [ tu.pathForTesting('client/chaos') ],
38+
};
39+
40+
function chaos (options) {
41+
let testCasesWithConfigs = {};
42+
let testCases = tu.scanTestPaths(testPaths.chaos, options);
43+
44+
// The chaos test suite is parameterized and each configuration runs 5min.
45+
// For the nightly tests we want to run a large number of possible parameter
46+
// combinations, but each file has a runtime limit of 15min and ATM the test
47+
// system is not designed to allow a test file to be run multiple times with
48+
// different configurations.
49+
// The hacky solution here is to build up a map of test cases with their respective
50+
// configurations and have n copies of the test file in the test case list,
51+
// where n is the number of configurations for this test. The new `preRun`
52+
// callback function is called before each testCase execution. At that point we
53+
// pop the next config from this test case's config list and set the global
54+
// variable `currentTestConfig` which is then used in the tests.
55+
// To allow the test cases to define the possible configs I introduced the
56+
// possibility to write test cases as modules. A jsunity test file that contains
57+
// "test-module-" in its filename is not executed directly, but instead must
58+
// export a "run" function that is called. Such a module can optionally define
59+
// a function "getConfigs" which must return an array of configurations.
60+
61+
testCases = _.flatMap(testCases, testCase => {
62+
if (testCase.includes("test-module-")) {
63+
const configProvider = require(testCase).getConfigs;
64+
if (configProvider) {
65+
const configs = configProvider();
66+
if (!Array.isArray(configs)) {
67+
throw "test case module " + testCase + " does not provide config list";
68+
}
69+
testCasesWithConfigs[testCase] = configs;
70+
return Array(configs.length).fill(testCase);
71+
}
72+
}
73+
return testCase;
74+
});
75+
76+
testCases = tu.splitBuckets(options, testCases);
77+
78+
let handlers = {
79+
preRun: (test) => {
80+
global.currentTestConfig = undefined;
81+
const configs = testCasesWithConfigs[test];
82+
if (Array.isArray(configs)) {
83+
if (configs.length === 0) {
84+
throw "Unexpected! Have no more configs for test case " + test;
85+
}
86+
global.currentTestConfig = configs.shift();
87+
}
88+
}
89+
};
90+
91+
return tu.performTests(options, testCases, 'chaos', tu.runInLocalArangosh, {}, handlers);
92+
}
93+
94+
exports.setup = function (testFns, defaultFns, opts, fnDocs, optionsDoc, allTestPaths) {
95+
Object.assign(allTestPaths, testPaths);
96+
testFns['chaos'] = chaos;
97+
98+
// intentionally not turned on by default, as the suite may take a lot of time
99+
// defaultFns.push('chaos');
100+
101+
for (var attrname in functionsDocumentation) { fnDocs[attrname] = functionsDocumentation[attrname]; }
102+
for (var i = 0; i < optionsDocumentation.length; i++) { optionsDoc.push(optionsDocumentation[i]); }
103+
};

‎js/client/modules/@arangodb/testutils/test-utils.js

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -233,7 +233,11 @@ function performTests (options, testList, testname, runFn, serverOptions, startS
233233
break;
234234
}
235235

236+
if (startStopHandlers && startStopHandlers.hasOwnProperty('preRun')) {
237+
startStopHandlers.preRun(te);
238+
}
236239
pu.getMemProfSnapshot(instanceInfo, options, memProfCounter++);
240+
237241
print('\n' + (new Date()).toISOString() + GREEN + " [============] " + runFn.info + ': Trying', te, '...', RESET);
238242
let reply = runFn(options, instanceInfo, te, env);
239243

@@ -934,7 +938,7 @@ function runInLocalArangosh (options, instanceInfo, file, addArgs) {
934938
} catch (ex) {
935939
let timeout = SetGlobalExecutionDeadlineTo(0.0);
936940
print(RED + 'test has thrown: ' + (timeout? "because of timeout in execution":""));
937-
print(ex);
941+
print(ex, ex.stack);
938942
print(RESET);
939943
return {
940944
timeout: timeout,

‎js/common/modules/@arangodb/replication-common.js

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,8 @@ exports.reconnectRetry = function(endpoint, databaseName, user, password) {
6161
try {
6262
arango.reconnect(endpoint, databaseName, user, password);
6363
return;
64-
} catch (ex) {
64+
} catch (e) {
65+
ex = e;
6566
print(RED + "connecting " + endpoint + " failed - retrying (" + ex.message + ")" + RESET);
6667
}
6768
sleepTime *= 2;

‎js/common/modules/jsunity.js

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -389,10 +389,15 @@ function RunTest (path, outputReply, filter) {
389389
var content;
390390
var f;
391391

392-
content = fs.read(path);
392+
if (path.includes("test-module-")) {
393+
content = `return require(${JSON.stringify(path)}).run({ runSetup, getOptions });`;
394+
} else {
395+
content = fs.read(path);
396+
}
393397

394-
content = `(function(){ require('jsunity').jsUnity.attachAssertions(); return (function() { require('jsunity').setTestFilter(${JSON.stringify(filter)}); const runSetup = false; const getOptions = false; ${content} }());
395-
});`;
398+
// NOTE: this is intentionally a single long line to ensure that any line information
399+
// refers to the correct line in the original source file.
400+
content = `(function(){ require('jsunity').jsUnity.attachAssertions(); return (function() { require('jsunity').setTestFilter(${JSON.stringify(filter)}); const runSetup = false; const getOptions = false; ${content} }()); });`;
396401
f = internal.executeScript(content, undefined, path);
397402

398403
if (f === undefined) {

‎tests/js/client/communication/test-chaos-cluster.js renamed to ‎tests/js/client/chaos/test-chaos-load-common.inc

Lines changed: 64 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,30 @@
11
/* jshint globalstrict:false, strict:false, maxlen: 200 */
22
/* global fail, assertTrue, assertFalse, assertEqual,
33
assertNotEqual, arango, print */
4+
5+
////////////////////////////////////////////////////////////////////////////////
6+
/// DISCLAIMER
7+
///
8+
/// Copyright 2014-2021 ArangoDB GmbH, Cologne, Germany
9+
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
10+
///
11+
/// Licensed under the Apache License, Version 2.0 (the "License")
12+
/// you may not use this file except in compliance with the License.
13+
/// You may obtain a copy of the License at
14+
///
15+
/// http://www.apache.org/licenses/LICENSE-2.0
16+
///
17+
/// Unless required by applicable law or agreed to in writing, software
18+
/// distributed under the License is distributed on an "AS IS" BASIS,
19+
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20+
/// See the License for the specific language governing permissions and
21+
/// limitations under the License.
22+
///
23+
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
24+
///
25+
/// @author Manuel Pöter
26+
////////////////////////////////////////////////////////////////////////////////
427

5-
// //////////////////////////////////////////////////////////////////////////////
6-
// / DISCLAIMER
7-
// /
8-
// / Copyright 2018 ArangoDB GmbH, Cologne, Germany
9-
// /
10-
// / Licensed under the Apache License, Version 2.0 (the "License")
11-
// / you may not use this file except in compliance with the License.
12-
// / You may obtain a copy of the License at
13-
// /
14-
// / http://www.apache.org/licenses/LICENSE-2.0
15-
// /
16-
// / Unless required by applicable law or agreed to in writing, software
17-
// / distributed under the License is distributed on an "AS IS" BASIS,
18-
// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19-
// / See the License for the specific language governing permissions and
20-
// / limitations under the License.
21-
// /
22-
// / Copyright holder is triAGENS GmbH, Cologne, Germany
23-
// /
24-
// / @author Manuel Pöter
25-
// //////////////////////////////////////////////////////////////////////////////
2628
'use strict';
2729
const _ = require('lodash');
2830
const jsunity = require('jsunity');
@@ -122,16 +124,15 @@ const clearAllFailurePoints = () => {
122124
function BaseChaosSuite(testOpts) {
123125
// generate a random collection name
124126
const cn = "UnitTests" + require("@arangodb/crypto").md5(internal.genRandomAlphaNumbers(32));
125-
const communication_cn = cn + "_comm";
126-
127+
const coordination_cn = cn + "_coord";
127128

128129
return {
129130
setUp: function () {
130131
db._drop(cn);
131-
db._drop(communication_cn);
132+
db._drop(coordination_cn);
132133
let rf = Math.max(2, getDBServers().length);
133134
db._create(cn, { numberOfShards: rf * 2, replicationFactor: rf });
134-
db._create(communication_cn);
135+
db._create(coordination_cn);
135136

136137
if (testOpts.withFailurePoints) {
137138
let servers = getDBServers();
@@ -160,12 +161,12 @@ function BaseChaosSuite(testOpts) {
160161
clearAllFailurePoints();
161162
db._drop(cn);
162163

163-
const shells = db[communication_cn].all();
164+
const shells = db[coordination_cn].all();
164165
if (shells.length > 0) {
165-
print("Found remaining docs in communication collection:");
166+
print("Found remaining docs in coordination collection:");
166167
print(shells);
167168
}
168-
db._drop(communication_cn);
169+
db._drop(coordination_cn);
169170
},
170171

171172
testWorkInParallel: function () {
@@ -176,8 +177,12 @@ function BaseChaosSuite(testOpts) {
176177
const key = () => "testmann" + Math.floor(Math.random() * 100000000);
177178
const docs = () => {
178179
let result = [];
179-
// TODO - optionally use only single-document operations
180-
const r = Math.floor(Math.random() * 2000) + 1;
180+
const max = 2000;
181+
const r = Math.floor(Math.random() * max) + 1;
182+
if (r > (max * 0.8)) {
183+
// we want ~20% of all requests to be single document operations
184+
r = 1;
185+
}
181186
for (let i = 0; i < r; ++i) {
182187
result.push({ _key: key() });
183188
}
@@ -208,26 +213,24 @@ function BaseChaosSuite(testOpts) {
208213

209214
let query = (...args) => db._query(...args);
210215
let trx = null;
211-
let isTransaction = false;
212216
if (testOpts.withStreamingTransactions && Math.random() < 0.5) {
213217
trx = db._createTransaction({ collections: { write: [c.name()] } });
214218
c = trx.collection(testOpts.collection);
215219
query = (...args) => trx.query(...args);
216-
isTransaction = true;
217220
}
218221

219-
const logAllOps = false;
222+
const logAllOps = false; // can be set to true for debugging purposes
220223
const log = (msg) => {
221224
if (logAllOps) {
222225
console.info(msg);
223226
}
224227
};
225-
const ops = isTransaction ? Math.floor(Math.random() * 5) + 1 : 1;
228+
const ops = trx === null ? 1 : Math.floor(Math.random() * 5) + 1;
226229
for (let op = 0; op < ops; ++op) {
227230
try {
228231
const d = Math.random();
229232
if (d >= 0.98 && testOpts.withTruncate) {
230-
console.warn("RUNNING TRUNCATE");
233+
log("RUNNING TRUNCATE");
231234
c.truncate();
232235
} else if (d >= 0.9) {
233236
let o = opts();
@@ -248,10 +251,12 @@ function BaseChaosSuite(testOpts) {
248251
let o = opts();
249252
let d = docs();
250253
log(`RUNNING INSERT WITH ${d.length} DOCS. OPTIONS: ${JSON.stringify(o)}`);
254+
d = d.length == 1 ? d[0] : d;
251255
c.insert(d, o);
252256
} else {
253257
let d = docs();
254258
log(`RUNNING REMOVE WITH ${d.length} DOCS`);
259+
d = d.length == 1 ? d[0] : d;
255260
c.remove(d);
256261
}
257262
} catch (err) {}
@@ -275,7 +280,7 @@ function BaseChaosSuite(testOpts) {
275280
}
276281

277282
// run the suite for 5 minutes
278-
runParallelArangoshTests(tests, 5 * 60, communication_cn);
283+
runParallelArangoshTests(tests, 5 * 60, coordination_cn);
279284

280285
print("Finished load test - waiting for cluster to get in sync before checking consistency.");
281286
clearAllFailurePoints();
@@ -285,44 +290,40 @@ function BaseChaosSuite(testOpts) {
285290
};
286291
}
287292

288-
function BuildChaosSuite(opts, suffix) {
289-
let suite = {};
290-
deriveTestSuite(BaseChaosSuite(opts), suite, suffix);
291-
return suite;
292-
}
293-
294-
const params = ["IntermediateCommits", "FailurePoints", "Truncate", "VaryingOverwriteMode", "StreamingTransactions"];
293+
// truncate is disabled because it does not work reliably ATM
294+
const params = ["IntermediateCommits", "FailurePoints", /*"Truncate",*/ "VaryingOverwriteMode", "StreamingTransactions"];
295295

296-
function addSuite(paramValues) {
296+
const makeConfig = (paramValues) => {
297297
let suffix = "";
298298
let opts = {};
299299
for (let j = 0; j < params.length; ++j) {
300300
suffix += paramValues[j] ? "_with_" : "_no_";
301301
suffix += params[j];
302302
opts["with" + params[j]] = paramValues[j];
303-
}
304-
let func = function() { return BuildChaosSuite(opts, suffix); };
303+
}
304+
return { suffix: suffix, options: opts };
305+
}
306+
307+
const run = () => {
308+
if (!global.currentTestConfig) {
309+
throw "Chaos test requires global currentTestConfig to be defined!";
310+
}
311+
const { options, suffix } = global.currentTestConfig;
312+
print("Running chaos test with config ", options);
313+
314+
let func = function() {
315+
let suite = {};
316+
deriveTestSuite(BaseChaosSuite(options), suite, suffix);
317+
return suite;
318+
};
305319
// define the function name as it shows up as suiteName
306320
Object.defineProperty(func, 'name', {value: "ChaosSuite" + suffix, writable: false});
307321

308322
jsunity.run(func);
309-
}
310323

311-
/*
312-
This code dynamically creates test suites for all parameter combinations.
313-
ATM we don't use it since we don't want to include ALL suites in the PR tests, but
314-
we do want to have them in the nightlies, but that will be done in a separate PR.
315-
for (let i = 0; i < (1 << params.length); ++i) {
316-
let paramValues = [];
317-
for (let j = params.length - 1; j >= 0; --j) {
318-
paramValues.push(Boolean(i & (1 << j)));
319-
}
320-
addSuite(paramValues);
324+
return jsunity.done();
321325
}
322-
*/
323-
324-
// ATM we only create a single suite with all options except Truncate, because there are still known issues.
325-
// Later we probably want to have all possible combinations, at least for the nightly builds.
326-
addSuite([true, true, false, true, true]);
327326

328-
return jsunity.done();
327+
module.exports.parameters = params;
328+
module.exports.makeConfig = makeConfig;
329+
module.exports.run = run;

0 commit comments

Comments
 (0)
0