8000 Add new chaos test suite. (#14406) · arangodb/arangodb@fb61d6a · GitHub
[go: up one dir, main page]
Skip to content

Commit fb61d6a

Browse files
mpoetermpoeter
authored
Add new chaos test suite. (#14406)
1 parent 0a0cfcb commit fb61d6a

File tree

7 files changed

+252
-68
lines changed
  • tests/js/client/chaos

    • 7 files changed

    +252
    -68
    lines changed

    ‎js/client/modules/@arangodb/testsuites/chaos.js

    Lines changed: 103 additions & 0 deletions
    Original file line numberDiff line numberDiff line change
    @@ -0,0 +1,103 @@
    1+
    /* jshint strict: false, sub: true */
    2+
    /* global */
    3+
    'use strict';
    4+
    5+
    ////////////////////////////////////////////////////////////////////////////////
    6+
    /// DISCLAIMER
    7+
    ///
    8+
    /// Copyright 2014-2021 ArangoDB GmbH, Cologne, Germany
    9+
    /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
    10+
    ///
    11+
    /// Licensed under the Apache License, Version 2.0 (the "License")
    12+
    /// you may not use this file except in compliance with the License.
    13+
    /// You may obtain a copy of the License at
    14+
    ///
    15+
    /// http://www.apache.org/licenses/LICENSE-2.0
    16+
    ///
    17+
    /// Unless required by applicable law or agreed to in writing, software
    18+
    /// distributed under the License is distributed on an "AS IS" BASIS,
    19+
    /// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    20+
    /// See the License for the specific language governing permissions and
    21+
    /// limitations under the License.
    22+
    ///
    23+
    /// Copyright holder is ArangoDB GmbH, Cologne, Germany
    24+
    ///
    25+
    /// @author Manuel Pöter
    26+
    ////////////////////////////////////////////////////////////////////////////////
    27+
    28+
    const functionsDocumentation = {
    29+
    'chaos': 'chaos tests'
    30+
    };
    31+
    const optionsDocumentation = [];
    32+
    33+
    const _ = require('lodash');
    34+
    const tu = require('@arangodb/testutils/test-utils');
    35+
    36+
    const testPaths = {
    37+
    'chaos': [ tu.pathForTesting('client/chaos') ],
    38+
    };
    39+
    40+
    function chaos (options) {
    41+
    let testCasesWithConfigs = {};
    42+
    let testCases = tu.scanTestPaths(testPaths.chaos, options);
    43+
    44+
    // The chaos test suite is parameterized and each configuration runs 5min.
    45+
    // For the nightly tests we want to run a large number of possible parameter
    46+
    // combinations, but each file has a runtime limit of 15min and ATM the test
    47+
    // system is not designed to allow a test file to be run multiple times with
    48+
    // different configurations.
    49+
    // The hacky solution here is to build up a map of test cases with their respective
    50+
    // configurations and have n copies of the test file in the test case list,
    51+
    // where n is the number of configurations for this test. The new `preRun`
    52+
    // callback function is called before each testCase execution. At that point we
    53+
    // pop the next config from this test case's config list and set the global
    54+
    // variable `currentTestConfig` which is then used in the tests.
    55+
    // To allow the test cases to define the possible configs I introduced the
    56+
    // possibility to write test cases as modules. A jsunity test file that contains
    57+
    // "test-module-" in its filename is not executed directly, but instead must
    58+
    // export a "run" function that is called. Such a module can optionally define
    59+
    // a function "getConfigs" which must return an array of configurations.
    60+
    61< 5957 /td>+
    testCases = _.flatMap(testCases, testCase => {
    62+
    if (testCase.includes("test-module-")) {
    63+
    const configProvider = require(testCase).getConfigs;
    64+
    if (configProvider) {
    65+
    const configs = configProvider();
    66+
    if (!Array.isArray(configs)) {
    67+
    throw "test case module " + testCase + " does not provide config list";
    68+
    }
    69+
    testCasesWithConfigs[testCase] = configs;
    70+
    return Array(configs.length).fill(testCase);
    71+
    }
    72+
    }
    73+
    return testCase;
    74+
    });
    75+
    76+
    testCases = tu.splitBuckets(options, testCases);
    77+
    78+
    let handlers = {
    79+
    preRun: (test) => {
    80+
    global.currentTestConfig = undefined;
    81+
    const configs = testCasesWithConfigs[test];
    82+
    if (Array.isArray(configs)) {
    83+
    if (configs.length === 0) {
    84+
    throw "Unexpected! Have no more configs for test case " + test;
    85+
    }
    86+
    global.currentTestConfig = configs.shift();
    87+
    }
    88+
    }
    89+
    };
    90+
    91+
    return tu.performTests(options, testCases, 'chaos', tu.runInLocalArangosh, {}, handlers);
    92+
    }
    93+
    94+
    exports.setup = function (testFns, defaultFns, opts, fnDocs, optionsDoc, allTestPaths) {
    95+
    Object.assign(allTestPaths, testPaths);
    96+
    testFns['chaos'] = chaos;
    97+
    98+
    // intentionally not turned on by default, as the suite may take a lot of time
    99+
    // defaultFns.push('chaos');
    100+
    101+
    for (var attrname in functionsDocumentation) { fnDocs[attrname] = functionsDocumentation[attrname]; }
    102+
    for (var i = 0; i < optionsDocumentation.length; i++) { optionsDoc.push(optionsDocumentation[i]); }
    103+
    };

    ‎js/client/modules/@arangodb/testutils/test-utils.js

    Lines changed: 5 additions & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -233,7 +233,11 @@ function performTests (options, testList, testname, runFn, serverOptions, startS
    233233
    break;
    234234
    }
    235235

    236+
    if (startStopHandlers && startStopHandlers.hasOwnProperty('preRun')) {
    237+
    startStopHandlers.preRun(te);
    238+
    }
    236239
    pu.getMemProfSnapshot(instanceInfo, options, memProfCounter++);
    240+
    237241
    print('\n' + (new Date()).toISOString() + GREEN + " [============] " + runFn.info + ': Trying', te, '...', RESET);
    238242
    let reply = runFn(options, instanceInfo, te, env);
    239243

    @@ -934,7 +938,7 @@ function runInLocalArangosh (options, instanceInfo, file, addArgs) {
    934938
    } catch (ex) {
    935939
    let timeout = SetGlobalExecutionDeadlineTo(0.0);
    936940
    print(RED + 'test has thrown: ' + (timeout? "because of timeout in execution":""));
    937-
    print(ex);
    941+
    print(ex, ex.stack);
    938942
    print(RESET);
    939943
    return {
    940944
    timeout: timeout,

    ‎js/common/modules/@arangodb/replication-common.js

    Lines changed: 2 additions & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -61,7 +61,8 @@ exports.reconnectRetry = function(endpoint, databaseName, user, password) {
    6161
    try {
    6262
    arango.reconnect(endpoint, databaseName, user, password);
    6363
    return;
    64-
    } catch (ex) {
    64+
    } catch (e) {
    65+
    ex = e;
    6566
    print(RED + "connecting " + endpoint + " failed - retrying (" + ex.message + ")" + RESET);
    6667
    }
    6768
    sleepTime *= 2;

    ‎js/common/modules/jsunity.js

    Lines changed: 8 additions & 3 deletions
    Original file line numberDiff line numberDiff line change
    @@ -389,10 +389,15 @@ function RunTest (path, outputReply, filter) {
    389389
    var content;
    390390
    var f;
    391391

    392-
    content = fs.read(path);
    392+
    if (path.includes("test-module-")) {
    393+
    content = `return require(${JSON.stringify(path)}).run({ runSetup, getOptions });`;
    394+
    } else {
    395+
    content = fs.read(path);
    396+
    }
    393397

    394-
    content = `(function(){ require('jsunity').jsUnity.attachAssertions(); return (function() { require('jsunity').setTestFilter(${JSON.stringify(filter)}); const runSetup = false; const getOptions = false; ${content} }());
    395-
    });`;
    398+
    // NOTE: this is intentionally a single long line to ensure that any line information
    399+
    // refers to the correct line in the original source file.
    400+
    content = `(function(){ require('jsunity').jsUnity.attachAssertions(); return (function() { require('jsunity').setTestFilter(${JSON.stringify(filter)}); const runSetup = false; const getOptions = false; ${content} }()); });`;
    396401
    f = internal.executeScript(content, undefined, path);
    397402

    398403
    if (f === undefined) {

    ‎tests/js/client/communication/test-chaos-cluster.js renamed to ‎tests/js/client/chaos/test-chaos-load-common.inc

    Lines changed: 64 additions & 63 deletions
    Original file line numberDiff line numberDiff line change
    @@ -1,28 +1,30 @@
    11
    /* jshint globalstrict:false, strict:false, maxlen: 200 */
    22
    /* global fail, assertTrue, assertFalse, assertEqual,
    33
    assertNotEqual, arango, print */
    4+
    5+
    ////////////////////////////////////////////////////////////////////////////////
    6+
    /// DISCLAIMER
    7+
    ///
    8+
    /// Copyright 2014-2021 ArangoDB GmbH, Cologne, Germany
    9+
    /// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
    10+
    ///
    11+
    /// Licensed under the Apache License, Version 2.0 (the "License")
    12+
    /// you may not use this file except in compliance with the License.
    13+
    /// You may obtain a copy of the License at
    14+
    ///
    15+
    /// http://www.apache.org/licenses/LICENSE-2.0
    16+
    ///
    17+
    /// Unless required by applicable law or agreed to in writing, software
    18+
    /// distributed under the License is distributed on an "AS IS" BASIS,
    19+
    /// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    20+
    /// See the License for the specific language governing permissions and
    21+
    /// limitations under the License.
    22+
    ///
    23+
    /// Copyright holder is ArangoDB GmbH, Cologne, Germany
    24+
    ///
    25+
    /// @author Manuel Pöter
    26+
    ////////////////////////////////////////////////////////////////////////////////
    427

    5-
    // //////////////////////////////////////////////////////////////////////////////
    6-
    // / DI 3D24 SCLAIMER
    7-
    // /
    8-
    // / Copyright 2018 ArangoDB GmbH, Cologne, Germany
    9-
    // /
    10-
    // / Licensed under the Apache License, Version 2.0 (the "License")
    11-
    // / you may not use this file except in compliance with the License.
    12-
    // / You may obtain a copy of the License at
    13-
    // /
    14-
    // / http://www.apache.org/licenses/LICENSE-2.0
    15-
    // /
    16-
    // / Unless required by applicable law or agreed to in writing, software
    17-
    // / distributed under the License is distributed on an "AS IS" BASIS,
    18-
    // / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    19-
    // / See the License for the specific language governing permissions and
    20-
    // / limitations under the License.
    21-
    // /
    22-
    // / Copyright holder is triAGENS GmbH, Cologne, Germany
    23-
    // /
    24-
    // / @author Manuel Pöter
    25-
    // //////////////////////////////////////////////////////////////////////////////
    2628
    'use strict';
    2729
    const _ = require('lodash');
    2830
    const jsunity = require('jsunity');
    @@ -122,16 +124,15 @@ const clearAllFailurePoints = () => {
    122124
    function BaseChaosSuite(testOpts) {
    123125
    // generate a random collection name
    124126
    const cn = "UnitTests" + require("@arangodb/crypto").md5(internal.genRandomAlphaNumbers(32));
    125-
    const communication_cn = cn + "_comm";
    126-
    127+
    const coordination_cn = cn + "_coord";
    127128

    128129
    return {
    129130
    setUp: function () {
    130131
    db._drop(cn);
    131-
    db._drop(communication_cn);
    132+
    db._drop(coordination_cn);
    132133
    let rf = Math.max(2, getDBServers().length);
    133134
    db._create(cn, { numberOfShards: rf * 2, replicationFactor: rf });
    134-
    db._create(communication_cn);
    135+
    db._create(coordination_cn);
    135136

    136137
    if (testOpts.withFailurePoints) {
    137138
    let servers = getDBServers();
    @@ -160,12 +161,12 @@ function BaseChaosSuite(testOpts) {
    160161
    clearAllFailurePoints();
    161162
    db._drop(cn);
    162163

    163-
    const shells = db[communication_cn].all();
    164+
    const shells = db[coordination_cn].all();
    164165
    if (shells.length > 0) {
    165-
    print("Found remaining docs in communication collection:");
    166+
    print("Found remaining docs in coordination collection:");
    166167
    print(shells);
    167168
    }
    168-
    db._drop(communication_cn);
    169+
    db._drop(coordination_cn);
    169170
    },
    170171

    171172
    testWorkInParallel: function () {
    @@ -176,8 +177,12 @@ function BaseChaosSuite(testOpts) {
    176177
    const key = () => "testmann" + Math.floor(Math.random() * 100000000);
    177178
    const docs = () => {
    178179
    let result = [];
    179-
    // TODO - optionally use only single-document operations
    180-
    const r = Math.floor(Math.random() * 2000) + 1;
    180+
    const max = 2000;
    181+
    const r = Math.floor(Math.random() * max) + 1;
    182+
    if (r > (max * 0.8)) {
    183+
    // we want ~20% of all requests to be single document operations
    184+
    r = 1;
    185+
    }
    181186
    for (let i = 0; i < r; ++i) {
    182187
    result.push({ _key: key() });
    183188
    }
    @@ -208,26 +213,24 @@ function BaseChaosSuite(testOpts) {
    208213

    209214
    let query = (...args) => db._query(...args);
    210215
    let trx = null;
    211-
    let isTransaction = false;
    212216
    if (testOpts.withStreamingTransactions && Math.random() < 0.5) {
    213217
    trx = db._createTransaction({ collections: { write: [c.name()] } });
    214218
    c = trx.collection(testOpts.collection);
    215219
    query = (...args) => trx.query(...args);
    216-
    isTransaction = true;
    217220
    }
    218221

    219-
    const logAllOps = false;
    222+
    const logAllOps = false; // can be set to true for debugging purposes
    220223
    const log = (msg) => {
    221224
    if (logAllOps) {
    222225
    console.info(msg);
    223226
    }
    224227
    };
    225-
    const ops = isTransaction ? Math.floor(Math.random() * 5) + 1 : 1;
    228+
    const ops = trx === null ? 1 : Math.floor(Math.random() * 5) + 1;
    226229
    for (let op = 0; op < ops; ++op) {
    227230
    try {
    228231
    const d = Math.random();
    229232
    if (d >= 0.98 && testOpts.withTruncate) {
    230-
    console.warn("RUNNING TRUNCATE");
    233+
    log("RUNNING TRUNCATE");
    231234
    c.truncate();
    232235
    } else if (d >= 0.9) {
    233236
    let o = opts();
    @@ -248,10 +251,12 @@ function BaseChaosSuite(testOpts) {
    248251
    let o = opts();
    249252
    let d = docs();
    250253
    log(`RUNNING INSERT WITH ${d.length} DOCS. OPTIONS: ${JSON.stringify(o)}`);
    254+
    d = d.length == 1 ? d[0] : d;
    251255
    c.insert(d, o);
    252256
    } else {
    253257
    let d = docs();
    254258
    log(`RUNNING REMOVE WITH ${d.length} DOCS`);
    259+
    d = d.length == 1 ? d[0] : d;
    255260
    c.remove(d);
    256261
    }
    257262
    } catch (err) {}
    @@ -275,7 +280,7 @@ function BaseChaosSuite(testOpts) {
    275280
    }
    276281

    277282
    // run the suite for 5 minutes
    278-
    runParallelArangoshTests(tests, 5 * 60, communication_cn);
    283+
    runParallelArangoshTests(tests, 5 * 60, coordination_cn);
    279284

    280285
    print("Finished load test - waiting for cluster to get in sync before checking consistency.");
    281286
    clearAllFailurePoints();
    @@ -285,44 +290,40 @@ function BaseChaosSuite(testOpts) {
    285290
    };
    286291
    }
    287292

    288-
    function BuildChaosSuite(opts, suffix) {
    289-
    let suite = {};
    290-
    deriveTestSuite(BaseChaosSuite(opts), suite, suffix);
    291-
    return suite;
    292-
    }
    293-
    294-
    const params = ["IntermediateCommits", "FailurePoints", "Truncate", "VaryingOverwriteMode", "StreamingTransactions"];
    293+
    // truncate is disabled because it does not work reliably ATM
    294+
    const params = ["IntermediateCommits", "FailurePoints", /*"Truncate",*/ "VaryingOverwriteMode", "StreamingTransactions"];
    295295

    296-
    function addSuite(paramValues) {
    296+
    const makeConfig = (paramValues) => {
    297297
    let suffix = "";
    298298
    let opts = {};
    299299
    for (let j = 0; j < params.length; ++j) {
    300300
    suffix += paramValues[j] ? "_with_" : "_no_";
    301301
    suffix += params[j];
    302302
    opts["with" + params[j]] = paramValues[j];
    303-
    }
    304-
    let func = function() { return BuildChaosSuite(opts, suffix); };
    303+
    }
    304+
    return { suffix: suffix, options: opts };
    305+
    }
    306+
    307+
    const run = () => {
    308+
    if (!global.currentTestConfig) {
    309+
    throw "Chaos test requires global currentTestConfig to be defined!";
    310+
    }
    311+
    const { options, suffix } = global.currentTestConfig;
    312+
    print("Running chaos test with config ", options);
    313+
    314+
    let func = function() {
    315+
    let suite = {};
    316+
    deriveTestSuite(BaseChaosSuite(options), suite, suffix);
    317+
    return suite;
    318+
    };
    305319
    // define the function name as it shows up as suiteName
    306320
    Object.defineProperty(func, 'name', {value: "ChaosSuite" + suffix, writable: false});
    307321

    308322
    jsunity.run(func);
    309-
    }
    310323

    311-
    /*
    312-
    This code dynamically creates test suites for all parameter combinations.
    313-
    ATM we don't use it since we don't want to include ALL suites in the PR tests, but
    314-
    we do want to have them in the nightlies, but that will be done in a separate PR.
    315-
    for (let i = 0; i < (1 << params.length); ++i) {
    316-
    let paramValues = [];
    317-
    for (let j = params.length - 1; j >= 0; --j) {
    318-
    paramValues.push(Boolean(i & (1 << j)));
    319-
    }
    320-
    addSuite(paramValues);
    324+
    return jsunity.done();
    321325
    }
    322-
    */
    323-
    324-
    // ATM we only create a single suite with all options except Truncate, because there are still known issues.
    325-
    // Later we probably want to have all possible combinations, at least for the nightly builds.
    326-
    addSuite([true, true, false, true, true]);
    327326

    328-
    return jsunity.done();
    327+
    module.exports.parameters = params;
    328+
    module.exports.makeConfig = makeConfig;
    329+
    module.exports.run = run;

    0 commit comments

    Comments
     (0)
    0