|
| 1 | +# Tests |
| 2 | +When creating tests, please bear in mind, that every test added will add additional time during the PR runs. |
| 3 | + |
| 4 | +So every added test *will* slow down PR runs. |
| 5 | + |
| 6 | +Please also note that test runs in the CI share the system with other parallel runs. |
| 7 | +So CI tests are not a place to max out I/O, memory, or such. |
| 8 | + |
| 9 | +So please use *BULK* inserts to slimline the creation of your test data. |
| 10 | +If you can use a common dataset for several tests, please use `setUpAll` and `tearDownAll` to reduce resource usage. |
10000
| 11 | +If splitting a testsuite between read only and writing tests is necessary for this, please do so. |
| 12 | + |
| 13 | +### Testsuites |
| 14 | +Testsuites live in `js/client/modules/@arangodb/testsuites/`. This folder is spidered during the startup of `testing.js` and all files in it are loaded automatically. |
| 15 | +Please think twice whether you need to create a new testsuite or your test will fit into one of the existing ones. All test suites need to be registered within Oskar in order to be executed during CI tests. |
| 16 | +The structure of each testsuite is as follows: |
| 17 | + |
| 18 | +``` |
| 19 | +const functionsDocumentation = { |
| 20 | + 'shell_server': 'shell server tests', |
| 21 | +}; |
| 22 | +const optionsDocumentation = [ |
| 23 | + ' - `skipAql`: if set to true the AQL tests are skipped', |
| 24 | +]; |
| 25 | +
|
| 26 | +const testPaths = { |
| 27 | + 'shell_server': [ tu.pathForTesting('common/shell'), tu.pathForTesting('server/shell') ], |
| 28 | +}; |
| 29 | +
|
| 30 | +function shellServer (options) { |
| 31 | + // spider all testcases, including twin paths in the enterprise directory: |
| 32 | + let testCases = tu.scanTestPaths(testPaths.shell_server, options); |
| 33 | + // split the testcases by `--testBuckets` parameter |
| 34 | + testCases = tu.splitBuckets(options, testCases); |
| 35 | +
|
| 36 | + // spawn the SUT, execute all tests via the `tu.runThere` function |
| 37 | + // alternatives are: |
| 38 | + // - runThere (use javascript shell exec to run the test *in* the server |
| 39 | + // - runInLocalArangosh (run in the very same arangosh as testing.js) |
| 40 | + // - runInArangosh (spawn a new arangosh, collect results via result file, more frequent process spawning) |
| 41 | + // - runInRSpec (launch tests via ruby rspec) |
| 42 | + // - runInDriverTest implemented inside the testsuite - see testsuites/driver.js as example |
| 43 | + let rc = tu.performTests(opts, testCases, 'shell_client, tu.runThere, /* startStopHandlers*/ ); |
| 44 | + // startStopHandlers can be implemented to add additional code during the startup / shutdown of the SUT: |
| 45 | + // - preStart - before starting the SUT, can be used to add additional params etc. |
| 46 | + // - postStart - invoked after the primary SUT is online. can be used to start other systems |
| 47 | + // - healthCheck - to be invoked after each test to check whether all is well |
| 48 | + // - preStop - just before the stop to do things that would hinder the shutdown |
| 49 | + // - postStop - after the stop to clean up things. |
| 50 | +
|
| 51 | + // pass up eventual errors that should prohibit flushing the SUT data |
| 52 | + options.cleanup = options.cleanup && opts.cleanup; |
| 53 | + return rc; |
| 54 | +} |
| 55 | +
|
| 56 | +// plugin hook. This is executed during loading of the modules. |
| 57 | +exports.setup = function (testFns, defaultFns, opts, fnDocs, optionsDoc, allTestPaths) { |
| 58 | + // add all used testpaths, so i.e. `auto` or `find` can find the suite for single `--test` files |
| 59 | + Object.assign(allTestPaths, testPaths); |
| 60 | + // Register the testsuite itself |
| 61 | + testFns['shell_server'] = shellServer; |
| 62 | + // enable it to be ran with `./scripts/unittest all` ; if not ommit - adds the [x] in help the list. |
| 63 | + defaultFns.push('shell_server'); |
| 64 | + // if your testsuite has CLI parameters, specify them including the default param here: |
| 65 | + opts['skipAql'] = false; |
| 66 | +
|
| 67 | + // |
| 68 | + for (var attrname in functionsDocumentation) { fnDocs[attrname] = functionsDocumentation[attrname]; } |
| 69 | + for (var i = 0; i < optionsDocumentation.length; i++) { optionsDoc.push(optionsDocumentation[i]); } |
| 70 | +}; |
| 71 | +``` |
| 72 | + |
| 73 | +### Existing Javascript tests |
| 74 | + |
| 75 | +#### [permission] => tests/js/client/permissions / [server_parameters] => tests/js/client/server_parameters / [server_permissions] => client/server_permissions |
| 76 | +This set of testsuites is here to revalidate startup parameters of arangod processes. |
| 77 | + |
| 78 | +The testfile persists of two parts, and the file is invoked twice: |
| 79 | + |
| 80 | +Controlling the startup parameters can be done with the `getOptions` part. In `server_permissions` this can also be used to setup data which may not be permitted during the later run: |
| 81 | +``` |
| 82 | +if (getOptions === true) { |
| 83 | + return { |
| 84 | + 'javascript.allow-port-testing': false |
| 85 | + }; |
| 86 | +} |
| 87 | +``` |
| 88 | +The later part of the test may contain regular jsunity testsuites. |
| 89 | + |
| 90 | +If you want to revalidate a parameter that is common to arangosh & arangod, you *should* use a test in the `permission` testsuite, |
| 91 | +since spawning an arangosh requires way less resources than spawning a full arangodb setup. |
| 92 | + |
| 93 | +If your testcase requires a setup with more permissions than the testcode itself, you should use `server_permissions`, since it will launch the SUT twice. |
| 94 | + |
| 95 | +You should however *prefer* `server_parameters` which will launch the SUT only once with the parameters you've provided. |
| 96 | + |
| 97 | +You should try to use the `-[non]cluster` filename filters to also control whether your test is ran on cluster too to reduce computing power. |
| 98 | + |
| 99 | +#### [dump*] to test dump and restore procedures |
| 100 | +The dump family of testsuites lives in `js/client/modules/@arangodb/testsuites/dump.js` |
| 101 | +and orchestrates a sequence: |
| 102 | +- launching a SUT |
| 103 | +- create test data within it |
| 104 | +- dump this test data |
| 105 | +- execute some manual sanity checks on the dump files |
| 106 | +- restore it to a fresh SUT |
| 107 | +- check data |
| 108 | + |
| 109 | +Since many create & check functions should be common to *all* suites, the default behavior is to enable them, and only exclude them if unsupported. |
| 110 | + |
| 111 | +The individual files to be used are controlled by code in the testsuite like this: |
| 112 | + |
| 113 | +``` |
| 114 | + let tstFiles = { |
| 115 | + dumpSetup: 'dump-setup' + c.cluster + '.js', |
| 116 | + dumpCheckDumpFiles: 'dump-check-dump-files-encrypted.js', |
| 117 | + dumpCleanup: 'cleanup-nothing.js', |
| 118 | + dumpAgain: 'dump' + c.cluster + '.js', |
| 119 | + dumpTearDown: 'dump-teardown' + c.cluster + '.js', |
| 120 | + foxxTest: 'check-foxx.js' |
| 121 | + }; |
| 122 | +``` |
| 123 | + |
| 124 | +So `tests/js/server/dump/dump-setup[-cluster].js` will be invoked during setting up the first SUT. |
| 125 | +It derives all its functions from `dump-setup-common.inc` and only invokes them in sequence. |
| 126 | + |
| 127 | +Revalidating will be done using `dump[-cluster].js`, which will obtain all its test functions from `dump-test.inc`. |
| 128 | + |
| 129 | +Testdata to be used during the first create etc can be put into `tests/js/server/dump`. |
| 130 | + |
| 131 | + |
| 132 | +#### [communication, chaos] testsuite to launch parallel load |
| 133 | +These testsuites are intended to launch several arangosh instances, in order to generate concurrent requests. |
| 134 | + |
| 135 | + |
| 136 | + |
| 137 | +#### [driver | go | java | js] run driver tests against the local testsuite |
| 138 | +These testsuites aim to integrate the respective drivers test with testing.js, |
| 139 | +so you can easily invoke them on your workstation. |
| 140 | + |
| 141 | +# more to come... |
| 142 | +#### shell_server |
| 143 | +tests/js/server |
| 144 | +tests/js/server/aql |
| 145 | +tests/js/server/import |
| 146 | +tests/js/server/recovery |
| 147 | +tests/js/server/replication |
| 148 | +tests/js/server/replication/fuzz |
| 149 | +tests/js/server/replication/aql |
| 150 | +tests/js/server/replication/random |
| 151 | +tests/js/server/replication/ongoing |
| 152 | +tests/js/server/replication/ongoing/frompresent |
| 153 | +tests/js/server/replication/ongoing/global |
| 154 | +tests/js/server/replication/ongoing/global/spec |
| 155 | +tests/js/server/replication/static |
| 156 | +tests/js/server/replication/sync |
| 157 | +tests/js/server/backup |
| 158 | +tests/js/server/agency-restart |
| 159 | +tests/js/server/paths |
| 160 | +tests/js/server/upgrade-data |
| 161 | +tests/js/server/stress |
| 162 | +tests/js/server/resilience |
| 163 | +tests/js/server/resilience/move-view |
| 164 | +tests/js/server/resilience/failover-view |
| 165 | +tests/js/server/resilience/transactions |
| 166 | +tests/js/server/resilience/failover |
| 167 | +tests/js/server/resilience/move |
| 168 | +tests/js/server/resilience/analyzers |
| 169 | +tests/js/server/resilience/failover-failure |
| 170 | +tests/js/server/resilience/sharddist |
| 171 | +tests/js/server/export |
| 172 | +tests/js/server/shell |
| 173 | +tests/js/common/aql |
| 174 | +tests/js/common/replication |
| 175 | +tests/js/common/http |
| 176 | +tests/js/common/shell |
| 177 | +tests/js/client |
| 178 | +tests/js/client/authentication |
| 179 | +tests/js/client/permissions |
| 180 | +tests/js/client/ldap |
| 181 | +tests/js/client/server_secrets |
| 182 | +tests/js/client/wal_cleanup |
| 183 | +tests/js/client/restart |
| 184 | +tests/js/client/assets |
| 185 | +tests/js/client/assets/queuetest |
| 186 | +tests/js/client/active-failover |
| 187 | +tests/js/client/load-balancing |
| 188 | +tests/js/client/communication |
| 189 | +tests/js/client/chaos |
| 190 | +tests/js/client/http |
| 191 | +tests/js/client/agency |
| 192 | +tests/js/client/replication2 |
| 193 | +tests/js/client/shell |
| 194 | +tests/js/client/server_permissions |
0 commit comments