10BC0 add REST API endpoint GET /_admin/debug/failat/all (#14687) · arangodb/arangodb@4e555c6 · GitHub
[go: up one dir, main page]
Skip to content

Commit 4e555c6

Browse files
jsteemannjsteemann
authored
add REST API endpoint GET /_admin/debug/failat/all (#14687)
1 parent a1e8b50 commit 4e555c6

File tree

9 files changed

+254
-56
lines changed

9 files changed

+254
-56
lines changed

‎CHANGELOG‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
devel
22
-----
33

4+
* Added REST API endpoint `/_admin/debug/failat/all` to retrieve the list
5+
of currently enabled failure points. This API is available only if
6+
failure testing is enabled, but not in production.
7+
48
* APM-60: optionally allow special characters and Unicode characters in
59
database names.
610

‎arangod/RestHandler/RestDebugHandler.cpp‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,13 @@ RestStatus RestDebugHandler::execute() {
5050
switch (type) {
5151
case rest::RequestType::GET: {
5252
VPackBuilder result;
53-
result.add(VPackValue(TRI_CanUseFailurePointsDebugging()));
53+
if (len == 2 && suffixes[1] == "all") {
54+
// return all currently set failure points
55+
TRI_GetFailurePointsDebugging(result);
56+
} else {
57+
// return whether we can use failure points or not
58+
result.add(VPackValue(TRI_CanUseFailurePointsDebugging()));
59+
}
5460
generateResult(rest::ResponseCode::OK, result.slice());
5561
return RestStatus::DONE;
5662
}

‎js/client/modules/@arangodb/test-helper.js‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -129,19 +129,22 @@ exports.debugClearFailAt = function (endpoint) {
129129
}
130130
};
131131

132-
133-
exports.debugClearFailAt = function (endpoint) {
132+
exports.debugGetFailurePoints = function (endpoint) {
134133
const primaryEndpoint = arango.getEndpoint();
135134
try {
136135
reconnectRetry(endpoint, db._name(), "root", "");
137-
let res = arango.DELETE_RAW('/_admin/debug/failat');
138-
if (res.code !== 200) {
139-
throw "Error removing failure points";
136+
let haveFailAt = arango.GET("/_admin/debug/failat") === true;
137+
if (haveFailAt) {
138+
let res = arango.GET_RAW('/_admin/debug/failat/all');
139+
if (res.code !== 200) {
140+
throw "Error checking failure points = " + JSON.stringify(res);
141+
}
142+
return res.parsedBody;
140143
}
141-
return true;
142144
} finally {
143145
reconnectRetry(primaryEndpoint, "_system", "root", "");
144146
}
147+
return [];
145148
};
146149

147150
exports.getChecksum = function (endpoint, name) {
@@ -177,8 +180,6 @@ exports.getMetric = function (endpoint, name) {
177180
}
178181
};
179182

180-
const arangosh = fs.join(global.ARANGOSH_PATH, 'arangosh' + pu.executableExt);
181-
182183
const debug = function (text) {
183184
console.warn(text);
184185
};
@@ -205,7 +206,7 @@ const runShell = function(args, prefix) {
205206
argv.push(options['javascript.module-directory'][o]);
206207
}
207208

208-
let result = internal.executeExternal(arangosh, argv, false /*usePipes*/);
209+
let result = internal.executeExternal(global.ARANGOSH_BIN, argv, false /*usePipes*/);
209210
assertTrue(result.hasOwnProperty('pid'));
210211
let status = internal.statusExternal(result.pid);
211212
assertEqual(status.status, "RUNNING");
@@ -246,7 +247,7 @@ while (++saveTries < 100) {
246247
exports.runShell = runShell;
247248

248249
exports.runParallelArangoshTests = function (tests, duration, cn) {
249-
assertTrue(fs.isFile(arangosh), "arangosh executable not found!");
250+
assertTrue(fs.isFile(global.ARANGOSH_BIN), "arangosh executable not found!");
250251

251252
assertFalse(db[cn].exists("stop"));
252253
let clients = [];

‎js/client/modules/@arangodb/testutils/process-utils.js‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ const internal = require('internal');
3434
const crashUtils = require('@arangodb/testutils/crash-utils');
3535
const crypto = require('@arangodb/crypto');
3636
const ArangoError = require('@arangodb').ArangoError;
37+
const debugGetFailurePoints = require('@arangodb/test-helper').debugGetFailurePoints;
3738

3839
/* Functions: */
3940
const toArgv = internal.toArgv;
@@ -372,6 +373,7 @@ function setupBinaries (builddir, buildType, configDir) {
372373
throw new Error('unable to locate ' + checkFiles[b]);
373374
}
374375
}
376+
global.ARANGOSH_BIN = ARANGOSH_BIN;
375377
}
376378

377379
// //////////////////////////////////////////////////////////////////////////////
@@ -1365,6 +1367,31 @@ function checkInstanceAlive (instanceInfo, options) {
13651367
return rc;
13661368
}
13671369

1370+
// //////////////////////////////////////////////////////////////////////////////
1371+
// / @brief checks whether any instance has failure points set
1372+
// //////////////////////////////////////////////////////////////////////////////
1373+
1374+
function checkServerFailurePoints(instanceInfo) {
1375+
let failurePoints = [];
1376+
instanceInfo.arangods.forEach(arangod => {
1377+
// we don't have JWT success atm, so if, skip:
1378+
if ((arangod.role !== "agent") &&
1379+
!arangod.args.hasOwnProperty('server.jwt-secret-folder') &&
1380+
!arangod.args.hasOwnProperty('server.jwt-secret')) {
1381+
let fp = debugGetFailurePoints(arangod.endpoint);
1382+
if (fp.length > 0) {
1383+
failurePoints.push({
1384+
"role": arangod.role,
1385+
"pid": arangod.pid,
1386+
"database.directory": arangod['database.directory'],
1387+
"failurePoints": fp
1388+
});
1389+
}
1390+
}
1391+
});
1392+
return failurePoints;
1393+
}
1394+
13681395
// //////////////////////////////////////////////////////////////////////////////
13691396
// / @brief waits for garbage collection using /_admin/execute
13701397
// //////////////////////////////////////////////////////////////////////////////
@@ -2478,6 +2505,7 @@ exports.run = {
24782505
exports.shutdownInstance = shutdownInstance;
24792506
exports.getProcessStats = getProcessStats;
24802507
exports.getDeltaProcessStats = getDeltaProcessStats;
2508+
exports.checkServerFailurePoints = checkServerFailurePoints;
24812509
exports.summarizeStats = summarizeStats;
24822510
exports.getMemProfSnapshot = getMemProfSnapshot;
24832511
exports.startArango = startArango;

‎js/client/modules/@arangodb/testutils/test-utils.js‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,7 @@ function performTests (options, testList, testname, runFn, serverOptions, startS
267267
break;
268268
}
269269
}
270+
270271
if (pu.arangod.check.instanceAlive(instanceInfo, options) &&
271272
healthCheck(options, serverOptions, instanceInfo, customInstanceInfos, startStopHandlers)) {
272273
continueTesting = true;
@@ -355,6 +356,18 @@ function performTests (options, testList, testname, runFn, serverOptions, startS
355356
};
356357
graphCount = graphs.count();
357358
}
359+
let failurePoints = pu.checkServerFailurePoints(instanceInfo);
360+
if (failurePoints.length > 0) {
361+
continueTesting = false;
362+
results[te] = {
363+
status: false,
364+
message: 'Cleanup of failure points missing - found failure points engaged: [ ' +
365+
JSON.stringify(failurePoints) +
366+
' ] - Original test status: ' +
367+
JSON.stringify(results[te])
368+
};
369+
370+
}< 3ADC /div>
358371
} else {
359372
serverDead = true;
360373
continueTesting = false;

‎lib/Basics/debugging.cpp‎

Lines changed: 35 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,9 @@
3939
#include "Logger/Logger.h"
4040
#include "Logger/LoggerStream.h"
4141

42+
#include <velocypack/Builder.h>
4243
#include <velocypack/StringRef.h>
44+
#include <velocypack/Value.h>
4345

4446
#ifdef TRI_HAVE_UNISTD_H
4547
#include <unistd.h>
@@ -149,14 +151,43 @@ void TRI_AddFailurePointDebugging(char const* value) {
149151

150152
/// @brief remove a failure point
151153
void TRI_RemoveFailurePointDebugging(char const* value) {
152-
WRITE_LOCKER(writeLocker, ::failurePointsLock);
153-
::failurePoints.erase(std::string(value));
154+
size_t numRemoved = 0;
155+
{
156+
WRITE_LOCKER(writeLocker, ::failurePointsLock);
157+
numRemoved = ::failurePoints.erase(std::string(value));
158+
}
159+
160+
if (numRemoved > 0) {
161+
LOG_TOPIC("5aacb", INFO, arangodb::Logger::FIXME)
162+
<< "cleared failure point " << value;
163+
}
154164
}
155165

156166
/// @brief clear all failure points
157167
void TRI_ClearFailurePointsDebugging() noexcept {
158-
WRITE_LOCKER(writeLocker, ::failurePointsLock);
159-
::failurePoints.clear();
168+
size_t numExisting = 0;
169+
{
170+
WRITE_LOCKER(writeLocker, ::failurePointsLock);
171+
numExisting = ::failurePoints.size();
172+
::failurePoints.clear();
173+
}
174+
175+
if (numExisting > 0) {
176+
LOG_TOPIC("ea4e7", INFO, arangodb::Logger::FIXME)
177+
<< "cleared " << numExisting << " failure point(s)";
178+
}
179+
}
180+
181+
/// @brief return all currently set failure points
182+
void TRI_GetFailurePointsDebugging(arangodb::velocypack::Builder& builder) {
183+
builder.openArray();
184+
{
185+
READ_LOCKER(readLocker, ::failurePointsLock);
186+
for (auto const& it : ::failurePoints) {
187+
builder.add(arangodb::velocypack::Value(it));
188+
}
189+
}
190+
builder.close();
160191
}
161192
#endif
162193

‎lib/Basics/debugging.h‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,10 @@
5353

5454
#endif
5555

56+
namespace arangodb::velocypack {
57+
class Builder;
58+
}
59+
5660
/// @brief intentionally cause a segmentation violation or other failures
5761
#ifdef ARANGODB_ENABLE_FAILURE_TESTS
5862
void TRI_TerminateDebugging(char const* value);
@@ -88,6 +92,13 @@ void TRI_ClearFailurePointsDebugging() noexcept;
8892
inline void TRI_ClearFailurePointsDebugging() noexcept {}
8993
#endif
9094

95+
/// @brief return all currently set failure points
96+
#ifdef ARANGODB_ENABLE_FAILURE_TESTS
97+
void TRI_GetFailurePointsDebugging(arangodb::velocypack::Builder& builder);
98+
#else
99+
inline void TRI_GetFailurePointsDebugging(arangodb::velocypack::Builder&) {}
100+
#endif
101+
91102
/// @brief returns whether failure point debugging can be used
92103
#ifdef ARANGODB_ENABLE_FAILURE_TESTS
93104
inline constexpr bool TRI_CanUseFailurePointsDebugging() { return true; }

‎tests/js/client/shell/shell-failat-api.js‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
/* jshint globalstrict:false, strict:false, maxlen: 200 */
2+
/* global fail, assertEqual, assertTrue, arango */
3+
4+
// //////////////////////////////////////////////////////////////////////////////
5+
// / DISCLAIMER
6+
// /
7+
// / Copyright 2018 ArangoDB GmbH, Cologne, Germany
8+
// /
9+
// / Licensed under the Apache License, Version 2.0 (the "License")
10+
// / you may not use this file except in compliance with the License.
11+
// / You may obtain a copy of the License at
12+
// /
13+
// / http://www.apache.org/licenses/LICENSE-2.0
14+
// /
15+
// / Unless required by applicable law or agreed to in writing, software
16+
// / distributed under the License is distributed on an "AS IS" BASIS,
17+
// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18+
// / See the License for the specific language governing permissions and
19+
// / limitations under the License.
20+
// /
21+
// / Copyright holder is triAGENS GmbH, Cologne, Germany
22+
// /
23+
// / @author Jan Steemann
24+
// //////////////////////////////////////////////////////////////////////////////
25+
26+
let jsunity = require('jsunity');
27+
28+
let canUseFailAt = function() {
29+
return arango.GET("/_admin/debug/failat") === true;
30+
};
31+
32+
function failAtApiSuite() {
33+
'use strict';
34+
return {
35+
setUp: function () {
36+
arango.DELETE("/_admin/debug/failat");
37+
},
38+
39+
tearDown: function () {
40+
arango.DELETE("/_admin/debug/failat");
41+
},
42+
43+
testCanUse: function () {
44+
let res = arango.GET("/_admin/debug/failat");
45+
assertEqual(typeof res, "boolean");
46+
assertTrue(res);
47+
},
48+
49+
testFailurePointsSet: function () {
50+
let res = arango.GET("/_admin/debug/failat/all");
51+
assertEqual([], res);
52+
53+
res = arango.PUT("/_admin/debug/failat/piff", {});
54+
assertEqual(typeof res, "boolean");
55+
assertTrue(res);
56+
res = arango.GET("/_admin/debug/failat/all");
57+
assertEqual(["piff"], res);
58+
59+
res = arango.PUT("/_admin/debug/failat/der-fuppes-wird-immer-schlimmer", {});
60+
assertEqual(typeof res, "boolean");
61+
assertTrue(res);
62+
res = arango.GET("/_admin/debug/failat/all");
63+
assertEqual(["der-fuppes-wird-immer-schlimmer", "piff"], res);
64+
65+
res = arango.DELETE("/_admin/debug/failat/piff");
66+
assertEqual(typeof res, "boolean");
67+
assertTrue(res);
68+
res = arango.GET("/_admin/debug/failat/all");
69+
assertEqual(["der-fuppes-wird-immer-schlimmer"], res);
70+
71+
res = arango.DELETE("/_admin/debug/failat");
72+
assertEqual(typeof res, "boolean");
73+
assertTrue(res);
74+
res = arango.GET("/_admin/debug/failat/all");
75+
assertEqual([], res);
76+
77+
res = arango.DELETE("/_admin/debug/failat/abc");
78+
assertEqual(typeof res, "boolean");
79+
assertTrue(res);
80+
res = arango.GET("/_admin/debug/failat/all");
81+
assertEqual([], res);
82+
},
83+
84+
};
85+
}
86+
87+
if (canUseFailAt()) {
88+
jsunity.run(failAtApiSuite);
89+
}
90+
return jsunity.done();

0 commit comments

Comments
 (0)
0