8000 Add test cases for _admin/metrics/v2 endpoint in cluster mode (#14446) · arangodb/arangodb@038485f · GitHub
[go: up one dir, main page]
Skip to content

Commit 038485f

Browse files
naushnikiKVS85
naushniki
and
KVS85
authored
Add test cases for _admin/metrics/v2 endpoint in cluster mode (#14446)
* add test cases for _admin/metrics/v2 endpoint in cluster mode * check that metrics from desired server are returned when requesting via coordinator use 2 coordinator servers for shell_client test suite ensure that server is recovered before going on with other tests * Fix jslint * Update shell-promtool-cluster.js Co-authored-by: Vadim Kondratyev <vadim@arangodb.com>
1 parent 2c73237 commit 038485f

File tree

3 files changed

+296
-3
lines changed

3 files changed

+296
-3
lines changed

‎README_maintainers.md

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,7 @@ Depending on the platform, ArangoDB tries to locate the temporary directory:
211211
### Local Cluster Startup
212212

213213
The scripts `scripts/startLocalCluster` helps you to quickly fire up a testing
214-
cluster on your local machine. `scripts/stopLocalCluster` stops it again.
214+
cluster on your local machine. `scripts/shutdownLocalCluster` stops it again.
215215

216216
`scripts/startLocalCluster [numDBServers numCoordinators [mode]]`
217217

@@ -692,10 +692,14 @@ To locate the suite(s) associated with a specific test file use:
692692

693693
./scripts/unittest find --test tests/js/common/shell/shell-aqlfunctions.js
694694

695-
Run all suite(s) associated with a specific test file:
695+
Run all suite(s) associated with a specific test file in single server mode:
696696

697697
./scripts/unittest auto --test tests/js/common/shell/shell-aqlfunctions.js
698698

699+
Run all suite(s) associated with a specific test file in cluster mode:
700+
701+
./scripts/unittest auto --cluster true --test tests/js/common/shell/shell-aqlfunctions.js
702+
699703
Run all C++ based Google Test (gtest) tests using the `arangodbtests` binary:
700704

701705
./scripts/unittest gtest

‎js/client/modules/@arangodb/testsuites/aql.js

Lines changed: 12 additions & 1 deletion
Origin 57AE al file line numberDiff line numberDiff line change
@@ -58,6 +58,16 @@ function ensureServers(options, numServers) {
5858
return options;
5959
}
6060

61+
/// ensure that we have enough coordinators in cluster tests
62+
function ensureCoordinators(options, numServers) {
63+
if (options.cluster && options.coordinators < numServers) {
64+
let localOptions = _.clone(options);
65+
localOptions.coordinators = numServers;
66+
return localOptions;
67+
}
68+
return options;
69+
}
70+
6171
// //////////////////////////////////////////////////////////////////////////////
6272
// / @brief TEST: shell_client
6373
// //////////////////////////////////////////////////////////////////////////////
@@ -67,7 +77,8 @@ function shellClient (options) {
6777

6878
testCases = tu.splitBuckets(options, testCases);
6979

70-
let opts = ensureServers(options, 3);
80+
var opts = ensureServers(options, 3);
81+
opts = ensureCoordinators(opts, 2);
7182
let rc = tu.performTests(opts, testCases, 'shell_client', tu.runInLocalArangosh);
7283
options.cleanup = options.cleanup && opts.cleanup;
7384
return rc;

‎tests/js/client/shell/shell-promtool-cluster.js

Lines changed: 278 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,278 @@
1+
/* jshint globalstrict:false, strict:false, maxlen: 200 */
2+
/* global print, assertNotEqual, assertFalse, assertTrue, assertEqual, fail, arango */
3+
4+
////////////////////////////////////////////////////////////////////////////////
5+
/// DISCLAIMER
6+
///
7+
/// Copyright 2014-2021 ArangoDB GmbH, Cologne, Germany
8+
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
9+
///
10+
/// Licensed under the Apache License, Version 2.0 (the "License");
11+
/// you may not use this file except in compliance with the License.
12+
/// You may obtain a copy of the License at
13+
///
14+
/// http://www.apache.org/licenses/LICENSE-2.0
15+
///
16+
/// Unless required by applicable law or agreed to in writing, software
17+
/// distributed under the License is distributed on an "AS IS" BASIS,
18+
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19+
/// See the License for the specific language governing permissions and
20+
/// limitations under the License.
21+
///
22+
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
23+
///
24+
/// @author Jan Steemann
25+
////////////////////////////////////////////////////////////////////////////////
26+
27+
const jsunity = require('jsunity');
28+
const internal = require('internal');
29+
const fs = require('fs');
30+
const pu = require('@arangodb/testutils/process-utils');
31+
const console = require('console');
32+
const request = require("@arangodb/request");
33+
const expect = require('chai').expect;
34+
const errors = require('@arangodb').errors;
35+
const suspendExternal = internal.suspendExternal;
36+
const continueExternal = internal.continueExternal;
37+
38+
// name of environment variable
39+
const PATH = 'PROMTOOL_PATH';
40+
41+
// detect the path to promtool
42+
let promtoolPath = internal.env[PATH];
43+
if (!promtoolPath) {
44+
promtoolPath = '.';
45+
}
46+
promtoolPath = fs.join(promtoolPath, 'promtool' + pu.executableExt);
47+
48+
const metricsUrlPath = "/_admin/metrics/v2";
49+
const serverIdPath = "/_admin/server/id";
50+
const healthUrl = "_admin/cluster/health";
51+
52+
function getServerId(server) {
53+
let res = request.get({
54+
url: server.url + serverIdPath
55+
});
56+
return res.json.id;
57+
}
58+
59+
function getServerShortName(server) {
60+
let serverId = getServerId(server);
61+
let clusterHealth = arango.GET_RAW(healthUrl).parsedBody.Health;
62+
let shortName = Object.keys(clusterHealth)
63+
.filter(x => {
64+
return x === serverId;
65+
})
66+
.map(x => clusterHealth[x]["ShortName"])[0];
67+
return shortName;
68+
}
69+
70+
function checkThatServerIsResponsive(server) {
71+
try {
72+
let serverName = getServerShortName(server);
73+
print("Checking if server " + serverName + " is responsive.");
74+
let res = request.get({
75+
url: server.url + metricsUrlPath
76+
});
77+
if (res.body.includes(serverName) && res.statusCode === 200) {
78+
print("Server " + serverName + " is OK!");
79+
return true;
80+
} else {
81+
print("Server " + serverName + " doesn't respond properly to requests.");
82+
return false;
83+
}
84+
} catch(error){
85+
return false;
86+
}
87+
}
88+
89+
function checkThatAllDbServersAreHealthy() {
90+
print("Checking that all DB servers are healthy.");
91+
let clusterHealth = arango.GET_RAW(healthUrl).parsedBody.Health;
92+
let dbServers = Object.keys(clusterHealth)
93+
.filter(x => {
94+
return clusterHealth[x]["Role"] === "DBServer";
95+
})
96+
.map(x => clusterHealth[x]);
97+
for(let i = 0; i < dbServers.length; i++) {
98+
print("Server "+ dbServers[i].ShortName + " status is " + dbServers[i]["Status"]);
99+
if(!(dbServers[i]["Status"] === "GOOD")){
100+
return false;
101+
}
102+
}
103+
return true;
104+
}
105+
106+
function validateMetrics(metrics) {
107+
let toRemove = [];
108+
try {
109+
// store output of /_admin/metrics/v2 into a temp file
110+
let input = fs.getTempFile();
111+
fs.writeFileSync(input, metrics);
112+
toRemove.push(input);
113+
114+
// this is where we will capture the output from promtool
115+
let output = fs.getTempFile();
116+
toRemove.push(output);
117+
118+
// build command string. this will be unsafe if input/output
119+
// contain quotation marks and such. but as these parameters are
120+
// under our control this is very unlikely
121+
let command = promtoolPath + ' check metrics < "' + input + '" > "' + output + '" 2>&1';
122+
// pipe contents of temp file into promtool
123+
let actualRc = internal.executeExternalAndWait('sh', ['-c', command]);
124+
assertTrue(actualRc.hasOwnProperty('exit'));
125+
assertEqual(0, actualRc.exit);
126+
127+
let promtoolResult = fs.readFileSync(output).toString();
128+
// no errors found means an empty result file
129+
assertEqual('', promtoolResult);
130+
} finally {
131+
// remove temp files
132+
toRemove.forEach((f) => {
133+
fs.remove(f);
134+
});
135+
}
136+
}
137+
138+
function checkMetricsBelongToServer(metrics, server) {
139+
let serverName = getServerShortName(server);
140+
let positiveRegex = new RegExp('(shortname="(' + serverName + ').*")');
141+
let negativeRegex = new RegExp('(shortname="(?!' + serverName + ').*")');
142+
let matchesServerName = metrics.match(positiveRegex);
143+
let matchesAnyOtherName = metrics.match(negativeRegex);
144+
assertNotEqual(null, matchesServerName, "Metrics must contain server name, but they don't");
145+
assertTrue(matchesServerName.length > 0, "Metrics must contain server name, but they don't");
146+
assertEqual(null, matchesAnyOtherName, "Metrics must NOT contain other servers' names.");
147+
}
148+
149+
function validateMetricsOnServer(server) {
150+
print("Querying server ", server.name);
151+
let res = request.get({
152+
url: server.url + metricsUrlPath
153+
});
154+
expect(res).to.be.an.instanceof(request.Response);
155+
expect(res).to.have.property('statusCode', 200);
156+
let body = String(res.body);
157+
validateMetrics(body);
158+
}
159+
160+
function validateMetricsViaCoordinator(coordinator, server) {
161+
let serverId = getServerId(server);
162+
let metricsUrl = coordinator.url + metricsUrlPath + "?serverId=" + serverId;
163+
let res = request.get({ url: metricsUrl });
164+
expect(res).to.be.an.instanceof(request.Response);
165+
expect(res).to.have.property('statusCode', 200);
166+
let body = String(res.body);
167+
validateMetrics(body);
168+
checkMetricsBelongToServer(body, server);
169+
}
170+
171+
function promtoolClusterSuite() {
172+
'use strict';
173+
174+
if (!internal.env.hasOwnProperty('INSTANCEINFO')) {
175+
throw new Error('env.INSTANCEINFO was not set by caller!');
176+
}
177+
const instanceinfo = JSON.parse(internal.env.INSTANCEINFO);
178+
let dbServers = instanceinfo.arangods.filter(arangod => arangod.role === "dbserver");
179+
let agents = instanceinfo.arangods.filter(arangod => arangod.role === "agent");
180+
let coordinators = instanceinfo.arangods.filter(arangod => arangod.role === "coordinator");
181+
182+
return {
183+
testMetricsOnDbServers: function () {
184+
print("Validating metrics from db servers...");
185+
for (let i = 0; i < dbServers.length; i++) {
186+
validateMetricsOnServer(dbServers[i]);
187+
}
188+
},
189+
testMetricsOnAgents: function () {
190+
print("Validating metrics from agents...");
191+
for (let i = 0; i < agents.length; i++) {
192+
validateMetricsOnServer(agents[i]);
193+
}
194+
},
195+
testMetricsOnCoordinators: function () {
196+
print("Validating metrics from coordinators...");
197+
for (let i = 0; i < coordinators.length; i++) {
198+
validateMetricsOnServer(coordinators[i]);
199+
}
200+
},
201+
testMetricsViaCoordinator: function () {
202+
//exclude agents, because agents cannot be queried via coordinator
203+
let servers = dbServers.concat(coordinators);
204+
for (let i = 0; i < servers.length; i++) {
205+
validateMetricsViaCoordinator(coordinators[0], servers[i]);
206+
}
207+
},
208+
testInvalidServerId: function () {
209+
//query metrics from coordinator, supplying invalid server id
210+
let coordinator = coordinators[0];
211+
let metricsUrl = coordinator.url + metricsUrlPath + "?serverId=" + "invalid-server-id";
212+
let res = request.get({ url: metricsUrl });
213+
expect(res).to.be.an.instanceof(request.Response);
214+
expect(res).to.have.property('statusCode', 404);
215+
expect(res.json.errorNum).to.equal(errors.ERROR_HTTP_BAD_PARAMETER.code);
216+
},
217+
testServerDoesntResponse: function () {
218+
//query metrics from coordinator, supplying id of a server, that is shut down
219+
let dbServer = dbServers[0];
220+
let serverId = getServerId(dbServer);
221+
assertTrue(suspendExternal(dbServer.pid));
222+
dbServer.suspended = true;
223+
try {
224+
let metricsUrl = metricsUrlPath + "?serverId=" + serverId;
225+
let res = arango.GET_RAW(metricsUrl);
226+
assertEqual(503, res.code);
227+
//Do not validate response body because errorNum and errorMessage can differ depending on whether there was an open connection
228+
//between coordinator and db server when the later stopped responding. This cannot be reproduced consistently in the test.
229+
// let body = res.parsedBody;
230+
// expect(body.errorNum).to.equal(errors.ERROR_CLUSTER_CONNECTION_LOST.code);
231+
} finally {
232+
let clusterHealthOk = false;
233+
assertTrue(continueExternal(dbServer.pid));
234+
for (let i = 0; i < 60; i++) {
235+
if (checkThatServerIsResponsive(dbServer)) {
236+
delete dbServer.suspended;
237+
break;
238+
}
239+
internal.sleep(1);
240+
}
241+
for (let i = 0; i < 60; i++) {
242+
if (checkThatAllDbServersAreHealthy()) {
243+
clusterHealthOk = true;
244+
break;
245+
}
246+
internal.sleep(1);
247+
}
248+
assertFalse(dbServer.suspended, "Couldn't recover server after suspension.");
249+
assertTrue(clusterHealthOk, "Some db servers are not ready according to " + healthUrl);
250+
}
251+
}
252+
};
253+
}
254+
255+
if (internal.platform === 'linux') {
256+
// this test intentionally only executes on Linux, and only if PROMTOOL_PATH
257+
// is set to the path containing the `promtool` executable. if the PROMTOOL_PATH
258+
// is set, but the executable cannot be found, the test will error out.
259+
// the test also requires `sh` to be a shell that supports input/output redirection,
260+
// and `true` to be an executable that returns exit code 0 (we use sh -c true` as a
261+
// test to check the shell functionality).
262+
if (fs.exists(promtoolPath)) {
263+
let actualRc = internal.executeExternalAndWait('sh', ['-c', 'true']);
264+
if (actualRc.hasOwnProperty('exit') && actualRc.exit === 0) {
265+
jsunity.run(promtoolClusterSuite);
266+
} else {
267+
console.warn('skipping test because no working sh can be found');
268+
}
269+
} else if (!internal.env.hasOwnProperty(PATH)) {
270+
console.warn('skipping test because promtool is not found. you can set ' + PATH + ' accordingly');
271+
} else {
272+
fail('promtool not found in PROMTOOL_PATH (' + internal.env[PATH] + ')');
273+
}
274+
} else {
275+
console.warn('skipping test because we are not on Linux');
276+
}
277+
278+
return jsunity.done();

0 commit comments

Comments
 (0)
0