8000 Fix shell-compaction-mmfiles-noncluster-timecritical and ungreylist (… · nginxpre/arangodb@705fab3 · GitHub
[go: up one dir, main page]
Skip to content

Commit 705fab3

Browse files
markuspfjsteemann
markuspf
authored and
jsteemann
committed
Fix shell-compaction-mmfiles-noncluster-timecritical and ungreylist (arangodb#10616)
1 parent 40e1d25 commit 705fab3

File tree

1 file changed

+78
-72
lines changed

1 file changed

+78
-72
lines changed

‎tests/js/server/shell/shell-compaction-mmfiles-noncluster-timecritical-grey.js renamed to ‎tests/js/server/shell/shell-compaction-mmfiles-noncluster.js

Lines changed: 78 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -32,11 +32,48 @@ var jsunity = require('jsunity');
3232
var internal = require('internal');
3333
var testHelper = require('@arangodb/test-helper').Helper;
3434
var ArangoCollection = require('@arangodb/arango-collection').ArangoCollection;
35+
const console = require('console');
3536

3637
// //////////////////////////////////////////////////////////////////////////////
3738
// / @brief test suite: collection
3839
// //////////////////////////////////////////////////////////////////////////////
3940

41+
// This function waits for the compactionState of collection to change
42+
// and a compaction to have occurred.
43+
//
44+
// We define a compaction to have occured when files have been combined.
45+
// Note that the compaction count only counts how many times the compaction
46+
// thread has run, but the thread might not have performed any action
47+
//
48+
// The function tries nrTries iterations in which it waits for wait seconds
49+
//
50+
// returns true if a compaction occurred, and false if there was no
51+
// compaction detected.
52+
function waitForCompaction(collection, initialFigures, nrTries, wait) {
53+
var lastTime = initialFigures["compactionStatus"]["time"];
54+
var lastMessage = initialFigures["compactionStatus"]["message"];
55+
var lastCount = initialFigures["compactionStatus"]["count"];
56+
57+
console.log("waiting for compaction to occur");
58+
var tries = 0;
59+
while (tries < nrTries) {
60+
var cur = collection.figures();
61+
if (cur["compactionStatus"]["time"] !== lastTime) {
62+
lastTime = cur["compactionStatus"]["time"];
63+
lastMessage = cur["compactionStatus"]["message"];
64+
lastCount = cur["compactionStatus"]["count"];
65+
console.log("compaction timestamp changed: " + lastTime + ", message: " + lastMessage);
66+
if (cur.compactionStatus.filesCombined > 0) {
67+
console.log("compaction occurred");
68+
return true;
69+
}
70+
}
71+
internal.wait(wait);
72+
tries++;
73+
}
74+
return false;
75+
}
76+
4077
function CompactionSuite () {
4178
'use strict';
4279
return {
@@ -46,14 +83,20 @@ function CompactionSuite () {
4683

4784
testShapesMovement: function () {
4885
var collectionName = 'example';
86+
const ndocs = 10000;
87+
const nrRubbish = 250000;
88+
4989
internal.db._drop(collectionName);
5090

5191
var cn = internal.db._create(collectionName, {
5292
'journalSize': 1048576
5393
});
5494
var x, i, j, doc;
5595

56-
for (i = 0; i < 1000; ++i) {
96+
97+
const initialFigures = cn.figures();
98+
99+
for (i = 0; i < ndocs; ++i) {
57100
doc = {
58101
_key: 'old' + i,
59102
a: i,
@@ -71,7 +114,7 @@ function CompactionSuite () {
71114
}
72115

73116
// now access the documents once, to build the shape accessors
74-
for (i = 0; i < 1000; ++i) {
117+
for (i = 0; i < ndocs; ++i) {
75118
doc = cn.document('old' + i);
76119

77120
assertTrue(doc.hasOwnProperty('a'));
@@ -92,23 +135,25 @@ function CompactionSuite () {
92135
}
93136

94137
// fill the datafile with rubbish
95-
for (i = 0; i < 10000; ++i) {
138+
for (i = 0; i < nrRubbish; ++i) {
96139
cn.save({
97140
_key: 'test' + i,
98141
value: 'thequickbrownfox'
99142
});
100143
}
101144

102-
for (i = 0; i < 10000; ++i) {
145+
for (i = 0; i < nrRubbish;< F438 /span> ++i) {
103146
cn.remove('test' + i);
104147
}
105148

106-
internal.wait(7, false);
107-
108-
assertEqual(1000, cn.count());
149+
/* we wait for a compaction to occur */
150+
if (waitForCompaction(cn, initialFigures, 100, 1) === false) {
151+
throw "No compaction occurred!";
152+
}
153+
assertEqual(ndocs, cn.count());
109154

110155
// now access the 'old' documents, which were probably moved
111-
for (i = 0; i < 1000; ++i) {
156+
for (i = 0; i < ndocs; ++i) {
112157
doc = cn.document('old' + i);
113158

114159
assertTrue(doc.hasOwnProperty('a'));
@@ -136,16 +181,19 @@ function CompactionSuite () {
136181
// //////////////////////////////////////////////////////////////////////////////
137182

138183
testShapes1: function () {
184+
const nrRubbish = 10000;
185+
const nrShapes = 10000;
139186
var cn = 'example';
140187
internal.db._drop(cn);
141188
var c1 = internal.db._create(cn, {
142189
'journalSize': 1048576
143190
});
191+
const initialFigures = c1.figures();
144192

145193
var i, doc;
146194

147195
// prefill with 'trash'
148-
for (i = 0; i < 1000; ++i) {
196+
for (i = 0; i < nrRubbish; ++i) {
149197
c1.save({
150198
_key: 'test' + i
151199
});
@@ -158,7 +206,7 @@ function CompactionSuite () {
158206
testHelper.rotate(c1);
159207

160208
// create lots of different shapes
161-
for (i = 0; i < 100; ++i) {
209+
for (i = 0; i < nrShapes; ++i) {
162210
doc = {
163211
_key: 'test' + i
164212
};
@@ -172,7 +220,9 @@ function CompactionSuite () {
172220
testHelper.rotate(c1);
173221
c1.truncate();
174222

175-
internal.wait(5, false);
223+
if (waitForCompaction(c1, initialFigures, 100, 1) === false) {
224+
throw "No compaction occurred!";
225+
}
176226

177227
for (i = 0; i < 100; ++i) {
178228
doc = {
@@ -208,6 +258,7 @@ function CompactionSuite () {
208258
var c1 = internal.db._create(cn, {
209259
'journalSize': 1048576
210260
});
261+
const initialFigures = c1.figures();
211262

212263
var i, doc;
213264
// prefill with 'trash'
@@ -284,6 +335,7 @@ function CompactionSuite () {
284335
var cn = 'example';
285336
internal.db._drop(cn);
286337
var c1 = internal.db._create(cn, { 'journalSize': 1048576 });
338+
const initialFigures = c1.figures();
287339

288340
var i, doc;
289341

@@ -383,6 +435,7 @@ function CompactionSuite () {
383435
var c1 = internal.db._create(cn, {
384436
'journalSize': 1048576
385437
});
438+
const initialFigures = c1.figures();
386439

387440
internal.wal.flush(true, true);
388441

@@ -562,30 +615,12 @@ function CompactionSuite () {
562615
internal.wal.flush(true, true);
563616
c1.rotate();
564617

618+
const initialFigures = c1.figures();
565619
c1.properties({
566620
doCompact: true
567621
});
568622

569-
// wait for compactor to run
570-
require('console').log('waiting for compactor to run');
571-
572-
// set max wait time
573-
if (internal.valgrind) {
574-
maxWait = 750;
575-
} else {
576-
maxWait = 90;
577-
}
578-
579-
var tries = 0;
580-
while (++tries < maxWait) {
581-
fig = c1.figures();
582-
583-
if (fig['dead']['deletion'] === 0 && fig['dead']['count'] === 0) {
584-
break;
585-
}
586-
587-
internal.wait(1, false);
588-
}
623+
waitForCompaction(c1, initialFigures, 100, 1);
589624

590625
fig = c1.figures();
591626
assertEqual(0, c1.count());
@@ -718,7 +753,7 @@ function CompactionSuite () {
718753
var maxWait;
719754
var waited;
720755
var cn = 'example';
721-
var n = 400;
756+
var n = 20000;
722757
var i;
723758
var payload = 'the quick brown fox jumped over the lazy dog. a quick dog jumped over the lazy fox';
724759

@@ -730,6 +765,7 @@ function CompactionSuite () {
730765
var c1 = internal.db._create(cn, {
731766
'journalSize': 1048576
732767
});
768+
const initialFigures = c1.figures();
733769

734770
for (i = 0; i < n; ++i) {
735771
c1.save({
@@ -752,7 +788,8 @@ function CompactionSuite () {
752788
var tries = 0;
753789
var fig;
754790

755-
while (++tries < 20) {
791+
console.log("waiting for alive.count === " + n/2 + " and dead.count === 0");
792+
while (++tries < 500) {
756793
fig = c1.figures();
757794
if (fig['alive']['count'] === n / 2 && fig['dead']['count'] === 0) {
758795
break;
@@ -772,27 +809,8 @@ function CompactionSuite () {
772809
internal.wait(0);
773810

774811
// wait for compactor to run
775-
require('console').log('waiting for compactor to run');
776-
777-
// set max wait time
778-
if (internal.valgrind) {
779-
maxWait = 750;
780-
} else {
781-
maxWait = 90;
782-
}
783-
784-
waited = 0;
785-
var lastValue = fig['dead']['deletion'];
786-
787-
while (waited < maxWait) {
788-
internal.wait(5);
789-
waited += 5;
790-
791-
fig = c1.figures();
792-
if (fig['dead']['deletion'] === lastValue) {
793-
break;
794-
}
795-
lastValue = fig['dead']['deletion'];
812+
if (waitForCompaction(c1, initialFigures, 100, 1) === false) {
813+
throw "No compaction occurred!";
796814
}
797815

798816
for (i = 0; i < n; i++) {
@@ -817,6 +835,8 @@ function CompactionSuite () {
817835

818836
waited = 0;
819837

838+
maxWait = 10000;
839+
console.log("Waiting for dead.deletion === 0 and dead.count === 0");
820840
while (waited < maxWait) {
821841
internal.wait(2);
822842
waited += 2;
@@ -861,6 +881,7 @@ function CompactionSuite () {
861881
var c1 = internal.db._create(cn, {
862882
'journalSize': 1048576
863883
});
884+
const initialFigures = c1.figures();
864885

865886
for (i = 0; i < n; ++i) {
866887
c1.save({ value: i,
@@ -909,26 +930,11 @@ function CompactionSuite () {
909930
assertEqual(0, fig['alive']['count']);
910931
assertEqual(n, fig['dead']['deletion']);
911932

912-
// wait for compactor to run
913-
require('console').log('waiting for compactor to run');
914-
915-
// set max wait time
916-
if (internal.valgrind) {
917-
maxWait = 750;
918-
} else {
919-
maxWait = 90;
933+
if (waitForCompaction(c1, initialFigures, 100, 1) === false) {
934+
throw "No compaction occurred!";
920935
}
921936

922-
tries = 0;
923-
while (++tries < maxWait) {
924-
fig = c1.figures();
925-
926-
if (fig['dead']['count'] === 0) {
927-
break;
928-
}
929-
930-
internal.wait(1, false);
931-
}
937+
fig = c1.figures();
932938

933939
assertEqual(0, c1.count());
934940
// all alive & dead markers should be gone

0 commit comments

Comments
 (0)
0