8000 BTS-999: Bug fix 3.8/bulk inserts (#17440) · arangodb/arangodb@c1b0bbd · GitHub
[go: up one dir, main page]
Skip to content

Commit c1b0bbd

Browse files
dothebartjsteemann
dothebart
and
jsteemann
authored
BTS-999: Bug fix 3.8/bulk inserts (#17440)
* BTS-1077: bulk test data creation (#17329) Co-authored-by: Jan <jsteemann@users.noreply.github.com> * setUpAll/tearDownAll for performance reasons (#17322) * bulk test, don't use graph module (#17319) * one more place missing bulk insert (#17189) * use setupAll, split writing tests (#17038) Co-authored-by: Jan <jsteemann@users.noreply.github.com>
1 parent 517aa19 commit c1b0bbd

4 files changed

+281
-168
lines changed

‎tests/js/common/shell/shell-community-sharding-compat-cluster.js

Lines changed: 72 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -73,9 +73,11 @@ function DocumentShardingSuite() {
7373
assertEqual(5, c.properties()["numberOfShards"]);
7474
assertEqual(["_key"], c.properties()["shardKeys"]);
7575

76+
let docs = [];
7677
for (let i = 0; i < 1000; ++i) {
77-
c.insert({ _key: "test" + i, value: i });
78+
docs.push({ _key: "test" + i, value: i });
7879
}
80+
c.insert(docs);
7981

8082
assertEqual([ 188, 192, 198, 204, 218 ], Object.values(c.count(true)).sort());
8183

@@ -91,8 +93,13 @@ function DocumentShardingSuite() {
9193
assertEqual(["value"], c.properties()["shardKeys"]);
9294

9395
let keys = [];
96+
let docs = [];
9497
for (let i = 0; i < 1000; ++i) {
95-
keys.push(c.insert({ value: i })._key);
98+
docs.push({ value: i });
99+
}
100+
docs = c.save(docs);
101+
for (let i = 0; i < 1000; ++i) {
102+
keys.push(docs[i]._key);
96103
}
97104

98105
assertEqual([ 0, 0, 0, 0, 1000 ], Object.values(c.count(true)).sort());
@@ -108,9 +115,11 @@ function DocumentShardingSuite() {
108115
assertEqual(5, c.properties()["numberOfShards"]);
109116
assertEqual(["_key"], c.properties()["shardKeys"]);
110117

118+
let docs = [];
111119
for (let i = 0; i < 1000; ++i) {
112-
c.insert({ _key: "test" + i, value: i });
120+
docs.push({ _key: "test" + i, value: i });
113121
}
122+
c.save(docs);
114123

115124
assertEqual([ 188, 192, 198, 204, 218 ], Object.values(c.count(true)).sort());
116125

@@ -129,9 +138,11 @@ function DocumentShardingSuite() {
129138
assertEqual(["_key"], c1.properties()["shardKeys"]);
130139
assertEqual(["_key"], c2.properties()["shardKeys"]);
131140

141+
let docs = [];
132142
for (let i = 0; i < 1000; ++i) {
133-
c2.insert({ _key: "test" + i, value: i });
143+
docs.push({ _key: "test" + i, value: i });
134144
}
145+
c2.save(docs);
135146

136147
assertEqual([ 188, 192, 198, 204, 218 ], Object.values(c2.count(true)).sort());
137148

@@ -151,8 +162,13 @@ function DocumentShardingSuite() {
151162
assertEqual(["two"], c2.properties()["shardKeys"]);
152163

153164
let keys = [];
165+
let docs = [];
154166
for (let i = 0; i < 1000; ++i) {
155-
keys.push(c2.insert({ two: i })._key);
167+
docs.push({ two: i });
168+
}
169+
docs = c2.save(docs);
170+
for (let i = 0; i < 1000; ++i) {
171+
keys.push(docs[i]._key);
156172
}
157173

158174
assertEqual([ 179, 192, 200, 207, 222 ], Object.values(c2.count(true)).sort());
@@ -173,8 +189,13 @@ function DocumentShardingSuite() {
173189
assertEqual(["three", "four"], c2.properties()["shardKeys"]);
174190

175191
let keys = [];
192+
let docs = [];
193+
for (let i = 0; i < 1000; ++i) {
194+
docs.push({ three: i, four: i + 1 });
195+
}
196+
docs = c2.save(docs);
176197
for (let i = 0; i < 1000; ++i) {
177-
keys.push(c2.insert({ three: i, four: i + 1 })._key);
198+
keys.push(docs[i]._key);
178199
}
179200

180201
assertEqual([ 187, 188, 203, 208, 214 ], Object.values(c2.count(true)).sort());
@@ -201,8 +222,13 @@ function DocumentShardingSuite() {
201222
assertEqual(["value"], c.properties()["shardKeys"]);
202223

203224
let keys = [];
225+
let docs = [];
226+
for (let i = 0; i < 1000; ++i) {
227+
docs.push({});
228+
}
229+
docs = c.save(docs);
204230
for (let i = 0; i < 1000; ++i) {
205-
keys.push(c.insert({ })._key);
231+
keys.push(docs[i]._key);
206232
}
207233

208234
assertEqual([ 0, 0, 0, 0, 1000 ], Object.values(c.count(true)).sort());
@@ -219,8 +245,13 @@ function DocumentShardingSuite() {
219245
assertEqual(["value"], c.properties()["shardKeys"]);
220246

221247
let keys = [];
248+
let docs = [];
249+
for (let i = 0; i < 1000; ++i) {
250+
docs.push({ value: i });
251+
}
252+
docs = c.save(docs);
222253
for (let i = 0; i < 1000; ++i) {
223-
keys.push(c.insert({ value: i })._key);
254+
keys.push(docs[i]._key);
224255
}
225256

226257
assertEqual([ 179, 192, 200, 207, 222 ], Object.values(c.count(true)).sort());
@@ -237,8 +268,13 @@ function DocumentShardingSuite() {
237268
assertEqual(["value"], c.properties()["shardKeys"]);
238269

239270
let keys = [];
271+
let docs = [];
240272
for (let i = 0; i < 1000; ++i) {
241-
keys.push(c.insert({ value: null })._key);
273+
docs.push({ value: null });
274+
}
275+
docs = c.save(docs);
276+
for (let i = 0; i < 1000; ++i) {
277+
keys.push(docs[i]._key);
242278
}
243279

244280
assertEqual([ 0, 0, 0, 0, 1000 ], Object.values(c.count(true)).sort());
@@ -259,8 +295,13 @@ function DocumentShardingSuite() {
259295
assertEqual(["value"], c2.properties()["shardKeys"]);
260296

261297
let keys = [];
298+
let docs = [];
262299
for (let i = 0; i < 1000; ++i) {
263-
keys.push(c2.insert({ value: i })._key);
300+
docs.push({ value: i });
301+
}
302+
docs = c2.save(docs);
303+
for (let i = 0; i < 1000; ++i) {
304+
keys.push(docs[i]._key);
264305
}
265306

266307
assertEqual([ 179, 192, 200, 207, 222 ], Object.values(c2.count(true)).sort());
@@ -313,9 +354,11 @@ function EdgeShardingSuite() {
313354
assertEqual(5, c.properties()["numberOfShards"]);
314355
assertEqual(["_key"], c.properties()["shardKeys"]);
315356

357+
let docs = [];
316358
for (let i = 0; i < 1000; ++i) {
317-
c.insert({ _key: "test" + i, value: i, _from: "v/test" + i, _to: "v/test" + i });
359+
docs.push({ _key: "test" + i, value: i, _from: "v/test" + i, _to: "v/test" + i });
318360
}
361+
c.save(docs);
319362

320363
assertEqual([ 188, 192, 198, 204, 218 ], Object.values(c.count(true)).sort());
321364

@@ -359,8 +402,13 @@ function EdgeShardingSuite() {
359402
assertEqual(["value"], c.properties()["shardKeys"]);
360403

361404
let keys = [];
405+
let docs = [];
406+
for (let i = 0; i < 1000; ++i) {
407+
docs.push({ value: i, _from: "v/test" + i, _to: "v/test" + i });
408+
}
409+
docs = c.save(docs);
362410
for (let i = 0; i < 1000; ++i) {
363-
keys.push(c.insert({ value: i, _from: "v/test" + i, _to: "v/test" + i })._key);
411+
keys.push(docs[i]._key);
364412
}
365413

366414
assertEqual([ 179, 192, 200, 207, 222 ], Object.values(c.count(true)).sort());
@@ -377,8 +425,13 @@ function EdgeShardingSuite() {
377425
assertEqual(["value"], c.properties()["shardKeys"]);
378426

379427
let keys = [];
428+
let docs = [];
429+
for (let i = 0; i < 1000; ++i) {
430+
docs.push({ value: null, _from: "v/test" + i, _to: "v/test" + i });
431+
}
432+
docs = c.save(docs);
380433
for (let i = 0; i < 1000; ++i) {
381-
keys.push(c.insert({ value: null, _from: "v/test" + i, _to: "v/test" + i })._key);
434+
keys.push(docs[i]._key);
382435
}
383436

384437
assertEqual([ 0, 0, 0, 0, 1000 ], Object.values(c.count(true)).sort());
@@ -399,8 +452,13 @@ function EdgeShardingSuite() {
399452
assertEqual(["value"], c2.properties()["shardKeys"]);
400453

401454
let keys = [];
455+
let docs = [];
456+
for (let i = 0; i < 1000; ++i) {
457+
docs.push({ value: i, _from: "v/test" + i, _to: "v/test" + i });
458+
}
459+
docs = c2.save(docs);
402460
for (let i = 0; i < 1000; ++i) {
403-
keys.push(c2.insert({ value: i, _from: "v/test" + i, _to: "v/test" + i })._key);
461+
keys.push(docs[i]._key);
404462
}
405463

406464
assertEqual([ 179, 192, 200, 207, 222 ], Object.values(c2.count(true)).sort());

‎tests/js/common/shell/shell-pregel-components.js

Lines changed: 27 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*jshint globalstrict:false, strict:false */
2-
/*global assertEqual, assertTrue, assertNotEqual, JSON */
2+
/*global assertEqual, assertTrue, JSON */
33
'use strict';
44

55
// //////////////////////////////////////////////////////////////////////////////
@@ -108,27 +108,27 @@ function componentsTestSuite() {
108108
console.log("Done inserting vertices, inserting edges");
109109

110110
let lcg = createRand();
111-
111+
let edges = [];
112+
let vertices = [];
112113
for (let c = 0; c < numComponents; c++) {
113-
let edges = [];
114-
for (let x = 0; x < m; x++) {
115-
let fromID = String(c) + ":" + Math.floor(lcg() * n);
116-
let toID = String(c) + ":" + Math.floor(lcg() * n);
117-
let from = vColl + '/' + fromID;
118-
let to = vColl + '/' + toID;
119-
edges.push({ _from: from, _to: to, vertex: String(fromID) });
120-
}
121-
db[eColl].insert(edges);
122-
123-
for (let x = 0; x < n; x++) {
124-
let fromID = String(c) + ":" + x;
125-
let toID = String(c) + ":" + (x + 1);
126-
let from = vColl + '/' + fromID;
127-
let to = vColl + '/' + toID;
128-
db[eColl].insert({ _from: from, _to: to, vertex: String(fromID) });
129-
}
114+
for (let x = 0; x < m; x++) {
115+
let fromID = String(c) + ":" + Math.floor(lcg() * n);
116+
let toID = String(c) + ":" + Math.floor(lcg() * n);
117+
let from = vColl + '/' + fromID;
118+
let to = vColl + '/' + toID;
119+
edges.push({_from: from, _to: to, vertex: String(fromID)});
120+
}
121+
122+
for (let x = 0; x < n; x++) {
123+
let fromID = String(c) + ":" + x;
124+
let toID = String(c) + ":" + (x + 1);
125+
let from = vColl + '/' + fromID;
126+
let to = vColl + '/' + toID;
127+
vertices.push({_from: from, _to: to, vertex: String(fromID)});
128+
}
130129
}
131-
130+
db[eColl].insert(edges);
131+
db[eColl].insert(vertices);
132132
console.log("Got %s edges", db[eColl].count());
133133
assertEqual(db[eColl].count(), numComponents * m + numComponents * n);
134134

@@ -181,15 +181,19 @@ function componentsTestSuite() {
181181
graph_module._drop(problematicGraphName, true);
182182
} catch (err) { }
183183

184-
const graph = graph_module._create(problematicGraphName, [graph_module._relation(e, v, v)]);
184+
graph_module._create(problematicGraphName, [graph_module._relation(e, v, v)]);
185185

186+
let vertdocs = [];
186187
vertices.forEach(vertex => {
187-
graph[v].save({ _key: vertex });
188+
vertdocs.push({_key: vertex});
188189
});
190+
db[v].save(vertdocs);
189191

192+
let edgedocs = [];
190193
edges.forEach(([from, to]) => {
191-
graph[e].save({ _from: `${v}/${from}`, _to: `${v}/${to}` });
194+
edgedocs.push({_from: `${v}/${from}`, _to: `${v}/${to}`});
192195
});
196+
db[e].save(edgedocs);
193197
}
194198
},
195199

‎tests/js/common/shell/shell-pregel-multicollection-edgecol.js

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@ function multiCollectionTestSuite() {
111111

112112
console.log("extended edge definition");
113113

114+
let cols = {};
114115
for (let c = 0; c < numComponents; c++) {
115116
let x = 0;
116117
let vertices = [];
@@ -122,9 +123,16 @@ function multiCollectionTestSuite() {
122123
vertices[i].push({ _key: String(c) + ":" + String(x++) });
123124
}
124125
for (let i = 0; i < cn; ++i) {
125-
db[`${vColl}_${i}`].insert(vertices[i]);
126+
let coln = `${vColl}_${i}`;
127+
if (!cols.hasOwnProperty(coln)) {
128+
cols[coln] = [];
129+
}
130+
cols[coln] = cols[coln].concat(vertices[i]);
126131
}
127132
}
133+
Object.keys(cols).forEach(coln => {
134+
db[coln].insert(cols[coln]);
135+
});
128136

129137
for (let i = 0; i < cn; ++i) {
130138
assertEqual(db[`${vColl}_${i}`].count(), numComponents * n / cn);
@@ -134,6 +142,7 @@ function multiCollectionTestSuite() {
134142

135143
let lcg = createRand();
136144

145+
let edgeCols = {};
137146
for (let c = 0; c < numComponents; c++) {
138147
let edges = [];
139148
for (let i = 0; i < cn; ++i) {
@@ -155,7 +164,11 @@ function multiCollectionTestSuite() {
155164
}
156165
for (let i = 0; i < cn; ++i) {
157166
for (let j = 0; j < cn; ++j) {
158-
db[`${eColl}_${i}_${j}`].insert(edges[i][j]);
167+
let coln = `${eColl}_${i}_${j}`;
168+
if (!edgeCols.hasOwnProperty(coln)) {
169+
edgeCols[coln] = [];
170+
}
171+
edgeCols[coln] = edgeCols[coln].concat(edges[i][j]);
159172
}
160173
}
161174

@@ -168,9 +181,16 @@ function multiCollectionTestSuite() {
168181
const vTo = (toId + c) % cn;
169182
const from = `${vColl}_${vFrom}/${fromKey}`;
170183
const to = `${vColl}_${vTo}/${toKey}`;
171-
db[`${eColl}_${vFrom}_${vTo}`].insert({ _from: from, _to: to, vertex: String(fromKey) });
184+
let coln = `${eColl}_${vFrom}_${vTo}`;
185+
if (!cols.hasOwnProperty(coln)) {
186+
cols[coln] = [];
187+
}
188+
edgeCols[coln] = edgeCols[coln].concat({ _from: from, _to: to, vertex: String(fromKey) });
172189
}
173190
}
191+
Object.keys(edgeCols).forEach(coln => {
192+
db[coln].insert(edgeCols[coln]);
193+
});
174194

175195
let count = 0;
176196
for (let i = 0; i < cn; ++i) {

0 commit comments

Comments
 (0)
0