8000 Fix creation of millions of VPackBuilders in edge import - #194 · arangodb/arangodb@4c18d25 · GitHub
[go: up one dir, main page]
Skip to content

Commit 4c18d25

Browse files
ObiWahnObiWahn
committed
Fix creation of millions of VPackBuilders in edge import - #194 
1 parent 4d73e13 commit 4c18d25

File tree

3 files changed

+72
-4
lines changed

3 files changed

+72
-4
lines changed

‎3rdParty/velocypack/include/velocypack/Collection.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,7 @@ class Collection {
199199
bool mergeValues, bool nullMeansRemove = false) {
200200
return merge(*left, *right, mergeValues, nullMeansRemove);
201201
}
202+
static Builder& merge(Builder& builder, Slice const& left, Slice const& right, bool mergeValues, bool nullMeansRemove = false);
202203

203204
static void visitRecursive(
204205
Slice const& slice, VisitationOrder order,

‎3rdParty/velocypack/src/Collection.cpp

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -403,6 +403,73 @@ Builder Collection::merge(Slice const& left, Slice const& right,
403403
return b;
404404
}
405405

406+
Builder& Collection::merge(Builder& builder, Slice const& left, Slice const& right,
407+
bool mergeValues, bool nullMeansRemove) {
408+
if (!left.isObject() || !right.isObject()) {
409+
throw Exception(Exception::InvalidValueType, "Expecting type Object");
410+
}
411+
412+
builder.add(Value(ValueType::Object));
413+
414+
std::unordered_map<std::string, Slice> rightValues;
415+
{
416+
ObjectIterator it(right);
417+
while (it.valid()) {
418+
rightValues.emplace(it.key(true).copyString(), it.value());
419+
it.next();
420+
}
421+
}
422+
423+
{
424+
ObjectIterator it(left);
425+
426+
while (it.valid()) {
427+
auto key = it.key(true).copyString();
428+
auto found = rightValues.find(key);
429+
430+
if (found == rightValues.end()) {
431+
// use left value
432+
builder.add(key, it.value());
433+
} else if (mergeValues && it.value().isObject() &&
434+
(*found).second.isObject()) {
435+
// merge both values
436+
auto& value = (*found).second;
437+
if (!nullMeansRemove || (!value.isNone() && !value.isNull())) {
438+
Collection::merge(builder, it.value(), value, true, nullMeansRemove);
439+
//Builder sub = Collection::merge(it.value(), value, true, nullMeansRemove);
440+
//builder.add(key, sub.slice());
441+
}
442+
// clear the value in the map so its not added again
443+
(*found).second = Slice();
444+
} else {
445+
// use right value
446+
auto& value = (*found).second;
447+
if (!nullMeansRemove || (!value.isNone() && !value.isNull())) {
448+
builder.add(key, value);
449+
}
450+
// clear the value in the map so its not added again
451+
(*found).second = Slice();
452+
}
453+
it.next();
454+
}
455+
}
456+
457+
// add remaining values that were only in right
458+
for (auto& it : rightValues) {
459+
auto& s = it.second;
460+
if (s.isNone()) {
461+
continue;
462+
}
463+
if (nullMeansRemove && s.isNull()) {
464+
continue;
465+
}
466+
builder.add(std::move(it.first), s);
467+
}
468+
469+
builder.close();
470+
return builder;
471+
}
472+
406473
template <Collection::VisitationOrder order>
407474
static bool doVisit(
408475
Slice const& slice,

‎arangod/RestHandler/RestImportHandler.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -201,7 +201,7 @@ int RestImportHandler::handleSingleDocument(SingleCollectionTransaction& trx,
201201

202202

203203
// document ok, now import it
204-
VPackBuilder newBuilder;
204+
transaction::BuilderLeaser newBuilder(&trx);
205205

206206
// add prefixes to _from and _to
207207
if (!_fromPrefix.empty() || !_toPrefix.empty()) {
@@ -239,10 +239,10 @@ int RestImportHandler::handleSingleDocument(SingleCollectionTransaction& trx,
239239
tempBuilder->close();
240240

241241
if (tempBuilder->slice().length() > 0) {
242-
newBuilder =
243-
VPackCollection::merge(slice, tempBuilder->slice(), false, false);
244-
slice = newBuilder.slice();
242+
VPackCollection::merge(*(newBuilder.builder()), slice, tempBuilder->slice(), false, false);
243+
slice = newBuilder->slice();
245244
}
245+
LOG_TOPIC(ERR, Logger::FIXME) << slice.toJson();
246246
}
247247

248248
try {

0 commit comments

Comments
 (0)
0