8000 Fix creation of millions of VPackBuilders in edge import - part 2 - #194 · arangodb/arangodb@5049062 · GitHub
[go: up one dir, main page]
Skip to content

Commit 5049062

Browse files
ObiWahnObiWahn
committed
Fix creation of millions of VPackBuilders in edge import - part 2 - #194 
1 parent c0e6538 commit 5049062

File tree

3 files changed

+62
-52
lines changed

3 files changed

+62
-52
lines changed

‎3rdParty/velocypack/include/velocypack/Parser.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ class Parser {
112112
}
113113
}
114114

115-
// This method produces a
115+
// This method produces a parser that does not own the builder
116116
explicit Parser(Builder& builder,
117117
Options const* options = &Options::Defaults)
118118
: _start(nullptr), _size(0), _pos(0), _nesting(0),

‎arangod/RestHandler/RestImportHandler.cpp

Lines changed: 57 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -174,10 +174,12 @@ std::string RestImportHandler::buildParseError(size_t i,
174174
////////////////////////////////////////////////////////////////////////////////
175175

176176
int RestImportHandler::handleSingleDocument(SingleCollectionTransaction& trx,
177+
VPackBuilder& tempBuilder,
177178
10000 RestImportResult& result,
178179
VPackBuilder& babies,
179180
VPackSlice slice,
180181
bool isEdgeCollection, size_t i) {
182+
181183
if (!slice.isObject()) {
182184
std::string part = VPackDumper::toString(slice);
183185
if (part.size() > 255) {
@@ -193,7 +195,7 @@ int RestImportHandler::handleSingleDocument(SingleCollectionTransaction& trx,
193195
registerError(result, errorMsg);
194196
return TRI_ERROR_ARANGO_DOCUMENT_TYPE_INVALID;
195197
}
196-
198+
197199
if (!isEdgeCollection) {
198200
babies.add(slice);
199201
return TRI_ERROR_NO_ERROR;
@@ -202,23 +204,22 @@ int RestImportHandler::handleSingleDocument(SingleCollectionTransaction& trx,
202204

203205
// document ok, now import it
204206
transaction::BuilderLeaser newBuilder(&trx);
207+
tempBuilder.clear();
205208

206209
// add prefixes to _from and _to
207210
if (!_fromPrefix.empty() || !_toPrefix.empty()) {
208-
transaction::BuilderLeaser tempBuilder(&trx);
209-
210-
tempBuilder->openObject();
211+
tempBuilder.openObject();
211212
if (!_fromPrefix.empty()) {
212213
VPackSlice from = slice.get(StaticStrings::FromString);
213214
if (from.isString()) {
214215
std::string f = from.copyString();
215216
if (f.find('/') == std::string::npos) {
216-
tempBuilder->add(StaticStrings::FromString,
217+
tempBuilder.add(StaticStrings::FromString,
217218
VPackValue(_fromPrefix + f));
218219
}
219220
} else if (from.isInteger()) {
220221
uint64_t f = from.getNumber<uint64_t>();
221-
tempBuilder->add(StaticStrings::FromString,
222+
tempBuilder.add(StaticStrings::FromString,
222223
VPackValue(_fromPrefix + std::to_string(f)));
223224
}
224225
}
@@ -227,19 +228,19 @@ int RestImportHandler::handleSingleDocument(SingleCollectionTransaction& trx,
227228
if (to.isString()) {
228229
std::string t = to.copyString();
229230
if (t.find('/') == std::string::npos) {
230-
tempBuilder->add(StaticStrings::ToString,
231+
tempBuilder.add(StaticStrings::ToString,
231232
VPackValue(_toPrefix + t));
232233
}
233234
} else if (to.isInteger()) {
234235
uint64_t t = to.getNumber<uint64_t>();
235-
tempBuilder->add(StaticStrings::ToString,
236+
tempBuilder.add(StaticStrings::ToString,
236237
VPackValue(_toPrefix + std::to_string(t)));
237238
}
238239
}
239-
tempBuilder->close();
240+
tempBuilder.close();
240241

241-
if (tempBuilder->slice().length() > 0) {
242-
VPackCollection::merge(*(newBuilder.builder()), slice, tempBuilder->slice(), false, false);
242+
if (tempBuilder.slice().length() > 0) {
243+
VPackCollection::merge(*(newBuilder.builder()), slice, tempBuilder.slice(), false, false);
243244
slice = newBuilder->slice();
244245
}
245246
}
@@ -383,6 +384,8 @@ bool RestImportHandler::createFromJson(std::string const& type) {
383384
VPackBuilder babies;
384385
babies.openArray();
385386

387+
VPackBuilder tmpBuilder;
388+
386389
if (linewise) {
387390
// http required here
388391
HttpRequest* req = dynamic_cast<HttpRequest*>(_request.get());
@@ -396,7 +399,8 @@ bool RestImportHandler::createFromJson(std::string const& type) {
396399
char const* ptr = body.c_str();
397400
char const* end = ptr + body.size();
398401
size_t i = 0;
399-
402+
403+
VPackBuilder lineBuilder;
400404
while (ptr < end) {
401405
// read line until done
402406
i++;
@@ -416,9 +420,6 @@ bool RestImportHandler::createFromJson(std::string const& type) {
416420
// now find end of line
417421
char const* pos = static_cast<char const*>(memchr(ptr, '\n', end - ptr));
418422
char const* oldPtr = nullptr;
419-
420-
std::shared_ptr<VPackBuilder> builder;
421-
422423
bool success = false;
423424

424425
if (pos == ptr) {
@@ -428,19 +429,20 @@ bool RestImportHandler::createFromJson(std::string const& type) {
428429
continue;
429430
}
430431

432+
tmpBuilder.clear();
431433
if (pos != nullptr) {
432434
// non-empty line
433435
*(const_cast<char*>(pos)) = '\0';
434436
TRI_ASSERT(ptr != nullptr);
435437
oldPtr = ptr;
436-
builder = parseVelocyPackLine(ptr, pos, success);
438+
parseVelocyPackLine(tmpBuilder, ptr, pos, success);
437439
ptr = pos + 1;
438440
} else {
439441
// last-line, non-empty
440442
TRI_ASSERT(pos == nullptr);
441443
TRI_ASSERT(ptr != nullptr);
442444
oldPtr = ptr;
443-
builder = parseVelocyPackLine(ptr, success);
445+
parseVelocyPackLine(tmpBuilder, ptr, success);
444446
ptr = end;
445447
}
446448

@@ -455,7 +457,7 @@ bool RestImportHandler::createFromJson(std::string const& type) {
455457
continue;
456458
}
457459

458-
res = handleSingleDocument(trx, result, babies, builder->slice(),
460+
res = handleSingleDocument(trx, lineBuilder, result, babies, tmpBuilder.slice(),
459461
isEdgeCollection, i);
460462

461463
if (res.fail()) {
@@ -491,11 +493,11 @@ bool RestImportHandler::createFromJson(std::string const& type) {
491493
}
492494

493495
VPackValueLength const n = documents.length();
494-
496+
VPackBuilder lineBuilder;
495497
for (VPackValueLength i = 0; i < n; ++i) {
496498
VPackSlice const slice = documents.at(i);
497499

498-
res = handleSingleDocument(trx, result, babies, slice, isEdgeCollection,
500+
res = handleSingleDocument(trx, lineBuilder, result, babies, slice, isEdgeCollection,
499501
static_cast<size_t>(i + 1));
500502

501503
if (res.fail()) {
@@ -597,10 +599,11 @@ bool RestImportHandler::createFromVPack(std::string const& type) {
597599

598600
VPackValueLength const n = documents.length();
599601

602+
VPackBuilder lineBuilder;
600603
for (VPackValueLength i = 0; i < n; ++i) {
601604
VPackSlice const slice = documents.at(i);
602605

603-
res = handleSingleDocument(trx, result, babies, slice, isEdgeCollection,
606+
res = handleSingleDocument(trx, lineBuilder, result, babies, slice, isEdgeCollection,
604607
static_cast<size_t>(i + 1));
605608

606609
if (res.fail()) {
@@ -715,18 +718,24 @@ bool RestImportHandler::createFromKeyValueList() {
715718

716719
*(const_cast<char*>(lineEnd)) = '\0';
717720
bool success = false;
718-
std::shared_ptr<VPackBuilder> parsedKeys;
721+
VPackBuilder parsedKeys;
719722
try {
720-
parsedKeys = parseVelocyPackLine(lineStart, lineEnd, success);
723+
parseVelocyPackLine(parsedKeys, lineStart, lineEnd, success);
721724
} catch (...) {
722725
// This throws if the body is not parseable
723726
generateError(rest::ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER,
724727
"no JSON string array found in first line");
725728
return false;
726729
}
727-
VPackSlice const keys = parsedKeys->slice();
730+
if (!success) {
731+
generateError(rest::ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER,
732+
"no JSON string array found in first line");
733+
return false;
734+
}
735+
736+
VPackSlice const keys = parsedKeys.slice();
728737

729-
if (!success || !checkKeys(keys)) {
738+
if (!checkKeys(keys)) {
730739
generateError(rest::ResponseCode::BAD, TRI_ERROR_HTTP_BAD_PARAMETER,
731740
"no JSON string array found in first line");
732741
return false;
@@ -760,10 +769,13 @@ bool RestImportHandler::createFromKeyValueList() {
760769
// Ignore the result ...
761770
}
762771

772+
VPackBuilder parsedValues;
763773
VPackBuilder babies;
764774
babies.openArray();
765775

766776
size_t i = static_cast<size_t>(lineNumber);
777+
VPackBuilder lineBuilder;
778+
VPackBuilder objectBuilder;
767779

768780
while (current != nullptr && current < bodyEnd) {
769781
i++;
@@ -803,8 +815,8 @@ bool RestImportHandler::createFromKeyValueList() {
803815
}
804816

805817
bool success;
806-
std::shared_ptr<VPackBuilder> parsedValues =
807-
parseVelocyPackLine(lineStart, lineEnd, success);
818+
parsedValues.clear();
819+
parseVelocyPackLine(parsedValues, lineStart, lineEnd, success);
808820

809821
// build the json object from the array
810822
std::string errorMsg;
@@ -813,11 +825,11 @@ bool RestImportHandler::createFromKeyValueList() {
813825
registerError(result, errorMsg);
814826
res = TRI_ERROR_INTERNAL;
815827
} else {
816-
VPackSlice const values = parsedValues->slice();
828+
VPackSlice const values = parsedValues.slice();
817829
try {
818-
std::shared_ptr<VPackBuilder> objectBuilder =
819-
createVelocyPackObject(keys, values, errorMsg, i);
820-
res = handleSingleDocument(trx, result, babies, objectBuilder->slice(),
830+
objectBuilder.clear();
831+
createVelocyPackObject(objectBuilder, keys, values, errorMsg, i);
832+
res = handleSingleDocument(trx, lineBuilder, result, babies, objectBuilder.slice(),
821833
isEdgeCollection, i);
822834
} catch (...) {
823835
// raise any error
@@ -1030,40 +1042,40 @@ void RestImportHandler::generateDocumentsCreated(
10301042
/// @brief parse a single document line
10311043
////////////////////////////////////////////////////////////////////////////////
10321044

1033-
std::shared_ptr<VPackBuilder> RestImportHandler::parseVelocyPackLine(
1045+
void RestImportHandler::parseVelocyPackLine(
1046+
VPackBuilder& builder,
10341047
std::string const& line, bool& success) {
10351048
try {
10361049
success = true;
1037-
return VPackParser::fromJson(line);
1050+
VPackParser parser(builder);
1051+
parser.parse(line);
10381052
} catch (VPackException const&) {
10391053
success = false;
1040-
VPackParser p;
1041-
return p.steal();
10421054
}
10431055
}
10441056

10451057
////////////////////////////////////////////////////////////////////////////////
10461058
/// @brief parse a single document line
10471059
////////////////////////////////////////////////////////////////////////////////
10481060

1049-
std::shared_ptr<VPackBuilder> RestImportHandler::parseVelocyPackLine(
1061+
void RestImportHandler::parseVelocyPackLine(VPackBuilder& builder,
10501062
char const* start, char const* end, bool& success) {
10511063
try {
1052-
std::string tmp(start, end);
1053-
return parseVelocyPackLine(tmp, success);
1064+
success = true;
1065+
VPackParser parser(builder);
1066+
parser.parse(start,std::distance(start, end));
10541067
} catch (std::exception const&) {
10551068
// The line is invalid and could not be transformed into a string
10561069
success = false;
1057-
VPackParser p;
1058-
return p.steal();
10591070
}
10601071
}
10611072

10621073
////////////////////////////////////////////////////////////////////////////////
10631074
/// @brief create a VelocyPack object from a key and value list
10641075
////////////////////////////////////////////////////////////////////////////////
10651076

1066-
std::shared_ptr<VPackBuilder> RestImportHandler::createVelocyPackObject(
1077+
void RestImportHandler::createVelocyPackObject(
1078+
VPackBuilder& result,
10671079
VPackSlice const& keys, VPackSlice const& values, std::string& errorMsg,
10681080
size_t lineNumber) {
10691081
if (!values.isArray()) {
@@ -1082,28 +1094,26 @@ std::shared_ptr<VPackBuilder> RestImportHandler::createVelocyPackObject(
10821094
THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_BAD_PARAMETER, errorMsg);
10831095
}
10841096

1085-
auto result = std::make_shared<VPackBuilder>();
1086-
result->openObject();
1097+
result.openObject();
10871098

10881099
while (itKeys.valid()) {
10891100
TRI_ASSERT(itValues.valid());
1090-
1101+
10911102
VPackSlice const key = itKeys.value();
10921103
VPackSlice const value = itValues.value();
10931104

10941105
if (key.isString() && !value.isNone() && !value.isNull()) {
10951106
VPackValueLength l;
10961107
char const* p = key.getString(l);
1097-
result->add(p, l, value);
1108+
result.add(p, l, value);
10981109
}
10991110

11001111
itKeys.next();
11011112
itValues.next();
11021113
}
11031114

1104-
result->close();
1115+
result.close();
11051116

1106-
return result;
11071117
}
11081118

11091119
////////////////////////////////////////////////////////////////////////////////

‎arangod/RestHandler/RestImportHandler.h

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ class RestImportHandler : public RestVocbaseBaseHandler {
9292
//////////////////////////////////////////////////////////////////////////////
9393

9494
int handleSingleDocument(SingleCollectionTransaction& trx,
95+
VPackBuilder& lineBuilder,
9596
RestImportResult& result,
9697
arangodb::velocypack::Builder& babies,
9798
arangodb::velocypack::Slice slice,
@@ -140,20 +141,19 @@ class RestImportHandler : public RestVocbaseBaseHandler {
140141
/// @brief parses a string
141142
//////////////////////////////////////////////////////////////////////////////
142143

143-
std::shared_ptr<VPackBuilder> parseVelocyPackLine(std::string const&, bool&);
144+
void parseVelocyPackLine(VPackBuilder&, std::string const&, bool&);
144145

145146
//////////////////////////////////////////////////////////////////////////////
146147
/// @brief parses a string
147148
//////////////////////////////////////////////////////////////////////////////
148149

149-
std::shared_ptr<VPackBuilder> parseVelocyPackLine(char const*, char const*,
150-
bool&);
150+
void parseVelocyPackLine(VPackBuilder&, char const*, char const*, bool&);
151151

152152
//////////////////////////////////////////////////////////////////////////////
153153
/// @brief builds a VPackBuilder object from a key and value list
154154
//////////////////////////////////////////////////////////////////////////////
155155

156-
std::shared_ptr<arangodb::velocypack::Builder> createVelocyPackObject(
156+
void createVelocyPackObject(VPackBuilder&,
157157
arangodb::velocypack::Slice const&, arangodb::velocypack::Slice const&,
158158
std::string&, size_t);
159159

0 commit comments

Comments
 (0)
0