8000 less dynamic memory allocations on index operations by jsteemann · Pull Request #6497 · arangodb/arangodb · GitHub
[go: up one dir, main page]
Skip to content

less dynamic memory allocations on index operations #6497

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Sep 14, 2018
Merged

less dynamic memory allocations on index operations #6497

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 31 additions & 28 deletions arangod/RocksDBEngine/RocksDBVPackIndex.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -357,8 +357,8 @@ bool RocksDBVPackIndex::implicitlyUnique() const {
int RocksDBVPackIndex::fillElement(VPackBuilder& leased,
LocalDocumentId const& documentId,
VPackSlice const& doc,
std::vector<RocksDBKey>& elements,
std::vector<uint64_t>& hashes) {
SmallVector<RocksDBKey>& elements,
SmallVector<uint64_t>& hashes) {
if (doc.isNone()) {
LOG_TOPIC(ERR, arangodb::Logger::FIXME)
<< "encountered invalid marker with slice of type None";
Expand Down Expand Up @@ -397,31 +397,30 @@ int RocksDBVPackIndex::fillElement(VPackBuilder& leased,
THROW_ARANGO_EXCEPTION(TRI_ERROR_OUT_OF_MEMORY);
}

elements.emplace_back();
RocksDBKey& key = elements.back();
if (_unique) {
// Unique VPack index values are stored as follows:
// - Key: 7 + 8-byte object ID of index + VPack array with index
// value(s) + separator (NUL) byte
// - Value: primary key
RocksDBKey key;
key.constructUniqueVPackIndexValue(_objectId, leased.slice());
elements.emplace_back(std::move(key));
} else {
// Non-unique VPack index values are stored as follows:
// - Key: 6 + 8-byte object ID of index + VPack array with index
// value(s) + revisionID
// - Value: empty
RocksDBKey key;
key.constructVPackIndexValue(_objectId, leased.slice(), documentId);
elements.emplace_back(std::move(key));
hashes.push_back(leased.slice().normalizedHash());
}
} else {
// other path for handling array elements, too

std::vector<VPackSlice> sliceStack;
SmallVector<VPackSlice>::allocator_type::arena_type sliceStackArena;
SmallVector<VPackSlice> sliceStack{sliceStackArena};

try {
buildIndexValues(leased, documentId, doc, 0, elements, sliceStack,
hashes);
buildIndexValues(leased, documentId, doc, 0, elements, hashes, sliceStack);
} catch (arangodb::basics::Exception const& ex) {
return ex.code();
} catch (std::bad_alloc const&) {
Expand All @@ -438,9 +437,9 @@ int RocksDBVPackIndex::fillElement(VPackBuilder& leased,
void RocksDBVPackIndex::addIndexValue(VPackBuilder& leased,
LocalDocumentId const& documentId,
VPackSlice const& document,
std::vector<RocksDBKey>& elements,
std::vector<VPackSlice>& sliceStack,
std::vector<uint64_t>& hashes) {
SmallVector<RocksDBKey>& elements,
SmallVector<uint64_t>& hashes,
SmallVector<VPackSlice>& sliceStack) {
leased.clear();
leased.openArray(true); // unindexed
for (VPackSlice const& s : sliceStack) {
Expand Down Expand Up @@ -471,14 +470,14 @@ void RocksDBVPackIndex::addIndexValue(VPackBuilder& leased,
void RocksDBVPackIndex::buildIndexValues(VPackBuilder& leased,
LocalDocumentId const& documentId,
VPackSlice const doc, size_t level,
std::vector<RocksDBKey>& elements,
std::vector<VPackSlice>& sliceStack,
std::vector<uint64_t>& hashes) {
SmallVector<RocksDBKey>& elements,
SmallVector<uint64_t>& hashes,
SmallVector<VPackSlice>& sliceStack) {
// Invariant: level == sliceStack.size()

// Stop the recursion:
if (level == _paths.size()) {
addIndexValue(leased, documentId, doc, elements, sliceStack, hashes);
addIndexValue(leased, documentId, doc, elements, hashes, sliceStack);
return;
}

Expand All @@ -492,8 +491,7 @@ void RocksDBVPackIndex::buildIndexValues(VPackBuilder& leased,
} else {
sliceStack.emplace_back(slice);
}
buildIndexValues(leased, documentId, doc, level + 1, elements, sliceStack,
hashes);
buildIndexValues(leased, documentId, doc, level + 1, elements, hashes, sliceStack);
sliceStack.pop_back();
return;
}
Expand All @@ -514,7 +512,7 @@ void RocksDBVPackIndex::buildIndexValues(VPackBuilder& leased,
for (size_t i = level; i < _paths.size(); i++) {
sliceStack.emplace_back(illegalSlice);
}
addIndexValue(leased, documentId, doc, elements, sliceStack, hashes);
addIndexValue(leased, documentId, doc, elements, hashes, sliceStack);
for (size_t i = level; i < _paths.size(); i++) {
sliceStack.pop_back();
}
Expand Down Expand Up @@ -549,8 +547,7 @@ void RocksDBVPackIndex::buildIndexValues(VPackBuilder& leased,
if (it == seen.end()) {
seen.insert(something);
sliceStack.emplace_back(something);
buildIndexValues(leased, documentId, doc, level + 1, elements, sliceStack,
hashes);
buildIndexValues(leased, documentId, doc, level + 1, elements, hashes, sliceStack);
sliceStack.pop_back();
} else if (_unique && !_deduplicate) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_ARANGO_UNIQUE_CONSTRAINT_VIOLATED);
Expand Down Expand Up @@ -616,8 +613,10 @@ Result RocksDBVPackIndex::insertInternal(transaction::Methods* trx,
LocalDocumentId const& documentId,
VPackSlice const& doc,
OperationMode mode) {
std::vector<RocksDBKey> elements;
std::vector<uint64_t> hashes;
SmallVector<RocksDBKey>::allocator_type::arena_type elementsArena;
SmallVector<RocksDBKey> elements{elementsArena};
SmallVector<uint64_t>::allocator_type::arena_type hashesArena;
SmallVector<uint64_t> hashes{hashesArena};
int res = TRI_ERROR_NO_ERROR;
{
// rethrow all types of exceptions from here...
Expand Down Expand Up @@ -733,9 +732,11 @@ Result RocksDBVPackIndex::updateInternal(
newDocumentId, newDoc, mode);
}

// more expansive method to
std::vector<RocksDBKey> elements;
std::vector<uint64_t> hashes;
// more expensive method to
SmallVector<RocksDBKey>::allocator_type::arena_type elementsArena;
SmallVector<RocksDBKey> elements{elementsArena};
SmallVector<uint64_t>::allocator_type::arena_type hashesArena;
SmallVector<uint64_t> hashes{hashesArena};
int res = TRI_ERROR_NO_ERROR;
{
// rethrow all types of exceptions from here...
Expand Down Expand Up @@ -777,8 +778,10 @@ Result RocksDBVPackIndex::removeInternal(transaction::Methods* trx,
LocalDocumentId const& documentId,
VPackSlice const& doc,
OperationMode mode) {
std::vector<RocksDBKey> elements;
std::vector<uint64_t> hashes;
SmallVector<RocksDBKey>::allocator_type::arena_type elementsArena;
SmallVector<RocksDBKey> elements{elementsArena};
SmallVector<uint64_t>::allocator_type::arena_type hashesArena;
SmallVector<uint64_t> hashes{hashesArena};
int res = TRI_ERROR_NO_ERROR;
{
// rethrow all types of exceptions from here...
Expand Down
19 changes: 10 additions & 9 deletions arangod/RocksDBEngine/RocksDBVPackIndex.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,9 @@
#ifndef ARANGOD_ROCKSDB_ROCKSDB_VPACK_INDEX_H
#define ARANGOD_ROCKSDB_ROCKSDB_VPACK_INDEX_H 1

#include "Aql/AstNode.h"
#include "Basics/Common.h"
#include "Aql/AstNode.h"
#include "Basics/SmallVector.h"
#include "Indexes/IndexIterator.h"
#include "RocksDBEngine/RocksDBCuckooIndexEstimator.h"
#include "RocksDBEngine/RocksDBIndex.h"
Expand Down Expand Up @@ -238,18 +239,18 @@ class RocksDBVPackIndex : public RocksDBIndex {
/// @brief helper function to insert a document into any index type
int fillElement(velocypack::Builder& leased,
LocalDocumentId const& documentId, VPackSlice const& doc,
std::vector<RocksDBKey>& elements,
std::vector<uint64_t>& hashes);
SmallVector<RocksDBKey>& elements,
SmallVector<uint64_t>& hashes);

/// @brief helper function 8000 to build the key and value for rocksdb from the
/// vector of slices
/// @param hashes list of VPackSlice hashes for the estimator.
void addIndexValue(velocypack::Builder& leased,
LocalDocumentId const& documentId,
VPackSlice const& document,
std::vector<RocksDBKey>& elements,
std::vector<VPackSlice>& sliceStack,
std::vector<uint64_t>& hashes);
SmallVector<RocksDBKey>& elements,
SmallVector<uint64_t>& hashes,
SmallVector<VPackSlice>& sliceStack);

/// @brief helper function to create a set of value combinations to insert
/// into the rocksdb index.
Expand All @@ -259,9 +260,9 @@ class RocksDBVPackIndex : public RocksDBIndex {
void buildIndexValues(velocypack::Builder& leased,
LocalDocumentId const& documentId,
VPackSlice const document, size_t level,
std::vector<RocksDBKey>& elements,
std::vector<VPackSlice>& sliceStack,
std::vector<uint64_t>& hashes);
SmallVector<RocksDBKey>& elements,
SmallVector<uint64_t>& hashes,
SmallVector<VPackSlice>& sliceStack);

private:
/// @brief the attribute paths
Expand Down
2 changes: 1 addition & 1 deletion arangod/RocksDBEngine/RocksDBValue.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ RocksDBValue::RocksDBValue(RocksDBEntryType type, LocalDocumentId const& docId,
switch (_type) {
case RocksDBEntryType::UniqueVPackIndexValue:
case RocksDBEntryType::PrimaryIndexValue: {
if(!revision){
if (!revision) {
_buffer.reserve(sizeof(uint64_t));
uint64ToPersistent(_buffer, docId.id()); // LocalDocumentId
} else {
Expand Down
0