8000 current version · lethalbrains/arangodb@b26aa2a · GitHub
[go: up one dir, main page]
Skip to content

Commit b26aa2a

Browse files
jsteemannjsteemann
committed
current version
1 parent 17971d0 commit b26aa2a

27 files changed

+383
-118
lines changed

‎arangod/Aql/IndexBlock.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -331,7 +331,7 @@ std::shared_ptr<arangodb::OperationCursor> IndexBlock::createCursor() {
331331
return ast->query()->trx()->indexScanForCondition(
332332
_collection->getName(), _indexes[_currentIndex], ast,
333333
conditionNode, outVariable, UINT64_MAX,
334-
TRI_DEFAULT_BATCH_SIZE, node->_reverse);
334+
Transaction::defaultBatchSize(), node->_reverse);
335335
DEBUG_END_BLOCK();
336336
}
337337

‎arangod/Indexes/EdgeIndex.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,8 @@ class EdgeIndex final : public Index {
141141
IndexType type() const override final {
142142
return Index::TRI_IDX_TYPE_EDGE_INDEX;
143143
}
144+
145+
bool canBeDropped() const override final { return false; }
144146

145147
bool isSorted() const override final { return false; }
146148

‎arangod/Indexes/FulltextIndex.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,8 @@ class FulltextIndex final : public Index {
4848
IndexType type() const override final {
4949
return Index::TRI_IDX_TYPE_FULLTEXT_INDEX;
5050
}
51+
52+
bool canBeDropped() const override final { return true; }
5153

5254
bool isSorted() const override final { return false; }
5355

‎arangod/Indexes/GeoIndex2.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,8 @@ class GeoIndex2 final : public Index {
7070

7171
return TRI_IDX_TYPE_GEO2_INDEX;
7272
}
73+
74+
bool canBeDropped() const override final { return true; }
7375

7476
bool isSorted() const override final { return false; }
7577

‎arangod/Indexes/HashIndex.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,8 @@ class HashIndex final : public PathBasedIndex {
101101
IndexType type() const override final {
102102
return Index::TRI_IDX_TYPE_HASH_INDEX;
103103
}
104+
105+
bool canBeDropped() const override final { return true; }
104106

105107
bool isSorted() const override final { return false; }
106108

‎arangod/Indexes/Index.cpp

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -444,6 +444,15 @@ int Index::cleanup() {
444444
return TRI_ERROR_NO_ERROR;
445445
}
446446

447+
////////////////////////////////////////////////////////////////////////////////
448+
/// @brief default implementation for drop
449+
////////////////////////////////////////////////////////////////////////////////
450+
451+
int Index::drop() {
452+
// do nothing
453+
return TRI_ERROR_NO_ERROR;
454+
}
455+
447456
////////////////////////////////////////////////////////////////////////////////
448457
/// @brief default implementation for sizeHint
449458
////////////////////////////////////////////////////////////////////////////////

‎arangod/Indexes/Index.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -329,6 +329,7 @@ class Index {
329329
static bool Compare(VPackSlice const& lhs, VPackSlice const& rhs);
330330

331331
virtual IndexType type() const = 0;
332+
virtual bool canBeDropped() const = 0;
332333

333334
//////////////////////////////////////////////////////////////////////////////
334335
/// @brief whether or not the index is sorted
@@ -357,6 +358,8 @@ class Index {
357358

358359
// a garbage collection function for the index
359360
virtual int cleanup();
361+
// called when the index is dropped
362+
virtual int drop();
360363

361364
// give index a hint about the expected size
362365
virtual int sizeHint(arangodb::Transaction*, size_t);

‎arangod/Indexes/PrimaryIndex.h

Lines changed: 2 additions & 0 deletions
< 10000 td data-grid-cell-id="diff-a952f703151ef93c55c0b92bae17e9692b4ef3e5e9fb2623b0610f2e92a0245b-128-128-1" data-selected="false" role="gridcell" style="background-color:var(--bgColor-default);text-align:center" tabindex="-1" valign="top" class="focusable-grid-cell diff-line-number position-relative diff-line-number-neutral left-side">128
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,8 @@ class PrimaryIndex final : public Index {
128
return Index::TRI_IDX_TYPE_PRIMARY_INDEX;
129129
}
130130

131+
bool canBeDropped() const override final { return false; }
132+
131133
bool isSorted() const override final { return false; }
132134

133135
bool hasSelectivityEstimate() const override final { return true; }

‎arangod/Indexes/RocksDBFeature.cpp

Lines changed: 120 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,24 @@
2222
////////////////////////////////////////////////////////////////////////////////
2323

2424
#include "RocksDBFeature.h"
25+
#include "Basics/Exceptions.h"
2526
#include "Basics/FileUtils.h"
2627
#include "Indexes/RocksDBKeyComparator.h"
2728
#include "Logger/Logger.h"
2829

2930
#include <rocksdb/db.h>
31+
#include <rocksdb/convenience.h>
3032
#include <rocksdb/filter_policy.h>
33+
#include <rocksdb/iterator.h>
3134
#include <rocksdb/options.h>
35+
#include <rocksdb/slice_transform.h>
3236
#include <rocksdb/table.h>
37+
#include <rocksdb/utilities/optimistic_transaction_db.h>
38+
#include <rocksdb/write_batch.h>
39+
40+
#include <velocypack/Builder.h>
41+
#include <velocypack/Slice.h>
42+
#include <velocypack/velocypack-aliases.h>
3343

3444
using namespace arangodb;
3545

@@ -47,6 +57,7 @@ int RocksDBFeature::initialize(std::string const& path) {
4757
LOG(INFO) << "initializing rocksdb";
4858

4959
_path = arangodb::basics::FileUtils::buildFilename(path, "rocksdb");
60+
5061
if (!arangodb::basics::FileUtils::isDirectory(_path)) {
5162
std::string systemErrorStr;
5263
long errorNo;
@@ -65,8 +76,9 @@ int RocksDBFeature::initialize(std::string const& path) {
6576

6677
rocksdb::BlockBasedTableOptions tableOptions;
6778
tableOptions.cache_index_and_filter_blocks = true;
68-
tableOptions.filter_policy.reset(rocksdb::NewBloomFilterPolicy(12));
79+
tableOptions.filter_policy.reset(rocksdb::NewBloomFilterPolicy(12, false));
6980

81+
_options.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(RocksDBIndex::minimalPrefixSize()));
7082
_options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(tableOptions));
7183
_options.create_if_missing = true;
7284
_options.max_open_files = -1;
@@ -84,7 +96,7 @@ int RocksDBFeature::initialize(std::string const& path) {
8496
//options.env->SetBackgroundThreads(num_threads, Env::Priority::HIGH);
8597
//options.env->SetBackgroundThreads(num_threads, Env::Priority::LOW);
8698

87-
rocksdb::Status status = rocksdb::DB::Open(_options, _path, &_db);
99+
rocksdb::Status status = rocksdb::OptimisticTransactionDB::Open(_options, _path, &_db);
88100

89101
if (! status.ok()) {
90102
return TRI_ERROR_INTERNAL;
@@ -94,29 +106,128 @@ int RocksDBFeature::initialize(std::string const& path) {
94106
}
95107

96108
int RocksDBFeature::shutdown() {
97-
LOG(INFO) << "shutting down rocksdb";
109+
LOG(TRACE) << "shutting down rocksdb";
98110

99111
// flush
100112
rocksdb::FlushOptions options;
101113
options.wait = true;
102-
rocksdb::Status status = _db->Flush(options);
114+
rocksdb::Status status = _db->GetBaseDB()->Flush(options);
103115

104116
if (! status.ok()) {
105117
LOG(ERR) << "error flushing rocksdb: " << status.ToString();
106118
}
107119

108-
status = _db->SyncWAL();
120+
return syncWal();
121+
}
122+
123+
RocksDBFeature* RocksDBFeature::instance() {
124+
if (Instance == nullptr) {
125+
Instance = new RocksDBFeature();
126+
}
127+
return Instance;
128+
}
129+
130+
int RocksDBFeature::syncWal() {
131+
LOG(TRACE) << "syncing rocksdb WAL";
132+
133+
rocksdb::Status status = instance()->db()->GetBaseDB()->SyncWAL();
134+
109135
if (! status.ok()) {
110136
LOG(ERR) << "error syncing rocksdb WAL: " << status.ToString();
137+
return TRI_ERROR_INTERNAL;
111138
}
112139

113140
return TRI_ERROR_NO_ERROR;
114141
}
115142

116-
RocksDBFeature* RocksDBFeature::instance() {
117-
if (Instance == nullptr) {
118-
Instance = new RocksDBFeature();
143+
int RocksDBFeature::dropDatabase(TRI_voc_tick_t databaseId) {
144+
return instance()->dropPrefix(RocksDBIndex::buildPrefix(databaseId));
145+
}
146+
147+
int RocksDBFeature::dropCollection(TRI_voc_tick_t databaseId, TRI_voc_cid_t collectionId) {
148+
return instance()->dropPrefix(RocksDBIndex::buildPrefix(databaseId, collectionId));
149+
}
150+
151+
int RocksDBFeature::dropIndex(TRI_voc_tick_t databaseId, TRI_voc_cid_t collectionId, TRI_idx_iid_t indexId) {
152+
return instance()->dropPrefix(RocksDBIndex::buildPrefix(databaseId, collectionId, indexId));
153+
}
154+
155+
int RocksDBFeature::dropPrefix(std::string const& prefix) {
156+
try {
157+
VPackBuilder builder;
158+
159+
// create lower and upper bound for deletion
160+
builder.openArray();
161+
builder.add(VPackSlice::minKeySlice());
162+
builder.close();
163+
164+
std::string l;
165+
l.reserve(prefix.size() + builder.slice().byteSize());
166+
l.append(prefix);
167+
l.append(builder.slice().startAs<char const>(), builder.slice().byteSize());
168+
169+
builder.clear();
170+
builder.openArray();
171+
builder.add(VPackSlice::maxKeySlice());
172+
builder.close();
173+
174+
std::string u;
175+
u.reserve(prefix.size() + builder.slice().byteSize());
176+
u.append(prefix);
177+
u.append(builder.slice().startAs<char const>(), builder.slice().byteSize());
178+
179+
#if 0
180+
for (size_t i = 0; i < prefix.size(); i += sizeof(TRI_idx_iid_t)) {
181+
char const* x = prefix.c_str() + i;
182+
LOG(TRACE) << "prefix part: " << std::to_string(*reinterpret_cast<uint64_t const*>(x));
183+
}
184+
#endif
185+
186+
LOG(TRACE) << "dropping range: " << VPackSlice(l.c_str() + prefix.size()).toJson() << " - " << VPackSlice(u.c_str() + prefix.size()).toJson();
187+
188+
rocksdb::Slice lower(l.c_str(), l.size());
189+
rocksdb::Slice upper(u.c_str(), u.size());
190+
191+
rocksdb::Status status = rocksdb::DeleteFilesInRange(_db->GetBaseDB(), _db->GetBaseDB()->DefaultColumnFamily(), &lower, &upper);
192+
193+
if (!status.ok()) {
194+
// if file deletion failed, we will still iterate over the remaining keys, so we
195+
// don't need to abort and raise an error here
196+
LOG(WARN) << "rocksdb file deletion failed";
197+
}
198+
199+
// go on and delete the remaining keys (delete files in range does not necessarily
200+
// find them all, just complete files
201+
202+
rocksdb::DB* db = _db->GetBaseDB();
203+
204+
rocksdb::WriteBatch batch;
205+
206+
std::unique_ptr<rocksdb::Iterator> it(db->NewIterator(rocksdb::ReadOptions()));
207+
208+
while (it->Valid()) {
209+
batch.Delete(it->key());
210+
it->Next();
211+
}
212+
213+
// now apply deletion batch
214+
status = db->Write(rocksdb::WriteOptions(), &batch);
215+
216+
if (!status.ok()) {
217+
LOG(WARN) << "rocksdb key deletion failed";
218+
return TRI_ERROR_INTERNAL;
219+
}
220+
221+
return TRI_ERROR_NO_ERROR;
222+
} catch (arangodb::basics::Exception const& ex) {
223+
LOG(ERR) << "caught exception during prefix deletion: " << ex.what();
224+
return ex.code();
225+
} catch (std::exception const& ex) {
226+
LOG(ERR) << "caught exception during prefix deletion: " << ex.what();
227+
return TRI_ERROR_INTERNAL;
228+
} catch (...) {
229+
LOG(ERR) << "caught unknown exception during prefix deletion";
230+
return TRI_ERROR_INTERNAL;
119231
}
120-
return Instance;
121232
}
122233

‎arangod/Indexes/RocksDBFeature.h

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,12 @@
2525
#define ARANGOD_INDEXES_ROCKS_DB_FEATURE_H 1
2626

2727
#include "Basics/Common.h"
28+
#include "VocBase/voc-types.h"
2829

2930
#include <rocksdb/options.h>
3031

3132
namespace rocksdb {
32-
class DB;
33+
class OptimisticTransactionDB;
3334
}
3435

3536
namespace arangodb {
@@ -40,21 +41,29 @@ class RocksDBFeature {
4041
RocksDBFeature();
4142
~RocksDBFeature();
4243

43-
inline rocksdb::DB* db() const { return _db; }
44+
inline rocksdb::OptimisticTransactionDB* db() const { return _db; }
4445
inline RocksDBKeyComparator* comparator() const { return _comparator; }
4546

4647
int initialize(std::string const&);
4748
int shutdown();
4849

4950
static RocksDBFeature* instance();
5051

52+
static int syncWal();
53+
static int dropDatabase(TRI_voc_tick_t);
54+
static int dropCollection(TRI_voc_tick_t, TRI_voc_cid_t);
55+
static int dropIndex(TRI_voc_tick_t, TRI_voc_cid_t, TRI_idx_iid_t);
56+
5157
private:
5258

53-
rocksdb::DB* _db;
54-
rocksdb::Options _options;
55-
RocksDBKeyComparator* _comparator;
56-
std::string _path;
59+
int dropPrefix(std::string const& prefix);
60+
61+
private:
5762

63+
rocksdb::OptimisticTransactionDB* _db;
64+
rocksdb::Options _options;
65+
RocksDBKeyComparator* _comparator;
66+
std::string _path;
5867
};
5968

6069
}

0 commit comments

Comments
 (0)
0