8000 added option `--rocksdb.total-write-buffer-size` (#7074) · strogo/arangodb@8303bc5 · GitHub
[go: up one dir, main page]
Skip to content

Commit 8303bc5

Browse files
jsteemannjsteemann
authored
added option --rocksdb.total-write-buffer-size (arangodb#7074)
1 parent ee1be96 commit 8303bc5

File tree

5 files changed

+27
-0
lines changed

5 files changed

+27
-0
lines changed

‎CHANGELOG

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
v3.3.20 (XXXX-XX-XX)
22
--------------------
33

4+
* added option `--rocksdb.total-write-buffer-size` to limit total memory usage
5+
across all RocksDB in-memory write buffers
6+
47
* in a cluster environment, the arangod process now exits if wrong credentials
58
are used during the startup process.
69

‎Documentation/Books/Manual/Administration/Configuration/RocksDB.md

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,13 @@ The maximum number of write buffers that built up in memory. If this number is
3333
reached before the buffers can be flushed, writes will be slowed or stalled.
3434
Default: 2.
3535

36+
`--rocksdb.total-write-buffer-size` (Hidden)
37+
38+
The total amount of data to build up in all in-memory buffers (backed by log
39+
files). This option, together with the block cache size configuration option,
40+
can be used to limit memory usage. If set to 0, the memory usage is not limited.
41+
Default: 0 (disabled).
42+
3643
`--rocksdb.min-write-buffer-number-to-merge`
3744

3845
Minimum number of write buffers that will be merged together when flushing to

‎arangod/RocksDBEngine/RocksDBEngine.cpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -430,6 +430,10 @@ void RocksDBEngine::start() {
430430
_listener.reset(new RocksDBThrottle);
431431
_options.listeners.push_back(_listener);
432432
}
433+
434+
if (opts->_totalWriteBufferSize > 0) {
435+
_options.db_write_buffer_size = opts->_totalWriteBufferSize;
436+
}
433437

434438
// this is cfFamilies.size() + 2 ... but _option needs to be set before
435439
// building cfFamilies
@@ -538,6 +542,7 @@ void RocksDBEngine::start() {
538542
}
539543
}
540544
}
545+
541546

542547
rocksdb::Status status = rocksdb::TransactionDB::Open(
543548
_options, transactionOptions, _path, cfFamilies, &cfHandles, &_db);

‎lib/ApplicationFeatures/RocksDBOptionFeature.cpp

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ RocksDBOptionFeature::RocksDBOptionFeature(
4545
application_features::ApplicationServer* server)
4646
: application_features::ApplicationFeature(server, "RocksDBOption"),
4747
_transactionLockTimeout(rocksDBTrxDefaults.transaction_lock_timeout),
48+
_totalWriteBufferSize(rocksDBDefaults.db_write_buffer_size),
4849
_writeBufferSize(rocksDBDefaults.write_buffer_size),
4950
_maxWriteBufferNumber(rocksDBDefaults.max_write_buffer_number),
5051
_maxTotalWalSize(80 << 20),
@@ -118,6 +119,10 @@ void RocksDBOptionFeature::collectOptions(
118119
" a transaction attempts to lock a document. A negative value "
119120
"is not recommended as it can lead to deadlocks (0 = no waiting, < 0 no timeout)",
120121
new Int64Parameter(&_transactionLockTimeout));
122+
123+
options->addHiddenOption("--rocksdb.total-write-buffer-size",
124+
"maximum total size of in-memory write buffers (0 = unbounded)",
125+
new UInt64Parameter(&_totalWriteBufferSize));
121126

122127
options->addOption("--rocksdb.write-buffer-size",
123128
"amount of data to build up in memory before converting "
@@ -275,6 +280,11 @@ void RocksDBOptionFeature::validateOptions(
275280
<< "invalid value for '--rocksdb.write-buffer-size'";
276281
FATAL_ERROR_EXIT();
277282
}
283+
if (_totalWriteBufferSize > 0 && _totalWriteBufferSize < 64 * 1024 * 1024) {
284+
LOG_TOPIC(FATAL, arangodb::Logger::FIXME)
285+
<< "invalid value for '--rocksdb.total-write-buffer-size'";
286+
FATAL_ERROR_EXIT();
287+
}
278288
if (_maxBytesForLevelMultiplier <= 0.0) {
279289
LOG_TOPIC(FATAL, arangodb::Logger::FIXME)
280290
<< "invalid value for '--rocksdb.max-bytes-for-level-multiplier'";
@@ -327,6 +337,7 @@ void RocksDBOptionFeature::start() {
327337

328338
LOG_TOPIC(TRACE, Logger::ROCKSDB) << "using RocksDB options:"
329339
<< " wal_dir: " << _walDirectory << "'"
340+
<< ", total_write_buffer_size: " << _totalWriteBufferSize
330341
<< ", write_buffer_size: " << _writeBufferSize
331342
<< ", max_write_buffer_number: " << _maxWriteBufferNumber
332343
<< ", max_total_wal_size: " << _maxTotalWalSize

‎lib/ApplicationFeatures/RocksDBOptionFeature.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ class RocksDBOptionFeature final
4949

5050
int64_t _transactionLockTimeout;
5151
std::string _walDirectory;
52+
uint64_t _totalWriteBufferSize;
5253
uint64_t _writeBufferSize;
5354
uint64_t _maxWriteBufferNumber;
5455
uint64_t _maxTotalWalSize;

0 commit comments

Comments
 (0)
0