8000 add support for LZ4 and LZ4HC compression in RocksDB (#16314) · rnshah9/arangodb@9310cc4 · GitHub
[go: up one dir, main page]
Skip to content

Commit 9310cc4

Browse files
jsteemannjsteemann
authored
add support for LZ4 and LZ4HC compression in RocksDB (arangodb#16314)
1 parent 981cea0 commit 9310cc4

File tree

5 files changed

+34
-3
lines changed

5 files changed

+34
-3
lines changed

‎3rdParty/rocksdb

‎CHANGELOG

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
devel
22
-----
33

4+
* Add support for LZ4 and LZ4HC compression in RocksDB.
5+
46
* Change default value of `--rocksdb.block-cache-shard-bits` to an automatic
57
default value that allows data blocks of at least 128MiB to be stored in each
68
cache shard if the block cache's strict capacity limit is used. The strict

‎arangod/RocksDBEngine/RocksDBEngine.cpp

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -751,12 +751,28 @@ void RocksDBEngine::start() {
751751
_options.max_subcompactions = opts._maxSubcompactions;
752752
_options.use_fsync = opts._useFSync;
753753

754+
rocksdb::CompressionType compressionType = rocksdb::kNoCompression;
755+
if (opts._compressionType == "none") {
756+
compressionType = rocksdb::kNoCompression;
757+
} else if (opts._compressionType == "snappy") {
758+
compressionType = rocksdb::kSnappyCompression;
759+
} else if (opts._compressionType == "lz4") {
760+
compressionType = rocksdb::kLZ4Compression;
761+
} else if (opts._compressionType == "lz4hc") {
762+
compressionType = rocksdb::kLZ4HCCompression;
763+
} else {
764+
TRI_ASSERT(false);
765+
LOG_TOPIC("74b7f", FATAL, arangodb::Logger::STARTUP)
766+
<< "unexpected compression type '" << opts._compressionType << "'";
767+
FATAL_ERROR_EXIT();
768+
}
769+
754770
// only compress levels >= 2
755771
_options.compression_per_level.resize(_options.num_levels);
756772
for (int level = 0; level < _options.num_levels; ++level) {
757773
_options.compression_per_level[level] =
758-
(((uint64_t)level >= opts._numUncompressedLevels)
759-
? rocksdb::kSnappyCompression
774+
((uint64_t(level) >= opts._numUncompressedLevels)
775+
? compressionType
760776
: rocksdb::kNoCompression);
761777
}
762778

‎arangod/RocksDBEngine/RocksDBOptionFeature.cpp

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,9 @@ using namespace arangodb::application_features;
5454
using namespace arangodb::options;
5555

5656
namespace {
57+
std::unordered_set<std::string> const compressionTypes = {
58+
{"snappy"}, {"lz4"}, {"lz4hc"}, {"none"}};
59+
5760
rocksdb::TransactionDBOptions rocksDBTrxDefaults;
5861
rocksdb::Options rocksDBDefaults;
5962
rocksdb::BlockBasedTableOptions rocksDBTableOptionsDefaults;
@@ -133,6 +136,7 @@ RocksDBOptionFeature::RocksDBOptionFeature(Server& server)
133136
_transactionLockStripes(
134137
std::max(NumberOfCores::getValue(), std::size_t(16))),
135138
_transactionLockTimeout(rocksDBTrxDefaults.transaction_lock_timeout),
139+
_compressionType("snappy"),
136140
_totalWriteBufferSize(rocksDBDefaults.db_write_buffer_size),
137141
_writeBufferSize(rocksDBDefaults.write_buffer_size),
138142
_maxWriteBufferNumber(8 + 2), // number of column families plus 2
@@ -221,6 +225,13 @@ void RocksDBOptionFeature::collectOptions(
221225
"regular data directory",
222226
new StringParameter(&_walDirectory));
223227

228+
options
229+
->addOption("--rocksdb.compression-type",
230+
"compression algorithm to use within RocksDB",
231+
new DiscreteValuesParameter<StringParameter>(
232+
&_compressionType, ::compressionTypes))
233+
.setIntroducedIn(31000);
234+
224235
options
225236
->addOption("--rocksdb.target-file-size-base",
226237
"per-file target file size for compaction (in bytes). the "
@@ -698,6 +709,7 @@ void RocksDBOptionFeature::start() {
698709
LOG_TOPIC("f66e4", TRACE, Logger::ROCKSDB)
699710
<< "using RocksDB options:"
700711
<< " wal_dir: '" << _walDirectory << "'"
712+
<< ", compression type: " << _compressionType
701713
<< ", write_buffer_size: " << _writeBufferSize
702714
<< ", total_write_buffer_size: " << _totalWriteBufferSize
703715
<< ", max_write_buffer_number: " << _maxWriteBufferNumber

‎arangod/RocksDBEngine/RocksDBOptionFeature.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ class RocksDBOptionFeature final : public ArangodFeature {
6363
uint64_t _transactionLockStripes;
6464
int64_t _transactionLockTimeout;
6565
std::string _walDirectory;
66+
std::string _compressionType;
6667
uint64_t _totalWriteBufferSize;
6768
uint64_t _writeBufferSize;
6869
// Update max_write_buffer_number above if you change number of families used

0 commit comments

Comments
 (0)
0