From caa5a8c23d9f955af2c6bb7620cee5b62acae7c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Gr=C3=A4tzer?= Date: Tue, 4 Feb 2020 16:42:42 +0100 Subject: [PATCH 01/11] Encryption key rotation --- .../RestHandler/RestAdminServerHandler.cpp | 6 +++++ arangod/RestHandler/RestAdminServerHandler.h | 1 + arangod/RocksDBEngine/RocksDBEngine.h | 7 ++++++ lib/Basics/StringUtils.h | 2 +- lib/Basics/VelocyPackHelper.cpp | 2 -- lib/V8/v8-buffer.cpp | 22 ++++++++++++------- lib/V8/v8-utils.cpp | 1 - 7 files changed, 29 insertions(+), 12 deletions(-) diff --git a/arangod/RestHandler/RestAdminServerHandler.cpp b/arangod/RestHandler/RestAdminServerHandler.cpp index eb1ecb1546a3..8f3e33d03e6f 100644 --- a/arangod/RestHandler/RestAdminServerHandler.cpp +++ b/arangod/RestHandler/RestAdminServerHandler.cpp @@ -58,6 +58,8 @@ RestStatus RestAdminServerHandler::execute() { handleTLS(); } else if (suffixes.size() == 1 && suffixes[0] == "jwt") { handleJWTSecretsReload(); + } else if (suffixes.size() == 1 && suffixes[0] == "encryption") { + handleEncryptionKeyRotation(); } else { generateError(rest::ResponseCode::NOT_FOUND, TRI_ERROR_HTTP_NOT_FOUND); } @@ -256,4 +258,8 @@ void RestAdminServerHandler::handleTLS() { void RestAdminServerHandler::handleJWTSecretsReload() { generateError(rest::ResponseCode::NOT_FOUND, TRI_ERROR_HTTP_NOT_FOUND); } + +void RestAdminServerHandler::handleEncryptionKeyRotation() { + generateError(rest::ResponseCode::NOT_FOUND, TRI_ERROR_HTTP_NOT_FOUND); +} #endif diff --git a/arangod/RestHandler/RestAdminServerHandler.h b/arangod/RestHandler/RestAdminServerHandler.h index f18e63344665..5ad9aad751c7 100644 --- a/arangod/RestHandler/RestAdminServerHandler.h +++ b/arangod/RestHandler/RestAdminServerHandler.h @@ -54,6 +54,7 @@ class RestAdminServerHandler : public RestBaseHandler { void writeModeResult(bool); void handleJWTSecretsReload(); + void handleEncryptionKeyRotation(); }; } // namespace arangodb diff --git a/arangod/RocksDBEngine/RocksDBEngine.h b/arangod/RocksDBEngine/RocksDBEngine.h index d35a23230810..7ee63fe305bd 100644 --- a/arangod/RocksDBEngine/RocksDBEngine.h +++ b/arangod/RocksDBEngine/RocksDBEngine.h @@ -350,11 +350,18 @@ class RocksDBEngine final : public StorageEngine { void startEnterprise(); void configureEnterpriseRocksDBOptions(rocksdb::Options& options); void validateJournalFiles() const; + + Result readUserEncryptionIV(std::string& out); enterprise::RocksDBEngineEEData _eeData; public: std::string const& getEncryptionKey(); + + std::string const& getEncryptedIVFile(); + + Result rotateEncryptionKey(); + #endif private: // activate generation of SHA256 files to parallel .sst files diff --git a/lib/Basics/StringUtils.h b/lib/Basics/StringUtils.h index d78befadd787..3c157b01e81b 100644 --- a/lib/Basics/StringUtils.h +++ b/lib/Basics/StringUtils.h @@ -165,7 +165,7 @@ static inline char tolower(char c) { } static inline unsigned char tolower(unsigned char c) { - return c + ((c - 65U < 26U) << 5); + return static_cast(c + ((c - 65U < 26U) << 5)); } static inline char toupper(char c) { diff --git a/lib/Basics/VelocyPackHelper.cpp b/lib/Basics/VelocyPackHelper.cpp index 4920f83ba3ea..6572de93f2cf 100644 --- a/lib/Basics/VelocyPackHelper.cpp +++ b/lib/Basics/VelocyPackHelper.cpp @@ -60,8 +60,6 @@ #include "Basics/memory.h" #include "Basics/system-compiler.h" #include "Logger/LogMacros.h" -#include "Logger/Logger.h" -#include "Logger/LoggerStream.h" extern "C" { unsigned long long XXH64(const void* input, size_t length, unsigned long long seed); diff --git a/lib/V8/v8-buffer.cpp b/lib/V8/v8-buffer.cpp index f91385f7e341..04838679b26a 100644 --- a/lib/V8/v8-buffer.cpp +++ b/lib/V8/v8-buffer.cpp @@ -712,7 +712,8 @@ void V8Buffer::replace(v8::Isolate* isolate, char* data, size_t length, memcpy(_data, data, _length); } - isolate->AdjustAmountOfExternalAllocatedMemory(sizeof(V8Buffer) + _length + SAFETY_OVERHEAD); + int64_t bytesToChange = (int64_t)(sizeof(V8Buffer) + _length + SAFETY_OVERHEAD); + isolate->AdjustAmountOfExternalAllocatedMemory(bytesToChange); } else { _data = NULL; } @@ -808,9 +809,12 @@ static void JS_Ucs2Slice(v8::FunctionCallbackInfo const& args) { if (!sliceArgs(isolate, args[0], args[1], parent, start, end)) { return; } + + std::vector buffer; + buffer.resize((end - start) / 2); + memcpy(buffer.data(), parent->_data + start, (end - start)); - uint16_t* data = (uint16_t*)(parent->_data + start); - TRI_V8_RETURN(TRI_V8_STRING_UTF16(isolate, data, (end - start) / 2)); + TRI_V8_RETURN(TRI_V8_STRING_UTF16(isolate, buffer.data(), buffer.size())); } //////////////////////////////////////////////////////////////////////////////// @@ -829,7 +833,7 @@ static void JS_HexSlice(v8::FunctionCallbackInfo const& args) { } char* src = parent->_data + start; - uint32_t dstlen = (end - start) * 2; + uint32_t dstlen = static_cast(end - start) * 2; if (dstlen == 0) { TRI_V8_RETURN(v8::String::Empty(isolate)); @@ -948,7 +952,7 @@ static void JS_Fill(v8::FunctionCallbackInfo const& args) { return; } - memset((void*)(parent->_data + start), value, end - start); + memset((void*)(parent->_data + start), value, static_cast(end - start)); TRI_V8_RETURN_UNDEFINED(); } @@ -1079,11 +1083,13 @@ static void JS_Ucs2Write(v8::FunctionCallbackInfo const& args) { : TRI_GET_UINT32(args[2]); max_length = MIN(buffer->_length - offset, max_length) / 2; - - uint16_t* p = (uint16_t*)(buffer->_data + offset); + + std::vector ucs2; + ucs2.resize(max_length); + memcpy(ucs2.data(), buffer->_data + offset, max_length); int written = - s->Write(isolate, p, 0, (int)max_length, + s->Write(isolate, ucs2.data(), 0, (int)max_length, (v8::String::HINT_MANY_WRITES_EXPECTED | v8::String::NO_NULL_TERMINATION)); TRI_V8_RETURN(v8::Integer::New(isolate, written * 2)); diff --git a/lib/V8/v8-utils.cpp b/lib/V8/v8-utils.cpp index 01ad782ea027..45855b55bd5b 100644 --- a/lib/V8/v8-utils.cpp +++ b/lib/V8/v8-utils.cpp @@ -2063,7 +2063,6 @@ static void JS_Load(v8::FunctionCallbackInfo const& args) { TRI_V8_LOG_THROW_EXCEPTION(tryCatch); } else { tryCatch.ReThrow(); - TRI_GET_GLOBALS(); v8g->_canceled = true; TRI_V8_RETURN_UNDEFINED(); } From e7331ad9eed0e22e1c897c1c03816aa65179f827 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Gr=C3=A4tzer?= Date: Tue, 11 Feb 2020 16:22:21 +0100 Subject: [PATCH 02/11] fix log id --- arangod/GeneralServer/H2CommTask.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arangod/GeneralServer/H2CommTask.cpp b/arangod/GeneralServer/H2CommTask.cpp index b32bc66a7990..6cc33e3092fb 100644 --- a/arangod/GeneralServer/H2CommTask.cpp +++ b/arangod/GeneralServer/H2CommTask.cpp @@ -153,7 +153,7 @@ template me->_streams.erase(stream_id); if (error_code != NGHTTP2_NO_ERROR) { - LOG_TOPIC("d04f7", DEBUG, Logger::REQUESTS) << " closing stream " + LOG_TOPIC("2824d", DEBUG, Logger::REQUESTS) << " closing stream " << stream_id << " with error '" << nghttp2_http2_strerror(error_code) << "' (" << error_code << ")"; } From 0b04bce5e5d0eb6720f9c78e83a4bd53a5083742 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Gr=C3=A4tzer?= Date: Tue, 11 Feb 2020 19:07:24 +0100 Subject: [PATCH 03/11] change some stuff --- arangod/RocksDBEngine/RocksDBEngine.cpp | 5 +++-- arangod/RocksDBEngine/RocksDBEngine.h | 16 ++++++++++++---- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/arangod/RocksDBEngine/RocksDBEngine.cpp b/arangod/RocksDBEngine/RocksDBEngine.cpp index 5bf6c387fd98..c153b1d45350 100644 --- a/arangod/RocksDBEngine/RocksDBEngine.cpp +++ b/arangod/RocksDBEngine/RocksDBEngine.cpp @@ -396,6 +396,7 @@ void RocksDBEngine::start() { auto& databasePathFeature = server().getFeature(); _path = databasePathFeature.subdirectoryName("engine-rocksdb"); + bool createdEngineDir = false; if (!basics::FileUtils::isDirectory(_path)) { std::string systemErrorStr; long errorNo; @@ -411,6 +412,7 @@ void RocksDBEngine::start() { << "': " << systemErrorStr; FATAL_ERROR_EXIT(); } + createdEngineDir = true; } // options imported set by RocksDBOptionFeature @@ -487,8 +489,7 @@ void RocksDBEngine::start() { _options.compaction_readahead_size = static_cast(opts._compactionReadaheadSize); #ifdef USE_ENTERPRISE - configureEnterpriseRocksDBOptions(_options); - startEnterprise(); + configureEnterpriseRocksDBOptions(_options, createdEngineDir); #endif _options.env->SetBackgroundThreads(static_cast(opts._numThreadsHigh), diff --git a/arangod/RocksDBEngine/RocksDBEngine.h b/arangod/RocksDBEngine/RocksDBEngine.h index 7ee63fe305bd..f0124051a6e9 100644 --- a/arangod/RocksDBEngine/RocksDBEngine.h +++ b/arangod/RocksDBEngine/RocksDBEngine.h @@ -347,21 +347,29 @@ class RocksDBEngine final : public StorageEngine { void collectEnterpriseOptions(std::shared_ptr); void validateEnterpriseOptions(std::shared_ptr); void prepareEnterprise(); - void startEnterprise(); - void configureEnterpriseRocksDBOptions(rocksdb::Options& options); + void configureEnterpriseRocksDBOptions(rocksdb::Options& options, bool createdEngineDir); void validateJournalFiles() const; - Result readUserEncryptionIV(std::string& out); + Result readUserEncryptionKey(std::string& out) const; enterprise::RocksDBEngineEEData _eeData; public: std::string const& getEncryptionKey(); - std::string const& getEncryptedIVFile(); + /// encryption file + std::string const& getEncryptionFilePath(); + /// encrypted key file path + std::string const& getEncryptedKeyFilePath(); + /// reload user-provided key, writes out the internal key file Result rotateEncryptionKey(); +private: + + /// reload encryption key, using new user provided key file + Result loadEncryptionKey(); + #endif private: // activate generation of SHA256 files to parallel .sst files From dec595cdfa55b1e7e419a79953fc28baa6b84aa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Gr=C3=A4tzer?= Date: Wed, 12 Feb 2020 15:53:00 +0100 Subject: [PATCH 04/11] add testing for re-encryption --- arangod/V8Server/v8-vocbase.cpp | 39 +++++++++++++++++++ .../modules/@arangodb/testsuites/recovery.js | 11 +++++- js/common/bootstrap/modules/internal.js | 9 +++++ 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/arangod/V8Server/v8-vocbase.cpp b/arangod/V8Server/v8-vocbase.cpp index 0de9a5255ab9..c846d82ee536 100644 --- a/arangod/V8Server/v8-vocbase.cpp +++ b/arangod/V8Server/v8-vocbase.cpp @@ -64,6 +64,7 @@ #include "RestServer/ConsoleThread.h" #include "RestServer/DatabaseFeature.h" #include "RestServer/QueryRegistryFeature.h" +#include "RocksDBEngine/RocksDBEngine.h" #include "Statistics/StatisticsFeature.h" #include "StorageEngine/EngineSelectorFeature.h" #include "StorageEngine/StorageEngine.h" @@ -1946,6 +1947,36 @@ static void JS_AgencyDump(v8::FunctionCallbackInfo const& args) { TRI_V8_TRY_CATCH_END } +#ifdef USE_ENTERPRISE + +//////////////////////////////////////////////////////////////////////////////// +/// @brief this is rotates the encryption keys, only for testing +//////////////////////////////////////////////////////////////////////////////// + +static void JS_EncryptionKeyReload(v8::FunctionCallbackInfo const& args) { + TRI_V8_TRY_CATCH_BEGIN(isolate); + v8::HandleScope scope(isolate); + + if (args.Length() != 0) { + TRI_V8_THROW_EXCEPTION_USAGE("encryptionKeyReload()"); + } + + if (!EngineSelectorFeature::isRocksDB()) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); + } + + auto* engine = EngineSelectorFeature::ENGINE; + auto res = static_cast(engine)->rotateEncryptionKey(); + if (res.fail()) { + TRI_V8_THROW_EXCEPTION(res); + } + + TRI_V8_RETURN_TRUE(); + TRI_V8_TRY_CATCH_END +} + +#endif + //////////////////////////////////////////////////////////////////////////////// /// @brief creates a TRI_vocbase_t global context //////////////////////////////////////////////////////////////////////////////// @@ -2133,6 +2164,14 @@ void TRI_InitV8VocBridge(v8::Isolate* isolate, v8::Handle context, TRI_AddGlobalFunctionVocbase(isolate, TRI_V8_ASCII_STRING(isolate, "AGENCY_DUMP"), JS_AgencyDump, true); + +#ifdef USE_ENTERPRISE + if (V8DealerFeature::DEALER && V8DealerFeature::DEALER->allowAdminExecute()) { + TRI_AddGlobalFunctionVocbase(isolate, + TRI_V8_ASCII_STRING(isolate, "ENCRYPTION_KEY_RELOAD"), + JS_EncryptionKeyReload, true); + } +#endif // ............................................................................. // create global variables diff --git a/js/client/modules/@arangodb/testsuites/recovery.js b/js/client/modules/@arangodb/testsuites/recovery.js index b47d65c2dc72..ba1d2fc848f0 100644 --- a/js/client/modules/@arangodb/testsuites/recovery.js +++ b/js/client/modules/@arangodb/testsuites/recovery.js @@ -92,7 +92,16 @@ function runArangodRecovery (params) { }); if (useEncryption) { - args['rocksdb.encryption-keyfile'] = tu.pathForTesting('server/recovery/encryption-keyfile'); + const key = '01234567890123456789012345678901'; + let keyDir = fs.join(fs.getTempPath(), 'arango_encryption'); + fs.makeDirectory(keyDir); + pu.cleanupDBDirectoriesAppend(keyDir); + + let keyfile = fs.join(keyDir, 'rocksdb-encryption-keyfile'); + fs.write(keyfile, key); + + args['rocksdb.encryption-keyfile'] = keyfile; + process.env["rocksdb-encryption-keyfile"] = keyfile; } params.args = args; diff --git a/js/common/bootstrap/modules/internal.js b/js/common/bootstrap/modules/internal.js index 9b9ac9bfc6f4..1b32146c8744 100644 --- a/js/common/bootstrap/modules/internal.js +++ b/js/common/bootstrap/modules/internal.js @@ -1731,6 +1731,15 @@ global.DEFINE_MODULE('internal', (function () { delete global.SYS_OPTIONS; } + // ////////////////////////////////////////////////////////////////////////////// + // / @brief options + // ////////////////////////////////////////////////////////////////////////////// + + if (typeof ENCRYPTION_KEY_RELOAD !== 'undefined') { + exports.encryptionKeyReload = global.ENCRYPTION_KEY_RELOAD; + delete global.ENCRYPTION_KEY_RELOAD; + } + let testsBasePaths = {}; exports.pathForTesting = function(path, prefix = 'js') { let fs = require('fs'); From 955fb0cf2c040a97ca84ad14b15f147ac2c04513 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Gr=C3=A4tzer?= Date: Wed, 12 Feb 2020 16:50:51 +0100 Subject: [PATCH 05/11] fix community builds --- arangod/RocksDBEngine/RocksDBEngine.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arangod/RocksDBEngine/RocksDBEngine.cpp b/arangod/RocksDBEngine/RocksDBEngine.cpp index c153b1d45350..4cbc82f1801c 100644 --- a/arangod/RocksDBEngine/RocksDBEngine.cpp +++ b/arangod/RocksDBEngine/RocksDBEngine.cpp @@ -406,13 +406,13 @@ void RocksDBEngine::start() { if (res == TRI_ERROR_NO_ERROR) { LOG_TOPIC("b2958", TRACE, arangodb::Logger::ENGINES) << "created RocksDB data directory '" << _path << "'"; + createdEngineDir = true; } else { LOG_TOPIC("a5ae3", FATAL, arangodb::Logger::ENGINES) << "unable to create RocksDB data directory '" << _path << "': " << systemErrorStr; FATAL_ERROR_EXIT(); } - createdEngineDir = true; } // options imported set by RocksDBOptionFeature @@ -490,6 +490,8 @@ void RocksDBEngine::start() { #ifdef USE_ENTERPRISE configureEnterpriseRocksDBOptions(_options, createdEngineDir); +#else + ((void)createdEngineDir); #endif _options.env->SetBackgroundThreads(static_cast(opts._numThreadsHigh), From 45346a4ad3fdb36e4a8c46cf0c5ab47fb0f2b5b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Gr=C3=A4tzer?= Date: Wed, 12 Feb 2020 20:53:21 +0100 Subject: [PATCH 06/11] fix some stuff --- lib/V8/v8-buffer.cpp | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/lib/V8/v8-buffer.cpp b/lib/V8/v8-buffer.cpp index 04838679b26a..f91385f7e341 100644 --- a/lib/V8/v8-buffer.cpp +++ b/lib/V8/v8-buffer.cpp @@ -712,8 +712,7 @@ void V8Buffer::replace(v8::Isolate* isolate, char* data, size_t length, memcpy(_data, data, _length); } - int64_t bytesToChange = (int64_t)(sizeof(V8Buffer) + _length + SAFETY_OVERHEAD); - isolate->AdjustAmountOfExternalAllocatedMemory(bytesToChange); + isolate->AdjustAmountOfExternalAllocatedMemory(sizeof(V8Buffer) + _length + SAFETY_OVERHEAD); } else { _data = NULL; } @@ -809,12 +808,9 @@ static void JS_Ucs2Slice(v8::FunctionCallbackInfo const& args) { if (!sliceArgs(isolate, args[0], args[1], parent, start, end)) { return; } - - std::vector buffer; - buffer.resize((end - start) / 2); - memcpy(buffer.data(), parent->_data + start, (end - start)); - TRI_V8_RETURN(TRI_V8_STRING_UTF16(isolate, buffer.data(), buffer.size())); + uint16_t* data = (uint16_t*)(parent->_data + start); + TRI_V8_RETURN(TRI_V8_STRING_UTF16(isolate, data, (end - start) / 2)); } //////////////////////////////////////////////////////////////////////////////// @@ -833,7 +829,7 @@ static void JS_HexSlice(v8::FunctionCallbackInfo const& args) { } char* src = parent->_data + start; - uint32_t dstlen = static_cast(end - start) * 2; + uint32_t dstlen = (end - start) * 2; if (dstlen == 0) { TRI_V8_RETURN(v8::String::Empty(isolate)); @@ -952,7 +948,7 @@ static void JS_Fill(v8::FunctionCallbackInfo const& args) { return; } - memset((void*)(parent->_data + start), value, static_cast(end - start)); + memset((void*)(parent->_data + start), value, end - start); TRI_V8_RETURN_UNDEFINED(); } @@ -1083,13 +1079,11 @@ static void JS_Ucs2Write(v8::FunctionCallbackInfo const& args) { : TRI_GET_UINT32(args[2]); max_length = MIN(buffer->_length - offset, max_length) / 2; - - std::vector ucs2; - ucs2.resize(max_length); - memcpy(ucs2.data(), buffer->_data + offset, max_length); + + uint16_t* p = (uint16_t*)(buffer->_data + offset); int written = - s->Write(isolate, ucs2.data(), 0, (int)max_length, + s->Write(isolate, p, 0, (int)max_length, (v8::String::HINT_MANY_WRITES_EXPECTED | v8::String::NO_NULL_TERMINATION)); TRI_V8_RETURN(v8::Integer::New(isolate, written * 2)); From df1d07ad8522f747708e67ddac2443e01d2d04ce Mon Sep 17 00:00:00 2001 From: Simran Brucherseifer Date: Tue, 18 Feb 2020 13:29:09 +0100 Subject: [PATCH 07/11] Add DocuBlock for /_admin/server/encryption --- .../post_admin_server_encryption.md | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 Documentation/DocuBlocks/Rest/Administration/post_admin_server_encryption.md diff --git a/Documentation/DocuBlocks/Rest/Administration/post_admin_server_encryption.md b/Documentation/DocuBlocks/Rest/Administration/post_admin_server_encryption.md new file mode 100644 index 000000000000..3b0ad9b8547b --- /dev/null +++ b/Documentation/DocuBlocks/Rest/Administration/post_admin_server_encryption.md @@ -0,0 +1,23 @@ + +@startDocuBlock post_admin_server_encryption +@brief Rotate encryption at rest key + +@RESTHEADER{POST /_admin/server/encryption, Rotate the encryption at rest key, handleEncryption:post} + +@RESTDESCRIPTION +Change the user supplied encryption at rest key by sending a request without +payload to this endpoint. The file supplied via `--rocksdb.encryption-keyfile` +will be reloaded and the internal encryption key will be re-encrypted with the +new user key. + +This is a protected API and can only be executed with superuser rights. + +@RESTRETURNCODES + +@RESTRETURNCODE{200} +This API will return HTTP 200 if everything is ok + +@RESTRETURNCODE{403} +This API will return HTTP 403 FORBIDDEN if it is not called with +superuser rights. +@endDocuBlock From 35185073598901c03897fa4ac9bda809475be2e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Gr=C3=A4tzer?= Date: Wed, 11 Mar 2020 18:16:05 +0100 Subject: [PATCH 08/11] adding rocksdb.encryption-keyfolder --- arangod/Aql/Expression.cpp | 2 +- arangod/RocksDBEngine/RocksDBEngine.h | 24 ++++++++++++------- arangod/RocksDBEngine/RocksDBKeyBounds.cpp | 10 ++++---- arangod/V8Server/v8-vocbase.cpp | 2 +- lib/Basics/FileUtils.cpp | 28 +++++++++++----------- lib/Basics/FileUtils.h | 6 ++--- lib/Basics/conversions.cpp | 18 +++++++------- 7 files changed, 48 insertions(+), 42 deletions(-) diff --git a/arangod/Aql/Expression.cpp b/arangod/Aql/Expression.cpp index e07b7f839653..de289ada4393 100644 --- a/arangod/Aql/Expression.cpp +++ b/arangod/Aql/Expression.cpp @@ -889,7 +889,7 @@ AqlValue Expression::executeSimpleExpressionFCallJS(AstNode const* node, _ast->query()->prepareV8Context(); std::string jsName; - size_t const n = static_cast(member->numMembers()); + size_t const n = member->numMembers(); size_t callArgs = (node->type == NODE_TYPE_FCALL_USER ? 2 : n); auto args = std::make_unique[]>(callArgs); diff --git a/arangod/RocksDBEngine/RocksDBEngine.h b/arangod/RocksDBEngine/RocksDBEngine.h index f0124051a6e9..580d26921d04 100644 --- a/arangod/RocksDBEngine/RocksDBEngine.h +++ b/arangod/RocksDBEngine/RocksDBEngine.h @@ -350,25 +350,31 @@ class RocksDBEngine final : public StorageEngine { void configureEnterpriseRocksDBOptions(rocksdb::Options& options, bool createdEngineDir); void validateJournalFiles() const; - Result readUserEncryptionKey(std::string& out) const; + Result readUserEncryptionKeys(std::map& outlist) const; enterprise::RocksDBEngineEEData _eeData; public: + + bool isEncryptionEnabled() const; + std::string const& getEncryptionKey(); - /// encryption file - std::string const& getEncryptionFilePath(); - /// encrypted key file path - std::string const& getEncryptedKeyFilePath(); + std::string getEncryptionTypeFile() const; + + std::string getKeyStoreFolder() const; + + std::vector userEncryptionKeys() const; - /// reload user-provided key, writes out the internal key file - Result rotateEncryptionKey(); + /// rotate user-provided keys, writes out the internal key files + Result rotateUserEncryptionKeys(); private: - /// reload encryption key, using new user provided key file - Result loadEncryptionKey(); + /// load encryption at rest key from keystore + Result decryptInternalKeystore(); + /// encrypt the internal keystore with all user keys + Result encryptInternalKeystore(); #endif private: diff --git a/arangod/RocksDBEngine/RocksDBKeyBounds.cpp b/arangod/RocksDBEngine/RocksDBKeyBounds.cpp index b33c1a8349d3..65ee1ec09de8 100644 --- a/arangod/RocksDBEngine/RocksDBKeyBounds.cpp +++ b/arangod/RocksDBEngine/RocksDBKeyBounds.cpp @@ -148,7 +148,7 @@ RocksDBKeyBounds RocksDBKeyBounds::FulltextIndexPrefix(uint64_t objectId, uint64ToPersistent(internals.buffer(), objectId); internals.buffer().append(word.data(), word.length()); - internals.push_back(0xFFU); + internals.push_back(static_cast(0xFFU)); // 0xFF is higher than any valud utf-8 character return b; } @@ -284,7 +284,7 @@ RocksDBKeyBounds::RocksDBKeyBounds(RocksDBEntryType type) : _type(type) { _internals.separate(); _internals.push_back(static_cast(_type)); - _internals.push_back(0xFFU); + _internals.push_back(static_cast(0xFFU)); break; } case RocksDBEntryType::CounterValue: @@ -325,7 +325,7 @@ RocksDBKeyBounds::RocksDBKeyBounds(RocksDBEntryType type, uint64_t first) _internals.reserve(2 * sizeof(uint64_t) + min.byteSize() + max.byteSize()); uint64ToPersistent(_internals.buffer(), first); - _internals.buffer().append((char*)(min.begin()), min.byteSize()); + _internals.buffer().append((char const*)(min.begin()), min.byteSize()); _internals.separate(); @@ -334,10 +334,10 @@ RocksDBKeyBounds::RocksDBKeyBounds(RocksDBEntryType type, uint64_t first) // for the upper bound we can use the object id + 1, which will always compare higher in a // bytewise comparison uint64ToPersistent(_internals.buffer(), first + 1); - _internals.buffer().append((char*)(min.begin()), min.byteSize()); + _internals.buffer().append((char const*)(min.begin()), min.byteSize()); } else { uint64ToPersistent(_internals.buffer(), first); - _internals.buffer().append((char*)(max.begin()), max.byteSize()); + _internals.buffer().append((char const*)(max.begin()), max.byteSize()); } break; } diff --git a/arangod/V8Server/v8-vocbase.cpp b/arangod/V8Server/v8-vocbase.cpp index c846d82ee536..7f8b6d7f6135 100644 --- a/arangod/V8Server/v8-vocbase.cpp +++ b/arangod/V8Server/v8-vocbase.cpp @@ -1966,7 +1966,7 @@ static void JS_EncryptionKeyReload(v8::FunctionCallbackInfo const& ar } auto* engine = EngineSelectorFeature::ENGINE; - auto res = static_cast(engine)->rotateEncryptionKey(); + auto res = static_cast(engine)->rotateUserEncryptionKeys(); if (res.fail()) { TRI_V8_THROW_EXCEPTION(res); } diff --git a/lib/Basics/FileUtils.cpp b/lib/Basics/FileUtils.cpp index 4b63c4e6fae3..b9a534b41499 100755 --- a/lib/Basics/FileUtils.cpp +++ b/lib/Basics/FileUtils.cpp @@ -330,7 +330,7 @@ bool createDirectory(std::string const& name, int mask, int* errorNumber) { *errorNumber = 0; } - int result = TRI_MKDIR(name.c_str(), mask); + auto result = TRI_MKDIR(name.c_str(), mask); int res = errno; if (result != 0 && res == EEXIST && isDirectory(name)) { @@ -365,19 +365,19 @@ bool copyRecursive(std::string const& source, std::string const& target, /// @brief will not copy files/directories for which the filter function /// returns true (now wrapper for version below with TRI_copy_recursive_e filter) -bool copyDirectoryRecursive(std::string const& source, std::string const& target, - std::function const& filter, - std::string& error) { - - // "auto lambda" will not work here - std::function lambda = - [&filter] (std::string const& pathname) -> TRI_copy_recursive_e { - return filter(pathname) ? TRI_COPY_IGNORE : TRI_COPY_COPY; - }; - - return copyDirectoryRecursive(source, target, lambda, error); - -} // copyDirectoryRecursive (bool filter()) +//bool copyDirectoryRecursive(std::string const& source, std::string const& target, +// std::function const& filter, +// std::string& error) { +// +// // "auto lambda" will not work here +// std::function lambda = +// [&filter] (std::string const& pathname) -> TRI_copy_recursive_e { +// return filter(pathname) ? TRI_COPY_IGNORE : TRI_COPY_COPY; +// }; +// +// return copyDirectoryRecursive(source, target, lambda, error); +// +//} // copyDirectoryRecursive (bool filter()) /// @brief will not copy files/directories for which the filter function diff --git a/lib/Basics/FileUtils.h b/lib/Basics/FileUtils.h index 599c51676230..4f40edb1af27 100644 --- a/lib/Basics/FileUtils.h +++ b/lib/Basics/FileUtils.h @@ -87,9 +87,9 @@ bool copyRecursive(std::string const& source, std::string const& target, /// @brief will not copy files/directories for which the filter function /// returns true (now wrapper for version below with TRI_copy_recursive_e filter) -bool copyDirectoryRecursive(std::string const& source, std::string const& target, - std::function const& filter, - std::string& error); +//bool copyDirectoryRecursive(std::string const& source, std::string const& target, +// std::function const& filter, +// std::string& error); enum TRI_copy_recursive_e { TRI_COPY_IGNORE, diff --git a/lib/Basics/conversions.cpp b/lib/Basics/conversions.cpp index 78fc875cadf0..80eb9527c683 100644 --- a/lib/Basics/conversions.cpp +++ b/lib/Basics/conversions.cpp @@ -206,7 +206,7 @@ size_t TRI_StringInt8InPlace(int8_t attr, char* buffer) { *p++ = (char)(attr % 10 + '0'); *p = '\0'; - return (p - buffer); + return static_cast(p - buffer); } //////////////////////////////////////////////////////////////////////////////// @@ -231,7 +231,7 @@ size_t TRI_StringUInt8InPlace(uint8_t attr, char* buffer) { *p++ = (char)(attr % 10 + '0'); *p = '\0'; - return (p - buffer); + return static_cast(p - buffer); } //////////////////////////////////////////////////////////////////////////////// @@ -272,7 +272,7 @@ size_t TRI_StringInt16InPlace(int16_t attr, char* buffer) { *p++ = (char)(attr % 10 + '0'); *p = '\0'; - return (p - buffer); + return static_cast(p - buffer); } //////////////////////////////////////////////////////////////////////////////// @@ -303,7 +303,7 @@ size_t TRI_StringUInt16InPlace(uint16_t attr, char* buffer) { *p++ = (char)(attr % 10 + '0'); *p = '\0'; - return (p - buffer); + return static_cast(p - buffer); } //////////////////////////////////////////////////////////////////////////////// @@ -359,7 +359,7 @@ size_t TRI_StringInt32InPlace(int32_t attr, char* buffer) { *p++ = (char)(attr % 10 + '0'); *p = '\0'; - return (p - buffer); + return static_cast(p - buffer); } //////////////////////////////////////////////////////////////////////////////// @@ -405,7 +405,7 @@ size_t TRI_StringUInt32InPlace(uint32_t attr, char* buffer) { *p++ = (char)(attr % 10 + '0'); *p = '\0'; - return (p - buffer); + return static_cast(p - buffer); } //////////////////////////////////////////////////////////////////////////////// @@ -498,7 +498,7 @@ size_t TRI_StringInt64InPlace(int64_t attr, char* buffer) { *p++ = (char)(attr % 10 + '0'); *p = '\0'; - return (p - buffer); + return static_cast(p - buffer); } //////////////////////////////////////////////////////////////////////////////// @@ -579,7 +579,7 @@ size_t TRI_StringUInt64InPlace(uint64_t attr, char* buffer) { *p++ = (char)(attr % 10 + '0'); *p = '\0'; - return (p - buffer); + return static_cast(p - buffer); } //////////////////////////////////////////////////////////////////////////////// @@ -619,7 +619,7 @@ size_t TRI_StringUInt32HexInPlace(uint32_t attr, char* buffer) { *p++ = HEX[attr % 0x10]; *p = '\0'; - return (p - buffer); + return static_cast(p - buffer); } //////////////////////////////////////////////////////////////////////////////// From 5ed670e22de69c2193acebf141fda689d8c69c72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Gr=C3=A4tzer?= Date: Thu, 12 Mar 2020 21:26:51 +0100 Subject: [PATCH 09/11] fix warning on gcc --- arangod/RocksDBEngine/RocksDBKeyBounds.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arangod/RocksDBEngine/RocksDBKeyBounds.cpp b/arangod/RocksDBEngine/RocksDBKeyBounds.cpp index 65ee1ec09de8..f98537e0221d 100644 --- a/arangod/RocksDBEngine/RocksDBKeyBounds.cpp +++ b/arangod/RocksDBEngine/RocksDBKeyBounds.cpp @@ -148,7 +148,7 @@ RocksDBKeyBounds RocksDBKeyBounds::FulltextIndexPrefix(uint64_t objectId, uint64ToPersistent(internals.buffer(), objectId); internals.buffer().append(word.data(), word.length()); - internals.push_back(static_cast(0xFFU)); + internals.push_back(static_cast(0xFFU)); // 0xFF is higher than any valud utf-8 character return b; } @@ -284,7 +284,7 @@ RocksDBKeyBounds::RocksDBKeyBounds(RocksDBEntryType type) : _type(type) { _internals.separate(); _internals.push_back(static_cast(_type)); - _internals.push_back(static_cast(0xFFU)); + _internals.push_back(static_cast(0xFFU)); break; } case RocksDBEntryType::CounterValue: From 4d07d2ac9e1a25f2fdb4934d879e3ffcc7e2a3df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Gr=C3=A4tzer?= Date: Fri, 13 Mar 2020 11:54:41 +0100 Subject: [PATCH 10/11] fix win32 --- js/client/modules/@arangodb/testsuites/recovery.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/js/client/modules/@arangodb/testsuites/recovery.js b/js/client/modules/@arangodb/testsuites/recovery.js index ba1d2fc848f0..80fb0d592e7b 100644 --- a/js/client/modules/@arangodb/testsuites/recovery.js +++ b/js/client/modules/@arangodb/testsuites/recovery.js @@ -94,7 +94,9 @@ function runArangodRecovery (params) { if (useEncryption) { const key = '01234567890123456789012345678901'; let keyDir = fs.join(fs.getTempPath(), 'arango_encryption'); - fs.makeDirectory(keyDir); + if (!fs.exists(keyDir)) { // needed on win32 + fs.makeDirectory(keyDir); + } pu.cleanupDBDirectoriesAppend(keyDir); let keyfile = fs.join(keyDir, 'rocksdb-encryption-keyfile'); From e128db1ae5d5e7cc40dcd94b62440a5bc7ab8c68 Mon Sep 17 00:00:00 2001 From: jsteemann Date: Fri, 13 Mar 2020 12:42:04 +0100 Subject: [PATCH 11/11] remove commented out code --- lib/Basics/FileUtils.cpp | 17 ----------------- lib/Basics/FileUtils.h | 7 ------- lib/Basics/datetime.cpp | 3 ++- 3 files changed, 2 insertions(+), 25 deletions(-) diff --git a/lib/Basics/FileUtils.cpp b/lib/Basics/FileUtils.cpp index b9a534b41499..2b10d878330c 100755 --- a/lib/Basics/FileUtils.cpp +++ b/lib/Basics/FileUtils.cpp @@ -363,23 +363,6 @@ bool copyRecursive(std::string const& source, std::string const& target, } // copyRecursive (bool filter()) -/// @brief will not copy files/directories for which the filter function -/// returns true (now wrapper for version below with TRI_copy_recursive_e filter) -//bool copyDirectoryRecursive(std::string const& source, std::string const& target, -// std::function const& filter, -// std::string& error) { -// -// // "auto lambda" will not work here -// std::function lambda = -// [&filter] (std::string const& pathname) -> TRI_copy_recursive_e { -// return filter(pathname) ? TRI_COPY_IGNORE : TRI_COPY_COPY; -// }; -// -// return copyDirectoryRecursive(source, target, lambda, error); -// -//} // copyDirectoryRecursive (bool filter()) - - /// @brief will not copy files/directories for which the filter function /// returns true bool copyRecursive(std::string const& source, std::string const& target, diff --git a/lib/Basics/FileUtils.h b/lib/Basics/FileUtils.h index 4f40edb1af27..f936de207327 100644 --- a/lib/Basics/FileUtils.h +++ b/lib/Basics/FileUtils.h @@ -85,19 +85,12 @@ bool copyRecursive(std::string const& source, std::string const& target, std::function const& filter, std::string& error); -/// @brief will not copy files/directories for which the filter function -/// returns true (now wrapper for version below with TRI_copy_recursive_e filter) -//bool copyDirectoryRecursive(std::string const& source, std::string const& target, -// std::function const& filter, -// std::string& error); - enum TRI_copy_recursive_e { TRI_COPY_IGNORE, TRI_COPY_COPY, TRI_COPY_LINK }; - /// @brief copies directories / files recursive /// will not copy files/directories for which the filter function /// returns true diff --git a/lib/Basics/datetime.cpp b/lib/Basics/datetime.cpp index 384fc0557e70..41e004a01e83 100644 --- a/lib/Basics/datetime.cpp +++ b/lib/Basics/datetime.cpp @@ -806,7 +806,8 @@ bool arangodb::basics::parseDateTime(arangodb::velocypack::StringRef dateTime, } } - // LOG_DEVEL << "year: " << result.year + // LOG_TOPIC("51643", TRACE, Logger::FIXME) + // << "year: " << result.year // << ", month: " << result.month // << ", day: " << result.day // << ", hour: " << result.hour