arangodb · jsteemann · Mar 13, 2020 · Feb 4, 2020 · Feb 11, 2020 · Feb 11, 2020
@@ -0,0 +1,23 @@
+
+@startDocuBlock post_admin_server_encryption
+@brief Rotate encryption at rest key
+
+@RESTHEADER{POST /_admin/server/encryption, Rotate the encryption at rest key, handleEncryption:post}
+
+@RESTDESCRIPTION
+Change the user supplied encryption at rest key by sending a request without
+payload to this endpoint. The file supplied via `--rocksdb.encryption-keyfile`
+will be reloaded and the internal encryption key will be re-encrypted with the
+new user key.
+
+This is a protected API and can only be executed with superuser rights.
+
+@RESTRETURNCODES
+
+@RESTRETURNCODE{200}
+This API will return HTTP 200 if everything is ok
+
+@RESTRETURNCODE{403}
+This API will return HTTP 403 FORBIDDEN if it is not called with
+superuser rights.
+@endDocuBlock
diff --git a/arangod/Aql/Expression.cpp b/arangod/Aql/Expression.cpp
@@ -889,7 +889,7 @@ AqlValue Expression::executeSimpleExpressionFCallJS(AstNode const* node,
     _ast->query()->prepareV8Context();
 
     std::string jsName;
-    size_t const n = static_cast<int>(member->numMembers());
+    size_t const n = member->numMembers();
     size_t callArgs = (node->type == NODE_TYPE_FCALL_USER ? 2 : n);
     auto args = std::make_unique<v8::Handle<v8::Value>[]>(callArgs);
 

diff --git a/arangod/GeneralServer/H2CommTask.cpp b/arangod/GeneralServer/H2CommTask.cpp
@@ -153,7 +153,7 @@ template <SocketType T>
   me->_streams.erase(stream_id);
 
   if (error_code != NGHTTP2_NO_ERROR) {
-    LOG_TOPIC("d04f7", DEBUG, Logger::REQUESTS) << "<http2> closing stream "
+    LOG_TOPIC("2824d", DEBUG, Logger::REQUESTS) << "<http2> closing stream "
        << stream_id << " with error '" << nghttp2_http2_strerror(error_code) << "' (" << error_code << ")";
   }
 

diff --git a/arangod/RestHandler/RestAdminServerHandler.cpp b/arangod/RestHandler/RestAdminServerHandler.cpp
@@ -58,6 +58,8 @@ RestStatus RestAdminServerHandler::execute() {
     handleTLS();
   } else if (suffixes.size() == 1 && suffixes[0] == "jwt") {
     handleJWTSecretsReload();
+  } else if (suffixes.size() == 1 && suffixes[0] == "encryption") {
+    handleEncryptionKeyRotation();
   } else {
     generateError(rest::ResponseCode::NOT_FOUND, TRI_ERROR_HTTP_NOT_FOUND);
   }
@@ -257,4 +259,8 @@ void RestAdminServerHandler::handleTLS() {
 void RestAdminServerHandler::handleJWTSecretsReload() {
   generateError(rest::ResponseCode::NOT_FOUND, TRI_ERROR_HTTP_NOT_FOUND);
 }
+
+void RestAdminServerHandler::handleEncryptionKeyRotation() {
+  generateError(rest::ResponseCode::NOT_FOUND, TRI_ERROR_HTTP_NOT_FOUND);
+}
 #endif
diff --git a/arangod/RestHandler/RestAdminServerHandler.h b/arangod/RestHandler/RestAdminServerHandler.h
@@ -54,6 +54,7 @@ class RestAdminServerHandler : public RestBaseHandler {
   void writeModeResult(bool);
 
   void handleJWTSecretsReload();
+  void handleEncryptionKeyRotation();
 };
 }  // namespace arangodb
 

diff --git a/arangod/RocksDBEngine/RocksDBEngine.cpp b/arangod/RocksDBEngine/RocksDBEngine.cpp
@@ -396,6 +396,7 @@ void RocksDBEngine::start() {
   auto& databasePathFeature = server().getFeature<DatabasePathFeature>();
   _path = databasePathFeature.subdirectoryName("engine-rocksdb");
 
+  bool createdEngineDir = false;
   if (!basics::FileUtils::isDirectory(_path)) {
     std::string systemErrorStr;
     long errorNo;
@@ -405,6 +406,7 @@ void RocksDBEngine::start() {
     if (res == TRI_ERROR_NO_ERROR) {
       LOG_TOPIC("b2958", TRACE, arangodb::Logger::ENGINES)
           << "created RocksDB data directory '" << _path << "'";
+      createdEngineDir = true;
     } else {
       LOG_TOPIC("a5ae3", FATAL, arangodb::Logger::ENGINES)
           << "unable to create RocksDB data directory '" << _path
@@ -487,8 +489,9 @@ void RocksDBEngine::start() {
   _options.compaction_readahead_size = static_cast<size_t>(opts._compactionReadaheadSize);
 
 #ifdef USE_ENTERPRISE
-  configureEnterpriseRocksDBOptions(_options);
-  startEnterprise();
+  configureEnterpriseRocksDBOptions(_options, createdEngineDir);
+#else
+  ((void)createdEngineDir);
 #endif
 
   _options.env->SetBackgroundThreads(static_cast<int>(opts._numThreadsHigh),

diff --git a/arangod/RocksDBEngine/RocksDBEngine.h b/arangod/RocksDBEngine/RocksDBEngine.h
@@ -347,14 +347,35 @@ class RocksDBEngine final : public StorageEngine {
   void collectEnterpriseOptions(std::shared_ptr<options::ProgramOptions>);
   void validateEnterpriseOptions(std::shared_ptr<options::ProgramOptions>);
   void prepareEnterprise();
-  void startEnterprise();
-  void configureEnterpriseRocksDBOptions(rocksdb::Options& options);
+  void configureEnterpriseRocksDBOptions(rocksdb::Options& options, bool createdEngineDir);
   void validateJournalFiles() const;
+
+  Result readUserEncryptionKeys(std::map<std::string, std::string>& outlist) const;
 
   enterprise::RocksDBEngineEEData _eeData;
 
 public:
+
+  bool isEncryptionEnabled() const;
+
   std::string const& getEncryptionKey();
+
+  std::string getEncryptionTypeFile() const;
+
+  std::string getKeyStoreFolder() const;
+
+  std::vector<std::string> userEncryptionKeys() const;
+
+  /// rotate user-provided keys, writes out the internal key files
+  Result rotateUserEncryptionKeys();
+
+private:
+
+  /// load encryption at rest key from keystore
+  Result decryptInternalKeystore();
+  /// encrypt the internal keystore with all user keys
+  Result encryptInternalKeystore();
+
 #endif
 private:
   // activate generation of SHA256 files to parallel .sst files

diff --git a/arangod/RocksDBEngine/RocksDBKeyBounds.cpp b/arangod/RocksDBEngine/RocksDBKeyBounds.cpp
@@ -148,7 +148,7 @@ RocksDBKeyBounds RocksDBKeyBounds::FulltextIndexPrefix(uint64_t objectId,
 
   uint64ToPersistent(internals.buffer(), objectId);
   internals.buffer().append(word.data(), word.length());
-  internals.push_back(0xFFU);
+  internals.push_back(static_cast<char>(0xFFU));
   // 0xFF is higher than any valud utf-8 character
   return b;
 }
@@ -284,7 +284,7 @@ RocksDBKeyBounds::RocksDBKeyBounds(RocksDBEntryType type) : _type(type) {
 
       _internals.separate();
       _internals.push_back(static_cast<char>(_type));
-      _internals.push_back(0xFFU);
+      _internals.push_back(static_cast<char>(0xFFU));
       break;
     }
     case RocksDBEntryType::CounterValue:
@@ -325,7 +325,7 @@ RocksDBKeyBounds::RocksDBKeyBounds(RocksDBEntryType type, uint64_t first)
       _internals.reserve(2 * sizeof(uint64_t) + min.byteSize() + max.byteSize());
 
       uint64ToPersistent(_internals.buffer(), first);
-      _internals.buffer().append((char*)(min.begin()), min.byteSize());
+      _internals.buffer().append((char const*)(min.begin()), min.byteSize());
 
       _internals.separate();
 
@@ -334,10 +334,10 @@ RocksDBKeyBounds::RocksDBKeyBounds(RocksDBEntryType type, uint64_t first)
         // for the upper bound we can use the object id + 1, which will always compare higher in a
         // bytewise comparison
         uint64ToPersistent(_internals.buffer(), first + 1);
-        _internals.buffer().append((char*)(min.begin()), min.byteSize());
+        _internals.buffer().append((char const*)(min.begin()), min.byteSize());
       } else {
         uint64ToPersistent(_internals.buffer(), first);
-        _internals.buffer().append((char*)(max.begin()), max.byteSize());
+        _internals.buffer().append((char const*)(max.begin()), max.byteSize());
       }
       break;
     }

diff --git a/arangod/V8Server/v8-vocbase.cpp b/arangod/V8Server/v8-vocbase.cpp
@@ -64,6 +64,7 @@
 #include "RestServer/ConsoleThread.h"
 #include "RestServer/DatabaseFeature.h"
 #include "RestServer/QueryRegistryFeature.h"
+#include "RocksDBEngine/RocksDBEngine.h"
 #include "Statistics/StatisticsFeature.h"
 #include "StorageEngine/EngineSelectorFeature.h"
 #include "StorageEngine/StorageEngine.h"
@@ -1946,6 +1947,36 @@ static void JS_AgencyDump(v8::FunctionCallbackInfo<v8::Value> const& args) {
   TRI_V8_TRY_CATCH_END
 }
 
+#ifdef USE_ENTERPRISE
+
+////////////////////////////////////////////////////////////////////////////////
+/// @brief this is rotates the encryption keys, only for testing
+////////////////////////////////////////////////////////////////////////////////
+
+static void JS_EncryptionKeyReload(v8::FunctionCallbackInfo<v8::Value> const& args) {
+  TRI_V8_TRY_CATCH_BEGIN(isolate);
+  v8::HandleScope scope(isolate);
+
+  if (args.Length() != 0) {
+    TRI_V8_THROW_EXCEPTION_USAGE("encryptionKeyReload()");
+  }
+
+  if (!EngineSelectorFeature::isRocksDB()) {
+    THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED);
+  }
+
+  auto* engine = EngineSelectorFeature::ENGINE;
+  auto res = static_cast<RocksDBEngine*>(engine)->rotateUserEncryptionKeys();
+  if (res.fail()) {
+    TRI_V8_THROW_EXCEPTION(res);
+  }
+
+  TRI_V8_RETURN_TRUE();
+  TRI_V8_TRY_CATCH_END
+}
+
+#endif
+
 ////////////////////////////////////////////////////////////////////////////////
 /// @brief creates a TRI_vocbase_t global context
 ////////////////////////////////////////////////////////////////////////////////
@@ -2133,6 +2164,14 @@ void TRI_InitV8VocBridge(v8::Isolate* isolate, v8::Handle<v8::Context> context,
   TRI_AddGlobalFunctionVocbase(isolate,
                                TRI_V8_ASCII_STRING(isolate, "AGENCY_DUMP"),
                                JS_AgencyDump, true);
+
+#ifdef USE_ENTERPRISE
+  if (V8DealerFeature::DEALER && V8DealerFeature::DEALER->allowAdminExecute()) {
+    TRI_AddGlobalFunctionVocbase(isolate,
+                                 TRI_V8_ASCII_STRING(isolate, "ENCRYPTION_KEY_RELOAD"),
+                                 JS_EncryptionKeyReload, true);
+  }
+#endif
 
   // .............................................................................
   // create global variables

diff --git a/js/client/modules/@arangodb/testsuites/recovery.js b/js/client/modules/@arangodb/testsuites/recovery.js
@@ -92,7 +92,18 @@ function runArangodRecovery (params) {
     });
 
     if (useEncryption) {
-      args['rocksdb.encryption-keyfile'] = tu.pathForTesting('server/recovery/encryption-keyfile');
+      const key = '01234567890123456789012345678901';
+      let keyDir = fs.join(fs.getTempPath(), 'arango_encryption');
+      if (!fs.exists(keyDir)) {  // needed on win32
+        fs.makeDirectory(keyDir);
+      }
+      pu.cleanupDBDirectoriesAppend(keyDir);
+
+      let keyfile = fs.join(keyDir, 'rocksdb-encryption-keyfile');
+      fs.write(keyfile, key);
+
+      args['rocksdb.encryption-keyfile'] = keyfile;
+      process.env["rocksdb-encryption-keyfile"] = keyfile;
     }
 
     params.args = args;

diff --git a/js/common/bootstrap/modules/internal.js b/js/common/bootstrap/modules/internal.js
@@ -1731,6 +1731,15 @@ global.DEFINE_MODULE('internal', (function () {
     delete global.SYS_OPTIONS;
   }
 
+  // //////////////////////////////////////////////////////////////////////////////
+  // / @brief options
+  // //////////////////////////////////////////////////////////////////////////////
+
+  if (typeof ENCRYPTION_KEY_RELOAD !== 'undefined') {
+    exports.encryptionKeyReload = global.ENCRYPTION_KEY_RELOAD;
+    delete global.ENCRYPTION_KEY_RELOAD;
+  }
+
   let testsBasePaths = {};
   exports.pathForTesting = function(path, prefix = 'js') {
     let fs = require('fs');

diff --git a/lib/Basics/FileUtils.cpp b/lib/Basics/FileUtils.cpp
@@ -330,7 +330,7 @@ bool createDirectory(std::string const& name, int mask, int* errorNumber) {
     *errorNumber = 0;
   }
 
-  int result = TRI_MKDIR(name.c_str(), mask);
+  auto result = TRI_MKDIR(name.c_str(), mask);
 
   int res = errno;
   if (result != 0 && res == EEXIST && isDirectory(name)) {
@@ -363,23 +363,6 @@ bool copyRecursive(std::string const& source, std::string const& target,
 } // copyRecursive (bool filter())
 
 
-/// @brief will not copy files/directories for which the filter function
-/// returns true (now wrapper for version below with TRI_copy_recursive_e filter)
-bool copyDirectoryRecursive(std::string const& source, std::string const& target,
-                   std::function<bool(std::string const&)> const& filter,
-                   std::string& error) {
-
-  // "auto lambda" will not work here
-  std::function<TRI_copy_recursive_e(std::string const&)> lambda =
-    [&filter] (std::string const& pathname) -> TRI_copy_recursive_e {
-    return filter(pathname) ? TRI_COPY_IGNORE : TRI_COPY_COPY;
-  };
-
-  return copyDirectoryRecursive(source, target, lambda, error);
-
-} // copyDirectoryRecursive (bool filter())
-
-
 /// @brief will not copy files/directories for which the filter function
 /// returns true
 bool copyRecursive(std::string const& source, std::string const& target,

diff --git a/lib/Basics/FileUtils.h b/lib/Basics/FileUtils.h
@@ -85,19 +85,12 @@ bool copyRecursive(std::string const& source, std::string const& target,
                    std::function<bool(std::string const&)> const& filter,
                    std::string& error);
 
-/// @brief will not copy files/directories for which the filter function
-/// returns true (now wrapper for version below with TRI_copy_recursive_e filter)
-bool copyDirectoryRecursive(std::string const& source, std::string const& target,
-                            std::function<bool(std::string const&)> const& filter,
-                            std::string& error);
-
 enum TRI_copy_recursive_e {
   TRI_COPY_IGNORE,
   TRI_COPY_COPY,
   TRI_COPY_LINK
 };
 
-
 /// @brief copies directories / files recursive
 /// will not copy files/directories for which the filter function
 /// returns true

diff --git a/lib/Basics/StringUtils.h b/lib/Basics/StringUtils.h
@@ -159,7 +159,7 @@ static inline char tolower(char c) {
 }
 
 static inline unsigned char tolower(unsigned char c) {
-  return c + ((c - 65U < 26U) << 5);
+  return static_cast<unsigned char>(c + ((c - 65U < 26U) << 5));
 }
 
 static inline char toupper(char c) {

diff --git a/lib/Basics/VelocyPackHelper.cpp b/lib/Basics/VelocyPackHelper.cpp
@@ -60,8 +60,6 @@
 #include "Basics/memory.h"
 #include "Basics/system-compiler.h"
 #include "Logger/LogMacros.h"
-#include "Logger/Logger.h"
-#include "Logger/LoggerStream.h"
 
 extern "C" {
 unsigned long long XXH64(const void* input, size_t length, unsigned long long seed);