Merge remote-tracking branch 'origin/devel' into feature/new-server-i…

…nfra
arangodb · neunhoef · Jan 8, 2019 · Jul 23, 2018 · Jul 24, 2018 · Jul 25, 2018
commit 20132a0fc6f8c7b2cd9cb6d680aea36da2081625
diff --git a/.clang-format b/.clang-format
@@ -2,3 +2,4 @@ BasedOnStyle: Google
 DerivePointerAlignment: false
 PointerAlignment: Left
 Standard: Cpp11
+SortUsingDeclarations: true
diff --git a/.gitignore b/.gitignore
@@ -29,12 +29,15 @@ build
 Thumbs.db
 
 enterprise
+upgrade-data-tests
 compile_commands.json
 instanceinfo.json
 testresult.json
 testsStarted
 soc-pokec-*
 
+Documentation/man
+
 build.sh
 /build*/
 Build64/

diff --git a/3rdParty/.gitignore b/3rdParty/.gitignore
@@ -1,11 +1,3 @@
 *.ao
 *.d
 *.pc
-
-libev/.libs/
-libev/config.h
-libev/ev.lo
-libev/event.lo
-libev/libev.la
-libev/libtool
-libev/stamp-h1
diff --git a/3rdParty/CMakeLists.txt b/3rdParty/CMakeLists.txt
@@ -164,7 +164,13 @@ if (USE_IRESEARCH)
   set(LZ4_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/lz4")
   set(SNOWBALL_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/snowball")
   set(UNWIND_ROOT "invalid")
+  set(BFD_ROOT "invalid")
 
+  set(IRESEARCH_PREGENERATED_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/iresearch.build/")
+  set(GENERATED_PARSER_SOURCES "${IRESEARCH_PREGENERATED_ROOT}/core/iql/parser.cc" CACHE PATH "use pregenerated source")
+  include_directories("${IRESEARCH_PREGENERATED_ROOT}/core/")
+  include_directories("${IRESEARCH_PREGENERATED_ROOT}/external/snowball/libstemmer/")
+
   if(MSVC)
     set(USE_IQL OFF CACHE BOOL "Use IQL" FORCE) # skip IQL on MSVC until IResearch upstream is updated
   endif()
@@ -179,24 +185,6 @@ if (USE_IRESEARCH)
     "${IRESEARCH_ROOT}/cmake" # to find iResearch dependencies
   )
 
-  # cmake will seemingly randomly delete files from ${IRESEARCH_BUILD_DIR} during configuration
-  # so must copy required files only after the cmake configuration phase is complete, not via file(COPY ...)
-  # this command must _always_ be executed because the subsequent cmake configuration runs will once again delete files
-  if(MSVC)
-    add_custom_command(OUTPUT "${IRESEARCH_BUILD_DIR}/copy_iresearch.build" # non-existent file
-      COMMAND ${CMAKE_COMMAND} -E copy_directory "${IRESEARCH_ROOT}.build/" "${IRESEARCH_BUILD_DIR}/"
-    )
-  else()
-    # 'cmake -E copy_directory' does not preserve timestamps, use 'cp' directly where avalable
-    add_custom_command(OUTPUT "${IRESEARCH_BUILD_DIR}/copy_iresearch.build" # non-existent file
-      COMMAND cp -rp "${IRESEARCH_ROOT}.build/*" "${IRESEARCH_BUILD_DIR}/" # MacOS requires '/*' suffix
-    )
-  endif()
-
-  add_custom_target("copy_iresearch.build" ALL
-    DEPENDS "${IRESEARCH_BUILD_DIR}/copy_iresearch.build"
-  )
-
   add_subdirectory("${IRESEARCH_ROOT}" "${IRESEARCH_BUILD_DIR}" EXCLUDE_FROM_ALL) # do not build unused targets
   set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH_ORIGINAL}) # restore CMAKE_MODULE_PATH
   unset(BUILD_SHARED_LIBS) # otherwise ZLib (below) does not build on win32
@@ -207,13 +195,6 @@ if (USE_IRESEARCH)
     # cmake does not build ICU without this (cmake does not inherit dependencies)
     add_dependencies(v8_build icu-build icudata-static icui18n-static icuuc-static)
   endif()
-
-  # ensure the 'copy_iresearch.build' target is executed before start of build of the 'iresearch' target
-  if(MSVC)
-    add_dependencies(iresearch-static-scrt "copy_iresearch.build") # use static + staticCRT to match ArangoDB
-  else()
-    add_dependencies(iresearch-static "copy_iresearch.build") # use static to match ArangoDB
-  endif()
 endif()
 
 
@@ -245,9 +226,11 @@ set(CURL_CONFIG_HAS_BEEN_RUN_BEFORE true PARENT_SCOPE)
 ## SNAPPY
 ################################################################################
 
-set(SNAPPY_VERSION "d53de18")
-set(SNAPPY_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/snappy/google-snappy-${SNAPPY_VERSION}")
-set(SNAPPY_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/snappy/google-snappy-${SNAPPY_VERSION}")
+set(SNAPPY_VERSION "1.1.7")
+set(SNAPPY_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/snappy/snappy-${SNAPPY_VERSION}")
+set(SNAPPY_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/snappy/snappy-${SNAPPY_VERSION}")
+set(SNAPPY_LIB "snappy")
+set(SNAPPY_LIB "${SNAPPY_LIB}" PARENT_SCOPE)
 add_subdirectory(${SNAPPY_SOURCE_DIR} EXCLUDE_FROM_ALL)
 
 ################################################################################
@@ -258,7 +241,7 @@ if (USE_JEMALLOC)
   add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/jemalloc)
   include_directories(SYSTEM "${JEMALLOC_HOME}/include")
   link_directories("${JEMALLOC_HOME}/lib")
-  set(SYS_LIBS ${SYS_LIBS} ${JEMALLOC_LIB})
+  set(SYS_LIBS ${SYS_LIBS} ${JEMALLOC_LIB} PARENT_SCOPE)
 endif ()
 
 ################################################################################

diff --git a/3rdParty/V8/v5.7.492.77/gypfiles/standalone.gypi b/3rdParty/V8/v5.7.492.77/gypfiles/standalone.gypi
@@ -783,7 +783,7 @@
           '-Wnon-virtual-dtor',
           '-fno-exceptions',
           '-fno-rtti',
-          '-std=gnu++11',
+          '-std=gnu++14',
         ],
         'ldflags': [ '-pth
F438
read', ],
         'conditions': [
@@ -832,7 +832,7 @@
           '-Wnon-virtual-dtor',
           '-fno-exceptions',
           '-fno-rtti',
-          '-std=gnu++11',
+          '-std=gnu++14',
         ],
         'conditions': [
           [ 'visibility=="hidden"', {
@@ -1096,7 +1096,7 @@
           ['clang==1', {
             'xcode_settings': {
               'GCC_VERSION': 'com.apple.compilers.llvm.clang.1_0',
-              'CLANG_CXX_LANGUAGE_STANDARD': 'c++11',  # -std=c++11
+              'CLANG_CXX_LANGUAGE_STANDARD': 'c++14',  # -std=c++14
             },
             'conditions': [
               ['clang_xcode==0', {
@@ -1134,7 +1134,7 @@
         },  # configurations
         'cflags': [ '-Wno-abi', '-Wall', '-W', '-Wno-unused-parameter'],
         'cflags_cc': [ '-Wnon-virtual-dtor', '-fno-rtti', '-fno-exceptions',
-                       '-std=gnu++11' ],
+                       '-std=gnu++14' ],
         'target_conditions': [
           ['_toolset=="target"', {
             'cflags!': [
@@ -1330,7 +1330,7 @@
     ['clang==0 and host_clang==1 and target_arch!="ia32" and target_arch!="x64"', {
       'target_conditions': [
         ['_toolset=="host"', {
-          'cflags_cc': [ '-std=gnu++11', ],
+          'cflags_cc': [ '-std=gnu++14', ],
         }],
       ],
       'target_defaults': {

diff --git a/3rdParty/fuerte/include/fuerte/FuerteLogger.h b/3rdParty/fuerte/include/fuerte/FuerteLogger.h
@@ -78,21 +78,21 @@
 #endif
 
 #if ENABLE_FUERTE_LOG_VSTTRACE > 0
-#define FUERTE_LOG_VSTTRACE std::cout
+#define FUERTE_LOG_VSTTRACE std::cout << "[vst] "
 #else
 #define FUERTE_LOG_VSTTRACE \
   if (0) std::cout
 #endif
 
 #if ENABLE_FUERTE_LOG_VSTCHUNKTRACE > 0
-#define FUERTE_LOG_VSTCHUNKTRACE std::cout
+#define FUERTE_LOG_VSTCHUNKTRACE std::cout << "[vst] "
 #else
 #define FUERTE_LOG_VSTCHUNKTRACE \
   if (0) std::cout
 #endif
 
 #if ENABLE_FUERTE_LOG_HTTPTRACE > 0
-#define FUERTE_LOG_HTTPTRACE std::cout
+#define FUERTE_LOG_HTTPTRACE std::cout << "[http] "
 #else
 #define FUERTE_LOG_HTTPTRACE \
   if (0) std::cout

diff --git a/3rdParty/fuerte/include/fuerte/connection.h b/3rdParty/fuerte/include/fuerte/connection.h
@@ -51,48 +51,47 @@ class Connection : public std::enable_shared_from_this<Connection> {
 
   /// @brief Send a request to the server and wait into a response it received.
   std::unique_ptr<Response> sendRequest(std::unique_ptr<Request> r);
-
-  /// @brief sed request synchronously, only save to use if the
-  virtual std::unique_ptr<Response> sendRequestSync(std::unique_ptr<Request> r) {
-    return sendRequest(std::move(r));
-  }
 
-  // Send a request to the server and wait into a response it received.
-  /*std::unique_ptr<Response> sendRequest(Request const& r) {
+  /// @brief Send a request to the server and wait into a response it received.
+  /// @param r request that is copied
+  std::unique_ptr<Response> sendRequest(Request const& r) {
     std::unique_ptr<Request> copy(new Request(r));
     return sendRequest(std::move(copy));
-  }*/
+  }
 
-  // Send a request to the server and return immediately.
-  // When a response is received or an error occurs, the corresponding
-  // callbackis called. The callback is executed on a specific
-  // IO-Thread for this connection.
+  /// @brief Send a request to the server and return immediately.
+  /// When a response is received or an error occurs, the corresponding
+  /// callbackis called. The callback is executed on a specific
+  /// IO-Thread for this connection.
   virtual MessageID sendRequest(std::unique_ptr<Request> r,
                                 RequestCallback cb) = 0;
 
-  // Send a request to the server and return immediately.
-  // When a response is received or an error occurs, the corresponding
-  // callback is called.
-  /*MessageID sendRequest(Request const& r, RequestCallback cb) {
+  /// @brief Send a request to the server and return immediately.
+  /// When a response is received or an error occurs, the corresponding
+  /// callbackis called. The callback is executed on a specific
+  /// IO-Thread for this connection.
+  MessageID sendRequest(Request const& r, RequestCallback cb) {
     std::unique_ptr<Request> copy(new Request(r));
     return sendRequest(std::move(copy), cb);
-  }*/
+  }
 
-  // Return the number of requests that have not yet finished.
+  /// @brief Return the number of requests that have not yet finished.
   virtual std::size_t requestsLeft() const = 0;
 
   /// @brief connection state
   virtual State state() const = 0;
-
-  /// @brief Activate the connection.
-  virtual void startConnection() = 0;
-  virtual void shutdownConnection(const ErrorCondition) = 0;
+
+  /// @brief cancel the connection, unusable afterwards
+  virtual void cancel() = 0;
 
   std::string endpoint() const;
 
  protected:
   Connection(detail::ConnectionConfiguration const& conf)
       : _config(conf) {}
+
+  /// @brief Activate the connection.
+  virtual void startConnection() = 0;
 
   // Invoke the configured ConnectionFailureCallback (if any)
   void onFailure(Error errorCode, const std::string& errorMessage) {

diff --git a/3rdParty/fuerte/include/fuerte/loop.h b/3rdParty/fuerte/include/fuerte/loop.h
@@ -51,15 +51,12 @@ class EventLoopService {
  public:
   // Initialize an EventLoopService with a given number of threads
   //  and a given number of io_context
-  EventLoopService(unsigned int threadCount = 1);
+  explicit EventLoopService(unsigned int threadCount = 1);
   virtual ~EventLoopService();
 
   // Prevent copying
   EventLoopService(EventLoopService const& other) = delete;
   EventLoopService& operator=(EventLoopService const& other) = delete;
-
-  /// forcebly stop all io contexts. service is unusable after
-  void forceStop();
 
   // io_service returns a reference to the boost io_service.
   std::shared_ptr<asio_io_context>& nextIOContext() {

diff --git a/3rdParty/fuerte/include/fuerte/types.h b/3rdParty/fuerte/include/fuerte/types.h
@@ -175,27 +175,32 @@ struct ConnectionConfiguration {
       : _socketType(SocketType::Tcp),
         _protocolType(ProtocolType::Vst),
         _vstVersion(vst::VST1_1),
-        _verifyHost(false),
         _host("localhost"),
+        _port("8529"),
+        _verifyHost(false),
         _connectionTimeout(60000),
+        _maxConnectRetries(3),
         _authenticationType(AuthenticationType::None),
         _user(""),
         _password(""),
         _jwtToken("") {}
 
+  ConnectionFailureCallback _onFailure;
   SocketType _socketType;  // tcp, ssl or unix
   ProtocolType _protocolType;  // vst or http
   vst::VSTVersion _vstVersion;
-  bool _verifyHost;
-
+
   std::string _host;
   std::string _port;
+  bool _verifyHost;
+
   std::chrono::milliseconds _connectionTimeout;
+  unsigned _maxConnectRetries;
+
   AuthenticationType _authenticationType;
   std::string _user;
   std::string _password;
   std::string _jwtToken;
-  ConnectionFailureCallback _onFailure;
 };
 }  // namespace detail
 }}}  // namespace arangodb::fuerte::v1

diff --git a/3rdParty/fuerte/include/fuerte/waitgroup.h b/3rdParty/fuerte/include/fuerte/waitgroup.h
@@ -51,7 +51,7 @@ class WaitGroup {
   // call done() when the event has finished.
   void add(unsigned int increment = 1) {
     assert(increment > 0);
-    std::lock_guard<std::mutex> lock(_mutex);
+    std::unique_lock<std::mutex> lock(_mutex);
     _counter += increment;
   }
 
@@ -61,6 +61,7 @@ class WaitGroup {
     std::unique_lock<std::mutex> lock(_mutex);
     _counter--;
     if (_counter == 0) {
+      //lock.unlock();
       _conditionVar.notify_all();
     }
   }

diff --git a/3rdParty/fuerte/src/AsioSockets.h b/3rdParty/fuerte/src/AsioSockets.h
@@ -38,8 +38,10 @@ struct Socket<SocketType::Tcp>  {
     : resolver(ctx), socket(ctx) {}
 
   ~Socket() {
-    resolver.cancel();
-    shutdown();
+    try {
+      resolver.cancel();
+      shutdown();
+    } catch(...) {}
   }
 
   template<typename CallbackT>
@@ -63,10 +65,10 @@ struct Socket<SocketType::Tcp>  {
   }
   void shutdown() {
     if (socket.is_open()) {
-      asio_ns::error_code error; // prevents exceptions
-      socket.cancel(error);
-      socket.shutdown(asio_ns::ip::tcp::socket::shutdown_both, error);
-      socket.close(error);
+      asio_ns::error_code ec; // prevents exceptions
+      socket.cancel(ec);
+      socket.shutdown(asio_ns::ip::tcp::socket::shutdown_both, ec);
+      socket.close(ec);
    }
   }
 
@@ -81,8 +83,10 @@ struct Socket<fuerte::SocketType::Ssl> {
   : resolver(ctx), socket(ctx, loop.sslContext()) {}
 
   ~Socket() {
-    resolver.cancel();
-    shutdown();
+    try {
+      resolver.cancel();
+      shutdown();
+    } catch(...) {}
   }
 
   template<typename CallbackT>
@@ -124,12 +128,12 @@ struct Socket<fuerte::SocketType::Ssl> {
     resolver.async_resolve({config._host, config._port}, rcb);
   }
   void shutdown() {
-    //socket.cancel();
     if (socket.lowest_layer().is_open()) {
-      asio_ns::error_code error;
-      socket.shutdown(error);
-      socket.lowest_layer().shutdown(asio_ns::ip::tcp::socket::shutdown_both, error);
-      socket.lowest_layer().close(error);
+      asio_ns::error_code ec;
+      socket.lowest_layer().cancel(ec);
+      socket.shutdown(ec);
+      socket.lowest_layer().shutdown(asio_ns::ip::tcp::socket::shutdown_both, ec);
+      socket.lowest_layer().close(ec);
     }
   }