arangodb · Dronplane · Oct 5, 2021 · Jun 10, 2021 · Jun 11, 2021 · Jun 11, 2021
diff --git a/.gitignore b/.gitignore
@@ -20,6 +20,7 @@ build
 
 *.deb
 *.rpm
+*.user
 
 .DS_Store
 *.swp
@@ -83,6 +84,8 @@ Installation/MacOSX/Bundle/Info.plist
 nbproject/
 .idea
 .vscode
+*.sublime-workspace
+*.sublime-project
 
 test.cpp.txt
 
@@ -115,3 +118,5 @@ last_compiled_version.sha
 scripts/perfanalysis
 perf*
 callgrind*
+
+Documentation/Metrics/allMetrics.yaml
diff --git a/3rdParty/CMakeLists.txt b/3rdParty/CMakeLists.txt
@@ -31,16 +31,28 @@ add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/zlib/zlib-1.2.11)
 ## SNAPPY
 ################################################################################
 
-set(SNAPPY_VERSION "1.1.8")
-set(SNAPPY_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/snappy/snappy-${SNAPPY_VERSION}")
+function (add_snappy)
+    # use the function to open a scope so CMAKE_CXX_FLAGS is only changed for
+    # snappy
+    add_c_flags_if_supported(CMAKE_C_FLAGS -Wno-suggest-override -Wno-sign-compare)
+    add_cxx_flags_if_supported(CMAKE_CXX_FLAGS -Wno-suggest-override -Wno-sign-compare)
+    set(SNAPPY_VERSION "1.1.9")
+    set(SNAPPY_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/snappy/snappy-${SNAPPY_VERSION}")
+    set(SNAPPY_SOURCE_DIR "${SNAPPY_SOURCE_DIR}" PARENT_SCOPE)
+    set(SNAPPY_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/snappy/snappy-${SNAPPY_VERSION}" PARENT_SCOPE)
+    set(SNAPPY_LIB "snappy" PARENT_SCOPE)
+    set(SNAPPY_FUZZING_BUILD OFF CACHE BOOL "build Snappy for fuzzing")
+    set(SNAPPY_BUILD_TESTS OFF CACHE BOOL "disable Snappy tests" FORCE)
+    set(SNAPPY_BUILD_BENCHMARKS OFF CACHE BOOL "disable Snappy benchmarks" FORCE)
+    set(SNAPPY_INSTALL OFF CACHE BOOL "disable Snappy installation")
+    set(SNAPPY_REQUIRE_AVX ON CACHE BOOL "target processors with AVX support" FORCE)
+    set(SNAPPY_REQUIRE_AVX2 OFF CACHE BOOL "target processors with AVX2 support" FORCE)
+    add_subdirectory(${SNAPPY_SOURCE_DIR} EXCLUDE_FROM_ALL)
+endfunction ()
+add_snappy()
 set(SNAPPY_SOURCE_DIR "${SNAPPY_SOURCE_DIR}" PARENT_SCOPE)
-set(SNAPPY_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/snappy/snappy-${SNAPPY_VERSION}")
 set(SNAPPY_BUILD_DIR "${SNAPPY_BUILD_DIR}" PARENT_SCOPE)
-set(SNAPPY_LIB "snappy")
 set(SNAPPY_LIB "${SNAPPY_LIB}" PARENT_SCOPE)
-set(SNAPPY_BUILD_TESTS OFF CACHE BOOL "disable Snappy tests")
-set(SNAPPY_INSTALL OFF CACHE BOOL "disable Snappy installation")
-add_subdirectory(${SNAPPY_SOURCE_DIR} EXCLUDE_FROM_ALL)
 
 ################################################################################
 ## LIBUNWIND
@@ -107,6 +119,16 @@ set(ARANGO_S2GEOMETRY_VERSION "${ARANGO_S2GEOMETRY_VERSION}" PARENT_SCOPE)
 add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/boost)
 set(BOOST_VERSION ${BOOST_VERSION} PARENT_SCOPE)
 
+################################################################################
+## Velocypack
+################################################################################
+
+set(HashType "xxhash" CACHE STRING "Hash type (fasthash, xxhash)" FORCE)
+set(BuildVelocyPackExamples OFF CACHE BOOL "Build VelocyPack Examples" FORCE)
+set(BuildTools OFF CACHE BOOL "Build VelocyPack Tools" FORCE)
+
+add_subdirectory(velocypack EXCLUDE_FROM_ALL)
+
 ################################################################################
 ## IResearch
 ################################################################################
@@ -138,7 +160,6 @@ set(LZ4_INCLUDE_DIR "${LZ4_ROOT}/lib" PARENT_SCOPE)
 set(SNOWBALL_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/snowball")
 set(UNWIND_ROOT "invalid")
 set(BFD_ROOT "invalid")
-
 find_package(IResearch REQUIRED) # set IRESEARCH_BUILD_DIR
 
 set(CMAKE_MACOSX_RPATH ON) # suppress cmake warning (use same value as cmake default)
@@ -251,10 +272,6 @@ target_include_directories(fuerte SYSTEM PRIVATE
 
 set(V8_INTERNAL_INCLUDE_DIR ${V8_INTERNAL_INCLUDE_DIR} PARENT_SCOPE)
 
-if(NOT TARGET velocypack)
-add_library(velocypack INTERFACE)
-target_include_directories(velocypack INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/velocypack/include")
-endif()
 
 set(TAOCPP_JSON_BUILD_TESTS    OFF CACHE BOOL "Build taocpp::json test programs"    FORCE)
 set(TAOCPP_JSON_BUILD_EXAMPLES OFF CACHE BOOL "Build taocpp::json example programs" FORCE)

diff --git a/3rdParty/README_maintainers.md b/3rdParty/README_maintainers.md
@@ -104,8 +104,6 @@ http://s2geometry.io/
 Compression library
 https://github.com/google/snappy
 
-We change the target `snappy` to `snapy-dyn` so cmake doesn't interfere targets with the static library (that we need)
-
 ## snowball
 
 http://snowball.tartarus.org/ stemming for IResearch. We use the latest provided cmake which we maintain.

diff --git a/3rdParty/V8/CMakeLists.txt b/3rdParty/V8/CMakeLists.txt
@@ -340,7 +340,7 @@ else ()
   set(V8_CFLAGS   "$ENV{V8_CFLAGS}")
   set(V8_CXXFLAGS "$ENV{V8_CXXFLAGS}")
   set(V8_LDFLAGS  "$ENV{V8_LDFLAGS}")
-  
+
   # build V8 without debug information
   if (USE_MINIMAL_DEBUGINFO)
     set(V8_CFLAGS   "${V8_CFLAGS} -g0")
@@ -353,6 +353,8 @@ else ()
     set(V8_CFLAGS   "${V8_CFLAGS} -fno-delete-null-pointer-checks")
     set(V8_CXXFLAGS "${V8_CXXFLAGS} -fno-delete-null-pointer-checks")
   endif()
+  add_c_flags_if_supported(V8_CFLAGS -Wno-suggest-override -Wno-implicit-const-int-float-conversion -Wno-non-virtual-dtor -Wno-final-dtor-non-final-class -Wno-implicit-const-int-float-conversion)
+  add_cxx_flags_if_supported(V8_CXXFLAGS -Wno-suggest-override -Wno-implicit-const-int-float-conversion -Wno-non-virtual-dtor -Wno-final-dtor-non-final-class -Wno-implicit-const-int-float-conversion)
 
   if (USE_DEBUG_V8)
     set(V8_TARGET_ARCH "${V8_PROC_ARCH}.debug")

diff --git a/3rdParty/boost/CMakeLists.txt b/3rdParty/boost/CMakeLists.txt
@@ -5,6 +5,9 @@ set(boost_version "1.71.0")
 set(BOOST_VERSION ${boost_version} PARENT_SCOPE)
 set(boost_src "${CMAKE_CURRENT_SOURCE_DIR}/${boost_version}")
 
+add_c_flags_if_supported(CMAKE_C_FLAGS -Wno-suggest-override)
+add_cxx_flags_if_supported(CMAKE_CXX_FLAGS -Wno-suggest-override)
+
 #create interface target for boost header only libraries
 message(STATUS "using 3rdParty BOOST")
 add_library(boost_boost INTERFACE)

diff --git a/3rdParty/fuerte/FuerteH1TCP.tla b/3rdParty/fuerte/FuerteH1TCP.tla
@@ -124,7 +124,7 @@ process(fuerte = "fuertethread") {
       or connectDone:    \* VERIFIED
         if (/\ Len(iocontext) >= 1
             /\ Head(iocontext) \in {"connect", "connectBAD"}) {
-          assert state \in {"Connecting", "Created", "Closed"};
+          assert state \in {"Connecting", "Closed"};
           alarm := "off";
           if (Head(iocontext) = "connect" /\ state = "Connecting") {
             assert active;
@@ -259,7 +259,7 @@ process(cancel = "cancelthread") {
 
 }
 *)
-\* BEGIN TRANSLATION - the hash of the PCal code: PCal-925ea62a5ed7f2fd7841f2836c7d799c
+\* BEGIN TRANSLATION - the hash of the PCal code: PCal-925ea62a5ed7f2fd7841f2836c7d799c (chksum(pcal) = "c5aa3da3" /\ chksum(tla) = "af7613cb")
 VARIABLES state, active, queueSize, alarm, asyncRunning, iocontext, reading, 
           writing, pc, stack
 
@@ -464,12 +464,12 @@ activate == /\ pc["fuertethread"] = "activate"
 connectDone == /\ pc["fuertethread"] = "connectDone"
                /\ IF /\ Len(iocontext) >= 1
                      /\ Head(iocontext) \in {"connect", "connectBAD"}
-                     THEN /\ Assert(state \in {"Connecting", "Created", "Closed"}, 
-                                    "Failure of assertion at line 127, column 11.")
+                     THEN /\ Assert(state \in {"Connecting", "Closed"}, 
+                                    "Failure of assertion at line 128, column 11.")
                           /\ alarm' = "off"
                           /\ IF Head(iocontext) = "connect" /\ state = "Connecting"
                                 THEN /\ Assert(active, 
-                                               "Failure of assertion at line 130, column 13.")
+                                               "Failure of assertion at line 131, column 13.")
       
F438
                               /\ iocontext' = Tail(iocontext)
                                      /\ state' = "Connected"
                                      /\ stack' = [stack EXCEPT !["fuertethread"] = << [ procedure |->  "asyncWriteNextRequest",
@@ -492,7 +492,7 @@ writeDone == /\ pc["fuertethread"] = "writeDone"
                    /\ Head(iocontext) \in {"write", "writeBAD"}
                    THEN /\ Assert(/\ state \in {"Connected", "Closed"}
                                   /\ writing, 
-                                  "Failure of assertion at line 145, column 11.")
+                                  "Failure of assertion at line 146, column 11.")
                         /\ writing' = FALSE
                         /\ IF Head(iocontext) = "write" /\ state = "Connected"
                               THEN /\ iocontext' = Tail(iocontext)
@@ -517,7 +517,7 @@ readDone == /\ pc["fuertethread"] = "readDone"
             /\ IF /\ Len(iocontext) >= 1
                   /\ Head(iocontext) \in {"read", "readBAD"}
                   THEN /\ Assert(state \in {"Connected", "Closed"}, 
-                                 "Failure of assertion at line 163, column 11.")
+                                 "Failure of assertion at line 164, column 11.")
                        /\ alarm' = "off"
                        /\ reading' = FALSE
                        /\ IF Head(iocontext) = "read" /\ state = "Connected"
@@ -714,6 +714,6 @@ NoSleepingBarber == /\ NothingForgottenOnQueue
 
 =============================================================================
 \* Modification History
-\* Last modified Fri Aug 21 16:03:16 CEST 2020 by neunhoef
+\* Last modified Thu Aug 19 14:45:44 CEST 2021 by neunhoef
 \* Last modified Wed Jul 22 12:06:32 CEST 2020 by simon
 \* Created Mi 22. Apr 22:46:19 CEST 2020 by neunhoef
diff --git a/3rdParty/fuerte/include/fuerte/connection.h b/3rdParty/fuerte/include/fuerte/connection.h
@@ -35,6 +35,7 @@ namespace arangodb { namespace fuerte { inline namespace v1 {
 // Connection is the base class for a connection between a client
 // and a server.
 // Different protocols (HTTP, VST) are implemented in derived classes.
+// For details of the setup see the top of src/GeneralConnection.h .
 class Connection : public std::enable_shared_from_this<Connection> {
   friend class ConnectionBuilder;
 
@@ -204,7 +205,7 @@ class ConnectionBuilder {
 
   /// @brief tcp, ssl or unix
   inline SocketType socketType() const { return _conf._socketType; }
-  /// @brief protocol typr
+  /// @brief protocol type
   inline ProtocolType protocolType() const { return _conf._protocolType; }
   void protocolType(ProtocolType pt) { _conf._protocolType = pt; }
 

diff --git a/3rdParty/fuerte/include/fuerte/types.h b/3rdParty/fuerte/include/fuerte/types.h
@@ -29,6 +29,7 @@
 #include <map>
 #include <memory>
 #include <string>
+#include <string_view>
 #include <vector>
 
 namespace arangodb { namespace fuerte { inline namespace v1 {
@@ -53,6 +54,7 @@ StatusCode constexpr StatusMethodNotAllowed = 405;
 StatusCode constexpr StatusNotAcceptable = 406;
 StatusCode constexpr StatusConflict = 409;
 StatusCode constexpr StatusPreconditionFailed = 412;
+StatusCode constexpr StatusMisdirectedRequest = 421;
 StatusCode constexpr StatusInternalError = 500;
 StatusCode constexpr StatusServiceUnavailable = 503;
 StatusCode constexpr StatusVersionNotSupported = 505;
@@ -116,7 +118,8 @@ enum class RestVerb {
   Options = 6
 };
 std::string to_string(RestVerb type);
-RestVerb from_string(std::string const&);
+RestVerb from_string(std::string_view type);
+RestVerb from_string(std::string const& type);
 
 // -----------------------------------------------------------------------------
 // --SECTION--                                                       MessageType

diff --git a/3rdParty/fuerte/src/AsioSockets.h b/3rdParty/fuerte/src/AsioSockets.h
@@ -40,12 +40,21 @@ void resolveConnect(detail::ConnectionConfiguration const& config,
       return;
     }
 
-    // A successful resolve operation is guaranteed to pass a
-    // non-empty range to the handler.
-    asio_ns::async_connect(socket, it,
-                           [done(std::move(done))](auto ec, auto it) mutable {
-                             std::forward<F>(done)(ec);
-                           });
+    try {
+      // A successful resolve operation is guaranteed to pass a
+      // non-empty range to the handler.
+      asio_ns::async_connect(socket, it,
+                             [done(std::move(done))](auto ec, auto it) mutable {
+                               std::forward<F>(done)(ec);
+                             });
+    } catch (std::bad_alloc const&) {
+      // definitely an OOM error
+      done(boost::system::errc::make_error_code(boost::system::errc::not_enough_memory));
+    } catch (...) {
+      // probably not an OOM error, but we don't know what it actually is.
+      // there is no code for a generic error that we could use here
+      done(boost::system::errc::make_error_code(boost::system::errc::not_enough_memory));
+    }
   };
 
   // windows does not like async_resolve
@@ -146,7 +155,13 @@ struct Socket<fuerte::SocketType::Ssl> {
             } else {
               socket.set_verify_mode(asio_ns::ssl::verify_none);
             }
+          } catch (std::bad_alloc const&) {
+            // definitely an OOM error
+            done(boost::system::errc::make_error_code(boost::system::errc::not_enough_memory));
+            return;
           } catch (boost::system::system_error const& exc) {
+            // probably not an OOM error, but we don't know what it actually is.
+            // there is no code for a generic error that we could use here
             done(exc.code());
             return;
           }

diff --git a/3rdParty/fuerte/src/GeneralConnection.h b/3rdParty/fuerte/src/GeneralConnection.h
@@ -24,6 +24,48 @@
 #ifndef ARANGO_CXX_DRIVER_GENERAL_CONNECTION_H
 #define ARANGO_CXX_DRIVER_GENERAL_CONNECTION_H 1
 
+/// Here is an overview over the class structure for connections in fuerte.
+/// There is a lot of inheritance and templating going on which can be a
+/// bit confusing at first sight.
+///
+/// The base class for all connections is `Connection` in `connection.h`
+/// which has `std::enable_shared_from_this<Connection>` to sort out
+/// memory allocation and object lifetimes. In this way, a callback
+/// closure can own a shared_ptr to the connection object it is working
+/// on. This base class basically only defines the interface for all
+/// other derived classes. There is not a lot implemented in there.
+///
+/// The class template GeneralConnection<ST, RT> implements code which
+/// is the same for all three cases HTTP/1, VST and HTTP/2. Here, ST
+/// stands for the socket type (which can be Tcp (unencrypted), Ssl (TLS
+/// encryption with openssl) and Unix (unix domain socket, no encryption).
+/// Note that all 9 combination between protocol version and socket type
+/// are possible. RT is a type which represents a request which is run
+/// on the connection. For HTTP/1, this will be a `RequestItem`, for
+/// HTTP/2 it will be a `Stream` and for VST it will be a `vst::RequestItem`.
+/// All three RTs are separate classes but behave similarly and have
+/// similar methods, such that the generic code can use them. They also
+/// have differences, since requests have to be handled differently
+/// in the three protocols.
+///
+/// For the VST and HTTP/2 cases there is an intermediate class template
+/// `MultiConnection<ST, RT>`, which inherits from `GeneralConnection<ST, RT>`
+/// and is needed, because - contrary to HTTP/1 - there can be more than
+/// one request in flight on a connection due to multiplexing. Furthermore,
+/// for VST a single request or response can be distributed across muliple
+/// "chunks" in the communication, and for HTTP/2, a single request or
+/// response can be distributed across multiple "frames".
+///
+/// Therefore, the `MultiConnection<ST, RT>` implementation has a member
+/// `_streams` which holds the current list of requests (instances of RT)
+/// which are currently in flight.
+///
+/// Finally, there are the three class templates
+///   `H1Connection` inheriting from `GeneralConnection<ST, RequestItem>`,
+///   `H2Connection` inheriting from `MultiConnection<ST, Stream>` and
+///   `VstConnection` inheriting from `MultiConnection<ST, vst::RequestItem>`.
+/// 
+
 #include <fuerte/connection.h>
 #include <fuerte/types.h>
 
@@ -403,7 +445,7 @@ struct MultiConnection : public GeneralConnection<ST, RT> {
       }
     }
 
-    const bool wasReading = this->_writing;
+    const bool wasReading = this->_reading;
     const bool wasWriting = this->_writing;
     if (wasReading || wasWriting) {
       FUERTE_ASSERT(_lastIO + kIOTimeout > now);

diff --git a/3rdParty/fuerte/src/H1Connection.cpp b/3rdParty/fuerte/src/H1Connection.cpp
@@ -194,15 +194,17 @@ void H1Connection<ST>::finishConnect() {
   // If it has already gone off, we might have a completion handler
   // already posted on the iocontext. However, this will not touch anything
   // if we have first set the state to `Connected`.
-  FUERTE_ASSERT(this->_active.load());
   auto exp = Connection::State::Connecting;
   if (this->_state.compare_exchange_strong(exp, Connection::State::Connected)) {
+    FUERTE_ASSERT(this->_active.load());
     this->asyncWriteNextRequest();  // starts writing if queue non-empty
   } else {
     FUERTE_LOG_ERROR << "finishConnect: found state other than 'Connecting': "
                      << static_cast<int>(exp);
-    FUERTE_ASSERT(false);
-    // If this happens, we probably have a sleeping barber
+    FUERTE_ASSERT(exp == Connection::State::Closed);
+    // If this happens, then the connection has been shut down before
+    // it could be fully connected, but the completion handler of the
+    // connect call was still scheduled. No more work to do.
   }
 }
 

diff --git a/3rdParty/fuerte/src/H2Connection.cpp b/3rdParty/fuerte/src/H2Connection.cpp
@@ -28,8 +28,6 @@
 #include <fuerte/types.h>
 #include <velocypack/velocypack-aliases.h>
 
-#include <regex>
-
 #include "debugging.h"
 
 namespace arangodb { namespace fuerte { inline namespace v1 { namespace http {
@@ -294,11 +292,11 @@ void H2Connection<T>::initNgHttp2Session() {
   }
 
   rv = nghttp2_session_client_new(&_session, callbacks, /*args*/ this);
+  nghttp2_session_callbacks_del(callbacks);
+
   if (rv != 0) {
-    nghttp2_session_callbacks_del(callbacks);
     throw std::runtime_error("out ouf memory");
   }
-  nghttp2_session_callbacks_del(callbacks);
 }
 
 // -----------------------------------------------------------------------------
@@ -465,7 +463,7 @@ void H2Connection<SocketType::Ssl>::finishConnect() {
 // queue the response onto the session, call only on IO thread
 template <SocketType T>
 void H2Connection<T>::queueHttp2Requests() {
-  int numQueued = 0;  // make sure we do send too many request
+  int numQueued = 0;  // make sure we do not send too many request
 
   Stream* tmp = nullptr;
   while (numQueued++ < 4 && this->_queue.pop(tmp)) {
@@ -723,7 +721,7 @@ template <SocketType T>
 void H2Connection<T>::abortRequests(fuerte::Error err, Clock::time_point now) {
   auto it = this->_streams.begin();
   while (it != this->_streams.end()) {
-    if (it->second->expires < now) {
+    if (it->second->expires <= now) {
       it->second->invokeOnError(err);
 
       if (now == Clock::time_point::max()) {