arangodb
diff --git a/‎CHANGELOG
Lines changed: 23 additions & 0 deletions
8000 b/‎CHANGELOG
Lines changed: 23 additions & 0 deletions
8000
diff --git a/‎arangod/RestHandler/RestReplicationHandler.cpp
Lines changed: 60 additions & 33 deletions b/‎arangod/RestHandler/RestReplicationHandler.cpp
Lines changed: 60 additions & 33 deletions
diff --git a/‎arangosh/Dump/DumpFeature.h
Lines changed: 1 addition & 1 deletion b/‎arangosh/Dump/DumpFeature.h
Lines changed: 1 addition & 1 deletion
@@ -1,6 +1,29 @@
 devel
 -----
 
+* Changed default value of arangodump's `--envelope` option from `true` to
+  `false`. This allows using higher parallelism in arangorestore when 
+  restoring large collection dumps. As a side-effect, this will also decrease 
+  the size of dumps taken with arangodump, and should slightly improve dump 
+  speed.
+
+* Improve parallelism capabilities of arangorestore.
+
+  arangorestore can now dispatch restoring data chunks of a collection to idle
+  background threads, so that multiple restore requests can be in flight for 
+  the same collection concurrently.
+
+  This can improve restore speed in situations when there are idle threads 
+  left (number of threads can be configured via arangorestore's `--threads` 
+  option) and the dump file for the collection is large.
+
+  The improved parallelism is only used when restoring dumps that are in the 
+  non-enveloped format. This format has been introduced with ArangoDB 3.8. 
+  The reason is that dumps in the non-enveloped format only contain the raw 
+  documents, which can be restored independent of each other, i.e. in any 
+  order. However, the enveloped format may contain documents and remove 
+  operations, which need to be restored in the original order. 
+
 * Fix BTS-374: thread race between ArangoSearch link unloading and storage
   engine WAL flushing.
 
 
@@ -1448,8 +1448,11 @@ Result RestReplicationHandler::parseBatch(transaction::Methods& trx,
   VPackStringRef bodyStr = _request->rawPayload();
   char const* ptr = bodyStr.data();
   char const* end = ptr + bodyStr.size();
-  
-  VPackBuilder builder(&basics::VelocyPackHelper::strictRequestValidationOptions);
+
+  VPackOptions builderOptions = basics::VelocyPackHelper::strictRequestValidationOptions;
+  builderOptions.paddingBehavior = VPackOptions::PaddingBehavior::UsePadding;
+
+  VPackBuilder builder(&builderOptions);
 
   // First parse and collect all markers, we assemble everything in one
   // large builder holding an array
@@ -1493,12 +1496,15 @@ Result RestReplicationHandler::parseBatch(transaction::Methods& trx,
 
         TRI_ASSERT(doc.isObject());
         bool checkKey = true;
+        bool checkRev = generateNewRevisionIds;
         for (auto it : VPackObjectIterator(doc, true)) {
           // only check for "_key" attribute here if we still have to.
           // once we have seen it, it will not show up again in the same document
           bool const isKey = checkKey && (arangodb::velocypack::StringRef(it.key) == StaticStrings::KeyString);
 
           if (isKey) {
+            // _key attribute
+
             // prevent checking for _key twice in the same document
             checkKey = false;
 
@@ -1515,17 +1521,23 @@ Result RestReplicationHandler::parseBatch(transaction::Methods& trx,
               // with MMFiles dumps from <= 3.6
               documentsToRemove.erase(it.value.copyString());
             }
-          }
+          
+            documentsToInsert.add(it.key);
+            documentsToInsert.add(it.value);
+          } else if (checkRev && arangodb::velocypack::StringRef(it.key) == StaticStrings::RevString) {
+            // _rev attribute
 
-          documentsToInsert.add(it.key);
+            // prevent checking for _rev twice in the same document
+            checkRev = false;
 
-          if (generateNewRevisionIds && 
-              !isKey && 
-              arangodb::velocypack::StringRef(it.key) == StaticStrings::RevString) {
             char ridBuffer[arangodb::basics::maxUInt64StringSize];
             RevisionId newRid = physical->newRevisionId();
+            
+            documentsToInsert.add(it.key);
             documentsToInsert.add(newRid.toValuePair(ridBuffer));
           } else {
+            // copy key/value verbatim
+            documentsToInsert.add(it.key);
             documentsToInsert.add(it.value);
           }
         }
@@ -1629,15 +1641,18 @@ Result RestReplicationHandler::processRestoreDataBatch(transaction::Methods& trx
                                                        std::string const& collectionName,
                                                        bool generateNewRevisionIds) {
   // we'll build all documents to insert in this builder
-  VPackBuilder documentsToInsert;
+  VPackOptions vpackOptions;
+  vpackOptions.paddingBehavior = VPackOptions::PaddingBehavior::UsePadding;
+
+  VPackBuilder documentsToInsert(&vpackOptions);
   std::unordered_set<std::string> documentsToRemove;
   Result res = parseBatch(trx, collectionName, documentsToInsert, documentsToRemove, generateNewRevisionIds);
   if (res.fail()) {
     return res;
   }
 
   OperationOptions options(_context);
-  options.silent = false;
+  options.silent = true;
   options.ignoreRevs = true;
   options.isRestore = true;
   options.waitForSync = false;
@@ -1679,6 +1694,42 @@ Result RestReplicationHandler::processRestoreDataBatch(transaction::Methods& trx
           << " documents for restore: " << opRes.result.errorMessage();
       return opRes.result;
     }
+
+    if (opRes.countErrorCodes.empty()) {
+      // no detailed errors reported. all good
+      return Result();
+    }
+
+    // at least one error occurred
+    if (opRes.slice().isArray()) {
+      // Now go through the individual results and check each errors
+      VPackArrayIterator itRequest(requestSlice);
+      VPackArrayIterator itResult(opRes.slice());
+
+      while (itRequest.valid()) {
+        VPackSlice result = *itResult;
+        VPackSlice error = result.get(StaticStrings::Error);
+        if (error.isTrue()) {
+          error = result.get(StaticStrings::ErrorNum);
+          if (error.isNumber()) {
+            auto code = ErrorCode{error.getNumericValue<int>()};
+            error = result.get(StaticStrings::ErrorMessage);
+            if (error.isString()) {
+              return { code, error.copyString() };
+            }
+            return { code };
+          }
+        }
+        itRequest.next();
+        itResult.next();
+      }
+    }
+    
+    // if we get here, we didn't have a detailed array with results.
+    // so we need to stick to the error code map, which is all we got
+    TRI_ASSERT(!opRes.countErrorCodes.empty());
+    ErrorCode ec = (*opRes.countErrorCodes.begin()).first;
+    return {ec};
   } catch (arangodb::basics::Exception const& ex) {
     LOG_TOPIC("8e8e1", WARN, Logger::CLUSTER)
         << "could not insert documents for restore exception: " << ex.what();
@@ -1692,30 +1743,6 @@ Result RestReplicationHandler::processRestoreDataBatch(transaction::Methods& trx
         << "could not insert documents for restore exception.";
     return Result(TRI_ERROR_INTERNAL);
   }
-
-  // Now go through the individual results and check each errors
-  VPackArrayIterator itRequest(requestSlice);
-  VPackArrayIterator itResult(opRes.slice());
-
-  while (itRequest.valid()) {
-    VPackSlice result = *itResult;
-    VPackSlice error = result.get(StaticStrings::Error);
-    if (error.isTrue()) {
-      error = result.get(StaticStrings::ErrorNum);
-      if (error.isNumber()) {
-        auto code = ErrorCode{error.getNumericValue<int>()};
-        error = result.get(StaticStrings::ErrorMessage);
-        if (error.isString()) {
-          return { code, error.copyString() };
-        }
-        return { code };
-      }
-    }
-    itRequest.next();
-    itResult.next();
-  }
-
-  return Result();
 }
 
 ////////////////////////////////////////////////////////////////////////////////
 
@@ -82,7 +82,7 @@ class DumpFeature final : public application_features::ApplicationFeature {
     bool overwrite{false};
     bool progress{true};
     bool useGzip{true};
-    bool useEnvelope{true};
+    bool useEnvelope{false};
   };
 
   /// @brief Stores stats about the overall dump progress