arangodb
diff --git a/‎CHANGELOG
Lines changed: 7 additions & 0 deletions b/‎CHANGELOG
Lines changed: 7 additions & 0 deletions
diff --git a/‎Documentation/DocuBlocks/Rest/Collections/1_structs.md
Lines changed: 4 additions & 0 deletions b/‎Documentation/DocuBlocks/Rest/Collections/1_structs.md
Lines changed: 4 additions & 0 deletions
diff --git a/‎Documentation/DocuBlocks/Rest/Graph/1_structs.md
Lines changed: 5 additions & 0 deletions b/‎Documentation/DocuBlocks/Rest/Graph/1_structs.md
Lines changed: 5 additions & 0 deletions
diff --git a/‎Documentation/DocuBlocks/Rest/Graph/general_graph_create_http_examples.md
Lines changed: 5 additions & 0 deletions b/‎Documentation/DocuBlocks/Rest/Graph/general_graph_create_http_examples.md
Lines changed: 5 additions & 0 deletions
diff --git a/‎Documentation/DocuBlocks/collectionProperties.md
Lines changed: 9 additions & 1 deletion b/‎Documentation/DocuBlocks/collectionProperties.md
Lines changed: 9 additions & 1 deletion
diff --git a/‎arangod/Agency/Job.cpp
Lines changed: 55 additions & 35 deletions b/‎arangod/Agency/Job.cpp
Lines changed: 55 additions & 35 deletions
diff --git a/‎arangod/Aql/RestAqlHandler.h
Lines changed: 1 addition & 3 deletions b/‎arangod/Aql/RestAqlHandler.h
Lines changed: 1 addition & 3 deletions
diff --git a/‎arangod/Cluster/ClusterCollectionCreationInfo.cpp
Lines changed: 16 additions & 15 deletions b/‎arangod/Cluster/ClusterCollectionCreationInfo.cpp
Lines changed: 16 additions & 15 deletions
diff --git a/‎arangod/Cluster/ClusterCollectionCreationInfo.h
Lines changed: 4 additions & 2 deletions b/‎arangod/Cluster/ClusterCollectionCreationInfo.h
Lines changed: 4 additions & 2 deletions
@@ -1,6 +1,13 @@
 v3.5.0-rc.5 (2019-XX-XX)
 ------------------------
 
+* MinReplicationFactor:
+  Collections can now be created with a minimal replication factor (minReplicationFactor), which defaults to 1.
+  If minReplicationFactor > 1 a collection will go into "read-only" mode as soon as it has less then minReplicationFactor
+  many insync followers. With this mechanism users can avoid to have collections diverge too much in case of failure scenarios.
+  minReplicationFactor can have the values: 1 <= minReplicationFactor <= replicationFactor.
+  Having minReplicationFactor == 1 ArangoDB behaves the same way as in any previous version.
+  
 * Fixed a query abort error with smart joins if both collections were restricted to a
   single shard using the "restrict-to-single-shard" optimizer rule.
 
 
@@ -43,6 +43,10 @@ determine the target shard for documents; *Cluster specific attribute.*
 @RESTSTRUCT{replicationFactor,collection_info,integer,optional,}
 contains how many copies of each shard are kept on different DBServers.; *Cluster specific attribute.*
 
+@RESTSTRUCT{minReplicationFactor,collection_info,integer,optional,}
+contains how many minimal copies of each shard need to be in sync on different DBServers.
+The shards will refuse to write, if we have less then these many copies in sync. *Cluster specific attribute.*
+
 @RESTSTRUCT{shardingStrategy,collection_info,string,optional,}
 the sharding strategy selected for the collection; *Cluster specific attribute.*
 One of 'hash' or 'enterprise-hash-smart-edge'
 
@@ -25,6 +25,11 @@ concurrent modifications to this graph.
 @RESTSTRUCT{replicationFactor,graph_representation,integer,required,}
 The replication factor used for every new collection in the graph.
 
+@RESTSTRUCT{minReplicationFactor,graph_representation,integer,optional,}
+The minimal replication factor used for every new collection in the graph.
+If one shard has less than minReplicationFactor copies, we cannot write
+to this shard, but to all others.
+
 @RESTSTRUCT{isSmart,graph_representation,boolean,required,}
 Flag if the graph is a SmartGraph (Enterprise Edition only) or not.
 
 
@@ -42,6 +42,11 @@ Cannot be modified later.
 @RESTSTRUCT{replicationFactor,post_api_gharial_create_opts,integer,required,}
 The replication factor used when initially creating collections for this graph.
 
+@RESTSTRUCT{minReplicationFactor,post_api_gharial_create_opts,integer,optional,}
+The minimal replication factor used for every new collection in the graph.
+If one shard has less than minReplicationFactor copies, we cannot write
+to this shard, but to all others.
+
 @RESTRETURNCODES
 
 @RESTRETURNCODE{201}
 
@@ -52,7 +52,11 @@ In a cluster setup, the result will also contain the following attributes:
   determine the target shard for documents.
 
 * *replicationFactor*: determines how many copies of each shard are kept 
-  on different DBServers.
+  on different DBServers. Has to be in the range of 1-10 *(Cluster only)*
+
+* *minReplicationFactor* : determines the number of minimal shard copies kept on
+  different DBServers, a shard will refuse to write if less than this amount
+  of copies are in sync. Has to be in the range of 1-replicationFactor *(Cluster only)*
 
 * *shardingStrategy*: the sharding strategy selected for the collection.
   This attribute will only be populated in cluster mode and is not populated
@@ -77,6 +81,10 @@ one or more of the following attribute(s):
   different DBServers, valid values are  integer numbers
   in the range of 1-10 *(Cluster only)*
 
+* *minReplicationFactor* : Change the number of minimal shard copies to be in sync on 
+  different DBServers, a shard will refuse to write if less than this amount
+  of copies are in sync. Has to be in the range of 1-replicationFactor *(Cluster only)*
+
 **Note**: some other collection properties, such as *type*, *isVolatile*,
 *keyOptions*, *numberOfShards* or *shardingStrategy* cannot be changed once 
 the collection is created.
 
@@ -95,21 +95,22 @@ bool Job::finish(std::string const& server, std::string const& shard,
     try {
       jobType = pending.slice()[0].get("type").copyString();
     } catch (std::exception const&) {
-      LOG_TOPIC("76352", WARN, Logger::AGENCY) << "Failed to obtain type of job " << _jobId;
+      LOG_TOPIC("76352", WARN, Logger::AGENCY)
+          << "Failed to obtain type of job " << _jobId;
     }
 
     // Additional payload, which is to be executed in the finish transaction
     Slice operations = Slice::emptyObjectSlice();
-    Slice preconditions  = Slice::emptyObjectSlice();
+    Slice preconditions = Slice::emptyObjectSlice();
 
     if (payload != nullptr) {
       Slice slice = payload->slice();
       TRI_ASSERT(slice.isObject() || slice.isArray());
-      if (slice.isObject()) {     // opers only
+      if (slice.isObject()) {  // opers only
         operations = slice;
         TRI_ASSERT(operations.isObject());
       } else {
-        TRI_ASSERT(slice.length() < 3); // opers + precs only
+        TRI_ASSERT(slice.length() < 3);  // opers + precs only
         if (slice.length() > 0) {
           operations = slice[0];
           TRI_ASSERT(operations.isObject());
@@ -125,7 +126,7 @@ bool Job::finish(std::string const& server, std::string const& shard,
     {
       VPackArrayBuilder guard(&finished);
 
-      { // operations --
+      {  // operations --
         VPackObjectBuilder operguard(&finished);
 
         addPutJobIntoSomewhere(finished, success ? "Finished" : "Failed",
@@ -148,15 +149,14 @@ bool Job::finish(std::string const& server, std::string const& shard,
           addReleaseShard(finished, shard);
         }
 
-      } // -- operations
+      }  // -- operations
 
-      if (preconditions.isObject() && preconditions.length() > 0) { // preconditions --
+      if (preconditions.isObject() && preconditions.length() > 0) {  // preconditions --
         VPackObjectBuilder precguard(&finished);
         for (auto const& prec : VPackObjectIterator(preconditions)) {
           finished.add(prec.key.copyString(), prec.value);
         }
-      } // -- preconditions
-
+      }  // -- preconditions
     }
 
     write_ret_t res = singleWriteTransaction(_agent, finished, false);
@@ -168,16 +168,16 @@ bool Job::finish(std::string const& server, std::string const& shard,
     }
   } catch (std::exception const& e) {
     LOG_TOPIC("1fead", WARN, Logger::AGENCY)
-      << "Caught exception in finish, message: " << e.what();
+        << "Caught exception in finish, message: " << e.what();
   } catch (...) {
     LOG_TOPIC("7762f", WARN, Logger::AGENCY)
-      << "Caught unspecified exception in finish.";
+        << "Caught unspecified exception in finish.";
   }
   return false;
 }
 
 std::string Job::randomIdleAvailableServer(Node const& snap,
-                                               std::vector<std::string> const& exclude) {
+                                           std::vector<std::string> const& exclude) {
   std::vector<std::string> as = availableServers(snap);
   std::string ret;
 
@@ -189,11 +189,11 @@ std::string Job::randomIdleAvailableServer(Node const& snap,
     for (auto const& srv : snap.hasAsChildren(healthPrefix).first) {
       // ignore excluded servers
       if (std::find(std::begin(exclude), std::end(exclude), srv.first) != std::end(exclude)) {
-        continue ;
+        continue;
       }
       // ignore servers not in availableServers above:
       if (std::find(std::begin(as), std::end(as), srv.first) == std::end(as)) {
-        continue ;
+        continue;
       }
 
       std::string const& status = (*srv.second).hasAsString("Status").first;
@@ -242,7 +242,7 @@ size_t Job::countGoodOrBadServersInList(Node const& snap, VPackSlice const& serv
   auto const& health = snap.hasAsChildren(healthPrefix);
   // Do we have a Health substructure?
   if (health.second) {
-    Node::Children const& healthData = health.first; // List of servers in Health
+    Node::Children const& healthData = health.first;  // List of servers in Health
     for (VPackSlice const serverName : VPackArrayIterator(serverList)) {
       if (serverName.isString()) {
         // serverName not a string? Then don't count
@@ -269,13 +269,14 @@ size_t Job::countGoodOrBadServersInList(Node const& snap, VPackSlice const& serv
 }
 
 // The following counts in a given server list how many of the servers are
- // in Status "GOOD" or "BAD".
-size_t Job::countGoodOrBadServersInList(Node const& snap, std::vector<std::string> const& serverList) {
+// in Status "GOOD" or "BAD".
+size_t Job::countGoodOrBadServersInList(Node const& snap,
+                                        std::vector<std::string> const& serverList) {
   size_t count = 0;
   auto const& health = snap.hasAsChildren(healthPrefix);
   // Do we have a Health substructure?
   if (health.second) {
-    Node::Children const& healthData = health.first; // List of servers in Health
+    Node::Children const& healthData = health.first;  // List of servers in Health
     for (auto& serverStr : serverList) {
       // Now look up this server:
       auto it = healthData.find(serverStr);
@@ -294,7 +295,8 @@ size_t Job::countGoodOrBadServersInList(Node const& snap, std::vector<std::strin
 }
 
 /// @brief Check if a server is cleaned or to be cleaned out:
-bool Job::isInServerList(Node const& snap, std::string const& prefix, std::string const& server, bool isArray) {
+bool Job::isInServerList(Node const& snap, std::string const& prefix,
+                         std::string const& server, bool isArray) {
   VPackSlice slice;
   bool found = false;
   if (isArray) {
@@ -309,7 +311,7 @@ bool Job::isInServerList(Node const& snap, std::string const& prefix, std::strin
       }
     }
   } else {  // an object
-    auto const& children  = snap.hasAsChildren(prefix);
+    auto const& children = snap.hasAsChildren(prefix);
     if (children.second) {
       for (auto const& srv : children.first) {
         if (srv.first == server) {
@@ -418,16 +420,15 @@ std::vector<Job::shard_t> Job::clones(Node const& snapshot, std::string const& d
 
   for (const auto& colptr : snapshot.hasAsChildren(databasePath).first) {  // collections
 
-    auto const &col = *colptr.second;
-    auto const &otherCollection = colptr.first;
+    auto const& col = *colptr.second;
+    auto const& otherCollection = colptr.first;
 
     if (otherCollection != collection && col.has("distributeShardsLike") &&  // use .has() form to prevent logging of missing
         col.hasAsSlice("distributeShardsLike").first.copyString() == collection) {
       auto const& theirshards = sortedShardList(col.hasAsNode("shards").first);
       if (theirshards.size() > 0) {  // do not care about virtual collections
         if (theirshards.size() == myshards.size()) {
-          ret.emplace_back(otherCollection,
-                           theirshards[steps]);
+          ret.emplace_back(otherCollection, theirshards[steps]);
         } else {
           LOG_TOPIC("3092e", ERR, Logger::SUPERVISION)
               << "Shard distribution of clone(" << otherCollection
@@ -452,25 +453,44 @@ std::string Job::findNonblockedCommonHealthyInSyncFollower(  // Which is in "GOO
 
   std::unordered_map<std::string, size_t> currentServers;
   for (const auto& clone : cs) {
-    auto currentShardPath = curColPrefix + db + "/" + clone.collection + "/" +
-                            clone.shard + "/servers";
-    auto plannedShardPath =
-        planColPrefix + db + "/" + clone.collection + "/shards/" + clone.shard;
-    size_t i = 0;
+    auto sharedPath = db + "/" + clone.collection + "/";
+    auto currentShardPath = curColPrefix + sharedPath + clone.shard + "/servers";
+    auto currentFailoverCandidatesPath =
+        curColPrefix + sharedPath + clone.shard + "/servers";
+    auto plannedShardPath = planColPrefix + sharedPath + "shards/" + clone.shard;
 
     // start up race condition  ... current might not have everything in plan
     if (!snap.has(currentShardPath) || !snap.has(plannedShardPath)) {
       --nclones;
       continue;
     }  // if
 
-    for (const auto& server :
-         VPackArrayIterator(snap.hasAsArray(currentShardPath).first)) {
-      auto id = server.copyString();
+    bool isArray = false;
+    VPackSlice serverList;
+    // If we do have failover candidates, we should use them
+    std::tie(serverList, isArray) = snap.hasAsArray(currentFailoverCandidatesPath);
+    if (!isArray) {
+      // We have old DBServers that do not report failover candidates,
+      // Need to rely on current
+      std::tie(serverList, isArray) = snap.hasAsArray(currentShardPath);
+      TRI_ASSERT(isArray);
+      if (!isArray) {
+        THROW_ARANGO_EXCEPTION_MESSAGE(
+            TRI_ERROR_SUPERVISION_GENERAL_FAILURE,
+            "Could not find common insync server for: " + currentShardPath +
+                ", value is not an array.");
+      }
+    }
+    // Guaranteed by if above
+    TRI_ASSERT(serverList.isArray());
+
+    size_t i = 0;
+    for (const auto& server : VPackArrayIterator(serverList)) {
       if (i++ == 0) {
         // Skip leader
         continue;
       }
+      auto id = server.copyString();
 
       if (!good[id]) {
         // Skip unhealthy servers
@@ -550,9 +570,9 @@ bool Job::abortable(Node const& snapshot, std::string const& jobId) {
   return false;
 }
 
-void Job::doForAllShards(Node const& snapshot, std::string& database,
-                         std::vector<shard_t>& shards,
-                         std::function<void(Slice plan, Slice current, std::string& planPath, std::string& curPath)> worker) {
+void Job::doForAllShards(
+    Node const& snapshot, std::string& database, std::vector<shard_t>& shards,
+    std::function<void(Slice plan, Slice current, std::string& planPath, std::string& curPath)> worker) {
   for (auto const& collShard : shards) {
     std::string shard = collShard.shard;
     std::string collection = collShard.collection;
 
@@ -49,9 +49,7 @@ class RestAqlHandler : public RestVocbaseBaseHandler {
 
  public:
   char const* name() const override final { return "RestAqlHandler"; }
-  RequestLane lane() const override final {
-    return RequestLane::CLUSTER_INTERNAL;
-  }
+  RequestLane lane() const override final { return RequestLane::CLUSTER_AQL; }
   RestStatus execute() override;
   RestStatus continueExecute() override;
 
 
@@ -28,26 +28,27 @@
 #include <velocypack/velocypack-aliases.h>
 
 arangodb::ClusterCollectionCreationInfo::ClusterCollectionCreationInfo(
-    std::string const cID, uint64_t shards, uint64_t repFac, bool waitForRep,
-    velocypack::Slice const& slice)
+    std::string const cID, uint64_t shards, uint64_t repFac, uint64_t minRepFac,
+    bool waitForRep, velocypack::Slice const& slice)
     : collectionID(std::move(cID)),
       numberOfShards(shards),
       replicationFactor(repFac),
+      minReplicationFactor(minRepFac),
       waitForReplication(waitForRep),
       json(slice),
       name(arangodb::basics::VelocyPackHelper::getStringValue(json, arangodb::StaticStrings::DataSourceName,
                                                               StaticStrings::Empty)),
       state(State::INIT) {
   if (numberOfShards == 0) {
-    // Nothing to do this cannot fail
-    // Deactivated this assertion, our testing mock for coordinator side
-    // tries to get away without other servers by initially adding only 0
-    // shard collections (non-smart). We do not want to loose these test.
-    // So we will loose this assertion for now.
-    /*
-    TRI_ASSERT(arangodb::basics::VelocyPackHelper::getBooleanValue(
-                                 json, arangodb::StaticStrings::IsSmart, false));
-    */
+// Nothing to do this cannot fail
+// Deactivated this assertion, our testing mock for coordinator side
+// tries to get away without other servers by initially adding only 0
+// shard collections (non-smart). We do not want to loose these test.
+// So we will loose this assertion for now.
+#ifndef ARANGODB_USE_GOOGLE_TESTS
+    TRI_ASSERT(arangodb::basics::VelocyPackHelper::getBooleanValue(json, arangodb::StaticStrings::IsSmart,
+                                                                   false));
+#endif
     state = State::DONE;
   }
   TRI_ASSERT(!name.empty());
@@ -68,10 +69,10 @@ VPackSlice arangodb::ClusterCollectionCreationInfo::isBuildingSlice() const {
 }
 
 bool arangodb::ClusterCollectionCreationInfo::needsBuildingFlag() const {
-    // Deactivated the smart graph check, our testing mock for coordinator side
-    // tries to get away without other servers by initially adding only 0
-    // shard collections (non-smart). We do not want to loose these test.
-    // So we will loose the more precise check for now.
+  // Deactivated the smart graph check, our testing mock for coordinator side
+  // tries to get away without other servers by initially adding only 0
+  // shard collections (non-smart). We do not want to loose these test.
+  // So we will loose the more precise check for now.
   /*
   return numberOfShards > 0 ||
          arangodb::basics::VelocyPackHelper::getBooleanValue(json, StaticStrings::IsSmart, false);
 
@@ -32,20 +32,22 @@ namespace arangodb {
 
 struct ClusterCollectionCreationInfo {
   enum State { INIT, FAILED, DONE };
-  ClusterCollectionCreationInfo(std::string const cID, uint64_t shards, uint64_t repFac,
+  ClusterCollectionCreationInfo(std::string const cID, uint64_t shards,
+                                uint64_t repFac, uint64_t minRepFac,
                                 bool waitForRep, velocypack::Slice const& slice);
 
   std::string const collectionID;
   uint64_t numberOfShards;
   uint64_t replicationFactor;
+  uint64_t minReplicationFactor;
   bool waitForReplication;
   velocypack::Slice const json;
   std::string name;
   State state;
 
  public:
   velocypack::Slice isBuildingSlice() const;
- 
+
  private:
   bool needsBuildingFlag() const;