arangodb · KVS85 · Sep 11, 2021 · Aug 26, 2021 · Aug 26, 2021 · Aug 26, 2021
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,6 +1,9 @@
 v3.8.2 (XXXX-XX-XX)
 -------------------
 
+* No runtime limits for shard move and server cleanout jobs, instead possibility
+  to cancel them.
+
 * (EE only) Bug-fix: If you created a ArangoSearch view on Satellite-
   Collections only and then join with a collection only having a single shard
   the cluster-one-shard-rule was falsely applied and could lead to empty view
@@ -48,7 +51,7 @@ v3.8.2 (XXXX-XX-XX)
   several clients concurrently inserts data and use custom analyzer with
   non-string return type.
 
-* Fix a rare shutdown race in RocksDBShaCalculatorThread. 
+* Fix a rare shutdown race in RocksDBShaCalculatorThread.
 
 * Reduce internal priority of AQL execution. This prevents possible deadlocks
   
8000
with modification operations in a cluster and replicationFactor >= 2, and can

diff --git a/arangod/Agency/CleanOutServer.cpp b/arangod/Agency/CleanOutServer.cpp
@@ -82,17 +82,8 @@ JOB_STATUS CleanOutServer::status() {
   }
 
   if (found > 0) {  // some subjob still running
-    // timeout here:
-    auto tmp_time =
-        _snapshot.hasAsString(pendingPrefix + _jobId + "/timeCreated");
-    std::string timeCreatedString = tmp_time.first;
-    Supervision::TimePoint timeCreated = stringToTimepoint(timeCreatedString);
-    Supervision::TimePoint now(std::chrono::system_clock::now());
-    if (now - timeCreated > std::chrono::duration<double>(86400.0)) { // 1 day
-      abort("job timed out");
-      return FAILED;
-    }
-    return PENDING;
+    // consider cancellation
+    return considerCancellation() ? FAILED : PENDING;
   }
 
   Node::Children const& failed = _snapshot.hasAsChildren(failedPrefix).first;
@@ -198,6 +189,11 @@ bool CleanOutServer::create(std::shared_ptr<VPackBuilder> envelope) {
 }
 
 bool CleanOutServer::start(bool& aborts) {
+
+  if (considerCancellation()) {
+    return false;
+  }
+
   // If anything throws here, the run() method catches it and finishes
   // the job.
 

diff --git a/arangod/Agency/Job.cpp b/arangod/Agency/Job.cpp
@@ -87,6 +87,17 @@ Job::~Job() = default;
 // this will be initialized in the AgencyFeature
 std::string Job::agencyPrefix = "arango";
 
+bool Job::considerCancellation() {
+  // Allow for cancellation of shard moves
+  auto [cancel,exists] = 
+    _snapshot.hasAsBool(std::string("/Target/") + jobStatus[_status] + "/" + _jobId + "/abort");
+  auto cancelled = exists && cancel;
+  if (cancelled) {
+    abort("Killed via API");
+  }
+  return cancelled;
+};
+
 bool Job::finish(std::string const& server, std::string const& shard,
                  bool success, std::string const& reason, query_t const payload) {
   try {  // protect everything, just in case

diff --git a/arangod/Agency/Job.h b/arangod/Agency/Job.h
@@ -49,6 +49,7 @@ namespace consensus {
 class Node;
 
 enum JOB_STATUS { TODO, PENDING, FINISHED, FAILED, NOTFOUND };
+const std::vector<std::string> jobStatus {"ToDo", "Pending", "Finished", "Failed"};
 const std::vector<std::string> pos({"/Target/ToDo/", "/Target/Pending/",
                                     "/Target/Finished/", "/Target/Failed/"});
 extern std::string const mapUniqueToShortID;
@@ -88,6 +89,8 @@ struct Job {
 
   virtual void run(bool& aborts) = 0;
 
+  bool considerCancellation();
+
   void runHelper(std::string const& server, std::string const& shard, bool& aborts) {
     if (_status == FAILED) {  // happens when the constructor did not work
       return;

diff --git a/arangod/Agency/MoveShard.cpp b/arangod/Agency/MoveShard.cpp
@@ -181,6 +181,11 @@ bool MoveShard::create(std::shared_ptr<VPackBuilder> envelope) {
 }
 
 bool MoveShard::start(bool&) {
+
+  if (considerCancellation()) {
+    return false;
+  }
+
   // If anything throws here, the run() method catches it and finishes
   // the job.
 
@@ -477,6 +482,13 @@ bool MoveShard::start(bool&) {
 }
 
 JOB_STATUS MoveShard::status() {
+
+  if (_status == PENDING || _status == TODO) {
+    if (considerCancellation()) {
+      return FAILED;
+    }
+  }
+
   if (_status != PENDING) {
     return _status;
   }
@@ -495,19 +507,6 @@ JOB_STATUS MoveShard::status() {
 
 JOB_STATUS MoveShard::pendingLeader() {
 
-  auto considerTimeout = [&]() -> bool {
-    // Not yet all in sync, consider timeout:
-    std::string timeCreatedString =
-        _snapshot.hasAsString(pendingPrefix + _jobId + "/timeCreated").first;
-    Supervision::TimePoint timeCreated = stringToTimepoint(timeCreatedString);
-    Supervision::TimePoint now(std::chrono::system_clock::now());
-    if (now - timeCreated > std::chrono::duration<double>(43200.0)) {  // 12h
-      abort("MoveShard timed out in pending leader");
-      return true;
-    }
-    return false;
-  };
-
   // Find the other shards in the same distributeShardsLike group:
   std::vector<Job::shard_t> shardsLikeMe =
       clones(_snapshot, _database, _collection, _shard);
@@ -556,9 +555,9 @@ JOB_STATUS MoveShard::pendingLeader() {
                      }
                    });
 
-    // Consider timeout:
+    // Consider cancellation:
     if (done < shardsLikeMe.size()) {
-      if (considerTimeout()) {
+      if (considerCancellation()) {
         return FAILED;
       }
       return PENDING;  // do not act
@@ -623,9 +622,9 @@ JOB_STATUS MoveShard::pendingLeader() {
                      }
                    });
 
-    // Consider timeout:
+    // Consider cancellation:
     if (done < shardsLikeMe.size()) {
-      if (considerTimeout()) {
+      if (considerCancellation()) {
         return FAILED;
       }
       return PENDING;  // do not act!
@@ -752,9 +751,9 @@ JOB_STATUS MoveShard::pendingLeader() {
                      }
                    });
 
-    // Consider timeout:
+    // Consider cancellation:
     if (done < shardsLikeMe.size()) {
-      if (considerTimeout()) {
+      if (considerCancellation()) {
         return FAILED;
       }
       return PENDING;  // do not act!
@@ -841,6 +840,7 @@ JOB_STATUS MoveShard::pendingLeader() {
 }
 
 JOB_STATUS MoveShard::pendingFollower() {
+
   // Check if any of the servers in the Plan are FAILED, if so,
   // we abort:
   std::string planPath =
@@ -867,14 +867,8 @@ JOB_STATUS MoveShard::pendingFollower() {
                    }
                  });
 
-  if (done < shardsLikeMe.size()) {
-    // Not yet all in sync, consider timeout:
-    std::string timeCreatedString =
-        _snapshot.hasAsString(pendingPrefix + _jobId + "/timeCreated").first;
-    Supervision::TimePoint timeCreated = stringToTimepoint(timeCreatedString);
-    Supervision::TimePoint now(std::chrono::system_clock::now());
-    if (now - timeCreated > std::chrono::duration<double>(10000.0)) {
-      abort("MoveShard timed out in pending follower");
+  if (done < shardsLikeMe.size()) { // Consider cancellation
+    if (considerCancellation()) {
       return FAILED;
     }
     return PENDING;
@@ -981,7 +975,7 @@ arangodb::Result MoveShard::abort(std::string const& reason) {
       }
     }
 
-    if (finish("", "", true, "job aborted (1): " + reason, todoPrec)) {
+    if (finish("", "", false, "job aborted (1): " + reason, todoPrec)) {
       return result;
     }
     _status = PENDING;