[3.7] BTS-446 Cluster machines fail to integrate on OSX jenkins runs (#14282)

kvahed · jsteemann · KVS85 · web-flow · commit b2adf44b78bb · 2021-06-20T00:36:55.000+03:00
* give the bootstrap process a little time

* Update CHANGELOG

Co-authored-by: Jan &lt;jsteemann@users.noreply.github.com&gt;

* Update CHANGELOG

Co-authored-by: Jan &lt;jsteemann@users.noreply.github.com&gt;
Co-authored-by: Vadim &lt;vadim@arangodb.com&gt;
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,6 +1,9 @@
 v3.7.13 (XXXX-XX-XX)
 --------------------
 
+* Fix BTS-446: When finding a not yet fully initialized agency, do not
+  immediately fatal exit. Keep trying for (very generous) 5 minutes.
+
 * Fix DEVSUP-753: now it is safe to call visit on exhausted disjunction
   iterator.
 
diff --git a/arangod/Cluster/ServerState.cpp b/arangod/Cluster/ServerState.cpp
@@ -664,10 +664,25 @@ bool ServerState::checkIfAgencyInitialized(AgencyComm& comm,
 //////////////////////////////////////////////////////////////////////////////
 
 bool ServerState::registerAtAgencyPhase1(AgencyComm& comm, ServerState::RoleEnum const& role) {
-  // if the agency is not initialized, there is no point in continuing.
-  if (!checkIfAgencyInitialized(comm, role)) {
-    return false;
-  }
+
+  // if the agency is not initialized, we'll give the bunch a little time to get their
+  // act together before calling it a day.
+
+  using namespace std::chrono;
+  using clock = steady_clock;
+  auto registrationTimeout = clock::now() + seconds(300);
+  auto backoff = milliseconds(75);
+  do {
+    if (checkIfAgencyInitialized(comm, role)) {
+      break;
+    } else if (clock::now() >= registrationTimeout) {
+      return false;
+    }
+    std::this_thread::sleep_for(backoff);
+    if (backoff < seconds(1)) {
+      backoff += backoff;
+    }
+  } while (!_server.isStopping());
 
   std::string const agencyListKey = roleToAgencyListKey(role);
   std::string const latestIdKey = "Latest" + roleToAgencyKey(role) + "Id";
