arangodb
diff --git a/‎3rdParty/V8/v7.9.317/src/codegen/source-position.h
Lines changed: 8 additions & 8 deletions b/‎3rdParty/V8/v7.9.317/src/codegen/source-position.h
Lines changed: 8 additions & 8 deletions
diff --git a/‎CHANGELOG
Lines changed: 9 additions & 1 deletion b/‎CHANGELOG
Lines changed: 9 additions & 1 deletion
diff --git a/‎VERSIONS
Lines changed: 1 addition & 1 deletion b/‎VERSIONS
Lines changed: 1 addition & 1 deletion
diff --git a/‎arangod/Agency/Job.cpp
Lines changed: 1 addition & 1 deletion b/‎arangod/Agency/Job.cpp
Lines changed: 1 addition & 1 deletion
diff --git a/‎arangod/Agency/Supervision.cpp
Lines changed: 69 additions & 26 deletions b/‎arangod/Agency/Supervision.cpp
Lines changed: 69 additions & 26 deletions
diff --git a/‎arangod/Aql/AqlFunctionFeature.cpp
Lines changed: 5 additions & 0 deletions b/‎arangod/Aql/AqlFunctionFeature.cpp
Lines changed: 5 additions & 0 deletions
diff --git a/‎arangod/Aql/AqlItemMatrix.cpp
Lines changed: 1 addition & 1 deletion b/‎arangod/Aql/AqlItemMatrix.cpp
Lines changed: 1 addition & 1 deletion
diff --git a/‎arangod/Aql/Expression.cpp
Lines changed: 32 additions & 9 deletions b/‎arangod/Aql/Expression.cpp
Lines changed: 32 additions & 9 deletions
@@ -146,23 +146,23 @@ class SourcePosition final {
 
   // The two below are only used if IsExternal() is true.
   using ExternalLineField = BitField64<int, 1, 20>;
-#ifdef DEBUG
+//#ifdef DEBUG
 // looks like debug build uses more external files
 // so extend bits for this field, but play safe for release builds 
   using ExternalFileIdField = BitField64<int, 21, 16>;
-#else
-  using ExternalFileIdField = BitField64<int, 21, 10>;
-#endif
+//#else
+//  using ExternalFileIdField = BitField64<int, 21, 10>;
+//#endif
   // ScriptOffsetField is only used if IsExternal() is false.
   using ScriptOffsetField = BitField64<int, 1, 30>;
 
   // InliningId is in the high bits for better compression in
   // SourcePositionTable.
-#ifdef DEBUG // also shift this field for above change  for ExternalFileIdField
+//#ifdef DEBUG // also shift this field for above change  for ExternalFileIdField
   using InliningIdField = BitField64<int, 37, 16>;
-#else
-  using InliningIdField = BitField64<int, 31, 16>;
-#endif
+//#else
+//  using InliningIdField = BitField64<int, 31, 16>;
+//#endif
   // Leaving the highest bit untouched to allow for signed conversion.
   uint64_t value_;
 };
 
@@ -1,11 +1,19 @@
 devel
 -----
 
+* Fixed a bug in the agency supervision, which ignored the `failoverCandidates`
+  field.
+
+* Fixed a bug in the agency supervision, which could declare an already FAILED
+  dbserver temporarily as GOOD again after an agency leader change.
+
+* Added INTERLEAVE AQL function.
+
 * Upgraded bundled RocksDB library to version 6.8.0.
 
 * Added AQL function `IN_RANGE`.
 
-* Added startup option `--ssl.prefer-http1-in-alpn` to optionally let the 
+* Added startup option `--ssl.prefer-http1-in-alpn` to optionally let the
   server prefer HTTP/1.1 over HTTP/2 in ALPN protocol negotiations.
 
 * Compilation issues with wrong cv-qualifiers and unnecessary
 
@@ -2,7 +2,7 @@ CXX_STANDARD "17"
 STARTER_REV "0.14.14"
 SYNCER_REV "0.7.3"
 GCC_LINUX "9.2.0"
-MSVC_WINDOWS "2017"
+MSVC_WINDOWS "2019"
 MACOS_MIN "10.14"
 OPENSSL_LINUX   "1.1.1d"
 OPENSSL_MACOS   "1.1.1d"
 
@@ -471,7 +471,7 @@ std::string Job::findNonblockedCommonHealthyInSyncFollower(  // Which is in "GOO
     auto sharedPath = db + "/" + clone.collection + "/";
     auto currentShardPath = curColPrefix + sharedPath + clone.shard + "/servers";
     auto currentFailoverCandidatesPath =
-        curColPrefix + sharedPath + clone.shard + "/servers";
+        curColPrefix + sharedPath + clone.shard + "/failoverCandidates";
     auto plannedShardPath = planColPrefix + sharedPath + "shards/" + clone.shard;
 
     // start up race condition  ... current might not have everything in plan
 
@@ -588,30 +588,66 @@ std::vector<check_t> Supervision::check(std::string const& type) {
       HealthRecord persist(shortName, endpoint, hostId, engine, serverVersion, externalEndpoint);
 
       // Get last health entries from transient and persistent key value stores
+      bool transientHealthRecordFound = true;
       if (_transient.has(healthPrefix + serverID)) {
         transist = _transient.hasAsNode(healthPrefix + serverID).first;
+      } else {
+        // In this case this is the first time we look at this server during our
+        // new leadership. So we do not touch the persisted health record and
+        // only create a new transient health record.
+        transientHealthRecordFound = false;
       }
       if (snapshot().has(healthPrefix + serverID)) {
         persist = snapshot().hasAsNode(healthPrefix + serverID).first;
       }
 
+      // Here is an important subtlety: We will derive the health status of this
+      // server a bit further down by looking at the time when we saw the last
+      // heartbeat. The heartbeat is stored in transient in `Sync/ServerStates`.
+      // It has a timestamp, however, this was taken on the other server, so we
+      // cannot trust this time stamp because of possible clock skew. Since we
+      // do not actually know when this heartbeat came in, we have to proceed as
+      // follows: We make a copy of the `syncTime` and store it into the health
+      // record in transient `Supervision/Health`. If we detect a difference, it
+      // is the first time we have seen the new heartbeat and we can then take
+      // a reading of our local system clock and use that time. However, if we
+      // do not yet have a previous reading in our transient health record,
+      // we must not touch the persisted health record at all. This is what the
+      // flag `transientHealthRecordFound` means.
+
       // New health record (start with old add current information from sync)
       // Sync.time is copied to Health.syncTime
       // Sync.status is copied to Health.syncStatus
-      std::string syncTime =
-          _transient.has(syncPrefix + serverID)
-              ? _transient.hasAsString(syncPrefix + serverID + "/time").first
-              : timepointToString(std::chrono::system_clock::time_point());
-      std::string syncStatus =
-          _transient.has(syncPrefix + serverID)
-              ? _transient.hasAsString(syncPrefix + serverID + "/status").first
-              : "UNKNOWN";
-
-      // Last change registered in sync (transient != sync)
-      // Either now or value in transient
-      auto lastAckedTime = (syncTime != transist.syncTime)
-                               ? startTimeLoop
-                               : stringToTimepoint(transist.lastAcked);
+      std::string syncTime;
+      std::string syncStatus;
+      bool heartBeatVisible = _transient.has(syncPrefix + serverID);
+      if (heartBeatVisible) {
+        syncTime = _transient.hasAsString(syncPrefix + serverID + "/time").first;
+        syncStatus = _transient.hasAsString(syncPrefix + serverID + "/status").first;
+      } else {
+        syncTime = timepointToString(std::chrono::system_clock::time_point()); // beginning of time
+        syncStatus = "UNKNOWN";
+      }
+
+      // Compute the time when we last discovered a new heartbeat from that server:
+      std::chrono::system_clock::time_point lastAckedTime;
+      if (heartBeatVisible) {
+        if (transientHealthRecordFound) {
+          lastAckedTime = (syncTime != transist.syncTime)
+                          ? startTimeLoop
+                          : stringToTimepoint(transist.lastAcked);
+        } else {
+          // in this case we do no really know when this heartbeat came in,
+          // however, it must have been after we became a leader, and since we
+          // do not have a transient health record yet, we just assume that we
+          // got it recently and set it to "now". Note that this will not make
+          // a FAILED server "GOOD" again, since we do not touch the persisted
+          // health record if we do not have a transient health record yet.
+          lastAckedTime = startTimeLoop;
+        }
+      } else {
+        lastAckedTime = std::chrono::system_clock::time_point();
+      }
       transist.lastAcked = timepointToString(lastAckedTime);
       transist.syncTime = syncTime;
       transist.syncStatus = syncStatus;
@@ -623,34 +659,41 @@ std::vector<check_t> Supervision::check(std::string const& type) {
       transist.hostId = hostId;
       transist.endpoint = endpoint;
 
-      // Calculate elapsed since lastAcked
-      auto elapsed = std::chrono::duration<double>(startTimeLoop - lastAckedTime);
+      // We have now computed a new transient health record under all circumstances.
+
+      bool changed;
+      if (transientHealthRecordFound) {
+        // Calculate elapsed since lastAcked
+        auto elapsed = std::chrono::duration<double>(startTimeLoop - lastAckedTime);
 
-      if (elapsed.count() <= _okThreshold) {
-        transist.status = Supervision::HEALTH_STATUS_GOOD;
-      } else if (elapsed.count() <= _gracePeriod) {
-        transist.status = Supervision::HEALTH_STATUS_BAD;
+        if (elapsed.count() <= _okThreshold) {
+          transist.status = Supervision::HEALTH_STATUS_GOOD;
+        } else if (elapsed.count() <= _gracePeriod) {
+          transist.status = Supervision::HEALTH_STATUS_BAD;
+        } else {
+          transist.status = Supervision::HEALTH_STATUS_FAILED;
+        }
+
+        // Status changed?
+        changed = transist.statusDiff(persist);
       } else {
-        transist.status = Supervision::HEALTH_STATUS_FAILED;
+        transist.status = persist.status;
+        changed = false;
       }
 
-      // Status changed?
-      bool changed = transist.statusDiff(persist);
-
       // Take necessary actions if any
       std::shared_ptr<VPackBuilder> envelope;
       if (changed) {
         LOG_TOPIC("bbbde", DEBUG, Logger::SUPERVISION)
             << "Status of server " << serverID << " has changed from "
             << persist.status << " to " << transist.status;
         handleOnStatus(_agent, snapshot(), persist, transist, serverID, _jobId, envelope);
+        persist = transist;  // Now copy Status, SyncStatus from transient to persited
       } else {
         LOG_TOPIC("44253", TRACE, Logger::SUPERVISION)
             << "Health of server " << machine.first << " remains " << transist.status;
       }
 
-      persist = transist;  // Now copy Status, SyncStatus from transient to persited
-
       // Transient report
       std::shared_ptr<Builder> tReport = std::make_shared<Builder>();
       {
 
@@ -316,6 +316,7 @@ void AqlFunctionFeature::addListFunctions() {
   add({"REMOVE_VALUES", ".,.", flags, &Functions::RemoveValues});
   add({"REMOVE_NTH", ".,.", flags, &Functions::RemoveNth});
   add({"REPLACE_NTH", ".,.,.|.", flags, &Functions::ReplaceNth});
+  add({"INTERLEAVE", ".,.|+", flags, &Functions::Interleave});
 
   // special flags:
   // CALL and APPLY will always run on the coordinator and are not deterministic
@@ -435,6 +436,10 @@ void AqlFunctionFeature::addMiscFunctions() {
   add({"DECODE_REV", ".", flags, &Functions::DecodeRev});
   add({"V8", ".", Function::makeFlags(FF::Deterministic, FF::Cacheable)});  // only function without a
                                                                             // C++ implementation
+                                                                            //
+  auto validationFlags = Function::makeFlags(FF::None);
+  add({"SCHEMA_GET", ".", validationFlags, &Functions::SchemaGet});
+  add({"SCHEMA_VALIDATE", ".,.", validationFlags, &Functions::SchemaValidate});
 
   // special flags:
   add({"VERSION", "", Function::makeFlags(FF::Deterministic), &Functions::Version});  // deterministic, not cacheable. only on
 
@@ -151,7 +151,7 @@ ShadowAqlItemRow AqlItemMatrix::popShadowRow() {
     TRI_ASSERT(blockPtr->size() >= _startIndexInFirstBlock);
     _size = blockPtr->size() - _startIndexInFirstBlock;
   }
-  // Remove all but the last
+  // Remove all but the last block
   _blocks.erase(_blocks.begin(), _blocks.end() - 1);
   TRI_ASSERT(_blocks.size() == 1);
   return shadowRow;
 
@@ -43,6 +43,7 @@
 #include "Basics/StringBuffer.h"
 #include "Basics/VPackStringBufferAdapter.h"
 #include "Basics/VelocyPackHelper.h"
+#include "Transaction/Context.h"
 #include "Transaction/Helpers.h"
 #include "Transaction/Methods.h"
 #include "V8/v8-globals.h"
@@ -261,18 +262,40 @@ bool Expression::findInArray(AqlValue const& left, AqlValue const& right,
     }
     // fall-through to linear search
   }
-
+  
   // use linear search
-  for (size_t i = 0; i < n; ++i) {
-    bool mustDestroy;
-    AqlValue a = right.at(i, mustDestroy, false);
-    AqlValueGuard guard(a, mustDestroy);
 
-    int compareResult = AqlValue::Compare(trx, left, a, false);
+  if (!right.isDocvec() && !right.isRange() &&
+      !left.isDocvec() && !left.isRange()) {
+    // optimization for the case in which rhs is a Velocypack array, and we 
+    // can simply use a VelocyPack iterator to walk through it. this will
+    // be a lot more efficient than using right.at(i) in case the array is
+    // of type Compact (without any index tables)
+    VPackSlice const lhs(left.slice());
+
+    VPackOptions const* options = trx->transactionContextPtr()->getVPackOptions();
+    VPackArrayIterator it(right.slice());
+    while (it.valid()) {
+      int compareResult = arangodb::basics::VelocyPackHelper::compare(lhs, it.value(), false, options);
+    
+      if (compareResult == 0) {
+        // item found in the list
+        return true;
+      }
+      it.next();
+    }
+  } else {
+    for (size_t i = 0; i < n; ++i) {
+      bool mustDestroy;
+      AqlValue a = right.at(i, mustDestroy, false);
+      AqlValueGuard guard(a, mustDestroy);
+
+      int compareResult = AqlValue::Compare(trx, left, a, false);
 
-    if (compareResult == 0) {
-      // item found in the list
-      return true;
+      if (compareResult == 0) {
+        // item found in the list
+        return true;
+      }
     }
   }
Original file line number	Diff line number	Diff line change
`@@ -151,7 +151,7 @@ ShadowAqlItemRow AqlItemMatrix::popShadowRow() {`
`151`	`151`	`TRI_ASSERT(blockPtr->size() >= _startIndexInFirstBlock);`
`152`	`152`	`_size = blockPtr->size() - _startIndexInFirstBlock;`
`153`	`153`	`}`
`154`		`- // Remove all but the last`
	`154`	`+ // Remove all but the last block`
`155`	`155`	`_blocks.erase(_blocks.begin(), _blocks.end() - 1);`
`156`	`156`	`TRI_ASSERT(_blocks.size() == 1);`
`157`	`157`	`return shadowRow;`