arangodb · jsteemann · Sep 15, 2021 · Sep 9, 2021 · Sep 9, 2021 · Sep 9, 2021
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,6 +1,9 @@
 devel
 -----
 
+* Fix active failover, so that the new host actually has working
+  foxx services. (BTS-558)
+
 * Fixed issue #14720: Bulk import ignores onDuplicate in 3.8.0.
   The "onDuplicate" attribute was ignored by the `/_api/import` REST API when
   not specifying the "type" URL parameter.

diff --git a/arangod/Cluster/HeartbeatThread.cpp b/arangod/Cluster/HeartbeatThread.cpp
@@ -50,6 +50,7 @@
 #include "Replication/GlobalReplicationApplier.h"
 #include "Replication/ReplicationFeature.h"
 #include "RestServer/DatabaseFeature.h"
+#include "RestServer/SystemDatabaseFeature.h"
 #include "RestServer/TtlFeature.h"
 #include "Scheduler/Scheduler.h"
 #include "Scheduler/SchedulerFeature.h"
@@ -58,6 +59,7 @@
 #include "Transaction/ClusterUtils.h"
 #include "Utils/Events.h"
 #include "VocBase/vocbase.h"
+#include "V8Server/V8DealerFeature.h"
 
 using namespace arangodb;
 using namespace arangodb::application_features;
@@ -905,6 +907,10 @@ void HeartbeatThread::runSingleServer() {
         ServerState::instance()->setFoxxmaster(_myId);
         auto prv = ServerState::setServerMode(ServerState::Mode::DEFAULT);
         if (prv == ServerState::Mode::REDIRECT) {
+          auto& sysDbFeature = server().getFeature<arangodb::SystemDatabaseFeature>();
+          auto database = sysDbFeature.use();
+          server().getFeature<V8DealerFeature>().loadJavaScriptFileInAllContexts(
+            database.get(), "server/leader.js", nullptr);
           LOG_TOPIC("98325", INFO, Logger::HEARTBEAT)
               << "Successful leadership takeover: "
               << "All your base are belong to us";

diff --git a/js/server/leader.js b/js/server/leader.js
@@ -0,0 +1,36 @@
+'use strict';
+
+// //////////////////////////////////////////////////////////////////////////////
+// / @brief active failover leadership change
+// /
+// / @file
+// /
+// / DISCLAIMER
+// /
+// / Copyright 2014 ArangoDB GmbH, Cologne, Germany
+// / Copyright 2011-2014 triAGENS GmbH, Cologne, Germany
+// /
+// / Licensed under the Apache License, Version 2.0 (the "License")
+// / you may not use this file except in compliance with the License.
+// / You may obtain a copy of the License at
+// /
+// /     http://www.apache.org/licenses/LICENSE-2.0
+// /
+// / Unless required by applicable law or agreed to in writing, software
+// / distributed under the License is distributed on an "AS IS" BASIS,
+// / WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// / See the License for the specific language governing permissions and
+// / limitations under the License.
+// /
+// / Copyright holder is ArangoDB GmbH, Cologne, Germany
+// /
+// / @author Alan Plum
+// / @author Copyright 2021, ArangoDB GmbH, Cologne, Germany
+// //////////////////////////////////////////////////////////////////////////////
+
+(function(){
+  "use strict";
+  if (require("internal").threadNumber === 0) {
+    require("@arangodb/foxx/manager").healAll();
+  }
+}());
diff --git a/js/server/modules/@arangodb/foxx/queues/manager.js b/js/server/modules/@arangodb/foxx/queues/manager.js
@@ -226,11 +226,6 @@ exports.manage = function () {
   }
 
   if (global.ArangoServerState.getFoxxmasterQueueupdate()) {
-    if (!isCluster) {
-      // On a Foxxmaster change FoxxmasterQueueupdate is set to true
-      // we use this to signify a Leader change to this server
-      foxxManager.healAll(true);
-    }
     // do not call again immediately
     global.ArangoServerState.setFoxxmasterQueueupdate(false);
 

diff --git a/tests/js/client/active-failover/basic.js b/tests/js/client/active-failover/basic.js
@@ -35,10 +35,13 @@ const request = require("@arangodb/request");
 const tasks = require("@arangodb/tasks");
 
 const arango = internal.arango;
-const compareTicks = require("@arangodb/replication").compareTicks;
-const wait = internal.wait;
 const db = internal.db;
+const fs = require('fs');
+const path = require('path');
+const utils = require('@arangodb/foxx/manager-utils');
+const wait = internal.wait;
 
+const compareTicks = require("@arangodb/replication").compareTicks;
 const suspendExternal = internal.suspendExternal;
 const continueExternal = internal.continueExternal;
 
@@ -293,10 +296,136 @@ function waitUntilHealthStatusIs(isHealthy, isFailed) {
   return false;
 }
 
+function loadFoxxIntoZip(path) {
+  let zip = utils.zipDirectory(path);
+  let content = fs.readFileSync(zip);
+  fs.remove(zip);
+  return {
+    type: 'inlinezip',
+    buffer: content
+  };
+}
+function checkFoxxService(readOnly) {
+  const onlyJson = {
+    'accept': 'application/json',
+    'accept-content-type': 'application/json'
+  };
+  let reply;
+  db._useDatabase("_system");
+
+  [
+    '/_db/_system/_admin/aardvark/index.html',
+    '/_db/_system/itz/index',
+    '/_db/_system/crud/xxx'
+  ].forEach(route => {
+    for (let i=0; i < 200; i++) {
+      try {
+        reply = arango.GET_RAW(route, onlyJson);
+        if (reply.code === 200) {
+          print(route + " OK");
+          return;
+        }
+        let msg = JSON.stringify(reply);
+        if (reply.hasOwnProperty('parsedBody')) {
+          msg = " '" + reply.parsedBody.errorNum + "' - " + reply.parsedBody.errorMessage;
+        }
+        print(route + " Not yet ready, retrying: " + msg);
+      } catch (e) {
+        print(route + " Caught - need to retry. " + JSON.stringify(e));
+      }
+      internal.sleep(3);
+    }
+    throw ("foxx route '" + route + "' not ready on time!");
+  });
+
+  print("Foxx: Itzpapalotl getting the root of the gods");
+  reply = arango.GET_RAW('/_db/_system/itz');
+  assertEqual(reply.code, "307", JSON.stringify(reply));
+
+  print('Foxx: Itzpapalotl getting index html with list of gods');
+  reply = arango.GET_RAW('/_db/_system/itz/index');
+  assertEqual(reply.code, "200", JSON.stringify(reply));
+
+  print("Foxx: Itzpapalotl summoning Chalchihuitlicue");
+  reply = arango.GET_RAW('/_db/_system/itz/Chalchihuitlicue/summon', onlyJson);
+  assertEqual(reply.code, "200", JSON.stringify(reply));
+  let parsedBody = JSON.parse(reply.body);
+  assertEqual(parsedBody.name, "Chalchihuitlicue");
+  assertTrue(parsedBody.summoned);
+
+  print("Foxx: crud testing get xxx");
+  reply = arango.GET_RAW('/_db/_system/crud/xxx', onlyJson);
+  assertEqual(reply.code, "200");
+  parsedBody = JSON.parse(reply.body);
+  assertEqual(parsedBody, []);
+
+  print("Foxx: crud testing POST xxx");
+
+  reply = arango.POST_RAW('/_db/_system/crud/xxx', {_key: "test"});
+  if (readOnly) {
+    assertEqual(reply.code, "400");
+  } else {
+    assertEqual(reply.code, "201");
+  }
+
+  print("Foxx: crud testing get xxx");
+  reply = arango.GET_RAW('/_db/_system/crud/xxx', onlyJson);
+  assertEqual(reply.code, "200");
+  parsedBody = JSON.parse(reply.body);
+  if (readOnly) {
+    assertEqual(parsedBody, []);
+  } else {
+    assertEqual(parsedBody.length, 1);
+  }
+
+  print('Foxx: crud testing delete document');
+  reply = arango.DELETE_RAW('/_db/_system/crud/xxx/' + 'test');
+  if (readOnly) {
+    assertEqual(reply.code, "400");
+  } else {
+    assertEqual(reply.code, "204");
+  }
+}
+
+function installFoxx(mountpoint, which, mode) {
+  let headers = {};
+  let content;
+  if (which.type === 'js') {
+    headers['content-type'] = 'application/javascript';
+    content = which.buffer;
+  } else if (which.type === 'dir') {
+    headers['content-type'] = 'application/zip';
+    var utils = require('@arangodb/foxx/manager-utils');
+    let zip = utils.zipDirectory(which.buffer);
+    content = fs.readFileSync(zip);
+    fs.remove(zip);
+  } else if (which.type === 'inlinezip') {
+    content = which.buffer;
+    headers['content-type'] = 'application/zip';
+  } else if (which.type === 'url') {
+    content = { source: which };
+  } else if (which.type === 'file') {
+    content = fs.readFileSync(which.buffer);
+  }
+  let devmode = '';
+  if (typeof which.devmode === "boolean") {
+    devmode = `&development=${which.devmode}`;
+  }
+  let crudResp;
+  if (mode === "upgrade") {
+    crudResp = arango.PATCH('/_api/foxx/service?mount=' + mountpoint + devmode, content, headers);
+  } else if (mode === "replace") {
+    crudResp = arango.PUT('/_api/foxx/service?mount=' + mountpoint + devmode, content, headers);
+  } else {
+    crudResp = arango.POST('/_api/foxx?mount=' + mountpoint + devmode, content, headers);
+  }
+  expect(crudResp).to.have.property('manifest');
+  return crudResp;
+}
+
 // Testsuite that quickly checks some of the basic premises of
 // the active failover functionality. It is designed as a quicker
 // variant of the node resilience tests (for active failover).
-// Things like Foxx resilience are not tested
 function ActiveFailoverSuite() {
   let servers = getClusterEndpoints();
   assertTrue(servers.length >= 4, "This test expects four single instances");
@@ -370,37 +499,76 @@ function ActiveFailoverSuite() {
     // Simple failover case: Leader is suspended, slave needs to
     // take over within a reasonable amount of time
     testFailover: function () {
+      const itzpapalotlPath = path.resolve(internal.pathForTesting('common'), 'test-data', 'apps', 'itzpapalotl');
+      const itzpapalotlZip = loadFoxxIntoZip(itzpapalotlPath);
+      installFoxx("/itz", itzpapalotlZip);
+
+      const minimalWorkingServicePath = path.resolve(internal.pathForTesting('common'), 'test-data', 'apps', 'crud');
+      const minimalWorkingZip = loadFoxxIntoZip(minimalWorkingServicePath);
+      installFoxx('/crud', minimalWorkingZip);
 
+      checkFoxxService(false);
       assertTrue(checkInSync(currentLead, servers));
       assertEqual(checkData(currentLead), 10000);
 
-      suspended = instanceinfo.arangods.filter(arangod => arangod.endpoint === currentLead);
-      suspended.forEach(arangod => {
-        print("Suspending Leader: ", arangod.endpoint);
-        assertTrue(suspendExternal(arangod.pid));
-      });
-
+      let suspended;
       let oldLead = currentLead;
-      // await failover and check that follower get in sync
-      currentLead = checkForFailover(currentLead);
-      assertNotEqual(currentLead, oldLead);
-      print("Failover to new leader : ", currentLead);
-
-      internal.wait(5); // settle down, heartbeat interval is 1s
-      assertEqual(checkData(currentLead), 10000);
-      print("New leader has correct data");
-
-      // check the remaining followers get in sync
-      assertTrue(checkInSync(currentLead, servers, oldLead));
-
-      // restart the old leader
-      suspended.forEach(arangod => {
-        print("Resuming: ", arangod.endpoint);
-        assertTrue(continueExternal(arangod.pid));
-      });
-      suspended = [];
-
-      assertTrue(checkInSync(currentLead, servers));
+      try {
+        suspended = instanceinfo.arangods.filter(arangod => arangod.endpoint === currentLead);
+        suspended.forEach(arangod => {
+          print("Suspending Leader: ", arangod.endpoint);
+          assertTrue(suspendExternal(arangod.pid));
+        });
+
+        // await failover and check that follower get in sync
+        currentLead = checkForFailover(currentLead);
+        assertNotEqual(currentLead, oldLead);
+        print("Failover to new leader : ", currentLead);
+
+        internal.wait(5); // settle down, heartbeat interval is 1s
+        assertEqual(checkData(currentLead), 10000);
+        print("New leader has correct data");
+
+        // check the remaining followers get in sync
+        assertTrue(checkInSync(currentLead, servers, oldLead));
+
+        connectToServer(currentLead);
+        checkFoxxService(false);
+
+      } finally {
+        // restart the old leader
+        suspended.forEach(arangod => {
+          print("Resuming: ", arangod.endpoint);
+          assertTrue(continueExternal(arangod.pid));
+        });
+        assertTrue(checkInSync(currentLead, servers));
+        // after its in sync, halt all others so it becomes the leader again
+        suspended = instanceinfo.arangods.filter(arangod =>
+          (arangod.endpoint !== oldLead) && (arangod.role === 'single'));
+        suspended.forEach(arangod => {
+          print("Suspending all but old Leader: ", arangod.endpoint);
+          assertTrue(suspendExternal(arangod.pid));
+        });
+        currentLead = checkForFailover(currentLead);
+        assertEqual(currentLead, oldLead);
+        connectToServer(currentLead);
+        // restart the other followers so the system is all up and running again
+        suspended.forEach(arangod => {
+          print("Resuming: ", arangod.endpoint);
+          assertTrue(continueExternal(arangod.pid));
+        });
+        assertTrue(checkInSync(currentLead, servers));
+        let stati = [];
+        ["/itz", "/crud"].forEach(mount => {
+          try {
+            print("Uninstalling " + mount);
+            let res = arango.DELETE(
+              "/_db/_system/_admin/aardvark/foxxes?teardown=true&mount=" + mount);
+            stati.push(res.error);
+          } catch (e) {}
+        });
+        assertEqual(stati, [false, false]);
+      }
     },
 
     // More complex case: We want to get the most up to date follower

diff --git a/tests/js/common/test-data/apps/crud/README.md b/tests/js/common/test-data/apps/crud/README.md
@@ -0,0 +1,9 @@
+# xx
+
+xx
+
+# License
+
+Copyright (c) 2021 xx
+
+License:  xx
diff --git a/tests/js/common/test-data/apps/crud/main.js b/tests/js/common/test-data/apps/crud/main.js
@@ -0,0 +1,4 @@
+'use strict';
+
+module.context.use('/xxx', require('./routes/xxx'), 'xxx');
+module.context.use('/yyyy', require('./routes/yyyy'), 'yyyy');
diff --git a/tests/js/common/test-data/apps/crud/manifest.json b/tests/js/common/test-data/apps/crud/manifest.json
@@ -0,0 +1,16 @@
+{
+    "name": "xx",
+    "version": "0.0.0",
+    "description": "xx",
+    "engines": {
+        "arangodb": "^3.0.0"
+    },
+    "author": "xx",
+    "license": " xx",
+    "main": "main.js",
+    "scripts": {
+        "setup": "scripts/setup.js",
+        "teardown": "scripts/teardown.js"
+    },
+    "tests": "test/**/*.js"
+}
diff --git a/tests/js/common/test-data/apps/crud/models/xxx.js b/tests/js/common/test-data/apps/crud/models/xxx.js
@@ -0,0 +1,19 @@
+'use strict';
+const _ = require('lodash');
+const joi = require('joi');
+
+module.exports = {
+  schema: {
+    // Describe the attributes with joi here
+    _key: joi.string()
+  },
+  forClient(obj) {
+    // Implement outgoing transformations here
+    obj = _.omit(obj, ['_id', '_rev', '_oldRev']);
+    return obj;
+  },
+  fromClient(obj) {
+    // Implement incoming transformations here
+    return obj;
+  }
+};