arangodb · KVS85 · Nov 19, 2019 · Nov 19, 2019 · Nov 19, 2019
diff --git a/arangod/IResearch/IResearchCommon.cpp b/arangod/IResearch/IResearchCommon.cpp
@@ -52,10 +52,12 @@ arangodb::LogTopic& logTopic() {
 /*static*/ std::string const StaticStrings::LinksField("links");
 /*static*/ std::string const StaticStrings::VersionField("version");
 /*static*/ std::string const StaticStrings::ViewIdField("view");
+/*static*/ std::string const StaticStrings::AnalyzerDefinitionsField("analyzerDefinitions");
 /*static*/ std::string const StaticStrings::AnalyzerFeaturesField("features");
 /*static*/ std::string const StaticStrings::AnalyzerNameField("name");
 /*static*/ std::string const StaticStrings::AnalyzerPropertiesField("properties");
 /*static*/ std::string const StaticStrings::AnalyzerTypeField("type");
+/*static*/ std::string const StaticStrings::PrimarySortField("primarySort");
 
 }  // namespace iresearch
 }  // namespace arangodb

diff --git a/arangod/IResearch/IResearchCommon.h b/arangod/IResearch/IResearchCommon.h
@@ -61,6 +61,12 @@ struct StaticStrings {
   ////////////////////////////////////////////////////////////////////////////////
   static std::string const ViewIdField;
 
+  ////////////////////////////////////////////////////////////////////////////////
+  /// @brief the name of the field in the IResearch Link definition denoting the
+  ///        referenced analyzer definitions
+  ////////////////////////////////////////////////////////////////////////////////
+  static std::string const AnalyzerDefinitionsField;
+
   ////////////////////////////////////////////////////////////////////////////////
   /// @brief the name of the field in the analyzer definition denoting the
   ///        corresponding analyzer name
@@ -84,6 +90,12 @@ struct StaticStrings {
   ///        corresponding analyzer features
   ////////////////////////////////////////////////////////////////////////////////
   static std::string const AnalyzerFeaturesField;
+
+  ////////////////////////////////////////////////////////////////////////////////
+  /// @brief the name of the field in the IResearch Link definition denoting the
+  ///        primary sort
+  ////////////////////////////////////////////////////////////////////////////////
+  static std::string const PrimarySortField;
 };
 
 }  // namespace iresearch

diff --git a/arangod/IResearch/IResearchLinkHelper.cpp b/arangod/IResearch/IResearchLinkHelper.cpp
@@ -783,7 +783,7 @@ namespace iresearch {
     std::string errorField;
 
     if (!linkDefinition.isNull()) { // have link definition
-      if (!meta.init(linkDefinition, false, errorField, &vocbase)) { // for db-server analyzer validation should have already applied on coordinator
+      if (!meta.init(linkDefinition, true, errorField, &vocbase)) { // for db-server analyzer validation should have already applied on coordinator
         return arangodb::Result( // result
           TRI_ERROR_BAD_PARAMETER, // code
           errorField.empty()

diff --git a/arangod/IResearch/IResearchView.cpp b/arangod/IResearch/IResearchView.cpp
@@ -363,12 +363,11 @@ arangodb::Result IResearchView::appendVelocyPackImpl(  // append JSON
   static const std::function<bool(irs::string_ref const& key)> persistenceAcceptor =
     [](irs::string_ref const&) -> bool { return true; };
 
-  auto& acceptor = 
-    (context == Serialization::Persistence || context == Serialization::PersistenceWithInProgress || context == Serialization::Inventory)
-    ? persistenceAcceptor
-    : propertiesAcceptor;
+  auto* acceptor = &propertiesAcceptor;
+
+  if (context == Serialization::Persistence || context == Serialization::PersistenceWithInProgress) {
+    acceptor = &persistenceAcceptor;
 
-    if (context == Serialization::Persistence || context == Serialization::PersistenceWithInProgress) {
     if (arangodb::ServerState::instance()->isSingleServer()) {
       auto res = arangodb::LogicalViewHelperStorageEngine::properties(builder, *this);
 
@@ -392,19 +391,14 @@ arangodb::Result IResearchView::appendVelocyPackImpl(  // append JSON
     sanitizedBuilder.openObject();
 
     if (!_meta.json(sanitizedBuilder) ||
-        !mergeSliceSkipKeys(builder, sanitizedBuilder.close().slice(), acceptor)) {
+        !mergeSliceSkipKeys(builder, sanitizedBuilder.close().slice(), *acceptor)) {
       return arangodb::Result(
           TRI_ERROR_INTERNAL,
           std::string("failure to generate definition while generating "
                       "properties jSON for arangosearch View in database '") +
               vocbase().name() + "'");
     }
 
-    if (context == Serialization::Inventory) {
-      // nothing more to output
-      return {};
-    }
-
     if (context == Serialization::Persistence || context == Serialization::PersistenceWithInProgress) {
       IResearchViewMetaState metaState;
 
@@ -461,7 +455,7 @@ arangodb::Result IResearchView::appendVelocyPackImpl(  // append JSON
       );
     }
 
-    auto visitor = [this, &linksBuilder, &res]( // visit collections
+    auto visitor = [this, &linksBuilder, &res, context]( // visit collections
       arangodb::TransactionCollection& trxCollection // transaction collection
     )->bool {
       auto collection = trxCollection.collection();
@@ -480,7 +474,7 @@ arangodb::Result IResearchView::appendVelocyPackImpl(  // append JSON
 
       linkBuilder.openObject();
 
-      if (!link->properties(linkBuilder, false).ok()) { // link definitions are not output if forPersistence
+      if (!link->properties(linkBuilder, Serialization::Inventory == context).ok()) { // link definitions are not output if forPersistence
         LOG_TOPIC("713ad", WARN, arangodb::iresearch::TOPIC)
           << "failed to generate json for arangosearch link '" << link->id() << "' while generating json for arangosearch view '" << name() << "'";
 

diff --git a/arangod/IResearch/IResearchViewCoordinator.cpp b/arangod/IResearch/IResearchViewCoordinator.cpp
@@ -181,28 +181,34 @@ arangodb::Result IResearchViewCoordinator::appendVelocyPackImpl(
     return {};
   }
 
-  static const std::function<bool(irs::string_ref const& key)> propertiesAcceptor =
+  static const std::function<bool(irs::string_ref const&)> propertiesAcceptor =
       [](irs::string_ref const& key) -> bool {
     return key != StaticStrings::VersionField; // ignored fields
   };
-  static const std::function<bool(irs::string_ref const& key)> persistenceAcceptor =
+  static const std::function<bool(irs::string_ref const&)> persistenceAcceptor =
     [](irs::string_ref const&) -> bool { return true; };
 
+  static const std::function<bool(irs::string_ref const&)> linkPropertiesAcceptor =
+    [](irs::string_ref const& key) -> bool {
+      return key != iresearch::StaticStrings::AnalyzerDefinitionsField
+          && key != iresearch::StaticStrings::PrimarySortField;
+  };
+
   auto* acceptor = &propertiesAcceptor;
 
   if (context == Serialization::Persistence || 
-      context == Serialization::PersistenceWithInProgress ||
-      context == Serialization::Inventory) {
+      context == Serialization::PersistenceWithInProgress) {
     auto res = arangodb::LogicalViewHelperClusterInfo::properties(builder, *this);
 
     if (!res.ok()) {
       return res;
     }
 
     acceptor = &persistenceAcceptor;
-    // links are not persisted, their definitions are part of the corresponding
-    // collections
-  } else if (context == Serialization::Properties) {
+  }
+
+  if (context == Serialization::Properties ||
+      context == Serialization::Inventory) {
     // verify that the current user has access on all linked collections
     auto* exec = ExecContext::CURRENT;
     if (exec) {
@@ -213,18 +219,31 @@ arangodb::Result IResearchViewCoordinator::appendVelocyPackImpl(
       }
     }
 
+    VPackBuilder tmp;
+
     ReadMutex mutex(_mutex);
     SCOPED_LOCK(mutex);  // '_collections' can be asynchronously modified
 
-    VPackBuilder links;
-    links.openObject();
-
+    builder.add(StaticStrings::LinksField, VPackValue(VPackValueType::Object));
     for (auto& entry : _collections) {
-      links.add(entry.second.first, entry.second.second.slice());
-    }
+      auto linkSlice = entry.second.second.slice();
+
+      if (context == Serialization::Properties) {
+        tmp.clear();
+        tmp.openObject();
+        if (!mergeSliceSkipKeys(tmp, linkSlice, linkPropertiesAcceptor)) {
+          return {
+            TRI_ERROR_INTERNAL,
+            "failed to generate externally visible link definition for arangosearch View '" + name() + "'"
+          };
+        }
 
-    links.close();
-    builder.add(StaticStrings::LinksField, links.slice());
+        linkSlice = tmp.close().slice();
+      }
+
+      builder.add(entry.second.first, linkSlice);
+    }
+    builder.close();
   }
 
   if (!builder.isOpenObject()) {
@@ -266,7 +285,7 @@ arangodb::Result IResearchViewCoordinator::link(IResearchLink const& link) {
 
   builder.openObject();
 
-  auto res = link.properties(builder, false); // generate user-visible definition, agency will not see links
+  auto res = link.properties(builder, true); // generate user-visible definition, agency will not see links
 
   if (!res.ok()) {
     return res;

diff --git a/arangod/Replication/DatabaseInitialSyncer.cpp b/arangod/Replication/DatabaseInitialSyncer.cpp
@@ -1359,6 +1359,7 @@ Result DatabaseInitialSyncer::handleCollectionsAndViews(VPackSlice const& collSl
                                                         bool incremental) {
   TRI_ASSERT(collSlices.isArray());
 
+  std::vector<std::pair<VPackSlice, VPackSlice>> systemCollections;
   std::vector<std::pair<VPackSlice, VPackSlice>> collections;
   for (VPackSlice it : VPackArrayIterator(collSlices)) {
     if (!it.isObject()) {
@@ -1414,10 +1415,46 @@ Result DatabaseInitialSyncer::handleCollectionsAndViews(VPackSlice const& collSl
       }
     }
 
-    collections.emplace_back(parameters, indexes);
+    if (masterName == StaticStrings::AnalyzersCollection) {
+      // _analyzers collection has to be restored before view creation
+      systemCollections.emplace_back(parameters, indexes);
+    } else {
+      collections.emplace_back(parameters, indexes);
+    }
+  }
+
+  // STEP 1: validate collection declarations from master
+  // ----------------------------------------------------------------------------------
+
+  // STEP 2: drop and re-create collections locally if they are also present on
+  // the master
+  //  ------------------------------------------------------------------------------------
+
+  // iterate over all collections from the master...
+  std::array<SyncPhase, 2> phases{{PHASE_VALIDATE, PHASE_DROP_CREATE}};
+  for (auto const& phase : phases) {
+    Result r = iterateCollections(systemCollections, incremental, phase);
+
+    if (r.fail()) {
+      return r;
+    }
+
+    r = iterateCollections(collections, incremental, phase);
+
+    if (r.fail()) {
+      return r;
+    }
   }
 
-  // STEP 1: now that the collections exist create the views
+  // STEP 3: restore data for system collections
+  // ----------------------------------------------------------------------------------
+  auto const res = iterateCollections(systemCollections, incremental, PHASE_DUMP);
+
+  if (res.fail()) {
+    return res;
+  }
+
+  // STEP 4: now that the collections exist create the views
   // this should be faster than re-indexing afterwards
   // ----------------------------------------------------------------------------------
 
@@ -1435,24 +1472,7 @@ Result DatabaseInitialSyncer::handleCollectionsAndViews(VPackSlice const& collSl
     _config.progress.set("view creation skipped because of configuration");
   }
 
-  // STEP 2: validate collection declarations from master
-  // ----------------------------------------------------------------------------------
-
-  // STEP 3: drop and re-create collections locally if they are also present on
-  // the master
-  //  ------------------------------------------------------------------------------------
-
-  // iterate over all collections from the master...
-  std::array<SyncPhase, 2> phases{{PHASE_VALIDATE, PHASE_DROP_CREATE}};
-  for (auto const& phase : phases) {
-    Result r = iterateCollections(collections, incremental, phase);
-
-    if (r.fail()) {
-      return r;
-    }
-  }
-
-  // STEP 4: sync collection data from master and create initial indexes
+  // STEP 5: sync collection data from master and create initial indexes
   // ----------------------------------------------------------------------------------
 
   // now load the data into the collections

diff --git a/arangod/VocBase/LogicalCollection.cpp b/arangod/VocBase/LogicalCollection.cpp
@@ -614,9 +614,6 @@ void LogicalCollection::toVelocyPackForClusterInventory(VPackBuilder& result,
       case Index::TRI_IDX_TYPE_PRIMARY_INDEX:
       case Index::TRI_IDX_TYPE_EDGE_INDEX:
         return false;
-      case Index::TRI_IDX_TYPE_IRESEARCH_LINK:
-        flags = Index::makeFlags(Index::Serialize::Internals);
-        return true;
       default:
         flags = Index::makeFlags();
         return !idx->isHidden() && !idx->inProgress();

diff --git a/arangod/VocBase/vocbase.cpp b/arangod/VocBase/vocbase.cpp
@@ -939,9 +939,6 @@ void TRI_vocbase_t::inventory(VPackBuilder& result, TRI_voc_tick_t maxTick,
           case Index::TRI_IDX_TYPE_PRIMARY_INDEX:
           case Index::TRI_IDX_TYPE_EDGE_INDEX:
             return false;
-          case Index::TRI_IDX_TYPE_IRESEARCH_LINK:
-            flags = Index::makeFlags(Index::Serialize::Internals);
-            return true;
           default:
             flags = Index::makeFlags(Index::Serialize::Basics);
             return !idx->isHidden();

diff --git a/arangosh/Restore/RestoreFeature.cpp b/arangosh/Restore/RestoreFeature.cpp
@@ -903,27 +903,10 @@ arangodb::Result processInputDirectory(
     std::sort(collections.begin(), collections.end(), ::sortCollectionsForCreation);
 
     std::unique_ptr<arangodb::RestoreFeature::JobData> usersData;
+    std::unique_ptr<arangodb::RestoreFeature::JobData> analyzersData;
     std::vector<std::unique_ptr<arangodb::RestoreFeature::JobData>> jobs;
     jobs.reserve(collections.size());
 
-    // Step 2: create views
-    // @note: done after collection population since views might depend on data
-    //        in restored collections
-    if (options.importStructure && !views.empty()) {
-      LOG_TOPIC("f723c", INFO, Logger::RESTORE) << "# Creating views...";
-
-      for (auto const& viewDefinition : views) {
-        LOG_TOPIC("c608d", DEBUG, Logger::RESTORE)
-          << "# Creating view: " << viewDefinition.toJson();
-
-        auto res = ::restoreView(httpClient, options, viewDefinition.slice());
-
-        if (!res.ok()) {
-          return res;
-        }
-          }
-    }
-
     bool didModifyFoxxCollection = false;
     // Step 3: create collections
     for (VPackBuilder const& b : collections) {
@@ -961,13 +944,43 @@ arangodb::Result processInputDirectory(
         // reason is that loading into the users collection may change the
         // credentials for the current arangorestore connection!
         usersData = std::move(jobData);
+      } else if (name.isString() && name.stringRef() == StaticStrings::AnalyzersCollection) {
+        // special treatment for _analyzers collection - this must be the very first
+        stats.totalCollections++;
+        analyzersData = std::move(jobData);
       } else {
         stats.totalCollections++;
         jobs.push_back(std::move(jobData));
       }
     }
-
-    // Step 4: fire up data transfer
+
+    // Step 4: restore data from _analyzers collection
+    if (analyzersData) {
+      // restore analyzers
+      if (!jobQueue.queueJob(std::move(analyzersData))) {
+        return Result(TRI_ERROR_OUT_OF_MEMORY, "unable to queue restore job");
+      }
+
+      jobQueue.waitForIdle();
+    }
+
+    // Step 5: create arangosearch views
+    if (options.importStructure && !views.empty()) {
+      LOG_TOPIC("f723c", INFO, Logger::RESTORE) << "# Creating views...";
+
+      for (auto const& viewDefinition : views) {
+        LOG_TOPIC("c608d", DEBUG, Logger::RESTORE)
+          << "# Creating view: " << viewDefinition.toJson();
+
+        auto res = ::restoreView(httpClient, options, viewDefinition.slice());
+
+        if (!res.ok()) {
+          return res;
+        }
+      }
+    }
+
+    // Step 6: fire up data transfer
     for (auto& job : jobs) {
       if (!jobQueue.queueJob(std::move(job))) {
         return Result(TRI_ERROR_OUT_OF_MEMORY, "unable to queue restore job");
@@ -1045,6 +1058,7 @@ arangodb::Result processInputDirectory(
         return firstError;
       }
     }
+
   } catch (std::exception const& ex) {
     return {TRI_ERROR_INTERNAL,
             std::string(