From ae075354862cfafa1994ebcb0bf793e1a4fafd5f Mon Sep 17 00:00:00 2001 From: Andrey Abramov Date: Tue, 19 Nov 2019 20:55:45 +0300 Subject: [PATCH 1/2] allow in-place analyzer creation via link definition (#10466) * allow in-place analyzer creation via link definition * add special handling for _analyzers collection * modify initial syncer * address review commments * fix accidentally broken test --- arangod/IResearch/IResearchCommon.cpp | 2 + arangod/IResearch/IResearchCommon.h | 12 + arangod/IResearch/IResearchLinkHelper.cpp | 2 +- arangod/IResearch/IResearchView.cpp | 20 +- .../IResearch/IResearchViewCoordinator.cpp | 49 ++- arangod/Replication/DatabaseInitialSyncer.cpp | 60 ++-- arangod/VocBase/LogicalCollection.cpp | 3 - arangod/VocBase/vocbase.cpp | 3 - arangosh/Restore/RestoreFeature.cpp | 54 ++-- tests/IResearch/IResearchView-test.cpp | 304 +++++++++++++++++- .../IResearchViewCoordinator-test.cpp | 260 ++++++++++++++- tests/js/server/dump/dump-mmfiles-cluster.js | 72 ++--- tests/js/server/dump/dump-rocksdb-cluster.js | 72 ++--- 13 files changed, 760 insertions(+), 153 deletions(-) diff --git a/arangod/IResearch/IResearchCommon.cpp b/arangod/IResearch/IResearchCommon.cpp index c024d20c5ce2..947b0d3d450d 100644 --- a/arangod/IResearch/IResearchCommon.cpp +++ b/arangod/IResearch/IResearchCommon.cpp @@ -52,10 +52,12 @@ arangodb::LogTopic& logTopic() { /*static*/ std::string const StaticStrings::LinksField("links"); /*static*/ std::string const StaticStrings::VersionField("version"); /*static*/ std::string const StaticStrings::ViewIdField("view"); +/*static*/ std::string const StaticStrings::AnalyzerDefinitionsField("analyzerDefinitions"); /*static*/ std::string const StaticStrings::AnalyzerFeaturesField("features"); /*static*/ std::string const StaticStrings::AnalyzerNameField("name"); /*static*/ std::string const StaticStrings::AnalyzerPropertiesField("properties"); /*static*/ std::string const StaticStrings::AnalyzerTypeField("type"); +/*static*/ std::string const StaticStrings::PrimarySortField("primarySort"); } // namespace iresearch } // namespace arangodb diff --git a/arangod/IResearch/IResearchCommon.h b/arangod/IResearch/IResearchCommon.h index b44cb5e1ffee..8c3cad04e80b 100644 --- a/arangod/IResearch/IResearchCommon.h +++ b/arangod/IResearch/IResearchCommon.h @@ -61,6 +61,12 @@ struct StaticStrings { //////////////////////////////////////////////////////////////////////////////// static std::string const ViewIdField; + //////////////////////////////////////////////////////////////////////////////// + /// @brief the name of the field in the IResearch Link definition denoting the + /// referenced analyzer definitions + //////////////////////////////////////////////////////////////////////////////// + static std::string const AnalyzerDefinitionsField; + //////////////////////////////////////////////////////////////////////////////// /// @brief the name of the field in the analyzer definition denoting the /// corresponding analyzer name @@ -84,6 +90,12 @@ struct StaticStrings { /// corresponding analyzer features //////////////////////////////////////////////////////////////////////////////// static std::string const AnalyzerFeaturesField; + + //////////////////////////////////////////////////////////////////////////////// + /// @brief the name of the field in the IResearch Link definition denoting the + /// primary sort + //////////////////////////////////////////////////////////////////////////////// + static std::string const PrimarySortField; }; } // namespace iresearch diff --git a/arangod/IResearch/IResearchLinkHelper.cpp b/arangod/IResearch/IResearchLinkHelper.cpp index 2009dfed0af1..f794572faced 100644 --- a/arangod/IResearch/IResearchLinkHelper.cpp +++ b/arangod/IResearch/IResearchLinkHelper.cpp @@ -783,7 +783,7 @@ namespace iresearch { std::string errorField; if (!linkDefinition.isNull()) { // have link definition - if (!meta.init(linkDefinition, false, errorField, &vocbase)) { // for db-server analyzer validation should have already applied on coordinator + if (!meta.init(linkDefinition, true, errorField, &vocbase)) { // for db-server analyzer validation should have already applied on coordinator return arangodb::Result( // result TRI_ERROR_BAD_PARAMETER, // code errorField.empty() diff --git a/arangod/IResearch/IResearchView.cpp b/arangod/IResearch/IResearchView.cpp index a2c47ca18970..87fb1f23c0b4 100644 --- a/arangod/IResearch/IResearchView.cpp +++ b/arangod/IResearch/IResearchView.cpp @@ -363,12 +363,11 @@ arangodb::Result IResearchView::appendVelocyPackImpl( // append JSON static const std::function persistenceAcceptor = [](irs::string_ref const&) -> bool { return true; }; - auto& acceptor = - (context == Serialization::Persistence || context == Serialization::PersistenceWithInProgress || context == Serialization::Inventory) - ? persistenceAcceptor - : propertiesAcceptor; + auto* acceptor = &propertiesAcceptor; + + if (context == Serialization::Persistence || context == Serialization::PersistenceWithInProgress) { + acceptor = &persistenceAcceptor; - if (context == Serialization::Persistence || context == Serialization::PersistenceWithInProgress) { if (arangodb::ServerState::instance()->isSingleServer()) { auto res = arangodb::LogicalViewHelperStorageEngine::properties(builder, *this); @@ -392,7 +391,7 @@ arangodb::Result IResearchView::appendVelocyPackImpl( // append JSON sanitizedBuilder.openObject(); if (!_meta.json(sanitizedBuilder) || - !mergeSliceSkipKeys(builder, sanitizedBuilder.close().slice(), acceptor)) { + !mergeSliceSkipKeys(builder, sanitizedBuilder.close().slice(), *acceptor)) { return arangodb::Result( TRI_ERROR_INTERNAL, std::string("failure to generate definition while generating " @@ -400,11 +399,6 @@ arangodb::Result IResearchView::appendVelocyPackImpl( // append JSON vocbase().name() + "'"); } - if (context == Serialization::Inventory) { - // nothing more to output - return {}; - } - if (context == Serialization::Persistence || context == Serialization::PersistenceWithInProgress) { IResearchViewMetaState metaState; @@ -461,7 +455,7 @@ arangodb::Result IResearchView::appendVelocyPackImpl( // append JSON ); } - auto visitor = [this, &linksBuilder, &res]( // visit collections + auto visitor = [this, &linksBuilder, &res, context]( // visit collections arangodb::TransactionCollection& trxCollection // transaction collection )->bool { auto collection = trxCollection.collection(); @@ -480,7 +474,7 @@ arangodb::Result IResearchView::appendVelocyPackImpl( // append JSON linkBuilder.openObject(); - if (!link->properties(linkBuilder, false).ok()) { // link definitions are not output if forPersistence + if (!link->properties(linkBuilder, Serialization::Inventory == context).ok()) { // link definitions are not output if forPersistence LOG_TOPIC("713ad", WARN, arangodb::iresearch::TOPIC) << "failed to generate json for arangosearch link '" << link->id() << "' while generating json for arangosearch view '" << name() << "'"; diff --git a/arangod/IResearch/IResearchViewCoordinator.cpp b/arangod/IResearch/IResearchViewCoordinator.cpp index d025bb533a2e..b25b45657071 100644 --- a/arangod/IResearch/IResearchViewCoordinator.cpp +++ b/arangod/IResearch/IResearchViewCoordinator.cpp @@ -181,18 +181,23 @@ arangodb::Result IResearchViewCoordinator::appendVelocyPackImpl( return {}; } - static const std::function propertiesAcceptor = + static const std::function propertiesAcceptor = [](irs::string_ref const& key) -> bool { return key != StaticStrings::VersionField; // ignored fields }; - static const std::function persistenceAcceptor = + static const std::function persistenceAcceptor = [](irs::string_ref const&) -> bool { return true; }; + static const std::function linkPropertiesAcceptor = + [](irs::string_ref const& key) -> bool { + return key != iresearch::StaticStrings::AnalyzerDefinitionsField + && key != iresearch::StaticStrings::PrimarySortField; + }; + auto* acceptor = &propertiesAcceptor; if (context == Serialization::Persistence || - context == Serialization::PersistenceWithInProgress || - context == Serialization::Inventory) { + context == Serialization::PersistenceWithInProgress) { auto res = arangodb::LogicalViewHelperClusterInfo::properties(builder, *this); if (!res.ok()) { @@ -200,9 +205,10 @@ arangodb::Result IResearchViewCoordinator::appendVelocyPackImpl( } acceptor = &persistenceAcceptor; - // links are not persisted, their definitions are part of the corresponding - // collections - } else if (context == Serialization::Properties) { + } + + if (context == Serialization::Properties || + context == Serialization::Inventory) { // verify that the current user has access on all linked collections auto* exec = ExecContext::CURRENT; if (exec) { @@ -213,18 +219,31 @@ arangodb::Result IResearchViewCoordinator::appendVelocyPackImpl( } } + VPackBuilder tmp; + ReadMutex mutex(_mutex); SCOPED_LOCK(mutex); // '_collections' can be asynchronously modified - VPackBuilder links; - links.openObject(); - + builder.add(StaticStrings::LinksField, VPackValue(VPackValueType::Object)); for (auto& entry : _collections) { - links.add(entry.second.first, entry.second.second.slice()); - } + auto linkSlice = entry.second.second.slice(); + + if (context == Serialization::Properties) { + tmp.clear(); + tmp.openObject(); + if (!mergeSliceSkipKeys(tmp, linkSlice, linkPropertiesAcceptor)) { + return { + TRI_ERROR_INTERNAL, + "failed to generate externally visible link definition for arangosearch View '" + name() + "'" + }; + } - links.close(); - builder.add(StaticStrings::LinksField, links.slice()); + linkSlice = tmp.close().slice(); + } + + builder.add(entry.second.first, linkSlice); + } + builder.close(); } if (!builder.isOpenObject()) { @@ -266,7 +285,7 @@ arangodb::Result IResearchViewCoordinator::link(IResearchLink const& link) { builder.openObject(); - auto res = link.properties(builder, false); // generate user-visible definition, agency will not see links + auto res = link.properties(builder, true); // generate user-visible definition, agency will not see links if (!res.ok()) { return res; diff --git a/arangod/Replication/DatabaseInitialSyncer.cpp b/arangod/Replication/DatabaseInitialSyncer.cpp index 3a27a9031f90..cb30c27f9e75 100644 --- a/arangod/Replication/DatabaseInitialSyncer.cpp +++ b/arangod/Replication/DatabaseInitialSyncer.cpp @@ -1359,6 +1359,7 @@ Result DatabaseInitialSyncer::handleCollectionsAndViews(VPackSlice const& collSl bool incremental) { TRI_ASSERT(collSlices.isArray()); + std::vector> systemCollections; std::vector> collections; for (VPackSlice it : VPackArrayIterator(collSlices)) { if (!it.isObject()) { @@ -1414,10 +1415,46 @@ Result DatabaseInitialSyncer::handleCollectionsAndViews(VPackSlice const& collSl } } - collections.emplace_back(parameters, indexes); + if (masterName == StaticStrings::AnalyzersCollection) { + // _analyzers collection has to be restored before view creation + systemCollections.emplace_back(parameters, indexes); + } else { + collections.emplace_back(parameters, indexes); + } + } + + // STEP 1: validate collection declarations from master + // ---------------------------------------------------------------------------------- + + // STEP 2: drop and re-create collections locally if they are also present on + // the master + // ------------------------------------------------------------------------------------ + + // iterate over all collections from the master... + std::array phases{{PHASE_VALIDATE, PHASE_DROP_CREATE}}; + for (auto const& phase : phases) { + Result r = iterateCollections(systemCollections, incremental, phase); + + if (r.fail()) { + return r; + } + + r = iterateCollections(collections, incremental, phase); + + if (r.fail()) { + return r; + } } - // STEP 1: now that the collections exist create the views + // STEP 3: restore data for system collections + // ---------------------------------------------------------------------------------- + auto const res = iterateCollections(systemCollections, incremental, PHASE_DUMP); + + if (res.fail()) { + return res; + } + + // STEP 4: now that the collections exist create the views // this should be faster than re-indexing afterwards // ---------------------------------------------------------------------------------- @@ -1435,24 +1472,7 @@ Result DatabaseInitialSyncer::handleCollectionsAndViews(VPackSlice const& collSl _config.progress.set("view creation skipped because of configuration"); } - // STEP 2: validate collection declarations from master - // ---------------------------------------------------------------------------------- - - // STEP 3: drop and re-create collections locally if they are also present on - // the master - // ------------------------------------------------------------------------------------ - - // iterate over all collections from the master... - std::array phases{{PHASE_VALIDATE, PHASE_DROP_CREATE}}; - for (auto const& phase : phases) { - Result r = iterateCollections(collections, incremental, phase); - - if (r.fail()) { - return r; - } - } - - // STEP 4: sync collection data from master and create initial indexes + // STEP 5: sync collection data from master and create initial indexes // ---------------------------------------------------------------------------------- // now load the data into the collections diff --git a/arangod/VocBase/LogicalCollection.cpp b/arangod/VocBase/LogicalCollection.cpp index d885ed6022ef..8ce5502eee49 100644 --- a/arangod/VocBase/LogicalCollection.cpp +++ b/arangod/VocBase/LogicalCollection.cpp @@ -614,9 +614,6 @@ void LogicalCollection::toVelocyPackForClusterInventory(VPackBuilder& result, case Index::TRI_IDX_TYPE_PRIMARY_INDEX: case Index::TRI_IDX_TYPE_EDGE_INDEX: return false; - case Index::TRI_IDX_TYPE_IRESEARCH_LINK: - flags = Index::makeFlags(Index::Serialize::Internals); - return true; default: flags = Index::makeFlags(); return !idx->isHidden() && !idx->inProgress(); diff --git a/arangod/VocBase/vocbase.cpp b/arangod/VocBase/vocbase.cpp index 6794a824cdaa..35c88c4eaeec 100644 --- a/arangod/VocBase/vocbase.cpp +++ b/arangod/VocBase/vocbase.cpp @@ -939,9 +939,6 @@ void TRI_vocbase_t::inventory(VPackBuilder& result, TRI_voc_tick_t maxTick, case Index::TRI_IDX_TYPE_PRIMARY_INDEX: case Index::TRI_IDX_TYPE_EDGE_INDEX: return false; - case Index::TRI_IDX_TYPE_IRESEARCH_LINK: - flags = Index::makeFlags(Index::Serialize::Internals); - return true; default: flags = Index::makeFlags(Index::Serialize::Basics); return !idx->isHidden(); diff --git a/arangosh/Restore/RestoreFeature.cpp b/arangosh/Restore/RestoreFeature.cpp index 1ff225a6741e..f1989eb6c2f8 100644 --- a/arangosh/Restore/RestoreFeature.cpp +++ b/arangosh/Restore/RestoreFeature.cpp @@ -903,27 +903,10 @@ arangodb::Result processInputDirectory( std::sort(collections.begin(), collections.end(), ::sortCollectionsForCreation); std::unique_ptr usersData; + std::unique_ptr analyzersData; std::vector> jobs; jobs.reserve(collections.size()); - // Step 2: create views - // @note: done after collection population since views might depend on data - // in restored collections - if (options.importStructure && !views.empty()) { - LOG_TOPIC("f723c", INFO, Logger::RESTORE) << "# Creating views..."; - - for (auto const& viewDefinition : views) { - LOG_TOPIC("c608d", DEBUG, Logger::RESTORE) - << "# Creating view: " << viewDefinition.toJson(); - - auto res = ::restoreView(httpClient, options, viewDefinition.slice()); - - if (!res.ok()) { - return res; - } - } - } - bool didModifyFoxxCollection = false; // Step 3: create collections for (VPackBuilder const& b : collections) { @@ -961,13 +944,43 @@ arangodb::Result processInputDirectory( // reason is that loading into the users collection may change the // credentials for the current arangorestore connection! usersData = std::move(jobData); + } else if (name.isString() && name.stringRef() == StaticStrings::AnalyzersCollection) { + // special treatment for _analyzers collection - this must be the very first + stats.totalCollections++; + analyzersData = std::move(jobData); } else { stats.totalCollections++; jobs.push_back(std::move(jobData)); } } - - // Step 4: fire up data transfer + + // Step 4: restore data from _analyzers collection + if (analyzersData) { + // restore analyzers + if (!jobQueue.queueJob(std::move(analyzersData))) { + return Result(TRI_ERROR_OUT_OF_MEMORY, "unable to queue restore job"); + } + + jobQueue.waitForIdle(); + } + + // Step 5: create arangosearch views + if (options.importStructure && !views.empty()) { + LOG_TOPIC("f723c", INFO, Logger::RESTORE) << "# Creating views..."; + + for (auto const& viewDefinition : views) { + LOG_TOPIC("c608d", DEBUG, Logger::RESTORE) + << "# Creating view: " << viewDefinition.toJson(); + + auto res = ::restoreView(httpClient, options, viewDefinition.slice()); + + if (!res.ok()) { + return res; + } + } + } + + // Step 6: fire up data transfer for (auto& job : jobs) { if (!jobQueue.queueJob(std::move(job))) { return Result(TRI_ERROR_OUT_OF_MEMORY, "unable to queue restore job"); @@ -1045,6 +1058,7 @@ arangodb::Result processInputDirectory( return firstError; } } + } catch (std::exception const& ex) { return {TRI_ERROR_INTERNAL, std::string( diff --git a/tests/IResearch/IResearchView-test.cpp b/tests/IResearch/IResearchView-test.cpp index 25452a6ad08f..918c98e88a05 100644 --- a/tests/IResearch/IResearchView-test.cpp +++ b/tests/IResearch/IResearchView-test.cpp @@ -502,7 +502,7 @@ TEST_F(IResearchViewTest, test_defaults) { std::string error; EXPECT_TRUE((slice.isObject())); EXPECT_TRUE((13U == slice.length())); - EXPECT_TRUE((slice.hasKey("globallyUniqueId") && slice.get("globallyUniqueId").isString() && false == slice.get("globallyUniqueId").copyString().empty())); + EXPECT_TRUE((slice.hasKey("globallyUniqueId") && slice.get("globallyUniqueId").isString() && !slice.get("globallyUniqueId").copyString().empty())); EXPECT_TRUE((slice.get("name").copyString() == "testView")); EXPECT_TRUE((slice.get("type").copyString() == arangodb::iresearch::DATA_SOURCE_TYPE.name())); EXPECT_TRUE((false == slice.hasKey("deleted"))); @@ -514,6 +514,308 @@ TEST_F(IResearchViewTest, test_defaults) { } } +TEST_F(IResearchViewTest, test_properties) { + // new view definition with links + auto collectionJson = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testCollection\", \"id\": 100 }"); + auto viewCreateJson = arangodb::velocypack::Parser::fromJson( + "{ \"name\": \"testView\", \"type\": \"arangosearch\", \"id\": 101, " + " \"links\": { " + " \"testCollection\": { " + " \"includeAllFields\":true, " + " \"analyzers\": [\"inPlace\"], " + " \"analyzerDefinitions\": [ { \"name\" : \"inPlace\", \"type\":\"identity\", \"properties\":{}, \"features\":[] } ]" + " } " + " } " + "}" + ); + + Vocbase vocbase(TRI_vocbase_type_e::TRI_VOCBASE_TYPE_NORMAL, testDBInfo(server.server())); + auto logicalCollection = vocbase.createCollection(collectionJson->slice()); + EXPECT_NE(nullptr, logicalCollection); + EXPECT_EQ(nullptr, vocbase.lookupView("testView")); + EXPECT_TRUE(logicalCollection->getIndexes().empty()); + arangodb::LogicalView::ptr logicalView; + EXPECT_TRUE(arangodb::iresearch::IResearchView::factory().create(logicalView, vocbase, viewCreateJson->slice()).ok()); + ASSERT_TRUE(logicalView); + std::set cids; + logicalView->visitCollections([&cids](TRI_voc_cid_t cid)->bool { cids.emplace(cid); return true; }); + EXPECT_EQ(1, cids.size()); + EXPECT_FALSE(logicalCollection->getIndexes().empty()); + + // check serialization for listing + { + arangodb::velocypack::Builder builder; + builder.openObject(); + logicalView->properties(builder, arangodb::LogicalDataSource::Serialization::List); + builder.close(); + + auto slice = builder.slice(); + EXPECT_TRUE(slice.isObject()); + EXPECT_EQ(4, slice.length()); + EXPECT_TRUE(slice.get("name").isString() && "testView" == slice.get("name").copyString()); + EXPECT_TRUE(slice.get("type").isString() && "arangosearch" == slice.get("type").copyString()); + EXPECT_TRUE(slice.get("id").isString() && "101" == slice.get("id").copyString()); + EXPECT_TRUE(slice.get("globallyUniqueId").isString() && !slice.get("globallyUniqueId").copyString().empty()); + } + + // check serialization for properties + { + VPackSlice tmpSlice, tmpSlice2; + + VPackBuilder builder; + builder.openObject(); + logicalView->properties(builder, arangodb::LogicalDataSource::Serialization::Properties); + builder.close(); + + auto slice = builder.slice(); + EXPECT_TRUE(slice.isObject()); + EXPECT_EQ(13, slice.length()); + EXPECT_TRUE(slice.get("name").isString() && "testView" == slice.get("name").copyString()); + EXPECT_TRUE(slice.get("type").isString() && "arangosearch" == slice.get("type").copyString()); + EXPECT_TRUE(slice.get("id").isString() && "101" == slice.get("id").copyString()); + EXPECT_TRUE(slice.get("globallyUniqueId").isString() && !slice.get("globallyUniqueId").copyString().empty()); + EXPECT_TRUE(slice.get("consolidationIntervalMsec").isNumber() && 10000 == slice.get("consolidationIntervalMsec").getNumber()); + EXPECT_TRUE(slice.get("cleanupIntervalStep").isNumber() && 2 == slice.get("cleanupIntervalStep").getNumber()); + EXPECT_TRUE(slice.get("commitIntervalMsec").isNumber() && 1000 == slice.get("commitIntervalMsec").getNumber()); + { // consolidation policy + tmpSlice = slice.get("consolidationPolicy"); + EXPECT_TRUE(tmpSlice.isObject() && 6 == tmpSlice.length()); + tmpSlice2 = tmpSlice.get("type"); + EXPECT_TRUE(tmpSlice2.isString() && std::string("tier") == tmpSlice2.copyString()); + tmpSlice2 = tmpSlice.get("segmentsMin"); + EXPECT_TRUE(tmpSlice2.isNumber() && 1 == tmpSlice2.getNumber()); + tmpSlice2 = tmpSlice.get("segmentsMax"); + EXPECT_TRUE(tmpSlice2.isNumber() && 10 == tmpSlice2.getNumber()); + tmpSlice2 = tmpSlice.get("segmentsBytesFloor"); + EXPECT_TRUE(tmpSlice2.isNumber() && (size_t(2) * (1 << 20)) == tmpSlice2.getNumber()); + tmpSlice2 = tmpSlice.get("segmentsBytesMax"); + EXPECT_TRUE(tmpSlice2.isNumber() && (size_t(5) * (1 << 30)) == tmpSlice2.getNumber()); + tmpSlice2 = tmpSlice.get("minScore"); + EXPECT_TRUE(tmpSlice2.isNumber() && (0. == tmpSlice2.getNumber())); + } + tmpSlice = slice.get("writebufferActive"); + EXPECT_TRUE(tmpSlice.isNumber() && 0 == tmpSlice.getNumber()); + tmpSlice = slice.get("writebufferIdle"); + EXPECT_TRUE(tmpSlice.isNumber() && 64 == tmpSlice.getNumber()); + tmpSlice = slice.get("writebufferSizeMax"); + EXPECT_TRUE(tmpSlice.isNumber() && 32 * (size_t(1) << 20) == tmpSlice.getNumber()); + tmpSlice = slice.get("primarySort"); + EXPECT_TRUE(tmpSlice.isArray()); + EXPECT_EQ(0, tmpSlice.length()); + { // links + tmpSlice = slice.get("links"); + EXPECT_TRUE(tmpSlice.isObject()); + EXPECT_EQ(1, tmpSlice.length()); + tmpSlice2 = tmpSlice.get("testCollection"); + EXPECT_TRUE(tmpSlice2.isObject()); + EXPECT_EQ(5, tmpSlice2.length()); + EXPECT_TRUE(tmpSlice2.get("analyzers").isArray() && + 1 == tmpSlice2.get("analyzers").length() && + "inPlace" == tmpSlice2.get("analyzers").at(0).copyString()); + EXPECT_TRUE(tmpSlice2.get("fields").isObject() && 0 == tmpSlice2.get("fields").length()); + EXPECT_TRUE(tmpSlice2.get("includeAllFields").isBool() && tmpSlice2.get("includeAllFields").getBool()); + EXPECT_TRUE(tmpSlice2.get("trackListPositions").isBool() && !tmpSlice2.get("trackListPositions").getBool()); + EXPECT_TRUE(tmpSlice2.get("storeValues").isString() && "none" == tmpSlice2.get("storeValues").copyString()); + } + } + + // check serialization for persistence + { + VPackSlice tmpSlice, tmpSlice2; + + arangodb::velocypack::Builder builder; + builder.openObject(); + logicalView->properties(builder, arangodb::LogicalDataSource::Serialization::Persistence); + builder.close(); + + auto slice = builder.slice(); + EXPECT_TRUE(slice.isObject()); + EXPECT_EQ(17, slice.length()); + EXPECT_TRUE(slice.get("name").isString() && "testView" == slice.get("name").copyString()); + EXPECT_TRUE(slice.get("type").isString() && "arangosearch" == slice.get("type").copyString()); + EXPECT_TRUE(slice.get("id").isString() && "101" == slice.get("id").copyString()); + EXPECT_TRUE(slice.get("planId").isString() && "101" == slice.get("planId").copyString()); + EXPECT_TRUE(slice.get("globallyUniqueId").isString() && !slice.get("globallyUniqueId").copyString().empty()); + EXPECT_TRUE(slice.get("consolidationIntervalMsec").isNumber() && 10000 == slice.get("consolidationIntervalMsec").getNumber()); + EXPECT_TRUE(slice.get("cleanupIntervalStep").isNumber() && 2 == slice.get("cleanupIntervalStep").getNumber()); + EXPECT_TRUE(slice.get("commitIntervalMsec").isNumber() && 1000 == slice.get("commitIntervalMsec").getNumber()); + EXPECT_TRUE(slice.get("deleted").isBool() && !slice.get("deleted").getBool()); + EXPECT_TRUE(slice.get("isSystem").isBool() && !slice.get("isSystem").getBool()); + EXPECT_TRUE(slice.get("collections").isArray() && + 1 == slice.get("collections").length() && + 100 == slice.get("collections").at(0).getNumber()); + + { // consolidation policy + tmpSlice = slice.get("consolidationPolicy"); + EXPECT_TRUE(tmpSlice.isObject() && 6 == tmpSlice.length()); + tmpSlice2 = tmpSlice.get("type"); + EXPECT_TRUE(tmpSlice2.isString() && std::string("tier") == tmpSlice2.copyString()); + tmpSlice2 = tmpSlice.get("segmentsMin"); + EXPECT_TRUE(tmpSlice2.isNumber() && 1 == tmpSlice2.getNumber()); + tmpSlice2 = tmpSlice.get("segmentsMax"); + EXPECT_TRUE(tmpSlice2.isNumber() && 10 == tmpSlice2.getNumber()); + tmpSlice2 = tmpSlice.get("segmentsBytesFloor"); + EXPECT_TRUE(tmpSlice2.isNumber() && (size_t(2) * (1 << 20)) == tmpSlice2.getNumber()); + tmpSlice2 = tmpSlice.get("segmentsBytesMax"); + EXPECT_TRUE(tmpSlice2.isNumber() && (size_t(5) * (1 << 30)) == tmpSlice2.getNumber()); + tmpSlice2 = tmpSlice.get("minScore"); + EXPECT_TRUE(tmpSlice2.isNumber() && (0. == tmpSlice2.getNumber())); + } + tmpSlice = slice.get("writebufferActive"); + EXPECT_TRUE(tmpSlice.isNumber() && 0 == tmpSlice.getNumber()); + tmpSlice = slice.get("writebufferIdle"); + EXPECT_TRUE(tmpSlice.isNumber() && 64 == tmpSlice.getNumber()); + tmpSlice = slice.get("writebufferSizeMax"); + EXPECT_TRUE(tmpSlice.isNumber() && 32 * (size_t(1) << 20) == tmpSlice.getNumber()); + tmpSlice = slice.get("primarySort"); + EXPECT_TRUE(tmpSlice.isArray()); + EXPECT_EQ(0, tmpSlice.length()); + tmpSlice = slice.get("version"); + EXPECT_TRUE(tmpSlice.isNumber() && 1 == tmpSlice.getNumber()); + } + + // check serialization for inventory + { + VPackSlice tmpSlice, tmpSlice2; + + arangodb::velocypack::Builder builder; + builder.openObject(); + logicalView->properties(builder, arangodb::LogicalDataSource::Serialization::Inventory); + builder.close(); + + auto slice = builder.slice(); + EXPECT_TRUE(slice.isObject()); + EXPECT_EQ(13, slice.length()); + EXPECT_TRUE(slice.get("name").isString() && "testView" == slice.get("name").copyString()); + EXPECT_TRUE(slice.get("type").isString() && "arangosearch" == slice.get("type").copyString()); + EXPECT_TRUE(slice.get("id").isString() && "101" == slice.get("id").copyString()); + EXPECT_TRUE(slice.get("globallyUniqueId").isString() && !slice.get("globallyUniqueId").copyString().empty()); + EXPECT_TRUE(slice.get("consolidationIntervalMsec").isNumber() && 10000 == slice.get("consolidationIntervalMsec").getNumber()); + EXPECT_TRUE(slice.get("cleanupIntervalStep").isNumber() && 2 == slice.get("cleanupIntervalStep").getNumber()); + EXPECT_TRUE(slice.get("commitIntervalMsec").isNumber() && 1000 == slice.get("commitIntervalMsec").getNumber()); + { // consolidation policy + tmpSlice = slice.get("consolidationPolicy"); + EXPECT_TRUE(tmpSlice.isObject() && 6 == tmpSlice.length()); + tmpSlice2 = tmpSlice.get("type"); + EXPECT_TRUE(tmpSlice2.isString() && std::string("tier") == tmpSlice2.copyString()); + tmpSlice2 = tmpSlice.get("segmentsMin"); + EXPECT_TRUE(tmpSlice2.isNumber() && 1 == tmpSlice2.getNumber()); + tmpSlice2 = tmpSlice.get("segmentsMax"); + EXPECT_TRUE(tmpSlice2.isNumber() && 10 == tmpSlice2.getNumber()); + tmpSlice2 = tmpSlice.get("segmentsBytesFloor"); + EXPECT_TRUE(tmpSlice2.isNumber() && (size_t(2) * (1 << 20)) == tmpSlice2.getNumber()); + tmpSlice2 = tmpSlice.get("segmentsBytesMax"); + EXPECT_TRUE(tmpSlice2.isNumber() && (size_t(5) * (1 << 30)) == tmpSlice2.getNumber()); + tmpSlice2 = tmpSlice.get("minScore"); + EXPECT_TRUE(tmpSlice2.isNumber() && (0. == tmpSlice2.getNumber())); + } + tmpSlice = slice.get("writebufferActive"); + EXPECT_TRUE(tmpSlice.isNumber() && 0 == tmpSlice.getNumber()); + tmpSlice = slice.get("writebufferIdle"); + EXPECT_TRUE(tmpSlice.isNumber() && 64 == tmpSlice.getNumber()); + tmpSlice = slice.get("writebufferSizeMax"); + EXPECT_TRUE(tmpSlice.isNumber() && 32 * (size_t(1) << 20) == tmpSlice.getNumber()); + tmpSlice = slice.get("primarySort"); + EXPECT_TRUE(tmpSlice.isArray()); + EXPECT_EQ(0, tmpSlice.length()); + { // links + tmpSlice = slice.get("links"); + EXPECT_TRUE(tmpSlice.isObject()); + EXPECT_EQ(1, tmpSlice.length()); + tmpSlice2 = tmpSlice.get("testCollection"); + EXPECT_TRUE(tmpSlice2.isObject()); + EXPECT_EQ(7, tmpSlice2.length()); + EXPECT_TRUE(tmpSlice2.get("analyzers").isArray() && + 1 == tmpSlice2.get("analyzers").length() && + "inPlace" == tmpSlice2.get("analyzers").at(0).copyString()); + EXPECT_TRUE(tmpSlice2.get("fields").isObject() && 0 == tmpSlice2.get("fields").length()); + EXPECT_TRUE(tmpSlice2.get("includeAllFields").isBool() && tmpSlice2.get("includeAllFields").getBool()); + EXPECT_TRUE(tmpSlice2.get("trackListPositions").isBool() && !tmpSlice2.get("trackListPositions").getBool()); + EXPECT_TRUE(tmpSlice2.get("storeValues").isString() && "none" == tmpSlice2.get("storeValues").copyString()); + + tmpSlice2 = tmpSlice2.get("analyzerDefinitions"); + ASSERT_TRUE(tmpSlice2.isArray()); + ASSERT_EQ(1, tmpSlice2.length()); + tmpSlice2 = tmpSlice2.at(0); + ASSERT_TRUE(tmpSlice2.isObject()); + EXPECT_EQ(4, tmpSlice2.length()); + EXPECT_TRUE(tmpSlice2.get("name").isString() && "inPlace" == tmpSlice2.get("name").copyString()); + EXPECT_TRUE(tmpSlice2.get("type").isString() && "identity" == tmpSlice2.get("type").copyString()); + EXPECT_TRUE(tmpSlice2.get("properties").isObject() && 0 == tmpSlice2.get("properties").length()); + EXPECT_TRUE(tmpSlice2.get("features").isArray() && 0 == tmpSlice2.get("features").length()); + } + } +} + +TEST_F(IResearchViewTest, test_vocbase_inventory) { + // new view definition with links + auto collectionJson = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testCollection\", \"id\": 100 }"); + auto viewCreateJson = arangodb::velocypack::Parser::fromJson( + "{ \"name\": \"testView\", \"type\": \"arangosearch\", \"id\": 101, " + " \"links\": { " + " \"testCollection\": { " + " \"incudeAllFields\":true, " + " \"analyzers\": [\"inPlace\"], " + " \"analyzerDefinitions\": [ { \"name\" : \"inPlace\", \"type\":\"identity\", \"properties\":{}, \"features\":[] } ]" + " } " + " } " + "}" + ); + + Vocbase vocbase(TRI_vocbase_type_e::TRI_VOCBASE_TYPE_NORMAL, testDBInfo(server.server())); + auto logicalCollection = vocbase.createCollection(collectionJson->slice()); + EXPECT_NE(nullptr, logicalCollection); + EXPECT_EQ(nullptr, vocbase.lookupView("testView")); + EXPECT_TRUE(logicalCollection->getIndexes().empty()); + arangodb::LogicalView::ptr logicalView; + EXPECT_TRUE(arangodb::iresearch::IResearchView::factory().create(logicalView, vocbase, viewCreateJson->slice()).ok()); + ASSERT_TRUE(logicalView); + std::set cids; + logicalView->visitCollections([&cids](TRI_voc_cid_t cid)->bool { cids.emplace(cid); return true; }); + EXPECT_EQ(1, cids.size()); + EXPECT_FALSE(logicalCollection->getIndexes().empty()); + + // check vocbase inventory + { + arangodb::velocypack::Builder builder; + builder.openObject(); + vocbase.inventory(builder, std::numeric_limits::max(), [](arangodb::LogicalCollection const*) { return true; }); + + auto slice = builder.close().slice(); + ASSERT_TRUE(slice.isObject()); + + // ensure links are not exposed as indices + auto collectionsSlice = slice.get("collections"); + ASSERT_TRUE(collectionsSlice.isArray()); + for (auto collectionSlice : VPackArrayIterator(collectionsSlice)) { + ASSERT_TRUE(collectionSlice.isObject()); + auto indexesSlice = collectionSlice.get("indexes"); + ASSERT_TRUE(indexesSlice.isArray()); + for (auto indexSlice : VPackArrayIterator(indexesSlice)) { + ASSERT_TRUE(indexSlice.isObject()); + ASSERT_TRUE(indexSlice.hasKey("type")); + ASSERT_TRUE(indexSlice.get("type").isString()); + ASSERT_NE("arangosearch", indexSlice.get("type").copyString()); + } + } + + // check views + auto viewsSlice = slice.get("views"); + ASSERT_TRUE(viewsSlice.isArray()); + ASSERT_EQ(1, viewsSlice.length()); + auto viewSlice = viewsSlice.at(0); + ASSERT_TRUE(viewSlice.isObject()); + + VPackBuilder viewDefinition; + viewDefinition.openObject(); + ASSERT_TRUE(logicalView->properties(viewDefinition, + arangodb::LogicalDataSource::Serialization::Inventory).ok()); + viewDefinition.close(); + + EXPECT_EQUAL_SLICES(viewDefinition.slice(), viewSlice); + } +} + TEST_F(IResearchViewTest, test_cleanup) { auto collectionJson = arangodb::velocypack::Parser::fromJson("{ \"name\": \"testCollection\" }"); auto linkJson = arangodb::velocypack::Parser::fromJson("{ \"view\": \"testView\", \"includeAllFields\": true }"); diff --git a/tests/IResearch/IResearchViewCoordinator-test.cpp b/tests/IResearch/IResearchViewCoordinator-test.cpp index e1fbbb40447f..ea61f4b506ee 100644 --- a/tests/IResearch/IResearchViewCoordinator-test.cpp +++ b/tests/IResearch/IResearchViewCoordinator-test.cpp @@ -748,7 +748,7 @@ TEST_F(IResearchViewCoordinatorTest, test_drop_with_link) { EXPECT_TRUE((false == logicalView->visitCollections( [](TRI_voc_cid_t) -> bool { return false; }))); - // simulate heartbeat thread (remove index from current) + // simulate heartbeat thread (remove index in current) { auto const path = "/Current/Collections/" + vocbase->name() + "/" + std::to_string(logicalCollection->id()) + @@ -959,6 +959,264 @@ TEST_F(IResearchViewCoordinatorTest, test_update_properties) { } } + +TEST_F(IResearchViewCoordinatorTest, test_properties) { + auto& ci = server.getFeature().clusterInfo(); + + TRI_vocbase_t* vocbase; // will be owned by DatabaseFeature + + createTestDatabase(vocbase); + + // create collections + std::shared_ptr logicalCollection; + { + auto const collectionId = "100"; + auto collectionJson = arangodb::velocypack::Parser::fromJson( + "{ \"id\": \"100\", \"planId\": \"100\", \"name\": \"testCollection\", " + "\"replicationFactor\": 1, \"type\": 1, \"shards\":{} }"); + + EXPECT_TRUE(ci.createCollectionCoordinator(vocbase->name(), collectionId, 0, 1, 1, false, + collectionJson->slice(), 0.0, false, nullptr).ok()); + + logicalCollection = ci.getCollection(vocbase->name(), collectionId); + ASSERT_NE(nullptr, logicalCollection); + } + + // new view definition with links + auto viewCreateJson = arangodb::velocypack::Parser::fromJson( + "{ \"name\": \"testView\", \"type\": \"arangosearch\", \"id\": 101 }" + ); + + auto viewUpdateJson = arangodb::velocypack::Parser::fromJson( + "{ \"links\": { " + " \"testCollection\": { " + " \"id\":1, " + " \"includeAllFields\":true, " + " \"analyzers\": [\"inPlace\"], " + " \"analyzerDefinitions\": [ { \"name\" : \"inPlace\", \"type\":\"identity\", \"properties\":{}, \"features\":[] } ]" + " } " + " } }"); + + auto viewId = std::to_string(ci.uniqid() + 1); // +1 because LogicalView creation will generate a new ID + EXPECT_TRUE(ci.createViewCoordinator(vocbase->name(), viewId, viewCreateJson->slice()).ok()); + auto logicalView = ci.getView(vocbase->name(), viewId); + ASSERT_NE(nullptr, logicalView); + + // simulate heartbeat thread (create index in current) + { + auto const path = "/Current/Collections/" + vocbase->name() + "/" + + std::to_string(logicalCollection->id()); + auto const value = arangodb::velocypack::Parser::fromJson( + "{ \"shard-id-does-not-matter\": { \"indexes\" : [ { \"id\": \"1\" } ] } }"); + EXPECT_TRUE(arangodb::AgencyComm().setValue(path, value->slice(), 0.0).successful()); + } + + ASSERT_TRUE(logicalView->properties(viewUpdateJson->slice(), false).ok()); + logicalView = ci.getView(vocbase->name(), viewId); + ASSERT_NE(nullptr, logicalView); + + // check serialization for listing + { + arangodb::velocypack::Builder builder; + builder.openObject(); + logicalView->properties(builder, arangodb::LogicalDataSource::Serialization::List); + builder.close(); + + auto slice = builder.slice(); + EXPECT_TRUE(slice.isObject()); + EXPECT_EQ(4, slice.length()); + EXPECT_TRUE(slice.get("name").isString() && "testView" == slice.get("name").copyString()); + EXPECT_TRUE(slice.get("type").isString() && "arangosearch" == slice.get("type").copyString()); + EXPECT_TRUE(slice.get("id").isString() && "101" == slice.get("id").copyString()); + EXPECT_TRUE(slice.get("globallyUniqueId").isString() && !slice.get("globallyUniqueId").copyString().empty()); + } + + // check serialization for properties + { + VPackSlice tmpSlice, tmpSlice2; + + VPackBuilder builder; + builder.openObject(); + logicalView->properties(builder, arangodb::LogicalDataSource::Serialization::Properties); + builder.close(); + + auto slice = builder.slice(); + EXPECT_TRUE(slice.isObject()); + EXPECT_EQ(13, slice.length()); + EXPECT_TRUE(slice.get("name").isString() && "testView" == slice.get("name").copyString()); + EXPECT_TRUE(slice.get("type").isString() && "arangosearch" == slice.get("type").copyString()); + EXPECT_TRUE(slice.get("id").isString() && "101" == slice.get("id").copyString()); + EXPECT_TRUE(slice.get("globallyUniqueId").isString() && !slice.get("globallyUniqueId").copyString().empty()); + EXPECT_TRUE(slice.get("consolidationIntervalMsec").isNumber() && 10000 == slice.get("consolidationIntervalMsec").getNumber()); + EXPECT_TRUE(slice.get("cleanupIntervalStep").isNumber() && 2 == slice.get("cleanupIntervalStep").getNumber()); + EXPECT_TRUE(slice.get("commitIntervalMsec").isNumber() && 1000 == slice.get("commitIntervalMsec").getNumber()); + { // consolidation policy + tmpSlice = slice.get("consolidationPolicy"); + EXPECT_TRUE(tmpSlice.isObject() && 6 == tmpSlice.length()); + tmpSlice2 = tmpSlice.get("type"); + EXPECT_TRUE(tmpSlice2.isString() && std::string("tier") == tmpSlice2.copyString()); + tmpSlice2 = tmpSlice.get("segmentsMin"); + EXPECT_TRUE(tmpSlice2.isNumber() && 1 == tmpSlice2.getNumber()); + tmpSlice2 = tmpSlice.get("segmentsMax"); + EXPECT_TRUE(tmpSlice2.isNumber() && 10 == tmpSlice2.getNumber()); + tmpSlice2 = tmpSlice.get("segmentsBytesFloor"); + EXPECT_TRUE(tmpSlice2.isNumber() && (size_t(2) * (1 << 20)) == tmpSlice2.getNumber()); + tmpSlice2 = tmpSlice.get("segmentsBytesMax"); + EXPECT_TRUE(tmpSlice2.isNumber() && (size_t(5) * (1 << 30)) == tmpSlice2.getNumber()); + tmpSlice2 = tmpSlice.get("minScore"); + EXPECT_TRUE(tmpSlice2.isNumber() && (0. == tmpSlice2.getNumber())); + } + tmpSlice = slice.get("writebufferActive"); + EXPECT_TRUE(tmpSlice.isNumber() && 0 == tmpSlice.getNumber()); + tmpSlice = slice.get("writebufferIdle"); + EXPECT_TRUE(tmpSlice.isNumber() && 64 == tmpSlice.getNumber()); + tmpSlice = slice.get("writebufferSizeMax"); + EXPECT_TRUE(tmpSlice.isNumber() && 32 * (size_t(1) << 20) == tmpSlice.getNumber()); + tmpSlice = slice.get("primarySort"); + EXPECT_TRUE(tmpSlice.isArray()); + EXPECT_EQ(0, tmpSlice.length()); + { // links + tmpSlice = slice.get("links"); + EXPECT_TRUE(tmpSlice.isObject()); + EXPECT_EQ(1, tmpSlice.length()); + tmpSlice2 = tmpSlice.get("testCollection"); + EXPECT_TRUE(tmpSlice2.isObject()); + EXPECT_EQ(5, tmpSlice2.length()); + EXPECT_TRUE(tmpSlice2.get("analyzers").isArray() && + 1 == tmpSlice2.get("analyzers").length() && + "inPlace" == tmpSlice2.get("analyzers").at(0).copyString()); + EXPECT_TRUE(tmpSlice2.get("fields").isObject() && 0 == tmpSlice2.get("fields").length()); + EXPECT_TRUE(tmpSlice2.get("includeAllFields").isBool() && tmpSlice2.get("includeAllFields").getBool()); + EXPECT_TRUE(tmpSlice2.get("trackListPositions").isBool() && !tmpSlice2.get("trackListPositions").getBool()); + EXPECT_TRUE(tmpSlice2.get("storeValues").isString() && "none" == tmpSlice2.get("storeValues").copyString()); + } + } + + // check serialization for persistence + { + VPackSlice tmpSlice, tmpSlice2; + + arangodb::velocypack::Builder builder; + builder.openObject(); + logicalView->properties(builder, arangodb::LogicalDataSource::Serialization::Persistence); + builder.close(); + + auto slice = builder.slice(); + EXPECT_TRUE(slice.isObject()); + EXPECT_EQ(16, slice.length()); + EXPECT_TRUE(slice.get("name").isString() && "testView" == slice.get("name").copyString()); + EXPECT_TRUE(slice.get("type").isString() && "arangosearch" == slice.get("type").copyString()); + EXPECT_TRUE(slice.get("id").isString() && "101" == slice.get("id").copyString()); + EXPECT_TRUE(slice.get("planId").isString() && "101" == slice.get("planId").copyString()); + EXPECT_TRUE(slice.get("globallyUniqueId").isString() && !slice.get("globallyUniqueId").copyString().empty()); + EXPECT_TRUE(slice.get("consolidationIntervalMsec").isNumber() && 10000 == slice.get("consolidationIntervalMsec").getNumber()); + EXPECT_TRUE(slice.get("cleanupIntervalStep").isNumber() && 2 == slice.get("cleanupIntervalStep").getNumber()); + EXPECT_TRUE(slice.get("commitIntervalMsec").isNumber() && 1000 == slice.get("commitIntervalMsec").getNumber()); + EXPECT_TRUE(slice.get("deleted").isBool() && !slice.get("deleted").getBool()); + EXPECT_TRUE(slice.get("isSystem").isBool() && !slice.get("isSystem").getBool()); + + { // consolidation policy + tmpSlice = slice.get("consolidationPolicy"); + EXPECT_TRUE(tmpSlice.isObject() && 6 == tmpSlice.length()); + tmpSlice2 = tmpSlice.get("type"); + EXPECT_TRUE(tmpSlice2.isString() && std::string("tier") == tmpSlice2.copyString()); + tmpSlice2 = tmpSlice.get("segmentsMin"); + EXPECT_TRUE(tmpSlice2.isNumber() && 1 == tmpSlice2.getNumber()); + tmpSlice2 = tmpSlice.get("segmentsMax"); + EXPECT_TRUE(tmpSlice2.isNumber() && 10 == tmpSlice2.getNumber()); + tmpSlice2 = tmpSlice.get("segmentsBytesFloor"); + EXPECT_TRUE(tmpSlice2.isNumber() && (size_t(2) * (1 << 20)) == tmpSlice2.getNumber()); + tmpSlice2 = tmpSlice.get("segmentsBytesMax"); + EXPECT_TRUE(tmpSlice2.isNumber() && (size_t(5) * (1 << 30)) == tmpSlice2.getNumber()); + tmpSlice2 = tmpSlice.get("minScore"); + EXPECT_TRUE(tmpSlice2.isNumber() && (0. == tmpSlice2.getNumber())); + } + tmpSlice = slice.get("writebufferActive"); + EXPECT_TRUE(tmpSlice.isNumber() && 0 == tmpSlice.getNumber()); + tmpSlice = slice.get("writebufferIdle"); + EXPECT_TRUE(tmpSlice.isNumber() && 64 == tmpSlice.getNumber()); + tmpSlice = slice.get("writebufferSizeMax"); + EXPECT_TRUE(tmpSlice.isNumber() && 32 * (size_t(1) << 20) == tmpSlice.getNumber()); + tmpSlice = slice.get("primarySort"); + EXPECT_TRUE(tmpSlice.isArray()); + EXPECT_EQ(0, tmpSlice.length()); + tmpSlice = slice.get("version"); + EXPECT_TRUE(tmpSlice.isNumber() && 1 == tmpSlice.getNumber()); + } + + // check serialization for inventory + { + VPackSlice tmpSlice, tmpSlice2; + + arangodb::velocypack::Builder builder; + builder.openObject(); + logicalView->properties(builder, arangodb::LogicalDataSource::Serialization::Inventory); + builder.close(); + + auto slice = builder.slice(); + EXPECT_TRUE(slice.isObject()); + EXPECT_EQ(13, slice.length()); + EXPECT_TRUE(slice.get("name").isString() && "testView" == slice.get("name").copyString()); + EXPECT_TRUE(slice.get("type").isString() && "arangosearch" == slice.get("type").copyString()); + EXPECT_TRUE(slice.get("id").isString() && "101" == slice.get("id").copyString()); + EXPECT_TRUE(slice.get("globallyUniqueId").isString() && !slice.get("globallyUniqueId").copyString().empty()); + EXPECT_TRUE(slice.get("consolidationIntervalMsec").isNumber() && 10000 == slice.get("consolidationIntervalMsec").getNumber()); + EXPECT_TRUE(slice.get("cleanupIntervalStep").isNumber() && 2 == slice.get("cleanupIntervalStep").getNumber()); + EXPECT_TRUE(slice.get("commitIntervalMsec").isNumber() && 1000 == slice.get("commitIntervalMsec").getNumber()); + { // consolidation policy + tmpSlice = slice.get("consolidationPolicy"); + EXPECT_TRUE(tmpSlice.isObject() && 6 == tmpSlice.length()); + tmpSlice2 = tmpSlice.get("type"); + EXPECT_TRUE(tmpSlice2.isString() && std::string("tier") == tmpSlice2.copyString()); + tmpSlice2 = tmpSlice.get("segmentsMin"); + EXPECT_TRUE(tmpSlice2.isNumber() && 1 == tmpSlice2.getNumber()); + tmpSlice2 = tmpSlice.get("segmentsMax"); + EXPECT_TRUE(tmpSlice2.isNumber() && 10 == tmpSlice2.getNumber()); + tmpSlice2 = tmpSlice.get("segmentsBytesFloor"); + EXPECT_TRUE(tmpSlice2.isNumber() && (size_t(2) * (1 << 20)) == tmpSlice2.getNumber()); + tmpSlice2 = tmpSlice.get("segmentsBytesMax"); + EXPECT_TRUE(tmpSlice2.isNumber() && (size_t(5) * (1 << 30)) == tmpSlice2.getNumber()); + tmpSlice2 = tmpSlice.get("minScore"); + EXPECT_TRUE(tmpSlice2.isNumber() && (0. == tmpSlice2.getNumber())); + } + tmpSlice = slice.get("writebufferActive"); + EXPECT_TRUE(tmpSlice.isNumber() && 0 == tmpSlice.getNumber()); + tmpSlice = slice.get("writebufferIdle"); + EXPECT_TRUE(tmpSlice.isNumber() && 64 == tmpSlice.getNumber()); + tmpSlice = slice.get("writebufferSizeMax"); + EXPECT_TRUE(tmpSlice.isNumber() && 32 * (size_t(1) << 20) == tmpSlice.getNumber()); + tmpSlice = slice.get("primarySort"); + EXPECT_TRUE(tmpSlice.isArray()); + EXPECT_EQ(0, tmpSlice.length()); + { // links + tmpSlice = slice.get("links"); + EXPECT_TRUE(tmpSlice.isObject()); + EXPECT_EQ(1, tmpSlice.length()); + tmpSlice2 = tmpSlice.get("testCollection"); + EXPECT_TRUE(tmpSlice2.isObject()); + EXPECT_EQ(7, tmpSlice2.length()); + EXPECT_TRUE(tmpSlice2.get("analyzers").isArray() && + 1 == tmpSlice2.get("analyzers").length() && + "inPlace" == tmpSlice2.get("analyzers").at(0).copyString()); + EXPECT_TRUE(tmpSlice2.get("fields").isObject() && 0 == tmpSlice2.get("fields").length()); + EXPECT_TRUE(tmpSlice2.get("includeAllFields").isBool() && tmpSlice2.get("includeAllFields").getBool()); + EXPECT_TRUE(tmpSlice2.get("trackListPositions").isBool() && !tmpSlice2.get("trackListPositions").getBool()); + EXPECT_TRUE(tmpSlice2.get("storeValues").isString() && "none" == tmpSlice2.get("storeValues").copyString()); + + tmpSlice2 = tmpSlice2.get("analyzerDefinitions"); + ASSERT_TRUE(tmpSlice2.isArray()); + ASSERT_EQ(1, tmpSlice2.length()); + tmpSlice2 = tmpSlice2.at(0); + ASSERT_TRUE(tmpSlice2.isObject()); + EXPECT_EQ(4, tmpSlice2.length()); + EXPECT_TRUE(tmpSlice2.get("name").isString() && "inPlace" == tmpSlice2.get("name").copyString()); + EXPECT_TRUE(tmpSlice2.get("type").isString() && "identity" == tmpSlice2.get("type").copyString()); + EXPECT_TRUE(tmpSlice2.get("properties").isObject() && 0 == tmpSlice2.get("properties").length()); + EXPECT_TRUE(tmpSlice2.get("features").isArray() && 0 == tmpSlice2.get("features").length()); + } + } +} + TEST_F(IResearchViewCoordinatorTest, test_overwrite_immutable_properties) { auto* ci = arangodb::ClusterInfo::instance(); ASSERT_NE(nullptr, ci); diff --git a/tests/js/server/dump/dump-mmfiles-cluster.js b/tests/js/server/dump/dump-mmfiles-cluster.js index 61ba2a557148..4993391c9c03 100644 --- a/tests/js/server/dump/dump-mmfiles-cluster.js +++ b/tests/js/server/dump/dump-mmfiles-cluster.js @@ -758,10 +758,7 @@ function dumpTestEnterpriseSuite () { let props = view.properties(); assertEqual("UnitTestsDumpSmartView", view.name()); assertTrue(props.hasOwnProperty("links")); - // FIXME: currently view restoration is broken - // we must restore 4 collections (virtual + 3 system) - //assertEqual(Object.keys(props.links).length, 4); // virtual collecion + 3 system collections - assertEqual(Object.keys(props.links).length, 1); + assertEqual(Object.keys(props.links).length, 4); // virtual collecion + 3 system collections // UnitTestDumpSmartEdges assertTrue(props.links.hasOwnProperty("UnitTestDumpSmartEdges")); @@ -773,40 +770,39 @@ function dumpTestEnterpriseSuite () { assertTrue("text_en", props.links.UnitTestDumpSmartEdges.fields.text.analyzers[0]); assertTrue("UnitTestsDumpView::smartCustom", props.links.UnitTestDumpSmartEdges.fields.text.analyzers[1]); - // FIXME: currently view restoration is broken for system collections - //// _to_UnitTestDumpSmartEdges - //assertTrue(props.links.hasOwnProperty("_to_UnitTestDumpSmartEdges")); - //assertTrue(props.links._to_UnitTestDumpSmartEdges.hasOwnProperty("includeAllFields")); - //assertTrue(props.links._to_UnitTestDumpSmartEdges.includeAllFields); - //assertTrue(props.links._to_UnitTestDumpSmartEdges.hasOwnProperty("fields")); - //assertEqual(Object.keys(props.links._to_UnitTestDumpSmartEdges.fields).length, 1); - //assertTrue(props.links._to_UnitTestDumpSmartEdges.fields.text.analyzers.length, 2); - //assertTrue("text_en", props.links._to_UnitTestDumpSmartEdges.fields.text.analyzers[0]); - //assertTrue("UnitTestsDumpView::smartCustom", props.links._to_UnitTestDumpSmartEdges.fields.text.analyzers[1]); - - //// _from_UnitTestDumpSmartEdges - //assertTrue(props.links.hasOwnProperty("_from_UnitTestDumpSmartEdges")); - //assertTrue(props.links._from_UnitTestDumpSmartEdges.hasOwnProperty("includeAllFields")); - //assertTrue(props.links._from_UnitTestDumpSmartEdges.includeAllFields); - //assertTrue(props.links._from_UnitTestDumpSmartEdges.hasOwnProperty("fields")); - //assertEqual(Object.keys(props.links._from_UnitTestDumpSmartEdges.fields).length, 1); - //assertTrue(props.links._from_UnitTestDumpSmartEdges.fields.text.analyzers.length, 2); - //assertTrue("text_en", props.links._from_UnitTestDumpSmartEdges.fields.text.analyzers[0]); - //assertTrue("UnitTestsDumpView::smartCustom", props.links._from_UnitTestDumpSmartEdges.fields.text.analyzers[1]); - - //// _local_UnitTestDumpSmartEdges - //assertTrue(props.links.hasOwnProperty("_local_UnitTestDumpSmartEdges")); - //assertTrue(props.links._local_UnitTestDumpSmartEdges.hasOwnProperty("includeAllFields")); - //assertTrue(props.links._local_UnitTestDumpSmartEdges.includeAllFields); - //assertTrue(props.links._local_UnitTestDumpSmartEdges.hasOwnProperty("fields")); - //assertEqual(Object.keys(props.links._local_UnitTestDumpSmartEdges.fields).length, 1); - //assertTrue(props.links._local_UnitTestDumpSmartEdges.fields.text.analyzers.length, 2); - //assertTrue("text_en", props.links._local_UnitTestDumpSmartEdges.fields.text.analyzers[0]); - //assertTrue("UnitTestsDumpView::smartCustom", props.links._local_UnitTestDumpSmartEdges.fields.text.analyzers[1]); - //assertEqual(props.consolidationIntervalMsec, 0); - //assertEqual(props.cleanupIntervalStep, 456); - //assertTrue(Math.abs(props.consolidationPolicy.threshold - 0.3) < 0.001); - //assertEqual(props.consolidationPolicy.type, "bytes_accum"); + // _to_UnitTestDumpSmartEdges + assertTrue(props.links.hasOwnProperty("_to_UnitTestDumpSmartEdges")); + assertTrue(props.links._to_UnitTestDumpSmartEdges.hasOwnProperty("includeAllFields")); + assertTrue(props.links._to_UnitTestDumpSmartEdges.includeAllFields); + assertTrue(props.links._to_UnitTestDumpSmartEdges.hasOwnProperty("fields")); + assertEqual(Object.keys(props.links._to_UnitTestDumpSmartEdges.fields).length, 1); + assertTrue(props.links._to_UnitTestDumpSmartEdges.fields.text.analyzers.length, 2); + assertTrue("text_en", props.links._to_UnitTestDumpSmartEdges.fields.text.analyzers[0]); + assertTrue("UnitTestsDumpView::smartCustom", props.links._to_UnitTestDumpSmartEdges.fields.text.analyzers[1]); + + // _from_UnitTestDumpSmartEdges + assertTrue(props.links.hasOwnProperty("_from_UnitTestDumpSmartEdges")); + assertTrue(props.links._from_UnitTestDumpSmartEdges.hasOwnProperty("includeAllFields")); + assertTrue(props.links._from_UnitTestDumpSmartEdges.includeAllFields); + assertTrue(props.links._from_UnitTestDumpSmartEdges.hasOwnProperty("fields")); + assertEqual(Object.keys(props.links._from_UnitTestDumpSmartEdges.fields).length, 1); + assertTrue(props.links._from_UnitTestDumpSmartEdges.fields.text.analyzers.length, 2); + assertTrue("text_en", props.links._from_UnitTestDumpSmartEdges.fields.text.analyzers[0]); + assertTrue("UnitTestsDumpView::smartCustom", props.links._from_UnitTestDumpSmartEdges.fields.text.analyzers[1]); + + // _local_UnitTestDumpSmartEdges + assertTrue(props.links.hasOwnProperty("_local_UnitTestDumpSmartEdges")); + assertTrue(props.links._local_UnitTestDumpSmartEdges.hasOwnProperty("includeAllFields")); + assertTrue(props.links._local_UnitTestDumpSmartEdges.includeAllFields); + assertTrue(props.links._local_UnitTestDumpSmartEdges.hasOwnProperty("fields")); + assertEqual(Object.keys(props.links._local_UnitTestDumpSmartEdges.fields).length, 1); + assertTrue(props.links._local_UnitTestDumpSmartEdges.fields.text.analyzers.length, 2); + assertTrue("text_en", props.links._local_UnitTestDumpSmartEdges.fields.text.analyzers[0]); + assertTrue("UnitTestsDumpView::smartCustom", props.links._local_UnitTestDumpSmartEdges.fields.text.analyzers[1]); + assertEqual(props.consolidationIntervalMsec, 0); + assertEqual(props.cleanupIntervalStep, 456); + assertTrue(Math.abs(props.consolidationPolicy.threshold - 0.3) < 0.001); + assertEqual(props.consolidationPolicy.type, "bytes_accum"); } }; diff --git a/tests/js/server/dump/dump-rocksdb-cluster.js b/tests/js/server/dump/dump-rocksdb-cluster.js index 87d7b17efb5b..728ead760372 100644 --- a/tests/js/server/dump/dump-rocksdb-cluster.js +++ b/tests/js/server/dump/dump-rocksdb-cluster.js @@ -752,10 +752,7 @@ function dumpTestEnterpriseSuite () { let props = view.properties(); assertEqual("UnitTestsDumpSmartView", view.name()); assertTrue(props.hasOwnProperty("links")); - // FIXME: currently view restoration is broken - // we must restore 4 collections (virtual + 3 system) - //assertEqual(Object.keys(props.links).length, 4); // virtual collecion + 3 system collections - assertEqual(Object.keys(props.links).length, 1); + assertEqual(Object.keys(props.links).length, 4); // virtual collecion + 3 system collections // UnitTestDumpSmartEdges assertTrue(props.links.hasOwnProperty("UnitTestDumpSmartEdges")); @@ -767,40 +764,39 @@ function dumpTestEnterpriseSuite () { assertTrue("text_en", props.links.UnitTestDumpSmartEdges.fields.text.analyzers[0]); assertTrue("UnitTestsDumpView::smartCustom", props.links.UnitTestDumpSmartEdges.fields.text.analyzers[1]); - // FIXME: currently view restoration is broken for system collections - //// _to_UnitTestDumpSmartEdges - //assertTrue(props.links.hasOwnProperty("_to_UnitTestDumpSmartEdges")); - //assertTrue(props.links._to_UnitTestDumpSmartEdges.hasOwnProperty("includeAllFields")); - //assertTrue(props.links._to_UnitTestDumpSmartEdges.includeAllFields); - //assertTrue(props.links._to_UnitTestDumpSmartEdges.hasOwnProperty("fields")); - //assertEqual(Object.keys(props.links._to_UnitTestDumpSmartEdges.fields).length, 1); - //assertTrue(props.links._to_UnitTestDumpSmartEdges.fields.text.analyzers.length, 2); - //assertTrue("text_en", props.links._to_UnitTestDumpSmartEdges.fields.text.analyzers[0]); - //assertTrue("UnitTestsDumpView::smartCustom", props.links._to_UnitTestDumpSmartEdges.fields.text.analyzers[1]); - - //// _from_UnitTestDumpSmartEdges - //assertTrue(props.links.hasOwnProperty("_from_UnitTestDumpSmartEdges")); - //assertTrue(props.links._from_UnitTestDumpSmartEdges.hasOwnProperty("includeAllFields")); - //assertTrue(props.links._from_UnitTestDumpSmartEdges.includeAllFields); - //assertTrue(props.links._from_UnitTestDumpSmartEdges.hasOwnProperty("fields")); - //assertEqual(Object.keys(props.links._from_UnitTestDumpSmartEdges.fields).length, 1); - //assertTrue(props.links._from_UnitTestDumpSmartEdges.fields.text.analyzers.length, 2); - //assertTrue("text_en", props.links._from_UnitTestDumpSmartEdges.fields.text.analyzers[0]); - //assertTrue("UnitTestsDumpView::smartCustom", props.links._from_UnitTestDumpSmartEdges.fields.text.analyzers[1]); - - //// _local_UnitTestDumpSmartEdges - //assertTrue(props.links.hasOwnProperty("_local_UnitTestDumpSmartEdges")); - //assertTrue(props.links._local_UnitTestDumpSmartEdges.hasOwnProperty("includeAllFields")); - //assertTrue(props.links._local_UnitTestDumpSmartEdges.includeAllFields); - //assertTrue(props.links._local_UnitTestDumpSmartEdges.hasOwnProperty("fields")); - //assertEqual(Object.keys(props.links._local_UnitTestDumpSmartEdges.fields).length, 1); - //assertTrue(props.links._local_UnitTestDumpSmartEdges.fields.text.analyzers.length, 2); - //assertTrue("text_en", props.links._local_UnitTestDumpSmartEdges.fields.text.analyzers[0]); - //assertTrue("UnitTestsDumpView::smartCustom", props.links._local_UnitTestDumpSmartEdges.fields.text.analyzers[1]); - //assertEqual(props.consolidationIntervalMsec, 0); - //assertEqual(props.cleanupIntervalStep, 456); - //assertTrue(Math.abs(props.consolidationPolicy.threshold - 0.3) < 0.001); - //assertEqual(props.consolidationPolicy.type, "bytes_accum"); + // _to_UnitTestDumpSmartEdges + assertTrue(props.links.hasOwnProperty("_to_UnitTestDumpSmartEdges")); + assertTrue(props.links._to_UnitTestDumpSmartEdges.hasOwnProperty("includeAllFields")); + assertTrue(props.links._to_UnitTestDumpSmartEdges.includeAllFields); + assertTrue(props.links._to_UnitTestDumpSmartEdges.hasOwnProperty("fields")); + assertEqual(Object.keys(props.links._to_UnitTestDumpSmartEdges.fields).length, 1); + assertTrue(props.links._to_UnitTestDumpSmartEdges.fields.text.analyzers.length, 2); + assertTrue("text_en", props.links._to_UnitTestDumpSmartEdges.fields.text.analyzers[0]); + assertTrue("UnitTestsDumpView::smartCustom", props.links._to_UnitTestDumpSmartEdges.fields.text.analyzers[1]); + + // _from_UnitTestDumpSmartEdges + assertTrue(props.links.hasOwnProperty("_from_UnitTestDumpSmartEdges")); + assertTrue(props.links._from_UnitTestDumpSmartEdges.hasOwnProperty("includeAllFields")); + assertTrue(props.links._from_UnitTestDumpSmartEdges.includeAllFields); + assertTrue(props.links._from_UnitTestDumpSmartEdges.hasOwnProperty("fields")); + assertEqual(Object.keys(props.links._from_UnitTestDumpSmartEdges.fields).length, 1); + assertTrue(props.links._from_UnitTestDumpSmartEdges.fields.text.analyzers.length, 2); + assertTrue("text_en", props.links._from_UnitTestDumpSmartEdges.fields.text.analyzers[0]); + assertTrue("UnitTestsDumpView::smartCustom", props.links._from_UnitTestDumpSmartEdges.fields.text.analyzers[1]); + + // _local_UnitTestDumpSmartEdges + assertTrue(props.links.hasOwnProperty("_local_UnitTestDumpSmartEdges")); + assertTrue(props.links._local_UnitTestDumpSmartEdges.hasOwnProperty("includeAllFields")); + assertTrue(props.links._local_UnitTestDumpSmartEdges.includeAllFields); + assertTrue(props.links._local_UnitTestDumpSmartEdges.hasOwnProperty("fields")); + assertEqual(Object.keys(props.links._local_UnitTestDumpSmartEdges.fields).length, 1); + assertTrue(props.links._local_UnitTestDumpSmartEdges.fields.text.analyzers.length, 2); + assertTrue("text_en", props.links._local_UnitTestDumpSmartEdges.fields.text.analyzers[0]); + assertTrue("UnitTestsDumpView::smartCustom", props.links._local_UnitTestDumpSmartEdges.fields.text.analyzers[1]); + assertEqual(props.consolidationIntervalMsec, 0); + assertEqual(props.cleanupIntervalStep, 456); + assertTrue(Math.abs(props.consolidationPolicy.threshold - 0.3) < 0.001); + assertEqual(props.consolidationPolicy.type, "bytes_accum"); } }; } From febeefd749c7c1bf562cc15116a83c7d107e87a9 Mon Sep 17 00:00:00 2001 From: Andrey Abramov Date: Tue, 19 Nov 2019 21:50:12 +0300 Subject: [PATCH 2/2] address compilation errors --- tests/IResearch/IResearchView-test.cpp | 4 ++-- tests/IResearch/IResearchViewCoordinator-test.cpp | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/IResearch/IResearchView-test.cpp b/tests/IResearch/IResearchView-test.cpp index 918c98e88a05..0c20735488e5 100644 --- a/tests/IResearch/IResearchView-test.cpp +++ b/tests/IResearch/IResearchView-test.cpp @@ -529,7 +529,7 @@ TEST_F(IResearchViewTest, test_properties) { "}" ); - Vocbase vocbase(TRI_vocbase_type_e::TRI_VOCBASE_TYPE_NORMAL, testDBInfo(server.server())); + Vocbase vocbase(TRI_vocbase_type_e::TRI_VOCBASE_TYPE_NORMAL, 1, "testVocbase"); auto logicalCollection = vocbase.createCollection(collectionJson->slice()); EXPECT_NE(nullptr, logicalCollection); EXPECT_EQ(nullptr, vocbase.lookupView("testView")); @@ -762,7 +762,7 @@ TEST_F(IResearchViewTest, test_vocbase_inventory) { "}" ); - Vocbase vocbase(TRI_vocbase_type_e::TRI_VOCBASE_TYPE_NORMAL, testDBInfo(server.server())); + Vocbase vocbase(TRI_vocbase_type_e::TRI_VOCBASE_TYPE_NORMAL, 1, "testVocbase"); auto logicalCollection = vocbase.createCollection(collectionJson->slice()); EXPECT_NE(nullptr, logicalCollection); EXPECT_EQ(nullptr, vocbase.lookupView("testView")); diff --git a/tests/IResearch/IResearchViewCoordinator-test.cpp b/tests/IResearch/IResearchViewCoordinator-test.cpp index ea61f4b506ee..eb30a5a5cc1e 100644 --- a/tests/IResearch/IResearchViewCoordinator-test.cpp +++ b/tests/IResearch/IResearchViewCoordinator-test.cpp @@ -961,7 +961,8 @@ TEST_F(IResearchViewCoordinatorTest, test_update_properties) { TEST_F(IResearchViewCoordinatorTest, test_properties) { - auto& ci = server.getFeature().clusterInfo(); + ASSERT_NE(nullptr, arangodb::ClusterInfo::instance()); + auto& ci = *arangodb::ClusterInfo::instance(); TRI_vocbase_t* vocbase; // will be owned by DatabaseFeature