8000 allow in-place analyzer creation via link definition (#10466) by gnusi · Pull Request #10481 · arangodb/arangodb · GitHub
[go: up one dir, main page]

Skip to content

allow in-place analyzer creation via link definition (#10466) #10481

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Nov 19, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions arangod/IResearch/IResearchCommon.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -52,10 +52,12 @@ arangodb::LogTopic& logTopic() {
/*static*/ std::string const StaticStrings::LinksField("links");
/*static*/ std::string const StaticStrings::VersionField("version");
/*static*/ std::string const StaticStrings::ViewIdField("view");
/*static*/ std::string const StaticStrings::AnalyzerDefinitionsField("analyzerDefinitions");
/*static*/ std::string const StaticStrings::AnalyzerFeaturesField("features");
/*static*/ std::string const StaticStrings::AnalyzerNameField("name");
/*static*/ std::string const StaticStrings::AnalyzerPropertiesField("properties");
/*static*/ std::string const StaticStrings::AnalyzerTypeField("type");
/*static*/ std::string const StaticStrings::PrimarySortField("primarySort");

} // namespace iresearch
} // namespace arangodb
Expand Down
12 changes: 12 additions & 0 deletions arangod/IResearch/IResearchCommon.h
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,12 @@ struct StaticStrings {
////////////////////////////////////////////////////////////////////////////////
static std::string const ViewIdField;

////////////////////////////////////////////////////////////////////////////////
/// @brief the name of the field in the IResearch Link definition denoting the
/// referenced analyzer definitions
////////////////////////////////////////////////////////////////////////////////
static std::string const AnalyzerDefinitionsField;

////////////////////////////////////////////////////////////////////////////////
/// @brief the name of the field in the analyzer definition denoting the
/// corresponding analyzer name
Expand All @@ -84,6 +90,12 @@ struct StaticStrings {
/// corresponding analyzer features
////////////////////////////////////////////////////////////////////////////////
static std::string const AnalyzerFeaturesField;

////////////////////////////////////////////////////////////////////////////////
/// @brief the name of the field in the IResearch Link definition denoting the
/// primary sort
////////////////////////////////////////////////////////////////////////////////
static std::string const PrimarySortField;
};

} // namespace iresearch
Expand Down
2 changes: 1 addition & 1 deletion arangod/IResearch/IResearchLinkHelper.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -783,7 +783,7 @@ namespace iresearch {
std::string errorField;

if (!linkDefinition.isNull()) { // have link definition
if (!meta.init(linkDefinition, false, errorField, &vocbase)) { // for db-server analyzer validation should have already applied on coordinator
if (!meta.init(linkDefinition, true, errorField, &vocbase)) { // for db-server analyzer validation should have already applied on coordinator
return arangodb::Result( // result
TRI_ERROR_BAD_PARAMETER, // code
errorField.empty()
Expand Down
20 changes: 7 additions & 13 deletions arangod/IResearch/IResearchView.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -363,12 +363,11 @@ arangodb::Result IResearchView::appendVelocyPackImpl( // append JSON
static const std::function<bool(irs::string_ref const& key)> persistenceAcceptor =
[](irs::string_ref const&) -> bool { return true; };

auto& acceptor =
(context == Serialization::Persistence || context == Serialization::PersistenceWithInProgress || context == Serialization::Inventory)
? persistenceAcceptor
: propertiesAcceptor;
auto* acceptor = &propertiesAcceptor;

if (context == Serialization::Persistence || context == Serialization::PersistenceWithInProgress) {
acceptor = &persistenceAcceptor;

if (context == Serialization::Persistence || context == Serialization::PersistenceWithInProgress) {
if (arangodb::ServerState::instance()->isSingleServer()) {
auto res = arangodb::LogicalViewHelperStorageEngine::properties(builder, *this);

Expand All @@ -392,19 +391,14 @@ arangodb::Result IResearchView::appendVelocyPackImpl( // append JSON
sanitizedBuilder.openObject();

if (!_meta.json(sanitizedBuilder) ||
!mergeSliceSkipKeys(builder, sanitizedBuilder.close().slice(), acceptor)) {
!mergeSliceSkipKeys(builder, sanitizedBuilder.close().slice(), *acceptor)) {
return arangodb::Result(
TRI_ERROR_INTERNAL,
std::string("failure to generate definition while generating "
"properties jSON for arangosearch View in database '") +
vocbase().name() + "'");
}

if (context == Serialization::Inventory) {
// nothing more to output
return {};
}

if (context == Serialization::Persistence || context == Serialization::PersistenceWithInProgress) {
IResearchViewMetaState metaState;

Expand Down Expand Up @@ -461,7 +455,7 @@ arangodb::Result IResearchView::appendVelocyPackImpl( // append JSON
);
}

auto visitor = [this, &linksBuilder, &res]( // visit collections
auto visitor = [this, &linksBuilder, &res, context]( // visit collections
arangodb::TransactionCollection& trxCollection // transaction collection
)->bool {
auto collection = trxCollection.collection();
Expand All @@ -480,7 +474,7 @@ arangodb::Result IResearchView::appendVelocyPackImpl( // append JSON

linkBuilder.openObject();

if (!link->properties(linkBuilder, false).ok()) { // link definitions are not output if forPersistence
if (!link->properties(linkBuilder, Serialization::Inventory == context).ok()) { // link definitions are not output if forPersistence
LOG_TOPIC("713ad", WARN, arangodb::iresearch::TOPIC)
<< "failed to generate json for arangosearch link '" << link->id() << "' while generating json for arangosearch view '" << name() << "'";

Expand Down
49 changes: 34 additions & 15 deletions arangod/IResearch/IResearchViewCoordinator.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -181,28 +181,34 @@ arangodb::Result IResearchViewCoordinator::appendVelocyPackImpl(
return {};
}

static const std::function<bool(irs::string_ref const& key)> propertiesAcceptor =
static const std::function<bool(irs::string_ref const&)> propertiesAcceptor =
[](irs::string_ref const& key) -> bool {
return key != StaticStrings::VersionField; // ignored fields
};
static const std::function<bool(irs::string_ref const& key)> persistenceAcceptor =
static const std::function<bool(irs::string_ref const&)> persistenceAcceptor =
[](irs::string_ref const&) -> bool { return true; };

static const std::function<bool(irs::string_ref const&)> linkPropertiesAcceptor =
[](irs::string_ref const& key) -> bool {
return key != iresearch::StaticStrings::AnalyzerDefinitionsField
&& key != iresearch::StaticStrings::PrimarySortField;
};

auto* acceptor = &propertiesAcceptor;

if (context == Serialization::Persistence ||
context == Serialization::PersistenceWithInProgress ||
context == Serialization::Inventory) {
context == Serialization::PersistenceWithInProgress) {
auto res = arangodb::LogicalViewHelperClusterInfo::properties(builder, *this);

if (!res.ok()) {
return res;
}

acceptor = &persistenceAcceptor;
// links are not persisted, their definitions are part of the corresponding
// collections
} else if (context == Serialization::Properties) {
}

if (context == Serialization::Properties ||
context == Serialization::Inventory) {
// verify that the current user has access on all linked collections
auto* exec = ExecContext::CURRENT;
if (exec) {
Expand All @@ -213,18 +219,31 @@ arangodb::Result IResearchViewCoordinator::appendVelocyPackImpl(
}
}

VPackBuilder tmp;

ReadMutex mutex(_mutex);
SCOPED_LOCK(mutex); // '_collections' can be asynchronously modified

VPackBuilder links;
links.openObject();

builder.add(StaticStrings::LinksField, VPackValue(VPackValueType::Object));
for (auto& entry : _collections) {
links.add(entry.second.first, entry.second.second.slice());
}
auto linkSlice = entry.second.second.slice();

if (context == Serialization::Properties) {
tmp.clear();
tmp.openObject();
if (!mergeSliceSkipKeys(tmp, linkSlice, linkPropertiesAcceptor)) {
return {
TRI_ERROR_INTERNAL,
"failed to generate externally visible link definition for arangosearch View '" + name() + "'"
};
}

links.close();
builder.add(StaticStrings::LinksField, links.slice());
linkSlice = tmp.close().slice();
}

builder.add(entry.second.first, linkSlice);
}
builder.close();
}

if (!builder.isOpenObject()) {
Expand Down Expand Up @@ -266,7 +285,7 @@ arangodb::Result IResearchViewCoordinator::link(IResearchLink const& link) {

builder.openObject();

auto res = link.properties(builder, false); // generate user-visible definition, agency will not see links
auto res = link.properties(builder, true); // generate user-visible definition, agency will not see links

if (!res.ok()) {
return res;
Expand Down
60 changes: 40 additions & 20 deletions arangod/Replication/DatabaseInitialSyncer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1359,6 +1359,7 @@ Result DatabaseInitialSyncer::handleCollectionsAndViews(VPackSlice const& collSl
bool incremental) {
TRI_ASSERT(collSlices.isArray());

std::vector<std::pair<VPackSlice, VPackSlice>> systemCollections;
std::vector<std::pair<VPackSlice, VPackSlice>> collections;
for (VPackSlice it : VPackArrayIterator(collSlices)) {
if (!it.isObject()) {
Expand Down Expand Up @@ -1414,10 +1415,46 @@ Result DatabaseInitialSyncer::handleCollectionsAndViews(VPackSlice const& collSl
}
}

collections.emplace_back(parameters, indexes);
if (masterName == StaticStrings::AnalyzersCollection) {
// _analyzers collection has to be restored before view creation
systemCollections.emplace_back(parameters, indexes);
} else {
collections.emplace_back(parameters, indexes);
}
}

// STEP 1: validate collection declarations from master
// ----------------------------------------------------------------------------------

// STEP 2: drop and re-create collections locally if they are also present on
// the master
// ------------------------------------------------------------------------------------

// iterate over all collections from the master...
std::array<SyncPhase, 2> phases{{PHASE_VALIDATE, PHASE_DROP_CREATE}};
for (auto const& phase : phases) {
Result r = iterateCollections(systemCollections, incremental, phase);

if (r.fail()) {
return r;
}

r = iterateCollections(collections, incremental, phase);

if (r.fail()) {
return r;
}
}

// STEP 1: now that the collections exist create the views
// STEP 3: restore data for system collections
// ----------------------------------------------------------------------------------
auto const res = iterateCollections(systemCollections, incremental, PHASE_DUMP);

if (res.fail()) {
return res;
}

// STEP 4: now that the collections exist create the views
// this should be faster than re-indexing afterwards
// ----------------------------------------------------------------------------------

Expand All @@ -1435,24 +1472,7 @@ Result DatabaseInitialSyncer::handleCollectionsAndViews(VPackSlice const& collSl
_config.progress.set("view creation skipped because of configuration");
}

// STEP 2: validate collection declarations from master
// ----------------------------------------------------------------------------------

// STEP 3: drop and re-create collections locally if they are also present on
// the master
// ------------------------------------------------------------------------------------

// iterate over all collections from the master...
std::array<SyncPhase, 2> phases{{PHASE_VALIDATE, PHASE_DROP_CREATE}};
for (auto const& phase : phases) {
Result r = iterateCollections(collections, incremental, phase);

if (r.fail()) {
return r;
}
}

// STEP 4: sync collection data from master and create initial indexes
// STEP 5: sync collection data from master and create initial indexes
// ----------------------------------------------------------------------------------

// now load the data into the collections
Expand Down
3 changes: 0 additions & 3 deletions arangod/VocBase/LogicalCollection.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -614,9 +614,6 @@ void LogicalCollection::toVelocyPackForClusterInventory(VPackBuilder& result,
case Index::TRI_IDX_TYPE_PRIMARY_INDEX:
case Index::TRI_IDX_TYPE_EDGE_INDEX:
return false;
case Index::TRI_IDX_TYPE_IRESEARCH_LINK:
flags = Index::makeFlags(Index::Serialize::Internals);
return true;
default:
flags = Index::makeFlags();
return !idx->isHidden() && !idx->inProgress();
Expand Down
3 changes: 0 additions & 3 deletions arangod/VocBase/vocbase.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -939,9 +939,6 @@ void TRI_vocbase_t::inventory(VPackBuilder& result, TRI_voc_tick_t maxTick,
case Index::TRI_IDX_TYPE_PRIMARY_INDEX:
case Index::TRI_IDX_TYPE_EDGE_INDEX:
return false;
case Index::TRI_IDX_TYPE_IRESEARCH_LINK:
flags = Index::makeFlags(Index::Serialize::Internals);
return true;
default:
flags = Index::makeFlags(Index::Serialize::Basics);
return !idx->isHidden();
Expand Down
54 changes: 34 additions & 20 deletions arangosh/Restore/RestoreFeature.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -903,27 +903,10 @@ arangodb::Result processInputDirectory(
std::sort(collections.begin(), collections.end(), ::sortCollectionsForCreation);

std::unique_ptr<arangodb::RestoreFeature::JobData> usersData;
std::unique_ptr<arangodb::RestoreFeature::JobData> analyzersData;
std::vector<std::unique_ptr<arangodb::RestoreFeature::JobData>> jobs;
jobs.reserve(collections.size());

// Step 2: create views
// @note: done after collection population since views might depend on data
// in restored collections
if (options.importStructure && !views.empty()) {
LOG_TOPIC("f723c", INFO, Logger::RESTORE) << "# Creating views...";

for (auto const& viewDefinition : views) {
LOG_TOPIC("c608d", DEBUG, Logger::RESTORE)
<< "# Creating view: " << viewDefinition.toJson();

auto res = ::restoreView(httpClient, options, viewDefinition.slice());

if (!res.ok()) {
return res;
}
}
}

bool didModifyFoxxCollection = false;
// Step 3: create collections
for (VPackBuilder const& b : collections) {
Expand Down Expand Up @@ -961,13 +944,43 @@ arangodb::Result processInputDirectory(
// reason is that loading into the users collection may change the
// credentials for the current arangorestore connection!
usersData = std::move(jobData);
} else if (name.isString() && name.stringRef() == StaticStrings::AnalyzersCollection) {
// special treatment for _analyzers collection - this must be the very first
stats.totalCollections++;
analyzersData = std::move(jobData);
} else {
stats.totalCollections++;
jobs.push_back(std::move(jobData));
}
}

// Step 4: fire up data transfer

// Step 4: restore data from _analyzers collection
if (analyzersData) {
// restore analyzers
if (!jobQueue.queueJob(std::move(analyzersData))) {
return Result(TRI_ERROR_OUT_OF_MEMORY, "unable to queue restore job");
}

jobQueue.waitForIdle();
}

// Step 5: create arangosearch views
if (options.importStructure && !views.empty()) {
LOG_TOPIC("f723c", INFO, Logger::RESTORE) << "# Creating views...";

for (auto const& viewDefinition : views) {
LOG_TOPIC("c608d", DEBUG, Logger::RESTORE)
<< "# Creating view: " << viewDefinition.toJson();

auto res = ::restoreView(httpClient, options, viewDefinition.slice());

if (!res.ok()) {
return res;
}
}
}

// Step 6: fire up data transfer
for (auto& job : jobs) {
if (!jobQueue.queueJob(std::move(job))) {
return Result(TRI_ERROR_OUT_OF_MEMORY, "unable to queue restore job");
Expand Down Expand Up @@ -1045,6 +1058,7 @@ arangodb::Result processInputDirectory(
return firstError;
}
}

} catch (std::exception const& ex) {
return {TRI_ERROR_INTERNAL,
std::string(
Expand Down
Loading
0