8000 issue 511.4: ensure IResearchLink can be initialized without view as … · HighAvailabilityLab/arangodb@8e7f0df · GitHub
[go: up one dir, main page]
Skip to content

Commit 8e7f0df

Browse files
vasiliy-arangodbAndrey Abramov
vasiliy-arangodb
authored and
Andrey Abramov
committed
issue 511.4: ensure IResearchLink can be initialized without view as required for db-server, minor code fixes and cleanup (arangodb#7786)
* issue 511.4: ensure IResearchLink can be initialized without view as required for db-server, minor code fixes and cleanup * remove unnessesary checks * revert last change due to test failures * fix typo in change revert * try to address random test failures
1 parent 0cff2bd commit 8e7f0df

19 files changed

+429
-526
lines changed

‎3rdParty/iresearch/core/index/index_writer.hpp

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -439,6 +439,21 @@ class IRESEARCH_API index_writer:
439439
}
440440
}; // segment_equal
441441

442+
//////////////////////////////////////////////////////////////////////////////
443+
/// @brief limits the the writer should use for segments
444+
//////////////////////////////////////////////////////////////////////////////
445+
// FIXME TODO inherit options from segment_limits
446+
struct segment_limits {
447+
size_t segment_count_max; // @see options::max_segment_count
448+
size_t segment_docs_max; // @see options::max_segment_docs
449+
size_t segment_memory_max; // @see options::max_segment_memory
450+
segment_limits(const options& opts) NOEXCEPT
451+
: segment_count_max(opts.segment_count_max),
452+
segment_docs_max(opts.segment_docs_max),
453+
segment_memory_max(opts.segment_memory_max) {
454+
}
455+
};
456+
442457
// works faster than std::unordered_set<string_ref>
443458
typedef std::unordered_set<
444459
const segment_meta*,
@@ -780,17 +795,6 @@ class IRESEARCH_API index_writer:
780795
void reset() NOEXCEPT;
781796
};
782797

783-
struct segment_limits {
784-
size_t segment_count_max; // @see options::max_segment_count
785-
size_t segment_docs_max; // @see options::max_segment_docs
786-
size_t segment_memory_max; // @see options::max_segment_memory
787-
segment_limits(const options& opts) NOEXCEPT
788-
: segment_count_max(opts.segment_count_max),
789-
segment_docs_max(opts.segment_docs_max),
790-
segment_memory_max(opts.segment_memory_max) {
791-
}
792-
};
793-
794798
typedef std::shared_ptr<
795799
std::pair<std::shared_ptr<index_meta>,
796800
file_refs_t

‎arangod/IResearch/IResearchFeature.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,9 @@ NS_END // arangodb
7272

7373
NS_LOCAL
7474

75+
typedef irs::async_utils::read_write_mutex::read_mutex ReadMutex;
76+
typedef irs::async_utils::read_write_mutex::write_mutex WriteMutex;
77+
7578
class IResearchLogTopic final : public arangodb::LogTopic {
7679
public:
7780
IResearchLogTopic(std::string const& name)

‎arangod/IResearch/IResearchLink.cpp

Lines changed: 162 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
#include "IResearchPrimaryKeyFilter.h"
2929
#include "IResearchView.h"
3030
#include "IResearchViewCoordinator.h"
31+
#include "Aql/QueryCache.h"
3132
#include "Basics/LocalTaskQueue.h"
3233
#include "Basics/StaticStrings.h"
3334
#include "Cluster/ClusterInfo.h"
@@ -144,21 +145,20 @@ size_t directoryMemory(
144145
////////////////////////////////////////////////////////////////////////////////
145146
irs::utf8_path getPersistedPath(
146147
arangodb::DatabasePathFeature const& dbPathFeature,
147-
arangodb::LogicalCollection const& collection,
148-
arangodb::LogicalView const& view
148+
arangodb::iresearch::IResearchLink const& link
149149
) {
150150
irs::utf8_path dataPath(dbPathFeature.directory());
151151
static const std::string subPath("databases");
152152
static const std::string dbPath("database-");
153153

154154
dataPath /= subPath;
155155
dataPath /= dbPath;
156-
dataPath += std::to_string(collection.vocbase().id());
156+
dataPath += std::to_string(link.collection().vocbase().id 8000 ());
157157
dataPath /= arangodb::iresearch::DATA_SOURCE_TYPE.name();
158158
dataPath += "-";
159-
dataPath += std::to_string(collection.id()); // has to be 'id' since this can be a per-shard collection
159+
dataPath += std::to_string(link.collection().id()); // has to be 'id' since this can be a per-shard collection
160160
dataPath += "_";
161-
dataPath += std::to_string(view.planId()); // has to be 'planId' since this is a cluster-wide view
161+
dataPath += std::to_string(link.id());
162162

163163
return dataPath;
164164
}
@@ -430,9 +430,20 @@ arangodb::Result IResearchLink::commit() {
430430
return arangodb::Result();
431431
}
432432

433-
if (_dataStore._reader != reader) {
434-
_dataStore._reader = reader; // update reader
433+
if (_dataStore._reader == reader) {
434+
return arangodb::Result(); // reader not modified
435435
}
436+
437+
_dataStore._reader = reader; // update reader
438+
439+
auto viewImpl = view();
440+
441+
// invalidate query cache if there were some data changes
442+
if (viewImpl) {
443+
arangodb::aql::QueryCache::instance()->invalidate(
444+
&(_collection.vocbase()), viewImpl->name()
445+
);
446+
}
436447
} catch (arangodb::basics::Exception const& e) {
437448
return arangodb::Result(
438449
e.code(),
@@ -622,84 +633,160 @@ arangodb::Result IResearchLink::init(
622633
);
623634
}
624635

625-
// we continue to support the old and new ID format
626-
auto idSlice = definition.get(StaticStrings::ViewIdField);
627-
auto viewId = idSlice.copyString();
636+
auto viewId = definition.get(StaticStrings::ViewIdField).copyString();
628637
auto& vocbase = _collection.vocbase();
629-
auto logicalView = arangodb::ServerState::instance()->isClusterRole()
630-
&& arangodb::ClusterInfo::instance()
631-
? arangodb::ClusterInfo::instance()->getView(vocbase.name(), viewId)
632-
: vocbase.lookupView(viewId)
633-
;
634-
635-
if (!logicalView
636-
|| arangodb::iresearch::DATA_SOURCE_TYPE != logicalView->type()) {
637-
return arangodb::Result(
638-
TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND,
639-
std::string("error finding view: '") + viewId + "' for link '" + std::to_string(_id) + "' : no such view"
640-
);
641-
}
642638

643-
if (arangodb::ServerState::instance()->isCoordinator()) {
644-
auto* view = LogicalView::cast<IResearchViewCoordinator>(logicalView.get());
639+
if (arangodb::ServerState::instance()->isCoordinator()) { // coordinator link
640+
auto* engine = arangodb::ClusterInfo::instance();
645641

646-
if (!view) {
642+
if (!engine) {
647643
return arangodb::Result(
648-
TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND,
649-
std::string("error finding view: '") + viewId + "' for link '" + std::to_string(_id) + "'"
644+
TRI_ERROR_INTERNAL,
645+
std::string("failure to get storage engine while initializing arangosearch link '") + std::to_string(_id) + "'"
650646
);
651647
}
652648

653-
if (!view->emplace(_collection.id(), _collection.name(), definition)) {
654-
return arangodb::Result(
655-
TRI_ERROR_INTERNAL,
656-
std::string("failed to link with view '") + view->name() + "' while initializing link '" + std::to_string(_id) + "'"
649+
auto logicalView = engine->getView(vocbase.name(), viewId);
650+
651+
// if there is no logicalView present yet then skip this step
652+
if (logicalView) {
653+
if (arangodb::iresearch::DATA_SOURCE_TYPE != logicalView->type()) {
654+
return arangodb::Result(
655+
TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND,
656+
std::string("error finding view: '") + viewId + "' for link '" + std::to_string(_id) + "' : no such view"
657+
);
658+
}
659+
660+
auto* view = arangodb::LogicalView::cast<IResearchViewCoordinator>(
661+
logicalView.get()
657662
);
663+
664+
if (!view) {
665+
return arangodb::Result(
666+
TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND,
667+
std::string("error finding view: '") + viewId + "' for link '" + std::to_string(_id) + "'"
668+
);
669+
}
670+
671+
viewId = view->guid(); // ensue that this is a GUID (required by operator==(IResearchView))
672+
673+
if (!view->emplace(_collection.id(), _collection.name(), definition)) {
674+
return arangodb::Result(
675+
TRI_ERROR_INTERNAL,
676+
std::string("failed to link with view '") + view->name() + "' while initializing link '" + std::to_string(_id) + "'"
677+
);
678+
}
658679
}
659-
} else {
660-
auto* view = LogicalView::cast<IResearchView>(logicalView.get());
680+
} else if (arangodb::ServerState::instance()->isDBServer()) { // db-server link
681+
auto* engine = arangodb::ClusterInfo::instance();
661682

662-
if (!view) {
683+
if (!engine) {
663684
return arangodb::Result(
664-
TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND,
665-
std::string("error finding view: '") + viewId + "' for link '" + std::to_string(_id) + "'"
685+
TRI_ERROR_INTERNAL,
686+
std::string("failure to get storage engine while initializing arangosearch link '") + std::to_string(_id) + "'"
666687
);
667688
}
668689

669-
if (arangodb::ServerState::instance()->isDBServer()
670-
&& _collection.id() == _collection.planId()
671-
&& _collection.isAStub()) { // cluster cluster-wide link
672-
auto shardIds = _collection.shardIds();
690+
auto clusterWideLink = _collection.id() == _collection.planId() && _collection.isAStub(); // cluster cluster-wide link
673691

674-
// go through all shard IDs of the collection and try to link any links
675-
// missing links will be populated when they are created in the per-shard collection
676-
if (shardIds) {
677-
for (auto& entry: *shardIds) {
678-
auto collection = vocbase.lookupCollection(entry.first); // per-shard collections are always in 'vocbase'
692+
if (!clusterWideLink) {
693+
auto res = initDataStore(); // prepare data-store which can then update options via the IResearchView::link(...) call
679694

680-
if (!collection) {
681-
continue; // missing collection should be created after Plan becomes Current
682-
}
695+
if (!res.ok()) {
696+
return res;
697+
}
698+
}
699+
700+
auto logicalView = engine->getView(vocbase.name(), viewId); // valid to call ClusterInfo (initialized in ClusterFFeature::prepare()) even from Databasefeature::start()
701+
702+
// if there is no logicalView present yet then skip this step
703+
if (logicalView) {
704+
if (arangodb::iresearch::DATA_SOURCE_TYPE != logicalView->type()) {
705+
unload(); // unlock the data store directory
706+
return arangodb::Result(
707+
TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND,
708+
std::string("error finding view: '") + viewId + "' for link '" + std::to_string(_id) + "' : no such view"
709+
);
710+
}
711+
712+
auto* view =
713+
arangodb::LogicalView::cast<IResearchView>(logicalView.get());
714+
715+
if (!view) {
716+
unload(); // unlock the data store directory
717+
return arangodb::Result(
718+
TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND,
719+
std::string("error finding view: '") + viewId + "' for link '" + std::to_string(_id) + "'"
720+
);
721+
}
722+
723+
viewId = view->guid(); // ensue that this is a GUID (required by operator==(IResearchView))
724+
725+
if (clusterWideLink) { // cluster cluster-wide link
726+
auto shardIds = _collection.shardIds();
727+
728+
// go through all shard IDs of the collection and try to link any links
729+
// missing links will be populated when they are created in the per-shard collection
730+
if (shardIds) {
731+
for (auto& entry: *shardIds) {
732+
auto collection = vocbase.lookupCollection(entry.first); // per-shard collections are always in 'vocbase'
733+
734+
if (!collection) {
735+
continue; // missing collection should be created after Plan becomes Current
736+
}
683737

684-
auto link = IResearchLinkHelper::find(*collection, *view);
738+
auto link = IResearchLinkHelper::find(*collection, *view);
685739

686-
if (link && !view->link(link->self())) {
687-
unload(); // unlock the directory
688-
return arangodb::Result(
689-
TRI_ERROR_INTERNAL,
690-
std::string("failed to link with view '") + view->name() + "' while initializing link '" + std::to_string(_id) + "', collection '" + collection->name() + "'"
691-
);
740+
if (link && !view->link(link->self())) {
741+
return arangodb::Result(
742+
TRI_ERROR_INTERNAL,
743+
std::string("failed to link with view '") + view->name() + "' while initializing link '" + std::to_string(_id) + "', collection '" + collection->name() + "'"
744+
);
745+
}
692746
}
693747
}
748+
} else { // cluster per-shard link
749+
if (!view->link(_asyncSelf)) {
750+
unload(); // unlock the data store directory
751+
return arangodb::Result(
752+
TRI_ERROR_INTERNAL,
753+
std::string("failed to link with view '") + view->name() + "' while initializing link '" + std::to_string(_id) + "'"
754+
);
755+
}
694756
}
695-
} else if (arangodb::ServerState::instance()->isSingleServer() // single-server link
696-
|| arangodb::ServerState::instance()->isDBServer()) { // cluster per-shard link
697-
auto res = initDataStore(*view);
757+
}
758+
} else if (arangodb::ServerState::instance()->isSingleServer()) { // single-server link
759+
auto res = initDataStore(); // prepare data-store which can then update options via the IResearchView::link(...) call
698760

699-
if (!res.ok()) {
700-
return res;
761+
if (!res.ok()) {
762+
return res;
763+
}
764+
765+
auto logicalView = vocbase.lookupView(viewId);
766+
767+
// if there is no logicalView present yet then skip this step
768+
if (logicalView) {
769+
if (arangodb::iresearch::DATA_SOURCE_TYPE != logicalView->type()) {
770+
unload(); // unlock the data store directory
771+
return arangodb::Result(
772+
TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND,
773+
std::string("error finding view: '") + viewId + "' for link '" + std::to_string(_id) + "' : no such view"
774+
);
775+
}
776+
777+
auto* view =
778+
arangodb::LogicalView::cast<IResearchView>(logicalView.get());
779+
780+
if (!view) {
781+
unload(); // unlock the data store directory
782+
return arangodb::Result(
783+
TRI_ERROR_ARANGO_DATA_SOURCE_NOT_FOUND,
784+
std::string("error finding view: '") + viewId + "' for link '" + std::to_string(_id) + "'"
785+
);
701786
}
702 10000 787

788+
viewId = view->guid(); // ensue that this is a GUID (required by operator==(IResearchView))
789+
703790
if (!view->link(_asyncSelf)) {
704791
unload(); // unlock the directory
705792
return arangodb::Result(
@@ -710,13 +797,13 @@ arangodb::Result IResearchLink::init(
710797
}
711798
}
712799

713-
const_cast<std::string&>(_viewGuid) = logicalView->guid(); // ensue that this is a GUID (required by operator==(IResearchView))
800+
const_cast<std::string&>(_viewGuid) = std::move(viewId);
714801
const_cast<IResearchLinkMeta&>(_meta) = std::move(meta);
715802

716803
return arangodb::Result();
717804
}
718805

719-
arangodb::Result IResearchLink::initDataStore(IResearchView const& view) {
806+
arangodb::Result IResearchLink::initDataStore() {
720807
_asyncSelf->reset(); // the data-store is being deallocated, link use is no longer valid (wait for all the view users to finish)
721808

722809
auto* dbPathFeature = arangodb::application_features::ApplicationServer::lookupFeature<
@@ -730,26 +817,6 @@ arangodb::Result IResearchLink::initDataStore(IResearchView const& view) {
730817
);
731818
}
732819

733-
auto viewMeta = view.meta();
734-
735-
if (!viewMeta) {
736-
return arangodb::Result(
737-
TRI_ERROR_INTERNAL,
738-
std::string("failed to get arangosearch view meta while initializing link '") + std::to_string(_id) + "'"
739-
);
740-
}
741-
742-
irs::index_writer::options options;
743-
744-
{
745-
SCOPED_LOCK(viewMeta->read());
746-
747-
options.lock_repository = false; // do not lock index, ArangoDB has it's own lock
748-
options.segment_count_max = viewMeta->_writebufferActive;
749-
options.segment_memory_max = viewMeta->_writebufferSizeMax;
750-
options.segment_pool_size = viewMeta->_writebufferIdle;
751-
}
752-
753820
auto format = irs::formats::get(IRESEARCH_STORE_FORMAT);
754821

755822
if (!format) {
@@ -761,7 +828,7 @@ arangodb::Result IResearchLink::initDataStore(IResearchView const& view) {
761828

762829
bool pathExists;
763830

764-
_dataStore._path = getPersistedPath(*dbPathFeature, _collection, view);
831+
_dataStore._path = getPersistedPath(*dbPathFeature, *this);
765832

766833
// must manually ensure that the data store directory exists (since not using a lockfile)
767834
if (_dataStore._path.exists_directory(pathExists)
@@ -783,6 +850,10 @@ arangodb::Result IResearchLink::initDataStore(IResearchView const& view) {
783850
);
784851
}
785852

853+
irs::index_writer::options options;
854+
855+
options.lock_repository = false; // do not lock index, ArangoDB has it's own lock
856+
786857
// create writer before reader to ensure data directory is present
787858
_dataStore._writer = irs::index_writer::make(
788859
*(_dataStore._directory), format, irs::OM_CREATE | irs::OM_APPEND, options
@@ -1028,6 +1099,13 @@ size_t IResearchLink::memory() const {
10281099
return size;
10291100
}
10301101

1102+
bool IResearchLink::properties(
1103+
irs::index_writer::segment_limits const& properties
1104+
) {
1105+
// FIXME TODO update the data-store options
1106+
return true;
1107+
}
1108+
10311109
arangodb::Result IResearchLink::remove(
10321110
arangodb::transaction::Methods& trx,
10331111
arangodb::LocalDocumentId const& documentId,

0 commit comments

Comments
 (0)
0