8000 Changes since last PR: remove 'dataPath' IResearchView configuration … · ashang/arangodb@423ce33 · GitHub
[go: up one dir, main page]
Skip to content

Commit 423ce33

Browse files
vasiliy-arangodbjsteemann
vasiliy-arangodb
authored and
jsteemann
committed
Changes since last PR: remove 'dataPath' IResearchView configuration property and use a hardcoded path similar to a LogicalCollection (arangodb#4683)
1 parent 7badc86 commit 423ce33

15 files changed

+137
-438
lines changed

‎Documentation/Books/Manual/Views/ArangoSearch.md

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -232,9 +232,6 @@ During view modification the following directives apply:
232232
* threshold: (optional; default: `0.85`)
233233
consolidate `IFF {threshold} > #segment_docs{valid} / (#segment_docs{valid} + #segment_docs{removed})`
234234

235-
* dataPath: <optional> (default: \<ArangoDB database path\>/arangosearch-\<view-id\>)
236-
the filesystem path where to store persisted view metadata
237-
238235
* locale: (optional; default: `C`)
239236
the default locale used for ordering processed attribute names
240237

‎Documentation/DocuBlocks/Rest/Views/JSF_patch_api_view_properties_iresearch.md

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -90,9 +90,6 @@ memory usage would continue to grow.
9090
For the case where there are a few inserts/updates, a higher value will impact performance and waste disk space for each
9191
commit call without any added benefits.
9292

93-
@RESTBODYPARAM{dataPath,string,optional,string}
94-
The filesystem path where to store persisted index data (default: *"<ArangoDB database path>/iresearch-<index id>"*).
95-
9693
@RESTBODYPARAM{locale,string,optional,string}
9794
The default locale used for queries 67ED on analyzed string values (default: *C*).
9895

‎Documentation/DocuBlocks/Rest/Views/JSF_post_api_view_iresearch.md

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -71,9 +71,6 @@ lot of disk space to be wasted.
7171
For the case where the consolidation policies rarely merge segments (i.e. few inserts/deletes), a higher value will impact
7272
performance without any added benefits.
7373

74-
@RESTSTRUCT{dataPath,JSF_post_api_view_props,string,optional,string}
75-
The filesystem path where to store persisted view data (default: *"<ArangoDB database path>/arangosearch-<view-id>"*).
76-
7774
@RESTSTRUCT{locale,JSF_post_api_view_props,string,optional,string}
7875
The default locale used for queries on analyzed string values (default: *C*).
7976

‎Documentation/DocuBlocks/Rest/Views/JSF_put_api_view_properties_iresearch.md

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -90,9 +90,6 @@ memory usage would continue to grow.
9090
For the case where there are a few inserts/updates, a higher value will impact performance and waste disk space for each
9191
commit call without any added benefits.
9292

93-
@RESTBODYPARAM{dataPath,string,optional,string}
94-
The filesystem path where to store persisted view data (default: *"<ArangoDB database path>/arangosearch-<view-id>"*).
95-
9693
@RESTBODYPARAM{locale,string,optional,string}
9794
The default locale used for queries on analyzed string values (default: *C*).
9895

‎arangod/IResearch/IResearchView.cpp

Lines changed: 50 additions & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@
3131
#include "utils/directory_utils.hpp"
3232
#include "utils/memory.hpp"
3333
#include "utils/misc.hpp"
34-
#include "utils/utf8_path.hpp"
3534

3635
#include "ApplicationServerHelper.h"
3736
#include "IResearchAttributes.h"
@@ -245,43 +244,6 @@ inline arangodb::FlushFeature* getFlushFeature() noexcept {
245244
return arangodb::iresearch::getFeature<arangodb::FlushFeature>("Flush");
246245
}
247246

248-
////////////////////////////////////////////////////////////////////////////////
249-
/// @brief construct an absolute path from the IResearchViewMeta configuration
250-
////////////////////////////////////////////////////////////////////////////////
251-
bool appendAbsolutePersistedDataPath(
252-
std::string& buf,
253-
arangodb::iresearch::IResearchView const& view,
254-
arangodb::iresearch::IResearchViewMeta const& meta
255-
) {
256-
auto& path = meta._dataPath;
257-
258-
if (path.empty()) {
259-
return false; // empty data path should have been set to a default value
260-
}
261-
262-
if (TRI_PathIsAbsolute(path)) {
263-
buf = path;
264-
265-
return true;
266-
}
267-
268-
auto* feature = arangodb::iresearch::getFeature<arangodb::DatabasePathFeature>("DatabasePath");
269-
270-
if (!feature) {
271-
return false;
272-
}
273-
274-
// get base path from DatabaseServerFeature (similar to MMFilesEngine)
275-
irs::utf8_path absPath(feature->directory());
276-
static std::string subPath("databases");
277-
278-
absPath/=subPath;
279-
absPath/=path;
280-
buf = absPath.utf8();
281-
282-
return true;
283-
}
284-
285247
////////////////////////////////////////////////////////////////////////////////
286248
/// @brief append link removal directives to the builder and return success
287249
////////////////////////////////////////////////////////////////////////////////
@@ -870,16 +832,22 @@ IResearchView::MemoryStore::MemoryStore() {
870832
_reader = irs::directory_reader::open(*_directory); // open after 'commit' for valid 'store'
871833
}
872834

835+
IResearchView::PersistedStore::PersistedStore(irs::utf8_path&& path)
836+
: _path(std::move(path)) {
837+
}
838+
873839
IResearchView::IResearchView(
874840
arangodb::LogicalView* view,
875-
arangodb::velocypack::Slice const& info
841+
arangodb::velocypack::Slice const& info,
842+
irs::utf8_path&& persistedPath
876843
) : ViewImplementation(view, info),
877844
FlushTransaction(toString(*this)),
878845
_asyncMetaRevision(1),
879846
_asyncSelf(irs::memory::make_unique<AsyncSelf>(this)),
880847
_asyncTerminate(false),
881848
_memoryNode(&_memoryNodes[0]), // set current memory node (arbitrarily 0)
882849
_toFlush(&_memoryNodes[1]), // set flush-pending memory node (not same as _memoryNode)
850+
_storePersisted(std::move(persistedPath)),
883851
_threadPool(0, 0), // 0 == create pool with no threads, i.e. not running anything
884852
_inRecovery(false) {
885853
// set up in-recovery insertion hooks
@@ -1207,7 +1175,11 @@ void IResearchView::drop() {
12071175
_storePersisted._directory.reset();
12081176
}
12091177

1210-
if (!TRI_IsDirectory(_meta._dataPath.c_str()) || TRI_ERROR_NO_ERROR == TRI_RemoveDirectory(_meta._dataPath.c_str())) {
1178+
bool exists;
1179+
1180+
// remove persisted data store directory if present
1181+
if (_storePersisted._path.exists_directory(exists)
1182+
&& (!exists || _storePersisted._path.remove())) {
12111183
return; // success
12121184
}
12131185
} catch (std::exception const& e) {
@@ -1690,12 +1662,40 @@ arangodb::Result IResearchView::link(
16901662
arangodb::velocypack::Slice const& info,
16911663
bool isNew
16921664
) {
1693-
PTR_NAMED(IResearchView, ptr, view, info);
1665+
if (!view) {
1666+
LOG_TOPIC(WARN, IResearchFeature::IRESEARCH)
1667+
<< "invalid LogicalView argument while constructing IResearch view";
1668+
return nullptr;
1669+
}
1670+
1671+
auto* feature =
1672+
arangodb::iresearch::getFeature<arangodb::DatabasePathFeature>("DatabasePath");
1673+
1674+
if (!feature) {
1675+
LOG_TOPIC(WARN, IResearchFeature::IRESEARCH)
1676+
<< "failure to find feature 'DatabasePath' while constructing IResearch view '" << view->id() << "'";
1677+
1678+
return nullptr;
1679+
}
1680+
1681+
// get base path from DatabaseServerFeature (similar to MMFilesEngine)
1682+
// the path is hardcoded to reside under:
1683+
// <DatabasePath>/<IResearchView::type()>-<view id>
1684+
// similar to the data path calculation for collections
1685+
irs::utf8_path dataPath(feature->directory());
1686+
static std::string subPath("databases");
1687+
1688+
dataPath /= subPath;
1689+
dataPath /= arangodb::iresearch::IResearchView::type();
1690+
dataPath += "-";
1691+
dataPath += std::to_string(view->id());
1692+
1693+
PTR_NAMED(IResearchView, ptr, view, info, std::move(dataPath));
16941694
auto& impl = reinterpret_cast<IResearchView&>(*ptr);
16951695
auto& json = info.isNone() ? emptyObjectSlice() : info; // if no 'info' then assume defaults
16961696
std::string error;
16971697

1698-
if (!view || !impl._meta.init(json, error, *view)) {
1698+
if (!impl._meta.init(json, error)) {
16991699
LOG_TOPIC(WARN, iresearch::IResearchFeature::IRESEARCH)
17001700
<< "failed to initialize iResearch view from definition, error: " << error;
17011701

@@ -1727,6 +1727,7 @@ size_t IResearchView::memory() const {
17271727

17281728
if (_storePersisted) {
17291729
size += directoryMemory(*(_storePersisted._directory), id());
1730+
size += _storePersisted._path.native().size() * sizeof(irs::utf8_path::native_char_t);
17301731
}
17311732

17321733
return size;
@@ -1750,14 +1751,12 @@ void IResearchView::open() {
17501751
return; // view already open
17511752
}
17521753

1753-
std::string absoluteDataPath;
1754-
17551754
try {
17561755
auto format = irs::formats::get(IRESEARCH_STORE_FORMAT);
17571756

1758-
if (format && appendAbsolutePersistedDataPath(absoluteDataPath, *this, _meta)) {
1757+
if (format) {
17591758
_storePersisted._directory =
1760-
irs::directory::make<irs::mmap_directory>(absoluteDataPath);
1759+
irs::directory::make<irs::mmap_directory>(_storePersisted._path.utf8());
17611760

17621761
if (_storePersisted._directory) {
17631762
// create writer before reader to ensure data directory is present
@@ -1800,10 +1799,10 @@ void IResearchView::open() {
18001799
}
18011800

18021801
LOG_TOPIC(WARN, iresearch::IResearchFeature::IRESEARCH)
1803-
<< "failed to open iResearch view '" << id() << "' at: " << absoluteDataPath;
1802+
<< "failed to open IResearch view '" << name() << "' at: " << _storePersisted._path.utf8();
18041803

18051804
throw std::runtime_error(
1806-
std::string("failed to open iResearch view '") + std::to_string(id()) + "' at: " + absoluteDataPath
1805+
std::string("failed to open iResearch view '") + name() + "' at: " + _storePersisted._path.utf8()
18071806
);
18081807
}
18091808

@@ -2052,7 +2051,7 @@ arangodb::Result IResearchView::updateProperties(
20522051

20532052
auto& initialMeta = partialUpdate ? _meta : IResearchViewMeta::DEFAULT();
20542053

2055-
if (!meta.init(slice, error, *_logicalView, initialMeta, &mask)) {
2054+
if (!meta.init(slice, error, initialMeta, &mask)) {
20562055
return arangodb::Result(TRI_ERROR_BAD_PARAMETER, std::move(error));
20572056
}
20582057

@@ -2123,75 +2122,6 @@ arangodb::Result IResearchView::updateProperties(
21232122
// reset non-updatable values to match current meta
21242123
meta._collections = _meta._collections;
21252124

2126-
// do not modify data path if not changes since will cause lock obtain failure
2127-
mask._dataPath = mask._dataPath && _meta._dataPath != meta._dataPath;
2128-
2129-
DataStore storePersisted; // renamed persisted data store
2130-
std::string srcDataPath = _meta._dataPath;
2131-
char const* dropDataPath = nullptr;
2132-
2133-
// copy directory to new location
2134-
if (mask._dataPath) {
2135-
std::string absoluteDataPath;
2136-
2137-
if (!appendAbsolutePersistedDataPath(absoluteDataPath, *this, meta)) {
2138-
return arangodb::Result(
2139-
TRI_ERROR_BAD_PARAMETER,
2140-
std::string("error generating absolute path for iResearch view '") + std::to_string(id()) + "' data path '" + meta._dataPath + "'"
2141-
);
2142-
}
2143-
2144-
auto res = createPersistedDataDirectory(
2145-
storePersisted._directory,
2146-
storePersisted._writer,
2147-
absoluteDataPath,
2148-
_storePersisted._reader, // reader from the original persisted data store
2149-
id()
2150-
);
2151-
2152-
if (!res.ok()) {
2153-
return res;
2154-
}
2155-
2156-
try {
2157-
storePersisted._reader = irs::directory_reader::open(*(storePersisted._directory));
2158-
storePersisted._segmentCount += storePersisted._reader.size(); // add commited segments (previously had 0)
2159-
dropDataPath = _storePersisted ? srcDataPath.c_str() : nullptr;
2160-
} catch (std::exception const& e) {
2161-
LOG_TOPIC(WARN, iresearch::IResearchFeature::IRESEARCH)
2162-
<< "caught exception while opening iResearch view '" << id()
2163-
<< "' data path '" + meta._dataPath + "': " << e.what();
2164-
IR_LOG_EXCEPTION();
2165-
return arangodb::Result(
2166-
TRI_ERROR_BAD_PARAMETER,
2167-
std::string("error opening iResearch view '") + std::to_string(id()) + "' data path '" + meta._dataPath + "'"
2168-
);
2169-
} catch (...) {
2170-
LOG_TOPIC(WARN, iresearch::IResearchFeature::IRESEARCH)
2171-
<< "caught exception while opening iResearch view '" << id() << "' data path '" + meta._dataPath + "'";
2172-
IR_LOG_EXCEPTION();
2173-
return arangodb::Result(
2174-
TRI_ERROR_BAD_PARAMETER,
2175-
std::string("error opening iResearch view '") + std::to_string(id()) + "' data path '" + meta._dataPath + "'"
2176-
);
2177-
}
2178-
2179-
if (!storePersisted._reader) {
2180-
LOG_TOPIC(WARN, iresearch::IResearchFeature::IRESEARCH)
2181-
<< "error while opening reader for iResearch view '" << id() << "' data path '" + meta._dataPath + "'";
2182-
return arangodb::Result(
2183-
TRI_ERROR_BAD_PARAMETER,
2184-
std::string("error opening reader for iResearch view '") + std::to_string(id()) + "' data path '" + meta._dataPath + "'"
2185-
);
2186-
}
2187-
}
2188-
2189-
_meta = std::move(meta);
2190-
2191-
if (mask._dataPath) {
2192-
_storePersisted = std::move(storePersisted);
2193-
}
2194-
21952125
if (mask._threadsMaxIdle) {
21962126
_threadPool.max_idle(meta._threadsMaxIdle);
21972127
}
@@ -2205,9 +2135,7 @@ arangodb::Result IResearchView::updateProperties(
22052135
_asyncCondition.notify_all(); // trigger reload of timeout settings for async jobs
22062136
}
22072137

2208-
if (dropDataPath) {
2209-
res = TRI_RemoveDirectory(dropDataPath); // ignore error (only done to tidy-up filesystem)
2210-
}
2138+
_meta = std::move(meta);
22112139
}
22122140

22132141
std::unordered_set<TRI_voc_cid_t> collections;
@@ -2373,4 +2301,4 @@ NS_END // arangodb
23732301

23742302
// -----------------------------------------------------------------------------
23752303
// --SECTION-- END-OF-FILE
2376-
// -----------------------------------------------------------------------------
2304+
// -----------------------------------------------------------------------------

‎arangod/IResearch/IResearchView.h

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
#include "index/index_writer.hpp"
3636
#include "index/directory_reader.hpp"
3737
#include "utils/async_utils.hpp"
38+
#include "utils/utf8_path.hpp"
3839

3940
NS_BEGIN(arangodb)
4041

@@ -304,6 +305,11 @@ class IResearchView final: public arangodb::ViewImplementation,
304305
MemoryStore(); // initialize _directory and _writer during allocation
305306
};
306307

308+
struct PersistedStore: public DataStore {
309+
const irs::utf8_path _path;
310+
PersistedStore(irs::utf8_path&& path);
311+
};
312+
307313
struct TidStore {
308314
mutable std::mutex _mutex; // for use with '_removals' (allow use in const functions)
309315
std::vector<std::shared_ptr<irs::filter>> _removals; // removal filters to be applied to during merge
@@ -329,7 +335,8 @@ class IResearchView final: public arangodb::ViewImplementation,
329335

330336
IResearchView(
331337
arangodb::LogicalView*,
332-
arangodb::velocypack::Slice const& info
338+
arangodb::velocypack::Slice const& info,
339+
irs::utf8_path&& persistedPath
333340
);
334341

335342
//////////////////////////////////////////////////////////////////////////////
@@ -361,7 +368,7 @@ class IResearchView final: public arangodb::ViewImplementation,
361368
MemoryStoreNode* _memoryNode; // points to the current memory store
362369
MemoryStoreNode* _toFlush; // points to memory store to be flushed
363370
MemoryStoreByTid _storeByTid;
364-
DataStore _storePersisted;
371+
PersistedStore _storePersisted;
365372
FlushCallback _flushCallback; // responsible for flush callback unregistration
366373
irs::async_utils::thread_pool _threadPool;
367374
std::function<void(arangodb::TransactionState& state)> _trxReadCallback; // for snapshot(...)

0 commit comments

Comments
 (0)
0