diff --git a/Documentation/Books/Manual/Indexing/Hash.md b/Documentation/Books/Manual/Indexing/Hash.md index 25753cda7578..9f6f20598f91 100644 --- a/Documentation/Books/Manual/Indexing/Hash.md +++ b/Documentation/Books/Manual/Indexing/Hash.md @@ -118,6 +118,30 @@ details, including the index-identifier, is returned. @endDocuBlock ensureHashIndexArray + + +For more information see [Creating Indexes in Background](IndexBasics.md#creating-indexes-in-background) + Ensure uniqueness of relations in edge collections -------------------------------------------------- diff --git a/Documentation/Books/Manual/Indexing/IndexBasics.md b/Documentation/Books/Manual/Indexing/IndexBasics.md index 76ec323d6a98..42591d21a1c9 100644 --- a/Documentation/Books/Manual/Indexing/IndexBasics.md +++ b/Documentation/Books/Manual/Indexing/IndexBasics.md @@ -22,6 +22,14 @@ are covered by an edge collection's edge index automatically. Using the system attribute `_id` in user-defined indexes is not possible, but indexing `_key`, `_rev`, `_from`, and `_to` is. + + ArangoDB provides the following index types: Primary Index @@ -243,31 +251,6 @@ Skiplist indexes support [indexing array values](#indexing-array-values) if the attribute name is extended with a [\*]`. -Persistent Index ----------------- - -The persistent index is a sorted index with persistence. The index entries are written to -disk when documents are stored or updated. That means the index entries do not need to be -rebuilt from the collection data when the server is restarted or the indexed collection -is initially loaded. Thus using persistent indexes may reduce collection loading times. - -The persistent index type can be used for secondary indexes at the moment. That means the -persistent index currently cannot be made the only index for a collection, because there -will always be the in-memory primary index for the collection in addition, and potentially -more indexes (such as the edges index for an edge collection). - -The index implementation is using the RocksDB engine, and it provides logarithmic complexity -for insert, update, and remove operations. As the persistent index is not an in-memory -index, it does not store pointers into the primary index as all the in-memory indexes do, -but instead it stores a document's primary key. To retrieve a document via a persistent -index via an index value lookup, there will therefore be an additional O(1) lookup into -the primary index to fetch the actual document. - -As the persistent index is sorted, it can be used for point lookups, range queries and sorting -operations, but only if either all index attributes are provided in a query, or if a leftmost -prefix of the index attributes is specified. - - Geo Index --------- @@ -307,6 +290,37 @@ minimum length will be included in the index. The fulltext index is used via dedicated functions in AQL or the simple queries, but will not be enabled for other types of queries or conditions. + +Persistent Index +---------------- + +{% hint 'warning' %} +this index should not be used anymore, instead use the rocksdb storage engine +with either the *skiplist* or *hash* index. +{% endhint %} + +The persistent index is a sorted index with persistence. The index entries are written to +disk when documents are stored or updated. That means the index entries do not need to be +rebuilt from the collection data when the server is restarted or the indexed collection +is initially loaded. Thus using persistent indexes may reduce collection loading times. + +The persistent index type can be used for secondary indexes at the moment. That means the +persistent index currently cannot be made the only index for a collection, because there +will always be the in-memory primary index for the collection in addition, and potentially +more indexes (such as the edges index for an edge collection). + +The index implementation is using the RocksDB engine, and it provides logarithmic complexity +for insert, update, and remove operations. As the persistent index is not an in-memory +index, it does not store pointers into the primary index as all the in-memory indexes do, +but instead it stores a document's primary key. To retrieve a document via a persistent +index via an index value lookup, there will therefore be an additional O(1) lookup into +the primary index to fetch the actual document. + +As the persistent index is sorted, it can be used for point lookups, range queries and sorting +operations, but only if either all index attributes are provided in a query, or if a leftmost +prefix of the index attributes is specified. + + Indexing attributes and sub-attributes -------------------------------------- @@ -534,3 +548,63 @@ optimizer may prefer the default edge index over vertex centric indexes based on the costs it estimates, even if a vertex centric index might in fact be faster. Vertex centric indexes are more likely to be chosen for highly connected graphs and with RocksDB storage engine. + + diff --git a/Documentation/Books/Manual/Indexing/Skiplist.md b/Documentation/Books/Manual/Indexing/Skiplist.md index a7c78f59d44a..656e93dfb7ff 100644 --- a/Documentation/Books/Manual/Indexing/Skiplist.md +++ b/Documentation/Books/Manual/Indexing/Skiplist.md @@ -185,3 +185,28 @@ and { "a" : { "c" : 1, "b" : 1 } } ``` will match. + + + diff --git a/arangod/Agency/Supervision.cpp b/arangod/Agency/Supervision.cpp index 5a56fd170343..f6ae4c9e69ca 100644 --- a/arangod/Agency/Supervision.cpp +++ b/arangod/Agency/Supervision.cpp @@ -1066,7 +1066,7 @@ void Supervision::readyOrphanedIndexCreations() { indexes = collection("indexes").getArray(); if (indexes.length() > 0) { for (auto const& planIndex : VPackArrayIterator(indexes)) { - if (planIndex.hasKey("isBuilding") && collection.has("shards")) { + if (planIndex.hasKey(StaticStrings::IndexIsBuilding) && collection.has("shards")) { auto const& planId = planIndex.get("id"); auto const& shards = collection("shards"); if (collection.has("numberOfShards") && @@ -1121,7 +1121,7 @@ void Supervision::readyOrphanedIndexCreations() { { VPackObjectBuilder props(envelope.get()); for (auto const& prop : VPackObjectIterator(planIndex)) { auto const& key = prop.key.copyString(); - if (key != "isBuilding") { + if (key != StaticStrings::IndexIsBuilding) { envelope->add(key, prop.value); } }} diff --git a/arangod/Aql/OptimizerRules.cpp b/arangod/Aql/OptimizerRules.cpp index b29a9295896a..e7485accb69f 100644 --- a/arangod/Aql/OptimizerRules.cpp +++ b/arangod/Aql/OptimizerRules.cpp @@ -6642,15 +6642,15 @@ static bool geoFuncArgCheck(ExecutionPlan* plan, AstNode const* args, info.collectionNodeToReplace = collNode; info.collectionNodeOutVar = collNode->outVariable(); info.collection = collNode->collection(); - std::shared_ptr coll = - collNode->collection()->getCollection(); - - // check for suitable indexes - for (std::shared_ptr idx : coll->getIndexes()) { + + // we should not access the LogicalCollection directly + Query* query = plan->getAst()->query(); + auto indexes = query->trx()->indexesForCollection(info.collection->name()); + // check for suitiable indexes + for (std::shared_ptr idx : indexes) { // check if current index is a geo-index - bool isGeo = - idx->type() == arangodb::Index::IndexType::TRI_IDX_TYPE_GEO_INDEX; - if (isGeo && idx->fields().size() == 1) { // individual fields + bool isGeo = idx->type() == arangodb::Index::IndexType::TRI_IDX_TYPE_GEO_INDEX; + if (isGeo && idx->fields().size() == 1) { // individual fields // check access paths of attributes in ast and those in index match if (idx->fields()[0] == attributeAccess.second) { if (info.index != nullptr && info.index != idx) { diff --git a/arangod/Aql/OptimizerRulesReplaceFunctions.cpp b/arangod/Aql/OptimizerRulesReplaceFunctions.cpp index 610b2c0d228c..9a28f284959f 100644 --- a/arangod/Aql/OptimizerRulesReplaceFunctions.cpp +++ b/arangod/Aql/OptimizerRulesReplaceFunctions.cpp @@ -195,7 +195,6 @@ std::pair getAttributeAccessFromIndex(Ast* ast, AstNode* doc for(auto& idx : indexes){ if(Index::isGeoIndex(idx->type())) { // we take the first index that is found - bool isGeo1 = idx->type() == Index::IndexType::TRI_IDX_TYPE_GEO1_INDEX; bool isGeo2 = idx->type() == Index::IndexType::TRI_IDX_TYPE_GEO2_INDEX; bool isGeo = idx->type() == Index::IndexType::TRI_IDX_TYPE_GEO_INDEX; diff --git a/arangod/Cluster/ClusterInfo.cpp b/arangod/Cluster/ClusterInfo.cpp index 87dd09e40b35..81ac076e5848 100644 --- a/arangod/Cluster/ClusterInfo.cpp +++ b/arangod/Cluster/ClusterInfo.cpp @@ -2433,14 +2433,11 @@ int ClusterInfo::ensureIndexCoordinator( // check index id uint64_t iid = 0; - VPackSlice const idSlice = slice.get(StaticStrings::IndexId); - if (idSlice.isString()) { - // use predefined index id + if (idSlice.isString()) { // use predefined index id iid = arangodb::basics::StringUtils::uint64(idSlice.copyString()); } - if (iid == 0) { - // no id set, create a new one! + if (iid == 0) { // no id set, create a new one! iid = uniqid(); } std::string const idString = arangodb::basics::StringUtils::itoa(iid); @@ -2629,14 +2626,14 @@ int ClusterInfo::ensureIndexCoordinatorInner( for (auto const& e : VPackObjectIterator(slice)) { TRI_ASSERT(e.key.isString()); std::string const& key = e.key.copyString(); - if (key != StaticStrings::IndexId && key != "isBuilding") { + if (key != StaticStrings::IndexId && key != StaticStrings::IndexIsBuilding) { ob->add(e.key); ob->add(e.value); } } if (numberOfShards > 0 && !slice.get(StaticStrings::IndexType).isEqualString("arangosearch")) { - ob->add("isBuilding", VPackValue(true)); + ob->add(StaticStrings::IndexIsBuilding, VPackValue(true)); } ob->add(StaticStrings::IndexId, VPackValue(idString)); } @@ -2709,7 +2706,7 @@ int ClusterInfo::ensureIndexCoordinatorInner( { VPackObjectBuilder o(&finishedPlanIndex); for (auto const& entry : VPackObjectIterator(newIndexBuilder.slice())) { auto const key = entry.key.copyString(); - if (key != "isBuilding" && key != "isNewlyCreated") { + if (key != StaticStrings::IndexIsBuilding && key != "isNewlyCreated") { finishedPlanIndex.add(entry.key.copyString(), entry.value); } } diff --git a/arangod/ClusterEngine/ClusterCollection.cpp b/arangod/ClusterEngine/ClusterCollection.cpp index a26d44d9a2f0..af9aa78bc4bf 100644 --- a/arangod/ClusterEngine/ClusterCollection.cpp +++ b/arangod/ClusterEngine/ClusterCollection.cpp @@ -360,42 +360,6 @@ void ClusterCollection::prepareIndexes( TRI_ASSERT(!_indexes.empty()); } -static std::shared_ptr findIndex( - velocypack::Slice const& info, - std::vector> const& indexes) { - TRI_ASSERT(info.isObject()); - - // extract type - VPackSlice value = info.get("type"); - - if (!value.isString()) { - // Compatibility with old v8-vocindex. - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, - "invalid index type definition"); - } - - std::string tmp = value.copyString(); - arangodb::Index::IndexType const type = arangodb::Index::type(tmp.c_str()); - - for (auto const& idx : indexes) { - if (idx->type() == type) { - // Only check relevant indexes - if (idx->matchesDefinition(info)) { - // We found an index for this definition. - return idx; - } - } - } - return nullptr; -} - -/// @brief Find index by definition -std::shared_ptr ClusterCollection::lookupIndex( - velocypack::Slice const& info) const { - READ_LOCKER(guard, _indexesLock); - return findIndex(info, _indexes); -} - std::shared_ptr ClusterCollection::createIndex( arangodb::velocypack::Slice const& info, bool restore, bool& created) { @@ -404,23 +368,19 @@ std::shared_ptr ClusterCollection::createIndex( WRITE_LOCKER(guard, _exclusiveLock); std::shared_ptr idx; - { - WRITE_LOCKER(guard, _indexesLock); - idx = findIndex(info, _indexes); - if (idx) { - created = false; - // We already have this index. - return idx; - } + WRITE_LOCKER(guard2, _indexesLock); + idx = lookupIndex(info); + if (idx) { + created = false; + // We already have this index. + return idx; } StorageEngine* engine = EngineSelectorFeature::ENGINE; TRI_ASSERT(engine != nullptr); // We are sure that we do not have an index of this type. - // We also hold the lock. - // Create it - + // We also hold the lock. Create it idx = engine->indexFactory().prepareIndexFromSlice( info, true, _logicalCollection, false ); diff --git a/arangod/ClusterEngine/ClusterCollection.h b/arangod/ClusterEngine/ClusterCollection.h index 59583af88ccf..b16fd0a1fed3 100644 --- a/arangod/ClusterEngine/ClusterCollection.h +++ b/arangod/ClusterEngine/ClusterCollection.h @@ -104,9 +104,6 @@ class ClusterCollection final : public PhysicalCollection { void prepareIndexes(arangodb::velocypack::Slice indexesSlice) override; - /// @brief Find index by definition - std::shared_ptr lookupIndex(velocypack::Slice const&) const override; - std::shared_ptr createIndex(arangodb::velocypack::Slice const& info, bool restore, bool& created) override; diff --git a/arangod/ClusterEngine/ClusterEngine.cpp b/arangod/ClusterEngine/ClusterEngine.cpp index d0b7785efcb6..843ea64c5f1a 100644 --- a/arangod/ClusterEngine/ClusterEngine.cpp +++ b/arangod/ClusterEngine/ClusterEngine.cpp @@ -289,11 +289,10 @@ void ClusterEngine::recoveryDone(TRI_vocbase_t& vocbase) { std::string ClusterEngine::createCollection( TRI_vocbase_t& vocbase, - TRI_voc_cid_t cid, LogicalCollection const& collection ) { - TRI_ASSERT(cid != 0); - TRI_UpdateTickServer(static_cast(cid)); + TRI_ASSERT(collection.id() != 0); + TRI_UpdateTickServer(static_cast(collection.id())); return std::string(); // no need to return a path } @@ -320,7 +319,6 @@ void ClusterEngine::destroyCollection( void ClusterEngine::changeCollection( TRI_vocbase_t& vocbase, - TRI_voc_cid_t id, LogicalCollection const& collection, bool doSync ) { diff --git a/arangod/ClusterEngine/ClusterEngine.h b/arangod/ClusterEngine/ClusterEngine.h index 926e44f9dbbe..7b94e407720c 100644 --- a/arangod/ClusterEngine/ClusterEngine.h +++ b/arangod/ClusterEngine/ClusterEngine.h @@ -239,7 +239,6 @@ class ClusterEngine final : public StorageEngine { public: std::string createCollection( TRI_vocbase_t& vocbase, - TRI_voc_cid_t id, LogicalCollection const& collection ) override; @@ -260,7 +259,6 @@ class ClusterEngine final : public StorageEngine { void changeCollection( TRI_vocbase_t& vocbase, - TRI_voc_cid_t id, LogicalCollection const& collection, bool doSync ) override; diff --git a/arangod/ClusterEngine/ClusterIndex.cpp b/arangod/ClusterEngine/ClusterIndex.cpp index b81b314878cf..32f8c6799e20 100644 --- a/arangod/ClusterEngine/ClusterIndex.cpp +++ b/arangod/ClusterEngine/ClusterIndex.cpp @@ -153,19 +153,6 @@ void ClusterIndex::updateClusterSelectivityEstimate(double estimate) { _clusterSelectivity = estimate; } -bool ClusterIndex::isPersistent() const { - if (_engineType == ClusterEngineType::MMFilesEngine) { - return _indexType == Index::TRI_IDX_TYPE_PERSISTENT_INDEX; - } else if (_engineType == ClusterEngineType::RocksDBEngine) { - return true; - } else if (_engineType == ClusterEngineType::MockEngine) { - return false; - } - TRI_ASSERT(false); - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, - "unsupported cluster storage engine"); -} - bool ClusterIndex::isSorted() const { if (_engineType == ClusterEngineType::MMFilesEngine) { return _indexType == Index::TRI_IDX_TYPE_SKIPLIST_INDEX || diff --git a/arangod/ClusterEngine/ClusterIndex.h b/arangod/ClusterEngine/ClusterIndex.h index 2e725ad37aba..e004557f55ec 100644 --- a/arangod/ClusterEngine/ClusterIndex.h +++ b/arangod/ClusterEngine/ClusterIndex.h @@ -49,6 +49,11 @@ class ClusterIndex : public Index { /// @brief return a VelocyPack representation of the index void toVelocyPack(velocypack::Builder& builder, std::underlying_type::type) const override; + + /// @brief if true this index should not be shown externally + bool isHidden() const override { + return false; // do not generally hide indexes + } IndexType type() const override { return _indexType; } @@ -56,8 +61,6 @@ class ClusterIndex : public Index { return Index::oldtypeName(_indexType); } - bool isPersistent() const override; - bool canBeDropped() const override { return _indexType != Index::TRI_IDX_TYPE_PRIMARY_INDEX && _indexType != Index::TRI_IDX_TYPE_EDGE_INDEX; diff --git a/arangod/ClusterEngine/ClusterIndexFactory.cpp b/arangod/ClusterEngine/ClusterIndexFactory.cpp index 1cd2ce2e849d..d8cae526224a 100644 --- a/arangod/ClusterEngine/ClusterIndexFactory.cpp +++ b/arangod/ClusterEngine/ClusterIndexFactory.cpp @@ -326,7 +326,7 @@ void ClusterIndexFactory::prepareIndexes( continue; } - if (basics::VelocyPackHelper::getBooleanValue(v, "isBuilding", false)) { + if (basics::VelocyPackHelper::getBooleanValue(v, StaticStrings::IndexIsBuilding, false)) { // This index is still being built. Do not add. continue; } diff --git a/arangod/IResearch/IResearchLink.cpp b/arangod/IResearch/IResearchLink.cpp index 85c435b5b666..a72fdd46831d 100644 --- a/arangod/IResearch/IResearchLink.cpp +++ b/arangod/IResearch/IResearchLink.cpp @@ -674,9 +674,9 @@ arangodb::Result IResearchLink::init( } } } else if (arangodb::ServerState::instance()->isDBServer()) { // db-server link - auto* engine = arangodb::ClusterInfo::instance(); + auto* ci = arangodb::ClusterInfo::instance(); - if (!engine) { + if (!ci) { return arangodb::Result( TRI_ERROR_INTERNAL, std::string("failure to get storage engine while initializing arangosearch link '") + std::to_string(_id) + "'" @@ -693,7 +693,7 @@ arangodb::Result IResearchLink::init( } } - auto logicalView = engine->getView(vocbase.name(), viewId); // valid to call ClusterInfo (initialized in ClusterFeature::prepare()) even from Databasefeature::start() + auto logicalView = ci->getView(vocbase.name(), viewId); // valid to call ClusterInfo (initialized in ClusterFeature::prepare()) even from Databasefeature::start() // if there is no logicalView present yet then skip this step if (logicalView) { @@ -1016,23 +1016,15 @@ arangodb::Result IResearchLink::insert( return arangodb::Result(); } -bool IResearchLink::isPersistent() const { - auto* engine = arangodb::EngineSelectorFeature::ENGINE; - - // FIXME TODO remove once MMFilesEngine will fillIndex(...) during recovery - // currently the index is created but fill is deffered untill the end of recovery - // at the end of recovery only non-persistent indexes are filled - if (engine && engine->inRecovery()) { - return false; - } - - return true; // records persisted into the iResearch view -} - bool IResearchLink::isSorted() const { return false; // iResearch does not provide a fixed default sort order } +bool IResearchLink::isHidden() const { + // hide links unless we are on a DBServer + return !arangodb::ServerState::instance()->isDBServer(); +} + bool IResearchLink::json(arangodb::velocypack::Builder& builder) const { if (!builder.isOpenObject() || !_meta.json(builder)) { return false; diff --git a/arangod/IResearch/IResearchLink.h b/arangod/IResearch/IResearchLink.h index b5dfb6fb33d2..898db924cd38 100644 --- a/arangod/IResearch/IResearchLink.h +++ b/arangod/IResearch/IResearchLink.h @@ -139,8 +139,9 @@ class IResearchLink { arangodb::Index::OperationMode mode ); // arangodb::Index override - bool isPersistent() const; // arangodb::Index override bool isSorted() const; // arangodb::Index override + + bool isHidden() const; // arangodb::Index override //////////////////////////////////////////////////////////////////////////////// /// @brief the identifier for this link diff --git a/arangod/IResearch/IResearchLinkCoordinator.h b/arangod/IResearch/IResearchLinkCoordinator.h index addd2ca3b4fd..dae75cad6720 100644 --- a/arangod/IResearch/IResearchLinkCoordinator.h +++ b/arangod/IResearch/IResearchLinkCoordinator.h @@ -83,8 +83,9 @@ class IResearchLinkCoordinator final return arangodb::Result(TRI_ERROR_NOT_IMPLEMENTED); } - virtual bool isPersistent() const override { - return IResearchLink::isPersistent(); + + bool isHidden() const override { + return true; // always hide links } // IResearch does not provide a fixed default sort order diff --git a/arangod/IResearch/IResearchMMFilesLink.cpp b/arangod/IResearch/IResearchMMFilesLink.cpp index a2dd547d2954..153081919eba 100644 --- a/arangod/IResearch/IResearchMMFilesLink.cpp +++ b/arangod/IResearch/IResearchMMFilesLink.cpp @@ -29,6 +29,8 @@ #include "Logger/Logger.h" #include "Logger/LogMacros.h" #include "MMFiles/MMFilesCollection.h" +#include "StorageEngine/EngineSelectorFeature.h" +#include "StorageEngine/StorageEngine.h" #include "VocBase/LogicalCollection.h" #include "IResearchMMFilesLink.h" @@ -120,7 +122,7 @@ struct IResearchMMFilesLink::IndexFactory: public arangodb::IndexTypeFactory { IResearchMMFilesLink::IResearchMMFilesLink( TRI_idx_iid_t iid, arangodb::LogicalCollection& collection -): Index(iid, collection, IResearchLinkHelper::emptyIndexSlice()), +): MMFilesIndex(iid, collection, IResearchLinkHelper::emptyIndexSlice()), IResearchLink(iid, collection) { TRI_ASSERT(!ServerState::instance()->isCoordinator()); _unique = false; // cannot be unique since multiple fields are indexed @@ -165,6 +167,20 @@ void IResearchMMFilesLink::toVelocyPack( builder.close(); } +bool IResearchMMFilesLink::isPersistent() const { + auto* engine = arangodb::EngineSelectorFeature::ENGINE; + + // FIXME TODO remove once MMFilesEngine will fillIndex(...) during recovery + // currently the index is created but fill is deffered untill the end of recovery + // at the end of recovery only non-persistent indexes are filled + if (engine && engine->inRecovery()) { + return false; + } + + return true; // records persisted into the iResearch view +} + + NS_END // iresearch NS_END // arangodb diff --git a/arangod/IResearch/IResearchMMFilesLink.h b/arangod/IResearch/IResearchMMFilesLink.h index cdcde99033fe..2b2a50196b69 100644 --- a/arangod/IResearch/IResearchMMFilesLink.h +++ b/arangod/IResearch/IResearchMMFilesLink.h @@ -26,7 +26,7 @@ #include "IResearchLink.h" -#include "Indexes/Index.h" +#include "MMFiles/MMFilesIndex.h" namespace arangodb { @@ -38,7 +38,7 @@ NS_BEGIN(arangodb) NS_BEGIN(iresearch) class IResearchMMFilesLink final - : public arangodb::Index, public IResearchLink { + : public arangodb::MMFilesIndex, public IResearchLink { public: void afterTruncate(TRI_voc_tick_t /*tick*/) override { IResearchLink::afterTruncate(); @@ -80,13 +80,15 @@ class IResearchMMFilesLink final return IResearchLink::insert(trx, documentId, doc, mode); } - virtual bool isPersistent() const override { - return IResearchLink::isPersistent(); - } + bool isPersistent() const override; virtual bool isSorted() const override { return IResearchLink::isSorted(); } + + bool isHidden() const override { + return IResearchLink::isHidden(); + } virtual arangodb::IndexIterator* iteratorForCondition( arangodb::transaction::Methods* trx, @@ -160,4 +162,4 @@ class IResearchMMFilesLink final NS_END // iresearch NS_END // arangodb -#endif \ No newline at end of file +#endif diff --git a/arangod/IResearch/IResearchRocksDBLink.cpp b/arangod/IResearch/IResearchRocksDBLink.cpp index 5655490cc2ff..4faa496df6c8 100644 --- a/arangod/IResearch/IResearchRocksDBLink.cpp +++ b/arangod/IResearch/IResearchRocksDBLink.cpp @@ -171,7 +171,7 @@ void IResearchRocksDBLink::toVelocyPack( )); } - if (arangodb::Index::hasFlag(flags, arangodb::Index::Serialize::ObjectId)) { + if (arangodb::Index::hasFlag(flags, arangodb::Index::Serialize::Internals)) { TRI_ASSERT(_objectId != 0); // If we store it, it cannot be 0 builder.add("objectId", VPackValue(std::to_string(_objectId))); } @@ -193,4 +193,4 @@ NS_END // arangodb // ----------------------------------------------------------------------------- // --SECTION-- END-OF-FILE -// ----------------------------------------------------------------------------- \ No newline at end of file +// ----------------------------------------------------------------------------- diff --git a/arangod/IResearch/IResearchRocksDBLink.h b/arangod/IResearch/IResearchRocksDBLink.h index c659c929a48e..d9718193fa36 100644 --- a/arangod/IResearch/IResearchRocksDBLink.h +++ b/arangod/IResearch/IResearchRocksDBLink.h @@ -84,6 +84,10 @@ class IResearchRocksDBLink final virtual bool isSorted() const override { return IResearchLink::isSorted(); } + + bool isHidden() const override { + return IResearchLink::isHidden(); + } virtual arangodb::IndexIterator* iteratorForCondition( arangodb::transaction::Methods* trx, @@ -158,4 +162,4 @@ class IResearchRocksDBLink final NS_END // iresearch NS_END // arangodb -#endif \ No newline at end of file +#endif diff --git a/arangod/IResearch/IResearchRocksDBRecoveryHelper.cpp b/arangod/IResearch/IResearchRocksDBRecoveryHelper.cpp index 8ec3d3be6779..db66c91a39d6 100644 --- a/arangod/IResearch/IResearchRocksDBRecoveryHelper.cpp +++ b/arangod/IResearch/IResearchRocksDBRecoveryHelper.cpp @@ -106,7 +106,7 @@ void ensureLink( auto const indexTypeSlice = indexSlice.get(arangodb::StaticStrings::IndexType); auto const indexTypeStr = indexTypeSlice.copyString(); - auto const indexType = arangodb::Index::type(indexTypeStr.c_str()); + auto const indexType = arangodb::Index::type(indexTypeStr); if (arangodb::Index::IndexType::TRI_IDX_TYPE_IRESEARCH_LINK != indexType) { // skip non iresearch link indexes @@ -405,4 +405,4 @@ void IResearchRocksDBRecoveryHelper::LogData(const rocksdb::Slice& blob) { // ----------------------------------------------------------------------------- // --SECTION-- END-OF-FILE -// ----------------------------------------------------------------------------- \ No newline at end of file +// ----------------------------------------------------------------------------- diff --git a/arangod/IResearch/IResearchView.cpp b/arangod/IResearch/IResearchView.cpp index eed1afdf4091..96ab4b50a9f1 100644 --- a/arangod/IResearch/IResearchView.cpp +++ b/arangod/IResearch/IResearchView.cpp @@ -557,7 +557,7 @@ arangodb::Result IResearchView::appendVelocyPackImpl( arangodb::velocypack::ObjectBuilder linksBuilderWrapper(&linksBuilder); for (auto& collectionName: state->collectionNames()) { - for (auto& index: trx.indexesForCollection(collectionName)) { + for (auto& index: trx.indexesForCollection(collectionName, /*withHidden*/true)) { if (index && arangodb::Index::IndexType::TRI_IDX_TYPE_IRESEARCH_LINK == index->type()) { // TODO FIXME find a better way to retrieve an IResearch Link // cannot use static_cast/reinterpret_cast since Index is not related to IResearchLink @@ -929,7 +929,9 @@ arangodb::Result IResearchView::properties( return res; } +#if USE_PLAN_CACHE arangodb::aql::PlanCache::instance()->invalidate(&vocbase()); +#endif arangodb::aql::QueryCache::instance()->invalidate(&vocbase()); return arangodb::ServerState::instance()->isSingleServer() @@ -1319,4 +1321,4 @@ void IResearchView::verifyKnownCollections() { // ----------------------------------------------------------------------------- // --SECTION-- END-OF-FILE -// ----------------------------------------------------------------------------- \ No newline at end of file +// ----------------------------------------------------------------------------- diff --git a/arangod/Indexes/Index.cpp b/arangod/Indexes/Index.cpp index 4f18bc4b93e1..193e51fa451d 100644 --- a/arangod/Indexes/Index.cpp +++ b/arangod/Indexes/Index.cpp @@ -133,6 +133,10 @@ void markAsNonNull(arangodb::aql::AstNode const* op, arangodb::aql::AstNode cons } } +bool typeMatch(char const* type, size_t len, char const* expected) { + return (len == ::strlen(expected)) && (::memcmp(type, expected, len) == 0); +} + } // namespace // If the Index is on a coordinator instance the index may not access the @@ -197,12 +201,11 @@ size_t Index::sortWeight(arangodb::aql::AstNode const* node) { /// @brief validate fields from slice void Index::validateFields(VPackSlice const& slice) { - auto allowExpansion = Index::allowExpansion( - Index::type(slice.get(arangodb::StaticStrings::IndexType).copyString()) - ); + VPackValueLength len; + const char *idxStr = slice.get(arangodb::StaticStrings::IndexType).getString(len); + auto allowExpansion = Index::allowExpansion(Index::type(idxStr, len)); auto fields = slice.get(arangodb::StaticStrings::IndexFields); - if (!fields.isArray()) { return; } @@ -220,40 +223,42 @@ void Index::validateFields(VPackSlice const& slice) { } /// @brief return the index type based on a type name -Index::IndexType Index::type(char const* type) { - if (::strcmp(type, "primary") == 0) { +Index::IndexType Index::type(char const* type, size_t len) { + if (::typeMatch(type, len, "primary")) { return TRI_IDX_TYPE_PRIMARY_INDEX; } - if (::strcmp(type, "edge") == 0) { + if (::typeMatch(type, len, "edge")) { return TRI_IDX_TYPE_EDGE_INDEX; } - if (::strcmp(type, "hash") == 0) { + if (::typeMatch(type, len, "hash")) { return TRI_IDX_TYPE_HASH_INDEX; } - if (::strcmp(type, "skiplist") == 0) { + if (::typeMatch(type, len, "skiplist")) { return TRI_IDX_TYPE_SKIPLIST_INDEX; } - if (::strcmp(type, "persistent") == 0 || ::strcmp(type, "rocksdb") == 0) { + if (::typeMatch(type, len, "persistent") || + ::typeMatch(type, len, "rocksdb")) { return TRI_IDX_TYPE_PERSISTENT_INDEX; } - if (::strcmp(type, "fulltext") == 0) { + if (::typeMatch(type, len, "fulltext")) { return TRI_IDX_TYPE_FULLTEXT_INDEX; } - if (::strcmp(type, "geo1") == 0) { + if (::typeMatch(type, len, "geo")) { + return TRI_IDX_TYPE_GEO_INDEX; + } + if (::typeMatch(type, len, "geo1")) { return TRI_IDX_TYPE_GEO1_INDEX; } - if (::strcmp(type, "geo2") == 0) { + if (::typeMatch(type, len, "geo2")) { return TRI_IDX_TYPE_GEO2_INDEX; } - if (::strcmp(type, "geo") == 0) { - return TRI_IDX_TYPE_GEO_INDEX; - } #ifdef USE_IRESEARCH - if (arangodb::iresearch::DATA_SOURCE_TYPE.name() == type) { + std::string const& tmp = arangodb::iresearch::DATA_SOURCE_TYPE.name(); + if (::typeMatch(type, len, tmp.c_str())) { return TRI_IDX_TYPE_IRESEARCH_LINK; } #endif - if (::strcmp(type, "noaccess") == 0) { + if (::typeMatch(type, len, "noaccess")) { return TRI_IDX_TYPE_NO_ACCESS_INDEX; } @@ -261,7 +266,7 @@ Index::IndexType Index::type(char const* type) { } Index::IndexType Index::type(std::string const& type) { - return Index::type(type.c_str()); + return Index::type(type.c_str(), type.size()); } /// @brief return the name of an index type @@ -942,4 +947,4 @@ std::ostream& operator<<(std::ostream& stream, arangodb::Index const& index) { // ----------------------------------------------------------------------------- // --SECTION-- END-OF-FILE -// ----------------------------------------------------------------------------- \ No newline at end of file +// ----------------------------------------------------------------------------- diff --git a/arangod/Indexes/Index.h b/arangod/Indexes/Index.h index 20e15070258d..54e48273436b 100644 --- a/arangod/Indexes/Index.h +++ b/arangod/Indexes/Index.h @@ -198,7 +198,7 @@ class Index { char const* oldtypeName() const { return oldtypeName(type()); } /// @brief return the index type based on a type name - static IndexType type(char const* type); + static IndexType type(char const* type, size_t len); static IndexType type(std::string const& type); @@ -235,7 +235,6 @@ class Index { static bool Compare(velocypack::Slice const& lhs, velocypack::Slice const& rhs); - virtual bool isPersistent() const { return false; } virtual bool canBeDropped() const = 0; /// @brief whether or not the index provides an iterator that can extract @@ -250,10 +249,13 @@ class Index { /// @brief whether or not the index is sorted virtual bool isSorted() const = 0; + + /// @brief if true this index should not be shown externally + virtual bool isHidden() const = 0; /// @brief whether or not the index has a selectivity estimate virtual bool hasSelectivityEstimate() const = 0; - + /// @brief return the selectivity estimate of the index /// must only be called if hasSelectivityEstimate() returns true /// @@ -281,10 +283,10 @@ class Index { Basics = 0, /// @brief serialize figures for index Figures = 2, - /// @brief serialize object ids for persistence - ObjectId = 4, /// @brief serialize selectivity estimates - Estimates = 8 + Estimates = 4, + /// @brief serialize object ids for persistence + Internals = 8, }; /// @brief helper for building flags diff --git a/arangod/MMFiles/MMFilesCollection.cpp b/arangod/MMFiles/MMFilesCollection.cpp index 0167d77b8edc..594d072a00a9 100644 --- a/arangod/MMFiles/MMFilesCollection.cpp +++ b/arangod/MMFiles/MMFilesCollection.cpp @@ -725,7 +725,6 @@ int MMFilesCollection::close() { engine->changeCollection( _logicalCollection.vocbase(), - _logicalCollection.id(), _logicalCollection, doSync ); @@ -1669,7 +1668,8 @@ void MMFilesCollection::fillIndex( return; } - if (idx->isPersistent() && skipPersistent) { + MMFilesIndex* midx = static_cast(idx); + if (midx->isPersistent() && skipPersistent) { return; } @@ -1714,7 +1714,8 @@ int MMFilesCollection::fillIndexes( if (idx->type() == Index::IndexType::TRI_IDX_TYPE_PRIMARY_INDEX) { continue; } - if (idx->isPersistent()) { + MMFilesIndex* midx = static_cast(idx); + if (midx->isPersistent()) { continue; } idx->unload(); // TODO: check is this safe? truncate not necessarily @@ -2209,35 +2210,6 @@ void MMFilesCollection::prepareIndexes(VPackSlice indexesSlice) { TRI_ASSERT(!_indexes.empty()); } -std::shared_ptr MMFilesCollection::lookupIndex( - VPackSlice const& info) const { - TRI_ASSERT(info.isObject()); - - // extract type - auto value = info.get(arangodb::StaticStrings::IndexType); - - if (!value.isString()) { - // Compatibility with old v8-vocindex. - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, "invalid index definition"); - } - - std::string tmp = value.copyString(); - arangodb::Index::IndexType const type = arangodb::Index::type(tmp.c_str()); - - {READ_LOCKER(guard, _indexesLock); - for (auto const& idx : _indexes) { - if (idx->type() == type) { - // Only check relevant indices - if (idx->matchesDefinition(info)) { - // We found an index for this definition. - return idx; - } - } - } - } - return nullptr; -} - std::shared_ptr MMFilesCollection::createIndex(arangodb::velocypack::Slice const& info, bool restore, bool& created) { SingleCollectionTransaction trx( @@ -2347,7 +2319,7 @@ int MMFilesCollection::saveIndex( std::shared_ptr builder; try { - builder = idx->toVelocyPack(Index::makeFlags(Index::Serialize::ObjectId)); + builder = idx->toVelocyPack(Index::makeFlags(Index::Serialize::Internals)); } catch (arangodb::basics::Exception const& ex) { LOG_TOPIC(ERR, arangodb::Logger::ENGINES) << "cannot save index definition: " << ex.what(); @@ -2429,7 +2401,8 @@ void MMFilesCollection::addIndexLocal(std::shared_ptr idx) { } // update statistics - if (idx->isPersistent()) { + MMFilesIndex* midx = static_cast(idx.get()); + if (midx->isPersistent()) { ++_persistentIndexes; } } @@ -2504,7 +2477,8 @@ bool MMFilesCollection::removeIndex(TRI_idx_iid_t iid) { _indexes.erase(_indexes.begin() + i); // update statistics - if (idx->isPersistent()) { + MMFilesIndex* midx = static_cast(idx.get()); + if (midx->isPersistent()) { --_persistentIndexes; } @@ -3276,7 +3250,7 @@ void MMFilesCollection::setCurrentVersion() { StorageEngine* engine = EngineSelectorFeature::ENGINE; engine->changeCollection(_logicalCollection.vocbase(), - _logicalCollection.id(), _logicalCollection, doSync); + _logicalCollection, doSync); } /// @brief creates a new entry in the primary index @@ -3326,7 +3300,8 @@ Result MMFilesCollection::insertSecondaryIndexes( auto idx = indexes[i]; TRI_ASSERT(idx->type() != Index::IndexType::TRI_IDX_TYPE_PRIMARY_INDEX); - if (!useSecondary && !idx->isPersistent()) { + MMFilesIndex* midx = static_cast(idx.get()); + if (!useSecondary && !midx->isPersistent()) { continue; } @@ -3375,7 +3350,8 @@ Result MMFilesCollection::deleteSecondaryIndexes( auto idx = indexes[i]; TRI_ASSERT(idx->type() != Index::IndexType::TRI_IDX_TYPE_PRIMARY_INDEX); - if (!useSecondary && !idx->isPersistent()) { + MMFilesIndex* midx = static_cast(idx.get()); + if (!useSecondary && !midx->isPersistent()) { continue; } diff --git a/arangod/MMFiles/MMFilesCollection.h b/arangod/MMFiles/MMFilesCollection.h index 00e93a364fcf..dc8fac335d65 100644 --- a/arangod/MMFiles/MMFilesCollection.h +++ b/arangod/MMFiles/MMFilesCollection.h @@ -247,9 +247,6 @@ class MMFilesCollection final : public PhysicalCollection { void prepareIndexes(arangodb::velocypack::Slice indexesSlice) override; - /// @brief Find index by definition - std::shared_ptr lookupIndex(velocypack::Slice const&) const override; - std::unique_ptr getAllIterator(transaction::Methods* trx) const override; std::unique_ptr getAnyIterator(transaction::Methods* trx) const override; void invokeOnAllElements( diff --git a/arangod/MMFiles/MMFilesEngine.cpp b/arangod/MMFiles/MMFilesEngine.cpp index e51bf5e5b5b2..2af2cd5eaa8f 100644 --- a/arangod/MMFiles/MMFilesEngine.cpp +++ b/arangod/MMFiles/MMFilesEngine.cpp @@ -923,11 +923,12 @@ void MMFilesEngine::waitUntilDeletion(TRI_voc_tick_t id, bool force, // to "createCollection" returns std::string MMFilesEngine::createCollection( TRI_vocbase_t& vocbase, - TRI_voc_cid_t id, LogicalCollection const& collection ) { auto path = databasePath(&vocbase); TRI_ASSERT(!path.empty()); + const TRI_voc_cid_t id = collection.id(); + TRI_ASSERT(id != 0); // sanity check if (sizeof(MMFilesDatafileHeaderMarker) + sizeof(MMFilesDatafileFooterMarker) > @@ -1245,11 +1246,10 @@ void MMFilesEngine::destroyCollection( // to "changeCollection" returns void MMFilesEngine::changeCollection( TRI_vocbase_t& vocbase, - TRI_voc_cid_t id, LogicalCollection const& collection, bool doSync ) { - saveCollectionInfo(&vocbase, id, &collection, doSync); + saveCollectionInfo(&vocbase, collection.id(), &collection, doSync); } // asks the storage engine to persist renaming of a collection diff --git a/arangod/MMFiles/MMFilesEngine.h b/arangod/MMFiles/MMFilesEngine.h index 0d63c2b86018..984960271730 100644 --- a/arangod/MMFiles/MMFilesEngine.h +++ b/arangod/MMFiles/MMFilesEngine.h @@ -262,7 +262,6 @@ class MMFilesEngine final : public StorageEngine { // to "createCollection" returns std::string createCollection( TRI_vocbase_t& vocbase, - TRI_voc_cid_t id, LogicalCollection const& collection ) override; @@ -302,7 +301,6 @@ class MMFilesEngine final : public StorageEngine { // to "changeCollection" returns void changeCollection( TRI_vocbase_t& vocbase, - TRI_voc_cid_t id, LogicalCollection const& collection, bool doSync ) override; diff --git a/arangod/MMFiles/MMFilesIndex.h b/arangod/MMFiles/MMFilesIndex.h index a4cb6e5834c3..356cda83d915 100644 --- a/arangod/MMFiles/MMFilesIndex.h +++ b/arangod/MMFiles/MMFilesIndex.h @@ -52,11 +52,19 @@ class MMFilesIndex : public Index { ) : Index(id, collection, info) {} + /// @brief if true this index should not be shown externally + virtual bool isHidden() const override { + return false; // do not generally hide MMFiles indexes + } void afterTruncate(TRI_voc_tick_t) override { // for mmfiles, truncating the index just unloads it unload(); } + + virtual bool isPersistent() const { + return false; + }; }; } diff --git a/arangod/RestHandler/RestIndexHandler.cpp b/arangod/RestHandler/RestIndexHandler.cpp index 327f37f4b845..b9fa51ebadea 100644 --- a/arangod/RestHandler/RestIndexHandler.cpp +++ b/arangod/RestHandler/RestIndexHandler.cpp @@ -93,10 +93,10 @@ RestStatus RestIndexHandler::getIndexes() { if (_request->parsedValue("withStats", false)) { flags = Index::makeFlags(Index::Serialize::Estimates, Index::Serialize::Figures); } - bool withLinks = _request->parsedValue("withLinks", false); + bool withHidden = _request->parsedValue("withHidden", false); VPackBuilder indexes; - Result res = methods::Indexes::getAll(coll.get(), flags, withLinks, indexes); + Result res = methods::Indexes::getAll(coll.get(), flags, withHidden, indexes); if (!res.ok()) { generateError(rest::ResponseCode::BAD, res.errorNumber(), res.errorMessage()); diff --git a/arangod/RestHandler/RestShutdownHandler.cpp b/arangod/RestHandler/RestShutdownHandler.cpp index f146a3fc6996..607539e2df9e 100644 --- a/arangod/RestHandler/RestShutdownHandler.cpp +++ b/arangod/RestHandler/RestShutdownHandler.cpp @@ -100,15 +100,16 @@ RestStatus RestShutdownHandler::execute() { } catch (...) { // Ignore the error } + auto self = shared_from_this(); rest::Scheduler* scheduler = SchedulerFeature::SCHEDULER; // don't block the response for workers waiting on this callback // this should allow workers to go into the IDLE state - scheduler->queue(RequestPriority::HIGH, [this] { + scheduler->queue(RequestPriority::HIGH, [self] { // Give the server 2 seconds to send the reply: std::this_thread::sleep_for(std::chrono::seconds(2)); // Go down: ApplicationServer::server->beginShutdown(); - }); + }); return RestStatus::DONE; } diff --git a/arangod/RocksDBEngine/CMakeLists.txt b/arangod/RocksDBEngine/CMakeLists.txt index 9a45522fd518..ca9d96de041f 100644 --- a/arangod/RocksDBEngine/CMakeLists.txt +++ b/arangod/RocksDBEngine/CMakeLists.txt @@ -40,6 +40,7 @@ endif() # add sources for rocksdb engine set(ROCKSDB_SOURCES RocksDBEngine/RocksDBBackgroundThread.cpp + RocksDBEngine/RocksDBBuilderIndex.cpp RocksDBEngine/RocksDBCollection.cpp RocksDBEngine/RocksDBCollectionMeta.cpp RocksDBEngine/RocksDBCommon.cpp diff --git a/arangod/RocksDBEngine/RocksDBBuilderIndex.cpp b/arangod/RocksDBEngine/RocksDBBuilderIndex.cpp new file mode 100644 index 000000000000..b6f40c84919b --- /dev/null +++ b/arangod/RocksDBEngine/RocksDBBuilderIndex.cpp @@ -0,0 +1,409 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2018 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Simon Grätzer +//////////////////////////////////////////////////////////////////////////////// + +#include "RocksDBBuilderIndex.h" +#include "Basics/VelocyPackHelper.h" +#include "RocksDBEngine/RocksDBColumnFamily.h" +#include "RocksDBEngine/RocksDBCollection.h" +#include "RocksDBEngine/RocksDBCommon.h" +#include "RocksDBEngine/RocksDBMethods.h" +#include "RocksDBEngine/RocksDBTransactionCollection.h" +#include "RocksDBEngine/RocksDBTransactionState.h" +#include "StorageEngine/EngineSelectorFeature.h" +#include "Transaction/StandaloneContext.h" +#include "Utils/SingleCollectionTransaction.h" +#include "VocBase/LogicalCollection.h" +#include "VocBase/ticks.h" + +#include +#include +#include +#include + +#include + +using namespace arangodb; +using namespace arangodb::rocksutils; + +RocksDBBuilderIndex::RocksDBBuilderIndex(std::shared_ptr const& wp) + : RocksDBIndex(wp->id(), wp->collection(), + wp->fields(), wp->unique(), + wp->sparse(), wp->columnFamily(), wp->objectId(), /*useCache*/false), + _wrapped(wp), _hasError(false) { + TRI_ASSERT(_wrapped); +} + +/// @brief return a VelocyPack representation of the index +void RocksDBBuilderIndex::toVelocyPack(VPackBuilder& builder, + std::underlying_type::type flags) const { + VPackBuilder inner; + _wrapped->toVelocyPack(inner, flags); + TRI_ASSERT(inner.slice().isObject()); + builder.openObject(); // FIXME refactor RocksDBIndex::toVelocyPack !! + builder.add(velocypack::ObjectIterator(inner.slice())); + if (Index::hasFlag(flags, Index::Serialize::Internals)) { + builder.add("_inprogress", VPackValue(true)); + } + builder.close(); +} + +/// insert index elements into the specified write batch. +Result RocksDBBuilderIndex::insertInternal(transaction::Methods& trx, RocksDBMethods* mthd, + LocalDocumentId const& documentId, + arangodb::velocypack::Slice const& slice, + OperationMode mode) { + TRI_ASSERT(false); // not enabled + Result r = _wrapped->insertInternal(trx, mthd, documentId, slice, mode); + if (r.is(TRI_ERROR_ARANGO_UNIQUE_CONSTRAINT_VIOLATED)) { + // these are expected errors; store in builder and suppress + bool expected = false; + if (!r.ok() && _hasError.compare_exchange_strong(expected, true)) { + std::lock_guard guard(_errorMutex); + _errorResult = r; + } + return Result(); + } + return r; +} + +/// remove index elements and put it in the specified write batch. +Result RocksDBBuilderIndex::removeInternal(transaction::Methods& trx, RocksDBMethods* mthd, + LocalDocumentId const& documentId, + arangodb::velocypack::Slice const& slice, + OperationMode mode) { + TRI_ASSERT(false); // not enabled + { + std::lock_guard guard(_removedDocsMutex); + _removedDocs.insert(documentId.id()); + } + { // wait for keys do be inserted, so we can remove them again + std::unique_lock guard(_lockedDocsMutex); + if (_lockedDocs.find(documentId.id()) != _lockedDocs.end()) { + _lockedDocsCond.wait(guard); + } + } + + Result r = _wrapped->removeInternal(trx, mthd, documentId, slice, mode); + if (r.is(TRI_ERROR_ARANGO_UNIQUE_CONSTRAINT_VIOLATED)) { + // these are expected errors; store in builder and suppress + bool expected = false; + if (!r.ok() && _hasError.compare_exchange_strong(expected, true)) { + std::lock_guard guard(_errorMutex); + _errorResult = r; + } + return Result(); + } + return r; +} + +namespace { + struct BuilderTrx : public arangodb::transaction::Methods { + BuilderTrx(std::shared_ptr const& transactionContext, + LogicalDataSource const& collection, + AccessMode::Type type) : + transaction::Methods(transactionContext), + _cid(collection.id()) { + // add the (sole) data-source + addCollection(collection.id(), collection.name(), type); + addHint(transaction::Hints::Hint::NO_DLD); + } + + /// @brief get the underlying transaction collection + RocksDBTransactionCollection* resolveTrxCollection() { + return static_cast(trxCollection(_cid)); + } + + private: + TRI_voc_cid_t _cid; +}; +} + +// Background index filler task +// FIXME simon: not used right now because rollbacks are not correctly handled yet +arangodb::Result RocksDBBuilderIndex::fillIndexBackground(std::function const& unlock) { + arangodb::Result res; + + // 1. Index everything under a snapshot iterator (get snapshot under exclusive coll lock) + // 2. Track deleted document IDs so we can avoid indexing them + // 3. Avoid conflicts on unique index keys by using rocksdb::Transaction snapshot conflict checking + // 4. Supress unique constraint violations / conflicts or client drivers + + auto lockedDocsGuard = scopeGuard([&] { // clear all the processed documents + std::lock_guard guard(_lockedDocsMutex); + _lockedDocs.clear(); + _lockedDocsCond.notify_all(); + }); + + // fillindex can be non transactional, we just need to clean up + RocksDBEngine* engine = rocksutils::globalRocksEngine(); + RocksDBCollection* rcoll = static_cast(_collection.getPhysical()); + rocksdb::DB* rootDB = engine->db()->GetRootDB(); + TRI_ASSERT(rootDB != nullptr); + + uint64_t numDocsWritten = 0; + + auto bounds = RocksDBKeyBounds::CollectionDocuments(rcoll->objectId()); + + rocksdb::Slice upper(bounds.end()); // exclusive upper bound + rocksdb::Status s; + rocksdb::WriteOptions wo; + wo.disableWAL = false; // TODO set to true eventually + + // create a read-snapshot under the guard + rocksdb::Snapshot const* snap = rootDB->GetSnapshot(); + auto snapGuard = scopeGuard([&] { + rootDB->ReleaseSnapshot(snap); + }); + TRI_ASSERT(snap != nullptr); + + rocksdb::ReadOptions ro; + ro.snapshot = snap; + ro.prefix_same_as_start = true; + ro.iterate_upper_bound = &upper; + ro.verify_checksums = false; + ro.fill_cache = false; + + rocksdb::ColumnFamilyHandle* docCF = bounds.columnFamily(); + std::unique_ptr it(rootDB->NewIterator(ro, docCF)); + + unlock(); // release indexes write lock + // FIXME use buildertrx + SingleCollectionTransaction trx(transaction::StandaloneContext::Create(_collection.vocbase()), + _collection, AccessMode::Type::WRITE); + res = trx.begin(); + if (res.fail()) { + return res; + } + auto state = RocksDBTransactionState::toState(&trx); + + // transaction used to perform actual indexing + rocksdb::TransactionOptions to; + to.lock_timeout = 100; // 100ms + std::unique_ptr rtrx(engine->db()->BeginTransaction(wo, to)); + if (this->unique()) { + rtrx->SetSnapshot(); // needed for unique index conflict detection + } else { + rtrx->DisableIndexing(); // we never check for existing index keys + } + RocksDBSideTrxMethods batched(state, rtrx.get()); + + RocksDBIndex* internal = _wrapped.get(); + TRI_ASSERT(internal != nullptr); + + // FIXE make selectivity estimates batch wise + it->Seek(bounds.start()); + while (it->Valid() && it->key().compare(upper) < 0) { + + if (_hasError.load(std::memory_order_acquire)) { + std::lock_guard guard(_errorMutex); + res = _errorResult; // a Writer got an error + break; + } + + LocalDocumentId docId = RocksDBKey::documentId(it->key()); + { + // must acquire both locks here to prevent interleaved operations + std::lock_guard guard(_removedDocsMutex); + std::lock_guard guard2(_lockedDocsMutex); + if (_removedDocs.find(docId.id()) != _removedDocs.end()) { + _removedDocs.erase(_removedDocs.find(docId.id())); + it->Next(); + continue; + } + _lockedDocs.insert(docId.id()); + } + + res = internal->insertInternal(trx, &batched, docId, + VPackSlice(it->value().data()), + Index::OperationMode::normal); + if (res.fail()) { + break; + } + numDocsWritten++; + + if (numDocsWritten % 200 == 0) { // commit buffered writes + s = rtrx->Commit(); + if (!s.ok()) { + res = rocksutils::convertStatus(s, rocksutils::StatusHint::index); + break; + } + { // clear all the processed documents + std::lock_guard guard(_lockedDocsMutex); + _lockedDocs.clear(); + _lockedDocsCond.notify_all(); + } + engine->db()->BeginTransaction(wo, to, rtrx.get()); // reuse transaction + if (this->unique()) { + rtrx->SetSnapshot(); + } + } + + it->Next(); + } + + // now actually write all remaining index keys + if (res.ok() && rtrx->GetNumPuts() > 0) { + s = rtrx->Commit(); + if (!s.ok()) { + res = rocksutils::convertStatus(s, rocksutils::StatusHint::index); + } + } + + if (res.ok()) { + res = trx.commit(); // required to commit selectivity estimates + } + + return res; +} + +// fast mode assuming exclusive access locked from outside +template +static arangodb::Result fillIndexFast(RocksDBIndex& ridx, + LogicalCollection& coll, + WriteBatchType& batch) { + Result res; + ::BuilderTrx trx(transaction::StandaloneContext::Create(coll.vocbase()), + coll, AccessMode::Type::EXCLUSIVE); + trx.addHint(transaction::Hints::Hint::LOCK_NEVER); // already locked + res = trx.begin(); + if (!res.ok()) { + THROW_ARANGO_EXCEPTION(res); + } + + RocksDBCollection* rcoll = static_cast(coll.getPhysical()); + auto state = RocksDBTransactionState::toState(&trx); + auto methds = RocksDBTransactionState::toMethods(&trx); + RocksDBTransactionCollection* trxColl = trx.resolveTrxCollection(); + + // fillindex can be non transactional, we just need to clean up + RocksDBEngine* engine = rocksutils::globalRocksEngine(); + rocksdb::DB* rootDB = engine->db()->GetRootDB(); + TRI_ASSERT(rootDB != nullptr); + + uint64_t numDocsWritten = 0; + // write batch will be reset every x documents + MethodsType batched(state, &batch); + + auto bounds = RocksDBKeyBounds::CollectionDocuments(rcoll->objectId()); + rocksdb::Slice upper(bounds.end()); + + rocksdb::Status s; + rocksdb::WriteOptions wo; + wo.disableWAL = false; // TODO set to true eventually + + const rocksdb::Snapshot* snap = rootDB->GetSnapshot(); + auto snapGuard = scopeGuard([&] { + rootDB->ReleaseSnapshot(snap); + }); + + rocksdb::ReadOptions ro; + ro.snapshot = snap; + ro.prefix_same_as_start = true; + ro.iterate_upper_bound = &upper; + ro.verify_checksums = false; + ro.fill_cache = false; + + rocksdb::ColumnFamilyHandle* docCF = RocksDBColumnFamily::documents(); + std::unique_ptr it = methds->NewIterator(ro, docCF); + + auto commitLambda = [&] { + if (batch.GetWriteBatch()->Count() > 0) { + s = rootDB->Write(wo, batch.GetWriteBatch()); + if (!s.ok()) { + res = rocksutils::convertStatus(s, rocksutils::StatusHint::index); + } + } + batch.Clear(); + + auto ops = trxColl->stealTrackedOperations(); + if (!ops.empty()) { + TRI_ASSERT(ridx.hasSelectivityEstimate() && ops.size() == 1); + auto it = ops.begin(); + ridx.estimator()->bufferUpdates(it->first, std::move(it->second.inserts), + std::move(it->second.removals)); + } + }; + + it->Seek(bounds.start()); + while (it->Valid()) { + TRI_ASSERT(it->key().compare(upper) < 0); + + res = ridx.insertInternal(trx, &batched, RocksDBKey::documentId(it->key()), + VPackSlice(it->value().data()), + Index::OperationMode::normal); + if (res.fail()) { + break; + } + numDocsWritten++; + + if (numDocsWritten % 200 == 0) { // commit buffered writes + commitLambda(); + if (res.fail()) { + break; + } + } + + it->Next(); + } + + if (res.ok()) { + commitLambda(); + } + batch.Clear(); + + if (res.ok()) { // required so iresearch commits + res = trx.commit(); + } + + // we will need to remove index elements created before an error + // occurred, this needs to happen since we are non transactional + if (res.fail()) { + RocksDBKeyBounds bounds = ridx.getBounds(); + arangodb::Result res2 = rocksutils::removeLargeRange(rocksutils::globalRocksDB(), bounds, + true, /*useRangeDel*/numDocsWritten > 25000); + if (res2.fail()) { + LOG_TOPIC(WARN, Logger::ENGINES) << "was not able to roll-back " + << "index creation: " << res2.errorMessage(); + } + } + + return res; +} + +/// non-transactional: fill index with existing documents +/// from this collection +arangodb::Result RocksDBBuilderIndex::fillIndexFast() { + RocksDBIndex* internal = _wrapped.get(); + TRI_ASSERT(internal != nullptr); + if (this->unique()) { + const rocksdb::Comparator* cmp = internal->columnFamily()->GetComparator(); + // unique index. we need to keep track of all our changes because we need to avoid + // duplicate index keys. must therefore use a WriteBatchWithIndex + rocksdb::WriteBatchWithIndex batch(cmp, 32 * 1024 * 1024); + return ::fillIndexFast(*internal, _collection, batch); + } else { + // non-unique index. all index keys will be unique anyway because they contain the document id + // we can therefore get away with a cheap WriteBatch + rocksdb::WriteBatch batch(32 * 1024 * 1024); + return ::fillIndexFast(*internal, _collection, batch); + } +} diff --git a/arangod/RocksDBEngine/RocksDBBuilderIndex.h b/arangod/RocksDBEngine/RocksDBBuilderIndex.h new file mode 100644 index 000000000000..ea638d38814c --- /dev/null +++ b/arangod/RocksDBEngine/RocksDBBuilderIndex.h @@ -0,0 +1,149 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Simon Grätzer +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGOD_ROCKSDB_ENGINE_ROCKSDB_BUILDER_INDEX_H +#define ARANGOD_ROCKSDB_ENGINE_ROCKSDB_BUILDER_INDEX_H 1 + +#include "RocksDBEngine/RocksDBIndex.h" + +#include + +namespace arangodb { + +/// Dummy index class that contains the logic to build indexes +/// without an exclusive lock. It wraps the actual index implementation +/// and adds some required synchronization logic on top +class RocksDBBuilderIndex final : public arangodb::RocksDBIndex { + + public: + /// @brief return a VelocyPack representation of the index + void toVelocyPack(velocypack::Builder& builder, + std::underlying_type::type) const override; + + char const* typeName() const override { + return _wrapped->typeName(); + } + + IndexType type() const override { + return _wrapped->type(); + } + + bool canBeDropped() const override { + return false; // TODO ?! + } + + /// @brief whether or not the index is sorted + bool isSorted() const override { + return _wrapped->isSorted(); + } + + /// @brief if true this index should not be shown externally + bool isHidden() const override { + return true; // do not show building indexes + } + + size_t memory() const override { + return _wrapped->memory(); + } + + Result drop() override { + return _wrapped->drop(); + } + + void afterTruncate(TRI_voc_tick_t tick) override { + _wrapped->afterTruncate(tick); + } + + void load() override { + _wrapped->load(); + } + + void unload() override { + _wrapped->unload(); + } + + /// @brief whether or not the index has a selectivity estimate + bool hasSelectivityEstimate() const override { + return false; + } + + /// insert index elements into the specified write batch. + Result insertInternal(transaction::Methods& trx, RocksDBMethods*, + LocalDocumentId const& documentId, + arangodb::velocypack::Slice const&, + OperationMode mode) override; + + /// remove index elements and put it in the specified write batch. + Result removeInternal(transaction::Methods& trx, RocksDBMethods*, + LocalDocumentId const& documentId, + arangodb::velocypack::Slice const&, + OperationMode mode) override; + + RocksDBBuilderIndex(std::shared_ptr const&); + + /// @brief get index estimator, optional + RocksDBCuckooIndexEstimator* estimator() override { + return _wrapped->estimator(); + } + void setEstimator(std::unique_ptr>) override { + TRI_ASSERT(false); + } + void recalculateEstimates() override { + _wrapped->recalculateEstimates(); + } + + /// @brief fill index, will exclusively lock the collection + Result fillIndexFast(); + + /// @brief fill the index, assume already locked exclusively + /// @param unlock called when collection lock can be released + Result fillIndexBackground(std::function const& unlock); + + virtual IndexIterator* iteratorForCondition( + transaction::Methods* trx, + ManagedDocumentResult* result, + aql::AstNode const* condNode, + aql::Variable const* var, + IndexIteratorOptions const& opts + ) override { + TRI_ASSERT(false); + return nullptr; + } + + private: + std::shared_ptr _wrapped; + + std::atomic _hasError; + std::mutex _errorMutex; + Result _errorResult; + + std::mutex _removedDocsMutex; + std::unordered_set _removedDocs; + + std::mutex _lockedDocsMutex; + std::condition_variable _lockedDocsCond; + std::unordered_set _lockedDocs; +}; +} // namespace arangodb + +#endif diff --git a/arangod/RocksDBEngine/RocksDBCollection.cpp b/arangod/RocksDBEngine/RocksDBCollection.cpp index b0a2a5598aa3..a08ebbf273a7 100644 --- a/arangod/RocksDBEngine/RocksDBCollection.cpp +++ b/arangod/RocksDBEngine/RocksDBCollection.cpp @@ -37,6 +37,7 @@ #include "Indexes/IndexIterator.h" #include "RestServer/DatabaseFeature.h" #include "RocksDBEngine/RocksDBPrimaryIndex.h" +#include "RocksDBEngine/RocksDBBuilderIndex.h" #include "RocksDBEngine/RocksDBCommon.h" #include "RocksDBEngine/RocksDBComparator.h" #include "RocksDBEngine/RocksDBEngine.h" @@ -48,16 +49,12 @@ #include "RocksDBEngine/RocksDBSettingsManager.h" #include "RocksDBEngine/RocksDBTransactionCollection.h" #include "RocksDBEngine/RocksDBTransactionState.h" -#include "RocksDBEngine/RocksDBValue.h" #include "StorageEngine/EngineSelectorFeature.h" #include "StorageEngine/StorageEngine.h" -#include "StorageEngine/TransactionState.h" #include "Transaction/Helpers.h" -#include "Transaction/StandaloneContext.h" #include "Utils/CollectionNameResolver.h" #include "Utils/Events.h" #include "Utils/OperationOptions.h" -#include "Utils/SingleCollectionTransaction.h" #include "VocBase/KeyGenerator.h" #include "VocBase/LocalDocumentId.h" #include "VocBase/LogicalCollection.h" @@ -67,7 +64,6 @@ #include #include -#include #include #include @@ -266,20 +262,20 @@ void RocksDBCollection::open(bool /*ignoreErrors*/) { void RocksDBCollection::prepareIndexes( arangodb::velocypack::Slice indexesSlice) { - WRITE_LOCKER(guard, _indexesLock); TRI_ASSERT(indexesSlice.isArray()); StorageEngine* engine = EngineSelectorFeature::ENGINE; std::vector> indexes; - - if (indexesSlice.length() == 0 && _indexes.empty()) { - engine->indexFactory().fillSystemIndexes(_logicalCollection, indexes); - } else { - engine->indexFactory().prepareIndexes( - _logicalCollection, indexesSlice, indexes - ); + { + READ_LOCKER(guard, _indexesLock); // link creation needs read-lock too + if (indexesSlice.length() == 0 && _indexes.empty()) { + engine->indexFactory().fillSystemIndexes(_logicalCollection, indexes); + } else { + engine->indexFactory().prepareIndexes(_logicalCollection, indexesSlice, indexes); + } } - + + WRITE_LOCKER(guard, _indexesLock); for (std::shared_ptr& idx : indexes) { addIndex(std::move(idx)); } @@ -288,80 +284,59 @@ void RocksDBCollection::prepareIndexes( (TRI_COL_TYPE_EDGE == _logicalCollection.type() && (_indexes[1]->type() != Index::IndexType::TRI_IDX_TYPE_EDGE_INDEX || _indexes[2]->type() != Index::IndexType::TRI_IDX_TYPE_EDGE_INDEX))) { - std::string msg = "got invalid indexes for collection '" - + _logicalCollection.name() + "'"; - LOG_TOPIC(ERR, arangodb::Logger::ENGINES) << msg; - + + std::string msg = "got invalid indexes for collection '" + + _logicalCollection.name() + "'"; + LOG_TOPIC(ERR, arangodb::Logger::ENGINES) << msg; #ifdef ARANGODB_ENABLE_MAINTAINER_MODE - for (auto it : _indexes) { - LOG_TOPIC(ERR, arangodb::Logger::ENGINES) << "- " << it.get(); - } -#endif - - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, msg); - } - - TRI_ASSERT(!_indexes.empty()); -} - -static std::shared_ptr findIndex( - velocypack::Slice const& info, - std::vector> const& indexes) { - TRI_ASSERT(info.isObject()); - - auto value = info.get(arangodb::StaticStrings::IndexType); // extract type - - if (!value.isString()) { - // Compatibility with old v8-vocindex. - THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, - "invalid index type definition"); - } - - std::string tmp = value.copyString(); - arangodb::Index::IndexType const type = arangodb::Index::type(tmp.c_str()); - - for (auto const& idx : indexes) { - if (idx->type() == type) { - // Only check relevant indexes - if (idx->matchesDefinition(info)) { - // We found an index for this definition. - return idx; - } + for (auto it : _indexes) { + LOG_TOPIC(ERR, arangodb::Logger::ENGINES) << "- " << it.get(); } +#endif + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, msg); } - return nullptr; -} -/// @brief Find index by definition -std::shared_ptr RocksDBCollection::lookupIndex( - velocypack::Slice const& info) const { - READ_LOCKER(guard, _indexesLock); - return findIndex(info, _indexes); + TRI_ASSERT(!_indexes.empty()); } std::shared_ptr RocksDBCollection::createIndex( - arangodb::velocypack::Slice const& info, bool restore, - bool& created) { + arangodb::velocypack::Slice const& info, + bool restore, bool& created) { TRI_ASSERT(info.isObject()); - SingleCollectionTransaction trx( // prevent concurrent dropping - transaction::StandaloneContext::Create(_logicalCollection.vocbase()), - _logicalCollection, - AccessMode::Type::EXCLUSIVE); - Result res = trx.begin(); + Result res; - if (!res.ok()) { + // Step 0. Lock all the things + TRI_vocbase_t& vocbase = _logicalCollection.vocbase(); + TRI_vocbase_col_status_e status; + res = vocbase.useCollection(&_logicalCollection, status); + if (res.fail()) { THROW_ARANGO_EXCEPTION(res); } - - std::shared_ptr idx = lookupIndex(info); - if (idx) { - created = false; // We already have this index. - return idx; + auto releaseGuard = scopeGuard([&] { + vocbase.releaseCollection(&_logicalCollection); + }); + res = lockWrite(); + if (res.fail()) { + THROW_ARANGO_EXCEPTION(res); } +// WRITE_LOCKER(indexGuard, _indexesLock); + auto unlockGuard = scopeGuard([&] { +// indexGuard.unlock(); // unlock in reverse order + this->unlockWrite(); + }); - RocksDBEngine* engine = static_cast(EngineSelectorFeature::ENGINE); + std::shared_ptr idx; + { // Step 1. Check for matching index + WRITE_LOCKER(guard, _indexesLock); + if ((idx = findIndex(info, _indexes)) != nullptr) { + created = false; // We already have this index. + return idx; + } + } - // We are sure that we do not have an index of this type. + RocksDBEngine* engine = static_cast(EngineSelectorFeature::ENGINE); + + // Step 2. We are sure that we do not have an index of this type. // We also hold the lock. Create it const bool generateKey = !restore; idx = engine->indexFactory().prepareIndexFromSlice( @@ -375,83 +350,135 @@ std::shared_ptr RocksDBCollection::createIndex( TRI_ASSERT(idx->type() != Index::IndexType::TRI_IDX_TYPE_PRIMARY_INDEX); TRI_ASSERT(idx->type() != Index::IndexType::TRI_IDX_TYPE_EDGE_INDEX); - std::shared_ptr other = PhysicalCollection::lookupIndex(idx->id()); - if (other) { // index already exists - return other; + { + READ_LOCKER(guard, _indexesLock); + for (auto const& other : _indexes) { // conflicting index exists + if (other->id() == idx->id()) { + return other; // index already exists + } + } } - res = fillIndexes(trx, idx); - - if (!res.ok()) { - THROW_ARANGO_EXCEPTION(res); - } - - // we need to sync the selectivity estimates - res = engine->settingsManager()->sync(false); - - if (res.fail()) { - LOG_TOPIC(WARN, Logger::ENGINES) << "could not sync settings: " - << res.errorMessage(); + auto buildIdx = std::make_shared(std::static_pointer_cast(idx)); + // Step 3. add index to collection entry (for removal after a crash) + if (!engine->inRecovery()) { // manually modify collection entry, other methods need lock + RocksDBKey key; // read collection info from database + key.constructCollection(_logicalCollection.vocbase().id(), _logicalCollection.id()); + rocksdb::PinnableSlice value; + rocksdb::Status s = engine->db()->Get(rocksdb::ReadOptions(), + RocksDBColumnFamily::definitions(), + key.string(), &value); + if (!s.ok()) { + res.reset(rocksutils::convertStatus(s)); + } else { + VPackBuilder builder; + builder.openObject(); + for (auto const& pair : VPackObjectIterator(VPackSlice(value.data()))) { + if (pair.key.isEqualString("indexes")) { + VPackArrayBuilder arrGuard(&builder, "indexes"); + builder.add(VPackArrayIterator(pair.value)); + buildIdx->toVelocyPack(builder, Index::makeFlags(Index::Serialize::Internals)); + continue; + } + builder.add(pair.key); + builder.add(pair.value); + } + builder.close(); + res = engine->writeCreateCollectionMarker(_logicalCollection.vocbase().id(), + _logicalCollection.id(), + builder.slice(), + RocksDBLogValue::Empty()); + } } + + const bool inBackground = false; // TODO simon: will be enabled in a seperate PR +#if 0 + bool inBackground = basics::VelocyPackHelper::getBooleanValue( + info, StaticStrings::IndexInBackground, false); +#endif - rocksdb::Status s = engine->db()->GetRootDB()->FlushWAL(true); - - if (!s.ok()) { - LOG_TOPIC(WARN, Logger::ENGINES) << "could not flush wal: " - << s.ToString(); + // Step 4. fill index + if (res.ok()) { + if (inBackground) { // allow concurrent inserts into index + _indexes.emplace_back(buildIdx); + res = buildIdx->fillIndexBackground([&] { + unlockGuard.fire(); // will be called at appropriate time + }); + } else { + res = buildIdx->fillIndexFast(); // will lock again internally + } } + // Step 5. cleanup + if (res.ok()) { + { + WRITE_LOCKER(guard, _indexesLock); + if (inBackground) { // swap in actual index + for (size_t i = 0; i < _indexes.size(); i++) { + if (_indexes[i]->id() == buildIdx->id()) { + _indexes[i] = idx; + break; + } + } + } else { + _indexes.push_back(idx); + } + } + +// // we should sync the selectivity estimates TODO fix +// res = engine->settingsManager()->sync(false); +// if (res.fail()) { // not critical +// LOG_TOPIC(WARN, Logger::ENGINES) << "could not sync settings: " +// << res.errorMessage(); +// res.reset(); +// } +// +// rocksdb::Status s = engine->db()->GetRootDB()->FlushWAL(true); +// if (!s.ok()) { // not critical +// LOG_TOPIC(WARN, Logger::ENGINES) << "could not flush wal: " +// << s.ToString(); +// } + #if USE_PLAN_CACHE - arangodb::aql::PlanCache::instance()->invalidate( - _logicalCollection->vocbase()); + arangodb::aql::PlanCache::instance()->invalidate(_logicalCollection->vocbase()); #endif - // Until here no harm is done if something fails. The shared_ptr will - // clean up, if left before - { - WRITE_LOCKER(guard, _indexesLock); - addIndex(idx); - } - if (!engine->inRecovery()) { - auto builder = _logicalCollection.toVelocyPackIgnore( - {"path", "statusString"}, true, /*forPersistence*/ true); - VPackBuilder indexInfo; - idx->toVelocyPack(indexInfo, Index::makeFlags(Index::Serialize::ObjectId)); - res = engine->writeCreateCollectionMarker( - _logicalCollection.vocbase().id(), - _logicalCollection.id(), - builder.slice(), - RocksDBLogValue::IndexCreate( + if (!engine->inRecovery()) { // write new collection marker + auto builder = _logicalCollection.toVelocyPackIgnore( + {"path", "statusString"}, true, /*forPersistence*/ true); + VPackBuilder indexInfo; + idx->toVelocyPack(indexInfo, Index::makeFlags(Index::Serialize::Internals)); + res = engine->writeCreateCollectionMarker( _logicalCollection.vocbase().id(), _logicalCollection.id(), - indexInfo.slice() - ) - ); + builder.slice(), + RocksDBLogValue::IndexCreate( + _logicalCollection.vocbase().id(), + _logicalCollection.id(), + indexInfo.slice() + ) + ); + } } + unlockGuard.fire(); // may have already been fired if (res.fail()) { - // We could not persist the index creation. Better abort - // Remove the Index in the local list again. - size_t i = 0; - WRITE_LOCKER(guard, _indexesLock); - for (auto index : _indexes) { - if (index == idx) { - _indexes.erase(_indexes.begin() + i); - break; + { // We could not create the index. Better abort + WRITE_LOCKER(guard, _indexesLock); + auto it = _indexes.begin(); + while (it != _indexes.end()) { + if ((*it)->id() == idx->id()) { + _indexes.erase(it); + break; + } + it++; } - ++i; } idx->drop(); THROW_ARANGO_EXCEPTION(res); } - res = trx.commit(); - if (res.fail()) { - THROW_ARANGO_EXCEPTION(res); - } - created = true; - return idx; } @@ -462,53 +489,53 @@ bool RocksDBCollection::dropIndex(TRI_idx_iid_t iid) { // invalid index id or primary index return true; } - - size_t i = 0; - WRITE_LOCKER(guard, _indexesLock); - for (std::shared_ptr index : _indexes) { - RocksDBIndex* cindex = static_cast(index.get()); - TRI_ASSERT(cindex != nullptr); - - if (iid == cindex->id()) { - auto rv = cindex->drop().errorNumber(); - - if (rv == TRI_ERROR_NO_ERROR) { - // trigger compaction before deleting the object - cindex->cleanup(); - + + std::shared_ptr toRemove; + { + size_t i = 0; + WRITE_LOCKER(guard, _indexesLock); + for (std::shared_ptr& idx : _indexes) { + if (iid == idx->id()) { + toRemove = std::move(idx); _indexes.erase(_indexes.begin() + i); - events::DropIndex("", std::to_string(iid), TRI_ERROR_NO_ERROR); - // toVelocyPackIgnore will take a read lock and we don't need the - // lock anymore, this branch always returns - guard.unlock(); - - auto engine = static_cast(EngineSelectorFeature::ENGINE); - engine->removeIndexMapping(cindex->objectId()); - - auto builder = _logicalCollection.toVelocyPackIgnore( - {"path", "statusString"}, true, true); - - // log this event in the WAL and in the collection meta-data - int res = engine->writeCreateCollectionMarker( - _logicalCollection.vocbase().id(), - _logicalCollection.id(), - builder.slice(), - RocksDBLogValue::IndexDrop( - _logicalCollection.vocbase().id(), _logicalCollection.id(), iid - ) - ); - - return res == TRI_ERROR_NO_ERROR; + break; } - - break; + ++i; } - ++i; } + if (!toRemove) { // index not found + // We tried to remove an index that does not exist + events::DropIndex("", std::to_string(iid), TRI_ERROR_ARANGO_INDEX_NOT_FOUND); + return false; + } + + READ_LOCKER(guard, _indexesLock); + + RocksDBIndex* cindex = static_cast(toRemove.get()); + TRI_ASSERT(cindex != nullptr); - // We tried to remove an index that does not exist - events::DropIndex("", std::to_string(iid), TRI_ERROR_ARANGO_INDEX_NOT_FOUND); - return false; + Result res = cindex->drop(); + if (res.ok()) { + events::DropIndex("", std::to_string(iid), TRI_ERROR_NO_ERROR); + + // trigger compaction before deleting the object + cindex->compact(); + + auto builder = _logicalCollection.toVelocyPackIgnore( + {"path", "statusString"}, true, true); + + // log this event in the WAL and in the collection meta-data + auto engine = static_cast(EngineSelectorFeature::ENGINE); + res = engine->writeCreateCollectionMarker( + _logicalCollection.vocbase().id(), + _logicalCollection.id(), + builder.slice(), + RocksDBLogValue::IndexDrop( + _logicalCollection.vocbase().id(), _logicalCollection.id(), iid + ) + ); + } + return res.ok(); } std::unique_ptr RocksDBCollection::getAllIterator(transaction::Methods* trx) const { @@ -1257,107 +1284,6 @@ void RocksDBCollection::addIndex(std::shared_ptr idx) { } } -template -static arangodb::Result fillIndex( - transaction::Methods& trx, - RocksDBIndex* ridx, - std::unique_ptr it, - WriteBatchType& batch, - RocksDBCollection* rcol -) { - auto state = RocksDBTransactionState::toState(&trx); - - // fillindex can be non transactional, we just need to clean up - rocksdb::DB* db = rocksutils::globalRocksDB()->GetRootDB(); - TRI_ASSERT(db != nullptr); - - uint64_t numDocsWritten = 0; - // write batch will be reset every x documents - MethodsType batched(state, &batch); - - arangodb::Result res; - auto cb = [&](LocalDocumentId const& documentId, VPackSlice slice) { - if (res.ok()) { - res = ridx->insertInternal(trx, &batched, documentId, slice, - Index::OperationMode::normal); - - if (res.ok()) { - numDocsWritten++; - } - } - }; - - rocksdb::WriteOptions wo; - - bool hasMore = true; - while (hasMore && res.ok()) { - hasMore = it->nextDocument(cb, 250); - - if (TRI_VOC_COL_STATUS_DELETED == it->collection()->status() - || it->collection()->deleted()) { - res = TRI_ERROR_INTERNAL; - } else if (application_features::ApplicationServer::isStopping()) { - res = TRI_ERROR_SHUTTING_DOWN; - } - - if (res.ok()) { - rocksdb::Status s = db->Write(wo, batch.GetWriteBatch()); - if (!s.ok()) { - res = rocksutils::convertStatus(s, rocksutils::StatusHint::index); - break; - } - } - - batch.Clear(); - } - - // we will need to remove index elements created before an error - // occurred, this needs to happen since we are non transactional - if (res.fail()) { - RocksDBKeyBounds bounds = ridx->getBounds(); - arangodb::Result res2 = rocksutils::removeLargeRange(rocksutils::globalRocksDB(), bounds, - true, /*useRangeDel*/numDocsWritten > 25000); - if (res2.fail()) { - LOG_TOPIC(WARN, Logger::ENGINES) << "was not able to roll-back " - << "index creation: " << res2.errorMessage(); - } - } - - return res; -} - -/// non-transactional: fill index with existing documents -/// from this collection -arangodb::Result RocksDBCollection::fillIndexes( - transaction::Methods& trx, - std::shared_ptr added -) { - TRI_ASSERT(trx.state()->collection( - _logicalCollection.id(), AccessMode::Type::EXCLUSIVE - )); - - std::unique_ptr it(new RocksDBAllIndexIterator( - &_logicalCollection, &trx, primaryIndex() - )); - - RocksDBIndex* ridx = static_cast(added.get()); - - if (ridx->unique()) { - // unique index. we need to keep track of all our changes because we need to avoid - // duplicate index keys. must therefore use a WriteBatchWithIndex - rocksdb::WriteBatchWithIndex batch(ridx->columnFamily()->GetComparator(), 32 * 1024 * 1024); - return fillIndex( - trx, ridx, std::move(it), batch, this); - } else { - // non-unique index. all index keys will be unique anyway because they contain the document id - // we can therefore get away with a cheap WriteBatch - rocksdb::WriteBatch batch(32 * 1024 * 1024); - return fillIndex( - trx, ridx, std::move(it), batch, this); - } - return Result(); -} - Result RocksDBCollection::insertDocument( arangodb::transaction::Methods* trx, LocalDocumentId const& documentId, VPackSlice const& doc, OperationOptions& options) const { @@ -1723,10 +1649,8 @@ int RocksDBCollection::lockWrite(double timeout) { } /// @brief write unlocks a collection -int RocksDBCollection::unlockWrite() { +void RocksDBCollection::unlockWrite() { _exclusiveLock.unlockWrite(); - - return TRI_ERROR_NO_ERROR; } /// @brief read locks a collection, with a timeout @@ -1777,9 +1701,8 @@ int RocksDBCollection::lockRead(double timeout) { } /// @brief read unlocks a collection -int RocksDBCollection::unlockRead() { +void RocksDBCollection::unlockRead() { _exclusiveLock.unlockRead(); - return TRI_ERROR_NO_ERROR; } // rescans the collection to update document count @@ -1830,6 +1753,7 @@ uint64_t RocksDBCollection::recalculateCounts() { rocksdb::Slice upper(bounds.end()); rocksdb::ReadOptions ro; + ro.snapshot = snapshot; ro.prefix_same_as_start = true; ro.iterate_upper_bound = &upper; ro.verify_checksums = false; @@ -1865,7 +1789,7 @@ void RocksDBCollection::compact() { READ_LOCKER(guard, _indexesLock); for (std::shared_ptr i : _indexes) { RocksDBIndex* index = static_cast(i.get()); - index->cleanup(); + index->compact(); } } diff --git a/arangod/RocksDBEngine/RocksDBCollection.h b/arangod/RocksDBEngine/RocksDBCollection.h index e5447df14c1a..0702fd019926 100644 --- a/arangod/RocksDBEngine/RocksDBCollection.h +++ b/arangod/RocksDBEngine/RocksDBCollection.h @@ -94,9 +94,6 @@ class RocksDBCollection final : public PhysicalCollection { void prepareIndexes(arangodb::velocypack::Slice indexesSlice) override; - /// @brief Find index by definition - std::shared_ptr lookupIndex(velocypack::Slice const&) const override; - std::shared_ptr createIndex(arangodb::velocypack::Slice const& info, bool restore, bool& created) override; @@ -193,9 +190,9 @@ class RocksDBCollection final : public PhysicalCollection { uint64_t objectId() const { return _objectId; } int lockWrite(double timeout = 0.0); - int unlockWrite(); + void unlockWrite(); int lockRead(double timeout = 0.0); - int unlockRead(); + void unlockRead(); /// recalculte counts for collection in case of failure uint64_t recalculateCounts(); @@ -216,11 +213,6 @@ class RocksDBCollection final : public PhysicalCollection { void figuresSpecific(std::shared_ptr&) override; void addIndex(std::shared_ptr idx); - arangodb::Result fillIndexes( - transaction::Methods& trx, - std::shared_ptr indexes - ); - // @brief return the primary index // WARNING: Make sure that this instance // is somehow protected. If it goes out of all scopes diff --git a/arangod/RocksDBEngine/RocksDBEngine.cpp b/arangod/RocksDBEngine/RocksDBEngine.cpp index 134ed6e75e3d..289006ee9417 100644 --- a/arangod/RocksDBEngine/RocksDBEngine.cpp +++ b/arangod/RocksDBEngine/RocksDBEngine.cpp @@ -1132,10 +1132,11 @@ int RocksDBEngine::writeCreateCollectionMarker(TRI_voc_tick_t databaseId, auto value = RocksDBValue::Collection(slice); rocksdb::WriteOptions wo; - // Write marker + key into RocksDB inside one batch rocksdb::WriteBatch batch; - batch.PutLogData(logValue.slice()); + if (logValue.slice().size() > 0) { + batch.PutLogData(logValue.slice()); + } batch.Put(RocksDBColumnFamily::definitions(), key.string(), value.string()); rocksdb::Status res = db->Write(wo, &batch); @@ -1184,14 +1185,13 @@ void RocksDBEngine::recoveryDone(TRI_vocbase_t& vocbase) { std::string RocksDBEngine::createCollection( TRI_vocbase_t& vocbase, - TRI_voc_cid_t cid, LogicalCollection const& collection ) { - auto builder = collection.toVelocyPackIgnore( - {"path", "statusString"}, /*translate cid*/ true, /*for persistence*/ true - ); - + const TRI_voc_cid_t cid = collection.id(); TRI_ASSERT(cid != 0); + + auto builder = collection.toVelocyPackIgnore({"path", "statusString"}, + /*translateCid*/ true, /*forPersist*/ true); TRI_UpdateTickServer(static_cast(cid)); int res = writeCreateCollectionMarker( @@ -1358,7 +1358,6 @@ void RocksDBEngine::destroyCollection( void RocksDBEngine::changeCollection( TRI_vocbase_t& vocbase, - TRI_voc_cid_t id, LogicalCollection const& collection, bool doSync ) { @@ -1367,9 +1366,9 @@ void RocksDBEngine::changeCollection( ); int res = writeCreateCollectionMarker( vocbase.id(), - id, + collection.id(), builder.slice(), - RocksDBLogValue::CollectionChange(vocbase.id(), id) + RocksDBLogValue::CollectionChange(vocbase.id(), collection.id()) ); if (res != TRI_ERROR_NO_ERROR) { diff --git a/arangod/RocksDBEngine/RocksDBEngine.h b/arangod/RocksDBEngine/RocksDBEngine.h index 3ef38e6e61d8..1df27f6d651c 100644 --- a/arangod/RocksDBEngine/RocksDBEngine.h +++ b/arangod/RocksDBEngine/RocksDBEngine.h @@ -235,7 +235,6 @@ class RocksDBEngine final : public StorageEngine { public: std::string createCollection( TRI_vocbase_t& vocbase, - TRI_voc_cid_t id, LogicalCollection const& collection ) override; @@ -256,7 +255,6 @@ class RocksDBEngine final : public StorageEngine { void changeCollection( TRI_vocbase_t& vocbase, - TRI_voc_cid_t id, LogicalCollection const& collection, bool doSync ) override; diff --git a/arangod/RocksDBEngine/RocksDBGeoIndex.cpp b/arangod/RocksDBEngine/RocksDBGeoIndex.cpp index db2114decad4..8a9e81e4f055 100644 --- a/arangod/RocksDBEngine/RocksDBGeoIndex.cpp +++ b/arangod/RocksDBEngine/RocksDBGeoIndex.cpp @@ -236,7 +236,6 @@ class RDBNearIterator final : public IndexIterator { geo_index::NearUtils _near; std::unique_ptr _iter; }; -typedef RDBNearIterator LegacyIterator; RocksDBGeoIndex::RocksDBGeoIndex( TRI_idx_iid_t iid, diff --git a/arangod/RocksDBEngine/RocksDBIndex.cpp b/arangod/RocksDBEngine/RocksDBIndex.cpp index 0dd3a851d796..1610291816bd 100644 --- a/arangod/RocksDBEngine/RocksDBIndex.cpp +++ b/arangod/RocksDBEngine/RocksDBIndex.cpp @@ -38,6 +38,10 @@ #include "VocBase/ticks.h" #include +#include +#include +#include + using namespace arangodb; using namespace arangodb::rocksutils; @@ -49,6 +53,12 @@ using namespace arangodb::rocksutils; uint64_t const arangodb::RocksDBIndex::ESTIMATOR_SIZE = 4096; +namespace { + inline uint64_t ensureObjectId(uint64_t oid) { + return (oid != 0) ? oid : TRI_NewTickServer(); + } +} + RocksDBIndex::RocksDBIndex( TRI_idx_iid_t id, LogicalCollection& collection, @@ -60,11 +70,11 @@ RocksDBIndex::RocksDBIndex( bool useCache ) : Index(id, collection, attributes, unique, sparse), - _objectId((objectId != 0) ? objectId : TRI_NewTickServer()), + _objectId(::ensureObjectId(objectId)), _cf(cf), _cache(nullptr), _cachePresent(false), - _cacheEnabled(useCache && !collection.system() && CacheManagerFeature::MANAGER != nullptr) { + _cacheEnabled(useCache && !collection.system() && CacheManagerFeature::MANAGER != nullptr){ TRI_ASSERT(cf != nullptr && cf != RocksDBColumnFamily::definitions()); if (_cacheEnabled) { @@ -86,29 +96,27 @@ RocksDBIndex::RocksDBIndex( bool useCache ) : Index(id, collection, info), - _objectId(basics::VelocyPackHelper::stringUInt64(info.get("objectId"))), + _objectId(::ensureObjectId(basics::VelocyPackHelper::stringUInt64(info.get("objectId")))), _cf(cf), _cache(nullptr), _cachePresent(false), _cacheEnabled(useCache && !collection.system() && CacheManagerFeature::MANAGER != nullptr) { TRI_ASSERT(cf != nullptr && cf != RocksDBColumnFamily::definitions()); - if (_objectId == 0) { - _objectId = TRI_NewTickServer(); - } - if (_cacheEnabled) { createCache(); } RocksDBEngine* engine = static_cast(EngineSelectorFeature::ENGINE); - engine->addIndexMapping( _objectId, collection.vocbase().id(), collection.id(), _iid ); } RocksDBIndex::~RocksDBIndex() { + auto engine = static_cast(EngineSelectorFeature::ENGINE); + engine->removeIndexMapping(_objectId); + if (useCache()) { try { TRI_ASSERT(_cache != nullptr); @@ -161,7 +169,7 @@ void RocksDBIndex::unload() { void RocksDBIndex::toVelocyPack(VPackBuilder& builder, std::underlying_type::type flags) const { Index::toVelocyPack(builder, flags); - if (Index::hasFlag(flags, Index::Serialize::ObjectId)) { + if (Index::hasFlag(flags, Index::Serialize::Internals)) { // If we store it, it cannot be 0 TRI_ASSERT(_objectId != 0); builder.add("objectId", VPackValue(std::to_string(_objectId))); @@ -282,7 +290,7 @@ size_t RocksDBIndex::memory() const { } /// compact the index, should reduce read amplification -void RocksDBIndex::cleanup() { +void RocksDBIndex::compact() { rocksdb::TransactionDB* db = rocksutils::globalRocksDB(); rocksdb::CompactRangeOptions opts; if (_cf != RocksDBColumnFamily::invalid()) { @@ -340,11 +348,3 @@ RocksDBKeyBounds RocksDBIndex::getBounds(Index::IndexType type, THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); } } - -RocksDBCuckooIndexEstimator* RocksDBIndex::estimator() { - return nullptr; -} - -void RocksDBIndex::setEstimator(std::unique_ptr>) { - // Nothing to do. -} diff --git a/arangod/RocksDBEngine/RocksDBIndex.h b/arangod/RocksDBEngine/RocksDBIndex.h index c6af1b9afcd7..808a83cb97cc 100644 --- a/arangod/RocksDBEngine/RocksDBIndex.h +++ b/arangod/RocksDBEngine/RocksDBIndex.h @@ -62,8 +62,13 @@ class RocksDBIndex : public Index { std::underlying_type::type) const override; uint64_t objectId() const { return _objectId; } + + /// @brief if true this index should not be shown externally + virtual bool isHidden() const override { + return false; // do not generally hide indexes + } - bool isPersistent() const override final { return true; } + size_t memory() const override; Result drop() override; @@ -92,9 +97,8 @@ class RocksDBIndex : public Index { void load() override; void unload() override; - size_t memory() const override; - - void cleanup(); + /// compact the index, should reduce read amplification + void compact(); /// @brief provides a size hint for the index Result sizeHint( @@ -149,11 +153,14 @@ class RocksDBIndex : public Index { static RocksDBKeyBounds getBounds(Index::IndexType type, uint64_t objectId, bool unique); - - virtual RocksDBCuckooIndexEstimator* estimator(); - virtual void setEstimator(std::unique_ptr>); + + /// @brief get index estimator, optional + virtual RocksDBCuckooIndexEstimator* estimator() { + return nullptr; + } + virtual void setEstimator(std::unique_ptr>) {} virtual void recalculateEstimates() {} - + protected: RocksDBIndex( TRI_idx_iid_t id, @@ -177,6 +184,8 @@ class RocksDBIndex : public Index { inline bool useCache() const { return (_cacheEnabled && _cachePresent); } void blackListKey(char const* data, std::size_t len); void blackListKey(StringRef& ref) { blackListKey(ref.data(), ref.size()); }; + + protected: uint64_t _objectId; rocksdb::ColumnFamilyHandle* _cf; diff --git a/arangod/RocksDBEngine/RocksDBIndexFactory.cpp b/arangod/RocksDBEngine/RocksDBIndexFactory.cpp index 883cd7d8e36a..5a68f38b2c99 100644 --- a/arangod/RocksDBEngine/RocksDBIndexFactory.cpp +++ b/arangod/RocksDBEngine/RocksDBIndexFactory.cpp @@ -151,6 +151,17 @@ static void ProcessIndexDeduplicateFlag(VPackSlice const definition, builder.add("deduplicate", VPackValue(dup)); } +//////////////////////////////////////////////////////////////////////////////// +/// @brief process the index in background flag and add it to the json +//////////////////////////////////////////////////////////////////////////////// + +static void ProcessIndexInBackgroundFlag(VPackSlice const definition, + VPackBuilder& builder) { + bool bck = basics::VelocyPackHelper::getBooleanValue(definition, + StaticStrings::IndexInBackground, false); + builder.add(StaticStrings::IndexInBackground, VPackValue(bck)); +} + //////////////////////////////////////////////////////////////////////////////// /// @brief enhances the json of a vpack index //////////////////////////////////////////////////////////////////////////////// @@ -163,6 +174,7 @@ static int EnhanceJsonIndexVPack(VPackSlice const definition, ProcessIndexSparseFlag(definition, builder, create); ProcessIndexUniqueFlag(definition, builder); ProcessIndexDeduplicateFlag(definition, builder); + ProcessIndexInBackgroundFlag(definition, builder); } return res; @@ -291,9 +303,9 @@ static int EnhanceJsonIndexFulltext(VPackSlice const definition, namespace { struct DefaultIndexFactory: public arangodb::IndexTypeFactory { - std::string const _type; + arangodb::Index::IndexType const _type; - DefaultIndexFactory(std::string const& type): _type(type) {} + DefaultIndexFactory(arangodb::Index::IndexType type): _type(type) {} virtual bool equal( arangodb::velocypack::Slice const& lhs, @@ -321,10 +333,8 @@ struct DefaultIndexFactory: public arangodb::IndexTypeFactory { } } - auto type = Index::type(_type); - - if (arangodb::Index::IndexType::TRI_IDX_TYPE_GEO1_INDEX == type|| - arangodb::Index::IndexType::TRI_IDX_TYPE_GEO_INDEX == type) { + if (arangodb::Index::IndexType::TRI_IDX_TYPE_GEO1_INDEX == _type|| + arangodb::Index::IndexType::TRI_IDX_TYPE_GEO_INDEX == _type) { // geoJson must be identical if present value = lhs.get("geoJson"); @@ -332,7 +342,7 @@ struct DefaultIndexFactory: public arangodb::IndexTypeFactory { && arangodb::basics::VelocyPackHelper::compare(value, rhs.get("geoJson"), false)) { return false; } - } else if (arangodb::Index::IndexType::TRI_IDX_TYPE_FULLTEXT_INDEX == type) { + } else if (arangodb::Index::IndexType::TRI_IDX_TYPE_FULLTEXT_INDEX == _type) { // minLength value = lhs.get("minLength"); @@ -346,7 +356,7 @@ struct DefaultIndexFactory: public arangodb::IndexTypeFactory { value = lhs.get(arangodb::StaticStrings::IndexFields); if (value.isArray()) { - if (arangodb::Index::IndexType::TRI_IDX_TYPE_HASH_INDEX == type) { + if (arangodb::Index::IndexType::TRI_IDX_TYPE_HASH_INDEX == _type) { arangodb::velocypack::ValueLength const nv = value.length(); // compare fields in arbitrary order @@ -386,7 +396,8 @@ struct DefaultIndexFactory: public arangodb::IndexTypeFactory { }; struct EdgeIndexFactory: public DefaultIndexFactory { - EdgeIndexFactory(std::string const& type): DefaultIndexFactory(type) {} + EdgeIndexFactory() + : DefaultIndexFactory(arangodb::Index::TRI_IDX_TYPE_EDGE_INDEX) {} virtual arangodb::Result instantiate( std::shared_ptr& index, @@ -436,7 +447,8 @@ struct EdgeIndexFactory: public DefaultIndexFactory { }; struct FulltextIndexFactory: public DefaultIndexFactory { - FulltextIndexFactory(std::string const& type): DefaultIndexFactory(type) {} + FulltextIndexFactory() + : DefaultIndexFactory(arangodb::Index::TRI_IDX_TYPE_FULLTEXT_INDEX) {} virtual arangodb::Result instantiate( std::shared_ptr& index, @@ -479,7 +491,8 @@ struct FulltextIndexFactory: public DefaultIndexFactory { }; struct GeoIndexFactory: public DefaultIndexFactory { - GeoIndexFactory(std::string const& type): DefaultIndexFactory(type) {} + GeoIndexFactory() + : DefaultIndexFactory(arangodb::Index::TRI_IDX_TYPE_GEO_INDEX) {} virtual arangodb::Result instantiate( std::shared_ptr& index, @@ -522,7 +535,8 @@ struct GeoIndexFactory: public DefaultIndexFactory { }; struct Geo1IndexFactory: public DefaultIndexFactory { - Geo1IndexFactory(std::string const& type): DefaultIndexFactory(type) {} + Geo1IndexFactory() + : DefaultIndexFactory(arangodb::Index::TRI_IDX_TYPE_GEO_INDEX) {} virtual arangodb::Result instantiate( std::shared_ptr& index, @@ -565,7 +579,8 @@ struct Geo1IndexFactory: public DefaultIndexFactory { }; struct Geo2IndexFactory: public DefaultIndexFactory { - Geo2IndexFactory(std::string const& type): DefaultIndexFactory(type) {} + Geo2IndexFactory() + : DefaultIndexFactory(arangodb::Index::TRI_IDX_TYPE_GEO_INDEX) {} virtual arangodb::Result instantiate( std::shared_ptr& index, @@ -606,9 +621,10 @@ struct Geo2IndexFactory: public DefaultIndexFactory { return EnhanceJsonIndexGeo2(definition, normalized, isCreation); } }; - -struct HashIndexFactory: public DefaultIndexFactory { - HashIndexFactory(std::string const& type): DefaultIndexFactory(type) {} + +template +struct SecondaryIndexFactory: public DefaultIndexFactory { + SecondaryIndexFactory() : DefaultIndexFactory(type) {} virtual arangodb::Result instantiate( std::shared_ptr& index, @@ -617,10 +633,7 @@ struct HashIndexFactory: public DefaultIndexFactory { TRI_idx_iid_t id, bool isClusterConstructor ) const override { - index = std::make_shared( - id, collection, definition - ); - + index = std::make_shared(id, collection, definition); return arangodb::Result(); } @@ -633,7 +646,7 @@ struct HashIndexFactory: public DefaultIndexFactory { normalized.add( arangodb::StaticStrings::IndexType, arangodb::velocypack::Value( - arangodb::Index::oldtypeName(arangodb::Index::TRI_IDX_TYPE_HASH_INDEX) + arangodb::Index::oldtypeName(type) ) ); @@ -650,51 +663,9 @@ struct HashIndexFactory: public DefaultIndexFactory { } }; -struct PersistentIndexFactory: public DefaultIndexFactory { - PersistentIndexFactory(std::string const& type): DefaultIndexFactory(type) {} - - virtual arangodb::Result instantiate( - std::shared_ptr& index, - arangodb::LogicalCollection& collection, - arangodb::velocypack::Slice const& definition, - TRI_idx_iid_t id, - bool isClusterConstructor - ) const override { - index = std::make_shared( - id, collection, definition - ); - - return arangodb::Result(); - } - - virtual arangodb::Result normalize( - arangodb::velocypack::Builder& normalized, - arangodb::velocypack::Slice definition, - bool isCreation - ) const override { - TRI_ASSERT(normalized.isOpenObject()); - normalized.add( - arangodb::StaticStrings::IndexType, - arangodb::velocypack::Value(arangodb::Index::oldtypeName( - arangodb::Index::TRI_IDX_TYPE_PERSISTENT_INDEX - )) - ); - - if (isCreation - && !ServerState::instance()->isCoordinator() - && !definition.hasKey("objectId")) { - normalized.add( - "objectId", - arangodb::velocypack::Value(std::to_string(TRI_NewTickServer())) - ); - } - - return EnhanceJsonIndexVPack(definition, normalized, isCreation); - } -}; - struct PrimaryIndexFactory: public DefaultIndexFactory { - PrimaryIndexFactory(std::string const& type): DefaultIndexFactory(type) {} + PrimaryIndexFactory() + : DefaultIndexFactory(arangodb::Index::TRI_IDX_TYPE_PRIMARY_INDEX) {} virtual arangodb::Result instantiate( std::shared_ptr& index, @@ -740,61 +711,21 @@ struct PrimaryIndexFactory: public DefaultIndexFactory { } }; -struct SkiplistIndexFactory: public DefaultIndexFactory { - SkiplistIndexFactory(std::string const& type): DefaultIndexFactory(type) {} - - virtual arangodb::Result instantiate( - std::shared_ptr& index, - arangodb::LogicalCollection& collection, - arangodb::velocypack::Slice const& definition, - TRI_idx_iid_t id, - bool isClusterConstructor - ) const override { - index = std::make_shared( - id, collection, definition - ); - - return arangodb::Result(); - } - - virtual arangodb::Result normalize( - arangodb::velocypack::Builder& normalized, - arangodb::velocypack::Slice definition, - bool isCreation - ) const override { - TRI_ASSERT(normalized.isOpenObject()); - normalized.add( - arangodb::StaticStrings::IndexType, - arangodb::velocypack::Value(arangodb::Index::oldtypeName( - arangodb::Index::TRI_IDX_TYPE_SKIPLIST_INDEX - )) - ); - - if (isCreation - && !ServerState::instance()->isCoordinator() - && !definition.hasKey("objectId")) { - normalized.add( - "objectId", - arangodb::velocypack::Value(std::to_string(TRI_NewTickServer())) - ); - } - - return EnhanceJsonIndexVPack(definition, normalized, isCreation); - } -}; - } RocksDBIndexFactory::RocksDBIndexFactory() { - static const EdgeIndexFactory edgeIndexFactory("edge"); - static const FulltextIndexFactory fulltextIndexFactory("fulltext"); - static const GeoIndexFactory geoIndexFactory("geo"); - static const Geo1IndexFactory geo1IndexFactory("geo1"); - static const Geo2IndexFactory geo2IndexFactory("geo2"); - static const HashIndexFactory hashIndexFactory("hash"); - static const PersistentIndexFactory persistentIndexFactory("persistent"); - static const PrimaryIndexFactory primaryIndexFactory("primary"); - static const SkiplistIndexFactory skiplistIndexFactory("skiplist"); + static const EdgeIndexFactory edgeIndexFactory; + static const FulltextIndexFactory fulltextIndexFactory; + static const GeoIndexFactory geoIndexFactory; + static const Geo1IndexFactory geo1IndexFactory; + static const Geo2IndexFactory geo2IndexFactory; + static const SecondaryIndexFactory hashIndexFactory; + static const SecondaryIndexFactory persistentIndexFactory; + static const SecondaryIndexFactory skiplistIndexFactory; + static const PrimaryIndexFactory primaryIndexFactory; emplace("edge", edgeIndexFactory); emplace("fulltext", fulltextIndexFactory); @@ -838,8 +769,8 @@ void RocksDBIndexFactory::prepareIndexes( bool splitEdgeIndex = false; TRI_idx_iid_t last = 0; - for (auto const& v : VPackArrayIterator(indexesSlice)) { - if (arangodb::basics::VelocyPackHelper::getBooleanValue(v, "error", + for (VPackSlice v : VPackArrayIterator(indexesSlice)) { + if (arangodb::basics::VelocyPackHelper::getBooleanValue(v, StaticStrings::Error, false)) { // We have an error here. // Do not add index. @@ -848,15 +779,14 @@ void RocksDBIndexFactory::prepareIndexes( } // check for combined edge index from MMFiles; must split! - auto value = v.get("type"); - - if (value.isString()) { - std::string tmp = value.copyString(); - arangodb::Index::IndexType const type = - arangodb::Index::type(tmp.c_str()); + auto typeSlice = v.get(StaticStrings::IndexType); + if (typeSlice.isString()) { + VPackValueLength len; + const char* tmp = typeSlice.getStringUnchecked(len); + arangodb::Index::IndexType const type = arangodb::Index::type(tmp, len); if (type == Index::IndexType::TRI_IDX_TYPE_EDGE_INDEX) { - VPackSlice fields = v.get("fields"); + VPackSlice fields = v.get(StaticStrings::IndexFields); if (fields.isArray() && fields.length() == 2) { VPackBuilder from; @@ -864,8 +794,8 @@ void RocksDBIndexFactory::prepareIndexes( from.openObject(); for (auto const& f : VPackObjectIterator(v)) { - if (arangodb::StringRef(f.key) == "fields") { - from.add(VPackValue("fields")); + if (arangodb::StringRef(f.key) == StaticStrings::IndexFields) { + from.add(VPackValue(StaticStrings::IndexFields)); from.openArray(); from.add(VPackValue(StaticStrings::FromString)); from.close(); @@ -880,18 +810,16 @@ void RocksDBIndexFactory::prepareIndexes( VPackBuilder to; to.openObject(); - for (auto const& f : VPackObjectIterator(v)) { - if (arangodb::StringRef(f.key) == "fields") { - to.add(VPackValue("fields")); + if (arangodb::StringRef(f.key) == StaticStrings::IndexFields) { + to.add(VPackValue(StaticStrings::IndexFields)); to.openArray(); to.add(VPackValue(StaticStrings::ToString)); to.close(); - } else if (arangodb::StringRef(f.key) == "id") { + } else if (arangodb::StringRef(f.key) == StaticStrings::IndexId) { auto iid = basics::StringUtils::uint64(f.value.copyString()) + 1; - last = iid; - to.add("id", VPackValue(std::to_string(iid))); + to.add(StaticStrings::IndexId, VPackValue(std::to_string(iid))); } else { to.add(f.key); to.add(f.value); @@ -930,9 +858,9 @@ void RocksDBIndexFactory::prepareIndexes( b.openObject(); for (auto const& f : VPackObjectIterator(v)) { - if (arangodb::StringRef(f.key) == "id") { + if (arangodb::StringRef(f.key) == StaticStrings::IndexId) { last++; - b.add("id", VPackValue(std::to_string(last))); + b.add(StaticStrings::IndexId, VPackValue(std::to_string(last))); } else { b.add(f.key); b.add(f.value); @@ -957,7 +885,6 @@ void RocksDBIndexFactory::prepareIndexes( } auto idx = prepareIndexFromSlice(v, false, col, true); - if (!idx) { LOG_TOPIC(ERR, arangodb::Logger::ENGINES) << "error creating index from definition '" << v.toString() << "'"; @@ -971,6 +898,12 @@ void RocksDBIndexFactory::prepareIndexes( << v.toJson() << "'"; } #endif + + if (basics::VelocyPackHelper::getBooleanValue(v, "_inprogress", false)) { + LOG_TOPIC(WARN, Logger::ENGINES) << "dropping failed index '" << idx->id() << "'"; + idx->drop(); + continue; + } indexes.emplace_back(std::move(idx)); } diff --git a/arangod/RocksDBEngine/RocksDBLogValue.cpp b/arangod/RocksDBEngine/RocksDBLogValue.cpp index 3e3f94a898f9..d3da40c9ca09 100644 --- a/arangod/RocksDBEngine/RocksDBLogValue.cpp +++ b/arangod/RocksDBEngine/RocksDBLogValue.cpp @@ -129,6 +129,10 @@ RocksDBLogValue RocksDBLogValue::SingleRemoveV2(TRI_voc_tick_t vocbaseId, return RocksDBLogValue(RocksDBLogType::SingleRemoveV2, vocbaseId, cid, rid); } +/*static*/ RocksDBLogValue RocksDBLogValue::Empty() { + return RocksDBLogValue(); +} + RocksDBLogValue::RocksDBLogValue(RocksDBLogType type, uint64_t val) : _buffer() { switch (type) { diff --git a/arangod/RocksDBEngine/RocksDBLogValue.h b/arangod/RocksDBEngine/RocksDBLogValue.h index f22eae87dbc4..ac9ef040377d 100644 --- a/arangod/RocksDBEngine/RocksDBLogValue.h +++ b/arangod/RocksDBEngine/RocksDBLogValue.h @@ -85,6 +85,9 @@ class RocksDBLogValue { static RocksDBLogValue SinglePut(TRI_voc_tick_t vocbaseId, TRI_voc_cid_t cid); static RocksDBLogValue SingleRemoveV2(TRI_voc_tick_t vocbaseId, TRI_voc_cid_t cid, TRI_voc_rid_t rid); + + // empty log value + static RocksDBLogValue Empty(); public: @@ -126,6 +129,7 @@ class RocksDBLogValue { rocksdb::Slice slice() const { return rocksdb::Slice(_buffer); } private: + explicit RocksDBLogValue() {} RocksDBLogValue(RocksDBLogType, uint64_t); RocksDBLogValue(RocksDBLogType, uint64_t, uint64_t); RocksDBLogValue(RocksDBLogType, uint64_t, uint64_t, uint64_t); diff --git a/arangod/RocksDBEngine/RocksDBMethods.cpp b/arangod/RocksDBEngine/RocksDBMethods.cpp index 418657c441a8..08b24b857d5e 100644 --- a/arangod/RocksDBEngine/RocksDBMethods.cpp +++ b/arangod/RocksDBEngine/RocksDBMethods.cpp @@ -383,3 +383,47 @@ std::unique_ptr RocksDBBatchedWithIndexMethods::NewIterator( return std::unique_ptr( _wb->NewIteratorWithBase(_db->NewIterator(ro, cf))); } + +// =================== RocksDBSideTrxMethods ==================== + +/// transaction wrapper, uses the provided rocksdb transaction +RocksDBSideTrxMethods::RocksDBSideTrxMethods(RocksDBTransactionState* state, + rocksdb::Transaction* trx) + : RocksDBMethods(state), _trx(trx) { + _ro.prefix_same_as_start = true; + _ro.fill_cache = false; + } + +rocksdb::Status RocksDBSideTrxMethods::Get(rocksdb::ColumnFamilyHandle* cf, rocksdb::Slice const& key, + std::string* val) { + TRI_ASSERT(cf != nullptr); + return _trx->Get(_ro, cf, key, val); +} + +rocksdb::Status RocksDBSideTrxMethods::Get(rocksdb::ColumnFamilyHandle* cf, rocksdb::Slice const& key, + rocksdb::PinnableSlice* val) { + TRI_ASSERT(cf != nullptr); + return _trx->Get(_ro, cf, key, val); +} + +rocksdb::Status RocksDBSideTrxMethods::Put(rocksdb::ColumnFamilyHandle* cf, + RocksDBKey const& key, rocksdb::Slice const& val) { + TRI_ASSERT(cf != nullptr); + return _trx->Put(cf, key.string(), val); +} +rocksdb::Status RocksDBSideTrxMethods::Delete(rocksdb::ColumnFamilyHandle* cf, + RocksDBKey const& key) { + TRI_ASSERT(cf != nullptr); + return _trx->Delete(cf, key.string()); +} + +rocksdb::Status RocksDBSideTrxMethods::SingleDelete(rocksdb::ColumnFamilyHandle* cf, + RocksDBKey const& key) { + TRI_ASSERT(cf != nullptr); + return _trx->SingleDelete(cf, key.string()); +} + +bool RocksDBSideTrxMethods::DisableIndexing() { + _trx->DisableIndexing(); + return true; +} diff --git a/arangod/RocksDBEngine/RocksDBMethods.h b/arangod/RocksDBEngine/RocksDBMethods.h index 10f07b2d194e..e00ab2ae81ef 100644 --- a/arangod/RocksDBEngine/RocksDBMethods.h +++ b/arangod/RocksDBEngine/RocksDBMethods.h @@ -245,6 +245,40 @@ class RocksDBBatchedWithIndexMethods final : public RocksDBMethods { rocksdb::TransactionDB* _db; rocksdb::WriteBatchWithIndex* _wb; }; + +/// transaction wrapper, uses the provided rocksdb transaction +class RocksDBSideTrxMethods final : public RocksDBMethods { +public: + explicit RocksDBSideTrxMethods(RocksDBTransactionState* state, + rocksdb::Transaction* trx); + + rocksdb::Status Get(rocksdb::ColumnFamilyHandle*, rocksdb::Slice const& key, + std::string* val) override; + rocksdb::Status Get(rocksdb::ColumnFamilyHandle*, rocksdb::Slice const& key, + rocksdb::PinnableSlice* val) override; + rocksdb::Status Put( + rocksdb::ColumnFamilyHandle*, RocksDBKey const& key, + rocksdb::Slice const& val) override; + rocksdb::Status Delete(rocksdb::ColumnFamilyHandle*, + RocksDBKey const& key) override; + rocksdb::Status SingleDelete(rocksdb::ColumnFamilyHandle*, + RocksDBKey const&) override; + + std::unique_ptr NewIterator(rocksdb::ReadOptions const&, + rocksdb::ColumnFamilyHandle*) override { + return nullptr; + } + + void SetSavePoint() override {} + rocksdb::Status RollbackToSavePoint() override { return rocksdb::Status::OK(); } + void PopSavePoint() override {} + + bool DisableIndexing() override; + +private: + rocksdb::Transaction* _trx; + rocksdb::ReadOptions _ro; +}; // INDEXING MAY ONLY BE DISABLED IN TOPLEVEL AQL TRANSACTIONS // THIS IS BECAUSE THESE TRANSACTIONS WILL EITHER READ FROM diff --git a/arangod/RocksDBEngine/RocksDBTransactionCollection.h b/arangod/RocksDBEngine/RocksDBTransactionCollection.h index 9fff50936489..7f60735c6830 100644 --- a/arangod/RocksDBEngine/RocksDBTransactionCollection.h +++ b/arangod/RocksDBEngine/RocksDBTransactionCollection.h @@ -94,6 +94,20 @@ class RocksDBTransactionCollection final : public TransactionCollection { /// @brief Every index can track hashes removed from this index /// Used to update the estimate after the trx commited void trackIndexRemove(uint64_t idxObjectId, uint64_t hash); + + /// @brief tracked index operations + struct IndexOperations { + std::vector inserts; + std::vector removals; + }; + typedef std::unordered_map OperationsMap; + + /// @brief steal the tracked operations from the map + OperationsMap stealTrackedOperations() { + OperationsMap empty; + _trackedIndexOperations.swap(empty); + return empty; + } private: /// @brief request a lock for a collection @@ -114,14 +128,9 @@ class RocksDBTransactionCollection final : public TransactionCollection { uint64_t _numRemoves; bool _usageLocked; - struct IndexOperations { - std::vector inserts; - std::vector removals; - }; - /// @brief A list where all indexes with estimates can store their operations /// Will be applied to the inserter on commit and not applied on abort - std::unordered_map _trackedIndexOperations; + OperationsMap _trackedIndexOperations; }; } diff --git a/arangod/RocksDBEngine/RocksDBTransactionState.cpp b/arangod/RocksDBEngine/RocksDBTransactionState.cpp index 0a81430675e0..c2a80c95b6d2 100644 --- a/arangod/RocksDBEngine/RocksDBTransactionState.cpp +++ b/arangod/RocksDBEngine/RocksDBTransactionState.cpp @@ -213,8 +213,8 @@ void RocksDBTransactionState::createTransaction() { _rocksTransaction->GetState() == rocksdb::Transaction::COMMITED || (_rocksTransaction->GetState() == rocksdb::Transaction::STARTED && _rocksTransaction->GetNumKeys() == 0)); - _rocksTransaction = - db->BeginTransaction(_rocksWriteOptions, trxOpts, _rocksTransaction); + rocksdb::WriteOptions wo; + _rocksTransaction = db->BeginTransaction(wo, trxOpts, _rocksTransaction); // add transaction begin marker if (!hasHint(transaction::Hints::Hint::SINGLE_OPERATION)) { @@ -258,7 +258,7 @@ arangodb::Result RocksDBTransactionState::internalCommit() { } Result result; - if (hasOperations()) { + if (hasOperations()) { // might not have ops for fillIndex // we are actually going to attempt a commit if (!hasHint(transaction::Hints::Hint::SINGLE_OPERATION)) { // add custom commit marker to increase WAL tailing reliability @@ -313,8 +313,9 @@ arangodb::Result RocksDBTransactionState::internalCommit() { // we do this only for Windows here, because all other platforms use the // RocksDB SyncThread to do the syncing if (waitForSync()) { - _rocksWriteOptions.sync = true; - _rocksTransaction->SetWriteOptions(_rocksWriteOptions); + rocksdb::WriteOptions wo; + wo.sync = true; + _rocksTransaction->SetWriteOptions(wo); } #endif @@ -368,15 +369,8 @@ arangodb::Result RocksDBTransactionState::internalCommit() { TRI_ASSERT(_rocksTransaction->GetNumKeys() == 0 && _rocksTransaction->GetNumPuts() == 0 && _rocksTransaction->GetNumDeletes() == 0); - - rocksdb::SequenceNumber seq = 0; - if (_rocksTransaction) { - seq = _rocksTransaction->GetSnapshot()->GetSequenceNumber(); - } else { - TRI_ASSERT(_readSnapshot); - seq = _readSnapshot->GetSequenceNumber(); - } - + // this is most likely the fill index case + rocksdb::SequenceNumber seq = _rocksTransaction->GetSnapshot()->GetSequenceNumber(); for (auto& trxColl : _collections) { TRI_IF_FAILURE("RocksDBCommitCounts") { continue; @@ -409,7 +403,6 @@ Result RocksDBTransactionState::commitTransaction(transaction::Methods* activeTr if (_rocksTransaction != nullptr) { res = internalCommit(); } - if (res.ok()) { updateStatus(transaction::Status::COMMITTED); cleanupTransaction(); // deletes trx @@ -576,11 +569,6 @@ Result RocksDBTransactionState::addOperation( return checkIntermediateCommit(currentSize, hasPerformedIntermediateCommit); } -RocksDBMethods* RocksDBTransactionState::rocksdbMethods() { - TRI_ASSERT(_rocksMethods); - return _rocksMethods.get(); -} - uint64_t RocksDBTransactionState::sequenceNumber() const { if (_rocksTransaction) { return static_cast( diff --git a/arangod/RocksDBEngine/RocksDBTransactionState.h b/arangod/RocksDBEngine/RocksDBTransactionState.h index 64266a4fc492..49ab743e5dc2 100644 --- a/arangod/RocksDBEngine/RocksDBTransactionState.h +++ b/arangod/RocksDBEngine/RocksDBTransactionState.h @@ -120,7 +120,10 @@ class RocksDBTransactionState final : public TransactionState { bool& hasPerformedIntermediateCommit); /// @brief return wrapper around rocksdb transaction - RocksDBMethods* rocksdbMethods(); + RocksDBMethods* rocksdbMethods() { + TRI_ASSERT(_rocksMethods); + return _rocksMethods.get(); + } /// @brief Rocksdb sequence number of snapshot. Works while trx /// has either a snapshot or a transaction @@ -182,8 +185,6 @@ class RocksDBTransactionState final : public TransactionState { /// @brief used for read-only trx and intermediate commits /// For intermediate commits this MUST ONLY be used for iteratos rocksdb::Snapshot const* _readSnapshot; - /// @brief shared write options used - rocksdb::WriteOptions _rocksWriteOptions; /// @brief shared read options which can be used by operations /// For intermediate commits iterators MUST use the _readSnapshot rocksdb::ReadOptions _rocksReadOptions; diff --git a/arangod/StorageEngine/PhysicalCollection.cpp b/arangod/StorageEngine/PhysicalCollection.cpp index 7a56c83e57ca..6083dc6aa907 100644 --- a/arangod/StorageEngine/PhysicalCollection.cpp +++ b/arangod/StorageEngine/PhysicalCollection.cpp @@ -105,6 +105,41 @@ bool PhysicalCollection::hasIndexOfType(arangodb::Index::IndexType type) const { return false; } +/// @brief Find index by definition +/*static*/ std::shared_ptr PhysicalCollection::findIndex( + VPackSlice const& info, + std::vector> const& indexes) { + TRI_ASSERT(info.isObject()); + + auto value = info.get(arangodb::StaticStrings::IndexType); // extract type + + if (!value.isString()) { + // Compatibility with old v8-vocindex. + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_INTERNAL, + "invalid index type definition"); + } + + VPackValueLength len; + const char* str = value.getStringUnchecked(len); + arangodb::Index::IndexType const type = arangodb::Index::type(str, len); + for (auto const& idx : indexes) { + if (idx->type() == type) { + // Only check relevant indexes + if (idx->matchesDefinition(info)) { + // We found an index for this definition. + return idx; + } + } + } + return nullptr; +} + +/// @brief Find index by definition +std::shared_ptr PhysicalCollection::lookupIndex(VPackSlice const& info) const { + READ_LOCKER(guard, _indexesLock); + return findIndex(info, _indexes); +} + std::shared_ptr PhysicalCollection::lookupIndex( TRI_idx_iid_t idxId) const { READ_LOCKER(guard, _indexesLock); diff --git a/arangod/StorageEngine/PhysicalCollection.h b/arangod/StorageEngine/PhysicalCollection.h index 24b098bb84a6..fb64eea41020 100644 --- a/arangod/StorageEngine/PhysicalCollection.h +++ b/arangod/StorageEngine/PhysicalCollection.h @@ -100,9 +100,11 @@ class PhysicalCollection { bool hasIndexOfType(arangodb::Index::IndexType type) const; + /// @brief find index by definition + static std::shared_ptr findIndex(velocypack::Slice const&, + std::vector> const&); /// @brief Find index by definition - virtual std::shared_ptr lookupIndex( - velocypack::Slice const&) const = 0; + std::shared_ptr lookupIndex(velocypack::Slice const&) const; /// @brief Find index by iid std::shared_ptr lookupIndex(TRI_idx_iid_t) const; @@ -117,8 +119,7 @@ class PhysicalCollection { /// @brief create or restore an index /// @param restore utilize specified ID, assume index has to be created virtual std::shared_ptr createIndex( - arangodb::velocypack::Slice const& info, bool restore, - bool& created) = 0; + arangodb::velocypack::Slice const& info, bool restore, bool& created) = 0; virtual bool dropIndex(TRI_idx_iid_t iid) = 0; diff --git a/arangod/StorageEngine/StorageEngine.h b/arangod/StorageEngine/StorageEngine.h index 498848e7afaf..764abcf84c3c 100644 --- a/arangod/StorageEngine/StorageEngine.h +++ b/arangod/StorageEngine/StorageEngine.h @@ -264,7 +264,6 @@ class StorageEngine : public application_features::ApplicationFeature { // to "createCollection" returns virtual std::string createCollection( TRI_vocbase_t& vocbase, - TRI_voc_cid_t id, LogicalCollection const& collection ) = 0; @@ -304,7 +303,6 @@ class StorageEngine : public application_features::ApplicationFeature { // to "changeCollection" returns virtual void changeCollection( TRI_vocbase_t& vocbase, - TRI_voc_cid_t id, LogicalCollection const& collection, bool doSync ) = 0; diff --git a/arangod/Transaction/Methods.cpp b/arangod/Transaction/Methods.cpp index 9303d781e2be..d4a001a230f6 100644 --- a/arangod/Transaction/Methods.cpp +++ b/arangod/Transaction/Methods.cpp @@ -3242,7 +3242,7 @@ Result transaction::Methods::unlockRecursive(TRI_voc_cid_t cid, /// @brief get list of indexes for a collection std::vector> transaction::Methods::indexesForCollection( - std::string const& collectionName) { + std::string const& collectionName, bool withHidden) { if (_state->isCoordinator()) { return indexesForCollectionCoordinator(collectionName); } @@ -3250,7 +3250,18 @@ std::vector> transaction::Methods::indexesForCollection( TRI_voc_cid_t cid = addCollectionAtRuntime(collectionName); LogicalCollection* document = documentCollection(trxCollection(cid)); - return document->getIndexes(); + std::vector> indexes = document->getIndexes(); + if (!withHidden) { + auto it = indexes.begin(); + while (it != indexes.end()) { + if ((*it)->isHidden()) { + it = indexes.erase(it); + } else { + it++; + } + } + } + return indexes; } /// @brief Lock all collections. Only works for selected sub-classes diff --git a/arangod/Transaction/Methods.h b/arangod/Transaction/Methods.h index 9d1fcd1fa006..50fd945cdb10 100644 --- a/arangod/Transaction/Methods.h +++ b/arangod/Transaction/Methods.h @@ -423,7 +423,7 @@ class Methods { /// @brief get all indexes for a collection name ENTERPRISE_VIRT std::vector> indexesForCollection( - std::string const&); + std::string const&, bool withHidden = false); /// @brief Lock all collections. Only works for selected sub-classes virtual int lockCollections(); diff --git a/arangod/V8Server/v8-vocindex.cpp b/arangod/V8Server/v8-vocindex.cpp index 1db700a0684a..4769226609fb 100644 --- a/arangod/V8Server/v8-vocindex.cpp +++ b/arangod/V8Server/v8-vocindex.cpp @@ -179,14 +179,13 @@ static void JS_GetIndexesVocbaseCol( flags = Index::makeFlags(Index::Serialize::Estimates, Index::Serialize::Figures); } - bool withLinks = false; - + bool withHidden = false; if (args.Length() > 1) { - withLinks = TRI_ObjectToBoolean(args[1]); + withHidden = TRI_ObjectToBoolean(args[1]); } VPackBuilder output; - auto res = methods::Indexes::getAll(collection, flags, withLinks, output); + auto res = methods::Indexes::getAll(collection, flags, withHidden, output); if (res.fail()) { TRI_V8_THROW_EXCEPTION(res); diff --git a/arangod/VocBase/LogicalCollection.cpp b/arangod/VocBase/LogicalCollection.cpp index 3faa1034463f..c4d7d06fd63f 100644 --- a/arangod/VocBase/LogicalCollection.cpp +++ b/arangod/VocBase/LogicalCollection.cpp @@ -499,7 +499,7 @@ Result LogicalCollection::rename(std::string&& newName) { TRI_ASSERT(engine != nullptr); name(std::move(newName)); - engine->changeCollection(vocbase(), id(), *this, doSync); + engine->changeCollection(vocbase(), *this, doSync); } catch (basics::Exception const& ex) { // Engine Rename somehow failed. Reset to old name name(std::move(oldName)); @@ -590,10 +590,10 @@ void LogicalCollection::toVelocyPackForClusterInventory(VPackBuilder& result, getIndexesVPack(result, Index::makeFlags(), [](arangodb::Index const* idx) { // we have to exclude the primary and the edge index here, because otherwise // at least the MMFiles engine will try to create it - // AND exclude arangosearch indexes + // AND exclude hidden indexes return (idx->type() != arangodb::Index::TRI_IDX_TYPE_PRIMARY_INDEX && idx->type() != arangodb::Index::TRI_IDX_TYPE_EDGE_INDEX && - idx->type() != arangodb::Index::TRI_IDX_TYPE_IRESEARCH_LINK); + !idx->isHidden()); }); result.add("planVersion", VPackValue(planVersion())); result.add("isReady", VPackValue(isReady)); @@ -642,10 +642,14 @@ arangodb::Result LogicalCollection::appendVelocyPack( // Indexes result.add(VPackValue("indexes")); auto flags = Index::makeFlags(); + // FIXME simon: hide links here, but increase chance of ASAN errors + /* auto filter = [&](arangodb::Index const* idx) { // hide hidden indexes + return (forPersistence || !idx->isHidden()); + };*/ if (forPersistence) { - flags = Index::makeFlags(Index::Serialize::ObjectId); + flags = Index::makeFlags(Index::Serialize::Internals); } - getIndexesVPack(result, flags); + getIndexesVPack(result, flags/*, filter*/); // Cluster Specific result.add(StaticStrings::IsSmart, VPackValue(_isSmart)); @@ -681,6 +685,9 @@ VPackBuilder LogicalCollection::toVelocyPackIgnore( full.openObject(); properties(full, translateCids, forPersistence); full.close(); + if (ignoreKeys.empty()) { + return full; + } return VPackCollection::remove(full.slice(), ignoreKeys); } @@ -795,7 +802,7 @@ arangodb::Result LogicalCollection::properties( ); } - engine->changeCollection(vocbase(), id(), *this, doSync); + engine->changeCollection(vocbase(), *this, doSync); if (DatabaseFeature::DATABASE != nullptr && DatabaseFeature::DATABASE->versionTracker() != nullptr) { @@ -871,7 +878,7 @@ void LogicalCollection::persistPhysicalCollection() { TRI_ASSERT(!ServerState::instance()->isCoordinator()); StorageEngine* engine = EngineSelectorFeature::ENGINE; - auto path = engine->createCollection(vocbase(), id(), *this); + auto path = engine->createCollection(vocbase(), *this); getPhysical()->setPath(path); } diff --git a/arangod/VocBase/Methods/Indexes.cpp b/arangod/VocBase/Methods/Indexes.cpp index f5daf29cd68d..eb15056a846a 100644 --- a/arangod/VocBase/Methods/Indexes.cpp +++ b/arangod/VocBase/Methods/Indexes.cpp @@ -83,7 +83,7 @@ Result Indexes::getIndex(LogicalCollection const* collection, VPackBuilder tmp; - Result res = Indexes::getAll(collection, Index::makeFlags(), false, tmp); + Result res = Indexes::getAll(collection, Index::makeFlags(), /*withHidden*/true, tmp); if (res.ok()) { for (VPackSlice const& index : VPackArrayIterator(tmp.slice())) { if (index.get("id").compareString(name) == 0) { @@ -98,7 +98,7 @@ Result Indexes::getIndex(LogicalCollection const* collection, /// @brief get all indexes, skips view links arangodb::Result Indexes::getAll(LogicalCollection const* collection, std::underlying_type::type flags, - bool withLinks, + bool withHidden, VPackBuilder& result) { VPackBuilder tmp; if (ServerState::instance()->isCoordinator()) { @@ -119,13 +119,9 @@ arangodb::Result Indexes::getAll(LogicalCollection const* collection, VPackBuilder tmpInner; auto c = ClusterInfo::instance()->getCollection(databaseName, cid); -#ifdef USE_IRESEARCH - c->getIndexesVPack(tmpInner, flags, [withLinks](arangodb::Index const* idx) { - return withLinks || idx->type() != Index::TRI_IDX_TYPE_IRESEARCH_LINK; + c->getIndexesVPack(tmpInner, flags, [&](arangodb::Index const* idx) { + return withHidden || !idx->isHidden(); }); -#else - c->getIndexesVPack(tmpInner, flags); -#endif tmp.openArray(); for (VPackSlice const& s : VPackArrayIterator(tmpInner.slice())) { @@ -169,11 +165,9 @@ arangodb::Result Indexes::getAll(LogicalCollection const* collection, tmp.openArray(true); for (std::shared_ptr const& idx : indexes) { -#ifdef USE_IRESEARCH - if (!withLinks && idx->type() == Index::TRI_IDX_TYPE_IRESEARCH_LINK) { + if (!withHidden && idx->isHidden()) { continue; } -#endif idx->toVelocyPack(tmp, flags); } tmp.close(); @@ -197,7 +191,7 @@ arangodb::Result Indexes::getAll(LogicalCollection const* collection, ); if (type.isString() && type.compareString("edge") == 0) { - VPackSlice fields = index.get("fields"); + VPackSlice fields = index.get(StaticStrings::IndexFields); TRI_ASSERT(fields.isArray() && fields.length() <= 2); if (fields.length() == 1) { // merge indexes @@ -240,7 +234,7 @@ arangodb::Result Indexes::getAll(LogicalCollection const* collection, if (fields[0].compareString(StaticStrings::FromString) == 0) { continue; } else if (fields[0].compareString(StaticStrings::ToString) == 0) { - merge.add("fields", VPackValue(VPackValueType::Array)); + merge.add(StaticStrings::IndexFields, VPackValue(VPackValueType::Array)); merge.add(VPackValue(StaticStrings::FromString)); merge.add(VPackValue(StaticStrings::ToString)); merge.close(); diff --git a/arangod/VocBase/Methods/Indexes.h b/arangod/VocBase/Methods/Indexes.h index c726284b8397..558e0133f7be 100644 --- a/arangod/VocBase/Methods/Indexes.h +++ b/arangod/VocBase/Methods/Indexes.h @@ -48,8 +48,7 @@ struct Indexes { /// @brief get all indexes, skips view links static arangodb::Result getAll(LogicalCollection const* collection, std::underlying_type::type, - bool skipLinks, - arangodb::velocypack::Builder&); + bool withHidden, arangodb::velocypack::Builder&); static arangodb::Result createIndex(LogicalCollection*, Index::IndexType, std::vector const&, diff --git a/arangod/VocBase/vocbase.cpp b/arangod/VocBase/vocbase.cpp index 5542db417b91..2b341df512c6 100644 --- a/arangod/VocBase/vocbase.cpp +++ b/arangod/VocBase/vocbase.cpp @@ -778,9 +778,7 @@ int TRI_vocbase_t::dropCollectionWorker(arangodb::LogicalCollection* collection, collection->deleted(true); try { - engine->changeCollection( - *this, collection->id(), *collection, doSync - ); + engine->changeCollection(*this, *collection, doSync); } catch (arangodb::basics::Exception const& ex) { collection->deleted(false); events::DropCollection(colName, ex.code()); @@ -1015,8 +1013,7 @@ void TRI_vocbase_t::inventory( collection->getIndexesVPack(result, Index::makeFlags(), [](arangodb::Index const* idx) { // we have to exclude the primary, edge index and links for dump / restore return (idx->type() != arangodb::Index::TRI_IDX_TYPE_PRIMARY_INDEX && - idx->type() != arangodb::Index::TRI_IDX_TYPE_EDGE_INDEX && - idx->type() != arangodb::Index::TRI_IDX_TYPE_IRESEARCH_LINK); + idx->type() != arangodb::Index::TRI_IDX_TYPE_EDGE_INDEX && !idx->isHidden()); }); result.add("parameters", VPackValue(VPackValueType::Object)); collection->toVelocyPackIgnore(result, { "objectId", "path", "statusString", "indexes" }, true, false); @@ -1753,7 +1750,9 @@ arangodb::Result TRI_vocbase_t::dropView( } // invalidate all entries in the plan and query cache now +#if USE_PLAN_CACHE arangodb::aql::PlanCache::instance()->invalidate(this); +#endif arangodb::aql::QueryCache::instance()->invalidate(this); unregisterView(*view); @@ -2237,4 +2236,4 @@ TRI_voc_rid_t TRI_StringToRid(char const* p, size_t len, bool& isOld, // ----------------------------------------------------------------------------- // --SECTION-- END-OF-FILE -// ----------------------------------------------------------------------------- \ No newline at end of file +// ----------------------------------------------------------------------------- diff --git a/js/client/modules/@arangodb/arango-collection.js b/js/client/modules/@arangodb/arango-collection.js index 36ebd7eb75ab..3d677f6ee287 100644 --- a/js/client/modules/@arangodb/arango-collection.js +++ b/js/client/modules/@arangodb/arango-collection.js @@ -611,10 +611,10 @@ ArangoCollection.prototype.refresh = function () { // / @brief gets all indexes // ////////////////////////////////////////////////////////////////////////////// -ArangoCollection.prototype.getIndexes = ArangoCollection.prototype.indexes = function (withStats, withLinks) { +ArangoCollection.prototype.getIndexes = ArangoCollection.prototype.indexes = function (withStats, withHidden) { let url = this._indexurl() + '&withStats=' + (withStats || false); - if (withLinks) { - url += '&withLinks=true'; + if (withHidden) { + url += '&withHidden=true'; } var requestResult = this._database._connection.GET(url); diff --git a/lib/Basics/RocksDBUtils.cpp b/lib/Basics/RocksDBUtils.cpp index 280e15eb1420..68979017db0f 100644 --- a/lib/Basics/RocksDBUtils.cpp +++ b/lib/Basics/RocksDBUtils.cpp @@ -138,7 +138,7 @@ arangodb::Result convertStatus(rocksdb::Status const& status, StatusHint hint, s // should actually not occur with our RocksDB configuration return {TRI_ERROR_RESOURCE_LIMIT, prefix + "failed to acquire lock due to lock number limit"+ postfix }; } - return {TRI_ERROR_ARANGO_CONFLICT}; + return {TRI_ERROR_ARANGO_CONFLICT, "write-write conflict"}; case rocksdb::Status::Code::kExpired: return {TRI_ERROR_INTERNAL, prefix + "key expired; TTL was set in error"+ postfix}; case rocksdb::Status::Code::kTryAgain: diff --git a/lib/Basics/StaticStrings.cpp b/lib/Basics/StaticStrings.cpp index 4fd3c118871e..6572cfd53b42 100644 --- a/lib/Basics/StaticStrings.cpp +++ b/lib/Basics/StaticStrings.cpp @@ -95,6 +95,8 @@ std::string const StaticStrings::IndexId("id"); std::string const StaticStrings::IndexSparse("sparse"); std::string const StaticStrings::IndexType("type"); std::string const StaticStrings::IndexUnique("unique"); +std::string const StaticStrings::IndexIsBuilding("isBuilding"); +std::string const StaticStrings::IndexInBackground("inBackground"); // HTTP headers std::string const StaticStrings::Accept("accept"); diff --git a/lib/Basics/StaticStrings.h b/lib/Basics/StaticStrings.h index da9b0c2dae54..7e38e6b007da 100644 --- a/lib/Basics/StaticStrings.h +++ b/lib/Basics/StaticStrings.h @@ -94,6 +94,8 @@ class StaticStrings { static std::string const IndexSparse; // index sparsness marker static std::string const IndexType; // index type static std::string const IndexUnique; // index uniqueness marker + static std::string const IndexIsBuilding; // index build in-process + static std::string const IndexInBackground; // index in background // HTTP headers static std::string const Accept; diff --git a/tests/IResearch/IResearchLink-test.cpp b/tests/IResearch/IResearchLink-test.cpp index 9c9dbe2660c1..ef65b9e8dfe2 100644 --- a/tests/IResearch/IResearchLink-test.cpp +++ b/tests/IResearch/IResearchLink-test.cpp @@ -214,7 +214,6 @@ SECTION("test_defaults") { CHECK((false == link->hasExpansion())); CHECK((false == link->hasSelectivityEstimate())); CHECK((false == link->implicitlyUnique())); - CHECK((true == link->isPersistent())); CHECK((false == link->isSorted())); CHECK((0 < link->memory())); CHECK((true == link->sparse())); @@ -266,7 +265,6 @@ SECTION("test_defaults") { CHECK((false == link->hasExpansion())); CHECK((false == link->hasSelectivityEstimate())); CHECK((false == link->implicitlyUnique())); - CHECK((true == link->isPersistent())); CHECK((false == link->isSorted())); CHECK((0 < link->memory())); CHECK((true == link->sparse())); diff --git a/tests/IResearch/IResearchLinkCoordinator-test.cpp b/tests/IResearch/IResearchLinkCoordinator-test.cpp index 944fe299ff13..0aaad381cc90 100644 --- a/tests/IResearch/IResearchLinkCoordinator-test.cpp +++ b/tests/IResearch/IResearchLinkCoordinator-test.cpp @@ -347,7 +347,6 @@ SECTION("test_create_drop") { CHECK((false == index->hasExpansion())); CHECK((false == index->hasSelectivityEstimate())); CHECK((false == index->implicitlyUnique())); - CHECK((true == index->isPersistent())); CHECK((false == index->isSorted())); CHECK((0 < index->memory())); CHECK((true == index->sparse())); @@ -456,7 +455,6 @@ SECTION("test_create_drop") { CHECK((false == index->hasExpansion())); CHECK((false == index->hasSelectivityEstimate())); CHECK((false == index->implicitlyUnique())); - CHECK((true == index->isPersistent())); CHECK((false == index->isSorted())); CHECK((0 < index->memory())); CHECK((true == index->sparse())); diff --git a/tests/IResearch/IResearchViewCoordinator-test.cpp b/tests/IResearch/IResearchViewCoordinator-test.cpp index fa6e27aefac1..361685889bcc 100644 --- a/tests/IResearch/IResearchViewCoordinator-test.cpp +++ b/tests/IResearch/IResearchViewCoordinator-test.cpp @@ -1192,7 +1192,6 @@ SECTION("test_update_links_partial_remove") { CHECK((false == index->hasExpansion())); CHECK((false == index->hasSelectivityEstimate())); CHECK((false == index->implicitlyUnique())); - CHECK((true == index->isPersistent())); CHECK((false == index->isSorted())); CHECK((0 < index->memory())); CHECK((true == index->sparse())); @@ -1238,7 +1237,6 @@ SECTION("test_update_links_partial_remove") { CHECK((false == index->hasExpansion())); CHECK((false == index->hasSelectivityEstimate())); CHECK((false == index->implicitlyUnique())); - CHECK((true == index->isPersistent())); CHECK((false == index->isSorted())); CHECK((0 < index->memory())); CHECK((true == index->sparse())); @@ -1284,7 +1282,6 @@ SECTION("test_update_links_partial_remove") { CHECK((false == index->hasExpansion())); CHECK((false == index->hasSelectivityEstimate())); CHECK((false == index->implicitlyUnique())); - CHECK((true == index->isPersistent())); CHECK((false == index->isSorted())); CHECK((0 < index->memory())); CHECK((true == index->sparse())); @@ -1417,7 +1414,6 @@ SECTION("test_update_links_partial_remove") { CHECK((false == index->hasExpansion())); CHECK((false == index->hasSelectivityEstimate())); CHECK((false == index->implicitlyUnique())); - CHECK((true == index->isPersistent())); CHECK((false == index->isSorted())); CHECK((0 < index->memory())); CHECK((true == index->sparse())); @@ -1464,7 +1460,6 @@ SECTION("test_update_links_partial_remove") { CHECK((false == index->hasExpansion())); CHECK((false == index->hasSelectivityEstimate())); CHECK((false == index->implicitlyUnique())); - CHECK((true == index->isPersistent())); CHECK((false == index->isSorted())); CHECK((0 < index->memory())); CHECK((true == index->sparse())); @@ -1737,7 +1732,6 @@ SECTION("test_update_links_partial_add") { CHECK((false == index->hasExpansion())); CHECK((false == index->hasSelectivityEstimate())); CHECK((false == index->implicitlyUnique())); - CHECK((true == index->isPersistent())); CHECK((false == index->isSorted())); CHECK((0 < index->memory())); CHECK((true == index->sparse())); @@ -1783,7 +1777,6 @@ SECTION("test_update_links_partial_add") { CHECK((false == index->hasExpansion())); CHECK((false == index->hasSelectivityEstimate())); CHECK((false == index->implicitlyUnique())); - CHECK((true == index->isPersistent())); CHECK((false == index->isSorted())); CHECK((0 < index->memory())); CHECK((true == index->sparse())); @@ -1931,7 +1924,6 @@ SECTION("test_update_links_partial_add") { CHECK((false == index->hasExpansion())); CHECK((false == index->hasSelectivityEstimate())); CHECK((false == index->implicitlyUnique())); - CHECK((true == index->isPersistent())); CHECK((false == index->isSorted())); CHECK((0 < index->memory())); CHECK((true == index->sparse())); @@ -1978,7 +1970,6 @@ SECTION("test_update_links_partial_add") { CHECK((false == index->hasExpansion())); CHECK((false == index->hasSelectivityEstimate())); CHECK((false == index->implicitlyUnique())); - CHECK((true == index->isPersistent())); CHECK((false == index->isSorted())); CHECK((0 < index->memory())); CHECK((true == index->sparse())); @@ -2025,7 +2016,6 @@ SECTION("test_update_links_partial_add") { CHECK((false == index->hasExpansion())); CHECK((false == index->hasSelectivityEstimate())); CHECK((false == index->implicitlyUnique())); - CHECK((true == index->isPersistent())); CHECK((false == index->isSorted())); CHECK((0 < index->memory())); CHECK((true == index->sparse())); @@ -2345,7 +2335,6 @@ SECTION("test_update_links_replace") { CHECK((false == index->hasExpansion())); CHECK((false == index->hasSelectivityEstimate())); CHECK((false == index->implicitlyUnique())); - CHECK((true == index->isPersistent())); CHECK((false == index->isSorted())); CHECK((0 < index->memory())); CHECK((true == index->sparse())); @@ -2391,7 +2380,6 @@ SECTION("test_update_links_replace") { CHECK((false == index->hasExpansion())); CHECK((false == index->hasSelectivityEstimate())); CHECK((false == index->implicitlyUnique())); - CHECK((true == index->isPersistent())); CHECK((false == index->isSorted())); CHECK((0 < index->memory())); CHECK((true == index->sparse())); @@ -2521,7 +2509,6 @@ SECTION("test_update_links_replace") { CHECK((false == index->hasExpansion())); CHECK((false == index->hasSelectivityEstimate())); CHECK((false == index->implicitlyUnique())); - CHECK((true == index->isPersistent())); CHECK((false == index->isSorted())); CHECK((0 < index->memory())); CHECK((true == index->sparse())); @@ -2643,7 +2630,6 @@ SECTION("test_update_links_replace") { CHECK((false == index->hasExpansion())); CHECK((false == index->hasSelectivityEstimate())); CHECK((false == index->implicitlyUnique())); - CHECK((true == index->isPersistent())); CHECK((false == index->isSorted())); CHECK((0 < index->memory())); CHECK((true == index->sparse())); @@ -2933,7 +2919,6 @@ SECTION("test_update_links_clear") { CHECK((false == index->hasExpansion())); CHECK((false == index->hasSelectivityEstimate())); CHECK((false == index->implicitlyUnique())); - CHECK((true == index->isPersistent())); CHECK((false == index->isSorted())); CHECK((0 < index->memory())); CHECK((true == index->sparse())); @@ -2980,7 +2965,6 @@ SECTION("test_update_links_clear") { CHECK((false == index->hasExpansion())); CHECK((false == index->hasSelectivityEstimate())); CHECK((false == index->implicitlyUnique())); - CHECK((true == index->isPersistent())); CHECK((false == index->isSorted())); CHECK((0 < index->memory())); CHECK((true == index->sparse())); @@ -3026,7 +3010,6 @@ SECTION("test_update_links_clear") { CHECK((false == index->hasExpansion())); CHECK((false == index->hasSelectivityEstimate())); CHECK((false == index->implicitlyUnique())); - CHECK((true == index->isPersistent())); CHECK((false == index->isSorted())); CHECK((0 < index->memory())); CHECK((true == index->sparse())); @@ -3294,7 +3277,6 @@ SECTION("test_drop_link") { CHECK((false == index->hasExpansion())); CHECK((false == index->hasSelectivityEstimate())); CHECK((false == index->implicitlyUnique())); - CHECK((true == index->isPersistent())); CHECK((false == index->isSorted())); CHECK((0 < index->memory())); CHECK((true == index->sparse())); diff --git a/tests/IResearch/StorageEngineMock.cpp b/tests/IResearch/StorageEngineMock.cpp index dbd9ba0493c1..8efc2965e500 100644 --- a/tests/IResearch/StorageEngineMock.cpp +++ b/tests/IResearch/StorageEngineMock.cpp @@ -168,6 +168,8 @@ class EdgeIndexMock final : public arangodb::Index { bool canBeDropped() const override { return false; } + bool isHidden() const override { return false; } + bool isSorted() const override { return false; } bool hasSelectivityEstimate() const override { return false; } @@ -529,7 +531,8 @@ int PhysicalCollectionMock::close() { return TRI_ERROR_NO_ERROR; // assume close successful } -std::shared_ptr PhysicalCollectionMock::createIndex(arangodb::velocypack::Slice const& info, bool restore, bool& created) { +std::shared_ptr PhysicalCollectionMock::createIndex(arangodb::velocypack::Slice const& info, + bool restore, bool& created) { before(); std::vector> docs; @@ -706,12 +709,6 @@ void PhysicalCollectionMock::invokeOnAllElements(arangodb::transaction::Methods* } } -std::shared_ptr PhysicalCollectionMock::lookupIndex(arangodb::velocypack::Slice const&) const { - before(); - TRI_ASSERT(false); - return nullptr; -} - arangodb::LocalDocumentId PhysicalCollectionMock::lookupKey(arangodb::transaction::Methods*, arangodb::velocypack::Slice const&) const { before(); TRI_ASSERT(false); @@ -1041,7 +1038,6 @@ void StorageEngineMock::addV8Functions() { void StorageEngineMock::changeCollection( TRI_vocbase_t& vocbase, - TRI_voc_cid_t id, arangodb::LogicalCollection const& collection, bool doSync ) { @@ -1074,7 +1070,6 @@ std::string StorageEngineMock::collectionPath( std::string StorageEngineMock::createCollection( TRI_vocbase_t& vocbase, - TRI_voc_cid_t id, arangodb::LogicalCollection const& collection ) { return ""; // physical path of the new collection diff --git a/tests/IResearch/StorageEngineMock.h b/tests/IResearch/StorageEngineMock.h index 8a97407da690..e71e8cb2d48e 100644 --- a/tests/IResearch/StorageEngineMock.h +++ b/tests/IResearch/StorageEngineMock.h @@ -59,7 +59,7 @@ class PhysicalCollectionMock: public arangodb::PhysicalCollection { PhysicalCollectionMock(arangodb::LogicalCollection& collection, arangodb::velocypack::Slice const& info); virtual PhysicalCollection* clone(arangodb::LogicalCollection& collection) const override; virtual int close() override; - virtual std::shared_ptr createIndex(arangodb::velocypack::Slice const& info, bool restore, bool& created) override; + virtual std::shared_ptr createIndex(arangodb::velocypack::Slice const& info, bool, bool&) override; virtual void deferDropCollection(std::function const& callback) override; virtual bool dropIndex(TRI_idx_iid_t iid) override; virtual void figuresSpecific(std::shared_ptr&) override; @@ -75,7 +75,6 @@ class PhysicalCollectionMock: public arangodb::PhysicalCollection { arangodb::KeyLockInfo* /*keyLockInfo*/, std::function callbackDuringLock) override; virtual void invokeOnAllElements(arangodb::transaction::Methods* trx, std::function callback) override; - virtual std::shared_ptr lookupIndex(arangodb::velocypack::Slice const&) const override; virtual arangodb::LocalDocumentId lookupKey(arangodb::transaction::Methods*, arangodb::velocypack::Slice const&) const override; virtual size_t memory() const override; virtual uint64_t numberDocuments(arangodb::transaction::Methods* trx) const override; @@ -171,10 +170,10 @@ class StorageEngineMock: public arangodb::StorageEngine { virtual void addOptimizerRules() override; virtual void addRestHandlers(arangodb::rest::RestHandlerFactory& handlerFactory) override; virtual void addV8Functions() override; - virtual void changeCollection(TRI_vocbase_t& vocbase, TRI_voc_cid_t id, arangodb::LogicalCollection const& collection, bool doSync) override; + virtual void changeCollection(TRI_vocbase_t& vocbase, arangodb::LogicalCollection const& collection, bool doSync) override; virtual arangodb::Result changeView(TRI_vocbase_t& vocbase, arangodb::LogicalView const& view, bool doSync) override; virtual std::string collectionPath(TRI_vocbase_t const& vocbase, TRI_voc_cid_t id) const override; - virtual std::string createCollection(TRI_vocbase_t& vocbase, TRI_voc_cid_t id, arangodb::LogicalCollection const& collection) override; + virtual std::string createCollection(TRI_vocbase_t& vocbase, arangodb::LogicalCollection const& collection) override; virtual std::unique_ptr createDatabase(TRI_voc_tick_t id, arangodb::velocypack::Slice const& args, int& status) override; virtual arangodb::Result createLoggerState(TRI_vocbase_t*, VPackBuilder&) override; virtual std::unique_ptr createPhysicalCollection(arangodb::LogicalCollection& collection, arangodb::velocypack::Slice const& info) override; diff --git a/tests/js/common/shell/shell-index-rocksdb-disabled.js b/tests/js/common/shell/shell-index-rocksdb-disabled.js new file mode 100644 index 000000000000..aec8935296f6 --- /dev/null +++ b/tests/js/common/shell/shell-index-rocksdb-disabled.js @@ -0,0 +1,385 @@ +/*jshint globalstrict:false, strict:false */ +/*global fail, assertEqual, assertNotEqual, assertTrue, assertFalse */ + +//////////////////////////////////////////////////////////////////////////////// +/// @brief test the index +/// +/// @file +/// +/// DISCLAIMER +/// +/// Copyright 2018 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author 2018 Simon Grätzer, Dan Larkin-York +//////////////////////////////////////////////////////////////////////////////// + +const jsunity = require("jsunity"); +const internal = require("internal"); +const errors = internal.errors; +const db = internal.db; + +function backgroundIndexSuite() { + 'use strict'; + const cn = "UnitTestsCollectionIdx"; + const tasks = require("@arangodb/tasks"); + const tasksCompleted = () => { + return 0 === tasks.get().filter((task) => { + return (task.id.match(/^UnitTest/) || task.name.match(/^UnitTest/)); + }).length; + }; + const waitForTasks = () => { + const time = require("internal").time; + const start = time(); + while (!tasksCompleted()) { + if (time() - start > 300) { // wait for 5 minutes maximum + fail("Timeout after 5 minutes"); + } + require("internal").wait(0.5, false); + } + require('internal').wal.flush(true, true); + // wait an extra second for good measure + require("internal").wait(1.0, false); + }; + + return { + + setUp : function () { + db._drop(cn); + db._create(cn); + }, + + tearDown : function () { + tasks.get().forEach(function(task) { + if (task.id.match(/^UnitTest/) || task.name.match(/^UnitTest/)) { + try { + tasks.unregister(task); + } + catch (err) { + } + } + }); + db._drop(cn); + }, + + testInsertParallelNonUnique: function () { + let c = require("internal").db._collection(cn); + // first lets add some initial documents + let x = 10; + while(x-- > 0) { + let docs = []; + for(let i = 0; i < 1000; i++) { + docs.push({value:i}); + } + c.save(docs); + } + + // lets insert the rest via tasks + let n = 9; + for (let i = 0; i < n; ++i) { + let command = `const c = require("internal").db._collection("${cn}"); + let x = 10; + while(x-- > 0) { + let docs = []; + for(let i = 0; i < 1000; i++) { + docs.push({value:i}) + } + c.save(docs); + }`; + tasks.register({ name: "UnitTestsIndexInsert" + i, command: command }); + } + + // create the index on the main thread + c.ensureIndex({type: 'hash', fields: ['value'], unique: false, inBackground: true}); + + // wait for insertion tasks to complete + waitForTasks(); + + // sanity checks + assertEqual(c.count(), 100000); + for (let i = 0; i < 1000; i++) { // 100 entries of each value [0,999] + let cursor = db._query("FOR doc IN @@coll FILTER doc.value == @val RETURN 1", + {'@coll': cn, 'val': i}, {count:true}); + assertEqual(cursor.count(), 100); + } + + internal.waitForEstimatorSync(); // make sure estimates are consistent + let indexes = c.getIndexes(true); + for (let i of indexes) { + switch (i.type) { + case 'primary': + break; + case 'hash': + assertEqual(i.selectivityEstimate, 0.01); + break; + default: + fail(); + } + } + }, + + testInsertParallelUnique: function () { + let c = require("internal").db._collection(cn); + // first lets add some initial documents + let x = 0; + while(x < 10000) { + let docs = []; + for(let i = 0; i < 1000; i++) { + docs.push({value: x++}); + } + c.save(docs); + } + + // lets insert the rest via tasks + for (let i = 1; i < 5; ++i) { + let command = `const c = require("internal").db._collection("${cn}"); + let x = ${i} * 10000; + while(x < ${i + 1} * 10000) { + let docs = []; + for(let i = 0; i < 1000; i++) { + docs.push({value: x++}) + } + c.save(docs); + }`; + tasks.register({ name: "UnitTestsIndexInsert" + i, command: command }); + } + + // create the index on the main thread + c.ensureIndex({type: 'hash', fields: ['value'], unique: true, inBackground: true }); + + // wait for insertion tasks to complete + waitForTasks(); + + // sanity checks + assertEqual(c.count(), 50000); + for (let i = 0; i < 50000; i++) { + const cursor = db._query("FOR doc IN @@coll FILTER doc.value == @val RETURN 1", + {'@coll': cn, 'val': i}, {count:true}); + assertEqual(cursor.count(), 1); + } + + let indexes = c.getIndexes(true); + for (let i of indexes) { + switch (i.type) { + case 'primary': + break; + case 'hash': + assertEqual(i.selectivityEstimate, 1.0); + break; + default: + fail(); + } + } + }, + + testInsertParallelUniqueConstraintViolation: function () { + let c = require("internal").db._collection(cn); + // first lets add some initial documents + let x = 0; + while(x < 10000) { + let docs = []; + for(let i = 0; i < 1000; i++) { + docs.push({value: x++}); + } + c.save(docs); + } + + // lets insert the rest via tasks + for (let i = 1; i < 5; ++i) { + let command = `const c = require("internal").db._collection("${cn}"); + let x = ${i} * 10000; + while(x < ${i + 1} * 10000) { + let docs = []; + for(let i = 0; i < 1000; i++) { + docs.push({value: x++}) + } + c.save(docs); + }`; + tasks.register({ name: "UnitTestsIndexInsert" + i, command: command }); + } + + // now insert a document that will cause a conflict while indexing + c.save({value: 1 }); + + try { + // create the index on the main thread + c.ensureIndex({type: 'hash', fields: ['value'], unique: true, inBackground: true}); + fail(); + } catch(err) { + assertEqual(errors.ERROR_ARANGO_UNIQUE_CONSTRAINT_VIOLATED.code, err.errorNum, err); + } + + // wait for insertion tasks to complete + waitForTasks(); + + // sanity checks + assertEqual(c.count(), 50001); + + let indexes = c.getIndexes(); + for (let i of indexes) { + switch (i.type) { + case 'primary': + break; + case 'hash': + default: + fail(); + } + } + }, + + testRemoveParallel: function () { + let c = require("internal").db._collection(cn); + // first lets add some initial documents + let x = 0; + while(x < 100000) { + let docs = []; + for(let i = 0; i < 1000; i++) { + docs.push({_key: "test_" + x, value: x++}); + } + c.save(docs); + } + + assertEqual(c.count(), 100000); + + // lets remove half via tasks + for (let i = 0; i < 10; ++i) { + let command = `const c = require("internal").db._collection("${cn}"); + if (!c) { + throw new Error('could not find collection'); + } + let x = ${i} * 10000; + while(x < ${i} * 10000 + 5000) { + let docs = []; + for(let i = 0; i < 1000; i++) { + docs.push("test_" + x++); + } + let removed = false; + while (!removed) { + const res = c.remove(docs); + removed = (res.filter(r => !r.error).length === 0); + } + }`; + tasks.register({ name: "UnitTestsIndexRemove" + i, command: command }); + } + + // create the index on the main thread + c.ensureIndex({type: 'hash', fields: ['value'], inBackground: true }); + + // wait for insertion tasks to complete + waitForTasks(); + + // sanity checks + assertEqual(c.count(), 50000); + for (let i = 0; i < 10; i++) { // check for remaining docs via index + for (let x = i * 10000 + 5000; x < (i+1) * 10000; x++) { + const cursor = db._query("FOR doc IN @@coll FILTER doc.value == @val RETURN 1", + {'@coll': cn, 'val': x}, {count:true}); + assertEqual(cursor.count(), 1); + } + } + for (let i = 0; i < 10; i++) { // check for removed docs via index + for (let x = i * 10000; x < i * 10000 + 5000; x++) { + const cursor = db._query("FOR doc IN @@coll FILTER doc.value == @val RETURN 1", + {'@coll': cn, 'val': x}, {count:true}); + assertEqual(cursor.count(), 0); + } + } + + let indexes = c.getIndexes(true); + for (let i of indexes) { + switch (i.type) { + case 'primary': + break; + case 'hash': + break; + default: + fail(); + } + } + }, + + testUpdateParallel: function () { + let c = require("internal").db._collection(cn); + // first lets add some initial documents + let x = 0; + while(x < 100000) { + let docs = []; + for(let i = 0; i < 1000; i++) { + docs.push({_key: "test_" + x, value: x++}); + } + c.save(docs); + } + + assertEqual(c.count(), 100000); + + // lets update all via tasks + for (let i = 0; i < 10; ++i) { + let command = `const c = require("internal").db._collection("${cn}"); + if (!c) { + throw new Error('could not find collection'); + } + let x = ${i * 10000}; + while(x < ${(i+1) * 10000}) { + let updated = false; + const current = x++; + const key = "test_" + current + const doc = {value: current + 100000}; + while (!updated) { + try { + const res = c.update(key, doc); + updated = true; + } catch (err) {} + } + }`; + tasks.register({ name: "UnitTestsIndexUpdate" + i, command: command }); + } + + // wait for insertion tasks to complete + waitForTasks(); + + // create the index on the main thread + c.ensureIndex({type: 'skiplist', fields: ['value'], inBackground: true }); + + // sanity checks + assertEqual(c.count(), 100000); + // check for new entries via index + const newCursor = db._query("FOR doc IN @@coll FILTER doc.value >= @val RETURN 1", + {'@coll': cn, 'val': 100000}, {count:true}); + assertEqual(newCursor.count(), 100000); + // check for old entries via index + const oldCursor = db._query("FOR doc IN @@coll FILTER doc.value < @val RETURN 1", + {'@coll': cn, 'val': 100000}, {count:true}); + assertEqual(oldCursor.count(), 0); + + let indexes = c.getIndexes(true); + for (let i of indexes) { + switch (i.type) { + case 'primary': + break; + case 'skiplist': + break; + default: + fail(); + } + } + }, + + }; +} + +jsunity.run(backgroundIndexSuite); + +return jsunity.done();