diff --git a/CHANGELOG b/CHANGELOG index 399604220486..3b3bf56d9557 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,23 @@ devel ----- +* Added multidimensional indexes which can be used to efficiently intersect + multiple range queries. They are currently limited to IEEE-754 double values. + Given documents of the form {x: 12.9, y: -284.0, z: 0.02} one can define a + multidimensional index using the new type 'zkd' on the fields ["x", "y", "z"]. + + The AQL optimizer will then consider this index when doing queries on multiple + ranges, for example: + + FOR p IN points + FILTER x0 <= p.x && p.x <= x1 + FILTER y0 <= p.y && p.y <= y1 + FILTER z0 <= p.z && p.z <= z1 + RETURN p + + The index implements the relation <=, == and >= natively. Strict relations are + emulated using post filtering. Ranges can be unbounded on one or both sides. + * No runtime limits for shard move and server cleanout jobs, instead possibility to cancel them. diff --git a/Documentation/DocuBlocks/Rest/Indexes/post_api_index_zkd.md b/Documentation/DocuBlocks/Rest/Indexes/post_api_index_zkd.md new file mode 100644 index 000000000000..9d635822d6cc --- /dev/null +++ b/Documentation/DocuBlocks/Rest/Indexes/post_api_index_zkd.md @@ -0,0 +1,68 @@ + +@startDocuBlock post_api_index_zkd +@brief creates a multi-dimensional index + +@RESTHEADER{POST /_api/index#multi-dim, Create multi-dimensional index, createIndex#multi-dim} + +@RESTQUERYPARAMETERS + +@RESTQUERYPARAM{collection,string,required} +The collection name. + +@RESTBODYPARAM{type,string,required,string} +must be equal to *"zkd"*. + +@RESTBODYPARAM{fields,array,required,string} +an array of attribute names used for each dimension. Array expansions are not allowed. + +@RESTBODYPARAM{unique,boolean,required,} +if *true*, then create a unique index. + +@RESTBODYPARAM{fieldValueTypes,string,required,string} +must be equal to *"double"*. Currently only doubles are supported as values. + +@RESTDESCRIPTION +Creates a multi-dimensional index for the collection *collection-name*, if +it does not already exist. The call expects an object containing the index +details. + +@RESTRETURNCODES + +@RESTRETURNCODE{200} +If the index already exists, then a *HTTP 200* is +returned. + +@RESTRETURNCODE{201} +If the index does not already exist and could be created, then a *HTTP 201* +is returned. + +@RESTRETURNCODE{404} +If the *collection-name* is unknown, then a *HTTP 404* is returned. + +@RESTRETURNCODE{400} +If the index definition is invalid, then a *HTTP 400* is returned. + +@EXAMPLES + +Creating a multi-dimensional index + +@EXAMPLE_ARANGOSH_RUN{RestIndexCreateNewFulltext} +var cn = "intervals"; +db._drop(cn); +db._create(cn); + + var url = "/_api/index?collection=" + cn; + var body = { + type: "zkd", + fields: [ "from", "to" ], + fieldValueTypes: "double" + }; + + var response = logCurlRequest('POST', url, body); + + assert(response.code === 201); + + logJsonResponse(response); +~ db._drop(cn); +@END_EXAMPLE_ARANGOSH_RUN +@endDocuBlock diff --git a/arangod/CMakeLists.txt b/arangod/CMakeLists.txt index b138f98a57e0..ec98c2149ea5 100644 --- a/arangod/CMakeLists.txt +++ b/arangod/CMakeLists.txt @@ -836,6 +836,8 @@ endif() include(ClusterEngine/CMakeLists.txt) include(RocksDBEngine/CMakeLists.txt) +add_library(arango_zkd STATIC Zkd/ZkdHelper.cpp) + add_library(arango_restart_action STATIC RestServer/RestartAction.cpp ) @@ -919,6 +921,7 @@ add_library(arango_rocksdb STATIC ${ROCKSDB_SOURCES} ${ADDITIONAL_LIB_ARANGO_ROCKSDB_SOURCES} ) +target_link_libraries(arango_rocksdb arango_zkd) add_dependencies(arango_rocksdb snappy) diff --git a/arangod/ClusterEngine/ClusterIndex.cpp b/arangod/ClusterEngine/ClusterIndex.cpp index bbd75a2a6767..4aa75e255cb7 100644 --- a/arangod/ClusterEngine/ClusterIndex.cpp +++ b/arangod/ClusterEngine/ClusterIndex.cpp @@ -27,6 +27,7 @@ #include "ClusterIndex.h" #include "Indexes/SimpleAttributeEqualityMatcher.h" #include "Indexes/SortedIndexAttributeMatcher.h" +#include "RocksDBEngine/RocksDBZkdIndex.h" #include "StorageEngine/EngineSelectorFeature.h" #include "VocBase/LogicalCollection.h" #include "VocBase/ticks.h" @@ -277,6 +278,9 @@ Index::FilterCosts ClusterIndex::supportsFilterCondition( return Index::supportsFilterCondition(allIndexes, node, reference, itemsInIndex); } + case TRI_IDX_TYPE_ZKD_INDEX: + return zkd::supportsFilterCondition(this, allIndexes, node, reference, itemsInIndex); + case TRI_IDX_TYPE_UNKNOWN: break; } @@ -315,6 +319,10 @@ Index::SortCosts ClusterIndex::supportsSortCondition(arangodb::aql::SortConditio break; } + case TRI_IDX_TYPE_ZKD_INDEX: + // Sorting not supported + return Index::SortCosts{}; + case TRI_IDX_TYPE_UNKNOWN: break; } @@ -359,7 +367,10 @@ aql::AstNode* ClusterIndex::specializeCondition(aql::AstNode* node, case TRI_IDX_TYPE_PERSISTENT_INDEX: { return SortedIndexAttributeMatcher::specializeCondition(this, node, reference); } - + + case TRI_IDX_TYPE_ZKD_INDEX: + return zkd::specializeCondition(this, node, reference); + case TRI_IDX_TYPE_UNKNOWN: break; } diff --git a/arangod/ClusterEngine/ClusterIndexFactory.cpp b/arangod/ClusterEngine/ClusterIndexFactory.cpp index f8c88a636a5c..b22c91575c2f 100644 --- a/arangod/ClusterEngine/ClusterIndexFactory.cpp +++ b/arangod/ClusterEngine/ClusterIndexFactory.cpp @@ -166,6 +166,7 @@ void ClusterIndexFactory::linkIndexFactories(application_features::ApplicationSe static const PrimaryIndexFactory primaryIndexFactory(server, "primary"); static const DefaultIndexFactory skiplistIndexFactory(server, "skiplist"); static const DefaultIndexFactory ttlIndexFactory(server, "ttl"); + static const DefaultIndexFactory zkdIndexFactory(server, "zkd"); factory.emplace(edgeIndexFactory._type, edgeIndexFactory); factory.emplace(fulltextIndexFactory._type, fulltextIndexFactory); @@ -177,6 +178,7 @@ void ClusterIndexFactory::linkIndexFactories(application_features::ApplicationSe factory.emplace(primaryIndexFactory._type, primaryIndexFactory); factory.emplace(skiplistIndexFactory._type, skiplistIndexFactory); factory.emplace(ttlIndexFactory._type, ttlIndexFactory); + factory.emplace(zkdIndexFactory._type, zkdIndexFactory); } ClusterIndexFactory::ClusterIndexFactory(application_features::ApplicationServer& server) diff --git a/arangod/Indexes/Index.cpp b/arangod/Indexes/Index.cpp index f50cfdccfc42..25e5981c166c 100644 --- a/arangod/Indexes/Index.cpp +++ b/arangod/Indexes/Index.cpp @@ -332,6 +332,9 @@ Index::IndexType Index::type(char const* type, size_t len) { if (::typeMatch(type, len, "geo2")) { return TRI_IDX_TYPE_GEO2_INDEX; } + if (::typeMatch(type, len, "zkd")) { + return TRI_IDX_TYPE_ZKD_INDEX; + } std::string const& tmp = arangodb::iresearch::DATA_SOURCE_TYPE.name(); if (::typeMatch(type, len, tmp.c_str())) { return TRI_IDX_TYPE_IRESEARCH_LINK; @@ -374,6 +377,8 @@ char const* Index::oldtypeName(Index::IndexType type) { return arangodb::iresearch::DATA_SOURCE_TYPE.name().c_str(); case TRI_IDX_TYPE_NO_ACCESS_INDEX: return "noaccess"; + case TRI_IDX_TYPE_ZKD_INDEX: + return "zkd"; case TRI_IDX_TYPE_UNKNOWN: { } } diff --git a/arangod/Indexes/Index.h b/arangod/Indexes/Index.h index 88042bec9490..aa1deed07caf 100644 --- a/arangod/Indexes/Index.h +++ b/arangod/Indexes/Index.h @@ -102,7 +102,8 @@ class Index { TRI_IDX_TYPE_TTL_INDEX, TRI_IDX_TYPE_PERSISTENT_INDEX, TRI_IDX_TYPE_IRESEARCH_LINK, - TRI_IDX_TYPE_NO_ACCESS_INDEX + TRI_IDX_TYPE_NO_ACCESS_INDEX, + TRI_IDX_TYPE_ZKD_INDEX }; /// @brief: helper struct returned by index methods that determine the costs diff --git a/arangod/Indexes/IndexFactory.cpp b/arangod/Indexes/IndexFactory.cpp index 671fffcdc02b..da4366f89950 100644 --- a/arangod/Indexes/IndexFactory.cpp +++ b/arangod/Indexes/IndexFactory.cpp @@ -301,8 +301,9 @@ std::shared_ptr IndexFactory::prepareIndexFromSlice(velocypack::Slice def /// same for both storage engines std::vector IndexFactory::supportedIndexes() const { - return std::vector{"primary", "edge", "hash", "skiplist", - "ttl", "persistent", "geo", "fulltext"}; + return std::vector{"primary", "edge", "hash", + "skiplist", "ttl", "persistent", + "geo", "fulltext", "zkd"}; } std::unordered_map IndexFactory::indexAliases() const { @@ -581,4 +582,34 @@ Result IndexFactory::enhanceJsonIndexFulltext(VPackSlice definition, return res; } +/// @brief enhances the json of a zkd index +Result IndexFactory::enhanceJsonIndexZkd(VPackSlice definition, + VPackBuilder& builder, bool create) { + if (auto fieldValueTypes = definition.get("fieldValueTypes"); + !fieldValueTypes.isString() || !fieldValueTypes.isEqualString("double")) { + return Result( + TRI_ERROR_BAD_PARAMETER, + "zkd index requires `fieldValueTypes` to be set to `double` - future " + "releases might lift this requirement"); + } + + builder.add("fieldValueTypes", VPackValue("double")); + Result res = processIndexFields(definition, builder, 1, INT_MAX, create, false); + + if (res.ok()) { + if (auto isSparse = definition.get(StaticStrings::IndexSparse).isTrue(); isSparse) { + return Result(TRI_ERROR_BAD_PARAMETER, + "zkd index does not support sparse property"); + } + + processIndexUniqueFlag(definition, builder); + + bool bck = basics::VelocyPackHelper::getBooleanValue(definition, StaticStrings::IndexInBackground, + false); + builder.add(StaticStrings::IndexInBackground, VPackValue(bck)); + } + + return res; +} + } // namespace arangodb diff --git a/arangod/Indexes/IndexFactory.h b/arangod/Indexes/IndexFactory.h index b335469c84a0..166e1b04cd21 100644 --- a/arangod/Indexes/IndexFactory.h +++ b/arangod/Indexes/IndexFactory.h @@ -158,11 +158,15 @@ class IndexFactory { static Result enhanceJsonIndexGeo(velocypack::Slice definition, velocypack::Builder& builder, bool create, int minFields, int maxFields); - + /// @brief enhances the json of a fulltext index static Result enhanceJsonIndexFulltext(velocypack::Slice definition, velocypack::Builder& builder, bool create); + /// @brief enhances the json of a zkd index + static Result enhanceJsonIndexZkd(arangodb::velocypack::Slice definition, + arangodb::velocypack::Builder& builder, bool create); + protected: /// @brief clear internal factory/normalizer maps void clear(); diff --git a/arangod/RocksDBEngine/CMakeLists.txt b/arangod/RocksDBEngine/CMakeLists.txt index e78b21eac931..8d3d679bce2a 100644 --- a/arangod/RocksDBEngine/CMakeLists.txt +++ b/arangod/RocksDBEngine/CMakeLists.txt @@ -72,9 +72,9 @@ set(ROCKSDB_SOURCES RocksDBEngine/RocksDBLogValue.cpp RocksDBEngine/RocksDBMetaCollection.cpp RocksDBEngine/RocksDBMetadata.cpp - RocksDBEngine/RocksDBTransactionMethods.cpp RocksDBEngine/RocksDBOptimizerRules.cpp RocksDBEngine/RocksDBOptionFeature.cpp + RocksDBEngine/RocksDBPersistedLog.cpp RocksDBEngine/RocksDBPrimaryIndex.cpp RocksDBEngine/RocksDBRecoveryManager.cpp RocksDBEngine/RocksDBReplicationCommon.cpp @@ -90,6 +90,7 @@ set(ROCKSDB_SOURCES RocksDBEngine/RocksDBSettingsManager.cpp RocksDBEngine/RocksDBSyncThread.cpp RocksDBEngine/RocksDBTransactionCollection.cpp + RocksDBEngine/RocksDBTransactionMethods.cpp RocksDBEngine/RocksDBTransactionState.cpp RocksDBEngine/RocksDBTtlIndex.cpp RocksDBEngine/RocksDBTypes.cpp @@ -98,6 +99,6 @@ set(ROCKSDB_SOURCES RocksDBEngine/RocksDBVPackIndex.cpp RocksDBEngine/RocksDBValue.cpp RocksDBEngine/RocksDBWalAccess.cpp - RocksDBEngine/RocksDBPersistedLog.cpp + RocksDBEngine/RocksDBZkdIndex.cpp ) set(ROCKSDB_SOURCES ${ROCKSDB_SOURCES} PARENT_SCOPE) diff --git a/arangod/RocksDBEngine/RocksDBColumnFamilyManager.cpp b/arangod/RocksDBEngine/RocksDBColumnFamilyManager.cpp index f30768ce6a29..781ba5d9867b 100644 --- a/arangod/RocksDBEngine/RocksDBColumnFamilyManager.cpp +++ b/arangod/RocksDBEngine/RocksDBColumnFamilyManager.cpp @@ -30,17 +30,19 @@ namespace arangodb { -std::array - RocksDBColumnFamilyManager::_internalNames = {"default", "Documents", - "PrimaryIndex", "EdgeIndex", - "VPackIndex", "GeoIndex", - "FulltextIndex", "ReplicatedLogs"}; +std::array RocksDBColumnFamilyManager::_internalNames = + {"default", "Documents", "PrimaryIndex", + "EdgeIndex", "VPackIndex", "GeoIndex", + "FulltextIndex", "ReplicatedLogs", "ZkdIndex"}; + std::array RocksDBColumnFamilyManager::_externalNames = - {"definitions", "documents", "primary", "edge", "vpack", "geo", "fulltext", "replicated-logs"}; + {"definitions", "documents", "primary", "edge", "vpack", + "geo", "fulltext", "replicated-logs", "zkd"}; std::array - RocksDBColumnFamilyManager::_handles = {nullptr, nullptr, nullptr, nullptr, - nullptr, nullptr, nullptr, nullptr}; + RocksDBColumnFamilyManager::_handles = {nullptr, nullptr, nullptr, + nullptr, nullptr, nullptr, + nullptr, nullptr, nullptr}; rocksdb::ColumnFamilyHandle* RocksDBColumnFamilyManager::_defaultHandle = nullptr; diff --git a/arangod/RocksDBEngine/RocksDBColumnFamilyManager.h b/arangod/RocksDBEngine/RocksDBColumnFamilyManager.h index 66c703a488af..0d646474df0f 100644 --- a/arangod/RocksDBEngine/RocksDBColumnFamilyManager.h +++ b/arangod/RocksDBEngine/RocksDBColumnFamilyManager.h @@ -46,6 +46,7 @@ struct RocksDBColumnFamilyManager { GeoIndex = 5, FulltextIndex = 6, ReplicatedLogs = 7, + ZkdIndex = 8, Invalid = 1024 // special placeholder }; @@ -56,7 +57,7 @@ struct RocksDBColumnFamilyManager { }; static constexpr size_t minNumberOfColumnFamilies = 7; - static constexpr size_t numberOfColumnFamilies = 8; + static constexpr size_t numberOfColumnFamilies = 9; static void initialize(); diff --git a/arangod/RocksDBEngine/RocksDBEngine.cpp b/arangod/RocksDBEngine/RocksDBEngine.cpp index 7ec438b0d314..810943020c42 100644 --- a/arangod/RocksDBEngine/RocksDBEngine.cpp +++ b/arangod/RocksDBEngine/RocksDBEngine.cpp @@ -707,7 +707,7 @@ void RocksDBEngine::start() { addFamily(RocksDBColumnFamilyManager::Family::GeoIndex); addFamily(RocksDBColumnFamilyManager::Family::FulltextIndex); addFamily(RocksDBColumnFamilyManager::Family::ReplicatedLogs); - + addFamily(RocksDBColumnFamilyManager::Family::ZkdIndex); size_t const minNumberOfColumnFamilies = RocksDBColumnFamilyManager::minNumberOfColumnFamilies; bool dbExisted = false; @@ -746,15 +746,17 @@ void RocksDBEngine::start() { << "found existing column families: " << names; auto const replicatedLogsName = RocksDBColumnFamilyManager::name( RocksDBColumnFamilyManager::Family::ReplicatedLogs); + auto const zkdIndexName = RocksDBColumnFamilyManager::name( + RocksDBColumnFamilyManager::Family::ReplicatedLogs); for (auto const& it : cfFamilies) { auto it2 = std::find(existingColumnFamilies.begin(), existingColumnFamilies.end(), it.name); if (it2 == existingColumnFamilies.end()) { - if (it.name == replicatedLogsName) { + if (it.name == replicatedLogsName || it.name == zkdIndexName) { LOG_TOPIC("293c3", INFO, Logger::STARTUP) - << "column family " << replicatedLogsName + << "column family " << it.name << " is missing and will be created."; continue; } @@ -836,6 +838,8 @@ void RocksDBEngine::start() { cfHandles[6]); RocksDBColumnFamilyManager::set(RocksDBColumnFamilyManager::Family::ReplicatedLogs, cfHandles[7]); + RocksDBColumnFamilyManager::set(RocksDBColumnFamilyManager::Family::ZkdIndex, + cfHandles[8]); TRI_ASSERT(RocksDBColumnFamilyManager::get(RocksDBColumnFamilyManager::Family::Definitions) ->GetID() == 0); diff --git a/arangod/RocksDBEngine/RocksDBIndex.cpp b/arangod/RocksDBEngine/RocksDBIndex.cpp index 688a1059910a..6b1e2d0e0fa5 100644 --- a/arangod/RocksDBEngine/RocksDBIndex.cpp +++ b/arangod/RocksDBEngine/RocksDBIndex.cpp @@ -378,6 +378,8 @@ RocksDBKeyBounds RocksDBIndex::getBounds(Index::IndexType type, uint64_t objectI return RocksDBKeyBounds::GeoIndex(objectId); case RocksDBIndex::TRI_IDX_TYPE_IRESEARCH_LINK: return RocksDBKeyBounds::DatabaseViews(objectId); + case RocksDBIndex::TRI_IDX_TYPE_ZKD_INDEX: + return RocksDBKeyBounds::ZkdIndex(objectId); case RocksDBIndex::TRI_IDX_TYPE_UNKNOWN: default: THROW_ARANGO_EXCEPTION(TRI_ERROR_NOT_IMPLEMENTED); diff --git a/arangod/RocksDBEngine/RocksDBIndexFactory.cpp b/arangod/RocksDBEngine/RocksDBIndexFactory.cpp index 50b2fa8c79dd..f8867666b3d2 100644 --- a/arangod/RocksDBEngine/RocksDBIndexFactory.cpp +++ b/arangod/RocksDBEngine/RocksDBIndexFactory.cpp @@ -35,6 +35,7 @@ #include "RocksDBEngine/RocksDBPrimaryIndex.h" #include "RocksDBEngine/RocksDBSkiplistIndex.h" #include "RocksDBEngine/RocksDBTtlIndex.h" +#include "RocksDBEngine/RocksDBZkdIndex.h" #include "RocksDBIndexFactory.h" #include "VocBase/LogicalCollection.h" #include "VocBase/ticks.h" @@ -255,6 +256,42 @@ struct SecondaryIndexFactory : public DefaultIndexFactory { } }; +struct ZkdIndexFactory : public DefaultIndexFactory { + ZkdIndexFactory(arangodb::application_features::ApplicationServer& server) + : DefaultIndexFactory(server, Index::TRI_IDX_TYPE_ZKD_INDEX) {} + + std::shared_ptr instantiate(arangodb::LogicalCollection& collection, + arangodb::velocypack::Slice definition, + IndexId id, + bool isClusterConstructor) const override { + if (auto isUnique = definition.get(StaticStrings::IndexUnique).isTrue(); isUnique) { + return std::make_shared(id, collection, definition); + } + + return std::make_shared(id, collection, definition); + } + + virtual arangodb::Result normalize( // normalize definition + arangodb::velocypack::Builder& normalized, // normalized definition (out-param) + arangodb::velocypack::Slice definition, // source definition + bool isCreation, // definition for index creation + TRI_vocbase_t const& vocbase // index vocbase + ) const override { + TRI_ASSERT(normalized.isOpenObject()); + normalized.add(arangodb::StaticStrings::IndexType, + arangodb::velocypack::Value( + arangodb::Index::oldtypeName(Index::TRI_IDX_TYPE_ZKD_INDEX))); + + if (isCreation && !ServerState::instance()->isCoordinator() && + !definition.hasKey("objectId")) { + normalized.add("objectId", + arangodb::velocypack::Value(std::to_string(TRI_NewTickServer()))); + } + + return IndexFactory::enhanceJsonIndexZkd(definition, normalized, isCreation); + } +}; + struct TtlIndexFactory : public DefaultIndexFactory { explicit TtlIndexFactory(application_features::ApplicationServer& server, Index::IndexType type) @@ -339,6 +376,7 @@ RocksDBIndexFactory::RocksDBIndexFactory(application_features::ApplicationServer server); static const TtlIndexFactory ttlIndexFactory(server, Index::TRI_IDX_TYPE_TTL_INDEX); static const PrimaryIndexFactory primaryIndexFactory(server); + static const ZkdIndexFactory zkdIndexFactory(server); emplace("edge", edgeIndexFactory); emplace("fulltext", fulltextIndexFactory); @@ -351,6 +389,7 @@ RocksDBIndexFactory::RocksDBIndexFactory(application_features::ApplicationServer emplace("rocksdb", persistentIndexFactory); emplace("skiplist", skiplistIndexFactory); emplace("ttl", ttlIndexFactory); + emplace("zkd", zkdIndexFactory); } /// @brief index name aliases (e.g. "persistent" => "hash", "skiplist" => diff --git a/arangod/RocksDBEngine/RocksDBKey.cpp b/arangod/RocksDBEngine/RocksDBKey.cpp index f0288fff84c5..10bb863437d8 100644 --- a/arangod/RocksDBEngine/RocksDBKey.cpp +++ b/arangod/RocksDBEngine/RocksDBKey.cpp @@ -76,6 +76,29 @@ bool RocksDBKey::containsLocalDocumentId(LocalDocumentId const& documentId) cons return false; } +void RocksDBKey::constructZkdIndexValue(uint64_t indexId, const zkd::byte_string& value) { + _type = RocksDBEntryType::ZkdIndexValue; + size_t keyLength = sizeof(uint64_t) + value.size(); + _buffer->clear(); + _buffer->reserve(keyLength); + uint64ToPersistent(*_buffer, indexId); + auto sv = std::string_view{reinterpret_cast(value.data()), value.size()}; + _buffer->append(sv.data(), sv.size()); + TRI_ASSERT(_buffer->size() == keyLength); +} + +void RocksDBKey::constructZkdIndexValue(uint64_t indexId, zkd::byte_string const& value, LocalDocumentId documentId) { + _type = RocksDBEntryType::ZkdIndexValue; + size_t keyLength = sizeof(uint64_t) + value.size() + sizeof(uint64_t); + _buffer->clear(); + _buffer->reserve(keyLength); + uint64ToPersistent(*_buffer, indexId); + auto sv = std::string_view{reinterpret_cast(value.data()), value.size()}; + _buffer->append(sv.data(), sv.size()); + uint64ToPersistent(*_buffer, documentId.id()); + TRI_ASSERT(_buffer->size() == keyLength); +} + void RocksDBKey::constructDatabase(TRI_voc_tick_t databaseId) { TRI_ASSERT(databaseId != 0); _type = RocksDBEntryType::Database; @@ -486,6 +509,17 @@ VPackSlice RocksDBKey::indexedVPack(char const* data, size_t size) { return VPackSlice(reinterpret_cast(data) + sizeof(uint64_t)); } +zkd::byte_string_view RocksDBKey::zkdIndexValue(char const* data, size_t size) { + TRI_ASSERT(data != nullptr); + TRI_ASSERT(size > 2 * sizeof(uint64_t)); + return zkd::byte_string_view(reinterpret_cast(data) + sizeof(uint64_t), + size - 2 * sizeof(uint64_t)); +} + +zkd::byte_string_view RocksDBKey::zkdIndexValue(const rocksdb::Slice& slice) { + return zkdIndexValue(slice.data(), slice.size()); +} + namespace arangodb { std::ostream& operator<<(std::ostream& stream, RocksDBKey const& key) { diff --git a/arangod/RocksDBEngine/RocksDBKey.h b/arangod/RocksDBEngine/RocksDBKey.h index c6e3872cac1d..ec8672962289 100644 --- a/arangod/RocksDBEngine/RocksDBKey.h +++ b/arangod/RocksDBEngine/RocksDBKey.h @@ -29,6 +29,7 @@ #include "VocBase/Identifiers/DataSourceId.h" #include "VocBase/Identifiers/LocalDocumentId.h" #include "VocBase/voc-types.h" +#include "Zkd/ZkdHelper.h" #include @@ -171,11 +172,18 @@ class RocksDBKey { ////////////////////////////////////////////////////////////////////////////// void constructRevisionTreeValue(uint64_t objectId); + ////////////////////////////////////////////////////////////////////////////// + /// @brief Create a fully-specified key for zkd index + ////////////////////////////////////////////////////////////////////////////// + void constructZkdIndexValue(uint64_t objectId, const zkd::byte_string& value); + void constructZkdIndexValue(uint64_t objectId, const zkd::byte_string& value, LocalDocumentId documentId); + ////////////////////////////////////////////////////////////////////////////// /// @brief Create a fully-specified key for revision tree for a collection ////////////////////////////////////////////////////////////////////////////// void constructLogEntry(uint64_t objectId, replication2::LogIndex idx); + public: ////////////////////////////////////////////////////////////////////////////// /// @brief Extracts the type from a key @@ -279,6 +287,12 @@ class RocksDBKey { ////////////////////////////////////////////////////////////////////////////// static uint64_t geoValue(rocksdb::Slice const& slice); + ////////////////////////////////////////////////////////////////////////////// + /// @brief Extracts the zkd index value + /// + /// May be called only on zkd index values + ////////////////////////////////////////////////////////////////////////////// + static zkd::byte_string_view zkdIndexValue(rocksdb::Slice const& slice); ////////////////////////////////////////////////////////////////////////////// /// @brief Extracts log index from key @@ -346,6 +360,7 @@ class RocksDBKey { static arangodb::velocypack::StringRef primaryKey(char const* data, size_t size); static arangodb::velocypack::StringRef vertexId(char const* data, size_t size); static VPackSlice indexedVPack(char const* data, size_t size); + static zkd::byte_string_view zkdIndexValue(char const* data, size_t size); private: static const char _stringSeparator; diff --git a/arangod/RocksDBEngine/RocksDBKeyBounds.cpp b/arangod/RocksDBEngine/RocksDBKeyBounds.cpp index 080687ba8d6d..b61516e3e644 100644 --- a/arangod/RocksDBEngine/RocksDBKeyBounds.cpp +++ b/arangod/RocksDBEngine/RocksDBKeyBounds.cpp @@ -101,6 +101,10 @@ RocksDBKeyBounds RocksDBKeyBounds::VPackIndex(uint64_t indexId, VPackSlice const return RocksDBKeyBounds(RocksDBEntryType::VPackIndexValue, indexId, left, right); } +RocksDBKeyBounds RocksDBKeyBounds::ZkdIndex(uint64_t indexId) { + return RocksDBKeyBounds(RocksDBEntryType::ZkdIndexValue, indexId, false); +} + /// used for seeking lookups RocksDBKeyBounds RocksDBKeyBounds::UniqueVPackIndex(uint64_t indexId, VPackSlice const& left, VPackSlice const& right) { @@ -228,7 +232,10 @@ rocksdb::ColumnFamilyHandle* RocksDBKeyBounds::columnFamily() const { return RocksDBColumnFamilyManager::get(RocksDBColumnFamilyManager::Family::FulltextIndex); case RocksDBEntryType::LegacyGeoIndexValue: case RocksDBEntryType::GeoIndexValue: + case RocksDBEntryType::UniqueZkdIndexValue: return RocksDBColumnFamilyManager::get(RocksDBColumnFamilyManager::Family::GeoIndex); + case RocksDBEntryType::ZkdIndexValue: + return RocksDBColumnFamilyManager::get(RocksDBColumnFamilyManager::Family::ZkdIndex); case RocksDBEntryType::LogEntry: return RocksDBColumnFamilyManager::get(RocksDBColumnFamilyManager::Family::ReplicatedLogs); case RocksDBEntryType::Database: @@ -385,6 +392,7 @@ RocksDBKeyBounds::RocksDBKeyBounds(RocksDBEntryType type, uint64_t first) RocksDBKeyBounds::RocksDBKeyBounds(RocksDBEntryType type, uint64_t first, bool second) : _type(type) { switch (_type) { + case RocksDBEntryType::ZkdIndexValue: case RocksDBEntryType::VPackIndexValue: case RocksDBEntryType::UniqueVPackIndexValue: { uint8_t const maxSlice[] = {0x02, 0x03, 0x1f}; diff --git a/arangod/RocksDBEngine/RocksDBKeyBounds.h b/arangod/RocksDBEngine/RocksDBKeyBounds.h index 154aa2c92321..69dacdcc84f6 100644 --- a/arangod/RocksDBEngine/RocksDBKeyBounds.h +++ b/arangod/RocksDBEngine/RocksDBKeyBounds.h @@ -177,6 +177,12 @@ class RocksDBKeyBounds { ////////////////////////////////////////////////////////////////////////////// static RocksDBKeyBounds FulltextIndexComplete(uint64_t, arangodb::velocypack::StringRef const&); + ////////////////////////////////////////////////////////////////////////////// + /// @brief Bounds for all index-entries belonging to a specified non-unique + /// index (hash, skiplist and permanent) + ////////////////////////////////////////////////////////////////////////////// + static RocksDBKeyBounds ZkdIndex(uint64_t indexId); + public: RocksDBKeyBounds(RocksDBKeyBounds const& other); RocksDBKeyBounds(RocksDBKeyBounds&& other) noexcept; diff --git a/arangod/RocksDBEngine/RocksDBOptionFeature.cpp b/arangod/RocksDBEngine/RocksDBOptionFeature.cpp index 7cb0a71a9172..4b16a830618a 100644 --- a/arangod/RocksDBEngine/RocksDBOptionFeature.cpp +++ b/arangod/RocksDBEngine/RocksDBOptionFeature.cpp @@ -118,7 +118,7 @@ RocksDBOptionFeature::RocksDBOptionFeature(application_features::ApplicationServ _transactionLockTimeout(rocksDBTrxDefaults.transaction_lock_timeout), _totalWriteBufferSize(rocksDBDefaults.db_write_buffer_size), _writeBufferSize(rocksDBDefaults.write_buffer_size), - _maxWriteBufferNumber(7 + 2), // number of column families plus 2 + _maxWriteBufferNumber(8 + 2), // number of column families plus 2 _maxWriteBufferSizeToMaintain(0), _maxTotalWalSize(80 << 20), _delayedWriteRate(rocksDBDefaults.delayed_write_rate), @@ -636,6 +636,7 @@ rocksdb::ColumnFamilyOptions RocksDBOptionFeature::columnFamilyOptions( case RocksDBColumnFamilyManager::Family::PrimaryIndex: case RocksDBColumnFamilyManager::Family::GeoIndex: case RocksDBColumnFamilyManager::Family::FulltextIndex: + case RocksDBColumnFamilyManager::Family::ZkdIndex: case RocksDBColumnFamilyManager::Family::ReplicatedLogs: { // fixed 8 byte object id prefix options.prefix_extractor = std::shared_ptr( diff --git a/arangod/RocksDBEngine/RocksDBTypes.cpp b/arangod/RocksDBEngine/RocksDBTypes.cpp index bcf9417e03a5..aaff39f7b904 100644 --- a/arangod/RocksDBEngine/RocksDBTypes.cpp +++ b/arangod/RocksDBEngine/RocksDBTypes.cpp @@ -109,6 +109,14 @@ static rocksdb::Slice ReplicatedLog( static RocksDBEntryType revisionTreeValue = RocksDBEntryType::RevisionTreeValue; static rocksdb::Slice RevisionTreeValue( reinterpret_cast::type*>(&revisionTreeValue), 1); + +static RocksDBEntryType zkdIndexValue = RocksDBEntryType::ZkdIndexValue; +static rocksdb::Slice ZdkIndexValue( + reinterpret_cast::type*>(&zkdIndexValue), 1); + +static RocksDBEntryType uniqueZkdIndexValue = RocksDBEntryType::UniqueZkdIndexValue; +static rocksdb::Slice UniqueZdkIndexValue( + reinterpret_cast::type*>(&uniqueZkdIndexValue), 1); } // namespace char const* arangodb::rocksDBEntryTypeName(arangodb::RocksDBEntryType type) { @@ -149,6 +157,10 @@ char const* arangodb::rocksDBEntryTypeName(arangodb::RocksDBEntryType type) { return "KeyGeneratorValue"; case arangodb::RocksDBEntryType::RevisionTreeValue: return "RevisionTreeValue"; + case arangodb::RocksDBEntryType::ZkdIndexValue: + return "ZkdIndexValue"; + case arangodb::RocksDBEntryType::UniqueZkdIndexValue: + return "UniqueZkdIndexValue"; case RocksDBEntryType::LogEntry: return "ReplicatedLogEntry"; case RocksDBEntryType::ReplicatedLog: @@ -251,6 +263,10 @@ rocksdb::Slice const& arangodb::rocksDBSlice(RocksDBEntryType const& type) { return KeyGeneratorValue; case RocksDBEntryType::RevisionTreeValue: return RevisionTreeValue; + case RocksDBEntryType::ZkdIndexValue: + return ZdkIndexValue; + case RocksDBEntryType::UniqueZkdIndexValue: + return UniqueZdkIndexValue; case RocksDBEntryType::LogEntry: return LogEntry; case RocksDBEntryType::ReplicatedLog: diff --git a/arangod/RocksDBEngine/RocksDBTypes.h b/arangod/RocksDBEngine/RocksDBTypes.h index a6cd73afc0e9..f587376b25cc 100644 --- a/arangod/RocksDBEngine/RocksDBTypes.h +++ b/arangod/RocksDBEngine/RocksDBTypes.h @@ -56,7 +56,9 @@ enum class RocksDBEntryType : char { // RevisionTreeValue = '@', // pre-3.8 GA revision trees. do not use or reuse! // RevisionTreeValue = '/', // pre-3.8 GA revision trees. do not use or reuse! RevisionTreeValue = '*', - ReplicatedLog = 'l' + ReplicatedLog = 'l', + ZkdIndexValue = 'z', + UniqueZkdIndexValue = 'Z', }; char const* rocksDBEntryTypeName(RocksDBEntryType); diff --git a/arangod/RocksDBEngine/RocksDBValue.cpp b/arangod/RocksDBEngine/RocksDBValue.cpp index 88f38213826d..3cdc69227c04 100644 --- a/arangod/RocksDBEngine/RocksDBValue.cpp +++ b/arangod/RocksDBEngine/RocksDBValue.cpp @@ -59,6 +59,15 @@ RocksDBValue RocksDBValue::VPackIndexValue() { return RocksDBValue(RocksDBEntryType::VPackIndexValue); } + +RocksDBValue RocksDBValue::ZkdIndexValue() { + return RocksDBValue(RocksDBEntryType::ZkdIndexValue); +} + +RocksDBValue RocksDBValue::UniqueZkdIndexValue(LocalDocumentId const& docId) { + return RocksDBValue(RocksDBEntryType::UniqueZkdIndexValue, docId, RevisionId::none()); +} + RocksDBValue RocksDBValue::UniqueVPackIndexValue(LocalDocumentId const& docId) { return RocksDBValue(RocksDBEntryType::UniqueVPackIndexValue, docId, RevisionId::none()); } @@ -170,6 +179,7 @@ RocksDBValue::RocksDBValue(RocksDBEntryType type, LocalDocumentId const& docId, : _type(type), _buffer() { switch (_type) { case RocksDBEntryType::UniqueVPackIndexValue: + case RocksDBEntryType::UniqueZkdIndexValue: case RocksDBEntryType::PrimaryIndexValue: { if (!revision) { _buffer.reserve(sizeof(uint64_t)); diff --git a/arangod/RocksDBEngine/RocksDBValue.h b/arangod/RocksDBEngine/RocksDBValue.h index 0997616e8cc2..e1c61ebcdf1a 100644 --- a/arangod/RocksDBEngine/RocksDBValue.h +++ b/arangod/RocksDBEngine/RocksDBValue.h @@ -62,6 +62,8 @@ class RocksDBValue { static RocksDBValue PrimaryIndexValue(LocalDocumentId const& docId, RevisionId revision); static RocksDBValue EdgeIndexValue(arangodb::velocypack::StringRef const& vertexId); static RocksDBValue VPackIndexValue(); + static RocksDBValue ZkdIndexValue(); + static RocksDBValue UniqueZkdIndexValue(LocalDocumentId const& docId); static RocksDBValue UniqueVPackIndexValue(LocalDocumentId const& docId); static RocksDBValue View(VPackSlice const& data); static RocksDBValue ReplicationApplierConfig(VPackSlice const& data); diff --git a/arangod/RocksDBEngine/RocksDBZkdIndex.cpp b/arangod/RocksDBEngine/RocksDBZkdIndex.cpp new file mode 100644 index 000000000000..27cc70f1f9fe --- /dev/null +++ b/arangod/RocksDBEngine/RocksDBZkdIndex.cpp @@ -0,0 +1,569 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2021 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Tobias Gödderz +/// @author Lars Maier +//////////////////////////////////////////////////////////////////////////////// + +#include "Zkd/ZkdHelper.h" + +#include +#include +#include +#include "RocksDBColumnFamilyManager.h" +#include "RocksDBMethods.h" +#include "RocksDBTransactionMethods.h" +#include "RocksDBZkdIndex.h" +#include "Transaction/Methods.h" + +using namespace arangodb; + +/*namespace { + +auto coordsToVector(zkd::byte_string_view bs, size_t dim) -> std::vector { + auto vs = zkd::transpose(bs, dim); + + std::vector res; + res.reserve(dim); + + for (auto&& v : vs) { + zkd::BitReader r(v); + std::ignore = r.next(); + res.push_back(zkd::from_bit_reader_fixed_length(r)); + } + return res; +} + +}*/ + +namespace arangodb { + +template +class RocksDBZkdIndexIterator final : public IndexIterator { + public: + RocksDBZkdIndexIterator(LogicalCollection* collection, RocksDBZkdIndexBase* index, + transaction::Methods* trx, zkd::byte_string min, + zkd::byte_string max, std::size_t dim, ReadOwnWrites readOwnWrites) + : IndexIterator(collection, trx, readOwnWrites), + _bound(RocksDBKeyBounds::ZkdIndex(index->objectId())), + _min(std::move(min)), + _max(std::move(max)), + _dim(dim), + _index(index) { + _cur = _min; + _upperBound = _bound.end(); + + RocksDBTransactionMethods* mthds = RocksDBTransactionState::toMethods(trx); + _iter = mthds->NewIterator(index->columnFamily(), [&](auto& opts) { + TRI_ASSERT(opts.prefix_same_as_start); + opts.iterate_upper_bound = &_upperBound; + }); + TRI_ASSERT(_iter != nullptr); + _iter->SeekToFirst(); + _compareResult.resize(_dim); + } + + char const* typeName() const override { return "rocksdb-zkd-index-iterator"; } + + protected: + bool nextImpl(LocalDocumentIdCallback const& callback, size_t limit) override { + for (auto i = size_t{0}; i < limit; ) { + switch (_iterState) { + case IterState::SEEK_ITER_TO_CUR: { + RocksDBKey rocks_key; + rocks_key.constructZkdIndexValue(_index->objectId(), _cur); + _iter->Seek(rocks_key.string()); + if (!_iter->Valid()) { + arangodb::rocksutils::checkIteratorStatus(_iter.get()); + _iterState = IterState::DONE; + } else { + TRI_ASSERT(_index->objectId() == RocksDBKey::objectId(_iter->key())); + _iterState = IterState::CHECK_CURRENT_ITER; + } + } break; + case IterState::CHECK_CURRENT_ITER: { + auto const rocksKey = _iter->key(); + auto const byteStringKey = RocksDBKey::zkdIndexValue(rocksKey); + if (!zkd::testInBox(byteStringKey, _min, _max, _dim)) { + _cur = byteStringKey; + + zkd::compareWithBoxInto(_cur, _min, _max, _dim, _compareResult); + auto const next = zkd::getNextZValue(_cur, _min, _max, _compareResult); + if (!next) { + _iterState = IterState::DONE; + } else { + _cur = next.value(); + _iterState = IterState::SEEK_ITER_TO_CUR; + } + } else { + auto const documentId = std::invoke([&]{ + if constexpr(isUnique) { + return RocksDBValue::documentId(_iter->value()); + } else { + return RocksDBKey::indexDocumentId(rocksKey); + } + }); + std::ignore = callback(documentId); + ++i; + _iter->Next(); + if (!_iter->Valid()) { + arangodb::rocksutils::checkIteratorStatus(_iter.get()); + _iterState = IterState::DONE; + } else { + // stay in ::CHECK_CURRENT_ITER + } + } + } break; + case IterState::DONE: + return false; + default: + TRI_ASSERT(false); + } + } + + return true; + } + private: + RocksDBKeyBounds _bound; + rocksdb::Slice _upperBound; + zkd::byte_string _cur; + const zkd::byte_string _min; + const zkd::byte_string _max; + const std::size_t _dim; + + enum class IterState { + SEEK_ITER_TO_CUR = 0, + CHECK_CURRENT_ITER, + DONE, + }; + IterState _iterState = IterState::SEEK_ITER_TO_CUR; + + std::unique_ptr _iter; + RocksDBZkdIndexBase* _index = nullptr; + + std::vector _compareResult; +}; + +} // namespace arangodb + +namespace { + +auto convertDouble(double x) -> zkd::byte_string { + zkd::BitWriter bw; + bw.append(zkd::Bit::ZERO); // add zero bit for `not infinity` + zkd::into_bit_writer_fixed_length(bw, x); + return std::move(bw).str(); +} + +auto nodeExtractDouble(aql::AstNode const* node) -> std::optional { + if (node != nullptr) { + return convertDouble(node->getDoubleValue()); + } + return std::nullopt; +} + +auto accessDocumentPath(VPackSlice doc, std::vector const& path) -> VPackSlice { + for (auto&& attrib : path) { + TRI_ASSERT(attrib.shouldExpand == false); + if (!doc.isObject()) { + return VPackSlice::noneSlice(); + } + + doc = doc.get(attrib.name); + } + + return doc; +} + +auto readDocumentKey(VPackSlice doc, + std::vector> const& fields) + -> zkd::byte_string { + std::vector v; + v.reserve(fields.size()); + + for (auto const& path : fields) { + VPackSlice value = accessDocumentPath(doc, path); + if (!value.isNumber()) { + THROW_ARANGO_EXCEPTION(TRI_ERROR_QUERY_INVALID_ARITHMETIC_VALUE); + } + auto dv = value.getNumericValue(); + if (std::isnan(dv)) { + THROW_ARANGO_EXCEPTION_MESSAGE(TRI_ERROR_QUERY_INVALID_ARITHMETIC_VALUE, "NaN is not allowed"); + } + v.emplace_back(convertDouble(dv)); + } + + return zkd::interleave(v); +} + +auto boundsForIterator(arangodb::Index const* index, const arangodb::aql::AstNode* node, + const arangodb::aql::Variable* reference, + const arangodb::IndexIteratorOptions& opts) + -> std::pair { + TRI_ASSERT(node->type == arangodb::aql::NODE_TYPE_OPERATOR_NARY_AND); + + std::unordered_map extractedBounds; + std::unordered_set unusedExpressions; + extractBoundsFromCondition(index, node, reference, extractedBounds, unusedExpressions); + + TRI_ASSERT(unusedExpressions.empty()); + + const size_t dim = index->fields().size(); + std::vector min; + min.resize(dim); + std::vector max; + max.resize(dim); + + static const auto ByteStringPosInfinity = zkd::byte_string{std::byte{0x80}}; + static const auto ByteStringNegInfinity = zkd::byte_string{std::byte{0}}; + + for (auto&& [idx, field] : enumerate(index->fields())) { + if (auto it = extractedBounds.find(idx); it != extractedBounds.end()) { + auto const& bounds = it->second; + min[idx] = nodeExtractDouble(bounds.lower.bound_value).value_or(ByteStringNegInfinity); + max[idx] = nodeExtractDouble(bounds.upper.bound_value).value_or(ByteStringPosInfinity); + } else { + min[idx] = ByteStringNegInfinity; + max[idx] = ByteStringPosInfinity; + } + } + + TRI_ASSERT(min.size() == dim); + TRI_ASSERT(max.size() == dim); + + return std::make_pair(zkd::interleave(min), zkd::interleave(max)); +} +} // namespace + + +void zkd::extractBoundsFromCondition( + arangodb::Index const* index, + const arangodb::aql::AstNode* condition, const arangodb::aql::Variable* reference, + std::unordered_map& extractedBounds, + std::unordered_set& unusedExpressions) { + TRI_ASSERT(condition->type == arangodb::aql::NODE_TYPE_OPERATOR_NARY_AND); + + auto const ensureBounds = [&](size_t idx) -> ExpressionBounds& { + if (auto it = extractedBounds.find(idx); it != std::end(extractedBounds)) { + return it->second; + } + return extractedBounds[idx]; + }; + + auto const useAsBound = [&](size_t idx, aql::AstNode* op_node, aql::AstNode* bounded_expr, + aql::AstNode* bound_value, bool asLower, bool isStrict) { + auto& bounds = ensureBounds(idx); + if (asLower) { + bounds.lower.op_node = op_node; + bounds.lower.bound_value = bound_value; + bounds.lower.bounded_expr = bounded_expr; + bounds.lower.isStrict = isStrict; + } else { + bounds.upper.op_node = op_node; + bounds.upper.bound_value = bound_value; + bounds.upper.bounded_expr = bounded_expr; + bounds.upper.isStrict = isStrict; + } + }; + + auto const checkIsBoundForAttribute = [&](aql::AstNode* op, aql::AstNode* access, + aql::AstNode* other, bool reverse) -> bool { + + std::unordered_set nonNullAttributes; // TODO only used in sparse case + if (!index->canUseConditionPart(access, other, op, reference, nonNullAttributes, false)) { + return false; + } + + std::pair> attributeData; + if (!access->isAttributeAccessForVariable(attributeData) || + attributeData.first != reference) { + // this access is not referencing this collection + return false; + } + + for (auto&& [idx, field] : enumerate(index->fields())) { + if (attributeData.second != field) { + continue; + } + + switch (op->type) { + case arangodb::aql::NODE_TYPE_OPERATOR_BINARY_EQ: + useAsBound(idx, op, access, other, true, false); + useAsBound(idx, op, access, other, false, false); + return true; + case arangodb::aql::NODE_TYPE_OPERATOR_BINARY_LE: + useAsBound(idx, op, access, other, reverse, false); + return true; + case arangodb::aql::NODE_TYPE_OPERATOR_BINARY_GE: + useAsBound(idx, op, access, other, !reverse, false); + return true; + case arangodb::aql::NODE_TYPE_OPERATOR_BINARY_LT: + useAsBound(idx, op, access, other, reverse, true); + return true; + case arangodb::aql::NODE_TYPE_OPERATOR_BINARY_GT: + useAsBound(idx, op, access, other, !reverse, true); + return true; + default: + break; + } + } + + return false; + }; + + + for (size_t i = 0; i < condition->numMembers(); ++i) { + bool ok = false; + auto op = condition->getMemberUnchecked(i); + switch (op->type) { + case arangodb::aql::NODE_TYPE_OPERATOR_BINARY_EQ: + case arangodb::aql::NODE_TYPE_OPERATOR_BINARY_LE: + case arangodb::aql::NODE_TYPE_OPERATOR_BINARY_GE: + case arangodb::aql::NODE_TYPE_OPERATOR_BINARY_LT: + case arangodb::aql::NODE_TYPE_OPERATOR_BINARY_GT: + ok |= checkIsBoundForAttribute(op, op->getMember(0), op->getMember(1), false); + ok |= checkIsBoundForAttribute(op, op->getMember(1), op->getMember(0), true); + break; + default: + break; + } + if (!ok) { + unusedExpressions.emplace(op); + } + } +} + +auto zkd::supportsFilterCondition( + arangodb::Index const* index,const std::vector>& allIndexes, + const arangodb::aql::AstNode* node, + const arangodb::aql::Variable* reference, + size_t itemsInIndex) -> Index::FilterCosts { + + TRI_ASSERT(node->type == arangodb::aql::NODE_TYPE_OPERATOR_NARY_AND); + + std::unordered_map extractedBounds; + std::unordered_set unusedExpressions; + extractBoundsFromCondition(index, node, reference, extractedBounds, unusedExpressions); + + if (extractedBounds.empty()) { + return {}; + } + + // TODO -- actually return costs + auto costs = Index::FilterCosts::defaultCosts(itemsInIndex / extractedBounds.size()); + costs.coveredAttributes = extractedBounds.size(); + costs.supportsCondition = true; + return costs; +} + +auto zkd::specializeCondition(arangodb::Index const* index, arangodb::aql::AstNode* condition, + const arangodb::aql::Variable* reference) -> aql::AstNode* { + std::unordered_map extractedBounds; + std::unordered_set unusedExpressions; + extractBoundsFromCondition(index, condition, reference, extractedBounds, unusedExpressions); + + std::vector children; + + for (size_t i = 0; i < condition->numMembers(); ++i) { + auto op = condition->getMemberUnchecked(i); + + if (unusedExpressions.find(op) == unusedExpressions.end()) { + switch (op->type) { + case arangodb::aql::NODE_TYPE_OPERATOR_BINARY_EQ: + case arangodb::aql::NODE_TYPE_OPERATOR_BINARY_LE: + case arangodb::aql::NODE_TYPE_OPERATOR_BINARY_GE: + children.emplace_back(op); + break; + case arangodb::aql::NODE_TYPE_OPERATOR_BINARY_LT: + op->type = aql::NODE_TYPE_OPERATOR_BINARY_LE; + children.emplace_back(op); + break; + case arangodb::aql::NODE_TYPE_OPERATOR_BINARY_GT: + op->type = aql::NODE_TYPE_OPERATOR_BINARY_GE; + children.emplace_back(op); + break; + default: + break; + } + } + } + + // must edit in place, no access to AST; TODO change so we can replace with + // copy + TEMPORARILY_UNLOCK_NODE(condition); + condition->clearMembers(); + + for (auto& it : children) { + TRI_ASSERT(it->type != arangodb::aql::NODE_TYPE_OPERATOR_BINARY_NE); + condition->addMember(it); + } + + + return condition; + +} + +arangodb::Result arangodb::RocksDBZkdIndexBase::insert( + arangodb::transaction::Methods& trx, arangodb::RocksDBMethods* methods, + const arangodb::LocalDocumentId& documentId, + arangodb::velocypack::Slice doc, const arangodb::OperationOptions& options, + bool performChecks) { + TRI_ASSERT(_unique == false); + TRI_ASSERT(_sparse == false); + + // TODO what about performChecks? + + auto key_value = readDocumentKey(doc, _fields); + + RocksDBKey rocks_key; + rocks_key.constructZkdIndexValue(objectId(), key_value, documentId); + + auto value = RocksDBValue::ZkdIndexValue(); + auto s = methods->PutUntracked(_cf, rocks_key, value.string()); + if (!s.ok()) { + return rocksutils::convertStatus(s); + } + + return {}; +} + +arangodb::Result arangodb::RocksDBZkdIndexBase::remove(arangodb::transaction::Methods& trx, + arangodb::RocksDBMethods* methods, + const arangodb::LocalDocumentId& documentId, + arangodb::velocypack::Slice doc) { + TRI_ASSERT(_unique == false); + TRI_ASSERT(_sparse == false); + + auto key_value = readDocumentKey(doc, _fields); + + RocksDBKey rocks_key; + rocks_key.constructZkdIndexValue(objectId(), key_value, documentId); + + auto s = methods->SingleDelete(_cf, rocks_key); + if (!s.ok()) { + return rocksutils::convertStatus(s); + } + + return {}; +} + +arangodb::RocksDBZkdIndexBase::RocksDBZkdIndexBase(arangodb::IndexId iid, + arangodb::LogicalCollection& coll, + arangodb::velocypack::Slice const& info) + : RocksDBIndex(iid, coll, info, + RocksDBColumnFamilyManager::get(RocksDBColumnFamilyManager::Family::ZkdIndex), + false) {} + +void arangodb::RocksDBZkdIndexBase::toVelocyPack( + arangodb::velocypack::Builder& builder, + std::underlying_type::type type) const { + VPackObjectBuilder ob(&builder); + RocksDBIndex::toVelocyPack(builder, type); +} + +arangodb::Index::FilterCosts arangodb::RocksDBZkdIndexBase::supportsFilterCondition( + const std::vector>& allIndexes, + const arangodb::aql::AstNode* node, + const arangodb::aql::Variable* reference, size_t itemsInIndex) const { + + return zkd::supportsFilterCondition(this, allIndexes, node, reference, itemsInIndex); +} + +arangodb::aql::AstNode* arangodb::RocksDBZkdIndexBase::specializeCondition( + arangodb::aql::AstNode* condition, const arangodb::aql::Variable* reference) const { + return zkd::specializeCondition(this, condition, reference); +} + +std::unique_ptr arangodb::RocksDBZkdIndexBase::iteratorForCondition( + arangodb::transaction::Methods* trx, const arangodb::aql::AstNode* node, + const arangodb::aql::Variable* reference, const arangodb::IndexIteratorOptions& opts, + ReadOwnWrites readOwnWrites) { + + auto&& [min, max] = boundsForIterator(this, node, reference, opts); + + return std::make_unique>(&_collection, this, trx, + std::move(min), std::move(max), + fields().size(), readOwnWrites); +} + + +std::unique_ptr arangodb::RocksDBUniqueZkdIndex::iteratorForCondition( + arangodb::transaction::Methods* trx, const arangodb::aql::AstNode* node, + const arangodb::aql::Variable* reference, const arangodb::IndexIteratorOptions& opts, + ReadOwnWrites readOwnWrites) { + + auto&& [min, max] = boundsForIterator(this, node, reference, opts); + + return std::make_unique>(&_collection, this, trx, + std::move(min), std::move(max), + fields().size(), readOwnWrites); +} + +arangodb::Result arangodb::RocksDBUniqueZkdIndex::insert( + arangodb::transaction::Methods& trx, arangodb::RocksDBMethods* methods, + const arangodb::LocalDocumentId& documentId, arangodb::velocypack::Slice doc, + const arangodb::OperationOptions& options, bool performChecks) { + TRI_ASSERT(_unique == true); + TRI_ASSERT(_sparse == false); + + // TODO what about performChecks + auto key_value = readDocumentKey(doc, _fields); + + RocksDBKey rocks_key; + rocks_key.constructZkdIndexValue(objectId(), key_value); + + if (!options.checkUniqueConstraintsInPreflight) { + transaction::StringLeaser leased(&trx); + rocksdb::PinnableSlice existing(leased.get()); + if (auto s = methods->GetForUpdate(_cf, rocks_key.string(), &existing); s.ok()) { // detected conflicting index entry + return {TRI_ERROR_ARANGO_UNIQUE_CONSTRAINT_VIOLATED}; + } else if (!s.IsNotFound()) { + return Result(rocksutils::convertStatus(s)); + } + } + + auto value = RocksDBValue::UniqueZkdIndexValue(documentId); + + if (auto s = methods->PutUntracked(_cf, rocks_key, value.string()); !s.ok()) { + return rocksutils::convertStatus(s); + } + + return {}; +} + +arangodb::Result arangodb::RocksDBUniqueZkdIndex::remove( + arangodb::transaction::Methods& trx, arangodb::RocksDBMethods* methods, + const arangodb::LocalDocumentId& documentId, + arangodb::velocypack::Slice doc) { + TRI_ASSERT(_unique == true); + TRI_ASSERT(_sparse == false); + + auto key_value = readDocumentKey(doc, _fields); + + RocksDBKey rocks_key; + rocks_key.constructZkdIndexValue(objectId(), key_value); + + auto s = methods->SingleDelete(_cf, rocks_key); + if (!s.ok()) { + return rocksutils::convertStatus(s); + } + + return {}; +} diff --git a/arangod/RocksDBEngine/RocksDBZkdIndex.h b/arangod/RocksDBEngine/RocksDBZkdIndex.h new file mode 100644 index 000000000000..979d15e05a23 --- /dev/null +++ b/arangod/RocksDBEngine/RocksDBZkdIndex.h @@ -0,0 +1,117 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2021 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Tobias Gödderz +/// @author Lars Maier +//////////////////////////////////////////////////////////////////////////////// + +#ifndef ARANGOD_ROCKSDB_ZKD_INDEX_H +#define ARANGOD_ROCKSDB_ZKD_INDEX_H + +#include "RocksDBEngine/RocksDBIndex.h" + +namespace arangodb { + +class RocksDBZkdIndexBase : public RocksDBIndex { + + public: + RocksDBZkdIndexBase(IndexId iid, LogicalCollection& coll, + arangodb::velocypack::Slice const& info); + void toVelocyPack(velocypack::Builder& builder, + std::underlying_type::type type) const override; + const char* typeName() const override { return "zkd"; }; + IndexType type() const override { return TRI_IDX_TYPE_ZKD_INDEX; }; + bool canBeDropped() const override { return true; } + bool isSorted() const override { return false; } + bool hasSelectivityEstimate() const override { return false; /* TODO */ } + Result insert(transaction::Methods& trx, RocksDBMethods* methods, + const LocalDocumentId& documentId, arangodb::velocypack::Slice doc, + const OperationOptions& options, bool performChecks) override; + Result remove(transaction::Methods& trx, RocksDBMethods* methods, + const LocalDocumentId& documentId, arangodb::velocypack::Slice doc) override; + + FilterCosts supportsFilterCondition(const std::vector>& allIndexes, + const arangodb::aql::AstNode* node, + const arangodb::aql::Variable* reference, + size_t itemsInIndex) const override; + + aql::AstNode* specializeCondition(arangodb::aql::AstNode* condition, + const arangodb::aql::Variable* reference) const override; + + std::unique_ptr iteratorForCondition(transaction::Methods* trx, + const aql::AstNode* node, + const aql::Variable* reference, + const IndexIteratorOptions& opts, + ReadOwnWrites readOwnWrites) override; +}; + +class RocksDBZkdIndex final : public RocksDBZkdIndexBase { + using RocksDBZkdIndexBase::RocksDBZkdIndexBase; +}; + +class RocksDBUniqueZkdIndex final : public RocksDBZkdIndexBase { + using RocksDBZkdIndexBase::RocksDBZkdIndexBase; + + Result insert(transaction::Methods& trx, RocksDBMethods* methods, + const LocalDocumentId& documentId, arangodb::velocypack::Slice doc, + const OperationOptions& options, bool performChecks) override; + Result remove(transaction::Methods& trx, RocksDBMethods* methods, + const LocalDocumentId& documentId, arangodb::velocypack::Slice doc) override; + + std::unique_ptr iteratorForCondition(transaction::Methods* trx, + const aql::AstNode* node, + const aql::Variable* reference, + const IndexIteratorOptions& opts, + ReadOwnWrites readOwnWrites) override; +}; + +namespace zkd { + +struct ExpressionBounds { + struct Bound { + aql::AstNode const* op_node = nullptr; + aql::AstNode const* bounded_expr = nullptr; + aql::AstNode const* bound_value = nullptr; + bool isStrict = false; + }; + + Bound lower; + Bound upper; +}; + +void extractBoundsFromCondition(arangodb::Index const* index, + const arangodb::aql::AstNode* condition, + const arangodb::aql::Variable* reference, + std::unordered_map& extractedBounds, + std::unordered_set& unusedExpressions); + +auto supportsFilterCondition(arangodb::Index const* index, + const std::vector>& allIndexes, + const arangodb::aql::AstNode* node, + const arangodb::aql::Variable* reference, + size_t itemsInIndex) -> Index::FilterCosts; + +auto specializeCondition(arangodb::Index const* index, arangodb::aql::AstNode* condition, + const arangodb::aql::Variable* reference) -> aql::AstNode*; +} + +} + +#endif // ARANGOD_ROCKSDB_ZKD_INDEX_H diff --git a/arangod/Zkd/ZkdHelper.cpp b/arangod/Zkd/ZkdHelper.cpp new file mode 100644 index 000000000000..381318b6875a --- /dev/null +++ b/arangod/Zkd/ZkdHelper.cpp @@ -0,0 +1,688 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2021 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Tobias Gödderz +/// @author Lars Maier +//////////////////////////////////////////////////////////////////////////////// +#include "ZkdHelper.h" + +#include "Basics/ScopeGuard.h" +#include "Basics/debugging.h" +#include "Containers/SmallVector.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace arangodb; +using namespace arangodb::zkd; + +zkd::byte_string zkd::operator"" _bs(const char* const str, std::size_t len) { + using namespace std::string_literals; + + std::string normalizedInput{}; + normalizedInput.reserve(len); + for (char const* p = str; *p != '\0'; ++p) { + switch (*p) { + case '0': + case '1': + normalizedInput += *p; + break; + case ' ': + case '\'': + // skip whitespace and single quotes + break; + default: + throw std::invalid_argument{"Unexpected character "s + *p + " in byte string: " + str}; + } + } + + if (normalizedInput.empty()) { + throw std::invalid_argument{"Empty byte string"}; + } + + auto result = byte_string{}; + + char const* p = normalizedInput.c_str(); + for (auto bitIdx = 0; *p != '\0'; bitIdx = 0) { + result += std::byte{0}; + for (; *p != '\0' && bitIdx < 8; ++bitIdx) { + switch (*p) { + case '0': + break; + case '1': { + auto const bitPos = 7 - bitIdx; + result.back() |= (std::byte{1} << bitPos); + break; + } + default: + throw std::invalid_argument{"Unexpected character "s + *p + " in byte string: " + str}; + } + + ++p; + // skip whitespace and single quotes + while (*p == ' ' || *p == '\'') { + ++p; + } + } + } + + return result; +} + +zkd::byte_string zkd::operator"" _bss(const char* str, std::size_t len) { + return byte_string{ reinterpret_cast(str), len}; +} + +zkd::BitReader::BitReader(zkd::BitReader::iterator begin, zkd::BitReader::iterator end) + : _current(begin), _end(end) {} + +auto zkd::BitReader::next() -> std::optional { + if (_nibble >= 8) { + if (_current == _end) { + return std::nullopt; + } + _value = *_current; + _nibble = 0; + ++_current; + } + + auto flag = std::byte{1u} << (7u - _nibble); + auto bit = (_value & flag) != std::byte{0} ? Bit::ONE : Bit::ZERO; + _nibble += 1; + return bit; +} + +auto zkd::BitReader::read_big_endian_bits(unsigned bits) -> uint64_t { + uint64_t result = 0; + for (size_t i = 0; i < bits; i++) { + uint64_t bit = uint64_t{1} << (bits - i - 1); + if (next_or_zero() == Bit::ONE) { + result |= bit; + } + } + return result; +} + +zkd::ByteReader::ByteReader(iterator begin, iterator end) + : _current(begin), _end(end) {} + +auto zkd::ByteReader::next() -> std::optional { + if (_current == _end) { + return std::nullopt; + } + return *(_current++); +} + +void zkd::BitWriter::append(Bit bit) { + if (bit == Bit::ONE) { + _value |= std::byte{1} << (7U - _nibble); + } + _nibble += 1; + if (_nibble == 8) { + _buffer.push_back(_value); + _value = std::byte{0}; + _nibble = 0; + } +} + +void zkd::BitWriter::write_big_endian_bits(uint64_t v, unsigned bits) { + for (size_t i = 0; i < bits; i++) { + auto b = (v & (uint64_t{1} << (bits - i - 1))) == 0 ? Bit::ZERO : Bit::ONE; + append(b); + } +} + +auto zkd::BitWriter::str() && -> zkd::byte_string { + if (_nibble > 0) { + _buffer.push_back(_value); + } + _nibble = 0; + _value = std::byte{0}; + return std::move(_buffer); +} + +void zkd::BitWriter::reserve(std::size_t amount) { + _buffer.reserve(amount); +} + +zkd::RandomBitReader::RandomBitReader(byte_string_view ref) : _ref(ref) {} + +auto zkd::RandomBitReader::getBit(std::size_t index) const -> Bit { + auto byte = index / 8; + auto nibble = index % 8; + + if (byte >= _ref.size()) { + return Bit::ZERO; + } + + auto b = (_ref[byte] >> (7 - nibble)) & 1_b; + return b != 0_b ? Bit::ONE : Bit::ZERO; +} + +auto zkd::RandomBitReader::bits() const -> std::size_t { + return 8 * _ref.size(); +} + +zkd::RandomBitManipulator::RandomBitManipulator(byte_string& ref) : _ref(ref) {} + +auto zkd::RandomBitManipulator::getBit(std::size_t index) const -> Bit { + auto byte = index / 8; + auto nibble = index % 8; + + if (byte >= _ref.size()) { + return Bit::ZERO; + } + + auto b = _ref[byte] & (1_b << (7 - nibble)); + return b != 0_b ? Bit::ONE : Bit::ZERO; +} + +auto zkd::RandomBitManipulator::setBit(std::size_t index, Bit value) -> void { + auto byte = index / 8; + auto nibble = index % 8; + + if (byte >= _ref.size()) { + _ref.resize(byte + 1); + } + auto bit = 1_b << (7 - nibble); + if (value == Bit::ONE) { + _ref[byte] |= bit; + } else { + _ref[byte] &= ~bit; + } +} + +auto zkd::RandomBitManipulator::bits() const -> std::size_t { + return 8 * _ref.size(); +} + +auto zkd::interleave(std::vector const& vec) -> zkd::byte_string { + std::size_t max_size = 0; + std::vector reader; + reader.reserve(vec.size()); + + for (auto const& str : vec) { + if (str.size() > max_size) { + max_size = str.size(); + } + reader.emplace_back(str); + } + + BitWriter bitWriter; + bitWriter.reserve(vec.size() * max_size); + + for (size_t i = 0; i < 8 * max_size; i++) { + for (auto& it : reader) { + auto b = it.next(); + bitWriter.append(b.value_or(Bit::ZERO)); + } + } + + return std::move(bitWriter).str(); +} + +auto zkd::transpose(byte_string_view bs, std::size_t dimensions) -> std::vector { + assert(dimensions > 0); + BitReader reader(bs); + std::vector writer; + writer.resize(dimensions); + + while (true) { + for (auto& w : writer) { + auto b = reader.next(); + if (!b.has_value()) { + goto break_loops; + } + w.append(b.value()); + } + } + break_loops: + + std::vector result; + std::transform(writer.begin(), writer.end(), std::back_inserter(result), [](auto& bs) { + return std::move(bs).str(); + }); + return result; +} + +auto zkd::compareWithBox(byte_string_view cur, byte_string_view min, byte_string_view max, std::size_t dimensions) +-> std::vector { + if (dimensions == 0) { + auto msg = std::string{"dimensions argument to "}; + msg += __func__; + msg += " must be greater than zero."; + throw std::invalid_argument{msg}; + } + std::vector result; + result.resize(dimensions); + zkd::compareWithBoxInto(cur, min, max, dimensions, result); + return result; +} + +void zkd::compareWithBoxInto(byte_string_view cur, byte_string_view min, byte_string_view max, + std::size_t dimensions, std::vector& result) { + + TRI_ASSERT(result.size() == dimensions); + std::fill(result.begin(), result.end(), CompareResult{}); + std::size_t max_size = std::max({cur.size(), min.size(), max.size()}); + + BitReader cur_reader(cur); + BitReader min_reader(min); + BitReader max_reader(max); + + auto const isLargerThanMin = [&result](auto const dim) { + return result[dim].saveMin != CompareResult::max; + }; + auto const isLowerThanMax = [&result](auto const dim) { + return result[dim].saveMax != CompareResult::max; + }; + + std::size_t step = 0; + std::size_t dim = 0; + + for (std::size_t i = 0; i < 8 * max_size; i++) { + TRI_ASSERT(step == i / dimensions); + TRI_ASSERT(dim == i % dimensions); + + auto cur_bit = cur_reader.next().value_or(Bit::ZERO); + auto min_bit = min_reader.next().value_or(Bit::ZERO); + auto max_bit = max_reader.next().value_or(Bit::ZERO); + + if (result[dim].flag == 0) { + if (!isLargerThanMin(dim)) { + if (cur_bit == Bit::ZERO && min_bit == Bit::ONE) { + result[dim].outStep = step; + result[dim].flag = -1; + } else if (cur_bit == Bit::ONE && min_bit == Bit::ZERO) { + result[dim].saveMin = step; + } + } + + if (!isLowerThanMax(dim)) { + if (cur_bit == Bit::ONE && max_bit == Bit::ZERO) { + result[dim].outStep = step; + result[dim].flag = 1; + } else if (cur_bit == Bit::ZERO && max_bit == Bit::ONE) { + result[dim].saveMax = step; + } + } + } + dim += 1; + if (dim >= dimensions) { + dim = 0; + step += 1; + } + } +} + +auto zkd::testInBox(byte_string_view cur, byte_string_view min, byte_string_view max, std::size_t dimensions) +-> bool { + + if (dimensions == 0) { + auto msg = std::string{"dimensions argument to "}; + msg += __func__; + msg += " must be greater than zero."; + throw std::invalid_argument{msg}; + } + + std::size_t max_size = std::max({cur.size(), min.size(), max.size()}); + + BitReader cur_reader(cur); + BitReader min_reader(min); + BitReader max_reader(max); + + ::arangodb::containers::SmallVector>::allocator_type::arena_type a; + ::arangodb::containers::SmallVector> isLargerLowerThanMinMax{a}; + isLargerLowerThanMinMax.resize(dimensions); + + unsigned dim = 0; + unsigned finished_dims = 2 * dimensions; + for (std::size_t i = 0; i < 8 * max_size; i++) { + + auto cur_bit = cur_reader.next().value_or(Bit::ZERO); + auto min_bit = min_reader.next().value_or(Bit::ZERO); + auto max_bit = max_reader.next().value_or(Bit::ZERO); + + if (!isLargerLowerThanMinMax[dim].first) { + if (cur_bit == Bit::ZERO && min_bit == Bit::ONE) { + return false; + } else if (cur_bit == Bit::ONE && min_bit == Bit::ZERO) { + isLargerLowerThanMinMax[dim].first = true; + if (--finished_dims == 0) { + break; + } + } + } + + if (!isLargerLowerThanMinMax[dim].second) { + if (cur_bit == Bit::ONE && max_bit == Bit::ZERO) { + return false; + } else if (cur_bit == Bit::ZERO && max_bit == Bit::ONE) { + isLargerLowerThanMinMax[dim].second = true; + if (--finished_dims == 0) { + break; + } + } + } + + dim += 1; + if (dim >= dimensions) { + dim = 0; + } + } + + return true; +} + +auto zkd::getNextZValue(byte_string_view cur, byte_string_view min, byte_string_view max, std::vector& cmpResult) +-> std::optional { + + auto result = byte_string{cur}; + + auto const dims = cmpResult.size(); + + auto minOutstepIter = std::min_element(cmpResult.begin(), cmpResult.end(), [&](auto const& a, auto const& b) { + if (a.flag == 0) { + return false; + } + if (b.flag == 0) { + return true; + } + return a.outStep < b.outStep; + }); + TRI_ASSERT(minOutstepIter->flag != 0); + auto const d = std::distance(cmpResult.begin(), minOutstepIter); + + RandomBitReader nisp(cur); + + std::size_t changeBP = dims * minOutstepIter->outStep + d; + + if (minOutstepIter->flag > 0) { + bool update_dims = false; + while (changeBP != 0 && !update_dims) { + --changeBP; + if (nisp.getBit(changeBP) == Bit::ZERO) { + auto dim = changeBP % dims; + auto step = changeBP / dims; + if (cmpResult[dim].saveMax <= step) { + cmpResult[dim].saveMin = step; + cmpResult[dim].flag = 0; + update_dims = true; + } + } + } + + if (!update_dims) { + return std::nullopt; + } + } + + { + RandomBitManipulator rbm(result); + TRI_ASSERT(rbm.getBit(changeBP) == Bit::ZERO); + rbm.setBit(changeBP, Bit::ONE); + TRI_ASSERT(rbm.getBit(changeBP) == Bit::ONE); + } + auto resultManipulator = RandomBitManipulator{result}; + auto minReader = RandomBitReader{min}; + + // Calculate the next bit position in dimension `dim` (regarding dims) + // after `bitPos` + auto const nextGreaterBitInDim = [dims](std::size_t const bitPos, std::size_t const dim) { + auto const posRem = bitPos % dims; + auto const posFloor = bitPos - posRem; + auto const result = dim > posRem ? (posFloor + dim) : posFloor + dims + dim; + // result must be a bit of dimension `dim` + TRI_ASSERT(result % dims == dim); + // result must be strictly greater than bitPos + TRI_ASSERT(bitPos < result); + // and the lowest bit with the above properties + TRI_ASSERT(result <= bitPos + dims); + return result; + }; + + for (std::size_t dim = 0; dim < dims; dim++) { + auto& cmpRes = cmpResult[dim]; + if (cmpRes.flag >= 0) { + auto bp = dims * cmpRes.saveMin + dim; + if (changeBP >= bp) { + // “set all bits of dim with bit positions > changeBP to 0” + for (std::size_t i = nextGreaterBitInDim(changeBP, dim); i < resultManipulator.bits(); i += dims) { + resultManipulator.setBit(i, Bit::ZERO); + } + } else { + // “set all bits of dim with bit positions > changeBP to the minimum of the query box in this dim” + for (std::size_t i = nextGreaterBitInDim(changeBP, dim); i < resultManipulator.bits(); i += dims) { + resultManipulator.setBit(i, minReader.getBit(i)); + } + } + } else { + // load the minimum for that dimension + for (std::size_t i = dim; i < resultManipulator.bits(); i += dims) { + resultManipulator.setBit(i, minReader.getBit(i)); + } + } + } + + return result; +} + +template +auto zkd::to_byte_string_fixed_length(T v) -> zkd::byte_string { + byte_string result; + static_assert(std::is_integral_v); + if constexpr (std::is_unsigned_v) { + result.reserve(sizeof(T)); + for (size_t i = 0; i < sizeof(T); i++) { + uint8_t b = 0xff & (v >> (56 - 8 * i)); + result.push_back(std::byte{b}); + } + } else { + // we have to add a byte + result.reserve(sizeof(T) + 1); + if (v < 0) { + result.push_back(0_b); + } else { + result.push_back(0xff_b); + } + for (size_t i = 0; i < sizeof(T); i++) { + uint8_t b = 0xff & (v >> (56 - 8 * i)); + result.push_back(std::byte{b}); + } + } + return result; +} + +template auto zkd::to_byte_string_fixed_length(uint64_t) -> zkd::byte_string; +template auto zkd::to_byte_string_fixed_length(int64_t) -> zkd::byte_string; +template auto zkd::to_byte_string_fixed_length(uint32_t) -> zkd::byte_string; +template auto zkd::to_byte_string_fixed_length(int32_t) -> zkd::byte_string; + +inline constexpr auto fp_infinity_expo_biased = (1u << 11) - 1; +inline constexpr auto fp_denorm_expo_biased = 0; +inline constexpr auto fp_min_expo_biased = std::numeric_limits::min_exponent - 1; + +auto zkd::destruct_double(double x) -> floating_point { + TRI_ASSERT(!std::isnan(x)); + + bool positive = true; + int exp = 0; + double base = frexp(x, &exp); + + // handle negative values + if (base < 0) { + positive = false; + base = - base; + } + + if (std::isinf(base)) { + // deal with +- infinity + return {positive, fp_infinity_expo_biased, 0}; + } + + auto int_base = uint64_t((uint64_t{1} << 53) * base); + + + if (exp < std::numeric_limits::min_exponent) { + // handle denormalized case + auto divide_by = std::numeric_limits::min_exponent - exp; + + exp = fp_denorm_expo_biased; + + int_base >>= divide_by; + return {positive, fp_denorm_expo_biased, int_base }; + } else { + //TRI_ASSERT(int_base & (uint64_t{1} << 53)); + uint64_t biased_exp = exp - fp_min_expo_biased; + if (int_base == 0) { + // handle zero case, assign smallest exponent + biased_exp = 0; + } + + return {positive, biased_exp, int_base}; + } +} + +auto zkd::construct_double(floating_point const& fp) -> double { + if (fp.exp == fp_infinity_expo_biased) { + // first handle infinity + auto base = std::numeric_limits::infinity(); + return fp.positive ? base : -base; + } + + uint64_t int_base = fp.base; + + + int exp = int(fp.exp) + fp_min_expo_biased; + + if (fp.exp != fp_denorm_expo_biased) { + int_base |= uint64_t{1} << 52; + } else { + exp = std::numeric_limits::min_exponent; + } + + double base = (double) int_base / double(uint64_t{1} << 53); + + if (!fp.positive) { + base = -base; + } + return std::ldexp(base, exp); +} + +std::ostream& zkd::operator<<(std::ostream& os, struct floating_point const& fp) { + os << (fp.positive ? "p" : "n") << fp.exp << "E" << fp.base; + return os; +} + +template<> +void zkd::into_bit_writer_fixed_length(BitWriter& bw, double x) { + auto [p, exp, base] = destruct_double(x); + + bw.append(p ? Bit::ONE : Bit::ZERO); + + if (!p) { + exp ^= (uint64_t{1} << 11) - 1; + base ^= (uint64_t{1} << 52) - 1; + } + + bw.write_big_endian_bits(exp, 11); + bw.write_big_endian_bits(base, 52); +} + +template<> +auto zkd::to_byte_string_fixed_length(double x) -> byte_string { + BitWriter bw; + zkd::into_bit_writer_fixed_length(bw, x); + return std::move(bw).str(); +} + + + +template<> +auto zkd::from_bit_reader_fixed_length(BitReader& r) -> double{ + bool isPositive = r.next_or_zero() == Bit::ONE; + + auto exp = r.read_big_endian_bits(11); + auto base = r.read_big_endian_bits(52); + if (!isPositive) { + exp ^= (uint64_t{1} << 11) - 1; + base ^= (uint64_t{1} << 52) - 1; + } + + return construct_double({isPositive, exp, base}); +} + +template +auto zkd::from_byte_string_fixed_length(byte_string_view bs) -> T { + T result = 0; + static_assert(std::is_integral_v); + if constexpr (std::is_unsigned_v) { + for (size_t i = 0; i < sizeof(T); i++) { + result |= std::to_integer(bs[i]) << (56 - 8 * i); + } + } else { + abort(); + } + + return result; +} + + +template auto zkd::from_byte_string_fixed_length(byte_string_view) -> uint64_t; + +template<> +auto zkd::from_byte_string_fixed_length(byte_string_view bs) -> double { + BitReader r(bs); + return from_bit_reader_fixed_length(r); +} + +std::ostream& operator<<(std::ostream& ostream, zkd::byte_string const& string) { + return ::operator<<(ostream, byte_string_view{string}); +} + +std::ostream& operator<<(std::ostream& ostream, byte_string_view string) { + ostream << "[0x "; + bool first = true; + for (auto const& it : string) { + if (!first) { + ostream << " "; + } + first = false; + ostream << std::hex << std::setfill('0') << std::setw(2) << std::to_integer(it); + } + ostream << "]"; + return ostream; +} + +std::ostream& zkd::operator<<(std::ostream& ostream, zkd::CompareResult const& cr) { + ostream << "CR{"; + ostream << "flag=" << cr.flag; + ostream << ", saveMin=" << cr.saveMin; + ostream << ", saveMax=" << cr.saveMax; + ostream << ", outStep=" << cr.outStep; + ostream << "}"; + return ostream; +} diff --git a/arangod/Zkd/ZkdHelper.h b/arangod/Zkd/ZkdHelper.h new file mode 100644 index 000000000000..9c65a41f1cca --- /dev/null +++ b/arangod/Zkd/ZkdHelper.h @@ -0,0 +1,170 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2021 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Tobias Gödderz +/// @author Lars Maier +//////////////////////////////////////////////////////////////////////////////// +#pragma once +#include +#include +#include +#include +#include +#include + +namespace arangodb::zkd { + +inline static std::byte operator"" _b(unsigned long long b) { + return std::byte{(unsigned char)b}; +} + +using byte_string = std::basic_string; +using byte_string_view = std::basic_string_view; + +byte_string operator"" _bs(const char* str, std::size_t len); +byte_string operator"" _bss(const char* str, std::size_t len); + +auto interleave(std::vector const& vec) -> byte_string; +auto transpose(byte_string_view bs, std::size_t dimensions) -> std::vector; + +struct alignas(32) CompareResult { + static constexpr auto max = std::numeric_limits::max(); + + signed flag = 0; + std::size_t outStep = CompareResult::max; + std::size_t saveMin = CompareResult::max; + std::size_t saveMax = CompareResult::max; +}; + +std::ostream& operator<<(std::ostream& ostream, CompareResult const& string); + +auto compareWithBox(byte_string_view cur, byte_string_view min, byte_string_view max, + std::size_t dimensions) -> std::vector; +void compareWithBoxInto(byte_string_view cur, byte_string_view min, byte_string_view max, + std::size_t dimensions, std::vector& result); +auto testInBox(byte_string_view cur, byte_string_view min, byte_string_view max, + std::size_t dimensions) -> bool; + +auto getNextZValue(byte_string_view cur, byte_string_view min, byte_string_view max, + std::vector& cmpResult) -> std::optional; + +template +auto to_byte_string_fixed_length(T) -> zkd::byte_string; +template +auto from_byte_string_fixed_length(byte_string_view) -> T; +template <> +byte_string to_byte_string_fixed_length(double x); + +enum class Bit { ZERO = 0, ONE = 1 }; + +class BitReader { + public: + using iterator = typename byte_string_view::const_iterator; + + explicit BitReader(iterator begin, iterator end); + explicit BitReader(byte_string const& str) + : BitReader(byte_string_view{str}) {} + explicit BitReader(byte_string_view v) : BitReader(v.cbegin(), v.cend()) {} + + auto next() -> std::optional; + auto next_or_zero() -> Bit { return next().value_or(Bit::ZERO); } + + auto read_big_endian_bits(unsigned bits) -> uint64_t; + + private: + iterator _current; + iterator _end; + std::byte _value{}; + std::size_t _nibble = 8; +}; + +class ByteReader { + public: + using iterator = typename byte_string::const_iterator; + + explicit ByteReader(iterator begin, iterator end); + + auto next() -> std::optional; + + private: + iterator _current; + iterator _end; +}; + +class BitWriter { + public: + void append(Bit bit); + void write_big_endian_bits(uint64_t, unsigned bits); + + auto str() && -> byte_string; + + void reserve(std::size_t amount); + + private: + std::size_t _nibble = 0; + std::byte _value = std::byte{0}; + byte_string _buffer; +}; + +struct RandomBitReader { + explicit RandomBitReader(byte_string_view ref); + + [[nodiscard]] auto getBit(std::size_t index) const -> Bit; + + [[nodiscard]] auto bits() const -> std::size_t; + + private: + byte_string_view _ref; +}; + +struct RandomBitManipulator { + explicit RandomBitManipulator(byte_string& ref); + + [[nodiscard]] auto getBit(std::size_t index) const -> Bit; + + auto setBit(std::size_t index, Bit value) -> void; + + [[nodiscard]] auto bits() const -> std::size_t; + + private: + byte_string& _ref; +}; + +template +void into_bit_writer_fixed_length(BitWriter&, T); +template +auto from_bit_reader_fixed_length(BitReader&) -> T; + +struct floating_point { + bool positive : 1; + uint64_t exp : 11; + uint64_t base : 52; +}; + +auto destruct_double(double x) -> floating_point; + +auto construct_double(floating_point const& fp) -> double; + +std::ostream& operator<<(std::ostream& os, struct floating_point const& fp); + +} // namespace arangodb::zkd + +std::ostream& operator<<(std::ostream& ostream, arangodb::zkd::byte_string const& string); +std::ostream& operator<<(std::ostream& ostream, arangodb::zkd::byte_string_view string); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 536af6686a06..7a3df028e0cb 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -289,7 +289,11 @@ set(ARANGODB_TESTS_SOURCES VocBase/LogicalView-test.cpp VocBase/VersionTest.cpp VPackDeserializer/BasicTests.cpp - ${ADDITIONAL_TEST_SOURCES}) + Zkd/Conversion.cpp + Zkd/Library.cpp + ${ADDITIONAL_TEST_SOURCES} +) + set(ARANGODB_REPLICATION2_TEST_SOURCES Replication2/ReplicatedLog/AppendEntriesBatchTest.cpp diff --git a/tests/Zkd/Conversion.cpp b/tests/Zkd/Conversion.cpp new file mode 100644 index 000000000000..431350c4d66b --- /dev/null +++ b/tests/Zkd/Conversion.cpp @@ -0,0 +1,211 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2021 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Tobias Gödderz +/// @author Lars Maier +//////////////////////////////////////////////////////////////////////////////// +#include "Zkd/ZkdHelper.h" + +#include "gtest/gtest.h" + +using namespace arangodb; +using namespace arangodb::zkd; + +TEST(Zkd_byte_string_conversion, uint64) { + auto tests = {std::pair{uint64_t{12}, byte_string{0_b, 0_b, 0_b, 0_b, 0_b, 0_b, 0_b, 12_b}}, + std::pair{uint64_t{0xAABBCCDD}, + byte_string{0_b, 0_b, 0_b, 0_b, 0xAA_b, 0xBB_b, 0xCC_b, 0xDD_b}}, + std::pair{uint64_t{0x0123456789ABCDEF}, + byte_string{0x01_b, 0x23_b, 0x45_b, 0x67_b, 0x89_b, + 0xAB_b, 0xCD_b, 0xEF_b}}}; + + for (auto&& [v, bs] : tests) { + auto result = to_byte_string_fixed_length(v); + EXPECT_EQ(result, bs); + } +} + +TEST(Zkd_byte_string_conversion, uint64_compare) { + auto tests = { + std::pair{uint64_t{12}, uint64_t{7}}, + std::pair{uint64_t{4567}, uint64_t{768735456}}, + std::pair{uint64_t{4567}, uint64_t{4567}}, + }; + + for (auto&& [a, b] : tests) { + auto a_bs = to_byte_string_fixed_length(a); + auto b_bs = to_byte_string_fixed_length(b); + + EXPECT_EQ(a < b, a_bs < b_bs) + << "byte string of " << a << " and " << b + << " does not compare equally: " << a_bs << " " << b_bs; + } +} + +TEST(Zkd_byte_string_conversion, int64) { + auto tests = {std::pair{int64_t{12}, byte_string{0xff_b, 0_b, 0_b, 0_b, 0_b, + 0_b, 0_b, 0_b, 12_b}}, + std::pair{int64_t{0xAABBCCDD}, byte_string{0xFF_b, 0_b, 0_b, 0_b, 0_b, 0xAA_b, + 0xBB_b, 0xCC_b, 0xDD_b}}, + std::pair{int64_t{-0x0123456789ABCDEF}, + byte_string{0x00_b, 0xFE_b, 0xDC_b, 0xBA_b, 0x98_b, + 0x76_b, 0x54_b, 0x32_b, 0x11_b}}}; + + for (auto&& [v, bs] : tests) { + auto result = to_byte_string_fixed_length(v); + EXPECT_EQ(result, bs); + } +} + +TEST(Zkd_byte_string_conversion, int64_compare) { + auto tests = { + std::pair{int64_t{12}, int64_t{453}}, + std::pair{int64_t{-12}, int64_t{453}}, + std::pair{int64_t{-1458792}, int64_t{453}}, + std::pair{int64_t{17819835131}, int64_t{-894564}}, + std::pair{int64_t{-12}, int64_t{-8}}, + std::pair{int64_t{-5646872}, int64_t{-5985646871}}, + std::pair{int64_t{-5985646871}, int64_t{-5985646871}}, + }; + + for (auto&& [a, b] : tests) { + auto a_bs = to_byte_string_fixed_length(a); + auto b_bs = to_byte_string_fixed_length(b); + + EXPECT_EQ(a < b, a_bs < b_bs) + << "byte string of " << a << " and " << b + << " does not compare equally: " << a_bs << " " << b_bs; + } +} + +auto const doubles_worth_testing = {0.0, + 0.1, + 0.2, + 0.3, + 0.4, + 1.0, + 10.0, + -1.0, + -0.001, + 1000., + -.00001, + -100.0, + 4.e-12, + 100000.0 + -5e+15, + std::numeric_limits::epsilon(), + std::numeric_limits::max(), + std::numeric_limits::min(), + std::numeric_limits::denorm_min(), + std::numeric_limits::infinity(), + -std::numeric_limits::infinity(), + std::numeric_limits::lowest()}; + +TEST(Zkd_byte_string_conversion, double_float_cmp) { + + for (auto&& a : doubles_worth_testing) { + for (auto&& b : doubles_worth_testing) { + auto a_bs = to_byte_string_fixed_length(a); + auto b_bs = to_byte_string_fixed_length(b); + + EXPECT_EQ(a < b, a_bs < b_bs) + << "byte string of " << a << " and " << b + << " does not compare equally: " << a_bs << " " << b_bs + << " cnvrt = " << destruct_double(a) << " b = " << destruct_double(b); + EXPECT_EQ(a == b, a_bs == b_bs) + << "byte string of " << a << " and " << b + << " does not compare equally: " << a_bs << " " << b_bs; + EXPECT_EQ(a > b, a_bs > b_bs) + << "byte string of " << a << " and " << b + << " does not compare equally: " << a_bs << " " << b_bs; + EXPECT_EQ(a >= b, a_bs >= b_bs) + << "byte string of " << a << " and " << b + << " does not compare equally: " << a_bs << " " << b_bs; + EXPECT_EQ(a <= b, a_bs <= b_bs) + << "byte string of " << a << " and " << b + << " does not compare equally: " << a_bs << " " << b_bs; + } + } +} + +TEST(Zkd_byte_string_conversion, bit_reader_test) { + auto s = "1110 10101"_bs; + + BitReader r(s); + auto v = r.read_big_endian_bits(4); + EXPECT_EQ(0b1110, v); + auto v2 = r.read_big_endian_bits(5); + EXPECT_EQ(0b10101, v2); +} + +TEST(Zkd_byte_string_conversion, bit_reader_test_different_sizes) { + auto s = "1"_bs; + + { + BitReader r(s); + auto v = r.read_big_endian_bits(1); + EXPECT_EQ(1, v); + } + { + BitReader r(s); + auto v = r.read_big_endian_bits(8); + EXPECT_EQ(1 << 7, v); + } + { + BitReader r(s); + auto v = r.read_big_endian_bits(16); + EXPECT_EQ(1ull << 15, v); + } + { + BitReader r(s); + auto v = r.read_big_endian_bits(32); + EXPECT_EQ(1ull << 31, v); + } + { + BitReader r(s); + auto v = r.read_big_endian_bits(64); + EXPECT_EQ(1ull << 63, v); + } +} + +TEST(Zkd_byte_string_conversion, construct_destruct_double) { + + for (auto const a : doubles_worth_testing) { + auto destructed = destruct_double(a); + auto reconstructed = construct_double(destructed); + ASSERT_EQ(a, reconstructed) << "testee: " << a << ", " + << "destructed: " << destructed << ", " + << "reconstructed: " << reconstructed; + } +} + +TEST(Zkd_byte_string_conversion, double_from_byte_string) { + + for (auto a : doubles_worth_testing) { + double a1; + memcpy(&a1, &a, sizeof(double)); + + auto a_bs = to_byte_string_fixed_length(a1); + auto b = from_byte_string_fixed_length(a_bs); + + EXPECT_EQ(a1, b) << "byte string of " << a1 << " is " << a_bs + << " and was read as " << b; + } +} diff --git a/tests/Zkd/Library.cpp b/tests/Zkd/Library.cpp new file mode 100644 index 000000000000..e8e85b3418bf --- /dev/null +++ b/tests/Zkd/Library.cpp @@ -0,0 +1,434 @@ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2014-2021 ArangoDB GmbH, Cologne, Germany +/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Tobias Gödderz +/// @author Lars Maier +//////////////////////////////////////////////////////////////////////////////// +#include +#include +#include +#include + +#include "Zkd/ZkdHelper.h" + +#include +#include + +using namespace arangodb; + +static std::ostream& operator<<(std::ostream& os, std::vector const& bsvec) { + os << "{"; + if (!bsvec.empty()) { + auto it = bsvec.begin(); + os << *it; + for (++it; it != bsvec.end(); ++it) { + os << ", "; + os << *it; + } + } + os << "}"; + + return os; +} + +static std::ostream& operator<<(std::ostream& os, + std::vector const& cmpResult) { + os << "{"; + if (!cmpResult.empty()) { + auto it = cmpResult.begin(); + os << *it; + for (++it; it != cmpResult.end(); ++it) { + os << ", "; + os << *it; + } + } + os << "}"; + + return os; +} + +#include "gtest/gtest.h" + +using namespace zkd; + +TEST(Zkd_byteStringLiteral, bs) { + EXPECT_THROW(""_bs, std::invalid_argument); + EXPECT_THROW(" "_bs, std::invalid_argument); + EXPECT_THROW("'"_bs, std::invalid_argument); + EXPECT_THROW("2"_bs, std::invalid_argument); + EXPECT_THROW("a"_bs, std::invalid_argument); + EXPECT_THROW("\0"_bs, std::invalid_argument); + EXPECT_THROW("02"_bs, std::invalid_argument); + EXPECT_THROW("12"_bs, std::invalid_argument); + EXPECT_THROW("0 2"_bs, std::invalid_argument); + EXPECT_THROW("1 2"_bs, std::invalid_argument); + + EXPECT_EQ(byte_string{std::byte{0x00}}, "0"_bs); + EXPECT_EQ(byte_string{std::byte{0x80}}, "1"_bs); + EXPECT_EQ((byte_string{std::byte{0x00}, std::byte{0x00}}), "00000000'0"_bs); + EXPECT_EQ((byte_string{std::byte{0x00}, std::byte{0x80}}), "00000000'1"_bs); + EXPECT_EQ((byte_string{std::byte{0x00}, std::byte{0x80}}), "0'00000001"_bs); + EXPECT_EQ((byte_string{std::byte{0x00}, std::byte{0x80}}), "0 00000001"_bs); + EXPECT_EQ((byte_string{std::byte{0x00}, std::byte{0x80}}), "0 000 000 01"_bs); + EXPECT_EQ((byte_string{std::byte{0x01}, std::byte{0x00}}), "00000001'0"_bs); + EXPECT_EQ((byte_string{std::byte{0x01}, std::byte{0x00}}), "0'00000010"_bs); + EXPECT_EQ((byte_string{std::byte{0x80}, std::byte{0x00}}), "1'00000000"_bs); + EXPECT_EQ((byte_string{std::byte{0xa8}, std::byte{0xa8}}), + "10101000'101010"_bs); + EXPECT_EQ((byte_string{std::byte{0x15}, std::byte{0x15}, std::byte{0}}), + "00010101'00010101'0"_bs); + + EXPECT_EQ(byte_string{std::byte{0x00}}, "00000000"_bs); + EXPECT_EQ((byte_string{std::byte{0x00}, std::byte{0x00}}), + "00000000 00000000"_bs); + EXPECT_EQ((byte_string{std::byte{0x00}, std::byte{1}}), + "00000000 00000001"_bs); + EXPECT_EQ((byte_string{std::byte{0x00}, std::byte{2}}), + "00000000 00000010"_bs); + EXPECT_EQ((byte_string{std::byte{1}, std::byte{0x00}}), + "00000001 00000000"_bs); + EXPECT_EQ((byte_string{std::byte{42}, std::byte{42}}), "00101010 00101010"_bs); + EXPECT_EQ((byte_string{std::byte{0x00}, std::byte{42}, std::byte{42}}), + "00000000 00101010 00101010"_bs); +} + +TEST(Zkd_interleave, d0) { + auto res = interleave({}); + ASSERT_EQ(byte_string{}, res); +} + +TEST(Zkd_interleave, d1_empty) { + auto res = interleave({{}}); + ASSERT_EQ(byte_string{}, res); +} + +TEST(Zkd_interleave, d1_multi) { + auto const testees = { + "00101010"_bs, + "00101010'00101010"_bs, + "00000001'00000010'00000011"_bs, + }; + for (auto const& testee : testees) { + auto const res = interleave({testee}); + EXPECT_EQ(testee, res); + } +} + +TEST(Zkd_interleave, d2_empty) { + auto res = interleave({{}, {}}); + ASSERT_EQ(byte_string{}, res); +} + +TEST(Zkd_interleave, d2_multi) { + auto const testees = { + std::pair{"01010101'10101010"_bs, + std::tuple{"00001111"_bs, "11110000"_bs}}, + {"01010101'01010101'00110011'00110011"_bs, + {"00000000'01010101"_bs, "11111111'01010101"_bs}}, + {"10101010'10101010'01010101'01010101"_bs, + {"11111111"_bs, "00000000'11111111"_bs}}, + {"01010111'01010111'00010001'00010001'01000100'01000100"_bs, + {"00010001"_bs, "11111111'01010101'10101010"_bs}}, + }; + for (auto const& it : testees) { + auto const& [expected, testee] = it; + auto const res = interleave({std::get<0>(testee), std::get<1>(testee)}); + EXPECT_EQ(expected, res); + } +} + +TEST(Zkd_transpose, d3_empty) { + auto res = transpose({}, 3); + ASSERT_EQ(res, (std::vector{{}, {}, {}})); +} + +TEST(Zkd_transpose, d3_multi) { + auto const testees = { + std::pair{"00011100"_bs, + std::vector{"01000000"_bs, "01000000"_bs, "01000000"_bs}}, + {"00011110"_bs, {"01100000"_bs, "01000000"_bs, "01000000"_bs}}, + {"10101010"_bs, {"10100000"_bs, "01000000"_bs, "10000000"_bs}}, + }; + for (auto const& it : testees) { + auto const& [testee, expected] = it; + auto res = transpose(testee, 3); + ASSERT_EQ(res, expected); + } +} + +TEST(Zkd_compareBox, d2_eq) { + auto min_v = interleave({"00000101"_bs, "01001101"_bs}); // 00 01 00 00 01 11 00 11 + auto max_v = interleave({"00100011"_bs, "01111001"_bs}); // 00 01 11 01 01 00 10 11 + auto v = interleave({"00001111"_bs, "01010110"_bs}); // 00 01 00 01 10 11 11 10 + + auto res = compareWithBox(v, min_v, max_v, 2); + + // 00 01 00 00 01 11 00 11 -- min (5, 77) + // 00 01 00 01 10 11 11 10 -- cur (15, 86) + // 00 01 11 01 01 00 10 11 -- max (35, 121) + + EXPECT_EQ(res[0].flag, 0); + EXPECT_EQ(res[0].saveMin, 4); + EXPECT_EQ(res[0].saveMax, 2); + EXPECT_EQ(res[0].outStep, CompareResult::max); + EXPECT_EQ(res[1].flag, 0); + EXPECT_EQ(res[1].saveMin, 3); + EXPECT_EQ(res[1].saveMax, 2); + EXPECT_EQ(res[1].outStep, CompareResult::max); +} + +TEST(Zkd_compareBox, d2_eq2) { + auto min_v = interleave({"00000010"_bs, "00000011"_bs}); // 00 00 00 00 00 00 11 01 + auto max_v = interleave({"00000110"_bs, "00000101"_bs}); // 00 00 00 00 00 11 10 01 + auto v = interleave({"00000011"_bs, "00000011"_bs}); // 00 00 00 00 00 00 11 11 + + auto res = compareWithBox(v, min_v, max_v, 2); + + EXPECT_EQ(res[0].flag, 0); + EXPECT_EQ(res[0].saveMin, 7); + EXPECT_EQ(res[0].saveMax, 5); + EXPECT_EQ(res[0].outStep, CompareResult::max); + EXPECT_EQ(res[1].flag, 0); + EXPECT_EQ(res[1].saveMin, CompareResult::max); + EXPECT_EQ(res[1].saveMax, 5); + EXPECT_EQ(res[1].outStep, CompareResult::max); +} + +TEST(Zkd_compareBox, d2_less) { + auto min_v = interleave({"00000101"_bs, "01001101"_bs}); + auto max_v = interleave({"00100011"_bs, "01111001"_bs}); + auto v = interleave({"00000011"_bs, "01010110"_bs}); + + auto res = compareWithBox(v, min_v, max_v, 2); + + EXPECT_EQ(res[0].flag, -1); + EXPECT_EQ(res[0].saveMin, CompareResult::max); + EXPECT_EQ(res[0].saveMax, 2); + EXPECT_EQ(res[0].outStep, 5); + EXPECT_EQ(res[1].flag, 0); + EXPECT_EQ(res[1].saveMin, 3); + EXPECT_EQ(res[1].saveMax, 2); + EXPECT_EQ(res[1].outStep, CompareResult::max); +} + +TEST(Zkd_compareBox, d2_x_less_y_greater) { + auto min_v = interleave({"00000100"_bs, "00000010"_bs}); // 00 00 00 00 00 10 01 00 + auto max_v = interleave({"00001000"_bs, "00000110"_bs}); // 00 00 00 00 10 01 01 00 + auto v = interleave({"00000011"_bs, "00010000"_bs}); // 00 00 00 01 00 00 10 10 + + auto res = compareWithBox(v, min_v, max_v, 2); + + EXPECT_EQ(res[0].flag, -1); + EXPECT_EQ(res[0].saveMin, -1); + EXPECT_EQ(res[0].saveMax, 4); + EXPECT_EQ(res[0].outStep, 5); + EXPECT_EQ(res[1].flag, 1); + EXPECT_EQ(res[1].saveMin, 3); + EXPECT_EQ(res[1].saveMax, -1); + EXPECT_EQ(res[1].outStep, 3); +} + +TEST(Zkd_compareBox, d3_x_less_y_greater_z_eq) { + auto min_v = interleave({"00000100"_bs, "00000010"_bs, "00000000"_bs}); // 000 000 000 000 000 100 010 000 + auto max_v = interleave({"00001000"_bs, "00000110"_bs, "00000010"_bs}); // 000 000 000 000 100 010 011 000 + auto v = interleave({"00000011"_bs, "00010000"_bs, "00000010"_bs}); // 000 000 000 010 000 000 101 100 + + auto res = compareWithBox(v, min_v, max_v, 3); + + EXPECT_EQ(res[0].flag, -1); + EXPECT_EQ(res[0].saveMin, -1); + EXPECT_EQ(res[0].saveMax, 4); + EXPECT_EQ(res[0].outStep, 5); + EXPECT_EQ(res[1].flag, 1); + EXPECT_EQ(res[1].saveMin, 3); + EXPECT_EQ(res[1].saveMax, -1); + EXPECT_EQ(res[1].outStep, 3); + EXPECT_EQ(res[2].flag, 0); + EXPECT_EQ(res[2].saveMin, 6); + EXPECT_EQ(res[2].saveMax, -1); + EXPECT_EQ(res[2].outStep, -1); +} + +TEST(Zkd_compareBox, testFigure41_3) { + // lower point of the box: (2, 2) + auto min_v = interleave({"00000010"_bs, "00000010"_bs}); + // upper point of the box: (5, 4) + auto max_v = interleave({"00000101"_bs, "00000100"_bs}); + + auto v = interleave({"00000110"_bs, "00000010"_bs}); // (6, 2) + auto res = compareWithBox(v, min_v, max_v, 2); + + EXPECT_EQ(res[0].flag, 1); + EXPECT_EQ(res[0].saveMin, 5); + EXPECT_EQ(res[0].saveMax, CompareResult::max); + EXPECT_EQ(res[0].outStep, 6); + EXPECT_EQ(res[1].flag, 0); + EXPECT_EQ(res[1].saveMin, CompareResult::max); + EXPECT_EQ(res[1].saveMax, 5); + EXPECT_EQ(res[1].outStep, CompareResult::max); +} + +TEST(Zkd_rocksdb, convert_bytestring) { + auto const data = { + "00011100"_bs, + "11111111'01010101"_bs, + }; + + for (auto const& it : data) { + auto const slice = + rocksdb::Slice(reinterpret_cast(it.c_str()), it.size()); + auto const string = + byte_string{reinterpret_cast(slice.data()), slice.size()}; + EXPECT_EQ(it, string); + EXPECT_EQ(it.size(), slice.size()); + EXPECT_EQ(it.size(), string.size()); + EXPECT_EQ(0, memcmp(it.data(), slice.data(), it.size())); + } +} + +static auto sliceFromString(byte_string const& str) -> rocksdb::Slice { + return rocksdb::Slice(reinterpret_cast(str.c_str()), str.size()); +} + +auto viewFromSlice(rocksdb::Slice slice) -> byte_string_view { + return byte_string_view{reinterpret_cast(slice.data()), + slice.size()}; +} + +TEST(Zkd_rocksdb, cmp_slice) { + enum class Cmp : int { LT = -1, EQ = 0, GT = 1 }; + auto const data = { + std::pair{Cmp::EQ, std::pair{"00101010"_bs, "00101010"_bs}}, + std::pair{Cmp::EQ, + std::pair{"00000001'00000010"_bs, "00000001'00000010"_bs}}, + std::pair{Cmp::LT, + std::pair{"00000001'00000001"_bs, "00000001'00000010"_bs}}, + std::pair{Cmp::GT, std::pair{"10000000"_bs, "01111111'11111111"_bs}}, + // TODO more tests + }; + + auto const* const cmp = rocksdb::BytewiseComparator(); + + for (auto const& it : data) { + auto const& [expected, testee] = it; + auto const& [left, right] = testee; + EXPECT_EQ(expected == Cmp::LT, cmp->Compare(sliceFromString(left), sliceFromString(right)) < 0) + << "left = " << left << ", right = " << right; + EXPECT_EQ(expected == Cmp::EQ, cmp->Compare(sliceFromString(left), sliceFromString(right)) == 0) + << "left = " << left << ", right = " << right; + EXPECT_EQ(expected == Cmp::EQ, cmp->Equal(sliceFromString(left), sliceFromString(right))) + << "left = " << left << ", right = " << right; + EXPECT_EQ(expected == Cmp::GT, cmp->Compare(sliceFromString(left), sliceFromString(right)) > 0) + << "left = " << left << ", right = " << right; + } +} + +TEST(Zkd_getNextZValue, testFigure41) { + // lower point of the box: (2, 2) + auto const pMin = interleave({"00000010"_bs, "00000010"_bs}); + // upper point of the box: (4, 5) + auto const pMax = interleave(std::vector{"00000100"_bs, "00000101"_bs}); + + auto test = [&pMin, &pMax](std::vector const& inputCoords, + std::optional> const& expectedCoords) { + auto const input = interleave(inputCoords); + auto const expected = std::invoke([&]() -> std::optional { + if (expectedCoords.has_value()) { + return interleave(expectedCoords.value()); + } else { + return std::nullopt; + } + }); + auto cmpResult = compareWithBox(input, pMin, pMax, 2); + // input should be outside the box: + auto sstr = std::stringstream{}; + if (expectedCoords.has_value()) { + sstr << expectedCoords.value(); + } else { + sstr << "n/a"; + } + ASSERT_TRUE(std::any_of(cmpResult.begin(), cmpResult.end(), + [](auto const& it) { return it.flag != 0; })) + << "with input=" << inputCoords << ", expected=" << sstr.str() + << ", result=" << cmpResult; + auto result = getNextZValue(input, pMin, pMax, cmpResult); + auto sstr2 = std::stringstream{}; + if (result.has_value()) { + sstr2 << result.value() << "/" << transpose(result.value(), 2); + } else { + sstr2 << "n/a"; + } + EXPECT_EQ(expected, result) + << "with input=" << inputCoords << ", expected=" << sstr.str() + << ", result=" << sstr2.str() << ", cmpResult=" << cmpResult; + // TODO should cmpResult be checked? + }; + + // z-curve inside the box [ (2, 2); (4, 5) ] goes through the following + // points. the value after -/> is outside the box. The next line continues + // with the next point on the curve inside the box. (2, 2) -> (2, 3) -> (3, 2) + // -> (3, 3) -/> (0, 4) (2, 4) -> (3, 4) -> (2, 5) -> (3, 5) -/> (2, 6) (4, 2) + // -> (4, 3) -/> (5, 2) (4, 4) -> (4, 5) -/> (5, 4) + + test({"00000000"_bs, "00000000"_bs}, {{"00000010"_bs, "00000010"_bs}}); + test({"00000000"_bs, "00000100"_bs}, {{"00000010"_bs, "00000100"_bs}}); + test({"00000010"_bs, "00000110"_bs}, {{"00000100"_bs, "00000010"_bs}}); + test({"00000101"_bs, "00000010"_bs}, {{"00000100"_bs, "00000100"_bs}}); + test({"00000101"_bs, "00000100"_bs}, std::nullopt); + + for (uint8_t xi = 0; xi < 8; ++xi) { + for (uint8_t yi = 0; yi < 8; ++yi) { + bool const inBox = 2 <= xi && xi <= 4 && 2 <= yi && yi <= 5; + auto const input = interleave({{std::byte{xi}}, {std::byte{yi}}}); + + auto cmpResult = compareWithBox(input, pMin, pMax, 2); + // assert that compareWithBox agrees with our `inBox` bool + ASSERT_EQ(inBox, std::all_of(cmpResult.begin(), cmpResult.end(), + [](auto const& it) { return it.flag == 0; })) + << "xi=" << int(xi) << ", yi=" << int(yi) << ", cmpResult=" << cmpResult; + if (!inBox) { + auto result = getNextZValue(input, pMin, pMax, cmpResult); + if (result.has_value()) { + // TODO make the check more precise, check that it's the correct point + auto res = compareWithBox(result.value(), pMin, pMax, 2); + EXPECT_TRUE(std::all_of(res.begin(), res.end(), + [](auto const& it) { return it.flag == 0; })); + } else { + } + // TODO add a check for the else branch, whether it's correct to not + // return a value. + } + } + } +} + +TEST(Zkd_testInBox, regression_1) { + auto cur = zkd::interleave({ + byte_string{0x5f_b, 0xf8_b, 0x00_b, 0x00_b, 0x00_b, 0x00_b, 0x00_b, 0x00_b, 0x00_b}, + byte_string{0x00_b, 0x00_b, 0x00_b, 0x00_b, 0x00_b, 0x00_b, 0x00_b, 0x00_b, 0x00_b} + }); + auto const& min = cur; + auto max = zkd::interleave({ + byte_string{0x60_b, 0x04_b, 0x00_b, 0x00_b, 0x00_b, 0x00_b, 0x00_b, 0x00_b, 0x00_b}, + byte_string{0x80_b, 0x00_b, 0x00_b, 0x00_b, 0x00_b, 0x00_b, 0x00_b, 0x00_b, 0x00_b} + }); + + ASSERT_TRUE(testInBox(cur, min, max, 2)); +} diff --git a/tests/js/server/aql/aql-optimizer-zkdindex-multi.js b/tests/js/server/aql/aql-optimizer-zkdindex-multi.js new file mode 100644 index 000000000000..e789345a852d --- /dev/null +++ b/tests/js/server/aql/aql-optimizer-zkdindex-multi.js @@ -0,0 +1,162 @@ +/* global AQL_EXPLAIN, AQL_EXECUTE */ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2021-2021 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Lars Maier +//////////////////////////////////////////////////////////////////////////////// + +'use strict'; + +const jsunity = require("jsunity"); +const arangodb = require("@arangodb"); +const db = arangodb.db; +const aql = arangodb.aql; +const {assertTrue, assertEqual} = jsunity.jsUnity.assertions; + +const useIndexes = 'use-indexes'; +const removeFilterCoveredByIndex = "remove-filter-covered-by-index"; + +const opCases = ["none", "eq", "le", "ge", "le2", "ge2", "lt", "gt", "legt"]; + +function conditionForVariable(op, name) { + if (op === "none") { + return "true"; + } else if (op === "eq") { + return name + " == 5"; + } else if (op === "le") { + return `${name} <= 5`; + } else if (op === "ge") { + return `${name} >= 5`; + } else if (op === "ge2") { + return `${name} >= 5 && ${name} <= 6`; + } else if (op === "le2") { + return `${name} <= 5 && ${name} >= 4`; + } else if (op === "lt") { + return `${name} < 5`; + } else if (op === "gt") { + return `${name} > 5`; + } else if (op === "legt") { + return `${name} <= 5 && ${name} > 4`; + } +} + +function resultSetForConditoin(op) { + const all = [...Array(11).keys()]; + if (op === "none") { + return all; + } else if (op === "eq") { + return all.filter((x) => x === 5); + } else if (op === "le") { + return all.filter((x) => x <= 5); + } else if (op === "ge") { + return all.filter((x) => x >= 5); + } else if (op === "ge2") { + return all.filter((x) => x >= 5 && x <= 6); + } else if (op === "le2") { + return all.filter((x) => x <= 5 && x >= 4); + } else if (op === "lt") { + return all.filter((x) => x < 5); + } else if (op === "gt") { + return all.filter((x) => x > 5); + } else if (op === "legt") { + return all.filter((x) => x <= 5 && x > 4); + } +} + +function productSet(x, y, z, w) { + let result = []; + for (let dx of resultSetForConditoin(x)) { + for (let dy of resultSetForConditoin(y)) { + for (let dz of resultSetForConditoin(z)) { + for (let dw of resultSetForConditoin(w)) { + result.unshift([dx, dy, dz, dw]); + } + } + } + } + return result; +} + +function optimizerRuleZkd2dIndexTestSuite() { + const colName = 'UnitTestZkdIndexMultiCollection'; + let col; + + let testObject = { + setUpAll: function () { + col = db._create(colName); + col.ensureIndex({type: 'zkd', name: 'zkdIndex', fields: ['x', 'y', 'z', 'a.w'], fieldValueTypes: 'double'}); + db._query(aql` + FOR x IN 0..10 + FOR y IN 0..10 + FOR z IN 0..10 + FOR w IN 0..10 + INSERT {x, y, z, a: {w}} INTO ${col} + `); + }, + + tearDownAll: function () { + col.drop(); + }, + }; + + for (let x of ["none", "eq"]) { + for (let y of ["none", "le", "gt"]) { + for (let z of opCases) { + for (let w of opCases) { + if (x === "none" && y === "none" && z === "none" && w === "none") { + continue; // does not use the index + } + + testObject[["testCase", x, y, z, w].join("_")] = function () { + const query = ` + FOR d IN ${colName} + FILTER ${conditionForVariable(x, "d.x")} + FILTER ${conditionForVariable(y, "d.y")} + FILTER ${conditionForVariable(z, "d.z")} + FILTER ${conditionForVariable(w, "d.a.w")} + RETURN [d.x, d.y, d.z, d.a.w] + `; + const explainRes = AQL_EXPLAIN(query); + const appliedRules = explainRes.plan.rules; + const nodeTypes = explainRes.plan.nodes.map(n => n.type).filter(n => !["GatherNode", "RemoteNode"].includes(n)); + assertEqual(["SingletonNode", "IndexNode", "CalculationNode", "ReturnNode"], nodeTypes); + assertTrue(appliedRules.includes(useIndexes)); + + const conds = [x, y, z, w]; + if (!conds.includes("lt") && !conds.includes("gt") && !conds.includes("legt")) { + assertTrue(appliedRules.includes(removeFilterCoveredByIndex)); + } + const executeRes = AQL_EXECUTE(query); + const res = executeRes.json; + const expected = productSet(x, y, z, w); + res.sort(); + expected.sort(); + assertEqual(expected, res, JSON.stringify({query})); + }; + } + } + } + } + + return testObject; +} + +jsunity.run(optimizerRuleZkd2dIndexTestSuite); + +return jsunity.done(); diff --git a/tests/js/server/aql/aql-optimizer-zkdindex-restricted.js b/tests/js/server/aql/aql-optimizer-zkdindex-restricted.js new file mode 100644 index 000000000000..8cb64c542670 --- /dev/null +++ b/tests/js/server/aql/aql-optimizer-zkdindex-restricted.js @@ -0,0 +1,74 @@ +/* global AQL_EXPLAIN, AQL_EXECUTE */ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2021-2021 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Tobias Gödderz +//////////////////////////////////////////////////////////////////////////////// + +'use strict'; + +const jsunity = require("jsunity"); +const arangodb = require("@arangodb"); +const internal = require("internal"); +const db = arangodb.db; +const {assertEqual} = jsunity.jsUnity.assertions; + +function optimizerRuleZkd2dIndexTestSuite() { + const colName = 'UnitTestZkdIndexCollection'; + let col; + + return { + setUpAll: function () { + col = db._create(colName); + }, + + tearDownAll: function () { + col.drop(); + }, + + testNoFieldValueTypes: function () { + try { + col.ensureIndex({type: 'zkd', name: 'zkdIndex', fields: ['x', 'y']}); + } catch (e) { + assertEqual(e.errorNum, internal.errors.ERROR_BAD_PARAMETER.code); + } + }, + + testSparseProperty: function () { + try { + col.ensureIndex({type: 'zkd', name: 'zkdIndex', fields: ['x', 'y'], fieldValueTypes: 'double', sparse: true}); + } catch (e) { + assertEqual(e.errorNum, internal.errors.ERROR_BAD_PARAMETER.code); + } + }, + + testArrayExpansions: function () { + try { + col.ensureIndex({type: 'zkd', name: 'zkdIndex', fields: ['x[*]', 'y'], fieldValueTypes: 'double'}); + } catch (e) { + assertEqual(e.errorNum, internal.errors.ERROR_BAD_PARAMETER.code); + } + } + + }; +} + +jsunity.run(optimizerRuleZkd2dIndexTestSuite); + +return jsunity.done(); diff --git a/tests/js/server/aql/aql-optimizer-zkdindex-unique.js b/tests/js/server/aql/aql-optimizer-zkdindex-unique.js new file mode 100644 index 000000000000..eb3ab7a14101 --- /dev/null +++ b/tests/js/server/aql/aql-optimizer-zkdindex-unique.js @@ -0,0 +1,116 @@ +/* global AQL_EXPLAIN, AQL_EXECUTE, fail */ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2021-2021 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Tobias Gödderz +//////////////////////////////////////////////////////////////////////////////// + +'use strict'; + +const jsunity = require("jsunity"); +const arangodb = require("@arangodb"); +const internal = require("internal"); +const db = arangodb.db; +const aql = arangodb.aql; +const {assertTrue, assertFalse, assertEqual} = jsunity.jsUnity.assertions; +const _ = require("lodash"); + +const useIndexes = 'use-indexes'; +const removeFilterCoveredByIndex = "remove-filter-covered-by-index"; +const moveFiltersIntoEnumerate = "move-filters-into-enumerate"; + +function optimizerRuleZkd2dIndexTestSuite() { + const colName = 'UnitTestZkdIndexCollection'; + let col; + + return { + setUpAll: function () { + col = db._create(colName); + col.ensureIndex({ + type: 'zkd', + name: 'zkdIndex', + fields: ['x', 'y'], + unique: true, + fieldValueTypes: 'double' + }); + // Insert 1001 points + // (-500, -499.5), (-499.1, -499.4), ..., (0, 0.5), ..., (499.9, 500.4), (500, 500.5) + db._query(aql` + FOR i IN 0..1000 + LET x = (i - 500) / 10 + LET y = x + 0.5 + INSERT {x, y, i} INTO ${col} + `); + }, + + tearDownAll: function () { + col.drop(); + }, + + testIndexAccess: function () { + const query = aql` + FOR d IN ${col} + FILTER 0 <= d.x && d.x <= 1 + RETURN d.x + `; + const explainRes = AQL_EXPLAIN(query.query, query.bindVars); + const appliedRules = explainRes.plan.rules; + const nodeTypes = explainRes.plan.nodes.map(n => n.type).filter(n => !["GatherNode", "RemoteNode"].includes(n)); + assertEqual(["SingletonNode", "IndexNode", "CalculationNode", "ReturnNode"], nodeTypes); + assertTrue(appliedRules.includes(useIndexes)); + assertTrue(appliedRules.includes(removeFilterCoveredByIndex)); + const executeRes = AQL_EXECUTE(query.query, query.bindVars); + const res = executeRes.json; + res.sort(); + assertEqual([0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1], res); + }, + + testIndexAccess2: function () { + const query = aql` + FOR d IN ${col} + FILTER 0 <= d.x && d.y <= 1 + RETURN d.x + `; + const explainRes = AQL_EXPLAIN(query.query, query.bindVars); + const appliedRules = explainRes.plan.rules; + const nodeTypes = explainRes.plan.nodes.map(n => n.type).filter(n => !["GatherNode", "RemoteNode"].includes(n)); + assertEqual(["SingletonNode", "IndexNode", "CalculationNode", "ReturnNode"], nodeTypes); + assertTrue(appliedRules.includes(useIndexes)); + assertTrue(appliedRules.includes(removeFilterCoveredByIndex)); + const executeRes = AQL_EXECUTE(query.query, query.bindVars); + const res = executeRes.json; + res.sort(); + assertEqual([0, 0.1, 0.2, 0.3, 0.4, 0.5], res); + }, + + testUniqueConstraint: function () { + col.save({x: 0, y: 0.50001}); + try { + col.save({x: 0, y: 0.5}); + fail(); + } catch (e) { + assertEqual(e.errorNum, internal.errors.ERROR_ARANGO_UNIQUE_CONSTRAINT_VIOLATED.code); + } + } + }; +} + +jsunity.run(optimizerRuleZkd2dIndexTestSuite); + +return jsunity.done(); diff --git a/tests/js/server/aql/aql-optimizer-zkdindex.js b/tests/js/server/aql/aql-optimizer-zkdindex.js new file mode 100644 index 000000000000..a75f21427856 --- /dev/null +++ b/tests/js/server/aql/aql-optimizer-zkdindex.js @@ -0,0 +1,222 @@ +/* global AQL_EXPLAIN, AQL_EXECUTE */ +//////////////////////////////////////////////////////////////////////////////// +/// DISCLAIMER +/// +/// Copyright 2021-2021 ArangoDB GmbH, Cologne, Germany +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// +/// Copyright holder is ArangoDB GmbH, Cologne, Germany +/// +/// @author Tobias Gödderz +//////////////////////////////////////////////////////////////////////////////// + +'use strict'; + +const jsunity = require("jsunity"); +const arangodb = require("@arangodb"); +const db = arangodb.db; +const aql = arangodb.aql; +const {assertTrue, assertFalse, assertEqual} = jsunity.jsUnity.assertions; +const _ = require("lodash"); + +const useIndexes = 'use-indexes'; +const removeFilterCoveredByIndex = "remove-filter-covered-by-index"; +const moveFiltersIntoEnumerate = "move-filters-into-enumerate"; + +function optimizerRuleZkd2dIndexTestSuite() { + const colName = 'UnitTestZkdIndexCollection'; + let col; + + return { + setUpAll: function () { + col = db._create(colName); + col.ensureIndex({type: 'zkd', name: 'zkdIndex', fields: ['x', 'y'], fieldValueTypes: 'double'}); + // Insert 1001 points + // (-500, -499.5), (-499.1, -499.4), ..., (0, 0.5), ..., (499.9, 500.4), (500, 500.5) + db._query(aql` + FOR i IN 0..1000 + LET x = (i - 500) / 10 + LET y = x + 0.5 + INSERT {x, y, i} INTO ${col} + `); + }, + + tearDownAll: function () { + col.drop(); + }, + + test1: function () { + const query = aql` + FOR d IN ${col} + FILTER d.i == 0 + RETURN d + `; + const res = AQL_EXPLAIN(query.query, query.bindVars); + const nodeTypes = res.plan.nodes.map(n => n.type).filter(n => !["GatherNode", "RemoteNode"].includes(n)); + const appliedRules = res.plan.rules; + assertEqual(["SingletonNode", "EnumerateCollectionNode", "ReturnNode"], nodeTypes); + assertFalse(appliedRules.includes(useIndexes)); + assertFalse(appliedRules.includes(removeFilterCoveredByIndex)); + }, + + test1_2: function () { + const query = aql` + FOR d IN ${col} + FILTER d.x >= 0 && d.i == 0 + RETURN d + `; + const res = AQL_EXPLAIN(query.query, query.bindVars); + const nodeTypes = res.plan.nodes.map(n => n.type).filter(n => !["GatherNode", "RemoteNode"].includes(n)); + const appliedRules = res.plan.rules; + assertEqual(["SingletonNode", "IndexNode", "ReturnNode"], nodeTypes); + assertTrue(appliedRules.includes(useIndexes)); + assertTrue(appliedRules.includes(removeFilterCoveredByIndex)); + assertTrue(appliedRules.includes(moveFiltersIntoEnumerate)); + }, + + test2: function () { + const query = aql` + FOR d IN ${col} + FILTER 0 <= d.x && d.x <= 1 + RETURN d.x + `; + const explainRes = AQL_EXPLAIN(query.query, query.bindVars); + const appliedRules = explainRes.plan.rules; + const nodeTypes = explainRes.plan.nodes.map(n => n.type).filter(n => !["GatherNode", "RemoteNode"].includes(n)); + assertEqual(["SingletonNode", "IndexNode", "CalculationNode", "ReturnNode"], nodeTypes); + assertTrue(appliedRules.includes(useIndexes)); + assertTrue(appliedRules.includes(removeFilterCoveredByIndex)); + const executeRes = AQL_EXECUTE(query.query, query.bindVars); + const res = executeRes.json; + res.sort(); + assertEqual([0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1], res); + }, + + test3: function () { + const query = aql` + FOR d IN ${col} + FILTER 0 <= d.x && d.y <= 1 + RETURN d.x + `; + const explainRes = AQL_EXPLAIN(query.query, query.bindVars); + const appliedRules = explainRes.plan.rules; + const nodeTypes = explainRes.plan.nodes.map(n => n.type).filter(n => !["GatherNode", "RemoteNode"].includes(n)); + assertEqual(["SingletonNode", "IndexNode", "CalculationNode", "ReturnNode"], nodeTypes); + assertTrue(appliedRules.includes(useIndexes)); + assertTrue(appliedRules.includes(removeFilterCoveredByIndex)); + const executeRes = AQL_EXECUTE(query.query, query.bindVars); + const res = executeRes.json; + res.sort(); + assertEqual([0, 0.1, 0.2, 0.3, 0.4, 0.5], res); + }, + + test4: function () { + const query = aql` + FOR d IN ${col} + FILTER 0 >= d.x + FILTER 10 <= d.y && d.y <= 16 + RETURN d.x + `; + const explainRes = AQL_EXPLAIN(query.query, query.bindVars); + const appliedRules = explainRes.plan.rules; + const nodeTypes = explainRes.plan.nodes.map(n => n.type).filter(n => !["GatherNode", "RemoteNode"].includes(n)); + assertEqual(["SingletonNode", "IndexNode", "CalculationNode", "ReturnNode"], nodeTypes); + assertTrue(appliedRules.includes(useIndexes)); + assertTrue(appliedRules.includes(removeFilterCoveredByIndex)); + const executeRes = AQL_EXECUTE(query.query, query.bindVars); + const res = executeRes.json; + res.sort(); + assertEqual([], res); + }, + + test5: function () { + const query = aql` + FOR d IN ${col} + FILTER 0 >= d.x || d.x >= 10 + FILTER d.y >= 0 && d.y <= 11 + RETURN d.x + `; + const explainRes = AQL_EXPLAIN(query.query, query.bindVars); + const appliedRules = explainRes.plan.rules; + const nodeTypes = explainRes.plan.nodes.map(n => n.type).filter(n => !["GatherNode", "RemoteNode"].includes(n)); + assertEqual(["SingletonNode", "IndexNode", "CalculationNode", "FilterNode", "CalculationNode", "ReturnNode"], nodeTypes); + assertTrue(appliedRules.includes(useIndexes)); + //assertTrue(appliedRules.includes(removeFilterCoveredByIndex)); -- TODO + const executeRes = AQL_EXECUTE(query.query, query.bindVars); + const res = executeRes.json; + res.sort(); + assertEqual([-0.1, -0.2, -0.3, -0.4, -0.5, 0, 10, 10.1, 10.2, 10.3, 10.4, 10.5].sort(), res); + }, + + test6: function () { + const query = aql` + FOR d IN ${col} + FILTER 0 == d.x + RETURN d.x + `; + const explainRes = AQL_EXPLAIN(query.query, query.bindVars); + const appliedRules = explainRes.plan.rules; + const nodeTypes = explainRes.plan.nodes.map(n => n.type).filter(n => !["GatherNode", "RemoteNode"].includes(n)); + assertEqual(["SingletonNode", "IndexNode", "CalculationNode", "ReturnNode"], nodeTypes); + assertTrue(appliedRules.includes(useIndexes)); + assertTrue(appliedRules.includes(removeFilterCoveredByIndex)); + const executeRes = AQL_EXECUTE(query.query, query.bindVars); + const res = executeRes.json; + res.sort(); + assertEqual([0].sort(), res); + }, + + test7: function () { + const query = aql` + FOR d IN ${col} + FILTER 0 == d.x && 0 == d.y + RETURN d.x + `; + const explainRes = AQL_EXPLAIN(query.query, query.bindVars); + const appliedRules = explainRes.plan.rules; + const nodeTypes = explainRes.plan.nodes.map(n => n.type).filter(n => !["GatherNode", "RemoteNode"].includes(n)); + assertEqual(["SingletonNode", "IndexNode", "CalculationNode", "ReturnNode"], nodeTypes); + assertTrue(appliedRules.includes(useIndexes)); + assertTrue(appliedRules.includes(removeFilterCoveredByIndex)); + const executeRes = AQL_EXECUTE(query.query, query.bindVars); + const res = executeRes.json; + res.sort(); + assertEqual([].sort(), res); + }, + + test8: function () { + const query = aql` + FOR d IN ${col} + FILTER 0 < d.x && d.y <= 1 + RETURN d.x + `; + const explainRes = AQL_EXPLAIN(query.query, query.bindVars); + const appliedRules = explainRes.plan.rules; + const nodeTypes = explainRes.plan.nodes.map(n => n.type).filter(n => !["GatherNode", "RemoteNode"].includes(n)); + assertEqual(["SingletonNode", "IndexNode", "CalculationNode", "ReturnNode"], nodeTypes); + assertTrue(appliedRules.includes(useIndexes)); + assertTrue(appliedRules.includes(removeFilterCoveredByIndex)); + assertTrue(appliedRules.includes(moveFiltersIntoEnumerate)); + const executeRes = AQL_EXECUTE(query.query, query.bindVars); + const res = executeRes.json; + res.sort(); + assertEqual([0.1, 0.2, 0.3, 0.4, 0.5].sort(), res); + }, + + }; +} + +jsunity.run(optimizerRuleZkd2dIndexTestSuite); + +return jsunity.done();