arangodb · gnusi · Apr 3, 2020 · Mar 25, 2020 · Mar 25, 2020 · Mar 25, 2020
diff --git a/3rdParty/iresearch/core/index/index_writer.cpp b/3rdParty/iresearch/core/index/index_writer.cpp
@@ -49,7 +49,7 @@ const size_t NON_UPDATE_RECORD = irs::integer_traits<size_t>::const_max; // non-
 
 const irs::column_info_provider_t DEFAULT_COLUMN_INFO = [](const irs::string_ref&) {
   // no compression, no encryption
-  return irs::column_info{ irs::compression::raw::type(), {}, false };
+  return irs::column_info{ irs::compression::none::type(), {}, false };
 };
 
 struct flush_segment_context {

diff --git a/3rdParty/iresearch/core/utils/compression.cpp b/3rdParty/iresearch/core/utils/compression.cpp
@@ -172,7 +172,7 @@ void init() {
 #ifndef IRESEARCH_DLL
   lz4::init();
   delta::init();
-  raw::init();
+  none::init();
 #endif
 }
 
@@ -192,16 +192,16 @@ bool visit(const std::function<bool(const string_ref&)>& visitor) {
 // --SECTION--                                                raw implementation
 // -----------------------------------------------------------------------------
 
-/*static*/ void raw::init() {
+/*static*/ void none::init() {
 #ifndef IRESEARCH_DLL
   // match registration below
-  REGISTER_COMPRESSION(raw, &raw::compressor, &raw::decompressor);
+  REGISTER_COMPRESSION(none, &none::compressor, &none::decompressor);
 #endif
 }
 
-DEFINE_COMPRESSION_TYPE(iresearch::compression::raw);
+DEFINE_COMPRESSION_TYPE(iresearch::compression::none);
 
-REGISTER_COMPRESSION(raw, &raw::compressor, &raw::decompressor);
+REGISTER_COMPRESSION(none, &none::compressor, &none::decompressor);
 
 NS_END // compression
 NS_END
diff --git a/3rdParty/iresearch/core/utils/compression.hpp b/3rdParty/iresearch/core/utils/compression.hpp
@@ -210,7 +210,7 @@ IRESEARCH_API bool visit(const std::function<bool(const string_ref&)>& visitor);
 /// @class raw
 /// @brief no compression
 ////////////////////////////////////////////////////////////////////////////////
-struct IRESEARCH_API raw {
+struct IRESEARCH_API none {
   DECLARE_COMPRESSION_TYPE();
 
   static void init();

diff --git a/3rdParty/iresearch/tests/index/index_column_tests.cpp b/3rdParty/iresearch/tests/index/index_column_tests.cpp
@@ -1969,7 +1969,7 @@ TEST_P(index_column_test_case, read_write_doc_attributes_sparse_column_dense_var
   // sparse_column<dense_block>
   irs::index_writer::init_options options;
   options.column_info = [](const irs::string_ref&) {
-    return irs::column_info{ irs::compression::raw::type(), irs::compression::options{}, true };
+    return irs::column_info{ irs::compression::none::type(), irs::compression::options{}, true };
   };
 
   static const irs::doc_id_t BLOCK_SIZE = 1024;
@@ -3118,7 +3118,7 @@ TEST_P(index_column_test_case, read_write_doc_attributes_sparse_column_dense_fix
 
   irs::index_writer::init_options options;
   options.column_info = [](const irs::string_ref&) {
-    return irs::column_info{ irs::compression::raw::type(), irs::compression::options{}, false };
+    return irs::column_info{ irs::compression::none::type(), irs::compression::options{}, false };
   };
 
   // border case for sparse fixed offset columns, e.g.

diff --git a/3rdParty/iresearch/tests/index/sorted_column_test.cpp b/3rdParty/iresearch/tests/index/sorted_column_test.cpp
@@ -160,7 +160,7 @@ TEST(sorted_column_test, insert_duplicates) {
 
     writer->prepare(dir, segment);
 
-    irs::sorted_column col({ irs::compression::raw::type(), {}, true });
+    irs::sorted_column col({ irs::compression::none::type(), {}, true });
     ASSERT_TRUE(col.empty());
     ASSERT_EQ(0, col.size());
     ASSERT_EQ(0, col.memory_active());

diff --git a/CHANGELOG b/CHANGELOG
@@ -1,5 +1,12 @@
 devel
 -----
+* Added JACCARD AQL function
+
+* storedValues property is removed from arngosearch link properties output.
+
+* Added primarySortCompression property to ArangoSearch Views
+
+* Added compression property to ArangoSearch View storedValues
 
 * Removed deprecated MMFiles storage engine and also the `arango-dfdb`
   (datafile debugger) executable that could be used to validate MMFiles

diff --git a/arangod/Aql/Functions.cpp b/arangod/Aql/Functions.cpp
@@ -5065,7 +5065,7 @@ AqlValue Functions::Jaccard(ExpressionContext* ctx, transaction::Methods* trx,
     count = 0;
   }
 
-  auto const jaccard = values.empty() ? 0.0 : double_t(cardinality) / values.size();
+  auto const jaccard = values.empty() ? 1.0 : double_t(cardinality) / values.size();
 
   return AqlValue{AqlValueHintDouble{jaccard}};
 }

diff --git a/arangod/CMakeLists.txt b/arangod/CMakeLists.txt
@@ -53,6 +53,7 @@ add_library(arango_iresearch
   IResearch/Containers.cpp IResearch/Containers.h
   IResearch/IResearchAnalyzerFeature.cpp IResearch/IResearchAnalyzerFeature.h
   IResearch/IResearchCommon.cpp IResearch/IResearchCommon.h
+  IResearch/IResearchCompression.cpp IResearch/IResearchCompression.h
   IResearch/IResearchKludge.cpp IResearch/IResearchKludge.h
   IResearch/IResearchLink.cpp IResearch/IResearchLink.h
   IResearch/IResearchLinkCoordinator.cpp IResearch/IResearchLinkCoordinator.h

diff --git a/arangod/IResearch/IResearchCompression.cpp b/arangod/IResearch/IResearchCompression.cpp
@@ -0,0 +1,69 @@
+////////////////////////////////////////////////////////////////////////////////
+/// DISCLAIMER
+///
+/// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Andrei Lobov
+////////////////////////////////////////////////////////////////////////////////
+
+#include "IResearchCompression.h"
+#include "Basics/debugging.h"
+#include <utils/lz4compression.hpp>
+#ifdef ARANGODB_USE_GOOGLE_TESTS
+#include "../tests/IResearch/IResearchTestCompressor.h"
+#endif
+
+namespace arangodb {
+namespace iresearch {
+
+irs::string_ref columnCompressionToString(irs::compression::type_id const* type) {
+  if (ADB_UNLIKELY(type == nullptr)) {
+    TRI_ASSERT(false);
+    return irs::string_ref::EMPTY;
+  }
+  auto const& mangled_name = type->name();
+  TRI_ASSERT(!mangled_name.empty());
+  auto demangled_start = mangled_name.end() - 1;
+  while (demangled_start != mangled_name.begin() && *(demangled_start-1) != ':') {
+    demangled_start--;
+  }
+  return irs::string_ref(demangled_start, std::distance(demangled_start, mangled_name.end()));
+}
+
+irs::compression::type_id const*  columnCompressionFromString(irs::string_ref const& c) {
+  TRI_ASSERT(!c.null());
+#ifdef ARANGODB_USE_GOOGLE_TESTS
+  if (c == "test") {
+    return &irs::compression::mock::test_compressor::type();
+  }
+#endif
+  if (c == "lz4") {
+    return &irs::compression::lz4::type();
+  }
+  if (c == "none") {
+    return &irs::compression::none::type();
+  } 
+  return nullptr;
+}
+
+irs::compression::type_id const& getDefaultCompression() {
+  return irs::compression::lz4::type();
+}
+
+}  // namespace iresearch
+}  // namespace arangodb
+
diff --git a/arangod/IResearch/IResearchCompression.h b/arangod/IResearch/IResearchCompression.h
@@ -0,0 +1,37 @@
+////////////////////////////////////////////////////////////////////////////////
+/// DISCLAIMER
+///
+/// Copyright 2020 ArangoDB GmbH, Cologne, Germany
+///
+/// Licensed under the Apache License, Version 2.0 (the "License");
+/// you may not use this file except in compliance with the License.
+/// You may obtain a copy of the License at
+///
+///     http://www.apache.org/licenses/LICENSE-2.0
+///
+/// Unless required by applicable law or agreed to in writing, software
+/// distributed under the License is distributed on an "AS IS" BASIS,
+/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+/// See the License for the specific language governing permissions and
+/// limitations under the License.
+///
+/// Copyright holder is ArangoDB GmbH, Cologne, Germany
+///
+/// @author Andrei Lobov
+////////////////////////////////////////////////////////////////////////////////
+#ifndef ARANGOD_IRESEARCH__IRESEARCH_COMPRESSION_H
+#define ARANGOD_IRESEARCH__IRESEARCH_COMPRESSION_H 1
+
+#include "utils/string.hpp"
+#include <utils/compression.hpp>
+
+namespace arangodb {
+namespace iresearch {
+
+irs::string_ref columnCompressionToString(irs::compression::type_id const* type);
+irs::compression::type_id const* columnCompressionFromString(irs::string_ref const& c);
+irs::compression::type_id const& getDefaultCompression();
+} // iresearch
+} // arangodb
+
+#endif // ARANGOD_IRESEARCH__IRESEARCH_COMPRESSION_H
diff --git a/arangod/IResearch/IResearchFeature.cpp b/arangod/IResearch/IResearchFeature.cpp
@@ -392,7 +392,7 @@ bool upgradeSingleServerArangoSearchView0_1(
 
 void registerFilters(arangodb::aql::AqlFunctionFeature& functions) {
   using arangodb::iresearch::addFunction;
-  
+
   auto flags =
       arangodb::aql::Function::makeFlags(arangodb::aql::Function::Flags::Deterministic,
                                          arangodb::aql::Function::Flags::Cacheable,

diff --git a/arangod/IResearch/IResearchFilterFactory.cpp b/arangod/IResearch/IResearchFilterFactory.cpp
@@ -2742,7 +2742,7 @@ arangodb::Result fromFuncNgramMatch(
     kludge::mangleStringField(name, analyzerPool);
 
     auto& ngramFilter = filter->add<irs::by_ngram_similarity>();
-    ngramFilter.field(st
7398
d::move(name)).threshold(threshold).boost(filterCtx.boost);;
+    ngramFilter.field(std::move(name)).threshold((float_t)threshold).boost(filterCtx.boost);;
 
     analyzer->reset(matchValue);
     irs::term_attribute const& token = *analyzer->attributes().get<irs::term_attribute>();

diff --git a/arangod/IResearch/IResearchLink.cpp b/arangod/IResearch/IResearchLink.cpp
@@ -25,7 +25,6 @@
 #include <store/mmap_directory.hpp>
 #include <store/store_utils.hpp>
 #include <utils/encryption.hpp>
-#include <utils/lz4compression.hpp>
 #include <utils/singleton.hpp>
 
 #include "IResearchLink.h"
@@ -37,6 +36,7 @@
 #include "Cluster/ClusterFeature.h"
 #include "Cluster/ClusterInfo.h"
 #include "IResearch/IResearchCommon.h"
+#include "IResearch/IResearchCompression.h"
 #include "IResearch/IResearchFeature.h"
 #include "IResearch/IResearchLinkHelper.h"
 #include "IResearch/IResearchPrimaryKeyFilter.h"
@@ -247,8 +247,7 @@ bool readTick(irs::bytes_ref const& payload, TRI_voc_tick_t& tick) noexcept {
   static_assert(
     // cppcheck-suppress duplicateExpression
     sizeof(uint64_t) == sizeof(TRI_voc_tick_t),
-    "sizeof(uint64_t) != sizeof(TRI_voc_tick_t)"
-  );
+    "sizeof(uint64_t) != sizeof(TRI_voc_tick_t)");
 
   if (payload.size() != sizeof(uint64_t)) {
     return false;
@@ -259,7 +258,6 @@ bool readTick(irs::bytes_ref const& payload, TRI_voc_tick_t& tick) noexcept {
 
   return true;
 }
-
 }  // namespace
 
 namespace arangodb {
@@ -305,12 +303,12 @@ IResearchLink::IResearchLink(arangodb::IndexId iid, LogicalCollection& collectio
   _trxCallback = [key](transaction::Methods& trx, transaction::Status status)->void {
     auto* state = trx.state();
     TRI_ASSERT(state != nullptr);
-    
+
     // check state of the top-most transaction only
     if (!state || !state->isTopLevelTransaction()) {
       return;  // NOOP
     }
-    
+
     auto prev = state->cookie(key, nullptr);  // get existing cookie
 
     if (prev) {
@@ -819,7 +817,9 @@ Result IResearchLink::init(
   auto viewId = definition.get(StaticStrings::ViewIdField).copyString();
   auto& vocbase = _collection.vocbase();
   bool const sorted = !meta._sort.empty();
-
+  auto const& storedValuesColumns = meta._storedValues.columns();
+  TRI_ASSERT(meta._sortCompression);
+  auto const& primarySortCompression = meta._sortCompression? *meta._sortCompression : getDefaultCompression();
   if (ServerState::ins
7398
tance()->isCoordinator()) { // coordinator link
     if (!vocbase.server().hasFeature<arangodb::ClusterFeature>()) {
       return {
@@ -874,7 +874,7 @@ Result IResearchLink::init(
     if (!clusterWideLink) {
       // prepare data-store which can then update options
       // via the IResearchView::link(...) call
-      auto const res = initDataStore(initCallback, sorted);
+      auto const res = initDataStore(initCallback, sorted, storedValuesColumns, primarySortCompression);
 
       if (!res.ok()) {
         return res;
@@ -911,7 +911,7 @@ Result IResearchLink::init(
         // missing links will be populated when they are created in the
         // per-shard collection
         if (shardIds) {
-          for (auto& entry: *shardIds) {
+          for (auto& entry : *shardIds) {
             auto collection = vocbase.lookupCollection(entry.first); // per-shard collections are always in 'vocbase'
 
             if (!collection) {
@@ -942,7 +942,7 @@ Result IResearchLink::init(
   } else if (ServerState::instance()->isSingleServer()) {  // single-server link
     // prepare data-store which can then update options
     // via the IResearchView::link(...) call
-    auto const res = initDataStore(initCallback, sorted);
+    auto const res = initDataStore(initCallback, sorted, storedValuesColumns, primarySortCompression);
 
     if (!res.ok()) {
       return res;
@@ -989,14 +989,17 @@ Result IResearchLink::init(
   return {};
 }
 
-Result IResearchLink::initDataStore(InitCallback const& initCallback, bool sorted) {
+Result IResearchLink::initDataStore(
+    InitCallback const& initCallback, bool sorted,
+    std::vector<IResearchViewStoredValues::StoredColumn> const& storedColumns,
+    irs::compression::type_id const& primarySortCompression) {
   _asyncTerminate.store(true); // mark long-running async jobs for terminatation
 
   if (_asyncFeature) {
     _asyncFeature->asyncNotify(); // trigger reload of settings for async jobs
   }
 
-  _flushSubscription.reset() ; // reset together with '_asyncSelf'
+  _flushSubscription.reset(); // reset together with '_asyncSelf'
   _asyncSelf->reset(); // the data-store is being deallocated, link use is no longer valid (wait for all the view users to finish)
 
   auto& server = _collection.vocbase().server();
@@ -1094,7 +1097,6 @@ Result IResearchLink::initDataStore(InitCallback const& initCallback, bool sorte
           << "link '" << id() << "', docs count '" << _dataStore._reader->docs_count()
           << "', live docs count '" << _dataStore._reader->live_docs_count()
           << "', recovery tick '" << _dataStore._recoveryTick << "'";
-
     } catch (irs::index_not_found const&) {
       // NOOP
     }
@@ -1108,18 +1110,33 @@ Result IResearchLink::initDataStore(InitCallback const& initCallback, bool sorte
   options.lock_repository = false; // do not lock index, ArangoDB has its own lock
   options.comparator = sorted ? &_comparer : nullptr; // set comparator if requested
 
+  // as meta is still not filled at this moment
+  // we need to store all compression mapping there
+  // as values provided may be temporary
+  std::map<std::string, irs::compression::type_id const&> compressionMap;
+  for (auto c : storedColumns) {
+    if (ADB_LIKELY(c.compression != nullptr)) {
+      compressionMap.emplace(c.name, *c.compression);
+    } else {
+      TRI_ASSERT(false);
+      compressionMap.emplace(c.name, getDefaultCompression());
+    }
+  }
   // setup columnstore compression/encryption if requested by storage engine
   auto const encrypt = (nullptr != irs::get_encryption(_dataStore._directory->attributes()));
-  if (encrypt) {
-    options.column_info = [](const irs::string_ref& name) -> irs::column_info {
-      // do not waste resources to encrypt primary key column
-      return { irs::compression::lz4::type(), {}, DocumentPrimaryKey::PK() != name };
-    };
-  } else {
-    options.column_info = [](const irs::string_ref& /*name*/) -> irs::column_info {
-      return { irs::compression::lz4::type(), {}, false };
+  options.column_info =
+    [encrypt, comprMap = std::move(compressionMap), &primarySortCompression](const irs::string_ref& name) -> irs::column_info {
+      if (name.null()) {
+        return { primarySortCompression, {}, encrypt };
+      }
+      auto compress = comprMap.find(name);
+      if (compress != comprMap.end()) {
+        // do not waste resources to encrypt primary key column
+        return { compress->second, {}, encrypt && (DocumentPrimaryKey::PK() != name) };
+      } else {
+        return { getDefaultCompression(), {}, encrypt && (DocumentPrimaryKey::PK() != name) };
+      }
     };
-  }
 
   auto openFlags = irs::OM_APPEND;
   if (!_dataStore._reader) {
@@ -1722,7 +1739,7 @@ Result IResearchLink::unload() {
     _asyncFeature->asyncNotify(); // trigger reload of settings for async jobs
   }
 
-  _flushSubscription.reset() ; // reset together with '_asyncSelf'
+  _flushSubscription.reset(); // reset together with '_asyncSelf'
   _asyncSelf->reset(); // the data-store is being deallocated, link use is no longer valid (wait for all the view users to finish)
 
   try {