update iresearch from upstream

arangodb · gnusi · Aug 18, 2021 · Jul 20, 2021 · Jul 20, 2021 · Jul 21, 2021
commit a5642180101004a84142d770fc9e89db04cb512c
diff --git a/3rdParty/iresearch/core/analysis/collation_token_stream.cpp b/3rdParty/iresearch/core/analysis/collation_token_stream.cpp
@@ -305,9 +305,16 @@ bool collation_token_stream::reset(const string_ref& data) {
   const icu::UnicodeString icu_token = icu::UnicodeString::fromUTF8(
     icu::StringPiece(data_utf8_ref.c_str(), static_cast<int32_t>(data_utf8_ref.size())));
 
-  const int32_t term_size = state_->collator->getSortKey(
+  int32_t term_size = state_->collator->getSortKey(
     icu_token, state_->term_buf, sizeof state_->term_buf);
-  assert(term_size >= 0);
+
+  // https://unicode-org.github.io/icu-docs/apidoc/released/icu4c/classicu_1_1Collator.html
+  // according to ICU docs sort keys are always zero-terminated,
+  // there is no reason to store terminal zero in term dictionary
+  assert(term_size > 0);
+
+  --term_size;
+  assert(0 == state_->term_buf[term_size]);
 
   if (term_size > static_cast<int32_t>(sizeof state_->term_buf)) {
     IR_FRMT_ERROR(

diff --git a/3rdParty/iresearch/tests/analysis/collation_token_stream_test.cpp b/3rdParty/iresearch/tests/analysis/collation_token_stream_test.cpp
@@ -88,7 +88,7 @@ TEST(collation_token_stream_test, check_tokens_utf8) {
     EXPECT_NE(nullptr, p);
     EXPECT_NE(0, size);
 
-    return { p, static_cast<size_t>(size) };
+    return { p, static_cast<size_t>(size)-1 };
   };
 
   {
@@ -105,6 +105,28 @@ TEST(collation_token_stream_test, check_tokens_utf8) {
     auto* inc = irs::get<irs::increment>(*stream);
     ASSERT_NE(nullptr, inc);
 
+    {
+      const irs::string_ref data{irs::string_ref::NIL};
+      ASSERT_TRUE(stream->reset(data));
+      ASSERT_TRUE(stream->next());
+      ASSERT_EQ(0, offset->start);
+      ASSERT_EQ(data.size(), offset->end);
+      ASSERT_EQ(get_collation_key(data), term->value);
+      ASSERT_EQ(1, inc->value);
+      ASSERT_FALSE(stream->next());
+    }
+
+    {
+      const irs::string_ref data{irs::string_ref::EMPTY};
+      ASSERT_TRUE(stream->reset(data));
+      ASSERT_TRUE(stream->next());
+      ASSERT_EQ(0, offset->start);
+      ASSERT_EQ(data.size(), offset->end);
+      ASSERT_EQ(get_collation_key(data), term->value);
+      ASSERT_EQ(1, inc->value);
+      ASSERT_FALSE(stream->next());
+    }
+
     {
       constexpr irs::string_ref data{"quick"};
       ASSERT_TRUE(stream->reset(data));
@@ -170,7 +192,7 @@ TEST(collation_token_stream_test, check_tokens) {
     EXPECT_NE(nullptr, p);
     EXPECT_NE(0, size);
 
-    return { p, static_cast<size_t>(size) };
+    return { p, static_cast<size_t>(size)-1 };
   };
 
   {

diff --git a/tests/js/common/aql/aql-view-arangosearch-feature.js b/tests/js/common/aql/aql-view-arangosearch-feature.js
@@ -1892,11 +1892,11 @@ function iResearchFeatureAqlTestSuite () {
             null,
             { }).toArray();
           assertEqual(1, result.length);
-          assertEqual(17, result[0][0].length);
+          assertEqual(16, result[0][0].length);
 
           const expected = [
             39, 6, 65533, 6, 120, 16, 65533, 26, 12,
-            1, 12, 1, 65533, 65533, 65533, 65533, 0 ];
+            1, 12, 1, 65533, 65533, 65533, 65533 ];
 
           assertTrue(
             _.isEqual(expected, result[0][0].split('').map(c => c.charCodeAt(0))));