8000 add test for concurrent access to the typed analyzer (#14714) · arangodb/arangodb@59964a3 · GitHub
[go: up one dir, main page]
Skip to content

Commit 59964a3

Browse files
author
Andrei Lobov
authored
add test for concurrent access to the typed analyzer (#14714)
* add test for concurrent access to the typed analyzer inside FieldIterator * Update CHANGELOG * Update CHANGELOG * fix build
1 parent 3e90e49 commit 59964a3

File tree

2 files changed

+161
-0
lines changed

2 files changed

+161
-0
lines changed

‎CHANGELOG‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
devel
22
-----
33

4+
* Fix rare case of invalid data could be inserted in the ArangoSearch index if
5+
several clients concurrently inserts data and use custom analyzer
6+
with non-string return type
7+
48
* Fix a rare shutdown race in RocksDBShaCalculatorThread.
59

610
* Added "Analyzers" view to web UI to let manage ArangoSearch analyzers

‎tests/IResearch/IResearchDocument-test.cpp‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -334,6 +334,72 @@ class TypedAnalyzer : public irs::analysis::analyzer {
334334

335335
REGISTER_ANALYZER_VPACK(TypedAnalyzer, TypedAnalyzer::make, TypedAnalyzer::normalize);
336336

337+
class TypedArrayAnalyzer : public irs::analysis::analyzer {
338+
public:
339+
static constexpr irs::string_ref type_name() noexcept {
340+
return "iresearch-document-typed-array";
341+
}
342+
343+
static ptr make(irs::string_ref const& args) {
344+
PTR_NAMED(TypedArrayAnalyzer, ptr, args);
345+
return ptr;
346+
}
347+
348+
static bool normalize(irs::string_ref const& args, std::string& out) {
349+
out.assign(args.c_str(), args.size());
350+
return true;
351+
}
352+
353+
explicit TypedArrayAnalyzer(irs::string_ref const&) : irs::analysis::analyzer(irs::type<TypedArrayAnalyzer>::get()) {
354+
_returnType.value = arangodb::iresearch::AnalyzerValueType::Number;
355+
}
356+
357+
virtual bool reset(irs::string_ref const& data) override {
358+
auto value = std::string(data);
359+
_values.clear();
360+
_current = 0;
361+
for (double d = 1; d < std::atoi(value.c_str()); ++d) {
362+
_values.push_back(d);
363+
}
364+
return true;
365+
}
366+
367+
virtual bool next() override {
368+
if (_current < _values.size()) {
369+
_typedValue = arangodb::aql::AqlValue(
370+
arangodb::aql::AqlValueHintDouble(_values[_current++]));
371+
_vpackTerm.value = _typedValue.slice();
372+
return true;
373+
} else {
374+
return false;
375+
}
376+
}
377+
378+
virtual irs::attribute* get_mutable(irs::type_info::type_id type) noexcept override {
379+
if (type == irs::type<irs::increment>::id()) {
380+
return &_inc;
381+
}
382+
if (type == irs::type<arangodb::iresearch::AnalyzerValueTypeAttribute>::id()) {
383+
return &_returnType;
384+
}
385+
if (type == irs::type<arangodb::iresearch::VPackTermAttribute>::id()) {
386+
return &_vpackTerm;
387+
}
388+
return nullptr;
389+
}
390+
391+
private:
392+
arangodb::iresearch::VPackTermAttribute _vpackTerm;
393+
irs::increment _inc;
394+
std::vector<double> _values;
395+
size_t _current{0};
396+
arangodb::iresearch::AnalyzerValueTypeAttribute _returnType;
397+
arangodb::aql::AqlValue _typedValue;
398+
};
399+
400+
REGISTER_ANALYZER_VPACK(TypedArrayAnalyzer, TypedArrayAnalyzer::make, TypedArrayAnalyzer::normalize);
401+
402+
337403
} // namespace
338404

339405
namespace std {
@@ -426,6 +492,14 @@ class IResearchDocumentTest
426492
arangodb::velocypack::Parser::fromJson("{ \"type\": \"string\" }")->slice(),
427493
arangodb::iresearch::Features{});
428494
EXPECT_TRUE(res.ok());
495+
496+
res = analyzers.emplace(
497+
result,
498+
arangodb::StaticStrings::SystemDatabase + "::iresearch-document-number-array",
499+
"iresearch-document-typed-array",
500+
arangodb::velocypack::Parser::fromJson("{ \"type\": \"number\" }")->slice(),
501+
arangodb::iresearch::Features{});
502+
EXPECT_TRUE(res.ok());
429503
}
430504
};
431505

@@ -2679,3 +2753,86 @@ TEST_F(IResearchDocumentTest, FieldIterator_dbServer_index_id_attr) {
26792753
ASSERT_TRUE(id_indexed);
26802754
}
26812755
}
2756+
2757+
TEST_F(IResearchDocumentTest, FieldIterator_concurrent_use_typed_analyzer) {
2758+
auto& sysDatabase = server.getFeature<arangodb::SystemDatabaseFeature>();
2759+
auto sysVocbase = sysDatabase.use();
2760+
auto& analyzers = server.template getFeature<arangodb::iresearch::IResearchAnalyzerFeature>();
2761+
arangodb::iresearch::IResearchLinkMeta linkMeta;
2762+
linkMeta._analyzers.clear();
2763+
linkMeta._analyzers.emplace_back(arangodb::iresearch::FieldMeta::Analyzer(
2764+
analyzers.get(arangodb::StaticStrings::SystemDatabase + "::iresearch-document-number-array",
2765+
arangodb::QueryAnalyzerRevisions::QUERY_LATEST),
2766+
"iresearch-document-number-array")); // add analyzer
2767+
ASSERT_TRUE(linkMeta._analyzers.front()._pool);
2768+
linkMeta._includeAllFields = true; // include all fields
2769+
linkMeta._primitiveOffset = linkMeta._analyzers.size();
2770+
std::string error;
2771+
std::vector<std::string> EMPTY;
2772+
arangodb::transaction::Methods trx(arangodb::transaction::StandaloneContext::Create(*sysVocbase),
2773+
EMPTY, EMPTY, EMPTY,
2774+
arangodb::transaction::Options());
2775+
arangodb::iresearch::FieldIterator it1(trx, irs::string_ref::EMPTY, arangodb::IndexId(0));
2776+
2777+
auto json = arangodb::velocypack::Parser::fromJson("{\"value\":\"3\"}");
2778+
auto json2 = arangodb::velocypack::Parser::fromJson("{\"value\":\"4\"}");
2779+
2780+
it1.reset(json->slice(), linkMeta);
2781+
ASSERT_TRUE(it1.valid());
2782+
2783+
arangodb::iresearch::FieldIterator it2(trx, irs::string_ref::EMPTY, arangodb::IndexId(0));
2784+
it2.reset(json2->slice(), linkMeta);
2785+
ASSERT_TRUE(it2.valid());
2786+
2787+
// exhaust first array member (1) of it1
2788+
{
2789+
irs::numeric_token_stream expected_tokens; expected_tokens.reset(1.);
2790+
auto& actual_tokens = (*it1).get_tokens();
2791+
auto actual_value = irs::get<irs::term_attribute>(actual_tokens);
2792+
auto expected_value = irs::get<irs::term_attribute>(expected_tokens);
2793+
while (actual_tokens.next()) {
2794+
ASSERT_TRUE(expected_tokens.next());
2795+
ASSERT_EQ(actual_value->value, expected_value->value);
2796+
}
2797+
ASSERT_FALSE(expected_tokens.next());
2798+
}
2799+
// exhaust first array member (1) of it2
2800+
{
2801+
irs::numeric_token_stream expected_tokens; expected_tokens.reset(1.);
2802+
auto& actual_tokens = (*it2).get_tokens();
2803+
auto actual_value = irs::get<irs::term_attribute>(actual_tokens);
2804+
auto expected_value = irs::get<irs::term_attribute>(expected_tokens);
2805+
while (actual_tokens.next()) {
2806+
ASSERT_TRUE(expected_tokens.next());
2807+
ASSERT_EQ(actual_value->value, expected_value->value);
2808+
}
2809+
ASSERT_FALSE(expected_tokens.next());
2810+
}
2811+
++it1; // now it1 should have it`s typed iterator pointing to 2
2812+
++it2; // now it2 should have it`s typed iterator pointing to 2 (not to 3!)
2813+
ASSERT_TRUE(it1.valid());
2814+
{
2815+
irs::numeric_token_stream expected_tokens; expected_tokens.reset(2.);
2816+
auto& actual_tokens = (*it1).get_tokens();
2817+
auto actual_value = irs::get<irs::term_attribute>(actual_tokens);
2818+
auto expected_value = irs::get<irs::term_attribute>(expected_tokens);
2819+
while (actual_tokens.next()) {
2820+
ASSERT_TRUE(expected_tokens.next());
2821+
ASSERT_EQ(actual_value->value, expected_value->value);
2822+
}
2823+
ASSERT_FALSE(expected_tokens.next());
2824+
}
2825+
ASSERT_TRUE(it2.valid());
2826+
{
2827+
irs::numeric_token_stream expected_tokens; expected_tokens.reset(2.);
2828+
auto& actual_tokens = (*it2).get_tokens();
2829+
auto actual_value = irs::get<irs::term_attribute>(actual_tokens);
2830+
auto expected_value = irs::get<irs::term_attribute>(expected_tokens);
2831+
while (actual_tokens.next()) {
2832+
ASSERT_TRUE(expected_tokens.next());
2833+
ASSERT_EQ(actual_value->value, expected_value->value);
2834+
}
2835+
ASSERT_FALSE(expected_tokens.next());
2836+
}
2837+
}
2838+

0 commit comments

Comments
 (0)
0