8000 Feature/internal issue #672 (#11370) · arangodb/arangodb@bce6f8f · GitHub
[go: up one dir, main page]
Skip to content

Commit bce6f8f

Browse files
author
Dronplane
authored
Feature/internal issue #672 (#11370)
* Tests now passes * More tests * Added compression settings * Fixed storage compression settings * reworked compression setting * added mock compressor * fixed linking * added primarySortCompression * Added tests * fix tests for mac * added primarySortCompression test * added primarySortCompression and storedValues compression to js tests. * more tests * jslint fixes * code cleanup. Jaccard function fix for empty arrays * Update CHANGELOG * Code cleanup. More tests * test fixes * fixed bug * test * adressed review comments * Fix after merge * Merge branch 'devel' into feature/internal-issue-#672 * fix build * cleanup * cleanup * fixed backslash * fix * fix typo
1 parent 0685746 commit bce6f8f

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+1522
-284
lines changed

‎3rdParty/iresearch/core/index/index_writer.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ const size_t NON_UPDATE_RECORD = irs::integer_traits<size_t>::const_max; // non-
4949

5050
const irs::column_info_provider_t DEFAULT_COLUMN_INFO = [](const irs::string_ref&) {
5151
// no compression, no encryption
52-
return irs::column_info{ irs::compression::raw::type(), {}, false };
52+
return irs::column_info{ irs::compression::none::type(), {}, false };
5353
};
5454

5555
struct flush_segment_context {

‎3rdParty/iresearch/core/utils/compression.cpp‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,7 @@ void init() {
172172
#ifndef IRESEARCH_DLL
173173
lz4::init();
174174
delta::init();
175-
raw::init();
175+
none::init();
176176
#endif
177177
}
178178

@@ -192,16 +192,16 @@ bool visit(const std::function<bool(const string_ref&)>& visitor) {
192192
// --SECTION-- raw implementation
193193
// -----------------------------------------------------------------------------
194194

195-
/*static*/ void raw::init() {
195+
/*static*/ void none::init() {
196196
#ifndef IRESEARCH_DLL
197197
// match registration below
198-
REGISTER_COMPRESSION(raw, &raw::compressor, &raw::decompressor);
198+
REGISTER_COMPRESSION(none, &none::compressor, &none::decompressor);
199199
#endif
200200
}
201201

202-
DEFINE_COMPRESSION_TYPE(iresearch::compression::raw);
202+
DEFINE_COMPRESSION_TYPE(iresearch::compression::none);
203203

204-
REGISTER_COMPRESSION(raw, &raw::compressor, &raw::decompressor);
204+
REGISTER_COMPRESSION(none, &none::compressor, &none::decompressor);
205205

206206
NS_END // compression
207207
NS_END

‎3rdParty/iresearch/core/utils/compression.hpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -210,7 +210,7 @@ IRESEARCH_API bool visit(const std::function<bool(const string_ref&)>& visitor);
210210
/// @class raw
211211
/// @brief no compression
212212
////////////////////////////////////////////////////////////////////////////////
213-
struct IRESEARCH_API raw {
213+
struct IRESEARCH_API none {
214214
DECLARE_COMPRESSION_TYPE();
215215

216216
static void init();

‎3rdParty/iresearch/tests/index/index_column_tests.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1969,7 +1969,7 @@ TEST_P(index_column_test_case, read_write_doc_attributes_sparse_column_dense_var
19691969
// sparse_column<dense_block>
19701970
irs::index_writer::init_options options;
19711971
options.column_info = [](const irs::string_ref&) {
1972-
return irs::column_info{ irs::compression::raw::type(), irs::compression::options{}, true };
1972+
return irs::column_info{ irs::compression::none::type(), irs::compression::options{}, true };
19731973
};
19741974

19751975
static const irs::doc_id_t BLOCK_SIZE = 1024;
@@ -3118,7 +3118,7 @@ TEST_P(index_column_test_case, read_write_doc_attributes_sparse_column_dense_fix
31183118

31193119
irs::index_writer::init_options options;
31203120
options.column_info = [](const irs::string_ref&) {
3121-
return irs::column_info{ irs::compression::raw::type(), irs::compression::options{}, false };
3121+
return irs::column_info{ irs::compression::none::type(), irs::compression::options{}, false };
31223122
};
31233123

31243124
// border case for sparse fixed offset columns, e.g.

‎3rdParty/iresearch/tests/index/sorted_column_test.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ TEST(sorted_column_test, insert_duplicates) {
160160

161161
writer->prepare(dir, segment);
162162

163-
irs::sorted_column col({ irs::compression::raw::type(), {}, true });
163+
irs::sorted_column col({ irs::compression::none::type(), {}, true });
164164
ASSERT_TRUE(col.empty());
165165
ASSERT_EQ(0, col.size());
166166
ASSERT_EQ(0, col.memory_active());

‎CHANGELOG‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,12 @@
11
devel
22
-----
3+
* Added JACCARD AQL function
4+
5+
* storedValues property is removed from arngosearch link properties output.
6+
7+
* Added primarySortCompression property to ArangoSearch Views
8+
9+
* Added compression property to ArangoSearch View storedValues
310

411
* Removed deprecated MMFiles storage engine and also the `arango-dfdb`
512
(datafile debugger) executable that could be used to validate MMFiles

‎arangod/Aql/Functions.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5065,7 +5065,7 @@ AqlValue Functions::Jaccard(ExpressionContext* ctx, transaction::Methods* trx,
50655065
count = 0;
50665066
}
50675067

5068-
auto const jaccard = values.empty() ? 0.0 : double_t(cardinality) / values.size();
5068+
auto const jaccard = values.empty() ? 1.0 : double_t(cardinality) / values.size();
50695069 65CE

50705070
return AqlValue{AqlValueHintDouble{jaccard}};
50715071
}

‎arangod/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ add_library(arango_iresearch
5353
IResearch/Containers.cpp IResearch/Containers.h
5454
IResearch/IResearchAnalyzerFeature.cpp IResearch/IResearchAnalyzerFeature.h
5555
IResearch/IResearchCommon.cpp IResearch/IResearchCommon.h
56+
IResearch/IResearchCompression.cpp IResearch/IResearchCompression.h
5657
IResearch/IResearchKludge.cpp IResearch/IResearchKludge.h
5758
IResearch/IResearchLink.cpp IResearch/IResearchLink.h
5859
IResearch/IResearchLinkCoordinator.cpp IResearch/IResearchLinkCoordinator.h

‎arangod/IResearch/IResearchCompression.cpp‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
////////////////////////////////////////////////////////////////////////////////
2+
/// DISCLAIMER
3+
///
4+
/// Copyright 2020 ArangoDB GmbH, Cologne, Germany
5+
///
6+
/// Licensed under the Apache License, Version 2.0 (the "License");
7+
/// you may not use this file except in compliance with the License.
8+
/// You may obtain a copy of the License at
9+
///
10+
/// http://www.apache.org/licenses/LICENSE-2.0
11+
///
12+
/// Unless required by applicable law or agreed to in writing, software
13+
/// distributed under the License is distributed on an "AS IS" BASIS,
14+
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
/// See the License for the specific language governing permissions and
16+
/// limitations under the License.
17+
///
18+
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
19+
///
20+
/// @author Andrei Lobov
21+
////////////////////////////////////////////////////////////////////////////////
22+
23+
#include "IResearchCompression.h"
24+
#include "Basics/debugging.h"
25+
#include <utils/lz4compression.hpp>
26+
#ifdef ARANGODB_USE_GOOGLE_TESTS
27+
#include "../tests/IResearch/IResearchTestCompressor.h"
28+
#endif
29+
30+
namespace arangodb {
31+
namespace iresearch {
32+
33+
irs::string_ref columnCompressionToString(irs::compression::type_id const* type) {
34+
if (ADB_UNLIKELY(type == nullptr)) {
35+
TRI_ASSERT(false);
36+
return irs::string_ref::EMPTY;
37+
}
38+
auto const& mangled_name = type->name();
39+
TRI_ASSERT(!mangled_name.empty());
40+
auto demangled_start = mangled_name.end() - 1;
41+
while (demangled_start != mangled_name.begin() && *(demangled_start-1) != ':') {
42+
demangled_start--;
43+
}
44+
return irs::string_ref(demangled_start, std::distance(demangled_start, mangled_name.end()));
45+
}
46+
47+
irs::compression::type_id const* columnCompressionFromString(irs::string_ref const& c) {
48+
TRI_ASSERT(!c.null());
49+
#ifdef ARANGODB_USE_GOOGLE_TESTS
50+
if (c == "test") {
51+
return &irs::compression::mock::test_compressor::type();
52+
}
53+
#endif
54+
if (c == "lz4") {
55+
return &irs::compression::lz4::type();
56+
}
57+
if (c == "none") {
58+
return &irs::compression::none::type();
59+
}
60+
return nullptr;
61+
}
62+
63+
irs::compression::type_id const& getDefaultCompression() {
64+
return irs::compression::lz4::type();
65+
}
66+
67+
} // namespace iresearch
68+
} // namespace arangodb
69+

‎arangod/IResearch/IResearchCompression.h‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
////////////////////////////////////////////////////////////////////////////////
2+
/// DISCLAIMER
3+
///
4+
/// Copyright 2020 ArangoDB GmbH, Cologne, Germany
5+
///
6+
/// Licensed under the Apache License, Version 2.0 (the "License");
7+
/// you may not use this file except in compliance with the License.
8+
/// You may obtain a copy of the License at
9+
///
10+
/// http://www.apache.org/licenses/LICENSE-2.0
11+
///
12+
/// Unless required by applicable law or agreed to in writing, software
13+
/// distributed under the License is distributed on an "AS IS" BASIS,
14+
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
/// See the License for the specific language governing permissions and
16+
/// limitations under the License.
17+
///
18+
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
19+
///
20+
/// @author Andrei Lobov
21+
////////////////////////////////////////////////////////////////////////////////
22+
#ifndef ARANGOD_IRESEARCH__IRESEARCH_COMPRESSION_H
23+
#define ARANGOD_IRESEARCH__IRESEARCH_COMPRESSION_H 1
24+
25+
#include "utils/string.hpp"
26+
#include <utils/compression.hpp>
27+
28+
namespace arangodb {
29+
namespace iresearch {
30+
31+
irs::string_ref columnCompressionToString(irs::compression::type_id const* type);
32+
irs::compression::type_id const* columnCompressionFromString(irs::string_ref const& c);
33+
irs::compression::type_id const& getDefaultCompression();
34+
} // iresearch
35+
} // arangodb
36+
37+
#endif // ARANGOD_IRESEARCH__IRESEARCH_COMPRESSION_H

0 commit comments

Comments
 (0)
0