8000 Merge branch 'devel' of https://github.com/arangodb/arangodb into bug… · arangodb/arangodb@2076d3f · GitHub
[go: up one dir, main page]
Skip to content

Commit 2076d3f

Browse files
ObiWahnObiWahn
committed
Merge branch 'devel' of https://github.com/arangodb/arangodb into bug-fix/validation-fixes-and-improvements
* 'devel' of https://github.com/arangodb/arangodb: (25 commits) Do not instantiate snipped if not collection is found on the server. (#11281) Add entries related to search features in 3.7 fix bug (#11279) Docs: Add DocuBlocks for document validation. (#11228) Feature/ngram similarity function (#11276) Fixed production check, removed assertion (#11273) fix compile warning Cluster Metrics (#11234) Feature/satellite graphs (#11015) fix newly created supervision bug with incremental updates (#11269) remove useless std::cout output fix yet more compile warnings Implement memory detection override. (#11268) implement a global deadline when running testcode in the local arangosh (#11123) Encryption key rotation (#11080) fix compile warnings fix compile warnings Feature/aql subquery execution block impl execute implementation (#10606) missing metrics (#10625) Bug fix/supervision server cleanup (#11187) ...
2 parents a1155cb + 98efcbc commit 2076d3f

File tree

356 files changed

+29956
-11468
lines changed
  • common
  • server/modules/@arangodb
  • lib
  • tests

    • Some content is hidden

    Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

    356 files changed

    +29956
    -11468
    lines changed

    ‎3rdParty/iresearch/core/search/wildcard_filter.cpp

    Lines changed: 29 additions & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -31,6 +31,27 @@
    3131
    #include "utils/automaton_utils.hpp"
    3232
    #include "utils/hash_utils.hpp"
    3333

    34+
    NS_LOCAL
    35+
    36+
    inline irs::bytes_ref unescape(const irs::bytes_ref& in, irs::bstring& out) {
    37+
    out.reserve(in.size());
    38+
    39+
    bool copy = true;
    40+
    std::copy_if(in.begin(), in.end(), std::back_inserter(out),
    41+
    [&copy](irs::byte_type c) {
    42+
    if (c == irs::WildcardMatch::ESCAPE) {
    43+
    copy = !copy;
    44+
    } else {
    45+
    copy = true;
    46+
    }
    47+
    return copy;
    48+
    });
    49+
    50+
    return out;
    51+
    }
    52+
    53+
    NS_END
    54+
    3455
    NS_ROOT
    3556

    3657
    DEFINE_FILTER_TYPE(by_wildcard)
    @@ -41,17 +62,24 @@ DEFINE_FACTORY_DEFAULT(by_wildcard)
    4162
    const order::prepared& order,
    4263
    boost_t boost,
    4364
    const string_ref& field,
    44-
    const bstring& term,
    65+
    bytes_ref term,
    4566
    size_t scored_terms_limit) {
    67+
    bstring buf;
    4668
    switch (wildcard_type(term)) {
    4769
    case WildcardType::INVALID:
    4870
    return prepared::empty();
    71+
    case WildcardType::TERM_ESCAPED:
    72+
    term = unescape(term, buf);
    73+
    [[fallthrough]];
    4974
    case WildcardType::TERM:
    5075
    return term_query::make(index, order, boost, field, term);
    5176
    case WildcardType::MATCH_ALL:
    5277
    return by_prefix::prepare(index, order, boost, field,
    5378
    bytes_ref::EMPTY, // empty prefix == match all
    5479
    scored_terms_limit);
    80+
    case WildcardType::PREFIX_ESCAPED:
    81+
    term = unescape(term, buf);
    82+
    [[fallthrough]];
    5583
    case WildcardType::PREFIX: {
    5684
    assert(!term.empty());
    5785
    const auto* begin = term.c_str();

    ‎3rdParty/iresearch/core/search/wildcard_filter.hpp

    Lines changed: 1 addition & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -43,7 +43,7 @@ class IRESEARCH_API by_wildcard final : public by_prefix {
    4343
    const order::prepared& order,
    4444
    boost_t boost,
    4545
    const string_ref& field,
    46-
    const bstring& term,
    46+
    bytes_ref term,
    4747
    size_t scored_terms_limit);
    4848

    4949
    explicit by_wildcard() noexcept;

    ‎3rdParty/iresearch/core/utils/wildcard_utils.cpp

    Lines changed: 8 additions & 2 deletions
    Original file line numberDiff line numberDiff line change
    @@ -32,6 +32,7 @@ WildcardType wildcard_type(const bytes_ref& expr) noexcept {
    3232
    }
    3333

    3434
    bool escaped = false;
    35+
    bool seen_escaped = false;
    3536
    size_t num_match_any_string = 0;
    3637
    size_t num_adjacent_match_any_string = 0;
    3738

    @@ -50,17 +51,20 @@ WildcardType wildcard_type(const bytes_ref& expr) noexcept {
    5051
    case WildcardMatch::ANY_STRING:
    5152
    num_adjacent_match_any_string += size_t(!escaped);
    5253
    num_match_any_string += size_t(!escaped);
    54+
    seen_escaped |= escaped;
    5355
    escaped = false;
    5456
    break;
    5557
    case WildcardMatch::ANY_CHAR:
    5658
    if (!escaped) {
    5759
    return WildcardType::WILDCARD;
    5860
    }
    61+
    seen_escaped = true;
    5962
    num_adjacent_match_any_string = 0;
    6063
    escaped = false;
    6164
    break;
    6265
    case WildcardMatch::ESCAPE:
    6366
    num_adjacent_match_any_string = 0;
    67+
    seen_escaped |= escaped;
    6468
    escaped = !escaped;
    6569
    break;
    6670
    default:
    @@ -73,15 +77,17 @@ WildcardType wildcard_type(const bytes_ref& expr) noexcept {
    7377
    }
    7478

    7579
    if (0 == num_match_any_string) {
    76-
    return WildcardType::TERM;
    80+
    return seen_escaped ? WildcardType::TERM_ESCAPED
    81+
    : WildcardType::TERM;
    7782
    }
    7883

    7984
    if (expr.size() == num_match_any_string) {
    8085
    return WildcardType::MATCH_ALL;
    8186
    }
    8287

    8388
    if (num_match_any_string == num_adjacent_match_any_string) {
    84-
    return WildcardType::PREFIX;
    89+
    return seen_escaped ? WildcardType::PREFIX_ESCAPED
    90+
    : WildcardType::PREFIX;
    8591
    }
    8692

    8793
    return WildcardType::WILDCARD;

    ‎3rdParty/iresearch/core/utils/wildcard_utils.hpp

    Lines changed: 7 additions & 5 deletions
    Original file line numberDiff line numberDiff line change
    @@ -28,11 +28,13 @@
    2828
    NS_ROOT
    2929

    3030
    enum class WildcardType {
    31-
    INVALID = 0, // invalid input sequence
    32-
    TERM, // foo
    33-
    MATCH_ALL, // *
    34-
    PREFIX, // foo*
    35-
    WILDCARD // f_o*
    31+
    INVALID = 0, // invalid input sequence
    32+
    TERM_ESCAPED, // f\*o
    33+
    TERM, // foo
    34+
    MATCH_ALL, // *
    35+
    PREFIX_ESCAPED, // fo\*
    36+
    PREFIX, // foo*
    37+
    WILDCARD // f_o*
    3638
    };
    3739

    3840
    IRESEARCH_API WildcardType wildcard_type(const bytes_ref& pattern) noexcept;

    ‎CHANGELOG

    Lines changed: 34 additions & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -1,13 +1,46 @@
    11
    devel
    22
    -----
    33

    4+
    * Added ability to store values in ArangoSearch views.
    5+
    6+
    * Added LIKE operator/function support to SEARCH clause.
    7+
    8+
    * Added NGRAM_SIMILARITY and NGRAM_POSITIONAL_SIMILARITY functions for
    9+
    calculating Ngram similarity.
    10+
    11+
    * Added ngram fuzzy search. Supported by NGRAM_MATCH filter function for SEARCH
    12+
    clause.
    13+
    14+
    * Allow to override the detected total amount of memory via an environment
    15+
    variable ARANGODB_OVERRIDE_DETECTED_TOTAL_MEMORY.
    16+
    17+
    * `splice-subqueries` optimization is not limited by any type of operation within the
    18+
    subquery any more. It can now be applied on every subquery and will be by default.
    19+
    However they may be a performance impact on some queries where splice-subqueries
    20+
    are not as performant as non-spliced subqueries. This is due to internal memory
    21+
    management right now and will be addressed in future versions. Spliced subqueries
    22+
    can be less performant if the query around the subquery is complex and requires
    23+
    lots of variables, or variables with large content, but the subquery itself
    24+
    does not require a lot of variables and produces many intermediate results
    25+
    s.t. good batching within the query does not pay off against memory overhead.
    26+
    27+
    * Supervision to clean up zombie servers after 24h, if no
    28+
    responsibility for shards.
    29+
    30+
    * Fix SORT RAND() LIMIT 1 optimization for RocksDB when only a projection of the
    31+
    attributes was used. When a projection was used and that projection was
    32+
    covered by an index (e.g. `_key` via the primary index), then the access
    33+
    pattern was transformed from random order collection seek to an index access,
    34+
    which always resulted in the same index entry to be returned and not a random
    35+
    one.
    36+
    437
    * Mark server startup options `--foxx.*`, `--frontend.*` and `--javascript.*`
    538
    as single server and Coordinator only for documentation (`--dump-options`).
    639

    740
    * Supervision hot backup and supervision maintenance modes ttl fix.
    841

    942
    * Fix a bug that leads to graph traversals yielding empty output when none of the
    10-
    output variables (vertex, edge, path) are used. This is relevant when a query
    43+
    output variables (vertex, edge, path) are used. This is relevant when a query
    1144
    is only interested in a COUNT of the outputs, for example.
    1245

    1346
    * MoveShard to check, if target is in sync follower before promotion

    ‎Documentation/DocuBlocks/Rest/Administration/post_admin_server_encryption.md

    Lines changed: 23 additions & 0 deletions
    Original file line numberDiff line numberDiff line change
    @@ -0,0 +1,23 @@
    1+
    2+
    @startDocuBlock post_admin_server_encryption
    3+
    @brief Rotate encryption at rest key
    4+
    5+
    @RESTHEADER{POST /_admin/server/encryption, Rotate the encryption at rest key, handleEncryption:post}
    6+
    7+
    @RESTDESCRIPTION
    8+
    Change the user supplied encryption at rest key by sending a request without
    9+
    payload to this endpoint. The file supplied via `--rocksdb.encryption-keyfile`
    10+
    will be reloaded and the internal encryption key will be re-encrypted with the
    11+
    new user key.
    12+
    13+
    This is a protected API and can only be executed with superuser rights.
    14+
    15+
    @RESTRETURNCODES
    16+
    17+
    @RESTRETURNCODE{200}
    18+
    This API will return HTTP 200 if everything is ok
    19+
    20+
    @RESTRETURNCODE{403}
    21+
    This API will return HTTP 403 FORBIDDEN if it is not called with
    22+
    superuser rights.
    23+
    @endDocuBlock

    ‎Documentation/DocuBlocks/Rest/Collections/post_api_collection.md

    Lines changed: 6 additions & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -54,12 +54,17 @@ This option is meaningful for the MMFiles storage engine only.
    5454
    additional options for key generation. If specified, then *keyOptions*
    5555
    should be a JSON array containing the following attributes:
    5656

    57+
    @RESTBODYPARAM{validation,object,optional,post_api_collection_opts}
    58+
    Optional object that specifies the collection level schema validation for
    59+
    documents. The attribute keys `rule`, `level` and `message` must follow the
    60+
    rules documented in [Document Schema Validation](https://www.arangodb.com/docs/devel/document-schema-validation.html)
    61+
    5762
    @RESTSTRUCT{type,post_api_collection_opts,string,required,string}
    5863
    specifies the type of the key generator. The currently available generators are
    5964
    *traditional*, *autoincrement*, *uuid* and *padded*.<br>
    6065
    The *traditional* key generator generates numerical keys in ascending order.
    6166
    The *autoincrement* key generator generates numerical keys in ascending order,
    62-
    the inital offset and the spacing can be configured
    67+
    the initial offset and the spacing can be configured
    6368
    The *padded* key generator generates keys of a fixed length (16 bytes) in
    6469
    ascending lexicographical sort order. This is ideal for usage with the _RocksDB_
    6570
    engine, which will slightly benefit keys that are inserted in lexicographically

    ‎Documentation/DocuBlocks/Rest/Collections/put_api_collection_properties.md

    Lines changed: 9 additions & 0 deletions
    Original file line numberDiff line numberDiff line change
    @@ -28,6 +28,10 @@ attribute(s)
    2828
    additional journals or datafiles that are created. Already
    2929
    existing journals or datafiles will not be affected.
    3030

    31+
    - *validation*: Object that specifies the collection level schema validation
    32+
    for documents. The attribute keys `rule`, `level` and `message` must follow
    33+
    the rules documented in [Document Schema Validation](https://www.arangodb.com/docs/devel/document-schema-validation.html)
    34+
    3135
    On success an object with the following attributes is returned:
    3236

    3337
    - *id*: The identifier of the collection.
    @@ -62,6 +66,11 @@ On success an object with the following attributes is returned:
    6266
    generating keys and supplying own key values in the *_key* attribute
    6367
    of documents is considered an error.
    6468

    69+
    * *validation* (optional, default is *null*, **rocksdb-only**):
    70+
    Object that specifies the collection level schema validation for documents.
    71+
    The attribute keys `rule`, `level` and `message` must follow the rules
    72+
    documented in [Document Schema Validation](https://www.arangodb.com/docs/devel/document-schema-validation.html)
    73+
    6574
    **Note**: except for *waitForSync*, *journalSize* and *name*, collection
    6675
    properties **cannot be changed** once a collection is created. To rename
    6776
    a collection, the rename endpoint must be used.

    ‎Documentation/DocuBlocks/Rest/Replication/get_api_replication_revisions_tree.md

    Lines changed: 70 additions & 0 deletions
    Original file line numberDiff line numberDiff line change
    @@ -0,0 +1,70 @@
    1+
    2+
    @startDocuBlock get_api_replication_revisions_tree
    3+
    @brief retrieves the Merkle tree associated with the collection
    4+
    5+
    @RESTHEADER{GET /_api/replication/revisions/tree, Return Merkle tree for a collection,handleCommandRevisionTree}
    6+
    7+
    @RESTQUERYPARAMETERS
    8+
    9+
    @RESTQUERYPARAM{collection,string,required}
    10+
    The name or id of the collection to query.
    11+
    12+
    @RESTQUERYPARAM{batchId,number,required}
    13+
    The id of the snapshot to use
    14+
    15+
    @RESTDESCRIPTION
    16+
    Returns the Merkle tree from the collection.
    17+
    18+
    The result will be JSON/VelocyPack in the following format:
    19+
    ```
    20+
    {
    21+
    version: <Number>,
    22+
    branchingFactor: <Number>
    23+
    maxDepth: <Number>,
    24+
    rangeMin: <String, revision>,
    25+
    rangeMax: <String, revision>,
    26+
    nodes: [
    27+
    { count: <Number>, hash: <String, revision> },
    28+
    { count: <Number>, hash: <String, revision> },
    29+
    ...
    30+
    { count: <Number>, hash: <String, revision> }
    31+
    ]
    32+
    }
    33+
    ```
    34+
    35+
    At the moment, there is only one version, 1, so this can safely be ignored for
    36+
    now.
    37+
    38+
    Each `<String, revision>` value type is a 64-bit value encoded as a string of
    39+
    11 characters, using the same encoding as our document `_rev` values. The
    40+
    reason for this is that 64-bit values cannot necessarily be represented in full
    41+
    in JavaScript, as it handles all numbers as floating point, and can only
    42+
    represent up to `2^53-1` faithfully.
    43+
    44+
    The node count should correspond to a full tree with the given `maxDepth` and
    45+
    `branchingFactor`. The nodes are laid out in level-order tree traversal, so the
    46+
    root is at index `0`, its children at indices `[1, branchingFactor]`, and so
    47+
    on.
    48+
    49+
    @RESTRETURNCODES
    50+
    51+
    @RESTRETURNCODE{200}
    52+
    is returned if the request was executed successfully and data was returned.
    53+
    54+
    @RESTRETURNCODE{401}
    55+
    is returned if necessary parameters are missing
    56+
    57+
    @RESTRETURNCODE{404}
    58+
    is returned when the collection or snapshot could not be found.
    59+
    60+
    @RESTRETURNCODE{405}
    61+
    is returned when an invalid HTTP method is used.
    62+
    63+
    @RESTRETURNCODE{500}
    64+
    is returned if an error occurred while assembling the response.
    65+
    66+
    @RESTRETURNCODE{501}
    67+
    is returned if called with mmfiles or on a collection which doesn't support
    68+
    sync-by-revision
    69+
    70+
    @endDocuBlock

    ‎Documentation/DocuBlocks/Rest/Replication/post_api_replication_revisions_tree.md

    Lines changed: 38 additions & 0 deletions
    Original file line numberDiff line numberDiff line change
    @@ -0,0 +1,38 @@
    1+
    2+
    @startDocuBlock post_api_replication_revisions_tree
    3+
    @brief rebuilds the Merkle tree associated with the collection
    4+
    5+
    @RESTHEADER{POST /_api/replication/revisions/tree, Rebuild Merkle tree for a collection,handleCommandRebuildRevisionTree}
    6+
    7+
    @RESTQUERYPARAMETERS
    8+
    9+
    @RESTQUERYPARAM{collection,string,required}
    10+
    The name or id of the collection to query.
    11+
    12+
    @RESTDESCRIPTION
    13+
    Rebuilds the Merkle tree for the collection.
    14+
    15+
    If successful, there will be no return body.
    16+
    17+
    @RESTRETURNCODES
    18+
    19+
    @RESTRETURNCODE{204}
    20+
    is returned if the request was executed successfully.
    21+
    22+
    @RESTRETURNCODE{401}
    23+
    is returned if necessary parameters are missing
    24+
    25+
    @RESTRETURNCODE{404}
    26+
    is returned when the collection or could not be found.
    27+
    28+
    @RESTRETURNCODE{405}
    29+
    is returned when an invalid HTTP method is used.
    30+
    31+
    @RESTRETURNCODE{500}
    32+
    is returned if an error occurred while assembling the response.
    33+
    34+
    @RESTRETURNCODE{501}
    35+
    is returned if called with mmfiles or on a collection which doesn't support
    36+
    sync-by-revision
    37+
    38+
    @endDocuBlock

    0 commit comments

    Comments
     (0)
    0