8000 Merge branch 'devel' of ssh://github.com/arangodb/ArangoDB into bug-f… · arangodb/arangodb@1442bb3 · GitHub
[go: up one dir, main page]
Skip to content

Commit 1442bb3

Browse files
mchackimchacki
committed
Merge branch 'devel' of ssh://github.com/arangodb/ArangoDB into bug-fix/potential-segfault-fix
2 parents 1a2f231 + eb1aa6e commit 1442bb3

File tree

142 files changed

+2146
-986
lines changed
  • HttpRequest.cpp
  • HttpResponse.h
  • VstResponse.h
  • tests

    • Some content is hidden

    Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

    142 files changed

    +2146
    -986
    lines changed

    ‎3rdParty/rocksdb/6.2/db/transaction_log_impl.cc

    Lines changed: 2 additions & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -202,7 +202,8 @@ void TransactionLogIteratorImpl::NextImpl(bool internal) {
    202202
    if (current_last_seq_ == versions_->LastSequence()) {
    203203
    current_status_ = Status::OK();
    204204
    } else {
    205-
    current_status_ = Status::Corruption("NO MORE DATA LEFT");
    205+
    const char* msg = "Create a new iterator to fetch the new tail.";
    206+
    current_status_ = Status::TryAgain(msg);
    206207
    }
    207208
    return;
    208209
    }

    ‎3rdParty/rocksdb/6.2/thirdparty.inc

    Lines changed: 9 additions & 9 deletions
    Original file line numberDiff line numberDiff line change
    @@ -69,20 +69,20 @@ if (WITH_SNAPPY)
    6969
    set(SNAPPY_INCLUDE $ENV{SNAPPY_INCLUDE})
    7070
    endif()
    7171

    72-
    if(DEFINED ENV{SNAPPY_LIB_DEBUG})
    73-
    set(SNAPPY_LIB_DEBUG $ENV{SNAPPY_LIB_DEBUG})
    74-
    endif()
    75-
    76-
    if(DEFINED ENV{SNAPPY_LIB_RELEASE})
    77-
    set(SNAPPY_LIB_RELEASE $ENV{SNAPPY_LIB_RELEASE})
    78-
    endif()
    72+
    # if(DEFINED ENV{SNAPPY_LIB_DEBUG})
    73+
    # set(SNAPPY_LIB_DEBUG $ENV{SNAPPY_LIB_DEBUG})
    74+
    # endif()
    75+
    #
    76+
    # if(DEFINED ENV{SNAPPY_LIB_RELEASE})
    77+
    # set(SNAPPY_LIB_RELEASE $ENV{SNAPPY_LIB_RELEASE})
    78+
    # endif()
    7979

    8080
    set(SNAPPY_CXX_FLAGS -DSNAPPY)
    81-
    set(SNAPPY_LIBS debug ${SNAPPY_LIB_DEBUG} optimized ${SNAPPY_LIB_RELEASE})
    81+
    # set(SNAPPY_LIBS debug ${SNAPPY_LIB_DEBUG} optimized ${SNAPPY_LIB_RELEASE})
    8282

    8383
    add_definitions(${SNAPPY_CXX_FLAGS})
    8484
    include_directories(${SNAPPY_INCLUDE})
    85-
    set (THIRDPARTY_LIBS ${THIRDPARTY_LIBS} ${SNAPPY_LIBS})
    85+
    set (THIRDPARTY_LIBS ${THIRDPARTY_LIBS} snappy)
    8686
    else ()
    8787
    message(STATUS "SNAPPY library is disabled")
    8888
    endif ()

    ‎3rdParty/velocypack/src/Builder.cpp

    Lines changed: 59 additions & 58 deletions
    Original file line numberDiff line numberDiff line change
    @@ -38,6 +38,8 @@ using namespace arangodb::velocypack;
    3838

    3939
    namespace {
    4040

    41+
    constexpr ValueLength LinearAttributeUniquenessCutoff = 4;
    42+
    4143
    // struct used when sorting index tables for objects:
    4244
    struct SortEntry {
    4345
    uint8_t const* nameStart;
    @@ -51,10 +53,10 @@ struct SortEntry {
    5153
    constexpr size_t minSortEntriesAllocation = 32;
    5254

    5355
    // thread-local, reusable buffer used for sorting medium to big index entries
    54-
    thread_local std::vector<SortEntry> sortEntries;
    56+
    thread_local std::unique_ptr<std::vector<SortEntry>> sortEntries;
    5557

    5658
    // thread-local, reusable set to track usage of duplicate keys
    57-
    thread_local std::unordered_set<StringRef> duplicateKeys;
    59+
    thread_local std::unique_ptr<std::unordered_set<StringRef>> duplicateKeys;
    5860

    5961
    // Find the actual bytes of the attribute name of the VPack value
    6062
    // at position base, also determine the length len of the attribute.
    @@ -81,6 +83,48 @@ uint8_t const* findAttrName(uint8_t const* base, uint64_t& len) {
    8183
    return findAttrName(arangodb::velocypack::Slice(base).makeKey().start(), len);
    8284
    }
    8385

    86+
    bool checkAttributeUniquenessUnsortedBrute(ObjectIterator& it) {
    87+
    std::array<StringRef, LinearAttributeUniquenessCutoff> keys;
    88+
    89+
    do {
    90+
    // key(true) guarantees a String as returned type
    91+
    StringRef key = it.key(true).stringRef();
    92+
    ValueLength index = it.index();
    93+
    // compare with all other already looked-at keys
    94+
    for (ValueLength i = 0; i < index; ++i) {
    95+
    if (VELOCYPACK_UNLIKELY(keys[i].equals(key))) {
    96+
    return false;
    97+
    }
    98+
    }
    99+
    keys[index] = key;
    100+
    it.next();
    101+
    102+
    } while (it.valid());
    103+
    104+
    return true;
    105+
    }
    106+
    107+
    bool checkAttributeUniquenessUnsortedSet(ObjectIterator& it) {
    108+
    if (::duplicateKeys == nullptr) {
    109+
    ::duplicateKeys.reset(new std::unordered_set<StringRef>());
    110+
    } else {
    111+
    ::duplicateKeys->clear();
    112+
    }
    113+
    do {
    114+
    Slice const key = it.key(true);
    115+
    // key(true) guarantees a String as returned type
    116+
    VELOCYPACK_ASSERT(key.isString());
    117+
    if (VELOCYPACK_UNLIKELY(!::duplicateKeys->emplace(key).second)) {
    118+
    // identical key
    119+
    return false;
    120+
    }
    121+
    it.next();
    122+
    } while (it.valid());
    123+
    124+
    return true;
    125+
    }
    126+
    127+
    84128
    } // namespace
    85129

    86130
    // create an empty Builder, using default Options
    @@ -250,20 +294,24 @@ void Builder::sortObjectIndexLong(uint8_t* objBase,
    250294
    std::vector<ValueLength>& offsets) {
    251295
    // start with clean sheet in case the previous run left something
    252296
    // in the vector (e.g. when bailing out with an exception)
    253-
    ::sortEntries.clear();
    297+
    if (::sortEntries == nullptr) {
    298+
    ::sortEntries.reset(new std::vector<SortEntry>());
    299+
    } else {
    300+
    ::sortEntries->clear();
    301+
    }
    254302

    255303
    std::size_t const n = offsets.size();
    256304
    VELOCYPACK_ASSERT(n > 1);
    257-
    ::sortEntries.reserve(std::max(::minSortEntriesAllocation, n));
    305+
    ::sortEntries->reserve(std::max(::minSortEntriesAllocation, n));
    258306
    for (std::size_t i = 0; i < n; i++) {
    259307
    SortEntry e;
    260308
    e.offset = offsets[i];
    261309
    e.nameStart = ::findAttrName(objBase + e.offset, e.nameSize);
    262-
    ::sortEntries.push_back(e);
    310+
    ::sortEntries->push_back(e);
    263311
    }
    264312
    VELOCYPACK_ASSERT(::sortEntries.size() == n);
    265-
    std::sort(::sortEntries.begin(), ::sortEntries.end(), [](SortEntry const& a,
    266-
    SortEntry const& b)
    313+
    std::sort(::sortEntries->begin(), ::sortEntries->end(), [](SortEntry const& a,
    314+
    SortEntry const& b)
    267315
    #ifdef VELOCYPACK_64BIT
    268316
    noexcept
    269317
    #endif
    @@ -279,18 +327,7 @@ void Builder::sortObjectIndexLong(uint8_t* objBase,
    279327

    280328
    // copy back the sorted offsets
    281329
    for (std::size_t i = 0; i < n; i++) {
    282-
    offsets[i] = ::sortEntries[i].offset;
    283-
    }
    284-
    285-
    if (::sortEntries.capacity() >= 4096) {
    286-
    // if we use around 100kb or more of memory, try to free up some memory
    287-
    if (::sortEntries.size() >= ::minSortEntriesAllocation) {
    288-
    // leave 32 elements in the vector, so we can hopefully avoid some reallocations later
    289-
    ::sortEntries.erase(::sortEntries.begin() + ::minSortEntriesAllocation, ::sortEntries.end());
    290-
    } else {
    291-
    ::sortEntries.clear();
    292-
    }
    293-
    ::sortEntries.shrink_to_fit();
    330+
    offsets[i] = (*::sortEntries)[i].offset;
    294331
    }
    295332
    }
    296333

    @@ -1053,48 +1090,12 @@ bool Builder::checkAttributeUniquenessUnsorted(Slice obj) const {
    10531090
    // objects with more attributes will use a validation routine that
    10541091
    // will use an std::unordered_set for O(1) lookups but with heap
    10551092
    // allocations
    1056-
    constexpr ValueLength LinearAttributeUniquenessCutoff = 4;
    1057-
    10581093
    ObjectIterator it(obj, true);
    10591094

    1060-
    if (it.size() <= LinearAttributeUniquenessCutoff) {
    1061-
    std::array<StringRef, LinearAttributeUniquenessCutoff> keys;
    1062-
    do {
    1063-
    // key(true) guarantees a String as returned type
    1064-
    StringRef key = it.key(true).stringRef();
    1065-
    ValueLength index = it.index();
    1066-
    // compare with all other already looked-at keys
    1067-
    for (ValueLength i = 0; i < index; ++i) {
    1068-
    if (VELOCYPACK_UNLIKELY(keys[i].equals(key))) {
    1069-
    return false;
    1070-
    }
    1071-
    }
    1072-
    keys[index] = key;
    1073-
    it.next();
    1074-
    } while (it.valid());
    1075-
    } else {
    1076-
    ::duplicateKeys.clear();
    1077-
    do {
    1078-
    Slice const key = it.key(true);
    1079-
    // key(true) guarantees a String as returned type
    1080-
    VELOCYPACK_ASSERT(key.isString());
    1081-
    if (VELOCYPACK_UNLIKELY(!::duplicateKeys.emplace(key).second)) {
    1082-
    // identical key
    1083-
    return false;
    1084-
    }
    1085-
    it.next();
    1086-
    } while (it.valid());
    1087-
    1088-
    // reclaim a bit of memory already if we have tracked a lot of keys.
    1089-
    // this will not free the set's top-level, but should free up the elements
    1090-
    // in the set
    1091-
    if (::duplicateKeys.size() >= 4096) {
    1092-
    ::duplicateKeys.clear();
    1093-
    }
    1095+
    if (it.size() <= ::LinearAttributeUniquenessCutoff) {
    1096+
    return ::checkAttributeUniquenessUnsortedBrute(it);
    10941097
    }
    1095-
    1096-
    // all keys unique
    1097-
    return true;
    1098+
    return ::checkAttributeUniquenessUnsortedSet(it);
    10981099
    }
    10991100

    11001101
    // Add all subkeys and subvalues into an object from an ObjectIterator

    ‎CHANGELOG

    Lines changed: 8 additions & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -1,6 +1,12 @@
    11
    devel
    22
    -----
    33

    4+
    * added option `--replication.max-parallel-tailing-invocations` to limit the maximum number
    5+
    of concurrent WAL tailing invocations.
    6+
    7+
    The option can be used to limit the usage of the WAL tailing APIs in order to control
    8+
    server load
    9+
    410
    * Speed up collection creation process in cluster, if not all agency callbacks are
    511
    delivered successfully.
    612

    @@ -34,6 +40,7 @@ devel
    3440

    3541
    * fixed agency bug with TTL object writes discovered in 3.4.6
    3642

    43+
    * fixed agency bug with election lock step
    3744

    3845
    v3.5.0-rc.3 (2019-05-31)
    3946
    ------------------------
    @@ -141,7 +148,7 @@ v3.5.0-rc.1 (2019-05-14)
    141148
    - `--foxx.store`: set to false disables Foxx UI
    142149
    - `--javascript.allow-port-testing`: enables internal.testPort()
    143150
    - `--javascript.allow-external-process-control`: enables external process control
    144-
    - `--javascript.harden`: disables getPid(), processStatistics() and logLevel()
    151+
    - `--javascript.harden`: disables getPid() and logLevel()
    145152
    - `--javascript.startup-options-whitelist`: control startup options visible in JavaScript
    146153
    - `--javascript.environment-variables-whitelist`: control environment variables visible in JavaScript
    147154
    - `--javascript.endpoints-whitelist`: control accessible endpoints in JavaScript

    ‎Documentation/Books/Manual/Security/SecurityOptions.md

    Lines changed: 0 additions & 3 deletions
    Original file line numberDiff line numberDiff line change
    @@ -133,9 +133,6 @@ extra options are available for locking down JavaScript access to server functio
    133133
    If set to `true`, this setting will deactivate the following JavaScript functions
    134134
    which may leak information about the environment:
    135135

    136-
    - `internal.clientStatistics()`
    137-
    - `internal.httpStatistics()`
    138-
    - `internal.processStatistics()`
    139136
    - `internal.getPid()`
    140137
    - `internal.logLevel()`.
    141138

    ‎VERSIONS

    Lines changed: 1 addition & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -1,5 +1,5 @@
    11
    STARTER_REV "0.14.5"
    2-
    SYNCER_REV "0.6.3"
    2+
    SYNCER_REV "0.6.4"
    33
    GCC_LINUX "8.3.0"
    44
    GSEARCH_ID_HTTP "010085642145132923492:fixi4yzeiz8"
    55
    GSEARCH_ID_AQL "010085642145132923492:6ymjhhr677k"

    ‎arangod/Agency/Constituent.cpp

    Lines changed: 14 additions & 0 deletions
    Original file line numberDiff line numberDiff line change
    @@ -391,6 +391,10 @@ bool Constituent::vote(term_t termOfPeer, std::string const& id,
    391391
    if (_votedFor != NO_LEADER) { // already voted in this term
    392392
    if (_votedFor == id) {
    393393
    LOG_TOPIC("41c49", DEBUG, Logger::AGENCY) << "repeating vote for " << id;
    394+
    // Set the last heart beat seen to now, to grant the other guy some time
    395+
    // to establish itself as a leader, before we call for another election:
    396+
    _lastHeartbeatSeen = TRI_microtime();
    397+
    LOG_TOPIC("658ba", TRACE, Logger::AGENCY) << "setting last heartbeat time to now, since we repeated a vote grant: " << _lastHeartbeatSeen;
    394398
    return true;
    395399
    }
    396400
    LOG_TOPIC("df508", DEBUG, Logger::AGENCY)
    @@ -406,6 +410,10 @@ bool Constituent::vote(term_t termOfPeer, std::string const& id,
    406410
    if (prevLogTerm > myLastLogEntry.term ||
    407411
    (prevLogTerm == myLastLogEntry.term && prevLogIndex >= myLastLogEntry.index)) {
    408412
    LOG_TOPIC("8d8da", DEBUG, Logger::AGENCY) << "voting for " << id << " in term " << _term;
    413+
    // Set the last heart beat seen to now, to grant the other guy some time
    414+
    // to establish itself as a leader, before we call for another election:
    415+
    _lastHeartbeatSeen = TRI_microtime();
    416+
    LOG_TOPIC("ffaac", TRACE, Logger::AGENCY) << "setting last heartbeat time to now, since we granted a vote: " << _lastHeartbeatSeen;
    409417
    termNoLock(_term, id);
    410418
    return true;
    411419
    }
    @@ -698,6 +706,12 @@ void Constituent::run() {
    698706

    699707
    } else if (role == CANDIDATE) {
    700708
    callElection(); // Run for office
    709+
    // Now we take this point of time as the next base point for a
    710+
    // potential next random timeout, since we have just cast a vote for
    711+
    // ourselves:
    712+
    _lastHeartbeatSeen = TRI_microtime();
    713+
    LOG_TOPIC("aeaef", TRACE, Logger::AGENCY) << "setting last heartbeat because we voted for us: " << _lastHeartbeatSeen;
    714+
    701715
    } else {
    702716
    double interval =
    703717
    0.25 * _agent->config().minPing() * _agent->config().timeoutMult();

    ‎arangod/Agency/Constituent.h

    Lines changed: 4 additions & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -156,7 +156,10 @@ class Constituent : public Thread {
    156156
    std::string _id; // My own id
    157157

    158158
    // Last time an AppendEntriesRPC message has arrived, this is used to
    159-
    // organize out-of-patience in the follower:
    159+
    // organize out-of-patience in the follower. Note that this variable is
    160+
    // also set to the current time when a vote is cast, either for ourselves
    161+
    // or for somebody else. The constituent calls for an election if and only
    162+
    // if the time since _lastHeartbeatSeen is greater than a random timeout:
    160163
    std::atomic<double> _lastHeartbeatSeen;
    161164

    162165
    role_t _role; // My role

    ‎arangod/Agency/RestAgencyHandler.cpp

    Lines changed: 2 additions & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -568,7 +568,7 @@ RestStatus RestAgencyHandler::handleConfig() {
    568568
    RestStatus RestAgencyHandler::handleState() {
    569569

    570570
    VPackBuilder body;
    571-
    {
    571+
    {
    572572
    VPackObjectBuilder o(&body);
    573573
    _agent->readDB(body);
    574574
    }
    @@ -583,6 +583,7 @@ RestStatus RestAgencyHandler::reportMethodNotAllowed() {
    583583
    }
    584584

    585585
    RestStatus RestAgencyHandler::execute() {
    586+
    response()->setAllowCompression(true);
    586587
    try {
    587588
    auto const& suffixes = _request->suffixes();
    588589
    if (suffixes.empty()) { // Empty request

    ‎arangod/Aql/IndexNode.cpp

    Lines changed: 3 additions & 10 deletions
    Original file line numberDiff line numberDiff line change
    @@ -429,18 +429,11 @@ CostEstimate IndexNode::estimateCost() const {
    429429
    auto root = _condition->root();
    430430

    431431
    for (size_t i = 0; i < _indexes.size(); ++i) {
    432-
    arangodb::aql::AstNode const* condition;
    433-
    if (root == nullptr || root->numMembers() <= i) {
    434-
    condition = nullptr;
    435-
    } else {
    436-
    condition = root->getMember(i);
    437-
    }
    432+
    Index::FilterCosts costs = Index::FilterCosts::defaultCosts(itemsInCollection);
    438433

    439-
    Index::UsageCosts costs;
    440-
    if (condition != nullptr) {
    434+
    if (root != nullptr && root->numMembers() > i) {
    435+
    arangodb::aql::AstNode const* condition = root->getMember(i);
    441436
    costs = _indexes[i].getIndex()->supportsFilterCondition(std::vector<std::shared_ptr<Index>>(), condition, _outVariable, itemsInCollection);
    442-
    } else {
    443-
    costs = Index::UsageCosts::defaultsForFiltering(itemsInCollection);
    444437
    }
    445438

    446439
    totalItems += costs.estimatedItems;

    0 commit comments

    Comments
     (0)
    0