Merge branch 'devel' of ssh://github.com/arangodb/ArangoDB into featu…

…re/AqlSubqueryExecutionBlockImplExecuteImplementation-expectedNumberOfRows
arangodb · mchacki · Mar 20, 2020 · Oct 22, 2019 · Oct 22, 2019 · Oct 22, 2019
commit a102483647b7f7a1dce902ab9ed512dad449b573
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,8 +1,22 @@
 devel
 -----
 
-<<<<<<< HEAD
-=======
+* Allow to override the detected total amount of memory via an environment
+  variable ARANGODB_OVERRIDE_DETECTED_TOTAL_MEMORY.
+
+* `splice-subqueries` optimization is not limited by any type of operation within the
+  subquery any more. It can now be applied on every subquery and will be by default.
+  However they may be a performance impact on some queries where splice-subqueries
+  are not as performant as non-spliced subqueries. This is due to internal memory
+  management right now and will be addressed in future versions. Spliced subqueries
+  can be less performant if the query around the subquery is complex and requires
+  lots of variables, or variables with large content, but the subquery itself
+  does not require a lot of variables and produces many intermediate results
+  s.t. good batching within the query does not pay off against memory overhead.
+
+* Supervision to clean up zombie servers after 24h, if no
+  responsibility for shards.
+
 * Fix SORT RAND() LIMIT 1 optimization for RocksDB when only a projection of the
   attributes was used. When a projection was used and that projection was
   covered by an index (e.g. `_key` via the primary index), then the access
@@ -13,7 +27,6 @@ devel
 * Mark server startup options `--foxx.*`, `--frontend.*` and `--javascript.*`
   as single server and Coordinator only for documentation (`--dump-options`).
 
->>>>>>> 7a85b8dc4087351a74eb2e8ef4a99f298a9d1519
 * Supervision hot backup and supervision maintenance modes ttl fix.
 
 * Fix a bug that leads to graph traversals yielding empty output when none of the

diff --git a/arangod/Aql/AqlItemBlockInputRange.cpp b/arangod/Aql/AqlItemBlockInputRange.cpp
@@ -196,4 +196,4 @@ auto AqlItemBlockInputRange::countShadowRows() const noexcept -> std::size_t {
   }
   auto const& block = getBlock();
   return block->getShadowRowIndexes().size();
-}
+}
diff --git a/arangod/Aql/ExecutionBlockImpl.cpp b/arangod/Aql/ExecutionBlockImpl.cpp
@@ -1279,9 +1279,10 @@ auto ExecutionBlockImpl<Executor>::executeFetcher(AqlCallStack& stack, AqlCallTy
         _lastRange.setDependency(dependency, range);
       }
       return {state, skipped, _lastRange};
+
     } else if constexpr (executorHasSideEffects<Executor>) {
       // If the executor has side effects, we cannot bypass any subqueries
-      // by skipping them. SO we need to fetch all shadow rows in order to
+      // by skipping them. So we need to fetch all shadow rows in order to
       // trigger this Executor with everthing from above.
       // NOTE: The Executor needs to discard shadowRows, and do the accouting.
       static_assert(std::is_same_v<AqlCall, std::decay_t<decltype(aqlCall)>>);
@@ -1694,7 +1695,7 @@ auto ExecutionBlockImpl<Executor>::executeFastForward(typename Fetcher::DataRang
     }
     case FastForwardVariant::FETCHER: {
       LOG_QUERY("fa327", DEBUG) << printTypeInfo() << " bypass unused rows.";
-      auto const dependency = inputRange.skipAllRemainingDataRows();
+      ADB_IGNORE_UNUSED auto const dependency = inputRange.skipAllRemainingDataRows();
       auto constexpr fastForwardCall = AqlCall{0, false, 0, AqlCall::LimitType::HARD};
       auto const call = std::invoke([&]() -> AqlCallType {
         if constexpr (std::is_same_v<AqlCallType, AqlCall>) {

diff --git a/arangod/Aql/IdExecutor.cpp b/arangod/Aql/IdExecutor.cpp
@@ -103,10 +103,6 @@ auto IdExecutor<UsedFetcher>::produceRows(AqlItemBlockInputRange& inputRange,
       THROW_ARANGO_EXCEPTION(TRI_ERROR_DEBUG);
     }
   }
-  if (_infos.doCount()) {
-    stats.addCounted(output.numRowsWritten());
-  }
-
   TRI_ASSERT(!inputRange.hasDataRow());
   if (_infos.doCount()) {
     stats.addCounted(output.numRowsWritten());

diff --git a/arangod/Aql/ReturnExecutor.cpp b/arangod/Aql/ReturnExecutor.cpp
@@ -136,4 +136,4 @@ auto ReturnExecutor::produceRows(AqlItemBlockInputRange& inputRange, OutputAqlIt
                                                            AqlCall const& call) const
     noexcept -> size_t {
   return input.countDataRows();
-}
+}
diff --git a/tests/Aql/EnumerateCollectionExecutorTest.cpp b/tests/Aql/EnumerateCollectionExecutorTest.cpp
@@ -244,8 +244,7 @@ TEST_F(EnumerateCollectionExecutorTest, the_skip_datarange) {
 // new framework tests
 
 // This is only to get a split-type. The Type is independent of actual template parameters
-using EnumerateCollectionTestHelper =
-    ExecutorTestHelper<1, 1>;
+using EnumerateCollectionTestHelper = ExecutorTestHelper<1, 1>;
 using EnumerateCollectionSplitType = EnumerateCollectionTestHelper::SplitType;
 using EnumerateCollectionInputParam = std::tuple<EnumerateCollectionSplitType>;
 
@@ -288,8 +287,7 @@ class EnumerateCollectionExecutorTestProduce
         useRawPointers(false),
         random(false),
         infos(1, 1, 2, {}, {}, engine, &aqlCollection, &outVariable, varUsedLater, nullptr,
-              projections, coveringIndexAttributePositions, useRawPointers, random) {
-  }
+              projections, coveringIndexAttributePositions, useRawPointers, random) {}
 
   auto makeInfos(RegisterId outputRegister = 0, RegisterId nrInputRegister = 1,
                  RegisterId nrOutputRegister = 1,
@@ -313,7 +311,8 @@ class EnumerateCollectionExecutorTestProduce
         R"aql(INSERT {_key: "testee1", value: 1, sortValue: 1, nestedObject: {value: 1} } INTO UnitTestCollection RETURN NEW)aql";
     SCOPED_TRACE(insertQuery);
     auto queryRes = arangodb::tests::executeQuery(vocbase, insertQuery, {});
-    queryResults.push_back(queryRes.data.get()->slice().at(0).toJson(queryRes.context->getVPackOptions()));
+    queryResults.push_back(queryRes.data.get()->slice().at(0).toJson(
+        queryRes.context->getVPackOptions()));
 
     for (size_t i = 2; i <= amount; i++) {
       std::string insertQueryPart1 = R"aql(INSERT {_key: "testee)aql";
@@ -323,7 +322,8 @@ class EnumerateCollectionExecutorTestProduce
       std::string finalQuery = insertQueryPart1 + insertQueryPart2 + insertQueryPart3;
       SCOPED_TRACE(finalQuery);
       auto queryResInner = arangodb::tests::executeQuery(vocbase, finalQuery, {});
-      queryResults.push_back(queryResInner.data.get()->slice().at(0).toJson(queryResInner.context->getVPackOptions()));
+      queryResults.push_back(queryResInner.data.get()->slice().at(0).toJson(
+          queryResInner.context->getVPackOptions()));
     }
 
     return queryRes.context->getVPackOptions();
@@ -348,7 +348,7 @@ TEST_P(EnumerateCollectionExecutorTestProduce, DISABLED_produce_all_documents) {
 
   uint64_t numberOfDocumentsToInsert = 10;
   std::vector<std::string> queryResults;
-  auto vpackOptions = insertDocuments(numberOfDocumentsToInsert, queryResults);
+  // auto vpackOptions = insertDocuments(numberOfDocumentsToInsert, queryResults);
   EXPECT_EQ(vocbase.lookupCollection("UnitTestCollection")
                 ->numberDocuments(fakedQuery->trx(), transaction::CountType::Normal),
             numberOfDocumentsToInsert);  // validate that our document inserts worked
@@ -383,7 +383,8 @@ TEST_P(EnumerateCollectionExecutorTestProduce, DISABLED_produce_5_documents) {
 
   uint64_t numberOfDocumentsToInsert = 10;
   std::vector<std::string> queryResults;
-  auto vpackOptions = insertDocuments(numberOfDocumentsToInsert, queryResults);
+  // auto vpackOptions = insertDocuments(numberOfDocumentsToInsert, queryResults);
+  std::ignore = insertDocuments(numberOfDocumentsToInsert, queryResults);
 
   ExecutorTestHelper<1, 1>(*fakedQuery)
       .setInputValue({{RowBuilder<1>{R"({ "cid" : "1337", "name": "UnitTestCollection" })"}}})
@@ -397,20 +398,21 @@ TEST_P(EnumerateCollectionExecutorTestProduce, DISABLED_produce_5_documents) {
       .run();
 }
    
-
 // DISABLED because we need to be able to compare real documents (currently not possible)
 TEST_P(EnumerateCollectionExecutorTestProduce, DISABLED_skip_5_documents_default) {
   auto [split] = GetParam();
 
   uint64_t numberOfDocumentsToInsert = 10;
   std::vector<std::string> queryResults;
-  auto vpackOptions = insertDocuments(numberOfDocumentsToInsert, queryResults);
+  std::ignore = insertDocuments(numberOfDocumentsToInsert, queryResults);
 
   ExecutorTestHelper<1, 1>(*fakedQuery)
       .setInputValue({{RowBuilder<1>{R"({ "cid" : "1337", "name":
-"UnitTestCollection" })"}}}) .setInputSplitType(split) .setCall(AqlCall{5,
-AqlCall::Infinity{}, AqlCall::Infinity{}, false}) .expectSkipped(5) .expectOutput({0},
-{{R"(null)"}, {R"(null)"}, {R"(null)"}, {R"(null)"}, {R"(null)"}})
+"UnitTestCollection" })"}}})
+      .setInputSplitType(split)
+      .setCall(AqlCall{5, AqlCall::Infinity{}, AqlCall::Infinity{}, false})
+      .expectSkipped(5)
+      .expectOutput({0}, {{R"(null)"}, {R"(null)"}, {R"(null)"}, {R"(null)"}, {R"(null)"}})
       .expectedState(ExecutionState::DONE)
       .setExecBlock<EnumerateCollectionExecutor>(std::move(makeInfos()))
       .run();

diff --git a/tests/Aql/SplicedSubqueryIntegrationTest.cpp b/tests/Aql/SplicedSubqueryIntegrationTest.cpp
@@ -470,7 +470,6 @@ TEST_P(SplicedSubqueryIntegrationTest, check_call_passes_subquery) {
 
 TEST_P(SplicedSubqueryIntegrationTest, check_skipping_subquery) {
   auto call = AqlCall{10};
-  LOG_DEVEL << call;
   auto pipeline = createSubquery(createAssertPipeline());
 
   executorTestHelper.setPipeline(std::move(pipeline))
@@ -485,7 +484,6 @@ TEST_P(SplicedSubqueryIntegrationTest, check_skipping_subquery) {
 
 TEST_P(SplicedSubqueryIntegrationTest, check_soft_limit_subquery) {
   auto call = AqlCall{0, false, 4, AqlCall::LimitType::SOFT};
-  LOG_DEVEL << call;
   auto pipeline = createSubquery(createAssertPipeline());
 
   ExecutorTestHelper<1, 2>{*fakedQuery}
@@ -497,4 +495,4 @@ TEST_P(SplicedSubqueryIntegrationTest, check_soft_limit_subquery) {
       .expectSkipped(0)
       .expectedState(ExecutionState::HASMORE)
       .run();
-};
+};
-Original file line number
+Diff line change
@@ Expand Up @@
       }
       auto const& block = getBlock();
       return block->getShadowRowIndexes().size();
-    }
+    }