8000 Feature/abort traversals by hkernbach · Pull Request #21765 · arangodb/arangodb · GitHub
[go: up one dir, main page]

Skip to content

Feature/abort traversals #21765

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 23 commits into
base: devel
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
532d00b
arangodb release 3.12.4
KVS85 Jan 23, 2025
af6afa9
first draft, added cancel of traversal in one sided enumerator, and t…
hkernbach Apr 28, 2025
4cfc171
added bidirectionalCircle as new graph in graph test suite
hkernbach May 6, 2025
db7f093
added hugeCompleteGraph
hkernbach May 6, 2025
a03dc03
added query cancel observer to twosided enumerator as well, so kpaths…
hkernbach May 13, 2025
a250d68
merge main into feature branch
hkernbach May 14, 2025
7692de9
only print debug creation info if set manually in constructor
hkernbach May 14, 2025
7ef978f
revert changelog merge issue
hkernbach May 14, 2025
67f8176
added additional check for cancelation of query in the twosidedenumer…
hkernbach May 14, 2025
8e05284
Merge remote-tracking branch 'origin/devel' into feature/abort-traver…
hkernbach May 14, 2025
2de1512
eslint
hkernbach May 14, 2025
aa29f4e
eslint
hkernbach May 16, 2025
6f21c2f
clang format
hkernbach May 16, 2025
af166b8
clang format
hkernbach May 16, 2025
f35a566
Merge remote-tracking branch 'origin/devel' into feature/abort-traver…
hkernbach May 16, 2025
e22c7c3
batchwise verted and edge insertion during fillGraph (integration tests)
hkernbach May 19, 2025
48c95dc
Merge remote-tracking branch 'origin/devel' into feature/abort-traver…
hkernbach May 19, 2025
033326a
try to help garbage collection v8
hkernbach May 19, 2025
2b79028
slightly increased timeout to 10s in clsuter, 5s in single server
hkernbach May 19, 2025
7a04512
added early exit to weighted two sided enumerator, yens still missing
hkernbach May 19, 2025
cc7991d
debug flag on
hkernbach May 19, 2025
5c4cb55
no need to call clear manually. will be called in destructor automat…
hkernbach May 20, 2025
7ff7f4f
format
hkernbach May 20, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 8 additions & 7 deletions arangod/Aql/ExecutionNode/EnumeratePathsNode.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -432,7 +432,7 @@ std::unique_ptr<ExecutionBlock> EnumeratePathsNode::createBlock(
TRI_ASSERT(pathType() != arangodb::graph::PathType::Type::ShortestPath);

arangodb::graph::TwoSidedEnumeratorOptions enumeratorOptions{
opts->getMinDepth(), opts->getMaxDepth(), pathType()};
opts->getMinDepth(), opts->getMaxDepth(), pathType(), opts->query()};
PathValidatorOptions validatorOptions(opts->tmpVar(),
opts->getExpressionCtx());

Expand Down Expand Up @@ -476,13 +476,14 @@ std::unique_ptr<ExecutionBlock> EnumeratePathsNode::createBlock(
SingleServerBaseProviderOptions forwardProviderOptions(
opts->tmpVar(), std::move(usedIndexes), opts->getExpressionCtx(), {},
opts->collectionToShard(), opts->getVertexProjections(),
opts->getEdgeProjections(), opts->produceVertices(), opts->useCache());
opts->getEdgeProjections(), opts->produceVertices(), opts->useCache(),
opts->query());

SingleServerBaseProviderOptions backwardProviderOptions(
opts->tmpVar(), std::move(reversedUsedIndexes),
opts->getExpressionCtx(), {}, opts->collectionToShard(),
opts->getVertexProjections(), opts->getEdgeProjections(),
opts->produceVertices(), opts->useCache());
opts->produceVertices(), opts->useCache(), opts->query());

using Provider = SingleServerProvider<SingleServerProviderStep>;
if (opts->query().queryOptions().getTraversalProfileLevel() ==
Expand Down Expand Up @@ -678,11 +679,11 @@ std::unique_ptr<ExecutionBlock> EnumeratePathsNode::createBlock(
} else { // Cluster case (on coordinator)
auto cache = std::make_shared<RefactoredClusterTraverserCache>(
opts->query().resourceMonitor());
ClusterBaseProviderOptions forwardProviderOptions(cache, engines(), false,
opts->produceVertices());
ClusterBaseProviderOptions forwardProviderOptions(
cache, engines(), false, opts->produceVertices(), opts->query());
forwardProviderOptions.setClearEdgeCacheOnClear(false);
ClusterBaseProviderOptions backwardProviderOptions(cache, engines(), true,
opts->produceVertices());
ClusterBaseProviderOptions backwardProviderOptions(
cache, engines(), true, opts->produceVertices(), opts->query());
backwardProviderOptions.setClearEdgeCacheOnClear(false);
// A comment is in order here: For all cases covered here
// (k-shortest-paths, all shortest paths, k-paths) we do not need to
Expand Down
15 changes: 8 additions & 7 deletions arangod/Aql/ExecutionNode/ShortestPathNode.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -419,7 +419,7 @@ std::unique_ptr<ExecutionBlock> ShortestPathNode::createBlock(

arangodb::graph::TwoSidedEnumeratorOptions enumeratorOptions{
0, std::numeric_limits<size_t>::max(),
arangodb::graph::PathType::Type::ShortestPath};
arangodb::graph::PathType::Type::ShortestPath, opts->query()};
PathValidatorOptions validatorOptions(opts->tmpVar(),
opts->getExpressionCtx());

Expand All @@ -437,13 +437,14 @@ std::unique_ptr<ExecutionBlock> ShortestPathNode::createBlock(
SingleServerBaseProviderOptions forwardProviderOptions(
opts->tmpVar(), std::move(usedIndexes), opts->getExpressionCtx(), {},
opts->collectionToShard(), opts->getVertexProjections(),
opts->getEdgeProjections(), opts->produceVertices(), opts->useCache());
opts->getEdgeProjections(), opts->produceVertices(), opts->useCache(),
opts->query());

SingleServerBaseProviderOptions backwardProviderOptions(
opts->tmpVar(), std::move(reversedUsedIndexes),
opts->getExpressionCtx(), {}, opts->collectionToShard(),
opts->getVertexProjections(), opts->getEdgeProjections(),
opts->produceVertices(), opts->useCache());
opts->produceVertices(), opts->useCache(), opts->query());

auto usesWeight =
checkWeight(forwardProviderOptions, backwardProviderOptions);
Expand Down Expand Up @@ -510,10 +511,10 @@ std::unique_ptr<ExecutionBlock> ShortestPathNode::createBlock(
using ClusterProvider = ClusterProvider<ClusterProviderStep>;
auto cache = std::make_shared<RefactoredClusterTraverserCache>(
opts->query().resourceMonitor());
ClusterBaseProviderOptions forwardProviderOptions(cache, engines(), false,
opts->produceVertices());
ClusterBaseProviderOptions backwardProviderOptions(cache, engines(), true,
opts->produceVertices());
ClusterBaseProviderOptions forwardProviderOptions(
cache, engines(), false, opts->produceVertices(), opts->query());
ClusterBaseProviderOptions backwardProviderOptions(
cache, engines(), true, opts->produceVertices(), opts->query());

auto usesWeight =
checkWeight(forwardProviderOptions, backwardProviderOptions);
Expand Down
10 changes: 6 additions & 4 deletions arangod/Aql/ExecutionNode/TraversalNode.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -817,8 +817,8 @@ std::unique_ptr<ExecutionBlock> TraversalNode::createBlock(
bool isSmart) const {
TraverserOptions* opts = this->options();

arangodb::graph::OneSidedEnumeratorOptions options{opts->minDepth,
opts->maxDepth};
arangodb::graph::OneSidedEnumeratorOptions options{
opts->minDepth, opts->maxDepth, opts->query()};
/*
* PathValidator Disjoint Helper (TODO [GraphRefactor]: Copy from createBlock)
* Clean this up as soon we clean up the whole TraversalNode as well.
Expand Down Expand Up @@ -915,7 +915,8 @@ ClusterBaseProviderOptions TraversalNode::getClusterBaseProviderOptions(
opts->produceVertices(),
&opts->getExpressionCtx(),
filterConditionVariables,
std::move(availableDepthsSpecificConditions)};
std::move(availableDepthsSpecificConditions),
opts->query()};
}

SingleServerBaseProviderOptions
Expand All @@ -936,7 +937,8 @@ TraversalNode::getSingleServerBaseProviderOptions(
opts->getVertexProjections(),
opts->getEdgeProjections(),
opts->produceVertices(),
opts->useCache()};
opts->useCache(),
opts->query()};
}

/// @brief creates corresponding ExecutionBlock
Expand Down
7 changes: 5 additions & 2 deletions arangod/Graph/Enumerators/OneSidedEnumerator.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -107,8 +107,11 @@ void OneSidedEnumerator<Configuration>::clearProvider() {
}

template<class Configuration>
auto OneSidedEnumerator<Configuration>::computeNeighbourhoodOfNextVertex()
-> void {
void OneSidedEnumerator<Configuration>::computeNeighbourhoodOfNextVertex() {
if (_options.isKilled()) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_QUERY_KILLED);
}

// Pull next element from Queue
// Do 1 step search
TRI_ASSERT(!_queue.isEmpty());
Expand Down
11 changes: 11 additions & 0 deletions arangod/Graph/Enumerators/TwoSidedEnumerator.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,10 @@ template<class QueueType, class PathStoreType, class ProviderType,
auto TwoSidedEnumerator<QueueType, PathStoreType, ProviderType, PathValidator>::
Ball::computeNeighbourhoodOfNextVertex(Ball& other, ResultList& results)
-> void {
if (_graphOptions.isKilled()) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_QUERY_KILLED);
}

// Pull next element from Queue
// Do 1 step search
TRI_ASSERT(!_queue.isEmpty());
Expand Down Expand Up @@ -451,6 +455,13 @@ void TwoSidedEnumerator<QueueType, PathStoreType, ProviderType,
PathValidator>::searchMoreResults() {
while (_results.empty() && !searchDone()) {
_resultsFetched = false;

// Check for kill signal before proceeding
// We will also do additional checks in computeNeighbourhoodOfNextVertex
if (_options.isKilled()) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_QUERY_KILLED);
}

if (_searchLeft) {
if (ADB_UNLIKELY(_left.doneWithDepth())) {
startNextDepth();
Expand Down
11 changes: 9 additions & 2 deletions arangod/Graph/Enumerators/WeightedTwoSidedEnumerator.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,10 @@ auto WeightedTwoSidedEnumerator<QueueType, PathStoreType, ProviderType,
PathValidator>::Ball::
computeNeighbourhoodOfNextVertex(Ball& other, CandidatesStore& candidates)
-> void {
if (_graphOptions.isKilled()) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_QUERY_KILLED);
}

ensureQueueHasProcessableElement();
auto tmp = _queue.pop();

Expand Down Expand Up @@ -466,7 +470,7 @@ void WeightedTwoSidedEnumerator<QueueType, PathStoreType, ProviderType,
template<class QueueType, class PathStoreType, class ProviderType,
class PathValidator>
bool WeightedTwoSidedEnumerator<QueueType, PathStoreType, ProviderType,
PathValidator>::isDone() const {
PathValidator>::isDone() {
if (!_candidatesStore.isEmpty()) {
return false;
}
Expand Down Expand Up @@ -861,7 +865,10 @@ WeightedTwoSidedEnumerator<QueueType, PathStoreType, ProviderType,
template<class QueueType, class PathStoreType, class ProviderType,
class PathValidator>
auto WeightedTwoSidedEnumerator<QueueType, PathStoreType, ProviderType,
PathValidator>::searchDone() const -> bool {
PathValidator>::searchDone() -> bool {
if (_options.isKilled()) {
THROW_ARANGO_EXCEPTION(TRI_ERROR_QUERY_KILLED);
}
if ((_left.noPathLeft() && _right.noPathLeft()) || isAlgorithmFinished()) {
return true;
}
Expand Down
4 changes: 2 additions & 2 deletions arangod/Graph/Enumerators/WeightedTwoSidedEnumerator.h
Original file line number Diff line number Diff line change
Expand Up @@ -345,7 +345,7 @@ class WeightedTwoSidedEnumerator {
* @return true There will be no further path.
* @return false There is a chance that there is more data available.
*/
[[nodiscard]] bool isDone() const;
[[nodiscard]] bool isDone();

/**
* @brief Reset to new source and target vertices.
Expand Down Expand Up @@ -410,7 +410,7 @@ class WeightedTwoSidedEnumerator {
_right.setForbiddenEdges(std::move(forbidden));
};

private : [[nodiscard]] auto searchDone() const -> bool;
private : [[nodiscard]] auto searchDone() -> bool;
// Ensure that we have fetched all vertices in the _results list. Otherwise,
// we will not be able to generate the resulting path
auto fetchResults() -> void;
Expand Down
5 changes: 3 additions & 2 deletions arangod/Graph/Options/OneSidedEnumeratorOptions.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,9 @@ using namespace arangodb;
using namespace arangodb::graph;

OneSidedEnumeratorOptions::OneSidedEnumeratorOptions(size_t minDepth,
size_t maxDepth)
: _minDepth(minDepth), _maxDepth(maxDepth) {}
size_t maxDepth,
aql::QueryContext& query)
: _minDepth(minDepth), _maxDepth(maxDepth), _observer(query) {}

OneSidedEnumeratorOptions::~OneSidedEnumeratorOptions() = default;

Expand Down
8 changes: 7 additions & 1 deletion arangod/Graph/Options/OneSidedEnumeratorOptions.h
Original file line number Diff line number Diff line change
Expand Up @@ -24,20 +24,26 @@

#pragma once

#include "Graph/Options/QueryContextObserver.h"
// TODO HEIKO: do not include h file here, use forward declaration instead
// TODO HEIKO: do not include h file here, use forward declaration instead
#include <cstddef>

namespace arangodb::graph {

struct OneSidedEnumeratorOptions {
public:
OneSidedEnumeratorOptions(size_t minDepth, size_t maxDepth);
OneSidedEnumeratorOptions(size_t minDepth, size_t maxDepth,
aql::QueryContext& query);
~OneSidedEnumeratorOptions();

[[nodiscard]] size_t getMinDepth() const noexcept;
[[nodiscard]] size_t getMaxDepth() const noexcept;
[[nodiscard]] bool isKilled() const noexcept { return _observer.isKilled(); }

private:
size_t const _minDepth;
size_t const _maxDepth;
QueryContextObserver _observer;
};
} // namespace arangodb::graph
52 changes: 52 additions & 0 deletions arangod/Graph/Options/QueryContextObserver.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2014-2024 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Business Source License 1.1 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// https://github.com/arangodb/arangodb/blob/devel/LICENSE
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author Michael Hackstein
/// @author Heiko Kernbach
////////////////////////////////////////////////////////////////////////////////

#pragma once

#include "Aql/QueryContext.h"

// This class serves as a wrapper around QueryContext to explicitly track where
// query killing is being used in the graph traversal code. It provides a single
// point of access to check if a query has been killed, making it easier to
// maintain and modify the query killing behavior if needed.
//
// While this adds a small layer of indirection, it helps with code clarity and
// maintainability. If profiling shows this wrapper causes significant overhead,
// we can remove it and use QueryContext directly.
//
// We can change this or discuss if this approach is not liked.

namespace arangodb::graph {

class QueryContextObserver {
public:
explicit QueryContextObserver(aql::QueryContext& query) : _query(query) {}

[[nodiscard]] bool isKilled() const { return _query.killed(); }

private:
aql::QueryContext& _query;
};

} // namespace arangodb::graph
8 changes: 6 additions & 2 deletions arangod/Graph/Options/TwoSidedEnumeratorOptions.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,12 @@ using namespace arangodb::graph;

TwoSidedEnumeratorOptions::TwoSidedEnumeratorOptions(size_t minDepth,
size_t maxDepth,
PathType::Type pathType)
: _minDepth(minDepth), _maxDepth(maxDepth), _pathType(pathType) {
PathType::Type pathType,
aql::QueryContext& query)
: _minDepth(minDepth),
_maxDepth(maxDepth),
_pathType(pathType),
_observer(query) {
if (getPathType() == PathType::Type::AllShortestPaths) {
setStopAtFirstDepth(true);
} else if (getPathType() == PathType::Type::ShortestPath) {
Expand Down
10 changes: 9 additions & 1 deletion arangod/Graph/Options/TwoSidedEnumeratorOptions.h
Original file line number Diff line number Diff line change
Expand Up @@ -25,17 +25,23 @@
#pragma once

#include "Graph/PathType.h"
#include "Graph/Options/QueryContextObserver.h"

#include <numeric>
#include <cstddef>

namespace arangodb {

namespace aql {
class QueryContext;
}

namespace graph {

struct TwoSidedEnumeratorOptions {
public:
TwoSidedEnumeratorOptions(size_t minDepth, size_t maxDepth,
PathType::Type pathType);
PathType::Type pathType, aql::QueryContext& query);

~TwoSidedEnumeratorOptions();

Expand All @@ -46,6 +52,7 @@ struct TwoSidedEnumeratorOptions {
[[nodiscard]] PathType::Type getPathType() const;
[[nodiscard]] bool getStopAtFirstDepth() const;
[[nodiscard]] bool onlyProduceOnePath() const;
[[nodiscard]] bool isKilled() const noexcept { return _observer.isKilled(); }

void setStopAtFirstDepth(bool stopAtFirstDepth);
void setOnlyProduceOnePath(bool onlyProduceOnePath);
Expand All @@ -57,6 +64,7 @@ struct TwoSidedEnumeratorOptions {
bool _stopAtFirstDepth{false};
bool _onlyProduceOnePath{false};
PathType::Type _pathType;
QueryContextObserver _observer;
};
} // namespace graph
} // namespace arangodb
Loading
0