8000 Feature/new server infra by maierlars · Pull Request #7733 · arangodb/arangodb · GitHub
[go: up one dir, main page]

Skip to content

Feature/new server infra #7733

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 42 commits into from
Jan 8, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
6f3593b
Decoupled IO handling from Scheduler.
Jul 23, 2018
1bd0db0
Fixed SSL start up bug.
Jul 24, 2018
0a3fe50
Replaced Scheduler with new worker farm implementation.
Jul 25, 2018
0400054
Added minimal statistics and info string for Scheduler.
Jul 25, 2018
bc0f94c
Added support for timed submissions.
Jul 27, 2018
fdd2033
Updated delayed submission api. Updated code that used timers.
Jul 31, 2018
3cb9b62
Extracted new Scheduler into a virtual parent class. The implementati…
Aug 1, 2018
bdc8d9b
Signal handler now working.
Aug 1, 2018
3796c3b
Changed threads names, `_stop` is atomic, check for failure during th…
Aug 2, 2018
5e6a3f2
Commented on source code and added TODOs.
Aug 3, 2018
e9ccd72
Merge branch 'devel' into feature/new-server-infra
Aug 3, 2018
0e99af6
Played around with start-stop-conditions
Aug 8, 2018
6fd7f26
Play around with start stop condition.
Aug 8, 2018
e65585c
start stop cond
Aug 8, 2018
c6ac00f
Sart Stop Conditions
Aug 8, 2018
a5516b2
Removed bad cv_status check.
Aug 9, 2018
3e12cb0
Merge remote-tracking branch 'origin/devel' into feature/new-server-i…
Aug 10, 2018
aa6672c
Merge remote-tracking branch 'origin/devel' into feature/new-server-i…
Aug 10, 2018
43fb2d9
Bug fix: now compare the actual objects instead of pointer values. Se…
Aug 10, 2018
c6a54c1
Moved most of the st 8000 uff now unrelated to the Scheduler to GeneralServ…
Aug 13, 2018
556f3e9
Instead of waiting for a thread to terminate, put it on a clean up li…
Aug 15, 2018
0ed5f82
Allow detaching long running threads.
Aug 15, 2018
20132a0
Merge remote-tracking branch 'origin/devel' into feature/new-server-i…
Dec 11, 2018
e2fc7e1
Fixed test mock.
Dec 12, 2018
ebcd63a
Merge remote-tracking branch 'origin/devel' into feature/new-server-i…
Dec 17, 2018
d186d14
Updated the WorkHandle logic. Removed post functions.
Dec 18, 2018
80c9f6a
Fixed crash when obtaining shared_ptr from this in destructor.
Dec 18, 2018
fdff089
Merge remote-tracking branch 'origin/devel' into feature/new-server-i…
Dec 18, 2018
a61a809
Added lost mutex.
Dec 18, 2018
8d4c003
Fixed memory leak.
Dec 18, 2018
3ff4806
Fixed merge bug.
Dec 18, 2018
e2fa338
Changed a lot of code to optimize the scheduler.
Dec 19, 2018
090e2cd
Merge remote-tracking branch 'origin/devel' into feature/new-server-i…
Dec 19, 2018
b37d5fd
Fixed bug of invalidated iterator. Dont remove task on shutdown at di…
Dec 19, 2018
2ccf743
Only by value calls to queue.
Dec 20, 2018
947fe2f
Added options again.
Dec 20, 2018
992e44c
Clean up of code.
Dec 20, 2018
b6bbc8c
UI Request Lane added.
Dec 20, 2018
1e8d3bd
Bug fixes in Scheduler.
Dec 21, 2018
4d6b0a0
Merge remote-tracking branch 'origin/devel' into feature/new-server-i…
Jan 7, 2019
83a775e
Applied reformat.
Jan 7, 2019
55e70a3
Use sigaction.
Jan 7, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions arangod/Aql/AqlFunctionFeature.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -428,8 +428,8 @@ void AqlFunctionFeature::addMiscFunctions() {
&Functions::CurrentUser}); // deterministic, but not cacheable
add({"CURRENT_DATABASE", "", Function::makeFlags(FF::Deterministic),
&Functions::CurrentDatabase}); // deterministic, but not cacheable
add({"CHECK_DOCUMENT", ".", Function::makeFlags(FF::CanRunOnDBServer),
&Functions::CheckDocument}); // not deterministic and not cacheable
add({"CHECK_DOCUMENT", ".", Function::makeFlags(FF::CanRunOnDBServer),
&Functions::CheckDocument}); // not deterministic and not cacheable
add({"COLLECTION_COUNT", ".h", Function::makeFlags(), &Functions::CollectionCount}); // not deterministic and not cacheable
add({"PREGEL_RESULT", ".", Function::makeFlags(FF::CanRunOnDBServer),
&Functions::PregelResult}); // not deterministic and not cacheable
Expand Down
2 changes: 1 addition & 1 deletion arangod/Aql/AstNode.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -978,7 +978,7 @@ void AstNode::toVelocyPackValue(VPackBuilder& builder) const {

if (type == NODE_TYPE_OBJECT) {
builder.openObject();

std::unordered_set<VPackStringRef> keys;
size_t const n = numMembers();

Expand Down
1 change: 0 additions & 1 deletion arangod/Aql/ClusterBlocks.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@
#include "Cluster/ClusterComm.h"
#include "Cluster/ClusterInfo.h"
#include "Cluster/ServerState.h"
#include "Scheduler/JobGuard.h"
#include "Scheduler/SchedulerFeature.h"
#include "Transaction/Methods.h"
#include "Transaction/StandaloneContext.h"
Expand Down
5 changes: 3 additions & 2 deletions arangod/Aql/Expression.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -672,7 +672,7 @@ AqlValue Expression::executeSimpleExpressionObject(AstNode const* node,
if (mustCheckUniqueness) {
std::string key(member->getString());

// track each individual object key
// track each individual object key
auto it = keys.find(key);

if (it != keys.end()) {
Expand All @@ -687,7 +687,8 @@ AqlValue Expression::executeSimpleExpressionObject(AstNode const* node,
keys.emplace(std::move(key));
}
} else {
builder->add(VPackValuePair(member->getStringValue(), member->getStringLength(), VPackValueType::String));
builder->add(VPackValuePair(member->getStringValue(),
member->getStringLength(), VPackValueType::String));
}

// value
Expand Down
7 changes: 3 additions & 4 deletions arangod/Aql/Functions.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -217,9 +217,9 @@ bool isValidDocument(VPackSlice slice) {

if (slice.isObject()) {
std::unordered_set<VPackStringRef> keys;

auto it = VPackObjectIterator(slice, true);

while (it.valid()) {
if (!keys.emplace(it.key().stringRef()).second) {
// duplicate key
Expand Down Expand Up @@ -6769,8 +6769,7 @@ AqlValue Functions::CollectionCount(ExpressionContext*, transaction::Methods* tr
}

/// @brief function CHECK_DOCUMENT
AqlValue Functions::CheckDocument(ExpressionContext*,
transaction::Methods* trx,
AqlValue Functions::CheckDocument(ExpressionContext*, transaction::Methods* trx,
VPackFunctionParameters const& parameters) {
AqlValue const& value = extractFunctionParameterValue(parameters, 0);
if (!value.isObject()) {
Expand Down
1 change: 0 additions & 1 deletion arangod/Aql/RestAqlHandler.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@
#include "Logger/Logger.h"
#include "Rest/HttpRequest.h"
#include "Rest/HttpResponse.h"
#include "Scheduler/JobGuard.h"
#include "Scheduler/Scheduler.h"
#include "Scheduler/SchedulerFeature.h"
#include "Transaction/Methods.h"
Expand Down
2 changes: 1 addition & 1 deletion arangod/Aql/Scopes.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ void Scopes::addVariable(Variable* variable) {
void Scopes::replaceVariable(Variable* variable) {
TRI_ASSERT(!_activeScopes.empty());
TRI_ASSERT(variable != nullptr);

_activeScopes.back()->addVariable(variable);
}

Expand Down
2 changes: 1 addition & 1 deletion arangod/Aql/SharedQueryState.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,6 @@ bool SharedQueryState::executeContinueCallback() const {
}
// do NOT use scheduler->post(), can have high latency that
// then backs up libcurl callbacks to other objects
scheduler->queue(RequestPriority::HIGH, _continueCallback);
scheduler->queue(RequestLane::CLIENT_AQL, _continueCallback);
return true;
}
79 changes: 42 additions & 37 deletions arangod/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -142,8 +142,8 @@ SET(ARANGOD_SOURCES
${ProductVersionFiles}
${ENTERPRISE_FILES}
Actions/ActionFeature.cpp
Actions/RestActionHandler.cpp
Actions/actions.cpp
Actions/RestActionHandler.cpp
Agency/ActiveFailoverJob.cpp
Agency/AddFollower.cpp
Agency/AgencyComm.cpp
Expand Down Expand Up @@ -190,11 +190,11 @@ SET(ARANGOD_SOURCES
Aql/ClusterBlocks.cpp
Aql/ClusterNodes.cpp
Aql/CollectBlock.cpp
Aql/CollectNode.cpp
Aql/CollectOptions.cpp
Aql/Collection.cpp
Aql/Collections.cpp
Aql/CollectionAccessingNode.cpp
Aql/Collections.cpp
Aql/CollectNode.cpp
Aql/CollectOptions.cpp
Aql/Condition.cpp
Aql/ConditionFinder.cpp
Aql/DocumentProducingBlock.cpp
Expand All @@ -212,8 +212,9 @@ SET(ARANGOD_SOURCES
Aql/FixedVarExpressionContext.cpp
Aql/Function.cpp
Aql/Functions.cpp
Aql/Graphs.cpp
Aql/grammar.cpp
Aql/GraphNode.cpp
Aql/Graphs.cpp
Aql/IndexBlock.cpp
Aql/IndexNode.cpp
Aql/ModificationBlocks.cpp
Expand All @@ -223,19 +224,19 @@ SET(ARANGOD_SOURCES
Aql/Optimizer.cpp
Aql/OptimizerRules.cpp
Aql/OptimizerRulesCluster.cpp
Aql/OptimizerRulesReplaceFunctions.cpp
Aql/OptimizerRulesFeature.cpp
Aql/OptimizerRulesReplaceFunctions.cpp
Aql/Parser.cpp
Aql/PlanCache.cpp
Aql/QueryExecutionState.cpp
Aql/QueryProfile.cpp
Aql/Quantifier.cpp
Aql/Query.cpp
Aql/QueryCache.cpp
Aql/QueryCursor.cpp
Aql/QueryExecutionState.cpp
Aql/QueryExpressionContext.cpp
Aql/QueryList.cpp
Aql/QueryOptions.cpp
Aql/QueryProfile.cpp
Aql/QueryRegistry.cpp
Aql/QueryResources.cpp
Aql/QueryString.cpp
Expand All @@ -244,32 +245,31 @@ SET(ARANGOD_SOURCES
Aql/RestAqlHandler.cpp
Aql/Scopes.cpp
Aql/SharedQueryState.cpp
Aql/ShortStringStorage.cpp
Aql/ShortestPathBlock.cpp
Aql/ShortestPathNode.cpp
Aql/ShortStringStorage.cpp
Aql/SortBlock.cpp
Aql/SortCondition.cpp
Aql/SortNode.cpp
Aql/SortRegister.cpp
Aql/SubqueryBlock.cpp
Aql/tokens.cpp
Aql/TraversalBlock.cpp
Aql/TraversalConditionFinder.cpp
Aql/TraversalNode.cpp
Aql/V8Executor.cpp
Aql/Variable.cpp
Aql/VariableGenerator.cpp
Aql/WakeupQueryCallback.cpp
Aql/grammar.cpp
Aql/tokens.cpp
Auth/Common.cpp
Auth/TokenCache.cpp
Auth/User.cpp
Auth/UserManager.cpp
Cache/BucketState.cpp
Cache/Cache.cpp
Cache/CachedValue.cpp
Cache/CacheManagerFeature.cpp
Cache/CacheManagerFeatureThreads.cpp
Cache/CachedValue.cpp
Cache/Finding.cpp
Cache/Manager.cpp
Cache/ManagerTasks.cpp
Expand Down Expand Up @@ -322,6 +322,8 @@ SET(ARANGOD_SOURCES
Cluster/TraverserEngineRegistry.cpp
Cluster/UpdateCollection.cpp
Cluster/v8-cluster.cpp
GeneralServer/Acceptor.cpp
GeneralServer/AcceptorTcp.cpp
GeneralServer/AsyncJobManager.cpp
GeneralServer/AuthenticationFeature.cpp
GeneralServer/GeneralCommTask.cpp
Expand All @@ -330,16 +332,20 @@ SET(ARANGOD_SOURCES
GeneralServer/GeneralServerFeature.cpp
GeneralServer/HttpCommTask.cpp
GeneralServer/IoTask.cpp
GeneralServer/ListenTask.cpp
GeneralServer/RestHandler.cpp
GeneralServer/RestHandlerFactory.cpp
GeneralServer/SocketSslTcp.cpp
GeneralServer/SocketTask.cpp
GeneralServer/Task.cpp
GeneralServer/VstCommTask.cpp
GeoIndex/Index.cpp
GeoIndex/Near.cpp
Graph/AttributeWeightShortestPathFinder.cpp
Graph/BaseOptions.cpp
Graph/BreadthFirstEnumerator.cpp
Graph/ConstantWeightShortestPathFinder.cpp
Graph/ClusterTraverserCache.cpp
Graph/ConstantWeightShortestPathFinder.cpp
Graph/EdgeCollectionInfo.cpp
Graph/Graph.cpp
Graph/GraphManager.cpp
Expand All @@ -350,10 +356,10 @@ SET(ARANGOD_SOURCES
Graph/ShortestPathResult.cpp
Graph/SingleServerEdgeCursor.cpp
Graph/SingleServerTraverser.cpp
Graph/Traverser.cpp
Graph/TraverserCache.cpp
Graph/TraverserCacheFactory.cpp
Graph/TraverserDocumentCache.cpp
Graph/Traverser.cpp
Graph/TraverserOptions.cpp
Indexes/Index.cpp
Indexes/IndexFactory.cpp
Expand All @@ -366,35 +372,36 @@ SET(ARANGOD_SOURCES
Pregel/AlgoRegistry.cpp
Pregel/Algos/AsyncSCC.cpp
Pregel/Algos/ConnectedComponents.cpp
Pregel/Algos/DMID/DMID.cpp
Pregel/Algos/EffectiveCloseness/EffectiveCloseness.cpp
Pregel/Algos/EffectiveCloseness/HLLCounter.cpp
Pregel/Algos/HITS.cpp
Pregel/Algos/LabelPropagation.cpp
Pregel/Algos/LineRank.cpp
Pregel/Algos/PageRank.cpp
Pregel/Algos/RecoveringPageRank.cpp
Pregel/Algos/ShortestPath.cpp
Pregel/Algos/SSSP.cpp
Pregel/Algos/SCC.cpp
Pregel/Algos/ShortestPath.cpp
Pregel/Algos/SLPA.cpp
Pregel/Algos/LabelPropagation.cpp
Pregel/Algos/DMID/DMID.cpp
Pregel/Algos/SSSP.cpp
Pregel/Conductor.cpp
Pregel/GraphStore.cpp
Pregel/IncomingCache.cpp
Pregel/OutgoingCache.cpp
Pregel/PregelFeature.cpp
Pregel/Recovery.cpp
Pregel/Utils.cpp
Pregel/Worker.cpp
Pregel/Worker-templates-native-types.cpp
Pregel/Worker-templates-algorithms.cpp
Pregel/Worker-templates-native-types.cpp
Pregel/Worker.cpp
Pregel/WorkerConfig.cpp
Replication/DatabaseReplicationApplier.cpp
Replication/GlobalReplicationApplier.cpp
Replication/GlobalInitialSyncer.cpp
Replication/common-defines.cpp
Replication/DatabaseInitialSyncer.cpp
Replication/DatabaseReplicationApplier.cpp
Replication/DatabaseTailingSyncer.cpp
Replication/GlobalInitialSyncer.cpp
Replication/GlobalInitialSyncer.cpp
Replication/GlobalRep 10000 licationApplier.cpp
Replication/GlobalTailingSyncer.cpp
Replication/InitialSyncer.cpp
Replication/ReplicationApplier.cpp
Expand All @@ -403,7 +410,6 @@ SET(ARANGOD_SOURCES
Replication/ReplicationFeature.cpp
Replication/Syncer.cpp
Replication/TailingSyncer.cpp
Replication/common-defines.cpp
Replication/utilities.cpp
RestHandler/RestAdminDatabaseHandler.cpp
RestHandler/RestAdminLogHandler.cpp
Expand Down Expand Up @@ -472,15 +478,9 @@ SET(ARANGOD_SOURCES
RestServer/UpgradeFeature.cpp
RestServer/ViewTypesFeature.cpp
RestServer/VocbaseContext.cpp
Scheduler/Acceptor.cpp
Scheduler/AcceptorTcp.cpp
Scheduler/JobGuard.cpp
Scheduler/ListenTask.cpp
Scheduler/Scheduler.cpp
Scheduler/SchedulerFeature.cpp
Scheduler/SocketSslTcp.cpp
Scheduler/SocketTask.cpp
Scheduler/Task.cpp
Scheduler/SupervisedScheduler.cpp
Sharding/ShardDistributionReporter.cpp
Sharding/ShardingFeature.cpp
Sharding/ShardingInfo.cpp
Expand Down Expand Up @@ -516,24 +516,29 @@ SET(ARANGOD_SOURCES
Utils/OperationCursor.cpp
Utils/SingleCollectionTransaction.cpp
V8Server/FoxxQueuesFeature.cpp
V8Server/V8Context.cpp
V8Server/V8DealerFeature.cpp
V8Server/v8-actions.cpp
V8Server/v8-collection-util.cpp
V8Server/v8-collection.cpp
V8Server/v8-dispatcher.cpp
V8Server/v8-general-graph.cpp
V8Server/v8-query.cpp
V8Server/v8-replication.cpp
V8Server/v8-statistics.cpp
V8Server/v8-users.cpp
V8Server/v8-user-structures.cpp
V8Server/v8-user-functions.cpp
V8Server/v8-user-structures.cpp
V8Server/v8-users.cpp
V8Server/v8-util.cpp
V8Server/v8-views.cpp
V8Server/v8-vocbase.cpp
V8Server/v8-voccursor.cpp
V8Server/v8-vocindex.cpp
V8Server/v8-general-graph.cpp
V8Server/V8Context.cpp
V8Server/V8DealerFeature.cpp
VocBase/KeyGenerator.cpp
VocBase/LogicalCollection.cpp
VocBase/LogicalDataSource.cpp
VocBase/LogicalView.cpp
VocBase/ManagedDocumentResult.cpp
VocBase/Methods/AqlUserFunctions.cpp
VocBase/Methods/Collections.cpp
VocBase/Methods/Databases.cpp
Expand All @@ -559,7 +564,7 @@ if (USE_MAINTAINER_MODE)
endif()

if (NOT MSVC)
set(ARANGOD_SOURCES ${ARANGOD_SOURCES} Scheduler/AcceptorUnixDomain.cpp Scheduler/SocketUnixDomain.cpp)
set(ARANGOD_SOURCES ${ARANGOD_SOURCES} GeneralServer/AcceptorUnixDomain.cpp GeneralServer/SocketUnixDomain.cpp)
endif()

include(ClusterEngine/CMakeLists.txt)
Expand Down
3 changes: 1 addition & 2 deletions arangod/Cache/CacheManagerFeature.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@ using namespace arangodb::application_features;
using namespace arangodb::basics;
using namespace arangodb::cache;
using namespace arangodb::options;
using namespace arangodb::rest;

namespace arangodb {

Expand Down Expand Up @@ -102,7 +101,7 @@ void CacheManagerFeature::start() {

auto scheduler = SchedulerFeature::SCHEDULER;
auto postFn = [scheduler](std::function<void()> fn) -> bool {
scheduler->queue(RequestPriority::LOW, fn);
scheduler->queue(RequestLane::INTERNAL_LOW, fn);
return true;
};
_manager.reset(new Manager(postFn, _cacheSize));
Expand Down
Loading
0