8000 Feature/hybrid smart graph api (#14037) · arangodb/arangodb@e951cba · GitHub
[go: up one dir, main page]
Skip to content

Commit e951cba

Browse files
hkernbachhkernbach
authored
Feature/hybrid smart graph api (#14037)
Added First Draft of Hybrid Smart Graph API
1 parent d396a1f commit e951cba

File tree

8 files changed

+150
-16
lines changed
  • Graph.h
  • GraphManager.cpp
  • GraphOperations.cpp
  • lib/Basics

    • 8 files changed

    +150
    -16
    lines changed

    ‎arangod/Aql/GraphNode.cpp

    Lines changed: 3 additions & 0 deletions
    Original file line numberDiff line numberDiff line change
    @@ -107,6 +107,7 @@ GraphNode::GraphNode(ExecutionPlan* plan, ExecutionNodeId id,
    107107
    _optionsBuilt(false),
    108108
    _isSmart(false),
    109109
    _isDisjoint(false),
    110+
    _isHybrid(false),
    110111
    _options(std::move(options)) {
    111112
    // Direction is already the correct Integer.
    112113
    // Is not inserted by user but by enum.
    @@ -810,6 +811,8 @@ bool GraphNode::isSmart() const { return _isSmart; }
    810811

    811812
    bool GraphNode::isDisjoint() const { return _isDisjoint; }
    812813

    814+
    bool GraphNode::isHybrid() const { return _isHybrid; }
    815+
    813816
    TRI_vocbase_t* GraphNode::vocbase() const { return _vocbase; }
    814817

    815818
    Variable const* GraphNode::vertexOutVariable() const {

    ‎arangod/Aql/GraphNode.h

    Lines changed: 6 additions & 0 deletions
    Original file line numberDiff line numberDiff line change
    @@ -123,6 +123,9 @@ class GraphNode : public ExecutionNode {
    123123
    /// @brief flag, if the graph is a Disjoint SmartGraph (Enterprise Edition only!)
    124124
    bool isDisjoint() const;
    125125

    126+
    /// @brief flag, if the graph is a Hybrid SmartGraph (Enterprise Edition only!)
    127+
    bool isHybrid() const;
    128+
    126129
    /// @brief return the database
    127130
    TRI_vocbase_t* vocbase() const;
    128131

    @@ -256,6 +259,9 @@ class GraphNode : public ExecutionNode {
    256259
    /// @brief flag, if graph is smart *and* disjoint (Enterprise Edition only!)
    257260
    bool _isDisjoint;
    258261

    262+
    /// @brief flag, if graph is smart *and* hybrid (Enterprise Edition only!)
    263+
    bool _isHybrid;
    264+
    259265
    /// @brief The directions edges are followed
    260266
    std::vector<TRI_edge_direction_e> _directions;
    261267

    ‎arangod/Graph/Graph.cpp

    Lines changed: 62 additions & 2 deletions
    Original file line numberDiff line numberDiff line change
    @@ -116,6 +116,19 @@ Graph::Graph(velocypack::Slice const& slice, ServerDefaults const& serverDefault
    116116
    TRI_ASSERT(!_rev.empty());
    117117

    118118
    if (slice.hasKey(StaticStrings::GraphEdgeDefinitions)) {
    119+
    // TODO Feature HybridSmartGraphs: Check why we're landing here and not in SmartGraphEE - Cleanup!
    120+
    if (slice.isObject()) {
    121+
    if (slice.hasKey(StaticStrings::GraphSatellites) &&
    122+
    slice.get(StaticStrings::GraphSatellites).isArray()) {
    123+
    124+
    for (VPackSlice it : VPackArrayIterator(slice.get(StaticStrings::GraphSatellites))) {
    125+
    if (!it.isString()) {
    126+
    THROW_ARANGO_EXCEPTION(TRI_ERROR_GRAPH_INVALID_PARAMETER);
    127+
    }
    128+
    _satelliteColls.emplace(it.copyString());
    129+
    }
    130+
    }
    131+
    }
    119132
    parseEdgeDefinitions(slice.get(StaticStrings::GraphEdgeDefinitions));
    120133
    }
    121134
    if (slice.hasKey(StaticStrings::GraphOrphans)) {
    @@ -146,6 +159,19 @@ Graph::Graph(TRI_vocbase_t& vocbase, std::string&& graphName,
    146159
    TRI_ASSERT(_rev.empty());
    147160

    148161
    if (info.hasKey(StaticStrings::GraphEdgeDefinitions)) {
    162+
    // TODO Feature HybridSmartGraphs: Check why we're landing here and not in SmartGraphEE - Cleanup!
    163+
    if (options.isObject()) {
    164+
    if (options.hasKey(StaticStrings::GraphSatellites) &&
    165+
    options.get(StaticStrings::GraphSatellites).isArray()) {
    166+
    167+
    for (VPackSlice it : VPackArrayIterator(options.get(StaticStrings::GraphSatellites))) {
    168+
    if (!it.isString()) {
    169+
    THROW_ARANGO_EXCEPTION(TRI_ERROR_GRAPH_INVALID_PARAMETER);
    170+
    }
    171+
    _satelliteColls.emplace(it.copyString());
    172+
    }
    173+
    }
    174+
    }
    149175
    parseEdgeDefinitions(info.get(StaticStrings::GraphEdgeDefinitions));
    150176
    }
    151177
    if (info.hasKey(StaticStrings::GraphOrphans)) {
    @@ -212,10 +238,18 @@ std::set<std::string> const& Graph::orphanCollections() const {
    212238
    return _orphanColls;
    213239
    }
    214240

    241+
    std::set<std::string> const& Graph::satelliteCollections() const {
    242+
    return _satelliteColls;
    243+
    }
    244+
    215245
    std::set<std::string> const& Graph::edgeCollections() const {
    216246
    return _edgeColls;
    217247
    }
    218248

    249+
    bool Graph::needsToBeSatellite(std::string const& edge) const {
    250+
    return false;
    251+
    }
    252+
    219253
    std::map<std::string, EdgeDefinition> const& Graph::edgeDefinitions() const {
    220254
    return _edgeDefs;
    221255
    }
    @@ -417,9 +451,10 @@ void EdgeDefinition::toVelocyPack(VPackBuilder& builder) const {
    417451
    builder.add(VPackValue(to));
    418452
    }
    419453
    builder.close(); // array
    454+
    builder.add("type", VPackValue(getType()));
    420455
    }
    421456

    422-
    ResultT<EdgeDefinition> EdgeDefinition::createFromVelocypack(VPackSlice edgeDefinition) {
    457+
    ResultT<EdgeDefinition> EdgeDefinition::createFromVelocypack(VPackSlice edgeDefinition, std::set<std::string> const& satCollections) {
    423458
    Result res = EdgeDefinition::validateEdgeDefinition(edgeDefinition);
    424459
    if (res.fail()) {
    425460
    return res;
    @@ -433,9 +468,11 @@ ResultT<EdgeDefinition> EdgeDefinition::createFromVelocypack(VPackSlice edgeDefi
    433468

    434469
    // duplicates in from and to shouldn't occur, but are safely ignored here
    435470
    for (VPackSlice it : VPackArrayIterator(from)) {
    471+
    TRI_ASSERT(it.isString());
    436472
    fromSet.emplace(it.copyString());
    437473
    }
    438474
    for (VPackSlice it : VPackArrayIterator(to)) {
    475+
    TRI_ASSERT(it.isString());
    439476
    toSet.emplace(it.copyString());
    440477
    }
    441478

    @@ -484,6 +521,19 @@ bool EdgeDefinition::renameCollection(std::string const& oldName, std::string co
    484521
    return renamed;
    485522
    }
    486523

    524+
    auto EdgeDefinition::getType() const -> EdgeDefinitionType {
    525+
    return _type;
    526+
    }
    527+
    528+
    auto EdgeDefinition::setType(EdgeDefinitionType type) -> bool {
    529+
    TRI_ASSERT(type != EdgeDefinitionType::DEFAULT);
    530+
    if (_type == EdgeDefinitionType::DEFAULT) {
    531+
    _type = type;
    532+
    return true;
    533+
    }
    534+
    return false;
    535+
    }
    536+
    487537
    bool EdgeDefinition::isFromVertexCollectionUsed(std::string const& collectionName) const {
    488538
    if (getFrom().find(collectionName) != getFrom().end()) {
    489539
    return true;
    @@ -522,6 +572,7 @@ void EdgeDefinition::addToBuilder(VPackBuilder& builder) const {
    522572
    builder.add(VPackValue(to));
    523573
    }
    524574
    builder.close(); // to
    575+
    builder.add(StaticStrings::GraphEdgeDefinitionType, VPackValue(getType()));
    525576

    526577
    builder.close(); // obj
    527578
    }
    @@ -590,7 +641,7 @@ ResultT<EdgeDefinition const*> Graph::addEdgeDefinition(EdgeDefinition const& ed
    590641
    }
    591642

    592643
    ResultT<EdgeDefinition const*> Graph::addEdgeDefinition(VPackSlice const& edgeDefinitionSlice) {
    593-
    auto res = EdgeDefinition::createFromVelocypack(edgeDefinitionSlice);
    644+
    auto res = EdgeDefinition::createFromVelocypack(edgeDefinitionSlice, satelliteCollections());
    594645

    595646
    if (res.fail()) {
    596647
    return std::move(res).result();
    @@ -724,6 +775,11 @@ bool Graph::isDisjoint() const {
    724775
    return false;
    725776
    }
    726777

    778+
    bool Graph::isHybrid() const {
    779+
    TRI_ASSERT(_satelliteColls.empty());
    780+
    return false;
    781+
    }
    782+
    727783
    bool Graph::isSatellite() const { return _isSatellite; }
    728784

    729785
    void Graph::createCollectionOptions(VPackBuilder& builder, bool waitForSync) const {
    @@ -743,6 +799,10 @@ void Graph::createCollectionOptions(VPackBuilder& builder, bool waitForSync) con
    743799
    builder.add(StaticStrings::ReplicationFactor, VPackValue(replicationFactor()));
    744800
    }
    745801

    802+
    void Graph::createSatelliteCollectionOptions(VPackBuilder& builder, bool waitForSync) const {
    803+
    TRI_ASSERT(false);
    804+
    }
    805+
    746806
    std::optional<std::reference_wrapper<const EdgeDefinition>> Graph::getEdgeDefinition(
    747807
    std::string const& collectionName) const {
    748808
    auto it = edgeDefinitions().find(collectionName);

    ‎arangod/Graph/Graph.h

    Lines changed: 38 additions & 3 deletions
    Original file line numberDiff line numberDiff line change
    @@ -46,12 +46,23 @@ struct ServerDefaults;
    4646
    namespace graph {
    4747

    4848
    class EdgeDefinition {
    49+
    public:
    50+
    enum EdgeDefinitionType {
    51+
    DEFAULT,
    52+
    SMART_TO_SMART,
    53+
    SAT_TO_SAT,
    54+
    SMART_TO_SAT,
    55+
    SAT_TO_SMART
    56+
    };
    57+
    4958
    public:
    5059
    EdgeDefinition(std::string edgeCollection_, std::set<std::string>&& from_,
    51-
    std::set<std::string>&& to_)
    60+
    std::set<std::string>&& to_,
    61+
    EdgeDefinitionType type = EdgeDefinitionType::DEFAULT)
    5262
    : _edgeCollection(std::move(edgeCollection_)),
    5363
    _from(std::move(from_)),
    54-
    _to(std::move(to_)) {}
    64+
    _to(std::move(to_)),
    65+
    _type(type) {}
    5566

    5667
    std::string const& getName() const { return _edgeCollection; }
    5768
    void setName(std::string const& newName) { _edgeCollection = newName; }
    @@ -70,7 +81,8 @@ class EdgeDefinition {
    7081
    /// types of values.
    7182
    static Result validateEdgeDefinition(const velocypack::Slice& edgeDefinition);
    7283

    73-
    static ResultT<EdgeDefinition> createFromVelocypack(velocypack::Slice edgeDefinition);
    84+
    static ResultT<EdgeDefinition> createFromVelocypack(velocypack::Slice edgeDefinition,
    85+
    std::set<std::string> const& satCollections);
    7486

    7587
    void toVelocyPack(velocypack::Builder&) const;
    7688

    @@ -82,10 +94,23 @@ class EdgeDefinition {
    8294

    8395
    bool renameCollection(std::string const& oldName, std::string const& newName);
    8496

    97+
    auto getType() const -> EdgeDefinitionType;
    98+
    99+
    /* @brief
    100+
    * Set type of the EdgeDefinition. Only allowed to be called once and only if
    101+
    * type is DEFAULT. If type has been set, it is not changeable anymore.
    102+
    *
    103+
    * @param type Type to be set
    104+
    *
    105+
    * @return True if type has been set, returns false in case type has not been set.
    106+
    */
    107+
    auto setType(EdgeDefinitionType type) -> bool;
    108+
    85109
    private:
    86110
    std::string _edgeCollection;
    87111
    std::set<std::string> _from;
    88112
    std::set<std::string> _to;
    113+
    EdgeDefinitionType _type;
    89114
    };
    90115

    91116
    class Graph {
    @@ -151,13 +176,18 @@ class Graph {
    151176

    152177
    virtual void createCollectionOptions(VPackBuilder& builder, bool waitForSync) const;
    153178

    179+
    virtual void createSatelliteCollectionOptions(VPackBuilder& builder, bool waitForSync) const;
    180+
    154181
    public:
    155182
    /// @brief get the cids of all vertexCollections
    156183
    std::set<std::string> const& vertexCollections() const;
    157184

    158185
    /// @brief get the cids of all orphanCollections
    159186
    std::set<std::string> const& orphanCollections() const;
    160187

    188+
    /// @brief get the cids of all satelliteCollections
    189+
    std::set<std::string> const& satelliteCollections() const;
    190+
    161191
    /// @brief get the cids of all edgeCollections
    162192
    std::set<std::string> const& edgeCollections() const;
    163193

    @@ -178,6 +208,8 @@ class Graph {
    178208
    virtual bool isSmart() const;
    179209
    virtual bool isDisjoint() const;
    180210
    virtual bool isSatellite() const;
    211+
    virtual bool isHybrid() const;
    212+
    virtual bool needsToBeSatellite(std::string const& edge) const;
    181213

    182214
    uint64_t numberOfShards() const;
    183215
    uint64_t replicationFactor() const;
    @@ -291,6 +323,9 @@ class Graph {
    291323
    /// @brief the names of all orphanCollections
    292324
    std::set<std::string> _orphanColls;
    293325

    326+
    /// @brief the names of all orphanCollections
    327+
    std::set<std::string> _satelliteColls;
    328+
    294329
    /// @brief the names of all edgeCollections
    295330
    std::set<std::string> _edgeColls;
    296331

    ‎arangod/Graph/GraphManager.cpp

    Lines changed: 34 additions & 8 deletions
    Original file line numberDiff line numberDiff line change
    @@ -459,8 +459,7 @@ OperationResult GraphManager::storeGraph(Graph const& graph, bool waitForSync,
    459459
    Result GraphManager::applyOnAllGraphs(std::function<Result(std::unique_ptr<Graph>)> const& callback) const {
    460460
    std::string const queryStr{"FOR g IN _graphs RETURN g"};
    461461
    arangodb::aql::Query query(transaction::StandaloneContext::Create(_vocbase),
    462-
    arangodb::aql::QueryString{queryStr},
    463-
    nullptr);
    462+
    arangodb::aql::QueryString{queryStr}, nullptr);
    464463
    query.queryOptions().skipAudit = true;
    465464
    aql::QueryResult queryResult = query.executeSync();
    466465

    @@ -505,7 +504,9 @@ Result GraphManager::applyOnAllGraphs(std::function<Result(std::unique_ptr<Graph
    505504
    Result GraphManager::ensureCollections(Graph* graph, bool waitForSync) const {
    506505
    // Validation Phase collect a list of collections to create
    507506
    std::unordered_set<std::string> documentCollectionsToCreate{};
    507+
    std::unordered_set<std::string> satelliteDocumentCollectionsToCreate{};
    508508
    std::unordered_set<std::string> edgeCollectionsToCreate{};
    509+
    std::unordered_set<std::string> satelliteEdgeCollectionsToCreate{};
    509510
    std::unordered_set<std::shared_ptr<LogicalCollection>> existentDocumentCollections{};
    510511
    std::unordered_set<std::shared_ptr<LogicalCollection>> existentEdgeCollections{};
    511512

    @@ -533,7 +534,11 @@ Result GraphManager::ensureCollections(Graph* graph, bool waitForSync) const {
    533534
    return res;
    534535
    } else {
    535536
    // not found the collection, need to create it later
    536-
    edgeCollectionsToCreate.emplace(edgeColl);
    537+
    if (graph->isHybrid() && graph->needsToBeSatellite(edgeColl)) { // check for satellites
    538+
    satelliteEdgeCollectionsToCreate.emplace(edgeColl);
    F438 539+
    } else {
    540+
    edgeCollectionsToCreate.emplace(edgeColl);
    541+
    }
    537542
    }
    538543
    }
    539544

    @@ -550,7 +555,11 @@ Result GraphManager::ensureCollections(Graph* graph, bool waitForSync) const {
    550555
    return res;
    551556
    } else {
    552557
    if (edgeCollectionsToCreate.find(vertexColl) == edgeCollectionsToCreate.end()) {
    553-
    documentCollectionsToCreate.emplace(vertexColl);
    558+
    if (graph->isHybrid() && graph->satelliteCollections().find(vertexColl) != graph->satelliteCollections().end()) {
    559+
    satelliteDocumentCollectionsToCreate.emplace(vertexColl);
    560+
    } else {
    561+
    documentCollectionsToCreate.emplace(vertexColl);
    562+
    }
    554563
    }
    555564
    }
    556565
    }
    @@ -610,6 +619,22 @@ Result GraphManager::ensureCollections(Graph* graph, bool waitForSync) const {
    610619
    collectionsToCreate.emplace_back(
    611620
    CollectionCreationInfo{edgeColl, TRI_COL_TYPE_EDGE, options});
    612621
    }
    622+
    623+
    // new builder
    624+
    VPackBuilder satOptionsBuilder;
    625+
    if (graph->isHybrid()) {
    626+
    satOptionsBuilder.openObject();
    627+
    graph->createSatelliteCollectionOptions(satOptionsBuilder, waitForSync);
    628+
    satOptionsBuilder.close();
    629+
    for (auto const& satColl : satelliteDocumentCollectionsToCreate) {
    630+
    collectionsToCreate.emplace_back(CollectionCreationInfo{satColl, TRI_COL_TYPE_DOCUMENT,
    631+
    satOptionsBuilder.slice()});
    632+
    }
    633+
    for (auto const& satEdgeColl : satelliteEdgeCollectionsToCreate) {
    634+
    collectionsToCreate.emplace_back(CollectionCreationInfo{satEdgeColl, TRI_COL_TYPE_EDGE,
    635+
    satOptionsBuilder.slice()});
    636+
    }
    637+
    }
    613638
    if (collectionsToCreate.empty()) {
    614639
    // NOTE: Empty graph is allowed.
    615640
    return TRI_ERROR_NO_ERROR;
    @@ -650,8 +675,7 @@ Result GraphManager::readGraphKeys(velocypack::Builder& builder) const {
    650675

    651676
    Result GraphManager::readGraphByQuery(velocypack::Builder& builder,
    652677
    std::string const& queryStr) const {
    653-
    arangodb::aql::Query query(ctx(), arangodb::aql::QueryString(queryStr),
    654-
    nullptr);
    678+
    arangodb::aql::Query query(ctx(), arangodb::aql::QueryString(queryStr), nullptr);
    655679
    query.queryOptions().skipAudit = true;
    656680

    657681
    LOG_TOPIC("f6782", DEBUG, arangodb::Logger::GRAPHS)
    @@ -670,7 +694,7 @@ Result GraphManager::readGraphByQuery(velocypack::Builder& builder,
    670694

    671695
    if (graphsSlice.isNone()) {
    672696
    return Result(TRI_ERROR_OUT_OF_MEMORY);
    673-
    }
    697+
    }
    674698
    if (!graphsSlice.isArray()) {
    675699
    LOG_TOPIC("338b7", ERR, arangodb::Logger::GRAPHS)
    676700
    << "cannot read graphs from _graphs collection";
    @@ -1037,7 +1061,9 @@ ResultT<std::unique_ptr<Graph>> GraphManager::buildGraphFromInput(std::string co
    10371061
    s = s.get(StaticStrings::ReplicationFactor);
    10381062
    if ((s.isNumber() && s.getNumber<int>() == 0) ||
    10391063
    (s.isString() && s.stringRef() == "satellite")) {
    1040-
    return Result{TRI_ERROR_BAD_PARAMETER, "invalid combination of 'isSmart' and 'satellite' replicationFactor"};
    1064+
    return Result{TRI_ERROR_BAD_PARAMETER,
    1065+
    "invalid combination of 'isSmart' and 'satellite' "
    1066+
    "replicationFactor"};
    10411067
    }
    10421068
    }
    10431069
    }

    ‎arangod/Graph/GraphOperations.cpp

    Lines changed: 1 addition & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -226,7 +226,7 @@ OperationResult GraphOperations::editEdgeDefinition(VPackSlice edgeDefinitionSli
    226226
    bool waitForSync,
    227227
    std::string const& edgeDefinitionName) {
    228228
    OperationOptions options(ExecContext::current());
    229-
    auto maybeEdgeDef = EdgeDefinition::createFromVelocypack(edgeDefinitionSlice);
    229+
    auto maybeEdgeDef = EdgeDefinition::createFromVelocypack(edgeDefinitionSlice, graph().satelliteCollections());
    230230
    if (!maybeEdgeDef) {
    231231
    return OperationResult{std::move(maybeEdgeDef).result(), options};
    232232
    }

    0 commit comments

    Comments
     (0)
    0