8000 Introduce usage of statement timeout. In the case then an user sets · postgrespro/aqo@d9bd3b5 · GitHub
[go: up one dir, main page]
Skip to content

Commit d9bd3b5

Browse files
danolivodanolivo
committed
Introduce usage of statement timeout. In the case then an user sets
statement timeout AQO add one more timeout right before this. If timeout is expired, AQO walks across the PlanState tree and learn on partially executed nodes. TODO: 1. We should somehow remember, that partial knowledge isn't real and use it only before first successful execution. 2. We can distinguish already finished nodes and partially finished nodes. For nodes, which really have time to finish execution we should store cardinality "AS IS". In other situation we should use some extrapolation formula. 3. Maybe we shouldn't change instrumentation during partial walk? 4. Think about parallel workers.
1 parent c538b70 commit d9bd3b5

File tree

18 files changed

+400
-50
lines changed
  • t
  • utils.c

    • 18 files changed

    +400
    -50
    lines changed

    ‎Makefile

    Lines changed: 1 addition & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -6,7 +6,7 @@ PGFILEDESC = "AQO - Adaptive Query Optimization"
    66
    MODULE_big = aqo
    77
    OBJS = aqo.o auto_tuning.o cardinality_estimation.o cardinality_hooks.o \
    88
    hash.o machine_learning.o path_utils.o postprocessing.o preprocessing.o \
    9-
    selectivity_cache.o storage.o utils.o $(WIN32RES)
    9+
    selectivity_cache.o storage.o utils.o learn_cache.o $(WIN32RES)
    1010

    1111
    TAP_TESTS = 1
    1212

    ‎README.md

    Lines changed: 1 addition & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -325,7 +325,7 @@ Dynamically generated constants are okay.
    325325

    326326
    ## License
    327327

    328-
    © [Postgres Professional](https://postgrespro.com/), 2016-2021. Licensed under
    328+
    © [Postgres Professional](https://postgrespro.com/), 2016-2022. Licensed under
    329329
    [The PostgreSQL License](LICENSE).
    330330

    331331
    ## Reference

    ‎aqo.c

    Lines changed: 6 additions & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -2,7 +2,7 @@
    22
    * aqo.c
    33
    * Adaptive query optimization extension
    44
    *
    5-
    * Copyright (c) 2016-2021, Postgres Professional
    5+
    * Copyright (c) 2016-2022, Postgres Professional
    66
    *
    77
    * IDENTIFICATION
    88
    * aqo/aqo.c
    @@ -21,6 +21,7 @@
    2121
    #include "cardinality_hooks.h"
    2222
    #include "path_utils.h"
    2323
    #include "preprocessing.h"
    24+
    #include "learn_cache.h"
    2425

    2526

    2627
    PG_MODULE_MAGIC;
    @@ -102,6 +103,7 @@ int njoins;
    102103
    post_parse_analyze_hook_type prev_post_parse_analyze_hook;
    103104
    planner_hook_type prev_planner_hook;
    104105
    ExecutorStart_hook_type prev_ExecutorStart_hook;
    106+
    ExecutorRun_hook_type prev_ExecutorRun;
    105107
    ExecutorEnd_hook_type prev_ExecutorEnd_hook;
    106108
    set_baserel_rows_estimate_hook_type prev_set_foreign_rows_estimate_hook;
    107109
    set_baserel_rows_estimate_hook_type prev_set_baserel_rows_estimate_hook;
    @@ -202,6 +204,8 @@ _PG_init(void)
    202204
    planner_hook = aqo_planner;
    203205
    prev_ExecutorStart_hook = ExecutorStart_hook;
    204206
    ExecutorStart_hook = aqo_ExecutorStart;
    207+
    prev_ExecutorRun = ExecutorRun_hook;
    208+
    ExecutorRun_hook = aqo_ExecutorRun;
    205209
    prev_ExecutorEnd_hook = ExecutorEnd_hook;
    206210
    ExecutorEnd_hook = aqo_ExecutorEnd;
    207211

    @@ -237,6 +241,7 @@ _PG_init(void)
    237241
    ALLOCSET_DEFAULT_SIZES);
    238242
    RegisterResourceReleaseCallback(aqo_free_callback, NULL);
    239243
    RegisterAQOPlanNodeMethods();
    244+
    lc_init();
    240245
    }
    241246

    242247
    PG_FUNCTION_INFO_V1(invalidate_deactivated_queries_cache);

    ‎aqo.h

    Lines changed: 12 additions & 3 deletions
    Original file line numberDiff line numberDiff line change
    @@ -105,7 +105,7 @@
    105105
    * Module storage.c is responsible for storage query settings and models
    106106
    * (i. e. all information which is used in extension).
    107107
    *
    108-
    * Copyright (c) 2016-2021, Postgres Professional
    108+
    * Copyright (c) 2016-2022, Postgres Professional
    109109
    *
    110110
    * IDENTIFICATION
    111111
    * aqo/aqo.h
    @@ -257,6 +257,7 @@ extern MemoryContext AQOMemoryContext;
    257257
    extern post_parse_analyze_hook_type prev_post_parse_analyze_hook;
    258258
    extern planner_hook_type prev_planner_hook;
    259259
    extern ExecutorStart_hook_type prev_ExecutorStart_hook;
    260+
    extern ExecutorRun_hook_type prev_ExecutorRun;
    260261
    extern ExecutorEnd_hook_type prev_ExecutorEnd_hook;
    261262
    extern set_baserel_rows_estimate_hook_type
    262263
    prev_set_foreign_rows_estimate_hook;
    @@ -284,9 +285,15 @@ extern bool find_query(uint64 qhash, QueryContextData *ctx);
    284285
    extern bool update_query(uint64 qhash, uint64 fhash,
    285286
    bool learn_aqo, bool use_aqo, bool auto_tuning);
    286287
    extern bool add_query_text(uint64 query_hash, const char *query_string);
    288+
    extern bool load_fss_ext(uint64 fs, int fss,
    289+
    int ncols, double **matrix, double *targets, int *rows,
    290+
    List **relids, bool isSafe);
    287291
    extern bool load_fss(uint64 fhash, int fss_hash,
    288292
    int ncols, double **matrix, double *targets, int *rows,
    289293
    List **relids);
    294+
    extern bool update_fss_ext(uint64 fhash, int fsshash, int nrows, int ncols,
    295+
    double **matrix, double *targets, List *relids,
    296+
    bool isTimedOut);
    290297
    extern bool update_fss(uint64 fhash, int fss_hash, int nrows, int ncols,
    291298
    double **matrix, double *targets, List *relids);
    292299
    QueryStat *get_aqo_stat(uint64 query_hash);
    @@ -312,8 +319,10 @@ double predict_for_relation(List *restrict_clauses, List *selectivities,
    312319
    List *relids, int *fss_hash);
    313320

    314321
    /* Query execution statistics collecting hooks */
    315-
    void aqo_ExecutorStart(QueryDesc *queryDesc, int eflags);
    316-
    void aqo_ExecutorEnd(QueryDesc *queryDesc);
    322+
    void aqo_ExecutorStart(QueryDesc *queryDesc, int eflags);
    323+
    void aqo_ExecutorRun(QueryDesc *queryDesc, ScanDirection direction,
    324+
    uint64 count, bool execute_once);
    325+
    void aqo_ExecutorEnd(QueryDesc *queryDesc);
    317326

    318327
    /* Machine learning techniques */
    319328
    extern double OkNNr_predict(int nrows, int ncols,

    ‎auto_tuning.c

    Lines changed: 1 addition & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -8,7 +8,7 @@
    88
    *
    99
    *******************************************************************************
    1010
    *
    11-
    * Copyright (c) 2016-2021, Postgres Professional
    11+
    * Copyright (c) 2016-2022, Postgres Professional
    1212
    *
    1313
    * IDENTIFICATION
    1414
    * aqo/auto_tuning.c

    ‎cardinality_estimation.c

    Lines changed: 3 additions & 3 deletions
    Original file line numberDiff line numberDiff line change
    @@ -8,7 +8,7 @@
    88
    *
    99
    *******************************************************************************
    1010
    *
    11-
    * Copyright (c) 2016-2021, Postgres Professional
    11+
    * Copyright (c) 2016-2022, Postgres Professional
    1212
    *
    1313
    * IDENTIFICATION
    1414
    * aqo/cardinality_estimation.c
    @@ -83,8 +83,8 @@ predict_for_relation(List *clauses, List *selectivities,
    8383
    for (i = 0; i < aqo_K; ++i)
    8484
    matrix[i] = palloc0(sizeof(**matrix) * nfeatures);
    8585

    86-
    if (load_fss(query_context.fspace_hash, *fss_hash, nfeatures, matrix,
    87-
    targets, &rows, NULL))
    86+
    if (load_fss_ext(query_context.fspace_hash, *fss_hash, nfeatures, matrix,
    87+
    targets, &rows, NULL, true))
    8888
    result = OkNNr_predict(rows, nfeatures, matrix, targets, features);
    8989
    else
    9090
    {

    ‎cardinality_hooks.c

    Lines changed: 3 additions & 2 deletions
    Original file line numberDiff line numberDiff line change
    @@ -18,7 +18,7 @@
    1818
    *
    1919
    *******************************************************************************
    2020
    *
    21-
    * Copyright (c) 2016-2021, Postgres Professional
    21+
    * Copyright (c) 2016-2022, Postgres Professional
    2222
    *
    2323
    * IDENTIFICATION
    2424
    * aqo/cardinality_hooks.c
    @@ -432,7 +432,8 @@ predict_num_groups(PlannerInfo *root, Path *subpath, List *group_exprs,
    432432

    433433
    *fss = get_grouped_exprs_hash(child_fss, group_exprs);
    434434

    435-
    if (!load_fss(query_context.fspace_hash, *fss, 0, NULL, &target, &rows, NULL))
    435+
    if (!load_fss_ext(query_context.fspace_hash, *fss, 0, NULL,
    436+
    &target, &rows, NULL, true))
    436437
    return -1;
    437438

    438439
    Assert(rows == 1);

    ‎hash.c

    Lines changed: 1 addition & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -12,7 +12,7 @@
    1212
    *
    1313
    *******************************************************************************
    1414
    *
    15-
    * Copyright (c) 2016-2021, Postgres Professional
    15+
    * Copyright (c) 2016-2022, Postgres Professional
    1616
    *
    1717
    * IDENTIFICATION
    1818
    * aqo/hash.c

    ‎learn_cache.c

    Lines changed: 157 additions & 0 deletions
    Original file line numberDiff line numberDiff line change
    @@ -0,0 +1,157 @@
    1+
    /*
    2+
    *******************************************************************************
    3+
    *
    4+
    *
    5+
    *
    6+
    *******************************************************************************
    7+
    *
    8+
    * Copyright (c) 2016-2022, Postgres Professional
    9+
    *
    10+
    * IDENTIFICATION
    11+
    * aqo/learn_cache.c
    12+
    *
    13+
    */
    14+
    15+
    #include "postgres.h"
    16+
    17+
    #include "aqo.h"
    18+
    #include "learn_cache.h"
    19+
    20+
    typedef struct
    21+
    {
    22+
    /* XXX we assume this struct contains no padding bytes */
    23+
    uint64 fs;
    24+
    int64 fss;
    25+
    } htab_key;
    26+
    27+
    typedef struct
    28+
    {
    29+
    htab_key key;
    30+
    31+
    /* Store ML data "AS IS". */
    32+
    int nrows;
    33+
    int ncols;
    34+
    double *matrix[aqo_K];
    35+
    double *targets;
    36+
    List *relids;
    37+
    } htab_entry;
    38+
    39+
    static HTAB *fss_htab = NULL;
    40+
    MemoryContext LearnCacheMemoryContext = NULL;
    41+
    42+
    void
    43+
    lc_init(void)
    44+
    {
    45+
    HASHCTL ctl;
    46+
    47+
    Assert(!LearnCacheMemoryContext);
    48+
    LearnCacheMemoryContext = AllocSetContextCreate(TopMemoryContext,
    49+
    "lcache context",
    50+
    ALLOCSET_DEFAULT_SIZES);
    51+
    52+
    ctl.keysize = sizeof(htab_key);
    53+
    ctl.entrysize = sizeof(htab_entry);
    54+
    ctl.hcxt = LearnCacheMemoryContext;
    55+
    56+
    fss_htab = hash_create("Remote Con hash", 32, &ctl, HASH_ELEM | HASH_BLOBS);
    57+
    }
    58+
    59+
    bool
    60+
    lc_update_fss(uint64 fs, int fss, int nrows, int ncols,
    61+
    double **matrix, double *targets, List *relids)
    62+
    {
    63+
    htab_key key = {fs, fss};
    64+
    htab_entry *entry;
    65+
    bool found;
    66+
    int i;
    67+
    MemoryContext memctx = MemoryContextSwitchTo(LearnCacheMemoryContext);
    68+
    69+
    Assert(fss_htab);
    70+
    71+
    entry = (htab_entry *) hash_search(fss_htab, &key, HASH_ENTER, &found);
    72+
    if (found)
    73+
    {
    74+
    /* Clear previous version of the cached data. */
    75+
    for (i = 0; i < entry->nrows; ++i)
    76+
    pfree(entry->matrix[i]);
    77+
    pfree(entry->targets);
    78+
    list_free(entry->relids);
    79+
    }
    80+
    81+
    entry->nrows = nrows;
    82+
    entry->ncols = ncols;
    83+
    for (i = 0; i < entry->nrows; ++i)
    84+
    {
    85+
    entry->matrix[i] = palloc(sizeof(double) * ncols);
    86+
    memcpy(entry->matrix[i], matrix[i], sizeof(double) * ncols);
    87+
    }
    88+
    entry->targets = palloc(sizeof(double) * nrows);
    89+
    memcpy(entry->targets, targets, sizeof(double) * nrows);
    90+
    entry->relids = list_copy(relids);
    91+
    92+
    MemoryContextSwitchTo(memctx);
    93+
    return true;
    94+
    }
    95+
    96+
    bool
    97+
    lc_has_fss(uint64 fs, int fss)
    98+
    {
    99+
    htab_key key = {fs, fss};
    100+
    bool found;
    101+
    102+
    Assert(fss_htab);
    103+
    104+
    (void) hash_search(fss_htab, &key, HASH_FIND, &found);
    105+
    if (!found)
    106+
    return false;
    107+
    return true;
    108+
    }
    109+
    110+
    bool
    111+
    lc_load_fss(uint64 fs, int fss, int ncols, double **matrix,
    112+
    double *targets, int *nrows, List **relids)
    113+
    {
    114+
    htab_key key = {fs, fss};
    115+
    htab_entry *entry;
    116+
    bool found;
    117+
    int i;
    118+
    119+
    Assert(fss_htab);
    120+
    121+
    entry = (htab_entry *) hash_search(fss_htab, &key, HASH_FIND, &found);
    122+
    if (!found)
    123+
    return false;
    124+
    125+
    *nrows = entry->nrows;
    126+
    Assert(entry->ncols == ncols);
    127+
    for (i = 0; i < entry->nrows; ++i)
    128+
    memcpy(matrix[i], entry->matrix[i], sizeof(double) * ncols);
    129+
    memcpy(targets, entry->targets, sizeof(double) * entry->nrows);
    130+
    if (relids)
    131+
    *relids = list_copy(entry->relids);
    132+
    return true;
    133+
    }
    134+
    135+
    /*
    136+
    * Remove record from fss cache. Should be done at learning stage of successfully
    137+
    * finished query execution.
    138+
    */
    139+
    void
    140+
    lc_remove_fss(uint64 fs, int fss)
    141+
    {
    142+
    htab_key key = {fs, fss};
    143+
    htab_entry *entry;
    144+
    bool found;
    145+
    int i;
    146+
    147+
    Assert(fss_htab);
    148+
    149+
    entry = (htab_entry *) hash_search(fss_htab, &key, HASH_FIND, &found);
    150+
    if (!found)
    151+
    return;
    152+
    153+
    for (i = 0; i < entry->nrows; ++i)
    154+
    pfree(entry->matrix[i]);
    155+
    pfree(entry->targets);
    156+
    hash_search(fss_htab, &key, HASH_REMOVE, NULL);
    157+
    }

    ‎learn_cache.h

    Lines changed: 15 additions & 0 deletions
    Original file line numberDiff line numberDiff line change
    @@ -0,0 +1,15 @@
    1+
    #ifndef LEARN_CACHE_H
    2+
    #define LEARN_CACHE_H
    3+
    4+
    #include "nodes/pg_list.h"
    5+
    6+
    extern void lc_init(void);
    7+
    extern bool lc_update_fss(uint64 fhash, int fsshash, int nrows, int ncols,
    8+
    double **matrix, double *targets, List *relids);
    9+
    extern bool lc_has_fss(uint64 fhash, int fss);
    10+
    extern bool lc_load_fss(uint64 fhash, int fsshash, int ncols,
    11+
    double **matrix, double *targets, int *nrows,
    12+
    List **relids);
    13+
    extern void lc_remove_fss(uint64 fhash, int fss_hash);
    14+
    15+
    #endif /* LEARN_CACHE_H */

    ‎machine_learning.c

    Lines changed: 1 addition & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -12,7 +12,7 @@
    1212
    *
    1313
    *******************************************************************************
    1414
    *
    15-
    * Copyright (c) 2016-2021, Postgres Professional
    15+
    * Copyright (c) 2016-2022, Postgres Professional
    1616
    *
    1717
    * IDENTIFICATION
    1818
    * aqo/machine_learning.c

    ‎path_utils.c

    Lines changed: 1 addition & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -5,7 +5,7 @@
    55
    *
    66
    *******************************************************************************
    77
    *
    8-
    * Copyright (c) 2016-2021, Postgres Professional
    8+
    * Copyright (c) 2016-2022, Postgres Professional
    99
    *
    1010
    * IDENTIFICATION
    1111
    * aqo/path_utils.c

    0 commit comments

    Comments
     (0)
    0