8000 Introduce usage of statement timeout. In the case then an user sets · postgrespro/aqo@d9bd3b5 · GitHub
[go: up one dir, main page]
Skip to content
8000

Commit d9bd3b5

Browse files
danolivodanolivo
committed
Introduce usage of statement timeout. In the case then an user sets
statement timeout AQO add one more timeout right before this. If timeout is expired, AQO walks across the PlanState tree and learn on partially executed nodes. TODO: 1. We should somehow remember, that partial knowledge isn't real and use it only before first successful execution. 2. We can distinguish already finished nodes and partially finished nodes. For nodes, which really have time to finish execution we should store cardinality "AS IS". In other situation we should use some extrapolation formula. 3. Maybe we shouldn't change instrumentation during partial walk? 4. Think about parallel workers.
1 parent c538b70 commit d9bd3b5

18 files changed

+400
-50
lines changed

‎Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ PGFILEDESC = "AQO - Adaptive Query Optimization"
66
MODULE_big = aqo
77
OBJS = aqo.o auto_tuning.o cardinality_estimation.o cardinality_hooks.o \
88
hash.o machine_learning.o path_utils.o postprocessing.o preprocessing.o \
9-
selectivity_cache.o storage.o utils.o $(WIN32RES)
9+
selectivity_cache.o storage.o utils.o learn_cache.o $(WIN32RES)
1010

1111
TAP_TESTS = 1
1212

‎README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -325,7 +325,7 @@ Dynamically generated constants are okay.
325325

326326
## License
327327

328-
© [Postgres Professional](https://postgrespro.com/), 2016-2021. Licensed under
328+
© [Postgres Professional](https://postgrespro.com/), 2016-2022. Licensed under
329329
[The PostgreSQL License](LICENSE).
330330

331331
## Reference

‎aqo.c

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
* aqo.c
33
* Adaptive query optimization extension
44
*
5-
* Copyright (c) 2016-2021, Postgres Professional
5+
* Copyright (c) 2016-2022, Postgres Professional
66
*
77
* IDENTIFICATION
88
* aqo/aqo.c
@@ -21,6 +21,7 @@
2121
#include "cardinality_hooks.h"
2222
#include "path_utils.h"
2323
#include "preprocessing.h"
24+
#include "learn_cache.h"
2425

2526

2627
PG_MODULE_MAGIC;
@@ -102,6 +103,7 @@ int njoins;
102103
post_parse_analyze_hook_type prev_post_parse_analyze_hook;
103104
planner_hook_type prev_planner_hook;
104105
ExecutorStart_hook_type prev_ExecutorStart_hook;
106+
ExecutorRun_hook_type prev_ExecutorRun;
105107
ExecutorEnd_hook_type prev_ExecutorEnd_hook;
106108
set_baserel_rows_estimate_hook_type prev_set_foreign_rows_estimate_hook;
107109
set_baserel_rows_estimate_hook_type prev_set_baserel_rows_estimate_hook;
@@ -202,6 +204,8 @@ _PG_init(void)
202204
planner_hook = aqo_planner;
203205
prev_ExecutorStart_hook = ExecutorStart_hook;
204206
ExecutorStart_hook = aqo_ExecutorStart;
207+
prev_ExecutorRun = ExecutorRun_hook;
208+
ExecutorRun_hook = aqo_ExecutorRun;
205209
prev_ExecutorEnd_hook = ExecutorEnd_hook;
206210
ExecutorEnd_hook = aqo_ExecutorEnd;
207211

@@ -237,6 +241,7 @@ _PG_init(void)
237241
ALLOCSET_DEFAULT_SIZES);
238242
RegisterResourceReleaseCallback(aqo_free_callback, NULL);
239243
RegisterAQOPlanNodeMethods();
244+
lc_init();
240245
}
241246

242247
PG_FUNCTION_INFO_V1(invalidate_deactivated_queries_cache);

‎aqo.h

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@
105105
* Module storage.c is responsible for storage query settings and models
106106
* (i. e. all information which is used in extension).
107107
*
108-
* Copyright (c) 2016-2021, Postgres Professional
108+
* Copyright (c) 2016-2022, Postgres Professional
109109
*
110110
* IDENTIFICATION
111111
* aqo/aqo.h
@@ -257,6 +257,7 @@ extern MemoryContext AQOMemoryContext;
257257
extern post_parse_analyze_hook_type prev_post_parse_analyze_hook;
258258
extern planner_hook_type prev_planner_hook;
259259
extern ExecutorStart_hook_type prev_ExecutorStart_hook;
260+
extern ExecutorRun_hook_type prev_ExecutorRun;
260261
extern ExecutorEnd_hook_type prev_ExecutorEnd_hook;
261262
extern set_baserel_rows_estimate_hook_type
262263
prev_set_foreign_rows_estimate_hook;
@@ -284,9 +285,15 @@ extern bool find_query(uint64 qhash, QueryContextData *ctx);
284285
extern bool update_query(uint64 qhash, uint64 fhash,
285286
bool learn_aqo, bool use_aqo, bool auto_tuning);
286287
extern bool add_query_text(uint64 query_hash, const char *query_string);
288+
extern bool load_fss_ext(uint64 fs, int fss,
289+
int ncols, double **matrix, double *targets, int *rows,
290+
List **relids, bool isSafe);
287291
extern bool load_fss(uint64 fhash, int fss_hash,
288292
int ncols, double **matrix, double *targets, int *rows,
289293
List **relids);
294+
extern bool update_fss_ext(uint64 fhash, int fsshash, int nrows, int ncols,
295+
double **matrix, double *targets, List *relids,
296+
bool isTimedOut);
290297
extern bool update_fss(uint64 fhash, int fss_hash, int nrows, int ncols,
291298
double **matrix, double *targets, List *relids);
292299
QueryStat *get_aqo_stat(uint64 query_hash);
@@ -312,8 +319,10 @@ double predict_for_relation(List *restrict_clauses, List *selectivities,
312319
List *relids, int *fss_hash);
313320

314321
/* Query execution statistics collecting hooks */
315-
void aqo_ExecutorStart(QueryDesc *queryDesc, int eflags);
316-
void aqo_ExecutorEnd(QueryDesc *queryDesc);
322+
void aqo_ExecutorStart(QueryDesc *queryDesc, int eflags);
323+
void aqo_ExecutorRun(QueryDesc *queryDesc, ScanDirection direction,
324+
uint64 count, bool execute_once);
325+
void aqo_ExecutorEnd(QueryDesc *queryDesc);
317326

318327
/* Machine learning techniques */
319328
extern double OkNNr_predict(int nrows, int ncols,

‎auto_tuning.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
*
99
*******************************************************************************
1010
*
11-
* Copyright (c) 2016-2021, Postgres Professional
11+
* Copyright (c) 2016-2022, Postgres Professional
1212
*
1313
* IDENTIFICATION
1414
* aqo/auto_tuning.c

‎cardinality_estimation.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
*
99
*******************************************************************************
1010
*
11-
* Copyright (c) 2016-2021, Postgres Professional
11+
* Copyright (c) 2016-2022, Postgres Professional
1212
*
1313
* IDENTIFICATION
1414
* aqo/cardinality_estimation.c
@@ -83,8 +83,8 @@ predict_for_relation(List *clauses, List *selectivities,
8383
for (i = 0; i < aqo_K; ++i)
8484
matrix[i] = palloc0(sizeof(**matrix) * nfeatures);
8585

86-
if (load_fss(query_context.fspace_hash, *fss_hash, nfeatures, matrix,
87-
targets, &rows, NULL))
86+
if (load_fss_ext(query_context.fspace_hash, *fss_hash, nfeatures, matrix,
87+
targets, &rows, NULL, true))
8888
result = OkNNr_predict(rows, nfeatures, matrix, targets, features);
8989
else
9090
{

‎cardinality_hooks.c

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
*
1919
*******************************************************************************
2020
*
21-
* Copyright (c) 2016-2021, Postgres Professional
21+
* Copyright (c) 2016-2022, Postgres Professional
2222
*
2323
* IDENTIFICATION
2424
* aqo/cardinality_hooks.c
@@ -432,7 +432,8 @@ predict_num_groups(PlannerInfo *root, Path *subpath, List *group_exprs,
432432

433433
*fss = get_grouped_exprs_hash(child_fss, group_exprs);
434434

435-
if (!load_fss(query_context.fspace_hash, *fss, 0, NULL, &target, &rows, NULL))
435+
if (!load_fss_ext(query_context.fspace_hash, *fss, 0, NULL,
436+
&target, &rows, NULL, true))
436437
return -1;
437438

438439
Assert(rows == 1);

‎hash.c

Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
*
1313
*******************************************************************************
1414
*
15-
* Copyright (c) 2016-2021, Postgres Professional
15+
* Copyright (c) 2016-2022, Postgres Professional
1616
*
1717
* IDENTIFICATION
1818
* aqo/hash.c

‎learn_cache.c

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
B41A /*
2+
*******************************************************************************
3+
*
4+
*
5+
*
6+
*******************************************************************************
7+
*
8+
* Copyright (c) 2016-2022, Postgres Professional
9+
*
10+
* IDENTIFICATION
11+
* aqo/learn_cache.c
12+
*
13+
*/
14+
15+
#include "postgres.h"
16+
17+
#include "aqo.h"
18+
#include "learn_cache.h"
19+
20+
typedef struct
21+
{
22+
/* XXX we assume this struct contains no padding bytes */
23+
uint64 fs;
24+
int64 fss;
25+
} htab_key;
26+
27+
typedef struct
28+
{
29+
htab_key key;
30+
31+
/* Store ML data "AS IS". */
32+
int nrows;
33+
int ncols;
34+
double *matrix[aqo_K];
35+
double *targets;
36+
List *relids;
37+
} htab_entry;
38+
39+
static HTAB *fss_htab = NULL;
40+
MemoryContext LearnCacheMemoryContext = NULL;
41+
42+
void
43+
lc_init(void)
44+
{
45+
HASHCTL ctl;
46+
47+
Assert(!LearnCacheMemoryContext);
48+
LearnCacheMemoryContext = AllocSetContextCreate(TopMemoryContext,
49+
"lcache context",
50+
ALLOCSET_DEFAULT_SIZES);
51+
52+
ctl.keysize = sizeof(htab_key);
53+
ctl.entrysize = sizeof(htab_entry);
54+
ctl.hcxt = LearnCacheMemoryContext;
55+
56+
fss_htab = hash_create("Remote Con hash", 32, &ctl, HASH_ELEM | HASH_BLOBS);
57+
}
58+
59+
bool
60+
lc_update_fss(uint64 fs, int fss, int nrows, int ncols,
61+
double **matrix, double *targets, List *relids)
62+
{
63+
htab_key key = {fs, fss};
64+
htab_entry *entry;
65+
bool found;
66+
int i;
67+
MemoryContext memctx = MemoryContextSwitchTo(LearnCacheMemoryContext);
68+
69+
Assert(fss_htab);
70+
71+
entry = (htab_entry *) hash_search(fss_htab, &key, HASH_ENTER, &found);
72+
if (found)
73+
{
74+
/* Clear previous version of the cached data. */
75+
for (i = 0; i < entry->nrows; ++i)
76+
pfree(entry->matrix[i]);
77+
pfree(entry->targets);
78+
list_free(entry->relids);
79+
}
80+
81+
entry->nrows = nrows;
82+
entry->ncols = ncols;
83+
for (i = 0; i < entry->nrows; ++i)
84+
{
85+
entry->matrix[i] = palloc(sizeof(double) * ncols);
86+
memcpy(entry->matrix[i], matrix[i], sizeof(double) * ncols);
87+
}
88+
entry->targets = palloc(sizeof(double) * nrows);
89+
memcpy(entry->targets, targets, sizeof(double) * nrows);
90+
entry->relids = list_copy(relids);
91+
92+
MemoryContextSwitchTo(memctx);
93+
return true;
94+
}
95+
96+
bool
97+
lc_has_fss(uint64 fs, int fss)
98+
{
99+
htab_key key = {fs, fss};
100+
bool found;
101+
102+
Assert(fss_htab);
103+
104+
(void) hash_search(fss_htab, &key, HASH_FIND, &found);
105+
if (!found)
106+
return false;
107+
return true;
108+
}
109+
110+
bool
111+
lc_load_fss(uint64 fs, int fss, int ncols, double **matrix,
112+
double *targets, int *nrows, List **relids)
113+
{
114+
htab_key key = {fs, fss};
115+
htab_entry *entry;
116+
bool found;
117+
int i;
118+
119+
Assert(fss_htab);
120+
121+
entry = (htab_entry *) hash_search(fss_htab, &key, HASH_FIND, &found);
122+
if (!found)
123+
return false;
124+
125+
*nrows = entry->nrows;
126+
Assert(entry->ncols == ncols);
127+
for (i = 0; i < entry->nrows; ++i)
128+
memcpy(matrix[i], entry->matrix[i], sizeof(double) * ncols);
129+
memcpy(targets, entry->targets, sizeof(double) * entry->nrows);
130+
if (relids)
131+
*relids = list_copy(entry->relids);
132+
return true;
133+
}
134+
135+
/*
136+
* Remove record from fss cache. Should be done at learning stage of successfully
137+
* finished query execution.
138+
*/
139+
void
140+
lc_remove_fss(uint64 fs, int fss)
141+
{
142+
htab_key key = {fs, fss};
143+
htab_entry *entry;
144+
bool found;
145+
int i;
146+
147+
Assert(fss_htab);
148+
149+
entry = (htab_entry *) hash_search(fss_htab, &key, HASH_FIND, &found);
150+
if (!found)
151+
return;
152+
153+
for (i = 0; i < entry->nrows; ++i)
154+
pfree(entry->matrix[i]);
155+
pfree(entry->targets);
156+
hash_search(fss_htab, &key, HASH_REMOVE, NULL);
157+
}

‎learn_cache.h

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#ifndef LEARN_CACHE_H
2+
#defin 10000 e LEARN_CACHE_H
3+
4+
#include "nodes/pg_list.h"
5+
6+
extern void lc_init(void);
7+
extern bool lc_update_fss(uint64 fhash, int fsshash, int nrows, int ncols,
8+
double **matrix, double *targets, List *relids);
9+
extern bool lc_has_fss(uint64 fhash, int fss);
10+
extern bool lc_load_fss(uint64 fhash, int fsshash, int ncols,
11+
double **matrix, double *targets, int *nrows,
12+
List **relids);
13+
extern void lc_remove_fss(uint64 fhash, int fss_hash);
14+
15+
#endif /* LEARN_CACHE_H */

‎machine_learning.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
*
1313
*******************************************************************************
1414
*
15-
* Copyright (c) 2016-2021, Postgres Professional
15+
* Copyright (c) 2016-2022, Postgres Professional
1616
*
1717
* IDENTIFICATION
1818
* aqo/machine_learning.c

‎path_utils.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
*
66
*******************************************************************************
77
*
8-
* Copyright (c) 2016-2021, Postgres Professional
8+
* Copyright (c) 2016-2022, Postgres Professional
99
*
1010
* IDENTIFICATION
1111
* aqo/path_utils.c

0 commit comments

Comments
 (0)
0