8000 Try reducing the memory overhead. · postgrespro/aqo@b0c63cc · GitHub
[go: up one dir, main page]
Skip to content

Commit b0c63cc

Browse files
Daniil AnisimovAnisimov-ds
Daniil Anisimov
authored and
Anisimov-ds
committed
Try reducing the memory overhead.
Free some allocated memory right after use. Reset AQOPredictMemCtx as soon as posible. Remove learning attempts on SubPlan nodes. Bugfix. Free allocated memory on save/load data. Add memory context for storage. Change copyright to 2016-2023.
1 parent e0f0d38 commit b0c63cc

13 files changed

+183
-49
lines changed

‎aqo.c

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
* aqo.c
33
* Adaptive query optimization extension
44
*
5-
* Copyright (c) 2016-2022, Postgres Professional
5+
* Copyright (c) 2016-2023, Postgres Professional
66
*
77
* IDENTIFICATION
88
* aqo/aqo.c
@@ -92,6 +92,9 @@ MemoryContext AQOPredictMemCtx = NULL;
9292
/* Is released at the end of learning */
9393
MemoryContext AQOLearnMemCtx = NULL;
9494

95+
/* Is released at the end of load/store routines */
96+
MemoryContext AQOStorageMemCtx = NULL;
97+
9598
/* Additional plan info */
9699
int njoins;
97100

@@ -348,6 +351,12 @@ _PG_init(void)
348351
AQOLearnMemCtx = AllocSetContextCreate(AQOTopMemCtx,
349352
"AQOLearnMemoryContext",
350353
ALLOCSET_DEFAULT_SIZES);
354+
/*
355+
* AQOStorageMemoryContext containe data for load/store routines.
356+
*/
357+
AQOStorageMemCtx = AllocSetContextCreate(AQOTopMemCtx,
358+
"AQOStorageMemoryContext",
359+
ALLOCSET_DEFAULT_SIZES);
351360
RegisterResourceReleaseCallback(aqo_free_callback, NULL);
352361
RegisterAQOPlanNodeMethods();
353362

‎aqo.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@
105105
* Module storage.c is responsible for storage query settings and models
106106
* (i. e. all information which is used in extension).
107107
*
108-
* Copyright (c) 2016-2022, Postgres Professional
108+
* Copyright (c) 2016-2023, Postgres Professional
109109
*
110110
* IDENTIFICATION
111111
* aqo/aqo.h
@@ -232,6 +232,7 @@ extern MemoryContext AQOTopMemCtx;
232232
extern MemoryContext AQOCacheMemCtx;
233233
extern MemoryContext AQOPredictMemCtx;
234234
extern MemoryContext AQOLearnMemCtx;
235+
extern MemoryContext AQOStorageMemCtx;
235236

236237
extern int aqo_statement_timeout;
237238

‎auto_tuning.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
*
99
*******************************************************************************
1010
*
11-
* Copyright (c) 2016-2022, Postgres Professional
11+
* Copyright (c) 2016-2023, Postgres Professional
1212
*
1313
* IDENTIFICATION
1414
* aqo/auto_tuning.c

‎cardinality_estimation.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
*
99
*******************************************************************************
1010
*
11-
* Copyright (c) 2016-2022, Postgres Professional
11+
* Copyright (c) 2016-2023, Postgres Professional
1212
*
1313
* IDENTIFICATION
1414
* aqo/cardinality_estimation.c

‎cardinality_hooks.c

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
*
1919
*******************************************************************************
2020
*
21-
* Copyright (c) 2016-2022, Postgres Professional
21+
* Copyright (c) 2016-2023, Postgres Professional
2222
*
2323
* IDENTIFICATION
2424
* aqo/cardinality_hooks.c
@@ -82,6 +82,7 @@ aqo_set_baserel_rows_estimate(PlannerInfo *root, RelOptInfo *rel)
8282
if (!query_context.use_aqo)
8383
{
8484
MemoryContextSwitchTo(old_ctx_m);
85+
MemoryContextReset(AQOPredictMemCtx);
8586
goto default_estimator;
8687
}
8788

@@ -100,6 +101,7 @@ aqo_set_baserel_rows_estimate(PlannerInfo *root, RelOptInfo *rel)
100101

101102
/* Return to the caller's memory context. */
102103
MemoryContextSwitchTo(old_ctx_m);
104+
MemoryContextReset(AQOPredictMemCtx);
103105

104106
if (predicted < 0)
105107
goto default_estimator;
@@ -191,12 +193,15 @@ aqo_get_parameterized_baserel_size(PlannerInfo *root,
191193
cache_selectivity(current_hash, rel->relid, rte->relid,
192194
*((double *) lfirst(l2)));
193195
}
196+
197+
pfree(args_hash);
198+
pfree(eclass_hash);
194199
}
195200

196201
if (!query_context.use_aqo)
197202
{
198203
MemoryContextSwitchTo(oldctx);
199-
204+
MemoryContextReset(AQOPredictMemCtx);
200205
goto default_estimator;
201206
}
202207

@@ -211,6 +216,7 @@ aqo_get_parameterized_baserel_size(PlannerInfo *root,
211216

212217
/* Return to the caller's memory context */
213218
MemoryContextSwitchTo(oldctx);
219+
MemoryContextReset(AQOPredictMemCtx);
214220

215221
predicted_ppi_rows = predicted;
216222
fss_ppi_hash = fss;
@@ -265,6 +271,7 @@ aqo_set_joinrel_size_estimates(PlannerInfo *root, RelOptInfo *rel,
265271
if (!query_context.use_aqo)
266272
{
267273
MemoryContextSwitchTo(old_ctx_m);
274+
MemoryContextReset(AQOPredictMemCtx);
268275
goto default_estimator;
269276
}
270277

@@ -284,6 +291,7 @@ aqo_set_joinrel_size_estimates(PlannerInfo *root, RelOptInfo *rel,
284291

285292
/* Return to the caller's memory context */
286293
MemoryContextSwitchTo(old_ctx_m);
294+
MemoryContextReset(AQOPredictMemCtx);
287295

288296
rel->fss_hash = fss;
289297

@@ -343,6 +351,7 @@ aqo_get_parameterized_joinrel_size(PlannerInfo *root,
343351
if (!query_context.use_aqo)
344352
{
345353
MemoryContextSwitchTo(old_ctx_m);
354+
MemoryContextReset(AQOPredictMemCtx);
346355
goto default_estimator;
347356
}
348357

@@ -359,6 +368,7 @@ aqo_get_parameterized_joinrel_size(PlannerInfo *root,
359368
&fss);
360369
/* Return to the caller's memory context */
361370
MemoryContextSwitchTo(old_ctx_m);
371+
MemoryContextReset(AQOPredictMemCtx);
362372

363373
predicted_ppi_rows = predicted;
364374
fss_ppi_hash = fss;
@@ -450,6 +460,7 @@ aqo_estimate_num_groups(PlannerInfo *root, List *groupExprs,
450460
grouped_rel->rows = predicted;
451461
grouped_rel->fss_hash = fss;
452462
MemoryContextSwitchTo(old_ctx_m);
463+
MemoryContextReset(AQOPredictMemCtx);
453464
return predicted;
454465
}
455466
else
@@ -460,6 +471,7 @@ aqo_estimate_num_groups(PlannerInfo *root, List *groupExprs,
460471
grouped_rel->predicted_cardinality = -1;
461472

462473
MemoryContextSwitchTo(old_ctx_m);
474+
MemoryContextReset(AQOPredictMemCtx);
463475

464476
default_estimator:
465477
if (aqo_estimate_num_groups_next)

‎expected/unsupported.out

Lines changed: 59 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -311,6 +311,59 @@ EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF)
311311
JOINS: 0
312312
(23 rows)
313313

314+
EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF)
315+
SELECT * FROM t WHERE
316+
x = (SELECT x FROM t t0 WHERE t0.x = t.x LIMIT 1) AND
317+
x IN (SELECT x FROM t t0 WHERE t0.x = t.x);
318+
QUERY PLAN
319+
-----------------------------------------------------------
320+
Seq Scan on t (actual rows=1000 loops=1)
321+
AQO not used
322+
Filter: ((x = (SubPlan 1)) AND (SubPlan 2))
323+
SubPlan 1
324+
-> Limit (actual rows=1 loops=1000)
325+
AQO not used
326+
-> Seq Scan on t t0 (actual rows=1 loops=1000)
327+
AQO not used
328+
Filter: (x = t.x)
329+
Rows Removed by Filter: 475
330+
SubPlan 2
331+
-> Seq Scan on t t0_1 (actual rows=1 loops=1000)
332+
AQO not used
333+
Filter: (x = t.x)
334+
Rows Removed by Filter: 475
335+
Using aqo: true
336+
AQO mode: LEARN
337+
JOINS: 0
338+
(18 rows)
339+
340+
-- No prediction for top SeqScan, because it fss is changed
341+
EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF)
342+
SELECT * FROM t WHERE
343+
x = (SELECT x FROM t t0 WHERE t0.x = t.x LIMIT 1) AND
344+
x IN (SELECT x FROM t t0 WHERE t0.x = t.x);
345+
QUERY PLAN
346+
-----------------------------------------------------------
347+
Seq Scan on t (actual rows=1000 loops=1)
348+
AQO not used
349+
Filter: ((SubPlan 2) AND (x = (SubPlan 1)))
350+
SubPlan 2
351+
-> Seq Scan on t t0_1 (actual rows=1 loops=1000)
352+
AQO: rows=1, error=0%
353+
Filter: (x = t.x)
354+
Rows Removed by Filter: 475
355+
SubPlan 1
356+
-> Limit (actual rows=1 loops=1000)
357+
AQO not used
358+
-> Seq Scan on t t0 (actual rows=1 loops=1000)
359+
AQO: rows=1, error=0%
360+
Filter: (x = t.x)
361+
Rows Removed by Filter: 475
362+
Using aqo: true
363+
AQO mode: LEARN
364+
JOINS: 0
365+
(18 rows)
366+
314367
-- It's OK to use the knowledge for a query with different constants.
315368
EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF)
316369
SELECT count(*) FROM t WHERE
@@ -580,6 +633,10 @@ ORDER BY (md5(query_text),error) DESC;
580633
-------+------------------------------------------------------------------------------------------------
581634
0.768 | SELECT count(*) FROM (SELECT count(*) FROM t1 GROUP BY (x,y)) AS q1;
582635
0.070 | SELECT count(*) FROM (SELECT * FROM t GROUP BY (x) HAVING x > 3) AS q1;
636+
1.554 | EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) +
637+
| SELECT * FROM t WHERE +
638+
| x = (SELECT x FROM t t0 WHERE t0.x = t.x LIMIT 1) AND +
639+
| x IN (SELECT x FROM t t0 WHERE t0.x = t.x);
583640
0.000 | SELECT count(*) FROM t WHERE x < 3 AND mod(x,3) = 1;
584641
0.000 | SELECT * FROM +
585642
| (SELECT * FROM t WHERE x < 0) AS t0 +
@@ -612,13 +669,13 @@ ORDER BY (md5(query_text),error) DESC;
612669
| JOIN +
613670
| (SELECT * FROM t WHERE x % 3 < (SELECT avg(x) FROM t t0 WHERE t0.x <> t.x)) AS q2 +
614671
| ON q1.x = q2.x+1;
615-
(13 rows)
672+
(14 rows)
616673

617674
DROP TABLE t,t1 CASCADE; -- delete all tables used in the test
618675
SELECT count(*) FROM aqo_data; -- Just to detect some changes in the logic. May some false positives really bother us here?
619676
count
620677
-------
621-
44
678+
48
622679
(1 row)
623680

624681
SELECT true AS success FROM aqo_cleanup();

‎hash.c

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
*
1313
*******************************************************************************
1414
*
15-
* Copyright (c) 2016-2022, Postgres Professional
15+
* Copyright (c) 2016-2023, Postgres Professional
1616
*
1717
* IDENTIFICATION
1818
* aqo/hash.c
@@ -157,6 +157,8 @@ get_grouped_exprs_hash(int child_fss, List *group_exprs)
157157
final_hashes[0] = child_fss;
158158
final_hashes[1] = get_int_array_hash(hashes, i);
159159

160+
pfree(hashes);
161+
160162
return get_int_array_hash(final_hashes, 2);
161163
}
162164

@@ -224,6 +226,7 @@ get_fss_for_object(List *relsigns, List *clauselist,
224226
clause_has_consts[i] = (args != NULL && has_consts(*args));
225227
i++;
226228
}
229+
pfree(args_hash);
227230

228231
idx = argsort(clause_hashes, n, sizeof(*clause_hashes), int_cmp);
229232
inverse_idx = inverse_permutation(idx, n);
@@ -234,6 +237,7 @@ get_fss_for_object(List *relsigns, List *clauselist,
234237
sorted_clauses[inverse_idx[i]] = clause_hashes[i];
235238
i++;
236239
}
240+
pfree(clause_hashes);
237241

238242
i = 0;
239243
foreach(lc, selectivities)
@@ -249,6 +253,7 @@ get_fss_for_object(List *relsigns, List *clauselist,
249253
}
250254
i++;
251255
}
256+
pfree(inverse_idx);
252257

253258
for (i = 0; i < n;)
254259
{
@@ -272,6 +277,8 @@ get_fss_for_object(List *relsigns, List *clauselist,
272277
sizeof(**features), double_cmp);
273278
i = j;
274279
}
280+
pfree(idx);
281+
pfree(clause_has_consts);
275282

276283
/*
277284
* Generate feature subspace hash.
@@ -281,6 +288,8 @@ get_fss_for_object(List *relsigns, List *clauselist,
281288
eclasses_hash = get_int_array_hash(eclass_hash, nargs);
282289
relations_hash = get_relations_hash(relsigns);
283290
fss_hash = get_fss_hash(clauses_hash, eclasses_hash, relations_hash);
291+
pfree(sorted_clauses);
292+
pfree(eclass_hash);
284293

285294
if (nfeatures != NULL)
286295
{
@@ -340,11 +349,17 @@ static int
340349
get_node_hash(Node *node)
341350
{
342351
char *str;
352+
char *no_consts;
353+
char *no_locations;
343354
int hash;
344355

345-
str = remove_locations(remove_consts(nodeToString(node)));
346-
hash = get_str_hash(str);
356+
str = nodeToString(node);
357+
no_consts = remove_consts(str);
347358
pfree(str);
359+
no_locations = remove_locations(no_consts);
360+
pfree(no_consts);
361+
hash = get_str_hash(no_locations);
362+
pfree(no_locations);
348363
return hash;
349364
}
350365

@@ -467,6 +482,7 @@ get_relations_hash(List *relsigns)
467482

468483
result = DatumGetInt32(hash_any((const unsigned char *) hashes,
469484
nhashes * sizeof(uint32)));
485+
pfree(hashes);
470486

471487
return result;
472488
}
@@ -479,9 +495,11 @@ static char *
479495
remove_consts(const char *str)
480496
{
481497
char *res;
498+
char *tmp;
482499

483-
res = replace_patterns(str, "{CONST", is_brace);
484-
res = replace_patterns(res, ":stmt_len", is_brace);
500+
tmp = replace_patterns(str, "{CONST", is_brace);
501+
res = replace_patterns(tmp, ":stmt_len", is_brace);
502+
pfree(tmp);
485503
return res;
486504
}
487505

@@ -683,6 +701,8 @@ get_eclasses(List *clauselist, int *nargs, int **args_hash, int **eclass_hash)
683701

684702
for (i = 0; i < *nargs; ++i)
685703
(*eclass_hash)[i] = e_hashes[disjoint_set_get_parent(p, i)];
704+
705+
pfree(e_hashes);
686706
}
687707

688708
/*

‎machine_learning.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
*
1313
*******************************************************************************
1414
*
15-
* Copyright (c) 2016-2022, Postgres Professional
15+
* Copyright (c) 2016-2023, Postgres Professional
1616
*
1717
* IDENTIFICATION
1818
* aqo/machine_learning.c

0 commit comments

Comments
 (0)
0