8000 Allow usage of huge maintenance_work_mem for GIN build. · postgrespro/postgres@30bb26b · GitHub
[go: up one dir, main page]
Skip to content

    • Commit 30bb26b

    Browse files
    feodorfeodor
    committed
    Allow usage of huge maintenance_work_mem for GIN build.
    Currently, in-memory posting list during GIN build process is limited 1GB because of using repalloc. The patch replaces call of repalloc to repalloc_huge. It increases limit of posting list from 180 millions (1GB / sizeof(ItemPointerData)) to 4 billions limited by maxcount/count fields in GinEntryAccumulator and subsequent calls. Check added. Also, fix accounting of allocatedMemory during build to prevent integer overflow with maintenance_work_mem > 4GB. Robert Abraham <robert.abraham86@googlemail.com> with additions by me
    1 parent 075ab42 commit 30bb26b

    File tree

    4 files changed

    +12
    -4
    lines changed

    4 files changed

    +12
    -4
    lines changed

    ‎src/backend/access/gin/ginbulk.c

    Lines changed: 9 additions & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -14,6 +14,8 @@
    1414

    1515
    #include "postgres.h"
    1616

    17+
    #include <limits.h>
    18+
    1719
    #include "access/gin_private.h"
    1820
    #include "utils/datum.h"
    1921
    #include "utils/memutils.h"
    @@ -36,10 +38,16 @@ ginCombineData(RBNode *existing, const RBNode *newdata, void *arg)
    3638
    */
    3739
    if (eo->count >= eo->maxcount)
    3840
    {
    41+
    if (eo->maxcount > INT_MAX)
    42+
    ereport(ERROR,
    43+
    (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
    44+
    errmsg("posting list is too long"),
    45+
    errhint("Reduce maintenance_work_mem")));
    46+
    3947
    accum->allocatedMemory -= GetMemoryChunkSpace(eo->list);
    4048
    eo->maxcount *= 2;
    4149
    eo->list = (ItemPointerData *)
    42-
    repalloc(eo->list, sizeof(ItemPointerData) * eo->maxcount);
    50+
    repalloc_huge(eo->list, sizeof(ItemPointerData) * eo->maxcount);
    4351
    accum->allocatedMemory += GetMemoryChunkSpace(eo->list);
    4452
    }
    4553

    ‎src/backend/access/gin/ginfast.c

    Lines changed: 1 addition & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -814,7 +814,7 @@ ginInsertCleanup(GinState *ginstate,
    814814
    */
    815815
    if (GinPageGetOpaque(page)->rightlink == InvalidBlockNumber ||
    816816
    (GinPageHasFullRow(page) &&
    817-
    (accum.allocatedMemory >= maintenance_work_mem * 1024L)))
    817+
    (accum.allocatedMemory >= (Size)maintenance_work_mem * 1024L)))
    818818
    {
    819819
    ItemPointerData *list;
    820820
    uint32 nlist;

    ‎src/backend/access/gin/gininsert.c

    Lines changed: 1 addition & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -281,7 +281,7 @@ ginBuildCallback(Relation index, HeapTuple htup, Datum *values,
    281281
    &htup->t_self);
    282282

    283283
    /* If we've maxed out our available memory, dump everything to the index */
    284-
    if (buildstate->accum.allocatedMemory >= maintenance_work_mem * 1024L)
    284+
    if (buildstate->accum.allocatedMemory >= (Size)maintenance_work_mem * 1024L)
    285285
    {
    286286
    ItemPointerData *list;
    287287
    Datum key;

    ‎src/include/access/gin_private.h

    Lines changed: 1 addition & 1 deletion
    Original file line numberDiff line numberDiff line change
    @@ -903,7 +903,7 @@ typedef struct GinEntryAccumulator
    903903
    typedef struct
    904904
    {
    905905
    GinState *ginstate;
    906-
    long allocatedMemory;
    906+
    Size allocatedMemory;
    907907
    GinEntryAccumulator *entryallocator;
    908908
    uint32 eas_used;
    909909
    RBTree *tree;

    0 commit comments

    Comments
     (0)
    0