8000 Keep WAL segments by the flushed value of the slot's restart LSN · postgres/postgres@dd3df0b · GitHub
[go: up one dir, main page]
Skip to content

Commit dd3df0b

Browse files
akorotkovakorotkov
committed
Keep WAL segments by the flushed value of the slot's restart LSN
The patch fixes the issue with the unexpected removal of old WAL segments after checkpoint, followed by an immediate restart. The issue occurs when a slot is advanced after the start of the checkpoint and before old WAL segments are removed at the end of the checkpoint. The idea of the patch is to get the minimal restart_lsn at the beginning of checkpoint (or restart point) creation and use this value when calculating the oldest LSN for WAL segments removal at the end of checkpoint. This idea was proposed by Tomas Vondra in the discussion. Unlike 291221c46575, this fix doesn't affect ABI and is intended for back branches. Discussion: https://postgr.es/m/flat/1d12d2-67235980-35-19a406a0%4063439497 Author: Vitaly Davydov <v.davydov@postgrespro.ru> Reviewed-by: Tomas Vondra <tomas@vondra.me> Reviewed-by: Alexander Korotkov <aekorotkov@gmail.com> Reviewed-by: Amit Kapila <amit.kapila16@gmail.com> Backpatch-through: 13
1 parent 38c8d29 commit dd3df0b

File tree

3 files changed

+60
-9
lines changed

3 files changed

+60
-9
lines changed

‎src/backend/access/transam/xlog.c

Lines changed: 47 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -920,7 +920,8 @@ static void LocalSetXLogInsertAllowed(void);
920920
static void CreateEndOfRecoveryRecord(void);
921921
static XLogRecPtr CreateOverwriteContrecordRecord(XLogRecPtr aborted_lsn);
922922
static void CheckPointGuts(XLogRecPtr checkPointRedo, int flags);
923-
static void KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo);
923+
static void KeepLogSeg(XLogRecPtr recptr, XLogRecPtr slotsMinLSN,
924+
XLogSegNo *logSegNo);
924925
static XLogRecPtr XLogGetReplicationSlotMinimumLSN(void);
925926

926927
static void AdvanceXLInsertBuffer(XLogRecPtr upto, bool opportunistic);
@@ -8921,6 +8922,7 @@ CreateCheckPoint(int flags)
89218922
XLogRecPtr last_important_lsn;
89228923
VirtualTransactionId *vxids;
89238924
int nvxids;
8925+
XLogRecPtr slotsMinReqLSN;
89248926

89258927
/*
89268928
* An end-of-recovery checkpoint is really a shutdown checkpoint, just
@@ -9140,6 +9142,15 @@ CreateCheckPoint(int flags)
91409142
*/
91419143
END_CRIT_SECTION();
91429144

9145+
/*
9146+
* Get the current minimum LSN to be used later in the WAL segment
9147+
* cleanup. We may clean up only WAL segments, which are not needed
9148+
* according to synchronized LSNs of replication slots. The slot's LSN
9149+
* might be advanced concurrently, so we call this before
9150+
* CheckPointReplicationSlots() synchronizes replication slots.
9151+
*/
9152+
slotsMinReqLSN = XLogGetReplicationSlotMinimumLSN();
9153+
91439154
/*
91449155
* In some cases there are groups of actions that must all occur on one
91459156
* side or the other of a checkpoint record. Before flushing the
@@ -9304,15 +9315,23 @@ CreateCheckPoint(int flags)
93049315
* prevent the disk holding the xlog from growing full.
93059316
*/
93069317
XLByteToSeg(RedoRecPtr, _logSegNo, wal_segment_size);
9307-
KeepLogSeg(recptr, &_logSegNo);
9318+
KeepLogSeg(recptr, slotsMinReqLSN, &_logSegNo);
93089319
if (InvalidateObsoleteReplicationSlots(_logSegNo))
93099320
{
9321+
/*
9322+
* Recalculate the current minimum LSN to be used in the WAL segment
9323+
* cleanup. Then, we must synchronize the replication slots again in
9324+
* order to make this LSN safe to use.
9325+
*/
9326+
slotsMinReqLSN = XLogGetReplicationSlotMinimumLSN();
9327+
CheckPointReplicationSlots();
9328+
93109329
/*
93119330
* Some slots have been invalidated; recalculate the old-segment
93129331
* horizon, starting again from RedoRecPtr.
93139332
*/
93149333
XLByteToSeg(RedoRecPtr, _logSegNo, wal_segment_size);
9315-
KeepLogSeg(recptr, &_logSegNo);
9334+
KeepLogSeg(recptr, slotsMinReqLSN, &_logSegNo);
93169335
}
93179336
_logSegNo--;
93189337
RemoveOldXlogFiles(_logSegNo, RedoRecPtr, recptr);
@@ -9534,6 +9553,7 @@ CreateRestartPoint(int flags)
95349553
XLogRecPtr endptr;
95359554
XLogSegNo _logSegNo;
95369555
TimestampTz xtime;
9556+
XLogRecPtr slotsMinReqLSN;
95379557

95389558
/*
95399559
* Acquire CheckpointLock to ensure only one restartpoint or checkpoint
@@ -9623,6 +9643,15 @@ CreateRestartPoint(int flags)
96239643
MemSet(&CheckpointStats, 0, sizeof(CheckpointStats));
96249644
CheckpointStats.ckpt_start_t = GetCurrentTimestamp();
96259645

9646+
/*
9647+
* Get the current minimum LSN to be used later in the WAL segment
9648+
* cleanup. We may clean up only WAL segments, which are not needed
9649+
* according to synchronized LSNs of replication slots. The slot's LSN
9650+
* might be advanced concurrently, so we call this before
9651+
* CheckPointReplicationSlots() synchronizes replication slots.
9652+
*/
9653+
slotsMinReqLSN = XLogGetReplicationSlotMinimumLSN();
9654+
96269655
if (log_checkpoints)
96279656
LogCheckpointStart(flags, true);
96289657

@@ -9708,15 +9737,23 @@ CreateRestartPoint(int flags)
97089737
receivePtr = GetWalRcvFlushRecPtr(NULL, NULL);
97099738
replayPtr = GetXLogReplayRecPtr(&replayTLI);
97109739
endptr = (receivePtr < replayPtr) ? replayPtr : receivePtr;
9711-
KeepLogSeg(endptr, &_logSegNo);
9740+
KeepLogSeg(endptr, slotsMinReqLSN, &_logSegNo);
97129741
if (InvalidateObsoleteReplicationSlots(_logSegNo))
97139742
{
9743+
/*
9744+
* Recalculate the current minimum LSN to be used in the WAL segment
9745+
* cleanup. Then, we must synchronize the replication slots again in
9746+
* order to make this LSN safe to use.
9747+
*/
9748+
slotsMinReqLSN = XLogGetReplicationSlotMinimumLSN();
9749+
CheckPointReplicationSlots();
9750+
97149751
/*
97159752
* Some slots have been invalidated; recalculate the old-segment
97169753
* horizon, starting again from RedoRecPtr.
97179754
*/
97189755
XLByteToSeg(RedoRecPtr, _logSegNo, wal_segment_size);
9719-
KeepLogSeg(endptr, &_logSegNo);
9756+
KeepLogSeg(endptr, slotsMinReqLSN, &_logSegNo);
97209757
}
97219758
_logSegNo--;
97229759

@@ -9818,6 +9855,7 @@ GetWALAvailability(XLogRecPtr targetLSN)
98189855
XLogSegNo oldestSegMaxWalSize; /* oldest segid kept by max_wal_size */
98199856
XLogSegNo oldestSlotSeg; /* oldest segid kept by slot */
98209857
uint64 keepSegs;
9858+
XLogRecPtr slotsMinReqLSN;
98219859

98229860
/*
98239861
* slot does not reserve WAL. Either deactivated, or has never been active
@@ -9831,8 +9869,9 @@ GetWALAvailability(XLogRecPtr targetLSN)
98319869
* oldestSlotSeg to the current segment.
98329870
*/
98339871
currpos = GetXLogWriteRecPtr();
9872+
slotsMinReqLSN = XLogGetReplicationSlotMinimumLSN();
98349873
XLByteToSeg(currpos, oldestSlotSeg, wal_segment_size);
9835-
KeepLogSeg(currpos, &oldestSlotSeg);
9874+
KeepLogSeg(currpos, slotsMinReqLSN, &oldestSlotSeg);
98369875

98379876
/*
98389877
* Find the oldest extant segment file. We get 1 until checkpoint removes
@@ -9893,7 +9932,7 @@ GetWALAvailability(XLogRecPtr targetLSN)
98939932
* invalidation is optionally done here, instead.
98949933
*/
98959934
static void
9896-
KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo)
9935+
KeepLogSeg(XLogRecPtr recptr, XLogRecPtr slotsMinReqLSN, XLogSegNo *logSegNo)
98979936
{
98989937
XLogSegNo currSegNo;
98999938
XLogSegNo segno;
@@ -9906,7 +9945,7 @@ KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo)
99069945
* Calculate how many segments are kept by slots first, adjusting for
99079946
* max_slot_wal_keep_size.
99089947
*/
9909-
keep = XLogGetReplicationSlotMinimumLSN();
9948+
keep = slotsMinReqLSN;
99109949
if (keep != InvalidXLogRecPtr && keep < recptr)
99119950
{
99129951
XLByteToSeg(keep, segno, wal_segment_size);

‎src/backend/replication/logical/logical.c

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1087,7 +1087,15 @@ LogicalConfirmReceivedLocation(XLogRecPtr lsn)
10871087

10881088
SpinLockRelease(&MyReplicationSlot->mutex);
10891089

1090-
/* first write new xmin to disk, so we know what's up after a crash */
1090+
/*
1091+
* First, write new xmin and restart_lsn to disk so we know what's up
1092+
* after a crash. Even when we do this, the checkpointer can see the
1093+
* updated restart_lsn value in the shared memory; then, a crash can
1094+
* happen before we manage to write that value to the disk. Thus,
1095+
* checkpointer still needs to make special efforts to keep WAL
1096+
* segments required by the restart_lsn written to the disk. See
1097+
* CreateCheckPoint() and CreateRestartPoint() for details.
1098+
*/
10911099
if (updated_xmin || updated_restart)
10921100
{
10931101
ReplicationSlotMarkDirty();

‎src/backend/replication/walsender.c

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1894,6 +1894,10 @@ PhysicalConfirmReceivedLocation(XLogRecPtr lsn)
18941894
* be energy wasted - the worst lost information can do here is give us
18951895
* wrong information in a statistics view - we'll just potentially be more
18961896
* conservative in removing files.
1897+
*
1898+
* Checkpointer makes special efforts to keep the WAL segments required by
1899+
* the restart_lsn written to the disk. See CreateCheckPoint() and
1900+
* CreateRestartPoint() for details.
18971901
*/
18981902
}
18991903

0 commit comments

Comments
 (0)
0