8000 Fix issues with wide tuples being updated and REPLICA IDENTITY FULL. · dirbacke/postgres@23a2b81 · GitHub
[go: up one dir, main page]
Skip to content

Commit 23a2b81

Browse files
anarazelanarazel
committed
Fix issues with wide tuples being updated and REPLICA IDENTITY FULL.
When replica identity full is being used with a wide tuple (above 2^16 bytes after compression) it lead to errors and/or crashes during decoding because the length field used to store such tuples doesn't fit into the variable used to store the width in the WAL record. To fix, discontinue use of xl_heap_header_len.t_len when decoding the old tuple version, instead compute length of the old tuple by subtracting the new tuple's length from the record length. In newer version of postgres this issue is moot because the length is stored by the new WAL machinery, instead of a xl_heap_header_len struct. A separate commit will forward-patch the regression test. Reported-By: "anderson" Discussion: http://postgr.es/m/20170105144819.f6i5o64vfvy4bn5i@alap3.anarazel.de
1 parent 8d05db3 commit 23a2b81

File tree

4 files changed

+32
-6
lines changed

4 files changed

+32
-6
lines changed

‎contrib/test_decoding/expected/toast.out

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -298,7 +298,13 @@ ALTER TABLE toasted_several REPLICA IDENTITY FULL;
298298
ALTER TABLE toasted_several ALTER COLUMN toasted_key SET STORAGE EXTERNAL;
299299
ALTER TABLE toasted_several ALTER COLUMN toasted_col1 SET STORAGE EXTERNAL;
300300
ALTER TABLE toasted_several ALTER COLUMN toasted_col2 SET STORAGE EXTERNAL;
301-
INSERT INTO toasted_several(toasted_key) VALUES(repeat('9876543210', 2000));
301+
INSERT INTO toasted_several(toasted_key) VALUES(repeat('9876543210', 10000));
302+
SELECT pg_column_size(toasted_key) > 2^16 FROM toasted_several;
303+
?column?
304+
----------
305+
t
306+
(1 row)
307+
302308
SELECT regexp_replace(data, '^(.{100}).*(.{100})$', '\1..\2') FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1');
303309
regexp_replace
304310
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------

‎contrib/test_decoding/sql/toast.sql

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -274,7 +274,8 @@ ALTER TABLE toasted_several ALTER COLUMN toasted_key SET STORAGE EXTERNAL;
274274
ALTER TABLE toasted_several ALTER COLUMN toasted_col1 SET STORAGE EXTERNAL;
275275
ALTER TABLE toasted_several ALTER COLUMN toasted_col2 SET STORAGE EXTERNAL;
276276

277-
INSERT INTO toasted_several(toasted_key) VALUES(repeat('9876543210', 2000));
277+
INSERT INTO toasted_several(toasted_key) VALUES(repeat('9876543210', 10000));
278+
SELECT pg_column_size(toasted_key) > 2^16 FROM toasted_several;
278279

279280
SELECT regexp_replace(data, '^(.{100}).*(.{100})$', '\1..\2') FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1');
280281

‎src/backend/access/heap/heapam.c

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7086,7 +7086,13 @@ log_heap_update(Relation reln, Buffer oldbuf,
70867086
/* We need to log a tuple identity */
70877087
if (old_key_tuple)
70887088
{
7089-
/* don't really need this, but its more comfy to decode */
7089+
/*
7090+
* This isn't needed, and can't actually capture the contents of
7091+
* the tuple accurately (because t_len isn't guaranteed to be big
7092+
* enough to contain old tuples which can be up to 1 GB long). But
7093+
* previous versions of 9.4 used this, so we can't change the WAL
7094+
* format.
7095+
*/
70907096
xlhdr_idx.header.t_infomask2 = old_key_tuple->t_data->t_infomask2;
70917097
xlhdr_idx.header.t_infomask = old_key_tuple->t_data->t_infomask;
70927098
xlhdr_idx.header.t_hoff = old_key_tuple->t_data->t_hoff;

‎src/backend/replication/logical/decode.c

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -653,6 +653,7 @@ DecodeUpdate(LogicalDecodingContext *ctx, XLogRecordBuffer *buf)
653653
xl_heap_update *xlrec;
654654
ReorderBufferChange *change;
655655
char *data;
656+
size_t remlen = r->xl_len;
656657

657658
xlrec = (xl_heap_update *) buf->record_data;
658659

@@ -666,6 +667,7 @@ DecodeUpdate(LogicalDecodingContext *ctx, XLogRecordBuffer *buf)
666667

667668
/* caution, remaining data in record is not aligned */
668669
data = buf->record_data + SizeOfHeapUpdate;
670+
remlen -= SizeOfHeapUpdate;
669671

670672
if (xlrec->flags & XLOG_HEAP_CONTAINS_NEW_TUPLE)
671673
{
@@ -677,6 +679,7 @@ DecodeUpdate(LogicalDecodingContext *ctx, XLogRecordBuffer *buf)
677679

678680
memcpy(&xlhdr, data, sizeof(xlhdr));
679681
data += offsetof(xl_heap_header_len, header);
682+
remlen -= offsetof(xl_heap_header_len, header);
680683

681684
datalen = xlhdr.t_len + SizeOfHeapHeader;
682685
tuplelen = xlhdr.t_len;
@@ -687,8 +690,10 @@ DecodeUpdate(LogicalDecodingContext *ctx, XLogRecordBuffer *buf)
687690
DecodeXLogTuple(data, datalen, change->data.tp.newtuple);
688691
/* skip over the rest of the tuple header */
689692
data += SizeOfHeapH 8000 eader;
693+
remlen -= SizeOfHeapHeader;
690694
/* skip over the tuple data */
691695
data += xlhdr.t_len;
696+
remlen -= xlhdr.t_len;
692697
}
693698

694699
if (xlrec->flags & XLOG_HEAP_CONTAINS_OLD)
@@ -699,17 +704,25 @@ DecodeUpdate(LogicalDecodingContext *ctx, XLogRecordBuffer *buf)
699704

700705
memcpy(&xlhdr, data, sizeof(xlhdr));
701706
data += offsetof(xl_heap_header_len, header);
707+
remlen -= offsetof(xl_heap_header_len, header);
702708

703-
/* t_len is inconsistent with other cases, see log_heap_update */
704-
tuplelen = xlhdr.t_len - offsetof(HeapTupleHeaderData, t_bits);
705-
datalen = tuplelen + SizeOfHeapHeader;
709+
/*
710+
* NB: Even though xl_heap_header_len contains the tuple's length,
711+
* it's length field is not wide enough. Use the whole record length
712+
* minus the new tuple's length instead. We can't remove the record
713+
* length from the WAL record format in 9.4 due to compatibility
714+
* concerns - later versions don't have it anyway.
715+
*/
716+
datalen = remlen;
717+
tuplelen = datalen - SizeOfHeapHeader;
706718

707719
change->data.tp.oldtuple =
708720
ReorderBufferGetTupleBuf(ctx->reorder, tuplelen);
709721

710722
DecodeXLogTuple(data, datalen, change->data.tp.oldtuple);
711723
#ifdef NOT_USED
712724
data += datalen;
725+
remlen -= datalen;
713726
#endif
714727
}
715728

0 commit comments

Comments
 (0)
0