8000 Adopt the GNU convention for handling tar-archive members exceeding 8GB. · SudhirLonkar/postgres@8f1559a · GitHub
[go: up one dir, main page]
Skip to content

Commit 8f1559a

Browse files
tglsfdctglsfdc
committed
Adopt the GNU convention for handling tar-archive members exceeding 8GB.
The POSIX standard for tar headers requires archive member sizes to be printed in octal with at most 11 digits, limiting the representable file size to 8GB. However, GNU tar and apparently most other modern tars support a convention in which oversized values can be stored in base-256, allowing any practical file to be a tar member. Adopt this convention to remove two limitations: * pg_dump with -Ft output format failed if the contents of any one table exceeded 8GB. * pg_basebackup failed if the data directory contained any file exceeding 8GB. (This would be a fatal problem for installations configured with a table segment size of 8GB or more, and it has also been seen to fail when large core dump files exist in the data directory.) File sizes under 8GB are still printed in octal, so that no compatibility issues are created except in cases that would have failed entirely before. In addition, this patch fixes several bugs in the same area: * In 9.3 and later, we'd defined tarCreateHeader's file-size argument as size_t, which meant that on 32-bit machines it would write a corrupt tar header for file sizes between 4GB and 8GB, even though no error was raised. This broke both "pg_dump -Ft" and pg_basebackup for such cases. * pg_restore from a tar archive would fail on tables of size between 4GB and 8GB, on machines where either "size_t" or "unsigned long" is 32 bits. This happened even with an archive file not affected by the previous bug. * pg_basebackup would fail if there were files of size between 4GB and 8GB, even on 64-bit machines. * In 9.3 and later, "pg_basebackup -Ft" failed entirely, for any file size, on 64-bit big-endian machines. In view of these potential data-loss bugs, back-patch to all supported branches, even though removal of the documented 8GB limit might otherwise be considered a new feature rather than a bug fix.
1 parent 60ba32c commit 8f1559a

File tree

7 files changed

+267
-293
lines changed

7 files changed

+267
-293
lines changed

‎doc/src/sgml/ref/pg_dump.sgml

Lines changed: 5 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -259,12 +259,12 @@ PostgreSQL documentation
259259
<listitem>
260260
<para>
261261
Output a <command>tar</command>-format archive suitable for input
262-
into <application>pg_restore</application>. The tar-format is
263-
compatible with the directory-format; extracting a tar-format
262+
into <application>pg_restore</application>. The tar format is
263+
compatible with the directory format: extracting a tar-format
264264
archive produces a valid directory-format archive.
265-
However, the tar-format does not support compression and has a
266-
limit of 8 GB on the size of individual tables. Also, the relative
267-
order of table data items cannot be changed during restore.
265+
However, the tar format does not support compression. Also, when
266+
using tar format the relative order of table data items cannot be
267+
changed during restore.
268268
</para>
269269
</listitem>
270270
</varlistentry>
@@ -932,15 +932,6 @@ CREATE DATABASE foo WITH TEMPLATE template0;
932932
catalogs might be left in the wrong state.
933933
</para>
934934

935-
<para>
936-
Members of tar archives are limited to a size less than 8 GB.
937-
(This is an inherent limitation of the tar file format.) Therefore
938-
this format cannot be used if the textual representation of any one table
939-
exceeds that size. The total size of a tar archive and any of the
940-
other output formats is not limited, except possibly by the
941-
operating system.
942-
</para>
943-
944935
<para>
945936
The dump file produced by <application>pg_dump</application>
946937
does not contain the statistics used by the optimizer to make

‎src/backend/replication/basebackup.c

Lines changed: 5 additions & 141 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
#include "utils/elog.h"
3434
#include "utils/memutils.h"
3535
#include "utils/ps_status.h"
36+
#include "pgtar.h"
3637

3738
typedef struct
3839
{
@@ -845,49 +846,6 @@ sendDir(char *path, int basepathlen, bool sizeonly, List *tablespaces)
845846
*/
846847

847848

848-
/*
849-
* Utility routine to print possibly larger than 32 bit integers in a
850-
* portable fashion. Filled with zeros.
851-
*/
852-
static void
853-
print_val(char *s, uint64 val, unsigned int base, size_t len)
854-
{
855-
int i;
856-
857-
for (i = len; i > 0; i--)
858-
{
859-
int digit = val % base;
860-
861-
s[i - 1] = '0' + digit;
862-
val = val / base;
863-
}
864-
}
865-
866-
/*
867-
* Maximum file size for a tar member: The limit inherent in the
868-
* format is 2^33-1 bytes (nearly 8 GB). But we don't want to exceed
869-
* what we can represent in pgoff_t.
870-
*/
871-
#define MAX_TAR_MEMBER_FILELEN (((int64) 1 << Min(33, sizeof(pgoff_t)*8 - 1)) - 1)
872-
873-
static int
874-
_tarChecksum(char *header)
875-
{
876-
int i,
877-
sum;
878-
879-
/*
880-
* Per POSIX, the checksum is the simple sum of all bytes in the header,
881-
* treating the bytes as unsigned, and treating the checksum field (at
882-
* offset 148) as though it contained 8 spaces.
883-
*/
884-
sum = 8 * ' '; /* presumed value for checksum field */
885-
for (i = 0; i < 512; i++)
886-
if (i < 148 || i >= 156)
887-
sum += 0xFF & header[i];
888-
return sum;
889-
}
890-
891849
/*
892850
* Given the member, write the TAR header & send the file.
893851
*
@@ -916,15 +874,6 @@ sendFile(char *readfilename, char *tarfilename, struct stat *statbuf,
916874
errmsg("could not open file \"%s\": %m", readfilename)));
917875
}
918876

919-
/*
920-
* Some compilers will throw a warning knowing this test can never be true
921-
* because pgoff_t can't exceed the compared maximum on their platform.
922-
*/
923-
if (statbuf->st_size > MAX_TAR_MEMBER_FILELEN)
924-
ereport(ERROR,
925-
(errmsg("archive member \"%s\" too large for tar format",
926-
tarfilename)));
927-
928877
_tarWriteHeader(tarfilename, NULL, statbuf);
929878

930879
while ((cnt = fread(buf, 1, Min(sizeof(buf), statbuf->st_size - len), fp)) > 0)
@@ -979,95 +928,10 @@ _tarWriteHeader(const char *filename, const char *linktarget,
979928
{
980929
char h[512];
981930

982-
/*
983-
* Note: most of the fields in a tar header are not supposed to be
984-
* null-terminated. We use sprintf, which will write a null after the
985-
* required bytes; that null goes into the first byte of the next field.
986-
* This is okay as long as we fill the fields in order.
987-
*/
988-
memset(h, 0, sizeof(h));
989-
990-
/* Name 100 */
991-
strlcpy(&h[0], filename, 100);
992-
if (linktarget != NULL || S_ISDIR(statbuf->st_mode))
993-
{
994-
/*
995-
* We only support symbolic links to directories, and this is
996-
* indicated in the tar format by adding a slash at the end of the
997-
* name, the same as for regular directories.
998-
*/
999-
int flen = strlen(filename);
1000-
1001-
flen = Min(flen, 99);
1002-
h[flen] = '/';
1003-
h[flen + 1] = '\0';
1004-
}
1005-
1006-
/* Mode 8 */
1007-
sprintf(&h[100], "%07o ", (int) statbuf->st_mode);
1008-
1009-
/* User ID 8 */
1010-
sprintf(&h[108], "%07o ", statbuf->st_uid);
1011-
1012-
/* Group 8 */
1013-
sprintf(&h[116], "%07o ", statbuf->st_gid);
1014-
1015-
/* File size 12 - 11 digits, 1 space; use print_val for 64 bit support */
1016-
if (linktarget != NULL || S_ISDIR(statbuf->st_mode))
1017-
/* Symbolic link or directory has size zero */
1018-
print_val(&h[124], 0, 8, 11);
1019-
else
1020-
print_val(&h[124], statbuf->st_size, 8, 11);
1021-
sprintf(&h[135], " ");
1022-
1023-
/* Mod Time 12 */
1024-
sprintf(&h[136], "%011o ", (int) statbuf->st_mtime);
1025-
1026-
/* Checksum 8 cannot be calculated until we've filled all other fields */
1027-
1028-
if (linktarget != NULL)
1029-
{
1030-
/* Type - Symbolic link */
1031-
sprintf(&h[156], "2");
1032-
/* Link Name 100 */
1033-
strlcpy(&h[157], linktarget, 100);
1034-
}
1035-
else if (S_ISDIR(statbuf->st_mode))
1036-
/* Type - directory */
1037-
sprintf(&h[156], "5");
1038-
else
1039-
/* Type - regular file */
1040-
sprintf(&h[156], "0");
1041-
1042-
/* Magic 6 */
1043-
sprintf(&h[257], "ustar");
1044-
1045-
/* Version 2 */
1046-
sprintf(&h[263], "00");
1047-
1048-
/* User 32 */
1049-
/* XXX: Do we need to care about setting correct username? */
1050-
strlcpy(&h[265], "postgres", 32);
1051-
1052-
/* Group 32 */
1053-
/* XXX: Do we need to care about setting correct group name? */
1054-
strlcpy(&h[297], "postgres", 32);
1055-
1056-
/* Major Dev 8 */
1057-
sprintf(&h[329], "%07o ", 0);
1058-
1059-
/* Minor Dev 8 */
1060-
sprintf(&h[337], "%07o ", 0);
1061-
1062-
/* Prefix 155 - not used, leave as nulls */
1063-
1064-
/*
1065-
* We mustn't overwrite the next field while inserting the checksum.
1066-
* Fortunately, the checksum can't exceed 6 octal digits, so we just write
1067-
* 6 digits, a space, and a null, which is legal per POSIX.
1068-
*/
1069-
sprintf(&h[148], "%06o ", _tarChecksum(h));
931+
tarCreateHeader(h, filename, linktarget, statbuf->st_size,
932+
statbuf->st_mode, statbuf->st_uid, statbuf->st_gid,
933+
statbuf->st_mtime,
934+
false /* write real POSIX header */);
1070935

1071-
/* Now send the completed header. */
1072936
pq_putmessage('d', h, 512);
1073937
}

‎src/bin/pg_basebackup/pg_basebackup.c

Lines changed: 4 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313

1414
#include "postgres_fe.h"
1515
#include "libpq-fe.h"
16+
#include "pgtar.h"
1617

1718
#include <unistd.h>
1819
#include <dirent.h>
@@ -502,7 +503,7 @@ ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum)
502503
{
503504
char current_path[MAXPGPATH];
504505
char filename[MAXPGPATH];
505-
int current_len_left;
506+
pgoff_t current_len_left = 0;
506507
int current_padding = 0;
507508
char *copybuf = NULL;
508509
FILE *file = NULL;
@@ -580,20 +581,10 @@ ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum)
580581
}
581582
totaldone += 512;
582583

583-
if (sscanf(copybuf + 124, "%11o", &current_len_left) != 1)
584-
{
585-
fprintf(stderr, _("%s: could not parse file size\n"),
586-
progname);
587-
disconnect_and_exit(1);
588-
}
584+
current_len_left = read_tar_number(&copybuf[124], 12);
589585

590586
/* Set permissions on the file */
591-
if (sscanf(&copybuf[100], "%07o ", &filemode) != 1)
592-
{
593-
fprintf(stderr, _("%s: could not parse file mode\n"),
594-
progname);
595-
disconnect_and_exit(1);
596-
}
587+
filemode = read_tar_number(&copybuf[100], 8);
597588

598589
/*
599590
* All files are padded up to 512 bytes

0 commit comments

Comments
 (0)
0