8000 Merge branch 'REL_2_5' into REL_2_5-PBCKP-304 · postgrespro/pg_probackup@e2b4549 · GitHub
[go: up one dir, main page]
Skip to content

Commit e2b4549

Browse files
funny-falconIvan Lazarev
funny-falcon
authored and
Ivan Lazarev
committed
Merge branch 'REL_2_5' into REL_2_5-PBCKP-304
2 parents 04e05b1 + b17669c commit e2b4549

File tree

16 files changed

+937
-303
lines changed

16 files changed

+937
-303
lines changed

‎packaging/Makefile.test

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -130,10 +130,10 @@ build/test_suse: build/test_suse_15.1 build/test_suse_15.2
130130
@echo Suse: done
131131

132132
build/test_suse_15.1: build/test_suse_15.1_9.6 build/test_suse_15.1_10 build/test_suse_15.1_11 build/test_suse_15.1_12 build/test_suse_15.1_13
133-
@echo Rhel 15.1: done
133+
@echo Suse 15.1: done
134134

135135
build/test_suse_15.2: build/test_suse_15.2_9.6 build/test_suse_15.2_10 build/test_suse_15.2_11 build/test_suse_15.2_12 build/test_suse_15.2_13 build/test_suse_15.2_14
136-
@echo Rhel 15.1: done
136+
@echo Suse 15.2: done
137137

138138
define test_suse
139139
docker rm -f $1_$2_probackup_$(PKG_NAME_SUFFIX)$(PBK_VERSION) >> /dev/null 2>&1 ; \

‎packaging/test/scripts/rpm.sh

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,12 @@ if [ ${DISTRIB} == 'centos' ] && [ ${DISTRIB_VERSION} == '8' ]; then
7777
dnf -qy module disable postgresql
7878
fi
7979

80+
# PGDG doesn't support install of PG-9.6 from repo package anymore
81+
if [ ${PG_VERSION} == '9.6' ] && [ ${DISTRIB_VERSION} == '7' ]; then
82+
# ugly hack: use repo settings from PG10
83+
sed -i 's/10/9.6/' /etc/yum.repos.d/pgdg-redhat-all.repo
84+
fi
85+
8086
yum install -y postgresql${PG_TOG}-server.x86_64
8187
export PGDATA=/var/lib/pgsql/${PG_VERSION}/data
8288

‎src/archive.c

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1375,11 +1375,11 @@ get_wal_file(const char *filename, const char *from_fullpath,
13751375
#ifdef HAVE_LIBZ
13761376
/* If requested file is regular WAL segment, then try to open it with '.gz' suffix... */
13771377
if (IsXLogFileName(filename))
1378-
rc = fio_send_file_gz(from_fullpath_gz, to_fullpath, out, &errmsg);
1378+
rc = fio_send_file_gz(from_fullpath_gz, out, &errmsg);
13791379
if (rc == FILE_MISSING)
13801380
#endif
13811381
/* ... failing that, use uncompressed */
1382-
rc = fio_send_file(from_fullpath, to_fullpath, out, NULL, &errmsg);
1382+
rc = fio_send_file(from_fullpath, out, false, NULL, &errmsg);
13831383

13841384
/* When not in prefetch mode, try to use partial file */
13851385
if (rc == FILE_MISSING && !prefetch_mode && IsXLogFileName(filename))
@@ -1389,13 +1389,13 @@ get_wal_file(const char *filename, const char *from_fullpath,
13891389
#ifdef HAVE_LIBZ
13901390
/* '.gz.partial' goes first ... */
13911391
snprintf(from_partial, sizeof(from_partial), "%s.gz.partial", from_fullpath);
1392-
rc = fio_send_file_gz(from_partial, to_fullpath, out, &errmsg);
1392+
rc = fio_send_file_gz(from_partial, out, &errmsg);
13931393
if (rc == FILE_MISSING)
13941394
#endif
13951395
{
13961396
/* ... failing that, use '.partial' */
13971397
snprintf(from_partial, sizeof(from_partial), "%s.partial", from_fullpath);
1398-
rc = fio_send_file(from_partial, to_fullpath, out, NULL, &errmsg);
1398+
rc = fio_send_file(from_partial, out, false, NULL, &errmsg);
13991399
}
14001400

14011401
if (rc == SEND_OK)

‎src/backup.c

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -946,10 +946,21 @@ check_server_version(PGconn *conn, PGNodeInfo *nodeInfo)
946946
*/
947947
#ifdef PGPRO_VERSION
948948
if (!res)
949+
{
949950
/* It seems we connected to PostgreSQL (not Postgres Pro) */
950-
elog(ERROR, "%s was built with Postgres Pro %s %s, "
951-
"but connection is made with F438 PostgreSQL %s",
952-
PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, nodeInfo->server_version_str);
951+
if(strcmp(PGPRO_EDITION, "1C") != 0)
952+
{
953+
elog(ERROR, "%s was built with Postgres Pro %s %s, "
954+
"but connection is made with PostgreSQL %s",
955+
PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, nodeInfo->server_version_str);
956+
}
957+
/* We have PostgresPro for 1C and connect to PostgreSQL or PostgresPro for 1C
958+
* Check the major version
959+
*/
960+
if (strcmp(nodeInfo->server_version_str, PG_MAJORVERSION) != 0)
961+
elog(ERROR, "%s was built with PostgrePro %s %s, but connection is made with %s",
962+
PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, nodeInfo->server_version_str);
963+
}
953964
else
954965
{
955966
if (strcmp(nodeInfo->server_version_str, PG_MAJORVERSION) != 0 &&

‎src/catalog.c

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1069,6 +1069,7 @@ get_backup_filelist(pgBackup *backup, bool strict)
10691069
char linked[MAXPGPATH];
10701070
char compress_alg_string[MAXPGPATH];
10711071
int64 write_size,
1072+
uncompressed_size,
10721073
mode, /* bit length of mode_t depends on platforms */
10731074
is_datafile,
10741075
is_cfs,
@@ -1132,8 +1133,30 @@ get_backup_filelist(pgBackup *backup, bool strict)
11321133
if (get_control_value_int64(buf, "hdr_size", &hdr_size, false))
11331134
file->hdr_size = (int) hdr_size;
11341135

1135-
if (file->external_dir_num == 0)
1136+
if (get_control_value_int64(buf, "full_size", &uncompressed_size, false))
1137+
file->uncompressed_size = uncompressed_size;
1138+
else
1139+
file->uncompressed_size = write_size;
1140+
if (!file->is_datafile || file->is_cfs)
1141+
file->size = file->uncompressed_size;
1142+
1143+
if (file->external_dir_num == 0 && S_ISREG(file->mode))
1144+
{
1145+
bool is_datafile = file->is_datafile;
11361146
set_forkname(file);
1147+
if (is_datafile != file->is_datafile)
1148+
{
1149+
if (is_datafile)
1150+
elog(WARNING, "File '%s' was stored as datafile, but looks like it is not",
1151+
file->rel_path);
1152+
else
1153+
elog(WARNING, "File '%s' was stored as non-datafile, but looks like it is",
1154+
file->rel_path);
1155+
/* Lets fail in tests */
1156+
Assert(file->is_datafile == file->is_datafile);
1157+
file->is_datafile = is_datafile;
1158+
}
1159+
}
11371160

11381161
parray_append(files, file);
11391162
}
@@ -2561,6 +2584,11 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
25612584
file->external_dir_num,
25622585
file->dbOid);
25632586

2587+
if (file->uncompressed_size != 0 &&
2588+
file->uncompressed_size != file->write_size)
2589+
len += sprintf(line+len, ",\"full_size\":\"" INT64_FORMAT "\"",
2590+
file->uncompressed_size);
2591+
25642592
if (file->is_datafile)
25652593
len += sprintf(line+len, ",\"segno\":\"%d\"", file->segno);
25662594

‎src/data.c

Lines changed: 40 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -799,13 +799,17 @@ backup_non_data_file(pgFile *file, pgFile *prev_file,
799799
* and its mtime is less than parent backup start time ... */
800800
if ((pg_strcasecmp(file->name, RELMAPPER_FILENAME) != 0) &&
801801
(prev_file && file->exists_in_prev &&
802+
file->size == prev_file->size &&
802803
file->mtime <= parent_backup_time))
803804
{
804805
/*
805806
* file could be deleted under our feets.
806807
* But then backup_non_data_file_internal will handle it safely
807808
*/
808-
file->crc = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, true);
809+
if (file->forkName != cfm)
810+
file->crc = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, true);
811+
else
812+
file->crc = fio_get_crc32_truncated(from_fullpath, FIO_DB_HOST, true);
809813

810814
/* ...and checksum is the same... */
811815
if (EQ_TRADITIONAL_CRC32(file->crc, prev_file->crc))
@@ -1330,7 +1334,12 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup,
13301334
if (already_exists)
13311335
{
13321336
/* compare checksums of already existing file and backup file */
1333-
pg_crc32 file_crc = fio_get_crc32(to_fullpath, FIO_DB_HOST, false, false);
1337+
pg_crc32 file_crc;
1338+
if (tmp_file->forkName == cfm &&
1339+
tmp_file->uncompressed_size > tmp_file->write_size)
1340+
file_crc = fio_get_crc32_truncated(to_fullpath, FIO_DB_HOST, false);
1341+
else
1342+
file_crc = fio_get_crc32(to_fullpath, FIO_DB_HOST, false, false);
13341343

13351344
if (file_crc == tmp_file->crc)
13361345
{
@@ -1387,10 +1396,12 @@ backup_non_data_file_internal(const char *from_fullpath,
13871396
const char *to_fullpath, pgFile *file,
13881397
bool missing_ok)
13891398
{
1390-
FILE *in = NULL;
13911399
FILE *out = NULL;
1392-
ssize_t read_len = 0;
1393-
char *buf = NULL;
1400+
char *errmsg = NULL;
1401+
int rc;
1402+
bool cut_zero_tail;
1403+
1404+
cut_zero_tail = file->forkName == cfm;
13941405

13951406
INIT_FILE_CRC32(true, file->crc);
13961407

@@ -1412,107 +1423,44 @@ backup_non_data_file_internal(const char *from_fullpath,
14121423

14131424
/* backup remote file */
14141425
if (fio_is_remote(FIO_DB_HOST))
1415-
{
1416-
char *errmsg = NULL;
1417-
int rc = fio_send_file(from_fullpath, to_fullpath, out, file, &errmsg);
1426+
rc = fio_send_file(from_fullpath, out, cut_zero_tail, file, &errmsg);
1427+
else
1428+
rc = fio_send_file_local(from_fullpath, out, cut_zero_tail, file, &errmsg);
14181429

1419-
/* handle errors */
1420-
if (rc == FILE_MISSING)
1421-
{
1422-
/* maybe deleted, it's not error in case of backup */
1423-
if (missing_ok)
1424-
{
1425-
elog(LOG, "File \"%s\" is not found", from_fullpath);
1426-
file->write_size = FILE_NOT_FOUND;
1427-
goto cleanup;
1428-
}
1429-
else
1430-
elog(ERROR, "File \"%s\" is not found", from_fullpath);
1431-
}
1432-
else if (rc == WRITE_FAILED)
1433-
elog(ERROR, "Cannot write to \"%s\": %s", to_fullpath, strerror(errno));
1434-
else if (rc != SEND_OK)
1430+
/* handle errors */
1431+
if (rc == FILE_MISSING)
1432+
{
1433+
/* maybe deleted, it's not error in case of backup */
1434+
if (missing_ok)
14351435
{
1436-
if (errmsg)
1437-
elog(ERROR, "%s", errmsg);
1438-
else
1439-
elog(ERROR, "Cannot access remote file \"%s\"", from_fullpath);
1436+
elog(LOG, "File \"%s\" is not found", from_fullpath);
1437+
file->write_size = FILE_NOT_FOUND;
1438+
goto cleanup;
14401439
}
1441-
1442-
pg_free(errmsg);
1440+
else
1441+
elog(ERROR, "File \"%s\" is not found", from_fullpath);
14 10000 431442
}
1444-
/* backup local file */
1445-
else
1443+
else if (rc == WRITE_FAILED)
1444+
elog(ERROR, "Cannot write to \"%s\": %s", to_fullpath, strerror(errno));
1445+
else if (rc != SEND_OK)
14461446
{
1447-
/* open source file for read */
1448-
in = fopen(from_fullpath, PG_BINARY_R);
1449-
if (in == NULL)
1450-
{
1451-
/* maybe deleted, it's not error in case of backup */
1452-
if (errno == ENOENT)
1453-
{
1454-
if (missing_ok)
1455-
{
1456-
elog(LOG, "File \"%s\" is not found", from_fullpath);
1457-
file->write_size = FILE_NOT_FOUND;
1458-
goto cleanup;
1459-
}
1460-
else
1461-
elog(ERROR, "File \"%s\" is not found", from_fullpath);
1462-
}
1463-
1464-
elog(ERROR, "Cannot open file \"%s\": %s", from_fullpath,
1465-
strerror(errno));
1466-
}
1467-
1468-
/* disable stdio buffering for local input/output files to avoid triple buffering */
1469-
setvbuf(in, NULL, _IONBF, BUFSIZ);
1470-
setvbuf(out, NULL, _IONBF, BUFSIZ);
1471-
1472-
/* allocate 64kB buffer */
1473-
buf = pgut_malloc(CHUNK_SIZE);
1474-
1475-
/* copy content and calc CRC */
1476-
for (;;)
1477-
{
1478-
read_len = fread(buf, 1, CHUNK_SIZE, in);
1479-
1480-
if (ferror(in))
1481-
elog(ERROR, "Cannot read from file \"%s\": %s",
1482-
from_fullpath, strerror(errno));
1483-
1484-
if (read_len > 0)
1485-
{
1486-
if (fwrite(buf, 1, read_len, out) != read_len)
1487-
elog(ERROR, "Cannot write to file \"%s\": %s", to_fullpath,
1488-
strerror(errno));
1489-
1490-
/* update CRC */
1491-
COMP_FILE_CRC32(true, file->crc, buf, read_len);
1492-
file->read_size += read_len;
1493-
}
1494-
1495-
if (feof(in))
1496-
break;
1497-
}
1447+
if (errmsg)
1448+
elog(ERROR, "%s", errmsg);
1449+
else
1450+
elog(ERROR, "Cannot access remote file \"%s\"", from_fullpath);
14981451
}
14991452

1500-
file->write_size = (int64) file->read_size;
1501-
1502-
if (file->write_size > 0)
1503-
file->uncompressed_size = file->write_size;
1453+
file->uncompressed_size = file->read_size;
15041454

15051455
cleanup:
1456+
if (errmsg != NULL)
1457+
pg_free(errmsg);
1458+
15061459
/* finish CRC calculation and store into pgFile */
15071460
FIN_FILE_CRC32(true, file->crc);
15081461

1509-
if (in && fclose(in))
1510-
elog(ERROR, "Cannot close the file \"%s\": %s", from_fullpath, strerror(errno));
1511-
15121462
if (out && fclose(out))
15131463
elog(ERROR, "Cannot close the file \"%s\": %s", to_fullpath, strerror(errno));
1514-
1515-
pg_free(buf);
15161464
}
15171465

15181466
/*

0 commit comments

Comments
 (0)
0