8000 Merge branch 'REL_2_5-some-fixes' into REL_2_5 · postgrespro/pg_probackup@ed47e49 · GitHub
[go: up one dir, main page]
Skip to content

Commit ed47e49

Browse files
funny-falconfunny-falcon
committed
Merge branch 'REL_2_5-some-fixes' into REL_2_5
2 parents c7e6d5c + 51a141c commit ed47e49

File tree

11 files changed

+89
-65
lines changed

11 files changed

+89
-65
lines changed

‎src/archive.c

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -512,8 +512,8 @@ push_file_internal_uncompressed(const char *wal_file_name, const char *pg_xlog_d
512512
pg_crc32 crc32_src;
513513
pg_crc32 crc32_dst;
514514

515-
crc32_src = fio_get_crc32(from_fullpath, FIO_DB_HOST, false);
516-
crc32_dst = fio_get_crc32(to_fullpath, FIO_BACKUP_HOST, false);
515+
crc32_src = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, false);
516+
crc32_dst = fio_get_crc32(to_fullpath, FIO_BACKUP_HOST, false, false);
517517

518518
if (crc32_src == crc32_dst)
519519
{
@@ -760,9 +760,8 @@ push_file_internal_gz(const char *wal_file_name, const char *pg_xlog_dir,
760760
pg_crc32 crc32_src;
761761
pg_crc32 crc32_dst;
762762

763-
/* TODO: what if one of them goes missing? */
764-
crc32_src = fio_get_crc32(from_fullpath, FIO_DB_HOST, false);
765-
crc32_dst = fio_get_crc32(to_fullpath_gz, FIO_BACKUP_HOST, true);
763+
crc32_src = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, false);
764+
crc32_dst = fio_get_crc32(to_fullpath_gz, FIO_BACKUP_HOST, true, false);
766765

767766
if (crc32_src == crc32_dst)
768767
{

‎src/catalog.c

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1132,6 +1132,9 @@ get_backup_filelist(pgBackup *backup, bool strict)
11321132
if (get_control_value_int64(buf, "hdr_size", &hdr_size, false))
11331133
file->hdr_size = (int) hdr_size;
11341134

1135+
if (file->external_dir_num == 0)
1136+
set_forkname(file);
1137+
11351138
parray_append(files, file);
11361139
}
11371140

@@ -2488,7 +2491,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
24882491
char control_path[MAXPGPATH];
24892492
char control_path_temp[MAXPGPATH];
24902493
size_t i = 0;
2491-
#define BUFFERSZ 1024*1024
2494+
#define BUFFERSZ (1024*1024)
24922495
char *buf;
24932496
int64 backup_size_on_disk = 0;
24942497
int64 uncompressed_size_on_disk = 0;

‎src/data.c

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -801,8 +801,11 @@ backup_non_data_file(pgFile *file, pgFile *prev_file,
801801
(prev_file && file->exists_in_prev &&
802802
file->mtime <= parent_backup_time))
803803
{
804-
805-
file->crc = fio_get_crc32(from_fullpath, FIO_DB_HOST, false);
804+
/*
805+
* file could be deleted under our feets.
806+
* But then backup_non_data_file_internal will handle it safely
807+
*/
808+
file->crc = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, true);
806809

807810
/* ...and checksum is the same... */
808811
if (EQ_TRADITIONAL_CRC32(file->crc, prev_file->crc))
@@ -1327,7 +1330,7 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup,
13271330
if (already_exists)
13281331
{
13291332
/* compare checksums of already existing file and backup file */
1330-
pg_crc32 file_crc = fio_get_crc32(to_fullpath, FIO_DB_HOST, false);
1333+
pg_crc32 file_crc = fio_get_crc32(to_fullpath, FIO_DB_HOST, false, false);
13311334

13321335
if (file_crc == tmp_file->crc)
13331336
{

‎src/dir.c

Lines changed: 42 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -758,57 +758,22 @@ dir_check_file(pgFile *file, bool backup_logs)
758758
return CHECK_FALSE;
759759
else if (isdigit(file->name[0]))
760760
{
761-
char *fork_name;
762-
int len;
763-
char suffix[MAXPGPATH];
761+
set_forkname(file);
764762

765-
fork_name = strstr(file->name, "_");
766-
if (fork_name)
767-
{
768-
/* Auxiliary fork of the relfile */
769-
if (strcmp(fork_name, "_vm") == 0)
770-
file->forkName = vm;
771-
772-
else if (strcmp(fork_name, "_fsm") == 0)
773-
file->forkName = fsm;
774-
775-
else if (strcmp(fork_name, "_cfm") == 0)
776-
file->forkName = cfm;
777-
778-
else if (strcmp(fork_name, "_ptrack") == 0)
779-
file->forkName = ptrack;
780-
781-
else if (strcmp(fork_name, "_init") == 0)
782-
file->forkName = init;
783-
784-
// extract relOid for certain forks
785-
if (file->forkName == vm ||
786-
file->forkName == fsm ||
787-
file->forkName == init ||
788-
file->forkName == cfm)
789-
{
790-
// sanity
791-
if (sscanf(file->name, "%u_*", &(file->relOid)) != 1)
792-
file->relOid = 0;
793-
}
763+
if (file->forkName == ptrack) /* Compatibility with left-overs from ptrack1 */
764+
return CHECK_FALSE;
765+
else if (file->forkName != none)
766+
return CHECK_TRUE;
794767

795-
/* Do not backup ptrack files */
796-
if (file->forkName == ptrack)
797-
return CHECK_FALSE;
798-
}
799-
else
768+
/* Set is_datafile flag */
800769
{
770+
char suffix[MAXFNAMELEN];
801771

802-
len = strlen(file->name);
803-
/* reloid.cfm */
804-
if (len > 3 && strcmp(file->name + len - 3, "cfm") == 0)
805-
return CHECK_TRUE;
806-
772+
/* check if file is datafile */
807773
sscanf_res = sscanf(file->name, "%u.%d.%s", &(file->relOid),
808774
&(file->segno), suffix);
809-
if (sscanf_res == 0)
810-
elog(ERROR, "Cannot parse file name \"%s\"", file->name);
811-
else if (sscanf_res == 1 || sscanf_res == 2)
775+
Assert(sscanf_res > 0); /* since first char is digit */
776+
if (sscanf_res == 1 || sscanf_res == 2)
812777
file->is_datafile = true;
813778
}
814779
}
@@ -1954,3 +1919,35 @@ pfilearray_clear_locks(parray *file_list)
19541919
pg_atomic_clear_flag(&file->lock);
19551920
}
19561921
}
1922+
1923+
/* Set forkName if possible */
1924+
void
1925+
set_forkname(pgFile *file)
1926+
{
1927+
int name_len = strlen(file->name);
1928+
1929+
/* Auxiliary fork of the relfile */
1930+
if (name_len > 3 && strcmp(file->name + name_len - 3, "_vm") == 0)
1931+
file->forkName = vm;
1932+
1933+
else if (name_len > 4 && strcmp(file->name + name_len - 4, "_fsm") == 0)
1934+
file->forkName = fsm;
1935+
1936+
else if (name_len > 4 && strcmp(file->name + name_len - 4, ".cfm") == 0)
1937+
file->forkName = cfm;
1938+
1939+
else if (name_len > 5 && strcmp(file->name + name_len - 5, "_init") == 0)
1940+
file->forkName = init;
1941+
1942+
else if (name_len > 7 && strcmp(file->name + name_len - 7, "_ptrack") == 0)
1943+
file->forkName = ptrack;
1944+
1945+
// extract relOid for certain forks
1946+
1947+
if ((file->forkName == vm ||
1948+
file->forkName == fsm ||
1949+
file->forkName == init ||
1950+
file->forkName == cfm) &&
1951+
(sscanf(file->name, "%u*", &(file->relOid)) != 1))
1952+
file->relOid = 0;
1953+
}

‎src/pg_probackup.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ bool perm_slot = false;
8888
/* backup options */
8989
bool backup_logs = false;
9090
bool smooth_checkpoint;
91-
char *remote_agent;
91+
bool remote_agent = false;
9292
static char *backup_note = NULL;
9393
/* catchup options */
9494
static char *catchup_source_pgdata = NULL;
@@ -361,6 +361,7 @@ main(int argc, char *argv[])
361361
elog(ERROR, "Version mismatch, pg_probackup binary with version '%s' "
362362
"is launched as an agent for pg_probackup binary with version '%s'",
363363
PROGRAM_VERSION, argv[2]);
364+
remote_agent = true;
364365
fio_communicate(STDIN_FILENO, STDOUT_FILENO);
365366
return 0;
366367
case HELP_CMD:

‎src/pg_probackup.h

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -215,6 +215,7 @@ typedef enum CompressAlg
215215

216216
typedef enum ForkName
217217
{
218+
none,
218219
vm,
219220
fsm,
220221
cfm,
@@ -798,7 +799,7 @@ extern bool perm_slot;
798799
extern bool smooth_checkpoint;
799800

800801
/* remote probackup options */
801-
extern char* remote_agent;
802+
extern bool remote_agent;
802803

803804
extern bool exclusive_backup;
804805

@@ -1091,6 +1092,7 @@ extern int pgCompareString(const void *str1, const void *str2);
10911092
extern int pgPrefixCompareString(const void *str1, const void *str2);
10921093
extern int pgCompareOid(const void *f1, const void *f2);
10931094
extern void pfilearray_clear_locks(parray *file_list);
1095+
extern void set_forkname(pgFile *file);
10941096

10951097
/* in data.c */
10961098
extern bool check_data_file(ConnectionArgs *arguments, pgFile *file,

‎src/utils/configuration.c

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -531,7 +531,6 @@ config_get_opt(int argc, char **argv, ConfigOption cmd_options[],
531531
opt = option_find(c, options);
532532

533533
if (opt
534-
&& !remote_agent
535534
&& opt->allowed < SOURCE_CMD && opt->allowed != SOURCE_CMD_STRICT)
536535
elog(ERROR, "Option %s cannot be specified in command line",
537536
opt->lname);

‎src/utils/file.c

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1355,9 +1355,15 @@ fio_sync(char const* path, fio_location location)
13551355
}
13561356
}
13571357

1358+
enum {
1359+
GET_CRC32_DECOMPRESS = 1,
1360+
GET_CRC32_MISSING_OK = 2
1361+
};
1362+
13581363
/* Get crc32 of file */
13591364
pg_crc32
1360-
fio_get_crc32(const char *file_path, fio_location location, bool decompress)
1365+
fio_get_crc32(const char *file_path, fio_location location,
1366+
bool decompress, bool missing_ok)
13611367
{
13621368
if (fio_is_remote(location))
13631369
{
@@ -1370,7 +1376,9 @@ fio_get_crc32(const char *file_path, fio_location location, bool decompress)
13701376
hdr.arg = 0;
13711377

13721378
if (decompress)
1373-
hdr.arg = 1;
1379+
hdr.arg = GET_CRC32_DECOMPRESS;
1380+
if (missing_ok)
1381+
hdr.arg |= GET_CRC32_MISSING_OK;
13741382

13751383
IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr));
13761384
IO_CHECK(fio_write_all(fio_stdout, file_path, path_len), path_len);
@@ -1381,9 +1389,9 @@ fio_get_crc32(const char *file_path, fio_location location, bool decompress)
13811389
else
13821390
{
13831391
if (decompress)
1384-
return pgFileGetCRCgz(file_path, true, true);
1392+
return pgFileGetCRCgz(file_path, true, missing_ok);
13851393
else
1386-
return pgFileGetCRC(file_path, true, true);
1394+
return pgFileGetCRC(file_path, true, missing_ok);
13871395
}
13881396
}
13891397

@@ -3380,10 +3388,10 @@ fio_communicate(int in, int out)
33803388
break;
33813389
case FIO_GET_CRC32:
33823390
/* calculate crc32 for a file */
3383-
if (hdr.arg == 1)
3384-
crc = pgFileGetCRCgz(buf, true, true);
3391+
if ((hdr.arg & GET_CRC32_DECOMPRESS))
3392+
crc = pgFileGetCRCgz(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0);
33853393
else
3386-
crc = pgFileGetCRC(buf, true, true);
3394+
crc = pgFileGetCRC(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0);
33873395
IO_CHECK(fio_write_all(out, &crc, sizeof(crc)), sizeof(crc));
33883396
break;
33893397
case FIO_GET_CHECKSUM_MAP:

‎src/utils/file.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,8 @@ extern int fio_truncate(int fd, off_t size);
120120
extern int fio_close(int fd);
121121
extern void fio_disconnect(void);
122122
extern int fio_sync(char const* path, fio_location location);
123-
extern pg_crc32 fio_get_crc32(const char *file_path, fio_location location, bool decompress);
123+
extern pg_crc32 fio_get_crc32(const char *file_path, fio_location location,
124+
bool decompress, bool missing_ok);
124125

125126
extern int fio_rename(char const* old_path, char const* new_path, fio_location location);
126127
extern int fio_symlink(char const* target, char const* link_path, bool overwrite, fio_location location);

‎tests/cfs_backup.py

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -995,6 +995,11 @@ def test_delete_random_cfm_file_from_tablespace_dir(self):
995995
"FROM generate_series(0,256) i".format('t1', tblspace_name)
996996
)
997997

998+
self.node.safe_psql(
999+
"postgres",
1000+
"CHECKPOINT"
1001+
)
1002+
9981003
list_cmf = find_by_extensions(
9991004
[self.get_tblspace_path(self.node, tblspace_name)],
10001005
['.cfm'])
@@ -1044,6 +1049,11 @@ def test_delete_random_data_file_from_tablespace_dir(self):
10441049
"FROM generate_series(0,256) i".format('t1', tblspace_name)
10451050
)
10461051

1052+
self.node.safe_psql(
1053+
"postgres",
1054+
"CHECKPOINT"
1055+
)
1056+
10471057
list_data_files = find_by_pattern(
10481058
[self.get_tblspace_path(self.node, tblspace_name)],
10491059
'^.*/\d+$')

‎tests/cfs_restore.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,7 @@ def test_restore_empty_tablespace_from_fullbackup(self):
103103
"postgres",
104104
"SELECT * FROM pg_tablespace WHERE spcname='{0}'".format(tblspace_name)
105105
)
106+
tblspace = str(tblspace)
106107
self.assertTrue(
107108
tblspace_name in tblspace and "compression=true" in tblspace,
108109
"ERROR: The tablespace not restored or it restored without compressions"

0 commit comments

Comments
 (0)
0