8000 Fix MERGE of compressed and uncompressed backups · postgrespro/pg_probackup@2837399 · GitHub
[go: up one dir, main page]
Skip to content

Commit 2837399

Browse files
za-arthurza-arthur
committed
Fix MERGE of compressed and uncompressed backups
1 parent 59ab21e commit 2837399

File tree

2 files changed

+51
-67
lines changed

2 files changed

+51
-67
lines changed

‎src/merge.c

Lines changed: 51 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,6 @@ do_merge(time_t backup_id)
131131
{
132132
pgBackup *from_backup = (pgBackup *) parray_get(backups, i - 1);
133133

134-
full_backup = (pgBackup *) parray_get(backups, i);
135134
merge_backups(full_backup, from_backup);
136135
}
137136

@@ -168,8 +167,6 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup)
168167
merge_files_arg *threads_args = NULL;
169168
int i;
170169
bool merge_isok = true;
171-
int64 to_data_bytes,
172-
to_wal_bytes;
173170

174171
elog(INFO, "Merging backup %s with backup %s", from_backup_id, to_backup_id);
175172

@@ -283,29 +280,37 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup)
283280
* Update to_backup metadata.
284281
*/
285282
to_backup->status = BACKUP_STATUS_OK;
283+
to_backup->parent_backup = INVALID_BACKUP_ID;
284+
to_backup->start_lsn = from_backup->start_lsn;
285+
to_backup->stop_lsn = from_backup->stop_lsn;
286+
to_backup->recovery_time = from_backup->recovery_time;
287+
to_backup->recovery_xid = from_backup->recovery_xid;
288+
/*
289+
* If one of the backups isn't "stream" backup then the target backup become
290+
* non-stream backup too.
291+
*/
292+
to_backup->stream = to_backup->stream && from_backup->stream;
286293
/* Compute summary of size of regular files in the backup */
287-
to_data_bytes = 0;
294+
to_backup->data_bytes = 0;
288295
for (i = 0; i < parray_num(files); i++)
289296
{
290297
pgFile *file = (pgFile *) parray_get(files, i);
291298

292299
if (S_ISDIR(file->mode))
293-
to_data_bytes += 4096;
300+
to_backup->data_bytes += 4096;
294301
/* Count the amount of the data actually copied */
295302
else if (S_ISREG(file->mode))
296-
to_data_bytes += file->write_size;
303+
to_backup->data_bytes += file->write_size;
297304
}
298305
/* compute size of wal files of this backup stored in the archive */
299306
if (!to_backup->stream)
300-
to_wal_bytes = instance_config.xlog_seg_size *
307+
to_backup->wal_bytes = instance_config.xlog_seg_size *
301308
(to_backup->stop_lsn / instance_config.xlog_seg_size -
302309
to_backup->start_lsn / instance_config.xlog_seg_size + 1);
303310
else
304-
to_wal_bytes = BYTES_INVALID;
311+
to_backup->wal_bytes = BYTES_INVALID;
305312

306313
write_backup_filelist(to_backup, files, from_database_path);
307-
to_backup->data_bytes = to_data_bytes;
308-
to_backup->wal_bytes = to_wal_bytes;
309314
write_backup(to_backup);
310315

311316
delete_source_backup:
@@ -330,27 +335,6 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup)
330335
}
331336
}
332337

333-
/*
334-
* Rename FULL backup directory.
335-
*/
336-
elog(INFO, "Rename %s to %s", to_backup_id, from_backup_id);
337-
if (rename(to_backup_path, from_backup_path) == -1)
338-
elog(ERROR, "Could not rename directory \"%s\" to \"%s\": %s",
339-
to_backup_path, from_backup_path, strerror(errno));
340-
341-
/*
342-
* Merging finished, now we can safely update ID of the destination backup.
343-
*/
344-
pgBackupCopy(to_backup, from_backup);
345-
/* Correct metadata */
346-
to_backup->backup_mode = BACKUP_MODE_FULL;
347-
to_backup->status = BACKUP_STATUS_OK;
348-
to_backup->parent_backup = INVALID_BACKUP_ID;
349-
/* Restore sizes */
350-
to_backup->data_bytes = to_data_bytes;
351-
to_backup->wal_bytes = to_wal_bytes;
352-
write_backup(to_backup);
353-
354338
/* Cleanup */
355339
if (threads)
356340
{
@@ -384,6 +368,8 @@ merge_files(void *arg)
384368
for (i = 0; i < num_files; i++)
385369
{
386370
pgFile *file = (pgFile *) parray_get(argument->files, i);
371+
pgFile *to_file;
372+
pgFile **res_file;
387373

388374
if (!pg_atomic_test_set_flag(&file->lock))
389375
continue;
@@ -392,17 +378,24 @@ merge_files(void *arg)
392378
if (interrupted)
393379
elog(ERROR, "Interrupted during merging backups");
394380

381+
/* Directories were created before */
382+
if (S_ISDIR(file->mode))
383+
continue;
384+
395385
if (progress)
396386
elog(INFO, "Progress: (%d/%d). Process file \"%s\"",
397387
i + 1, num_files, file->path);
398388

389+
res_file = parray_bsearch(argument->to_files, file,
390+
pgFileComparePathDesc);
391+
to_file = (res_file) ? *res_file : NULL;
392+
399393
/*
400394
* Skip files which haven't changed since previous backup. But in case
401395
* of DELTA backup we should consider n_blocks to truncate the target
402396
* backup.
403397
*/
404-
if (file->write_size == BYTES_INVALID &&
405-
file->n_blocks == -1)
398+
if (file->write_size == BYTES_INVALID && file->n_blocks == -1)
406399
{
407400
elog(VERBOSE, "Skip merging file \"%s\", the file didn't change",
408401
file->path);
@@ -411,27 +404,16 @@ merge_files(void *arg)
411404
* If the file wasn't changed in PAGE backup, retreive its
412405
* write_size from previous FULL backup.
413406
*/
414-
if (S_ISREG(file->mode))
407+
if (to_file)
415408
{
416-
pgFile **res_file;
417-
418-
res_file = parray_bsearch(argument->to_files, file,
419-
pgFileComparePathDesc);
420-
if (res_file && *res_file)
421-
{
422-
file->compress_alg = (*res_file)->compress_alg;
423-
file->write_size = (*res_file)->write_size;
424-
file->crc = (*res_file)->crc;
425-
}
409+
file->compress_alg = to_file->compress_alg;
410+
file->write_size = to_file->write_size;
411+
file->crc = to_file->crc;
426412
}
427413

428414
continue;
429415
}
430416

431-
/* Directories were created before */
432-
if (S_ISDIR(file->mode))
433-
continue;
434-
435417
/*
436418
* Move the file. We need to decompress it and compress again if
437419
* necessary.
@@ -447,7 +429,7 @@ merge_files(void *arg)
447429
file->path + to_root_len + 1);
448430

449431
/*
450-
* We need more complicate algorithm if target file exists and it is
432+
* We need more complicate algorithm if target file should be
451433
* compressed.
452434
*/
453435
if (to_backup->compress_alg == PGLZ_COMPRESS ||
@@ -462,33 +444,36 @@ merge_files(void *arg)
462444

463445
/*
464446
* Merge files:
465-
* - decompress first file
466-
* - decompress second file and merge with first decompressed file
447+
* - if target file exists restore and decompress it to the temp
448+
* path
449+
* - decompress source file if necessary and merge it with the
450+
* target decompressed file
467451
* - compress result file
468452
*/
469453

470-
elog(VERBOSE, "File is compressed, decompress to the temporary file \"%s\"",
471-
tmp_file_path);
472-
473-
prev_path = file->path;
474454
/*
475-
* We need to decompress target file only if it exists.
455+
* We need to decompress target file if it exists.
476456
*/
477-
if (fileExists(to_path_tmp))
457+
if (to_file)
478458
{
459+
elog(VERBOSE, "Merge target and source files into the temporary path \"%s\"",
460+
tmp_file_path);
461+
479462
/*
480463
* file->path points to the file in from_root directory. But we
481464
* need the file in directory to_root.
482465
*/
483-
file->path = to_path_tmp;
484-
485-
/* Decompress first/target file */
486-
restore_data_file(tmp_file_path, file, false, false,
466+
prev_path = to_file->path;
467+
to_file->path = to_path_tmp;
468+
/* Decompress target file into temporary one */
469+
restore_data_file(tmp_file_path, to_file, false, false,
487470
parse_program_version(to_backup->program_version));
488-
489-
file->path = prev_path;
471+
to_file->path = prev_path;
490472
}
491-
/* Merge second/source file with first/target file */
473+
else
474+
elog(VERBOSE, "Restore source file into the temporary path \"%s\"",
475+
tmp_file_path);
476+
/* Merge source file with target file */
492477
restore_data_file(tmp_file_path, file,
493478
from_backup->backup_mode == BACKUP_MODE_DIFF_DELTA,
494479
false,
@@ -497,7 +482,8 @@ merge_files(void *arg)
497482
elog(VERBOSE, "Compress file and save it to the directory \"%s\"",
498483
argument->to_root);
499484

500-
/* Again we need change path */
485+
/* Again we need to change path */
486+
prev_path = file->path;
501487
file->path = tmp_file_path;
502488
/* backup_data_file() requires file size to calculate nblocks */
503489
file->size = pgFileSize(file->path);

‎tests/merge.py

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -875,8 +875,6 @@ def test_continue_failed_merge_with_corrupted_delta_backup(self):
875875
repr(self.output), self.cmd))
876876
except ProbackupException as e:
877877
self.assertTrue(
878-
"WARNING: Backup {0} data files are corrupted".format(
879-
backup_id) in e.message and
880878
"ERROR: Merging of backup {0} failed".format(
881879
backup_id) in e.message,
882880
'\n Unexpected Error Message: {0}\n CMD: {1}'.format(

0 commit comments

Comments
 (0)
0