8000 tests: prettify pgdata_content and compare_pgdata · postgrespro/pg_probackup@1617eb3 · GitHub
[go: up one dir, main page]
Skip to content

Commit 1617eb3

Browse files
funny-falconfunny-falcon
committed
tests: prettify pgdata_content and compare_pgdata
1 parent 20667e9 commit 1617eb3

File tree

1 file changed

+116
-118
lines changed

1 file changed

+116
-118
lines changed

‎tests/helpers/ptrack_helpers.py

Lines changed: 116 additions & 118 deletions
Original file line numberDiff line numberDiff line change
@@ -1772,7 +1772,8 @@ def pgdata_content(self, pgdata, ignore_ptrack=True, exclude_dirs=None):
17721772

17731773
file_fullpath = os.path.join(root, file)
17741774
file_relpath = os.path.relpath(file_fullpath, pgdata)
1775-
directory_dict['files'][file_relpath] = {'is_datafile': False}
1775+
cfile = ContentFile(file.isdigit())
1776+
directory_dict['files'][file_relpath] = cfile
17761777
with open(file_fullpath, 'rb') as f:
17771778
# truncate cfm's content's zero tail
17781779
if file_relpath.endswith('.cfm'):
@@ -1792,14 +1793,12 @@ def pgdata_content(self, pgdata, ignore_ptrack=True, exclude_dirs=None):
17921793
b = f.read(64*1024)
17931794
if not b: break
17941795
digest.update(b)
1795-
directory_dict['files'][file_relpath]['md5'] = digest.hexdigest()
1796+
cfile.md5 = digest.hexdigest()
17961797

17971798
# crappy algorithm
1798-
if file.isdigit():
1799-
directory_dict['files'][file_relpath]['is_datafile'] = True
1799+
if cfile.is_datafile:
18001800
size_in_pages = os.path.getsize(file_fullpath)/8192
1801-
directory_dict['files'][file_relpath][
1802-
'md5_per_page'] = self.get_md5_per_page_for_fork(
1801+
cfile.md5_per_page = self.get_md5_per_page_for_fork(
18031802
file_fullpath, size_in_pages
18041803
)
18051804

@@ -1809,18 +1808,16 @@ def pgdata_content(self, pgdata, ignore_ptrack=True, exclude_dirs=None):
18091808
parent = os.path.dirname(directory_relpath)
18101809
if parent in directory_dict['dirs']:
18111810
del directory_dict['dirs'][parent]
1812-
directory_dict['dirs'][directory_relpath] = {}
1811+
directory_dict['dirs'][directory_relpath] = ContentDir()
18131812

18141813
# get permissions for every file and directory
1815-
for file in directory_dict['dirs']:
1814+
for file, cfile in directory_dict['dirs'].items():
18161815
full_path = os.path.join(pgdata, file)
1817-
directory_dict['dirs'][file]['mode'] = os.stat(
1818-
full_path).st_mode
1816+
cfile.mode = os.stat(full_path).st_mode
18191817

1820-
for file in directory_dict['files']:
1818+
for file, cdir in directory_dict['files'].items():
18211819
full_path = os.path.join(pgdata, file)
1822-
directory_dict['files'][file]['mode'] = os.stat(
1823-
full_path).st_mode
1820+
cdir.mode = os.stat(full_path).st_mode
18241821

18251822
return directory_dict
18261823

@@ -1852,123 +1849,117 @@ def compare_pgdata(self, original_pgdata, restored_pgdata, exclusion_dict = dict
18521849
error_message = 'Restored PGDATA is not equal to original!\n'
18531850

18541851
# Compare directories
1855-
for directory in restored_pgdata['dirs']:
1856-
if directory not in original_pgdata['dirs']:
1852+
restored_dirs = set(restored_pgdata['dirs'])
1853+
original_dirs = set(restored_pgdata['dirs'])
1854+
1855+
for directory in sorted(restored_dirs - original_dirs):
1856+
fail = True
1857+
error_message += '\nDirectory was not present'
1858+
error_message += ' in original PGDATA: {0}\n'.format(
1859+
os.path.join(restored_pgdata['pgdata'], directory))
1860+
1861+
for directory in sorted(original_dirs - restored_dirs):
1862+
fail = True
1863+
error_message += '\nDirectory dissappeared'
1864+
error_message += ' in restored PGDATA: {0}\n'.format(
1865+
os.path.join(restored_pgdata['pgdata'], directory))
1866+
1867+
for directory in sorted(original_dirs & restored_dirs):
1868+
original = original_pgdata['dirs'][directory]
1869+
restored = restored_pgdata['dirs'][directory]
1870+
if original.mode != restored.mode:
18571871
fail = True
1858-
error_message += '\nDirectory was not present'
1859-
error_message += ' in original PGDATA: {0}\n'.format(
1860-
os.path.join(restored_pgdata['pgdata'], directory))
1861-
else:
1862-
if (
1863-
restored_pgdata['dirs'][directory]['mode'] !=
1864-
original_pgdata['dirs'][directory]['mode']
1865-
):
1866-
fail = True
1867-
error_message += '\nDir permissions mismatch:\n'
1868-
error_message += ' Dir old: {0} Permissions: {1}\n'.format(
1869-
os.path.join(original_pgdata['pgdata'], directory),
1870-
original_pgdata['dirs'][directory]['mode'])
1871-
error_message += ' Dir new: {0} Permissions: {1}\n'.format(
1872-
os.path.join(restored_pgdata['pgdata'], directory),
1873-
restored_pgdata['dirs'][directory]['mode'])
1874-
1875-
for directory in original_pgdata['dirs']:
1876-
if directory not in restored_pgdata['dirs']:
1877-
fail = True
1878-
error_message += '\nDirectory dissappeared'
1879-
error_message += ' in restored PGDATA: {0}\n'.format(
1880-
os.path.join(restored_pgdata['pgdata'], directory))
1881-
1882-
for file in restored_pgdata['files']:
1872+
error_message += '\nDir permissions mismatch:\n'
1873+
error_message += ' Dir old: {0} Permissions: {1}\n'.format(
1874+
os.path.join(original_pgdata['pgdata'], directory),
1875+
original.mode)
1876+
error_message += ' Dir new: {0} Permissions: {1}\n'.format(
1877+
os.path.join(restored_pgdata['pgdata'], directory),
1878+
restored.mode)
1879+
1880+
restored_files = set(restored_pgdata['files'])
1881+
original_files = set(restored_pgdata['files'])
1882+
1883+
for file in sorted(restored_files - original_files):
18831884
# File is present in RESTORED PGDATA
18841885
# but not present in ORIGINAL
18851886
# only backup_label is allowed
1886-
if file not in original_pgdata['files']:
1887-
fail = True
1888-
error_message += '\nFile is not present'
1889-
error_message += ' in original PGDATA: {0}\n'.format(
1890-
os.path.join(restored_pgdata['pgdata'], file))
1891-
1892-
for file in original_pgdata['files']:
1893-
if file in restored_pgdata['files']:
1887+
fail = True
1888+
error_message += '\nFile is not present'
1889+
error_message += ' in original PGDATA: {0}\n'.format(
1890+
os.path.join(restored_pgdata['pgdata'], file))
1891+
1892+
for file in sorted(original_files - restored_files):
1893+
error_message += (
1894+
'\nFile disappearance.\n '
1895+
'File: {0}\n').format(
1896+
os.path.join(restored_pgdata['pgdata'], file)
1897+
)
1898+
fail = True
18941899

1895-
if (
1896-
restored_pgdata['files'][file]['mode'] !=
1897-
original_pgdata['files'][file]['mode']
1898-
):
1900+
for file in sorted(original_files & restored_files):
1901+
original = original_pgdata['files'][file]
1902+
restored = restored_pgdata['files'][file]
1903+
if restored.mode != original.mode:
1904+
fail = True
1905+
error_message += '\nFile permissions mismatch:\n'
1906+
error_message += ' File_old: {0} Permissions: {1:o}\n'.format(
1907+
os.path.join(original_pgdata['pgdata'], file),
1908+
original.mode)
1909+
error_message += ' File_new: {0} Permissions: {1:o}\n'.format(
1910+
os.path.join(restored_pgdata['pgdata'], file),
1911+
restored.mode)
1912+
1913+
if original.md5 != restored.md5:
1914+
if file not in exclusion_dict:
18991915
fail = True
1900-
error_message += '\nFile permissions mismatch:\n'
1901-
error_message += ' File_old: {0} Permissions: {1:o}\n'.format(
1916+
error_message += (
1917+
'\nFile Checksum mismatch.\n'
1918+
'File_old: {0}\nChecksum_old: {1}\n'
1919+
'File_new: {2}\nChecksum_new: {3}\n').format(
19021920
os.path.join(original_pgdata['pgdata'], file),
1903-
original_pgdata['files'][file]['mode'])
1904-
error_message += ' File_new: {0} Permissions: {1:o}\n'.format(
1921+
original.md5,
19051922
os.path.join(restored_pgdata['pgdata'], file),
1906-
restored_pgdata['files'][file]['mode'])
1923+
restored.md5
1924+
)
19071925

1908-
if (
1909-
original_pgdata['files'][file]['md5'] !=
1910-
restored_pgdata['files'][file]['md5']
1911-
):
1912-
if file not in exclusion_dict:
1913-
fail = True
1914-
error_message += (
1915-
'\nFile Checksum mismatch.\n'
1916-
'File_old: {0}\nChecksum_old: {1}\n'
1917-
'File_new: {2}\nChecksum_new: {3}\n').format(
1918-
F987 os.path.join(original_pgdata['pgdata'], file),
1919-
original_pgdata['files'][file]['md5'],
1920-
os.path.join(restored_pgdata['pgdata'], file),
1921-
restored_pgdata['files'][file]['md5']
1922-
)
1926+
if not original.is_datafile:
1927+
continue
19231928

1924-
if original_pgdata['files'][file]['is_datafile']:
1925-
for page in original_pgdata['files'][file]['md5_per_page']:
1926-
if page not in restored_pgdata['files'][file]['md5_per_page']:
1927-
error_message += (
1928-
'\n Page {0} dissappeared.\n '
1929-
'File: {1}\n').format(
1930-
page,
1931-
os.path.join(
1932-
restored_pgdata['pgdata'],
1933-
file
1934-
)
1935-
)
1936-
continue
1937-
1938-
if not (file in exclusion_dict and page in exclusion_dict[file]):
1939-
if (
1940-
original_pgdata['files'][file]['md5_per_page'][page] !=
1941-
restored_pgdata['files'][file]['md5_per_page'][page]
1942-
):
1943-
fail = True
1944-
error_message += (
1945-
'\n Page checksum mismatch: {0}\n '
1946-
' PAGE Checksum_old: {1}\n '
1947-
' PAGE Checksum_new: {2}\n '
1948-
' File: {3}\n'
1949-
).format(
1950-
page,
1951-
original_pgdata['files'][file][
1952-
'md5_per_page'][page],
1953-
restored_pgdata['files'][file][
1954-
'md5_per_page'][page],
1955-
os.path.join(
1956-
restored_pgdata['pgdata'], file)
1957-
)
1958-
for page in restored_pgdata['files'][file]['md5_per_page']:
1959-
if page not in original_pgdata['files'][file]['md5_per_page']:
1960-
error_message += '\n Extra page {0}\n File: {1}\n'.format(
1961-
page,
1962-
os.path.join(
1963-
restored_pgdata['pgdata'], file))
1929+
original_pages = set(original.md5_per_page)
1930+
restored_pages = set(restored.md5_per_page)
19641931

1965-
else:
1966-
error_message += (
1967-
'\nFile disappearance.\n '
1968-
'File: {0}\n').format(
1969-
os.path.join(restored_pgdata['pgdata'], file)
1932+
for page in sorted(original_pages - restored_pages):
1933+
error_message += '\n Page {0} dissappeared.\n File: {1}\n'.format(
1934+
page,
1935+
os.path.join(restored_pgdata['pgdata'], file)
19701936
)
1971-
fail = True
1937+
1938+
1939+
for page in sorted(restored_pages - original_pages):
1940+
error_message += '\n Extra page {0}\n File: {1}\n'.format(
1941+
page,
1942+
os.path.join(restored_pgdata['pgdata'], file))
1943+
1944+
for page in sorted(original_pages & restored_pages):
1945+
if file in exclusion_dict and page in exclusion_dict[file]:
1946+
continue
1947+
1948+
if original.md5_per_page[page] != restored.md5_per_page[page]:
1949+
fail = True
1950+
error_message += (
1951+
'\n Page checksum mismatch: {0}\n '
1952+
' PAGE Checksum_old: {1}\n '
1953+
' PAGE Checksum_new: {2}\n '
1954+
' File: {3}\n'
1955+
).format(
1956+
page,
1957+
original.md5_per_page[page],
1958+
restored.md5_per_page[page],
1959+
os.path.join(
1960+
restored_pgdata['pgdata'], file)
1961+
)
1962+
19721963
self.assertFalse(fail, error_message)
19731964

19741965
def gdb_attach(self, pid):
@@ -2221,3 +2212,10 @@ def _execute(self, cmd, running=True):
22212212
# if running and line.startswith('*running'):
22222213
break
22232214
return output
2215+
class ContentFile(object):
2216+
__slots__ = ('is_datafile', 'mode', 'md5', 'md5_per_page')
2217+
def __init__(self, is_datafile: bool):
2218+
self.is_datafile = is_datafile
2219+
2220+
class ContentDir(object):
2221+
__slots__ = ('mode')

0 commit comments

Comments
 (0)
0