8000 tests: table_checksum needs no sorting in fact · postgrespro/pg_probackup@02e3fb0 · GitHub
[go: up one dir, main page]
Skip to content

Commit 02e3fb0

Browse files
funny-falconfunny-falcon
committed
tests: table_checksum needs no sorting in fact
since we are compare table content exactly
1 parent 8d8a92c commit 02e3fb0

File tree

6 files changed

+33
-39
lines changed

6 files changed

+33
-39
lines changed

‎tests/archive_test.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2215,7 +2215,7 @@ def test_multi_timeline_recovery_prefetching(self):
22152215
node.slow_start()
22162216

22172217
node.pgbench_init(scale=20)
2218-
result = node.table_checksum("pgbench_accounts", "aid")
2218+
result = node.ta 10000 ble_checksum("pgbench_accounts")
22192219
node.stop()
22202220
node.cleanup()
22212221

@@ -2240,7 +2240,7 @@ def test_multi_timeline_recovery_prefetching(self):
22402240

22412241
node.slow_start()
22422242

2243-
result_new = node.table_checksum("pgbench_accounts", "aid")
2243+
result_new = node.table_checksum("pgbench_accounts")
22442244

22452245
self.assertEqual(result, result_new)
22462246

‎tests/cfs_backup_test.py

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -761,7 +761,7 @@ def test_multiple_segments(self):
761761
't_heap', tblspace_name)
762762
)
763763

764-
full_result = self.node.table_checksum("t_heap", "id")
764+
full_result = self.node.table_checksum("t_heap")
765765

766766
try:
767767
backup_id_full = self.backup_node(
@@ -783,7 +783,7 @@ def test_multiple_segments(self):
783783
't_heap')
784784
)
785785

786-
page_result = self.node.table_checksum("t_heap", "id")
786+
page_result = self.node.table_checksum("t_heap")
787787

788788
try:
789789
backup_id_page = self.backup_node(
@@ -824,7 +824,7 @@ def test_multiple_segments(self):
824824
self.node.slow_start()
825825
self.assertEqual(
826826
full_result,
827-
self.node.table_checksum("t_heap", "id"),
827+
self.node.table_checksum("t_heap"),
828828
'Lost data after restore')
829829

830830
# CHECK PAGE BACKUP
@@ -843,7 +843,7 @@ def test_multiple_segments(self):
843843
self.node.slow_start()
844844
self.assertEqual(
845845
page_result,
846-
self.node.table_checksum("t_heap", "id"),
846+
self.node.table_checksum("t_heap"),
847847
'Lost data after restore')
848848

849849
# @unittest.expectedFailure
@@ -877,8 +877,8 @@ def test_multiple_segments_in_multiple_tablespaces(self):
877877
"FROM generate_series(0,1005000) i".format(
878878
't_heap_2', tblspace_name_2))
879879

880-
full_result_1 = self.node.table_checksum("t_heap_1", "id")
881-
full_result_2 = self.node.table_checksum("t_heap_2", "id")
880+
full_result_1 = self.node.table_checksum("t_heap_1")
881+
full_result_2 = self.node.table_checksum("t_heap_2")
882882

883883
try:
884884
backup_id_full = self.backup_node(
@@ -909,8 +909,8 @@ def test_multiple_segments_in_multiple_tablespaces(self):
909909
't_heap_2')
910910
)
911911

912-
page_result_1 = self.node.table_checksum("t_heap_1", "id")
913-
page_result_2 = self.node.table_checksum("t_heap_2", "id")
912+
page_result_1 = self.node.table_checksum("t_heap_1")
913+
page_result_2 = self.node.table_checksum("t_heap_2")
914914

915915
try:
916916
backup_id_page = self.backup_node(
@@ -951,11 +951,11 @@ def test_multiple_segments_in_multiple_tablespaces(self):
951951

952952
self.assertEqual(
953953
full_result_1,
954-
self.node.table_checksum("t_heap_1", "id"),
954+
self.node.table_checksum("t_heap_1"),
955955
'Lost data after restore')
956956
self.assertEqual(
957957
full_result_2,
958-
self.node.table_checksum("t_heap_2", "id"),
958+
self.node.table_checksum("t_heap_2"),
959959
'Lost data after restore')
960960

961961
# CHECK PAGE BACKUP
@@ -972,11 +972,11 @@ def test_multiple_segments_in_multiple_tablespaces(self):
972972

973973
self.assertEqual(
974974
page_result_1,
975-
self.node.table_checksum("t_heap_1", "id"),
975+
self.node.table_checksum("t_heap_1"),
976976
'Lost data after restore')
977977
self.assertEqual(
978978
page_result_2,
979-
self.node.table_checksum("t_heap_2", "id"),
979+
self.node.table_checksum("t_heap_2"),
980980
'Lost data after restore')
981981

982982
# @unittest.expectedFailure

‎tests/helpers/ptrack_helpers.py

Lines changed: 10 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515
from time import sleep
1616
import re
1717
import json
18-
from hashlib import md5
1918
import random
2019

2120
idx_ptrack = {
@@ -202,30 +201,28 @@ def kill(self, someone = None):
202201
os.kill(self.auxiliary_pids[someone][0], sig)
203202
self.is_started = False
204203

205-
def table_checksum(self, table, sort, dbname="postgres"):
206-
curname = "cur_"+str(random.randint(0,2**48))
207-
208-
sum = md5(b"\x01")
209-
204+
def table_checksum(self, table, dbname="postgres"):
210205
con = self.connect(dbname=dbname)
211206

212-
con.execute(f"""
213-
DECLARE {curname} NO SCROLL CURSOR FOR
214-
SELECT t::text FROM {table} as t ORDER BY {sort};
215-
""")
207+
curname = "cur_"+str(random.randint(0,2**48))
208+
209+
con.execute("""
210+
DECLARE %s NO SCROLL CURSOR FOR
211+
SELECT t::text FROM %s as t
212+
""" % (curname, table))
216213

214+
sum = hashlib.md5()
217215
while True:
218-
rows = con.execute(f"FETCH FORWARD 5000 FROM {curname}")
216+
rows = con.execute("FETCH FORWARD 5000 FROM %s" % curname)
219217
if not rows:
220218
break
221219
for row in rows:
220+
# hash uses SipHash since Python3.4, therefore it is good enough
222221
sum.update(row[0].encode('utf8'))
223-
sum.update(b'\x00')
224222

225223
con.execute(f"CLOSE {curname}; ROLLBACK;")
226224

227225
con.close()
228-
sum.update(b'\x02')
229226
return sum.hexdigest()
230227

231228
class ProbackupTest(object):

‎tests/page_test.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1191,7 +1191,7 @@ def test_multi_timeline_page(self):
11911191

11921192
pgdata = self.pgdata_content(node.data_dir)
11931193

1194-
result = node.table_checksum("pgbench_accounts", "aid")
1194+
result = node.table_checksum("pgbench_accounts")
11951195

11961196
node_restored = self.make_simple_node(
11971197
base_dir=os.path.join(self.module_name, self.fname, 'node_restored'))
@@ -1203,7 +1203,7 @@ def test_multi_timeline_page(self):
12031203
self.set_auto_conf(node_restored, {'port': node_restored.port})
12041204
node_restored.slow_start()
12051205

1206-
result_new = node_restored.table_checksum("pgbench_accounts", "aid")
1206+
result_new = node_restored.table_checksum("pgbench_accounts")
12071207

12081208
self.assertEqual(result, result_new)
12091209

‎tests/ptrack_test.py

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -375,7 +375,7 @@ def test_ptrack_eat_my_data(self):
375375

376376
self.switch_wal_segment(node)
377377

378-
result = node.table_checksum("pgbench_accounts", "aid")
378+
result = node.table_checksum("pgbench_accounts")
379379

380380
node_restored.cleanup()
381381
self.restore_node(backup_dir, 'node', node_restored)
@@ -396,7 +396,7 @@ def test_ptrack_eat_my_data(self):
396396
# Logical comparison
397397
self.assertEqual(
398398
result,
399-
node.table_checksum("pgbench_accounts", "aid"),
399+
node.table_checksum("pgbench_accounts"),
400400
'Data loss')
401401

402402
# @unittest.skip("skip")
@@ -2037,8 +2037,7 @@ def test_ptrack_multiple_segments(self):
20372037

20382038
# CREATE TABLE
20392039
node.pgbench_init(scale=100, options=['--tablespace=somedata'])
2040-
result = node.table_checksum("pgbench_accounts", "aid",
2041-
dbname="postgres")
2040+
result = node.table_checksum("pgbench_accounts")
20422041
# FULL BACKUP
20432042
self.backup_node(backup_dir, 'node', node, options=['--stream'])
20442043

@@ -2075,8 +2074,7 @@ def test_ptrack_multiple_segments(self):
20752074

20762075
# GET LOGICAL CONTENT FROM NODE
20772076
# it`s stupid, because hint`s are ignored by ptrack
2078-
result = node.table_checksum("pgbench_accounts", "aid",
2079-
dbname="postgres")
2077+
result = node.table_checksum("pgbench_accounts")
20802078
# FIRTS PTRACK BACKUP
20812079
self.backup_node(
20822080
backup_dir, 'node', node, backup_type='ptrack', options=['--stream'])
21092107
restored_node, {'port': restored_node.port})
21102108
restored_node.slow_start()
21112109

2112-
result_new = restored_node.table_checksum("pgbench_accounts", "aid",
2113-
dbname="postgres")
2110+
result_new = restored_node.table_checksum("pgbench_accounts")
21142111

21152112
# COMPARE RESTORED FILES
21162113
self.assertEqual(result, result_new, 'data is lost')

‎tests/replica_test.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -326,7 +326,7 @@ def test_replica_archive_page_backup(self):
326326

327327
self.switch_wal_segment(master)
328328

329-
before = master.table_checksum("pgbench_accounts", "aid")
329+
before = master.table_checksum("pgbench_accounts")
330330

331331
self.validate_pb(backup_dir, 'replica')
332332
self.assertEqual(
@@ -342,7 +342,7 @@ def test_replica_archive_page_backup(self):
342342
node.slow_start()
343343

344344
# CHECK DATA CORRECTNESS
345-
after = master.table_checksum("pgbench_accounts", "aid")
345+
after = master.table_checksum("pgbench_accounts")
346346
self.assertEqual(
347347
before, after, 'Restored data is not equal to original')
348348

0 commit comments

Comments
 (0)
0