10000 Add filtering capability for cross-version pg_upgrade tests · postgrespro/postgres@62be9e4 · GitHub
[go: up one dir, main page]
Skip to content

Commit 62be9e4

Browse files
michaelpqmichaelpq
committed
Add filtering capability for cross-version pg_upgrade tests
This commit expands the TAP tests of pg_upgrade when running these with different major versions for the "old" cluster (to-be-upgraded) and the "new" cluster (upgraded-to), by backporting some of the buildfarm facilities directory into the script: - Remove comments from the dump files, avoiding version-dependent information. - Remove empty lines from the dump files. - Use --extra-float-digits=0 in the pg_dump command, when using an "old" cluster with version equal to or lower than v11. - Use --wal-segsize and --allow-group-access in initdb only when the "old" cluster is equal to or higher than v11. This allows the tests to pass down to v14 with the main regression test suite, while v9.5~13 still generate some diffs, but these are minimal compared to what happened before this commit. Much more could be done, especially around dump differences with function and procedures (these can also be avoided with direct manipulation of the dumps loaded, for example, in a way similar to the buildfarm), but at least the basics are in place now. Reviewed-by: Justin Pryzby, Anton A. Melnikov Discussion: https://postgr.es/m/Yox1ME99GhAemMq1@paquier.xyz
1 parent 908e171 commit 62be9e4

File tree

1 file changed

+70
-33
lines changed

1 file changed

+70
-33
lines changed

‎src/bin/pg_upgrade/t/002_pg_upgrade.pl

Lines changed: 70 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,27 @@ sub generate_db
3030
"created database with ASCII characters from $from_char to $to_char");
3131
}
3232

33+
# Filter the contents of a dump before its use in a content comparison.
34+
# This returns the path to the filtered dump.
35+
sub filter_dump
36+
{
37+
my ($node, $dump_file) = @_;
38+
my $dump_contents = slurp_file($dump_file);
39+
40+
# Remove the comments.
41+
$dump_contents =~ s/^\-\-.*//mgx;
42+
# Remove empty lines.
43+
$dump_contents =~ s/^\n//mgx;
44+
45+
my $dump_file_filtered = "${dump_file}_filtered";
46+
open(my $dh, '>', $dump_file_filtered)
47+
|| die "opening $dump_file_filtered";
48+
print $dh $dump_contents;
49+
close($dh);
50+
51+
return $dump_file_filtered;
52+
}
53+
3354
# The test of pg_upgrade requires two clusters, an old one and a new one
3455
# that gets upgraded. Before running the upgrade, a logical dump of the
3556
# old cluster is taken, and a second logical dump of the new one is taken
@@ -49,8 +70,10 @@ sub generate_db
4970
die "olddump or oldinstall is undefined";
5071
}
5172

52-
# Temporary location for the dumps taken
53-
my $tempdir = PostgreSQL::Test::Utils::tempdir;
73+
# Paths to the dumps taken during the tests.
74+
my $tempdir = PostgreSQL::Test::Utils::tempdir;
75+
my $dump1_file = "$tempdir/dump1.sql";
76+
my $dump2_file = "$tempdir/dump2.sql";
5477

5578
# Initialize node to upgrade
5679
my $oldnode =
@@ -60,7 +83,10 @@ sub generate_db
6083
# To increase coverage of non-standard segment size and group access without
6184
# increasing test runtime, run these tests with a custom setting.
6285
# --allow-group-access and --wal-segsize have been added in v11.
63-
$oldnode->init(extra => [ '--wal-segsize', '1', '--allow-group-access' ]);
86+
my %node_params = ();
87+
$node_params{extra} = [ '--wal-segsize', '1', '--allow-group-access' ]
88+
if $oldnode->pg_version >= 11;
89+
$oldnode->init(%node_params);
6490
$oldnode->start;
6591

6692
# The default location of the source code is the root of this directory.
@@ -129,37 +155,38 @@ sub generate_db
129155
is($rc, 0, 'regression tests pass');
130156
}
131157

158+
# Initialize a new node for the upgrade.
159+
my $newnode = PostgreSQL::Test::Cluster->new('new_node');
160+
$newnode->init(%node_params);
161+
162+
my $newbindir = $newnode->config_data('--bindir');
163+
my $oldbindir = $oldnode->config_data('--bindir');
164+
132165
# Before dumping, get rid of objects not existing or not supported in later
133166
# versions. This depends on the version of the old server used, and matters
134167
# only if different major versions are used for the dump.
135168
if (defined($ENV{oldinstall}))
136169
{
137-
# Note that upgrade_adapt.sql from the new version is used, to
138-
# cope with an upgrade to this version.
139-
$oldnode->command_ok(
170+
# Note that upgrade_adapt.sql and psql from the new version are used,
171+
# to cope with an upgrade to this version.
172+
$newnode->command_ok(
140173
[
141174
'psql', '-X',
142-
'-f', "$srcdir/src/bin/pg_upgrade/upgrade_adapt.sql",
143-
'regression'
175+
'-f', "$srcdir/src/bin/pg_upgrade/upgrade_adapt.sql",
176+
'-d', $oldnode->connstr('regression'),
144177
],
145178
'ran adapt script');
146179
}
147180

148-
# Initialize a new node for the upgrade.
149-
my $newnode = PostgreSQL::Test::Cluster->new('new_node');
150-
$newnode->init(extra => [ '--wal-segsize', '1', '--allow-group-access' ]);
151-
my $newbindir = $newnode->config_data('--bindir');
152-
my $oldbindir = $oldnode->config_data('--bindir');
153-
154181
# Take a dump before performing the upgrade as a base comparison. Note
155182
# that we need to use pg_dumpall from the new node here.
156-
$newnode->command_ok(
157-
[
158-
'pg_dumpall', '--no-sync',
159-
'-d', $oldnode->connstr('postgres'),
160-
'-f', "$tempdir/dump1.sql"
161-
],
162-
'dump before running pg_upgrade');
183+
my @dump_command = (
184+
'pg_dumpall', '--no-sync', '-d', $oldnode->connstr('postgres'),
185+
'-f', $dump1_file);
186+
# --extra-float-digits is needed when upgrading from a version older than 11.
187+
push(@dump_command, '--extra-float-digits', '0')
188+
if ($oldnode->pg_version < 12);
189+
$newnode->command_ok(\@dump_command, 'dump before running pg_upgrade');
163190

164191
# After dumping, update references to the old source tree's regress.so
165192
# to point to the new tree.
@@ -173,7 +200,7 @@ sub generate_db
173200
chomp($output);
174201
my @libpaths = split("\n", $output);
175202

176-
my $dump_data = slurp_file("$tempdir/dump1.sql");
203+
my $dump_data = slurp_file($dump1_file);
177204

178205
my $newregresssrc = "$srcdir/src/test/regress";
179206
foreach (@libpaths)
@@ -183,7 +210,7 @@ sub generate_db
183210
$dump_data =~ s/$libpath/$newregresssrc/g;
184211
}
185212

186-
open my $fh, ">", "$tempdir/dump1.sql" or die "could not open dump file";
213+
open my $fh, ">", $dump1_file or die "could not open dump file";
187214
print $fh $dump_data;
188215
close $fh;
189216

@@ -284,24 +311,34 @@ sub generate_db
284311
}
285312

286313
# Second dump from the upgraded instance.
287-
$newnode->command_ok(
288-
[
289-
'pg_dumpall', '--no-sync',
290-
'-d', $newnode->connstr('postgres'),
291-
'-f', "$tempdir/dump2.sql"
292-
],
293-
'dump after running pg_upgrade');
314+
@dump_command = (
315+
'pg_dumpall', '--no-sync', '-d', $newnode->connstr('postgres'),
316+
'-f', $dump2_file);
317+
# --extra-float-digits is needed when upgrading from a version older than 11.
318+
push(@dump_command, '--extra-float-digits', '0')
319+
if ($oldnode->pg_version < 12);
320+
$newnode->command_ok(\@dump_command, 'dump after running pg_upgrade');
321+
322+
# No need to apply filters on the dumps if working on the same version
323+
# for the old and new nodes.
324+
my $dump1_filtered = $dump1_file;
325+
my $dump2_filtered = $dump2_file;
326+
if ($oldnode->pg_version != $newnode->pg_version)
327+
{
328+
$dump1_filtered = filter_dump($oldnode, $dump1_file);
329+
$dump2_filtered = filter_dump($newnode, $dump2_file);
330+
}
294331

295332
# Compare the two dumps, there should be no differences.
296-
my $compare_res = compare("$tempdir/dump1.sql", "$tempdir/dump2.sql");
333+
my $compare_res = compare($dump1_filtered, $dump2_filtered);
297334
is($compare_res, 0, 'old and new dumps match after pg_upgrade');
298335

299336
# Provide more context if the dumps do not match.
300337
if ($compare_res != 0)
301338
{
302339
my ($stdout, $stderr) =
303-
run_command([ 'diff', "$tempdir/dump1.sql", "$tempdir/dump2.sql" ]);
304-
print "=== diff of $tempdir/dump1.sql and $tempdir/dump2.sql\n";
340+
run_command([ 'diff', $dump1_filtered, $dump2_filtered ]);
341+
print "=== diff of $dump1_filtered and $dump2_filtered\n";
305342
print "=== stdout ===\n";
306343
print $stdout;
307344
print "=== stderr ===\n";

0 commit comments

Comments
 (0)
0