diff --git a/src/bin/pg_combinebackup/meson.build b/src/bin/pg_combinebackup/meson.build index ba1c8cfa3d045..015f8bf9b5175 100644 --- a/src/bin/pg_combinebackup/meson.build +++ b/src/bin/pg_combinebackup/meson.build @@ -40,6 +40,7 @@ tests += { 't/010_hardlink.pl', 't/011_ib_truncation.pl', 't/012_vm_consistency.pl', + 't/013_stale_vm_detection.pl', ], } } diff --git a/src/bin/pg_combinebackup/t/002_compare_backups.pl b/src/bin/pg_combinebackup/t/002_compare_backups.pl index b509296a94a3f..4d4d1e31cb682 100644 --- a/src/bin/pg_combinebackup/t/002_compare_backups.pl +++ b/src/bin/pg_combinebackup/t/002_compare_backups.pl @@ -7,6 +7,11 @@ use PostgreSQL::Test::Utils; use Test::More; +use FindBin; +use lib $FindBin::RealBin; + +use CombineBackupTest; + my $tempdir = PostgreSQL::Test::Utils::tempdir_short(); # Can be changed to test the other modes. @@ -45,6 +50,9 @@ INSERT INTO will_change_in_ts VALUES (1, 'initial test row'); CREATE TABLE will_get_dropped_in_ts (a int, b text); INSERT INTO will_get_dropped_in_ts VALUES (1, 'initial test row'); +CREATE TABLE indexed_table (a int PRIMARY KEY, b text); +INSERT INTO indexed_table SELECT g, 'row ' || g FROM generate_series(1, 1000) g; +VACUUM (FREEZE) indexed_table; EOM # Read list of tablespace OIDs. There should be just one. @@ -83,6 +91,10 @@ VACUUM FULL will_get_rewritten; DROP DATABASE db_will_get_dropped; CREATE DATABASE db_newly_created; +UPDATE indexed_table SET b = 'updated row ' || a WHERE a % 100 = 0; +DELETE FROM indexed_table WHERE a % 137 = 0; +INSERT INTO indexed_table SELECT g, 'new row ' || g FROM generate_series(2001, 2100) g; +VACUUM (FREEZE) indexed_table; EOM # Take an incremental backup. @@ -131,6 +143,7 @@ recovery_target_lsn = '$lsn' recovery_target_action = 'promote' archive_mode = 'off' +autovacuum = 'off' }); $pitr1->start(); @@ -150,6 +163,7 @@ recovery_target_lsn = '$lsn' recovery_target_action = 'promote' archive_mode = 'off' +autovacuum = 'off' }); $pitr2->start(); @@ -202,4 +216,16 @@ return $_[0] ne $_[1]; }); +# A logical dump can only see live row contents, so the comparison above is +# blind to physical-level problems that pg_combinebackup could introduce: +# stale visibility map bits, index corruption, or other damage to data that +# a sequential scan does not consult. Run additional physical consistency +# checks against each restored cluster, so that if only one of them fails, +# the divergence is attributable. See CombineBackupTest.pm for the details; +# t/013_stale_vm_detection.pl is the negative control demonstrating that +# these checks really do fire on a corrupted cluster. +check_physical_consistency($pitr1, 'full backup PITR', 'indexed_table', 'a'); +check_physical_consistency($pitr2, 'incremental backup PITR', + 'indexed_table', 'a'); + done_testing(); diff --git a/src/bin/pg_combinebackup/t/012_vm_consistency.pl b/src/bin/pg_combinebackup/t/012_vm_consistency.pl index 6bf47ebec3739..c8ecea0593660 100644 --- a/src/bin/pg_combinebackup/t/012_vm_consistency.pl +++ b/src/bin/pg_combinebackup/t/012_vm_consistency.pl @@ -11,6 +11,11 @@ use PostgreSQL::Test::Utils; use Test::More; +use FindBin; +use lib $FindBin::RealBin; + +use CombineBackupTest; + my $tempdir = PostgreSQL::Test::Utils::tempdir_short(); my $mode = $ENV{PG_TEST_PG_COMBINEBACKUP_MODE} || '--copy'; @@ -166,6 +171,11 @@ sub validate_restored_vm "SELECT count(*) FROM pg_check_visible('$test->{table}')"); is($corrupt_tids, '0', "$test->{label} test: no VM corruption detected by pg_check_visible"); + + my $corrupt_frozen_tids = $restored->safe_psql('postgres', + "SELECT count(*) FROM pg_check_frozen('$test->{table}')"); + is($corrupt_frozen_tids, '0', + "$test->{label} test: no VM corruption detected by pg_check_frozen"); } # Create and populate the tables, then vacuum freeze them to set the VM bits @@ -177,6 +187,15 @@ sub validate_restored_vm check_vacuumed_vm($primary, $test); } +# Also create a multi-page table with an index, so that on the restored +# cluster we can compare a forced index-only scan (which trusts the +# visibility map) against a forced sequential scan (which does not). +$primary->safe_psql('postgres', q{ + CREATE TABLE vm_ios_test (id int PRIMARY KEY, val text); + INSERT INTO vm_ios_test SELECT g, 'row ' || g FROM generate_series(1, 1000) g; + VACUUM (FREEZE) vm_ios_test; +}); + # Take a full backup my $full_name = 'full'; my $full_path = $primary->backup_dir . "/$full_name"; @@ -221,6 +240,17 @@ sub validate_restored_vm } } +# Modify the index-only-scan test table so that its heap, index and +# visibility map all change within the incremental backup window, then +# re-vacuum so the VM bits are set again and an index-only scan on the +# restored cluster will actually consult them. +$primary->safe_psql('postgres', q{ + UPDATE vm_ios_test SET val = 'updated row ' || id WHERE id % 100 = 0; + DELETE FROM vm_ios_test WHERE id % 137 = 0; + INSERT INTO vm_ios_test SELECT g, 'new row ' || g FROM generate_series(2001, 2100) g; + VACUUM (FREEZE) vm_ios_test; +}); + # Take an incremental backup. This will have the changes made in the # modification step. my $incr_name = 'incr'; @@ -252,6 +282,13 @@ sub validate_restored_vm validate_restored_vm($restored, $test); } +# Run the full set of physical consistency checks against the restored +# cluster: pg_amcheck across all databases, a pg_visibility sweep over every +# user relation in every connectable database, and a forced index-only-scan +# vs seqscan comparison on vm_ios_test. See CombineBackupTest.pm. +check_physical_consistency($restored, 'restored cluster', 'vm_ios_test', + 'id'); + $restored->stop; $primary->stop; diff --git a/src/bin/pg_combinebackup/t/013_stale_vm_detection.pl b/src/bin/pg_combinebackup/t/013_stale_vm_detection.pl new file mode 100644 index 0000000000000..aa0b57e9e89f7 --- /dev/null +++ b/src/bin/pg_combinebackup/t/013_stale_vm_detection.pl @@ -0,0 +1,160 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group +# +# Negative control for the physical-consistency oracle used by +# 002_compare_backups.pl and 012_vm_consistency.pl (CombineBackupTest.pm): +# prove that the checks really do fire on a corrupted cluster. Restore a +# combined (full + incremental) backup into a scratch cluster, verify the +# oracle passes there, then fabricate stale all-visible/all-frozen bits by +# writing directly into the table's visibility map fork and assert that +# both the pg_visibility sweep and the index-only-scan-vs-seqscan +# comparison detect the damage. Finally, overwrite part of a btree page +# and assert that pg_amcheck detects that as well. +# +# Only the scratch restored cluster is ever corrupted; nothing here +# touches the clusters used for real comparisons in the other tests. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +use FindBin; +use lib $FindBin::RealBin; + +use CombineBackupTest; + +my $mode = $ENV{PG_TEST_PG_COMBINEBACKUP_MODE} || '--copy'; + +note "testing using mode $mode"; + +# Data checksums must be disabled: we fabricate stale VM bits by writing +# directly into the VM fork without recomputing page checksums, which is +# also how a hypothetical pg_combinebackup bug would manifest (a valid +# page image whose bits are simply wrong). +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 1, no_data_checksums => 1); +$primary->append_conf('postgresql.conf', <start; + +# A thousand rows, vacuum-frozen so the VM bits are set and an index-only +# scan will consult them. Table t is the corruption target; t_control is +# an identical table used for the intact-cluster sanity checks. The two +# must be distinct: an index(-only) scan over t before its VM is +# corrupted would heap-fetch the dead tuples and set LP_DEAD kill bits on +# their index entries, after which later index-only scans would skip them +# regardless of the (corrupted) visibility map. +$primary->safe_psql('postgres', q{ + CREATE TABLE t (id int PRIMARY KEY, val text); + INSERT INTO t SELECT g, 'row ' || g FROM generate_series(1, 1000) g; + VACUUM (FREEZE) t; + CREATE TABLE t_control (id int PRIMARY KEY, val text); + INSERT INTO t_control SELECT g, 'row ' || g FROM generate_series(1, 1000) g; + VACUUM (FREEZE) t_control; +}); + +# Take a full backup. +my $full_path = $primary->backup_dir . '/full'; +$primary->command_ok( + [ + 'pg_basebackup', '--no-sync', + '--pgdata' => $full_path, + '--checkpoint' => 'fast', + ], + 'full backup'); + +# Delete some rows, clearing the VM bits of every affected heap page and +# leaving dead tuples behind that are still present in the index. The +# "% 10" predicate is not indexable, so this cannot set index kill bits. +$primary->safe_psql('postgres', 'DELETE FROM t WHERE id % 10 = 0'); +$primary->safe_psql('postgres', 'DELETE FROM t_control WHERE id % 10 = 0'); + +# Take an incremental backup containing those changes. +my $incr_path = $primary->backup_dir . '/incr'; +$primary->command_ok( + [ + 'pg_basebackup', '--no-sync', + '--pgdata' => $incr_path, + '--checkpoint' => 'fast', + '--incremental' => $full_path . '/backup_manifest', + ], + 'incremental backup'); + +# Restore the combined backup into a scratch cluster. +my $restored = PostgreSQL::Test::Cluster->new('restored'); +$restored->init_from_backup( + $primary, 'incr', + combine_with_prior => ['full'], + combine_mode => $mode); +$restored->append_conf('postgresql.conf', 'autovacuum = off'); +$restored->start; + +# Sanity check: on the intact restored cluster, the oracle passes. Note +# that the index-only-scan check runs on t_control, not t, to avoid +# setting index kill bits on t (see above). +is(vm_sweep_errors($restored, 'postgres'), + '', 'no stale VM bits on intact restored cluster'); +my ($ios_plan, $ios_result, $seq_result) = + ios_vs_seqscan($restored, 'postgres', 't_control', 'id'); +like( + $ios_plan, + qr/Index Only Scan/, + 'aggregate query uses an index-only scan when forced'); +is($ios_result, $seq_result, + 'index-only scan and seqscan agree on intact restored cluster'); + +# Now fabricate stale VM bits: mark the first 16 heap pages all-visible +# and all-frozen by setting the first four bytes of the VM data area +# (which follows the standard 24-byte page header). The table's heap is +# smaller than 16 pages, so this covers every page, including the ones +# holding dead tuples. +my $relpath = + $restored->safe_psql('postgres', "SELECT pg_relation_filepath('t')"); +my $vmfile = $restored->data_dir . '/' . $relpath . '_vm'; +ok(-f $vmfile, 'visibility map fork exists'); + +$restored->stop; +open my $vmfh, '+<:raw', $vmfile or die "open $vmfile: $!"; +seek($vmfh, 24, 0) or die "seek $vmfile: $!"; +print $vmfh "\xff" x 4; +close $vmfh or die "close $vmfile: $!"; +$restored->start; + +# The pg_visibility sweep must now report corruption ... +my $vm_errors = vm_sweep_errors($restored, 'postgres'); +isnt($vm_errors, '', 'pg_visibility sweep detects the stale VM bits'); +note "vm sweep reported: $vm_errors"; + +# ... and the index-only scan must diverge from the seqscan, since it +# now trusts all-visible bits covering dead tuples. +($ios_plan, $ios_result, $seq_result) = + ios_vs_seqscan($restored, 'postgres', 't', 'id'); +isnt($ios_result, $seq_result, + 'index-only scan diverges from seqscan with stale VM bits'); +note "index-only scan returned [$ios_result], seqscan [$seq_result]"; + +# Also prove the pg_amcheck leg of the oracle: overwrite part of a btree +# page and assert pg_amcheck fails. (pg_amcheck by itself does not detect +# stale VM bits, which is why the pg_visibility sweep above is needed.) +my $idxpath = + $restored->safe_psql('postgres', "SELECT pg_relation_filepath('t_pkey')"); +my $idxfile = $restored->data_dir . '/' . $idxpath; + +$restored->stop; +open my $idxfh, '+<:raw', $idxfile or die "open $idxfile: $!"; +seek($idxfh, 8192 + 100, 0) or die "seek $idxfile: $!"; +print $idxfh "\xde\xad\xbe\xef" x 25; +close $idxfh or die "close $idxfile: $!"; +$restored->start; + +$restored->command_fails( + [ 'pg_amcheck', '--all', '--install-missing', '--heapallindexed' ], + 'pg_amcheck detects the corrupted btree page'); + +$restored->stop; +$primary->stop; + +done_testing(); diff --git a/src/bin/pg_combinebackup/t/CombineBackupTest.pm b/src/bin/pg_combinebackup/t/CombineBackupTest.pm new file mode 100644 index 0000000000000..d841d249e5b8e --- /dev/null +++ b/src/bin/pg_combinebackup/t/CombineBackupTest.pm @@ -0,0 +1,155 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +=pod + +=head1 NAME + +CombineBackupTest - helper routines for pg_combinebackup TAP tests + +=head1 SYNOPSIS + + use FindBin; + use lib $FindBin::RealBin; + use CombineBackupTest; + + check_physical_consistency($node, 'some label', 'indexed_table', 'a'); + +=head1 DESCRIPTION + +Physical-consistency checks ("oracle") run against restored clusters. +A logical dump can only see live row contents through sequential scans, +so comparing pg_dumpall output is blind to physical-level problems that +pg_combinebackup could introduce: stale visibility map bits, index +corruption, or other damage to data that a sequential scan does not +consult. The routines here check those directly. + +The individual building blocks (vm_sweep_errors, ios_vs_seqscan) are +also exported so that a negative-control test can assert that the +checks really do fire on corrupted clusters. + +=cut + +package CombineBackupTest; + +use strict; +use warnings FATAL => 'all'; + +use Exporter 'import'; +use Test::More; + +our @EXPORT = qw( + connectable_databases + vm_sweep_errors + ios_vs_seqscan + check_physical_consistency +); + +# Return the names of all connectable databases in the cluster. +sub connectable_databases +{ + my ($node) = @_; + my $datnames = $node->safe_psql('postgres', + "SELECT datname FROM pg_database WHERE datallowconn ORDER BY datname"); + return split(/\n/, $datnames); +} + +# Ask pg_visibility to cross-check the visibility map against the heap for +# every user relation (tables, materialized views, toast tables) in the +# given database. Returns an empty string if no problems were found, else +# one line per corrupt relation listing the number of TIDs flagged by +# pg_check_visible() and pg_check_frozen(). +sub vm_sweep_errors +{ + my ($node, $dbname) = @_; + + $node->safe_psql($dbname, 'CREATE EXTENSION IF NOT EXISTS pg_visibility'); + return $node->safe_psql($dbname, q{ + SELECT relation, bad_visible, bad_frozen + FROM (SELECT c.oid::regclass AS relation, + (SELECT count(*) FROM pg_check_visible(c.oid)) AS bad_visible, + (SELECT count(*) FROM pg_check_frozen(c.oid)) AS bad_frozen + FROM pg_class c + WHERE c.relkind IN ('r', 'm', 't') + AND c.oid >= 16384) s + WHERE bad_visible > 0 OR bad_frozen > 0 + }); +} + +# Run the same aggregate over the given table with a forced index-only scan +# (which trusts the visibility map) and a forced sequential scan (which does +# not). Returns the forced index-only-scan plan and both results, so the +# caller can assert that the plan really is an index-only scan and that the +# two results agree (or, in a negative test, disagree). +sub ios_vs_seqscan +{ + my ($node, $dbname, $table, $column) = @_; + + my $query = + "SELECT count(*), sum($column), min($column), max($column) FROM $table"; + my $ios_plan = $node->safe_psql( + $dbname, qq{ + SET enable_seqscan = off; + SET enable_bitmapscan = off; + SET enable_indexonlyscan = on; + EXPLAIN (COSTS OFF) $query; + }); + my $ios_result = $node->safe_psql( + $dbname, qq{ + SET enable_seqscan = off; + SET enable_bitmapscan = off; + SET enable_indexonlyscan = on; + $query; + }); + my $seq_result = $node->safe_psql( + $dbname, qq{ + SET enable_indexscan = off; + SET enable_indexonlyscan = off; + SET enable_bitmapscan = off; + $query; + }); + return ($ios_plan, $ios_result, $seq_result); +} + +# Run the full set of physical consistency checks against a restored +# cluster: +# +# 1. pg_amcheck over every database, verifying heap and index consistency. +# --heapallindexed also verifies that every heap tuple is found in the +# expected indexes. +# +# 2. A pg_visibility sweep over every user relation in every connectable +# database, asserting that no stale all-visible or all-frozen bits +# exist. Stale VM bits are invisible to a logical dump but corrupt +# index-only scan results and can cause future heap corruption once +# vacuum trusts them. +# +# 3. A forced index-only scan vs forced sequential scan comparison over +# $ios_table.$ios_column (in database "postgres"), which is the +# user-visible symptom of stale all-visible bits. +sub check_physical_consistency +{ + my ($node, $label, $ios_table, $ios_column) = @_; + local $Test::Builder::Level = $Test::Builder::Level + 1; + + $node->command_ok( + [ 'pg_amcheck', '--all', '--install-missing', '--heapallindexed' ], + "$label: pg_amcheck reports no corruption"); + + foreach my $dbname (connectable_databases($node)) + { + is(vm_sweep_errors($node, $dbname), + '', "$label: no stale VM bits in database $dbname"); + } + + my ($ios_plan, $ios_result, $seq_result) = + ios_vs_seqscan($node, 'postgres', $ios_table, $ios_column); + like( + $ios_plan, + qr/Index Only Scan/, + "$label: aggregate query uses an index-only scan when forced"); + is($ios_result, $seq_result, + "$label: index-only scan and seqscan agree"); + return; +} + +1;