From 1d6c2d9a72a9b6a064353daf0ca958e3c786175e Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 18 Jul 2026 02:11:55 +0000 Subject: [PATCH 1/2] Strengthen restored-cluster oracle in pg_combinebackup tests The incremental-backup comparison test 002_compare_backups.pl verified restored clusters only by comparing pg_dumpall output. A logical dump sees just live row contents through sequential scans, so it is blind to exactly the kind of physical damage pg_combinebackup could plausibly introduce: stale visibility map bits, index corruption, and other page-level problems. That oracle gap is why the ed62d26 class of VM WAL-logging bugs could survive this test. 012_vm_consistency.pl checks VM state for specific operations, but its restored-cluster verification had similar gaps. Upgrade the oracle in both tests. After each restored cluster is started, additionally: * Run pg_amcheck --all --install-missing --heapallindexed to verify heap and index consistency in every database. * Cross-check the visibility map against the heap with pg_visibility's pg_check_visible()/pg_check_frozen() over every user relation (including toast tables), asserting zero corrupt TIDs. * Run the same aggregate under a forced index-only scan (which trusts the VM) and a forced sequential scan (which does not) and require identical results -- the user-visible symptom of stale all-visible bits. Both tests gain a dedicated indexed table whose heap, index, and VM change within the incremental backup window and which is re-VACUUMed so the restored VM bits are actually consulted. In 002_compare_backups.pl the checks run against both the full-backup restore and the incremental (combined) restore, so any divergence is attributable to pg_combinebackup. Autovacuum is disabled on the restored nodes to keep the checks deterministic. A throwaway sanity experiment (not committed) confirmed the oracle bites: with manually-set stale VM bits, pg_check_visible and pg_check_frozen each reported 100 corrupt TIDs and the forced index-only scan returned rows the seqscan did not, and pg_amcheck detected a manually corrupted btree page. pg_amcheck alone does not detect stale VM bits, which is why the pg_visibility checks are also needed. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01PtxA6B4d47ietk5dGsseuP --- .../pg_combinebackup/t/002_compare_backups.pl | 81 +++++++++++++++++++ .../pg_combinebackup/t/012_vm_consistency.pl | 81 +++++++++++++++++++ 2 files changed, 162 insertions(+) diff --git a/src/bin/pg_combinebackup/t/002_compare_backups.pl b/src/bin/pg_combinebackup/t/002_compare_backups.pl index b509296a94a3f..d49cb0ad7c48f 100644 --- a/src/bin/pg_combinebackup/t/002_compare_backups.pl +++ b/src/bin/pg_combinebackup/t/002_compare_backups.pl @@ -45,6 +45,9 @@ INSERT INTO will_change_in_ts VALUES (1, 'initial test row'); CREATE TABLE will_get_dropped_in_ts (a int, b text); INSERT INTO will_get_dropped_in_ts VALUES (1, 'initial test row'); +CREATE TABLE indexed_table (a int PRIMARY KEY, b text); +INSERT INTO indexed_table SELECT g, 'row ' || g FROM generate_series(1, 1000) g; +VACUUM (FREEZE) indexed_table; EOM # Read list of tablespace OIDs. There should be just one. @@ -83,6 +86,10 @@ VACUUM FULL will_get_rewritten; DROP DATABASE db_will_get_dropped; CREATE DATABASE db_newly_created; +UPDATE indexed_table SET b = 'updated row ' || a WHERE a % 100 = 0; +DELETE FROM indexed_table WHERE a % 137 = 0; +INSERT INTO indexed_table SELECT g, 'new row ' || g FROM generate_series(2001, 2100) g; +VACUUM (FREEZE) indexed_table; EOM # Take an incremental backup. @@ -131,6 +138,7 @@ recovery_target_lsn = '$lsn' recovery_target_action = 'promote' archive_mode = 'off' +autovacuum = 'off' }); $pitr1->start(); @@ -150,6 +158,7 @@ recovery_target_lsn = '$lsn' recovery_target_action = 'promote' archive_mode = 'off' +autovacuum = 'off' }); $pitr2->start(); @@ -202,4 +211,76 @@ return $_[0] ne $_[1]; }); +# A logical dump can only see live row contents, so the comparison above is +# blind to physical-level problems that pg_combinebackup could introduce: +# stale visibility map bits, index corruption, or other damage to data that +# a sequential scan does not consult. Run additional physical consistency +# checks against each restored cluster, so that if only one of them fails, +# the divergence is attributable. +sub check_physical_consistency +{ + my ($node, $label) = @_; + + # Check heap and index consistency in every database. --heapallindexed + # also verifies that every heap tuple is found in the expected indexes. + $node->command_ok( + [ 'pg_amcheck', '--all', '--install-missing', '--heapallindexed' ], + "$label: pg_amcheck reports no corruption"); + + # Ask pg_visibility to cross-check the visibility map against the heap + # for every user relation. Stale all-visible or all-frozen bits are + # invisible to a logical dump but corrupt index-only scan results and + # can cause future heap corruption once vacuum trusts them. + $node->safe_psql('postgres', + 'CREATE EXTENSION IF NOT EXISTS pg_visibility'); + my $vm_errors = $node->safe_psql('postgres', q{ + SELECT relation, bad_visible, bad_frozen + FROM (SELECT c.oid::regclass AS relation, + (SELECT count(*) FROM pg_check_visible(c.oid)) AS bad_visible, + (SELECT count(*) FROM pg_check_frozen(c.oid)) AS bad_frozen + FROM pg_class c + WHERE c.relkind IN ('r', 'm', 't') + AND c.oid >= 16384) s + WHERE bad_visible > 0 OR bad_frozen > 0 + }); + is($vm_errors, '', + "$label: pg_check_visible/pg_check_frozen report no problems"); + + # Run the same aggregate with a forced index-only scan (which trusts the + # visibility map) and a forced sequential scan (which does not), and + # require identical answers. This is the user-visible symptom of stale + # all-visible bits. + my $ios_query = 'SELECT count(*), sum(a), min(a), max(a) FROM indexed_table'; + my $ios_plan = $node->safe_psql( + 'postgres', qq{ + SET enable_seqscan = off; + SET enable_bitmapscan = off; + SET enable_indexonlyscan = on; + EXPLAIN (COSTS OFF) $ios_query; + }); + like( + $ios_plan, + qr/Index Only Scan/, + "$label: aggregate query uses an index-only scan when forced"); + my $ios_result = $node->safe_psql( + 'postgres', qq{ + SET enable_seqscan = off; + SET enable_bitmapscan = off; + SET enable_indexonlyscan = on; + $ios_query; + }); + my $seq_result = $node->safe_psql( + 'postgres', qq{ + SET enable_indexscan = off; + SET enable_indexonlyscan = off; + SET enable_bitmapscan = off; + $ios_query; + }); + is($ios_result, $seq_result, + "$label: index-only scan and seqscan agree"); +} + +check_physical_consistency($pitr1, 'full backup PITR'); +check_physical_consistency($pitr2, 'incremental backup PITR'); + done_testing(); diff --git a/src/bin/pg_combinebackup/t/012_vm_consistency.pl b/src/bin/pg_combinebackup/t/012_vm_consistency.pl index 6bf47ebec3739..a0a20195d7dfe 100644 --- a/src/bin/pg_combinebackup/t/012_vm_consistency.pl +++ b/src/bin/pg_combinebackup/t/012_vm_consistency.pl @@ -166,6 +166,11 @@ sub validate_restored_vm "SELECT count(*) FROM pg_check_visible('$test->{table}')"); is($corrupt_tids, '0', "$test->{label} test: no VM corruption detected by pg_check_visible"); + + my $corrupt_frozen_tids = $restored->safe_psql('postgres', + "SELECT count(*) FROM pg_check_frozen('$test->{table}')"); + is($corrupt_frozen_tids, '0', + "$test->{label} test: no VM corruption detected by pg_check_frozen"); } # Create and populate the tables, then vacuum freeze them to set the VM bits @@ -177,6 +182,15 @@ sub validate_restored_vm check_vacuumed_vm($primary, $test); } +# Also create a multi-page table with an index, so that on the restored +# cluster we can compare a forced index-only scan (which trusts the +# visibility map) against a forced sequential scan (which does not). +$primary->safe_psql('postgres', q{ + CREATE TABLE vm_ios_test (id int PRIMARY KEY, val text); + INSERT INTO vm_ios_test SELECT g, 'row ' || g FROM generate_series(1, 1000) g; + VACUUM (FREEZE) vm_ios_test; +}); + # Take a full backup my $full_name = 'full'; my $full_path = $primary->backup_dir . "/$full_name"; @@ -221,6 +235,17 @@ sub validate_restored_vm } } +# Modify the index-only-scan test table so that its heap, index and +# visibility map all change within the incremental backup window, then +# re-vacuum so the VM bits are set again and an index-only scan on the +# restored cluster will actually consult them. +$primary->safe_psql('postgres', q{ + UPDATE vm_ios_test SET val = 'updated row ' || id WHERE id % 100 = 0; + DELETE FROM vm_ios_test WHERE id % 137 = 0; + INSERT INTO vm_ios_test SELECT g, 'new row ' || g FROM generate_series(2001, 2100) g; + VACUUM (FREEZE) vm_ios_test; +}); + # Take an incremental backup. This will have the changes made in the # modification step. my $incr_name = 'incr'; @@ -252,6 +277,62 @@ sub validate_restored_vm validate_restored_vm($restored, $test); } +# Cross-check the visibility map against the heap for every user relation +# (including vm_ios_test and any toast tables), asserting that no stale +# all-visible or all-frozen bits survived the restore. +my $vm_errors = $restored->safe_psql('postgres', q{ + SELECT relation, bad_visible, bad_frozen + FROM (SELECT c.oid::regclass AS relation, + (SELECT count(*) FROM pg_check_visible(c.oid)) AS bad_visible, + (SELECT count(*) FROM pg_check_frozen(c.oid)) AS bad_frozen + FROM pg_class c + WHERE c.relkind IN ('r', 'm', 't') + AND c.oid >= 16384) s + WHERE bad_visible > 0 OR bad_frozen > 0 +}); +is($vm_errors, '', + 'no stale VM bits in any user relation on restored cluster'); + +# Check heap and index consistency in every database of the restored +# cluster. --heapallindexed also verifies every heap tuple is found in the +# expected indexes. +$restored->command_ok( + [ 'pg_amcheck', '--all', '--install-missing', '--heapallindexed' ], + 'pg_amcheck reports no corruption on restored cluster'); + +# Run the same aggregate with a forced index-only scan (which trusts the +# visibility map) and a forced sequential scan (which does not), and require +# identical answers. This is the user-visible symptom of stale all-visible +# bits. +my $ios_query = 'SELECT count(*), sum(id), min(id), max(id) FROM vm_ios_test'; +my $ios_plan = $restored->safe_psql( + 'postgres', qq{ + SET enable_seqscan = off; + SET enable_bitmapscan = off; + SET enable_indexonlyscan = on; + EXPLAIN (COSTS OFF) $ios_query; +}); +like( + $ios_plan, + qr/Index Only Scan/, + 'aggregate query uses an index-only scan when forced'); +my $ios_result = $restored->safe_psql( + 'postgres', qq{ + SET enable_seqscan = off; + SET enable_bitmapscan = off; + SET enable_indexonlyscan = on; + $ios_query; +}); +my $seq_result = $restored->safe_psql( + 'postgres', qq{ + SET enable_indexscan = off; + SET enable_indexonlyscan = off; + SET enable_bitmapscan = off; + $ios_query; +}); +is($ios_result, $seq_result, + 'index-only scan and seqscan agree on restored cluster'); + $restored->stop; $primary->stop; From a07274ecdd1cb5534c98e815d42ff9990f5af845 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 18 Jul 2026 15:24:12 +0000 Subject: [PATCH 2/2] Address review: shared helper, all-DB sweep, committed negative control Per review of the restored-cluster oracle changes: * Factor the physical-consistency checks into a shared local module, t/CombineBackupTest.pm, loaded via the FindBin pattern used by pg_rewind's RewindTest.pm, and use it from both 002_compare_backups.pl and 012_vm_consistency.pl. * Run the pg_visibility sweep in every connectable database (pg_database WHERE datallowconn) rather than only "postgres", matching the comment's claim of covering every user relation. * Add t/013_stale_vm_detection.pl, a committed negative control proving the oracle actually fires. It restores a combined (full + incremental) backup into a scratch cluster, verifies the checks pass there, then fabricates stale all-visible/all-frozen bits by writing 0xFF into the table's VM fork: the pg_visibility sweep must report the corruption (t|100|100 corrupt TIDs) and a forced index-only scan must diverge from a forced seqscan (1000|500500|1|1000 vs 900|450000|1|999). It then overwrites part of a btree page and asserts pg_amcheck fails, proving the amcheck leg too. Only the scratch cluster is corrupted. The corruption target table must not be index-scanned before its VM is corrupted: an index-only scan on the intact cluster would heap-fetch the dead tuples and set LP_DEAD kill bits in the index, hiding the divergence afterwards. The intact-cluster sanity checks therefore use a separate identical control table. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01PtxA6B4d47ietk5dGsseuP --- src/bin/pg_combinebackup/meson.build | 1 + .../pg_combinebackup/t/002_compare_backups.pl | 77 ++------- .../pg_combinebackup/t/012_vm_consistency.pl | 66 ++------ .../t/013_stale_vm_detection.pl | 160 ++++++++++++++++++ .../pg_combinebackup/t/CombineBackupTest.pm | 155 +++++++++++++++++ 5 files changed, 338 insertions(+), 121 deletions(-) create mode 100644 src/bin/pg_combinebackup/t/013_stale_vm_detection.pl create mode 100644 src/bin/pg_combinebackup/t/CombineBackupTest.pm diff --git a/src/bin/pg_combinebackup/meson.build b/src/bin/pg_combinebackup/meson.build index ba1c8cfa3d045..015f8bf9b5175 100644 --- a/src/bin/pg_combinebackup/meson.build +++ b/src/bin/pg_combinebackup/meson.build @@ -40,6 +40,7 @@ tests += { 't/010_hardlink.pl', 't/011_ib_truncation.pl', 't/012_vm_consistency.pl', + 't/013_stale_vm_detection.pl', ], } } diff --git a/src/bin/pg_combinebackup/t/002_compare_backups.pl b/src/bin/pg_combinebackup/t/002_compare_backups.pl index d49cb0ad7c48f..4d4d1e31cb682 100644 --- a/src/bin/pg_combinebackup/t/002_compare_backups.pl +++ b/src/bin/pg_combinebackup/t/002_compare_backups.pl @@ -7,6 +7,11 @@ use PostgreSQL::Test::Utils; use Test::More; +use FindBin; +use lib $FindBin::RealBin; + +use CombineBackupTest; + my $tempdir = PostgreSQL::Test::Utils::tempdir_short(); # Can be changed to test the other modes. @@ -216,71 +221,11 @@ # stale visibility map bits, index corruption, or other damage to data that # a sequential scan does not consult. Run additional physical consistency # checks against each restored cluster, so that if only one of them fails, -# the divergence is attributable. -sub check_physical_consistency -{ - my ($node, $label) = @_; - - # Check heap and index consistency in every database. --heapallindexed - # also verifies that every heap tuple is found in the expected indexes. - $node->command_ok( - [ 'pg_amcheck', '--all', '--install-missing', '--heapallindexed' ], - "$label: pg_amcheck reports no corruption"); - - # Ask pg_visibility to cross-check the visibility map against the heap - # for every user relation. Stale all-visible or all-frozen bits are - # invisible to a logical dump but corrupt index-only scan results and - # can cause future heap corruption once vacuum trusts them. - $node->safe_psql('postgres', - 'CREATE EXTENSION IF NOT EXISTS pg_visibility'); - my $vm_errors = $node->safe_psql('postgres', q{ - SELECT relation, bad_visible, bad_frozen - FROM (SELECT c.oid::regclass AS relation, - (SELECT count(*) FROM pg_check_visible(c.oid)) AS bad_visible, - (SELECT count(*) FROM pg_check_frozen(c.oid)) AS bad_frozen - FROM pg_class c - WHERE c.relkind IN ('r', 'm', 't') - AND c.oid >= 16384) s - WHERE bad_visible > 0 OR bad_frozen > 0 - }); - is($vm_errors, '', - "$label: pg_check_visible/pg_check_frozen report no problems"); - - # Run the same aggregate with a forced index-only scan (which trusts the - # visibility map) and a forced sequential scan (which does not), and - # require identical answers. This is the user-visible symptom of stale - # all-visible bits. - my $ios_query = 'SELECT count(*), sum(a), min(a), max(a) FROM indexed_table'; - my $ios_plan = $node->safe_psql( - 'postgres', qq{ - SET enable_seqscan = off; - SET enable_bitmapscan = off; - SET enable_indexonlyscan = on; - EXPLAIN (COSTS OFF) $ios_query; - }); - like( - $ios_plan, - qr/Index Only Scan/, - "$label: aggregate query uses an index-only scan when forced"); - my $ios_result = $node->safe_psql( - 'postgres', qq{ - SET enable_seqscan = off; - SET enable_bitmapscan = off; - SET enable_indexonlyscan = on; - $ios_query; - }); - my $seq_result = $node->safe_psql( - 'postgres', qq{ - SET enable_indexscan = off; - SET enable_indexonlyscan = off; - SET enable_bitmapscan = off; - $ios_query; - }); - is($ios_result, $seq_result, - "$label: index-only scan and seqscan agree"); -} - -check_physical_consistency($pitr1, 'full backup PITR'); -check_physical_consistency($pitr2, 'incremental backup PITR'); +# the divergence is attributable. See CombineBackupTest.pm for the details; +# t/013_stale_vm_detection.pl is the negative control demonstrating that +# these checks really do fire on a corrupted cluster. +check_physical_consistency($pitr1, 'full backup PITR', 'indexed_table', 'a'); +check_physical_consistency($pitr2, 'incremental backup PITR', + 'indexed_table', 'a'); done_testing(); diff --git a/src/bin/pg_combinebackup/t/012_vm_consistency.pl b/src/bin/pg_combinebackup/t/012_vm_consistency.pl index a0a20195d7dfe..c8ecea0593660 100644 --- a/src/bin/pg_combinebackup/t/012_vm_consistency.pl +++ b/src/bin/pg_combinebackup/t/012_vm_consistency.pl @@ -11,6 +11,11 @@ use PostgreSQL::Test::Utils; use Test::More; +use FindBin; +use lib $FindBin::RealBin; + +use CombineBackupTest; + my $tempdir = PostgreSQL::Test::Utils::tempdir_short(); my $mode = $ENV{PG_TEST_PG_COMBINEBACKUP_MODE} || '--copy'; @@ -277,61 +282,12 @@ sub validate_restored_vm validate_restored_vm($restored, $test); } -# Cross-check the visibility map against the heap for every user relation -# (including vm_ios_test and any toast tables), asserting that no stale -# all-visible or all-frozen bits survived the restore. -my $vm_errors = $restored->safe_psql('postgres', q{ - SELECT relation, bad_visible, bad_frozen - FROM (SELECT c.oid::regclass AS relation, - (SELECT count(*) FROM pg_check_visible(c.oid)) AS bad_visible, - (SELECT count(*) FROM pg_check_frozen(c.oid)) AS bad_frozen - FROM pg_class c - WHERE c.relkind IN ('r', 'm', 't') - AND c.oid >= 16384) s - WHERE bad_visible > 0 OR bad_frozen > 0 -}); -is($vm_errors, '', - 'no stale VM bits in any user relation on restored cluster'); - -# Check heap and index consistency in every database of the restored -# cluster. --heapallindexed also verifies every heap tuple is found in the -# expected indexes. -$restored->command_ok( - [ 'pg_amcheck', '--all', '--install-missing', '--heapallindexed' ], - 'pg_amcheck reports no corruption on restored cluster'); - -# Run the same aggregate with a forced index-only scan (which trusts the -# visibility map) and a forced sequential scan (which does not), and require -# identical answers. This is the user-visible symptom of stale all-visible -# bits. -my $ios_query = 'SELECT count(*), sum(id), min(id), max(id) FROM vm_ios_test'; -my $ios_plan = $restored->safe_psql( - 'postgres', qq{ - SET enable_seqscan = off; - SET enable_bitmapscan = off; - SET enable_indexonlyscan = on; - EXPLAIN (COSTS OFF) $ios_query; -}); -like( - $ios_plan, - qr/Index Only Scan/, - 'aggregate query uses an index-only scan when forced'); -my $ios_result = $restored->safe_psql( - 'postgres', qq{ - SET enable_seqscan = off; - SET enable_bitmapscan = off; - SET enable_indexonlyscan = on; - $ios_query; -}); -my $seq_result = $restored->safe_psql( - 'postgres', qq{ - SET enable_indexscan = off; - SET enable_indexonlyscan = off; - SET enable_bitmapscan = off; - $ios_query; -}); -is($ios_result, $seq_result, - 'index-only scan and seqscan agree on restored cluster'); +# Run the full set of physical consistency checks against the restored +# cluster: pg_amcheck across all databases, a pg_visibility sweep over every +# user relation in every connectable database, and a forced index-only-scan +# vs seqscan comparison on vm_ios_test. See CombineBackupTest.pm. +check_physical_consistency($restored, 'restored cluster', 'vm_ios_test', + 'id'); $restored->stop; $primary->stop; diff --git a/src/bin/pg_combinebackup/t/013_stale_vm_detection.pl b/src/bin/pg_combinebackup/t/013_stale_vm_detection.pl new file mode 100644 index 0000000000000..aa0b57e9e89f7 --- /dev/null +++ b/src/bin/pg_combinebackup/t/013_stale_vm_detection.pl @@ -0,0 +1,160 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group +# +# Negative control for the physical-consistency oracle used by +# 002_compare_backups.pl and 012_vm_consistency.pl (CombineBackupTest.pm): +# prove that the checks really do fire on a corrupted cluster. Restore a +# combined (full + incremental) backup into a scratch cluster, verify the +# oracle passes there, then fabricate stale all-visible/all-frozen bits by +# writing directly into the table's visibility map fork and assert that +# both the pg_visibility sweep and the index-only-scan-vs-seqscan +# comparison detect the damage. Finally, overwrite part of a btree page +# and assert that pg_amcheck detects that as well. +# +# Only the scratch restored cluster is ever corrupted; nothing here +# touches the clusters used for real comparisons in the other tests. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +use FindBin; +use lib $FindBin::RealBin; + +use CombineBackupTest; + +my $mode = $ENV{PG_TEST_PG_COMBINEBACKUP_MODE} || '--copy'; + +note "testing using mode $mode"; + +# Data checksums must be disabled: we fabricate stale VM bits by writing +# directly into the VM fork without recomputing page checksums, which is +# also how a hypothetical pg_combinebackup bug would manifest (a valid +# page image whose bits are simply wrong). +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 1, no_data_checksums => 1); +$primary->append_conf('postgresql.conf', <start; + +# A thousand rows, vacuum-frozen so the VM bits are set and an index-only +# scan will consult them. Table t is the corruption target; t_control is +# an identical table used for the intact-cluster sanity checks. The two +# must be distinct: an index(-only) scan over t before its VM is +# corrupted would heap-fetch the dead tuples and set LP_DEAD kill bits on +# their index entries, after which later index-only scans would skip them +# regardless of the (corrupted) visibility map. +$primary->safe_psql('postgres', q{ + CREATE TABLE t (id int PRIMARY KEY, val text); + INSERT INTO t SELECT g, 'row ' || g FROM generate_series(1, 1000) g; + VACUUM (FREEZE) t; + CREATE TABLE t_control (id int PRIMARY KEY, val text); + INSERT INTO t_control SELECT g, 'row ' || g FROM generate_series(1, 1000) g; + VACUUM (FREEZE) t_control; +}); + +# Take a full backup. +my $full_path = $primary->backup_dir . '/full'; +$primary->command_ok( + [ + 'pg_basebackup', '--no-sync', + '--pgdata' => $full_path, + '--checkpoint' => 'fast', + ], + 'full backup'); + +# Delete some rows, clearing the VM bits of every affected heap page and +# leaving dead tuples behind that are still present in the index. The +# "% 10" predicate is not indexable, so this cannot set index kill bits. +$primary->safe_psql('postgres', 'DELETE FROM t WHERE id % 10 = 0'); +$primary->safe_psql('postgres', 'DELETE FROM t_control WHERE id % 10 = 0'); + +# Take an incremental backup containing those changes. +my $incr_path = $primary->backup_dir . '/incr'; +$primary->command_ok( + [ + 'pg_basebackup', '--no-sync', + '--pgdata' => $incr_path, + '--checkpoint' => 'fast', + '--incremental' => $full_path . '/backup_manifest', + ], + 'incremental backup'); + +# Restore the combined backup into a scratch cluster. +my $restored = PostgreSQL::Test::Cluster->new('restored'); +$restored->init_from_backup( + $primary, 'incr', + combine_with_prior => ['full'], + combine_mode => $mode); +$restored->append_conf('postgresql.conf', 'autovacuum = off'); +$restored->start; + +# Sanity check: on the intact restored cluster, the oracle passes. Note +# that the index-only-scan check runs on t_control, not t, to avoid +# setting index kill bits on t (see above). +is(vm_sweep_errors($restored, 'postgres'), + '', 'no stale VM bits on intact restored cluster'); +my ($ios_plan, $ios_result, $seq_result) = + ios_vs_seqscan($restored, 'postgres', 't_control', 'id'); +like( + $ios_plan, + qr/Index Only Scan/, + 'aggregate query uses an index-only scan when forced'); +is($ios_result, $seq_result, + 'index-only scan and seqscan agree on intact restored cluster'); + +# Now fabricate stale VM bits: mark the first 16 heap pages all-visible +# and all-frozen by setting the first four bytes of the VM data area +# (which follows the standard 24-byte page header). The table's heap is +# smaller than 16 pages, so this covers every page, including the ones +# holding dead tuples. +my $relpath = + $restored->safe_psql('postgres', "SELECT pg_relation_filepath('t')"); +my $vmfile = $restored->data_dir . '/' . $relpath . '_vm'; +ok(-f $vmfile, 'visibility map fork exists'); + +$restored->stop; +open my $vmfh, '+<:raw', $vmfile or die "open $vmfile: $!"; +seek($vmfh, 24, 0) or die "seek $vmfile: $!"; +print $vmfh "\xff" x 4; +close $vmfh or die "close $vmfile: $!"; +$restored->start; + +# The pg_visibility sweep must now report corruption ... +my $vm_errors = vm_sweep_errors($restored, 'postgres'); +isnt($vm_errors, '', 'pg_visibility sweep detects the stale VM bits'); +note "vm sweep reported: $vm_errors"; + +# ... and the index-only scan must diverge from the seqscan, since it +# now trusts all-visible bits covering dead tuples. +($ios_plan, $ios_result, $seq_result) = + ios_vs_seqscan($restored, 'postgres', 't', 'id'); +isnt($ios_result, $seq_result, + 'index-only scan diverges from seqscan with stale VM bits'); +note "index-only scan returned [$ios_result], seqscan [$seq_result]"; + +# Also prove the pg_amcheck leg of the oracle: overwrite part of a btree +# page and assert pg_amcheck fails. (pg_amcheck by itself does not detect +# stale VM bits, which is why the pg_visibility sweep above is needed.) +my $idxpath = + $restored->safe_psql('postgres', "SELECT pg_relation_filepath('t_pkey')"); +my $idxfile = $restored->data_dir . '/' . $idxpath; + +$restored->stop; +open my $idxfh, '+<:raw', $idxfile or die "open $idxfile: $!"; +seek($idxfh, 8192 + 100, 0) or die "seek $idxfile: $!"; +print $idxfh "\xde\xad\xbe\xef" x 25; +close $idxfh or die "close $idxfile: $!"; +$restored->start; + +$restored->command_fails( + [ 'pg_amcheck', '--all', '--install-missing', '--heapallindexed' ], + 'pg_amcheck detects the corrupted btree page'); + +$restored->stop; +$primary->stop; + +done_testing(); diff --git a/src/bin/pg_combinebackup/t/CombineBackupTest.pm b/src/bin/pg_combinebackup/t/CombineBackupTest.pm new file mode 100644 index 0000000000000..d841d249e5b8e --- /dev/null +++ b/src/bin/pg_combinebackup/t/CombineBackupTest.pm @@ -0,0 +1,155 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +=pod + +=head1 NAME + +CombineBackupTest - helper routines for pg_combinebackup TAP tests + +=head1 SYNOPSIS + + use FindBin; + use lib $FindBin::RealBin; + use CombineBackupTest; + + check_physical_consistency($node, 'some label', 'indexed_table', 'a'); + +=head1 DESCRIPTION + +Physical-consistency checks ("oracle") run against restored clusters. +A logical dump can only see live row contents through sequential scans, +so comparing pg_dumpall output is blind to physical-level problems that +pg_combinebackup could introduce: stale visibility map bits, index +corruption, or other damage to data that a sequential scan does not +consult. The routines here check those directly. + +The individual building blocks (vm_sweep_errors, ios_vs_seqscan) are +also exported so that a negative-control test can assert that the +checks really do fire on corrupted clusters. + +=cut + +package CombineBackupTest; + +use strict; +use warnings FATAL => 'all'; + +use Exporter 'import'; +use Test::More; + +our @EXPORT = qw( + connectable_databases + vm_sweep_errors + ios_vs_seqscan + check_physical_consistency +); + +# Return the names of all connectable databases in the cluster. +sub connectable_databases +{ + my ($node) = @_; + my $datnames = $node->safe_psql('postgres', + "SELECT datname FROM pg_database WHERE datallowconn ORDER BY datname"); + return split(/\n/, $datnames); +} + +# Ask pg_visibility to cross-check the visibility map against the heap for +# every user relation (tables, materialized views, toast tables) in the +# given database. Returns an empty string if no problems were found, else +# one line per corrupt relation listing the number of TIDs flagged by +# pg_check_visible() and pg_check_frozen(). +sub vm_sweep_errors +{ + my ($node, $dbname) = @_; + + $node->safe_psql($dbname, 'CREATE EXTENSION IF NOT EXISTS pg_visibility'); + return $node->safe_psql($dbname, q{ + SELECT relation, bad_visible, bad_frozen + FROM (SELECT c.oid::regclass AS relation, + (SELECT count(*) FROM pg_check_visible(c.oid)) AS bad_visible, + (SELECT count(*) FROM pg_check_frozen(c.oid)) AS bad_frozen + FROM pg_class c + WHERE c.relkind IN ('r', 'm', 't') + AND c.oid >= 16384) s + WHERE bad_visible > 0 OR bad_frozen > 0 + }); +} + +# Run the same aggregate over the given table with a forced index-only scan +# (which trusts the visibility map) and a forced sequential scan (which does +# not). Returns the forced index-only-scan plan and both results, so the +# caller can assert that the plan really is an index-only scan and that the +# two results agree (or, in a negative test, disagree). +sub ios_vs_seqscan +{ + my ($node, $dbname, $table, $column) = @_; + + my $query = + "SELECT count(*), sum($column), min($column), max($column) FROM $table"; + my $ios_plan = $node->safe_psql( + $dbname, qq{ + SET enable_seqscan = off; + SET enable_bitmapscan = off; + SET enable_indexonlyscan = on; + EXPLAIN (COSTS OFF) $query; + }); + my $ios_result = $node->safe_psql( + $dbname, qq{ + SET enable_seqscan = off; + SET enable_bitmapscan = off; + SET enable_indexonlyscan = on; + $query; + }); + my $seq_result = $node->safe_psql( + $dbname, qq{ + SET enable_indexscan = off; + SET enable_indexonlyscan = off; + SET enable_bitmapscan = off; + $query; + }); + return ($ios_plan, $ios_result, $seq_result); +} + +# Run the full set of physical consistency checks against a restored +# cluster: +# +# 1. pg_amcheck over every database, verifying heap and index consistency. +# --heapallindexed also verifies that every heap tuple is found in the +# expected indexes. +# +# 2. A pg_visibility sweep over every user relation in every connectable +# database, asserting that no stale all-visible or all-frozen bits +# exist. Stale VM bits are invisible to a logical dump but corrupt +# index-only scan results and can cause future heap corruption once +# vacuum trusts them. +# +# 3. A forced index-only scan vs forced sequential scan comparison over +# $ios_table.$ios_column (in database "postgres"), which is the +# user-visible symptom of stale all-visible bits. +sub check_physical_consistency +{ + my ($node, $label, $ios_table, $ios_column) = @_; + local $Test::Builder::Level = $Test::Builder::Level + 1; + + $node->command_ok( + [ 'pg_amcheck', '--all', '--install-missing', '--heapallindexed' ], + "$label: pg_amcheck reports no corruption"); + + foreach my $dbname (connectable_databases($node)) + { + is(vm_sweep_errors($node, $dbname), + '', "$label: no stale VM bits in database $dbname"); + } + + my ($ios_plan, $ios_result, $seq_result) = + ios_vs_seqscan($node, 'postgres', $ios_table, $ios_column); + like( + $ios_plan, + qr/Index Only Scan/, + "$label: aggregate query uses an index-only scan when forced"); + is($ios_result, $seq_result, + "$label: index-only scan and seqscan agree"); + return; +} + +1;