From 8177c629c3216fb630c2ca94accac99281a50434 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 18 Jul 2026 13:01:00 +0000 Subject: [PATCH 1/2] Add physical differential oracle test for incremental backups Incremental backups depend on the WAL summarizer, which only learns about relation blocks that WAL records properly register. Any code path that modifies a page without registering its buffer (or without WAL-logging the change at all) is a summarizer blind spot: pg_basebackup omits the block from the incremental backup and pg_combinebackup silently reconstructs a stale page from an older backup. Bugs of this class - most recently unregistered visibility map buffers, fixed in ed62d26 - were previously only detectable by enumerating each affected operation (as 012_vm_consistency.pl does). This adds a generic physical oracle that catches the whole class instead of known instances. The new test runs a mixed workload (heap with indexes, TOAST, VM bit set/clear cycles, bulk COPY, TRUNCATE and re-extension, index build/drop, table drop, an unlogged table for init fork coverage) with churn running concurrently with a chain of rate-limited incremental backups F0 <- I1 <- I2. It then quiesces the cluster (VACUUM, a hint-bit settle sweep over catalogs and user tables, CHECKPOINT) and takes a final incremental I3 plus a reference full backup R back to back. pg_waldump audits the WAL window spanning both final backups: if any record in it carries a block reference, the two backups may not describe the same cluster state and the pair is retaken (bounded retries), so the comparison is deterministic by construction rather than by luck. pg_combinebackup(F0, I1, I2, I3) must then be physically identical to R, block by block, for every main, _vm and _init fork under base/ and global/. FSM forks are skipped (not fully WAL-logged by design). Only pd_lsn and pd_checksum are masked; hint bits are compared strictly, which is sound because data checksums (on by default) force hint-bit changes to be WAL-logged. In practice even the masked bytes match; the mask exists as documented insurance. Any divergence is reported with file, relation name, block number and differing byte ranges. Closes #74 Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01PtxA6B4d47ietk5dGsseuP --- src/bin/pg_combinebackup/meson.build | 1 + .../t/013_physical_differential.pl | 552 ++++++++++++++++++ 2 files changed, 553 insertions(+) create mode 100644 src/bin/pg_combinebackup/t/013_physical_differential.pl diff --git a/src/bin/pg_combinebackup/meson.build b/src/bin/pg_combinebackup/meson.build index ba1c8cfa3d045..4fd4ff5b146b4 100644 --- a/src/bin/pg_combinebackup/meson.build +++ b/src/bin/pg_combinebackup/meson.build @@ -40,6 +40,7 @@ tests += { 't/010_hardlink.pl', 't/011_ib_truncation.pl', 't/012_vm_consistency.pl', + 't/013_physical_differential.pl', ], } } diff --git a/src/bin/pg_combinebackup/t/013_physical_differential.pl b/src/bin/pg_combinebackup/t/013_physical_differential.pl new file mode 100644 index 0000000000000..69c1e8815eced --- /dev/null +++ b/src/bin/pg_combinebackup/t/013_physical_differential.pl @@ -0,0 +1,552 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group +# +# Physical oracle for incremental backup completeness. +# +# Incremental backups rely on the WAL summarizer, which only knows about +# buffers that WAL records properly register. Any code path that modifies a +# relation page without registering the buffer (or without WAL-logging the +# change at all) is invisible to the summarizer, and pg_combinebackup will +# silently reconstruct a stale page from an older backup. Past bugs of this +# class (e.g. visibility map buffers not registered by heap operations) were +# only caught operation-by-operation. This test is a generic oracle: it +# runs a mixed workload with churn concurrent to a chain of incremental +# backups, then quiesces the cluster and takes -- back to back, with no +# intervening relation writes -- a final incremental backup and a reference +# full backup. The combined result of the backup chain must then be +# physically identical, relation block by relation block, to the reference +# full backup. +# +# Comparison scope and masking: +# - main, _vm and _init relation forks under base/ and global/ are compared +# byte-for-byte per 8 KB block. +# - _fsm forks are skipped: the free space map is not fully WAL-logged by +# design, so it is legitimately allowed to diverge. +# - Within a block, only pd_lsn and pd_checksum (the first 10 bytes of the +# page header) are masked. Everything else, including hint bits in tuple +# infomasks, must match: data checksums are enabled by default, so hint +# bit changes that reach disk are WAL-logged as full-page hints and hence +# visible to the WAL summarizer. Blocks that differ only in the masked +# bytes are counted and reported as a note, not a failure. +# +# Quiescence discipline: after the workload stops, a settle sequence +# (VACUUM, catalog/table scans to set any remaining hint bits, CHECKPOINT) +# runs before the final backup pair. The WAL range spanning both final +# backups is then checked with pg_waldump: if any WAL record in that range +# references a relation block, the reference backup might not describe the +# same cluster state as the combined backup, so the pair is retaken (with a +# bounded number of attempts) rather than risking a false verdict. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +use constant BLCKSZ => 8192; +# pd_lsn (8 bytes) + pd_checksum (2 bytes) +use constant MASKED_HEADER_BYTES => 10; +# blocks per 1 GB segment file with default BLCKSZ +use constant RELSEG_SIZE => 131072; + +my $mode = $ENV{PG_TEST_PG_COMBINEBACKUP_MODE} || '--copy'; +note "testing using mode $mode"; + +# Set up a primary with WAL summarization. Autovacuum is disabled so that +# all relation modifications come from the (bounded) workload below and the +# final quiesce is actually quiescent. Data checksums are left at their +# default (enabled); the no-masking-of-hint-bits policy above depends on it. +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 1); +$primary->append_conf('postgresql.conf', <start; + +ok( $primary->safe_psql('postgres', 'SHOW data_checksums') eq 'on', + 'data checksums are enabled (hint bit writes are WAL-logged)'); + +# Seed workload: a heap with indexes fed by a sequence, a TOAST-heavy table, +# a table whose VM bits will be set and cleared repeatedly, a table for bulk +# COPY extension, a table that gets truncated and re-extended, and an +# unlogged table (so that an init fork is part of the comparison). +$primary->safe_psql('postgres', <<'EOSQL'); +CREATE SEQUENCE heap_seq; +CREATE TABLE t_heap (id bigint PRIMARY KEY DEFAULT nextval('heap_seq'), + grp int, val text); +INSERT INTO t_heap (grp, val) + SELECT g % 100, 'seed-' || g FROM generate_series(1, 20000) g; +CREATE INDEX t_heap_grp_idx ON t_heap (grp); + +CREATE TABLE t_toast (id int PRIMARY KEY, big text); +INSERT INTO t_toast + SELECT g, (SELECT string_agg(md5(g::text || s::text), '') + FROM generate_series(1, 300) s) + FROM generate_series(1, 20) g; + +CREATE TABLE t_vm (id int, val text); +INSERT INTO t_vm SELECT g, 'vm-' || g FROM generate_series(1, 5000) g; +VACUUM (FREEZE) t_vm; + +CREATE TABLE t_copy (id bigint); +CREATE TABLE t_trunc (id int, val text); +INSERT INTO t_trunc SELECT g, 'tr-' || g FROM generate_series(1, 5000) g; + +CREATE UNLOGGED TABLE t_unlogged (id int, val text); +INSERT INTO t_unlogged SELECT g, 'ul-' || g FROM generate_series(1, 1000) g; + +CREATE TABLE t_dropped AS + SELECT g AS id, 'drop-' || g AS val FROM generate_series(1, 2000) g; +EOSQL + +my $backup_dir = $primary->backup_dir; + +# Take the full backup F0 that anchors the incremental chain. +my $f0_path = $backup_dir . '/f0'; +$primary->command_ok( + [ + 'pg_basebackup', '--no-sync', + '--pgdata' => $f0_path, + '--checkpoint' => 'fast', + ], + 'full backup F0'); + +# Helper: run a churn script in a background session, concurrently with a +# foreground action (an incremental backup). The DO block is submitted +# asynchronously; \echo output tells us it has started. After the +# foreground action completes we synchronize on a trivial query, which can +# only run once the DO block has finished. +my $bg = $primary->background_psql('postgres'); + +sub churn_concurrently_with +{ + my ($phase, $churn_sql, $action) = @_; + + $bg->query_until( + qr/${phase}_started/, qq{\\echo ${phase}_started +$churn_sql +}); + $action->(); + my $sync = $bg->query_safe(qq{SELECT '${phase}_synced'}); + like($sync, qr/${phase}_synced/, "$phase churn completed"); + return; +} + +# Phase 1 churn: scattered updates, inserts and deletes on the heap, TOAST +# value rewrites. Runs concurrently with incremental backup I1. +my $phase1_churn = <<'EOSQL'; +DO $$ +DECLARE i int; +BEGIN + FOR i IN 1..40 LOOP + UPDATE t_heap SET val = val || 'x' WHERE grp = (i * 7) % 100; + INSERT INTO t_heap (grp, val) + SELECT (i * 13 + g) % 100, 'p1-' || i + FROM generate_series(1, 100) g; + DELETE FROM t_heap + WHERE grp = (i * 11) % 100 AND id % 5 = i % 5; + UPDATE t_toast SET big = md5(i::text) || big + WHERE id = (i % 20) + 1; + COMMIT; + PERFORM pg_sleep(0.05); + END LOOP; +END +$$; +EOSQL + +my $i1_path = $backup_dir . '/i1'; +churn_concurrently_with( + 'phase1', + $phase1_churn, + sub { + $primary->command_ok( + [ + 'pg_basebackup', '--no-sync', + '--pgdata' => $i1_path, + '--checkpoint' => 'fast', + # rate-limit so the backup genuinely overlaps the churn + '--max-rate' => '4M', + '--incremental' => $f0_path . '/backup_manifest', + ], + 'incremental backup I1 (concurrent churn)'); + }); + +# Interlude between I1 and I2: set and clear VM bits, bulk COPY extension, +# TRUNCATE and re-extend, index creation. +$primary->safe_psql('postgres', + "COPY t_copy FROM STDIN;\n" . join("\n", 1 .. 60000) . "\n\\.\n"); +$primary->safe_psql('postgres', <<'EOSQL'); +DELETE FROM t_vm WHERE id % 7 = 0; +VACUUM t_vm; +VACUUM (FREEZE) t_heap; +TRUNCATE t_trunc; +INSERT INTO t_trunc SELECT g, 'tr2-' || g FROM generate_series(1, 8000) g; +CREATE INDEX t_trunc_idx ON t_trunc (id); +EOSQL + +# Phase 2 churn: operations that clear freshly-set VM bits (the historical +# summarizer blind spot): updates, deletes, tuple locks, plus more heap and +# TOAST churn. Runs concurrently with incremental backup I2. +my $phase2_churn = <<'EOSQL'; +DO $$ +DECLARE i int; +BEGIN + FOR i IN 1..40 LOOP + UPDATE t_vm SET val = val || 'y' WHERE id % 97 = i; + DELETE FROM t_vm WHERE id % 89 = i; + PERFORM * FROM t_heap WHERE grp = (i * 3) % 100 FOR UPDATE; + UPDATE t_heap SET val = 'p2-' || i WHERE grp = (i * 17) % 100; + INSERT INTO t_copy SELECT g FROM generate_series(1, 200) g; + UPDATE t_toast SET big = big || md5((i * 31)::text) + WHERE id = (i % 20) + 1; + COMMIT; + PERFORM pg_sleep(0.05); + END LOOP; +END +$$; +EOSQL + +my $i2_path = $backup_dir . '/i2'; +churn_concurrently_with( + 'phase2', + $phase2_churn, + sub { + $primary->command_ok( + [ + 'pg_basebackup', '--no-sync', + '--pgdata' => $i2_path, + '--checkpoint' => 'fast', + # rate-limit so the backup genuinely overlaps the churn + '--max-rate' => '4M', + '--incremental' => $i1_path . '/backup_manifest', + ], + 'incremental backup I2 (concurrent churn)'); + }); + +# Post-I2 activity: index churn, more VM cycling, unlogged churn, and a +# table drop, all of which land in the final incremental I3. +$primary->safe_psql('postgres', <<'EOSQL'); +DROP INDEX t_heap_grp_idx; +CREATE INDEX t_heap_grp_idx2 ON t_heap (grp); +DELETE FROM t_vm WHERE id % 11 = 0; +VACUUM (FREEZE) t_vm; +TRUNCATE t_unlogged; +INSERT INTO t_unlogged SELECT g, 'ul2-' || g FROM generate_series(1, 1000) g; +DROP TABLE t_dropped; +EOSQL + +# End of workload. +$bg->quit; + +# Quiesce and take the final backup pair: incremental I3 (completing the +# chain) and reference full backup R, back to back. Both must capture the +# identical set of relation blocks. The settle sequence makes the cluster +# truly quiescent: VACUUM sets hint bits and VM bits everywhere and +# reconciles catalogs, the SELECT sweep sets hint bits on the tuples the +# VACUUM itself created, and CHECKPOINT flushes everything so that nothing +# is left for a later checkpoint (e.g. R's own start checkpoint) to write. +# +# Even so, we verify rather than hope: any WAL record carrying a block +# reference between the start of I3 and the end of R means a relation page +# may have changed between the two backups, invalidating the comparison. +# In that case the pair is retaken. +my $settle_sweep = q{ +SELECT count(*) FROM pg_class; +SELECT count(*) FROM pg_attribute; +SELECT count(*) FROM pg_type; +SELECT count(*) FROM pg_index; +SELECT count(*) FROM pg_depend; +SELECT count(*) FROM pg_shdepend; +SELECT count(*) FROM pg_database; +SELECT count(*) FROM pg_authid; +SELECT count(*) FROM pg_auth_members; +SELECT count(*) FROM pg_statistic; +SELECT count(*) FROM pg_sequence; +SELECT count(*) FROM t_heap; +SELECT count(*) FROM t_toast; +SELECT count(*) FROM t_vm; +SELECT count(*) FROM t_copy; +SELECT count(*) FROM t_trunc; +SELECT count(*) FROM t_unlogged; +}; + +sub insert_lsn +{ + return $primary->safe_psql('postgres', + 'SELECT pg_current_wal_insert_lsn()'); +} + +sub parse_lsn +{ + my ($lsn) = @_; + my ($hi, $lo) = split m{/}, $lsn; + return (hex($hi) << 32) + hex($lo); +} + +my $max_attempts = 3; +my ($i3_path, $r_path); +my $accepted = 0; + +for my $attempt (1 .. $max_attempts) +{ + $primary->safe_psql('postgres', 'VACUUM'); + $primary->safe_psql('postgres', $settle_sweep); + $primary->safe_psql('postgres', 'CHECKPOINT'); + + my $lsn_start = insert_lsn(); + + $i3_path = $backup_dir . "/i3_attempt$attempt"; + $primary->command_ok( + [ + 'pg_basebackup', '--no-sync', + '--pgdata' => $i3_path, + '--checkpoint' => 'fast', + '--incremental' => $i2_path . '/backup_manifest', + ], + "final incremental backup I3 (attempt $attempt)"); + + my $lsn_mid = insert_lsn(); + + $r_path = $backup_dir . "/r_attempt$attempt"; + $primary->command_ok( + [ + 'pg_basebackup', '--no-sync', + '--pgdata' => $r_path, + '--checkpoint' => 'fast', + ], + "reference full backup R (attempt $attempt)"); + + my $lsn_end = insert_lsn(); + + note "attempt $attempt: I3 window start $lsn_start, " + . "between backups $lsn_mid, after R $lsn_end"; + + # The backup-end WAL switch leaves the insert LSN pointing into a + # segment whose file may not exist yet. Write and flush one trivial + # record just past the audited window: it materializes that segment + # and serves as a coverage sentinel proving pg_waldump really read + # through $lsn_end. + $primary->safe_psql('postgres', 'SELECT txid_current()'); + + # Backups themselves emit WAL (checkpoint and backup-end records), so + # the insert LSN moving is expected; what must NOT appear in this + # window is any record referencing a relation block. pg_waldump is + # run without --end (an end LSN exactly at a segment boundary confuses + # its page reader) and always terminates with an end-of-WAL error; + # instead, we require that the records it printed reach $lsn_end. + my ($wal_out, $wal_err) = run_command( + [ + 'pg_waldump', + '--path' => $primary->data_dir . '/pg_wal', + '--start' => $lsn_start, + ]); + + my $end_n = parse_lsn($lsn_end); + my $max_n = 0; + my @blkref_lines; + foreach my $line (split /\n/, $wal_out) + { + next unless $line =~ m{lsn: ([0-9A-Fa-f]+/[0-9A-Fa-f]+), prev}; + my $n = parse_lsn($1); + $max_n = $n if $n > $max_n; + push @blkref_lines, $line if $n < $end_n && $line =~ /blkref/; + } + + if (@blkref_lines) + { + diag "attempt $attempt: relation blocks referenced in WAL " + . "between final backups; not quiescent:"; + diag join("\n", + @blkref_lines[ 0 .. ($#blkref_lines > 9 ? 9 : $#blkref_lines) ]); + } + elsif ($max_n < $end_n) + { + diag "attempt $attempt: pg_waldump did not cover the full window " + . "(reached " . sprintf('%X/%08X', $max_n >> 32, $max_n & 0xFFFFFFFF) + . ", needed $lsn_end): $wal_err"; + } + else + { + $accepted = 1; + ok(1, "final backup pair captured identical cluster state " + . "(attempt $attempt)"); + last; + } +} + +ok($accepted, + "quiescent final backup pair obtained within $max_attempts attempts") + or die 'cannot obtain a quiescent reference point, aborting'; + +# Map relfilenode paths to relation names for better diagnostics. +my %relname_by_path; +{ + my $rows = $primary->safe_psql('postgres', + q{SELECT pg_relation_filepath(oid), relname + FROM pg_class WHERE pg_relation_filepath(oid) IS NOT NULL}); + foreach my $row (split /\n/, $rows) + { + my ($path, $name) = split /\|/, $row; + $relname_by_path{$path} = $name; + } +} + +$primary->stop; + +# Reconstruct the combined backup C = F0 + I1 + I2 + I3. +my $c_path = $backup_dir . '/combined'; +$primary->command_ok( + [ + 'pg_combinebackup', $mode, '--no-sync', + '--output' => $c_path, + $f0_path, $i1_path, $i2_path, $i3_path, + ], + 'pg_combinebackup reconstructs the chain'); + +# +# Physical comparison of C against R. +# + +# Return a hash of relation data files (relative paths) under base/ and +# global/ in a backup directory: main, _vm and _init forks, including +# extra segment files; _fsm is excluded by design (not fully WAL-logged). +sub collect_relation_files +{ + my ($root) = @_; + my %files; + my @dirs = ('global'); + if (-d "$root/base") + { + push @dirs, map { "base/$_" } grep { /^\d+$/ } slurp_dir("$root/base"); + } + foreach my $dir (@dirs) + { + next unless -d "$root/$dir"; + foreach my $f (slurp_dir("$root/$dir")) + { + next unless $f =~ /^\d+(?:_(vm|init|fsm))?(?:\.\d+)?$/; + my $fork = $1 // 'main'; + next if $fork eq 'fsm'; + $files{"$dir/$f"} = 1; + } + } + return \%files; +} + +# Describe the divergence within one block: the differing byte ranges +# (computed on the masked images, so pd_lsn/pd_checksum noise is excluded) +# with hex dumps of the raw bytes. +sub describe_block_diff +{ + my ($rel, $relname, $blkno, $cbuf, $rbuf, $cmask, $rmask) = @_; + + my $absblk = $blkno; + $absblk += $1 * RELSEG_SIZE if $rel =~ /\.(\d+)$/; + + my $msg = "DIVERGENCE: $rel ($relname) block $absblk:"; + my $xor = $cmask ^ $rmask; + my $nranges = 0; + while ($xor =~ /[^\0]+/g) + { + my ($s, $e) = ($-[0], $+[0] - 1); + my $len = $e - $s + 1; + my $show = $len > 16 ? 16 : $len; + $msg .= sprintf( + "\n bytes %d..%d differ; combined=%s%s reference=%s%s", + $s, $e, + unpack('H*', substr($cbuf, $s, $show)), + $len > $show ? '...' : '', + unpack('H*', substr($rbuf, $s, $show)), + $len > $show ? '...' : ''); + last if ++$nranges >= 5; + } + return $msg; +} + +sub compare_relation_file +{ + my ($c_file, $r_file, $rel, $relname, $reports, $stats) = @_; + + open my $cf, '<:raw', $c_file or die "open $c_file: $!"; + open my $rf, '<:raw', $r_file or die "open $r_file: $!"; + my $c_size = -s $cf; + my $r_size = -s $rf; + my $ndiv = 0; + + if ($c_size != $r_size) + { + $ndiv++; + push @$reports, + "DIVERGENCE: $rel ($relname): size mismatch: " + . "combined $c_size vs reference $r_size bytes"; + } + + my $minsize = $c_size < $r_size ? $c_size : $r_size; + my $nblocks = int($minsize / BLCKSZ); + for my $blk (0 .. $nblocks - 1) + { + my ($cbuf, $rbuf); + read($cf, $cbuf, BLCKSZ) == BLCKSZ or die "short read: $c_file"; + read($rf, $rbuf, BLCKSZ) == BLCKSZ or die "short read: $r_file"; + $stats->{blocks}++; + next if $cbuf eq $rbuf; + + my ($cmask, $rmask) = ($cbuf, $rbuf); + substr($cmask, 0, MASKED_HEADER_BYTES) = "\0" x MASKED_HEADER_BYTES; + substr($rmask, 0, MASKED_HEADER_BYTES) = "\0" x MASKED_HEADER_BYTES; + if ($cmask eq $rmask) + { + $stats->{lsn_only}++; + next; + } + + $ndiv++; + push @$reports, + describe_block_diff($rel, $relname, $blk, $cbuf, $rbuf, + $cmask, $rmask) + if @$reports < 50; + } + close $cf; + close $rf; + return $ndiv; +} + +my $c_files = collect_relation_files($c_path); +my $r_files = collect_relation_files($r_path); + +my @only_c = grep { !$r_files->{$_} } sort keys %$c_files; +my @only_r = grep { !$c_files->{$_} } sort keys %$r_files; +is(scalar(@only_c) + scalar(@only_r), + 0, 'combined and reference backups contain the same relation files') + or diag "only in combined: @only_c\nonly in reference: @only_r"; + +my @common = sort grep { $r_files->{$_} } keys %$c_files; +my %stats = (blocks => 0, lsn_only => 0); +my @reports; +my $ndivergent = 0; + +foreach my $rel (@common) +{ + (my $base = $rel) =~ s/_(?:vm|init)//; + $base =~ s/\.\d+$//; + my $relname = $relname_by_path{$base} // '?'; + $ndivergent += compare_relation_file("$c_path/$rel", "$r_path/$rel", + $rel, $relname, \@reports, \%stats); +} + +# Guard against the oracle trivially passing because it looked at nothing. +cmp_ok(scalar(@common), '>', 20, + 'compared a meaningful number of relation files (' . scalar(@common) . ')'); +cmp_ok($stats{blocks}, '>', 500, + "compared a meaningful number of blocks ($stats{blocks})"); + +is($ndivergent, 0, + 'combined backup is block-identical to reference full backup ' + . "($stats{blocks} blocks in " . scalar(@common) . ' files)') + or diag join("\n", @reports); +note "$stats{lsn_only} block(s) differed only in masked pd_lsn/pd_checksum" + if $stats{lsn_only}; + +done_testing(); From 52e346d194c8d7e5d09c2e518478378eeb12d40f Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 18 Jul 2026 15:24:53 +0000 Subject: [PATCH 2/2] Address review: framing, normalization structure, committed kill test Review feedback on the physical differential oracle raised three points; this addresses all of them without changing the oracle's verdict logic. 1. Framing. The header no longer implies the final incremental and the reference full backup share a stop LSN -- they cannot, since each emits its own checkpoint/backup-end/switch records. Instead the comment now states the actual equivalence argument: the two backups are comparable because no relation block changes on disk between them, which is audited (pg_waldump over the window spanning both backups must show no block references), not assumed. The audit is conservative-sufficient because with data checksums on, a relation page can only reach disk modified if some WAL record referencing it was inserted first; the hint-bit FPI loophole is closed by having I3's own start checkpoint inside the window. The alternative reviewer-suggested design (restore both paths, replay to a common recovery_target_lsn, then compare) was considered and rejected: WAL replay applies full-page images that would repair exactly the stale blocks this oracle exists to catch, so the zero-replay comparison is the stronger oracle for the summarizer-blind-spot class. 2. Normalization structure. Blanket masking is replaced by a per-block normalizer dispatched on fork and access method ("fork:am" -> "fork:*" -> "*:*"), with the relation's relkind/AM resolved from pg_class and included in divergence reports. The dispatch currently maps everything to the page-header mask (pd_lsn + pd_checksum), which is sufficient under this test's documented preconditions (quiesced audited window, checksums on, heap/btree/sequence/toast only); the comment now spells out per fork and AM what could legitimately diverge under relaxed conditions (heap infomask hint bits and pd_prune_xid, btree LP_DEAD hints and btpo_cycleid) so that AM-aware maskers slot in locally instead of requiring a comparator rewrite. 3. Committed kill test. The previously ad-hoc corruption check is now part of the test: after the main comparison, one byte is flipped (outside the masked header) in copies of a heap main-fork block and a visibility-map block from the combined output, and the comparator must report exactly one divergent block each, attributed to the correct file, block and byte offset. This keeps the oracle honest against future masking or file-collection regressions that would otherwise turn the main comparison into a trivial pass. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01PtxA6B4d47ietk5dGsseuP --- .../t/013_physical_differential.pl | 226 ++++++++++++++---- 1 file changed, 184 insertions(+), 42 deletions(-) diff --git a/src/bin/pg_combinebackup/t/013_physical_differential.pl b/src/bin/pg_combinebackup/t/013_physical_differential.pl index 69c1e8815eced..4ac3ab83a3fe8 100644 --- a/src/bin/pg_combinebackup/t/013_physical_differential.pl +++ b/src/bin/pg_combinebackup/t/013_physical_differential.pl @@ -10,34 +10,45 @@ # class (e.g. visibility map buffers not registered by heap operations) were # only caught operation-by-operation. This test is a generic oracle: it # runs a mixed workload with churn concurrent to a chain of incremental -# backups, then quiesces the cluster and takes -- back to back, with no -# intervening relation writes -- a final incremental backup and a reference -# full backup. The combined result of the backup chain must then be -# physically identical, relation block by relation block, to the reference -# full backup. +# backups, then quiesces the cluster and takes a final incremental backup +# and a reference full backup, back to back. The combined result of the +# backup chain must then be physically identical, relation block by +# relation block, to the reference full backup. # -# Comparison scope and masking: -# - main, _vm and _init relation forks under base/ and global/ are compared -# byte-for-byte per 8 KB block. -# - _fsm forks are skipped: the free space map is not fully WAL-logged by -# design, so it is legitimately allowed to diverge. -# - Within a block, only pd_lsn and pd_checksum (the first 10 bytes of the -# page header) are masked. Everything else, including hint bits in tuple -# infomasks, must match: data checksums are enabled by default, so hint -# bit changes that reach disk are WAL-logged as full-page hints and hence -# visible to the WAL summarizer. Blocks that differ only in the masked -# bytes are counted and reported as a note, not a failure. +# Why the comparison is valid (equivalence argument): the two final backups +# are taken separately and do NOT share a stop LSN -- each emits its own +# checkpoint, backup-end and WAL-switch records. What makes them +# comparable is that no *relation block* changes on disk between them. +# This is not assumed, it is audited: pg_waldump scans the WAL window +# spanning both backups (from just before I3's start checkpoint to just +# after R completes) and the pair is rejected if any record in the window +# carries a block reference. That audit is conservative-sufficient +# because, with data checksums enabled (the default, asserted below), a +# relation page can only reach disk in modified form if some WAL record +# referencing it was inserted first: normal modifications are WAL-logged in +# the same critical section that dirties the buffer, and hint-bit-only +# modifications emit a full-page hint record the first time a page is +# touched after a checkpoint -- and since I3's own start checkpoint is +# inside the audited window, a hint-bit change during the window cannot +# hide behind an earlier FPI without that FPI also being in the window. +# The known exceptions -- FSM forks (not fully WAL-logged) and unlogged +# relation main forks (never backed up) -- are excluded from the +# comparison below. Deliberately, NO WAL replay happens between backup +# and comparison: replaying to a common recovery target would let full-page +# images repair exactly the stale blocks this oracle exists to catch. +# +# Comparison scope and normalization: see $normalizers below. # # Quiescence discipline: after the workload stops, a settle sequence # (VACUUM, catalog/table scans to set any remaining hint bits, CHECKPOINT) -# runs before the final backup pair. The WAL range spanning both final -# backups is then checked with pg_waldump: if any WAL record in that range -# references a relation block, the reference backup might not describe the -# same cluster state as the combined backup, so the pair is retaken (with a -# bounded number of attempts) rather than risking a false verdict. +# runs before the final backup pair, so that in the common case the audited +# window is genuinely empty of block references and the pair is accepted on +# the first attempt; otherwise it is retaken (bounded attempts) rather than +# risking a false verdict. use strict; use warnings FATAL => 'all'; +use File::Copy qw(copy); use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; @@ -370,7 +381,7 @@ sub parse_lsn else { $accepted = 1; - ok(1, "final backup pair captured identical cluster state " + ok(1, "final backup pair captured the same relation-block state " . "(attempt $attempt)"); last; } @@ -380,16 +391,21 @@ sub parse_lsn "quiescent final backup pair obtained within $max_attempts attempts") or die 'cannot obtain a quiescent reference point, aborting'; -# Map relfilenode paths to relation names for better diagnostics. -my %relname_by_path; +# Map relfilenode paths to relation name, relkind and access method, both +# for diagnostics and so the comparator can select AM-aware normalization. +# Relations of other databases (template0/template1) stay unmapped and use +# the default normalizer. +my %relinfo_by_path; { my $rows = $primary->safe_psql('postgres', - q{SELECT pg_relation_filepath(oid), relname - FROM pg_class WHERE pg_relation_filepath(oid) IS NOT NULL}); + q{SELECT pg_relation_filepath(c.oid), c.relname, c.relkind, + coalesce(a.amname, '-') + FROM pg_class c LEFT JOIN pg_am a ON a.oid = c.relam + WHERE pg_relation_filepath(c.oid) IS NOT NULL}); foreach my $row (split /\n/, $rows) { - my ($path, $name) = split /\|/, $row; - $relname_by_path{$path} = $name; + my ($path, $name, $kind, $am) = split /\|/, $row; + $relinfo_by_path{$path} = { name => $name, kind => $kind, am => $am }; } } @@ -435,17 +451,68 @@ sub collect_relation_files return \%files; } +# Per-fork / per-AM block normalization: each entry maps a block image to +# its comparable form by masking bytes that may legitimately differ between +# two representations of the same logical page state. Keys are tried as +# "fork:am", then "fork:*", then "*:*". +# +# What could legitimately differ in general, and why only the page header +# needs masking under THIS test's conditions (quiesced cluster, data +# checksums on, heap + btree + sequence + toast relations only): +# +# - all forks: pd_lsn may differ when the same tuple-level state was +# reached via different WAL histories, and pd_checksum follows any bit +# difference. Under the audited-empty-window discipline the on-disk +# image is literally the same bytes, so even these normally match; they +# are masked as insurance and counted when the mask was needed. +# - heap main fork: infomask hint bits (HEAP_XMIN/XMAX_COMMITTED etc.) and +# pd_prune_xid can differ on a live standby-vs-primary style comparison; +# here checksums=on forces hint-bit changes through WAL (full-page +# hints), so the summarizer sees them and no infomask masking is needed. +# A comparison mode that relaxes those preconditions must add an +# infomask/prune-xid normalizer here (a page-parsing masker, cf. heap's +# rm_mask), not widen the global mask. +# - btree main fork: LP_DEAD line-pointer hints and the btpo_cycleid; same +# reasoning as heap applies under checksums=on. +# - _vm fork: pure bitmap payload behind the page header; no unlogged +# content when the referencing heap operations are WAL-correct (that +# correctness is precisely what this oracle checks). +# - _init forks: written once at creation, no legitimate divergence. +# - _fsm forks: intentionally not WAL-logged; they are excluded from the +# comparison entirely (see collect_relation_files) rather than masked. +sub mask_page_header +{ + my ($buf) = @_; + substr($buf, 0, MASKED_HEADER_BYTES) = "\0" x MASKED_HEADER_BYTES; + return $buf; +} + +my %normalizers = ( + # Everything currently uses the page-header mask only; AM-specific + # entries (e.g. 'main:heap', 'main:btree', 'vm:*') slot in here if a + # future variant of this test relaxes the preconditions above. + '*:*' => \&mask_page_header,); + +sub normalize_block +{ + my ($fork, $relinfo, $buf) = @_; + my $am = defined $relinfo ? $relinfo->{am} : '-'; + my $fn = $normalizers{"$fork:$am"} + // $normalizers{"$fork:*"} // $normalizers{'*:*'}; + return $fn->($buf); +} + # Describe the divergence within one block: the differing byte ranges -# (computed on the masked images, so pd_lsn/pd_checksum noise is excluded) +# (computed on the normalized images, so masked-byte noise is excluded) # with hex dumps of the raw bytes. sub describe_block_diff { - my ($rel, $relname, $blkno, $cbuf, $rbuf, $cmask, $rmask) = @_; + my ($rel, $desc, $blkno, $cbuf, $rbuf, $cmask, $rmask) = @_; my $absblk = $blkno; $absblk += $1 * RELSEG_SIZE if $rel =~ /\.(\d+)$/; - my $msg = "DIVERGENCE: $rel ($relname) block $absblk:"; + my $msg = "DIVERGENCE: $rel ($desc) block $absblk:"; my $xor = $cmask ^ $rmask; my $nranges = 0; while ($xor =~ /[^\0]+/g) @@ -465,9 +532,24 @@ sub describe_block_diff return $msg; } +# Parse a relation file's relative path into (base path, fork). +sub parse_rel_path +{ + my ($rel) = @_; + my $fork = 'main'; + (my $base = $rel) =~ s/_(vm|init)// and $fork = $1; + $base =~ s/\.\d+$//; + return ($base, $fork); +} + sub compare_relation_file { - my ($c_file, $r_file, $rel, $relname, $reports, $stats) = @_; + my ($c_file, $r_file, $rel, $relinfo, $reports, $stats) = @_; + + my (undef, $fork) = parse_rel_path($rel); + my $desc = defined $relinfo + ? "$relinfo->{name} kind=$relinfo->{kind} am=$relinfo->{am} fork=$fork" + : "? fork=$fork"; open my $cf, '<:raw', $c_file or die "open $c_file: $!"; open my $rf, '<:raw', $r_file or die "open $r_file: $!"; @@ -479,7 +561,7 @@ sub compare_relation_file { $ndiv++; push @$reports, - "DIVERGENCE: $rel ($relname): size mismatch: " + "DIVERGENCE: $rel ($desc): size mismatch: " . "combined $c_size vs reference $r_size bytes"; } @@ -493,9 +575,8 @@ sub compare_relation_file $stats->{blocks}++; next if $cbuf eq $rbuf; - my ($cmask, $rmask) = ($cbuf, $rbuf); - substr($cmask, 0, MASKED_HEADER_BYTES) = "\0" x MASKED_HEADER_BYTES; - substr($rmask, 0, MASKED_HEADER_BYTES) = "\0" x MASKED_HEADER_BYTES; + my $cmask = normalize_block($fork, $relinfo, $cbuf); + my $rmask = normalize_block($fork, $relinfo, $rbuf); if ($cmask eq $rmask) { $stats->{lsn_only}++; @@ -504,7 +585,7 @@ sub compare_relation_file $ndiv++; push @$reports, - describe_block_diff($rel, $relname, $blk, $cbuf, $rbuf, + describe_block_diff($rel, $desc, $blk, $cbuf, $rbuf, $cmask, $rmask) if @$reports < 50; } @@ -529,11 +610,9 @@ sub compare_relation_file foreach my $rel (@common) { - (my $base = $rel) =~ s/_(?:vm|init)//; - $base =~ s/\.\d+$//; - my $relname = $relname_by_path{$base} // '?'; + my ($base, undef) = parse_rel_path($rel); $ndivergent += compare_relation_file("$c_path/$rel", "$r_path/$rel", - $rel, $relname, \@reports, \%stats); + $rel, $relinfo_by_path{$base}, \@reports, \%stats); } # Guard against the oracle trivially passing because it looked at nothing. @@ -546,7 +625,70 @@ sub compare_relation_file 'combined backup is block-identical to reference full backup ' . "($stats{blocks} blocks in " . scalar(@common) . ' files)') or diag join("\n", @reports); -note "$stats{lsn_only} block(s) differed only in masked pd_lsn/pd_checksum" +note "$stats{lsn_only} block(s) differed only in normalizer-masked bytes" if $stats{lsn_only}; +# +# Oracle self-check ("kill test"): prove the comparator actually bites. +# Flip one byte -- outside the masked page-header bytes -- in copies of one +# heap main-fork block and one visibility-map block taken from the combined +# output, and require the comparator to report exactly that divergence with +# correct file and block attribution. This guards against the main +# comparison rotting into a trivial pass (e.g. a future masking or +# file-collection bug that silently compares nothing). +# +my %path_by_name = + map { $relinfo_by_path{$_}{name} => $_ } keys %relinfo_by_path; + +my $killdir = $backup_dir . '/killtest'; +mkdir $killdir or die "mkdir $killdir: $!"; + +my @kills = ( + { + rel => $path_by_name{t_heap}, + blk => 3, + off => 4000, + what => 'heap main fork', + }, + { + rel => $path_by_name{t_vm} . '_vm', + blk => 0, + off => 100, + what => 'visibility map fork', + }); + +foreach my $kill (@kills) +{ + my $rel = $kill->{rel}; + ok($c_files->{$rel} && $r_files->{$rel}, + "kill test: $kill->{what} target $rel present in both backups"); + + # Corrupt a copy; the combined output itself must stay pristine. + (my $flat = $rel) =~ s{/}{_}g; + my $copy = "$killdir/$flat"; + copy("$c_path/$rel", $copy) or die "copy $c_path/$rel: $!"; + + my $pos = $kill->{blk} * BLCKSZ + $kill->{off}; + open my $fh, '+<:raw', $copy or die "open $copy: $!"; + seek($fh, $pos, 0) or die "seek $copy: $!"; + read($fh, my $byte, 1) == 1 or die "read $copy: $!"; + seek($fh, $pos, 0) or die "seek $copy: $!"; + print $fh ($byte ^ chr(0xFF)); + close $fh or die "close $copy: $!"; + + my ($base, undef) = parse_rel_path($rel); + my @kreports; + my %kstats = (blocks => 0, lsn_only => 0); + my $kdiv = compare_relation_file($copy, "$r_path/$rel", + $rel, $relinfo_by_path{$base}, \@kreports, \%kstats); + + is($kdiv, 1, + "kill test: $kill->{what}: exactly one divergent block detected"); + like( + join("\n", @kreports), + qr/DIVERGENCE: \Q$rel\E .*block $kill->{blk}:.*bytes $kill->{off}\.\.$kill->{off} differ/s, + "kill test: $kill->{what}: divergence attributed to block " + . "$kill->{blk} offset $kill->{off}"); +} + done_testing();