diff --git a/src/bin/pg_verifybackup/meson.build b/src/bin/pg_verifybackup/meson.build index 0b21db9f1b53c..781cf60b37cde 100644 --- a/src/bin/pg_verifybackup/meson.build +++ b/src/bin/pg_verifybackup/meson.build @@ -38,6 +38,7 @@ tests += { 't/008_untar.pl', 't/009_extract.pl', 't/010_client_untar.pl', + 't/011_manifest_mutation.pl', ], }, } diff --git a/src/bin/pg_verifybackup/t/011_manifest_mutation.pl b/src/bin/pg_verifybackup/t/011_manifest_mutation.pl new file mode 100644 index 0000000000000..95f367d81859e --- /dev/null +++ b/src/bin/pg_verifybackup/t/011_manifest_mutation.pl @@ -0,0 +1,203 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +# Regression coverage for "corrupt file in, clean error out", applied to the +# backup manifest: take a real base backup, then apply deterministic mutations +# to its real backup_manifest (truncation at structural boundaries, byte/bit +# flips of JSON structure at fixed anchor points, and bogus size / checksum +# fields) and confirm that pg_verifybackup rejects every mutant with a nonzero +# exit status and a clean error message, and never dies from a signal (crash / +# assertion failure / core dump). +# +# The existing 005_bad_manifest.pl feeds pg_verifybackup hand-written tiny +# manifests to exercise individual JSON parse errors; 003_corruption.pl mutates +# the backup *data* and appends a byte to the manifest. What was missing, and +# what this test adds, is mutating a real, full-sized manifest at the byte level +# (truncations at fixed offsets and structural bit flips) and asserting the +# tool fails cleanly without crashing. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# Set up an instance and populate it with a little data. +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 1); +$primary->start; +$primary->safe_psql('postgres', <<'EOM'); +CREATE TABLE t1 (a int, b text); +INSERT INTO t1 SELECT g, repeat('x', 100) FROM generate_series(1, 1000) g; +CHECKPOINT; +EOM + +# Take a base backup with a manifest and confirm it verifies cleanly. +my $backup_path = $primary->backup_dir . '/mutation'; +$primary->command_ok( + [ + 'pg_basebackup', + '--pgdata' => $backup_path, + '--no-sync', + '--checkpoint' => 'fast', + ], + 'base backup ok'); +command_ok([ 'pg_verifybackup', $backup_path ], 'intact backup verified'); + +my $manifest_path = "$backup_path/backup_manifest"; +my $orig = slurp_file($manifest_path); +my $orig_len = length($orig); +note("real backup manifest is $orig_len bytes"); + +# --------------------------------------------------------------------------- +# Build the mutation matrix. Every mutation is deterministic: fixed truncation +# lengths and fixed anchor points located structurally in the real manifest, +# with fixed replacement bytes. No randomness. +# +# Note on expected errors: the manifest carries its own SHA256 "Manifest- +# Checksum" covering all preceding bytes, so any byte change that leaves the +# JSON well-formed is caught as a "manifest checksum mismatch", while changes +# that break the JSON structure are caught earlier as a parse error. Both are +# clean, signal-free failures, which is the property under test. +# --------------------------------------------------------------------------- +my @mutations; + +# --- Truncation at structural boundaries ----------------------------------- +push @mutations, + { + name => 'truncate to 0 bytes (empty manifest)', + data => '', + err => qr/backup manifest/, + }, + { + name => 'truncate to 1 byte (lone opening brace)', + data => substr($orig, 0, 1), + err => qr/could not parse backup manifest/, + }, + { + name => 'truncate at midpoint (mid-JSON)', + data => substr($orig, 0, int($orig_len / 2)), + err => qr/pg_verifybackup: error:/, + }, + { + name => 'truncate final newline', + data => substr($orig, 0, $orig_len - 1), + err => qr/pg_verifybackup: error:/, + }; + +# --- Bit / byte flips of JSON structure at fixed anchor points ------------- + +# Flip the top-level opening brace to an opening bracket: well-formed prefix, +# but not the object the parser expects. +push @mutations, + { + name => 'corrupt top-level object start', + data => replace_at($orig, 0, '['), + err => qr/could not parse backup manifest/, + }; + +# Corrupt the version key so it is no longer recognized. +{ + my $off = index($orig, 'PostgreSQL-Backup-Manifest-Version'); + die "could not locate version key" if $off < 0; + push @mutations, + { + name => 'corrupt version key name', + data => replace_at($orig, $off, 'Q'), + err => qr/could not parse backup manifest/, + }; +} + +# Corrupt the structural brace that opens the "Files" array's first object. +{ + my $off = index($orig, '{ "Path": '); + if ($off >= 0) + { + push @mutations, + { + name => 'corrupt file object start', + data => replace_at($orig, $off, 'Z'), + err => qr/could not parse backup manifest/, + }; + } +} + +# --- Bogus size / checksum fields (valid JSON -> checksum mismatch) --------- + +# Change one digit of a file's Size field. +if ($orig =~ /"Size": (\d)/) +{ + my $off = $-[1]; # offset of the first Size digit + my $newdigit = (substr($orig, $off, 1) eq '9') ? '1' : '9'; + push @mutations, + { + name => 'bogus file size', + data => replace_at($orig, $off, $newdigit), + err => qr/manifest checksum mismatch/, + }; +} + +# Flip one hex character of a file's Checksum value. +if ($orig =~ /"Checksum": "([0-9a-f])/) +{ + my $off = $-[1]; + my $newc = (substr($orig, $off, 1) eq 'a') ? 'b' : 'a'; + push @mutations, + { + name => 'bogus file checksum', + data => replace_at($orig, $off, $newc), + err => qr/manifest checksum mismatch/, + }; +} + +# Flip one hex character inside the manifest's own SHA256 checksum. +if ($orig =~ /"Manifest-Checksum": "([0-9a-f])/) +{ + my $off = $-[1]; + my $newc = (substr($orig, $off, 1) eq 'a') ? 'b' : 'a'; + push @mutations, + { + name => 'corrupt manifest checksum value', + data => replace_at($orig, $off, $newc), + err => qr/manifest checksum mismatch/, + }; +} + +# --------------------------------------------------------------------------- +# Run pg_verifybackup against each mutated manifest and require a clean +# failure. We restore the pristine manifest into memory for each iteration, so +# mutations do not compound. +# --------------------------------------------------------------------------- +for my $m (@mutations) +{ + # Overwrite the backup's manifest with the mutated bytes. + open(my $fh, '>:raw', $manifest_path) || die "open $manifest_path: $!"; + print $fh $m->{data}; + close($fh); + + my ($stdout, $stderr) = ('', ''); + print("# Running: pg_verifybackup $backup_path\n"); + IPC::Run::run([ 'pg_verifybackup', $backup_path ], + '>' => \$stdout, '2>' => \$stderr); + my $child = $?; + + # The cardinal requirement: never a crash / signal / core dump. + my $signal = $child & 127; + is($signal, 0, "no crash on: $m->{name}") + or diag("pg_verifybackup died from signal $signal on '$m->{name}'; " + . "stderr: $stderr"); + + # And it must reject the manifest with a nonzero exit and a clean message. + my $exit_code = $child >> 8; + isnt($exit_code, 0, "nonzero exit on: $m->{name}"); + like($stderr, $m->{err}, "clean error message on: $m->{name}"); +} + +done_testing(); + +# Return a copy of $str with the single byte at $off replaced by $repl. +sub replace_at +{ + my ($str, $off, $repl) = @_; + substr($str, $off, 1) = $repl; + return $str; +} diff --git a/src/bin/pg_walsummary/meson.build b/src/bin/pg_walsummary/meson.build index d012275402bba..7b10082a2db7d 100644 --- a/src/bin/pg_walsummary/meson.build +++ b/src/bin/pg_walsummary/meson.build @@ -25,6 +25,7 @@ tests += { 'tests': [ 't/001_basic.pl', 't/002_blocks.pl', + 't/003_corrupt.pl', ], } } diff --git a/src/bin/pg_walsummary/t/003_corrupt.pl b/src/bin/pg_walsummary/t/003_corrupt.pl new file mode 100644 index 0000000000000..06fca051bdfe2 --- /dev/null +++ b/src/bin/pg_walsummary/t/003_corrupt.pl @@ -0,0 +1,315 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +# Regression coverage for "corrupt file in, clean error out": generate a real +# WAL summary file, then apply deterministic mutations to it (truncation at +# structural boundaries, bit flips in header / fork-number / chunk-size fields, +# and oversized length/count values) and confirm that pg_walsummary rejects +# every mutant with a nonzero exit status and a clean error message, and never +# dies from a signal (crash / assertion failure / core dump). +# +# This exercises the on-disk validation in src/common/blkreftable.c, including +# the fork-number and chunk-size sanity checks added by commit ee654419d5. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# The on-disk block reference table format (see src/common/blkreftable.c): +# +# uint32 magic (BLOCKREFTABLE_MAGIC == 0x652b137b) +# repeated serialized entries, each: +# uint32 rlocator.spcOid +# uint32 rlocator.dbOid +# uint32 rlocator.relNumber +# int32 forknum +# uint32 limit_block +# uint32 nchunks +# uint16 chunk_usage[nchunks] (chunk length array) +# uint16 chunk_data[...] (per chunk: chunk_usage[j] entries) +# an all-zero 24-byte serialized entry acts as the terminator +# uint32 crc +# +# All integers are stored in native byte order; the test reads and writes the +# file on the same machine that produced it, so native byte order is correct. +use constant { + BLOCKREFTABLE_MAGIC => 0x652b137b, + ENTRY_LEN => 24, # sizeof(BlockRefTableSerializedEntry) + OFF_FORKNUM => 12, # within a serialized entry + OFF_LIMIT_BLOCK => 16, + OFF_NCHUNKS => 20, + MAX_FORKNUM => 3, # INIT_FORKNUM + MAX_ENTRIES_PER_CHUNK => 4096, # BLOCKS_PER_CHUNK / BLOCKS_PER_ENTRY +}; + +# Set up a database instance with WAL summarization enabled. +my $node = PostgreSQL::Test::Cluster->new('node'); +$node->init(has_archiving => 1, allows_streaming => 1); +$node->append_conf('postgresql.conf', 'summarize_wal = on'); +$node->start; + +# Generate WAL that touches many blocks so that the resulting summary file +# contains at least one relation fork with real chunk data (not just a limit +# block), which lets us exercise the chunk-size validation path. +$node->safe_psql('postgres', <<'EOM'); +CREATE TABLE mytable (a int, b text); +INSERT INTO mytable +SELECT g, repeat('x', 200) FROM generate_series(1, 5000) g; +UPDATE mytable SET b = repeat('y', 200) WHERE a % 3 = 0; +VACUUM FREEZE mytable; +CHECKPOINT; +EOM + +my $base_lsn = $node->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn()'); +$node->safe_psql('postgres', 'CHECKPOINT'); + +# Wait until a summary covering our activity has been written to disk. +$node->poll_query_until('postgres', <= '$base_lsn' +) +EOM + or die "timed out waiting for WAL summarization to catch up"; + +# Stop the server so the set of summary files is stable while we mutate copies. +$node->stop; + +# Pick the largest summary file on disk; it is the most likely to contain a +# relation fork with chunk data. +my $summary_dir = $node->data_dir . '/pg_wal/summaries'; +opendir(my $dh, $summary_dir) || die "opendir $summary_dir: $!"; +my @summaries = sort grep { /\.summary$/ } readdir($dh); +closedir($dh); +die "no WAL summary files were generated" unless @summaries; + +my ($orig_file, $orig_size) = ('', -1); +for my $f (@summaries) +{ + my $path = "$summary_dir/$f"; + my $sz = -s $path; + ($orig_file, $orig_size) = ($path, $sz) if $sz > $orig_size; +} +note("using WAL summary file $orig_file ($orig_size bytes)"); + +# Read the pristine summary file into memory. +my $orig = slurp_file($orig_file); +my $orig_len = length($orig); + +# Sanity check: pg_walsummary reads the unmodified file cleanly. +{ + my $good = "$PostgreSQL::Test::Utils::tmp_check/good.summary"; + write_summary($good, $orig); + my ($ret, $stdout, $stderr) = run_walsummary($good); + is($ret, 0, "pristine summary file parses successfully"); +} + +# Parse the entry structure so we can target specific fields precisely. +my $magic = unpack('L', substr($orig, 0, 4)); +is($magic, BLOCKREFTABLE_MAGIC, "summary file has expected magic number"); + +my $first_entry_off = 4; +my $first_chunk_size_off; # offset of first chunk_usage[] value, if any +my $mid_chunk_off; # a byte offset that lands inside chunk data + +{ + my $pos = 4; + while ($pos + ENTRY_LEN <= $orig_len) + { + my $hdr = substr($orig, $pos, ENTRY_LEN); + last if $hdr eq ("\0" x ENTRY_LEN); # terminator + + my $nchunks = unpack('L', substr($hdr, OFF_NCHUNKS, 4)); + my $entry_start = $pos; + $pos += ENTRY_LEN; + + if ($nchunks > 0 && !defined $first_chunk_size_off) + { + $first_chunk_size_off = $pos; + } + + # Read chunk_usage[] and advance past the chunk data. + my @usage = unpack("S$nchunks", substr($orig, $pos, 2 * $nchunks)); + $pos += 2 * $nchunks; + for my $u (@usage) + { + # The first non-empty chunk gives us a genuine "mid-chunk" offset. + $mid_chunk_off = $pos + 1 + if $u > 0 && !defined $mid_chunk_off; + $pos += 2 * $u; + } + } +} + +note( + defined $first_chunk_size_off + ? "first chunk_usage[] value at offset $first_chunk_size_off" + : "no relation fork with chunk data found"); + +# --------------------------------------------------------------------------- +# Build the mutation matrix. Each mutation is deterministic: fixed offsets +# (computed from the real file structure) and fixed values, no randomness. +# --------------------------------------------------------------------------- +my @mutations; + +# --- Truncation at structural boundaries ----------------------------------- +push @mutations, + { + name => 'truncate to 0 bytes (empty file)', + data => '', + err => qr/ends unexpectedly/, + }, + { + name => 'truncate mid-magic (2 bytes)', + data => substr($orig, 0, 2), + err => qr/ends unexpectedly/, + }, + { + name => 'truncate after magic (no entries)', + data => substr($orig, 0, 4), + err => qr/ends unexpectedly/, + }, + { + name => 'truncate mid-header (partial first entry)', + data => substr($orig, 0, 4 + 12), + err => qr/ends unexpectedly/, + }, + { + name => 'truncate mid-CRC (drop trailing bytes)', + data => substr($orig, 0, $orig_len - 3), + err => qr/ends unexpectedly/, + }; + +push @mutations, + { + name => 'truncate mid-chunk (inside chunk data)', + data => substr($orig, 0, $mid_chunk_off), + err => qr/ends unexpectedly/, + } + if defined $mid_chunk_off; + +# --- Bit flips in header / fork-number / chunk-size fields ----------------- +push @mutations, + { + name => 'corrupt magic number', + data => mutate_u32($orig, 0, BLOCKREFTABLE_MAGIC ^ 0x01), + err => qr/wrong magic number/, + }, + { + name => 'invalid fork number (too large)', + data => mutate_i32($orig, $first_entry_off + OFF_FORKNUM, 16), + err => qr/invalid fork number/, + }, + { + name => 'invalid fork number (negative)', + data => mutate_i32($orig, $first_entry_off + OFF_FORKNUM, -5), + err => qr/invalid fork number/, + }, + { + name => 'corrupt limit_block (CRC mismatch)', + data => mutate_u32( + $orig, $first_entry_off + OFF_LIMIT_BLOCK, + unpack('L', substr($orig, $first_entry_off + OFF_LIMIT_BLOCK, 4)) ^ + 0xFFFF), + err => qr/wrong checksum/, + }; + +push @mutations, + { + name => 'oversized chunk size value', + data => mutate_u16($orig, $first_chunk_size_off, 0xFFFF), + err => qr/chunk \d+ has invalid size/, + } + if defined $first_chunk_size_off; + +# --- Oversized length / count values --------------------------------------- +push @mutations, + { + name => 'oversized nchunks (overflows allocation limit)', + data => mutate_u32($orig, $first_entry_off + OFF_NCHUNKS, 0xFFFFFFFF), + err => qr/oversized chunk size array/, + }, + { + # Large but below the allocation cap: passes the oversize check, then the + # read of the chunk-size array runs off the end of the (short) file. + name => 'large nchunks (chunk array runs off end of file)', + data => mutate_u32($orig, $first_entry_off + OFF_NCHUNKS, 0x00100000), + err => qr/ends unexpectedly/, + }; + +# --------------------------------------------------------------------------- +# Run pg_walsummary on each mutant and require a clean failure. +# --------------------------------------------------------------------------- +my $i = 0; +for my $m (@mutations) +{ + my $path = sprintf("%s/mutant_%02d.summary", + $PostgreSQL::Test::Utils::tmp_check, $i++); + write_summary($path, $m->{data}); + + my ($ret, $stdout, $stderr) = run_walsummary($path); + + # The cardinal requirement: never a crash / signal / core dump. + my $signal = $ret & 127; + is($signal, 0, "no crash on: $m->{name}") + or diag("pg_walsummary died from signal $signal on '$m->{name}'; " + . "stderr: $stderr"); + + # And it must reject the file with a nonzero exit and a clean message. + my $exit_code = $ret >> 8; + isnt($exit_code, 0, "nonzero exit on: $m->{name}"); + like($stderr, $m->{err}, "clean error message on: $m->{name}"); +} + +done_testing(); + +# --------------------------------------------------------------------------- +# Helpers. +# --------------------------------------------------------------------------- + +# Overwrite 4 bytes at $off with a native-order uint32. +sub mutate_u32 +{ + my ($data, $off, $val) = @_; + substr($data, $off, 4) = pack('L', $val); + return $data; +} + +# Overwrite 4 bytes at $off with a native-order int32. +sub mutate_i32 +{ + my ($data, $off, $val) = @_; + substr($data, $off, 4) = pack('l', $val); + return $data; +} + +# Overwrite 2 bytes at $off with a native-order uint16. +sub mutate_u16 +{ + my ($data, $off, $val) = @_; + substr($data, $off, 2) = pack('S', $val); + return $data; +} + +sub write_summary +{ + my ($path, $data) = @_; + open(my $fh, '>:raw', $path) || die "open $path: $!"; + print $fh $data; + close($fh); + return; +} + +# Run pg_walsummary on a file, returning ($child_error, $stdout, $stderr). +# $child_error is Perl's $? so callers can inspect both the signal (low 7 bits) +# and the exit code (>> 8) -- command_fails_like() would hide a signal death. +sub run_walsummary +{ + my ($path) = @_; + my ($stdout, $stderr) = ('', ''); + print("# Running: pg_walsummary -i $path\n"); + IPC::Run::run([ 'pg_walsummary', '-i', $path ], + '>' => \$stdout, '2>' => \$stderr); + return ($?, $stdout, $stderr); +}