diff --git a/doc/src/sgml/custom-rmgr.sgml b/doc/src/sgml/custom-rmgr.sgml index 3032b2dc0d2df..5feda955c7f19 100644 --- a/doc/src/sgml/custom-rmgr.sgml +++ b/doc/src/sgml/custom-rmgr.sgml @@ -35,7 +35,10 @@ * record, if available (e.g. the last block). * * rm_mask takes as input a page modified by the resource manager and masks - * out bits that shouldn't be flagged by wal_consistency_checking. + * out bits that shouldn't be flagged by wal_consistency_checking. The fork + * number of the block is passed as well, since resource managers may + * register blocks of different forks (e.g. heap records can reference + * visibility map pages) which require different masking rules. * * RmgrTable[] is indexed by RmgrId values (see rmgrlist.h). If rm_name is * NULL, the corresponding RmgrTable entry is considered invalid. @@ -48,7 +51,8 @@ typedef struct RmgrData const char *(*rm_identify) (uint8 info); void (*rm_startup) (void); void (*rm_cleanup) (void); - void (*rm_mask) (char *pagedata, BlockNumber blkno); + void (*rm_mask) (char *pagedata, BlockNumber blkno, + ForkNumber forknum); void (*rm_decode) (struct LogicalDecodingContext *ctx, struct XLogRecordBuffer *buf); } RmgrData; diff --git a/src/backend/access/brin/brin_xlog.c b/src/backend/access/brin/brin_xlog.c index a07f1d11ddeeb..4aef48298b28d 100644 --- a/src/backend/access/brin/brin_xlog.c +++ b/src/backend/access/brin/brin_xlog.c @@ -339,7 +339,7 @@ brin_redo(XLogReaderState *record) * Mask a BRIN page before doing consistency checks. */ void -brin_mask(char *pagedata, BlockNumber blkno) +brin_mask(char *pagedata, BlockNumber blkno, ForkNumber forknum) { Page page = (Page) pagedata; PageHeader pagehdr = (PageHeader) page; diff --git a/src/backend/access/common/bufmask.c b/src/backend/access/common/bufmask.c index 5f63d04c9cbf6..312a575fac74a 100644 --- a/src/backend/access/common/bufmask.c +++ b/src/backend/access/common/bufmask.c @@ -55,8 +55,18 @@ mask_page_hint_bits(Page page) PageClearHasFreeLinePointers(page); /* - * PD_ALL_VISIBLE is masked during WAL consistency checking. XXX: It is - * worth investigating if we could stop doing this. + * PD_ALL_VISIBLE is masked during WAL consistency checking. + * + * All changes to PD_ALL_VISIBLE are WAL-logged nowadays, so in principle + * primary and standby should agree on it. However, when the only change + * to a heap page is setting PD_ALL_VISIBLE (and checksums/wal_log_hints + * are disabled), log_heap_prune_and_freeze() intentionally skips the FPI + * and does not stamp the heap page with the record's LSN. The flag's + * on-disk state is therefore not strictly LSN-ordered, and around + * recovery restarts the primary's page image and the standby's replayed + * page can transiently disagree on it without indicating corruption. So + * we conservatively continue to mask it. XXX: It is worth investigating + * if we could stop doing this. */ PageClearAllVisible(page); } diff --git a/src/backend/access/gin/ginxlog.c b/src/backend/access/gin/ginxlog.c index b1fee3c281f33..48bbee5a27b9e 100644 --- a/src/backend/access/gin/ginxlog.c +++ b/src/backend/access/gin/ginxlog.c @@ -788,7 +788,7 @@ gin_xlog_cleanup(void) * Mask a GIN page before running consistency checks on it. */ void -gin_mask(char *pagedata, BlockNumber blkno) +gin_mask(char *pagedata, BlockNumber blkno, ForkNumber forknum) { Page page = (Page) pagedata; PageHeader pagehdr = (PageHeader) page; diff --git a/src/backend/access/gist/gistxlog.c b/src/backend/access/gist/gistxlog.c index ae538dc81ca1e..b8f70a6ea6c13 100644 --- a/src/backend/access/gist/gistxlog.c +++ b/src/backend/access/gist/gistxlog.c @@ -445,7 +445,7 @@ gist_xlog_cleanup(void) * Mask a Gist page before running consistency checks on it. */ void -gist_mask(char *pagedata, BlockNumber blkno) +gist_mask(char *pagedata, BlockNumber blkno, ForkNumber forknum) { Page page = (Page) pagedata; diff --git a/src/backend/access/hash/hash_xlog.c b/src/backend/access/hash/hash_xlog.c index e9a2b9aa9a723..2b596c4837bc2 100644 --- a/src/backend/access/hash/hash_xlog.c +++ b/src/backend/access/hash/hash_xlog.c @@ -1091,7 +1091,7 @@ hash_redo(XLogReaderState *record) * Mask a hash page before performing consistency checks on it. */ void -hash_mask(char *pagedata, BlockNumber blkno) +hash_mask(char *pagedata, BlockNumber blkno, ForkNumber forknum) { Page page = (Page) pagedata; HashPageOpaque opaque; diff --git a/src/backend/access/heap/heapam_xlog.c b/src/backend/access/heap/heapam_xlog.c index 7a7bc7ea740aa..a6e17a324adec 100644 --- a/src/backend/access/heap/heapam_xlog.c +++ b/src/backend/access/heap/heapam_xlog.c @@ -1346,11 +1346,30 @@ heap2_redo(XLogReaderState *record) * Mask a heap page before performing consistency checks on it. */ void -heap_mask(char *pagedata, BlockNumber blkno) +heap_mask(char *pagedata, BlockNumber blkno, ForkNumber forknum) { Page page = (Page) pagedata; OffsetNumber off; + /* + * Heap records can reference visibility map pages in addition to heap + * pages. A visibility map page stores the all-visible/all-frozen bitmap + * in the space between pd_lower and pd_upper, so the heap masking rules + * below must not be applied to it: in particular, mask_unused_space() + * would wipe out the entire bitmap, making the consistency check on VM + * pages vacuous. VM bits are only ever set or cleared while holding + * exclusive content lock on the VM page with the changes WAL-logged, so + * only the LSN and checksum can legitimately differ between the primary + * and standby versions of a VM page. + */ + if (forknum == VISIBILITYMAP_FORKNUM) + { + mask_page_lsn_and_checksum(page); + return; + } + + Assert(forknum == MAIN_FORKNUM); + mask_page_lsn_and_checksum(page); mask_page_hint_bits(page); diff --git a/src/backend/access/nbtree/nbtxlog.c b/src/backend/access/nbtree/nbtxlog.c index dff7d286fc834..19973db2743d0 100644 --- a/src/backend/access/nbtree/nbtxlog.c +++ b/src/backend/access/nbtree/nbtxlog.c @@ -1078,7 +1078,7 @@ btree_xlog_cleanup(void) * Mask a btree page before performing consistency checks on it. */ void -btree_mask(char *pagedata, BlockNumber blkno) +btree_mask(char *pagedata, BlockNumber blkno, ForkNumber forknum) { Page page = (Page) pagedata; BTPageOpaque maskopaq; diff --git a/src/backend/access/spgist/spgxlog.c b/src/backend/access/spgist/spgxlog.c index 55e8066a77b35..c3ded7d0f86d1 100644 --- a/src/backend/access/spgist/spgxlog.c +++ b/src/backend/access/spgist/spgxlog.c @@ -983,7 +983,7 @@ spg_xlog_cleanup(void) * Mask a SpGist page before performing consistency checks on it. */ void -spg_mask(char *pagedata, BlockNumber blkno) +spg_mask(char *pagedata, BlockNumber blkno, ForkNumber forknum) { Page page = (Page) pagedata; PageHeader pagehdr = (PageHeader) page; diff --git a/src/backend/access/transam/generic_xlog.c b/src/backend/access/transam/generic_xlog.c index 7f82186d0d6ee..794eae2bfdd31 100644 --- a/src/backend/access/transam/generic_xlog.c +++ b/src/backend/access/transam/generic_xlog.c @@ -536,7 +536,7 @@ generic_redo(XLogReaderState *record) * Mask a generic page before performing consistency checks on it. */ void -generic_mask(char *page, BlockNumber blkno) +generic_mask(char *page, BlockNumber blkno, ForkNumber forknum) { mask_page_lsn_and_checksum(page); diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c index a9ebac2d0ef31..b7d43372c18d4 100644 --- a/src/backend/access/transam/xlogrecovery.c +++ b/src/backend/access/transam/xlogrecovery.c @@ -2523,8 +2523,8 @@ verifyBackupPageConsistency(XLogReaderState *record) */ if (rmgr.rm_mask != NULL) { - rmgr.rm_mask(replay_image_masked, blkno); - rmgr.rm_mask(primary_image_masked, blkno); + rmgr.rm_mask(replay_image_masked, blkno, forknum); + rmgr.rm_mask(primary_image_masked, blkno, forknum); } /* Time to compare the primary and replay images. */ diff --git a/src/backend/commands/sequence_xlog.c b/src/backend/commands/sequence_xlog.c index fcb3230cf3be0..b48b3c74871b4 100644 --- a/src/backend/commands/sequence_xlog.c +++ b/src/backend/commands/sequence_xlog.c @@ -73,7 +73,7 @@ seq_redo(XLogReaderState *record) * Mask a Sequence page before performing consistency checks on it. */ void -seq_mask(char *page, BlockNumber blkno) +seq_mask(char *page, BlockNumber blkno, ForkNumber forknum) { mask_page_lsn_and_checksum(page); diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 3908529872a31..eadf3470f48cd 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -5454,9 +5454,21 @@ RelationCopyStorageUsingBuffer(RelFileLocator srclocator, memcpy(dstPage, srcPage, BLCKSZ); MarkBufferDirty(dstBuf); - /* WAL-log the copied page. */ + /* + * WAL-log the copied page. + * + * We must not claim the standard page layout for visibility map or + * FSM pages: they keep their whole payload between pd_lower and + * pd_upper, so treating them as standard would make the full-page + * image omit that region as a "hole", and replay would zero it, + * silently emptying the fork's contents on a standby or after crash + * recovery. Main and init fork pages use the standard layout, so + * hole compression is safe for them. + */ if (use_wal) - log_newpage_buffer(dstBuf, true); + log_newpage_buffer(dstBuf, + forkNum != VISIBILITYMAP_FORKNUM && + forkNum != FSM_FORKNUM); END_CRIT_SECTION(); diff --git a/src/include/access/brin_xlog.h b/src/include/access/brin_xlog.h index 1c60dc8adb9c5..00b205cb8d281 100644 --- a/src/include/access/brin_xlog.h +++ b/src/include/access/brin_xlog.h @@ -146,6 +146,6 @@ typedef struct xl_brin_desummarize extern void brin_redo(XLogReaderState *record); extern void brin_desc(StringInfo buf, XLogReaderState *record); extern const char *brin_identify(uint8 info); -extern void brin_mask(char *pagedata, BlockNumber blkno); +extern void brin_mask(char *pagedata, BlockNumber blkno, ForkNumber forknum); #endif /* BRIN_XLOG_H */ diff --git a/src/include/access/generic_xlog.h b/src/include/access/generic_xlog.h index 93d93f73ddcb0..4e8c2bd5ef704 100644 --- a/src/include/access/generic_xlog.h +++ b/src/include/access/generic_xlog.h @@ -40,6 +40,6 @@ extern void GenericXLogAbort(GenericXLogState *state); extern void generic_redo(XLogReaderState *record); extern const char *generic_identify(uint8 info); extern void generic_desc(StringInfo buf, XLogReaderState *record); -extern void generic_mask(char *page, BlockNumber blkno); +extern void generic_mask(char *page, BlockNumber blkno, ForkNumber forknum); #endif /* GENERIC_XLOG_H */ diff --git a/src/include/access/ginxlog.h b/src/include/access/ginxlog.h index 74fdf9b3d7ed4..50a45c6c98653 100644 --- a/src/include/access/ginxlog.h +++ b/src/include/access/ginxlog.h @@ -214,6 +214,6 @@ extern void gin_desc(StringInfo buf, XLogReaderState *record); extern const char *gin_identify(uint8 info); extern void gin_xlog_startup(void); extern void gin_xlog_cleanup(void); -extern void gin_mask(char *pagedata, BlockNumber blkno); +extern void gin_mask(char *pagedata, BlockNumber blkno, ForkNumber forknum); #endif /* GINXLOG_H */ diff --git a/src/include/access/gistxlog.h b/src/include/access/gistxlog.h index 1c2cf6e813afe..05c52459eee18 100644 --- a/src/include/access/gistxlog.h +++ b/src/include/access/gistxlog.h @@ -112,6 +112,6 @@ extern void gist_desc(StringInfo buf, XLogReaderState *record); extern const char *gist_identify(uint8 info); extern void gist_xlog_startup(void); extern void gist_xlog_cleanup(void); -extern void gist_mask(char *pagedata, BlockNumber blkno); +extern void gist_mask(char *pagedata, BlockNumber blkno, ForkNumber forknum); #endif diff --git a/src/include/access/hash_xlog.h b/src/include/access/hash_xlog.h index 149aed69dc5c9..46ad0a5cc019e 100644 --- a/src/include/access/hash_xlog.h +++ b/src/include/access/hash_xlog.h @@ -265,6 +265,6 @@ typedef struct xl_hash_vacuum_one_page extern void hash_redo(XLogReaderState *record); extern void hash_desc(StringInfo buf, XLogReaderState *record); extern const char *hash_identify(uint8 info); -extern void hash_mask(char *pagedata, BlockNumber blkno); +extern void hash_mask(char *pagedata, BlockNumber blkno, ForkNumber forknum); #endif /* HASH_XLOG_H */ diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h index 3f79c389a90a7..a1ea4e1ba5f24 100644 --- a/src/include/access/heapam_xlog.h +++ b/src/include/access/heapam_xlog.h @@ -512,7 +512,7 @@ extern void HeapTupleHeaderAdvanceConflictHorizon(HeapTupleHeader tuple, extern void heap_redo(XLogReaderState *record); extern void heap_desc(StringInfo buf, XLogReaderState *record); extern const char *heap_identify(uint8 info); -extern void heap_mask(char *pagedata, BlockNumber blkno); +extern void heap_mask(char *pagedata, BlockNumber blkno, ForkNumber forknum); extern void heap2_redo(XLogReaderState *record); extern void heap2_desc(StringInfo buf, XLogReaderState *record); extern const char *heap2_identify(uint8 info); diff --git a/src/include/access/nbtxlog.h b/src/include/access/nbtxlog.h index 3a78ec27fe8ce..5761662e0206f 100644 --- a/src/include/access/nbtxlog.h +++ b/src/include/access/nbtxlog.h @@ -356,7 +356,7 @@ typedef struct xl_btree_newroot extern void btree_redo(XLogReaderState *record); extern void btree_xlog_startup(void); extern void btree_xlog_cleanup(void); -extern void btree_mask(char *pagedata, BlockNumber blkno); +extern void btree_mask(char *pagedata, BlockNumber blkno, ForkNumber forknum); /* * prototypes for functions in nbtdesc.c diff --git a/src/include/access/spgxlog.h b/src/include/access/spgxlog.h index c257408be669e..9fafdcd3810dd 100644 --- a/src/include/access/spgxlog.h +++ b/src/include/access/spgxlog.h @@ -254,6 +254,6 @@ extern void spg_desc(StringInfo buf, XLogReaderState *record); extern const char *spg_identify(uint8 info); extern void spg_xlog_startup(void); extern void spg_xlog_cleanup(void); -extern void spg_mask(char *pagedata, BlockNumber blkno); +extern void spg_mask(char *pagedata, BlockNumber blkno, ForkNumber forknum); #endif /* SPGXLOG_H */ diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index be718993401bc..593e836d4b897 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -343,7 +343,10 @@ struct XLogRecordBuffer; * record, if available (e.g. the last block). * * rm_mask takes as input a page modified by the resource manager and masks - * out bits that shouldn't be flagged by wal_consistency_checking. + * out bits that shouldn't be flagged by wal_consistency_checking. The fork + * number of the block is passed as well, since resource managers may + * register blocks of different forks (e.g. heap records can reference + * visibility map pages) which require different masking rules. * * RmgrTable[] is indexed by RmgrId values (see rmgrlist.h). If rm_name is * NULL, the corresponding RmgrTable entry is considered invalid. @@ -356,7 +359,8 @@ typedef struct RmgrData const char *(*rm_identify) (uint8 info); void (*rm_startup) (void); void (*rm_cleanup) (void); - void (*rm_mask) (char *pagedata, BlockNumber blkno); + void (*rm_mask) (char *pagedata, BlockNumber blkno, + ForkNumber forknum); void (*rm_decode) (struct LogicalDecodingContext *ctx, struct XLogRecordBuffer *buf); } RmgrData; diff --git a/src/include/commands/sequence_xlog.h b/src/include/commands/sequence_xlog.h index b0495f41b43d7..397996abd5bf6 100644 --- a/src/include/commands/sequence_xlog.h +++ b/src/include/commands/sequence_xlog.h @@ -40,6 +40,6 @@ typedef struct xl_seq_rec extern void seq_redo(XLogReaderState *record); extern void seq_desc(StringInfo buf, XLogReaderState *record); extern const char *seq_identify(uint8 info); -extern void seq_mask(char *page, BlockNumber blkno); +extern void seq_mask(char *page, BlockNumber blkno, ForkNumber forknum); #endif /* SEQUENCE_XLOG_H */ diff --git a/src/test/recovery/Makefile b/src/test/recovery/Makefile index d41aaaf8ae13d..a07395ac011f9 100644 --- a/src/test/recovery/Makefile +++ b/src/test/recovery/Makefile @@ -9,8 +9,10 @@ # #------------------------------------------------------------------------- -EXTRA_INSTALL=contrib/pg_prewarm \ +EXTRA_INSTALL=contrib/pg_freespacemap \ + contrib/pg_prewarm \ contrib/pg_stat_statements \ + contrib/pg_visibility \ contrib/test_decoding \ src/test/modules/injection_points diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build index ad0d85f41897e..866b576b11c6f 100644 --- a/src/test/recovery/meson.build +++ b/src/test/recovery/meson.build @@ -63,6 +63,7 @@ tests += { 't/052_checkpoint_segment_missing.pl', 't/053_standby_login_event_trigger.pl', 't/054_unlogged_sequence_promotion.pl', + 't/055_create_database_walog.pl', ], }, } diff --git a/src/test/recovery/t/055_create_database_walog.pl b/src/test/recovery/t/055_create_database_walog.pl new file mode 100644 index 0000000000000..9d0671c8f2466 --- /dev/null +++ b/src/test/recovery/t/055_create_database_walog.pl @@ -0,0 +1,86 @@ + +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Test that CREATE DATABASE ... STRATEGY=WAL_LOG WAL-logs the auxiliary +# relation forks in a way that recovery can reproduce them. +# +# The WAL_LOG strategy copies the template database block by block and +# WAL-logs every page. Visibility map and free space map pages store +# their entire payload between pd_lower and pd_upper, so they must not be +# logged as standard-layout pages: the full-page-image "hole" optimization +# would elide the payload and replay would zero it, silently emptying the +# copied database's VM and FSM on a standby (or after crash recovery). +# Verify that a standby ends up with the same VM and FSM contents as the +# primary for a database copied with STRATEGY=WAL_LOG. + +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node_primary = PostgreSQL::Test::Cluster->new('primary'); +$node_primary->init(allows_streaming => 1); +# Keep the VM and FSM of the tables involved stable during the test. +$node_primary->append_conf('postgresql.conf', 'autovacuum = off'); +$node_primary->start; + +# Create a template database whose table has both visibility map bits set +# and free space recorded in the FSM. +$node_primary->safe_psql('postgres', 'CREATE DATABASE src_db'); +$node_primary->safe_psql( + 'src_db', qq[ + CREATE EXTENSION pg_visibility; + CREATE EXTENSION pg_freespacemap; + CREATE TABLE tab_copied (a int, filler text); + INSERT INTO tab_copied + SELECT g, repeat('x', 100) FROM generate_series(1, 5000) g; + DELETE FROM tab_copied WHERE a % 5 = 0; + ]); +# Set all-visible/all-frozen bits in the VM and record the space freed by +# the deletions in the FSM. +$node_primary->safe_psql('src_db', 'VACUUM (FREEZE) tab_copied'); + +# Create a standby. +my $backup_name = 'my_backup'; +$node_primary->backup($backup_name); +my $node_standby = PostgreSQL::Test::Cluster->new('standby'); +$node_standby->init_from_backup($node_primary, $backup_name, + has_streaming => 1); +$node_standby->start; +$node_primary->wait_for_catchup($node_standby, 'replay'); + +# Copy the template database using the WAL_LOG strategy and wait for the +# standby to replay the copy. +$node_primary->safe_psql('postgres', + 'CREATE DATABASE dst_db TEMPLATE src_db STRATEGY WAL_LOG'); +$node_primary->wait_for_catchup($node_standby, 'replay'); + +my $vm_query = + "SELECT all_visible, all_frozen FROM pg_visibility_map_summary('tab_copied')"; +my $fsm_query = + "SELECT count(*), COALESCE(sum(avail), 0) FROM pg_freespace('tab_copied')"; + +my $primary_vm = $node_primary->safe_psql('dst_db', $vm_query); +my $primary_fsm = $node_primary->safe_psql('dst_db', $fsm_query); + +# Sanity checks: the copied table's VM and FSM must have interesting +# contents on the primary, otherwise the comparisons below would pass +# vacuously. +my ($all_visible) = split(/\|/, $primary_vm); +my (undef, $fsm_avail) = split(/\|/, $primary_fsm); +cmp_ok($all_visible, '>', 0, + 'copied table has visibility map bits set on primary'); +cmp_ok($fsm_avail, '>', 0, 'copied table has free space recorded on primary'); + +# The standby must agree with the primary on the copied database's VM and +# FSM contents. +is( $node_standby->safe_psql('dst_db', $vm_query), + $primary_vm, + 'standby has same visibility map contents as primary after WAL_LOG copy'); +is( $node_standby->safe_psql('dst_db', $fsm_query), + $primary_fsm, + 'standby has same free space map contents as primary after WAL_LOG copy'); + +done_testing();