Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions doc/src/sgml/custom-rmgr.sgml
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,10 @@
* record, if available (e.g. the last block).
*
* rm_mask takes as input a page modified by the resource manager and masks
* out bits that shouldn't be flagged by wal_consistency_checking.
* out bits that shouldn't be flagged by wal_consistency_checking. The fork
* number of the block is passed as well, since resource managers may
* register blocks of different forks (e.g. heap records can reference
* visibility map pages) which require different masking rules.
*
* RmgrTable[] is indexed by RmgrId values (see rmgrlist.h). If rm_name is
* NULL, the corresponding RmgrTable entry is considered invalid.
Expand All @@ -48,7 +51,8 @@ typedef struct RmgrData
const char *(*rm_identify) (uint8 info);
void (*rm_startup) (void);
void (*rm_cleanup) (void);
void (*rm_mask) (char *pagedata, BlockNumber blkno);
void (*rm_mask) (char *pagedata, BlockNumber blkno,
ForkNumber forknum);
void (*rm_decode) (struct LogicalDecodingContext *ctx,
struct XLogRecordBuffer *buf);
} RmgrData;
Expand Down
2 changes: 1 addition & 1 deletion src/backend/access/brin/brin_xlog.c
Original file line number Diff line number Diff line change
Expand Up @@ -339,7 +339,7 @@ brin_redo(XLogReaderState *record)
* Mask a BRIN page before doing consistency checks.
*/
void
brin_mask(char *pagedata, BlockNumber blkno)
brin_mask(char *pagedata, BlockNumber blkno, ForkNumber forknum)
{
Page page = (Page) pagedata;
PageHeader pagehdr = (PageHeader) page;
Expand Down
14 changes: 12 additions & 2 deletions src/backend/access/common/bufmask.c
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,18 @@ mask_page_hint_bits(Page page)
PageClearHasFreeLinePointers(page);

/*
* PD_ALL_VISIBLE is masked during WAL consistency checking. XXX: It is
* worth investigating if we could stop doing this.
* PD_ALL_VISIBLE is masked during WAL consistency checking.
*
* All changes to PD_ALL_VISIBLE are WAL-logged nowadays, so in principle
* primary and standby should agree on it. However, when the only change
* to a heap page is setting PD_ALL_VISIBLE (and checksums/wal_log_hints
* are disabled), log_heap_prune_and_freeze() intentionally skips the FPI
* and does not stamp the heap page with the record's LSN. The flag's
* on-disk state is therefore not strictly LSN-ordered, and around
* recovery restarts the primary's page image and the standby's replayed
* page can transiently disagree on it without indicating corruption. So
* we conservatively continue to mask it. XXX: It is worth investigating
* if we could stop doing this.
*/
PageClearAllVisible(page);
}
Expand Down
2 changes: 1 addition & 1 deletion src/backend/access/gin/ginxlog.c
Original file line number Diff line number Diff line change
Expand Up @@ -788,7 +788,7 @@ gin_xlog_cleanup(void)
* Mask a GIN page before running consistency checks on it.
*/
void
gin_mask(char *pagedata, BlockNumber blkno)
gin_mask(char *pagedata, BlockNumber blkno, ForkNumber forknum)
{
Page page = (Page) pagedata;
PageHeader pagehdr = (PageHeader) page;
Expand Down
2 changes: 1 addition & 1 deletion src/backend/access/gist/gistxlog.c
Original file line number Diff line number Diff line change
Expand Up @@ -445,7 +445,7 @@ gist_xlog_cleanup(void)
* Mask a Gist page before running consistency checks on it.
*/
void
gist_mask(char *pagedata, BlockNumber blkno)
gist_mask(char *pagedata, BlockNumber blkno, ForkNumber forknum)
{
Page page = (Page) pagedata;

Expand Down
2 changes: 1 addition & 1 deletion src/backend/access/hash/hash_xlog.c
Original file line number Diff line number Diff line change
Expand Up @@ -1091,7 +1091,7 @@ hash_redo(XLogReaderState *record)
* Mask a hash page before performing consistency checks on it.
*/
void
hash_mask(char *pagedata, BlockNumber blkno)
hash_mask(char *pagedata, BlockNumber blkno, ForkNumber forknum)
{
Page page = (Page) pagedata;
HashPageOpaque opaque;
Expand Down
21 changes: 20 additions & 1 deletion src/backend/access/heap/heapam_xlog.c
Original file line number Diff line number Diff line change
Expand Up @@ -1346,11 +1346,30 @@ heap2_redo(XLogReaderState *record)
* Mask a heap page before performing consistency checks on it.
*/
void
heap_mask(char *pagedata, BlockNumber blkno)
heap_mask(char *pagedata, BlockNumber blkno, ForkNumber forknum)
{
Page page = (Page) pagedata;
OffsetNumber off;

/*
* Heap records can reference visibility map pages in addition to heap
* pages. A visibility map page stores the all-visible/all-frozen bitmap
* in the space between pd_lower and pd_upper, so the heap masking rules
* below must not be applied to it: in particular, mask_unused_space()
* would wipe out the entire bitmap, making the consistency check on VM
* pages vacuous. VM bits are only ever set or cleared while holding
* exclusive content lock on the VM page with the changes WAL-logged, so
* only the LSN and checksum can legitimately differ between the primary
* and standby versions of a VM page.
*/
if (forknum == VISIBILITYMAP_FORKNUM)
{
mask_page_lsn_and_checksum(page);
return;
}

Assert(forknum == MAIN_FORKNUM);

mask_page_lsn_and_checksum(page);

mask_page_hint_bits(page);
Expand Down
2 changes: 1 addition & 1 deletion src/backend/access/nbtree/nbtxlog.c
Original file line number Diff line number Diff line change
Expand Up @@ -1078,7 +1078,7 @@ btree_xlog_cleanup(void)
* Mask a btree page before performing consistency checks on it.
*/
void
btree_mask(char *pagedata, BlockNumber blkno)
btree_mask(char *pagedata, BlockNumber blkno, ForkNumber forknum)
{
Page page = (Page) pagedata;
BTPageOpaque maskopaq;
Expand Down
2 changes: 1 addition & 1 deletion src/backend/access/spgist/spgxlog.c
Original file line number Diff line number Diff line change
Expand Up @@ -983,7 +983,7 @@ spg_xlog_cleanup(void)
* Mask a SpGist page before performing consistency checks on it.
*/
void
spg_mask(char *pagedata, BlockNumber blkno)
spg_mask(char *pagedata, BlockNumber blkno, ForkNumber forknum)
{
Page page = (Page) pagedata;
PageHeader pagehdr = (PageHeader) page;
Expand Down
2 changes: 1 addition & 1 deletion src/backend/access/transam/generic_xlog.c
Original file line number Diff line number Diff line change
Expand Up @@ -536,7 +536,7 @@ generic_redo(XLogReaderState *record)
* Mask a generic page before performing consistency checks on it.
*/
void
generic_mask(char *page, BlockNumber blkno)
generic_mask(char *page, BlockNumber blkno, ForkNumber forknum)
{
mask_page_lsn_and_checksum(page);

Expand Down
4 changes: 2 additions & 2 deletions src/backend/access/transam/xlogrecovery.c
Original file line number Diff line number Diff line change
Expand Up @@ -2523,8 +2523,8 @@ verifyBackupPageConsistency(XLogReaderState *record)
*/
if (rmgr.rm_mask != NULL)
{
rmgr.rm_mask(replay_image_masked, blkno);
rmgr.rm_mask(primary_image_masked, blkno);
rmgr.rm_mask(replay_image_masked, blkno, forknum);
rmgr.rm_mask(primary_image_masked, blkno, forknum);
}

/* Time to compare the primary and replay images. */
Expand Down
2 changes: 1 addition & 1 deletion src/backend/commands/sequence_xlog.c
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ seq_redo(XLogReaderState *record)
* Mask a Sequence page before performing consistency checks on it.
*/
void
seq_mask(char *page, BlockNumber blkno)
seq_mask(char *page, BlockNumber blkno, ForkNumber forknum)
{
mask_page_lsn_and_checksum(page);

Expand Down
16 changes: 14 additions & 2 deletions src/backend/storage/buffer/bufmgr.c
Original file line number Diff line number Diff line change
Expand Up @@ -5454,9 +5454,21 @@ RelationCopyStorageUsingBuffer(RelFileLocator srclocator,
memcpy(dstPage, srcPage, BLCKSZ);
MarkBufferDirty(dstBuf);

/* WAL-log the copied page. */
/*
* WAL-log the copied page.
*
* We must not claim the standard page layout for visibility map or
* FSM pages: they keep their whole payload between pd_lower and
* pd_upper, so treating them as standard would make the full-page
* image omit that region as a "hole", and replay would zero it,
* silently emptying the fork's contents on a standby or after crash
* recovery. Main and init fork pages use the standard layout, so
* hole compression is safe for them.
*/
if (use_wal)
log_newpage_buffer(dstBuf, true);
log_newpage_buffer(dstBuf,
forkNum != VISIBILITYMAP_FORKNUM &&
forkNum != FSM_FORKNUM);

END_CRIT_SECTION();

Expand Down
2 changes: 1 addition & 1 deletion src/include/access/brin_xlog.h
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,6 @@ typedef struct xl_brin_desummarize
extern void brin_redo(XLogReaderState *record);
extern void brin_desc(StringInfo buf, XLogReaderState *record);
extern const char *brin_identify(uint8 info);
extern void brin_mask(char *pagedata, BlockNumber blkno);
extern void brin_mask(char *pagedata, BlockNumber blkno, ForkNumber forknum);

#endif /* BRIN_XLOG_H */
2 changes: 1 addition & 1 deletion src/include/access/generic_xlog.h
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,6 @@ extern void GenericXLogAbort(GenericXLogState *state);
extern void generic_redo(XLogReaderState *record);
extern const char *generic_identify(uint8 info);
extern void generic_desc(StringInfo buf, XLogReaderState *record);
extern void generic_mask(char *page, BlockNumber blkno);
extern void generic_mask(char *page, BlockNumber blkno, ForkNumber forknum);

#endif /* GENERIC_XLOG_H */
2 changes: 1 addition & 1 deletion src/include/access/ginxlog.h
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,6 @@ extern void gin_desc(StringInfo buf, XLogReaderState *record);
extern const char *gin_identify(uint8 info);
extern void gin_xlog_startup(void);
extern void gin_xlog_cleanup(void);
extern void gin_mask(char *pagedata, BlockNumber blkno);
extern void gin_mask(char *pagedata, BlockNumber blkno, ForkNumber forknum);

#endif /* GINXLOG_H */
2 changes: 1 addition & 1 deletion src/include/access/gistxlog.h
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,6 @@ extern void gist_desc(StringInfo buf, XLogReaderState *record);
extern const char *gist_identify(uint8 info);
extern void gist_xlog_startup(void);
extern void gist_xlog_cleanup(void);
extern void gist_mask(char *pagedata, BlockNumber blkno);
extern void gist_mask(char *pagedata, BlockNumber blkno, ForkNumber forknum);

#endif
2 changes: 1 addition & 1 deletion src/include/access/hash_xlog.h
Original file line number Diff line number Diff line change
Expand Up @@ -265,6 +265,6 @@ typedef struct xl_hash_vacuum_one_page
extern void hash_redo(XLogReaderState *record);
extern void hash_desc(StringInfo buf, XLogReaderState *record);
extern const char *hash_identify(uint8 info);
extern void hash_mask(char *pagedata, BlockNumber blkno);
extern void hash_mask(char *pagedata, BlockNumber blkno, ForkNumber forknum);

#endif /* HASH_XLOG_H */
2 changes: 1 addition & 1 deletion src/include/access/heapam_xlog.h
Original file line number Diff line number Diff line change
Expand Up @@ -512,7 +512,7 @@ extern void HeapTupleHeaderAdvanceConflictHorizon(HeapTupleHeader tuple,
extern void heap_redo(XLogReaderState *record);
extern void heap_desc(StringInfo buf, XLogReaderState *record);
extern const char *heap_identify(uint8 info);
extern void heap_mask(char *pagedata, BlockNumber blkno);
extern void heap_mask(char *pagedata, BlockNumber blkno, ForkNumber forknum);
extern void heap2_redo(XLogReaderState *record);
extern void heap2_desc(StringInfo buf, XLogReaderState *record);
extern const char *heap2_identify(uint8 info);
Expand Down
2 changes: 1 addition & 1 deletion src/include/access/nbtxlog.h
Original file line number Diff line number Diff line change
Expand Up @@ -356,7 +356,7 @@ typedef struct xl_btree_newroot
extern void btree_redo(XLogReaderState *record);
extern void btree_xlog_startup(void);
extern void btree_xlog_cleanup(void);
extern void btree_mask(char *pagedata, BlockNumber blkno);
extern void btree_mask(char *pagedata, BlockNumber blkno, ForkNumber forknum);

/*
* prototypes for functions in nbtdesc.c
Expand Down
2 changes: 1 addition & 1 deletion src/include/access/spgxlog.h
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,6 @@ extern void spg_desc(StringInfo buf, XLogReaderState *record);
extern const char *spg_identify(uint8 info);
extern void spg_xlog_startup(void);
extern void spg_xlog_cleanup(void);
extern void spg_mask(char *pagedata, BlockNumber blkno);
extern void spg_mask(char *pagedata, BlockNumber blkno, ForkNumber forknum);

#endif /* SPGXLOG_H */
8 changes: 6 additions & 2 deletions src/include/access/xlog_internal.h
Original file line number Diff line number Diff line change
Expand Up @@ -343,7 +343,10 @@ struct XLogRecordBuffer;
* record, if available (e.g. the last block).
*
* rm_mask takes as input a page modified by the resource manager and masks
* out bits that shouldn't be flagged by wal_consistency_checking.
* out bits that shouldn't be flagged by wal_consistency_checking. The fork
* number of the block is passed as well, since resource managers may
* register blocks of different forks (e.g. heap records can reference
* visibility map pages) which require different masking rules.
*
* RmgrTable[] is indexed by RmgrId values (see rmgrlist.h). If rm_name is
* NULL, the corresponding RmgrTable entry is considered invalid.
Expand All @@ -356,7 +359,8 @@ typedef struct RmgrData
const char *(*rm_identify) (uint8 info);
void (*rm_startup) (void);
void (*rm_cleanup) (void);
void (*rm_mask) (char *pagedata, BlockNumber blkno);
void (*rm_mask) (char *pagedata, BlockNumber blkno,
ForkNumber forknum);
void (*rm_decode) (struct LogicalDecodingContext *ctx,
struct XLogRecordBuffer *buf);
} RmgrData;
Expand Down
2 changes: 1 addition & 1 deletion src/include/commands/sequence_xlog.h
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,6 @@ typedef struct xl_seq_rec
extern void seq_redo(XLogReaderState *record);
extern void seq_desc(StringInfo buf, XLogReaderState *record);
extern const char *seq_identify(uint8 info);
extern void seq_mask(char *page, BlockNumber blkno);
extern void seq_mask(char *page, BlockNumber blkno, ForkNumber forknum);

#endif /* SEQUENCE_XLOG_H */
4 changes: 3 additions & 1 deletion src/test/recovery/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,10 @@
#
#-------------------------------------------------------------------------

EXTRA_INSTALL=contrib/pg_prewarm \
EXTRA_INSTALL=contrib/pg_freespacemap \
contrib/pg_prewarm \
contrib/pg_stat_statements \
contrib/pg_visibility \
contrib/test_decoding \
src/test/modules/injection_points

Expand Down
1 change: 1 addition & 0 deletions src/test/recovery/meson.build
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ tests += {
't/052_checkpoint_segment_missing.pl',
't/053_standby_login_event_trigger.pl',
't/054_unlogged_sequence_promotion.pl',
't/055_create_database_walog.pl',
],
},
}
86 changes: 86 additions & 0 deletions src/test/recovery/t/055_create_database_walog.pl
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@

# Copyright (c) 2026, PostgreSQL Global Development Group

# Test that CREATE DATABASE ... STRATEGY=WAL_LOG WAL-logs the auxiliary
# relation forks in a way that recovery can reproduce them.
#
# The WAL_LOG strategy copies the template database block by block and
# WAL-logs every page. Visibility map and free space map pages store
# their entire payload between pd_lower and pd_upper, so they must not be
# logged as standard-layout pages: the full-page-image "hole" optimization
# would elide the payload and replay would zero it, silently emptying the
# copied database's VM and FSM on a standby (or after crash recovery).
# Verify that a standby ends up with the same VM and FSM contents as the
# primary for a database copied with STRATEGY=WAL_LOG.

use strict;
use warnings FATAL => 'all';

use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;

my $node_primary = PostgreSQL::Test::Cluster->new('primary');
$node_primary->init(allows_streaming => 1);
# Keep the VM and FSM of the tables involved stable during the test.
$node_primary->append_conf('postgresql.conf', 'autovacuum = off');
$node_primary->start;

# Create a template database whose table has both visibility map bits set
# and free space recorded in the FSM.
$node_primary->safe_psql('postgres', 'CREATE DATABASE src_db');
$node_primary->safe_psql(
'src_db', qq[
CREATE EXTENSION pg_visibility;
CREATE EXTENSION pg_freespacemap;
CREATE TABLE tab_copied (a int, filler text);
INSERT INTO tab_copied
SELECT g, repeat('x', 100) FROM generate_series(1, 5000) g;
DELETE FROM tab_copied WHERE a % 5 = 0;
]);
# Set all-visible/all-frozen bits in the VM and record the space freed by
# the deletions in the FSM.
$node_primary->safe_psql('src_db', 'VACUUM (FREEZE) tab_copied');

# Create a standby.
my $backup_name = 'my_backup';
$node_primary->backup($backup_name);
my $node_standby = PostgreSQL::Test::Cluster->new('standby');
$node_standby->init_from_backup($node_primary, $backup_name,
has_streaming => 1);
$node_standby->start;
$node_primary->wait_for_catchup($node_standby, 'replay');

# Copy the template database using the WAL_LOG strategy and wait for the
# standby to replay the copy.
$node_primary->safe_psql('postgres',
'CREATE DATABASE dst_db TEMPLATE src_db STRATEGY WAL_LOG');
$node_primary->wait_for_catchup($node_standby, 'replay');

my $vm_query =
"SELECT all_visible, all_frozen FROM pg_visibility_map_summary('tab_copied')";
my $fsm_query =
"SELECT count(*), COALESCE(sum(avail), 0) FROM pg_freespace('tab_copied')";

my $primary_vm = $node_primary->safe_psql('dst_db', $vm_query);
my $primary_fsm = $node_primary->safe_psql('dst_db', $fsm_query);

# Sanity checks: the copied table's VM and FSM must have interesting
# contents on the primary, otherwise the comparisons below would pass
# vacuously.
my ($all_visible) = split(/\|/, $primary_vm);
my (undef, $fsm_avail) = split(/\|/, $primary_fsm);
cmp_ok($all_visible, '>', 0,
'copied table has visibility map bits set on primary');
cmp_ok($fsm_avail, '>', 0, 'copied table has free space recorded on primary');

# The standby must agree with the primary on the copied database's VM and
# FSM contents.
is( $node_standby->safe_psql('dst_db', $vm_query),
$primary_vm,
'standby has same visibility map contents as primary after WAL_LOG copy');
is( $node_standby->safe_psql('dst_db', $fsm_query),
$primary_fsm,
'standby has same free space map contents as primary after WAL_LOG copy');

done_testing();