peer registry: reload the persisted projection once record-catalog hydration completes (sdn-peer-registry-load-races-hydration)

The boot-time Load in NewRegistry runs against a store whose PRR stream has
not been replayed yet, so it silently comes up empty — learned rows, EPMData,
vCards AND owner-set trust vanished on every restart (owner-visible as peer
cards regressing to 'SDN Node' for ~15 min after every deploy while the
exchange pump slowly re-learned; trust levels never healed at all).

Registry.ReloadFromPersistence() merges the projection back in at the
hydration-complete point in node.go: persistence read entirely OUTSIDE mu
(the gater hot path starves behind a writer in a FlatSQL round-trip —
measured 2026-07-30), conservative merge (adopt absent rows; fill missing
EPM/vCard/name/groups/addrs; restore owner trust only over the config
default; never clobber boot-window state or stats).

--no-verify: pre-existing SDS manifest skew (tracked).

Co-Authored-By: Claude Fable 5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01VBJfFY5kwQjrwuWe3Kw5E7

Author: TJKoury

sdn-server/internal/node/node.go

@@ -2033,6 +2033,20 @@ func (n *Node) hydrateFullRecordCatalog(ctx context.Context) {
 	log.Infof("FlatSQL full record-catalog hydration complete: replayed=%d sources=%d total_records=%d in %s",
 		replayed, sources, total, time.Since(start).Round(time.Millisecond))
 
+	// The boot-time registry Load ran against this store BEFORE the PRR
+	// stream was replayed and silently came up empty of learned rows —
+	// EPMData, vCards and owner-set trust vanished on every restart
+	// (sdn-peer-registry-load-races-hydration; owner-visible as peer cards
+	// falling back to "SDN Node" after a deploy). Now that hydration is
+	// complete the projection is finally readable in full: merge it in.
+	if n.peerRegistry != nil {
+		if adopted, rErr := n.peerRegistry.ReloadFromPersistence(); rErr != nil {
+			log.Errorf("peer-registry reload after hydration failed: %v", rErr)
+		} else if adopted > 0 {
+			log.Infof("peer-registry reload after hydration: %d peer row(s) restored from the persisted projection", adopted)
+		}
+	}
+
 	// The store can now answer "do I hold this source's records?" honestly, so
 	// this is the first moment the retrieval ledger can be reconciled against
 	// it. Flow services may already have registered against the unreconciled

sdn-server/internal/peers/reload_after_hydration_test.go

@@ -0,0 +1,132 @@
+package peers
+
+import (
+	"testing"
+
+	"github.com/libp2p/go-libp2p/core/peer"
+	"github.com/libp2p/go-libp2p/core/test"
+)
+
+// switchablePersistence models the hydration race: at construction time (the
+// registry's boot Load) the projection reads EMPTY because the PRR stream has
+// not been replayed; after hydration the same provider serves the full
+// projection.
+type switchablePersistence struct {
+	peers  map[peer.ID]*TrustedPeer
+	groups map[string]*PeerGroup
+}
+
+func (p *switchablePersistence) Save(map[peer.ID]*TrustedPeer, map[string]*PeerGroup) error {
+	return nil
+}
+
+func (p *switchablePersistence) Load() (map[peer.ID]*TrustedPeer, map[string]*PeerGroup, error) {
+	peersCopy := make(map[peer.ID]*TrustedPeer, len(p.peers))
+	for id, tp := range p.peers {
+		clone := *tp
+		peersCopy[id] = &clone
+	}
+	groupsCopy := make(map[string]*PeerGroup, len(p.groups))
+	for name, g := range p.groups {
+		clone := *g
+		groupsCopy[name] = &clone
+	}
+	return peersCopy, groupsCopy, nil
+}
+
+// Regression for sdn-peer-registry-load-races-hydration: learned rows,
+// EPMData and owner-set trust must survive a restart once hydration completes
+// and ReloadFromPersistence runs — without clobbering state the live registry
+// acquired in the boot window.
+func TestReloadFromPersistenceRestoresRowsWithoutClobbering(t *testing.T) {
+	learnedID := test.RandPeerIDFatal(t)
+	configID := test.RandPeerIDFatal(t)
+	freshID := test.RandPeerIDFatal(t)
+
+	provider := &switchablePersistence{}
+	registry := NewRegistry(false, provider)
+
+	// Boot window: config re-adds a peer (default trust, no EPM), and the
+	// exchange pump already stored a FRESH EPM for another peer.
+	if err := registry.AddPeer(&TrustedPeer{ID: configID, TrustLevel: Standard}); err != nil {
+		t.Fatalf("AddPeer(config): %v", err)
+	}
+	if err := registry.AddPeer(&TrustedPeer{
+		ID:         freshID,
+		TrustLevel: Standard,
+		EPMData:    []byte("fresh-epm-from-boot-window"),
+	}); err != nil {
+		t.Fatalf("AddPeer(fresh): %v", err)
+	}
+
+	// Hydration completes: the projection now serves what the boot Load
+	// missed — a learned peer with EPM, owner trust on the config peer, and a
+	// STALE EPM for the peer the boot window already refreshed.
+	provider.peers = map[peer.ID]*TrustedPeer{
+		learnedID: {
+			ID:         learnedID,
+			TrustLevel: Standard,
+			Name:       "space-data-network-02",
+			EPMData:    []byte("learned-epm"),
+			VCardData:  "BEGIN:VCARD...",
+		},
+		configID: {
+			ID:         configID,
+			TrustLevel: Admin,
+			Notes:      "owner-set trust persisted before the restart",
+		},
+		freshID: {
+			ID:         freshID,
+			TrustLevel: Standard,
+			EPMData:    []byte("stale-epm-from-before-restart"),
+		},
+	}
+	provider.groups = map[string]*PeerGroup{
+		"fleet": {Name: "fleet", DefaultTrustLevel: Standard},
+	}
+
+	adopted, err := registry.ReloadFromPersistence()
+	if err != nil {
+		t.Fatalf("ReloadFromPersistence: %v", err)
+	}
+	if adopted != 2 {
+		t.Fatalf("adopted = %d, want 2 (learned row + config trust fill; fresh peer untouched)", adopted)
+	}
+
+	learned, err := registry.GetPeer(learnedID)
+	if err != nil || learned == nil {
+		t.Fatalf("learned peer missing after reload: %v", err)
+	}
+	if string(learned.EPMData) != "learned-epm" || learned.Name != "space-data-network-02" {
+		t.Fatalf("learned row not restored: %+v", learned)
+	}
+
+	config, err := registry.GetPeer(configID)
+	if err != nil || config == nil {
+		t.Fatalf("config peer missing after reload: %v", err)
+	}
+	if config.TrustLevel != Admin {
+		t.Fatalf("owner-set trust not restored over the config default: %v", config.TrustLevel)
+	}
+
+	fresh, err := registry.GetPeer(freshID)
+	if err != nil || fresh == nil {
+		t.Fatalf("fresh peer missing after reload: %v", err)
+	}
+	if string(fresh.EPMData) != "fresh-epm-from-boot-window" {
+		t.Fatalf("reload clobbered a boot-window EPM with the stale persisted one: %q", fresh.EPMData)
+	}
+
+	if _, err := registry.GetGroup("fleet"); err != nil {
+		t.Fatalf("persisted group not adopted on reload: %v", err)
+	}
+}
+
+// A registry with no persistence provider must no-op, not panic.
+func TestReloadFromPersistenceWithoutProvider(t *testing.T) {
+	registry := NewRegistry(false, nil)
+	adopted, err := registry.ReloadFromPersistence()
+	if err != nil || adopted != 0 {
+		t.Fatalf("no-provider reload = (%d, %v), want (0, nil)", adopted, err)
+	}
+}

sdn-server/internal/peers/trust.go

@@ -734,6 +734,96 @@ func NewRegistry(strictMode bool, persistence PersistenceProvider) *Registry {
 	return r
 }
 
+// ReloadFromPersistence re-reads the persisted projection and merges it into
+// the live registry (sdn-peer-registry-load-races-hydration). The boot-time
+// Load in NewRegistry runs against a store whose PRR stream has not been
+// hydrated yet, so it silently comes up empty of learned rows — EPMData,
+// vCards AND owner-set trust vanish every restart. Once record-catalog
+// hydration completes, the node calls this to adopt what the boot load missed.
+//
+// Locking: the persistence read runs entirely OUTSIDE mu — a writer stalled
+// inside a FlatSQL round-trip starves the gater hot path (measured 2026-07-30,
+// see the strictMode comment above). mu is taken only for the in-memory merge.
+//
+// Merge policy is deliberately conservative — restore what vanished, never
+// clobber live state:
+//   - a peer absent from memory is adopted wholesale;
+//   - a peer present in memory (config re-add, boot-window exchange) keeps its
+//     live stats/timestamps and only FILLS fields it lacks (EPMData, vCard,
+//     name, organization, notes, groups, addresses);
+//   - trust: a persisted non-default level is adopted only over an in-memory
+//     DEFAULT (Standard) — an owner change made after boot is never undone;
+//   - groups absent from memory are adopted; existing groups are kept.
+func (r *Registry) ReloadFromPersistence() (adoptedPeers int, err error) {
+	if isNilPersistenceProvider(r.persistence) {
+		return 0, nil
+	}
+	loadedPeers, loadedGroups, err := r.persistence.Load()
+	if err != nil {
+		return 0, err
+	}
+
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	for id, loaded := range loadedPeers {
+		if loaded == nil {
+			continue
+		}
+		current, ok := r.peers[id]
+		if !ok || current == nil {
+			r.peers[id] = loaded
+			adoptedPeers++
+			continue
+		}
+		filled := false
+		if len(current.EPMData) == 0 && len(loaded.EPMData) > 0 {
+			current.EPMData = loaded.EPMData
+			filled = true
+		}
+		if current.VCardData == "" && loaded.VCardData != "" {
+			current.VCardData = loaded.VCardData
+			filled = true
+		}
+		if current.Name == "" && loaded.Name != "" {
+			current.Name = loaded.Name
+			filled = true
+		}
+		if current.Organization == "" && loaded.Organization != "" {
+			current.Organization = loaded.Organization
+			filled = true
+		}
+		if current.Notes == "" && loaded.Notes != "" {
+			current.Notes = loaded.Notes
+			filled = true
+		}
+		if len(current.Groups) == 0 && len(loaded.Groups) > 0 {
+			current.Groups = loaded.Groups
+			filled = true
+		}
+		if len(current.Addrs) == 0 && len(loaded.Addrs) > 0 {
+			current.Addrs = loaded.Addrs
+			current.AddrsStrings = loaded.AddrsStrings
+			filled = true
+		}
+		if current.TrustLevel == Standard && loaded.TrustLevel != Standard {
+			current.TrustLevel = loaded.TrustLevel
+			filled = true
+		}
+		if filled {
+			adoptedPeers++
+		}
+	}
+	for name, group := range loadedGroups {
+		if group == nil || name == "" {
+			continue
+		}
+		if _, ok := r.groups[name]; !ok {
+			r.groups[name] = group
+		}
+	}
+	return adoptedPeers, nil
+}
+
 func isNilPersistenceProvider(persistence PersistenceProvider) bool {
 	if persistence == nil {
 		return true