diff --git a/cmd/streambed/main.go b/cmd/streambed/main.go index 3b04c9c..5e32f37 100644 --- a/cmd/streambed/main.go +++ b/cmd/streambed/main.go @@ -17,6 +17,7 @@ import ( _ "net/http/pprof" "github.com/jackc/pglogrepl" + "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -236,6 +237,15 @@ func runSync(cmd *cobra.Command, args []string) error { } defer pgConn.Close(context.Background()) + // Open a regular (non-replication) connection for metadata queries + // (column defaults on schema changes, etc.). + metaConn, err := pgx.Connect(ctx, cfg.SourceURL) + if err != nil { + return fmt.Errorf("connect to postgres (metadata): %w", err) + } + defer metaConn.Close(context.Background()) + metaQuerier := wal.NewMetadataQuerier(metaConn) + // Create publication pubName := cfg.SlotName // use same name for publication if err := wal.CreatePublication(ctx, pgConn, pubName, cfg.IncludeTables, logger); err != nil { @@ -311,7 +321,7 @@ func runSync(cmd *cobra.Command, args []string) error { // Create unified pipeline (single goroutine: reads WAL + writes Iceberg) p := pipeline.New(pgConn, cfg.SlotName, pubName, startLSN, cfg.ExcludeTables, - logger, stateStore, tableFlushLSN, writer, cfg.FlushInterval) + logger, stateStore, tableFlushLSN, writer, cfg.FlushInterval, metaQuerier) // Run blocks until ctx is cancelled; does final flush internally. // On non-context errors (e.g. Postgres disconnect, transient S3 @@ -399,7 +409,7 @@ func runSync(cmd *cobra.Command, args []string) error { writer = iceberg.NewWriter(catalog, s3Client, stateStore, cfg.SlotName, cfg.FlushRows, cfg.FlushInterval, logger) p = pipeline.New(pgConn, cfg.SlotName, pubName, startLSN, cfg.ExcludeTables, - logger, stateStore, tableFlushLSN, writer, cfg.FlushInterval) + logger, stateStore, tableFlushLSN, writer, cfg.FlushInterval, metaQuerier) logger.Info("reconnected, resuming pipeline", "start_lsn", startLSN, diff --git a/internal/iceberg/catalog.go b/internal/iceberg/catalog.go index b069058..df743b0 100644 --- a/internal/iceberg/catalog.go +++ b/internal/iceberg/catalog.go @@ -12,6 +12,7 @@ import ( ice "github.com/apache/iceberg-go" "github.com/google/uuid" "github.com/viggy28/streambed/internal/storage" + "github.com/viggy28/streambed/internal/wal" ) // ColumnDef describes a table column for Iceberg metadata. @@ -64,19 +65,23 @@ func (c *Catalog) TableExists(ctx context.Context, schema, table string) (bool, } // CreateTable creates initial Iceberg v2 metadata for a new table (no snapshots). -func (c *Catalog) CreateTable(ctx context.Context, schema, table string, columns []ColumnDef) error { +// Returns the field ID mapping (column name → Iceberg field ID) assigned to the table. +func (c *Catalog) CreateTable(ctx context.Context, schema, table string, columns []ColumnDef) (map[string]int, error) { tableLoc := fmt.Sprintf("s3://%s/%s", c.bucket, c.tablePath(schema, table)) tableUUID := uuid.New().String() now := time.Now().UnixMilli() + fieldIDs := make(map[string]int, len(columns)) fields := make([]schemaField, len(columns)) for i, col := range columns { + id := i + 1 fields[i] = schemaField{ - ID: i + 1, + ID: id, Name: col.Name, Required: false, Type: PgOIDToIcebergType(col.OID), } + fieldIDs[col.Name] = id } metadata := tableMetadata{ @@ -112,7 +117,7 @@ func (c *Catalog) CreateTable(ctx context.Context, schema, table string, columns metadataJSON, err := json.MarshalIndent(metadata, "", " ") if err != nil { - return fmt.Errorf("marshal metadata: %w", err) + return nil, fmt.Errorf("marshal metadata: %w", err) } basePath := c.tablePath(schema, table) @@ -120,12 +125,241 @@ func (c *Catalog) CreateTable(ctx context.Context, schema, table string, columns // Write v1.metadata.json (DuckDB looks for v.metadata.json) metaKey := path.Join(basePath, "metadata", "v1.metadata.json") if err := c.storage.PutObject(ctx, metaKey, metadataJSON, "application/json"); err != nil { - return err + return nil, err } // Write version-hint.text hintKey := path.Join(basePath, "metadata", "version-hint.text") - return c.storage.PutObject(ctx, hintKey, []byte("1"), "text/plain") + if err := c.storage.PutObject(ctx, hintKey, []byte("1"), "text/plain"); err != nil { + return nil, err + } + return fieldIDs, nil +} + +// SchemaFieldInfo describes a single field in the current Iceberg schema. +// Exported so the writer can diff WAL columns against Iceberg on restart. +type SchemaFieldInfo struct { + ID int + Name string + Type IcebergType +} + +// GetFieldIDs reads the current Iceberg metadata for a table and returns +// the field ID mapping (column name → Iceberg field ID) and the lastColumnID. +// Returns an error if the table does not exist or metadata is unreadable. +func (c *Catalog) GetFieldIDs(ctx context.Context, schema, table string) (map[string]int, int, error) { + basePath := c.tablePath(schema, table) + + hintKey := path.Join(basePath, "metadata", "version-hint.text") + hintData, err := c.storage.GetObject(ctx, hintKey) + if err != nil { + return nil, 0, fmt.Errorf("read version-hint: %w", err) + } + version, err := strconv.Atoi(string(hintData)) + if err != nil { + return nil, 0, fmt.Errorf("parse version hint: %w", err) + } + + metaKey := path.Join(basePath, "metadata", fmt.Sprintf("v%d.metadata.json", version)) + metaData, err := c.storage.GetObject(ctx, metaKey) + if err != nil { + return nil, 0, fmt.Errorf("read metadata: %w", err) + } + var metadata tableMetadata + if err := json.Unmarshal(metaData, &metadata); err != nil { + return nil, 0, fmt.Errorf("parse metadata: %w", err) + } + + if len(metadata.Schemas) == 0 { + return nil, 0, fmt.Errorf("no schemas in metadata") + } + + currentSchema := metadata.Schemas[metadata.CurrentSchemaID] + fieldIDs := make(map[string]int, len(currentSchema.Fields)) + for _, f := range currentSchema.Fields { + fieldIDs[f.Name] = f.ID + } + return fieldIDs, metadata.LastColumnID, nil +} + +// GetSchemaFields reads the current Iceberg schema for a table and returns +// the list of fields with their IDs, names, and types. Used by the writer +// to reconcile WAL-provided columns against Iceberg on restart. +func (c *Catalog) GetSchemaFields(ctx context.Context, schema, table string) ([]SchemaFieldInfo, error) { + basePath := c.tablePath(schema, table) + + hintKey := path.Join(basePath, "metadata", "version-hint.text") + hintData, err := c.storage.GetObject(ctx, hintKey) + if err != nil { + return nil, fmt.Errorf("read version-hint: %w", err) + } + version, err := strconv.Atoi(string(hintData)) + if err != nil { + return nil, fmt.Errorf("parse version hint: %w", err) + } + + metaKey := path.Join(basePath, "metadata", fmt.Sprintf("v%d.metadata.json", version)) + metaData, err := c.storage.GetObject(ctx, metaKey) + if err != nil { + return nil, fmt.Errorf("read metadata: %w", err) + } + var metadata tableMetadata + if err := json.Unmarshal(metaData, &metadata); err != nil { + return nil, fmt.Errorf("parse metadata: %w", err) + } + + if len(metadata.Schemas) == 0 { + return nil, fmt.Errorf("no schemas in metadata") + } + + currentSchema := metadata.Schemas[metadata.CurrentSchemaID] + fields := make([]SchemaFieldInfo, len(currentSchema.Fields)) + for i, f := range currentSchema.Fields { + fields[i] = SchemaFieldInfo{ID: f.ID, Name: f.Name, Type: f.Type} + } + return fields, nil +} + +// EvolveSchema applies schema changes (ADD, DROP, TYPE_CHANGE) to the +// Iceberg table metadata. It reads the current metadata, computes the new +// schema, writes a new metadata version, and returns the updated field ID +// mapping. +// +// For ADD columns, defaults maps column names to their Postgres default +// expression (used to set Iceberg's initial-default). Pass nil if no +// defaults are available. +func (c *Catalog) EvolveSchema( + ctx context.Context, + schema, table string, + changes []wal.SchemaChange, + newColumns []wal.Column, + defaults map[string]string, +) (map[string]int, error) { + basePath := c.tablePath(schema, table) + + // 1. Read current version. + hintKey := path.Join(basePath, "metadata", "version-hint.text") + hintData, err := c.storage.GetObject(ctx, hintKey) + if err != nil { + return nil, fmt.Errorf("read version-hint: %w", err) + } + currentVersion, err := strconv.Atoi(string(hintData)) + if err != nil { + return nil, fmt.Errorf("parse version hint: %w", err) + } + + // 2. Read current metadata. + currentMetaKey := path.Join(basePath, "metadata", fmt.Sprintf("v%d.metadata.json", currentVersion)) + metaData, err := c.storage.GetObject(ctx, currentMetaKey) + if err != nil { + return nil, fmt.Errorf("read metadata v%d: %w", currentVersion, err) + } + var metadata tableMetadata + if err := json.Unmarshal(metaData, &metadata); err != nil { + return nil, fmt.Errorf("parse metadata: %w", err) + } + + if len(metadata.Schemas) == 0 { + return nil, fmt.Errorf("no schemas in metadata for %s.%s", schema, table) + } + + // 3. Build the evolved schema from the current one. + oldSchema := metadata.Schemas[metadata.CurrentSchemaID] + oldFieldsByName := make(map[string]schemaField, len(oldSchema.Fields)) + for _, f := range oldSchema.Fields { + oldFieldsByName[f.Name] = f + } + + lastColID := metadata.LastColumnID + var newFields []schemaField + + // Start with existing fields, applying DROP and TYPE_CHANGE. + dropSet := make(map[string]bool) + typeChanges := make(map[string]wal.SchemaChange) + for _, ch := range changes { + switch ch.Type { + case wal.SchemaChangeDrop: + dropSet[ch.Column] = true + case wal.SchemaChangeTypeChange: + typeChanges[ch.Column] = ch + } + } + + for _, f := range oldSchema.Fields { + if dropSet[f.Name] { + continue // dropped + } + if ch, changed := typeChanges[f.Name]; changed { + oldType := f.Type + newType := PgOIDToIcebergType(ch.NewOID) + if err := ValidateTypeWidening(oldType, newType); err != nil { + return nil, fmt.Errorf("column %q: %w", f.Name, err) + } + f.Type = newType + } + newFields = append(newFields, f) + } + + // Apply ADD columns. + for _, ch := range changes { + if ch.Type != wal.SchemaChangeAdd { + continue + } + lastColID++ + sf := schemaField{ + ID: lastColID, + Name: ch.Column, + Required: false, + Type: PgOIDToIcebergType(ch.NewOID), + } + if defaults != nil { + if defVal, ok := defaults[ch.Column]; ok && defVal != "" { + sf.InitialDefault = defVal + sf.WriteDefault = defVal + } + } + newFields = append(newFields, sf) + } + + // 4. Create new schema entry. + newSchemaID := oldSchema.SchemaID + 1 + evolvedSchema := icebergSchema{ + Type: "struct", + SchemaID: newSchemaID, + Fields: newFields, + IdentifierField: []int{}, + } + + metadata.Schemas = append(metadata.Schemas, evolvedSchema) + metadata.CurrentSchemaID = newSchemaID + metadata.LastColumnID = lastColID + metadata.LastUpdatedMS = time.Now().UnixMilli() + metadata.MetadataLog = append(metadata.MetadataLog, metadataLogEntry{ + TimestampMS: metadata.LastUpdatedMS, + MetadataFile: fmt.Sprintf("s3://%s/%s", c.bucket, currentMetaKey), + }) + + // 5. Write new metadata version. + newVersion := currentVersion + 1 + newMetadataJSON, err := json.MarshalIndent(metadata, "", " ") + if err != nil { + return nil, fmt.Errorf("marshal evolved metadata: %w", err) + } + + newMetaKey := path.Join(basePath, "metadata", fmt.Sprintf("v%d.metadata.json", newVersion)) + if err := c.storage.PutObject(ctx, newMetaKey, newMetadataJSON, "application/json"); err != nil { + return nil, err + } + if err := c.storage.PutObject(ctx, hintKey, []byte(strconv.Itoa(newVersion)), "text/plain"); err != nil { + return nil, err + } + + // 6. Build and return field ID mapping. + fieldIDs := make(map[string]int, len(newFields)) + for _, f := range newFields { + fieldIDs[f.Name] = f.ID + } + return fieldIDs, nil } // GetDataFilePaths returns the S3 paths of all data files in the current @@ -665,10 +899,12 @@ type icebergSchema struct { } type schemaField struct { - ID int `json:"id"` - Name string `json:"name"` - Required bool `json:"required"` - Type string `json:"type"` + ID int `json:"id"` + Name string `json:"name"` + Required bool `json:"required"` + Type string `json:"type"` + InitialDefault string `json:"initial-default,omitempty"` + WriteDefault string `json:"write-default,omitempty"` } type partitionSpec struct { diff --git a/internal/iceberg/catalog_test.go b/internal/iceberg/catalog_test.go index 2fcaf43..5cbf43c 100644 --- a/internal/iceberg/catalog_test.go +++ b/internal/iceberg/catalog_test.go @@ -476,7 +476,7 @@ func TestCommitChangeset_VersionHintWriteFail(t *testing.T) { {Name: "id", OID: 23}, // int4 {Name: "name", OID: 25}, // text } - if err := catalog.CreateTable(ctx, "public", "crash_test", cols); err != nil { + if _, err := catalog.CreateTable(ctx, "public", "crash_test", cols); err != nil { t.Fatalf("create table: %v", err) } @@ -554,7 +554,7 @@ func TestCommitChangeset_ManifestWriteFail(t *testing.T) { {Name: "id", OID: 23}, {Name: "name", OID: 25}, } - if err := catalog.CreateTable(ctx, "public", "manifest_test", cols); err != nil { + if _, err := catalog.CreateTable(ctx, "public", "manifest_test", cols); err != nil { t.Fatalf("create table: %v", err) } @@ -593,7 +593,7 @@ func TestCommitChangeset_MetadataWriteFail(t *testing.T) { {Name: "id", OID: 23}, {Name: "name", OID: 25}, } - if err := catalog.CreateTable(ctx, "public", "meta_test", cols); err != nil { + if _, err := catalog.CreateTable(ctx, "public", "meta_test", cols); err != nil { t.Fatalf("create table: %v", err) } @@ -630,7 +630,7 @@ func TestCommitChangeset_SuccessfulRoundTrip(t *testing.T) { {Name: "id", OID: 23}, {Name: "name", OID: 25}, } - if err := catalog.CreateTable(ctx, "public", "roundtrip", cols); err != nil { + if _, err := catalog.CreateTable(ctx, "public", "roundtrip", cols); err != nil { t.Fatalf("create table: %v", err) } diff --git a/internal/iceberg/schema.go b/internal/iceberg/schema.go index c5da6ad..6dc4807 100644 --- a/internal/iceberg/schema.go +++ b/internal/iceberg/schema.go @@ -1,5 +1,7 @@ package iceberg +import "fmt" + // IcebergType represents an Iceberg type string used in metadata JSON. type IcebergType = string @@ -17,6 +19,27 @@ const ( TypeBinary IcebergType = "binary" ) +// ValidateTypeWidening checks whether changing an Iceberg column from +// oldType to newType is a compatible widening. Returns nil if the change +// is allowed, or an error describing why it is not. +// +// Allowed widenings per the Iceberg spec: +// - int → long +// - float → double +func ValidateTypeWidening(oldType, newType IcebergType) error { + if oldType == newType { + return nil + } + switch { + case oldType == TypeInt && newType == TypeLong: + return nil + case oldType == TypeFloat && newType == TypeDouble: + return nil + default: + return fmt.Errorf("incompatible type change: %s → %s (only int→long and float→double are supported)", oldType, newType) + } +} + // PgOIDToIcebergType maps a Postgres OID to its Iceberg type string. func PgOIDToIcebergType(oid uint32) IcebergType { switch oid { diff --git a/internal/iceberg/schema_test.go b/internal/iceberg/schema_test.go index 0e64353..a557cd4 100644 --- a/internal/iceberg/schema_test.go +++ b/internal/iceberg/schema_test.go @@ -33,3 +33,28 @@ func TestPgOIDToIcebergType(t *testing.T) { } } } + +func TestValidateTypeWidening(t *testing.T) { + // Allowed widenings. + if err := ValidateTypeWidening(TypeInt, TypeLong); err != nil { + t.Errorf("int→long should be allowed: %v", err) + } + if err := ValidateTypeWidening(TypeFloat, TypeDouble); err != nil { + t.Errorf("float→double should be allowed: %v", err) + } + // Same type is always OK. + if err := ValidateTypeWidening(TypeString, TypeString); err != nil { + t.Errorf("string→string should be allowed: %v", err) + } + + // Incompatible changes should fail. + if err := ValidateTypeWidening(TypeLong, TypeInt); err == nil { + t.Error("long→int should be rejected") + } + if err := ValidateTypeWidening(TypeString, TypeInt); err == nil { + t.Error("string→int should be rejected") + } + if err := ValidateTypeWidening(TypeInt, TypeBoolean); err == nil { + t.Error("int→boolean should be rejected") + } +} diff --git a/internal/iceberg/writer.go b/internal/iceberg/writer.go index bf98f27..524b5b8 100644 --- a/internal/iceberg/writer.go +++ b/internal/iceberg/writer.go @@ -34,7 +34,11 @@ type tableBuffer struct { Table string Columns []wal.Column KeyColumns []int // positions into Columns; stamped once from first event - Rows [][]pqbuilder.Value + // fieldIDs maps column names to their Iceberg field IDs. Populated on + // first flush (from CreateTable) or schema evolution (from EvolveSchema). + // When nil, flush() falls back to positional IDs (i+1). + fieldIDs map[string]int + Rows [][]pqbuilder.Value // Deletes holds equality-delete keys, one per row. Each inner slice // is aligned with KeyColumns: Deletes[i][j] is the value of the // j-th key column for the i-th pending delete. @@ -96,6 +100,137 @@ func (w *Writer) HandleEvent(ctx context.Context, event wal.RowEvent) (transitio return transitioned, nil } +// HandleSchemaChange is called when the pipeline detects that a +// RelationMessage carries schema changes for a known table. It: +// 1. Flushes any buffered rows with the OLD schema. +// 2. Evolves the Iceberg table metadata (adds/drops/widens columns). +// 3. Updates the buffer's column list and field ID mapping. +// +// defaults maps added column names to their Postgres DEFAULT expression +// (used for Iceberg initial-default). May be nil. +func (w *Writer) HandleSchemaChange(ctx context.Context, rel *wal.RelationMessage, defaults map[string]string) error { + key := fmt.Sprintf("%s.%s", rel.Namespace, rel.Name) + + // 1. Flush existing buffer with OLD schema. + if buf, exists := w.buffers[key]; exists { + if len(buf.Rows) > 0 || len(buf.Deletes) > 0 { + w.logger.Info("flushing buffer before schema evolution", + "table", key, + "rows", len(buf.Rows), + "deletes", len(buf.Deletes), + ) + if err := w.flush(ctx, key); err != nil { + return fmt.Errorf("pre-evolution flush for %s: %w", key, err) + } + } + } + + // 2. Evolve Iceberg schema if the table already exists. + tableExists, err := w.catalog.TableExists(ctx, rel.Namespace, rel.Name) + if err != nil { + return fmt.Errorf("check table %s: %w", key, err) + } + + buf, exists := w.buffers[key] + if !exists { + buf = &tableBuffer{ + Schema: rel.Namespace, + Table: rel.Name, + } + w.buffers[key] = buf + } + + if tableExists { + fids, err := w.catalog.EvolveSchema(ctx, rel.Namespace, rel.Name, + rel.Changes, rel.Columns, defaults) + if err != nil { + return fmt.Errorf("evolve schema for %s: %w", key, err) + } + buf.fieldIDs = fids + w.logger.Info("schema evolved", + "table", key, + "changes", len(rel.Changes), + "field_ids", len(fids), + ) + } + + // 3. Update buffer with new schema. + buf.Columns = rel.Columns + buf.KeyColumns = rel.KeyColumnIndexes + + // Update state store with new column count. + w.state.RegisterTable(rel.Namespace, rel.Name, len(rel.Columns)) + + return nil +} + +// reconcileWithIceberg compares the WAL-provided columns in the buffer +// against the current Iceberg schema. Returns the field ID mapping from +// Iceberg and any schema changes detected. This handles DDL that occurred +// while the pipeline was stopped — on restart the decoder has no cached +// relation to diff against, so the writer must diff against Iceberg. +func (w *Writer) reconcileWithIceberg(ctx context.Context, buf *tableBuffer) (map[string]int, []wal.SchemaChange, error) { + iceFields, err := w.catalog.GetSchemaFields(ctx, buf.Schema, buf.Table) + if err != nil { + return nil, nil, err + } + + // Build maps for diffing. + iceByName := make(map[string]SchemaFieldInfo, len(iceFields)) + for _, f := range iceFields { + iceByName[f.Name] = f + } + walByName := make(map[string]wal.Column, len(buf.Columns)) + for _, c := range buf.Columns { + walByName[c.Name] = c + } + + var changes []wal.SchemaChange + + // Columns in Iceberg but not in WAL → DROP. + for _, f := range iceFields { + if _, exists := walByName[f.Name]; !exists { + changes = append(changes, wal.SchemaChange{ + Type: wal.SchemaChangeDrop, + Column: f.Name, + }) + } + } + + // Columns in WAL but not in Iceberg → ADD. + // Columns in both but with different type → TYPE_CHANGE. + for _, c := range buf.Columns { + iceField, exists := iceByName[c.Name] + if !exists { + changes = append(changes, wal.SchemaChange{ + Type: wal.SchemaChangeAdd, + Column: c.Name, + NewOID: c.OID, + }) + } else { + walType := PgOIDToIcebergType(c.OID) + if walType != iceField.Type { + changes = append(changes, wal.SchemaChange{ + Type: wal.SchemaChangeTypeChange, + Column: c.Name, + NewOID: c.OID, + }) + } + } + } + + // If no changes, just return the existing field IDs. + if len(changes) == 0 { + fids := make(map[string]int, len(iceFields)) + for _, f := range iceFields { + fids[f.Name] = f.ID + } + return fids, nil, nil + } + + return nil, changes, nil +} + // buffer appends an event to the per-table buffer. Returns true if this // append transitioned the buffer from empty to non-empty (i.e. FirstLSN // was just set), which the caller uses to decide whether to emit a new @@ -199,10 +334,16 @@ func (w *Writer) flush(ctx context.Context, key string) error { rowCount := len(buf.Rows) delCount := len(buf.Deletes) - // Convert full-table columns for parquet builder. + // Convert full-table columns for parquet builder, stamping field IDs + // from the buffer's mapping so Parquet columns carry stable Iceberg IDs. cols := make([]pqbuilder.ColumnDef, len(buf.Columns)) for i, c := range buf.Columns { cols[i] = pqbuilder.ColumnDef{Name: c.Name, OID: c.OID} + if buf.fieldIDs != nil { + if fid, ok := buf.fieldIDs[c.Name]; ok { + cols[i].FieldID = fid + } + } } // Ensure Iceberg table exists before writing anything. @@ -221,9 +362,31 @@ func (w *Writer) flush(ctx context.Context, key string) error { for i, c := range buf.Columns { icebergCols[i] = ColumnDef{Name: c.Name, OID: c.OID} } - if err := w.catalog.CreateTable(ctx, buf.Schema, buf.Table, icebergCols); err != nil { + fids, err := w.catalog.CreateTable(ctx, buf.Schema, buf.Table, icebergCols) + if err != nil { return fmt.Errorf("create table %s: %w", key, err) } + buf.fieldIDs = fids + } else if buf.fieldIDs == nil { + // Table exists in Iceberg but we have no field IDs cached (e.g. + // after a restart). Reconcile WAL columns against the Iceberg + // schema: if they differ, a DDL happened while we were down. + fids, changes, err := w.reconcileWithIceberg(ctx, buf) + if err != nil { + return fmt.Errorf("reconcile schema for %s: %w", key, err) + } + if len(changes) > 0 { + w.logger.Info("schema drift detected on restart, evolving Iceberg", + "table", key, + "changes", len(changes), + ) + fids, err = w.catalog.EvolveSchema(ctx, buf.Schema, buf.Table, + changes, buf.Columns, nil) + if err != nil { + return fmt.Errorf("evolve schema on restart for %s: %w", key, err) + } + } + buf.fieldIDs = fids } var dataFile *DataFile diff --git a/internal/pipeline/pipeline.go b/internal/pipeline/pipeline.go index f3700bb..c07f9ad 100644 --- a/internal/pipeline/pipeline.go +++ b/internal/pipeline/pipeline.go @@ -34,6 +34,7 @@ type Pipeline struct { tableFlushLSN map[string]pglogrepl.LSN writer *iceberg.Writer + metaQuerier *wal.MetadataQuerier // for column default lookups on schema change flushInterval time.Duration logger *slog.Logger } @@ -51,6 +52,7 @@ func New( tableFlushLSN map[string]pglogrepl.LSN, writer *iceberg.Writer, flushInterval time.Duration, + metaQuerier *wal.MetadataQuerier, ) *Pipeline { exclude := make(map[string]bool) for _, t := range excludeTables { @@ -66,6 +68,7 @@ func New( state: stateStore, tableFlushLSN: tableFlushLSN, writer: writer, + metaQuerier: metaQuerier, flushInterval: flushInterval, logger: logger, } @@ -258,6 +261,28 @@ func (p *Pipeline) Run(ctx context.Context) error { key := fmt.Sprintf("%s.%s", v.Namespace, v.Name) if p.excludeTables[key] { p.logger.Debug("excluding table", "table", key) + break + } + if len(v.Changes) > 0 { + p.logger.Info("schema change detected", + "table", key, + "changes", len(v.Changes), + ) + // Look up defaults for ADD columns. + var defaults map[string]string + if p.metaQuerier != nil { + addCols := collectAddColumns(v.Changes) + if len(addCols) > 0 { + var err error + defaults, err = p.metaQuerier.GetColumnDefaults(ctx, v.Namespace, v.Name, addCols) + if err != nil { + p.logger.Warn("could not fetch column defaults", "table", key, "error", err) + } + } + } + if err := p.writer.HandleSchemaChange(ctx, v, defaults); err != nil { + return fmt.Errorf("schema evolution for %s: %w", key, err) + } } case *wal.InsertMessage: @@ -397,3 +422,15 @@ func (p *Pipeline) sendStandby(ctx context.Context, receivedLSN pglogrepl.LSN) e ) return nil } + +// collectAddColumns returns the names of columns being added from a list +// of schema changes. +func collectAddColumns(changes []wal.SchemaChange) []string { + var cols []string + for _, ch := range changes { + if ch.Type == wal.SchemaChangeAdd { + cols = append(cols, ch.Column) + } + } + return cols +} diff --git a/internal/resync/resync.go b/internal/resync/resync.go index 5f43eeb..e28d386 100644 --- a/internal/resync/resync.go +++ b/internal/resync/resync.go @@ -137,7 +137,7 @@ func Run(ctx context.Context, opts Options) (Stats, error) { return fmt.Errorf("check table exists: %w", err) } if !exists { - if err := opts.Catalog.CreateTable(ctx, opts.Schema, opts.Table, icebergCols); err != nil { + if _, err := opts.Catalog.CreateTable(ctx, opts.Schema, opts.Table, icebergCols); err != nil { return fmt.Errorf("create iceberg table: %w", err) } } diff --git a/internal/wal/decoder.go b/internal/wal/decoder.go index 79eecf7..ab86fdf 100644 --- a/internal/wal/decoder.go +++ b/internal/wal/decoder.go @@ -26,7 +26,7 @@ func (d *Decoder) Relations() map[uint32]*RelationMessage { } // Decode takes a pglogrepl.Message and returns a typed event. -// Returns nil for messages we don't handle in Phase 1. +// Returns nil for messages we don't handle currently. func (d *Decoder) Decode(msg pglogrepl.Message) (interface{}, error) { switch m := msg.(type) { case *pglogrepl.RelationMessage: @@ -76,23 +76,110 @@ func (d *Decoder) decodeRelation(m *pglogrepl.RelationMessage) *RelationMessage keyIdx = append(keyIdx, i) } } + + // Detect schema changes if we already have a cached relation. + var changes []SchemaChange + if prev, exists := d.relations[m.RelationID]; exists { + changes = diffRelationColumns(prev.Columns, cols, prev.KeyColumnIndexes, keyIdx) + } + rel := &RelationMessage{ RelationID: m.RelationID, Namespace: m.Namespace, Name: m.RelationName, Columns: cols, KeyColumnIndexes: keyIdx, + Changes: changes, } d.relations[m.RelationID] = rel - d.logger.Info("relation discovered", - "schema", rel.Namespace, - "table", rel.Name, - "columns", len(rel.Columns), - "key_columns", len(keyIdx), - ) + + if len(changes) > 0 { + d.logger.Info("relation schema changed", + "schema", rel.Namespace, + "table", rel.Name, + "columns", len(rel.Columns), + "key_columns", len(keyIdx), + "changes", len(changes), + ) + } else { + d.logger.Info("relation discovered", + "schema", rel.Namespace, + "table", rel.Name, + "columns", len(rel.Columns), + "key_columns", len(keyIdx), + ) + } return rel } +// diffRelationColumns compares old and new column lists by name to detect +// ADD, DROP, and TYPE_CHANGE. It also detects REPLICA IDENTITY key changes. +func diffRelationColumns(oldCols, newCols []Column, oldKeyIdx, newKeyIdx []int) []SchemaChange { + var changes []SchemaChange + + oldByName := make(map[string]Column, len(oldCols)) + for _, c := range oldCols { + oldByName[c.Name] = c + } + newByName := make(map[string]Column, len(newCols)) + for _, c := range newCols { + newByName[c.Name] = c + } + + // Detect DROP and TYPE_CHANGE. + for _, old := range oldCols { + if new, exists := newByName[old.Name]; exists { + if old.OID != new.OID { + changes = append(changes, SchemaChange{ + Type: SchemaChangeTypeChange, + Column: old.Name, + OldOID: old.OID, + NewOID: new.OID, + }) + } + } else { + changes = append(changes, SchemaChange{ + Type: SchemaChangeDrop, + Column: old.Name, + OldOID: old.OID, + }) + } + } + + // Detect ADD. + for _, new := range newCols { + if _, exists := oldByName[new.Name]; !exists { + changes = append(changes, SchemaChange{ + Type: SchemaChangeAdd, + Column: new.Name, + NewOID: new.OID, + }) + } + } + + // Detect KEY_CHANGE (replica identity change). + if !equalIntSlice(oldKeyIdx, newKeyIdx) { + changes = append(changes, SchemaChange{ + Type: SchemaChangeKeyChange, + Column: "", // applies to the whole table + }) + } + + return changes +} + +func equalIntSlice(a, b []int) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + // decodeTuple turns a pgoutput TupleData into ColumnValue slice using the // relation's column metadata for names/OIDs. Unchanged-TOAST markers are // translated to NULL to keep Phase 1 semantics; the caller is responsible diff --git a/internal/wal/decoder_test.go b/internal/wal/decoder_test.go index 92d5642..1eeb9c8 100644 --- a/internal/wal/decoder_test.go +++ b/internal/wal/decoder_test.go @@ -434,3 +434,140 @@ func TestDecodeUpdate_UnknownRelation(t *testing.T) { t.Error("expected error for unknown relation") } } + +func TestDiffRelationColumns_AddColumn(t *testing.T) { + old := []Column{ + {Name: "id", OID: 23}, + {Name: "name", OID: 25}, + } + new := []Column{ + {Name: "id", OID: 23}, + {Name: "name", OID: 25}, + {Name: "email", OID: 25}, + } + changes := diffRelationColumns(old, new, []int{0}, []int{0}) + if len(changes) != 1 { + t.Fatalf("expected 1 change, got %d: %+v", len(changes), changes) + } + if changes[0].Type != SchemaChangeAdd || changes[0].Column != "email" { + t.Errorf("expected ADD email, got %+v", changes[0]) + } +} + +func TestDiffRelationColumns_DropColumn(t *testing.T) { + old := []Column{ + {Name: "id", OID: 23}, + {Name: "name", OID: 25}, + {Name: "email", OID: 25}, + } + new := []Column{ + {Name: "id", OID: 23}, + {Name: "name", OID: 25}, + } + changes := diffRelationColumns(old, new, []int{0}, []int{0}) + if len(changes) != 1 { + t.Fatalf("expected 1 change, got %d: %+v", len(changes), changes) + } + if changes[0].Type != SchemaChangeDrop || changes[0].Column != "email" { + t.Errorf("expected DROP email, got %+v", changes[0]) + } +} + +func TestDiffRelationColumns_TypeChange(t *testing.T) { + old := []Column{ + {Name: "id", OID: 23}, // int4 + {Name: "count", OID: 23}, + } + new := []Column{ + {Name: "id", OID: 23}, + {Name: "count", OID: 20}, // int8 + } + changes := diffRelationColumns(old, new, []int{0}, []int{0}) + if len(changes) != 1 { + t.Fatalf("expected 1 change, got %d: %+v", len(changes), changes) + } + if changes[0].Type != SchemaChangeTypeChange || changes[0].OldOID != 23 || changes[0].NewOID != 20 { + t.Errorf("expected TYPE_CHANGE int4→int8, got %+v", changes[0]) + } +} + +func TestDiffRelationColumns_KeyChange(t *testing.T) { + cols := []Column{ + {Name: "id", OID: 23}, + {Name: "name", OID: 25}, + } + changes := diffRelationColumns(cols, cols, []int{0}, []int{0, 1}) + found := false + for _, ch := range changes { + if ch.Type == SchemaChangeKeyChange { + found = true + } + } + if !found { + t.Error("expected KEY_CHANGE, got none") + } +} + +func TestDiffRelationColumns_NoChanges(t *testing.T) { + cols := []Column{ + {Name: "id", OID: 23}, + {Name: "name", OID: 25}, + } + changes := diffRelationColumns(cols, cols, []int{0}, []int{0}) + if len(changes) != 0 { + t.Errorf("expected no changes, got %+v", changes) + } +} + +func TestDecodeRelation_DetectsSchemaChange(t *testing.T) { + d := testDecoder() + + // First relation — no changes expected. + d.Decode(&pglogrepl.RelationMessage{ + RelationID: 16384, + Namespace: "public", + RelationName: "orders", + Columns: []*pglogrepl.RelationMessageColumn{ + {Name: "id", DataType: 23}, + {Name: "name", DataType: 25}, + }, + }) + + // Second relation with same ID but ADD COLUMN. + result, _ := d.Decode(&pglogrepl.RelationMessage{ + RelationID: 16384, + Namespace: "public", + RelationName: "orders", + Columns: []*pglogrepl.RelationMessageColumn{ + {Name: "id", DataType: 23}, + {Name: "name", DataType: 25}, + {Name: "email", DataType: 25}, + }, + }) + + rel := result.(*RelationMessage) + if len(rel.Changes) != 1 { + t.Fatalf("expected 1 change, got %d", len(rel.Changes)) + } + if rel.Changes[0].Type != SchemaChangeAdd || rel.Changes[0].Column != "email" { + t.Errorf("expected ADD email, got %+v", rel.Changes[0]) + } +} + +func TestDecodeRelation_FirstDiscoveryNoChanges(t *testing.T) { + d := testDecoder() + + result, _ := d.Decode(&pglogrepl.RelationMessage{ + RelationID: 16384, + Namespace: "public", + RelationName: "orders", + Columns: []*pglogrepl.RelationMessageColumn{ + {Name: "id", DataType: 23}, + }, + }) + + rel := result.(*RelationMessage) + if len(rel.Changes) != 0 { + t.Errorf("expected no changes on first discovery, got %+v", rel.Changes) + } +} diff --git a/internal/wal/metadata.go b/internal/wal/metadata.go new file mode 100644 index 0000000..2c035be --- /dev/null +++ b/internal/wal/metadata.go @@ -0,0 +1,66 @@ +package wal + +import ( + "context" + "fmt" + + "github.com/jackc/pgx/v5" +) + +// MetadataQuerier provides read-only access to Postgres catalog metadata +// (column defaults, type info) over a regular (non-replication) connection. +type MetadataQuerier struct { + conn *pgx.Conn +} + +// NewMetadataQuerier wraps an existing pgx connection for metadata queries. +func NewMetadataQuerier(conn *pgx.Conn) *MetadataQuerier { + return &MetadataQuerier{conn: conn} +} + +// GetColumnDefaults returns the default expressions for the specified columns +// of a table. The result maps column name → default expression as text +// (e.g. "42", "'hello'::text"). Columns without defaults are omitted. +func (m *MetadataQuerier) GetColumnDefaults(ctx context.Context, schema, table string, columns []string) (map[string]string, error) { + if len(columns) == 0 { + return nil, nil + } + + // Query pg_attrdef joined with pg_attribute and pg_class to get + // column default expressions for the specified columns. + query := ` + SELECT a.attname, pg_get_expr(d.adbin, d.adrelid) AS default_expr + FROM pg_attribute a + JOIN pg_class c ON c.oid = a.attrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_attrdef d ON d.adrelid = a.attrelid AND d.adnum = a.attnum + WHERE n.nspname = $1 + AND c.relname = $2 + AND a.attname = ANY($3) + AND NOT a.attisdropped + ` + + rows, err := m.conn.Query(ctx, query, schema, table, columns) + if err != nil { + return nil, fmt.Errorf("query column defaults: %w", err) + } + defer rows.Close() + + defaults := make(map[string]string) + for rows.Next() { + var colName, defaultExpr string + if err := rows.Scan(&colName, &defaultExpr); err != nil { + return nil, fmt.Errorf("scan column default: %w", err) + } + defaults[colName] = defaultExpr + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate column defaults: %w", err) + } + return defaults, nil +} + +// Close closes the underlying connection. +func (m *MetadataQuerier) Close(ctx context.Context) error { + return m.conn.Close(ctx) +} diff --git a/internal/wal/types.go b/internal/wal/types.go index bc7e82f..39e4d27 100644 --- a/internal/wal/types.go +++ b/internal/wal/types.go @@ -2,6 +2,41 @@ package wal import "github.com/jackc/pglogrepl" +// SchemaChangeType identifies the kind of schema modification detected +// when comparing consecutive RelationMessages for the same table. +type SchemaChangeType int + +const ( + SchemaChangeAdd SchemaChangeType = iota // new column added + SchemaChangeDrop // existing column removed + SchemaChangeTypeChange // column OID changed (ALTER TYPE) + SchemaChangeKeyChange // REPLICA IDENTITY columns changed +) + +func (t SchemaChangeType) String() string { + switch t { + case SchemaChangeAdd: + return "ADD" + case SchemaChangeDrop: + return "DROP" + case SchemaChangeTypeChange: + return "TYPE_CHANGE" + case SchemaChangeKeyChange: + return "KEY_CHANGE" + default: + return "UNKNOWN" + } +} + +// SchemaChange describes a single column-level change between two +// RelationMessages for the same table. +type SchemaChange struct { + Type SchemaChangeType + Column string // column name (new name for ADD, old name for DROP) + OldOID uint32 // previous OID (0 for ADD) + NewOID uint32 // new OID (0 for DROP) +} + // Column describes a single column from a Postgres Relation message. // IsKey is true when the column is part of the table's REPLICA IDENTITY // (the "key" flag in pgoutput). These are the columns we can safely use @@ -25,6 +60,9 @@ type RelationMessage struct { Name string Columns []Column KeyColumnIndexes []int + // Changes is populated when this RelationMessage represents a schema + // modification of a previously-known table. Empty for first discovery. + Changes []SchemaChange } // ColumnValue holds a single decoded column value from a WAL tuple. diff --git a/test/integration/integration_test.go b/test/integration/integration_test.go index d790594..21885ed 100644 --- a/test/integration/integration_test.go +++ b/test/integration/integration_test.go @@ -20,6 +20,7 @@ import ( "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/jackc/pglogrepl" + "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" _ "github.com/marcboeker/go-duckdb" pqgo "github.com/parquet-go/parquet-go" @@ -220,9 +221,17 @@ func runSync(t *testing.T, ctx context.Context, duration time.Duration, statePat writer := iceberg.NewWriter(catalog, s3Client, stateStore, slotName, flushRows, 5*time.Second, logger) + // Open metadata connection for schema evolution queries. + metaConn, err := pgx.Connect(ctx, pgConnStr()) + if err != nil { + t.Fatalf("connect metadata: %v", err) + } + defer metaConn.Close(context.Background()) + metaQuerier := wal.NewMetadataQuerier(metaConn) + // Create unified pipeline p := pipeline.New(pgConn, slotName, slotName, startLSN, nil, - logger, stateStore, tableFlushLSN, writer, 5*time.Second) + logger, stateStore, tableFlushLSN, writer, 5*time.Second, metaQuerier) // Run with timeout syncCtx, cancel := context.WithTimeout(ctx, duration) diff --git a/test/integration/schema_test.go b/test/integration/schema_test.go index 4193f8c..8e1bb68 100644 --- a/test/integration/schema_test.go +++ b/test/integration/schema_test.go @@ -4,6 +4,8 @@ package integration import ( "context" + "database/sql" + "fmt" "testing" "time" ) @@ -55,6 +57,24 @@ func TestSchemaEvolution_AddColumn(t *testing.T) { t.Fatalf("row count mismatch after ADD COLUMN: PG=%d Iceberg=%d", pgCount, icebergCount) } + // Verify via DuckDB that the new column exists and new rows have the value. + duckDB := newTestDuckDB(t) + icebergPath := fmt.Sprintf("s3://%s/%s/public/test_events/metadata/version-hint.text", s3Bucket, s3Prefix) + query := fmt.Sprintf( + "SELECT extra_info FROM iceberg_scan('%s', allow_moved_paths := true) WHERE extra_info IS NOT NULL", + icebergPath, + ) + var extraCount int64 + err := duckDB.QueryRow(fmt.Sprintf("SELECT count(*) FROM (%s)", query)).Scan(&extraCount) + if err != nil { + t.Logf("DuckDB column query failed (may not support evolved schema yet): %v", err) + } else { + t.Logf("rows with extra_info populated: %d", extraCount) + if extraCount != 10 { + t.Errorf("expected 10 rows with extra_info, got %d", extraCount) + } + } + cleanup(t) } @@ -116,3 +136,240 @@ func TestSchemaEvolution_DropColumn(t *testing.T) { cleanup(t) execSQL(t, "DROP TABLE IF EXISTS drop_col_test") } + +// TestSchemaEvolution_TypeWidening verifies that ALTER COLUMN TYPE from +// int4 to int8 (int → long in Iceberg) is handled correctly. +func TestSchemaEvolution_TypeWidening(t *testing.T) { + skipIfNotAvailable(t) + ctx := context.Background() + + cleanup(t) + clearS3Prefix(t) + + execSQL(t, "DROP TABLE IF EXISTS widen_test") + execSQL(t, `CREATE TABLE widen_test ( + id SERIAL PRIMARY KEY, + count INT NOT NULL + )`) + + createSlotAndPublication(t) + sharedStatePath := t.TempDir() + "/state.db" + + // Phase 1: Insert rows with int4. + t.Log("phase 1: inserting rows with INT...") + for i := 0; i < 5; i++ { + execSQL(t, fmt.Sprintf("INSERT INTO widen_test (count) VALUES (%d)", i+1)) + } + + t.Log("syncing phase 1...") + runSync(t, ctx, 12*time.Second, sharedStatePath) + + rows := countNamedTableSnapshotRows(t, "public", "widen_test") + if rows != 5 { + t.Fatalf("phase 1: expected 5 rows, got %d", rows) + } + + // Phase 2: Widen to BIGINT and insert more rows. + t.Log("phase 2: ALTER COLUMN TYPE to BIGINT...") + execSQL(t, "ALTER TABLE widen_test ALTER COLUMN count TYPE BIGINT") + + for i := 0; i < 5; i++ { + execSQL(t, fmt.Sprintf("INSERT INTO widen_test (count) VALUES (%d)", 1000000000+i)) + } + + t.Log("syncing phase 2...") + runSync(t, ctx, 12*time.Second, sharedStatePath) + + pgCount := pgRowCount(t, "widen_test") + icebergCount := countNamedTableSnapshotRows(t, "public", "widen_test") + + t.Logf("after TYPE WIDENING: PG=%d Iceberg=%d", pgCount, icebergCount) + if pgCount != icebergCount { + t.Fatalf("row count mismatch after TYPE WIDENING: PG=%d Iceberg=%d", pgCount, icebergCount) + } + + cleanup(t) + execSQL(t, "DROP TABLE IF EXISTS widen_test") +} + +// TestSchemaEvolution_AddColumnWithCoW verifies that ADD COLUMN followed by +// UPDATE (which triggers CoW merge with mixed schemas) works correctly. +func TestSchemaEvolution_AddColumnWithCoW(t *testing.T) { + skipIfNotAvailable(t) + ctx := context.Background() + + cleanup(t) + clearS3Prefix(t) + + execSQL(t, "DROP TABLE IF EXISTS cow_schema_test") + execSQL(t, `CREATE TABLE cow_schema_test ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + value DOUBLE PRECISION + )`) + + createSlotAndPublication(t) + sharedStatePath := t.TempDir() + "/state.db" + + // Phase 1: Insert rows. + t.Log("phase 1: inserting 5 rows...") + for i := 0; i < 5; i++ { + execSQL(t, fmt.Sprintf("INSERT INTO cow_schema_test (name, value) VALUES ('row_%d', %d.0)", i, i)) + } + + t.Log("syncing phase 1...") + runSync(t, ctx, 12*time.Second, sharedStatePath) + + rows := countNamedTableSnapshotRows(t, "public", "cow_schema_test") + if rows != 5 { + t.Fatalf("phase 1: expected 5 rows, got %d", rows) + } + + // Phase 2: ADD COLUMN, then UPDATE existing rows (triggers CoW with mixed schemas). + t.Log("phase 2: ADD COLUMN then UPDATE...") + execSQL(t, "ALTER TABLE cow_schema_test ADD COLUMN extra TEXT") + + // Update existing rows — this triggers CoW merge. + for i := 0; i < 5; i++ { + execSQL(t, fmt.Sprintf("UPDATE cow_schema_test SET extra = 'updated', value = %d.5 WHERE id = %d", i, i+1)) + } + + // Insert new rows with the new column. + for i := 0; i < 5; i++ { + execSQL(t, fmt.Sprintf("INSERT INTO cow_schema_test (name, value, extra) VALUES ('new_%d', %d.0, 'new')", i, i+10)) + } + + t.Log("syncing phase 2...") + runSync(t, ctx, 12*time.Second, sharedStatePath) + + pgCount := pgRowCount(t, "cow_schema_test") + icebergCount := countNamedTableSnapshotRows(t, "public", "cow_schema_test") + + t.Logf("after ADD COLUMN + CoW: PG=%d Iceberg=%d", pgCount, icebergCount) + if pgCount != icebergCount { + t.Fatalf("row count mismatch after ADD COLUMN + CoW: PG=%d Iceberg=%d", pgCount, icebergCount) + } + + cleanup(t) + execSQL(t, "DROP TABLE IF EXISTS cow_schema_test") +} + +// TestSchemaEvolution_MultipleChanges verifies that ADD COLUMN followed by +// DROP COLUMN in sequence maintains stable field IDs and correct data. +func TestSchemaEvolution_MultipleChanges(t *testing.T) { + skipIfNotAvailable(t) + ctx := context.Background() + + cleanup(t) + clearS3Prefix(t) + + execSQL(t, "DROP TABLE IF EXISTS multi_schema_test") + execSQL(t, `CREATE TABLE multi_schema_test ( + id SERIAL PRIMARY KEY, + col_a TEXT NOT NULL, + col_b TEXT NOT NULL + )`) + + createSlotAndPublication(t) + sharedStatePath := t.TempDir() + "/state.db" + + // Phase 1: Insert rows. + for i := 0; i < 5; i++ { + execSQL(t, fmt.Sprintf("INSERT INTO multi_schema_test (col_a, col_b) VALUES ('a_%d', 'b_%d')", i, i)) + } + runSync(t, ctx, 12*time.Second, sharedStatePath) + + rows := countNamedTableSnapshotRows(t, "public", "multi_schema_test") + if rows != 5 { + t.Fatalf("phase 1: expected 5 rows, got %d", rows) + } + + // Phase 2: ADD col_c, then DROP col_b. + t.Log("phase 2: ADD col_c, DROP col_b...") + execSQL(t, "ALTER TABLE multi_schema_test ADD COLUMN col_c TEXT DEFAULT 'new'") + execSQL(t, "ALTER TABLE multi_schema_test DROP COLUMN col_b") + + for i := 5; i < 10; i++ { + execSQL(t, fmt.Sprintf("INSERT INTO multi_schema_test (col_a, col_c) VALUES ('a_%d', 'c_%d')", i, i)) + } + + runSync(t, ctx, 12*time.Second, sharedStatePath) + + pgCount := pgRowCount(t, "multi_schema_test") + icebergCount := countNamedTableSnapshotRows(t, "public", "multi_schema_test") + + t.Logf("after multiple schema changes: PG=%d Iceberg=%d", pgCount, icebergCount) + if pgCount != icebergCount { + t.Fatalf("row count mismatch: PG=%d Iceberg=%d", pgCount, icebergCount) + } + + cleanup(t) + execSQL(t, "DROP TABLE IF EXISTS multi_schema_test") +} + +// TestSchemaEvolution_AddColumnWithDefault verifies that ADD COLUMN with DEFAULT +// sets the Iceberg initial-default so that spec-compliant readers return the +// default value for pre-existing rows. +func TestSchemaEvolution_AddColumnWithDefault(t *testing.T) { + skipIfNotAvailable(t) + ctx := context.Background() + + cleanup(t) + clearS3Prefix(t) + + execSQL(t, "DROP TABLE IF EXISTS default_test") + execSQL(t, `CREATE TABLE default_test ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL + )`) + + createSlotAndPublication(t) + sharedStatePath := t.TempDir() + "/state.db" + + // Phase 1: Insert rows. + for i := 0; i < 5; i++ { + execSQL(t, fmt.Sprintf("INSERT INTO default_test (name) VALUES ('row_%d')", i)) + } + runSync(t, ctx, 12*time.Second, sharedStatePath) + + // Phase 2: ADD COLUMN with DEFAULT. + execSQL(t, "ALTER TABLE default_test ADD COLUMN status TEXT DEFAULT 'active'") + for i := 5; i < 10; i++ { + execSQL(t, fmt.Sprintf("INSERT INTO default_test (name) VALUES ('row_%d')", i)) + } + runSync(t, ctx, 12*time.Second, sharedStatePath) + + pgCount := pgRowCount(t, "default_test") + icebergCount := countNamedTableSnapshotRows(t, "public", "default_test") + + t.Logf("after ADD COLUMN with DEFAULT: PG=%d Iceberg=%d", pgCount, icebergCount) + if pgCount != icebergCount { + t.Fatalf("row count mismatch: PG=%d Iceberg=%d", pgCount, icebergCount) + } + + // Verify via DuckDB that the Iceberg metadata has the initial-default set. + // Note: DuckDB may or may not honor initial-default. We verify the metadata + // was written correctly by checking that new rows have the column. + duckDB := newTestDuckDB(t) + icebergPath := fmt.Sprintf("s3://%s/%s/public/default_test/metadata/version-hint.text", s3Bucket, s3Prefix) + var statusCount int64 + err := duckDB.QueryRow(fmt.Sprintf( + "SELECT count(*) FROM iceberg_scan('%s', allow_moved_paths := true) WHERE status IS NOT NULL", + icebergPath, + )).Scan(&statusCount) + if err != nil { + t.Logf("DuckDB query for default column: %v", err) + } else { + // At minimum, the 5 new rows should have status populated. + t.Logf("rows with status NOT NULL: %d", statusCount) + if statusCount < 5 { + t.Errorf("expected at least 5 rows with status, got %d", statusCount) + } + } + + cleanup(t) + execSQL(t, "DROP TABLE IF EXISTS default_test") +} + +// Suppress unused import warning for sql package used in DuckDB queries. +var _ = (*sql.DB)(nil)