From 6b4721df0d91d63467378b50ef0cc84802e6dcb6 Mon Sep 17 00:00:00 2001 From: Ashley Jeffs Date: Tue, 21 Jul 2026 15:24:15 +0100 Subject: [PATCH 01/12] iceberg: add copy-on-write merge strategy to the output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The row-level upsert/delete path writes merge-on-read equality-delete files, which several query engines can't read — notably Snowflake and the Databricks Unity Catalog, and UC also rejects tables carrying identifier-field-ids. This adds an opt-in `merge_strategy: copy-on-write` that rewrites whole data files so the table only ever holds plain data files any engine can read, and skips registering identifier-field-ids on auto-created tables. The default stays merge-on-read, so existing configs are unaffected. copy-on-write trades write amplification for read compatibility (each mutating batch rewrites every data file holding a touched key), so it's a batch/moderate-throughput mode. The prototype supports flat-primitive schemas with int/long/string/boolean merge keys on unpartitioned tables; other cases fail loudly or stay on merge-on-read. New columns in a mutating batch trigger schema evolution (parity with the append path), and files left behind by a failed copy-on-write commit are cleaned up. --- .../components/pages/outputs/iceberg.adoc | 26 +- internal/impl/iceberg/committer.go | 172 ++++++ internal/impl/iceberg/config.go | 10 +- internal/impl/iceberg/cow.go | 430 ++++++++++++++ .../iceberg/cow_amplification_bench_test.go | 347 +++++++++++ internal/impl/iceberg/cow_test.go | 556 ++++++++++++++++++ .../cow_row_operation_integration_test.go | 137 +++++ internal/impl/iceberg/output_iceberg.go | 13 + internal/impl/iceberg/router.go | 8 +- internal/impl/iceberg/row_operation_test.go | 6 + internal/impl/iceberg/writer.go | 40 +- 11 files changed, 1736 insertions(+), 9 deletions(-) create mode 100644 internal/impl/iceberg/cow.go create mode 100644 internal/impl/iceberg/cow_amplification_bench_test.go create mode 100644 internal/impl/iceberg/cow_test.go create mode 100644 internal/impl/iceberg/integration/cow_row_operation_integration_test.go diff --git a/docs/modules/components/pages/outputs/iceberg.adoc b/docs/modules/components/pages/outputs/iceberg.adoc index aa869cf885..4e55392e19 100644 --- a/docs/modules/components/pages/outputs/iceberg.adoc +++ b/docs/modules/components/pages/outputs/iceberg.adoc @@ -122,6 +122,7 @@ output: case_sensitive_columns: true row_operation: insert identifier_fields: [] + merge_strategy: merge-on-read storage: aws_s3: bucket: my-iceberg-data # No default (required) @@ -231,7 +232,9 @@ By default this output is append-only — every message becomes a new row (`row_ `row_operation` supports interpolation, so the operation can be driven by the data itself — for example by mapping a change-data-capture stream's operation field — but no CDC-specific format is assumed (see the change-data-capture example below). It is named `row_operation` to distinguish it from Iceberg's snapshot-level operation. -`upsert` and `delete` require `identifier_fields` and use Iceberg merge-on-read equality deletes, which require table format version 2. A version-1 table is automatically upgraded to version 2 on the first `upsert`/`delete`; *this upgrade is irreversible*. +`upsert` and `delete` require `identifier_fields`. By default (`merge_strategy: merge-on-read`) they use Iceberg merge-on-read equality deletes, which require table format version 2. A version-1 table is automatically upgraded to version 2 on the first `upsert`/`delete`; *this upgrade is irreversible*. + +*Merge strategy.* `merge_strategy` controls how mutations are materialised. `merge-on-read` (the default) writes equality-delete files and is the streaming path, but only catalog-only/Flink-world engines can read the result. `copy-on-write` instead rewrites whole data files so the table only ever holds plain data files that every engine can read — including engine-backed catalogs such as Snowflake and the Databricks Unity Catalog — at the cost of heavy write amplification (each mutating batch rewrites every data file holding a touched key). Treat `copy-on-write` as a batch / moderate-throughput mode: sort the table by the identifier key and use large batches. Under `copy-on-write` the `identifier_fields` are the connector-side merge key only and are not registered as the table's Iceberg identifier-field-ids, so auto-created tables carry no identifier-field spec and columns are not forced required. This prototype's `copy-on-write` path supports tables whose columns are all flat, primitive types; a table with nested (struct/list/map) columns, or a merge key that is not an `int`/`long`/`string`/`boolean` column, is rejected with a clear error at write time. *Identifier fields.* `identifier_fields` must reference existing table columns of a primitive, non-floating-point type. A static `upsert`/`delete` is validated at startup; an interpolated `row_operation` is validated per message at write time, so an empty `identifier_fields` is not caught until the first `upsert`/`delete` message arrives. Identifier columns of a temporal type (`timestamp`, `timestamptz`, `date`, `time`) must arrive as time values, not bare numbers — a numeric epoch is ambiguous as a delete key and is rejected at write time; convert it to a timestamp upstream. If the table is partitioned, every partition source column must be one of the `identifier_fields`, since equality deletes are partition-scoped. @@ -683,6 +686,27 @@ identifier_fields: - user_id ``` +=== `merge_strategy` + +How `upsert` and `delete` are materialised on disk. + +* `merge-on-read` (the default) writes Iceberg v2 equality-delete files. Deletes are applied at read time, so writes stay cheap and streaming-friendly, but only catalog-only/Flink-world engines can read the result — engine-backed catalogs such as Snowflake and the Databricks Unity Catalog cannot read equality deletes. +* `copy-on-write` rewrites whole data files so the table only ever contains plain data files (no delete files), which every engine can read — including Snowflake and Databricks Unity Catalog. The trade-off is heavy write amplification: each mutating batch rewrites every data file that contains a touched key. This is a batch / moderate-throughput mode, not a streaming one. Sort the table by the identifier key and use large batches so each rewrite touches as few files as possible. + +Under `copy-on-write` the `identifier_fields` are used only connector-side as the merge key and are *not* registered as the table's Iceberg identifier-field-ids, so auto-created tables carry no identifier-field spec (this is what lets the Databricks Unity Catalog accept the `CREATE TABLE`). `merge-on-read` continues to register them. + +See the <> section above for more detail. + + +*Type*: `string` + +*Default*: `"merge-on-read"` + +Options: +`merge-on-read` +, `copy-on-write` +. + === `storage` Storage backend configuration for data files. Exactly one of `aws_s3`, `gcp_cloud_storage`, or `azure_blob_storage` must be specified. diff --git a/internal/impl/iceberg/committer.go b/internal/impl/iceberg/committer.go index 56749d749c..b07573ea9f 100644 --- a/internal/impl/iceberg/committer.go +++ b/internal/impl/iceberg/committer.go @@ -12,12 +12,16 @@ import ( "context" "errors" "fmt" + "io/fs" "strconv" + "strings" "sync" "time" + "github.com/apache/arrow-go/v18/arrow/array" "github.com/apache/iceberg-go" "github.com/apache/iceberg-go/catalog/rest" + iceio "github.com/apache/iceberg-go/io" "github.com/apache/iceberg-go/table" "github.com/redpanda-data/benthos/v4/public/service" @@ -42,6 +46,18 @@ type CommitInput struct { SchemaID int } +// OverwriteInput describes a copy-on-write mutation applied as one atomic +// snapshot. Filter selects the existing rows to remove. NewReader, when +// non-nil, is a factory that builds the rows to (re)write; it is a factory +// rather than a reader because array.RecordReader is consumed once and the +// commit stage may run more than once on retry. A nil NewReader is a +// delete-only mutation (no rows written). +type OverwriteInput struct { + Filter iceberg.BooleanExpression + NewReader func() (array.RecordReader, error) + SchemaID int +} + // CommitConfig holds configuration for the committer. type CommitConfig struct { ManifestMergeEnabled bool @@ -184,6 +200,162 @@ func (c *committer) commitRowDelta(ctx context.Context, input CommitInput) error return nil } +// commitOverwrite applies a copy-on-write mutation as one atomic snapshot, +// outside the batcher so it is never coalesced with another commit. When +// input.NewReader is nil it is a delete-only mutation (txn.Delete); otherwise +// it is an overwrite that deletes the rows matching input.Filter and appends +// the reader's rows in a single snapshot (txn.Overwrite). Both produce only +// plain data files — no equality- or positional-delete files — so the result +// is readable by engine-backed catalogs (Snowflake, Databricks Unity Catalog). +func (c *committer) commitOverwrite(ctx context.Context, input OverwriteInput) error { + c.commitMu.Lock() + defer c.commitMu.Unlock() + + currentSchemaID := c.currentSchemaID() + if input.SchemaID != currentSchemaID { + return &StaleSchemaError{WriterSchemaID: input.SchemaID, CurrentSchemaID: currentSchemaID} + } + + // Copy-on-write writes its rewritten and new data files to storage before the + // catalog commit (inside txn.Overwrite/Delete), and — unlike the writer- + // authored append/row-delta paths — we never hold their paths. Snapshot the + // data files present beforehand so a failed commit's leftovers can be + // removed. nil means the filesystem can't be listed, so cleanup is skipped. + before := c.dataFilePaths(ctx) + + // retryOnUnknownState is false, matching commitRowDelta: the overwrite is + // not yet idempotent across a reload, so retrying a possibly-landed commit + // could duplicate it. + err := c.commitLocked(ctx, false, func(txn *table.Transaction, props iceberg.Properties, _ bool) error { + // txn.Delete branches on the table's write.delete.mode; the library + // default is already copy-on-write, but set it explicitly for safety so + // the delete-only path can never fall into merge-on-read. txn.Overwrite + // is always copy-on-write regardless of the property. + if c.table.Properties()[table.WriteDeleteModeKey] != table.WriteModeCopyOnWrite { + if err := txn.SetProperties(iceberg.Properties{table.WriteDeleteModeKey: table.WriteModeCopyOnWrite}); err != nil { + return fmt.Errorf("setting %s: %w", table.WriteDeleteModeKey, err) + } + } + if input.NewReader == nil { + return txn.Delete(ctx, input.Filter, props) + } + rdr, err := input.NewReader() + if err != nil { + return err + } + defer rdr.Release() + return txn.Overwrite(ctx, rdr, props, table.WithOverwriteFilter(input.Filter)) + }) + if err != nil { + // Clean up orphaned files only when the commit definitely did not land. + // On an unknown/ambiguous state the written files may belong to a + // snapshot that committed server-side, so removing them would corrupt the + // table — leave those for Iceberg orphan-file maintenance. + if before != nil && !errors.Is(err, rest.ErrCommitStateUnknown) { + c.cleanupOrphanedOverwriteFiles(ctx, before) + } + return err + } + c.logger.Debugf("Committed copy-on-write mutation (delete-only=%t)", input.NewReader == nil) + return nil +} + +// dataFilePaths returns the set of .parquet paths currently under the table's +// data directory, or nil if the filesystem doesn't support listing (in which +// case copy-on-write orphan cleanup is skipped). Best-effort: any walk error +// yields nil. The scan is O(files), but the copy-on-write commit it guards +// already scans every file's metadata, so it does not change that path's order. +func (c *committer) dataFilePaths(ctx context.Context) map[string]struct{} { + fsys, err := c.table.FS(ctx) + if err != nil { + return nil + } + lister, ok := fsys.(iceio.ListableIO) + if !ok { + return nil + } + paths := make(map[string]struct{}) + if walkErr := lister.WalkDir(c.table.Location()+"/data", func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() && strings.HasSuffix(p, ".parquet") { + paths[p] = struct{}{} + } + return nil + }); walkErr != nil { + return nil + } + return paths +} + +// cleanupOrphanedOverwriteFiles removes .parquet files a failed copy-on-write +// commit left under the data directory: those that appeared since the `before` +// snapshot and are not referenced by the current snapshot. The reference check +// is a safety net so a file a committed snapshot still points to is never +// deleted. Best-effort — errors are logged, not returned. The caller must have +// established that the failed commit did not land. +func (c *committer) cleanupOrphanedOverwriteFiles(ctx context.Context, before map[string]struct{}) { + fsys, err := c.table.FS(ctx) + if err != nil { + return + } + lister, ok := fsys.(iceio.ListableIO) + if !ok { + return + } + referenced := c.referencedDataFilePaths(ctx) + _ = lister.WalkDir(c.table.Location()+"/data", func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(p, ".parquet") { + return nil + } + if _, existed := before[p]; existed { + return nil + } + if _, ref := referenced[p]; ref { + return nil + } + if rmErr := fsys.Remove(p); rmErr != nil { + c.logger.Warnf("Failed to remove orphaned copy-on-write file %s: %v", p, rmErr) + } else { + c.logger.Debugf("Removed orphaned copy-on-write file %s", p) + } + return nil + }) +} + +// referencedDataFilePaths returns the paths referenced by the table's current +// snapshot, used to guard orphan cleanup. Best-effort: on any error it returns +// what it has so far, and an empty set simply disables the reference guard +// (leaving the appeared-since-`before` guard to do the work). +func (c *committer) referencedDataFilePaths(ctx context.Context) map[string]struct{} { + refs := make(map[string]struct{}) + snap := c.table.CurrentSnapshot() + if snap == nil { + return refs + } + fsys, err := c.table.FS(ctx) + if err != nil { + return refs + } + manifests, err := snap.Manifests(fsys) + if err != nil { + return refs + } + for _, m := range manifests { + for entry, err := range m.Entries(fsys, true) { + if err != nil { + return refs + } + refs[entry.DataFile().FilePath()] = struct{}{} + } + } + return refs +} + // commitLocked stages a transaction via stage and commits it, retrying on // concurrent-commit conflicts and reloading table metadata between attempts. // The stage callback's reloaded argument is false on the first attempt and diff --git a/internal/impl/iceberg/config.go b/internal/impl/iceberg/config.go index a25c475e10..c4ae504772 100644 --- a/internal/impl/iceberg/config.go +++ b/internal/impl/iceberg/config.go @@ -43,6 +43,7 @@ const ( // the Iceberg spec's identifier-field-ids (primary-key columns). ioFieldRowOperation = "row_operation" ioFieldIdentifierFields = "identifier_fields" + ioFieldMergeStrategy = "merge_strategy" // Storage fields - common ioFieldStorage = "storage" @@ -117,7 +118,9 @@ const rowOperationDocs = "\n" + "\n" + "`row_operation` supports interpolation, so the operation can be driven by the data itself — for example by mapping a change-data-capture stream's operation field — but no CDC-specific format is assumed (see the change-data-capture example below). It is named `row_operation` to distinguish it from Iceberg's snapshot-level operation.\n" + "\n" + - "`upsert` and `delete` require `identifier_fields` and use Iceberg merge-on-read equality deletes, which require table format version 2. A version-1 table is automatically upgraded to version 2 on the first `upsert`/`delete`; *this upgrade is irreversible*.\n" + + "`upsert` and `delete` require `identifier_fields`. By default (`merge_strategy: merge-on-read`) they use Iceberg merge-on-read equality deletes, which require table format version 2. A version-1 table is automatically upgraded to version 2 on the first `upsert`/`delete`; *this upgrade is irreversible*.\n" + + "\n" + + "*Merge strategy.* `merge_strategy` controls how mutations are materialised. `merge-on-read` (the default) writes equality-delete files and is the streaming path, but only catalog-only/Flink-world engines can read the result. `copy-on-write` instead rewrites whole data files so the table only ever holds plain data files that every engine can read — including engine-backed catalogs such as Snowflake and the Databricks Unity Catalog — at the cost of heavy write amplification (each mutating batch rewrites every data file holding a touched key). Treat `copy-on-write` as a batch / moderate-throughput mode: sort the table by the identifier key and use large batches. Under `copy-on-write` the `identifier_fields` are the connector-side merge key only and are not registered as the table's Iceberg identifier-field-ids, so auto-created tables carry no identifier-field spec and columns are not forced required. This prototype's `copy-on-write` path supports tables whose columns are all flat, primitive types; a table with nested (struct/list/map) columns, or a merge key that is not an `int`/`long`/`string`/`boolean` column, is rejected with a clear error at write time.\n" + "\n" + "*Identifier fields.* `identifier_fields` must reference existing table columns of a primitive, non-floating-point type. A static `upsert`/`delete` is validated at startup; an interpolated `row_operation` is validated per message at write time, so an empty `identifier_fields` is not caught until the first `upsert`/`delete` message arrives. Identifier columns of a temporal type (`timestamp`, `timestamptz`, `date`, `time`) must arrive as time values, not bare numbers — a numeric epoch is ambiguous as a delete key and is rejected at write time; convert it to a timestamp upstream. If the table is partitioned, every partition source column must be one of the `identifier_fields`, since equality deletes are partition-scoped.\n" + "\n" + @@ -278,6 +281,11 @@ array:list Default([]string{}). Advanced(), + service.NewStringEnumField(ioFieldMergeStrategy, string(mergeStrategyMOR), string(mergeStrategyCOW)). + Description("How `upsert` and `delete` are materialised on disk.\n\n* `merge-on-read` (the default) writes Iceberg v2 equality-delete files. Deletes are applied at read time, so writes stay cheap and streaming-friendly, but only catalog-only/Flink-world engines can read the result — engine-backed catalogs such as Snowflake and the Databricks Unity Catalog cannot read equality deletes.\n* `copy-on-write` rewrites whole data files so the table only ever contains plain data files (no delete files), which every engine can read — including Snowflake and Databricks Unity Catalog. The trade-off is heavy write amplification: each mutating batch rewrites every data file that contains a touched key. This is a batch / moderate-throughput mode, not a streaming one. Sort the table by the identifier key and use large batches so each rewrite touches as few files as possible.\n\nUnder `copy-on-write` the `identifier_fields` are used only connector-side as the merge key and are *not* registered as the table's Iceberg identifier-field-ids, so auto-created tables carry no identifier-field spec (this is what lets the Databricks Unity Catalog accept the `CREATE TABLE`). `merge-on-read` continues to register them.\n\nSee the <> section above for more detail."). + Default(string(mergeStrategyMOR)). + Advanced(), + // Storage configuration - one of s3, gcs, or azure must be specified service.NewObjectField(ioFieldStorage, // S3 storage configuration diff --git a/internal/impl/iceberg/cow.go b/internal/impl/iceberg/cow.go new file mode 100644 index 0000000000..fd499be9a0 --- /dev/null +++ b/internal/impl/iceberg/cow.go @@ -0,0 +1,430 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md + +package iceberg + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "math" + "strconv" + "strings" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/table" + + "github.com/redpanda-data/benthos/v4/public/service" +) + +// writeCOW materialises a mutating batch as copy-on-write: it rewrites whole +// data files so the table only ever contains plain data files (no equality- or +// positional-delete files), which engine-backed catalogs such as Snowflake and +// the Databricks Unity Catalog can read. +// +// It reuses splitByOperation's parse + last-writer-wins per-key collapse: +// - inserts = insert-op rows plus upsert-op rows (the rows to (re)write) +// - deletes = one message per keyed op (upsert OR delete) — the keys whose +// prior versions must be removed +// +// The whole batch is applied as a single atomic snapshot: +// - only inserts (no keyed ops): plain append fast path (no rewrite) +// - only deletes (no rows to write): txn.Delete(filter) +// - otherwise: txn.Overwrite(reader, WithOverwriteFilter(filter)), which +// deletes every existing row matching filter and appends the new rows in +// one snapshot. +func (w *writer) writeCOW(ctx context.Context, batch service.MessageBatch) error { + inserts, deletes, counts, err := w.splitByOperation(batch) + if err != nil { + return fmt.Errorf("splitting batch by row operation: %w", err) + } + + // Fast path: no keyed operations — this is a plain append, so reuse the + // data-file path and the append commit. No file rewrite is needed. + if len(deletes) == 0 { + if len(inserts) == 0 { + return nil + } + files, err := w.writeDataFiles(ctx, inserts) + if err != nil { + return fmt.Errorf("writing data files: %w", err) + } + if err := w.committer.Commit(ctx, CommitInput{Files: files, SchemaID: w.table.Schema().ID}); err != nil { + w.cleanupFiles(ctx, files) + return fmt.Errorf("committing: %w", err) + } + w.metrics.incrInserted(counts.inserted) + return nil + } + + // The remaining paths rewrite data files. Partitioned copy-on-write is not + // yet validated in this prototype: iceberg-go's Overwrite routes rewritten + // rows to partitions internally, but we have only proven the unpartitioned + // case end-to-end, so fail loudly rather than risk mis-partitioned rewrites. + spec := w.table.Spec() + if spec.NumFields() > 0 { + return errors.New("copy-on-write merge_strategy does not support upsert/delete on partitioned tables in this prototype; use merge-on-read, or an unpartitioned table") + } + + // The rewrite builds records through the Arrow JSON round-trip, so the whole + // table schema must be faithfully representable that way. + tableSchema := w.table.Schema() + if err := checkCOWSchemaSupported(tableSchema); err != nil { + return err + } + + filter, err := w.buildCOWFilter(tableSchema, deletes) + if err != nil { + return fmt.Errorf("building copy-on-write filter: %w", err) + } + + input := OverwriteInput{Filter: filter, SchemaID: tableSchema.ID} + + // Only deletes: delete the matching rows, write nothing new. + if len(inserts) == 0 { + if err := w.committer.commitOverwrite(ctx, input); err != nil { + return fmt.Errorf("committing copy-on-write delete: %w", err) + } + w.metrics.incrDeleted(counts.deleted) + return nil + } + + // Detect columns present in the rows to write but absent from the table + // schema and surface them as a schema-evolution error, so the router adds the + // columns and retries — exactly as the append path does. The rewrite below + // projects rows onto the current schema via array.RecordFromJSON, so without + // this an unknown column's value would be silently dropped. + if err := w.cowDetectNewColumns(tableSchema, inserts); err != nil { + return err + } + + // Deletes + new rows: one atomic overwrite. The reader factory rebuilds the + // reader on every attempt because array.RecordReader is consumed once and + // the commit stage can run multiple times on retry. + factory, err := w.buildCOWRecordFactory(tableSchema, inserts) + if err != nil { + return fmt.Errorf("building copy-on-write records: %w", err) + } + input.NewReader = factory + if err := w.committer.commitOverwrite(ctx, input); err != nil { + return fmt.Errorf("committing copy-on-write overwrite: %w", err) + } + w.metrics.incrInserted(counts.inserted) + w.metrics.incrUpserted(counts.upserted) + w.metrics.incrDeleted(counts.deleted) + return nil +} + +// checkCOWSchemaSupported rejects table schemas the prototype's copy-on-write +// path cannot faithfully round-trip through Arrow. The custom shredder handles +// nested/complex types on the append path, but the copy-on-write rewrite builds +// records via array.RecordFromJSON, which we have only verified for flat, +// primitive columns. Rather than silently mis-write, fail loudly with an +// actionable message. +func checkCOWSchemaSupported(s *iceberg.Schema) error { + for _, f := range s.Fields() { + if _, ok := f.Type.(iceberg.PrimitiveType); !ok { + return fmt.Errorf("copy-on-write merge_strategy does not support column %q of non-primitive type %s; this prototype only supports tables whose columns are all flat primitive types (use merge-on-read for nested schemas)", f.Name, f.Type) + } + if !cowSupportedColumnType(f.Type) { + return fmt.Errorf("copy-on-write merge_strategy does not support column %q of type %s; supported column types are boolean, int, long, float, double, string, date, time, timestamp, timestamptz, decimal, and uuid", f.Name, f.Type) + } + } + return nil +} + +// cowSupportedColumnType reports whether a primitive iceberg type is known to +// round-trip faithfully through deleteKeyJSONValue + array.RecordFromJSON. The +// set is deliberately conservative for the prototype; binary/fixed are excluded +// because we have not verified their JSON encoding. +func cowSupportedColumnType(t iceberg.Type) bool { + switch t.(type) { + case iceberg.BooleanType, + iceberg.Int32Type, iceberg.Int64Type, + iceberg.Float32Type, iceberg.Float64Type, + iceberg.StringType, + iceberg.DateType, iceberg.TimeType, + iceberg.TimestampType, iceberg.TimestampTzType, + iceberg.DecimalType, + iceberg.UUIDType: + return true + default: + return false + } +} + +// buildCOWFilter builds the boolean expression selecting every row whose merge +// key appears in the keyed (upsert/delete) messages. For a single identifier +// column it is `col IN (v1, v2, ...)`; for a composite key it is an OR of +// per-tuple ANDs — `(a=a1 AND b=b1) OR (a=a2 AND b=b2) ...` — which is the +// correct semantics (an AND of per-column INs would match the cross product). +// +// For the prototype, merge-key columns are restricted to int/long/string/ +// boolean so the filter literals are unambiguous; other key types return a +// clear error. +func (w *writer) buildCOWFilter(tableSchema *iceberg.Schema, keyed service.MessageBatch) (iceberg.BooleanExpression, error) { + idFields, err := w.cowKeyFields(tableSchema) + if err != nil { + return nil, err + } + + if len(idFields) == 1 { + f := idFields[0] + lits := make([]iceberg.Literal, 0, len(keyed)) + for i, msg := range keyed { + v, err := w.lookupKeyValue(msg, f, i) + if err != nil { + return nil, err + } + lit, err := cowKeyLiteral(f.Type, f.Name, v) + if err != nil { + return nil, err + } + lits = append(lits, lit) + } + // SetPredicate collapses duplicate literals and reduces to EqualTo/ + // AlwaysFalse for the degenerate cases. + return iceberg.SetPredicate(iceberg.OpIn, iceberg.Reference(f.Name), lits), nil + } + + clauses := make([]iceberg.BooleanExpression, 0, len(keyed)) + for i, msg := range keyed { + ands := make([]iceberg.BooleanExpression, 0, len(idFields)) + for _, f := range idFields { + v, err := w.lookupKeyValue(msg, f, i) + if err != nil { + return nil, err + } + lit, err := cowKeyLiteral(f.Type, f.Name, v) + if err != nil { + return nil, err + } + ands = append(ands, iceberg.LiteralPredicate(iceberg.OpEQ, iceberg.Reference(f.Name), lit)) + } + var clause iceberg.BooleanExpression + if len(ands) == 1 { + clause = ands[0] + } else { + clause = iceberg.NewAnd(ands[0], ands[1], ands[2:]...) + } + clauses = append(clauses, clause) + } + + if len(clauses) == 1 { + return clauses[0], nil + } + return iceberg.NewOr(clauses[0], clauses[1], clauses[2:]...), nil +} + +// cowKeyFields resolves the configured identifier_fields against the table +// schema, validating that each is a primitive column supported as a copy-on- +// write merge key. +func (w *writer) cowKeyFields(tableSchema *iceberg.Schema) ([]iceberg.NestedField, error) { + if len(w.rowOpCfg.IdentifierFields) == 0 { + return nil, fmt.Errorf("%s is required for upsert/delete", ioFieldIdentifierFields) + } + fields := make([]iceberg.NestedField, 0, len(w.rowOpCfg.IdentifierFields)) + for _, name := range w.rowOpCfg.IdentifierFields { + field, ok := tableSchema.FindFieldByName(name) + if !ok && !w.caseSensitive { + field, ok = tableSchema.FindFieldByNameCaseInsensitive(name) + } + if !ok { + return nil, fmt.Errorf("%s column %q not found in table schema", ioFieldIdentifierFields, name) + } + fields = append(fields, field) + } + return fields, nil +} + +// lookupKeyValue extracts a single identifier column's value from a message, +// erroring on a missing or null key (a null merge key cannot select rows). +func (w *writer) lookupKeyValue(msg *service.Message, field iceberg.NestedField, idx int) (any, error) { + structured, err := msg.AsStructured() + if err != nil { + return nil, fmt.Errorf("reading structured message %d for merge key: %w", idx, err) + } + row, ok := structured.(map[string]any) + if !ok { + return nil, fmt.Errorf("message %d for upsert/delete must be an object, got %T", idx, structured) + } + v, ok := lookupField(row, field.Name, w.caseSensitive) + if !ok || v == nil { + return nil, fmt.Errorf("%s %q is missing or null in message %d", ioFieldIdentifierFields, field.Name, idx) + } + return v, nil +} + +// cowKeyLiteral builds an iceberg filter literal for a merge-key value. Only +// int/long/string/boolean key columns are supported by the prototype's +// copy-on-write filter; other types return a clear, actionable error. +func cowKeyLiteral(t iceberg.Type, name string, v any) (iceberg.Literal, error) { + switch t.(type) { + case iceberg.Int32Type: + n, err := cowValueToInt64(v) + if err != nil { + return nil, fmt.Errorf("%s %q: %w", ioFieldIdentifierFields, name, err) + } + return iceberg.NewLiteral(int32(n)), nil + case iceberg.Int64Type: + n, err := cowValueToInt64(v) + if err != nil { + return nil, fmt.Errorf("%s %q: %w", ioFieldIdentifierFields, name, err) + } + return iceberg.NewLiteral(n), nil + case iceberg.StringType: + s, ok := v.(string) + if !ok { + return nil, fmt.Errorf("%s %q: string column given %T", ioFieldIdentifierFields, name, v) + } + return iceberg.NewLiteral(s), nil + case iceberg.BooleanType: + b, ok := v.(bool) + if !ok { + return nil, fmt.Errorf("%s %q: boolean column given %T", ioFieldIdentifierFields, name, v) + } + return iceberg.NewLiteral(b), nil + default: + return nil, fmt.Errorf("copy-on-write merge_strategy does not support merge key column %q of type %s; supported merge-key types are int, long, string, and boolean (use merge-on-read for other key types)", name, t) + } +} + +// cowValueToInt64 converts a JSON-decoded value into an int64 without silent +// precision loss, mirroring deleteKeyJSONValue's integer handling. +func cowValueToInt64(v any) (int64, error) { + switch n := v.(type) { + case json.Number: + return n.Int64() + case int: + return int64(n), nil + case int32: + return int64(n), nil + case int64: + return n, nil + case float64: + if n != math.Trunc(n) { + return 0, fmt.Errorf("integer column given non-integer value %v", n) + } + if math.Abs(n) >= 1<<53 { + return 0, fmt.Errorf("integer column given value %v outside the range representable exactly as a float64 (provide it as an integer or string)", n) + } + return int64(n), nil + case string: + return strconv.ParseInt(n, 10, 64) + default: + return 0, fmt.Errorf("unsupported value type %T for integer column", v) + } +} + +// cowDetectNewColumns returns a BatchSchemaEvolutionError naming any top-level +// field present in the rows to write but absent from the table schema. The +// copy-on-write rewrite projects rows onto the current schema, so an unknown +// column would otherwise be dropped without trace; returning this error lets the +// router evolve the table and retry, matching the shredder-based append path +// (writer.go writeDataFiles). Copy-on-write is gated to flat-primitive schemas, +// so every new field is at the schema root. +func (w *writer) cowDetectNewColumns(tableSchema *iceberg.Schema, rows service.MessageBatch) error { + var newErrs []*UnknownFieldError + seen := make(map[string]struct{}) + for i, msg := range rows { + structured, err := msg.AsStructured() + if err != nil { + return fmt.Errorf("reading structured message %d: %w", i, err) + } + row, ok := structured.(map[string]any) + if !ok { + return fmt.Errorf("message %d must be an object, got %T", i, structured) + } + for name, v := range row { + _, known := tableSchema.FindFieldByName(name) + if !known && !w.caseSensitive { + _, known = tableSchema.FindFieldByNameCaseInsensitive(name) + } + if known { + continue + } + dedup := name + if !w.caseSensitive { + dedup = strings.ToLower(name) + } + if _, dup := seen[dedup]; dup { + continue + } + seen[dedup] = struct{}{} + newErrs = append(newErrs, NewUnknownFieldError(nil, name, v)) + } + } + if len(newErrs) > 0 { + return NewBatchSchemaEvolutionError(newErrs) + } + return nil +} + +// buildCOWRecordFactory projects the rows to (re)write into JSON matching the +// full table schema and returns a factory that rebuilds a fresh RecordReader on +// each call. A factory (rather than a single reader) is required because +// array.RecordReader is consumed once and the commit stage can run multiple +// times on retry. +func (w *writer) buildCOWRecordFactory(tableSchema *iceberg.Schema, rows service.MessageBatch) (func() (array.RecordReader, error), error) { + arrowSc, err := table.SchemaToArrowSchema(tableSchema, nil, true, false) + if err != nil { + return nil, fmt.Errorf("building arrow schema: %w", err) + } + + fields := tableSchema.Fields() + encoded := make([]map[string]any, 0, len(rows)) + for i, msg := range rows { + structured, err := msg.AsStructured() + if err != nil { + return nil, fmt.Errorf("reading structured message %d: %w", i, err) + } + row, ok := structured.(map[string]any) + if !ok { + return nil, fmt.Errorf("message %d must be an object, got %T", i, structured) + } + out := make(map[string]any, len(fields)) + for _, field := range fields { + v, ok := lookupField(row, field.Name, w.caseSensitive) + if !ok || v == nil { + // Absent/null columns are left out so Arrow reads them as null. + continue + } + jv, err := deleteKeyJSONValue(field.Type, v) + if err != nil { + return nil, fmt.Errorf("column %q in message %d: %w", field.Name, i, err) + } + out[field.Name] = jv + } + encoded = append(encoded, out) + } + + jsonRows, err := json.Marshal(encoded) + if err != nil { + return nil, fmt.Errorf("encoding rows: %w", err) + } + + return func() (array.RecordReader, error) { + rec, _, err := array.RecordFromJSON(memory.DefaultAllocator, arrowSc, bytes.NewReader(jsonRows)) + if err != nil { + return nil, fmt.Errorf("building records: %w", err) + } + rdr, err := array.NewRecordReader(arrowSc, []arrow.RecordBatch{rec}) + if err != nil { + rec.Release() + return nil, fmt.Errorf("building record reader: %w", err) + } + // NewRecordReader retains rec, so drop our reference. + rec.Release() + return rdr, nil + }, nil +} diff --git a/internal/impl/iceberg/cow_amplification_bench_test.go b/internal/impl/iceberg/cow_amplification_bench_test.go new file mode 100644 index 0000000000..70d70ce357 --- /dev/null +++ b/internal/impl/iceberg/cow_amplification_bench_test.go @@ -0,0 +1,347 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md + +package iceberg + +// De-risking harness (NOT production code): characterises the write +// amplification of copy-on-write (COW) row-level mutations in iceberg-go, to +// decide whether COW is viable for streaming CDC in the connect iceberg output. +// +// Run with: +// +// go test -run TestCOWWriteAmplification -v ./internal/impl/iceberg/ +// +// It seeds a table with M data files, each holding R rows with contiguous, +// non-overlapping id ranges (simulating a sorted/clustered key — the realistic +// CDC-on-ordered-key case), then applies a delete touching K keys and measures +// how much of the table COW rewrites, comparing against merge-on-read (MOR). + +import ( + "context" + "fmt" + "io/fs" + "math/rand" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/table" + "github.com/google/uuid" + "github.com/stretchr/testify/require" +) + +// cowArrowSchema is the Arrow schema matching the (id int64, payload string) +// table below. Field names match the iceberg schema so Append can bind them. +var cowArrowSchema = arrow.NewSchema([]arrow.Field{ + {Name: "id", Type: arrow.PrimitiveTypes.Int64, Nullable: false}, + {Name: "payload", Type: arrow.BinaryTypes.String, Nullable: true}, +}, nil) + +// newAmpTable builds an unpartitioned v2 table (id int64, payload string) backed +// by an in-memory catalog and the local filesystem. deleteMode, when non-empty, +// is baked into table metadata as write.delete.mode. +func newAmpTable(tb testing.TB, deleteMode string) (*table.Table, *memCatalog) { + tb.Helper() + location := filepath.ToSlash(tb.TempDir()) + + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}, + iceberg.NestedField{ID: 2, Name: "payload", Type: iceberg.PrimitiveTypes.String, Required: false}, + ) + props := iceberg.Properties{table.PropertyFormatVersion: "2"} + if deleteMode != "" { + props[table.WriteDeleteModeKey] = deleteMode + } + meta, err := table.NewMetadata(sc, iceberg.UnpartitionedSpec, table.UnsortedSortOrder, location, props) + require.NoError(tb, err) + + cat := &memCatalog{ + meta: meta, + metadataLocation: fmt.Sprintf("%s/metadata/00001-%s.metadata.json", location, uuid.New()), + ident: table.Identifier{"default", "amp"}, + location: location, + } + return cat.snapshot(), cat +} + +// appendDataFile writes ONE real parquet data file holding `rows` rows with +// contiguous ids [startID, startID+rows) and a per-row random payload (so the +// file has realistic size and real id min/max stats), committing it as its own +// snapshot. Returns the latest table handle. +func appendDataFile(tb testing.TB, ctx context.Context, tbl *table.Table, rng *rand.Rand, startID int64, rows, payloadBytes int) *table.Table { + tb.Helper() + mem := memory.NewGoAllocator() + bldr := array.NewRecordBuilder(mem, cowArrowSchema) + defer bldr.Release() + + idB := bldr.Field(0).(*array.Int64Builder) + payB := bldr.Field(1).(*array.StringBuilder) + buf := make([]byte, payloadBytes) + for i := range rows { + idB.Append(startID + int64(i)) + for j := range buf { + buf[j] = byte('a' + rng.Intn(26)) + } + payB.Append(string(buf)) + } + + rec := bldr.NewRecordBatch() + defer rec.Release() + rdr, err := array.NewRecordReader(cowArrowSchema, []arrow.RecordBatch{rec}) + require.NoError(tb, err) + defer rdr.Release() + + tx := tbl.NewTransaction() + require.NoError(tb, tx.Append(ctx, rdr, nil)) + next, err := tx.Commit(ctx) + require.NoError(tb, err) + return next +} + +// parquetStats returns the count and total on-disk byte size of all parquet +// files under dir. +func parquetStats(tb testing.TB, dir string) (count int, bytes int64) { + tb.Helper() + require.NoError(tb, filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() && strings.HasSuffix(p, ".parquet") { + info, ierr := d.Info() + if ierr != nil { + return ierr + } + count++ + bytes += info.Size() + } + return nil + })) + return count, bytes +} + +// countDeleteManifestFiles returns the number of delete files referenced by the +// current snapshot (delete-content manifests). Zero after a COW mutation; > 0 +// after a MOR mutation. +func countDeleteManifestFiles(tb testing.TB, ctx context.Context, tbl *table.Table) int { + tb.Helper() + snap := tbl.CurrentSnapshot() + if snap == nil { + return 0 + } + fsys, err := tbl.FS(ctx) + require.NoError(tb, err) + manifests, err := snap.Manifests(fsys) + require.NoError(tb, err) + + n := 0 + for _, m := range manifests { + if m.ManifestContent() != iceberg.ManifestContentDeletes { + continue + } + for entry, err := range m.Entries(fsys, true) { + require.NoError(tb, err) + _ = entry + n++ + } + } + return n +} + +func summaryInt(tb testing.TB, s *table.Summary, key string) int64 { + tb.Helper() + v, ok := s.Properties[key] + if !ok || v == "" { + return 0 + } + n, err := strconv.ParseInt(v, 10, 64) + require.NoError(tb, err) + return n +} + +type ampResult struct { + name string + mode string + m, r, k int + scatter string + + seedFiles int + seedBytes int64 + + // snapshot summary counters for the mutation + addedDataFiles int64 + addedFilesSize int64 + deletedDataFiles int64 + removedFilesSize int64 + addedRecords int64 + deletedRecords int64 + addedDeleteFiles int64 + addedPosDelFiles int64 + addedEqDelFiles int64 + + deleteManifestFiles int + elapsed time.Duration +} + +// buildKeys returns the set of ids to mutate. For "within" all K keys live in a +// single (middle) file; for "perfile" the K keys are spread one-per-file across +// K distinct files (worst case for COW). +func buildKeys(m, r, k int, scatter string) []int64 { + keys := make([]int64, 0, k) + switch scatter { + case "within": + fileIdx := int64(m / 2) + base := fileIdx * int64(r) + for i := range k { + keys = append(keys, base+int64(i)) + } + case "perfile": + for i := range k { + keys = append(keys, int64(i)*int64(r)) // first id of file i + } + } + return keys +} + +func runAmpScenario(tb testing.TB, ctx context.Context, mode string, m, r, k, payloadBytes int, scatter string) ampResult { + tb.Helper() + rng := rand.New(rand.NewSource(int64(m*1_000_000 + r*1000 + k))) + + tbl, cat := newAmpTable(tb, mode) + for f := range m { + tbl = appendDataFile(tb, ctx, tbl, rng, int64(f)*int64(r), r, payloadBytes) + } + + seedFiles, seedBytes := parquetStats(tb, cat.location) + + keys := buildKeys(m, r, k, scatter) + filter := iceberg.IsIn(iceberg.Reference("id"), keys...) + + tx := tbl.NewTransaction() + start := time.Now() + require.NoError(tb, tx.Delete(ctx, filter, nil)) + next, err := tx.Commit(ctx) + require.NoError(tb, err) + elapsed := time.Since(start) + + snap := next.CurrentSnapshot() + require.NotNil(tb, snap) + require.NotNil(tb, snap.Summary) + + res := ampResult{ + name: fmt.Sprintf("M=%d R=%d K=%d %s", m, r, k, scatter), mode: mode, + m: m, r: r, k: k, scatter: scatter, + seedFiles: seedFiles, seedBytes: seedBytes, + addedDataFiles: summaryInt(tb, snap.Summary, "added-data-files"), + addedFilesSize: summaryInt(tb, snap.Summary, "added-files-size"), + deletedDataFiles: summaryInt(tb, snap.Summary, "deleted-data-files"), + removedFilesSize: summaryInt(tb, snap.Summary, "removed-files-size"), + addedRecords: summaryInt(tb, snap.Summary, "added-records"), + deletedRecords: summaryInt(tb, snap.Summary, "deleted-records"), + addedDeleteFiles: summaryInt(tb, snap.Summary, "added-delete-files"), + addedPosDelFiles: summaryInt(tb, snap.Summary, "added-position-delete-files"), + addedEqDelFiles: summaryInt(tb, snap.Summary, "added-equality-delete-files"), + deleteManifestFiles: countDeleteManifestFiles(tb, ctx, next), + elapsed: elapsed, + } + return res +} + +// TestCOWWriteAmplification is the headline harness. It sweeps M/K/scatter for +// both COW and MOR delete modes and prints a table of measured amplification. +func TestCOWWriteAmplification(t *testing.T) { + if testing.Short() { + t.Skip("amplification harness is slow; skipped under -short") + } + ctx := t.Context() + + const ( + R = 1000 // rows per seeded file + payloadBytes = 128 // per-row payload size -> realistic file sizes + ) + + type scen struct { + m, k int + scatter string + } + scenarios := []scen{ + // K keys all within ONE file (best case). + {10, 1, "within"}, + {50, 1, "within"}, + {200, 1, "within"}, + {200, 10, "within"}, + {200, 100, "within"}, + // K keys spread ONE-PER-FILE across K distinct files (worst case). + {10, 1, "perfile"}, + {10, 10, "perfile"}, + {50, 1, "perfile"}, + {50, 10, "perfile"}, + {50, 50, "perfile"}, + {200, 1, "perfile"}, + {200, 10, "perfile"}, + {200, 100, "perfile"}, + } + + var results []ampResult + for _, s := range scenarios { + for _, mode := range []string{table.WriteModeCopyOnWrite, table.WriteModeMergeOnRead} { + results = append(results, runAmpScenario(t, ctx, mode, s.m, R, s.k, payloadBytes, s.scatter)) + } + } + + // Correctness sanity, per the engine-agnostic property: COW must produce + // zero delete files; MOR must produce delete files. + for _, r := range results { + if r.mode == table.WriteModeCopyOnWrite { + require.Zerof(t, r.deleteManifestFiles, "COW %s must produce zero delete files (found %d)", r.name, r.deleteManifestFiles) + require.Zerof(t, r.addedDeleteFiles, "COW %s must not report added-delete-files", r.name) + } else { + require.Positivef(t, r.deleteManifestFiles, "MOR %s must produce delete files", r.name) + } + } + + // ---- Report ---- + t.Log("") + t.Logf("Seed layout: R=%d rows/file, payload=%d bytes/row, contiguous non-overlapping id ranges per file (clustered key)", R, payloadBytes) + t.Log("") + t.Logf("%-28s %-13s | %6s %10s | %6s %10s %6s | %5s %8s | %8s %8s | %8s | %9s", + "scenario", "mode", "files", "seedBytes", "+files", "+bytes", "-files", "-recs", "-bytes", "delFiles", "amp(x)", "wall", "tblRewr%") + t.Log(strings.Repeat("-", 160)) + + for _, r := range results { + perRowBytes := float64(r.seedBytes) / float64(r.m*r.r) + logicalBytes := perRowBytes * float64(r.k) + + var ampX float64 + var writtenBytes int64 + if r.mode == table.WriteModeCopyOnWrite { + writtenBytes = r.addedFilesSize + } else { + writtenBytes = r.addedFilesSize // delete-file bytes written + } + if logicalBytes > 0 { + ampX = float64(writtenBytes) / logicalBytes + } + tblRewrPct := 100 * float64(r.removedFilesSize) / float64(r.seedBytes) + + t.Logf("%-28s %-13s | %6d %10d | %6d %10d %6d | %5d %8d | %8d %8.1f | %8s | %8.2f", + r.name, r.mode, + r.seedFiles, r.seedBytes, + r.addedDataFiles, r.addedFilesSize, r.deletedDataFiles, + r.deletedRecords, r.removedFilesSize, + r.deleteManifestFiles, ampX, r.elapsed.Round(time.Microsecond).String(), tblRewrPct) + } + t.Log("") + t.Log("amp(x) = bytes written by the mutation / bytes logically changed (~K rows)") + t.Log("tblRewr% = removed-files-size / total seeded bytes (fraction of table COW rewrote)") +} diff --git a/internal/impl/iceberg/cow_test.go b/internal/impl/iceberg/cow_test.go new file mode 100644 index 0000000000..53bbe2cb2a --- /dev/null +++ b/internal/impl/iceberg/cow_test.go @@ -0,0 +1,556 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md + +package iceberg + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "testing" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/table" + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/redpanda-data/benthos/v4/public/service" +) + +// --- merge_strategy config parsing -------------------------------------------- + +func TestParseMergeStrategyConfig(t *testing.T) { + t.Run("defaults to merge-on-read", func(t *testing.T) { + cfg, err := parseTestRowOpConfig(t, "") + require.NoError(t, err) + assert.Equal(t, mergeStrategyMOR, cfg.MergeStrategy) + }) + + t.Run("explicit copy-on-write", func(t *testing.T) { + cfg, err := parseTestRowOpConfig(t, "merge_strategy: copy-on-write\nrow_operation: upsert\nidentifier_fields: [id]\n") + require.NoError(t, err) + assert.Equal(t, mergeStrategyCOW, cfg.MergeStrategy) + }) + + t.Run("explicit merge-on-read", func(t *testing.T) { + cfg, err := parseTestRowOpConfig(t, "merge_strategy: merge-on-read\n") + require.NoError(t, err) + assert.Equal(t, mergeStrategyMOR, cfg.MergeStrategy) + }) + + t.Run("invalid value rejected", func(t *testing.T) { + // An unknown enum value is rejected somewhere along the parse pipeline + // (spec validation or the defensive switch in parseRowOpConfig). + conf, yamlErr := icebergOutputConfig().ParseYAML(` +catalog: + url: http://localhost:8181/api/catalog +namespace: ns +table: t +storage: + aws_s3: + bucket: bucket +merge_strategy: sideways +`, nil) + if yamlErr != nil { + return // rejected at spec-validation time + } + _, err := parseRowOpConfig(conf) + require.Error(t, err) + assert.Contains(t, err.Error(), ioFieldMergeStrategy) + }) +} + +// --- conditional identifier-field registration -------------------------------- + +func cowRouter(strategy mergeStrategy) *Router { + return &Router{ + caseSensitive: true, + resolver: newTypeResolver("", nil, true, nil), + rowOpCfg: RowOpConfig{IdentifierFields: []string{"id"}, MergeStrategy: strategy}, + } +} + +func TestSchemaWithIdentifierFieldsMORRegisters(t *testing.T) { + r := cowRouter(mergeStrategyMOR) + record := map[string]any{"id": int64(1), "name": "a"} + sc, err := r.buildSchemaWithResolver(record, structuredMsg(t, record), tableKey{namespace: "ns", table: "t"}) + require.NoError(t, err) + + idField, ok := sc.FindFieldByName("id") + require.True(t, ok) + assert.Equal(t, []int{idField.ID}, sc.IdentifierFieldIDs, "merge-on-read must register identifier-field-ids") + assert.True(t, idField.Required, "identifier columns must be marked required under merge-on-read") +} + +func TestSchemaWithIdentifierFieldsCOWDoesNotRegister(t *testing.T) { + r := cowRouter(mergeStrategyCOW) + record := map[string]any{"id": int64(1), "name": "a"} + sc, err := r.buildSchemaWithResolver(record, structuredMsg(t, record), tableKey{namespace: "ns", table: "t"}) + require.NoError(t, err) + + assert.Empty(t, sc.IdentifierFieldIDs, "copy-on-write must not register identifier-field-ids") + idField, ok := sc.FindFieldByName("id") + require.True(t, ok) + assert.False(t, idField.Required, "copy-on-write must not force identifier columns required") +} + +// --- schema-support gate ------------------------------------------------------- + +func TestCheckCOWSchemaSupported(t *testing.T) { + t.Run("flat primitives ok", func(t *testing.T) { + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64}, + iceberg.NestedField{ID: 2, Name: "name", Type: iceberg.PrimitiveTypes.String}, + iceberg.NestedField{ID: 3, Name: "ts", Type: iceberg.PrimitiveTypes.Timestamp}, + ) + require.NoError(t, checkCOWSchemaSupported(sc)) + }) + + t.Run("nested struct rejected", func(t *testing.T) { + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64}, + iceberg.NestedField{ID: 2, Name: "nested", Type: &iceberg.StructType{ + FieldList: []iceberg.NestedField{{ID: 3, Name: "inner", Type: iceberg.PrimitiveTypes.String}}, + }}, + ) + err := checkCOWSchemaSupported(sc) + require.Error(t, err) + assert.Contains(t, err.Error(), "non-primitive") + }) + + t.Run("binary rejected", func(t *testing.T) { + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "b", Type: iceberg.PrimitiveTypes.Binary}, + ) + err := checkCOWSchemaSupported(sc) + require.Error(t, err) + assert.Contains(t, err.Error(), "supported column types") + }) +} + +// --- filter construction ------------------------------------------------------- + +func cowWriter(t testing.TB, tbl *table.Table, idFields ...string) *writer { + t.Helper() + return &writer{ + table: tbl, + caseSensitive: true, + rowOpCfg: RowOpConfig{ + // Drive the operation from metadata, as a real change-data-capture + // mapping does, so the routing field never leaks into the row body. + Operation: mustInterp(t, `${! metadata("op") }`), + IdentifierFields: idFields, + MergeStrategy: mergeStrategyCOW, + }, + logger: service.MockResources().Logger(), + } +} + +// cowMsg builds a message whose body is the row image and whose "op" metadata +// drives the row_operation. +func cowMsg(t testing.TB, op string, row map[string]any) *service.Message { + t.Helper() + msg := structuredMsg(t, row) + msg.MetaSetMut("op", op) + return msg +} + +func TestBuildCOWFilterSingleKey(t *testing.T) { + tbl, _ := newTestTable(t) // schema: id int64 + w := cowWriter(t, tbl, "id") + + keyed := service.MessageBatch{ + structuredMsg(t, map[string]any{"id": 2}), + structuredMsg(t, map[string]any{"id": 4}), + } + filter, err := w.buildCOWFilter(tbl.Schema(), keyed) + require.NoError(t, err) + assert.Equal(t, iceberg.OpIn, filter.Op(), "two distinct keys on one column is an IN predicate") +} + +func TestBuildCOWFilterCompositeKey(t *testing.T) { + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "tenant", Type: iceberg.PrimitiveTypes.String}, + iceberg.NestedField{ID: 2, Name: "id", Type: iceberg.PrimitiveTypes.Int64}, + ) + tbl := newTypedKeyTableFromSchema(t, sc) + w := cowWriter(t, tbl, "tenant", "id") + + keyed := service.MessageBatch{ + structuredMsg(t, map[string]any{"tenant": "a", "id": 1}), + structuredMsg(t, map[string]any{"tenant": "b", "id": 2}), + } + filter, err := w.buildCOWFilter(sc, keyed) + require.NoError(t, err) + // Two tuples => OR-of-ANDs; the top-level operator is OR. + assert.Equal(t, iceberg.OpOr, filter.Op(), "composite key over two tuples must be an OR of ANDs") +} + +func TestBuildCOWFilterUnsupportedKeyType(t *testing.T) { + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "ts", Type: iceberg.PrimitiveTypes.Timestamp}, + ) + tbl := newTypedKeyTableFromSchema(t, sc) + w := cowWriter(t, tbl, "ts") + _, err := w.buildCOWFilter(sc, service.MessageBatch{structuredMsg(t, map[string]any{"ts": 1})}) + require.Error(t, err) + assert.Contains(t, err.Error(), "merge key") +} + +// --- record factory ------------------------------------------------------------ + +func TestBuildCOWRecordFactoryRebuildsReader(t *testing.T) { + tbl, _ := newTestTable(t) // id int64 + w := cowWriter(t, tbl, "id") + factory, err := w.buildCOWRecordFactory(tbl.Schema(), service.MessageBatch{ + structuredMsg(t, map[string]any{"id": 1}), + structuredMsg(t, map[string]any{"id": 2}), + }) + require.NoError(t, err) + + // The factory must return an independent, fully-consumable reader each time + // (the commit stage can run more than once on retry). + for attempt := range 2 { + rdr, err := factory() + require.NoError(t, err) + rows := int64(0) + for rdr.Next() { + rows += rdr.RecordBatch().NumRows() + } + rdr.Release() + assert.EqualValues(t, 2, rows, "attempt %d must see all rows", attempt) + } +} + +// TestCOWMutationDetectsNewColumn pins the fix for silent schema-evolution data +// loss: a copy-on-write upsert carrying a column absent from the table schema +// must surface a BatchSchemaEvolutionError (so the router evolves the table and +// retries), not silently drop the column via the Arrow projection. No committer +// is wired — detection must fire before any commit. +func TestCOWMutationDetectsNewColumn(t *testing.T) { + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}, + iceberg.NestedField{ID: 2, Name: "payload", Type: iceberg.PrimitiveTypes.String}, + ) + tbl, _ := newCOWTable(t, sc) + w := cowWriter(t, tbl, "id") + + err := w.Write(t.Context(), service.MessageBatch{ + cowMsg(t, "upsert", map[string]any{"id": 1, "payload": "x", "extra": "new-column"}), + }) + require.Error(t, err) + var evo *BatchSchemaEvolutionError + require.ErrorAs(t, err, &evo, "an unknown column must trigger schema evolution, not a silent drop") + assert.Contains(t, err.Error(), "extra") +} + +// TestCommitOverwriteCleansUpOrphansOnFailure pins the fix for orphaned +// copy-on-write files: iceberg-go's Overwrite writes rewritten/new parquet files +// before the catalog commit, so a definitively-failed commit must leave none +// behind. TestCOWUpsertDeleteRoundTrip is the control proving the overwrite path +// does write files, so a return to the seed count here is genuine cleanup rather +// than a vacuous pass. +func TestCommitOverwriteCleansUpOrphansOnFailure(t *testing.T) { + ctx := t.Context() + logger := service.MockResources().Logger() + + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}, + iceberg.NestedField{ID: 2, Name: "payload", Type: iceberg.PrimitiveTypes.String}, + ) + seedTbl, cat := newCOWTable(t, sc) + seedTbl = appendCOWRows(t, ctx, seedTbl, map[int64]string{1: "one", 2: "two", 3: "three"}) + seedCount := countParquetFiles(t, seedTbl.Location()) + require.Positive(t, seedCount, "seeding must have written data files") + + // A non-retryable failure guarantees the mutation's commit does not land, so + // the files the overwrite wrote are genuine orphans. + fc := &flakyCatalog{memCatalog: cat, failuresLeft: 1 << 30, failErr: errors.New("storage unavailable")} + comm, err := NewCommitter(fc.snapshot(), CommitConfig{MaxRetries: 2}, func(context.Context) (*table.Table, error) { return fc.snapshot(), nil }, logger) + require.NoError(t, err) + defer comm.Close() + w := cowWriter(t, fc.snapshot(), "id") + w.committer = comm + + err = w.Write(ctx, service.MessageBatch{cowMsg(t, "upsert", map[string]any{"id": 2, "payload": "TWO"})}) + require.Error(t, err) + + assert.Equal(t, seedCount, countParquetFiles(t, seedTbl.Location()), + "the failed copy-on-write commit's parquet files must be cleaned up, leaving only the seed files") +} + +// --- committer-level round trip ------------------------------------------------ + +// newTypedKeyTableFromSchema builds an unpartitioned v2 table for the given +// schema, backed by an in-memory catalog and the local filesystem. +func newTypedKeyTableFromSchema(t testing.TB, sc *iceberg.Schema) *table.Table { + t.Helper() + tbl, _ := newCOWTable(t, sc) + return tbl +} + +func newCOWTable(t testing.TB, sc *iceberg.Schema) (*table.Table, *memCatalog) { + t.Helper() + return newAmpTableWithSchema(t, sc) +} + +// TestCOWUpsertDeleteRoundTrip drives a full copy-on-write upsert+delete batch +// through the writer and committer against an in-memory catalog, then asserts +// (a) the resulting table has the correct final rows and (b) the table contains +// ONLY plain data files — zero delete files — which is what makes it readable by +// engine-backed catalogs. +func TestCOWUpsertDeleteRoundTrip(t *testing.T) { + ctx := t.Context() + + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}, + iceberg.NestedField{ID: 2, Name: "payload", Type: iceberg.PrimitiveTypes.String, Required: false}, + ) + seedTbl, cat := newCOWTable(t, sc) + + // Seed rows id=1,2,3. + seedTbl = appendCOWRows(t, ctx, seedTbl, map[int64]string{1: "one", 2: "two", 3: "three"}) + + // Build a writer whose committer shares the catalog. + comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) + require.NoError(t, err) + defer comm.Close() + w := cowWriter(t, seedTbl, "id") + w.committer = comm + + // upsert id=2 (payload->TWO), delete id=3, upsert id=4 (new row). + batch := service.MessageBatch{ + cowMsg(t, "upsert", map[string]any{"id": 2, "payload": "TWO"}), + cowMsg(t, "delete", map[string]any{"id": 3}), + cowMsg(t, "upsert", map[string]any{"id": 4, "payload": "FOUR"}), + } + require.NoError(t, w.Write(ctx, batch)) + + final := cat.snapshot() + + // (a) zero delete files: every manifest must be data content. + assert.Zero(t, countDeleteManifestFiles(t, ctx, final), "copy-on-write must leave no delete files") + assertAllManifestsData(t, ctx, final) + + // (b) correct final state. + got := scanRows(t, ctx, final) + want := map[int64]string{1: "one", 2: "TWO", 4: "FOUR"} + assert.Equal(t, want, got) +} + +func TestCOWOnlyDeletesFastPath(t *testing.T) { + ctx := t.Context() + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}, + iceberg.NestedField{ID: 2, Name: "payload", Type: iceberg.PrimitiveTypes.String, Required: false}, + ) + seedTbl, cat := newCOWTable(t, sc) + seedTbl = appendCOWRows(t, ctx, seedTbl, map[int64]string{1: "one", 2: "two"}) + + comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) + require.NoError(t, err) + defer comm.Close() + w := cowWriter(t, seedTbl, "id") + w.committer = comm + + require.NoError(t, w.Write(ctx, service.MessageBatch{ + cowMsg(t, "delete", map[string]any{"id": 1}), + })) + + final := cat.snapshot() + assert.Zero(t, countDeleteManifestFiles(t, ctx, final), "delete-only copy-on-write must leave no delete files") + assert.Equal(t, table.OpDelete, final.CurrentSnapshot().Summary.Operation) + assert.Equal(t, map[int64]string{2: "two"}, scanRows(t, ctx, final)) +} + +func TestCOWOnlyInsertsUsesAppend(t *testing.T) { + ctx := t.Context() + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}, + iceberg.NestedField{ID: 2, Name: "payload", Type: iceberg.PrimitiveTypes.String, Required: false}, + ) + seedTbl, cat := newCOWTable(t, sc) + + comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) + require.NoError(t, err) + defer comm.Close() + w := cowWriter(t, seedTbl, "id") + w.committer = comm + + // The shredder append path writes into a data/ subdir that LocalFS will not + // create implicitly. + require.NoError(t, os.MkdirAll(filepath.Join(seedTbl.Location(), "data"), 0o755)) + + // A batch of only inserts (unkeyed) must take the plain append path. + require.NoError(t, w.Write(ctx, service.MessageBatch{ + cowMsg(t, "insert", map[string]any{"id": 10, "payload": "ten"}), + cowMsg(t, "insert", map[string]any{"id": 11, "payload": "eleven"}), + })) + + final := cat.snapshot() + require.NotNil(t, final.CurrentSnapshot()) + assert.Equal(t, table.OpAppend, final.CurrentSnapshot().Summary.Operation, "insert-only batch must append, not overwrite") + assert.Equal(t, map[int64]string{10: "ten", 11: "eleven"}, scanRows(t, ctx, final)) +} + +// --- partition gate ------------------------------------------------------------ + +// newPartitionedCOWTable builds a partitioned v2 table for the given schema and +// spec, backed by an in-memory catalog and the local filesystem. It mirrors +// newAmpTableWithSchema but installs a real partition spec so +// tbl.Spec().NumFields() > 0. +func newPartitionedCOWTable(t testing.TB, sc *iceberg.Schema, spec iceberg.PartitionSpec) *table.Table { + t.Helper() + location := filepath.ToSlash(t.TempDir()) + meta, err := table.NewMetadata(sc, &spec, table.UnsortedSortOrder, location, + iceberg.Properties{table.PropertyFormatVersion: "2"}) + require.NoError(t, err) + cat := &memCatalog{ + meta: meta, + metadataLocation: fmt.Sprintf("%s/metadata/00001-%s.metadata.json", location, uuid.New()), + ident: table.Identifier{"default", "cow_partitioned"}, + location: location, + } + return cat.snapshot() +} + +// TestCOWPartitionedTableRejectsMutation pins the copy-on-write partition gate: +// writeCOW must refuse a mutating (upsert/delete) batch on a partitioned table +// rather than risk a mis-partitioned rewrite, and it must do so with an +// actionable error. Only the file-rewrite paths are gated, so an upsert — a +// keyed operation — is the trigger that reaches the gate. +func TestCOWPartitionedTableRejectsMutation(t *testing.T) { + ctx := t.Context() + + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}, + iceberg.NestedField{ID: 2, Name: "region", Type: iceberg.PrimitiveTypes.String, Required: true}, + iceberg.NestedField{ID: 3, Name: "payload", Type: iceberg.PrimitiveTypes.String, Required: false}, + ) + // Partition by region (identity transform) so spec.NumFields() > 0. + spec := iceberg.NewPartitionSpec(iceberg.PartitionField{ + SourceIDs: []int{2}, FieldID: 1000, Name: "region", Transform: iceberg.IdentityTransform{}, + }) + tbl := newPartitionedCOWTable(t, sc, spec) + tblSpec := tbl.Spec() + require.Positive(t, tblSpec.NumFields(), "table must be partitioned for this test to be meaningful") + + w := cowWriter(t, tbl, "id") + + // An upsert is a keyed operation, so writeCOW reaches the file-rewrite path + // where the partition gate lives. No committer is wired because the gate must + // fire before any commit is attempted. + err := w.Write(ctx, service.MessageBatch{ + cowMsg(t, "upsert", map[string]any{"id": 2, "region": "eu", "payload": "TWO"}), + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "does not support upsert/delete on partitioned tables") +} + +// --- test helpers -------------------------------------------------------------- + +// newAmpTableWithSchema builds an unpartitioned v2 table for the given schema, +// backed by an in-memory catalog and the local filesystem. write.delete.mode is +// deliberately left unset, mirroring a connector-auto-created table, so the +// committer's explicit copy-on-write property-set path is exercised. +func newAmpTableWithSchema(t testing.TB, sc *iceberg.Schema) (*table.Table, *memCatalog) { + t.Helper() + location := filepath.ToSlash(t.TempDir()) + props := iceberg.Properties{ + table.PropertyFormatVersion: "2", + } + meta, err := table.NewMetadata(sc, iceberg.UnpartitionedSpec, table.UnsortedSortOrder, location, props) + require.NoError(t, err) + cat := &memCatalog{ + meta: meta, + metadataLocation: fmt.Sprintf("%s/metadata/00001-%s.metadata.json", location, uuid.New()), + ident: table.Identifier{"default", "cow"}, + location: location, + } + return cat.snapshot(), cat +} + +// appendCOWRows appends a batch of (id, payload) rows as one plain-data-file +// snapshot and returns the updated table handle. +func appendCOWRows(t testing.TB, ctx context.Context, tbl *table.Table, rows map[int64]string) *table.Table { + t.Helper() + arrowSc, err := table.SchemaToArrowSchema(tbl.Schema(), nil, false, false) + require.NoError(t, err) + + list := make([]map[string]any, 0, len(rows)) + for id, pay := range rows { + list = append(list, map[string]any{"id": strconv.FormatInt(id, 10), "payload": pay}) + } + b, err := json.Marshal(list) + require.NoError(t, err) + + rec, _, err := array.RecordFromJSON(memory.DefaultAllocator, arrowSc, bytes.NewReader(b)) + require.NoError(t, err) + rdr, err := array.NewRecordReader(arrowSc, []arrow.RecordBatch{rec}) + require.NoError(t, err) + rec.Release() + defer rdr.Release() + + tx := tbl.NewTransaction() + require.NoError(t, tx.Append(ctx, rdr, nil)) + next, err := tx.Commit(ctx) + require.NoError(t, err) + return next +} + +// scanRows scans the table into an id->payload map, honouring any deletes. +func scanRows(t testing.TB, ctx context.Context, tbl *table.Table) map[int64]string { + t.Helper() + at, err := tbl.Scan().ToArrowTable(ctx) + require.NoError(t, err) + defer at.Release() + + out := map[int64]string{} + tr := array.NewTableReader(at, 0) + defer tr.Release() + for tr.Next() { + rec := tr.RecordBatch() + idIdx := rec.Schema().FieldIndices("id")[0] + payIdx := rec.Schema().FieldIndices("payload")[0] + idArr := rec.Column(idIdx).(*array.Int64) + payArr := rec.Column(payIdx).(*array.String) + for r := 0; r < int(rec.NumRows()); r++ { + pay := "" + if payArr.IsValid(r) { + pay = payArr.Value(r) + } + out[idArr.Value(r)] = pay + } + } + return out +} + +// assertAllManifestsData asserts every manifest in the current snapshot is +// data-content (no delete manifests), i.e. the table holds only plain data +// files. +func assertAllManifestsData(t testing.TB, ctx context.Context, tbl *table.Table) { + t.Helper() + snap := tbl.CurrentSnapshot() + require.NotNil(t, snap) + fsys, err := tbl.FS(ctx) + require.NoError(t, err) + manifests, err := snap.Manifests(fsys) + require.NoError(t, err) + for _, m := range manifests { + assert.Equal(t, iceberg.ManifestContentData, m.ManifestContent(), "expected only data manifests") + } +} diff --git a/internal/impl/iceberg/integration/cow_row_operation_integration_test.go b/internal/impl/iceberg/integration/cow_row_operation_integration_test.go new file mode 100644 index 0000000000..cc5f4ebbca --- /dev/null +++ b/internal/impl/iceberg/integration/cow_row_operation_integration_test.go @@ -0,0 +1,137 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md + +package iceberg + +import ( + "context" + "testing" + + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/table" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/redpanda-data/benthos/v4/public/service" + "github.com/redpanda-data/benthos/v4/public/service/integration" + + icebergimpl "github.com/redpanda-data/connect/v4/internal/impl/iceberg" +) + +// countManifestsByContent loads the table's current snapshot and tallies its +// manifests by content kind. copy-on-write must leave only data manifests and +// zero delete manifests, which is what makes the result readable by +// engine-backed catalogs (Snowflake, Databricks Unity Catalog) that cannot +// apply Iceberg v2 delete files. +func countManifestsByContent(t *testing.T, ctx context.Context, tbl *table.Table) (dataManifests, deleteManifests int) { + t.Helper() + snap := tbl.CurrentSnapshot() + require.NotNil(t, snap, "table must have a current snapshot") + + fsys, err := tbl.FS(ctx) + require.NoError(t, err) + manifests, err := snap.Manifests(fsys) + require.NoError(t, err) + + for _, m := range manifests { + if m.ManifestContent() == iceberg.ManifestContentDeletes { + deleteManifests++ + } else { + dataManifests++ + } + } + return dataManifests, deleteManifests +} + +// TestCOWRowOperationsIntegration drives an insert -> upsert -> delete round +// trip through the iceberg output configured with merge_strategy: +// copy-on-write, then asserts against a real Iceberg REST catalog that (a) the +// final table state is correct (id=3 deleted, id=2 updated, id=4 inserted, id=1 +// untouched) via DuckDB, and (b) the table holds ONLY plain data files — zero +// delete files. The zero-delete-files property is the entire point of +// copy-on-write: a merge-on-read run of the same operations would leave +// equality-delete manifests, so this assertion is what distinguishes the two. +func TestCOWRowOperationsIntegration(t *testing.T) { + integration.CheckSkip(t) + + ctx := context.Background() + infra := setupTestInfra(t, ctx) + + const ns, tbl = "cow_row_ops_ns", "cow_row_ops_test" + + operation, err := service.NewInterpolatedString(`${! meta("op") }`) + require.NoError(t, err) + + router := infra.NewRouter(t, ns, tbl, + WithSchemaEvolution(icebergimpl.SchemaEvolutionConfig{Enabled: true}), + WithRowOperation(icebergimpl.RowOpConfig{ + Operation: operation, + IdentifierFields: []string{"id"}, + // The feature under test: rewrite whole data files instead of + // writing Iceberg v2 delete files. + MergeStrategy: icebergimpl.MergeStrategyCOW, + }), + ) + + // Seed three rows. id is a string so the auto-created column is a valid + // (non-floating-point) copy-on-write merge key. + produceMessages(t, ctx, router, service.MessageBatch{ + opMsg(t, "insert", `{"id": "1", "value": "one"}`), + opMsg(t, "insert", `{"id": "2", "value": "two"}`), + opMsg(t, "insert", `{"id": "3", "value": "three"}`), + }) + + // One mutating batch: upsert id=2 (new value), delete id=3, and upsert id=4 + // (a brand-new row). This exercises the combined overwrite+delete path — the + // interesting copy-on-write path that rewrites data files in a single atomic + // snapshot. + produceMessages(t, ctx, router, service.MessageBatch{ + opMsg(t, "upsert", `{"id": "2", "value": "two-updated"}`), + opMsg(t, "delete", `{"id": "3"}`), + opMsg(t, "upsert", `{"id": "4", "value": "four"}`), + }) + + // (a) Final state via DuckDB. Select the key column per the DuckDB Iceberg + // projection quirk (a projection that omits the key can misread deletes). + type row struct { + ID string `json:"id"` + Value string `json:"value"` + } + rows := querySQL[row](t, ctx, infra, + `SELECT id, value FROM iceberg_cat."cow_row_ops_ns"."cow_row_ops_test" ORDER BY id;`) + + require.Len(t, rows, 3, "expected id=1, id=2, id=4 (id=3 deleted, id=2 not duplicated)") + assert.Equal(t, "1", rows[0].ID) + assert.Equal(t, "one", rows[0].Value, "id=1 must be untouched") + assert.Equal(t, "2", rows[1].ID) + assert.Equal(t, "two-updated", rows[1].Value, "upsert must replace the prior value for id=2") + assert.Equal(t, "4", rows[2].ID) + assert.Equal(t, "four", rows[2].Value, "upsert of a new key must insert id=4") + + // (b) Zero delete files. Load the committed table through the REST catalog + // and inspect its snapshot manifests directly. The assertion is non-vacuous: + // we require at least one data manifest (proving we actually read real + // manifest content off MinIO) AND exactly zero delete manifests. A + // merge-on-read run of the identical upsert/delete batch would have produced + // delete-content manifests here. + client := infra.NewCatalogClient(t, ns) + loaded, err := client.LoadTable(ctx, tbl) + require.NoError(t, err) + + dataManifests, deleteManifests := countManifestsByContent(t, ctx, loaded) + assert.Positive(t, dataManifests, "expected at least one data manifest to inspect") + assert.Zero(t, deleteManifests, "copy-on-write must leave zero delete manifests") + + // Belt and braces: the mutating batch must have landed as an overwrite (whole + // data-file rewrite), not as a delete-file append. This proves the mutation + // was materialised the copy-on-write way rather than the table merely + // happening to have no delete files. + require.NotNil(t, loaded.CurrentSnapshot()) + assert.Equal(t, table.OpOverwrite, loaded.CurrentSnapshot().Summary.Operation, + "the upsert+delete batch must commit as an overwrite under copy-on-write") +} diff --git a/internal/impl/iceberg/output_iceberg.go b/internal/impl/iceberg/output_iceberg.go index 2005e376f7..ae5822d4c0 100644 --- a/internal/impl/iceberg/output_iceberg.go +++ b/internal/impl/iceberg/output_iceberg.go @@ -578,6 +578,19 @@ func parseRowOpConfig(conf *service.ParsedConfig) (RowOpConfig, error) { return cfg, err } + // merge_strategy carries Default(merge-on-read) and is a validated enum, so + // FieldString returns a known value without a Contains guard. + strategy, err := conf.FieldString(ioFieldMergeStrategy) + if err != nil { + return cfg, err + } + switch mergeStrategy(strategy) { + case mergeStrategyMOR, mergeStrategyCOW: + cfg.MergeStrategy = mergeStrategy(strategy) + default: + return cfg, fmt.Errorf("invalid %s %q: must be %q or %q", ioFieldMergeStrategy, strategy, mergeStrategyMOR, mergeStrategyCOW) + } + if static, ok := op.Static(); ok { parsed, err := parseRowOperation(static) if err != nil { diff --git a/internal/impl/iceberg/router.go b/internal/impl/iceberg/router.go index 10cfd1f8dc..0608a76751 100644 --- a/internal/impl/iceberg/router.go +++ b/internal/impl/iceberg/router.go @@ -496,8 +496,14 @@ func (r *Router) buildSchemaWithResolver(record map[string]any, msg *service.Mes // With no identifier_fields configured the schema is created exactly as before // (all columns optional, no identifier-field-ids), so append-only table // creation is unchanged. +// +// Under copy-on-write (merge_strategy: copy-on-write) the identifier_fields are +// the connector-side merge key only and are deliberately NOT registered as the +// table's Iceberg identifier-field-ids, nor are the columns forced required: +// this is what lets engine-backed catalogs (e.g. the Databricks Unity Catalog) +// accept the CREATE TABLE. Merge-on-read (the default) keeps registering them. func (r *Router) schemaWithIdentifierFields(fields []iceberg.NestedField) (*iceberg.Schema, error) { - if len(r.rowOpCfg.IdentifierFields) == 0 { + if len(r.rowOpCfg.IdentifierFields) == 0 || r.rowOpCfg.MergeStrategy == mergeStrategyCOW { return iceberg.NewSchema(0, fields...), nil } diff --git a/internal/impl/iceberg/row_operation_test.go b/internal/impl/iceberg/row_operation_test.go index ed02d00546..494e01ac9c 100644 --- a/internal/impl/iceberg/row_operation_test.go +++ b/internal/impl/iceberg/row_operation_test.go @@ -514,6 +514,12 @@ iceberg: {"static upsert at default in-flight", " row_operation: upsert\n identifier_fields: [id]\n", true}, {"static upsert with in-flight 1", " row_operation: upsert\n identifier_fields: [id]\n max_in_flight: 1\n", false}, {"dynamic operation at default in-flight", " row_operation: '${! metadata(\"op\") }'\n identifier_fields: [id]\n", true}, + // The ordering guard is merge_strategy-agnostic: copy-on-write commits + // are serialized per-commit, but with more than one batch in flight two + // batches can still land out of order and let a stale overwrite win, so + // the lint must fire for a mutating copy-on-write config too. + {"copy-on-write upsert at default in-flight", " row_operation: upsert\n identifier_fields: [id]\n merge_strategy: copy-on-write\n", true}, + {"copy-on-write upsert with in-flight 1", " row_operation: upsert\n identifier_fields: [id]\n merge_strategy: copy-on-write\n max_in_flight: 1\n", false}, } for _, tc := range cases { diff --git a/internal/impl/iceberg/writer.go b/internal/impl/iceberg/writer.go index 3ffc08c477..b3df53832c 100644 --- a/internal/impl/iceberg/writer.go +++ b/internal/impl/iceberg/writer.go @@ -61,15 +61,36 @@ func parseRowOperation(s string) (rowOperation, error) { } } +// mergeStrategy selects how upsert/delete mutations are materialised on disk. +type mergeStrategy string + +const ( + // mergeStrategyMOR (merge-on-read) writes Iceberg v2 equality-delete files. + // This is the default and preserves all pre-existing behaviour. + mergeStrategyMOR mergeStrategy = "merge-on-read" + // mergeStrategyCOW (copy-on-write) rewrites whole data files so the table + // only ever contains plain data files (no delete files), readable by + // engine-backed catalogs such as Snowflake and the Databricks Unity Catalog. + mergeStrategyCOW mergeStrategy = "copy-on-write" +) + +// MergeStrategyCOW is the copy-on-write merge strategy, exported so a +// RowOpConfig can select it programmatically (the merge_strategy type itself is +// internal). This mirrors the merge_strategy: copy-on-write YAML value. +const MergeStrategyCOW = mergeStrategyCOW + // RowOpConfig configures per-message row-level operations. type RowOpConfig struct { // Operation resolves per message to insert, upsert, or delete. When it // resolves to the empty string the default (insert) is assumed. Operation *service.InterpolatedString - // IdentifierFields are the table column names forming the equality-delete - // key (the Iceberg identifier fields) used by upsert and delete. Empty for - // append-only (insert) workloads. + // IdentifierFields are the table column names forming the merge key (the + // Iceberg identifier fields / equality-delete key) used by upsert and + // delete. Empty for append-only (insert) workloads. IdentifierFields []string + // MergeStrategy selects how upsert/delete are materialised: merge-on-read + // (equality deletes, the default) or copy-on-write (whole-file rewrite). + MergeStrategy mergeStrategy } // mutating reports whether the configuration can ever produce a non-insert @@ -152,9 +173,16 @@ func (w *writer) Write(ctx context.Context, batch service.MessageBatch) error { return nil } - // Row-level mutations: split the batch by operation, write inserted rows as - // data files and deleted/upserted keys as equality-delete files, then commit - // them together so a single snapshot reflects the whole batch. + // Copy-on-write: rewrite whole data files (no delete files) so the result is + // readable by engine-backed catalogs. Handled on its own path. + if w.rowOpCfg.MergeStrategy == mergeStrategyCOW { + return w.writeCOW(ctx, batch) + } + + // Row-level mutations (merge-on-read): split the batch by operation, write + // inserted rows as data files and deleted/upserted keys as equality-delete + // files, then commit them together so a single snapshot reflects the whole + // batch. inserts, deletes, counts, err := w.splitByOperation(batch) if err != nil { return fmt.Errorf("splitting batch by row operation: %w", err) From 753ae1ae09d071deca2d537bda97da0892e2935a Mon Sep 17 00:00:00 2001 From: Ashley Jeffs Date: Tue, 21 Jul 2026 20:45:18 +0100 Subject: [PATCH 02/12] iceberg: broaden copy-on-write to the full support matrix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends the opt-in copy-on-write merge strategy from the initial flat-primitive/unpartitioned prototype to the breadth merge-on-read already supports, and tightens a couple of edges: - Column types: adds binary/fixed and nested struct/list/map (whose leaves must be supported primitives), via a recursive value massager that applies the same canonicalisation at every depth. This fixes silent truncation of integers nested beyond 2^53 and reshapes maps to the Arrow map JSON encoding. - Merge-key types: adds date/time/timestamp/timestamptz/uuid, each with a round-trip test that proves the filter matched the intended row (not just "no error"). Decimal merge keys are rejected up front with a clear message — an upstream substrait conversion panics on a decimal filter literal — pointing at merge-on-read instead. - Partitioned tables: supported. Copy-on-write rewrites by filter and appends value-routed rows, so it needs no partition-vs-identifier constraint and can even move a key across partitions. - Format version: copy-on-write no longer forces the irreversible v1->v2 upgrade, since it only ever writes plain data files. Also adds scale and memory benchmarks and a docs pass: a merge-on-read vs copy-on-write decision guide, a worked Snowflake/Databricks example, a maintenance runbook, and the current limitations. --- .../components/pages/outputs/iceberg.adoc | 95 ++++- internal/impl/iceberg/committer.go | 8 +- internal/impl/iceberg/config.go | 87 +++- internal/impl/iceberg/cow.go | 274 +++++++++++-- .../iceberg/cow_amplification_bench_test.go | 159 ++++++++ .../iceberg/cow_merge_key_roundtrip_test.go | 227 +++++++++++ internal/impl/iceberg/cow_test.go | 356 +++++++++++++++-- .../impl/iceberg/cow_type_roundtrip_test.go | 374 ++++++++++++++++++ .../cow_partitioned_integration_test.go | 130 ++++++ internal/impl/iceberg/router.go | 8 +- 10 files changed, 1634 insertions(+), 84 deletions(-) create mode 100644 internal/impl/iceberg/cow_merge_key_roundtrip_test.go create mode 100644 internal/impl/iceberg/cow_type_roundtrip_test.go create mode 100644 internal/impl/iceberg/integration/cow_partitioned_integration_test.go diff --git a/docs/modules/components/pages/outputs/iceberg.adoc b/docs/modules/components/pages/outputs/iceberg.adoc index 4e55392e19..a370ebcf49 100644 --- a/docs/modules/components/pages/outputs/iceberg.adoc +++ b/docs/modules/components/pages/outputs/iceberg.adoc @@ -232,13 +232,11 @@ By default this output is append-only — every message becomes a new row (`row_ `row_operation` supports interpolation, so the operation can be driven by the data itself — for example by mapping a change-data-capture stream's operation field — but no CDC-specific format is assumed (see the change-data-capture example below). It is named `row_operation` to distinguish it from Iceberg's snapshot-level operation. -`upsert` and `delete` require `identifier_fields`. By default (`merge_strategy: merge-on-read`) they use Iceberg merge-on-read equality deletes, which require table format version 2. A version-1 table is automatically upgraded to version 2 on the first `upsert`/`delete`; *this upgrade is irreversible*. +`upsert` and `delete` require `identifier_fields`. How those mutations are materialised on disk — and, critically, which query engines can then read the table — is controlled by `merge_strategy`. See <> below for the decision guide, support matrix, and maintenance guidance. -*Merge strategy.* `merge_strategy` controls how mutations are materialised. `merge-on-read` (the default) writes equality-delete files and is the streaming path, but only catalog-only/Flink-world engines can read the result. `copy-on-write` instead rewrites whole data files so the table only ever holds plain data files that every engine can read — including engine-backed catalogs such as Snowflake and the Databricks Unity Catalog — at the cost of heavy write amplification (each mutating batch rewrites every data file holding a touched key). Treat `copy-on-write` as a batch / moderate-throughput mode: sort the table by the identifier key and use large batches. Under `copy-on-write` the `identifier_fields` are the connector-side merge key only and are not registered as the table's Iceberg identifier-field-ids, so auto-created tables carry no identifier-field spec and columns are not forced required. This prototype's `copy-on-write` path supports tables whose columns are all flat, primitive types; a table with nested (struct/list/map) columns, or a merge key that is not an `int`/`long`/`string`/`boolean` column, is rejected with a clear error at write time. +*Identifier fields.* `identifier_fields` must reference existing table columns of a primitive, non-floating-point type. A static `upsert`/`delete` is validated at startup; an interpolated `row_operation` is validated per message at write time, so an empty `identifier_fields` is not caught until the first `upsert`/`delete` message arrives. Identifier columns of a temporal type (`timestamp`, `timestamptz`, `date`, `time`) must arrive as time values, not bare numbers — a numeric epoch is ambiguous as a delete key and is rejected at write time; convert it to a timestamp upstream. If the table is partitioned, `merge-on-read` additionally requires every partition source column to be one of the `identifier_fields`, since equality deletes are partition-scoped; `copy-on-write` carries no such restriction (it rewrites whole files by filter and routes new rows by value, and can even move a key across partitions). See the support matrix for the merge-key types each strategy accepts. -*Identifier fields.* `identifier_fields` must reference existing table columns of a primitive, non-floating-point type. A static `upsert`/`delete` is validated at startup; an interpolated `row_operation` is validated per message at write time, so an empty `identifier_fields` is not caught until the first `upsert`/`delete` message arrives. Identifier columns of a temporal type (`timestamp`, `timestamptz`, `date`, `time`) must arrive as time values, not bare numbers — a numeric epoch is ambiguous as a delete key and is rejected at write time; convert it to a timestamp upstream. If the table is partitioned, every partition source column must be one of the `identifier_fields`, since equality deletes are partition-scoped. - -When this output auto-creates a table (via `schema_evolution`), the `identifier_fields` columns are created as *required* and registered as the table's Iceberg identifier-field-ids, so downstream engines and other writers see the primary key. A consequence is that a null or missing value in an identifier column is rejected on write, even for `insert`. Identifier columns must therefore be present at creation — in the first message or declared via `schema_metadata`. Pre-existing tables are never modified. +Under `merge-on-read`, when this output auto-creates a table (via `schema_evolution`), the `identifier_fields` columns are created as *required* and registered as the table's Iceberg identifier-field-ids, so downstream engines and other writers see the primary key. A consequence is that a null or missing value in an identifier column is rejected on write, even for `insert`; identifier columns must therefore be present at creation — in the first message or declared via `schema_metadata`. Under `copy-on-write` the identifier fields are used only as the connector-side merge key and are *not* registered as identifier-field-ids, so auto-created columns are not forced required (this is also what lets engine-backed catalogs such as the Databricks Unity Catalog, which rejects identifier-field-ids at table creation, accept the `CREATE TABLE`). Pre-existing tables are never modified. *Batching and ordering.* Within a single batch the last `upsert`/`delete` per `identifier_fields` key wins. Each batch containing an `upsert`/`delete` is committed as its own snapshot (these commits are never coalesced, which is required for correctness), so a high-throughput mutation workload produces one snapshot per batch. Size batches accordingly and run regular table maintenance (snapshot expiry and compaction) to keep metadata manageable. Pure `insert`-only batches keep the original append fast path, which does coalesce commits. @@ -249,6 +247,38 @@ Ordering only holds *within* a batch. With more than one batch in flight, concur `insert` is an unconditional append and is *not* keyed or de-duplicated. For keyed data (including change-data-capture), map create/read events to `upsert`, never `insert` — mixing `insert` with `upsert`/`delete` on the same key in one batch produces duplicate rows. ==== +[[merge-strategies]] +=== Merge strategies: merge-on-read vs copy-on-write + +`merge_strategy` controls how `upsert`/`delete` mutations are written, which sets both the write cost and — most importantly — which query engines can read the table. + +* `merge-on-read` (the default) writes Iceberg v2 equality-delete files and applies them at read time. Writes stay cheap and streaming-friendly, but only catalog-native / Flink-world engines (Apache Polaris, Flink, Trino, Spark) can read equality deletes. Engine-backed catalogs — Snowflake and the Databricks Unity Catalog — cannot. It requires table format version 2, so a version-1 table is automatically upgraded to version 2 on the first `upsert`/`delete`; *this upgrade is irreversible*. +* `copy-on-write` instead rewrites whole data files so the table only ever holds plain data files — no equality- or positional-delete files. Every engine that reads Iceberg data files can read the result: Snowflake and the Databricks Unity Catalog as well as Polaris, Flink and Trino. Because it writes only plain data files it works on version-1 or version-2 tables and never forces the irreversible v1->v2 upgrade. + +*Which to choose.* + +* Choose `merge-on-read` for streaming or high-throughput mutation into a lake read by Polaris, Flink, Trino or Spark, where write cost must stay low. +* Choose `copy-on-write` when the table must be correct on Snowflake or the Databricks Unity Catalog (or any engine that cannot read equality deletes), and the workload is batch or moderate-throughput so the write amplification is acceptable. + +*Copy-on-write support matrix.* + +* *Column types:* all flat primitives (`boolean`, `int`, `long`, `float`, `double`, `string`, `date`, `time`, `timestamp`, `timestamptz`, `decimal`, `uuid`, `binary`, `fixed`), and nested `struct`/`list`/`map` columns whose leaves are all supported primitives. +* *Merge-key (`identifier_fields`) types:* `boolean`, `int`, `long`, `string`, `date`, `time`, `timestamp`, `timestamptz` and `uuid`. A `decimal` merge key is *not* supported and errors with a message pointing you at `merge-on-read` (an upstream limitation in the Iceberg library's overwrite filter); `decimal` as a non-key column is fine. +* *Partitioned tables:* supported, with no requirement that the partition columns be a subset of `identifier_fields`. A `copy-on-write` `upsert` can even move a key from one partition to another. +* *Table format:* version 1 or version 2, with no forced upgrade. + +*Write amplification and throughput.* `copy-on-write` rewrites every data file that contains a touched key: a batch of K keys scattered over M files rewrites roughly K/M of the table, and touching even a single key in a file rewrites that whole file — so a one-row change to a 512 MB file rewrites all 512 MB. To keep amplification low, sort the table by the identifier key so a batch's keys cluster into as few files as possible, and use large batches. This is a batch / moderate-throughput mode, not a streaming one. + +*Memory.* Under `copy-on-write` the whole new-row batch is materialised in memory as a single Arrow record while the batch commits, so a keyed batch's memory scales with its total row bytes. Size keyed batches to stay within the process memory budget rather than making them arbitrarily large. + +*Maintenance.* Because every mutating batch rewrites files and adds a snapshot, a high-churn `copy-on-write` workload accumulates data files and snapshots quickly. Run regular table maintenance: compaction (rewrite / bin-pack data files), snapshot expiry, and orphan-file removal. Orphan-file removal matters specifically because a `copy-on-write` commit that fails *ambiguously* (the catalog may or may not have recorded it) can leave newly-written data files unreferenced; the connector cleans these up best-effort, but periodic orphan-file removal is the definitive backstop. + +*Copy-on-write limitations.* + +* A `decimal` merge key is not supported — use `merge-on-read` for a decimal key (a `decimal` non-key column is fine). +* Schema evolution covers new *top-level* columns only; new fields appearing inside an existing nested `struct`/`list`/`map` column are not auto-surfaced for evolution. +* It is a batch / moderate-throughput mode: expect heavy write amplification under scattered, high-frequency keyed mutations. + == Performance @@ -300,6 +330,53 @@ output: region: us-east-1 ``` +-- +CDC into a Snowflake- or Databricks-readable table (copy-on-write):: ++ +-- + +Materialize a change-data-capture stream into an Iceberg table that must be read by an engine-backed catalog such as Snowflake or the Databricks Unity Catalog, which cannot read merge-on-read equality deletes. `merge_strategy: copy-on-write` rewrites whole data files so the table only ever holds plain data files that every engine can read. As with any keyed workload it requires `max_in_flight: 1`; because copy-on-write rewrites every file containing a touched key, prefer large batches and a table sorted by the identifier key, and run regular compaction and snapshot expiry. + +```yaml +input: + redpanda: + seed_brokers: [ localhost:9092 ] + topics: [ dbserver.inventory.customers ] + consumer_group: iceberg_sink_cow + +pipeline: + processors: + - mapping: | + meta op = match this.op { + "d" => "delete", + _ => "upsert", + } + root = this.after | this.before + +output: + iceberg: + catalog: + url: http://localhost:8181/api/catalog + namespace: inventory + table: customers + row_operation: ${! metadata("op") } + identifier_fields: [ id ] + # copy-on-write produces only plain data files, so Snowflake and the + # Databricks Unity Catalog can read the result (unlike equality deletes). + merge_strategy: copy-on-write + # Keyed writes must stay ordered; a single batch in flight prevents a stale + # update from overwriting a newer one for the same key. + max_in_flight: 1 + # Amortise the file rewrites over larger commits. + batching: + count: 5000 + period: 30s + storage: + aws_s3: + bucket: my-iceberg-data + region: us-east-1 +``` + -- ====== @@ -690,12 +767,10 @@ identifier_fields: How `upsert` and `delete` are materialised on disk. -* `merge-on-read` (the default) writes Iceberg v2 equality-delete files. Deletes are applied at read time, so writes stay cheap and streaming-friendly, but only catalog-only/Flink-world engines can read the result — engine-backed catalogs such as Snowflake and the Databricks Unity Catalog cannot read equality deletes. -* `copy-on-write` rewrites whole data files so the table only ever contains plain data files (no delete files), which every engine can read — including Snowflake and Databricks Unity Catalog. The trade-off is heavy write amplification: each mutating batch rewrites every data file that contains a touched key. This is a batch / moderate-throughput mode, not a streaming one. Sort the table by the identifier key and use large batches so each rewrite touches as few files as possible. - -Under `copy-on-write` the `identifier_fields` are used only connector-side as the merge key and are *not* registered as the table's Iceberg identifier-field-ids, so auto-created tables carry no identifier-field spec (this is what lets the Databricks Unity Catalog accept the `CREATE TABLE`). `merge-on-read` continues to register them. +* `merge-on-read` (the default) writes Iceberg v2 equality-delete files. Deletes are applied at read time, so writes stay cheap and streaming-friendly, but only catalog-native / Flink-world engines can read the result — engine-backed catalogs such as Snowflake and the Databricks Unity Catalog cannot read equality deletes. +* `copy-on-write` rewrites whole data files so the table only ever contains plain data files (no delete files), which every engine can read — including Snowflake and Databricks Unity Catalog. It works on version-1 or version-2 tables and never forces the irreversible v1->v2 upgrade. The trade-off is heavy write amplification: each mutating batch rewrites every data file that contains a touched key, so it is a batch / moderate-throughput mode. Sort the table by the identifier key and use large batches so each rewrite touches as few files as possible. -See the <> section above for more detail. +See the <> section above for the full decision guide, copy-on-write support matrix (column and merge-key types, partitioning, table format), and maintenance guidance. *Type*: `string` diff --git a/internal/impl/iceberg/committer.go b/internal/impl/iceberg/committer.go index b07573ea9f..a374f92c46 100644 --- a/internal/impl/iceberg/committer.go +++ b/internal/impl/iceberg/committer.go @@ -63,6 +63,12 @@ type CommitConfig struct { ManifestMergeEnabled bool MaxSnapshotAge time.Duration MaxRetries int + // SkipFormatUpgrade leaves the table at its existing format version instead + // of upgrading to v2. Set for copy-on-write, which only ever writes plain + // data files (no v2 delete files) and so works on a v1 table — avoiding an + // unnecessary, irreversible v1->v2 upgrade. Merge-on-read/append leave this + // false: their equality-delete path requires v2. + SkipFormatUpgrade bool } // StaleSchemaError is returned when data was written with a schema @@ -381,7 +387,7 @@ func (c *committer) commitLocked(ctx context.Context, retryOnUnknownState bool, for range c.cfg.MaxRetries { attempt++ txn := c.table.NewTransaction() - if c.table.Metadata().Version() < CurrentIcebergVersion { + if !c.cfg.SkipFormatUpgrade && c.table.Metadata().Version() < CurrentIcebergVersion { c.upgradeWarnOnce.Do(func() { c.logger.Warnf("Upgrading iceberg table to format version %d to support row-level deletes; this change is irreversible", CurrentIcebergVersion) }) diff --git a/internal/impl/iceberg/config.go b/internal/impl/iceberg/config.go index c4ae504772..85da15dc36 100644 --- a/internal/impl/iceberg/config.go +++ b/internal/impl/iceberg/config.go @@ -118,13 +118,11 @@ const rowOperationDocs = "\n" + "\n" + "`row_operation` supports interpolation, so the operation can be driven by the data itself — for example by mapping a change-data-capture stream's operation field — but no CDC-specific format is assumed (see the change-data-capture example below). It is named `row_operation` to distinguish it from Iceberg's snapshot-level operation.\n" + "\n" + - "`upsert` and `delete` require `identifier_fields`. By default (`merge_strategy: merge-on-read`) they use Iceberg merge-on-read equality deletes, which require table format version 2. A version-1 table is automatically upgraded to version 2 on the first `upsert`/`delete`; *this upgrade is irreversible*.\n" + + "`upsert` and `delete` require `identifier_fields`. How those mutations are materialised on disk — and, critically, which query engines can then read the table — is controlled by `merge_strategy`. See <> below for the decision guide, support matrix, and maintenance guidance.\n" + "\n" + - "*Merge strategy.* `merge_strategy` controls how mutations are materialised. `merge-on-read` (the default) writes equality-delete files and is the streaming path, but only catalog-only/Flink-world engines can read the result. `copy-on-write` instead rewrites whole data files so the table only ever holds plain data files that every engine can read — including engine-backed catalogs such as Snowflake and the Databricks Unity Catalog — at the cost of heavy write amplification (each mutating batch rewrites every data file holding a touched key). Treat `copy-on-write` as a batch / moderate-throughput mode: sort the table by the identifier key and use large batches. Under `copy-on-write` the `identifier_fields` are the connector-side merge key only and are not registered as the table's Iceberg identifier-field-ids, so auto-created tables carry no identifier-field spec and columns are not forced required. This prototype's `copy-on-write` path supports tables whose columns are all flat, primitive types; a table with nested (struct/list/map) columns, or a merge key that is not an `int`/`long`/`string`/`boolean` column, is rejected with a clear error at write time.\n" + + "*Identifier fields.* `identifier_fields` must reference existing table columns of a primitive, non-floating-point type. A static `upsert`/`delete` is validated at startup; an interpolated `row_operation` is validated per message at write time, so an empty `identifier_fields` is not caught until the first `upsert`/`delete` message arrives. Identifier columns of a temporal type (`timestamp`, `timestamptz`, `date`, `time`) must arrive as time values, not bare numbers — a numeric epoch is ambiguous as a delete key and is rejected at write time; convert it to a timestamp upstream. If the table is partitioned, `merge-on-read` additionally requires every partition source column to be one of the `identifier_fields`, since equality deletes are partition-scoped; `copy-on-write` carries no such restriction (it rewrites whole files by filter and routes new rows by value, and can even move a key across partitions). See the support matrix for the merge-key types each strategy accepts.\n" + "\n" + - "*Identifier fields.* `identifier_fields` must reference existing table columns of a primitive, non-floating-point type. A static `upsert`/`delete` is validated at startup; an interpolated `row_operation` is validated per message at write time, so an empty `identifier_fields` is not caught until the first `upsert`/`delete` message arrives. Identifier columns of a temporal type (`timestamp`, `timestamptz`, `date`, `time`) must arrive as time values, not bare numbers — a numeric epoch is ambiguous as a delete key and is rejected at write time; convert it to a timestamp upstream. If the table is partitioned, every partition source column must be one of the `identifier_fields`, since equality deletes are partition-scoped.\n" + - "\n" + - "When this output auto-creates a table (via `schema_evolution`), the `identifier_fields` columns are created as *required* and registered as the table's Iceberg identifier-field-ids, so downstream engines and other writers see the primary key. A consequence is that a null or missing value in an identifier column is rejected on write, even for `insert`. Identifier columns must therefore be present at creation — in the first message or declared via `schema_metadata`. Pre-existing tables are never modified.\n" + + "Under `merge-on-read`, when this output auto-creates a table (via `schema_evolution`), the `identifier_fields` columns are created as *required* and registered as the table's Iceberg identifier-field-ids, so downstream engines and other writers see the primary key. A consequence is that a null or missing value in an identifier column is rejected on write, even for `insert`; identifier columns must therefore be present at creation — in the first message or declared via `schema_metadata`. Under `copy-on-write` the identifier fields are used only as the connector-side merge key and are *not* registered as identifier-field-ids, so auto-created columns are not forced required (this is also what lets engine-backed catalogs such as the Databricks Unity Catalog, which rejects identifier-field-ids at table creation, accept the `CREATE TABLE`). Pre-existing tables are never modified.\n" + "\n" + "*Batching and ordering.* Within a single batch the last `upsert`/`delete` per `identifier_fields` key wins. Each batch containing an `upsert`/`delete` is committed as its own snapshot (these commits are never coalesced, which is required for correctness), so a high-throughput mutation workload produces one snapshot per batch. Size batches accordingly and run regular table maintenance (snapshot expiry and compaction) to keep metadata manageable. Pure `insert`-only batches keep the original append fast path, which does coalesce commits.\n" + "\n" + @@ -133,7 +131,39 @@ const rowOperationDocs = "\n" + "[CAUTION]\n" + "====\n" + "`insert` is an unconditional append and is *not* keyed or de-duplicated. For keyed data (including change-data-capture), map create/read events to `upsert`, never `insert` — mixing `insert` with `upsert`/`delete` on the same key in one batch produces duplicate rows.\n" + - "====\n" + "====\n" + + "\n" + + "[[merge-strategies]]\n" + + "=== Merge strategies: merge-on-read vs copy-on-write\n" + + "\n" + + "`merge_strategy` controls how `upsert`/`delete` mutations are written, which sets both the write cost and — most importantly — which query engines can read the table.\n" + + "\n" + + "* `merge-on-read` (the default) writes Iceberg v2 equality-delete files and applies them at read time. Writes stay cheap and streaming-friendly, but only catalog-native / Flink-world engines (Apache Polaris, Flink, Trino, Spark) can read equality deletes. Engine-backed catalogs — Snowflake and the Databricks Unity Catalog — cannot. It requires table format version 2, so a version-1 table is automatically upgraded to version 2 on the first `upsert`/`delete`; *this upgrade is irreversible*.\n" + + "* `copy-on-write` instead rewrites whole data files so the table only ever holds plain data files — no equality- or positional-delete files. Every engine that reads Iceberg data files can read the result: Snowflake and the Databricks Unity Catalog as well as Polaris, Flink and Trino. Because it writes only plain data files it works on version-1 or version-2 tables and never forces the irreversible v1->v2 upgrade.\n" + + "\n" + + "*Which to choose.*\n" + + "\n" + + "* Choose `merge-on-read` for streaming or high-throughput mutation into a lake read by Polaris, Flink, Trino or Spark, where write cost must stay low.\n" + + "* Choose `copy-on-write` when the table must be correct on Snowflake or the Databricks Unity Catalog (or any engine that cannot read equality deletes), and the workload is batch or moderate-throughput so the write amplification is acceptable.\n" + + "\n" + + "*Copy-on-write support matrix.*\n" + + "\n" + + "* *Column types:* all flat primitives (`boolean`, `int`, `long`, `float`, `double`, `string`, `date`, `time`, `timestamp`, `timestamptz`, `decimal`, `uuid`, `binary`, `fixed`), and nested `struct`/`list`/`map` columns whose leaves are all supported primitives.\n" + + "* *Merge-key (`identifier_fields`) types:* `boolean`, `int`, `long`, `string`, `date`, `time`, `timestamp`, `timestamptz` and `uuid`. A `decimal` merge key is *not* supported and errors with a message pointing you at `merge-on-read` (an upstream limitation in the Iceberg library's overwrite filter); `decimal` as a non-key column is fine.\n" + + "* *Partitioned tables:* supported, with no requirement that the partition columns be a subset of `identifier_fields`. A `copy-on-write` `upsert` can even move a key from one partition to another.\n" + + "* *Table format:* version 1 or version 2, with no forced upgrade.\n" + + "\n" + + "*Write amplification and throughput.* `copy-on-write` rewrites every data file that contains a touched key: a batch of K keys scattered over M files rewrites roughly K/M of the table, and touching even a single key in a file rewrites that whole file — so a one-row change to a 512 MB file rewrites all 512 MB. To keep amplification low, sort the table by the identifier key so a batch's keys cluster into as few files as possible, and use large batches. This is a batch / moderate-throughput mode, not a streaming one.\n" + + "\n" + + "*Memory.* Under `copy-on-write` the whole new-row batch is materialised in memory as a single Arrow record while the batch commits, so a keyed batch's memory scales with its total row bytes. Size keyed batches to stay within the process memory budget rather than making them arbitrarily large.\n" + + "\n" + + "*Maintenance.* Because every mutating batch rewrites files and adds a snapshot, a high-churn `copy-on-write` workload accumulates data files and snapshots quickly. Run regular table maintenance: compaction (rewrite / bin-pack data files), snapshot expiry, and orphan-file removal. Orphan-file removal matters specifically because a `copy-on-write` commit that fails *ambiguously* (the catalog may or may not have recorded it) can leave newly-written data files unreferenced; the connector cleans these up best-effort, but periodic orphan-file removal is the definitive backstop.\n" + + "\n" + + "*Copy-on-write limitations.*\n" + + "\n" + + "* A `decimal` merge key is not supported — use `merge-on-read` for a decimal key (a `decimal` non-key column is fine).\n" + + "* Schema evolution covers new *top-level* columns only; new fields appearing inside an existing nested `struct`/`list`/`map` column are not auto-surfaced for evolution.\n" + + "* It is a batch / moderate-throughput mode: expect heavy write amplification under scattered, high-frequency keyed mutations.\n" // icebergOutputConfig returns the configuration spec for the Iceberg output. func icebergOutputConfig() *service.ConfigSpec { @@ -282,7 +312,7 @@ array:list Advanced(), service.NewStringEnumField(ioFieldMergeStrategy, string(mergeStrategyMOR), string(mergeStrategyCOW)). - Description("How `upsert` and `delete` are materialised on disk.\n\n* `merge-on-read` (the default) writes Iceberg v2 equality-delete files. Deletes are applied at read time, so writes stay cheap and streaming-friendly, but only catalog-only/Flink-world engines can read the result — engine-backed catalogs such as Snowflake and the Databricks Unity Catalog cannot read equality deletes.\n* `copy-on-write` rewrites whole data files so the table only ever contains plain data files (no delete files), which every engine can read — including Snowflake and Databricks Unity Catalog. The trade-off is heavy write amplification: each mutating batch rewrites every data file that contains a touched key. This is a batch / moderate-throughput mode, not a streaming one. Sort the table by the identifier key and use large batches so each rewrite touches as few files as possible.\n\nUnder `copy-on-write` the `identifier_fields` are used only connector-side as the merge key and are *not* registered as the table's Iceberg identifier-field-ids, so auto-created tables carry no identifier-field spec (this is what lets the Databricks Unity Catalog accept the `CREATE TABLE`). `merge-on-read` continues to register them.\n\nSee the <> section above for more detail."). + Description("How `upsert` and `delete` are materialised on disk.\n\n* `merge-on-read` (the default) writes Iceberg v2 equality-delete files. Deletes are applied at read time, so writes stay cheap and streaming-friendly, but only catalog-native / Flink-world engines can read the result — engine-backed catalogs such as Snowflake and the Databricks Unity Catalog cannot read equality deletes.\n* `copy-on-write` rewrites whole data files so the table only ever contains plain data files (no delete files), which every engine can read — including Snowflake and Databricks Unity Catalog. It works on version-1 or version-2 tables and never forces the irreversible v1->v2 upgrade. The trade-off is heavy write amplification: each mutating batch rewrites every data file that contains a touched key, so it is a batch / moderate-throughput mode. Sort the table by the identifier key and use large batches so each rewrite touches as few files as possible.\n\nSee the <> section above for the full decision guide, copy-on-write support matrix (column and merge-key types, partitioning, table format), and maintenance guidance."). Default(string(mergeStrategyMOR)). Advanced(), @@ -489,6 +519,49 @@ output: aws_s3: bucket: my-iceberg-data region: us-east-1 +`, + ). + Example( + "CDC into a Snowflake- or Databricks-readable table (copy-on-write)", + "Materialize a change-data-capture stream into an Iceberg table that must be read by an engine-backed catalog such as Snowflake or the Databricks Unity Catalog, which cannot read merge-on-read equality deletes. `merge_strategy: copy-on-write` rewrites whole data files so the table only ever holds plain data files that every engine can read. As with any keyed workload it requires `max_in_flight: 1`; because copy-on-write rewrites every file containing a touched key, prefer large batches and a table sorted by the identifier key, and run regular compaction and snapshot expiry.", + ` +input: + redpanda: + seed_brokers: [ localhost:9092 ] + topics: [ dbserver.inventory.customers ] + consumer_group: iceberg_sink_cow + +pipeline: + processors: + - mapping: | + meta op = match this.op { + "d" => "delete", + _ => "upsert", + } + root = this.after | this.before + +output: + iceberg: + catalog: + url: http://localhost:8181/api/catalog + namespace: inventory + table: customers + row_operation: ${! metadata("op") } + identifier_fields: [ id ] + # copy-on-write produces only plain data files, so Snowflake and the + # Databricks Unity Catalog can read the result (unlike equality deletes). + merge_strategy: copy-on-write + # Keyed writes must stay ordered; a single batch in flight prevents a stale + # update from overwriting a newer one for the same key. + max_in_flight: 1 + # Amortise the file rewrites over larger commits. + batching: + count: 5000 + period: 30s + storage: + aws_s3: + bucket: my-iceberg-data + region: us-east-1 `, ) } diff --git a/internal/impl/iceberg/cow.go b/internal/impl/iceberg/cow.go index fd499be9a0..e400f1926b 100644 --- a/internal/impl/iceberg/cow.go +++ b/internal/impl/iceberg/cow.go @@ -12,11 +12,11 @@ import ( "bytes" "context" "encoding/json" - "errors" "fmt" "math" "strconv" "strings" + "time" "github.com/apache/arrow-go/v18/arrow" "github.com/apache/arrow-go/v18/arrow/array" @@ -67,14 +67,24 @@ func (w *writer) writeCOW(ctx context.Context, batch service.MessageBatch) error return nil } - // The remaining paths rewrite data files. Partitioned copy-on-write is not - // yet validated in this prototype: iceberg-go's Overwrite routes rewritten - // rows to partitions internally, but we have only proven the unpartitioned - // case end-to-end, so fail loudly rather than risk mis-partitioned rewrites. - spec := w.table.Spec() - if spec.NumFields() > 0 { - return errors.New("copy-on-write merge_strategy does not support upsert/delete on partitioned tables in this prototype; use merge-on-read, or an unpartitioned table") - } + // The remaining paths rewrite data files. Partitioned tables are supported: + // iceberg-go's Overwrite/Delete route rows to partitions correctly end-to-end. + // - New/rewritten rows: recordsToDataFiles sends a partitioned spec through + // the partitioned fanout writer (partitioned_fanout_writer.go), which + // derives each row's partition tuple from the actual source-column value + // via PartitionField.Transform.Apply — so every transform (identity, + // bucket, truncate, year/month/day/hour) routes correctly. This is not the + // stats-inference path (fileToDataFile) that panics on non-order-preserving + // transforms; that path is only used by AddFiles. + // - Deletions: classifyFilesForFilteredDeletions evaluates the filter against + // every data file's stats across all partitions. A merge key on a + // non-partition column projects to AlwaysTrue in the partition space, so no + // partition is pruned and matching rows are found in every partition. + // Unlike merge-on-read equality deletes (which are partition-scoped and so + // require the partition source columns to be a subset of identifier_fields, see + // writer.deleteRecordFields), copy-on-write rewrites whole files by filter and + // appends real rows routed by value, so it carries no such constraint — the + // merge key need not include (or be) the partition column. // The rewrite builds records through the Arrow JSON round-trip, so the whole // table schema must be faithfully representable that way. @@ -125,28 +135,68 @@ func (w *writer) writeCOW(ctx context.Context, batch service.MessageBatch) error return nil } -// checkCOWSchemaSupported rejects table schemas the prototype's copy-on-write -// path cannot faithfully round-trip through Arrow. The custom shredder handles -// nested/complex types on the append path, but the copy-on-write rewrite builds -// records via array.RecordFromJSON, which we have only verified for flat, -// primitive columns. Rather than silently mis-write, fail loudly with an -// actionable message. +// checkCOWSchemaSupported rejects table schemas the copy-on-write path cannot +// faithfully round-trip through Arrow. The rewrite builds records via +// array.RecordFromJSON from the JSON produced by cowMassage; that projection is +// recursive, so nested struct/list/map columns are supported as long as every +// leaf is a supported primitive. Each type kind is checked by walking the type +// tree; any unsupported leaf fails loudly with an actionable message rather than +// risking a silent mis-write. func checkCOWSchemaSupported(s *iceberg.Schema) error { for _, f := range s.Fields() { - if _, ok := f.Type.(iceberg.PrimitiveType); !ok { - return fmt.Errorf("copy-on-write merge_strategy does not support column %q of non-primitive type %s; this prototype only supports tables whose columns are all flat primitive types (use merge-on-read for nested schemas)", f.Name, f.Type) - } - if !cowSupportedColumnType(f.Type) { - return fmt.Errorf("copy-on-write merge_strategy does not support column %q of type %s; supported column types are boolean, int, long, float, double, string, date, time, timestamp, timestamptz, decimal, and uuid", f.Name, f.Type) + if err := checkCOWTypeSupported(f.Name, f.Type); err != nil { + return err } } return nil } -// cowSupportedColumnType reports whether a primitive iceberg type is known to -// round-trip faithfully through deleteKeyJSONValue + array.RecordFromJSON. The -// set is deliberately conservative for the prototype; binary/fixed are excluded -// because we have not verified their JSON encoding. +// checkCOWTypeSupported recurses an iceberg type, accepting nested +// struct/list/map whose leaves are all supported primitives and rejecting any +// unsupported leaf. path names the column position (dotted for nested fields) so +// the error points at the offending leaf. +func checkCOWTypeSupported(path string, t iceberg.Type) error { + switch tt := t.(type) { + case *iceberg.StructType: + for _, f := range tt.FieldList { + if err := checkCOWTypeSupported(path+"."+f.Name, f.Type); err != nil { + return err + } + } + return nil + case *iceberg.ListType: + return checkCOWTypeSupported(path+".element", tt.Element) + case *iceberg.MapType: + if err := checkCOWTypeSupported(path+".key", tt.KeyType); err != nil { + return err + } + return checkCOWTypeSupported(path+".value", tt.ValueType) + default: + if _, ok := t.(iceberg.PrimitiveType); !ok { + return fmt.Errorf("copy-on-write merge_strategy does not support column %q of unsupported non-primitive type %s (use merge-on-read for this schema)", path, t) + } + if !cowSupportedColumnType(t) { + return fmt.Errorf("copy-on-write merge_strategy does not support column %q of type %s; supported leaf types are boolean, int, long, float, double, string, date, time, timestamp, timestamptz, decimal, uuid, binary, and fixed", path, t) + } + return nil + } +} + +// cowSupportedColumnType reports whether a primitive iceberg type round-trips +// faithfully through deleteKeyJSONValue + array.RecordFromJSON, as used by +// cowMassage/buildCOWRecordFactory. Every type in this set is guarded by a +// faithful round-trip in TestCOWColumnTypeRoundTrip (cow_type_roundtrip_test.go). +// +// binary and fixed are included: deleteKeyJSONValue passes a []byte through +// unchanged, json.Marshal base64-encodes it, and the Arrow Binary / +// FixedSizeBinary JSON readers base64-decode it back to the exact bytes. +// +// Nested struct/list/map are supported by recursing the type tree (see +// checkCOWTypeSupported) down to these primitive leaves: cowMassage produces the +// correct JSON shape at every depth — integers are emitted as strings at every +// leaf (fixing the historical >2^53 nested truncation) and maps are reshaped to +// Arrow's array-of-{key,value}-entries encoding. See cow_type_roundtrip_test.go +// for the round-trip evidence at each nesting. func cowSupportedColumnType(t iceberg.Type) bool { switch t.(type) { case iceberg.BooleanType, @@ -156,7 +206,8 @@ func cowSupportedColumnType(t iceberg.Type) bool { iceberg.DateType, iceberg.TimeType, iceberg.TimestampType, iceberg.TimestampTzType, iceberg.DecimalType, - iceberg.UUIDType: + iceberg.UUIDType, + iceberg.BinaryType, iceberg.FixedType: return true default: return false @@ -169,9 +220,12 @@ func cowSupportedColumnType(t iceberg.Type) bool { // per-tuple ANDs — `(a=a1 AND b=b1) OR (a=a2 AND b=b2) ...` — which is the // correct semantics (an AND of per-column INs would match the cross product). // -// For the prototype, merge-key columns are restricted to int/long/string/ -// boolean so the filter literals are unambiguous; other key types return a -// clear error. +// Merge-key columns may be int/long/string/boolean, the temporal types +// (date/time/timestamp/timestamptz), or uuid. Every key literal is built so its +// encoding matches how buildCOWRecordFactory stores the same value (see +// cowKeyLiteral). decimal is intentionally excluded (a vendored-library bug +// panics on a decimal overwrite filter — use merge-on-read for a decimal key); +// other key types return a clear error. func (w *writer) buildCOWFilter(tableSchema *iceberg.Schema, keyed service.MessageBatch) (iceberg.BooleanExpression, error) { idFields, err := w.cowKeyFields(tableSchema) if err != nil { @@ -265,9 +319,36 @@ func (w *writer) lookupKeyValue(msg *service.Message, field iceberg.NestedField, return v, nil } -// cowKeyLiteral builds an iceberg filter literal for a merge-key value. Only -// int/long/string/boolean key columns are supported by the prototype's -// copy-on-write filter; other types return a clear, actionable error. +// cowKeyLiteral builds an iceberg filter literal for a merge-key value. +// +// The overriding invariant is that the literal's encoding MUST match how +// buildCOWRecordFactory stores the same value, or the overwrite filter selects +// no rows and the upsert/delete silently becomes a no-op (the CON-490 hazard). +// The rewrite stores every value by running it through deleteKeyJSONValue and +// then array.RecordFromJSON, so this function derives each literal from that +// same canonicalisation: +// +// - int/long/string/boolean: built directly, mirroring the append path. +// - date/time/uuid: canonicalised by deleteKeyJSONValue to the exact string +// the data path stores, then parsed into the typed literal by iceberg's own +// StringLiteral.To — so filter and storage share an encoding by construction +// (date days, microsecond time-of-day, uuid bytes). +// - timestamp/timestamptz: deleteKeyJSONValue requires a time.Time and rejects +// a bare number (a numeric timestamp is ambiguous — the exact CON-490 silent +// no-match), so it is reused for that validation. The literal is then built +// directly from the time.Time as UnixMicro, because StringLiteral.To's +// timestamp parser does not accept the RFC3339 form the data path stores; +// both encode microseconds since the epoch, so they still agree. +// +// decimal is deliberately NOT a supported merge key: iceberg-go's overwrite +// applies the filter through its substrait conversion, which panics on a decimal +// literal (toDecimalLiteral asserts *iceberg.DecimalType, but DecimalLiteral.Type +// returns a value DecimalType — a bug in the vendored library). Rather than let +// that panic reach a real table, decimal keys are rejected here with an +// actionable error. decimal remains valid as a merge-on-read equality-delete key +// (that path does not go through substrait). +// +// Other key types return a clear, actionable error. func cowKeyLiteral(t iceberg.Type, name string, v any) (iceberg.Literal, error) { switch t.(type) { case iceberg.Int32Type: @@ -294,8 +375,37 @@ func cowKeyLiteral(t iceberg.Type, name string, v any) (iceberg.Literal, error) return nil, fmt.Errorf("%s %q: boolean column given %T", ioFieldIdentifierFields, name, v) } return iceberg.NewLiteral(b), nil + case iceberg.TimestampType, iceberg.TimestampTzType: + // Reuse deleteKeyJSONValue purely for its validation: it requires a + // time.Time and rejects a bare number with an actionable error. + if _, err := deleteKeyJSONValue(t, v); err != nil { + return nil, fmt.Errorf("%s %q: %w", ioFieldIdentifierFields, name, err) + } + tm := v.(time.Time) + return iceberg.NewLiteral(iceberg.Timestamp(tm.UTC().UnixMicro())), nil + case iceberg.DecimalType: + // See the doc comment: an overwrite filter on a decimal column panics + // inside iceberg-go's substrait conversion, so refuse loudly here. + return nil, fmt.Errorf("copy-on-write merge_strategy does not support merge key column %q of type %s; decimal is not a supported copy-on-write merge key (a known limitation in the underlying iceberg library's overwrite filter) — use merge-on-read for a decimal key", name, t) + case iceberg.DateType, iceberg.TimeType, iceberg.UUIDType: + jv, err := deleteKeyJSONValue(t, v) + if err != nil { + return nil, fmt.Errorf("%s %q: %w", ioFieldIdentifierFields, name, err) + } + s, ok := jv.(string) + if !ok { + // deleteKeyJSONValue canonicalises all of these to a string; a + // non-string means the incoming value could not be canonicalised + // (e.g. a non-string uuid value), which cannot key a row. + return nil, fmt.Errorf("%s %q: %s column requires a value convertible to its canonical string form, got %T", ioFieldIdentifierFields, name, t, v) + } + lit, err := iceberg.StringLiteral(s).To(t) + if err != nil { + return nil, fmt.Errorf("%s %q: %w", ioFieldIdentifierFields, name, err) + } + return lit, nil default: - return nil, fmt.Errorf("copy-on-write merge_strategy does not support merge key column %q of type %s; supported merge-key types are int, long, string, and boolean (use merge-on-read for other key types)", name, t) + return nil, fmt.Errorf("copy-on-write merge_strategy does not support merge key column %q of type %s; supported merge-key types are boolean, int, long, string, date, time, timestamp, timestamptz, and uuid (use merge-on-read for other key types)", name, t) } } @@ -331,8 +441,10 @@ func cowValueToInt64(v any) (int64, error) { // copy-on-write rewrite projects rows onto the current schema, so an unknown // column would otherwise be dropped without trace; returning this error lets the // router evolve the table and retry, matching the shredder-based append path -// (writer.go writeDataFiles). Copy-on-write is gated to flat-primitive schemas, -// so every new field is at the schema root. +// (writer.go writeDataFiles). Only top-level columns are detected here; new +// fields appearing inside an existing nested struct/list/map are not surfaced +// for evolution (nested schema evolution is out of scope for copy-on-write) — +// they are projected onto the current nested type by cowMassage. func (w *writer) cowDetectNewColumns(tableSchema *iceberg.Schema, rows service.MessageBatch) error { var newErrs []*UnknownFieldError seen := make(map[string]struct{}) @@ -399,7 +511,7 @@ func (w *writer) buildCOWRecordFactory(tableSchema *iceberg.Schema, rows service // Absent/null columns are left out so Arrow reads them as null. continue } - jv, err := deleteKeyJSONValue(field.Type, v) + jv, err := w.cowMassage(field.Type, v) if err != nil { return nil, fmt.Errorf("column %q in message %d: %w", field.Name, i, err) } @@ -428,3 +540,95 @@ func (w *writer) buildCOWRecordFactory(tableSchema *iceberg.Schema, rows service return rdr, nil }, nil } + +// cowMassage recursively projects a CDC value onto the JSON shape that +// SchemaToArrowSchema + array.RecordFromJSON expects for the given iceberg type, +// at every depth of the type tree. It is the nested generalisation of the flat +// deleteKeyJSONValue projection and exists so that copy-on-write can faithfully +// (re)write struct/list/map columns rather than either corrupting them or +// rejecting them outright. +// +// Each type kind is handled as follows: +// +// - primitive: delegate to deleteKeyJSONValue, which applies the int->string, +// temporal, decimal and uuid canonicalisation at every leaf. Doing this at +// every leaf (not just the top level) is what fixes the historical silent +// truncation of integers nested beyond 2^53: a nested int64 is emitted as a +// JSON string, which the Arrow Int32/Int64 JSON builder parses back exactly, +// instead of decoding through a lossy float64. +// - struct: the value is a map[string]any keyed by field name; recurse per +// struct field, honouring the writer's case sensitivity, and emit a +// map[string]any. Absent/null fields are omitted so Arrow reads them as null, +// mirroring the top-level behaviour. +// - list: the value is a []any; recurse per element with the element type and +// emit a []any (nil elements pass through as JSON null). +// - map: the CDC value is a map[string]any (the natural {"k": v} shape), but +// Arrow encodes a map as List> (see arrow.MapOf: the entry +// struct fields are literally named "key" and "value", with value nullable). +// So reshape to []any of {"key": k, "value": v} objects, recursing the key +// and value types. A null map value stays null under its "value" key. +func (w *writer) cowMassage(t iceberg.Type, v any) (any, error) { + switch tt := t.(type) { + case *iceberg.StructType: + m, ok := v.(map[string]any) + if !ok { + return nil, fmt.Errorf("struct value must be an object, got %T", v) + } + out := make(map[string]any, len(tt.FieldList)) + for _, f := range tt.FieldList { + fv, ok := lookupField(m, f.Name, w.caseSensitive) + if !ok || fv == nil { + // Absent/null field: omit so Arrow reads it as null. + continue + } + mv, err := w.cowMassage(f.Type, fv) + if err != nil { + return nil, fmt.Errorf("struct field %q: %w", f.Name, err) + } + out[f.Name] = mv + } + return out, nil + case *iceberg.ListType: + l, ok := v.([]any) + if !ok { + return nil, fmt.Errorf("list value must be an array, got %T", v) + } + out := make([]any, len(l)) + for i, e := range l { + if e == nil { + out[i] = nil + continue + } + me, err := w.cowMassage(tt.Element, e) + if err != nil { + return nil, fmt.Errorf("list element %d: %w", i, err) + } + out[i] = me + } + return out, nil + case *iceberg.MapType: + m, ok := v.(map[string]any) + if !ok { + return nil, fmt.Errorf("map value must be an object, got %T", v) + } + entries := make([]any, 0, len(m)) + for k, mv := range m { + mk, err := w.cowMassage(tt.KeyType, k) + if err != nil { + return nil, fmt.Errorf("map key %q: %w", k, err) + } + var vv any + if mv != nil { + vv, err = w.cowMassage(tt.ValueType, mv) + if err != nil { + return nil, fmt.Errorf("map value for key %q: %w", k, err) + } + } + entries = append(entries, map[string]any{"key": mk, "value": vv}) + } + return entries, nil + default: + // Primitive leaf: apply the same canonicalisation the flat path uses. + return deleteKeyJSONValue(t, v) + } +} diff --git a/internal/impl/iceberg/cow_amplification_bench_test.go b/internal/impl/iceberg/cow_amplification_bench_test.go index 70d70ce357..0267959265 100644 --- a/internal/impl/iceberg/cow_amplification_bench_test.go +++ b/internal/impl/iceberg/cow_amplification_bench_test.go @@ -27,6 +27,7 @@ import ( "io/fs" "math/rand" "path/filepath" + "runtime" "strconv" "strings" "testing" @@ -345,3 +346,161 @@ func TestCOWWriteAmplification(t *testing.T) { t.Log("amp(x) = bytes written by the mutation / bytes logically changed (~K rows)") t.Log("tblRewr% = removed-files-size / total seeded bytes (fraction of table COW rewrote)") } + +// TestCOWWriteAmplificationScale confirms the K/M model holds at larger, +// production-like per-file sizes (up to a few MB per data file) and measures the +// per-MB rewrite cost, from which 128-512 MB file behaviour extrapolates +// linearly. TestCOWWriteAmplification above characterises tiny (~80 KB) files; +// this one holds M fixed and grows R so each data file reaches ~1-4 MB, then: +// +// - "within" (K=1): touches one key in one file, so exactly ONE file (1/M of +// the table) is rewritten regardless of file size — the K/M model. +// - "perfile" (K=M): touches one key in every file, so ALL M files (the whole +// table) are rewritten. +// +// For each case it reports the bytes COW rewrote (removed-files-size), which +// tracks the touched files' on-disk size, and the wall-clock MB/s of the +// rewrite. We deliberately do NOT seed literal 512 MB files (far too slow for a +// unit test); the per-MB cost measured here is the extrapolation constant. +func TestCOWWriteAmplificationScale(t *testing.T) { + if testing.Short() { + t.Skip("amplification harness is slow; skipped under -short") + } + ctx := t.Context() + + const ( + M = 4 // data files in the table + payloadBytes = 512 // larger payload so a few thousand rows already spans MBs + ) + // rows/file chosen so each file lands around ~1, ~2 and ~4 MB on disk. + rowCounts := []int{2000, 4000, 8000} + + type scen struct { + scatter string + k int + want int64 // files expected to be rewritten + } + scatters := []scen{ + {"within", 1, 1}, // K/M: one key, one file rewritten + {"perfile", M, M}, // whole table: one key per file, all files rewritten + } + + t.Log("") + t.Logf("Scale layout: M=%d files, payload=%d bytes/row, contiguous clustered id ranges per file", M, payloadBytes) + t.Log("") + t.Logf("%-22s | %8s %9s | %8s | %9s %9s | %9s | %8s | %8s", + "scenario", "fileMB", "seedMB", "filesRw", "rewroteMB", "wroteMB", "wall", "MB/s", "tblRewr%") + t.Log(strings.Repeat("-", 120)) + + for _, R := range rowCounts { + for _, s := range scatters { + res := runAmpScenario(t, ctx, table.WriteModeCopyOnWrite, M, R, s.k, payloadBytes, s.scatter) + + // K/M model: the number of files rewritten equals the number of + // distinct files that held a touched key, independent of file size. + require.Equalf(t, s.want, res.deletedDataFiles, + "COW must rewrite exactly %d file(s) for %s (K/M model)", s.want, res.name) + require.Zerof(t, res.deleteManifestFiles, "COW must produce zero delete files") + + const mb = 1024.0 * 1024.0 + fileMB := float64(res.seedBytes) / float64(M) / mb + seedMB := float64(res.seedBytes) / mb + rewroteMB := float64(res.removedFilesSize) / mb + wroteMB := float64(res.addedFilesSize) / mb + var mbPerSec float64 + if res.elapsed > 0 { + mbPerSec = rewroteMB / res.elapsed.Seconds() + } + tblRewrPct := 100 * float64(res.removedFilesSize) / float64(res.seedBytes) + + t.Logf("%-22s | %8.2f %9.2f | %8d | %9.2f %9.2f | %9s | %8.1f | %8.2f", + res.name, fileMB, seedMB, res.deletedDataFiles, rewroteMB, wroteMB, + res.elapsed.Round(time.Millisecond).String(), mbPerSec, tblRewrPct) + } + } + + t.Log("") + t.Log("K/M model: filesRw = number of files holding a touched key; a K-key batch scattered") + t.Log("over M files rewrites ~K/M of the table. rewroteMB tracks those files' on-disk size,") + t.Log("so a 1-key touch in a 512 MB file rewrites the whole 512 MB. Extrapolate cost via MB/s.") +} + +// TestCOWRecordFactoryMemory quantifies the peak memory of the copy-on-write +// new-row path so we can state a batch-size guideline. buildCOWRecordFactory +// projects the whole batch into JSON (retained by the returned closure for +// retries) and every factory() call materialises the entire batch as one +// in-memory Arrow record via array.RecordFromJSON. Both scale linearly with the +// batch's total row bytes, so an over-large keyed batch can dominate RSS. +// +// It measures, for growing batch sizes: total bytes allocated (churn) building +// the factory + one reader, and the heap retained while that reader is live +// (the closure's JSON plus the live Arrow record), and derives bytes/row. +func TestCOWRecordFactoryMemory(t *testing.T) { + if testing.Short() { + t.Skip("memory harness allocates hundreds of MB; skipped under -short") + } + + tbl, _ := newAmpTable(t, table.WriteModeCopyOnWrite) // schema: id int64, payload string + w := cowWriter(t, tbl, "id") + sc := tbl.Schema() + + const payloadBytes = 256 // per-row payload; representative CDC row body + + t.Log("") + t.Logf("Row layout: id int64 + payload string (%d bytes/row); COW materialises the whole batch as Arrow", payloadBytes) + t.Log("") + t.Logf("%8s | %10s | %12s %10s | %12s %10s", + "rows", "rawMB", "churnMB", "churn B/row", "retainedMB", "ret B/row") + t.Log(strings.Repeat("-", 84)) + + rng := rand.New(rand.NewSource(42)) + for _, n := range []int{10_000, 50_000, 100_000} { + rows := make([]map[string]any, n) + buf := make([]byte, payloadBytes) + for i := range rows { + for j := range buf { + buf[j] = byte('a' + rng.Intn(26)) + } + rows[i] = map[string]any{"id": int64(i), "payload": string(buf)} + } + batch := toBatch(t, rows) + rawBytes := int64(n) * int64(payloadBytes+8) // payload + int64 id, logical size + + runtime.GC() + var before runtime.MemStats + runtime.ReadMemStats(&before) + + factory, err := w.buildCOWRecordFactory(sc, batch) + require.NoError(t, err) + rdr, err := factory() + require.NoError(t, err) + var got int64 + for rdr.Next() { + got += rdr.RecordBatch().NumRows() + } + + // Read while the reader (and the closure's JSON) is still alive so the + // retained-heap delta reflects what a live COW commit holds. + var live runtime.MemStats + runtime.ReadMemStats(&live) + require.EqualValues(t, n, got) + rdr.Release() + + const mb = 1024.0 * 1024.0 + churn := live.TotalAlloc - before.TotalAlloc + retained := max(int64(live.HeapAlloc)-int64(before.HeapAlloc), 0) + + t.Logf("%8d | %10.2f | %12.2f %10.1f | %12.2f %10.1f", + n, float64(rawBytes)/mb, + float64(churn)/mb, float64(churn)/float64(n), + float64(retained)/mb, float64(retained)/float64(n)) + + runtime.KeepAlive(factory) + } + + t.Log("") + t.Log("churnMB = total bytes allocated to build the JSON + one Arrow record (transient, GC-reclaimed)") + t.Log("retainedMB = live heap held during a commit: the closure's JSON batch + the materialised Arrow record") + t.Log("Guideline: budget ~retained B/row per row of a keyed batch (plus the inbound message bodies);") + t.Log("size batches so this stays within the process memory budget, since the whole batch is held at once.") +} diff --git a/internal/impl/iceberg/cow_merge_key_roundtrip_test.go b/internal/impl/iceberg/cow_merge_key_roundtrip_test.go new file mode 100644 index 0000000000..69825351bc --- /dev/null +++ b/internal/impl/iceberg/cow_merge_key_roundtrip_test.go @@ -0,0 +1,227 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md + +package iceberg + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/table" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/redpanda-data/benthos/v4/public/service" +) + +// This file proves the copy-on-write merge-key (filter-literal) path for every +// key type broadened in Tier 1.2 that could be made safe: date, time, +// timestamp, timestamptz, and uuid. Each case drives a REAL copy-on-write +// upsert+delete batch through writer.Write (which builds the overwrite filter +// via buildCOWFilter -> cowKeyLiteral) and asserts that the filter matched the +// INTENDED rows — the upserted key's row was rewritten, the deleted key's row +// was removed, and an untouched key survived unchanged. decimal is proven GATED +// (see TestCOWDecimalMergeKeyGated) because iceberg-go's overwrite filter panics +// on a decimal literal. +// +// The load-bearing guard is against the CON-490 silent-no-match bug: if the +// filter literal's encoding disagreed with the stored value, the overwrite +// would match nothing, leaving a duplicate of the upserted key and failing to +// delete — which these assertions (exact final row set, keyed identity) +// catch. A test that only checked "no error" would NOT catch that. + +// seedMergeKeyRows appends the given rows as one plain-data-file snapshot, +// encoding each value exactly as the copy-on-write rewrite would (via +// buildCOWRecordFactory), and returns the updated table handle. +func seedMergeKeyRows(t testing.TB, ctx context.Context, tbl *table.Table, cat *memCatalog, rows []map[string]any) *table.Table { + t.Helper() + w := cowWriter(t, cat.snapshot(), "k") + factory, err := w.buildCOWRecordFactory(tbl.Schema(), toBatch(t, rows)) + require.NoError(t, err) + rdr, err := factory() + require.NoError(t, err) + tx := tbl.NewTransaction() + require.NoError(t, tx.Append(ctx, rdr, nil)) + rdr.Release() + next, err := tx.Commit(ctx) + require.NoError(t, err) + return next +} + +// scanKeyPayload scans the table into a map keyed by the canonical JSON form of +// the "k" column, valued by the "payload" string. JSON is a type-agnostic, +// deterministic form for identifying which key each surviving row carries. +func scanKeyPayload(t testing.TB, ctx context.Context, tbl *table.Table) map[string]string { + t.Helper() + at, err := tbl.Scan().ToArrowTable(ctx) + require.NoError(t, err) + defer at.Release() + + out := map[string]string{} + tr := array.NewTableReader(at, 0) + defer tr.Release() + for tr.Next() { + rec := tr.RecordBatch() + kArr := rec.Column(rec.Schema().FieldIndices("k")[0]) + pArr := rec.Column(rec.Schema().FieldIndices("payload")[0]).(*array.String) + for r := 0; r < int(rec.NumRows()); r++ { + b, err := json.Marshal(kArr.GetOneForMarshal(r)) + require.NoError(t, err) + pay := "" + if pArr.IsValid(r) { + pay = pArr.Value(r) + } + out[string(b)] = pay + } + } + return out +} + +// invertByPayload maps payload -> key JSON, asserting payloads are unique. +func invertByPayload(t testing.TB, m map[string]string) map[string]string { + t.Helper() + out := make(map[string]string, len(m)) + for k, pay := range m { + _, dup := out[pay] + require.False(t, dup, "payloads must be unique to identify rows") + out[pay] = k + } + return out +} + +func TestCOWMergeKeyRoundTrip(t *testing.T) { + ctx := t.Context() + + // timeOf builds a UTC time.Time; date-only / time-only cases just fix the + // irrelevant component. + ts := func(y int, mo time.Month, d, h, mi, s, ns int) time.Time { + return time.Date(y, mo, d, h, mi, s, ns, time.UTC) + } + + cases := []struct { + name string + keyType iceberg.Type + k1 any // untouched + k2 any // upserted (payload one/two/three -> TWO) + k3 any // deleted + }{ + { + name: "date", + keyType: iceberg.PrimitiveTypes.Date, + k1: ts(2026, 1, 1, 0, 0, 0, 0), + k2: ts(2026, 6, 15, 0, 0, 0, 0), + k3: ts(2026, 12, 31, 0, 0, 0, 0), + }, + { + name: "time", + keyType: iceberg.PrimitiveTypes.Time, + k1: ts(2000, 1, 1, 1, 2, 3, 0), + k2: ts(2000, 1, 1, 12, 13, 14, 500000000), // 12:13:14.5 + k3: ts(2000, 1, 1, 23, 59, 59, 999999000), // microsecond precision + }, + { + name: "timestamp", + keyType: iceberg.PrimitiveTypes.Timestamp, + k1: ts(2026, 1, 1, 0, 0, 0, 0), + k2: ts(2026, 6, 15, 10, 20, 30, 123456000), // microsecond precision + k3: ts(2026, 12, 31, 23, 59, 59, 0), + }, + { + name: "timestamptz", + keyType: iceberg.PrimitiveTypes.TimestampTz, + k1: ts(2026, 1, 1, 0, 0, 0, 0), + k2: ts(2026, 6, 15, 10, 20, 30, 123456000), + k3: ts(2026, 12, 31, 23, 59, 59, 0), + }, + { + name: "uuid", + keyType: iceberg.PrimitiveTypes.UUID, + k1: "f47ac10b-58cc-0372-8567-0e02b2c3d479", + k2: "6ba7b810-9dad-11d1-80b4-00c04fd430c8", + k3: "00000000-0000-0000-0000-000000000001", + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "k", Type: c.keyType, Required: true}, + iceberg.NestedField{ID: 2, Name: "payload", Type: iceberg.PrimitiveTypes.String}, + ) + tbl, cat := newCOWTable(t, sc) + + // Seed three distinct-key rows. + tbl = seedMergeKeyRows(t, ctx, tbl, cat, []map[string]any{ + {"k": c.k1, "payload": "one"}, + {"k": c.k2, "payload": "two"}, + {"k": c.k3, "payload": "three"}, + }) + + seedMap := scanKeyPayload(t, ctx, tbl) + require.Len(t, seedMap, 3, "seed must produce three rows") + seedByPay := invertByPayload(t, seedMap) + + // Drive a real copy-on-write upsert(k2)+delete(k3) batch. + comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) + require.NoError(t, err) + defer comm.Close() + w := cowWriter(t, cat.snapshot(), "k") + w.committer = comm + + require.NoError(t, w.Write(ctx, service.MessageBatch{ + cowMsg(t, "upsert", map[string]any{"k": c.k2, "payload": "TWO"}), + cowMsg(t, "delete", map[string]any{"k": c.k3}), + })) + + final := cat.snapshot() + + // Copy-on-write must never leave delete files. + assert.Zero(t, countDeleteManifestFiles(t, ctx, final), "copy-on-write must leave no delete files") + + finalMap := scanKeyPayload(t, ctx, final) + // Exactly the untouched row and the upserted row remain. If the + // filter had matched nothing (the silent-no-match bug), we would see + // a duplicate k2 ("two" AND "TWO") and a surviving k3 ("three"). + require.Len(t, finalMap, 2, "exactly the untouched and upserted rows must remain") + finalByPay := invertByPayload(t, finalMap) + + // The row now carrying "TWO" must be keyed by k2 (proves the upsert + // rewrote the intended key, not a different or no row). + assert.Equal(t, seedByPay["two"], finalByPay["TWO"], "upserted row must carry the k2 key") + // The untouched row must be exactly k1, still "one". + assert.Equal(t, seedByPay["one"], finalByPay["one"], "untouched row must keep the k1 key and value") + // k3 must be gone and the stale k2 value must not linger. + assert.NotContains(t, finalByPay, "three", "deleted key k3 must be removed") + assert.NotContains(t, finalByPay, "two", "the pre-upsert k2 value must be overwritten, not duplicated") + }) + } +} + +// TestCOWDecimalMergeKeyGated pins the deliberate gate on decimal merge keys. +// iceberg-go's overwrite filter routes a decimal literal through its substrait +// conversion, which panics (toDecimalLiteral asserts *iceberg.DecimalType while +// DecimalLiteral.Type returns a value DecimalType). Rather than let that panic +// reach a real overwrite, cowKeyLiteral rejects a decimal key up front with an +// actionable error. This is the "could not make safe" case for Tier 1.2; the +// decimal *column* type (Tier 1.1) and decimal merge-on-read keys are +// unaffected. +func TestCOWDecimalMergeKeyGated(t *testing.T) { + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "k", Type: iceberg.DecimalTypeOf(10, 2), Required: true}, + ) + tbl := newTypedKeyTableFromSchema(t, sc) + w := cowWriter(t, tbl, "k") + _, err := w.buildCOWFilter(sc, service.MessageBatch{structuredMsg(t, map[string]any{"k": "1.00"})}) + require.Error(t, err) + assert.Contains(t, err.Error(), "decimal is not a supported copy-on-write merge key") + assert.Contains(t, err.Error(), "merge-on-read") +} diff --git a/internal/impl/iceberg/cow_test.go b/internal/impl/iceberg/cow_test.go index 53bbe2cb2a..8988fb9c45 100644 --- a/internal/impl/iceberg/cow_test.go +++ b/internal/impl/iceberg/cow_test.go @@ -120,25 +120,86 @@ func TestCheckCOWSchemaSupported(t *testing.T) { require.NoError(t, checkCOWSchemaSupported(sc)) }) - t.Run("nested struct rejected", func(t *testing.T) { + t.Run("nested struct accepted", func(t *testing.T) { + // Nested struct/list/map are now supported: the gate recurses the type + // tree and cowMassage produces the correct JSON shape at every depth. The + // faithful round-trips live in cow_type_roundtrip_test.go. sc := iceberg.NewSchema(0, iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64}, iceberg.NestedField{ID: 2, Name: "nested", Type: &iceberg.StructType{ FieldList: []iceberg.NestedField{{ID: 3, Name: "inner", Type: iceberg.PrimitiveTypes.String}}, }}, ) - err := checkCOWSchemaSupported(sc) - require.Error(t, err) - assert.Contains(t, err.Error(), "non-primitive") + require.NoError(t, checkCOWSchemaSupported(sc)) }) - t.Run("binary rejected", func(t *testing.T) { + t.Run("binary and fixed accepted", func(t *testing.T) { + // binary and fixed are flat primitives that round-trip faithfully through + // the Arrow JSON base64 encoding (see TestCOWColumnTypeRoundTrip), so the + // gate accepts them. sc := iceberg.NewSchema(0, iceberg.NestedField{ID: 1, Name: "b", Type: iceberg.PrimitiveTypes.Binary}, + iceberg.NestedField{ID: 2, Name: "f", Type: iceberg.FixedTypeOf(16)}, + ) + require.NoError(t, checkCOWSchemaSupported(sc)) + }) + + t.Run("nested list accepted", func(t *testing.T) { + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64}, + iceberg.NestedField{ID: 2, Name: "l", Type: &iceberg.ListType{ + ElementID: 3, Element: iceberg.PrimitiveTypes.String, ElementRequired: false, + }}, + ) + require.NoError(t, checkCOWSchemaSupported(sc)) + }) + + t.Run("map accepted", func(t *testing.T) { + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64}, + iceberg.NestedField{ID: 2, Name: "m", Type: &iceberg.MapType{ + KeyID: 3, KeyType: iceberg.PrimitiveTypes.String, + ValueID: 4, ValueType: iceberg.PrimitiveTypes.Int64, ValueRequired: false, + }}, + ) + require.NoError(t, checkCOWSchemaSupported(sc)) + }) + + t.Run("deeply nested primitives accepted", func(t *testing.T) { + // struct>>> — every leaf is a supported + // primitive, so the recursive gate accepts the whole tree. + sc := iceberg.NewSchema(0, + cowIDField(), + iceberg.NestedField{ID: 2, Name: "deep", Type: &iceberg.StructType{FieldList: []iceberg.NestedField{ + {ID: 3, Name: "items", Type: &iceberg.ListType{ + ElementID: 4, ElementRequired: false, Element: &iceberg.MapType{ + KeyID: 5, KeyType: iceberg.PrimitiveTypes.String, + ValueID: 6, ValueRequired: false, ValueType: &iceberg.StructType{FieldList: []iceberg.NestedField{ + {ID: 7, Name: "n", Type: iceberg.PrimitiveTypes.Int64}, + }}, + }, + }}, + }}}, + ) + require.NoError(t, checkCOWSchemaSupported(sc)) + }) + + t.Run("unsupported leaf inside nested type rejected", func(t *testing.T) { + // A genuinely unsupported leaf (timestamp_ns is not in the supported set) + // nested inside a list-of-struct still fails loudly, and the error names + // the dotted path to the offending leaf. + sc := iceberg.NewSchema(0, + cowIDField(), + iceberg.NestedField{ID: 2, Name: "events", Type: &iceberg.ListType{ + ElementID: 3, ElementRequired: false, Element: &iceberg.StructType{FieldList: []iceberg.NestedField{ + {ID: 4, Name: "at", Type: iceberg.PrimitiveTypes.TimestampNs}, + }}, + }}, ) err := checkCOWSchemaSupported(sc) require.Error(t, err) - assert.Contains(t, err.Error(), "supported column types") + assert.Contains(t, err.Error(), "events.element.at") + assert.Contains(t, err.Error(), "timestamp_ns") }) } @@ -201,14 +262,38 @@ func TestBuildCOWFilterCompositeKey(t *testing.T) { } func TestBuildCOWFilterUnsupportedKeyType(t *testing.T) { + // binary is a supported COW *column* type but not a sensible merge key, so + // it must be rejected by the filter path with an actionable error. sc := iceberg.NewSchema(0, - iceberg.NestedField{ID: 1, Name: "ts", Type: iceberg.PrimitiveTypes.Timestamp}, + iceberg.NestedField{ID: 1, Name: "k", Type: iceberg.PrimitiveTypes.Binary}, ) tbl := newTypedKeyTableFromSchema(t, sc) - w := cowWriter(t, tbl, "ts") - _, err := w.buildCOWFilter(sc, service.MessageBatch{structuredMsg(t, map[string]any{"ts": 1})}) + w := cowWriter(t, tbl, "k") + _, err := w.buildCOWFilter(sc, service.MessageBatch{structuredMsg(t, map[string]any{"k": []byte{0x01}})}) require.Error(t, err) - assert.Contains(t, err.Error(), "merge key") + assert.Contains(t, err.Error(), "does not support merge key column") +} + +// TestBuildCOWFilterBareNumberTemporalKeyRejected pins the CON-490 guard on the +// merge-key path: a temporal key given as a bare number is ambiguous (the data +// path cannot reproduce how a number would be interpreted), so it must be +// rejected loudly rather than silently building a literal that matches nothing. +func TestBuildCOWFilterBareNumberTemporalKeyRejected(t *testing.T) { + for _, typ := range []iceberg.Type{ + iceberg.PrimitiveTypes.Timestamp, + iceberg.PrimitiveTypes.TimestampTz, + iceberg.PrimitiveTypes.Date, + iceberg.PrimitiveTypes.Time, + } { + t.Run(typ.String(), func(t *testing.T) { + sc := iceberg.NewSchema(0, iceberg.NestedField{ID: 1, Name: "k", Type: typ}) + tbl := newTypedKeyTableFromSchema(t, sc) + w := cowWriter(t, tbl, "k") + _, err := w.buildCOWFilter(sc, service.MessageBatch{structuredMsg(t, map[string]any{"k": 1})}) + require.Error(t, err) + assert.Contains(t, err.Error(), "requires a time value") + }) + } } // --- record factory ------------------------------------------------------------ @@ -308,6 +393,48 @@ func newCOWTable(t testing.TB, sc *iceberg.Schema) (*table.Table, *memCatalog) { return newAmpTableWithSchema(t, sc) } +// TestCOWv1TableStaysV1 pins Tier 3.1: copy-on-write writes only plain data +// files, so it works on a v1 table and must NOT trigger the irreversible v1->v2 +// upgrade the merge-on-read path forces. The mutation must still round-trip. +func TestCOWv1TableStaysV1(t *testing.T) { + ctx := t.Context() + location := filepath.ToSlash(t.TempDir()) + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: false}, + iceberg.NestedField{ID: 2, Name: "payload", Type: iceberg.PrimitiveTypes.String, Required: false}, + ) + meta, err := table.NewMetadata(sc, iceberg.UnpartitionedSpec, table.UnsortedSortOrder, + location, iceberg.Properties{table.PropertyFormatVersion: "1"}) + require.NoError(t, err) + cat := &memCatalog{ + meta: meta, + metadataLocation: fmt.Sprintf("%s/metadata/00001-%s.metadata.json", location, uuid.New()), + ident: table.Identifier{"default", "t"}, + location: location, + } + tbl := cat.snapshot() + require.EqualValues(t, 1, tbl.Metadata().Version(), "precondition: table starts at v1") + + tbl = appendCOWRows(t, ctx, tbl, map[int64]string{1: "one", 2: "two", 3: "three"}) + require.EqualValues(t, 1, cat.snapshot().Metadata().Version(), "seeding must not upgrade the table") + + comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3, SkipFormatUpgrade: true}, reloadFn(cat), service.MockResources().Logger()) + require.NoError(t, err) + defer comm.Close() + w := cowWriter(t, tbl, "id") + w.committer = comm + + require.NoError(t, w.Write(ctx, service.MessageBatch{ + cowMsg(t, "upsert", map[string]any{"id": 2, "payload": "TWO"}), + cowMsg(t, "delete", map[string]any{"id": 3}), + })) + + final := cat.snapshot() + assert.EqualValues(t, 1, final.Metadata().Version(), "copy-on-write must not upgrade a v1 table to v2") + assertAllManifestsData(t, ctx, final) + assert.Equal(t, map[int64]string{1: "one", 2: "TWO"}, scanRows(t, ctx, final)) +} + // TestCOWUpsertDeleteRoundTrip drives a full copy-on-write upsert+delete batch // through the writer and committer against an in-memory catalog, then asserts // (a) the resulting table has the correct final rows and (b) the table contains @@ -407,13 +534,13 @@ func TestCOWOnlyInsertsUsesAppend(t *testing.T) { assert.Equal(t, map[int64]string{10: "ten", 11: "eleven"}, scanRows(t, ctx, final)) } -// --- partition gate ------------------------------------------------------------ +// --- partitioned copy-on-write -------------------------------------------------- // newPartitionedCOWTable builds a partitioned v2 table for the given schema and // spec, backed by an in-memory catalog and the local filesystem. It mirrors // newAmpTableWithSchema but installs a real partition spec so // tbl.Spec().NumFields() > 0. -func newPartitionedCOWTable(t testing.TB, sc *iceberg.Schema, spec iceberg.PartitionSpec) *table.Table { +func newPartitionedCOWTable(t testing.TB, sc *iceberg.Schema, spec iceberg.PartitionSpec) (*table.Table, *memCatalog) { t.Helper() location := filepath.ToSlash(t.TempDir()) meta, err := table.NewMetadata(sc, &spec, table.UnsortedSortOrder, location, @@ -425,15 +552,70 @@ func newPartitionedCOWTable(t testing.TB, sc *iceberg.Schema, spec iceberg.Parti ident: table.Identifier{"default", "cow_partitioned"}, location: location, } - return cat.snapshot() + return cat.snapshot(), cat +} + +// appendPartitionedCOWRows appends (id, region, payload) rows as one plain-data- +// file snapshot, routed to partitions by iceberg-go's partitioned fanout writer, +// and returns the updated table handle. +func appendPartitionedCOWRows(t testing.TB, ctx context.Context, tbl *table.Table, rows []map[string]any) *table.Table { + t.Helper() + arrowSc, err := table.SchemaToArrowSchema(tbl.Schema(), nil, false, false) + require.NoError(t, err) + + b, err := json.Marshal(rows) + require.NoError(t, err) + + rec, _, err := array.RecordFromJSON(memory.DefaultAllocator, arrowSc, bytes.NewReader(b)) + require.NoError(t, err) + rdr, err := array.NewRecordReader(arrowSc, []arrow.RecordBatch{rec}) + require.NoError(t, err) + rec.Release() + defer rdr.Release() + + tx := tbl.NewTransaction() + require.NoError(t, tx.Append(ctx, rdr, nil)) + next, err := tx.Commit(ctx) + require.NoError(t, err) + return next +} + +// scanPartitionedRows scans the table into id -> {region, payload}, honouring any +// deletes. +func scanPartitionedRows(t testing.TB, ctx context.Context, tbl *table.Table) map[int64][2]string { + t.Helper() + at, err := tbl.Scan().ToArrowTable(ctx) + require.NoError(t, err) + defer at.Release() + + out := map[int64][2]string{} + tr := array.NewTableReader(at, 0) + defer tr.Release() + for tr.Next() { + rec := tr.RecordBatch() + idArr := rec.Column(rec.Schema().FieldIndices("id")[0]).(*array.Int64) + regArr := rec.Column(rec.Schema().FieldIndices("region")[0]).(*array.String) + payArr := rec.Column(rec.Schema().FieldIndices("payload")[0]).(*array.String) + for r := 0; r < int(rec.NumRows()); r++ { + pay := "" + if payArr.IsValid(r) { + pay = payArr.Value(r) + } + out[idArr.Value(r)] = [2]string{regArr.Value(r), pay} + } + } + return out } -// TestCOWPartitionedTableRejectsMutation pins the copy-on-write partition gate: -// writeCOW must refuse a mutating (upsert/delete) batch on a partitioned table -// rather than risk a mis-partitioned rewrite, and it must do so with an -// actionable error. Only the file-rewrite paths are gated, so an upsert — a -// keyed operation — is the trigger that reaches the gate. -func TestCOWPartitionedTableRejectsMutation(t *testing.T) { +// TestCOWPartitionedUpsertDeleteRoundTrip proves copy-on-write works end-to-end +// on a partitioned table (partition by region, merge key id — a NON-partition +// column). A single mutating batch touches multiple partitions: it updates a row +// in eu, deletes a row in eu, and inserts a new row in apac. The result must have +// the correct per-partition state AND zero delete files (the copy-on-write +// invariant). The merge key is not the partition column, which merge-on-read +// could not support (equality deletes are partition-scoped) — copy-on-write can, +// because it rewrites whole files by filter and re-routes appended rows by value. +func TestCOWPartitionedUpsertDeleteRoundTrip(t *testing.T) { ctx := t.Context() sc := iceberg.NewSchema(0, @@ -445,20 +627,136 @@ func TestCOWPartitionedTableRejectsMutation(t *testing.T) { spec := iceberg.NewPartitionSpec(iceberg.PartitionField{ SourceIDs: []int{2}, FieldID: 1000, Name: "region", Transform: iceberg.IdentityTransform{}, }) - tbl := newPartitionedCOWTable(t, sc, spec) - tblSpec := tbl.Spec() - require.Positive(t, tblSpec.NumFields(), "table must be partitioned for this test to be meaningful") + seedTbl, cat := newPartitionedCOWTable(t, sc, spec) + seedSpec := seedTbl.Spec() + require.Positive(t, seedSpec.NumFields(), "table must be partitioned for this test to be meaningful") + + // Seed rows across three partitions. + seedTbl = appendPartitionedCOWRows(t, ctx, seedTbl, []map[string]any{ + {"id": 1, "region": "us", "payload": "one"}, + {"id": 2, "region": "eu", "payload": "two"}, + {"id": 3, "region": "eu", "payload": "three"}, + {"id": 4, "region": "apac", "payload": "four"}, + }) - w := cowWriter(t, tbl, "id") + comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) + require.NoError(t, err) + defer comm.Close() + w := cowWriter(t, seedTbl, "id") + w.committer = comm - // An upsert is a keyed operation, so writeCOW reaches the file-rewrite path - // where the partition gate lives. No committer is wired because the gate must - // fire before any commit is attempted. - err := w.Write(ctx, service.MessageBatch{ + // One batch spanning multiple partitions: upsert id=2 (eu), delete id=3 (eu), + // upsert id=5 (new row in apac). + require.NoError(t, w.Write(ctx, service.MessageBatch{ cowMsg(t, "upsert", map[string]any{"id": 2, "region": "eu", "payload": "TWO"}), + cowMsg(t, "delete", map[string]any{"id": 3, "region": "eu"}), + cowMsg(t, "upsert", map[string]any{"id": 5, "region": "apac", "payload": "FIVE"}), + })) + + final := cat.snapshot() + + // (a) zero delete files: the copy-on-write invariant. + assert.Zero(t, countDeleteManifestFiles(t, ctx, final), "copy-on-write must leave no delete files") + assertAllManifestsData(t, ctx, final) + + // (b) correct final state, per partition. + got := scanPartitionedRows(t, ctx, final) + want := map[int64][2]string{ + 1: {"us", "one"}, // untouched + 2: {"eu", "TWO"}, // upserted in place (no duplicate) + 4: {"apac", "four"}, // untouched + 5: {"apac", "FIVE"}, // inserted into a different partition + } + assert.Equal(t, want, got, "id=3 must be deleted; id=2 updated once; id=5 landed in apac") +} + +// TestCOWPartitionKeyChangeRoundTrip covers the case merge-on-read cannot: an +// upsert that moves a keyed row to a DIFFERENT partition. Copy-on-write deletes +// the old row wherever it lives (the filter matches across all partitions) and +// appends the new row into its new partition, leaving exactly one row. +func TestCOWPartitionKeyChangeRoundTrip(t *testing.T) { + ctx := t.Context() + + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}, + iceberg.NestedField{ID: 2, Name: "region", Type: iceberg.PrimitiveTypes.String, Required: true}, + iceberg.NestedField{ID: 3, Name: "payload", Type: iceberg.PrimitiveTypes.String, Required: false}, + ) + spec := iceberg.NewPartitionSpec(iceberg.PartitionField{ + SourceIDs: []int{2}, FieldID: 1000, Name: "region", Transform: iceberg.IdentityTransform{}, }) - require.Error(t, err) - assert.Contains(t, err.Error(), "does not support upsert/delete on partitioned tables") + seedTbl, cat := newPartitionedCOWTable(t, sc, spec) + + seedTbl = appendPartitionedCOWRows(t, ctx, seedTbl, []map[string]any{ + {"id": 1, "region": "us", "payload": "one"}, + }) + + comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) + require.NoError(t, err) + defer comm.Close() + w := cowWriter(t, seedTbl, "id") + w.committer = comm + + // Move id=1 from us to eu via upsert. + require.NoError(t, w.Write(ctx, service.MessageBatch{ + cowMsg(t, "upsert", map[string]any{"id": 1, "region": "eu", "payload": "ONE"}), + })) + + final := cat.snapshot() + assert.Zero(t, countDeleteManifestFiles(t, ctx, final), "copy-on-write must leave no delete files") + got := scanPartitionedRows(t, ctx, final) + assert.Equal(t, map[int64][2]string{1: {"eu", "ONE"}}, got, + "the row must move to eu with no stale copy left in us") +} + +// TestCOWBucketPartitionRoundTrip exercises a NON-order-preserving transform +// (bucket) on the copy-on-write write path. The partitioned fanout writer derives +// each row's partition from Transform.Apply on the actual value, so bucket works +// exactly like identity — this is distinct from the stats-inference path +// (fileToDataFile), which panics on non-order-preserving transforms but is only +// used by AddFiles, never by the record-writing path copy-on-write uses. +func TestCOWBucketPartitionRoundTrip(t *testing.T) { + ctx := t.Context() + + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}, + iceberg.NestedField{ID: 2, Name: "region", Type: iceberg.PrimitiveTypes.String, Required: true}, + iceberg.NestedField{ID: 3, Name: "payload", Type: iceberg.PrimitiveTypes.String, Required: false}, + ) + // Partition by bucket(4, region) — a non-order-preserving transform. + spec := iceberg.NewPartitionSpec(iceberg.PartitionField{ + SourceIDs: []int{2}, FieldID: 1000, Name: "region_bucket", Transform: iceberg.BucketTransform{NumBuckets: 4}, + }) + seedTbl, cat := newPartitionedCOWTable(t, sc, spec) + + seedTbl = appendPartitionedCOWRows(t, ctx, seedTbl, []map[string]any{ + {"id": 1, "region": "us", "payload": "one"}, + {"id": 2, "region": "eu", "payload": "two"}, + {"id": 3, "region": "apac", "payload": "three"}, + }) + + comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) + require.NoError(t, err) + defer comm.Close() + w := cowWriter(t, seedTbl, "id") + w.committer = comm + + require.NoError(t, w.Write(ctx, service.MessageBatch{ + cowMsg(t, "upsert", map[string]any{"id": 2, "region": "eu", "payload": "TWO"}), + cowMsg(t, "delete", map[string]any{"id": 3, "region": "apac"}), + cowMsg(t, "upsert", map[string]any{"id": 4, "region": "us", "payload": "FOUR"}), + })) + + final := cat.snapshot() + assert.Zero(t, countDeleteManifestFiles(t, ctx, final), "copy-on-write must leave no delete files") + assertAllManifestsData(t, ctx, final) + got := scanPartitionedRows(t, ctx, final) + want := map[int64][2]string{ + 1: {"us", "one"}, + 2: {"eu", "TWO"}, + 4: {"us", "FOUR"}, + } + assert.Equal(t, want, got, "id=3 deleted; id=2 updated; id=4 inserted — all bucket-partitioned") } // --- test helpers -------------------------------------------------------------- diff --git a/internal/impl/iceberg/cow_type_roundtrip_test.go b/internal/impl/iceberg/cow_type_roundtrip_test.go new file mode 100644 index 0000000000..04c9ff2eb7 --- /dev/null +++ b/internal/impl/iceberg/cow_type_roundtrip_test.go @@ -0,0 +1,374 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md + +package iceberg + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/table" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/redpanda-data/benthos/v4/public/service" +) + +// This file is the guard for the copy-on-write column-type support gate +// (checkCOWSchemaSupported / cowSupportedColumnType in cow.go). Each accepted +// type has a faithful round-trip proven here; each rejected type has evidence +// here of exactly why it cannot be accepted safely. +// +// cowMutateDirect exercises the real mutating machinery — buildCOWRecordFactory +// (JSON projection), buildCOWFilter, and committer.commitOverwrite (which calls +// iceberg-go's txn.Overwrite) — but bypasses checkCOWSchemaSupported so the true +// round-trip behaviour of any column type can be observed independently of the +// gate. The merge key is always int64 "id" so the filter path is exercised +// unchanged; the column under test is a non-key column "v". + +// cowMutateDirect seeds `seed` as a plain append, then applies `upsert` as a +// copy-on-write overwrite, and returns the resulting table (or the first error +// from the encode/commit path). +func cowMutateDirect(t testing.TB, ctx context.Context, sc *iceberg.Schema, seed, upsert []map[string]any) (*table.Table, error) { + t.Helper() + tbl, cat := newCOWTable(t, sc) + w := cowWriter(t, cat.snapshot(), "id") + + if len(seed) > 0 { + factory, err := w.buildCOWRecordFactory(sc, toBatch(t, seed)) + if err != nil { + return nil, err + } + rdr, err := factory() + if err != nil { + return nil, err + } + tx := tbl.NewTransaction() + if err := tx.Append(ctx, rdr, nil); err != nil { + rdr.Release() + return nil, err + } + rdr.Release() + if _, err := tx.Commit(ctx); err != nil { + return nil, err + } + } + + comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) + require.NoError(t, err) + defer comm.Close() + w = cowWriter(t, cat.snapshot(), "id") + w.committer = comm + + sc = cat.snapshot().Schema() + filter, err := w.buildCOWFilter(sc, toBatch(t, upsert)) + if err != nil { + return nil, err + } + factory, err := w.buildCOWRecordFactory(sc, toBatch(t, upsert)) + if err != nil { + return nil, err + } + if err := w.committer.commitOverwrite(ctx, OverwriteInput{Filter: filter, NewReader: factory, SchemaID: sc.ID}); err != nil { + return nil, err + } + return cat.snapshot(), nil +} + +func toBatch(t testing.TB, rows []map[string]any) service.MessageBatch { + t.Helper() + b := make(service.MessageBatch, 0, len(rows)) + for _, r := range rows { + b = append(b, structuredMsg(t, r)) + } + return b +} + +// cowReadColJSON reads column `col` for the row whose int64 "id" == id and +// returns its value as canonical JSON (via the Arrow array's GetOneForMarshal), +// plus whether the value was present (non-null). JSON is a type-agnostic, +// lossless-comparable form for the round-trip assertions. +func cowReadColJSON(t testing.TB, ctx context.Context, tbl *table.Table, col string, id int64) (string, bool) { + t.Helper() + at, err := tbl.Scan().ToArrowTable(ctx) + require.NoError(t, err) + defer at.Release() + tr := array.NewTableReader(at, 0) + defer tr.Release() + for tr.Next() { + rec := tr.RecordBatch() + idArr := rec.Column(rec.Schema().FieldIndices("id")[0]).(*array.Int64) + carr := rec.Column(rec.Schema().FieldIndices(col)[0]) + for r := 0; r < int(rec.NumRows()); r++ { + if idArr.Value(r) != id { + continue + } + if carr.IsNull(r) { + return "", false + } + b, err := json.Marshal(carr.GetOneForMarshal(r)) + require.NoError(t, err) + return string(b), true + } + } + return "", false +} + +func cowIDField() iceberg.NestedField { + return iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true} +} + +func cowSingleColSchema(id int, name string, typ iceberg.Type) *iceberg.Schema { + return iceberg.NewSchema(0, cowIDField(), iceberg.NestedField{ID: id, Name: name, Type: typ}) +} + +// TestCOWColumnTypeRoundTrip proves each column type accepted by +// cowSupportedColumnType round-trips faithfully through a real copy-on-write +// overwrite: seed row id=1 with one value, upsert id=1 to a known value, then +// read id=1 back and assert the stored value is exactly the intended value. Row +// id=2 is seeded and left untouched to confirm the rewrite preserves other rows. +func TestCOWColumnTypeRoundTrip(t *testing.T) { + ctx := t.Context() + + uuidVal := "f47ac10b-58cc-0372-8567-0e02b2c3d479" + tsVal := time.Date(2026, 7, 21, 10, 20, 30, 123456000, time.UTC) // microsecond precision + dateVal := time.Date(2026, 7, 21, 0, 0, 0, 0, time.UTC) + timeVal := time.Date(2000, 1, 1, 13, 14, 15, 123456000, time.UTC) + + cases := []struct { + name string + schema *iceberg.Schema + seedV any + upV any + wantJSON string // canonical JSON of the faithfully-stored value + }{ + {"boolean", cowSingleColSchema(2, "v", iceberg.PrimitiveTypes.Bool), false, true, `true`}, + {"int32", cowSingleColSchema(2, "v", iceberg.PrimitiveTypes.Int32), int64(1), int64(2147483647), `2147483647`}, + // > 2^53: proves the top-level integer-to-string massaging preserves full int64 precision. + {"int64", cowSingleColSchema(2, "v", iceberg.PrimitiveTypes.Int64), int64(1), int64(9007199254740993), `9007199254740993`}, + {"float32", cowSingleColSchema(2, "v", iceberg.PrimitiveTypes.Float32), float64(1), float64(1.5), `1.5`}, + {"float64", cowSingleColSchema(2, "v", iceberg.PrimitiveTypes.Float64), float64(1), float64(1.5), `1.5`}, + {"string", cowSingleColSchema(2, "v", iceberg.PrimitiveTypes.String), "a", "hello", `"hello"`}, + {"date", cowSingleColSchema(2, "v", iceberg.PrimitiveTypes.Date), dateVal, dateVal, `"2026-07-21"`}, + {"time", cowSingleColSchema(2, "v", iceberg.PrimitiveTypes.Time), timeVal, timeVal, `"13:14:15.123456"`}, + {"timestamp", cowSingleColSchema(2, "v", iceberg.PrimitiveTypes.Timestamp), tsVal, tsVal, `"2026-07-21T10:20:30.123456Z"`}, + {"timestamptz", cowSingleColSchema(2, "v", iceberg.PrimitiveTypes.TimestampTz), tsVal, tsVal, `"2026-07-21T10:20:30.123456Z"`}, + {"decimal", cowSingleColSchema(2, "v", iceberg.DecimalTypeOf(10, 2)), "1.00", "123.45", `"123.45"`}, + {"uuid", cowSingleColSchema(2, "v", iceberg.PrimitiveTypes.UUID), uuidVal, uuidVal, `"` + uuidVal + `"`}, + // []byte -> json.Marshal base64 -> Arrow Binary base64-decodes: DEADBEEF. + {"binary", cowSingleColSchema(2, "v", iceberg.PrimitiveTypes.Binary), []byte{0x01}, []byte{0xDE, 0xAD, 0xBE, 0xEF}, `"3q2+7w=="`}, + {"fixed", cowSingleColSchema(2, "v", iceberg.FixedTypeOf(4)), []byte{0, 0, 0, 0}, []byte{1, 2, 3, 4}, `"AQIDBA=="`}, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + seed := []map[string]any{{"id": int64(1), "v": c.seedV}, {"id": int64(2), "v": c.seedV}} + upsert := []map[string]any{{"id": int64(1), "v": c.upV}} + final, err := cowMutateDirect(t, ctx, c.schema, seed, upsert) + require.NoError(t, err) + + got, present := cowReadColJSON(t, ctx, final, "v", 1) + require.True(t, present, "upserted value must be present") + assert.JSONEq(t, c.wantJSON, got, "upserted %s value must round-trip faithfully", c.name) + + got2, present2 := cowReadColJSON(t, ctx, final, "v", 2) + require.True(t, present2, "untouched row must survive the rewrite") + _ = got2 + }) + } +} + +// TestCOWNestedStructAndListRoundTrip proves nested struct and list columns +// round-trip faithfully through a real copy-on-write overwrite, at several +// compositions: a flat struct, a list, a struct-of-list, a struct-of-struct, and +// a list-of-struct. cowMassage projects each value onto the Arrow JSON shape +// recursively, so these are now supported (not gated). The >2^53-nested-int and +// map fidelity cases are covered by their own tests below. +func TestCOWNestedStructAndListRoundTrip(t *testing.T) { + ctx := t.Context() + + t.Run("struct", func(t *testing.T) { + sc := iceberg.NewSchema(0, cowIDField(), iceberg.NestedField{ID: 2, Name: "v", Type: &iceberg.StructType{FieldList: []iceberg.NestedField{ + {ID: 3, Name: "a", Type: iceberg.PrimitiveTypes.Int64}, + {ID: 4, Name: "b", Type: iceberg.PrimitiveTypes.String}, + }}}) + seed := []map[string]any{{"id": int64(1), "v": map[string]any{"a": int64(0), "b": "seed"}}} + upsert := []map[string]any{{"id": int64(1), "v": map[string]any{"a": int64(7), "b": "hi"}}} + final, err := cowMutateDirect(t, ctx, sc, seed, upsert) + require.NoError(t, err) + got, present := cowReadColJSON(t, ctx, final, "v", 1) + require.True(t, present) + assert.JSONEq(t, `{"a":7,"b":"hi"}`, got) + }) + + t.Run("list", func(t *testing.T) { + sc := iceberg.NewSchema(0, cowIDField(), iceberg.NestedField{ID: 2, Name: "v", Type: &iceberg.ListType{ + ElementID: 3, Element: iceberg.PrimitiveTypes.String, ElementRequired: false, + }}) + seed := []map[string]any{{"id": int64(1), "v": []any{"x"}}} + upsert := []map[string]any{{"id": int64(1), "v": []any{"a", "b", "c"}}} + final, err := cowMutateDirect(t, ctx, sc, seed, upsert) + require.NoError(t, err) + got, present := cowReadColJSON(t, ctx, final, "v", 1) + require.True(t, present) + assert.JSONEq(t, `["a","b","c"]`, got) + }) + + t.Run("struct_of_list", func(t *testing.T) { + sc := iceberg.NewSchema(0, cowIDField(), iceberg.NestedField{ID: 2, Name: "v", Type: &iceberg.StructType{FieldList: []iceberg.NestedField{ + {ID: 3, Name: "tags", Type: &iceberg.ListType{ElementID: 4, Element: iceberg.PrimitiveTypes.String, ElementRequired: false}}, + {ID: 5, Name: "name", Type: iceberg.PrimitiveTypes.String}, + }}}) + seed := []map[string]any{{"id": int64(1), "v": map[string]any{"tags": []any{"x"}, "name": "seed"}}} + upsert := []map[string]any{{"id": int64(1), "v": map[string]any{"tags": []any{"p", "q"}, "name": "hi"}}} + final, err := cowMutateDirect(t, ctx, sc, seed, upsert) + require.NoError(t, err) + got, present := cowReadColJSON(t, ctx, final, "v", 1) + require.True(t, present) + assert.JSONEq(t, `{"tags":["p","q"],"name":"hi"}`, got) + }) + + t.Run("struct_of_struct", func(t *testing.T) { + sc := iceberg.NewSchema(0, cowIDField(), iceberg.NestedField{ID: 2, Name: "v", Type: &iceberg.StructType{FieldList: []iceberg.NestedField{ + {ID: 3, Name: "inner", Type: &iceberg.StructType{FieldList: []iceberg.NestedField{ + {ID: 4, Name: "a", Type: iceberg.PrimitiveTypes.Int64}, + {ID: 5, Name: "b", Type: iceberg.PrimitiveTypes.String}, + }}}, + {ID: 6, Name: "label", Type: iceberg.PrimitiveTypes.String}, + }}}) + seed := []map[string]any{{"id": int64(1), "v": map[string]any{"inner": map[string]any{"a": int64(0), "b": "s"}, "label": "seed"}}} + upsert := []map[string]any{{"id": int64(1), "v": map[string]any{"inner": map[string]any{"a": int64(42), "b": "deep"}, "label": "hi"}}} + final, err := cowMutateDirect(t, ctx, sc, seed, upsert) + require.NoError(t, err) + got, present := cowReadColJSON(t, ctx, final, "v", 1) + require.True(t, present) + assert.JSONEq(t, `{"inner":{"a":42,"b":"deep"},"label":"hi"}`, got) + }) + + t.Run("list_of_struct", func(t *testing.T) { + sc := iceberg.NewSchema(0, cowIDField(), iceberg.NestedField{ID: 2, Name: "v", Type: &iceberg.ListType{ + ElementID: 3, ElementRequired: false, Element: &iceberg.StructType{FieldList: []iceberg.NestedField{ + {ID: 4, Name: "a", Type: iceberg.PrimitiveTypes.Int64}, + {ID: 5, Name: "b", Type: iceberg.PrimitiveTypes.String}, + }}, + }}) + seed := []map[string]any{{"id": int64(1), "v": []any{map[string]any{"a": int64(0), "b": "s"}}}} + upsert := []map[string]any{{"id": int64(1), "v": []any{ + map[string]any{"a": int64(1), "b": "one"}, + map[string]any{"a": int64(2), "b": "two"}, + }}} + final, err := cowMutateDirect(t, ctx, sc, seed, upsert) + require.NoError(t, err) + got, present := cowReadColJSON(t, ctx, final, "v", 1) + require.True(t, present) + assert.JSONEq(t, `[{"a":1,"b":"one"},{"a":2,"b":"two"}]`, got) + }) +} + +// TestCOWMapColumnRoundTrip proves the map type now round-trips faithfully. A CDC +// map value arrives as a JSON object ({"k":v}); cowMassage reshapes it to Arrow's +// List> encoding (an array of {"key":...,"value":...} entries), +// which is exactly the shape the Arrow map JSON reader — and read-back marshaller +// — use, so the value round-trips exactly. +func TestCOWMapColumnRoundTrip(t *testing.T) { + ctx := t.Context() + + t.Run("string_to_primitive", func(t *testing.T) { + sc := iceberg.NewSchema(0, cowIDField(), iceberg.NestedField{ID: 2, Name: "v", Type: &iceberg.MapType{ + KeyID: 3, KeyType: iceberg.PrimitiveTypes.String, + ValueID: 4, ValueType: iceberg.PrimitiveTypes.Int64, ValueRequired: false, + }}) + seed := []map[string]any{{"id": int64(1), "v": map[string]any{"k0": int64(0)}}} + upsert := []map[string]any{{"id": int64(1), "v": map[string]any{"k1": int64(1)}}} + final, err := cowMutateDirect(t, ctx, sc, seed, upsert) + require.NoError(t, err) + got, present := cowReadColJSON(t, ctx, final, "v", 1) + require.True(t, present) + // Arrow marshals a map back as an array of {"key","value"} entries. + assert.JSONEq(t, `[{"key":"k1","value":1}]`, got) + }) + + t.Run("string_to_struct", func(t *testing.T) { + sc := iceberg.NewSchema(0, cowIDField(), iceberg.NestedField{ID: 2, Name: "v", Type: &iceberg.MapType{ + KeyID: 3, KeyType: iceberg.PrimitiveTypes.String, + ValueID: 4, ValueRequired: false, ValueType: &iceberg.StructType{FieldList: []iceberg.NestedField{ + {ID: 5, Name: "a", Type: iceberg.PrimitiveTypes.Int64}, + {ID: 6, Name: "b", Type: iceberg.PrimitiveTypes.String}, + }}, + }}) + seed := []map[string]any{{"id": int64(1), "v": map[string]any{"k0": map[string]any{"a": int64(0), "b": "s"}}}} + upsert := []map[string]any{{"id": int64(1), "v": map[string]any{"k1": map[string]any{"a": int64(9), "b": "nine"}}}} + final, err := cowMutateDirect(t, ctx, sc, seed, upsert) + require.NoError(t, err) + got, present := cowReadColJSON(t, ctx, final, "v", 1) + require.True(t, present) + assert.JSONEq(t, `[{"key":"k1","value":{"a":9,"b":"nine"}}]`, got) + }) +} + +// TestCOWNestedIntegerBeyond2Pow53RoundTrip is the load-bearing fidelity test: an +// int64 nested inside a struct beyond 2^53 must now round-trip EXACTLY. cowMassage +// applies deleteKeyJSONValue at every leaf, so the nested int is emitted as a JSON +// string and parsed back by the Arrow Int64 builder without the float64 truncation +// that the old flat projection suffered. This is the reverse of the previous +// TestCOWNestedIntegerBeyond2Pow53IsLossy, which documented the corruption that +// kept struct/list gated. +func TestCOWNestedIntegerBeyond2Pow53RoundTrip(t *testing.T) { + ctx := t.Context() + sc := iceberg.NewSchema(0, cowIDField(), iceberg.NestedField{ID: 2, Name: "v", Type: &iceberg.StructType{FieldList: []iceberg.NestedField{ + {ID: 3, Name: "a", Type: iceberg.PrimitiveTypes.Int64}, + }}}) + const big = int64(9007199254740993) // 2^53 + 1 + seed := []map[string]any{{"id": int64(1), "v": map[string]any{"a": int64(0)}}} + upsert := []map[string]any{{"id": int64(1), "v": map[string]any{"a": big}}} + final, err := cowMutateDirect(t, ctx, sc, seed, upsert) + require.NoError(t, err) + got, present := cowReadColJSON(t, ctx, final, "v", 1) + require.True(t, present) + assert.JSONEq(t, `{"a":9007199254740993}`, got, + "nested int64 beyond 2^53 must round-trip faithfully via the recursive per-leaf massage") +} + +// TestCOWNestedNullAndAbsentFields proves that null and absent nested fields read +// back as null: an omitted struct field, an explicit null struct field, and null +// list elements are all preserved through the copy-on-write rewrite. +func TestCOWNestedNullAndAbsentFields(t *testing.T) { + ctx := t.Context() + + t.Run("absent_and_null_struct_fields", func(t *testing.T) { + sc := iceberg.NewSchema(0, cowIDField(), iceberg.NestedField{ID: 2, Name: "v", Type: &iceberg.StructType{FieldList: []iceberg.NestedField{ + {ID: 3, Name: "a", Type: iceberg.PrimitiveTypes.Int64}, + {ID: 4, Name: "b", Type: iceberg.PrimitiveTypes.String}, + }}}) + seed := []map[string]any{{"id": int64(1), "v": map[string]any{"a": int64(0), "b": "s"}}} + // "a" explicitly null, "b" absent -> both read back as null. + upsert := []map[string]any{{"id": int64(1), "v": map[string]any{"a": nil}}} + final, err := cowMutateDirect(t, ctx, sc, seed, upsert) + require.NoError(t, err) + got, present := cowReadColJSON(t, ctx, final, "v", 1) + require.True(t, present) + assert.JSONEq(t, `{"a":null,"b":null}`, got) + }) + + t.Run("null_list_elements", func(t *testing.T) { + sc := iceberg.NewSchema(0, cowIDField(), iceberg.NestedField{ID: 2, Name: "v", Type: &iceberg.ListType{ + ElementID: 3, Element: iceberg.PrimitiveTypes.String, ElementRequired: false, + }}) + seed := []map[string]any{{"id": int64(1), "v": []any{"x"}}} + upsert := []map[string]any{{"id": int64(1), "v": []any{"a", nil, "c"}}} + final, err := cowMutateDirect(t, ctx, sc, seed, upsert) + require.NoError(t, err) + got, present := cowReadColJSON(t, ctx, final, "v", 1) + require.True(t, present) + assert.JSONEq(t, `["a",null,"c"]`, got) + }) +} diff --git a/internal/impl/iceberg/integration/cow_partitioned_integration_test.go b/internal/impl/iceberg/integration/cow_partitioned_integration_test.go new file mode 100644 index 0000000000..d0036d66ba --- /dev/null +++ b/internal/impl/iceberg/integration/cow_partitioned_integration_test.go @@ -0,0 +1,130 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md + +package iceberg + +import ( + "context" + "fmt" + "testing" + + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/catalog" + "github.com/apache/iceberg-go/table" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/redpanda-data/benthos/v4/public/service" + "github.com/redpanda-data/benthos/v4/public/service/integration" + + icebergimpl "github.com/redpanda-data/connect/v4/internal/impl/iceberg" +) + +// TestCOWPartitionedRowOperationsIntegration drives an insert -> upsert/delete +// round trip through the iceberg output configured with merge_strategy: +// copy-on-write against a PARTITIONED table (partition by region, identity +// transform), keyed on id — a column that is NOT the partition column. +// +// This is the case merge-on-read cannot serve: equality deletes are +// partition-scoped, so merge-on-read requires every partition source column to +// be an identifier field. Copy-on-write has no such constraint — it rewrites +// whole data files by filter (which matches across all partitions) and appends +// the new rows routed to their partitions by value. +// +// The single mutating batch spans multiple partitions and asserts: +// - correct final per-partition state via DuckDB (id=1 untouched in us, id=2 +// updated in eu, id=3 deleted from eu, id=5 inserted into apac), +// - ZERO delete files (the copy-on-write invariant), and +// - the batch committed as an overwrite (a whole-file rewrite, not a delete +// append). +func TestCOWPartitionedRowOperationsIntegration(t *testing.T) { + integration.CheckSkip(t) + + ctx := context.Background() + infra := setupTestInfra(t, ctx) + + const ns, tbl = "cow_part_ns", "cow_part_test" + infra.CreateNamespace(t, ns) + + // Create a partitioned table up front. All columns are flat strings so the + // copy-on-write path (validated for flat primitives) and its string merge key + // are both satisfied. Note the schema carries NO identifier-field-ids: under + // copy-on-write, identifier_fields are the connector-side merge key only. + client := infra.NewCatalogClient(t, ns) + sc := iceberg.NewSchema(1, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.StringType{}, Required: true}, + iceberg.NestedField{ID: 2, Name: "region", Type: iceberg.StringType{}, Required: true}, + iceberg.NestedField{ID: 3, Name: "value", Type: iceberg.StringType{}, Required: false}, + ) + spec := iceberg.NewPartitionSpec(iceberg.PartitionField{ + SourceIDs: []int{2}, FieldID: 1000, Name: "region", Transform: iceberg.IdentityTransform{}, + }) + _, err := client.CreateTable(ctx, tbl, sc, catalog.WithPartitionSpec(&spec)) + require.NoError(t, err) + + operation, err := service.NewInterpolatedString(`${! meta("op") }`) + require.NoError(t, err) + + router := infra.NewRouter(t, ns, tbl, + WithRowOperation(icebergimpl.RowOpConfig{ + Operation: operation, + // Merge key is id only — deliberately NOT the partition column, to prove + // copy-on-write is free of the merge-on-read partition-subset constraint. + IdentifierFields: []string{"id"}, + MergeStrategy: icebergimpl.MergeStrategyCOW, + })) + + // Seed four rows spread across three partitions (us, eu, apac). + produceMessages(t, ctx, router, service.MessageBatch{ + opStructMsg("insert", map[string]any{"id": "1", "region": "us", "value": "one"}), + opStructMsg("insert", map[string]any{"id": "2", "region": "eu", "value": "two"}), + opStructMsg("insert", map[string]any{"id": "3", "region": "eu", "value": "three"}), + opStructMsg("insert", map[string]any{"id": "4", "region": "apac", "value": "four"}), + }) + + // One mutating batch touching keys in multiple partitions: + // - upsert id=2 (eu): new value, same partition + // - delete id=3 (eu) + // - upsert id=5 (apac): a brand-new key in a different partition + produceMessages(t, ctx, router, service.MessageBatch{ + opStructMsg("upsert", map[string]any{"id": "2", "region": "eu", "value": "two-updated"}), + opStructMsg("delete", map[string]any{"id": "3", "region": "eu"}), + opStructMsg("upsert", map[string]any{"id": "5", "region": "apac", "value": "five"}), + }) + + // (a) Final state via DuckDB. Select the partition + key columns per the + // DuckDB Iceberg projection quirk (a projection that omits them can misread). + type row struct { + ID string `json:"id"` + Region string `json:"region"` + Value string `json:"value"` + } + rows := querySQL[row](t, ctx, infra, + fmt.Sprintf(`SELECT id, region, value FROM iceberg_cat."%s"."%s" ORDER BY id;`, ns, tbl)) + + require.Len(t, rows, 4, "expected id=1,2,4,5 (id=3 deleted, id=2 not duplicated)") + assert.Equal(t, row{"1", "us", "one"}, rows[0], "id=1 must be untouched in us") + assert.Equal(t, row{"2", "eu", "two-updated"}, rows[1], "id=2 must be upserted in place in eu") + assert.Equal(t, row{"4", "apac", "four"}, rows[2], "id=4 must be untouched in apac") + assert.Equal(t, row{"5", "apac", "five"}, rows[3], "id=5 must be inserted into apac") + + // (b) Zero delete files. Load the committed table through the REST catalog and + // inspect its snapshot manifests directly. + loaded, err := client.LoadTable(ctx, tbl) + require.NoError(t, err) + + dataManifests, deleteManifests := countManifestsByContent(t, ctx, loaded) + assert.Positive(t, dataManifests, "expected at least one data manifest to inspect") + assert.Zero(t, deleteManifests, "copy-on-write must leave zero delete manifests on a partitioned table") + + // (c) The mutating batch must have landed as an overwrite, not a delete-file + // append — proving it was materialised the copy-on-write way. + require.NotNil(t, loaded.CurrentSnapshot()) + assert.Equal(t, table.OpOverwrite, loaded.CurrentSnapshot().Summary.Operation, + "the upsert+delete batch must commit as an overwrite under copy-on-write") +} diff --git a/internal/impl/iceberg/router.go b/internal/impl/iceberg/router.go index 0608a76751..c553352bfb 100644 --- a/internal/impl/iceberg/router.go +++ b/internal/impl/iceberg/router.go @@ -731,8 +731,12 @@ func (r *Router) createWriter(ctx context.Context, key tableKey) (*writer, error return rc.LoadTable(ctx, key.table) } - // Create committer with its own table reference - comm, err := NewCommitter(committerTbl, r.commitCfg, reloadTable, r.logger) + // Create committer with its own table reference. Copy-on-write writes only + // plain data files, so it works on a v1 table and must not trigger the + // irreversible v1->v2 upgrade the merge-on-read path needs. + commitCfg := r.commitCfg + commitCfg.SkipFormatUpgrade = r.rowOpCfg.MergeStrategy == mergeStrategyCOW + comm, err := NewCommitter(committerTbl, commitCfg, reloadTable, r.logger) if err != nil { return nil, fmt.Errorf("creating committer: %w", err) } From 88b2f75deedf7fbd5ac2cc6eef3c2aa9610d7436 Mon Sep 17 00:00:00 2001 From: Ashley Jeffs Date: Tue, 21 Jul 2026 21:58:31 +0100 Subject: [PATCH 03/12] =?UTF-8?q?iceberg:=20harden=20copy-on-write=20?= =?UTF-8?q?=E2=80=94=20retry=20idempotency,=20concurrency,=20polish?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Retry idempotency: an ambiguous catalog response (ErrCommitStateUnknown, e.g. a 5xx/timeout after the write) previously failed the batch on the mutation paths, since blindly retrying a possibly-landed commit could duplicate it. Each mutation commit now stamps a per-call commit-id token into its snapshot summary; on a retry after a reload, if a snapshot already carries the token the prior attempt landed, so the commit returns success instead of re-applying. This makes both copy-on-write (Overwrite/Delete) and merge-on-read (RowDelta) safe to retry on an unknown state. The append path is unchanged — it is already idempotent via file-path dedupe. Also: - multi-writer concurrency test: two committers doing copy-on-write on one table converge via the optimistic-concurrency conflict + retry path with no lost updates (race-clean). - partition-transform integration coverage: truncate plus day/month temporal transforms round-trip under copy-on-write (identity and bucket were already covered), leaving zero delete files. - a one-time startup log when copy-on-write is configured with a mutating operation, noting the write-amplification trade-off and the sort-by-key / large-batch mitigations. --- internal/impl/iceberg/committer.go | 115 +++++++-- internal/impl/iceberg/committer_test.go | 15 ++ internal/impl/iceberg/cow_concurrency_test.go | 228 ++++++++++++++++++ internal/impl/iceberg/cow_polish_test.go | 124 ++++++++++ internal/impl/iceberg/cow_test.go | 72 ++++++ ...ow_transform_partition_integration_test.go | 201 +++++++++++++++ internal/impl/iceberg/output_iceberg.go | 22 ++ .../impl/iceberg/row_operation_commit_test.go | 100 ++++++++ internal/impl/iceberg/writer.go | 15 ++ 9 files changed, 877 insertions(+), 15 deletions(-) create mode 100644 internal/impl/iceberg/cow_concurrency_test.go create mode 100644 internal/impl/iceberg/cow_polish_test.go create mode 100644 internal/impl/iceberg/integration/cow_transform_partition_integration_test.go diff --git a/internal/impl/iceberg/committer.go b/internal/impl/iceberg/committer.go index a374f92c46..4262f6fe40 100644 --- a/internal/impl/iceberg/committer.go +++ b/internal/impl/iceberg/committer.go @@ -13,6 +13,7 @@ import ( "errors" "fmt" "io/fs" + "slices" "strconv" "strings" "sync" @@ -23,6 +24,7 @@ import ( "github.com/apache/iceberg-go/catalog/rest" iceio "github.com/apache/iceberg-go/io" "github.com/apache/iceberg-go/table" + "github.com/google/uuid" "github.com/redpanda-data/benthos/v4/public/service" "github.com/redpanda-data/connect/v4/internal/asyncroutine" @@ -33,6 +35,18 @@ import ( // For now we assume everything works with at least v2 const CurrentIcebergVersion = 2 +// commitIDProp is a namespaced idempotency token written into a mutation +// commit's snapshot summary. iceberg-go copies any custom key in the snapshot +// props into the committed snapshot's Summary.Properties (snapshot_producers.go +// summary() does maps.Copy(summaryProps, props)), and Summary marshals/unmarshals +// through the catalog, so the token survives a table reload. On a retry after an +// ambiguous catalog response (ErrCommitStateUnknown) the committer can then look +// for the token in a reloaded snapshot: if present, the prior attempt actually +// landed server-side, so the retry returns success instead of applying the +// mutation a second time. This makes copy-on-write (Overwrite/Delete) and +// merge-on-read (RowDelta) commits safe to retry on an unknown state. +const commitIDProp = "redpanda-connect.commit-id" + // CommitInput holds data files and the schema ID they were written with. // // Files are inserted (appended) data files. DeleteFiles are equality-delete @@ -143,7 +157,11 @@ func (c *committer) doCommit(ctx context.Context, inputs []CommitInput) ([]struc allFiles = append(allFiles, input.Files...) } - if err := c.commitLocked(ctx, true, func(txn *table.Transaction, props iceberg.Properties, reloaded bool) error { + // The append path is idempotent across a reload via dropAlreadyCommitted + // (keyed on file paths), not via a commit-id token, so it passes an empty + // commitID and keeps its existing dedupe behaviour while still retrying on an + // unknown state. + if err := c.commitLocked(ctx, "", true, func(txn *table.Transaction, props iceberg.Properties, reloaded bool) error { files := allFiles if reloaded { // A prior attempt can land server-side yet report failure (a lost @@ -185,12 +203,15 @@ func (c *committer) commitRowDelta(ctx context.Context, input CommitInput) error if input.SchemaID != currentSchemaID { return &StaleSchemaError{WriterSchemaID: input.SchemaID, CurrentSchemaID: currentSchemaID} } - // retryOnUnknownState is false here: this path does not yet dedupe against a - // reloaded snapshot on retry, so retrying a possibly-landed commit could - // duplicate it. RowDelta commits carry equality-delete files alongside - // inserts, so idempotent replay must reconcile both; that is tracked - // separately. - if err := c.commitLocked(ctx, false, func(txn *table.Transaction, props iceberg.Properties, _ bool) error { + // A stable commit-id, generated once before the retry loop, makes this + // merge-on-read commit idempotent across a reload: commitLocked stamps it into + // the snapshot summary and, on a retry after a failed or ambiguous + // (ErrCommitStateUnknown) response, detects a prior attempt that actually + // landed by finding the id in a reloaded snapshot — returning success instead + // of applying the RowDelta (and its equality deletes) a second time. That is + // why retryOnUnknownState is safe to enable here. + commitID := uuid.NewString() + if err := c.commitLocked(ctx, commitID, true, func(txn *table.Transaction, props iceberg.Properties, _ bool) error { // RowDelta derives the snapshot operation automatically // (append/delete/overwrite). rd := txn.NewRowDelta(props) @@ -229,10 +250,15 @@ func (c *committer) commitOverwrite(ctx context.Context, input OverwriteInput) e // removed. nil means the filesystem can't be listed, so cleanup is skipped. before := c.dataFilePaths(ctx) - // retryOnUnknownState is false, matching commitRowDelta: the overwrite is - // not yet idempotent across a reload, so retrying a possibly-landed commit - // could duplicate it. - err := c.commitLocked(ctx, false, func(txn *table.Transaction, props iceberg.Properties, _ bool) error { + // A stable commit-id, generated once before the retry loop, makes this + // copy-on-write commit idempotent across a reload: commitLocked stamps it into + // the snapshot summary and, on a retry after a failed or ambiguous + // (ErrCommitStateUnknown) response, detects a prior attempt that actually + // landed by finding the id in a reloaded snapshot — returning success instead + // of re-applying the overwrite. That is why retryOnUnknownState is safe to + // enable here. + commitID := uuid.NewString() + err := c.commitLocked(ctx, commitID, true, func(txn *table.Transaction, props iceberg.Properties, _ bool) error { // txn.Delete branches on the table's write.delete.mode; the library // default is already copy-on-write, but set it explicitly for safety so // the delete-only path can never fall into merge-on-read. txn.Overwrite @@ -370,16 +396,39 @@ func (c *committer) referencedDataFilePaths(ctx context.Context) map[string]stru // // retryOnUnknownState controls whether an ErrCommitStateUnknown result (the // commit may have landed server-side, e.g. a 5xx/timeout response) is retried. -// It is only safe to set when stage is idempotent across a reload — i.e. it -// drops files the reloaded snapshot already references — otherwise a retry of a -// commit that actually landed would duplicate it. Callers must hold c.commitMu. -func (c *committer) commitLocked(ctx context.Context, retryOnUnknownState bool, stage func(txn *table.Transaction, props iceberg.Properties, reloaded bool) error) error { +// It is only safe to set when stage is idempotent across a reload — otherwise a +// retry of a commit that actually landed would duplicate it. Two mechanisms +// provide that idempotency: +// - a non-empty commitID: it is stamped into the snapshot summary +// (commitIDProp) so that, after a reload, committedSnapshotHasID can detect a +// prior attempt that landed and short-circuit to success. Used by the +// mutation paths (copy-on-write Overwrite/Delete and merge-on-read RowDelta), +// whose stage callbacks are not path-idempotent on their own. +// - stage dropping files the reloaded snapshot already references (the append +// path's dropAlreadyCommitted), in which case commitID is empty. +// +// Callers must hold c.commitMu. +func (c *committer) commitLocked(ctx context.Context, commitID string, retryOnUnknownState bool, stage func(txn *table.Transaction, props iceberg.Properties, reloaded bool) error) error { props := iceberg.Properties{ table.ManifestMergeEnabledKey: strconv.FormatBool(c.cfg.ManifestMergeEnabled), } if c.cfg.MaxSnapshotAge > 0 { props[table.MaxSnapshotAgeMsKey] = strconv.FormatInt(c.cfg.MaxSnapshotAge.Milliseconds(), 10) } + // Stamp the idempotency token into the snapshot props so the committed + // snapshot carries it. props is reused across attempts, so this holds for + // every stage attempt. iceberg-go copies it verbatim into Summary.Properties. + if commitID != "" { + props[commitIDProp] = commitID + } + + // Record the snapshot current before our first attempt so the post-reload + // idempotency scan can stop once it walks past it: any snapshot our commit + // created is strictly newer than this one. + var startSnapshotID int64 = -1 + if snap := c.table.CurrentSnapshot(); snap != nil { + startSnapshotID = snap.SnapshotID + } var commitErr error attempt := 0 @@ -411,6 +460,16 @@ func (c *committer) commitLocked(ctx context.Context, retryOnUnknownState bool, if reloadedTbl, reloadErr := c.reloadTable(ctx); reloadErr == nil { c.table = reloadedTbl reloaded = true + // Idempotency: a failed or ambiguous response may still have + // landed the commit server-side. If the reloaded table already + // carries our commit-id, the prior attempt succeeded — return + // success rather than re-applying the mutation (which would + // duplicate it). Only the mutation paths pass a commitID; the + // append path relies on dropAlreadyCommitted in its stage instead. + if commitID != "" && c.committedSnapshotHasID(commitID, startSnapshotID) { + c.logger.Debugf("Commit %s already landed on a prior attempt (found in reloaded snapshot); treating retry as success", commitID) + return nil + } } else { c.logger.Warnf("Failed to reload table during commit retry: %v", reloadErr) } @@ -492,6 +551,32 @@ func (c *committer) dropAlreadyCommitted(ctx context.Context, files []iceberg.Da return remaining, nil } +// committedSnapshotHasID reports whether any snapshot in c.table's current +// metadata carries commitID under commitIDProp in its summary. The caller must +// have just reloaded c.table. It backs the mutation paths' idempotent retry: a +// commit that failed or returned an ambiguous state may still have landed +// server-side, and finding our (UUID) commit-id in a reloaded snapshot proves it +// did — so the retry returns success instead of applying the mutation twice. +// +// Snapshots are scanned newest-first (Metadata().Snapshots() is oldest-first, so +// we walk it in reverse) because a just-landed commit is the most recent. The +// scan stops once it reaches stopAtSnapshotID — the snapshot that was current +// when the commit began — since any snapshot our attempt created is strictly +// newer than that. commitIDs are unique per call, so only a snapshot our own +// attempt produced can match. +func (c *committer) committedSnapshotHasID(commitID string, stopAtSnapshotID int64) bool { + snaps := c.table.Metadata().Snapshots() + for _, s := range slices.Backward(snaps) { + if s.SnapshotID == stopAtSnapshotID { + break + } + if s.Summary != nil && s.Summary.Properties[commitIDProp] == commitID { + return true + } + } + return false +} + func (c *committer) incrCommitFailure() { c.metrics.incrCommitFailure() } diff --git a/internal/impl/iceberg/committer_test.go b/internal/impl/iceberg/committer_test.go index e02a191c37..7c7c3678a3 100644 --- a/internal/impl/iceberg/committer_test.go +++ b/internal/impl/iceberg/committer_test.go @@ -240,6 +240,21 @@ func countDataFileRefs(tb testing.TB, ctx context.Context, tbl *table.Table, pat return n } +// countSnapshotsWithCommitID counts how many snapshots in the table's metadata +// carry the mutation idempotency token (commitIDProp) in their summary. Only the +// copy-on-write and merge-on-read mutation paths stamp it — plain appends and +// seed writes do not — so for an exactly-once mutation this is exactly 1. A +// duplicate-apply bug (a landed commit re-applied on retry) shows up as 2. +func countSnapshotsWithCommitID(tbl *table.Table) int { + n := 0 + for _, s := range tbl.Metadata().Snapshots() { + if s.Summary != nil && s.Summary.Properties[commitIDProp] != "" { + n++ + } + } + return n +} + func newScriptedCommitter(tb testing.TB, outcomes ...commitOutcome) (*committer, *scriptedCatalog) { tb.Helper() _, mem := newTestTable(tb) diff --git a/internal/impl/iceberg/cow_concurrency_test.go b/internal/impl/iceberg/cow_concurrency_test.go new file mode 100644 index 0000000000..afa1f5a1cc --- /dev/null +++ b/internal/impl/iceberg/cow_concurrency_test.go @@ -0,0 +1,228 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md + +package iceberg + +import ( + "context" + "fmt" + "path/filepath" + "sync" + "testing" + + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/catalog/rest" + iceio "github.com/apache/iceberg-go/io" + "github.com/apache/iceberg-go/table" + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/redpanda-data/benthos/v4/public/service" +) + +// occCatalog is an in-memory table.CatalogIO that, unlike memCatalog, actually +// enforces optimistic concurrency: on every CommitTable it validates the +// transaction's requirements (notably assert-ref-snapshot-id) against the +// current metadata and rejects a stale commit with rest.ErrCommitFailed — the +// same signal a real REST catalog returns on a 409. This is what turns two +// committers racing on the same table into a genuine stage -> conflict -> +// reload -> re-stage -> success exchange rather than two blind last-write-wins +// applies. All access to meta and the commit counter is guarded by mu, so the +// type is safe for concurrent use (and clean under -race). +type occCatalog struct { + mu sync.Mutex + meta table.Metadata + metadataLocation string + ident table.Identifier + location string + calls int // total CommitTable invocations, including rejected ones +} + +func (c *occCatalog) LoadTable(context.Context, table.Identifier) (*table.Table, error) { + return c.snapshot(), nil +} + +func (c *occCatalog) CommitTable(_ context.Context, _ table.Identifier, reqs []table.Requirement, updates []table.Update) (table.Metadata, string, error) { + c.mu.Lock() + defer c.mu.Unlock() + c.calls++ + + // Enforce optimistic concurrency exactly as a real catalog does: if any + // requirement no longer holds against the current metadata (e.g. main has + // advanced since this transaction was built), reject with ErrCommitFailed so + // the committer reloads and retries. + for _, r := range reqs { + if err := r.Validate(c.meta); err != nil { + return nil, "", fmt.Errorf("%w: %v", rest.ErrCommitFailed, err) + } + } + + meta, err := table.UpdateTableMetadata(c.meta, updates, "") + if err != nil { + return nil, "", err + } + c.meta = meta + return meta, c.metadataLocation, nil +} + +func (c *occCatalog) snapshot() *table.Table { + c.mu.Lock() + defer c.mu.Unlock() + return table.New( + c.ident, + c.meta, + c.metadataLocation, + func(context.Context) (iceio.IO, error) { return iceio.LocalFS{}, nil }, + c, + ) +} + +func (c *occCatalog) callCount() int { + c.mu.Lock() + defer c.mu.Unlock() + return c.calls +} + +// newOCCTable builds an unpartitioned v2 table for sc, backed by an +// OCC-enforcing in-memory catalog and the local filesystem. +func newOCCTable(t testing.TB, sc *iceberg.Schema) *occCatalog { + t.Helper() + location := filepath.ToSlash(t.TempDir()) + meta, err := table.NewMetadata(sc, iceberg.UnpartitionedSpec, table.UnsortedSortOrder, + location, iceberg.Properties{table.PropertyFormatVersion: "2"}) + require.NoError(t, err) + return &occCatalog{ + meta: meta, + metadataLocation: fmt.Sprintf("%s/metadata/00001-%s.metadata.json", location, uuid.New()), + ident: table.Identifier{"default", "cow_conc"}, + location: location, + } +} + +// countTableRows returns the total live row count of the table, used to detect a +// duplicate that a keyed map (like scanRows) would silently hide by overwriting. +func countTableRows(t testing.TB, ctx context.Context, tbl *table.Table) int { + t.Helper() + at, err := tbl.Scan().ToArrowTable(ctx) + require.NoError(t, err) + defer at.Release() + return int(at.NumRows()) +} + +// TestCOWConcurrentCommittersConverge proves item 2.3: two committers applying +// copy-on-write mutations to the SAME table concurrently converge correctly via +// the OCC-conflict + retry path, with no lost updates and no duplicates. +// +// Both committers are constructed from the same seed snapshot and never reload +// before their first attempt, so whichever loses the race into the catalog's +// CommitTable lock is guaranteed to find main already advanced and get +// ErrCommitFailed — deterministically exercising stage -> conflict -> reload -> +// re-stage -> success regardless of goroutine scheduling. The writes still run +// genuinely concurrently (two goroutines), so the test is meaningful under +// -race. Exactly-once is checked two ways: exactly two snapshots carry a +// mutation commit-id (one per committer, so no landed commit was re-applied), +// and the catalog saw exactly three commit applications for the two mutations +// (one clean + one conflicted-then-retried). +func TestCOWConcurrentCommittersConverge(t *testing.T) { + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}, + iceberg.NestedField{ID: 2, Name: "payload", Type: iceberg.PrimitiveTypes.String, Required: false}, + ) + logger := service.MockResources().Logger() + + // mkWriter builds an independent writer+committer pair over the shared + // catalog, both anchored to the catalog's current (seed) snapshot. A healthy + // MaxRetries lets the loser of the race reload and re-stage. + mkWriter := func(t *testing.T, occ *occCatalog) *writer { + t.Helper() + comm, err := NewCommitter(occ.snapshot(), CommitConfig{MaxRetries: 10}, + func(context.Context) (*table.Table, error) { return occ.snapshot(), nil }, logger) + require.NoError(t, err) + t.Cleanup(comm.Close) + w := cowWriter(t, occ.snapshot(), "id") + w.committer = comm + return w + } + + // runConcurrently fires both writes at once and fails on either error. + runConcurrently := func(t *testing.T, ctx context.Context, w1, w2 *writer, b1, b2 service.MessageBatch) { + t.Helper() + var wg sync.WaitGroup + errs := make([]error, 2) + wg.Add(2) + go func() { defer wg.Done(); errs[0] = w1.Write(ctx, b1) }() + go func() { defer wg.Done(); errs[1] = w2.Write(ctx, b2) }() + wg.Wait() + require.NoError(t, errs[0], "committer 1 write") + require.NoError(t, errs[1], "committer 2 write") + } + + t.Run("different keys both land", func(t *testing.T) { + ctx := t.Context() + occ := newOCCTable(t, sc) + + // Seed id=1,2 in a single data file (so each mutation rewrites the shared + // file, forcing the two committers into genuine contention). + seed := appendCOWRows(t, ctx, occ.snapshot(), map[int64]string{1: "one", 2: "two"}) + _ = seed + base := occ.callCount() + + w1 := mkWriter(t, occ) + w2 := mkWriter(t, occ) + + runConcurrently(t, ctx, w1, w2, + service.MessageBatch{cowMsg(t, "upsert", map[string]any{"id": 1, "payload": "ONE"})}, + service.MessageBatch{cowMsg(t, "upsert", map[string]any{"id": 2, "payload": "TWO"})}, + ) + + final := occ.snapshot() + assert.Equal(t, map[int64]string{1: "ONE", 2: "TWO"}, scanRows(t, ctx, final), + "both upserts must survive; neither may be lost to a stale overwrite") + assert.Equal(t, 2, countTableRows(t, ctx, final), "exactly two rows, no duplicates") + assert.Zero(t, countDeleteManifestFiles(t, ctx, final), "copy-on-write must leave no delete files") + assertAllManifestsData(t, ctx, final) + + assert.Equal(t, 2, countSnapshotsWithCommitID(final), "each mutation committed exactly once") + assert.Equal(t, 3, occ.callCount()-base, + "one clean commit + one conflicted-then-retried commit = 3 catalog applications") + }) + + t.Run("same key last writer wins with no duplicate", func(t *testing.T) { + ctx := t.Context() + occ := newOCCTable(t, sc) + + seed := appendCOWRows(t, ctx, occ.snapshot(), map[int64]string{1: "seed"}) + _ = seed + base := occ.callCount() + + w1 := mkWriter(t, occ) + w2 := mkWriter(t, occ) + + runConcurrently(t, ctx, w1, w2, + service.MessageBatch{cowMsg(t, "upsert", map[string]any{"id": 1, "payload": "A"})}, + service.MessageBatch{cowMsg(t, "upsert", map[string]any{"id": 1, "payload": "B"})}, + ) + + final := occ.snapshot() + // Exactly one row for the contended key: the retry re-stages its overwrite + // against the winner's snapshot, deleting that row and re-appending its own, + // so there is never a duplicate. Which value wins depends on the race, but + // it must be one of the two and there must be exactly one. + assert.Equal(t, 1, countTableRows(t, ctx, final), "exactly one row for the contended key — no duplicate") + got := scanRows(t, ctx, final) + require.Len(t, got, 1) + assert.Contains(t, []string{"A", "B"}, got[1], "the surviving value must be one of the two writers'") + assert.Zero(t, countDeleteManifestFiles(t, ctx, final), "copy-on-write must leave no delete files") + assertAllManifestsData(t, ctx, final) + + assert.Equal(t, 2, countSnapshotsWithCommitID(final), "each mutation committed exactly once") + assert.Equal(t, 3, occ.callCount()-base, + "one clean commit + one conflicted-then-retried commit = 3 catalog applications") + }) +} diff --git a/internal/impl/iceberg/cow_polish_test.go b/internal/impl/iceberg/cow_polish_test.go new file mode 100644 index 0000000000..24560bbb23 --- /dev/null +++ b/internal/impl/iceberg/cow_polish_test.go @@ -0,0 +1,124 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md + +package iceberg + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/redpanda-data/benthos/v4/public/service" +) + +// TestCOWAmplificationWarning pins item 3.4: the one-time startup guidance about +// copy-on-write's write-amplification characteristic must fire for a mutating +// copy-on-write config and stay silent otherwise (merge-on-read, or an +// append-only insert config regardless of merge_strategy). It is deliberately +// separate from the max_in_flight ordering warning, which is about correctness. +func TestCOWAmplificationWarning(t *testing.T) { + cases := []struct { + name string + cfg RowOpConfig + wantWarn bool + }{ + { + name: "cow static upsert warns", + cfg: RowOpConfig{Operation: mustInterp(t, "upsert"), IdentifierFields: []string{"id"}, MergeStrategy: mergeStrategyCOW}, + wantWarn: true, + }, + { + name: "cow static delete warns", + cfg: RowOpConfig{Operation: mustInterp(t, "delete"), IdentifierFields: []string{"id"}, MergeStrategy: mergeStrategyCOW}, + wantWarn: true, + }, + { + name: "cow dynamic operation warns", + cfg: RowOpConfig{Operation: mustInterp(t, `${! metadata("op") }`), IdentifierFields: []string{"id"}, MergeStrategy: mergeStrategyCOW}, + wantWarn: true, + }, + { + name: "cow static insert stays silent", + cfg: RowOpConfig{Operation: mustInterp(t, "insert"), MergeStrategy: mergeStrategyCOW}, + wantWarn: false, + }, + { + name: "merge-on-read mutating stays silent", + cfg: RowOpConfig{Operation: mustInterp(t, "upsert"), IdentifierFields: []string{"id"}, MergeStrategy: mergeStrategyMOR}, + wantWarn: false, + }, + { + name: "unset operation stays silent", + cfg: RowOpConfig{MergeStrategy: mergeStrategyCOW}, + wantWarn: false, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + msg, ok := tc.cfg.cowAmplificationWarning() + assert.Equal(t, tc.wantWarn, ok) + if tc.wantWarn { + assert.Contains(t, msg, "copy-on-write") + assert.Contains(t, msg, "sort the table by the identifier key", "the message must name the sort mitigation") + assert.Contains(t, msg, "large batches", "the message must name the batching mitigation") + } else { + assert.Empty(t, msg) + } + }) + } +} + +// TestSplitByOperationCOWCountsFeedMetrics pins item 3.2 at the reachable seam: +// the per-operation counts that drive iceberg_row_operations_total{operation=...} +// are computed by splitByOperation. Because the emitted counter *values* are not +// readable from a unit test (see the CON-490 note in output_iceberg.go), this +// guards the numbers that would be handed to incrInserted/incrUpserted/ +// incrDeleted instead — including the last-writer-wins per-key collapse, so a +// counter can never over-count a repeatedly-mutated key. +func TestSplitByOperationCOWCountsFeedMetrics(t *testing.T) { + tbl, _ := newTestTable(t) // schema: id int64 + w := cowWriter(t, tbl, "id") + + t.Run("distinct ops counted after collapse", func(t *testing.T) { + inserts, deletes, counts, err := w.splitByOperation(service.MessageBatch{ + cowMsg(t, "insert", map[string]any{"id": 10}), + cowMsg(t, "insert", map[string]any{"id": 11}), + cowMsg(t, "upsert", map[string]any{"id": 2}), + cowMsg(t, "delete", map[string]any{"id": 3}), + cowMsg(t, "upsert", map[string]any{"id": 4}), + }) + require.NoError(t, err) + // inserted counts insert-op rows; upserted/deleted count keyed ops after + // per-key collapse. These are exactly the arguments passed to the incr* + // methods in cow.go's writeCOW. + assert.EqualValues(t, 2, counts.inserted, "two insert-op rows") + assert.EqualValues(t, 2, counts.upserted, "two distinct upsert keys") + assert.EqualValues(t, 1, counts.deleted, "one delete key") + // Sanity on the batch split the same counts describe: inserts carry the + // insert rows plus the upsert rows to (re)write; deletes carry one message + // per keyed op. + assert.Len(t, inserts, 4, "2 inserts + 2 upserts to write") + assert.Len(t, deletes, 3, "2 upsert keys + 1 delete key to remove") + }) + + t.Run("repeated key collapses to one op so the counter cannot over-count", func(t *testing.T) { + // Three mutations of id=2 in one batch collapse to the last (a delete), so + // the metrics reflect one committed operation for the key, not three. + _, _, counts, err := w.splitByOperation(service.MessageBatch{ + cowMsg(t, "upsert", map[string]any{"id": 2}), + cowMsg(t, "upsert", map[string]any{"id": 2}), + cowMsg(t, "delete", map[string]any{"id": 2}), + }) + require.NoError(t, err) + assert.EqualValues(t, 0, counts.inserted) + assert.EqualValues(t, 0, counts.upserted, "the trailing delete wins, so no upsert is counted") + assert.EqualValues(t, 1, counts.deleted, "the key collapses to a single delete") + }) +} diff --git a/internal/impl/iceberg/cow_test.go b/internal/impl/iceberg/cow_test.go index 8988fb9c45..01981b5fc1 100644 --- a/internal/impl/iceberg/cow_test.go +++ b/internal/impl/iceberg/cow_test.go @@ -378,6 +378,78 @@ func TestCommitOverwriteCleansUpOrphansOnFailure(t *testing.T) { "the failed copy-on-write commit's parquet files must be cleaned up, leaving only the seed files") } +// TestCommitOverwriteIdempotentOnUnknownState pins the copy-on-write half of the +// commit-id idempotency guarantee. A copy-on-write overwrite is safe to retry +// after an ambiguous (ErrCommitStateUnknown) catalog response because the +// commit-id stamped into the snapshot summary lets the retry tell a landed +// overwrite from a lost one. Every path must leave the mutation applied exactly +// once — no duplicate snapshot, correct final rows. +func TestCommitOverwriteIdempotentOnUnknownState(t *testing.T) { + logger := service.MockResources().Logger() + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}, + iceberg.NestedField{ID: 2, Name: "payload", Type: iceberg.PrimitiveTypes.String, Required: false}, + ) + + // setup seeds id=1,2,3 through a plain catalog, then wraps it in a + // scriptedCatalog so only the mutation under test is subject to the scripted + // outcome. The writer and committer share that scripted catalog. + setup := func(t *testing.T, outcome commitOutcome) (*scriptedCatalog, *writer) { + ctx := t.Context() + seedTbl, mem := newCOWTable(t, sc) + _ = appendCOWRows(t, ctx, seedTbl, map[int64]string{1: "one", 2: "two", 3: "three"}) + cat := &scriptedCatalog{memCatalog: mem, outcomes: []commitOutcome{outcome}} + comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, func(context.Context) (*table.Table, error) { return cat.snapshot(), nil }, logger) + require.NoError(t, err) + t.Cleanup(comm.Close) + w := cowWriter(t, cat.snapshot(), "id") + w.committer = comm + return cat, w + } + + want := map[int64]string{1: "one", 2: "TWO", 3: "three"} + upsert := func() service.MessageBatch { + return service.MessageBatch{cowMsg(t, "upsert", map[string]any{"id": 2, "payload": "TWO"})} + } + + // (A) landed-but-reported-unknown: the first CommitTable applies the overwrite + // server-side, then reports ErrCommitStateUnknown. The retry must find the + // commit-id in the reloaded snapshot and short-circuit to success without + // re-committing (CommitTable called exactly once). + t.Run("landed then unknown applies once", func(t *testing.T) { + ctx := t.Context() + cat, w := setup(t, commitLandThenUnknown) + require.NoError(t, w.Write(ctx, upsert())) + assert.Equal(t, 1, cat.calls, "a landed overwrite must not be re-committed after an unknown-state response") + assert.Equal(t, 1, countSnapshotsWithCommitID(cat.snapshot()), "overwrite applied exactly once") + assert.Equal(t, want, scanRows(t, ctx, cat.snapshot())) + }) + + // (B) not-landed-unknown: the first CommitTable returns ErrCommitStateUnknown + // WITHOUT applying, so the commit-id is absent on reload and the retry must + // re-apply and succeed — still exactly once. + t.Run("unknown without landing re-applies once", func(t *testing.T) { + ctx := t.Context() + cat, w := setup(t, commitUnknownNoLand) + require.NoError(t, w.Write(ctx, upsert())) + assert.Equal(t, 2, cat.calls, "an overwrite that did not land must be retried") + assert.Equal(t, 1, countSnapshotsWithCommitID(cat.snapshot()), "overwrite committed exactly once on the retry") + assert.Equal(t, want, scanRows(t, ctx, cat.snapshot())) + }) + + // Clean conflict (ErrCommitFailed, nothing landed): the commit-id is absent on + // reload, so the genuine-conflict retry still re-applies exactly once — the + // idempotency check must not over-filter a legitimate retry. + t.Run("clean conflict re-applies once", func(t *testing.T) { + ctx := t.Context() + cat, w := setup(t, commitConflict) + require.NoError(t, w.Write(ctx, upsert())) + assert.Equal(t, 2, cat.calls, "a genuine conflict must be retried") + assert.Equal(t, 1, countSnapshotsWithCommitID(cat.snapshot()), "overwrite committed exactly once after the conflict") + assert.Equal(t, want, scanRows(t, ctx, cat.snapshot())) + }) +} + // --- committer-level round trip ------------------------------------------------ // newTypedKeyTableFromSchema builds an unpartitioned v2 table for the given diff --git a/internal/impl/iceberg/integration/cow_transform_partition_integration_test.go b/internal/impl/iceberg/integration/cow_transform_partition_integration_test.go new file mode 100644 index 0000000000..692dcc6635 --- /dev/null +++ b/internal/impl/iceberg/integration/cow_transform_partition_integration_test.go @@ -0,0 +1,201 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md + +package iceberg + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/catalog" + "github.com/apache/iceberg-go/table" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/redpanda-data/benthos/v4/public/service" + "github.com/redpanda-data/benthos/v4/public/service/integration" + + icebergimpl "github.com/redpanda-data/connect/v4/internal/impl/iceberg" +) + +// TestCOWPartitionTransformExtrasIntegration exercises copy-on-write upsert/ +// delete on tables partitioned by transforms that Tier 1.3 reasoned equivalent +// to identity/bucket (same PartitionField.Transform.Apply code path through the +// partitioned fanout writer) but had not been driven end-to-end: a truncate +// transform and the temporal day and month transforms. +// +// Each sub-test seeds rows across several partitions, then applies one mutating +// batch (upsert an existing key in place, delete a key, upsert a brand-new key +// in another partition) and asserts: +// - the correct final per-partition row set via DuckDB (deleted key gone, +// upserted key updated exactly once with no duplicate, new key present), and +// - ZERO delete files — the copy-on-write invariant that makes the table +// readable by engine-backed catalogs — plus the batch committing as an +// overwrite. +// +// The merge key is `id`, deliberately NOT the partition source column, which is +// only possible under copy-on-write. +func TestCOWPartitionTransformExtrasIntegration(t *testing.T) { + integration.CheckSkip(t) + + ctx := context.Background() + infra := setupTestInfra(t, ctx) + + // t15/t16/t17/t18 are distinct days within Jan 2024; m1/m2/m3 are distinct + // months. Seeds pass timestamps as numeric microseconds (the append/shredder + // path), mutations as time.Time (the copy-on-write rewrite path requires a + // real time value for temporal columns). + day := func(d int) time.Time { return time.Date(2024, 1, d, 12, 0, 0, 0, time.UTC) } + month := func(m int) time.Time { return time.Date(2024, time.Month(m), 10, 12, 0, 0, 0, time.UTC) } + + newRouter := func(t *testing.T, ns, tbl string) *icebergimpl.Router { + operation, err := service.NewInterpolatedString(`${! meta("op") }`) + require.NoError(t, err) + return infra.NewRouter(t, ns, tbl, WithRowOperation(icebergimpl.RowOpConfig{ + Operation: operation, + IdentifierFields: []string{"id"}, + MergeStrategy: icebergimpl.MergeStrategyCOW, + })) + } + + // assertCOWClean asserts the committed table holds zero delete files and that + // the mutating batch landed as an overwrite (a whole-file rewrite). + assertCOWClean := func(t *testing.T, ns, tbl string) { + client := infra.NewCatalogClient(t, ns) + loaded, err := client.LoadTable(ctx, tbl) + require.NoError(t, err) + dataManifests, deleteManifests := countManifestsByContent(t, ctx, loaded) + assert.Positive(t, dataManifests, "expected at least one data manifest to inspect") + assert.Zero(t, deleteManifests, "copy-on-write must leave zero delete manifests") + require.NotNil(t, loaded.CurrentSnapshot()) + assert.Equal(t, table.OpOverwrite, loaded.CurrentSnapshot().Summary.Operation, + "the upsert+delete batch must commit as an overwrite under copy-on-write") + } + + type idVal struct { + ID string `json:"id"` + Value string `json:"value"` + } + + t.Run("truncate", func(t *testing.T) { + const ns, tbl = "cow_trunc_ns", "cow_trunc_test" + infra.CreateNamespace(t, ns) + + client := infra.NewCatalogClient(t, ns) + sc := iceberg.NewSchema(1, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.StringType{}, Required: true}, + iceberg.NestedField{ID: 2, Name: "code", Type: iceberg.StringType{}, Required: true}, + iceberg.NestedField{ID: 3, Name: "value", Type: iceberg.StringType{}, Required: false}, + ) + // Partition by truncate(3, code): the first three characters bucket rows. + spec := iceberg.NewPartitionSpec(iceberg.PartitionField{ + SourceIDs: []int{2}, FieldID: 1000, Name: "code_trunc", Transform: iceberg.TruncateTransform{Width: 3}, + }) + _, err := client.CreateTable(ctx, tbl, sc, catalog.WithPartitionSpec(&spec)) + require.NoError(t, err) + + router := newRouter(t, ns, tbl) + + // Seed: AAA partition has id=1,2; BBB has id=3; CCC has id=4. + produceMessages(t, ctx, router, service.MessageBatch{ + opStructMsg("insert", map[string]any{"id": "1", "code": "AAA111", "value": "one"}), + opStructMsg("insert", map[string]any{"id": "2", "code": "AAA222", "value": "two"}), + opStructMsg("insert", map[string]any{"id": "3", "code": "BBB111", "value": "three"}), + opStructMsg("insert", map[string]any{"id": "4", "code": "CCC111", "value": "four"}), + }) + + // Mutate: upsert id=2 in place (AAA), delete id=3 (BBB), upsert new id=5 (AAA). + produceMessages(t, ctx, router, service.MessageBatch{ + opStructMsg("upsert", map[string]any{"id": "2", "code": "AAA222", "value": "two-updated"}), + opStructMsg("delete", map[string]any{"id": "3", "code": "BBB111"}), + opStructMsg("upsert", map[string]any{"id": "5", "code": "AAA333", "value": "five"}), + }) + + rows := querySQL[idVal](t, ctx, infra, + fmt.Sprintf(`SELECT id, value FROM iceberg_cat."%s"."%s" ORDER BY id;`, ns, tbl)) + require.Len(t, rows, 4, "id=3 deleted, id=2 not duplicated, id=5 added") + assert.Equal(t, []idVal{ + {"1", "one"}, {"2", "two-updated"}, {"4", "four"}, {"5", "five"}, + }, rows) + + // Per-partition check: the AAA truncate partition now holds id=1,2,5. + aaa := querySQL[countResult](t, ctx, infra, + fmt.Sprintf(`SELECT COUNT(*) as count FROM iceberg_cat."%s"."%s" WHERE code LIKE 'AAA%%';`, ns, tbl)) + require.Len(t, aaa, 1) + assert.Equal(t, 3, aaa[0].Count, "the AAA truncate partition must contain id=1,2,5") + + assertCOWClean(t, ns, tbl) + }) + + // temporal runs the same upsert/delete shape against a table partitioned by a + // temporal transform on a timestamptz column, parameterised by the transform + // and the three timestamps (existing-key, deleted-key, new-key partitions). + temporal := func(t *testing.T, ns, tbl string, transform iceberg.Transform, tsExisting, tsDeleted, tsNew time.Time) { + infra.CreateNamespace(t, ns) + + client := infra.NewCatalogClient(t, ns) + sc := iceberg.NewSchema(1, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.StringType{}, Required: true}, + iceberg.NestedField{ID: 2, Name: "ts", Type: iceberg.TimestampTzType{}, Required: false}, + iceberg.NestedField{ID: 3, Name: "value", Type: iceberg.StringType{}, Required: false}, + ) + spec := iceberg.NewPartitionSpec(iceberg.PartitionField{ + SourceIDs: []int{2}, FieldID: 1000, Name: "ts_part", Transform: transform, + }) + _, err := client.CreateTable(ctx, tbl, sc, catalog.WithPartitionSpec(&spec)) + require.NoError(t, err) + + router := newRouter(t, ns, tbl) + + // Seed (numeric micros): id=1,2 in the existing-key partition, id=3 in the + // deleted-key partition, id=4 in the existing-key partition. + produceMessages(t, ctx, router, service.MessageBatch{ + opStructMsg("insert", map[string]any{"id": "1", "ts": tsExisting.UnixMicro(), "value": "one"}), + opStructMsg("insert", map[string]any{"id": "2", "ts": tsExisting.UnixMicro(), "value": "two"}), + opStructMsg("insert", map[string]any{"id": "3", "ts": tsDeleted.UnixMicro(), "value": "three"}), + opStructMsg("insert", map[string]any{"id": "4", "ts": tsExisting.UnixMicro(), "value": "four"}), + }) + + // Mutate (time.Time, as the copy-on-write path requires for temporals): + // upsert id=2 in place, delete id=3, upsert new id=5 into a new partition. + produceMessages(t, ctx, router, service.MessageBatch{ + opStructMsg("upsert", map[string]any{"id": "2", "ts": tsExisting, "value": "two-updated"}), + opStructMsg("delete", map[string]any{"id": "3"}), + opStructMsg("upsert", map[string]any{"id": "5", "ts": tsNew, "value": "five"}), + }) + + rows := querySQL[idVal](t, ctx, infra, + fmt.Sprintf(`SELECT id, value FROM iceberg_cat."%s"."%s" ORDER BY id;`, ns, tbl)) + require.Len(t, rows, 4, "id=3 deleted, id=2 not duplicated, id=5 added") + assert.Equal(t, []idVal{ + {"1", "one"}, {"2", "two-updated"}, {"4", "four"}, {"5", "five"}, + }, rows) + + // Per-partition check: the existing-key partition holds id=1,2,4; the new + // partition holds id=5; the deleted partition is empty. + existing := querySQL[countResult](t, ctx, infra, + fmt.Sprintf(`SELECT COUNT(*) as count FROM iceberg_cat."%s"."%s" WHERE value IN ('one','two-updated','four');`, ns, tbl)) + require.Len(t, existing, 1) + assert.Equal(t, 3, existing[0].Count) + + assertCOWClean(t, ns, tbl) + } + + t.Run("day", func(t *testing.T) { + // id=1,2,4 on Jan 15; id=3 (deleted) on Jan 16; new id=5 on Jan 18. + temporal(t, "cow_day_ns", "cow_day_test", iceberg.DayTransform{}, day(15), day(16), day(18)) + }) + + t.Run("month", func(t *testing.T) { + // id=1,2,4 in January; id=3 (deleted) in February; new id=5 in March. + temporal(t, "cow_month_ns", "cow_month_test", iceberg.MonthTransform{}, month(1), month(2), month(3)) + }) +} diff --git a/internal/impl/iceberg/output_iceberg.go b/internal/impl/iceberg/output_iceberg.go index ae5822d4c0..f2d35b2f07 100644 --- a/internal/impl/iceberg/output_iceberg.go +++ b/internal/impl/iceberg/output_iceberg.go @@ -85,6 +85,19 @@ func newOpMetrics(m *service.Metrics) *opMetrics { } } +// NOTE(CON-490, item 3.2): there is no importable seam here to assert emitted +// counter *values* in a unit test. service.MockResources() backs its Metrics +// with metrics.Noop() (which discards writes), and the only readable in-memory +// implementation (metrics.NewLocal) lives in benthos-internal +// internal/component/metrics, which a different module cannot import; nor is +// there a public constructor for *service.Metrics that accepts a recording +// MetricsExporter. Reading a value would therefore require either putting +// opMetrics behind an interface (the refactor deliberately avoided) or standing +// up a full stream + Prometheus scrape (disproportionate). Instead the row-count +// values that feed rowOps are pinned by TestSplitByOperationCOWCountsFeedMetrics +// on the writer side; the incr* label wiring is covered by the incr* methods' +// own trivial mapping. Revisit if benthos exposes a public readable metrics mock. + func (m *opMetrics) incrInserted(n int64) { if m != nil && n > 0 { m.rowOps.Incr(n, "insert") @@ -163,6 +176,15 @@ func newIcebergOutputFromConfig(conf *service.ParsedConfig, mgr *service.Resourc } } + // Copy-on-write trades write amplification for engine-readable (delete-file- + // free) tables. Surface that characteristic and its mitigations once at + // startup so it is not a surprise in production. This is distinct from the + // max_in_flight ordering warning above (which is about correctness, not cost) + // and stays silent for merge-on-read and append-only configs. + if msg, ok := rowOpCfg.cowAmplificationWarning(); ok { + mgr.Logger().Infof("%s", msg) + } + // Parse parquet config var writerOpts []parquet.WriterOption if conf.Contains(ioFieldParquet) { diff --git a/internal/impl/iceberg/row_operation_commit_test.go b/internal/impl/iceberg/row_operation_commit_test.go index 4908064895..9bb75d8de1 100644 --- a/internal/impl/iceberg/row_operation_commit_test.go +++ b/internal/impl/iceberg/row_operation_commit_test.go @@ -415,6 +415,106 @@ func TestWriteCleansUpFilesOnCommitFailure(t *testing.T) { }) } +// morUpsertInput builds an upsert-shaped merge-on-read CommitInput (one data +// file plus one equality-delete file for the given id) against tbl. It mirrors +// the shape TestCommitUpsertProducesOverwriteSnapshot uses, so the RowDelta +// commit derives an overwrite snapshot. +func morUpsertInput(t testing.TB, ctx context.Context, tbl *table.Table, id int) CommitInput { + t.Helper() + w := newDeleteWriter(t, tbl) + deleteFiles, err := w.writeEqualityDeletes(ctx, service.MessageBatch{structuredMsg(t, map[string]any{"id": id})}) + require.NoError(t, err) + dataFile := synthDataFile(t, tbl.Spec(), fmt.Sprintf("%s/data/mor-%s.parquet", tbl.Location(), uuid.New())) + return CommitInput{Files: []iceberg.DataFile{dataFile}, DeleteFiles: deleteFiles, SchemaID: tbl.Schema().ID} +} + +// TestCommitRowDeltaIdempotentOnUnknownState pins the merge-on-read half of the +// commit-id idempotency guarantee: a RowDelta commit is safe to retry after an +// ambiguous (ErrCommitStateUnknown) catalog response because the commit-id +// stamped into the snapshot summary lets the retry tell a landed commit from a +// lost one. Both danger paths must leave the mutation applied exactly once. +func TestCommitRowDeltaIdempotentOnUnknownState(t *testing.T) { + logger := service.MockResources().Logger() + + // (A) landed-but-reported-unknown: the first CommitTable applies the RowDelta + // server-side, then reports ErrCommitStateUnknown. The retry must find the + // commit-id in the reloaded snapshot and return success WITHOUT committing a + // second time — exactly one snapshot carries the token, and CommitTable is + // called exactly once (no re-apply). + t.Run("landed then unknown applies once", func(t *testing.T) { + ctx := t.Context() + _, mem := newTestTable(t) + cat := &scriptedCatalog{memCatalog: mem, outcomes: []commitOutcome{commitLandThenUnknown}} + c, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, func(context.Context) (*table.Table, error) { return cat.snapshot(), nil }, logger) + require.NoError(t, err) + defer c.Close() + + require.NoError(t, c.Commit(ctx, morUpsertInput(t, ctx, cat.snapshot(), 2))) + + assert.Equal(t, 1, cat.calls, "a landed commit must not be re-committed after an unknown-state response") + assert.Equal(t, 1, countSnapshotsWithCommitID(cat.snapshot()), + "exactly one snapshot must carry the commit-id (mutation applied once)") + }) + + // (B) not-landed-unknown: the first CommitTable returns ErrCommitStateUnknown + // WITHOUT applying. The commit-id is therefore absent on reload, so the retry + // must re-apply and succeed — still exactly once. + t.Run("unknown without landing re-applies once", func(t *testing.T) { + ctx := t.Context() + _, mem := newTestTable(t) + cat := &scriptedCatalog{memCatalog: mem, outcomes: []commitOutcome{commitUnknownNoLand}} + c, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, func(context.Context) (*table.Table, error) { return cat.snapshot(), nil }, logger) + require.NoError(t, err) + defer c.Close() + + require.NoError(t, c.Commit(ctx, morUpsertInput(t, ctx, cat.snapshot(), 2))) + + assert.Equal(t, 2, cat.calls, "a commit that did not land must be retried") + assert.Equal(t, 1, countSnapshotsWithCommitID(cat.snapshot()), + "the mutation must be committed exactly once on the successful retry") + }) + + // Clean conflict (ErrCommitFailed, nothing landed): the commit-id is absent on + // reload, so the genuine-conflict retry still re-applies exactly once — the + // idempotency check must not over-filter a legitimate retry. + t.Run("clean conflict re-applies once", func(t *testing.T) { + ctx := t.Context() + _, mem := newTestTable(t) + cat := &scriptedCatalog{memCatalog: mem, outcomes: []commitOutcome{commitConflict}} + c, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, func(context.Context) (*table.Table, error) { return cat.snapshot(), nil }, logger) + require.NoError(t, err) + defer c.Close() + + require.NoError(t, c.Commit(ctx, morUpsertInput(t, ctx, cat.snapshot(), 2))) + + assert.Equal(t, 2, cat.calls, "a genuine conflict must be retried") + assert.Equal(t, 1, countSnapshotsWithCommitID(cat.snapshot()), + "the mutation must be committed exactly once after the conflict") + }) +} + +// TestCommitRowDeltaWritesCommitIDToSummary is the direct round-trip test for the +// idempotency token: a normal (non-flaky) merge-on-read commit must write a +// commit-id into the snapshot summary that is still readable after a catalog +// reload (and is a valid UUID). +func TestCommitRowDeltaWritesCommitIDToSummary(t *testing.T) { + ctx := t.Context() + tbl, cat := newTestTable(t) + c, err := NewCommitter(tbl, CommitConfig{MaxRetries: 1}, reloadFn(cat), service.MockResources().Logger()) + require.NoError(t, err) + defer c.Close() + + require.NoError(t, c.Commit(ctx, morUpsertInput(t, ctx, tbl, 2))) + + snap := cat.snapshot().CurrentSnapshot() + require.NotNil(t, snap) + require.NotNil(t, snap.Summary) + id := snap.Summary.Properties[commitIDProp] + require.NotEmpty(t, id, "the mutation snapshot must carry the commit-id after reload") + _, err = uuid.Parse(id) + assert.NoError(t, err, "the commit-id must be a valid UUID") +} + // BenchmarkCommitterAppend measures the append fast path (no delete files), // which existing append-only users hit. It is the baseline for confirming the // row-operation work did not regress the commit hot path: the only added cost diff --git a/internal/impl/iceberg/writer.go b/internal/impl/iceberg/writer.go index b3df53832c..736277d99e 100644 --- a/internal/impl/iceberg/writer.go +++ b/internal/impl/iceberg/writer.go @@ -107,6 +107,21 @@ func (c RowOpConfig) mutating() bool { return true } +// cowAmplificationWarning returns one-time startup guidance (and ok=true) when +// the configuration uses copy-on-write for a mutating (upsert/delete) workload, +// and ok=false otherwise (append-only, or merge-on-read). Copy-on-write rewrites +// every data file that contains a touched identifier key, so an operator who has +// opted into it for a keyed workload benefits from being pointed once at the two +// mitigations that keep that write amplification bounded — sorting the table by +// the identifier key and using large batches. It stays silent for a purely +// static insert (no mutation, so no amplification) and for merge-on-read. +func (c RowOpConfig) cowAmplificationWarning() (string, bool) { + if c.MergeStrategy != mergeStrategyCOW || !c.mutating() { + return "", false + } + return "merge_strategy: copy-on-write rewrites every data file that contains a touched identifier_fields key, so a scattered keyed workload can rewrite a large fraction of the table per batch. To keep this write amplification bounded, sort the table by the identifier key so each batch's keys cluster into as few data files as possible, and use large batches. See the copy-on-write section of the iceberg output docs for details.", true +} + // writer handles writing batches of messages to a single Iceberg table. type writer struct { table *table.Table From 72365dad5dc16ddc94e3a5dc649ee009c1bcf971 Mon Sep 17 00:00:00 2001 From: Ashley Jeffs Date: Wed, 22 Jul 2026 10:43:47 +0100 Subject: [PATCH 04/12] =?UTF-8?q?iceberg:=20address=20copy-on-write=20revi?= =?UTF-8?q?ew=20=E2=80=94=20temporal=20correctness,=20cleanup,=20coverage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Acts on a full review of the copy-on-write merge strategy. Two behavioural fixes and a large coverage expansion; no change to the merge-on-read or append defaults. Correctness: - The copy-on-write rewrite rejected numeric-epoch temporal values in data columns, diverging from the insert path and ignoring schema_metadata (so a CDC stream carrying epoch-number timestamps would create the table, ingest the first insert, then fail every upsert). The rewrite now runs numeric temporals through the same schema-metadata/unit-aware conversion the shredder uses, via a shared helper with an equivalence test that keeps the two paths from drifting. Temporal merge keys stay strict (a bare number is still rejected up front) so the filter literal can never disagree with the stored value. - No-timezone `timestamp` columns were written to parquet as UTC-adjusted (isAdjustedToUTC=true), so they read back as `timestamptz` and a copy-on-write file rewrite failed with "cannot promote timestamptz to timestamp". No-tz timestamps are now written with isAdjustedToUTC=false per the Iceberg spec; `timestamptz` is unchanged. (Tables whose data files were written before this fix keep the old annotation and would need their data rewritten — noted in the tests.) - Orphaned parquet files written by a copy-on-write attempt that hit a clean conflict and then succeeded on retry are now cleaned up (previously only the final-failure path cleaned up); cleanup is still skipped on an ambiguous (possibly-landed) commit. max_retries gained a lower bound. Boolean merge keys join decimal as gated (an upstream filter-rewrite limitation); boolean/decimal remain fine as ordinary columns. Coverage: numeric/production merge-key input shapes, case-insensitive matching, malformed nested input, the schema-evolution evolve-and-retry loop, the terminal-unknown cleanup-skip and lost-ack retry paths, runtime-warning wiring, and a set of copy-on-write integration tests that read back through DuckDB (an independent engine) — merge-key types, nested struct/list/map, schema evolution, v1 tables, delete-only and cross-partition key moves, and multi-file rewrites. --- .../components/pages/outputs/iceberg.adoc | 2 +- internal/impl/iceberg/committer.go | 97 +++- internal/impl/iceberg/committer_test.go | 146 +++++ internal/impl/iceberg/config.go | 8 +- internal/impl/iceberg/cow.go | 121 +++- .../iceberg/cow_merge_key_roundtrip_test.go | 196 +++++++ .../cow_schema_evolution_disabled_test.go | 104 ++++ .../cow_temporal_data_roundtrip_test.go | 238 ++++++++ internal/impl/iceberg/cow_test.go | 516 +++++++++++++++++- internal/impl/iceberg/icebergx/parquet.go | 15 +- .../impl/iceberg/icebergx/parquet_test.go | 32 +- .../cow_delete_and_move_integration_test.go | 155 ++++++ .../cow_format_version_integration_test.go | 105 ++++ ...ow_multifile_composite_integration_test.go | 194 +++++++ .../cow_nested_schema_integration_test.go | 150 +++++ ...ow_row_operation_types_integration_test.go | 222 ++++++++ .../cow_schema_evolution_integration_test.go | 106 ++++ ...w_temporal_data_column_integration_test.go | 131 +++++ internal/impl/iceberg/output_iceberg_test.go | 136 +++++ .../impl/iceberg/row_operation_commit_test.go | 20 + internal/impl/iceberg/shredder/temporal.go | 79 +++ .../impl/iceberg/shredder/temporal_test.go | 74 +++ internal/impl/iceberg/writer.go | 21 +- 23 files changed, 2798 insertions(+), 70 deletions(-) create mode 100644 internal/impl/iceberg/cow_schema_evolution_disabled_test.go create mode 100644 internal/impl/iceberg/cow_temporal_data_roundtrip_test.go create mode 100644 internal/impl/iceberg/integration/cow_delete_and_move_integration_test.go create mode 100644 internal/impl/iceberg/integration/cow_format_version_integration_test.go create mode 100644 internal/impl/iceberg/integration/cow_multifile_composite_integration_test.go create mode 100644 internal/impl/iceberg/integration/cow_nested_schema_integration_test.go create mode 100644 internal/impl/iceberg/integration/cow_row_operation_types_integration_test.go create mode 100644 internal/impl/iceberg/integration/cow_schema_evolution_integration_test.go create mode 100644 internal/impl/iceberg/integration/cow_temporal_data_column_integration_test.go create mode 100644 internal/impl/iceberg/output_iceberg_test.go diff --git a/docs/modules/components/pages/outputs/iceberg.adoc b/docs/modules/components/pages/outputs/iceberg.adoc index a370ebcf49..563ce9beab 100644 --- a/docs/modules/components/pages/outputs/iceberg.adoc +++ b/docs/modules/components/pages/outputs/iceberg.adoc @@ -263,7 +263,7 @@ Ordering only holds *within* a batch. With more than one batch in flight, concur *Copy-on-write support matrix.* * *Column types:* all flat primitives (`boolean`, `int`, `long`, `float`, `double`, `string`, `date`, `time`, `timestamp`, `timestamptz`, `decimal`, `uuid`, `binary`, `fixed`), and nested `struct`/`list`/`map` columns whose leaves are all supported primitives. -* *Merge-key (`identifier_fields`) types:* `boolean`, `int`, `long`, `string`, `date`, `time`, `timestamp`, `timestamptz` and `uuid`. A `decimal` merge key is *not* supported and errors with a message pointing you at `merge-on-read` (an upstream limitation in the Iceberg library's overwrite filter); `decimal` as a non-key column is fine. +* *Merge-key (`identifier_fields`) types:* `int`, `long`, `string`, `date`, `time`, `timestamp`, `timestamptz` and `uuid`. `decimal` and `boolean` merge keys are *not* supported and error with a message pointing you at `merge-on-read` (an upstream limitation in the Iceberg library's overwrite filter — it cannot apply a `decimal` or `boolean` predicate when rewriting files); both are fine as non-key columns. * *Partitioned tables:* supported, with no requirement that the partition columns be a subset of `identifier_fields`. A `copy-on-write` `upsert` can even move a key from one partition to another. * *Table format:* version 1 or version 2, with no forced upgrade. diff --git a/internal/impl/iceberg/committer.go b/internal/impl/iceberg/committer.go index 4262f6fe40..08f1d5e319 100644 --- a/internal/impl/iceberg/committer.go +++ b/internal/impl/iceberg/committer.go @@ -114,6 +114,14 @@ type committer struct { // NewCommitter creates a new committer for a specific table. func NewCommitter(tbl *table.Table, cfg CommitConfig, reloadTable func(ctx context.Context) (*table.Table, error), logger *service.Logger) (*committer, error) { + // Defensively clamp MaxRetries to at least 1: commitLocked's retry loop is + // `for range cfg.MaxRetries`, so a zero or negative value would never run a + // single attempt and return a "committing transaction after 0 attempts" + // error wrapping a nil cause. Config lint rejects it at startup (config.go), + // but callers constructing a committer directly get the same safety here. + if cfg.MaxRetries < 1 { + cfg.MaxRetries = 1 + } c := &committer{ table: tbl, cfg: cfg, @@ -161,7 +169,7 @@ func (c *committer) doCommit(ctx context.Context, inputs []CommitInput) ([]struc // (keyed on file paths), not via a commit-id token, so it passes an empty // commitID and keeps its existing dedupe behaviour while still retrying on an // unknown state. - if err := c.commitLocked(ctx, "", true, func(txn *table.Transaction, props iceberg.Properties, reloaded bool) error { + if _, err := c.commitLocked(ctx, "", true, func(txn *table.Transaction, props iceberg.Properties, reloaded bool) error { files := allFiles if reloaded { // A prior attempt can land server-side yet report failure (a lost @@ -211,7 +219,7 @@ func (c *committer) commitRowDelta(ctx context.Context, input CommitInput) error // of applying the RowDelta (and its equality deletes) a second time. That is // why retryOnUnknownState is safe to enable here. commitID := uuid.NewString() - if err := c.commitLocked(ctx, commitID, true, func(txn *table.Transaction, props iceberg.Properties, _ bool) error { + if _, err := c.commitLocked(ctx, commitID, true, func(txn *table.Transaction, props iceberg.Properties, _ bool) error { // RowDelta derives the snapshot operation automatically // (append/delete/overwrite). rd := txn.NewRowDelta(props) @@ -245,9 +253,12 @@ func (c *committer) commitOverwrite(ctx context.Context, input OverwriteInput) e // Copy-on-write writes its rewritten and new data files to storage before the // catalog commit (inside txn.Overwrite/Delete), and — unlike the writer- - // authored append/row-delta paths — we never hold their paths. Snapshot the - // data files present beforehand so a failed commit's leftovers can be - // removed. nil means the filesystem can't be listed, so cleanup is skipped. + // authored append/row-delta paths — we never hold their paths. commitLocked + // re-runs the stage (and so re-writes fresh parquet) on EACH attempt, so even + // a commit that ultimately succeeds can leave earlier attempts' files behind. + // Snapshot the data files present beforehand so any attempt's leftovers can be + // diffed out once the commit resolves. nil means the filesystem can't be + // listed, so cleanup is skipped. before := c.dataFilePaths(ctx) // A stable commit-id, generated once before the retry loop, makes this @@ -258,7 +269,7 @@ func (c *committer) commitOverwrite(ctx context.Context, input OverwriteInput) e // of re-applying the overwrite. That is why retryOnUnknownState is safe to // enable here. commitID := uuid.NewString() - err := c.commitLocked(ctx, commitID, true, func(txn *table.Transaction, props iceberg.Properties, _ bool) error { + retried, err := c.commitLocked(ctx, commitID, true, func(txn *table.Transaction, props iceberg.Properties, _ bool) error { // txn.Delete branches on the table's write.delete.mode; the library // default is already copy-on-write, but set it explicitly for safety so // the delete-only path can never fall into merge-on-read. txn.Overwrite @@ -278,14 +289,36 @@ func (c *committer) commitOverwrite(ctx context.Context, input OverwriteInput) e defer rdr.Release() return txn.Overwrite(ctx, rdr, props, table.WithOverwriteFilter(input.Filter)) }) + + // Diff-based orphan cleanup runs after commitLocked resolves. commitLocked + // re-runs the stage on every retry, and each stage attempt writes a fresh set + // of parquet files, so a clean-conflict-then-success sequence lands the winning + // snapshot's files but leaves the earlier attempt's files orphaned — running + // cleanup only on error (as we used to) would leak them. + // + // Two guards keep this safe: + // - Terminal ErrCommitStateUnknown is never cleaned: the commit may have + // landed server-side, so its files could belong to a committed snapshot and + // deleting them would corrupt the table. Those are left for Iceberg + // orphan-file maintenance. + // - On SUCCESS we only clean when the commit was retried. A first-attempt + // success wrote exactly the files it committed (no orphans of ours), and — + // critically — under concurrent committers on the same table our snapshot + // view does not reference a racing committer's just-written files, so a + // first-attempt winner running diff cleanup would delete another committer's + // live/in-flight files. Only a retried commit can have left our own + // orphans, and by the time it succeeds c.table has been reloaded onto the + // latest committed state, so referencedDataFilePaths protects every live + // file while removing only our earlier attempts' leftovers. On FAILURE we + // still clean unconditionally (our commit did not land, so its files are + // genuine orphans), preserving the original failure-path behaviour. + // + // Cleanup is best-effort: skipped when the filesystem can't be listed + // (before == nil). + if before != nil && !errors.Is(err, rest.ErrCommitStateUnknown) && (err != nil || retried) { + c.cleanupOrphanedOverwriteFiles(ctx, before) + } if err != nil { - // Clean up orphaned files only when the commit definitely did not land. - // On an unknown/ambiguous state the written files may belong to a - // snapshot that committed server-side, so removing them would corrupt the - // table — leave those for Iceberg orphan-file maintenance. - if before != nil && !errors.Is(err, rest.ErrCommitStateUnknown) { - c.cleanupOrphanedOverwriteFiles(ctx, before) - } return err } c.logger.Debugf("Committed copy-on-write mutation (delete-only=%t)", input.NewReader == nil) @@ -321,12 +354,17 @@ func (c *committer) dataFilePaths(ctx context.Context) map[string]struct{} { return paths } -// cleanupOrphanedOverwriteFiles removes .parquet files a failed copy-on-write -// commit left under the data directory: those that appeared since the `before` -// snapshot and are not referenced by the current snapshot. The reference check -// is a safety net so a file a committed snapshot still points to is never -// deleted. Best-effort — errors are logged, not returned. The caller must have -// established that the failed commit did not land. +// cleanupOrphanedOverwriteFiles removes .parquet files a copy-on-write commit +// left orphaned under the data directory: those that appeared since the `before` +// snapshot and are not referenced by the current snapshot. It runs after the +// commit resolves whether it succeeded or failed — a retried commit re-writes +// files on each attempt, so even a successful commit can leave an earlier +// attempt's files behind. The reference check against the current snapshot is +// what makes the success case safe: a file the committed snapshot still points +// to is never deleted, so only genuinely unreferenced leftovers are removed. +// Best-effort — errors are logged, not returned. The caller must have +// established that the commit did not terminate in an ambiguous (possibly- +// landed) state, since deleting a possibly-committed file would corrupt the table. func (c *committer) cleanupOrphanedOverwriteFiles(ctx context.Context, before map[string]struct{}) { fsys, err := c.table.FS(ctx) if err != nil { @@ -407,8 +445,15 @@ func (c *committer) referencedDataFilePaths(ctx context.Context) map[string]stru // - stage dropping files the reloaded snapshot already references (the append // path's dropAlreadyCommitted), in which case commitID is empty. // +// The first return value is retried: true when more than one attempt ran, i.e. +// the stage executed more than once and so may have written data files that are +// not part of the final committed snapshot. commitOverwrite uses this to decide +// whether success-path orphan cleanup is warranted: a first-attempt success has +// no such leftovers, and cleaning then would be unsafe under concurrent +// committers (see commitOverwrite). +// // Callers must hold c.commitMu. -func (c *committer) commitLocked(ctx context.Context, commitID string, retryOnUnknownState bool, stage func(txn *table.Transaction, props iceberg.Properties, reloaded bool) error) error { +func (c *committer) commitLocked(ctx context.Context, commitID string, retryOnUnknownState bool, stage func(txn *table.Transaction, props iceberg.Properties, reloaded bool) error) (bool, error) { props := iceberg.Properties{ table.ManifestMergeEnabledKey: strconv.FormatBool(c.cfg.ManifestMergeEnabled), } @@ -441,11 +486,11 @@ func (c *committer) commitLocked(ctx context.Context, commitID string, retryOnUn c.logger.Warnf("Upgrading iceberg table to format version %d to support row-level deletes; this change is irreversible", CurrentIcebergVersion) }) if err := txn.UpgradeFormatVersion(CurrentIcebergVersion); err != nil { - return fmt.Errorf("upgrading version: %w", err) + return attempt > 1, fmt.Errorf("upgrading version: %w", err) } } if err := stage(txn, props, reloaded); err != nil { - return err + return attempt > 1, err } tbl, err := txn.Commit(ctx) // ErrCommitFailed is a clean conflict (our commit did not land), so a @@ -468,7 +513,7 @@ func (c *committer) commitLocked(ctx context.Context, commitID string, retryOnUn // append path relies on dropAlreadyCommitted in its stage instead. if commitID != "" && c.committedSnapshotHasID(commitID, startSnapshotID) { c.logger.Debugf("Commit %s already landed on a prior attempt (found in reloaded snapshot); treating retry as success", commitID) - return nil + return attempt > 1, nil } } else { c.logger.Warnf("Failed to reload table during commit retry: %v", reloadErr) @@ -480,13 +525,13 @@ func (c *committer) commitLocked(ctx context.Context, commitID string, retryOnUn c.table = reloaded } c.incrCommitFailure() - return fmt.Errorf("committing transaction: %w", err) + return attempt > 1, fmt.Errorf("committing transaction: %w", err) } c.table = tbl - return nil + return attempt > 1, nil } c.incrCommitFailure() - return fmt.Errorf("committing transaction after %d attempts: %w", attempt, commitErr) + return attempt > 1, fmt.Errorf("committing transaction after %d attempts: %w", attempt, commitErr) } // dropAlreadyCommitted returns the subset of files whose paths are not already diff --git a/internal/impl/iceberg/committer_test.go b/internal/impl/iceberg/committer_test.go index 7c7c3678a3..dbd1f7b37d 100644 --- a/internal/impl/iceberg/committer_test.go +++ b/internal/impl/iceberg/committer_test.go @@ -12,13 +12,17 @@ import ( "context" "fmt" "path/filepath" + "strconv" + "strings" "testing" + "time" "github.com/apache/iceberg-go" "github.com/apache/iceberg-go/catalog/rest" iceio "github.com/apache/iceberg-go/io" "github.com/apache/iceberg-go/table" "github.com/google/uuid" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/redpanda-data/benthos/v4/public/service" @@ -331,6 +335,148 @@ func TestCommitterRetriesUnknownStateWithoutLanding(t *testing.T) { "a file that did not land before an unknown-state response must be committed once on retry") } +// TestStaleSchemaErrorOnAllEntryPoints (T-16) proves every commit entry point +// rejects an input whose SchemaID no longer matches the table's current schema +// with a StaleSchemaError carrying the right field values, and exercises the +// error's message. +func TestStaleSchemaErrorOnAllEntryPoints(t *testing.T) { + ctx := t.Context() + + assertStale := func(t *testing.T, err error, writerID, currentID int) { + t.Helper() + require.Error(t, err) + var se *StaleSchemaError + require.ErrorAs(t, err, &se) + assert.Equal(t, writerID, se.WriterSchemaID) + assert.Equal(t, currentID, se.CurrentSchemaID) + } + + t.Run("doCommit (append path)", func(t *testing.T) { + tbl, cat := newTestTable(t) + c, err := NewCommitter(tbl, CommitConfig{MaxRetries: 1}, reloadFn(cat), service.MockResources().Logger()) + require.NoError(t, err) + defer c.Close() + cur := c.currentSchemaID() + df := synthDataFile(t, tbl.Spec(), fmt.Sprintf("%s/data/stale-%s.parquet", tbl.Location(), uuid.New())) + assertStale(t, c.Commit(ctx, CommitInput{Files: []iceberg.DataFile{df}, SchemaID: cur + 1}), cur+1, cur) + }) + + t.Run("commitRowDelta (merge-on-read path)", func(t *testing.T) { + tbl, cat := newTestTable(t) + c, err := NewCommitter(tbl, CommitConfig{MaxRetries: 1}, reloadFn(cat), service.MockResources().Logger()) + require.NoError(t, err) + defer c.Close() + cur := c.currentSchemaID() + // A delete file present routes c.Commit through commitRowDelta. + w := newDeleteWriter(t, tbl) + dels, err := w.writeEqualityDeletes(ctx, service.MessageBatch{structuredMsg(t, map[string]any{"id": 2})}) + require.NoError(t, err) + assertStale(t, c.Commit(ctx, CommitInput{DeleteFiles: dels, SchemaID: cur + 1}), cur+1, cur) + }) + + t.Run("commitOverwrite (copy-on-write path)", func(t *testing.T) { + tbl, cat := newTestTable(t) + c, err := NewCommitter(tbl, CommitConfig{MaxRetries: 1}, reloadFn(cat), service.MockResources().Logger()) + require.NoError(t, err) + defer c.Close() + cur := c.currentSchemaID() + // The schema check fires before any filesystem or reader work, so an empty + // OverwriteInput is enough to reach it. + assertStale(t, c.commitOverwrite(ctx, OverwriteInput{SchemaID: cur + 1}), cur+1, cur) + }) + + t.Run("Error message", func(t *testing.T) { + e := &StaleSchemaError{WriterSchemaID: 5, CurrentSchemaID: 3} + assert.Equal(t, "stale schema: data written with schema 5 but table is at schema 3", e.Error()) + }) +} + +// TestCommitStampsMaxSnapshotAge (T-20) proves a committer configured with a +// non-zero MaxSnapshotAge stamps MaxSnapshotAgeMsKey into the committed snapshot's +// summary (via the shared props map in commitLocked). +func TestCommitStampsMaxSnapshotAge(t *testing.T) { + ctx := t.Context() + tbl, cat := newTestTable(t) + const age = 48 * time.Hour + c, err := NewCommitter(tbl, CommitConfig{MaxRetries: 1, MaxSnapshotAge: age}, reloadFn(cat), service.MockResources().Logger()) + require.NoError(t, err) + defer c.Close() + + df := synthDataFile(t, tbl.Spec(), fmt.Sprintf("%s/data/age-%s.parquet", tbl.Location(), uuid.New())) + require.NoError(t, c.Commit(ctx, CommitInput{Files: []iceberg.DataFile{df}, SchemaID: c.currentSchemaID()})) + + snap := cat.snapshot().CurrentSnapshot() + require.NotNil(t, snap) + require.NotNil(t, snap.Summary) + assert.Equal(t, strconv.FormatInt(age.Milliseconds(), 10), snap.Summary.Properties[table.MaxSnapshotAgeMsKey], + "the committed snapshot summary must carry the configured max snapshot age") +} + +// TestNewCommitterClampsMaxRetries (CORR-4) proves a configured MaxRetries below 1 +// is clamped up to 1, so the retry loop runs at least one attempt rather than +// returning a "committing transaction after 0 attempts" error wrapping nil. +func TestNewCommitterClampsMaxRetries(t *testing.T) { + ctx := t.Context() + for _, n := range []int{0, -3} { + t.Run(fmt.Sprintf("max_retries_%d", n), func(t *testing.T) { + tbl, cat := newTestTable(t) + c, err := NewCommitter(tbl, CommitConfig{MaxRetries: n}, reloadFn(cat), service.MockResources().Logger()) + require.NoError(t, err) + defer c.Close() + assert.Equal(t, 1, c.cfg.MaxRetries, "MaxRetries must be clamped to at least 1") + + df := synthDataFile(t, tbl.Spec(), fmt.Sprintf("%s/data/clamp-%s.parquet", tbl.Location(), uuid.New())) + require.NoError(t, c.Commit(ctx, CommitInput{Files: []iceberg.DataFile{df}, SchemaID: c.currentSchemaID()}), + "a clamped committer must still perform the commit") + }) + } +} + +// TestMaxRetriesLint (CORR-4) pins the config lint rule: max_retries below 1 must +// produce a lint error, while the default and any value >= 1 must not. +func TestMaxRetriesLint(t *testing.T) { + linter := service.GlobalEnvironment().NewComponentConfigLinter() + base := func(extra string) string { + return ` +iceberg: + catalog: + url: http://localhost:8181/api/catalog + namespace: ns + table: t + storage: + aws_s3: + bucket: b +` + extra + } + cases := []struct { + name string + extra string + wantLint bool + }{ + {"default (no commit block)", "", false}, + {"explicit valid", " commit:\n max_retries: 2\n", false}, + {"zero", " commit:\n max_retries: 0\n", true}, + {"negative", " commit:\n max_retries: -1\n", true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + lints, err := linter.LintOutputYAML([]byte(base(tc.extra))) + require.NoError(t, err) + var found []string + for _, l := range lints { + if strings.Contains(l.Error(), "max_retries") { + found = append(found, l.Error()) + } + } + if tc.wantLint { + assert.NotEmpty(t, found, "expected a max_retries lint, got: %v", lints) + } else { + assert.Empty(t, found, "expected no max_retries lint, got: %v", found) + } + }) + } +} + // BenchmarkAddDataFilesDupCheck measures the cost of Transaction.AddDataFiles // against a snapshot pre-seeded with N existing data files, with iceberg-go's // duplicate-path check on vs off. It is the local reproduction harness for diff --git a/internal/impl/iceberg/config.go b/internal/impl/iceberg/config.go index 85da15dc36..95eb8568ff 100644 --- a/internal/impl/iceberg/config.go +++ b/internal/impl/iceberg/config.go @@ -149,7 +149,7 @@ const rowOperationDocs = "\n" + "*Copy-on-write support matrix.*\n" + "\n" + "* *Column types:* all flat primitives (`boolean`, `int`, `long`, `float`, `double`, `string`, `date`, `time`, `timestamp`, `timestamptz`, `decimal`, `uuid`, `binary`, `fixed`), and nested `struct`/`list`/`map` columns whose leaves are all supported primitives.\n" + - "* *Merge-key (`identifier_fields`) types:* `boolean`, `int`, `long`, `string`, `date`, `time`, `timestamp`, `timestamptz` and `uuid`. A `decimal` merge key is *not* supported and errors with a message pointing you at `merge-on-read` (an upstream limitation in the Iceberg library's overwrite filter); `decimal` as a non-key column is fine.\n" + + "* *Merge-key (`identifier_fields`) types:* `int`, `long`, `string`, `date`, `time`, `timestamp`, `timestamptz` and `uuid`. `decimal` and `boolean` merge keys are *not* supported and error with a message pointing you at `merge-on-read` (an upstream limitation in the Iceberg library's overwrite filter — it cannot apply a `decimal` or `boolean` predicate when rewriting files); both are fine as non-key columns.\n" + "* *Partitioned tables:* supported, with no requirement that the partition columns be a subset of `identifier_fields`. A `copy-on-write` `upsert` can even move a key from one partition to another.\n" + "* *Table format:* version 1 or version 2, with no forced upgrade.\n" + "\n" + @@ -456,7 +456,11 @@ array:list Default("24h"), service.NewIntField(ioFieldMaxCommitRetries). Description("Maximum number of times to retry a failed transaction commit."). - Default(3), + Default(3). + // A commit needs at least one attempt: the retry loop runs + // `max_retries` times, so 0 or a negative value would never + // attempt the commit at all. Require >= 1. + LintRule(`root = if this < 1 { [ "max_retries must be at least 1" ] }`), ).Description("Commit behavior configuration."). Advanced(). Optional(), diff --git a/internal/impl/iceberg/cow.go b/internal/impl/iceberg/cow.go index e400f1926b..8496693df4 100644 --- a/internal/impl/iceberg/cow.go +++ b/internal/impl/iceberg/cow.go @@ -24,7 +24,9 @@ import ( "github.com/apache/iceberg-go" "github.com/apache/iceberg-go/table" + "github.com/redpanda-data/benthos/v4/public/schema" "github.com/redpanda-data/benthos/v4/public/service" + "github.com/redpanda-data/connect/v4/internal/impl/iceberg/shredder" ) // writeCOW materialises a mutating batch as copy-on-write: it rewrites whole @@ -220,12 +222,13 @@ func cowSupportedColumnType(t iceberg.Type) bool { // per-tuple ANDs — `(a=a1 AND b=b1) OR (a=a2 AND b=b2) ...` — which is the // correct semantics (an AND of per-column INs would match the cross product). // -// Merge-key columns may be int/long/string/boolean, the temporal types +// Merge-key columns may be int/long/string, the temporal types // (date/time/timestamp/timestamptz), or uuid. Every key literal is built so its // encoding matches how buildCOWRecordFactory stores the same value (see -// cowKeyLiteral). decimal is intentionally excluded (a vendored-library bug -// panics on a decimal overwrite filter — use merge-on-read for a decimal key); -// other key types return a clear error. +// cowKeyLiteral). decimal and boolean are intentionally excluded (iceberg-go's +// overwrite filter cannot apply either — decimal panics, boolean is +// unimplemented; use merge-on-read for those keys); other key types return a +// clear error. func (w *writer) buildCOWFilter(tableSchema *iceberg.Schema, keyed service.MessageBatch) (iceberg.BooleanExpression, error) { idFields, err := w.cowKeyFields(tableSchema) if err != nil { @@ -328,7 +331,7 @@ func (w *writer) lookupKeyValue(msg *service.Message, field iceberg.NestedField, // then array.RecordFromJSON, so this function derives each literal from that // same canonicalisation: // -// - int/long/string/boolean: built directly, mirroring the append path. +// - int/long/string: built directly, mirroring the append path. // - date/time/uuid: canonicalised by deleteKeyJSONValue to the exact string // the data path stores, then parsed into the typed literal by iceberg's own // StringLiteral.To — so filter and storage share an encoding by construction @@ -370,11 +373,14 @@ func cowKeyLiteral(t iceberg.Type, name string, v any) (iceberg.Literal, error) } return iceberg.NewLiteral(s), nil case iceberg.BooleanType: - b, ok := v.(bool) - if !ok { - return nil, fmt.Errorf("%s %q: boolean column given %T", ioFieldIdentifierFields, name, v) - } - return iceberg.NewLiteral(b), nil + // Like decimal, a boolean merge key cannot be applied by iceberg-go's + // copy-on-write rewrite: rewriteFilesWithFilter evaluates the (negated) + // key predicate against each data file's rows to keep survivors, and that + // row-level evaluation is unimplemented for BOOL ("not implemented: + // unsupported type BOOL"), failing the whole overwrite. Refuse up front + // with an actionable error rather than let it surface mid-commit. boolean + // remains valid as a non-key column and as a merge-on-read key. + return nil, fmt.Errorf("copy-on-write merge_strategy does not support merge key column %q of type %s; boolean is not a supported copy-on-write merge key (a known limitation in the underlying iceberg library's overwrite filter) — use merge-on-read for a boolean key", name, t) case iceberg.TimestampType, iceberg.TimestampTzType: // Reuse deleteKeyJSONValue purely for its validation: it requires a // time.Time and rejects a bare number with an actionable error. @@ -405,7 +411,7 @@ func cowKeyLiteral(t iceberg.Type, name string, v any) (iceberg.Literal, error) } return lit, nil default: - return nil, fmt.Errorf("copy-on-write merge_strategy does not support merge key column %q of type %s; supported merge-key types are boolean, int, long, string, date, time, timestamp, timestamptz, and uuid (use merge-on-read for other key types)", name, t) + return nil, fmt.Errorf("copy-on-write merge_strategy does not support merge key column %q of type %s; supported merge-key types are int, long, string, date, time, timestamp, timestamptz, and uuid (use merge-on-read for other key types)", name, t) } } @@ -493,6 +499,16 @@ func (w *writer) buildCOWRecordFactory(tableSchema *iceberg.Schema, rows service return nil, fmt.Errorf("building arrow schema: %w", err) } + // Resolve the batch's schema metadata into the same fieldID -> schema.Common + // map the insert path (messagesToParquet) hands the shredder, so a numeric + // temporal DATA column is interpreted with the identical unit-aware + // conversion rather than being rejected. We sample rows[0] and apply it to + // every row, matching messagesToParquet's batch[0] assumption (Connect's + // iceberg router groups by table before this point, so a batch shares one + // schema). A parse failure is non-fatal: we log and fall back to the + // schema-agnostic conversion, exactly as the insert path does. + fieldCommons := w.cowFieldCommons(tableSchema, rows) + fields := tableSchema.Fields() encoded := make([]map[string]any, 0, len(rows)) for i, msg := range rows { @@ -511,7 +527,7 @@ func (w *writer) buildCOWRecordFactory(tableSchema *iceberg.Schema, rows service // Absent/null columns are left out so Arrow reads them as null. continue } - jv, err := w.cowMassage(field.Type, v) + jv, err := w.cowMassage(field.Type, field.ID, v, fieldCommons) if err != nil { return nil, fmt.Errorf("column %q in message %d: %w", field.Name, i, err) } @@ -541,6 +557,31 @@ func (w *writer) buildCOWRecordFactory(tableSchema *iceberg.Schema, rows service }, nil } +// cowFieldCommons resolves the batch's schema_metadata into a leaf-field-ID -> +// schema.Common map, reusing exactly the parse + walk the insert path uses +// (typeResolver.parseSchemaMetadata + buildShredderFieldCommons). Returns nil +// when no resolver/metadata is configured, when the sampled message carries no +// metadata, or when the metadata does not parse — in every such case cowMassage +// falls back to the historical schema-agnostic conversion, matching +// messagesToParquet. Sampling rows[0] mirrors the insert path's batch[0] +// assumption. +func (w *writer) cowFieldCommons(tableSchema *iceberg.Schema, rows service.MessageBatch) map[int]*schema.Common { + if w.resolver == nil || len(rows) == 0 { + return nil + } + common, err := w.resolver.parseSchemaMetadata(rows[0]) + if err != nil { + if w.logger != nil { + w.logger.Warnf("parsing schema metadata for copy-on-write rewrite: %v (falling back to schema-agnostic conversion)", err) + } + return nil + } + if common == nil { + return nil + } + return buildShredderFieldCommons(tableSchema, common, w.caseSensitive) +} + // cowMassage recursively projects a CDC value onto the JSON shape that // SchemaToArrowSchema + array.RecordFromJSON expects for the given iceberg type, // at every depth of the type tree. It is the nested generalisation of the flat @@ -548,14 +589,25 @@ func (w *writer) buildCOWRecordFactory(tableSchema *iceberg.Schema, rows service // (re)write struct/list/map columns rather than either corrupting them or // rejecting them outright. // +// fieldID names the current leaf's iceberg field ID and fieldCommons carries the +// batch's schema metadata keyed by leaf field ID (see cowFieldCommons); together +// they let a temporal leaf interpret a numeric epoch value using the declared +// unit, exactly as the insert path's shredder does. +// // Each type kind is handled as follows: // -// - primitive: delegate to deleteKeyJSONValue, which applies the int->string, -// temporal, decimal and uuid canonicalisation at every leaf. Doing this at -// every leaf (not just the top level) is what fixes the historical silent -// truncation of integers nested beyond 2^53: a nested int64 is emitted as a -// JSON string, which the Arrow Int32/Int64 JSON builder parses back exactly, -// instead of decoding through a lossy float64. +// - primitive: for a temporal column (date/time/timestamp/timestamptz) a bare +// numeric value is first resolved to a time.Time via +// shredder.NumericTemporalToTime — the SAME unit-aware conversion the insert +// path applies — so copy-on-write accepts the same numeric-epoch inputs as +// inserts (CORR-1) instead of hard-erroring, and honours +// require_schema_metadata identically. A time.Time value is unchanged. The +// (possibly converted) value is then handed to deleteKeyJSONValue, which +// applies the int->string, temporal, decimal and uuid canonicalisation at +// every leaf. Doing this at every leaf (not just the top level) is what +// fixes the historical silent truncation of integers nested beyond 2^53: a +// nested int64 is emitted as a JSON string, which the Arrow Int32/Int64 JSON +// builder parses back exactly, instead of decoding through a lossy float64. // - struct: the value is a map[string]any keyed by field name; recurse per // struct field, honouring the writer's case sensitivity, and emit a // map[string]any. Absent/null fields are omitted so Arrow reads them as null, @@ -567,7 +619,7 @@ func (w *writer) buildCOWRecordFactory(tableSchema *iceberg.Schema, rows service // struct fields are literally named "key" and "value", with value nullable). // So reshape to []any of {"key": k, "value": v} objects, recursing the key // and value types. A null map value stays null under its "value" key. -func (w *writer) cowMassage(t iceberg.Type, v any) (any, error) { +func (w *writer) cowMassage(t iceberg.Type, fieldID int, v any, fieldCommons map[int]*schema.Common) (any, error) { switch tt := t.(type) { case *iceberg.StructType: m, ok := v.(map[string]any) @@ -581,7 +633,7 @@ func (w *writer) cowMassage(t iceberg.Type, v any) (any, error) { // Absent/null field: omit so Arrow reads it as null. continue } - mv, err := w.cowMassage(f.Type, fv) + mv, err := w.cowMassage(f.Type, f.ID, fv, fieldCommons) if err != nil { return nil, fmt.Errorf("struct field %q: %w", f.Name, err) } @@ -599,7 +651,7 @@ func (w *writer) cowMassage(t iceberg.Type, v any) (any, error) { out[i] = nil continue } - me, err := w.cowMassage(tt.Element, e) + me, err := w.cowMassage(tt.Element, tt.ElementID, e, fieldCommons) if err != nil { return nil, fmt.Errorf("list element %d: %w", i, err) } @@ -613,13 +665,13 @@ func (w *writer) cowMassage(t iceberg.Type, v any) (any, error) { } entries := make([]any, 0, len(m)) for k, mv := range m { - mk, err := w.cowMassage(tt.KeyType, k) + mk, err := w.cowMassage(tt.KeyType, tt.KeyID, k, fieldCommons) if err != nil { return nil, fmt.Errorf("map key %q: %w", k, err) } var vv any if mv != nil { - vv, err = w.cowMassage(tt.ValueType, mv) + vv, err = w.cowMassage(tt.ValueType, tt.ValueID, mv, fieldCommons) if err != nil { return nil, fmt.Errorf("map value for key %q: %w", k, err) } @@ -628,7 +680,28 @@ func (w *writer) cowMassage(t iceberg.Type, v any) (any, error) { } return entries, nil default: - // Primitive leaf: apply the same canonicalisation the flat path uses. + // Primitive leaf. For a temporal column, resolve a numeric epoch value to + // a time.Time using the shredder's unit-aware conversion (honouring the + // field's schema metadata and require_schema_metadata) so the encoded + // value matches what the insert path stores; a time.Time passes through + // untouched. Non-temporal leaves and time.Time values fall straight + // through to the shared canonicalisation. + if tm, ok, err := shredder.NumericTemporalToTime(v, t, fieldCommons[fieldID], w.requireSchemaMetadata); err != nil { + return nil, err + } else if ok { + v = tm + } + // Iceberg date/time/timestamp columns are microsecond resolution, and the + // Arrow JSON readers for those columns reject sub-microsecond strings. A + // time.Time carrying nanoseconds is therefore truncated to microseconds so + // it both encodes and matches exactly what the insert path stores (the + // shredder uses UnixMicro, which truncates identically) — and, for a merge + // key, what cowKeyLiteral's UnixMicro literal filters on. Truncation is the + // only lossy step, and it is consistent across filter and storage, so it + // never causes a silent no-match. + if tm, ok := v.(time.Time); ok { + v = tm.Truncate(time.Microsecond) + } return deleteKeyJSONValue(t, v) } } diff --git a/internal/impl/iceberg/cow_merge_key_roundtrip_test.go b/internal/impl/iceberg/cow_merge_key_roundtrip_test.go index 69825351bc..3dd97bb86e 100644 --- a/internal/impl/iceberg/cow_merge_key_roundtrip_test.go +++ b/internal/impl/iceberg/cow_merge_key_roundtrip_test.go @@ -11,6 +11,7 @@ package iceberg import ( "context" "encoding/json" + "fmt" "testing" "time" @@ -206,6 +207,201 @@ func TestCOWMergeKeyRoundTrip(t *testing.T) { } } +// driveCOWKeyed seeds a "k"/"payload" table with seed rows, then runs a single +// copy-on-write batch of the given upsert and (optional) delete messages through +// writer.Write, returning the final key-JSON -> payload map. It is the shared +// driver for the merge-key representation and type round-trip tests below. +func driveCOWKeyed(t testing.TB, ctx context.Context, sc *iceberg.Schema, seed []map[string]any, batch service.MessageBatch) map[string]string { + t.Helper() + tbl, cat := newCOWTable(t, sc) + _ = seedMergeKeyRows(t, ctx, tbl, cat, seed) + + comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) + require.NoError(t, err) + defer comm.Close() + w := cowWriter(t, cat.snapshot(), "k") + w.committer = comm + require.NoError(t, w.Write(ctx, batch)) + + final := cat.snapshot() + assert.Zero(t, countDeleteManifestFiles(t, ctx, final), "copy-on-write must leave no delete files") + return scanKeyPayload(t, ctx, final) +} + +// TestCOWInt64KeyRepresentationsRoundTrip (T-2) drives real copy-on-write +// upsert+delete on an int64-keyed table where the mutating message supplies the +// key in each JSON-decoded representation an upstream might produce — +// json.Number beyond 2^53, float64, and string. Each must match the intended +// seeded row (no duplicate, no missed delete), proving cowValueToInt64's +// canonicalisation agrees with the stored integer encoding. +func TestCOWInt64KeyRepresentationsRoundTrip(t *testing.T) { + ctx := t.Context() + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "k", Type: iceberg.PrimitiveTypes.Int64, Required: true}, + iceberg.NestedField{ID: 2, Name: "payload", Type: iceberg.PrimitiveTypes.String}, + ) + + cases := []struct { + name string + upKey, delKey any + upWant int64 // canonical key expected to carry "UP" + }{ + // 2^53+1 and 2^53+3, exact only as json.Number/string, never float64. + {"json.Number beyond 2^53", json.Number("9007199254740993"), json.Number("9007199254740995"), 9007199254740993}, + {"float64 within 2^53", float64(42), float64(43), 42}, + {"string", "100", "101", 100}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + // Seed the two touched keys plus a distinct untouched key. + seed := []map[string]any{ + {"k": c.upWant, "payload": "up-old"}, + {"k": mustInt64(t, c.delKey), "payload": "del"}, + {"k": int64(7), "payload": "untouched"}, + } + final := driveCOWKeyed(t, ctx, sc, seed, service.MessageBatch{ + cowMsg(t, "upsert", map[string]any{"k": c.upKey, "payload": "UP"}), + cowMsg(t, "delete", map[string]any{"k": c.delKey}), + }) + byPay := invertByPayload(t, final) + require.Len(t, final, 2, "exactly the upserted and untouched rows must remain") + assert.Contains(t, byPay, "UP") + assert.Contains(t, byPay, "untouched") + assert.NotContains(t, byPay, "up-old", "the upsert must overwrite in place, not duplicate") + assert.NotContains(t, byPay, "del", "the delete must remove the intended row") + assert.JSONEq(t, fmt.Sprintf("%d", c.upWant), byPay["UP"], "the upserted row must carry the intended key") + }) + } +} + +// mustInt64 converts a test key representation to its canonical int64 so the seed +// stores the same logical key the mutating message references. +func mustInt64(t testing.TB, v any) int64 { + t.Helper() + switch n := v.(type) { + case json.Number: + i, err := n.Int64() + require.NoError(t, err) + return i + case float64: + return int64(n) + case string: + var i int64 + _, err := fmt.Sscan(n, &i) + require.NoError(t, err) + return i + default: + t.Fatalf("unsupported key representation %T", v) + return 0 + } +} + +// TestCOWFloatKeyValueRejected pins that a non-integer or out-of-exact-range +// float64 int-key value is rejected loudly by the filter path rather than +// silently corrupting the merge key (cowValueToInt64). +func TestCOWFloatKeyValueRejected(t *testing.T) { + sc := iceberg.NewSchema(0, iceberg.NestedField{ID: 1, Name: "k", Type: iceberg.PrimitiveTypes.Int64, Required: true}) + tbl := newTypedKeyTableFromSchema(t, sc) + w := cowWriter(t, tbl, "k") + + for _, v := range []float64{1.5, 1e300} { + _, err := w.buildCOWFilter(sc, service.MessageBatch{structuredMsg(t, map[string]any{"k": v})}) + require.Error(t, err, "float64 %v must be rejected as an int64 key", v) + } +} + +// TestCOWInt32KeyRoundTrip proves an int32 merge key round-trips through a real +// copy-on-write upsert+delete (previously untested as a key type). +func TestCOWInt32KeyRoundTrip(t *testing.T) { + ctx := t.Context() + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "k", Type: iceberg.PrimitiveTypes.Int32, Required: true}, + iceberg.NestedField{ID: 2, Name: "payload", Type: iceberg.PrimitiveTypes.String}, + ) + final := driveCOWKeyed(t, ctx, sc, + []map[string]any{ + {"k": int64(1), "payload": "one"}, + {"k": int64(2), "payload": "two"}, + {"k": int64(3), "payload": "three"}, + }, + service.MessageBatch{ + cowMsg(t, "upsert", map[string]any{"k": int64(2), "payload": "TWO"}), + cowMsg(t, "delete", map[string]any{"k": int64(3)}), + }) + byPay := invertByPayload(t, final) + require.Len(t, final, 2) + assert.Equal(t, "2", byPay["TWO"]) + assert.Equal(t, "1", byPay["one"]) + assert.NotContains(t, byPay, "three") + assert.NotContains(t, byPay, "two") +} + +// TestCOWBooleanMergeKeyGated pins the deliberate gate on boolean merge keys. +// Investigating the "boolean-key round-trip" ask (T-2) surfaced that a boolean +// key cannot round-trip: iceberg-go's copy-on-write rewrite +// (rewriteFilesWithFilter) evaluates the negated key predicate against each data +// file's rows, and that row-level evaluation is unimplemented for BOOL, failing +// the whole overwrite mid-commit ("not implemented: unsupported type BOOL"). +// Rather than let that reach a real table, cowKeyLiteral rejects a boolean key +// up front — the same treatment as decimal. boolean remains fine as a non-key +// column (see TestCOWColumnTypeRoundTrip) and as a merge-on-read key. +func TestCOWBooleanMergeKeyGated(t *testing.T) { + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "k", Type: iceberg.PrimitiveTypes.Bool, Required: true}, + ) + tbl := newTypedKeyTableFromSchema(t, sc) + w := cowWriter(t, tbl, "k") + _, err := w.buildCOWFilter(sc, service.MessageBatch{structuredMsg(t, map[string]any{"k": true})}) + require.Error(t, err) + assert.Contains(t, err.Error(), "boolean is not a supported copy-on-write merge key") + assert.Contains(t, err.Error(), "merge-on-read") +} + +// TestCOWSubMicrosecondTimestampKeyTruncation (CORR-5) documents and verifies the +// truncation behaviour of a timestamp merge key carrying sub-microsecond +// (nanosecond) precision. Iceberg TIMESTAMP is microsecond-resolution, so the +// nanoseconds are dropped — but the invariant that matters is that the filter +// literal and the stored value truncate IDENTICALLY, so the key still matches its +// row. cowKeyLiteral builds the literal from time.Time.UnixMicro (truncating), +// while the stored value goes RFC3339Nano -> Arrow timestamp[us] (also +// truncating); this test proves they agree by matching the intended row exactly. +// +// Consequence (documented, not a bug): two instants differing only below the +// microsecond collapse to the same key. That is inherent to a microsecond column +// and is consistent between filter and storage, so it never causes the CON-490 +// silent-no-match — it only means sub-microsecond precision is not part of the +// key identity. +func TestCOWSubMicrosecondTimestampKeyTruncation(t *testing.T) { + ctx := t.Context() + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "k", Type: iceberg.PrimitiveTypes.Timestamp, Required: true}, + iceberg.NestedField{ID: 2, Name: "payload", Type: iceberg.PrimitiveTypes.String}, + ) + // A key with nanosecond precision beyond microseconds (…123456789). + key := time.Date(2026, 6, 15, 10, 20, 30, 123456789, time.UTC) + other := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + + final := driveCOWKeyed(t, ctx, sc, + []map[string]any{ + {"k": key, "payload": "old"}, + {"k": other, "payload": "untouched"}, + }, + service.MessageBatch{ + // Upsert the SAME nanosecond-precision instant: it must match the + // seeded row, proving filter and storage truncate identically. + cowMsg(t, "upsert", map[string]any{"k": key, "payload": "NEW"}), + }) + byPay := invertByPayload(t, final) + require.Len(t, final, 2, "the sub-microsecond key must match its row (no duplicate) — filter and storage truncate identically") + assert.Contains(t, byPay, "NEW") + assert.Contains(t, byPay, "untouched") + assert.NotContains(t, byPay, "old") + // The stored key is the microsecond truncation of the nanosecond input. + wantKeyJSON, err := json.Marshal(key.UTC().Truncate(time.Microsecond)) + require.NoError(t, err) + assert.JSONEq(t, string(wantKeyJSON), byPay["NEW"], "the stored key is the nanosecond input truncated to microseconds") +} + // TestCOWDecimalMergeKeyGated pins the deliberate gate on decimal merge keys. // iceberg-go's overwrite filter routes a decimal literal through its substrait // conversion, which panics (toDecimalLiteral asserts *iceberg.DecimalType while diff --git a/internal/impl/iceberg/cow_schema_evolution_disabled_test.go b/internal/impl/iceberg/cow_schema_evolution_disabled_test.go new file mode 100644 index 0000000000..0512b94e2f --- /dev/null +++ b/internal/impl/iceberg/cow_schema_evolution_disabled_test.go @@ -0,0 +1,104 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md + +package iceberg + +import ( + "testing" + + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/table" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/redpanda-data/benthos/v4/public/service" +) + +// TestCOWUpsertUnknownColumnSchemaEvolutionDisabled is the copy-on-write +// analogue of the merge-on-read "schema evolution disabled" case in +// integration/schema_evolution_test.go (SchemaEvolutionDisabled_FailsOnMissingTable): +// with schema evolution off, an inbound record carrying something the table +// schema does not have must fail loudly and leave the table byte-for-byte +// unchanged rather than silently dropping data or half-writing an overwrite. +// +// This is driven at the writer + in-memory-catalog seam rather than through the +// Router: the Router only ever talks to a live catalog via +// catalogx.NewCatalogClient (there is no in-memory catalog seam on the Router), +// and schema evolution is purely a Router concern — the writer's job is to +// surface a *BatchSchemaEvolutionError, and the Router either recovers from it +// (schemaEvoCfg.Enabled) or propagates it untouched (disabled). So a writer that +// returns the evolution error *before committing anything* is exactly what the +// disabled Router surfaces to the pipeline. We wire a real committer against a +// seeded table so we can additionally prove the "leaves the table unchanged" +// half of the guarantee: no schema change, no new snapshot, no orphaned +// overwrite parquet files. +func TestCOWUpsertUnknownColumnSchemaEvolutionDisabled(t *testing.T) { + ctx := t.Context() + logger := service.MockResources().Logger() + + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}, + iceberg.NestedField{ID: 2, Name: "payload", Type: iceberg.PrimitiveTypes.String}, + ) + seedTbl, cat := newCOWTable(t, sc) + seedTbl = appendCOWRows(t, ctx, seedTbl, map[int64]string{1: "one", 2: "two", 3: "three"}) + + // Capture the pre-write state to compare against afterward. + seedRows := scanRows(t, ctx, cat.snapshot()) + require.Equal(t, map[int64]string{1: "one", 2: "two", 3: "three"}, seedRows, + "precondition: seed rows are present") + seedSnapshot := cat.snapshot().CurrentSnapshot() + require.NotNil(t, seedSnapshot, "precondition: seeding produced a snapshot") + seedParquet := countParquetFiles(t, seedTbl.Location()) + require.Positive(t, seedParquet, "precondition: seeding wrote data files") + + // A real committer over the in-memory catalog — so if the write erroneously + // reached the overwrite commit, it would land a new snapshot we could detect. + comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3, SkipFormatUpgrade: true}, reloadFn(cat), logger) + require.NoError(t, err) + defer comm.Close() + w := cowWriter(t, cat.snapshot(), "id") + w.committer = comm + + // A copy-on-write upsert carrying the column "extra", which the table schema + // does not contain. With schema evolution disabled this must not be admitted. + err = w.Write(ctx, service.MessageBatch{ + cowMsg(t, "upsert", map[string]any{"id": 2, "payload": "TWO", "extra": "surprise"}), + }) + + // 1. Fails loudly, surfacing the unknown column and the need to evolve. + require.Error(t, err, "an unknown column with evolution disabled must fail, not silently drop the column") + var evo *BatchSchemaEvolutionError + require.ErrorAs(t, err, &evo, "the failure must be a schema-evolution error naming the unknown column") + assert.Contains(t, err.Error(), "extra", "the error must name the offending column") + + // 2. Table schema unchanged — the unknown column was not silently added. + after := cat.snapshot() + assert.Len(t, after.Schema().Fields(), 2, "schema must not have gained a column") + _, hasExtra := after.Schema().FindFieldByName("extra") + assert.False(t, hasExtra, "the unknown column must not have been added to the schema") + + // 3. No new snapshot — no partial write and no orphaned overwrite snapshot. + nowSnapshot := after.CurrentSnapshot() + require.NotNil(t, nowSnapshot) + assert.Equal(t, seedSnapshot.SnapshotID, nowSnapshot.SnapshotID, + "the failed write must not have committed a new snapshot") + + // 4. Row data unchanged — id=2 still holds its seeded value, nothing dropped. + assert.Equal(t, seedRows, scanRows(t, ctx, after), + "table rows must be exactly as seeded (in particular id=2 keeps payload \"two\")") + + // 5. No orphaned overwrite parquet files left on disk. + assert.Equal(t, seedParquet, countParquetFiles(t, seedTbl.Location()), + "a failed copy-on-write write must not leave orphaned parquet files") + + // Sanity: the operation record snapshot's op is still whatever seeding left, + // never an overwrite from this failed write. + assert.NotEqual(t, table.OpOverwrite, nowSnapshot.Summary.Operation, + "the current snapshot must not be an overwrite produced by the failed write") +} diff --git a/internal/impl/iceberg/cow_temporal_data_roundtrip_test.go b/internal/impl/iceberg/cow_temporal_data_roundtrip_test.go new file mode 100644 index 0000000000..1de0ee8334 --- /dev/null +++ b/internal/impl/iceberg/cow_temporal_data_roundtrip_test.go @@ -0,0 +1,238 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md + +package iceberg + +import ( + "context" + "testing" + "time" + + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/table" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/redpanda-data/benthos/v4/public/schema" + "github.com/redpanda-data/benthos/v4/public/service" +) + +// This file is the CORR-1 proof: a copy-on-write UPSERT carrying a numeric-epoch +// value in a temporal DATA column must be interpreted with the SAME schema +// metadata the insert path uses, landing at the correct instant instead of being +// rejected (the old behaviour) or corrupted into year ~56755 (the seconds +// misinterpretation). The equivalence between the copy-on-write and insert unit +// conversions is pinned at the unit level by +// shredder.TestNumericTemporalToTimeMatchesConvert; here we prove it end-to-end +// through writer.Write. + +const cowTSMetaKey = "schema" + +// cowWriterWithResolver is cowWriter plus a resolver keyed on cowTSMetaKey, so a +// message's schema_metadata drives the temporal unit interpretation exactly as +// the insert path's shredder does. requireMeta toggles strict mode +// (require_schema_metadata). +func cowWriterWithResolver(t testing.TB, tbl *table.Table, requireMeta bool, idFields ...string) *writer { + t.Helper() + w := cowWriter(t, tbl, idFields...) + w.resolver = newTypeResolver(cowTSMetaKey, nil, true, service.MockResources().Logger()) + w.requireSchemaMetadata = requireMeta + // messagesToParquet (insert fast path) logs coerce decisions into this map; + // NewWriter initialises it in production, so mirror that here for safety. + w.coerceLoggedFieldIDs = map[int]struct{}{} + return w +} + +// tsMillisMeta describes an {id BIGINT, ts timestamp-millis} record. +func tsMillisMeta() *schema.Common { + return &schema.Common{ + Type: schema.Object, + Children: []schema.Common{ + {Name: "id", Type: schema.Int64}, + { + Name: "ts", Type: schema.Timestamp, + // AdjustToUTC:false so the metadata maps to a plain (non-tz) + // TIMESTAMP, matching the table column below. The unit scaling — + // the CORR-1 crux — is independent of AdjustToUTC. + Logical: &schema.LogicalParams{Timestamp: &schema.TimestampParams{Unit: schema.TimeUnitMillis, AdjustToUTC: false}}, + }, + }, + } +} + +// cowMsgMeta builds a copy-on-write message with the given row_operation and, +// when meta is non-nil, the schema_metadata that disambiguates numeric temporal +// units. +func cowMsgMeta(t testing.TB, op string, meta *schema.Common, row map[string]any) *service.Message { + t.Helper() + msg := cowMsg(t, op, row) + if meta != nil { + msg.MetaSetMut(cowTSMetaKey, meta.ToAny()) + } + return msg +} + +// seedCOWRows appends rows encoded exactly as the copy-on-write rewrite would +// (via buildCOWRecordFactory), returning the updated handle. Values are supplied +// as native Go values (e.g. time.Time), matching how a seed insert would arrive. +func seedCOWRows(t testing.TB, ctx context.Context, tbl *table.Table, cat *memCatalog, idField string, rows []map[string]any) *table.Table { + t.Helper() + w := cowWriter(t, cat.snapshot(), idField) + factory, err := w.buildCOWRecordFactory(tbl.Schema(), toBatch(t, rows)) + require.NoError(t, err) + rdr, err := factory() + require.NoError(t, err) + tx := tbl.NewTransaction() + require.NoError(t, tx.Append(ctx, rdr, nil)) + rdr.Release() + next, err := tx.Commit(ctx) + require.NoError(t, err) + return next +} + +// readTimestampMicros returns the raw microseconds-since-epoch stored in an +// Iceberg TIMESTAMP column for the row whose int64 "id" == id. Reading the raw +// Arrow int64 (rather than a formatted string) makes the assertion exact and +// unit-explicit. +func readTimestampMicros(t testing.TB, ctx context.Context, tbl *table.Table, col string, id int64) (int64, bool) { + t.Helper() + at, err := tbl.Scan().ToArrowTable(ctx) + require.NoError(t, err) + defer at.Release() + tr := array.NewTableReader(at, 0) + defer tr.Release() + for tr.Next() { + rec := tr.RecordBatch() + idArr := rec.Column(rec.Schema().FieldIndices("id")[0]).(*array.Int64) + tsArr := rec.Column(rec.Schema().FieldIndices(col)[0]).(*array.Timestamp) + for r := 0; r < int(rec.NumRows()); r++ { + if idArr.Value(r) != id { + continue + } + if tsArr.IsNull(r) { + return 0, false + } + return int64(tsArr.Value(r)), true + } + } + return 0, false +} + +// TestCOWNumericEpochTimestampDataColumnUpsert is the CORR-1 proof. A timestamp +// DATA column receives a bare numeric millis value under schema_metadata +// declaring timestamp-millis. The copy-on-write upsert must store it as +// millis*1000 microseconds — the exact result the insert path produces for the +// identical input — and must NOT reject it or misread it as unix seconds. +func TestCOWNumericEpochTimestampDataColumnUpsert(t *testing.T) { + ctx := t.Context() + + const epochMillis = int64(1_730_000_000_000) // 2024-10-27T03:33:20Z + const wantMicros = epochMillis * 1_000 // correct: millis -> micros + const secondsMisread = epochMillis * 1_000_000 + + meta := tsMillisMeta() + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}, + iceberg.NestedField{ID: 2, Name: "ts", Type: iceberg.PrimitiveTypes.Timestamp}, + ) + + seedTbl, cat := newCOWTable(t, sc) + sentinel := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) + _ = seedCOWRows(t, ctx, seedTbl, cat, "id", []map[string]any{ + {"id": int64(1), "ts": sentinel}, + {"id": int64(2), "ts": sentinel}, + }) + + comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) + require.NoError(t, err) + defer comm.Close() + w := cowWriterWithResolver(t, cat.snapshot(), false, "id") + w.committer = comm + + // COW UPSERT of id=1 with a numeric millis value + schema metadata. + require.NoError(t, w.Write(ctx, service.MessageBatch{ + cowMsgMeta(t, "upsert", meta, map[string]any{"id": int64(1), "ts": epochMillis}), + })) + + final := cat.snapshot() + assert.Zero(t, countDeleteManifestFiles(t, ctx, final), "copy-on-write must leave no delete files") + + got, present := readTimestampMicros(t, ctx, final, "ts", 1) + require.True(t, present, "the upserted row must be present (upsert must not be rejected)") + assert.Equal(t, wantMicros, got, "numeric millis must be stored as millis*1000 micros, matching the insert path") + assert.NotEqual(t, secondsMisread, got, "must not be misinterpreted as unix seconds (year ~56755 corruption)") + // Sanity: the stored instant is the intended one. + assert.Equal(t, time.UnixMilli(epochMillis).UTC(), time.UnixMicro(got).UTC()) + + // The untouched row must survive the rewrite unchanged. + got2, present2 := readTimestampMicros(t, ctx, final, "ts", 2) + require.True(t, present2, "untouched row must survive") + assert.Equal(t, sentinel.UnixMicro(), got2) + + // wantMicros (= millis*1000) is exactly what the shredder insert path + // produces for this input: convertTimestamp scales millis->micros via + // scaleTimestampNumeric, the very helper NumericTemporalToTime reuses. That + // insert==copy-on-write equivalence is pinned unit-for-unit by + // shredder.TestNumericTemporalToTimeMatchesConvert, so asserting wantMicros + // here is asserting the copy-on-write result mirrors the insert path. +} + +// TestCOWNumericEpochTimestampRequireSchemaMetadata pins the +// require_schema_metadata (strict) behaviour for the copy-on-write data path: a +// numeric temporal DATA value is accepted when metadata is present and rejected +// with a coherent error when it is absent — mirroring the shredder insert path. +func TestCOWNumericEpochTimestampRequireSchemaMetadata(t *testing.T) { + ctx := t.Context() + const epochMillis = int64(1_730_000_000_000) + + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}, + iceberg.NestedField{ID: 2, Name: "ts", Type: iceberg.PrimitiveTypes.Timestamp}, + ) + + t.Run("accepted with metadata", func(t *testing.T) { + seedTbl, cat := newCOWTable(t, sc) + sentinel := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) + _ = seedCOWRows(t, ctx, seedTbl, cat, "id", []map[string]any{{"id": int64(1), "ts": sentinel}}) + + comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) + require.NoError(t, err) + defer comm.Close() + w := cowWriterWithResolver(t, cat.snapshot(), true, "id") + w.committer = comm + + require.NoError(t, w.Write(ctx, service.MessageBatch{ + cowMsgMeta(t, "upsert", tsMillisMeta(), map[string]any{"id": int64(1), "ts": epochMillis}), + })) + got, present := readTimestampMicros(t, ctx, cat.snapshot(), "ts", 1) + require.True(t, present) + assert.Equal(t, epochMillis*1_000, got) + }) + + t.Run("rejected without metadata", func(t *testing.T) { + seedTbl, cat := newCOWTable(t, sc) + sentinel := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) + _ = seedCOWRows(t, ctx, seedTbl, cat, "id", []map[string]any{{"id": int64(1), "ts": sentinel}}) + + comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) + require.NoError(t, err) + defer comm.Close() + w := cowWriterWithResolver(t, cat.snapshot(), true, "id") + w.committer = comm + + // No schema metadata on the message: strict mode must reject the numeric. + err = w.Write(ctx, service.MessageBatch{ + cowMsgMeta(t, "upsert", nil, map[string]any{"id": int64(1), "ts": epochMillis}), + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "require_schema_metadata=true") + // And the message must not be misleadingly framed as an identifier column. + assert.NotContains(t, err.Error(), "identifier column") + }) +} diff --git a/internal/impl/iceberg/cow_test.go b/internal/impl/iceberg/cow_test.go index 01981b5fc1..ec773057a9 100644 --- a/internal/impl/iceberg/cow_test.go +++ b/internal/impl/iceberg/cow_test.go @@ -18,11 +18,14 @@ import ( "path/filepath" "strconv" "testing" + "time" "github.com/apache/arrow-go/v18/arrow" "github.com/apache/arrow-go/v18/arrow/array" "github.com/apache/arrow-go/v18/arrow/memory" "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/catalog/rest" + iceio "github.com/apache/iceberg-go/io" "github.com/apache/iceberg-go/table" "github.com/google/uuid" "github.com/stretchr/testify/assert" @@ -53,8 +56,15 @@ func TestParseMergeStrategyConfig(t *testing.T) { }) t.Run("invalid value rejected", func(t *testing.T) { - // An unknown enum value is rejected somewhere along the parse pipeline - // (spec validation or the defensive switch in parseRowOpConfig). + // FINDING (T-20/G4): the merge_strategy StringEnumField enum is NOT + // enforced by ParseYAML — parsing an unknown value succeeds without error, + // so the enum constraint is advisory (surfaced only by a separate config + // lint pass, which ParseYAML does not run). The only hard rejection at + // runtime is the defensive switch in parseRowOpConfig. This subtest pins + // exactly that: ParseYAML admits the bogus value, and parseRowOpConfig is + // what rejects it. If the StringEnumField ever starts rejecting at + // ParseYAML time, the first assertion flips and this test should be + // tightened to require the earlier rejection. conf, yamlErr := icebergOutputConfig().ParseYAML(` catalog: url: http://localhost:8181/api/catalog @@ -65,11 +75,11 @@ storage: bucket: bucket merge_strategy: sideways `, nil) - if yamlErr != nil { - return // rejected at spec-validation time - } + require.NoError(t, yamlErr, + "the merge_strategy enum is not enforced at ParseYAML time; if this ever changes, tighten this test to require the earlier rejection") + _, err := parseRowOpConfig(conf) - require.Error(t, err) + require.Error(t, err, "parseRowOpConfig's defensive switch must reject the unknown merge_strategy") assert.Contains(t, err.Error(), ioFieldMergeStrategy) }) } @@ -221,6 +231,38 @@ func cowWriter(t testing.TB, tbl *table.Table, idFields ...string) *writer { } } +// cowWriterCI is cowWriter with case-insensitive matching, for the case-fold +// tests. +func cowWriterCI(t testing.TB, tbl *table.Table, idFields ...string) *writer { + t.Helper() + w := cowWriter(t, tbl, idFields...) + w.caseSensitive = false + return w +} + +// countRowsWithID counts how many rows carry the given int64 key in column +// idCol — used to detect duplicates that an id->value map would silently +// collapse. +func countRowsWithID(t testing.TB, ctx context.Context, tbl *table.Table, idCol string, id int64) int { + t.Helper() + at, err := tbl.Scan().ToArrowTable(ctx) + require.NoError(t, err) + defer at.Release() + tr := array.NewTableReader(at, 0) + defer tr.Release() + n := 0 + for tr.Next() { + rec := tr.RecordBatch() + idArr := rec.Column(rec.Schema().FieldIndices(idCol)[0]).(*array.Int64) + for r := 0; r < int(rec.NumRows()); r++ { + if idArr.Value(r) == id { + n++ + } + } + } + return n +} + // cowMsg builds a message whose body is the row image and whose "op" metadata // drives the row_operation. func cowMsg(t testing.TB, op string, row map[string]any) *service.Message { @@ -448,6 +490,20 @@ func TestCommitOverwriteIdempotentOnUnknownState(t *testing.T) { assert.Equal(t, 1, countSnapshotsWithCommitID(cat.snapshot()), "overwrite committed exactly once after the conflict") assert.Equal(t, want, scanRows(t, ctx, cat.snapshot())) }) + + // (T-13) landed-but-reported-failed (a lost ack on a 409): the first CommitTable + // applies the overwrite server-side, then reports ErrCommitFailed as if it had + // been a clean conflict. The retry must find the commit-id in the reloaded + // snapshot and short-circuit to success WITHOUT re-committing — exactly one + // CommitTable call, one snapshot with the token, correct rows. + t.Run("landed then failed applies once", func(t *testing.T) { + ctx := t.Context() + cat, w := setup(t, commitLandThenFail) + require.NoError(t, w.Write(ctx, upsert())) + assert.Equal(t, 1, cat.calls, "a landed overwrite must not be re-committed after a lost-ack conflict") + assert.Equal(t, 1, countSnapshotsWithCommitID(cat.snapshot()), "overwrite applied exactly once") + assert.Equal(t, want, scanRows(t, ctx, cat.snapshot())) + }) } // --- committer-level round trip ------------------------------------------------ @@ -831,6 +887,212 @@ func TestCOWBucketPartitionRoundTrip(t *testing.T) { assert.Equal(t, want, got, "id=3 deleted; id=2 updated; id=4 inserted — all bucket-partitioned") } +// TestCOWCaseInsensitiveUpsert (T-3) proves copy-on-write matches a merge key +// case-insensitively end-to-end: the table column is "Id", the mutating messages +// key it as "ID" and "id", and caseSensitive is false. The upsert must rewrite +// the intended row (no duplicate, no missed match), exercising both cowKeyFields +// (filter side) and lookupField in buildCOWRecordFactory (storage side) under +// case folding. +func TestCOWCaseInsensitiveUpsert(t *testing.T) { + ctx := t.Context() + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "Id", Type: iceberg.PrimitiveTypes.Int64, Required: true}, + iceberg.NestedField{ID: 2, Name: "payload", Type: iceberg.PrimitiveTypes.String}, + ) + seedTbl, cat := newCOWTable(t, sc) + + // Seed id=1,2 (canonical "Id" casing) via a case-insensitive writer. + seedW := cowWriterCI(t, cat.snapshot(), "Id") + factory, err := seedW.buildCOWRecordFactory(seedTbl.Schema(), toBatch(t, []map[string]any{ + {"Id": int64(1), "payload": "one"}, + {"Id": int64(2), "payload": "two"}, + })) + require.NoError(t, err) + rdr, err := factory() + require.NoError(t, err) + tx := seedTbl.NewTransaction() + require.NoError(t, tx.Append(ctx, rdr, nil)) + rdr.Release() + _, err = tx.Commit(ctx) + require.NoError(t, err) + + comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) + require.NoError(t, err) + defer comm.Close() + w := cowWriterCI(t, cat.snapshot(), "Id") + w.committer = comm + + // Upsert keyed as "ID", another as "id" — both must fold onto column "Id". + require.NoError(t, w.Write(ctx, service.MessageBatch{ + cowMsg(t, "upsert", map[string]any{"ID": int64(1), "payload": "ONE"}), + cowMsg(t, "upsert", map[string]any{"id": int64(2), "payload": "TWO"}), + })) + + final := cat.snapshot() + assert.Zero(t, countDeleteManifestFiles(t, ctx, final), "copy-on-write must leave no delete files") + assert.Equal(t, 1, countRowsWithID(t, ctx, final, "Id", 1), "id=1 must be rewritten in place, not duplicated") + assert.Equal(t, 1, countRowsWithID(t, ctx, final, "Id", 2), "id=2 must be rewritten in place, not duplicated") + assert.Equal(t, map[int64]string{1: "ONE", 2: "TWO"}, scanRowsBy(t, ctx, final, "Id")) +} + +// scanRowsBy scans an {idCol int64, payload string} table into id->payload. +func scanRowsBy(t testing.TB, ctx context.Context, tbl *table.Table, idCol string) map[int64]string { + t.Helper() + at, err := tbl.Scan().ToArrowTable(ctx) + require.NoError(t, err) + defer at.Release() + out := map[int64]string{} + tr := array.NewTableReader(at, 0) + defer tr.Release() + for tr.Next() { + rec := tr.RecordBatch() + idArr := rec.Column(rec.Schema().FieldIndices(idCol)[0]).(*array.Int64) + payArr := rec.Column(rec.Schema().FieldIndices("payload")[0]).(*array.String) + for r := 0; r < int(rec.NumRows()); r++ { + pay := "" + if payArr.IsValid(r) { + pay = payArr.Value(r) + } + out[idArr.Value(r)] = pay + } + } + return out +} + +// TestCOWDetectNewColumnsCaseFold (T-3) proves cowDetectNewColumns folds case: a +// message field "Extra" against table column "extra" is NOT flagged as new, +// while a genuinely absent "brand_new" IS flagged for schema evolution. +func TestCOWDetectNewColumnsCaseFold(t *testing.T) { + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "Id", Type: iceberg.PrimitiveTypes.Int64, Required: true}, + iceberg.NestedField{ID: 2, Name: "extra", Type: iceberg.PrimitiveTypes.String}, + ) + tbl := newTypedKeyTableFromSchema(t, sc) + w := cowWriterCI(t, tbl, "Id") + + // "Extra" folds onto "extra": not a new column. + require.NoError(t, w.cowDetectNewColumns(sc, service.MessageBatch{ + structuredMsg(t, map[string]any{"Id": int64(1), "Extra": "x"}), + })) + + // "brand_new" has no case-folded match: flagged for evolution. + err := w.cowDetectNewColumns(sc, service.MessageBatch{ + structuredMsg(t, map[string]any{"Id": int64(1), "brand_new": "y"}), + }) + require.Error(t, err) + var evo *BatchSchemaEvolutionError + require.ErrorAs(t, err, &evo) + assert.Contains(t, err.Error(), "brand_new") +} + +// TestCOWMassageMalformedInput (T-18) pins cowMassage's shape-mismatch branches: +// a scalar where a struct is expected, a map where a list is expected, and a +// scalar where a map is expected each return a descriptive error rather than +// panicking or silently mis-encoding. +func TestCOWMassageMalformedInput(t *testing.T) { + w := &writer{caseSensitive: true} + + structT := &iceberg.StructType{FieldList: []iceberg.NestedField{{ID: 3, Name: "a", Type: iceberg.PrimitiveTypes.Int64}}} + _, err := w.cowMassage(structT, 2, "scalar-not-object", nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "struct value must be an object") + + listT := &iceberg.ListType{ElementID: 3, Element: iceberg.PrimitiveTypes.String, ElementRequired: false} + _, err = w.cowMassage(listT, 2, map[string]any{"x": 1}, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "list value must be an array") + + mapT := &iceberg.MapType{KeyID: 3, KeyType: iceberg.PrimitiveTypes.String, ValueID: 4, ValueType: iceberg.PrimitiveTypes.Int64, ValueRequired: false} + _, err = w.cowMassage(mapT, 2, "scalar-not-object", nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "map value must be an object") + + // A nested malformed leaf is reported with its path context. + _, err = w.cowMassage(structT, 2, map[string]any{"a": []any{"list-into-int"}}, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "struct field \"a\"") +} + +// TestCOWWriteEmptyBatchNoOp (T-19) proves an empty batch through writeCOW is a +// no-op: it must not create a snapshot or touch the committer (nil here would +// panic if used). +func TestCOWWriteEmptyBatchNoOp(t *testing.T) { + ctx := t.Context() + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}, + ) + tbl, cat := newCOWTable(t, sc) + w := cowWriter(t, tbl, "id") // no committer wired + + require.NoError(t, w.writeCOW(ctx, service.MessageBatch{})) + assert.Nil(t, cat.snapshot().CurrentSnapshot(), "an empty batch must not produce a snapshot") +} + +// TestCOWUnsupportedColumnGateThroughWrite (T-19) proves a copy-on-write mutating +// write against a table with an unsupported column type surfaces the schema-gate +// error through Write (before any commit), rather than corrupting or dropping the +// column. timestamp_ns is a valid Iceberg type but outside the copy-on-write +// supported set. +func TestCOWUnsupportedColumnGateThroughWrite(t *testing.T) { + ctx := t.Context() + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}, + iceberg.NestedField{ID: 2, Name: "at", Type: iceberg.PrimitiveTypes.TimestampNs}, + ) + // timestamp_ns is only valid in a v3 table, so build one directly (the v2 + // helper would reject the schema before we could reach the copy-on-write gate). + location := filepath.ToSlash(t.TempDir()) + meta, err := table.NewMetadata(sc, iceberg.UnpartitionedSpec, table.UnsortedSortOrder, location, + iceberg.Properties{table.PropertyFormatVersion: "3"}) + require.NoError(t, err) + cat := &memCatalog{ + meta: meta, + metadataLocation: fmt.Sprintf("%s/metadata/00001-%s.metadata.json", location, uuid.New()), + ident: table.Identifier{"default", "cow_ns"}, + location: location, + } + tbl := cat.snapshot() + w := cowWriter(t, tbl, "id") // no committer: the gate must fire first + + err = w.Write(ctx, service.MessageBatch{ + cowMsg(t, "upsert", map[string]any{"id": int64(1), "at": time.Now()}), + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "does not support column") + assert.Contains(t, err.Error(), "timestamp_ns") +} + +// TestCOWInsertPlusUpsertSameKeyDuplicates (CORR-6) pins the documented contract +// that an insert and an upsert of the SAME key in one batch produce a duplicate: +// insert is an unconditional append and is deliberately not keyed, so it is not +// collapsed against the upsert (splitByOperation). Operators must map create +// events to upsert, not insert, for keyed data. This test fixes that behaviour +// so a future change to it is a conscious decision. +func TestCOWInsertPlusUpsertSameKeyDuplicates(t *testing.T) { + ctx := t.Context() + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}, + iceberg.NestedField{ID: 2, Name: "payload", Type: iceberg.PrimitiveTypes.String}, + ) + seedTbl, cat := newCOWTable(t, sc) + + comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) + require.NoError(t, err) + defer comm.Close() + w := cowWriter(t, seedTbl, "id") + w.committer = comm + + // insert id=1 and upsert id=1 in the same batch. + require.NoError(t, w.Write(ctx, service.MessageBatch{ + cowMsg(t, "insert", map[string]any{"id": int64(1), "payload": "A"}), + cowMsg(t, "upsert", map[string]any{"id": int64(1), "payload": "B"}), + })) + + final := cat.snapshot() + assert.Equal(t, 2, countRowsWithID(t, ctx, final, "id", 1), + "insert + upsert of the same key in one batch is not de-duplicated — the contract in splitByOperation") +} + // --- test helpers -------------------------------------------------------------- // newAmpTableWithSchema builds an unpartitioned v2 table for the given schema, @@ -909,6 +1171,248 @@ func scanRows(t testing.TB, ctx context.Context, tbl *table.Table) map[int64]str return out } +// TestCommitOverwriteNoLeakOnConflictThenSuccess (T-7, CORR-2) proves the fix for +// the orphan-parquet leak on a clean-conflict-then-success retry. commitLocked +// re-runs the overwrite stage on every attempt, writing a fresh set of parquet +// files each time; here attempt 1 loses a clean 409 (nothing lands) and attempt 2 +// succeeds. The winning snapshot's files must survive while attempt 1's files are +// cleaned, so the final on-disk parquet count is exactly the seed files (protected +// because they existed before the commit) plus the winning snapshot's files — no +// leak. Before CORR-2 (cleanup ran only on the error path) attempt 1's files +// leaked and this count was higher. +func TestCommitOverwriteNoLeakOnConflictThenSuccess(t *testing.T) { + ctx := t.Context() + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}, + iceberg.NestedField{ID: 2, Name: "payload", Type: iceberg.PrimitiveTypes.String, Required: false}, + ) + seedTbl, mem := newCOWTable(t, sc) + seedTbl = appendCOWRows(t, ctx, seedTbl, map[int64]string{1: "one", 2: "two", 3: "three"}) + seedCount := countParquetFiles(t, seedTbl.Location()) + require.Positive(t, seedCount) + + // attempt 1 = clean conflict (nothing lands), attempt 2 = success. + cat := &scriptedCatalog{memCatalog: mem, outcomes: []commitOutcome{commitConflict}} + comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, + func(context.Context) (*table.Table, error) { return cat.snapshot(), nil }, service.MockResources().Logger()) + require.NoError(t, err) + defer comm.Close() + w := cowWriter(t, cat.snapshot(), "id") + w.committer = comm + + require.NoError(t, w.Write(ctx, service.MessageBatch{cowMsg(t, "upsert", map[string]any{"id": 2, "payload": "TWO"})})) + assert.Equal(t, 2, cat.calls, "a clean conflict must force a second attempt") + + final := cat.snapshot() + assert.Equal(t, map[int64]string{1: "one", 2: "TWO", 3: "three"}, scanRows(t, ctx, final)) + + // On-disk parquet must be exactly the seed files (in the before-snapshot, so + // protected) plus the files the winning snapshot references (disjoint from the + // seed, since the overwrite rewrote them into fresh files). If attempt 1's + // files had leaked, the count would be strictly larger. + referenced := comm.referencedDataFilePaths(ctx) + require.NotEmpty(t, referenced) + assert.Equal(t, seedCount+len(referenced), countParquetFiles(t, final.Location()), + "attempt 1's orphaned parquet must have been cleaned even though the overall commit succeeded") +} + +// TestCommitOverwritePreservesFilesOnTerminalUnknown (T-6, CORR-2) proves the +// durability-critical skip: when a copy-on-write commit exhausts its retries with +// ErrCommitStateUnknown, commitOverwrite must return the wrapped unknown error AND +// leave the written parquet files in place. A possibly-landed commit's files may +// belong to a snapshot that committed server-side, so deleting them could corrupt +// the table — they are left for Iceberg orphan-file maintenance instead. +func TestCommitOverwritePreservesFilesOnTerminalUnknown(t *testing.T) { + ctx := t.Context() + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}, + iceberg.NestedField{ID: 2, Name: "payload", Type: iceberg.PrimitiveTypes.String, Required: false}, + ) + seedTbl, mem := newCOWTable(t, sc) + seedTbl = appendCOWRows(t, ctx, seedTbl, map[int64]string{1: "one", 2: "two", 3: "three"}) + seedCount := countParquetFiles(t, seedTbl.Location()) + + // Every attempt returns ErrCommitStateUnknown WITHOUT landing, so the commit-id + // never appears on reload and the loop runs to exhaustion. (A landed unknown + // would instead be detected by the commit-id and short-circuit to success, so + // it could not exhaust — hence commitUnknownNoLand here.) + const maxRetries = 3 + outcomes := make([]commitOutcome, maxRetries) + for i := range outcomes { + outcomes[i] = commitUnknownNoLand + } + cat := &scriptedCatalog{memCatalog: mem, outcomes: outcomes} + comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: maxRetries}, + func(context.Context) (*table.Table, error) { return cat.snapshot(), nil }, service.MockResources().Logger()) + require.NoError(t, err) + defer comm.Close() + w := cowWriter(t, cat.snapshot(), "id") + w.committer = comm + + err = w.Write(ctx, service.MessageBatch{cowMsg(t, "upsert", map[string]any{"id": 2, "payload": "TWO"})}) + require.Error(t, err) + assert.ErrorIs(t, err, rest.ErrCommitStateUnknown, "the terminal error must be ErrCommitStateUnknown") + assert.Equal(t, maxRetries, cat.calls, "the commit must exhaust every retry") + assert.Greater(t, countParquetFiles(t, seedTbl.Location()), seedCount, + "the overwrite's parquet files must be preserved (NOT cleaned) on a possibly-landed unknown state") +} + +// TestCommitOverwriteResumesAfterReloadFailures (T-12, CORR-3) proves the commit-id +// idempotency check resumes correctly even when the reload after a lost-ack +// conflict fails several times before recovering. Attempt 1 lands the overwrite +// server-side but reports a conflict; the next reloads fail, and each retry cleanly +// conflicts (nothing lands) so no double-apply is possible. Once a reload finally +// succeeds, the token is found in the reloaded snapshot and the commit returns +// success — the mutation lands exactly once, with no duplicate rows. +func TestCommitOverwriteResumesAfterReloadFailures(t *testing.T) { + ctx := t.Context() + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}, + iceberg.NestedField{ID: 2, Name: "payload", Type: iceberg.PrimitiveTypes.String, Required: false}, + ) + seedTbl, mem := newCOWTable(t, sc) + _ = appendCOWRows(t, ctx, seedTbl, map[int64]string{1: "one", 2: "two", 3: "three"}) + + // attempt 1 lands but reports ErrCommitFailed; later attempts cleanly conflict. + cat := &scriptedCatalog{memCatalog: mem, outcomes: []commitOutcome{commitLandThenFail, commitConflict, commitConflict}} + + // Reload fails its first two calls, then recovers. Commits are serialized under + // the committer's lock and reload runs inside it, so a plain counter is safe. + var reloadCalls int + reload := func(context.Context) (*table.Table, error) { + reloadCalls++ + if reloadCalls <= 2 { + return nil, errors.New("catalog reload unavailable") + } + return cat.snapshot(), nil + } + comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 5}, reload, service.MockResources().Logger()) + require.NoError(t, err) + defer comm.Close() + w := cowWriter(t, cat.snapshot(), "id") + w.committer = comm + + require.NoError(t, w.Write(ctx, service.MessageBatch{cowMsg(t, "upsert", map[string]any{"id": 2, "payload": "TWO"})})) + + final := cat.snapshot() + assert.Equal(t, 1, countSnapshotsWithCommitID(final), + "the mutation must land exactly once even though the first reloads failed") + assert.Equal(t, map[int64]string{1: "one", 2: "TWO", 3: "three"}, scanRows(t, ctx, final), + "no duplicate rows: the landed commit was detected once reload recovered") +} + +// nonListableFS forwards reads and writes to the local filesystem but deliberately +// omits WalkDir, so it satisfies iceio.IO / WriteFileIO but NOT iceio.ListableIO. +// It lets a copy-on-write commit write real parquet while forcing the orphan- +// cleanup path down its can't-list branch. +type nonListableFS struct{ inner iceio.LocalFS } + +func (f nonListableFS) Open(name string) (iceio.File, error) { return f.inner.Open(name) } +func (f nonListableFS) Create(name string) (iceio.FileWriter, error) { return f.inner.Create(name) } +func (f nonListableFS) WriteFile(name string, p []byte) error { return f.inner.WriteFile(name, p) } +func (f nonListableFS) Remove(name string) error { return f.inner.Remove(name) } + +// TestCommitOverwriteGracefulWithoutListableFS (T-14) proves graceful degradation +// when the filesystem cannot be listed: a failing copy-on-write commit must return +// its error without panicking, and orphan cleanup is silently skipped (dataFilePaths +// returns nil for a non-listable FS, so the before-snapshot guard short-circuits) +// rather than attempting a WalkDir it cannot perform. +func TestCommitOverwriteGracefulWithoutListableFS(t *testing.T) { + ctx := t.Context() + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}, + iceberg.NestedField{ID: 2, Name: "payload", Type: iceberg.PrimitiveTypes.String, Required: false}, + ) + // Seed via the normal (listable) handle so real data files exist to rewrite. + seedTbl, mem := newCOWTable(t, sc) + seedTbl = appendCOWRows(t, ctx, seedTbl, map[int64]string{1: "one", 2: "two", 3: "three"}) + seedCount := countParquetFiles(t, seedTbl.Location()) + + // A catalog that always fails the commit with a non-retryable error, and a + // table handle whose FS is writable but non-listable. + fc := &flakyCatalog{memCatalog: mem, failuresLeft: 1 << 30, failErr: errors.New("storage unavailable")} + nlSnap := table.New(fc.ident, fc.meta, fc.metadataLocation, + func(context.Context) (iceio.IO, error) { return nonListableFS{}, nil }, fc) + comm, err := NewCommitter(nlSnap, CommitConfig{MaxRetries: 2}, + func(context.Context) (*table.Table, error) { return nlSnap, nil }, service.MockResources().Logger()) + require.NoError(t, err) + defer comm.Close() + w := cowWriter(t, nlSnap, "id") + w.committer = comm + + err = w.Write(ctx, service.MessageBatch{cowMsg(t, "upsert", map[string]any{"id": 2, "payload": "TWO"})}) + require.Error(t, err, "the failing commit must surface an error, not panic") + + // Cleanup was skipped (FS not listable), so the overwrite's parquet remains. + assert.Greater(t, countParquetFiles(t, seedTbl.Location()), seedCount, + "a non-listable FS must skip cleanup (no WalkDir), leaving the written files in place") +} + +// TestCleanupOverwriteReferenceGuard (T-15) proves cleanup never deletes a file the +// current snapshot still references, even when that file "appeared since" the +// before-snapshot. With an empty before set every file counts as appeared-since, so +// only the referenced[p] guard can protect the live seed files; a genuinely +// unreferenced orphan is still removed. +func TestCleanupOverwriteReferenceGuard(t *testing.T) { + ctx := t.Context() + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}, + iceberg.NestedField{ID: 2, Name: "payload", Type: iceberg.PrimitiveTypes.String, Required: false}, + ) + seedTbl, cat := newCOWTable(t, sc) + seedTbl = appendCOWRows(t, ctx, seedTbl, map[int64]string{1: "one", 2: "two"}) + + comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 1}, reloadFn(cat), service.MockResources().Logger()) + require.NoError(t, err) + defer comm.Close() + + referenced := comm.referencedDataFilePaths(ctx) + require.NotEmpty(t, referenced, "the seeded snapshot must reference at least one data file") + + // A genuine orphan under data/ that no snapshot references. + orphan := filepath.Join(seedTbl.Location(), "data", "orphan-"+uuid.NewString()+".parquet") + require.NoError(t, os.WriteFile(orphan, []byte("not a real parquet"), 0o644)) + + // Empty before set: only the reference guard can save the live seed files. + comm.cleanupOrphanedOverwriteFiles(ctx, map[string]struct{}{}) + + for p := range referenced { + _, statErr := os.Stat(p) + assert.NoError(t, statErr, "a file referenced by the current snapshot must survive cleanup: %s", p) + } + _, statErr := os.Stat(orphan) + assert.True(t, os.IsNotExist(statErr), "an unreferenced orphan must be removed") +} + +// TestCommitOverwriteReturnsNewReaderError (T-17) proves a factory error from +// OverwriteInput.NewReader is surfaced by commitOverwrite (the stage fails before +// any file is written), and because that error is not an ambiguous unknown state, +// the cleanup path runs — with no new files written it leaves the seed untouched. +func TestCommitOverwriteReturnsNewReaderError(t *testing.T) { + ctx := t.Context() + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}, + iceberg.NestedField{ID: 2, Name: "payload", Type: iceberg.PrimitiveTypes.String, Required: false}, + ) + seedTbl, cat := newCOWTable(t, sc) + seedTbl = appendCOWRows(t, ctx, seedTbl, map[int64]string{1: "one", 2: "two"}) + seedCount := countParquetFiles(t, seedTbl.Location()) + + comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) + require.NoError(t, err) + defer comm.Close() + + readerErr := errors.New("reader factory boom") + err = comm.commitOverwrite(ctx, OverwriteInput{ + Filter: nil, // unused: NewReader errors before the filter is applied + NewReader: func() (array.RecordReader, error) { return nil, readerErr }, + SchemaID: comm.currentSchemaID(), + }) + require.ErrorIs(t, err, readerErr, "the NewReader factory error must propagate") + assert.Equal(t, seedCount, countParquetFiles(t, seedTbl.Location()), + "cleanup runs on a non-unknown failure; with no new files written the seed is untouched") +} + // assertAllManifestsData asserts every manifest in the current snapshot is // data-content (no delete manifests), i.e. the table holds only plain data // files. diff --git a/internal/impl/iceberg/icebergx/parquet.go b/internal/impl/iceberg/icebergx/parquet.go index 87c9172a1d..1f8ff84831 100644 --- a/internal/impl/iceberg/icebergx/parquet.go +++ b/internal/impl/iceberg/icebergx/parquet.go @@ -1,5 +1,5 @@ /* - * Copyright 2025 Redpanda Data, Inc. + * Copyright 2026 Redpanda Data, Inc. * * Licensed as a Redpanda Enterprise file under the Redpanda Community * License (the "License"); you may not use this file except in compliance with @@ -145,9 +145,18 @@ func icebergTypeToParquet(t iceberg.Type) (parquet.Node, error) { case iceberg.TimeType: return parquet.Time(parquet.Microsecond), nil case iceberg.TimestampType: - return parquet.Timestamp(parquet.Microsecond), nil + // A no-timezone Iceberg `timestamp` must be written with the parquet + // logical-type annotation isAdjustedToUTC=false (per the Iceberg spec). + // parquet.Timestamp defaults this to true, which would round-trip back + // through iceberg-go as `timestamptz` and break copy-on-write file + // rewrites (the strict rewrite visitor refuses timestamptz -> timestamp). + // This mirrors iceberg-go's own Arrow writer, which encodes a no-tz + // timestamp with an empty Arrow time zone (isAdjustedToUTC=false). + return parquet.TimestampAdjusted(parquet.Microsecond, false), nil case iceberg.TimestampTzType: - return parquet.Timestamp(parquet.Microsecond), nil + // A `timestamptz` is UTC-adjusted: isAdjustedToUTC=true (parquet.Timestamp's + // default). iceberg-go reads this back as arrow timestamp[tz=UTC] -> timestamptz. + return parquet.TimestampAdjusted(parquet.Microsecond, true), nil case iceberg.UUIDType: return parquet.UUID(), nil case iceberg.DecimalType: diff --git a/internal/impl/iceberg/icebergx/parquet_test.go b/internal/impl/iceberg/icebergx/parquet_test.go index 6ee5ef8903..02d7539ecd 100644 --- a/internal/impl/iceberg/icebergx/parquet_test.go +++ b/internal/impl/iceberg/icebergx/parquet_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2025 Redpanda Data, Inc. + * Copyright 2026 Redpanda Data, Inc. * * Licensed as a Redpanda Enterprise file under the Redpanda Community * License (the "License"); you may not use this file except in compliance with @@ -18,6 +18,36 @@ import ( "github.com/stretchr/testify/require" ) +// TestIcebergTimestampParquetAnnotation pins the parquet logical-type annotation +// for the two Iceberg timestamp types. A no-timezone `timestamp` must be written +// with isAdjustedToUTC=false and `timestamptz` with isAdjustedToUTC=true, so that +// iceberg-go reads each back as the matching Arrow/Iceberg type (an empty vs. UTC +// time zone). Getting `timestamp` wrong (the historical default of true) makes it +// round-trip as `timestamptz` and breaks copy-on-write file rewrites. +func TestIcebergTimestampParquetAnnotation(t *testing.T) { + cases := []struct { + name string + typ iceberg.Type + wantAdjustedTZ bool + }{ + {"timestamp", iceberg.TimestampType{}, false}, + {"timestamptz", iceberg.TimestampTzType{}, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + node, err := icebergTypeToParquet(tc.typ) + require.NoError(t, err) + + lt := node.Type().LogicalType() + require.NotNil(t, lt, "expected a logical type annotation") + require.NotNil(t, lt.Timestamp, "expected a TIMESTAMP logical type") + assert.Equal(t, tc.wantAdjustedTZ, lt.Timestamp.IsAdjustedToUTC, + "isAdjustedToUTC annotation mismatch for %s", tc.name) + require.NotNil(t, lt.Timestamp.Unit.Micros, "expected microsecond precision") + }) + } +} + func TestBuildParquetSchema_SimpleFlat(t *testing.T) { // Schema: { id: int64, name: string } schema := iceberg.NewSchema(1, diff --git a/internal/impl/iceberg/integration/cow_delete_and_move_integration_test.go b/internal/impl/iceberg/integration/cow_delete_and_move_integration_test.go new file mode 100644 index 0000000000..2f5f390761 --- /dev/null +++ b/internal/impl/iceberg/integration/cow_delete_and_move_integration_test.go @@ -0,0 +1,155 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md + +package iceberg + +import ( + "context" + "fmt" + "testing" + + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/catalog" + "github.com/apache/iceberg-go/table" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/redpanda-data/benthos/v4/public/service" + "github.com/redpanda-data/benthos/v4/public/service/integration" + + icebergimpl "github.com/redpanda-data/connect/v4/internal/impl/iceberg" +) + +// TestCOWDeleteOnlyBatchIntegration drives a mutating batch that is ALL deletes +// through copy-on-write, exercising commitOverwrite's txn.Delete branch +// (input.NewReader == nil) end-to-end against a real catalog. +// +// A delete-only copy-on-write mutation rewrites the surviving rows of every +// touched data file and commits a data-only snapshot; iceberg-go stamps this as +// an OpDelete snapshot (not OpOverwrite — only txn.Overwrite yields the latter), +// so this test asserts OpDelete while still requiring the copy-on-write +// invariant of zero delete-content manifests. DuckDB confirms the surviving row. +func TestCOWDeleteOnlyBatchIntegration(t *testing.T) { + integration.CheckSkip(t) + ctx := context.Background() + infra := setupTestInfra(t, ctx) + + const ns, tbl = "cow_delonly_ns", "cow_delonly_test" + infra.CreateNamespace(t, ns) + + client := infra.NewCatalogClient(t, ns) + _, err := client.CreateTable(ctx, tbl, iceberg.NewSchema(1, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.StringType{}, Required: true}, + iceberg.NestedField{ID: 2, Name: "value", Type: iceberg.StringType{}, Required: false}, + )) + require.NoError(t, err) + + operation, err := service.NewInterpolatedString(`${! meta("op") }`) + require.NoError(t, err) + router := infra.NewRouter(t, ns, tbl, + WithRowOperation(icebergimpl.RowOpConfig{ + Operation: operation, + IdentifierFields: []string{"id"}, + MergeStrategy: icebergimpl.MergeStrategyCOW, + })) + + // Seed three rows in one data file. + produceMessages(t, ctx, router, service.MessageBatch{ + opStructMsg("insert", map[string]any{"id": "1", "value": "one"}), + opStructMsg("insert", map[string]any{"id": "2", "value": "two"}), + opStructMsg("insert", map[string]any{"id": "3", "value": "three"}), + }) + + // One batch of ONLY deletes: remove id=2 and id=3. No rows to write, so + // writeCOW takes the delete-only branch (txn.Delete over the filter). + produceMessages(t, ctx, router, service.MessageBatch{ + opStructMsg("delete", map[string]any{"id": "2"}), + opStructMsg("delete", map[string]any{"id": "3"}), + }) + + type row struct { + ID string `json:"id"` + Value string `json:"value"` + } + rows := querySQL[row](t, ctx, infra, + fmt.Sprintf(`SELECT id, value FROM iceberg_cat."%s"."%s" ORDER BY id;`, ns, tbl)) + require.Len(t, rows, 1, "only id=1 must survive an all-deletes batch") + assert.Equal(t, row{"1", "one"}, rows[0]) + + // Copy-on-write delete-only invariant: zero delete manifests, OpDelete op. + assertCOWSnapshot(t, ctx, infra, ns, tbl, table.OpDelete) +} + +// TestCOWCrossPartitionKeyMoveIntegration verifies that a copy-on-write upsert +// which changes an existing key's partition value MOVES the row across +// partitions: the old-partition copy must be gone and the row must land in the +// new partition. This is distinct from inserting a brand-new key, and is only +// tractable under copy-on-write because the merge key (id) is not the partition +// column (region) — the rewrite deletes the old row by filter across all +// partitions and re-appends it routed by its new partition value. +func TestCOWCrossPartitionKeyMoveIntegration(t *testing.T) { + integration.CheckSkip(t) + ctx := context.Background() + infra := setupTestInfra(t, ctx) + + const ns, tbl = "cow_move_ns", "cow_move_test" + infra.CreateNamespace(t, ns) + + client := infra.NewCatalogClient(t, ns) + sc := iceberg.NewSchema(1, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.StringType{}, Required: true}, + iceberg.NestedField{ID: 2, Name: "region", Type: iceberg.StringType{}, Required: true}, + iceberg.NestedField{ID: 3, Name: "value", Type: iceberg.StringType{}, Required: false}, + ) + spec := iceberg.NewPartitionSpec(iceberg.PartitionField{ + SourceIDs: []int{2}, FieldID: 1000, Name: "region", Transform: iceberg.IdentityTransform{}, + }) + _, err := client.CreateTable(ctx, tbl, sc, catalog.WithPartitionSpec(&spec)) + require.NoError(t, err) + + operation, err := service.NewInterpolatedString(`${! meta("op") }`) + require.NoError(t, err) + router := infra.NewRouter(t, ns, tbl, + WithRowOperation(icebergimpl.RowOpConfig{ + Operation: operation, + IdentifierFields: []string{"id"}, // merge key is NOT the partition column + MergeStrategy: icebergimpl.MergeStrategyCOW, + })) + + // Seed: id=1 in us, id=2 in eu. + produceMessages(t, ctx, router, service.MessageBatch{ + opStructMsg("insert", map[string]any{"id": "1", "region": "us", "value": "one"}), + opStructMsg("insert", map[string]any{"id": "2", "region": "eu", "value": "two"}), + }) + + // Upsert id=1 changing its region us -> eu: the SAME key moves partitions. + produceMessages(t, ctx, router, service.MessageBatch{ + opStructMsg("upsert", map[string]any{"id": "1", "region": "eu", "value": "one-moved"}), + }) + + // Full state: id=1 now in eu, id=2 still in eu, and NO id=1 left in us. + type row struct { + ID string `json:"id"` + Region string `json:"region"` + Value string `json:"value"` + } + rows := querySQL[row](t, ctx, infra, + fmt.Sprintf(`SELECT id, region, value FROM iceberg_cat."%s"."%s" ORDER BY id;`, ns, tbl)) + require.Len(t, rows, 2, "id=1 must move (not duplicate); id=2 untouched") + assert.Equal(t, row{"1", "eu", "one-moved"}, rows[0], "id=1 must now live in eu with the new value") + assert.Equal(t, row{"2", "eu", "two"}, rows[1]) + + // The old-partition copy must be gone: no id=1 remains in us. + usID1 := querySQL[countResult](t, ctx, infra, + fmt.Sprintf(`SELECT COUNT(*) AS count FROM iceberg_cat."%s"."%s" WHERE region = 'us';`, ns, tbl)) + require.Len(t, usID1, 1) + assert.Equal(t, 0, usID1[0].Count, "the old us-partition copy of id=1 must be gone after the move") + + // Copy-on-write invariant. + assertCOWSnapshot(t, ctx, infra, ns, tbl, table.OpOverwrite) +} diff --git a/internal/impl/iceberg/integration/cow_format_version_integration_test.go b/internal/impl/iceberg/integration/cow_format_version_integration_test.go new file mode 100644 index 0000000000..b804cfa392 --- /dev/null +++ b/internal/impl/iceberg/integration/cow_format_version_integration_test.go @@ -0,0 +1,105 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md + +package iceberg + +import ( + "context" + "fmt" + "testing" + + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/catalog" + "github.com/apache/iceberg-go/table" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/redpanda-data/benthos/v4/public/service" + "github.com/redpanda-data/benthos/v4/public/service/integration" + + icebergimpl "github.com/redpanda-data/connect/v4/internal/impl/iceberg" +) + +// TestCOWFormatVersion1Integration creates a format-version-1 table through the +// REST catalog and runs a copy-on-write upsert+delete against it, asserting the +// table is NOT force-upgraded to v2. +// +// This is the whole point of copy-on-write for legacy tables: it only ever +// writes plain data files (no Iceberg v2 delete files), so it can operate on a +// v1 table and must leave it at v1. The merge-on-read path, by contrast, +// requires equality-delete files and so irreversibly upgrades v1 -> v2. The +// production router already wires this: it sets CommitConfig.SkipFormatUpgrade +// whenever merge_strategy is copy-on-write (router.go), and the test harness +// exercises that same production NewRouter, so no test-only wiring is needed +// here — this test guards that behaviour end-to-end. +// +// Asserted: the loaded table is still v1, DuckDB reads the correct final state, +// and there are zero delete files. +func TestCOWFormatVersion1Integration(t *testing.T) { + integration.CheckSkip(t) + ctx := context.Background() + infra := setupTestInfra(t, ctx) + + const ns, tbl = "cow_v1_ns", "cow_v1_test" + infra.CreateNamespace(t, ns) + + // Create an explicit format-version-1 table via the REST catalog. + client := infra.NewCatalogClient(t, ns) + created, err := client.CreateTable(ctx, tbl, + iceberg.NewSchema(1, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.StringType{}, Required: true}, + iceberg.NestedField{ID: 2, Name: "value", Type: iceberg.StringType{}, Required: false}, + ), + catalog.WithProperties(iceberg.Properties{table.PropertyFormatVersion: "1"}), + ) + require.NoError(t, err) + require.Equal(t, 1, created.Metadata().Version(), "table must be created at format version 1") + + operation, err := service.NewInterpolatedString(`${! meta("op") }`) + require.NoError(t, err) + router := infra.NewRouter(t, ns, tbl, + WithRowOperation(icebergimpl.RowOpConfig{ + Operation: operation, + IdentifierFields: []string{"id"}, + MergeStrategy: icebergimpl.MergeStrategyCOW, + })) + + // Seed three rows (append fast path). + produceMessages(t, ctx, router, service.MessageBatch{ + opStructMsg("insert", map[string]any{"id": "1", "value": "one"}), + opStructMsg("insert", map[string]any{"id": "2", "value": "two"}), + opStructMsg("insert", map[string]any{"id": "3", "value": "three"}), + }) + + // One mutating batch: upsert id=2, delete id=3 — a copy-on-write overwrite. + produceMessages(t, ctx, router, service.MessageBatch{ + opStructMsg("upsert", map[string]any{"id": "2", "value": "two-updated"}), + opStructMsg("delete", map[string]any{"id": "3"}), + }) + + // The table must STILL be v1 — copy-on-write must not trigger the v1->v2 + // upgrade the merge-on-read path needs. + loaded, err := client.LoadTable(ctx, tbl) + require.NoError(t, err) + assert.Equal(t, 1, loaded.Metadata().Version(), + "copy-on-write must leave the table at format version 1 (no forced v2 upgrade)") + + // Correct final state via DuckDB. + type row struct { + ID string `json:"id"` + Value string `json:"value"` + } + rows := querySQL[row](t, ctx, infra, + fmt.Sprintf(`SELECT id, value FROM iceberg_cat."%s"."%s" ORDER BY id;`, ns, tbl)) + require.Len(t, rows, 2, "id=3 deleted, id=2 not duplicated") + assert.Equal(t, row{"1", "one"}, rows[0]) + assert.Equal(t, row{"2", "two-updated"}, rows[1]) + + // Copy-on-write invariant: zero delete files, overwrite op. + assertCOWSnapshot(t, ctx, infra, ns, tbl, table.OpOverwrite) +} diff --git a/internal/impl/iceberg/integration/cow_multifile_composite_integration_test.go b/internal/impl/iceberg/integration/cow_multifile_composite_integration_test.go new file mode 100644 index 0000000000..f6ba59a326 --- /dev/null +++ b/internal/impl/iceberg/integration/cow_multifile_composite_integration_test.go @@ -0,0 +1,194 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md + +package iceberg + +import ( + "context" + "fmt" + "testing" + + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/table" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/redpanda-data/benthos/v4/public/service" + "github.com/redpanda-data/benthos/v4/public/service/integration" + + icebergimpl "github.com/redpanda-data/connect/v4/internal/impl/iceberg" +) + +// TestCOWMultiFileAndCompositeIntegration bundles three lower-risk copy-on-write +// e2e checks that share one infra: +// +// - multifile: seed several separate data files, then a mutating batch that +// touches keys in only one file; the untouched files' rows must survive +// unchanged (a whole-file rewrite must be scoped to the matched files). +// - composite: a two-column merge key round trip, exercising buildCOWFilter's +// OR-of-per-tuple-ANDs filter shape rather than the single-column IN. +// - collapse: two upserts to the same key in one batch must yield a single +// (latest-wins) row, exercising the same-batch per-key collapse under the +// rewrite path. +// +// Each asserts the final state via DuckDB plus the copy-on-write invariant +// (zero delete manifests + overwrite operation). +func TestCOWMultiFileAndCompositeIntegration(t *testing.T) { + integration.CheckSkip(t) + ctx := context.Background() + infra := setupTestInfra(t, ctx) + + newRouter := func(t *testing.T, ns, tbl string, idFields ...string) *icebergimpl.Router { + operation, err := service.NewInterpolatedString(`${! meta("op") }`) + require.NoError(t, err) + return infra.NewRouter(t, ns, tbl, WithRowOperation(icebergimpl.RowOpConfig{ + Operation: operation, + IdentifierFields: idFields, + MergeStrategy: icebergimpl.MergeStrategyCOW, + })) + } + + t.Run("multifile", func(t *testing.T) { + const ns, tbl = "cow_multifile_ns", "cow_multifile_test" + infra.CreateNamespace(t, ns) + client := infra.NewCatalogClient(t, ns) + _, err := client.CreateTable(ctx, tbl, iceberg.NewSchema(1, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.StringType{}, Required: true}, + iceberg.NestedField{ID: 2, Name: "value", Type: iceberg.StringType{}, Required: false}, + )) + require.NoError(t, err) + + router := newRouter(t, ns, tbl, "id") + + // Three separate seed batches -> three separate data files. ids are + // zero-padded so their string min/max cleanly separates the files. + produceMessages(t, ctx, router, service.MessageBatch{ + opStructMsg("insert", map[string]any{"id": "01", "value": "one"}), + opStructMsg("insert", map[string]any{"id": "02", "value": "two"}), + opStructMsg("insert", map[string]any{"id": "03", "value": "three"}), + }) + produceMessages(t, ctx, router, service.MessageBatch{ + opStructMsg("insert", map[string]any{"id": "04", "value": "four"}), + opStructMsg("insert", map[string]any{"id": "05", "value": "five"}), + opStructMsg("insert", map[string]any{"id": "06", "value": "six"}), + }) + produceMessages(t, ctx, router, service.MessageBatch{ + opStructMsg("insert", map[string]any{"id": "07", "value": "seven"}), + opStructMsg("insert", map[string]any{"id": "08", "value": "eight"}), + opStructMsg("insert", map[string]any{"id": "09", "value": "nine"}), + }) + + // Mutate keys only in the middle file: upsert id=05, delete id=06. The + // first and third files must be left entirely intact. + produceMessages(t, ctx, router, service.MessageBatch{ + opStructMsg("upsert", map[string]any{"id": "05", "value": "five-updated"}), + opStructMsg("delete", map[string]any{"id": "06"}), + }) + + type row struct { + ID string `json:"id"` + Value string `json:"value"` + } + rows := querySQL[row](t, ctx, infra, + fmt.Sprintf(`SELECT id, value FROM iceberg_cat."%s"."%s" ORDER BY id;`, ns, tbl)) + require.Len(t, rows, 8, "id=06 deleted, id=05 updated in place, all others intact") + assert.Equal(t, []row{ + {"01", "one"}, + {"02", "two"}, + {"03", "three"}, + {"04", "four"}, + {"05", "five-updated"}, + {"07", "seven"}, + {"08", "eight"}, + {"09", "nine"}, + }, rows, "untouched files' rows must survive the scoped rewrite unchanged") + + assertCOWSnapshot(t, ctx, infra, ns, tbl, table.OpOverwrite) + }) + + t.Run("composite", func(t *testing.T) { + const ns, tbl = "cow_composite_ns", "cow_composite_test" + infra.CreateNamespace(t, ns) + client := infra.NewCatalogClient(t, ns) + _, err := client.CreateTable(ctx, tbl, iceberg.NewSchema(1, + iceberg.NestedField{ID: 1, Name: "tenant", Type: iceberg.StringType{}, Required: true}, + iceberg.NestedField{ID: 2, Name: "id", Type: iceberg.StringType{}, Required: true}, + iceberg.NestedField{ID: 3, Name: "val", Type: iceberg.StringType{}, Required: false}, + )) + require.NoError(t, err) + + router := newRouter(t, ns, tbl, "tenant", "id") + + produceMessages(t, ctx, router, service.MessageBatch{ + opStructMsg("insert", map[string]any{"tenant": "t1", "id": "x", "val": "a"}), + opStructMsg("insert", map[string]any{"tenant": "t1", "id": "y", "val": "b"}), + opStructMsg("insert", map[string]any{"tenant": "t2", "id": "x", "val": "c"}), + }) + // delete (t1,y); upsert (t2,x). (t1,x) untouched. The composite filter must + // distinguish (t1,x) from (t2,x) — an AND-of-INs would match the cross + // product and wrongly touch (t1,x). + produceMessages(t, ctx, router, service.MessageBatch{ + opStructMsg("delete", map[string]any{"tenant": "t1", "id": "y"}), + opStructMsg("upsert", map[string]any{"tenant": "t2", "id": "x", "val": "c2"}), + }) + + type row struct { + Tenant string `json:"tenant"` + ID string `json:"id"` + Val string `json:"val"` + } + rows := querySQL[row](t, ctx, infra, + fmt.Sprintf(`SELECT tenant, id, val FROM iceberg_cat."%s"."%s" ORDER BY tenant, id;`, ns, tbl)) + require.Len(t, rows, 2) + assert.Equal(t, row{"t1", "x", "a"}, rows[0], "(t1,x) must be untouched") + assert.Equal(t, row{"t2", "x", "c2"}, rows[1], "(t2,x) must be upserted, not confused with (t1,x)") + + assertCOWSnapshot(t, ctx, infra, ns, tbl, table.OpOverwrite) + }) + + t.Run("collapse", func(t *testing.T) { + const ns, tbl = "cow_collapse_ns", "cow_collapse_test" + infra.CreateNamespace(t, ns) + client := infra.NewCatalogClient(t, ns) + _, err := client.CreateTable(ctx, tbl, iceberg.NewSchema(1, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.StringType{}, Required: true}, + iceberg.NestedField{ID: 2, Name: "val", Type: iceberg.StringType{}, Required: false}, + )) + require.NoError(t, err) + + router := newRouter(t, ns, tbl, "id") + + // Seed k and g so the mutating batch genuinely rewrites existing rows and + // commits as an overwrite (an overwrite whose filter matches no existing + // data is optimised into a plain append by iceberg-go). + produceMessages(t, ctx, router, service.MessageBatch{ + opStructMsg("insert", map[string]any{"id": "k", "val": "seed"}), + opStructMsg("insert", map[string]any{"id": "g", "val": "seed"}), + }) + + // Two upserts of "k" plus an upsert-then-delete of "g", all in one batch. + // The per-key collapse must leave one row for k (latest wins) and none for g. + produceMessages(t, ctx, router, service.MessageBatch{ + opStructMsg("upsert", map[string]any{"id": "k", "val": "v1"}), + opStructMsg("upsert", map[string]any{"id": "k", "val": "v2"}), + opStructMsg("upsert", map[string]any{"id": "g", "val": "g1"}), + opStructMsg("delete", map[string]any{"id": "g"}), + }) + + type row struct { + ID string `json:"id"` + Val string `json:"val"` + } + rows := querySQL[row](t, ctx, infra, + fmt.Sprintf(`SELECT id, val FROM iceberg_cat."%s"."%s" ORDER BY id;`, ns, tbl)) + require.Len(t, rows, 1, "k must appear once (two same-batch upserts collapse); g must be deleted") + assert.Equal(t, row{"k", "v2"}, rows[0], "the later upsert of k must win") + + assertCOWSnapshot(t, ctx, infra, ns, tbl, table.OpOverwrite) + }) +} diff --git a/internal/impl/iceberg/integration/cow_nested_schema_integration_test.go b/internal/impl/iceberg/integration/cow_nested_schema_integration_test.go new file mode 100644 index 0000000000..05e27579ea --- /dev/null +++ b/internal/impl/iceberg/integration/cow_nested_schema_integration_test.go @@ -0,0 +1,150 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md + +package iceberg + +import ( + "context" + "fmt" + "testing" + + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/table" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/redpanda-data/benthos/v4/public/service" + "github.com/redpanda-data/benthos/v4/public/service/integration" + + icebergimpl "github.com/redpanda-data/connect/v4/internal/impl/iceberg" +) + +// TestCOWNestedSchemaIntegration pre-creates a table with a nested struct (that +// itself contains a nested int64 field), a list, and a map, seeds +// a row, then copy-on-write-upserts the same key with entirely new nested values +// and reads the result back through DuckDB — which unnests struct/list/map +// independently of iceberg-go. +// +// This closes the second-highest-risk gap: the recursive cowMassage projection +// is what re-encodes nested struct/list/map values into the JSON shape +// array.RecordFromJSON expects at every depth. All unit coverage of cowMassage +// reads back through iceberg-go's own Arrow scan (writer and reader are the same +// library), so a self-consistent-but-wrong nested encoding — most dangerously +// the historical silent truncation of an integer nested beyond 2^53, which +// cowMassage fixes by emitting integers as JSON strings at every leaf — would +// pass. DuckDB parses the parquet itself, so a wrong nested encoding surfaces +// here as a wrong nested value or a truncated nested int. +// +// Both the struct's `big` field and the map value struct's `score` field carry +// values > 2^53 to prove the nested-int fix end-to-end through an independent +// reader. +func TestCOWNestedSchemaIntegration(t *testing.T) { + integration.CheckSkip(t) + ctx := context.Background() + infra := setupTestInfra(t, ctx) + + const ns, tbl = "cow_nested_ns", "cow_nested_test" + infra.CreateNamespace(t, ns) + + // id (merge key), a nested struct, a list, and a map. + client := infra.NewCatalogClient(t, ns) + sc := iceberg.NewSchema(1, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.StringType{}, Required: true}, + iceberg.NestedField{ID: 2, Name: "info", Required: false, Type: &iceberg.StructType{ + FieldList: []iceberg.NestedField{ + {ID: 10, Name: "name", Type: iceberg.StringType{}, Required: false}, + {ID: 11, Name: "big", Type: iceberg.Int64Type{}, Required: false}, + }, + }}, + iceberg.NestedField{ID: 3, Name: "tags", Required: false, Type: &iceberg.ListType{ + ElementID: 20, Element: iceberg.StringType{}, ElementRequired: false, + }}, + iceberg.NestedField{ID: 4, Name: "attrs", Required: false, Type: &iceberg.MapType{ + KeyID: 30, KeyType: iceberg.StringType{}, + ValueID: 31, ValueRequired: false, + ValueType: &iceberg.StructType{FieldList: []iceberg.NestedField{ + {ID: 40, Name: "score", Type: iceberg.Int64Type{}, Required: false}, + }}, + }}, + ) + _, err := client.CreateTable(ctx, tbl, sc) + require.NoError(t, err) + + operation, err := service.NewInterpolatedString(`${! meta("op") }`) + require.NoError(t, err) + router := infra.NewRouter(t, ns, tbl, + WithRowOperation(icebergimpl.RowOpConfig{ + Operation: operation, + IdentifierFields: []string{"id"}, + MergeStrategy: icebergimpl.MergeStrategyCOW, + })) + + // Seed id=1 (append fast path). + produceMessages(t, ctx, router, service.MessageBatch{ + opStructMsg("insert", map[string]any{ + "id": "1", + "info": map[string]any{"name": "alice", "big": int64(9007199254740993)}, + "tags": []any{"a", "b"}, + "attrs": map[string]any{ + "x": map[string]any{"score": int64(100)}, + }, + }), + }) + + // Copy-on-write upsert of id=1 changing every nested value. The new nested + // ints are > 2^53 to prove the nested-int encoding survives an independent + // reader. + produceMessages(t, ctx, router, service.MessageBatch{ + opStructMsg("upsert", map[string]any{ + "id": "1", + "info": map[string]any{"name": "alice2", "big": int64(9007199254740995)}, + "tags": []any{"c", "d", "e"}, + "attrs": map[string]any{ + "y": map[string]any{"score": int64(9007199254740997)}, + }, + }), + }) + + // Read the nested values back through DuckDB, which unnests the struct/list/ + // map itself. Selecting the leaf columns flattens the projection so it parses + // into a Go struct. + type nestedRow struct { + ID string `json:"id"` + Name string `json:"name"` + Big int64 `json:"big"` + NTags int `json:"ntags"` + Tag0 string `json:"tag0"` + AKey string `json:"akey"` + AScore int64 `json:"ascore"` + } + rows := querySQL[nestedRow](t, ctx, infra, fmt.Sprintf(` + SELECT + id, + info.name AS name, + info.big AS big, + len(tags) AS ntags, + tags[1] AS tag0, + map_keys(attrs)[1] AS akey, + (map_values(attrs)[1]).score AS ascore + FROM iceberg_cat."%s"."%s";`, ns, tbl)) + + require.Len(t, rows, 1, "the upsert must replace id=1, not duplicate it") + got := rows[0] + assert.Equal(t, "1", got.ID) + assert.Equal(t, "alice2", got.Name, "nested struct string must be the upserted value") + assert.Equal(t, int64(9007199254740995), got.Big, + "nested int64 > 2^53 must survive the copy-on-write rewrite without truncation") + assert.Equal(t, 3, got.NTags, "list must hold the upserted three elements") + assert.Equal(t, "c", got.Tag0, "first list element must be the upserted value") + assert.Equal(t, "y", got.AKey, "map key must be the upserted key") + assert.Equal(t, int64(9007199254740997), got.AScore, + "nested int64 > 2^53 inside a map value must survive without truncation") + + // Copy-on-write invariant. + assertCOWSnapshot(t, ctx, infra, ns, tbl, table.OpOverwrite) +} diff --git a/internal/impl/iceberg/integration/cow_row_operation_types_integration_test.go b/internal/impl/iceberg/integration/cow_row_operation_types_integration_test.go new file mode 100644 index 0000000000..4638b0df3f --- /dev/null +++ b/internal/impl/iceberg/integration/cow_row_operation_types_integration_test.go @@ -0,0 +1,222 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md + +package iceberg + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/table" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/redpanda-data/benthos/v4/public/service" + "github.com/redpanda-data/benthos/v4/public/service/integration" + + icebergimpl "github.com/redpanda-data/connect/v4/internal/impl/iceberg" +) + +// assertCOWSnapshot loads the committed table through the REST catalog and +// asserts the copy-on-write invariant: at least one real data manifest, ZERO +// delete manifests (what makes the result readable by engine-backed catalogs), +// and the current snapshot's operation equals wantOp. It is the shared +// end-of-test check reused by the copy-on-write e2e tests in this package. +func assertCOWSnapshot(t *testing.T, ctx context.Context, infra *testInfrastructure, ns, tbl string, wantOp table.Operation) { + t.Helper() + client := infra.NewCatalogClient(t, ns) + loaded, err := client.LoadTable(ctx, tbl) + require.NoError(t, err) + dataManifests, deleteManifests := countManifestsByContent(t, ctx, loaded) + assert.Positive(t, dataManifests, "expected at least one data manifest to inspect") + assert.Zero(t, deleteManifests, "copy-on-write must leave zero delete manifests") + require.NotNil(t, loaded.CurrentSnapshot()) + assert.Equal(t, wantOp, loaded.CurrentSnapshot().Summary.Operation, + "unexpected snapshot operation under copy-on-write") +} + +// TestCOWRowOperationKeyTypesIntegration is the copy-on-write analog of +// TestRowOperationKeyTypesIntegration: for every supported non-string merge-key +// type it pre-creates a table keyed on that type, seeds two rows, then runs a +// single mutating batch (upsert k1 in place + delete k2) under +// merge_strategy: copy-on-write and asserts — via DuckDB, an INDEPENDENT reader +// of the committed table — the exact final row set. +// +// This is the highest-value gap closed by these tests. The copy-on-write filter +// literal (cowKeyLiteral) and the rewrite's re-encoding of the surviving key +// (cowMassage -> deleteKeyJSONValue -> Arrow) are the riskiest code: a wrong +// encoding makes the overwrite filter select no rows, so the delete/upsert +// silently becomes a no-op (the CON-490 hazard) or the rewritten key is +// corrupted. All existing unit round-trips read back through iceberg-go's own +// Arrow scan, so a self-consistent-but-wrong encoding would pass. DuckDB reads +// the parquet + manifests itself, so a wrong encoding shows up here as a wrong +// row count, a stale row surviving, or a key that fails DuckDB's own typed +// equality against the literal we expect. +// +// The int64 case deliberately uses values > 2^53 (not representable exactly as +// float64) to prove the integer-as-string encoding survives the rewrite. The +// per-type WHERE ... = query asks DuckDB to confirm the +// surviving key equals the exact value we upserted, using DuckDB's own type +// system rather than iceberg-go's. +// +// Note: boolean and decimal are GATED as copy-on-write merge keys (the vendored +// iceberg-go overwrite filter cannot apply either — see cowKeyLiteral), so they +// are intentionally not exercised as keys here. +func TestCOWRowOperationKeyTypesIntegration(t *testing.T) { + integration.CheckSkip(t) + ctx := context.Background() + infra := setupTestInfra(t, ctx) + const ns = "cow_keytypes" + infra.CreateNamespace(t, ns) + + cases := []struct { + name string + tbl string + keyType iceberg.Type + k1, k2 any + // lit is the DuckDB typed literal equal to k1 (the surviving/upserted + // key). The final row must satisfy k = lit under DuckDB's own typing. + lit string + // skip, when non-empty, documents a genuine defect this case surfaces and + // keeps the case visible without failing the suite (see the comment on the + // timestamp case below). + skip string + }{ + { + name: "int64-big", + tbl: "k_int64", + keyType: iceberg.Int64Type{}, + // 2^53+1 and 2^53+2: exact as int64, NOT exact as float64. + k1: int64(9007199254740993), + k2: int64(9007199254740994), + lit: "9007199254740993", + }, + { + name: "date", + tbl: "k_date", + keyType: iceberg.DateType{}, + k1: time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC), + k2: time.Date(2024, 1, 16, 0, 0, 0, 0, time.UTC), + lit: "DATE '2024-01-15'", + }, + { + name: "time", + tbl: "k_time", + keyType: iceberg.TimeType{}, + // UTC time-of-day; the copy-on-write encoder canonicalises in UTC so + // filter and storage agree. + k1: time.Date(1970, 1, 1, 12, 30, 45, 0, time.UTC), + k2: time.Date(1970, 1, 1, 13, 30, 45, 0, time.UTC), + lit: "TIME '12:30:45'", + }, + { + name: "timestamp", + tbl: "k_ts", + keyType: iceberg.TimestampType{}, + k1: time.Date(2024, 1, 15, 12, 30, 45, 0, time.UTC), + k2: time.Date(2024, 1, 16, 12, 30, 45, 0, time.UTC), + lit: "TIMESTAMP '2024-01-15 12:30:45'", + // Previously skipped: a copy-on-write upsert/delete on a table with a + // no-timezone `timestamp` column used to fail at commit with + // "failed to rewrite file ...: cannot promote timestamptz to timestamp". + // The append path (icebergx/parquet.go) wrote no-tz `timestamp` columns + // with the parquet annotation isAdjustedToUTC=true, so iceberg-go read the + // existing file back as `timestamptz` and the copy-on-write rewrite's + // strict schema visitor refused to promote it to the table's declared + // no-tz `timestamp`. Fixed by writing no-tz `timestamp` with + // isAdjustedToUTC=false, matching iceberg-go's own Arrow writer. NOTE: this + // only fixes tables whose data files were written after the fix; a table + // with pre-existing old-encoding (isAdjustedToUTC=true) files still fails + // copy-on-write and would need its data rewritten. + }, + { + name: "timestamptz", + tbl: "k_tstz", + keyType: iceberg.TimestampTzType{}, + k1: time.Date(2024, 1, 15, 12, 30, 45, 0, time.UTC), + k2: time.Date(2024, 1, 16, 12, 30, 45, 0, time.UTC), + lit: "TIMESTAMPTZ '2024-01-15 12:30:45+00'", + }, + { + name: "uuid", + tbl: "k_uuid", + keyType: iceberg.UUIDType{}, + k1: "f47ac10b-58cc-4372-a567-0e02b2c3d479", + k2: "1b4e28ba-2fa1-11d2-883f-0016d3cca427", + lit: "UUID 'f47ac10b-58cc-4372-a567-0e02b2c3d479'", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if tc.skip != "" { + t.Skip(tc.skip) + } + // Pre-create the table keyed on the typed column. The schema carries no + // identifier-field-ids: under copy-on-write, identifier_fields are the + // connector-side merge key only. + client := infra.NewCatalogClient(t, ns) + _, err := client.CreateTable(ctx, tc.tbl, iceberg.NewSchema(1, + iceberg.NestedField{ID: 1, Name: "k", Type: tc.keyType, Required: true}, + iceberg.NestedField{ID: 2, Name: "val", Type: iceberg.StringType{}, Required: false}, + )) + require.NoError(t, err) + + operation, err := service.NewInterpolatedString(`${! meta("op") }`) + require.NoError(t, err) + router := infra.NewRouter(t, ns, tc.tbl, + WithRowOperation(icebergimpl.RowOpConfig{ + Operation: operation, + IdentifierFields: []string{"k"}, + MergeStrategy: icebergimpl.MergeStrategyCOW, + })) + + // Seed k1, k2 (append fast path — no keyed ops in this batch). + produceMessages(t, ctx, router, service.MessageBatch{ + opStructMsg("insert", map[string]any{"k": tc.k1, "val": "a"}), + opStructMsg("insert", map[string]any{"k": tc.k2, "val": "b"}), + }) + + // One mutating batch: upsert k1 (new value) and delete k2. Contains both + // a row to (re)write and keys to remove, so it lands as a single + // copy-on-write overwrite. A wrong key-literal encoding makes the filter + // match nothing: k2 would survive (count 2) or the upsert would duplicate + // k1 (count 2). + produceMessages(t, ctx, router, service.MessageBatch{ + opStructMsg("upsert", map[string]any{"k": tc.k1, "val": "a2"}), + opStructMsg("delete", map[string]any{"k": tc.k2}), + }) + + // (a) Exactly one surviving row (k2 deleted, k1 not duplicated), + // observed by DuckDB. Select the key column per the projection quirk. + type valRow struct { + Val string `json:"val"` + } + rows := querySQL[valRow](t, ctx, infra, + fmt.Sprintf(`SELECT k, val FROM iceberg_cat."%s"."%s";`, ns, tc.tbl)) + require.Lenf(t, rows, 1, "expected exactly one surviving row; got %d", len(rows)) + assert.Equal(t, "a2", rows[0].Val, "surviving row must be the upserted value") + + // (b) The surviving key equals the exact value we upserted, per DuckDB's + // own typed comparison. This is the load-bearing check for the >2^53 + // int64 case: a truncated re-encode would fail k = 9007199254740993. + match := querySQL[countResult](t, ctx, infra, + fmt.Sprintf(`SELECT COUNT(*) AS count FROM iceberg_cat."%s"."%s" WHERE k = %s AND val = 'a2';`, + ns, tc.tbl, tc.lit)) + require.Len(t, match, 1) + assert.Equal(t, 1, match[0].Count, + "DuckDB must find the surviving row keyed by the exact upserted value %s", tc.lit) + + // (c) Copy-on-write invariant: zero delete manifests + overwrite op. + assertCOWSnapshot(t, ctx, infra, ns, tc.tbl, table.OpOverwrite) + }) + } +} diff --git a/internal/impl/iceberg/integration/cow_schema_evolution_integration_test.go b/internal/impl/iceberg/integration/cow_schema_evolution_integration_test.go new file mode 100644 index 0000000000..4bed253b3a --- /dev/null +++ b/internal/impl/iceberg/integration/cow_schema_evolution_integration_test.go @@ -0,0 +1,106 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md + +package iceberg + +import ( + "context" + "fmt" + "testing" + + "github.com/apache/iceberg-go/table" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/redpanda-data/benthos/v4/public/service" + "github.com/redpanda-data/benthos/v4/public/service/integration" + + icebergimpl "github.com/redpanda-data/connect/v4/internal/impl/iceberg" +) + +// TestCOWSchemaEvolutionIntegration drives schema evolution through the +// copy-on-write mutation path: batch 1 seeds {id,value}; batch 2 upserts an +// existing key carrying an extra new column. The copy-on-write path detects the +// unknown column (cowDetectNewColumns), returns a schema-evolution error, and +// the router evolves the table and retries the overwrite. This proves the +// evolve-and-retry loop works for the whole-file-rewrite path, not only the +// append path. +// +// Asserted via DuckDB (an independent reader): +// - the table evolved (the new column is present and selectable), +// - the upserted row carries the new value, +// - the prior row is intact (its new column reads back null), +// - zero delete files and the mutation committed as an overwrite. +func TestCOWSchemaEvolutionIntegration(t *testing.T) { + integration.CheckSkip(t) + ctx := context.Background() + infra := setupTestInfra(t, ctx) + + const ns, tbl = "cow_evo_ns", "cow_evo_test" + infra.CreateNamespace(t, ns) + + operation, err := service.NewInterpolatedString(`${! meta("op") }`) + require.NoError(t, err) + router := infra.NewRouter(t, ns, tbl, + WithSchemaEvolution(icebergimpl.SchemaEvolutionConfig{Enabled: true}), + WithRowOperation(icebergimpl.RowOpConfig{ + Operation: operation, + IdentifierFields: []string{"id"}, + MergeStrategy: icebergimpl.MergeStrategyCOW, + })) + + // Batch 1: auto-create the table with {id, value} and seed two rows. id is a + // string so the auto-created column is a valid copy-on-write merge key. + produceMessages(t, ctx, router, service.MessageBatch{ + opStructMsg("insert", map[string]any{"id": "1", "value": "one"}), + opStructMsg("insert", map[string]any{"id": "2", "value": "two"}), + }) + + // Confirm the seeded schema has exactly {id, value} before evolution. + cols := querySQL[ColumnInfo](t, ctx, infra, + fmt.Sprintf(`DESCRIBE iceberg_cat."%s"."%s";`, ns, tbl)) + require.Len(t, cols, 2, "table should start with {id, value}") + + // Batch 2: upsert id=2 carrying a brand-new column `extra`. Under + // copy-on-write this is an overwrite (delete old id=2 + append the rewritten + // row); the new column forces a schema-evolution round trip first. + produceMessages(t, ctx, router, service.MessageBatch{ + opStructMsg("upsert", map[string]any{"id": "2", "value": "two-v2", "extra": "hello"}), + }) + + // The table must have evolved: `extra` is now present. + cols = querySQL[ColumnInfo](t, ctx, infra, + fmt.Sprintf(`DESCRIBE iceberg_cat."%s"."%s";`, ns, tbl)) + colNames := make(map[string]string, len(cols)) + for _, c := range cols { + colNames[c.ColumnName] = c.ColumnType + } + require.Contains(t, colNames, "extra", "table must have evolved to add the `extra` column") + + // Final state via DuckDB, selecting the evolved column too. + type row struct { + ID string `json:"id"` + Value string `json:"value"` + Extra *string `json:"extra"` + } + rows := querySQL[row](t, ctx, infra, + fmt.Sprintf(`SELECT id, value, extra FROM iceberg_cat."%s"."%s" ORDER BY id;`, ns, tbl)) + require.Len(t, rows, 2, "id=2 must be upserted in place, not duplicated") + + assert.Equal(t, "1", rows[0].ID) + assert.Equal(t, "one", rows[0].Value, "prior row must be intact") + assert.Nil(t, rows[0].Extra, "the pre-evolution row reads back null for the new column") + + assert.Equal(t, "2", rows[1].ID) + assert.Equal(t, "two-v2", rows[1].Value, "upsert must replace id=2's value") + require.NotNil(t, rows[1].Extra) + assert.Equal(t, "hello", *rows[1].Extra, "the evolved column value must read back via DuckDB") + + // Copy-on-write invariant. + assertCOWSnapshot(t, ctx, infra, ns, tbl, table.OpOverwrite) +} diff --git a/internal/impl/iceberg/integration/cow_temporal_data_column_integration_test.go b/internal/impl/iceberg/integration/cow_temporal_data_column_integration_test.go new file mode 100644 index 0000000000..3855213cb6 --- /dev/null +++ b/internal/impl/iceberg/integration/cow_temporal_data_column_integration_test.go @@ -0,0 +1,131 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md + +package iceberg + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/table" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/redpanda-data/benthos/v4/public/service" + "github.com/redpanda-data/benthos/v4/public/service/integration" + + icebergimpl "github.com/redpanda-data/connect/v4/internal/impl/iceberg" +) + +// TestCOWTemporalDataColumnIntegration proves the no-tz `timestamp` on-disk-format +// fix end-to-end for temporal DATA columns (as opposed to merge keys, which +// TestCOWRowOperationKeyTypesIntegration/timestamp covers). A single table carries +// BOTH a no-timezone `timestamp` column and a `timestamptz` column so DuckDB — an +// independent reader — can confirm the two are stored and typed distinctly. +// +// The flow: append two rows, then a single copy-on-write batch upserts id=1 (new +// temporal values in both columns) and deletes id=2. Before the fix, the append +// path wrote the no-tz `timestamp` column with the parquet annotation +// isAdjustedToUTC=true, so iceberg-go read the existing data file back as +// `timestamptz` and the copy-on-write file rewrite failed with "cannot promote +// timestamptz to timestamp". With the fix the no-tz column is written with +// isAdjustedToUTC=false and the rewrite succeeds. +// +// The assertions: +// - exactly one surviving row (id=2 deleted, id=1 not duplicated); +// - DuckDB matches the surviving row using a no-tz TIMESTAMP literal for the +// `ts` column and a TIMESTAMPTZ literal for the `tstz` column — i.e. the +// instant round-trips through the copy-on-write rewrite; +// - DuckDB reports the two columns with distinct SQL types (TIMESTAMP vs +// TIMESTAMP WITH TIME ZONE), proving the annotation is honoured on read. +func TestCOWTemporalDataColumnIntegration(t *testing.T) { + integration.CheckSkip(t) + ctx := context.Background() + infra := setupTestInfra(t, ctx) + + const ns, tbl = "cow_temporal_datacol_ns", "cow_temporal_datacol_test" + infra.CreateNamespace(t, ns) + + client := infra.NewCatalogClient(t, ns) + _, err := client.CreateTable(ctx, tbl, iceberg.NewSchema(1, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.StringType{}, Required: true}, + // A no-timezone `timestamp` data column: this is the column whose append + // encoding used to break copy-on-write. + iceberg.NestedField{ID: 2, Name: "ts", Type: iceberg.TimestampType{}, Required: false}, + // A `timestamptz` column in the SAME table, so DuckDB can show the two are + // typed distinctly and the regression case stays covered. + iceberg.NestedField{ID: 3, Name: "tstz", Type: iceberg.TimestampTzType{}, Required: false}, + )) + require.NoError(t, err) + + operation, err := service.NewInterpolatedString(`${! meta("op") }`) + require.NoError(t, err) + router := infra.NewRouter(t, ns, tbl, + WithRowOperation(icebergimpl.RowOpConfig{ + Operation: operation, + IdentifierFields: []string{"id"}, + MergeStrategy: icebergimpl.MergeStrategyCOW, + })) + + tsSeed := time.Date(2024, 1, 15, 12, 30, 45, 0, time.UTC) + tsUpsert := time.Date(2024, 3, 20, 8, 15, 0, 0, time.UTC) + + // Seed id=1, id=2 via the append fast path (no keyed ops in this batch). + produceMessages(t, ctx, router, service.MessageBatch{ + opStructMsg("insert", map[string]any{"id": "1", "ts": tsSeed, "tstz": tsSeed}), + opStructMsg("insert", map[string]any{"id": "2", "ts": tsSeed, "tstz": tsSeed}), + }) + + // One mutating batch: upsert id=1 with new temporal values and delete id=2. + // This lands as a single copy-on-write overwrite that must rewrite the + // surviving rows of the existing data file — the step that previously failed + // for the no-tz `timestamp` column. + produceMessages(t, ctx, router, service.MessageBatch{ + opStructMsg("upsert", map[string]any{"id": "1", "ts": tsUpsert, "tstz": tsUpsert}), + opStructMsg("delete", map[string]any{"id": "2"}), + }) + + // (a) Exactly one surviving row (id=2 deleted, id=1 not duplicated). + type idRow struct { + ID string `json:"id"` + } + rows := querySQL[idRow](t, ctx, infra, + fmt.Sprintf(`SELECT id FROM iceberg_cat."%s"."%s";`, ns, tbl)) + require.Lenf(t, rows, 1, "expected exactly one surviving row; got %d", len(rows)) + assert.Equal(t, "1", rows[0].ID, "surviving row must be the upserted id=1") + + // (b) The surviving row's temporal values round-tripped through the rewrite, + // per DuckDB's own typed comparison: a no-tz TIMESTAMP literal for `ts` and a + // TIMESTAMPTZ literal for `tstz`. + match := querySQL[countResult](t, ctx, infra, fmt.Sprintf( + `SELECT COUNT(*) AS count FROM iceberg_cat."%s"."%s" `+ + `WHERE ts = TIMESTAMP '2024-03-20 08:15:00' AND tstz = TIMESTAMPTZ '2024-03-20 08:15:00+00';`, + ns, tbl)) + require.Len(t, match, 1) + assert.Equal(t, 1, match[0].Count, + "DuckDB must find the surviving row at the exact upserted instant in both temporal columns") + + // (c) DuckDB reports the two columns with distinct SQL types: the no-tz column + // as TIMESTAMP and the tz column as TIMESTAMP WITH TIME ZONE. This is the + // direct proof that the isAdjustedToUTC annotation is honoured on read. + type typeRow struct { + TSType string `json:"ts_type"` + TSTZType string `json:"tstz_type"` + } + types := querySQL[typeRow](t, ctx, infra, fmt.Sprintf( + `SELECT typeof(ts) AS ts_type, typeof(tstz) AS tstz_type FROM iceberg_cat."%s"."%s";`, ns, tbl)) + require.Len(t, types, 1) + assert.Equal(t, "TIMESTAMP", types[0].TSType, "no-tz column must read back as a no-timezone TIMESTAMP") + assert.Equal(t, "TIMESTAMP WITH TIME ZONE", types[0].TSTZType, "tz column must read back as TIMESTAMP WITH TIME ZONE") + + // (d) Copy-on-write invariant: zero delete manifests + overwrite op. + assertCOWSnapshot(t, ctx, infra, ns, tbl, table.OpOverwrite) +} diff --git a/internal/impl/iceberg/output_iceberg_test.go b/internal/impl/iceberg/output_iceberg_test.go new file mode 100644 index 0000000000..bf499081ff --- /dev/null +++ b/internal/impl/iceberg/output_iceberg_test.go @@ -0,0 +1,136 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md + +package iceberg + +import ( + "bytes" + "log/slog" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/redpanda-data/benthos/v4/public/service" +) + +// captureIcebergOutput parses the given extra top-level YAML on top of a minimal +// valid iceberg output config and constructs the output through +// newIcebergOutputFromConfig against a mock Resources whose logger writes into a +// buffer. It returns everything logged during construction so the two runtime +// notices wired in newIcebergOutputFromConfig can be asserted at their real +// call site (rather than only through the pure helpers they delegate to). The +// capture seam is service.NewLoggerFromSlog + MockResourcesOptUseLogger — the +// same one internal/impl/kafka's hooks_test.go uses. +func captureIcebergOutput(t *testing.T, extra string) string { + t.Helper() + + conf, err := icebergOutputConfig().ParseYAML(` +catalog: + url: http://localhost:8181/api/catalog +namespace: ns +table: t +storage: + aws_s3: + bucket: bucket +`+extra, nil) + require.NoError(t, err) + + var buf bytes.Buffer + logger := service.NewLoggerFromSlog(slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{ + Level: slog.LevelDebug, + }))) + mgr := service.MockResources(service.MockResourcesOptUseLogger(logger)) + + // NewRouter (and hence newIcebergOutputFromConfig) does no catalog I/O, so + // this succeeds without a live catalog — construction is exactly where the + // two notices fire. + _, err = newIcebergOutputFromConfig(conf, mgr) + require.NoError(t, err) + + return buf.String() +} + +// orderingWarned / amplificationInformed key off substrings unique to each +// message (output_iceberg.go). The ordering WARNING is about correctness under +// concurrent commits; the amplification INFO is about copy-on-write write cost. +func orderingWarned(out string) bool { + return strings.Contains(out, "concurrent batches may commit out of order") +} + +func amplificationInformed(out string) bool { + return strings.Contains(out, "write amplification bounded") +} + +// TestRuntimeNoticeWiring pins the two startup log notices wired in +// newIcebergOutputFromConfig (currently 0% covered): the max_in_flight ordering +// WARNING and the copy-on-write write-amplification INFO. The pure helpers +// (mutating / cowAmplificationWarning) are unit-tested separately in +// cow_polish_test.go; this exercises the WIRING that decides whether each fires. +func TestRuntimeNoticeWiring(t *testing.T) { + cases := []struct { + name string + extra string + wantOrdering bool + wantAmp bool + }{ + { + // copy-on-write + mutating + concurrent: both the ordering warning + // (correctness) and the amplification info (cost) must fire. + name: "cow mutating max_in_flight 4 warns and informs", + extra: "row_operation: upsert\nidentifier_fields: [id]\nmerge_strategy: copy-on-write\nmax_in_flight: 4\n", + wantOrdering: true, + wantAmp: true, + }, + { + // copy-on-write + mutating + serialised: no ordering hazard, so only + // the amplification info fires. + name: "cow mutating max_in_flight 1 informs only", + extra: "row_operation: upsert\nidentifier_fields: [id]\nmerge_strategy: copy-on-write\nmax_in_flight: 1\n", + wantOrdering: false, + wantAmp: true, + }, + { + // merge-on-read + mutating + concurrent: the ordering warning fires, + // but amplification is a copy-on-write-only concern so it stays silent. + name: "mor mutating max_in_flight 4 warns only", + extra: "row_operation: upsert\nidentifier_fields: [id]\nmerge_strategy: merge-on-read\nmax_in_flight: 4\n", + wantOrdering: true, + wantAmp: false, + }, + { + // append-only (explicit insert) under copy-on-write + concurrent: + // neither fires — a non-mutating config has no ordering hazard and no + // amplification. + name: "append-only insert cow max_in_flight 4 silent", + extra: "row_operation: insert\nmerge_strategy: copy-on-write\nmax_in_flight: 4\n", + wantOrdering: false, + wantAmp: false, + }, + { + // append-only via the default (row_operation unset) under + // copy-on-write + concurrent: still silent, proving the default takes + // the non-mutating path. + name: "append-only default cow max_in_flight 4 silent", + extra: "merge_strategy: copy-on-write\nmax_in_flight: 4\n", + wantOrdering: false, + wantAmp: false, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + out := captureIcebergOutput(t, tc.extra) + assert.Equal(t, tc.wantOrdering, orderingWarned(out), + "ordering warning firing mismatch; captured log:\n%s", out) + assert.Equal(t, tc.wantAmp, amplificationInformed(out), + "amplification info firing mismatch; captured log:\n%s", out) + }) + } +} diff --git a/internal/impl/iceberg/row_operation_commit_test.go b/internal/impl/iceberg/row_operation_commit_test.go index 9bb75d8de1..ed1177282c 100644 --- a/internal/impl/iceberg/row_operation_commit_test.go +++ b/internal/impl/iceberg/row_operation_commit_test.go @@ -491,6 +491,26 @@ func TestCommitRowDeltaIdempotentOnUnknownState(t *testing.T) { assert.Equal(t, 1, countSnapshotsWithCommitID(cat.snapshot()), "the mutation must be committed exactly once after the conflict") }) + + // (T-13) landed-but-reported-failed (a lost ack on a 409): the first CommitTable + // applies the RowDelta server-side, then reports ErrCommitFailed as if it had + // been a clean conflict. The retry must find the commit-id in the reloaded + // snapshot and short-circuit to success WITHOUT committing a second time — + // exactly one CommitTable call, exactly one snapshot carrying the token. + t.Run("landed then failed applies once", func(t *testing.T) { + ctx := t.Context() + _, mem := newTestTable(t) + cat := &scriptedCatalog{memCatalog: mem, outcomes: []commitOutcome{commitLandThenFail}} + c, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, func(context.Context) (*table.Table, error) { return cat.snapshot(), nil }, logger) + require.NoError(t, err) + defer c.Close() + + require.NoError(t, c.Commit(ctx, morUpsertInput(t, ctx, cat.snapshot(), 2))) + + assert.Equal(t, 1, cat.calls, "a landed commit must not be re-committed after a lost-ack conflict") + assert.Equal(t, 1, countSnapshotsWithCommitID(cat.snapshot()), + "exactly one snapshot must carry the commit-id (mutation applied once)") + }) } // TestCommitRowDeltaWritesCommitIDToSummary is the direct round-trip test for the diff --git a/internal/impl/iceberg/shredder/temporal.go b/internal/impl/iceberg/shredder/temporal.go index 76dc373f9f..960494c9ee 100644 --- a/internal/impl/iceberg/shredder/temporal.go +++ b/internal/impl/iceberg/shredder/temporal.go @@ -14,12 +14,91 @@ import ( "math" "time" + "github.com/apache/iceberg-go" "github.com/parquet-go/parquet-go" "github.com/redpanda-data/benthos/v4/public/bloblang" "github.com/redpanda-data/benthos/v4/public/schema" ) +// secondsPerDay is the number of seconds in a UTC day, used to turn a +// days-since-epoch DATE value into an absolute instant. +const secondsPerDay = 86400 + +// NumericTemporalToTime interprets a bare numeric value destined for a temporal +// Iceberg column (DATE / TIME / TIMESTAMP / TIMESTAMPTZ) exactly as the +// shredder's insert path (convertLeafValue -> convertDate / convertTime / +// convertTimestamp) would, returning the equivalent UTC time.Time. +// +// It exists so the copy-on-write rewrite path (which encodes values through +// deleteKeyJSONValue + array.RecordFromJSON, and historically required a +// time.Time) can share the shredder's unit interpretation instead of diverging +// from it. The unit scaling is delegated to the very same helpers the insert +// path uses — numericToTimeMicros for TIME and scaleTimestampNumeric for +// TIMESTAMP, with DATE treated as already-days like convertDate — so the two +// paths cannot drift on how a bare number is scaled. That drift was the source +// of the year-50000 corruption when copy-on-write rejected the numeric the +// insert path happily interpreted. TestNumericTemporalToTimeMatchesConvert pins +// the equivalence. +// +// ok is false when value is not a bare number (NaN/±Inf included, so the +// caller's existing type check rejects them) or typ is not one of the four +// supported temporal types; the caller then keeps its existing time.Time-only +// handling. strict mirrors [RecordShredder.StrictTemporalMode]: when true and +// the metadata required to disambiguate the unit is absent, it returns the same +// error the insert path returns, so copy-on-write honours require_schema_metadata +// identically to inserts. +// +// This handles only the microsecond temporal types, which are exactly the ones +// copy-on-write supports as columns; the V3 nanosecond variants are out of +// scope (they are not copy-on-write-supported column types). +func NumericTemporalToTime(value any, typ iceberg.Type, common *schema.Common, strict bool) (time.Time, bool, error) { + n, isNum := numericInt64(value) + switch typ.(type) { + case iceberg.DateType: + if !isNum { + return time.Time{}, false, nil + } + if strict && (common == nil || common.Type != schema.Date) { + return time.Time{}, true, errors.New("date column received numeric value without matching schema.Common (Type=Date); require_schema_metadata=true demands a Date metadata entry to disambiguate the unit") + } + // DATE numerics are days since the epoch (convertDate applies no unit + // scaling); midnight UTC of that day formats back to the same day. + return time.Unix(n*secondsPerDay, 0).UTC(), true, nil + case iceberg.TimeType: + if !isNum { + return time.Time{}, false, nil + } + if strict && (common == nil || common.Type != schema.TimeOfDay || common.Logical == nil || common.Logical.TimeOfDay == nil) { + return time.Time{}, true, errors.New("time column received numeric value without matching schema.Common (Type=TimeOfDay with Logical.TimeOfDay); require_schema_metadata=true demands a TimeOfDay metadata entry with declared unit") + } + micros := numericToTimeMicros(n, common) + // 1970-01-01 + micros; its UTC wall-clock is the time-of-day, which the + // writer formats as HH:MM:SS.ffffff. + return time.UnixMicro(micros).UTC(), true, nil + case iceberg.TimestampType, iceberg.TimestampTzType: + if !isNum { + return time.Time{}, false, nil + } + if common != nil && common.Type == schema.Timestamp { + micros := scaleTimestampNumeric(n, common.EffectiveTimestamp().Unit, false) + return time.UnixMicro(micros).UTC(), true, nil + } + if strict { + return time.Time{}, true, errors.New("timestamp column received numeric value with no Timestamp schema metadata; require_schema_metadata=true demands a schema.Common with Type=Timestamp to disambiguate the unit") + } + // No metadata: match convertTimestamp's bloblang seconds-default + // fallback so the two paths agree even absent metadata. + t, err := bloblang.ValueAsTimestamp(value) + if err != nil { + return time.Time{}, true, err + } + return t.UTC(), true, nil + default: + return time.Time{}, false, nil + } +} + // commonForField returns the upstream schema.Common registered for the given // iceberg field ID, or nil when no metadata has been registered for that // field. Returns nil when the shredder itself has no metadata at all. diff --git a/internal/impl/iceberg/shredder/temporal_test.go b/internal/impl/iceberg/shredder/temporal_test.go index cb4aff52ac..06405da438 100644 --- a/internal/impl/iceberg/shredder/temporal_test.go +++ b/internal/impl/iceberg/shredder/temporal_test.go @@ -586,6 +586,80 @@ func TestCoerceTemporalRejectedInStrictMode(t *testing.T) { }) } +// TestNumericTemporalToTimeMatchesConvert is the anti-drift guard tying the +// copy-on-write numeric->time.Time helper (NumericTemporalToTime) to the insert +// path's numeric->parquet conversion (convertDate/convertTime/convertTimestamp). +// For every temporal type and unit, the time.Time the helper returns must encode +// to exactly the same iceberg-internal integer the insert path stores. If a +// future change scales one path differently from the other — the divergence that +// caused the year-50000 corruption — this test fails. +func TestNumericTemporalToTimeMatchesConvert(t *testing.T) { + daysOf := func(tm time.Time) int64 { + secs := tm.UTC().Unix() + d := secs / secondsPerDay + if secs < 0 && secs%secondsPerDay != 0 { + d-- + } + return d + } + + t.Run("timestamp all units", func(t *testing.T) { + const n = int64(1_730_000_000_000) + for _, u := range []schema.TimeUnit{schema.TimeUnitSeconds, schema.TimeUnitMillis, schema.TimeUnitMicros, schema.TimeUnitNanos} { + common := tsCommon(u, true) + pq, err := convertTimestamp(n, common, false, false) + require.NoError(t, err) + tm, ok, err := NumericTemporalToTime(n, iceberg.TimestampTzType{}, common, false) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, pq.Int64(), tm.UnixMicro(), "timestamp unit %v must agree between insert and copy-on-write", u) + } + }) + + t.Run("time all units", func(t *testing.T) { + const n = int64(45_296_000_000) // fits within a day at every unit up to micros + for _, u := range []schema.TimeUnit{schema.TimeUnitSeconds, schema.TimeUnitMillis, schema.TimeUnitMicros, schema.TimeUnitNanos} { + common := todCommon(u) + pq, err := convertTime(n, common, false) + require.NoError(t, err) + tm, ok, err := NumericTemporalToTime(n, iceberg.TimeType{}, common, false) + require.NoError(t, err) + require.True(t, ok) + // time.UnixMicro round-trips the micros-of-day the insert path stores. + assert.Equal(t, pq.Int64(), tm.UnixMicro(), "time unit %v must agree between insert and copy-on-write", u) + } + }) + + t.Run("date is days since epoch", func(t *testing.T) { + const days = int64(20289) // 2025-07-15 + common := &schema.Common{Type: schema.Date} + pq, err := convertDate(days, common, false) + require.NoError(t, err) + tm, ok, err := NumericTemporalToTime(days, iceberg.DateType{}, common, false) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, int64(pq.Int32()), daysOf(tm), "date must agree between insert and copy-on-write") + }) + + t.Run("non-numeric and non-temporal return ok=false", func(t *testing.T) { + _, ok, err := NumericTemporalToTime("not-a-number", iceberg.TimestampType{}, nil, false) + require.NoError(t, err) + assert.False(t, ok, "a non-numeric temporal value is left for the caller's time.Time handling") + + _, ok, err = NumericTemporalToTime(int64(5), iceberg.StringType{}, nil, false) + require.NoError(t, err) + assert.False(t, ok, "a non-temporal type is not handled here") + }) + + t.Run("strict rejects numeric without metadata like the insert path", func(t *testing.T) { + for _, typ := range []iceberg.Type{iceberg.DateType{}, iceberg.TimeType{}, iceberg.TimestampType{}, iceberg.TimestampTzType{}} { + _, _, err := NumericTemporalToTime(int64(1_730_000_000_000), typ, nil, true) + require.Error(t, err, "strict mode must reject a numeric temporal lacking metadata for %s", typ) + assert.Contains(t, err.Error(), "require_schema_metadata=true") + } + }) +} + func tsCommon(u schema.TimeUnit, utc bool) *schema.Common { return &schema.Common{ Type: schema.Timestamp, diff --git a/internal/impl/iceberg/writer.go b/internal/impl/iceberg/writer.go index 736277d99e..e63db48aa3 100644 --- a/internal/impl/iceberg/writer.go +++ b/internal/impl/iceberg/writer.go @@ -506,25 +506,32 @@ func deleteKeyJSONValue(t iceberg.Type, v any) (any, error) { default: return nil, fmt.Errorf("unsupported value type %T for decimal column", v) } - // Temporal keys must arrive as time.Time. A bare number is ambiguous (the - // data path interprets it via schema_metadata or a seconds fallback, which - // the delete path cannot reproduce), so accepting one would silently fail to - // match the intended rows — reject it loudly instead. + // Temporal values must arrive as time.Time here. A bare number is ambiguous + // (its unit — seconds/millis/micros — cannot be recovered without schema + // metadata), so accepting one blindly would silently mismatch what the + // insert path wrote. Merge-key callers keep this strict on purpose (an + // unambiguous key must round-trip exactly, the CON-490 guarantee); the + // copy-on-write data-column path (cowMassage) resolves a numeric temporal to + // a time.Time via the shredder's unit-aware conversion BEFORE calling this, + // so a numeric only reaches here for a merge key or a genuinely + // unconvertible value. The message is deliberately neutral about + // key-vs-column: callers add that context (identifier_fields for keys, the + // column name for data). case iceberg.DateType: if tm, ok := v.(time.Time); ok { return tm.UTC().Format("2006-01-02"), nil } - return nil, fmt.Errorf("date identifier column requires a time value, got %T", v) + return nil, fmt.Errorf("date column requires a time value, got %T", v) case iceberg.TimeType: if tm, ok := v.(time.Time); ok { return tm.UTC().Format("15:04:05.999999999"), nil } - return nil, fmt.Errorf("time identifier column requires a time value, got %T", v) + return nil, fmt.Errorf("time column requires a time value, got %T", v) case iceberg.TimestampType, iceberg.TimestampTzType: if tm, ok := v.(time.Time); ok { return tm.UTC().Format(time.RFC3339Nano), nil } - return nil, fmt.Errorf("timestamp identifier column requires a time value, got %T (a bare numeric timestamp is ambiguous as a delete key — convert it to a timestamp upstream)", v) + return nil, fmt.Errorf("timestamp column requires a time value, got %T (a bare numeric timestamp is ambiguous without schema metadata)", v) default: return v, nil } From febce7425b2a8c6f8dfbab92c0b7ffce784e6dd2 Mon Sep 17 00:00:00 2001 From: Ashley Jeffs Date: Tue, 28 Jul 2026 09:46:05 +0100 Subject: [PATCH 05/12] iceberg: pin timestamp encoding per table so existing tables never change The earlier spec fix (no-timezone `timestamp` columns written with isAdjustedToUTC=false) would silently change the on-disk encoding of existing tables, leaving them with mixed parquet annotations after an upgrade. This makes the choice per-table, permanent, and visible via the table property `redpanda-connect.timestamp-encoding`: - Tables created by this output are pinned to `spec` at creation. - An existing table without the property is bootstrapped on first write by probing a data file's parquet footer: files carrying the old UTC-adjusted annotation pin the table to `legacy`, so its encoding never changes and never mixes; an empty table, a schema without no-tz timestamp columns, or spec-encoded files pin `spec`. The resolved value is stamped onto the table so every future writer agrees; probe or stamp failures are hard errors rather than guesses. - `legacy` writes byte-identical output to pre-fix releases; `timestamptz` columns are unaffected in both modes. Mutating copy-on-write against a legacy-pinned table with no-tz timestamp columns now fails upfront with a migration message (compact the data files, then set the property to `spec`) instead of surfacing iceberg-go's "cannot promote timestamptz to timestamp" mid-commit. Insert-only copy-on-write and merge-on-read keep writing the table's own encoding and are unaffected. --- .../components/pages/outputs/iceberg.adoc | 14 + internal/impl/iceberg/config.go | 16 +- internal/impl/iceberg/cow.go | 31 ++ internal/impl/iceberg/icebergx/parquet.go | 32 +- .../impl/iceberg/icebergx/parquet_test.go | 80 ++- .../iceberg/icebergx/timestamp_encoding.go | 83 ++++ .../timestamp_encoding_integration_test.go | 288 +++++++++++ internal/impl/iceberg/router.go | 56 ++- internal/impl/iceberg/timestamp_encoding.go | 200 ++++++++ .../impl/iceberg/timestamp_encoding_test.go | 465 ++++++++++++++++++ internal/impl/iceberg/writer.go | 18 +- 11 files changed, 1247 insertions(+), 36 deletions(-) create mode 100644 internal/impl/iceberg/icebergx/timestamp_encoding.go create mode 100644 internal/impl/iceberg/integration/timestamp_encoding_integration_test.go create mode 100644 internal/impl/iceberg/timestamp_encoding.go create mode 100644 internal/impl/iceberg/timestamp_encoding_test.go diff --git a/docs/modules/components/pages/outputs/iceberg.adoc b/docs/modules/components/pages/outputs/iceberg.adoc index 563ce9beab..30ce09775d 100644 --- a/docs/modules/components/pages/outputs/iceberg.adoc +++ b/docs/modules/components/pages/outputs/iceberg.adoc @@ -278,6 +278,20 @@ Ordering only holds *within* a batch. With more than one batch in flight, concur * A `decimal` merge key is not supported — use `merge-on-read` for a decimal key (a `decimal` non-key column is fine). * Schema evolution covers new *top-level* columns only; new fields appearing inside an existing nested `struct`/`list`/`map` column are not auto-surfaced for evolution. * It is a batch / moderate-throughput mode: expect heavy write amplification under scattered, high-frequency keyed mutations. +* Tables pinned to the legacy timestamp encoding whose schema contains a no-timezone `timestamp` column reject `upsert`/`delete` — see <> for why and for the migration path. + +[[timestamp-encoding]] +=== Timestamp encoding on existing tables + +Older versions of this output annotated no-timezone `timestamp` columns in the parquet files they wrote with `isAdjustedToUTC=true` — the annotation the Iceberg spec reserves for `timestamptz`. The stored microsecond instants are correct, and appends and most readers are unaffected, but the annotation makes some readers treat the column as UTC-adjusted, and it prevents `copy-on-write` from rewriting those files (the file's annotation reads back as `timestamptz`, which cannot be written into a `timestamp` column). Current versions write the spec-correct `isAdjustedToUTC=false`. + +To guarantee an existing table never ends up with a mix of the two annotations, the encoding is pinned *per table* via the table property `redpanda-connect.timestamp-encoding` (`spec` or `legacy`): + +* Tables created by this output carry `redpanda-connect.timestamp-encoding: spec` from creation. +* For an existing table without the property, the output resolves the encoding automatically on first contact and stamps the result onto the table: if the schema has no no-timezone `timestamp` column, or the table has no data files, it resolves `spec`; otherwise the output inspects one data file's parquet footer and adopts whatever that file already contains (`legacy` for `isAdjustedToUTC=true`). A table that cannot be probed (unreadable file) fails the write rather than risk mixing annotations. +* Once stamped, the property is authoritative and the probe never runs again. An unrecognised property value is a hard error. + +A table pinned `legacy` keeps receiving the legacy annotation on every new file — byte-identical to what previous releases wrote — so appends and `merge-on-read` continue working unchanged forever. The one restriction is mutating `copy-on-write` (`upsert`/`delete`): it must rewrite existing files, which the legacy annotation prevents, so such writes fail upfront with an actionable error (pure `insert` batches still work). To migrate a legacy table to the spec encoding: rewrite/compact the table's data files with an engine that writes the spec annotation (e.g. Spark's `rewrite_data_files`), then set the table property `redpanda-connect.timestamp-encoding` to `spec`. Alternatively, keep the table on `merge-on-read`. == Performance diff --git a/internal/impl/iceberg/config.go b/internal/impl/iceberg/config.go index 95eb8568ff..199394b450 100644 --- a/internal/impl/iceberg/config.go +++ b/internal/impl/iceberg/config.go @@ -163,7 +163,21 @@ const rowOperationDocs = "\n" + "\n" + "* A `decimal` merge key is not supported — use `merge-on-read` for a decimal key (a `decimal` non-key column is fine).\n" + "* Schema evolution covers new *top-level* columns only; new fields appearing inside an existing nested `struct`/`list`/`map` column are not auto-surfaced for evolution.\n" + - "* It is a batch / moderate-throughput mode: expect heavy write amplification under scattered, high-frequency keyed mutations.\n" + "* It is a batch / moderate-throughput mode: expect heavy write amplification under scattered, high-frequency keyed mutations.\n" + + "* Tables pinned to the legacy timestamp encoding whose schema contains a no-timezone `timestamp` column reject `upsert`/`delete` — see <> for why and for the migration path.\n" + + "\n" + + "[[timestamp-encoding]]\n" + + "=== Timestamp encoding on existing tables\n" + + "\n" + + "Older versions of this output annotated no-timezone `timestamp` columns in the parquet files they wrote with `isAdjustedToUTC=true` — the annotation the Iceberg spec reserves for `timestamptz`. The stored microsecond instants are correct, and appends and most readers are unaffected, but the annotation makes some readers treat the column as UTC-adjusted, and it prevents `copy-on-write` from rewriting those files (the file's annotation reads back as `timestamptz`, which cannot be written into a `timestamp` column). Current versions write the spec-correct `isAdjustedToUTC=false`.\n" + + "\n" + + "To guarantee an existing table never ends up with a mix of the two annotations, the encoding is pinned *per table* via the table property `redpanda-connect.timestamp-encoding` (`spec` or `legacy`):\n" + + "\n" + + "* Tables created by this output carry `redpanda-connect.timestamp-encoding: spec` from creation.\n" + + "* For an existing table without the property, the output resolves the encoding automatically on first contact and stamps the result onto the table: if the schema has no no-timezone `timestamp` column, or the table has no data files, it resolves `spec`; otherwise the output inspects one data file's parquet footer and adopts whatever that file already contains (`legacy` for `isAdjustedToUTC=true`). A table that cannot be probed (unreadable file) fails the write rather than risk mixing annotations.\n" + + "* Once stamped, the property is authoritative and the probe never runs again. An unrecognised property value is a hard error.\n" + + "\n" + + "A table pinned `legacy` keeps receiving the legacy annotation on every new file — byte-identical to what previous releases wrote — so appends and `merge-on-read` continue working unchanged forever. The one restriction is mutating `copy-on-write` (`upsert`/`delete`): it must rewrite existing files, which the legacy annotation prevents, so such writes fail upfront with an actionable error (pure `insert` batches still work). To migrate a legacy table to the spec encoding: rewrite/compact the table's data files with an engine that writes the spec annotation (e.g. Spark's `rewrite_data_files`), then set the table property `redpanda-connect.timestamp-encoding` to `spec`. Alternatively, keep the table on `merge-on-read`.\n" // icebergOutputConfig returns the configuration spec for the Iceberg output. func icebergOutputConfig() *service.ConfigSpec { diff --git a/internal/impl/iceberg/cow.go b/internal/impl/iceberg/cow.go index 8496693df4..818bd99232 100644 --- a/internal/impl/iceberg/cow.go +++ b/internal/impl/iceberg/cow.go @@ -26,6 +26,7 @@ import ( "github.com/redpanda-data/benthos/v4/public/schema" "github.com/redpanda-data/benthos/v4/public/service" + "github.com/redpanda-data/connect/v4/internal/impl/iceberg/icebergx" "github.com/redpanda-data/connect/v4/internal/impl/iceberg/shredder" ) @@ -69,6 +70,18 @@ func (w *writer) writeCOW(ctx context.Context, batch service.MessageBatch) error return nil } + // Mutating (upsert/delete) copy-on-write rewrites existing data files, and + // the rewrite cannot read back files whose no-tz `timestamp` columns carry + // the legacy UTC-adjusted annotation (iceberg-go reads them as timestamptz + // and refuses the timestamptz -> timestamp "promotion"). Fail upfront with + // an actionable error — before any file writes — rather than surface the + // library's cryptic one mid-commit. Insert-only batches on such a table are + // fine (they took the append fast path above and keep writing the table's + // own legacy encoding), as is merge-on-read (no file rewrites). + if err := w.checkCOWTimestampEncoding(); err != nil { + return err + } + // The remaining paths rewrite data files. Partitioned tables are supported: // iceberg-go's Overwrite/Delete route rows to partitions correctly end-to-end. // - New/rewritten rows: recordsToDataFiles sends a partitioned spec through @@ -137,6 +150,24 @@ func (w *writer) writeCOW(ctx context.Context, batch service.MessageBatch) error return nil } +// checkCOWTimestampEncoding guards mutating copy-on-write against tables +// pinned to the legacy timestamp encoding: their data files annotate no-tz +// `timestamp` columns with isAdjustedToUTC=true, which the copy-on-write +// rewrite cannot read back losslessly (iceberg-go maps the annotation to +// timestamptz and its strict rewrite visitor refuses timestamptz -> +// timestamp). Tables without any no-tz timestamp column are unaffected — +// there is no column the encodings disagree on. +func (w *writer) checkCOWTimestampEncoding() error { + if w.tsEncoding != icebergx.TimestampEncodingLegacy || !icebergx.SchemaHasNoTZTimestamp(w.table.Schema()) { + return nil + } + return fmt.Errorf( + "table %s uses the legacy UTC-adjusted parquet encoding for its `timestamp` columns (table property %s=legacy), which copy-on-write cannot rewrite; "+ + "compact/rewrite the table's data files with an engine that writes the spec encoding and set the table property %s=spec, or use merge_strategy: merge-on-read", + strings.Join(w.table.Identifier(), "."), icebergx.TimestampEncodingProperty, icebergx.TimestampEncodingProperty, + ) +} + // checkCOWSchemaSupported rejects table schemas the copy-on-write path cannot // faithfully round-trip through Arrow. The rewrite builds records via // array.RecordFromJSON from the JSON produced by cowMassage; that projection is diff --git a/internal/impl/iceberg/icebergx/parquet.go b/internal/impl/iceberg/icebergx/parquet.go index 1f8ff84831..6f260580db 100644 --- a/internal/impl/iceberg/icebergx/parquet.go +++ b/internal/impl/iceberg/icebergx/parquet.go @@ -22,12 +22,15 @@ import ( ) // BuildParquetSchema builds a parquet schema from an iceberg schema and returns -// a mapping from field ID to column index. -func BuildParquetSchema(schema *iceberg.Schema) (_ *parquet.Schema, fieldIDToColIdx map[int]int, err error) { +// a mapping from field ID to column index. tsEncoding selects the +// isAdjustedToUTC annotation written for no-timezone `timestamp` columns — +// it must match the encoding of the table's existing data files so a table +// never carries mixed annotations (see TimestampEncodingProperty). +func BuildParquetSchema(schema *iceberg.Schema, tsEncoding TimestampEncoding) (_ *parquet.Schema, fieldIDToColIdx map[int]int, err error) { group := make(parquet.Group) for _, field := range schema.Fields() { - node, err := icebergFieldToParquet(field) + node, err := icebergFieldToParquet(field, tsEncoding) if err != nil { return nil, nil, fmt.Errorf("field %s: %w", field.Name, err) } @@ -107,8 +110,8 @@ func schemaLeaves(root iceberg.Type, fieldID int, path []string) iter.Seq[schema } // icebergFieldToParquet converts an iceberg field to a parquet node. -func icebergFieldToParquet(field iceberg.NestedField) (parquet.Node, error) { - node, err := icebergTypeToParquet(field.Type) +func icebergFieldToParquet(field iceberg.NestedField, tsEncoding TimestampEncoding) (parquet.Node, error) { + node, err := icebergTypeToParquet(field.Type, tsEncoding) if err != nil { return nil, err } @@ -124,7 +127,7 @@ func icebergFieldToParquet(field iceberg.NestedField) (parquet.Node, error) { } // icebergTypeToParquet converts an iceberg type to a parquet node. -func icebergTypeToParquet(t iceberg.Type) (parquet.Node, error) { +func icebergTypeToParquet(t iceberg.Type, tsEncoding TimestampEncoding) (parquet.Node, error) { switch t := t.(type) { case iceberg.BooleanType: return parquet.Leaf(parquet.BooleanType), nil @@ -152,7 +155,14 @@ func icebergTypeToParquet(t iceberg.Type) (parquet.Node, error) { // rewrites (the strict rewrite visitor refuses timestamptz -> timestamp). // This mirrors iceberg-go's own Arrow writer, which encodes a no-tz // timestamp with an empty Arrow time zone (isAdjustedToUTC=false). - return parquet.TimestampAdjusted(parquet.Microsecond, false), nil + // + // EXCEPT for tables pinned to the legacy encoding + // (TimestampEncodingLegacy): released connector versions wrote + // isAdjustedToUTC=true, so tables holding such files must keep + // receiving it — a table must never carry mixed annotations for one + // column. The per-table choice is resolved from + // TimestampEncodingProperty (see that constant's doc). + return parquet.TimestampAdjusted(parquet.Microsecond, tsEncoding == TimestampEncodingLegacy), nil case iceberg.TimestampTzType: // A `timestamptz` is UTC-adjusted: isAdjustedToUTC=true (parquet.Timestamp's // default). iceberg-go reads this back as arrow timestamp[tz=UTC] -> timestamptz. @@ -164,7 +174,7 @@ func icebergTypeToParquet(t iceberg.Type) (parquet.Node, error) { case *iceberg.StructType: group := make(parquet.Group, len(t.Fields())) for _, f := range t.Fields() { - node, err := icebergFieldToParquet(f) + node, err := icebergFieldToParquet(f, tsEncoding) if err != nil { return nil, err } @@ -172,7 +182,7 @@ func icebergTypeToParquet(t iceberg.Type) (parquet.Node, error) { } return group, nil case *iceberg.ListType: - elem, err := icebergTypeToParquet(t.Element) + elem, err := icebergTypeToParquet(t.Element, tsEncoding) if err != nil { return nil, err } @@ -182,12 +192,12 @@ func icebergTypeToParquet(t iceberg.Type) (parquet.Node, error) { elem = parquet.FieldID(elem, t.ElementID) return parquet.List(elem), nil case *iceberg.MapType: - key, err := icebergTypeToParquet(t.KeyType) + key, err := icebergTypeToParquet(t.KeyType, tsEncoding) if err != nil { return nil, err } key = parquet.FieldID(key, t.KeyID) - val, err := icebergTypeToParquet(t.ValueType) + val, err := icebergTypeToParquet(t.ValueType, tsEncoding) if err != nil { return nil, err } diff --git a/internal/impl/iceberg/icebergx/parquet_test.go b/internal/impl/iceberg/icebergx/parquet_test.go index 02d7539ecd..2e5da1428f 100644 --- a/internal/impl/iceberg/icebergx/parquet_test.go +++ b/internal/impl/iceberg/icebergx/parquet_test.go @@ -28,14 +28,22 @@ func TestIcebergTimestampParquetAnnotation(t *testing.T) { cases := []struct { name string typ iceberg.Type + encoding TimestampEncoding wantAdjustedTZ bool }{ - {"timestamp", iceberg.TimestampType{}, false}, - {"timestamptz", iceberg.TimestampTzType{}, true}, + // Spec encoding: the Iceberg-spec-correct annotations. This is what + // every table created by the connector from now on gets. + {"timestamp spec", iceberg.TimestampType{}, TimestampEncodingSpec, false}, + {"timestamptz spec", iceberg.TimestampTzType{}, TimestampEncodingSpec, true}, + // Legacy encoding: no-tz `timestamp` keeps the pre-fix + // isAdjustedToUTC=true so existing tables never become mixed; + // `timestamptz` is UTC-adjusted in BOTH modes. + {"timestamp legacy", iceberg.TimestampType{}, TimestampEncodingLegacy, true}, + {"timestamptz legacy", iceberg.TimestampTzType{}, TimestampEncodingLegacy, true}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - node, err := icebergTypeToParquet(tc.typ) + node, err := icebergTypeToParquet(tc.typ, tc.encoding) require.NoError(t, err) lt := node.Type().LogicalType() @@ -48,6 +56,54 @@ func TestIcebergTimestampParquetAnnotation(t *testing.T) { } } +func TestParseTimestampEncoding(t *testing.T) { + enc, err := ParseTimestampEncoding("spec") + require.NoError(t, err) + assert.Equal(t, TimestampEncodingSpec, enc) + + enc, err = ParseTimestampEncoding("legacy") + require.NoError(t, err) + assert.Equal(t, TimestampEncodingLegacy, enc) + + // Unknown values must fail loud: guessing could mix parquet annotations + // within a single table. + _, err = ParseTimestampEncoding("bogus") + require.Error(t, err) + assert.Contains(t, err.Error(), TimestampEncodingProperty) + assert.Contains(t, err.Error(), "bogus") + + _, err = ParseTimestampEncoding("") + require.Error(t, err) +} + +func TestSchemaHasNoTZTimestamp(t *testing.T) { + t.Run("none", func(t *testing.T) { + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64}, + iceberg.NestedField{ID: 2, Name: "tstz", Type: iceberg.PrimitiveTypes.TimestampTz}, + ) + assert.False(t, SchemaHasNoTZTimestamp(sc), "timestamptz alone must not count") + }) + + t.Run("top level", func(t *testing.T) { + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "ts", Type: iceberg.PrimitiveTypes.Timestamp}, + ) + assert.True(t, SchemaHasNoTZTimestamp(sc)) + }) + + t.Run("nested", func(t *testing.T) { + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "events", Type: &iceberg.ListType{ + ElementID: 2, Element: &iceberg.StructType{FieldList: []iceberg.NestedField{ + {ID: 3, Name: "at", Type: iceberg.PrimitiveTypes.Timestamp}, + }}, + }}, + ) + assert.True(t, SchemaHasNoTZTimestamp(sc), "a nested no-tz timestamp leaf must count") + }) +} + func TestBuildParquetSchema_SimpleFlat(t *testing.T) { // Schema: { id: int64, name: string } schema := iceberg.NewSchema(1, @@ -55,7 +111,7 @@ func TestBuildParquetSchema_SimpleFlat(t *testing.T) { iceberg.NestedField{ID: 2, Name: "name", Type: iceberg.PrimitiveTypes.String, Required: false}, ) - pqSchema, fieldToCol, err := BuildParquetSchema(schema) + pqSchema, fieldToCol, err := BuildParquetSchema(schema, TimestampEncodingSpec) require.NoError(t, err) require.NotNil(t, pqSchema) @@ -92,7 +148,7 @@ func TestBuildParquetSchema_NestedStruct(t *testing.T) { }, ) - pqSchema, fieldToCol, err := BuildParquetSchema(schema) + pqSchema, fieldToCol, err := BuildParquetSchema(schema, TimestampEncodingSpec) require.NoError(t, err) require.NotNil(t, pqSchema) @@ -131,7 +187,7 @@ func TestBuildParquetSchema_List(t *testing.T) { }, ) - pqSchema, fieldToCol, err := BuildParquetSchema(schema) + pqSchema, fieldToCol, err := BuildParquetSchema(schema, TimestampEncodingSpec) require.NoError(t, err) require.NotNil(t, pqSchema) @@ -167,7 +223,7 @@ func TestBuildParquetSchema_Map(t *testing.T) { }, ) - pqSchema, fieldToCol, err := BuildParquetSchema(schema) + pqSchema, fieldToCol, err := BuildParquetSchema(schema, TimestampEncodingSpec) require.NoError(t, err) require.NotNil(t, pqSchema) @@ -211,7 +267,7 @@ func TestBuildParquetSchema_ListOfStructs(t *testing.T) { }, ) - pqSchema, fieldToCol, err := BuildParquetSchema(schema) + pqSchema, fieldToCol, err := BuildParquetSchema(schema, TimestampEncodingSpec) require.NoError(t, err) require.NotNil(t, pqSchema) @@ -260,7 +316,7 @@ func TestBuildParquetSchema_DeeplyNested(t *testing.T) { }, ) - pqSchema, fieldToCol, err := BuildParquetSchema(schema) + pqSchema, fieldToCol, err := BuildParquetSchema(schema, TimestampEncodingSpec) require.NoError(t, err) require.NotNil(t, pqSchema) @@ -312,7 +368,7 @@ func TestBuildParquetSchema_NestedListsInStruct(t *testing.T) { }, ) - pqSchema, fieldToCol, err := BuildParquetSchema(schema) + pqSchema, fieldToCol, err := BuildParquetSchema(schema, TimestampEncodingSpec) require.NoError(t, err) require.NotNil(t, pqSchema) @@ -369,7 +425,7 @@ func TestBuildParquetSchema_ComplexMixed(t *testing.T) { }, ) - pqSchema, fieldToCol, err := BuildParquetSchema(schema) + pqSchema, fieldToCol, err := BuildParquetSchema(schema, TimestampEncodingSpec) require.NoError(t, err) require.NotNil(t, pqSchema) @@ -416,7 +472,7 @@ func TestBuildParquetSchema_AllPrimitiveTypes(t *testing.T) { iceberg.NestedField{ID: 12, Name: "uuid_col", Type: iceberg.PrimitiveTypes.UUID, Required: false}, ) - pqSchema, fieldToCol, err := BuildParquetSchema(schema) + pqSchema, fieldToCol, err := BuildParquetSchema(schema, TimestampEncodingSpec) require.NoError(t, err) require.NotNil(t, pqSchema) diff --git a/internal/impl/iceberg/icebergx/timestamp_encoding.go b/internal/impl/iceberg/icebergx/timestamp_encoding.go new file mode 100644 index 0000000000..b8ad9cad57 --- /dev/null +++ b/internal/impl/iceberg/icebergx/timestamp_encoding.go @@ -0,0 +1,83 @@ +/* + * Copyright 2026 Redpanda Data, Inc. + * + * Licensed as a Redpanda Enterprise file under the Redpanda Community + * License (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md + */ + +package icebergx + +import ( + "fmt" + + "github.com/apache/iceberg-go" +) + +// TimestampEncodingProperty is the Iceberg table property that pins how this +// connector annotates no-timezone `timestamp` columns in the parquet files it +// writes. It exists because a released version of the connector wrote them +// with the parquet logical-type annotation isAdjustedToUTC=true (the +// "legacy" encoding, spec-incorrect but harmless for append-only reads), +// and silently switching an existing table to the spec-correct +// isAdjustedToUTC=false would leave it with mixed annotations — and break +// copy-on-write rewrites of the old files. The property makes the choice +// per-table, permanent and visible: new tables are created with "spec", +// existing tables are pinned to whatever their data files already contain. +const TimestampEncodingProperty = "redpanda-connect.timestamp-encoding" + +// TimestampEncoding selects the parquet isAdjustedToUTC annotation written +// for no-timezone iceberg `timestamp` columns. `timestamptz` columns are +// always written UTC-adjusted regardless of the encoding. +type TimestampEncoding int + +const ( + // TimestampEncodingSpec writes no-tz `timestamp` columns with + // isAdjustedToUTC=false, as the Iceberg spec requires. The zero value: + // every new table gets this. + TimestampEncodingSpec TimestampEncoding = iota + // TimestampEncodingLegacy writes no-tz `timestamp` columns with + // isAdjustedToUTC=true, byte-identical to what pre-fix connector + // versions produced, so existing tables never become mixed. + TimestampEncodingLegacy +) + +// String returns the property value form of the encoding ("spec" / "legacy"). +func (e TimestampEncoding) String() string { + switch e { + case TimestampEncodingLegacy: + return "legacy" + default: + return "spec" + } +} + +// ParseTimestampEncoding parses a TimestampEncodingProperty value. Unknown +// values are a hard error: guessing here could silently mix parquet +// annotations within one table. +func ParseTimestampEncoding(s string) (TimestampEncoding, error) { + switch s { + case "spec": + return TimestampEncodingSpec, nil + case "legacy": + return TimestampEncodingLegacy, nil + default: + return 0, fmt.Errorf("invalid table property %s value %q: must be %q or %q", TimestampEncodingProperty, s, TimestampEncodingSpec, TimestampEncodingLegacy) + } +} + +// SchemaHasNoTZTimestamp reports whether any leaf column of the schema +// (including nested struct/list/map leaves) is a no-timezone `timestamp`. +// Only those columns are affected by the timestamp encoding; a schema +// without them is encoding-agnostic. +func SchemaHasNoTZTimestamp(schema *iceberg.Schema) bool { + st := schema.AsStruct() + for leaf := range schemaLeaves(&st, -1, nil) { + if _, ok := leaf.Type.(iceberg.TimestampType); ok { + return true + } + } + return false +} diff --git a/internal/impl/iceberg/integration/timestamp_encoding_integration_test.go b/internal/impl/iceberg/integration/timestamp_encoding_integration_test.go new file mode 100644 index 0000000000..e43dbb4b35 --- /dev/null +++ b/internal/impl/iceberg/integration/timestamp_encoding_integration_test.go @@ -0,0 +1,288 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md + +package iceberg + +import ( + "context" + "fmt" + "net/http" + "strings" + "testing" + "time" + + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/table" + "github.com/parquet-go/parquet-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/redpanda-data/benthos/v4/public/schema" + "github.com/redpanda-data/benthos/v4/public/service" + "github.com/redpanda-data/benthos/v4/public/service/integration" + + icebergimpl "github.com/redpanda-data/connect/v4/internal/impl/iceberg" + "github.com/redpanda-data/connect/v4/internal/impl/iceberg/icebergx" +) + +// tsAnnotationsPerDataFile opens every current-snapshot data file's parquet +// footer via the table's own filesystem and returns, per file path, fieldID -> +// isAdjustedToUTC for each leaf annotated with a TIMESTAMP logical type. +func tsAnnotationsPerDataFile(t *testing.T, ctx context.Context, tbl *table.Table) map[string]map[int]bool { + t.Helper() + out := map[string]map[int]bool{} + snap := tbl.CurrentSnapshot() + if snap == nil { + return out + } + fsys, err := tbl.FS(ctx) + require.NoError(t, err) + manifests, err := snap.Manifests(fsys) + require.NoError(t, err) + for _, m := range manifests { + if m.ManifestContent() != iceberg.ManifestContentData { + continue + } + for entry, err := range m.Entries(fsys, true) { + require.NoError(t, err) + path := entry.DataFile().FilePath() + f, err := fsys.Open(path) + require.NoError(t, err) + info, err := f.Stat() + require.NoError(t, err) + pf, err := parquet.OpenFile(f, info.Size(), parquet.SkipPageIndex(true), parquet.SkipBloomFilters(true)) + require.NoError(t, err) + ann := map[int]bool{} + for _, el := range pf.Metadata().Schema { + if lt, ok := el.LogicalType.Get(); ok && lt.Timestamp != nil { + ann[int(el.FieldID)] = lt.Timestamp.IsAdjustedToUTC + } + } + require.NoError(t, f.Close()) + out[path] = ann + } + } + return out +} + +// removeTableProperty removes a table property through the raw REST commit +// endpoint (iceberg-go's Transaction has no remove-properties surface). Used +// to turn a table stamped by this test suite back into a faithful simulation +// of a pre-upgrade table: legacy-annotated data files, no pinning property. +func removeTableProperty(t *testing.T, infra *testInfrastructure, ns, tblName, prop string) { + t.Helper() + body := fmt.Sprintf(`{"requirements":[],"updates":[{"action":"remove-properties","removals":[%q]}]}`, prop) + req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, + fmt.Sprintf("%s/v1/namespaces/%s/tables/%s", infra.RestURL, ns, tblName), strings.NewReader(body)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode, "remove-properties commit failed") +} + +// TestTimestampEncodingLegacyTableIntegration proves, end-to-end against a real +// REST catalog + S3 + DuckDB, that an EXISTING table whose data files carry the +// legacy UTC-adjusted `timestamp` annotation is (a) probe-detected and pinned +// `legacy` when the property is absent, (b) kept uniformly legacy-annotated by +// every subsequent append (never mixed), (c) still correct to an independent +// reader, and (d) protected from mutating copy-on-write by the upfront guard +// error instead of iceberg-go's cryptic mid-commit failure. +func TestTimestampEncodingLegacyTableIntegration(t *testing.T) { + integration.CheckSkip(t) + ctx := context.Background() + infra := setupTestInfra(t, ctx) + + const ns, tblName = "ts_enc_legacy_ns", "ts_enc_legacy_test" + infra.CreateNamespace(t, ns) + client := infra.NewCatalogClient(t, ns) + + // A pre-existing table (not created by the connector): id + a no-timezone + // `timestamp` column + a `timestamptz` column, no pinning property. + _, err := client.CreateTable(ctx, tblName, iceberg.NewSchema(1, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.StringType{}, Required: true}, + iceberg.NestedField{ID: 2, Name: "ts", Type: iceberg.TimestampType{}, Required: false}, + iceberg.NestedField{ID: 3, Name: "tstz", Type: iceberg.TimestampTzType{}, Required: false}, + )) + require.NoError(t, err) + + seed := time.Date(2024, 1, 15, 12, 30, 45, 0, time.UTC) + rowMsg := func(id string) *service.Message { + m := service.NewMessage(nil) + m.SetStructured(map[string]any{"id": id, "ts": seed, "tstz": seed}) + return m + } + + // --- Produce genuine legacy files via the connector's own legacy mode. --- + // Pin the table `legacy`, append through the router (the property-present + // resolution path), and confirm the produced file is annotated exactly as + // pre-fix releases wrote it: isAdjustedToUTC=true on BOTH columns. + { + tbl, err := client.LoadTable(ctx, tblName) + require.NoError(t, err) + txn := tbl.NewTransaction() + require.NoError(t, txn.SetProperties(iceberg.Properties{ + icebergx.TimestampEncodingProperty: "legacy", + })) + _, err = txn.Commit(ctx) + require.NoError(t, err) + } + routerA := infra.NewRouter(t, ns, tblName) + produceMessages(t, ctx, routerA, service.MessageBatch{rowMsg("1")}) + + tbl, err := client.LoadTable(ctx, tblName) + require.NoError(t, err) + for path, ann := range tsAnnotationsPerDataFile(t, ctx, tbl) { + assert.Equal(t, map[int]bool{2: true, 3: true}, ann, + "legacy-pinned append must write the pre-fix annotation: %s", path) + } + + // --- Simulate the real upgrade scenario. --- + // Strip the property so the table looks exactly like one written entirely + // by a pre-upgrade connector: legacy files, no pin. A FRESH router (new + // process) must footer-probe the file, resolve `legacy`, stamp the + // property, and keep appending the legacy annotation — uniform, not mixed. + removeTableProperty(t, infra, ns, tblName, icebergx.TimestampEncodingProperty) + tbl, err = client.LoadTable(ctx, tblName) + require.NoError(t, err) + require.NotContains(t, tbl.Properties(), icebergx.TimestampEncodingProperty, + "precondition: the pinning property must be absent before the bootstrap") + + routerB := infra.NewRouter(t, ns, tblName) + produceMessages(t, ctx, routerB, service.MessageBatch{rowMsg("2")}) + + tbl, err = client.LoadTable(ctx, tblName) + require.NoError(t, err) + assert.Equal(t, "legacy", tbl.Properties()[icebergx.TimestampEncodingProperty], + "the footer-probe bootstrap must stamp the resolved encoding onto the table") + + // --- An insert-only copy-on-write batch is fine on a legacy table. --- + operation, err := service.NewInterpolatedString(`${! meta("op") }`) + require.NoError(t, err) + cowRouter := infra.NewRouter(t, ns, tblName, + WithRowOperation(icebergimpl.RowOpConfig{ + Operation: operation, + IdentifierFields: []string{"id"}, + MergeStrategy: icebergimpl.MergeStrategyCOW, + })) + produceMessages(t, ctx, cowRouter, service.MessageBatch{ + opStructMsg("insert", map[string]any{"id": "3", "ts": seed, "tstz": seed}), + }) + + // --- Uniformity: EVERY data file (pre-existing, probed append, copy-on- + // write insert) carries the legacy annotation. --- + tbl, err = client.LoadTable(ctx, tblName) + require.NoError(t, err) + anns := tsAnnotationsPerDataFile(t, ctx, tbl) + require.Len(t, anns, 3, "expected one data file per appended batch") + for path, ann := range anns { + assert.Equal(t, map[int]bool{2: true, 3: true}, ann, + "a legacy table's files must stay uniformly legacy-annotated: %s", path) + } + + // --- DuckDB (an independent reader) sees all rows at the correct instant. + // Under the legacy annotation both columns read as UTC-adjusted timestamps, + // exactly as they did from pre-fix releases. --- + match := querySQL[countResult](t, ctx, infra, fmt.Sprintf( + `SELECT COUNT(*) AS count FROM iceberg_cat."%s"."%s" `+ + `WHERE ts = TIMESTAMPTZ '2024-01-15 12:30:45+00' AND tstz = TIMESTAMPTZ '2024-01-15 12:30:45+00';`, + ns, tblName)) + require.Len(t, match, 1) + assert.Equal(t, 3, match[0].Count, "every row must be stored at the exact seeded instant") + + // --- Mutating copy-on-write must fail upfront with the actionable guard + // error, not iceberg-go's "cannot promote timestamptz to timestamp". --- + err = cowRouter.Route(ctx, service.MessageBatch{ + opStructMsg("upsert", map[string]any{"id": "1", "ts": seed.Add(time.Hour), "tstz": seed.Add(time.Hour)}), + }) + require.Error(t, err, "a copy-on-write upsert on a legacy table must be rejected") + assert.Contains(t, err.Error(), "legacy UTC-adjusted parquet encoding") + assert.Contains(t, err.Error(), icebergx.TimestampEncodingProperty+"=spec") + assert.NotContains(t, err.Error(), "cannot promote", "the guard must fire before the library's cryptic failure") +} + +// TestTimestampEncodingNewTableIntegration proves a table auto-created by the +// connector is pinned to the spec encoding at birth: the creation commit +// carries redpanda-connect.timestamp-encoding=spec, its no-tz `timestamp` +// column is annotated isAdjustedToUTC=false on disk, and DuckDB types the +// column as a plain TIMESTAMP holding the exact written instant. +func TestTimestampEncodingNewTableIntegration(t *testing.T) { + integration.CheckSkip(t) + ctx := context.Background() + infra := setupTestInfra(t, ctx) + + const ns, tblName = "ts_enc_new_ns", "ts_enc_new_test" + infra.CreateNamespace(t, ns) + client := infra.NewCatalogClient(t, ns) + + // Schema metadata declaring ts as a NO-timezone timestamp, so the + // auto-created column is `timestamp` rather than the `timestamptz` that + // bare time.Time inference produces. + commonSchema := schema.Common{ + Type: schema.Object, Name: "Event", + Children: []schema.Common{ + {Name: "id", Type: schema.String}, + { + Name: "ts", Optional: true, Type: schema.Timestamp, + Logical: &schema.LogicalParams{ + Timestamp: &schema.TimestampParams{Unit: schema.TimeUnitMicros, AdjustToUTC: false}, + }, + }, + }, + } + + seed := time.Date(2024, 3, 20, 8, 15, 0, 0, time.UTC) + msg := service.NewMessage(nil) + msg.SetStructured(map[string]any{"id": "1", "ts": seed}) + msg.MetaSetMut("schema", commonSchema.ToAny()) + + router := infra.NewRouter(t, ns, tblName, + WithSchemaEvolution(icebergimpl.SchemaEvolutionConfig{ + Enabled: true, + SchemaMetadata: "schema", + })) + produceMessages(t, ctx, router, service.MessageBatch{msg}) + + tbl, err := client.LoadTable(ctx, tblName) + require.NoError(t, err) + + // The pin must be present from creation. + assert.Equal(t, "spec", tbl.Properties()[icebergx.TimestampEncodingProperty], + "tables created by the connector must be pinned spec at creation") + + // The no-tz column must be spec-annotated on disk. + tsField, ok := tbl.Schema().FindFieldByName("ts") + require.True(t, ok) + require.IsType(t, iceberg.TimestampType{}, tsField.Type, "schema metadata must yield a no-tz timestamp column") + anns := tsAnnotationsPerDataFile(t, ctx, tbl) + require.Len(t, anns, 1) + for path, ann := range anns { + assert.Equal(t, map[int]bool{tsField.ID: false}, ann, + "a spec table's no-tz timestamp column must be isAdjustedToUTC=false: %s", path) + } + + // DuckDB types the column as plain TIMESTAMP and reads the exact instant. + type typeRow struct { + ColumnName string `json:"column_name"` + ColumnType string `json:"column_type"` + } + cols := querySQL[typeRow](t, ctx, infra, + fmt.Sprintf(`DESCRIBE iceberg_cat."%s"."%s";`, ns, tblName)) + typeOf := map[string]string{} + for _, c := range cols { + typeOf[c.ColumnName] = c.ColumnType + } + assert.Equal(t, "TIMESTAMP", typeOf["ts"], "spec encoding must read as a plain (no-tz) TIMESTAMP") + + match := querySQL[countResult](t, ctx, infra, fmt.Sprintf( + `SELECT COUNT(*) AS count FROM iceberg_cat."%s"."%s" WHERE ts = TIMESTAMP '2024-03-20 08:15:00';`, + ns, tblName)) + require.Len(t, match, 1) + assert.Equal(t, 1, match[0].Count, "the written instant must round-trip exactly") +} diff --git a/internal/impl/iceberg/router.go b/internal/impl/iceberg/router.go index c553352bfb..ce5f8644ab 100644 --- a/internal/impl/iceberg/router.go +++ b/internal/impl/iceberg/router.go @@ -67,6 +67,13 @@ const maxSchemaEvolutionRetries = 10 type tableEntry struct { mu sync.RWMutex writer *writer + + // tsEncoding caches the table's resolved timestamp encoding (see + // resolveTimestampEncoding) so the footer-probe/stamp bootstrap runs at + // most once per table per process — writer re-creation (schema evolution, + // error recovery) reuses it. Guarded by mu; valid when tsEncodingResolved. + tsEncoding icebergx.TimestampEncoding + tsEncodingResolved bool } // Router routes message batches to per-table writers. @@ -259,7 +266,7 @@ func (r *Router) doWrite(ctx context.Context, key tableKey, entry *tableEntry, b entry.mu.Unlock() continue } - w, err := r.createWriter(ctx, key) + w, err := r.createWriter(ctx, key, entry) if err != nil { entry.mu.Unlock() return err @@ -381,6 +388,12 @@ func (r *Router) createTable(ctx context.Context, key tableKey, batch service.Me location := tableLocationFor(r.schemaEvoCfg.TableLocation, nsParts, key.table) createOpts = append(createOpts, catalog.WithLocation(location)) } + // Pin the timestamp encoding at birth: tables created by this connector + // are spec-encoded from their first file, so the footer-probe bootstrap + // (resolveTimestampEncoding) never has to run for them. + createOpts = append(createOpts, catalog.WithProperties(iceberg.Properties{ + icebergx.TimestampEncodingProperty: icebergx.TimestampEncodingSpec.String(), + })) // Create the table _, err = client.CreateTable(ctx, key.table, schema, createOpts...) @@ -394,6 +407,11 @@ func (r *Router) createTable(ctx context.Context, key tableKey, batch service.Me return err } + // The property is stamped in the creation commit, so the resolution is + // already known — cache it to spare createWriter a parse. + entry.tsEncoding = icebergx.TimestampEncodingSpec + entry.tsEncodingResolved = true + r.logger.Infof("Created table: %s.%s with %d columns", key.namespace, key.table, len(schema.Fields())) // Invalidate cached writer so it gets recreated with the new table r.closeWriter(entry) @@ -695,8 +713,8 @@ func (*Router) closeWriter(entry *tableEntry) { } // createWriter creates a new writer for a table. -// Caller must ensure this is only called when entry.writer is nil. -func (r *Router) createWriter(ctx context.Context, key tableKey) (*writer, error) { +// Caller must hold entry.mu.Lock() and ensure entry.writer is nil. +func (r *Router) createWriter(ctx context.Context, key tableKey, entry *tableEntry) (*writer, error) { // Parse namespace into parts nsParts := strings.Split(key.namespace, ".") @@ -715,11 +733,6 @@ func (r *Router) createWriter(ctx context.Context, key tableKey) (*writer, error return nil, err } - committerTbl, err := client.LoadTable(ctx, key.table) - if err != nil { - return nil, err - } - // reloadTable creates a fresh catalog client and reloads the table, // allowing the committer to recover from stale metadata or auth errors. reloadTable := func(ctx context.Context) (*table.Table, error) { @@ -731,6 +744,31 @@ func (r *Router) createWriter(ctx context.Context, key tableKey) (*writer, error return rc.LoadTable(ctx, key.table) } + // Resolve the table's timestamp encoding before any file is written, so + // every parquet file this writer produces matches the table's existing + // annotation. When the pinning property is present on the freshly loaded + // table it is authoritative and re-read on every writer (re)creation — + // this is metadata already in hand, and it means an operator who migrates + // a table (rewrite files, flip the property to spec) is honoured at the + // next writer recreation rather than being overridden by a stale cache. + // The entry cache (guarded by entry.mu, which we hold) only short-circuits + // the property-ABSENT bootstrap, so the footer probe and pinning stamp run + // at most once per table per process. + if _, propPresent := writerTbl.Properties()[icebergx.TimestampEncodingProperty]; propPresent || !entry.tsEncodingResolved { + enc, stampedTbl, err := resolveTimestampEncoding(ctx, writerTbl, reloadTable, r.logger) + if err != nil { + return nil, err + } + entry.tsEncoding = enc + entry.tsEncodingResolved = true + writerTbl = stampedTbl + } + + committerTbl, err := client.LoadTable(ctx, key.table) + if err != nil { + return nil, err + } + // Create committer with its own table reference. Copy-on-write writes only // plain data files, so it works on a v1 table and must not trigger the // irreversible v1->v2 upgrade the merge-on-read path needs. @@ -745,7 +783,7 @@ func (r *Router) createWriter(ctx context.Context, key tableKey) (*writer, error // Create writer with its own table reference and the committer. // The resolver is passed so the writer can use schema metadata to // interpret numeric inputs into time-typed columns at shredding time. - w := NewWriter(writerTbl, comm, r.caseSensitive, r.writerOpts, r.resolver, r.schemaEvoCfg.RequireSchemaMetadata, r.rowOpCfg, r.logger) + w := NewWriter(writerTbl, comm, r.caseSensitive, r.writerOpts, r.resolver, r.schemaEvoCfg.RequireSchemaMetadata, r.rowOpCfg, entry.tsEncoding, r.logger) w.metrics = r.metrics r.logger.Debugf("Created writer for table %s.%s", key.namespace, key.table) diff --git a/internal/impl/iceberg/timestamp_encoding.go b/internal/impl/iceberg/timestamp_encoding.go new file mode 100644 index 0000000000..e305b47829 --- /dev/null +++ b/internal/impl/iceberg/timestamp_encoding.go @@ -0,0 +1,200 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md + +package iceberg + +import ( + "context" + "fmt" + + "github.com/apache/iceberg-go" + iceio "github.com/apache/iceberg-go/io" + "github.com/apache/iceberg-go/table" + "github.com/parquet-go/parquet-go" + + "github.com/redpanda-data/benthos/v4/public/service" + "github.com/redpanda-data/connect/v4/internal/impl/iceberg/icebergx" +) + +// resolveTimestampEncoding determines how no-timezone `timestamp` columns must +// be annotated in the parquet files written to tbl, guaranteeing an existing +// table never sees its encoding change or become mixed (a released connector +// version wrote the spec-incorrect isAdjustedToUTC=true "legacy" annotation). +// +// Resolution order: +// +// 1. Table property redpanda-connect.timestamp-encoding present → use it. +// An unknown value is a hard error (guessing risks a mixed table). +// 2. Property absent → bootstrap by probing the table's own files +// (probeTimestampEncoding), then STAMP the resolved value onto the table +// as a SetProperties commit so the decision is permanent and visible to +// every future writer. +// +// reload re-loads the table from the catalog; it is used to resolve a stamp +// race (two writers bootstrapping the same table concurrently). The returned +// table is tbl with the stamp applied when one was committed, otherwise tbl +// unchanged, so callers keep working with fresh metadata. +func resolveTimestampEncoding(ctx context.Context, tbl *table.Table, reload func(context.Context) (*table.Table, error), logger *service.Logger) (icebergx.TimestampEncoding, *table.Table, error) { + if v, ok := tbl.Properties()[icebergx.TimestampEncodingProperty]; ok { + enc, err := icebergx.ParseTimestampEncoding(v) + if err != nil { + return 0, nil, fmt.Errorf("table %v: %w", tbl.Identifier(), err) + } + return enc, tbl, nil + } + + enc, err := probeTimestampEncoding(ctx, tbl) + if err != nil { + return 0, nil, fmt.Errorf("resolving timestamp encoding for table %v: %w", tbl.Identifier(), err) + } + + stamped, err := stampTimestampEncoding(ctx, tbl, enc, reload) + if err != nil { + return 0, nil, fmt.Errorf("stamping timestamp encoding %q on table %v: %w", enc, tbl.Identifier(), err) + } + if logger != nil { + logger.Infof("Pinned table %v to timestamp encoding %q (table property %s)", tbl.Identifier(), enc, icebergx.TimestampEncodingProperty) + } + return enc, stamped, nil +} + +// probeTimestampEncoding bootstraps the encoding for a table that predates the +// pinning property, by inspecting what the table actually contains: +// +// - schema has no no-tz `timestamp` column → spec. There is nothing the two +// encodings disagree on, and it is correct for any timestamp column added +// later by schema evolution, since no existing file can carry that column. +// - no data files (empty table / no snapshot) → spec. +// - otherwise → open a current-snapshot data file's parquet footer and read +// the isAdjustedToUTC annotation off a no-tz timestamp column: +// true → legacy, false → spec. Files that don't carry any such column +// (e.g. written before the column was evolved in) are skipped; if no +// parquet file carries one, resolve spec for the same reason as the +// no-column case. Any read/parse failure is a hard error — guessing here +// could silently mix annotations within the table. +func probeTimestampEncoding(ctx context.Context, tbl *table.Table) (icebergx.TimestampEncoding, error) { + schema := tbl.Schema() + if !icebergx.SchemaHasNoTZTimestamp(schema) { + return icebergx.TimestampEncodingSpec, nil + } + snap := tbl.CurrentSnapshot() + if snap == nil { + return icebergx.TimestampEncodingSpec, nil + } + fsys, err := tbl.FS(ctx) + if err != nil { + return 0, fmt.Errorf("getting table filesystem: %w", err) + } + manifests, err := snap.Manifests(fsys) + if err != nil { + return 0, fmt.Errorf("listing current-snapshot manifests: %w", err) + } + for _, m := range manifests { + if m.ManifestContent() != iceberg.ManifestContentData { + continue + } + for entry, err := range m.Entries(fsys, true) { + if err != nil { + return 0, fmt.Errorf("reading manifest entries: %w", err) + } + df := entry.DataFile() + if df.FileFormat() != iceberg.ParquetFile { + // A non-parquet data file was never written by this connector, + // so it cannot carry the legacy encoding; skip it. + continue + } + enc, found, err := probeParquetFooterEncoding(fsys, df.FilePath(), schema) + if err != nil { + return 0, fmt.Errorf("probing parquet footer of %s: %w", df.FilePath(), err) + } + if found { + return enc, nil + } + // The file has no no-tz timestamp leaf (it predates the column); + // keep scanning. In the common case the very first file decides. + } + } + // Data files exist but none carries a no-tz timestamp column: the column + // was added by evolution after every current file was written, so there is + // no legacy-annotated file to stay consistent with. + return icebergx.TimestampEncodingSpec, nil +} + +// probeParquetFooterEncoding opens one parquet file's footer via the table's +// filesystem and inspects the logical-type annotation of the first leaf that +// is a no-tz `timestamp` column in the iceberg schema (matched by field ID). +// found is false when the file carries no such leaf. +func probeParquetFooterEncoding(fsys iceio.IO, path string, schema *iceberg.Schema) (enc icebergx.TimestampEncoding, found bool, err error) { + f, err := fsys.Open(path) + if err != nil { + return 0, false, err + } + defer f.Close() + info, err := f.Stat() + if err != nil { + return 0, false, err + } + // Only the footer is needed: skip the page-index and bloom-filter + // sections so the probe stays a couple of small ranged reads. + pf, err := parquet.OpenFile(f, info.Size(), parquet.SkipPageIndex(true), parquet.SkipBloomFilters(true)) + if err != nil { + return 0, false, err + } + for _, el := range pf.Metadata().Schema { + lt, ok := el.LogicalType.Get() + if !ok || lt.Timestamp == nil { + continue + } + field, ok := schema.FindFieldByID(int(el.FieldID)) + if !ok { + continue + } + if _, noTZ := field.Type.(iceberg.TimestampType); !noTZ { + continue + } + if lt.Timestamp.IsAdjustedToUTC { + return icebergx.TimestampEncodingLegacy, true, nil + } + return icebergx.TimestampEncodingSpec, true, nil + } + return 0, false, nil +} + +// stampTimestampEncoding commits the resolved encoding onto the table as the +// redpanda-connect.timestamp-encoding property, making the bootstrap decision +// permanent and visible. Two writers may race to stamp the same table: on a +// commit failure the table is reloaded and, if the property appeared +// meanwhile with our value, the race is benign and the reloaded table is +// used. A property that appeared with a DIFFERENT value is a hard error — +// both writers probed the same files, so a disagreement means something is +// wrong and writing could mix annotations. +func stampTimestampEncoding(ctx context.Context, tbl *table.Table, enc icebergx.TimestampEncoding, reload func(context.Context) (*table.Table, error)) (*table.Table, error) { + txn := tbl.NewTransaction() + if err := txn.SetProperties(iceberg.Properties{icebergx.TimestampEncodingProperty: enc.String()}); err != nil { + return nil, err + } + stamped, commitErr := txn.Commit(ctx) + if commitErr == nil { + return stamped, nil + } + // The commit can fail because a concurrent writer stamped first (or wrote + // anything else to the table). Reload and check whether the property is + // now present and agrees with our resolution. + if reload != nil { + reloaded, reloadErr := reload(ctx) + if reloadErr == nil { + if v, ok := reloaded.Properties()[icebergx.TimestampEncodingProperty]; ok { + if v == enc.String() { + return reloaded, nil + } + return nil, fmt.Errorf("concurrent writer pinned %s=%q but this writer resolved %q (commit error: %v)", icebergx.TimestampEncodingProperty, v, enc, commitErr) + } + } + } + return nil, commitErr +} diff --git a/internal/impl/iceberg/timestamp_encoding_test.go b/internal/impl/iceberg/timestamp_encoding_test.go new file mode 100644 index 0000000000..f7a8211de7 --- /dev/null +++ b/internal/impl/iceberg/timestamp_encoding_test.go @@ -0,0 +1,465 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md + +package iceberg + +import ( + "context" + "errors" + "fmt" + "maps" + "os" + "path/filepath" + "testing" + "time" + + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/table" + "github.com/google/uuid" + "github.com/parquet-go/parquet-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/redpanda-data/benthos/v4/public/service" + "github.com/redpanda-data/connect/v4/internal/impl/iceberg/icebergx" +) + +// encTestSchema is the canonical schema for timestamp-encoding tests: an id +// column, a no-tz `timestamp` column (the one the encodings disagree on) and a +// `timestamptz` column (identical in both encodings). +func encTestSchema() *iceberg.Schema { + return iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: false}, + iceberg.NestedField{ID: 2, Name: "ts", Type: iceberg.PrimitiveTypes.Timestamp, Required: false}, + iceberg.NestedField{ID: 3, Name: "tstz", Type: iceberg.PrimitiveTypes.TimestampTz, Required: false}, + ) +} + +// newEncTable builds an unpartitioned v2 table for sc over an in-memory +// catalog and the local filesystem, with extra table properties merged in. +func newEncTable(t testing.TB, sc *iceberg.Schema, extra iceberg.Properties) (*table.Table, *memCatalog) { + t.Helper() + location := filepath.ToSlash(t.TempDir()) + props := iceberg.Properties{table.PropertyFormatVersion: "2"} + maps.Copy(props, extra) + meta, err := table.NewMetadata(sc, iceberg.UnpartitionedSpec, table.UnsortedSortOrder, location, props) + require.NoError(t, err) + cat := &memCatalog{ + meta: meta, + metadataLocation: fmt.Sprintf("%s/metadata/00001-%s.metadata.json", location, uuid.New()), + ident: table.Identifier{"default", "enc"}, + location: location, + } + return cat.snapshot(), cat +} + +var encSeedTime = time.Date(2024, 1, 15, 12, 30, 45, 0, time.UTC) + +// seedEncTimestampFile appends one real parquet data file to the table through +// the connector's own shredder append path (writer.writeDataFiles) using the +// given timestamp encoding, and returns its path. Seeding with +// TimestampEncodingLegacy therefore produces a file byte-annotated exactly as +// pre-fix connector releases wrote it. +func seedEncTimestampFile(t testing.TB, ctx context.Context, cat *memCatalog, enc icebergx.TimestampEncoding) string { + t.Helper() + tbl := cat.snapshot() + // LocalFS does not create the data/ subdir implicitly. + require.NoError(t, os.MkdirAll(filepath.Join(tbl.Location(), "data"), 0o755)) + w := &writer{table: tbl, caseSensitive: true, tsEncoding: enc, logger: service.MockResources().Logger()} + files, err := w.writeDataFiles(ctx, service.MessageBatch{structuredMsg(t, map[string]any{ + "id": int64(1), "ts": encSeedTime, "tstz": encSeedTime, + })}) + require.NoError(t, err) + require.Len(t, files, 1) + tx := tbl.NewTransaction() + require.NoError(t, tx.AddDataFiles(ctx, files, nil, table.WithoutAutoNameMapping(), table.WithoutDuplicateCheck())) + _, err = tx.Commit(ctx) + require.NoError(t, err) + return files[0].FilePath() +} + +// footerTimestampAdjusted opens a parquet file's footer and returns fieldID -> +// isAdjustedToUTC for every leaf carrying a TIMESTAMP logical type. +func footerTimestampAdjusted(t testing.TB, path string) map[int]bool { + t.Helper() + f, err := os.Open(path) + require.NoError(t, err) + defer f.Close() + info, err := f.Stat() + require.NoError(t, err) + pf, err := parquet.OpenFile(f, info.Size(), parquet.SkipPageIndex(true), parquet.SkipBloomFilters(true)) + require.NoError(t, err) + out := map[int]bool{} + for _, el := range pf.Metadata().Schema { + if lt, ok := el.LogicalType.Get(); ok && lt.Timestamp != nil { + out[int(el.FieldID)] = lt.Timestamp.IsAdjustedToUTC + } + } + return out +} + +// --- append-path uniformity ------------------------------------------------ + +// TestAppendWritesResolvedTimestampEncoding pins the writer-side guarantee: a +// table resolved to an encoding gets EVERY new file annotated with it. In +// legacy mode the no-tz column keeps isAdjustedToUTC=true (identical to +// pre-fix output, so an existing table never becomes mixed); in spec mode it +// is false. `timestamptz` is UTC-adjusted in both. +func TestAppendWritesResolvedTimestampEncoding(t *testing.T) { + ctx := t.Context() + + t.Run("legacy", func(t *testing.T) { + _, cat := newEncTable(t, encTestSchema(), nil) + path := seedEncTimestampFile(t, ctx, cat, icebergx.TimestampEncodingLegacy) + ann := footerTimestampAdjusted(t, path) + assert.Equal(t, map[int]bool{2: true, 3: true}, ann, + "legacy mode must annotate the no-tz timestamp column isAdjustedToUTC=true (pre-fix bytes) and timestamptz true") + }) + + t.Run("spec", func(t *testing.T) { + _, cat := newEncTable(t, encTestSchema(), nil) + path := seedEncTimestampFile(t, ctx, cat, icebergx.TimestampEncodingSpec) + ann := footerTimestampAdjusted(t, path) + assert.Equal(t, map[int]bool{2: false, 3: true}, ann, + "spec mode must annotate the no-tz timestamp column isAdjustedToUTC=false and timestamptz true") + }) +} + +// --- resolution: property present ------------------------------------------- + +func TestResolveTimestampEncodingPropertyPresent(t *testing.T) { + ctx := t.Context() + + t.Run("spec", func(t *testing.T) { + tbl, cat := newEncTable(t, encTestSchema(), iceberg.Properties{icebergx.TimestampEncodingProperty: "spec"}) + enc, out, err := resolveTimestampEncoding(ctx, tbl, reloadFn(cat), nil) + require.NoError(t, err) + assert.Equal(t, icebergx.TimestampEncodingSpec, enc) + assert.Same(t, tbl, out, "a present property must be used as-is, with no stamp commit") + }) + + t.Run("legacy", func(t *testing.T) { + tbl, cat := newEncTable(t, encTestSchema(), iceberg.Properties{icebergx.TimestampEncodingProperty: "legacy"}) + enc, _, err := resolveTimestampEncoding(ctx, tbl, reloadFn(cat), nil) + require.NoError(t, err) + assert.Equal(t, icebergx.TimestampEncodingLegacy, enc) + }) + + t.Run("unknown value fails loud", func(t *testing.T) { + tbl, cat := newEncTable(t, encTestSchema(), iceberg.Properties{icebergx.TimestampEncodingProperty: "sideways"}) + _, _, err := resolveTimestampEncoding(ctx, tbl, reloadFn(cat), nil) + require.Error(t, err) + assert.Contains(t, err.Error(), icebergx.TimestampEncodingProperty) + assert.Contains(t, err.Error(), "sideways") + }) + + // The property, when present, must win WITHOUT touching any data file: + // files pinned legacy stay legacy even if unreadable. + t.Run("no probe when property present", func(t *testing.T) { + _, cat := newEncTable(t, encTestSchema(), iceberg.Properties{icebergx.TimestampEncodingProperty: "legacy"}) + path := seedEncTimestampFile(t, ctx, cat, icebergx.TimestampEncodingLegacy) + require.NoError(t, os.Remove(path), "removing the data file so any probe attempt would fail") + enc, _, err := resolveTimestampEncoding(ctx, cat.snapshot(), reloadFn(cat), nil) + require.NoError(t, err, "resolution must not read data files when the property is present") + assert.Equal(t, icebergx.TimestampEncodingLegacy, enc) + }) +} + +// --- resolution: property absent (footer-probe bootstrap + stamp) ------------ + +func TestResolveTimestampEncodingBootstrap(t *testing.T) { + ctx := t.Context() + logger := service.MockResources().Logger() + + stampedValue := func(cat *memCatalog) string { + return cat.snapshot().Properties()[icebergx.TimestampEncodingProperty] + } + + t.Run("no no-tz timestamp columns resolves spec", func(t *testing.T) { + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64}, + iceberg.NestedField{ID: 2, Name: "tstz", Type: iceberg.PrimitiveTypes.TimestampTz}, + ) + tbl, cat := newEncTable(t, sc, nil) + enc, out, err := resolveTimestampEncoding(ctx, tbl, reloadFn(cat), logger) + require.NoError(t, err) + assert.Equal(t, icebergx.TimestampEncodingSpec, enc) + assert.Equal(t, "spec", stampedValue(cat), "the decision must be stamped onto the table") + assert.Equal(t, "spec", out.Properties()[icebergx.TimestampEncodingProperty], "the returned table must carry the stamp") + }) + + t.Run("empty table resolves spec", func(t *testing.T) { + tbl, cat := newEncTable(t, encTestSchema(), nil) + enc, _, err := resolveTimestampEncoding(ctx, tbl, reloadFn(cat), logger) + require.NoError(t, err) + assert.Equal(t, icebergx.TimestampEncodingSpec, enc) + assert.Equal(t, "spec", stampedValue(cat)) + }) + + t.Run("legacy data file resolves legacy", func(t *testing.T) { + _, cat := newEncTable(t, encTestSchema(), nil) + seedEncTimestampFile(t, ctx, cat, icebergx.TimestampEncodingLegacy) + enc, _, err := resolveTimestampEncoding(ctx, cat.snapshot(), reloadFn(cat), logger) + require.NoError(t, err) + assert.Equal(t, icebergx.TimestampEncodingLegacy, enc, + "a table whose files carry isAdjustedToUTC=true must pin legacy") + assert.Equal(t, "legacy", stampedValue(cat)) + }) + + t.Run("spec data file resolves spec", func(t *testing.T) { + _, cat := newEncTable(t, encTestSchema(), nil) + seedEncTimestampFile(t, ctx, cat, icebergx.TimestampEncodingSpec) + enc, _, err := resolveTimestampEncoding(ctx, cat.snapshot(), reloadFn(cat), logger) + require.NoError(t, err) + assert.Equal(t, icebergx.TimestampEncodingSpec, enc) + assert.Equal(t, "spec", stampedValue(cat)) + }) + + t.Run("unreadable data file fails loud", func(t *testing.T) { + tbl, cat := newEncTable(t, encTestSchema(), nil) + // Register a data file whose content is not parquet: the probe must + // error rather than guess an encoding. + junkPath := filepath.Join(tbl.Location(), "data", "junk.parquet") + require.NoError(t, os.MkdirAll(filepath.Dir(junkPath), 0o755)) + require.NoError(t, os.WriteFile(junkPath, []byte("not parquet"), 0o644)) + tx := tbl.NewTransaction() + require.NoError(t, tx.AddDataFiles(ctx, []iceberg.DataFile{synthDataFile(t, tbl.Spec(), filepath.ToSlash(junkPath))}, nil, + table.WithoutAutoNameMapping(), table.WithoutDuplicateCheck())) + _, err := tx.Commit(ctx) + require.NoError(t, err) + + _, _, err = resolveTimestampEncoding(ctx, cat.snapshot(), reloadFn(cat), logger) + require.Error(t, err, "an unreadable footer must fail resolution, not guess") + assert.Contains(t, err.Error(), "probing parquet footer") + assert.Empty(t, stampedValue(cat), "no stamp may be committed when the probe fails") + }) + + t.Run("stamp makes re-resolution probe-free", func(t *testing.T) { + _, cat := newEncTable(t, encTestSchema(), nil) + path := seedEncTimestampFile(t, ctx, cat, icebergx.TimestampEncodingLegacy) + + enc, _, err := resolveTimestampEncoding(ctx, cat.snapshot(), reloadFn(cat), logger) + require.NoError(t, err) + require.Equal(t, icebergx.TimestampEncodingLegacy, enc) + require.Equal(t, "legacy", stampedValue(cat)) + + // Remove the data file from disk: a second resolution (e.g. a new + // process) must ride the stamped property and never probe again. + require.NoError(t, os.Remove(path)) + enc, _, err = resolveTimestampEncoding(ctx, cat.snapshot(), reloadFn(cat), logger) + require.NoError(t, err, "re-resolution must use the stamp, not the (now missing) file") + assert.Equal(t, icebergx.TimestampEncodingLegacy, enc) + }) + + // A data file that predates the timestamp column (added later by schema + // evolution) has no timestamp leaf to inspect; when no file carries the + // column the table resolves spec — no legacy file can exist for it. + t.Run("files without the timestamp column resolve spec", func(t *testing.T) { + idOnly := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: false}, + ) + _, cat := newEncTable(t, idOnly, nil) + tbl := cat.snapshot() + require.NoError(t, os.MkdirAll(filepath.Join(tbl.Location(), "data"), 0o755)) + w := &writer{table: tbl, caseSensitive: true, logger: service.MockResources().Logger()} + files, err := w.writeDataFiles(ctx, service.MessageBatch{structuredMsg(t, map[string]any{"id": int64(1)})}) + require.NoError(t, err) + tx := tbl.NewTransaction() + require.NoError(t, tx.AddDataFiles(ctx, files, nil, table.WithoutAutoNameMapping(), table.WithoutDuplicateCheck())) + _, err = tx.Commit(ctx) + require.NoError(t, err) + + // Evolve the schema: add the ts column AFTER the file was written. + tx = cat.snapshot().NewTransaction() + us := tx.UpdateSchema(true, false) + us.AddColumn([]string{"ts"}, iceberg.PrimitiveTypes.Timestamp, "", false, nil) + require.NoError(t, us.Commit()) + _, err = tx.Commit(ctx) + require.NoError(t, err) + + enc, _, err := resolveTimestampEncoding(ctx, cat.snapshot(), reloadFn(cat), logger) + require.NoError(t, err) + assert.Equal(t, icebergx.TimestampEncodingSpec, enc) + assert.Equal(t, "spec", stampedValue(cat)) + }) +} + +// --- stamping race ------------------------------------------------------------ + +// TestStampTimestampEncodingRace covers two writers bootstrapping the same +// table concurrently: our SetProperties commit loses, and the outcome depends +// on what the winner stamped. +func TestStampTimestampEncodingRace(t *testing.T) { + ctx := t.Context() + // Schema without timestamp columns: the probe trivially resolves spec with + // no file access, isolating the stamping behaviour under test. + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64}, + ) + + failingTable := func(t *testing.T) *table.Table { + _, cat := newEncTable(t, sc, nil) + flaky := &flakyCatalog{memCatalog: cat, failuresLeft: 1 << 30, failErr: errors.New("commit conflict")} + return flaky.snapshot() + } + + t.Run("winner stamped the same value", func(t *testing.T) { + winner, _ := newEncTable(t, sc, iceberg.Properties{icebergx.TimestampEncodingProperty: "spec"}) + reload := func(context.Context) (*table.Table, error) { return winner, nil } + enc, out, err := resolveTimestampEncoding(ctx, failingTable(t), reload, nil) + require.NoError(t, err, "losing the stamp race to an agreeing writer must be benign") + assert.Equal(t, icebergx.TimestampEncodingSpec, enc) + assert.Same(t, winner, out, "the reloaded (stamped) table must be adopted") + }) + + t.Run("winner stamped a different value", func(t *testing.T) { + winner, _ := newEncTable(t, sc, iceberg.Properties{icebergx.TimestampEncodingProperty: "legacy"}) + reload := func(context.Context) (*table.Table, error) { return winner, nil } + _, _, err := resolveTimestampEncoding(ctx, failingTable(t), reload, nil) + require.Error(t, err, "a disagreeing concurrent stamp must fail loud") + assert.Contains(t, err.Error(), "concurrent writer") + }) + + t.Run("no concurrent stamp propagates the commit error", func(t *testing.T) { + bare, _ := newEncTable(t, sc, nil) + reload := func(context.Context) (*table.Table, error) { return bare, nil } + _, _, err := resolveTimestampEncoding(ctx, failingTable(t), reload, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "commit conflict") + }) +} + +// --- copy-on-write guard -------------------------------------------------------- + +// TestCOWLegacyTimestampGuard pins the guard that keeps mutating copy-on-write +// off legacy-pinned tables with no-tz timestamp columns: iceberg-go reads their +// UTC-adjusted files back as timestamptz and the rewrite fails with a cryptic +// "cannot promote timestamptz to timestamp" mid-commit — the guard converts +// that into an upfront, actionable error before any file is written. +func TestCOWLegacyTimestampGuard(t *testing.T) { + ctx := t.Context() + + newCOWEncWriter := func(t *testing.T, cat *memCatalog, enc icebergx.TimestampEncoding) *writer { + t.Helper() + comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3, SkipFormatUpgrade: true}, reloadFn(cat), service.MockResources().Logger()) + require.NoError(t, err) + t.Cleanup(comm.Close) + w := cowWriter(t, cat.snapshot(), "id") + w.committer = comm + w.tsEncoding = enc + return w + } + + t.Run("mutating write on legacy table errors upfront", func(t *testing.T) { + _, cat := newEncTable(t, encTestSchema(), nil) + seedEncTimestampFile(t, ctx, cat, icebergx.TimestampEncodingLegacy) + w := newCOWEncWriter(t, cat, icebergx.TimestampEncodingLegacy) + + err := w.Write(ctx, service.MessageBatch{ + cowMsg(t, "upsert", map[string]any{"id": int64(1), "ts": encSeedTime, "tstz": encSeedTime}), + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "legacy UTC-adjusted parquet encoding", + "the guard must name the problem") + assert.Contains(t, err.Error(), icebergx.TimestampEncodingProperty+"=spec", + "the guard must give the migration path") + assert.Contains(t, err.Error(), "merge-on-read", + "the guard must offer the strategy alternative") + + // Delete-only mutations rewrite files too and must hit the same guard. + err = w.Write(ctx, service.MessageBatch{cowMsg(t, "delete", map[string]any{"id": int64(1)})}) + require.Error(t, err) + assert.Contains(t, err.Error(), "legacy UTC-adjusted parquet encoding") + }) + + t.Run("insert-only batch on legacy table succeeds", func(t *testing.T) { + _, cat := newEncTable(t, encTestSchema(), nil) + seedEncTimestampFile(t, ctx, cat, icebergx.TimestampEncodingLegacy) + w := newCOWEncWriter(t, cat, icebergx.TimestampEncodingLegacy) + + require.NoError(t, w.Write(ctx, service.MessageBatch{ + cowMsg(t, "insert", map[string]any{"id": int64(2), "ts": encSeedTime, "tstz": encSeedTime}), + }), "insert-only copy-on-write is a plain append and must not be guarded") + + // Uniformity: the appended file must carry the legacy annotation. + final := cat.snapshot() + snap := final.CurrentSnapshot() + require.NotNil(t, snap) + fsys, err := final.FS(ctx) + require.NoError(t, err) + manifests, err := snap.Manifests(fsys) + require.NoError(t, err) + checked := 0 + for _, m := range manifests { + for entry, err := range m.Entries(fsys, true) { + require.NoError(t, err) + ann := footerTimestampAdjusted(t, entry.DataFile().FilePath()) + assert.Equal(t, map[int]bool{2: true, 3: true}, ann, + "every file of a legacy table must stay legacy-annotated: %s", entry.DataFile().FilePath()) + checked++ + } + } + assert.Equal(t, 2, checked, "expected the seed file and the appended file") + }) + + t.Run("mutating write on spec table unaffected", func(t *testing.T) { + _, cat := newEncTable(t, encTestSchema(), nil) + seedEncTimestampFile(t, ctx, cat, icebergx.TimestampEncodingSpec) + w := newCOWEncWriter(t, cat, icebergx.TimestampEncodingSpec) + + upsertAt := encSeedTime.Add(time.Hour) + require.NoError(t, w.Write(ctx, service.MessageBatch{ + cowMsg(t, "upsert", map[string]any{"id": int64(1), "ts": upsertAt, "tstz": upsertAt}), + }), "copy-on-write over spec-encoded files must keep working") + assert.Equal(t, 1, countRowsWithID(t, ctx, cat.snapshot(), "id", 1), "the upsert must not duplicate the row") + }) + + t.Run("legacy table without timestamp columns unaffected", func(t *testing.T) { + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: false}, + iceberg.NestedField{ID: 2, Name: "payload", Type: iceberg.PrimitiveTypes.String, Required: false}, + ) + _, cat := newEncTable(t, sc, nil) + seed := appendCOWRows(t, ctx, cat.snapshot(), map[int64]string{1: "one"}) + _ = seed + w := newCOWEncWriter(t, cat, icebergx.TimestampEncodingLegacy) + + require.NoError(t, w.Write(ctx, service.MessageBatch{ + cowMsg(t, "upsert", map[string]any{"id": int64(1), "payload": "ONE"}), + }), "the guard must only fire when the schema actually has a no-tz timestamp column") + assert.Equal(t, map[int64]string{1: "ONE"}, scanRows(t, ctx, cat.snapshot())) + }) + + t.Run("merge-on-read on legacy table unaffected", func(t *testing.T) { + _, cat := newEncTable(t, encTestSchema(), nil) + seedEncTimestampFile(t, ctx, cat, icebergx.TimestampEncodingLegacy) + + comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) + require.NoError(t, err) + defer comm.Close() + w := &writer{ + table: cat.snapshot(), + committer: comm, + caseSensitive: true, + tsEncoding: icebergx.TimestampEncodingLegacy, + rowOpCfg: RowOpConfig{ + Operation: mustInterp(t, `${! metadata("op") }`), + IdentifierFields: []string{"id"}, + MergeStrategy: mergeStrategyMOR, + }, + logger: service.MockResources().Logger(), + } + + require.NoError(t, w.Write(ctx, service.MessageBatch{ + cowMsg(t, "upsert", map[string]any{"id": int64(1), "ts": encSeedTime.Add(time.Hour), "tstz": encSeedTime}), + }), "merge-on-read must keep working on a legacy table (no file rewrites)") + snap := cat.snapshot().CurrentSnapshot() + require.NotNil(t, snap) + assert.Equal(t, table.OpOverwrite, snap.Summary.Operation, "the upsert must land as a row-delta overwrite") + }) +} diff --git a/internal/impl/iceberg/writer.go b/internal/impl/iceberg/writer.go index e63db48aa3..beaed229fa 100644 --- a/internal/impl/iceberg/writer.go +++ b/internal/impl/iceberg/writer.go @@ -134,6 +134,14 @@ type writer struct { metrics *opMetrics logger *service.Logger + // tsEncoding is the table's resolved timestamp encoding: how no-timezone + // `timestamp` columns are annotated in the parquet files this writer + // produces. It is resolved once per table (from the + // redpanda-connect.timestamp-encoding property, footer-probed and stamped + // when absent — see resolveTimestampEncoding) so a table's data files + // never mix annotations. The zero value is the spec encoding. + tsEncoding icebergx.TimestampEncoding + // coerceLoggedFieldIDs tracks the iceberg field IDs we have already // logged a coerce-on-write notice for, so that a long-running writer // emits the divergence between schema metadata and existing column @@ -150,7 +158,10 @@ type writer struct { // to interpret numeric inputs into time-typed columns; pass nil to disable. // requireSchemaMetadata enables shredder strict mode — see // [shredder.RecordShredder.StrictTemporalMode]. -func NewWriter(tbl *table.Table, comm *committer, caseSensitive bool, writerOpts []parquet.WriterOption, resolver *typeResolver, requireSchemaMetadata bool, rowOpCfg RowOpConfig, logger *service.Logger) *writer { +// tsEncoding is the table's resolved timestamp encoding (see +// resolveTimestampEncoding); it must match the annotation carried by the +// table's existing data files. +func NewWriter(tbl *table.Table, comm *committer, caseSensitive bool, writerOpts []parquet.WriterOption, resolver *typeResolver, requireSchemaMetadata bool, rowOpCfg RowOpConfig, tsEncoding icebergx.TimestampEncoding, logger *service.Logger) *writer { return &writer{ table: tbl, committer: comm, @@ -159,6 +170,7 @@ func NewWriter(tbl *table.Table, comm *committer, caseSensitive bool, writerOpts resolver: resolver, requireSchemaMetadata: requireSchemaMetadata, rowOpCfg: rowOpCfg, + tsEncoding: tsEncoding, logger: logger, coerceLoggedFieldIDs: map[int]struct{}{}, } @@ -273,7 +285,7 @@ func (w *writer) writeDataFiles(ctx context.Context, batch service.MessageBatch) } // Build field ID mappings for stats extraction and partition data - _, fieldToCol, err := icebergx.BuildParquetSchema(w.table.Schema()) + _, fieldToCol, err := icebergx.BuildParquetSchema(w.table.Schema(), w.tsEncoding) if err != nil { return nil, fmt.Errorf("building parquet schema: %w", err) } @@ -705,7 +717,7 @@ func (w *writer) messagesToParquet(batch service.MessageBatch) ([]partitionFile, spec := w.table.Spec() // Build parquet schema and field ID to column index mapping - pqSchema, fieldToCol, err := icebergx.BuildParquetSchema(schema) + pqSchema, fieldToCol, err := icebergx.BuildParquetSchema(schema, w.tsEncoding) if err != nil { return nil, fmt.Errorf("building parquet schema: %w", err) } From b501e539adda0f52da84ca50c7c67e239d1a9a87 Mon Sep 17 00:00:00 2001 From: Ashley Jeffs Date: Wed, 29 Jul 2026 10:07:42 +0100 Subject: [PATCH 06/12] iceberg: add Databricks Unity Catalog e2e harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A scripted setup/test/teardown harness for validating the iceberg output against a real Databricks Unity Catalog, following the existing e2e pattern (terraform + Taskfile + flag-driven tests driving the real router). Terraform provisions an isolated random-suffixed catalog, a pre-created schema, a 2X-Small serverless SQL warehouse with one-minute auto-stop, and the grants an external Iceberg REST client needs (including EXTERNAL USE SCHEMA); teardown is a terraform destroy. The tests cover the questions that only a real engine-backed catalog can answer: a copy-on-write insert/upsert/delete round-trip read back through Databricks' own SQL engine (via the statement-execution API, asserting exact final state, timestamp types, and zero delete files); a reproduction of Unity Catalog rejecting table creation that registers identifier-field-ids (which fails loudly in reverse if that behaviour ever changes); a non-fatal diagnostic recording how Unity Catalog actually handles a merge-on-read equality-delete commit; and a flag-gated commit-latency benchmark. The auth token travels via environment only — never terraform state, outputs, or flags. Everything skips cleanly when unconfigured, and the attributes that cannot be verified without live workspace access are marked in comments and the README. --- .../impl/iceberg/e2e/databricks/README.md | 93 +++ .../impl/iceberg/e2e/databricks/Taskfile.yml | 61 ++ .../impl/iceberg/e2e/databricks/e2e_test.go | 607 ++++++++++++++++++ .../iceberg/e2e/databricks/terraform/main.tf | 134 ++++ .../e2e/databricks/terraform/outputs.tf | 19 + .../e2e/databricks/terraform/terraform.yml | 22 + .../e2e/databricks/terraform/variables.tf | 34 + 7 files changed, 970 insertions(+) create mode 100644 internal/impl/iceberg/e2e/databricks/README.md create mode 100644 internal/impl/iceberg/e2e/databricks/Taskfile.yml create mode 100644 internal/impl/iceberg/e2e/databricks/e2e_test.go create mode 100644 internal/impl/iceberg/e2e/databricks/terraform/main.tf create mode 100644 internal/impl/iceberg/e2e/databricks/terraform/outputs.tf create mode 100644 internal/impl/iceberg/e2e/databricks/terraform/terraform.yml create mode 100644 internal/impl/iceberg/e2e/databricks/terraform/variables.tf diff --git a/internal/impl/iceberg/e2e/databricks/README.md b/internal/impl/iceberg/e2e/databricks/README.md new file mode 100644 index 0000000000..a6e7d2f92d --- /dev/null +++ b/internal/impl/iceberg/e2e/databricks/README.md @@ -0,0 +1,93 @@ +# Databricks Unity Catalog e2e — iceberg copy-on-write + +Validates the iceberg output's `merge_strategy: copy-on-write` against a real +Databricks Unity Catalog: writes go through the UC Iceberg REST endpoint +(`https:///api/2.1/unity-catalog/iceberg-rest`), reads come back through +a serverless SQL warehouse via the SQL Statement Execution API. + +## Prerequisites + +- A Databricks workspace on **Premium or above** with **serverless SQL + warehouses enabled**, attached to a Unity Catalog metastore. +- **Metastore external access** must be on for the Iceberg REST endpoint to + work. Flipping it needs a **METASTORE ADMIN** (usually a one-time manual + action; ask your account/metastore admin if that isn't you): + + ```sh + databricks metastores summary # note the metastore id + databricks metastores update --json '{"external_access_enabled": true}' + ``` + + Alternatively set `manage_external_access = true` (plus `metastore_id`) and + terraform runs that CLI call for you. +- **Storage root check**: if `databricks metastores summary` shows no default + storage root (common on auto-provisioned metastores), catalog creation needs + an explicit one — set `TF_VAR_storage_root=s3://bucket/prefix` (or the + `storage_root` variable) before applying. + +## Auth + +Quick-start with a personal access token (User Settings → Developer → Access +tokens in the workspace UI, or `databricks tokens create`): + +```sh +export DATABRICKS_HOST="https://dbc-abc123.cloud.databricks.com" +export DATABRICKS_TOKEN="dapi..." +export TF_VAR_workspace_host="$DATABRICKS_HOST" +``` + +Both terraform and the tests read the token from `DATABRICKS_TOKEN` only — it +is never a terraform variable/output, taskfile var, or test flag, so it can't +end up in state or logs. + +OAuth2 (M2M service principal) also works for terraform, but the tests use the +PAT bearer token for the Iceberg REST client deliberately: community reports +intermittent 500s using OAuth2 tokens against the UC IRC endpoint. + +## Running + +```sh +task terraform:apply # catalog + schema + serverless warehouse + grants +task test # the e2e suite (skips itself if unconfigured) +task terraform:destroy # tear everything down +``` + +`task bench` runs the commit-latency measurements; `task full` chains +apply → test → destroy (destroy is deferred, so it still runs when tests +fail — but if apply itself dies partway, run `task terraform:destroy` +manually). + +Tests use unique per-run table names and drop their tables via +`DROP TABLE IF EXISTS` on cleanup, so repeated `task test` runs don't need a +terraform re-apply. + +## Cost + +Minimal: one 2X-Small serverless SQL warehouse with `auto_stop_mins = 1` +(statement submission auto-restarts it), and a few thousand tiny rows at most +(the bench writes ~20k rows total). Destroy when done and nothing keeps +billing. + +## Permission asterisks (and fallbacks) + +1. **`EXTERNAL USE SCHEMA`** is not part of `ALL PRIVILEGES` and only the + *catalog owner* can grant it. Terraform's principal creates the catalog and + so owns it, which is why the self-grant in `main.tf` should work. Fallback + if the grant fails: have the catalog owner run + `GRANT EXTERNAL USE SCHEMA ON CATALOG TO ` in a SQL + editor. +2. **`external_access_enabled`** on the metastore needs METASTORE ADMIN. + Fallback: the one-line CLI call above, run once by an admin — after that + `manage_external_access` can stay `false` forever. + +## Unverified until the first live run + +Written before live credentials existed; verify these on first contact: + +- the `databricks_grants` privilege string `EXTERNAL_USE_SCHEMA` (and whether + UC tolerates the redundant self-grant to the owner) — `terraform/main.tf`; +- the `null_resource` local-exec `databricks metastores update` invocation — + `terraform/main.tf`; +- the UC Iceberg REST behaviours the tests probe (CREATE TABLE acceptance, + identifier-field-ids rejection wording, set-properties commits for the + timestamp-encoding pin, equality-delete commit handling) — `e2e_test.go`. diff --git a/internal/impl/iceberg/e2e/databricks/Taskfile.yml b/internal/impl/iceberg/e2e/databricks/Taskfile.yml new file mode 100644 index 0000000000..1ae3cdb845 --- /dev/null +++ b/internal/impl/iceberg/e2e/databricks/Taskfile.yml @@ -0,0 +1,61 @@ +version: '3' + +# The bearer token is read by go test (and terraform) directly from the +# DATABRICKS_TOKEN environment variable — it is never a taskfile var, test +# flag, or terraform output, so it cannot leak into logs or state. + +vars: + GIT_ROOT: + sh: git rev-parse --show-toplevel + DBX_HOST: + sh: cd terraform && terraform output -raw workspace_host 2>/dev/null || echo "" + DBX_CATALOG: + sh: cd terraform && terraform output -raw catalog_name 2>/dev/null || echo "" + DBX_SCHEMA: + sh: cd terraform && terraform output -raw schema_name 2>/dev/null || echo "" + DBX_WAREHOUSE_ID: + sh: cd terraform && terraform output -raw warehouse_id 2>/dev/null || echo "" + +includes: + terraform: + taskfile: ./terraform/terraform.yml + dir: terraform + +tasks: + test: + desc: Run Databricks Unity Catalog copy-on-write e2e tests + dir: '{{.GIT_ROOT}}' + cmds: + - >- + go test -v -timeout 20m + -run TestDatabricksE2E + ./internal/impl/iceberg/e2e/databricks/... + -databricks.host={{.DBX_HOST}} + -databricks.catalog={{.DBX_CATALOG}} + -databricks.schema={{.DBX_SCHEMA}} + -databricks.warehouse-id={{.DBX_WAREHOUSE_ID}} + + bench: + desc: Run the copy-on-write commit latency bench (tiny data, costs a little warehouse time) + dir: '{{.GIT_ROOT}}' + cmds: + - >- + go test -v -timeout 30m + -run TestDatabricksE2E_CommitLatencyBench + ./internal/impl/iceberg/e2e/databricks/... + -databricks.host={{.DBX_HOST}} + -databricks.catalog={{.DBX_CATALOG}} + -databricks.schema={{.DBX_SCHEMA}} + -databricks.warehouse-id={{.DBX_WAREHOUSE_ID}} + -databricks.bench + + full: + desc: Provision, test, tear down. Destroy is deferred so it runs even when tests fail (needs task >= 3.15); if apply itself dies partway, run `task terraform:destroy` manually. + cmds: + - task: terraform:init + - task: terraform:apply + - defer: + task: terraform:destroy + # Spawned as a fresh `task` process (not a task: reference) so the + # DBX_* vars above are re-evaluated AFTER apply created the outputs. + - task test diff --git a/internal/impl/iceberg/e2e/databricks/e2e_test.go b/internal/impl/iceberg/e2e/databricks/e2e_test.go new file mode 100644 index 0000000000..ba54bcc486 --- /dev/null +++ b/internal/impl/iceberg/e2e/databricks/e2e_test.go @@ -0,0 +1,607 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/connect/blob/main/licenses/rcl.md + +// Package databrickse2e validates the iceberg output's copy-on-write +// (merge_strategy: copy-on-write) feature against a REAL Databricks Unity +// Catalog, reached over its Iceberg REST endpoint +// (https:///api/2.1/unity-catalog/iceberg-rest). Written rows are read +// back through a serverless SQL warehouse via the SQL Statement Execution API +// (implemented with plain net/http — no extra module dependencies). +// +// Infrastructure (catalog, schema, SQL warehouse, grants) is provisioned by +// ./terraform — see README.md. The UC schema is pre-created there because +// client-side namespace creation against UC's Iceberg REST catalog is +// unverified; the tests never call CreateNamespace. +// +// UNVERIFIED-WITHOUT-LIVE-ACCESS: these tests were written before live +// Databricks credentials were available. The exact UC Iceberg REST behaviours +// they probe (CREATE TABLE acceptance, rejection wording for +// identifier-field-ids, set-properties commits for the timestamp-encoding +// pin, equality-delete commit handling) are asserted per official docs and +// field reports and must be confirmed on the first live run. +package databrickse2e + +import ( + "bytes" + "context" + "encoding/json" + "flag" + "fmt" + "io" + "net/http" + "os" + "strconv" + "strings" + "testing" + "time" + + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/table" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/redpanda-data/benthos/v4/public/service" + + icebergimpl "github.com/redpanda-data/connect/v4/internal/impl/iceberg" + "github.com/redpanda-data/connect/v4/internal/impl/iceberg/catalogx" +) + +var ( + dbxHost = flag.String("databricks.host", "", "Databricks workspace host, e.g. dbc-abc123.cloud.databricks.com (scheme optional)") + dbxCatalog = flag.String("databricks.catalog", "", "Unity Catalog catalog name (used as the Iceberg REST warehouse)") + dbxSchema = flag.String("databricks.schema", "e2e", "Unity Catalog schema (the Iceberg namespace); must be pre-created by terraform") + dbxWarehouseID = flag.String("databricks.warehouse-id", "", "SQL warehouse ID used to query written data back") + dbxBench = flag.Bool("databricks.bench", false, "run the commit latency bench test (costs a little warehouse time)") +) + +// dbxToken returns the PAT bearer token. It is deliberately sourced from the +// environment only — never a flag, terraform variable, or output — so it can +// never end up in logs, task output, or terraform state. +func dbxToken() string { + return os.Getenv("DATABRICKS_TOKEN") +} + +func skipIfNotConfigured(t *testing.T) { + t.Helper() + if *dbxHost == "" || *dbxCatalog == "" || *dbxWarehouseID == "" { + t.Skip("set -databricks.host, -databricks.catalog, -databricks.warehouse-id flags to run Databricks e2e tests") + } + if dbxToken() == "" { + t.Skip("set the DATABRICKS_TOKEN environment variable to run Databricks e2e tests") + } +} + +// normalizeHost accepts a bare host or a full https:// URL and returns the +// bare host without scheme or trailing slash. +func normalizeHost(h string) string { + h = strings.TrimPrefix(h, "https://") + h = strings.TrimPrefix(h, "http://") + return strings.TrimSuffix(h, "/") +} + +// redact removes the bearer token from a string before it is logged or +// embedded in an error. The token only ever travels in Authorization headers +// (which are never logged), so this is belt-and-braces for response bodies. +func redact(s string) string { + if tok := dbxToken(); tok != "" { + return strings.ReplaceAll(s, tok, "[REDACTED]") + } + return s +} + +// buildCatalogConfig points catalogx at the Unity Catalog Iceberg REST +// endpoint. The UC catalog name is passed as the Iceberg `warehouse`; UC's +// /v1/config response supplies the prefix. Auth is a PAT bearer token — +// recommended over OAuth2 for this endpoint (community reports intermittent +// 500s with OAuth2 M2M against the UC IRC). +func buildCatalogConfig() catalogx.Config { + return catalogx.Config{ + URL: fmt.Sprintf("https://%s/api/2.1/unity-catalog/iceberg-rest", normalizeHost(*dbxHost)), + Warehouse: *dbxCatalog, + AuthType: "bearer", + BearerToken: dbxToken(), + } +} + +func newCatalogClient(t *testing.T, ctx context.Context) *catalogx.Client { + t.Helper() + client, err := catalogx.NewCatalogClient(ctx, buildCatalogConfig(), []string{*dbxSchema}) + require.NoError(t, err) + t.Cleanup(func() { _ = client.Close() }) + return client +} + +// newRouter mirrors the polaris-aws sibling but takes a RowOpConfig so both +// copy-on-write and merge-on-read modes are testable. +func newRouter(t *testing.T, namespace, tableName string, schemaEvo bool, rowOp icebergimpl.RowOpConfig) *icebergimpl.Router { + t.Helper() + namespaceStr, err := service.NewInterpolatedString(namespace) + require.NoError(t, err) + tableStr, err := service.NewInterpolatedString(tableName) + require.NoError(t, err) + + logger := service.MockResources().Logger() + commitCfg := icebergimpl.CommitConfig{ + ManifestMergeEnabled: true, + MaxSnapshotAge: 24 * time.Hour, + MaxRetries: 3, + } + schemaEvoCfg := icebergimpl.SchemaEvolutionConfig{ + Enabled: schemaEvo, + } + router := icebergimpl.NewRouter(buildCatalogConfig(), namespaceStr, tableStr, true, schemaEvoCfg, commitCfg, rowOp, nil, logger) + t.Cleanup(func() { router.Close() }) + return router +} + +// cowRowOp builds the copy-on-write row-operation config mirroring the YAML +// +// row_operation: ${! meta("op") } +// identifier_fields: [id] +// merge_strategy: copy-on-write +func cowRowOp(t *testing.T) icebergimpl.RowOpConfig { + t.Helper() + op, err := service.NewInterpolatedString(`${! meta("op") }`) + require.NoError(t, err) + return icebergimpl.RowOpConfig{ + Operation: op, + IdentifierFields: []string{"id"}, + MergeStrategy: icebergimpl.MergeStrategyCOW, + } +} + +// morRowOp is the same but with the default merge-on-read strategy (the zero +// MergeStrategy value behaves as merge-on-read). +func morRowOp(t *testing.T) icebergimpl.RowOpConfig { + t.Helper() + op, err := service.NewInterpolatedString(`${! meta("op") }`) + require.NoError(t, err) + return icebergimpl.RowOpConfig{ + Operation: op, + IdentifierFields: []string{"id"}, + } +} + +// opRow builds a structured message whose `op` metadata drives the +// row_operation interpolation, mirroring how a CDC source would map its +// operation onto the iceberg output. Structured (not raw JSON) so integer ids +// stay int64 and time.Time values map onto timestamp columns. +func opRow(op string, fields map[string]any) *service.Message { + m := service.NewMessage(nil) + m.SetStructured(fields) + m.MetaSetMut("op", op) + return m +} + +func produce(t *testing.T, ctx context.Context, router *icebergimpl.Router, batch service.MessageBatch) { + t.Helper() + require.NoError(t, router.Route(ctx, batch)) + time.Sleep(2 * time.Second) +} + +func uniqueTableName(prefix string) string { + return fmt.Sprintf("%s_%d", prefix, time.Now().UnixNano()) +} + +// fqTable returns the backtick-quoted three-level Databricks SQL name. +func fqTable(tableName string) string { + return fmt.Sprintf("`%s`.`%s`.`%s`", *dbxCatalog, *dbxSchema, tableName) +} + +// --- SQL Statement Execution API (query-back through the warehouse) --- + +// sqlPollTimeout bounds polling after the initial server-side wait. Statement +// submission auto-starts a stopped serverless warehouse, and serverless +// cold-start is typically a few seconds, so 30s wait + 60s poll is generous. +const sqlPollTimeout = 60 * time.Second + +type sqlStatementResponse struct { + StatementID string `json:"statement_id"` + Status struct { + State string `json:"state"` + Error struct { + Message string `json:"message"` + } `json:"error"` + } `json:"status"` + Manifest struct { + Schema struct { + Columns []struct { + Name string `json:"name"` + TypeName string `json:"type_name"` + } `json:"columns"` + } `json:"schema"` + } `json:"manifest"` + Result struct { + DataArray [][]*string `json:"data_array"` + } `json:"result"` +} + +func dbxSQLRequest(ctx context.Context, method, url string, payload any) (*sqlStatementResponse, error) { + var body io.Reader + if payload != nil { + b, err := json.Marshal(payload) + if err != nil { + return nil, err + } + body = bytes.NewReader(b) + } + req, err := http.NewRequestWithContext(ctx, method, url, body) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", "Bearer "+dbxToken()) + req.Header.Set("Content-Type", "application/json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("%s %s: %w", method, url, err) + } + defer resp.Body.Close() + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("%s %s: reading body: %w", method, url, err) + } + if resp.StatusCode >= 300 { + return nil, fmt.Errorf("%s %s failed (%d): %s", method, url, resp.StatusCode, redact(string(respBody))) + } + var out sqlStatementResponse + if err := json.Unmarshal(respBody, &out); err != nil { + return nil, fmt.Errorf("%s %s: decoding response: %w", method, url, err) + } + return &out, nil +} + +// runSQL executes a statement on the configured SQL warehouse and returns the +// result rows keyed by column name (NULL values become ""). It submits with a +// 30s server-side wait, then polls until SUCCEEDED/FAILED or sqlPollTimeout. +func runSQL(ctx context.Context, statement string) ([]map[string]string, error) { + base := "https://" + normalizeHost(*dbxHost) + "/api/2.0/sql/statements" + resp, err := dbxSQLRequest(ctx, http.MethodPost, base, map[string]any{ + "statement": statement, + "warehouse_id": *dbxWarehouseID, + "wait_timeout": "30s", + "on_wait_timeout": "CONTINUE", + }) + if err != nil { + return nil, err + } + + deadline := time.Now().Add(sqlPollTimeout) + for resp.Status.State == "PENDING" || resp.Status.State == "RUNNING" { + if time.Now().After(deadline) { + return nil, fmt.Errorf("statement %s still %s after %v: %q", resp.StatementID, resp.Status.State, sqlPollTimeout, statement) + } + time.Sleep(2 * time.Second) + resp, err = dbxSQLRequest(ctx, http.MethodGet, base+"/"+resp.StatementID, nil) + if err != nil { + return nil, err + } + } + if resp.Status.State != "SUCCEEDED" { + return nil, fmt.Errorf("statement %s finished %s: %s (statement: %q)", + resp.StatementID, resp.Status.State, redact(resp.Status.Error.Message), statement) + } + + rows := make([]map[string]string, 0, len(resp.Result.DataArray)) + for _, raw := range resp.Result.DataArray { + row := make(map[string]string, len(resp.Manifest.Schema.Columns)) + for i, col := range resp.Manifest.Schema.Columns { + if i < len(raw) && raw[i] != nil { + row[col.Name] = *raw[i] + } else { + row[col.Name] = "" + } + } + rows = append(rows, row) + } + return rows, nil +} + +// sqlQuery is runSQL with a hard failure on error. +func sqlQuery(t *testing.T, ctx context.Context, statement string) []map[string]string { + t.Helper() + rows, err := runSQL(ctx, statement) + require.NoError(t, err) + return rows +} + +// dropTable best-effort drops a test table through the warehouse so repeated +// runs never need a terraform re-apply. +func dropTable(t *testing.T, tableName string) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + if _, err := runSQL(ctx, "DROP TABLE IF EXISTS "+fqTable(tableName)); err != nil { + t.Logf("warning: failed to drop table %s: %v", tableName, err) + } +} + +// countManifestsByContent loads the table's current snapshot and tallies its +// manifests by content kind — copy-on-write must leave only data manifests +// and zero delete manifests, which is what makes the result readable by the +// Unity Catalog (which cannot apply Iceberg v2 delete files). +func countManifestsByContent(t *testing.T, ctx context.Context, tbl *table.Table) (dataManifests, deleteManifests int) { + t.Helper() + snap := tbl.CurrentSnapshot() + require.NotNil(t, snap, "table must have a current snapshot") + + fsys, err := tbl.FS(ctx) + require.NoError(t, err) + manifests, err := snap.Manifests(fsys) + require.NoError(t, err) + + for _, m := range manifests { + if m.ManifestContent() == iceberg.ManifestContentDeletes { + deleteManifests++ + } else { + dataManifests++ + } + } + return dataManifests, deleteManifests +} + +// TestDatabricksE2E_COWRoundTrip is THE release-gate proof: an insert → +// upsert → delete round trip through merge_strategy: copy-on-write against a +// real Unity Catalog, read back through a real Databricks SQL warehouse. +// +// The table is pre-created via the Iceberg REST catalog with an explicit +// schema (id long, name string, ts timestamp, tstz timestamptz) so the +// no-timezone `timestamp` column exercises the spec timestamp encoding +// against Databricks' reader — auto-created columns from time.Time values +// would all be timestamptz. Keep it UNPARTITIONED: UC ignores/re-clusters +// Iceberg partition specs. +// +// UNVERIFIED-WITHOUT-LIVE-ACCESS: the first write also stamps the +// redpanda-connect.timestamp-encoding table property through a set-properties +// commit; UC accepting that commit is one of the behaviours this test proves. +func TestDatabricksE2E_COWRoundTrip(t *testing.T) { + skipIfNotConfigured(t) + ctx := t.Context() + + tableName := uniqueTableName("cow_e2e") + t.Cleanup(func() { dropTable(t, tableName) }) + + client := newCatalogClient(t, ctx) + // All columns optional and NO identifier-field-ids — the same shape the + // router's copy-on-write auto-create produces, and the shape UC accepts. + _, err := client.CreateTable(ctx, tableName, iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.Int64Type{}}, + iceberg.NestedField{ID: 2, Name: "name", Type: iceberg.StringType{}}, + iceberg.NestedField{ID: 3, Name: "ts", Type: iceberg.TimestampType{}}, + iceberg.NestedField{ID: 4, Name: "tstz", Type: iceberg.TimestampTzType{}}, + )) + require.NoError(t, err, "CREATE TABLE via the UC Iceberg REST catalog must succeed") + + seed := time.Date(2026, 1, 15, 12, 30, 45, 0, time.UTC) + row := func(op string, id int64, name string, at time.Time) *service.Message { + return opRow(op, map[string]any{"id": id, "name": name, "ts": at, "tstz": at}) + } + + router := newRouter(t, *dbxSchema, tableName, true, cowRowOp(t)) + + // Seed three rows. + produce(t, ctx, router, service.MessageBatch{ + row("insert", 1, "one", seed), + row("insert", 2, "two", seed), + row("insert", 3, "three", seed), + }) + + // One mutating batch: upsert id=2, delete id=3, upsert id=4 (new row) — + // the combined overwrite+delete path that rewrites data files in a single + // atomic snapshot. + produce(t, ctx, router, service.MessageBatch{ + row("upsert", 2, "two-updated", seed.Add(time.Hour)), + opRow("delete", map[string]any{"id": int64(3)}), + row("upsert", 4, "four", seed.Add(2*time.Hour)), + }) + + // Read back THROUGH THE WAREHOUSE — this is Databricks' own reader + // consuming what the connector wrote. ts (TIMESTAMP_NTZ) round-trips as a + // naive wall-clock string; tstz (TIMESTAMP) is compared as an epoch so the + // assertion is independent of the warehouse session timezone. + rows := sqlQuery(t, ctx, fmt.Sprintf( + "SELECT id, name, date_format(ts, 'yyyy-MM-dd HH:mm:ss') AS ts, CAST(unix_timestamp(tstz) AS STRING) AS tstz_unix FROM %s ORDER BY id", + fqTable(tableName))) + t.Logf("warehouse read-back: %v", rows) + + require.Len(t, rows, 3, "expected id=1, id=2, id=4 (id=3 deleted, id=2 not duplicated)") + expect := []struct { + id, name string + at time.Time + }{ + {"1", "one", seed}, + {"2", "two-updated", seed.Add(time.Hour)}, + {"4", "four", seed.Add(2 * time.Hour)}, + } + for i, e := range expect { + assert.Equal(t, e.id, rows[i]["id"]) + assert.Equal(t, e.name, rows[i]["name"]) + assert.Equal(t, e.at.UTC().Format("2006-01-02 15:04:05"), rows[i]["ts"], "ts (timestamp_ntz) must round-trip naively for id=%s", e.id) + assert.Equal(t, strconv.FormatInt(e.at.Unix(), 10), rows[i]["tstz_unix"], "tstz (timestamptz) must round-trip as the same instant for id=%s", e.id) + } + + // Log and assert how Databricks reports the two timestamp flavours. + descRows := sqlQuery(t, ctx, "DESCRIBE TABLE "+fqTable(tableName)) + colTypes := map[string]string{} + for _, r := range descRows { + colTypes[r["col_name"]] = r["data_type"] + } + t.Logf("DESCRIBE TABLE column types: %v", colTypes) + assert.Equal(t, "timestamp_ntz", strings.ToLower(colTypes["ts"]), "no-tz iceberg timestamp should surface as TIMESTAMP_NTZ") + assert.Equal(t, "timestamp", strings.ToLower(colTypes["tstz"]), "iceberg timestamptz should surface as TIMESTAMP") + + // Zero delete files, via the catalog's own manifests: non-vacuous (at + // least one data manifest) AND exactly zero delete manifests. A + // merge-on-read run of the same batch would have left delete manifests — + // this property is the entire point of copy-on-write on Databricks. + loaded, err := client.LoadTable(ctx, tableName) + require.NoError(t, err) + dataManifests, deleteManifests := countManifestsByContent(t, ctx, loaded) + assert.Positive(t, dataManifests, "expected at least one data manifest to inspect") + assert.Zero(t, deleteManifests, "copy-on-write must leave zero delete manifests") + + require.NotNil(t, loaded.CurrentSnapshot()) + assert.Equal(t, table.OpOverwrite, loaded.CurrentSnapshot().Summary.Operation, + "the upsert+delete batch must commit as an overwrite under copy-on-write") + + // Finally: the router's own copy-on-write auto-create (no + // identifier-field-ids registered) must be accepted by UC — this is the + // exact CREATE TABLE the original field report saw rejected under + // merge-on-read, and the no-registration rationale behind copy-on-write. + autoTable := uniqueTableName("cow_autocreate") + t.Cleanup(func() { dropTable(t, autoTable) }) + autoRouter := newRouter(t, *dbxSchema, autoTable, true, cowRowOp(t)) + produce(t, ctx, autoRouter, service.MessageBatch{ + opRow("insert", map[string]any{"id": int64(1), "name": "auto"}), + }) + autoRows := sqlQuery(t, ctx, fmt.Sprintf("SELECT id, name FROM %s", fqTable(autoTable))) + require.Len(t, autoRows, 1, "COW auto-created table must be readable through the warehouse") + assert.Equal(t, "1", autoRows[0]["id"]) + assert.Equal(t, "auto", autoRows[0]["name"]) +} + +// TestDatabricksE2E_IdentifierFieldsRejected reproduces the original field +// report: under merge-on-read with identifier_fields, the router registers +// the identifier-field-ids on CREATE TABLE, and Unity Catalog rejects that +// ("Table with identifier columns is not allowed. [ErrorCode: 2014]" — the +// wording may drift, so the assertion is deliberately loose). +// +// If UC ever ACCEPTS this creation, the test fails loudly: it would mean UC +// behaviour has changed and the copy-on-write no-registration rationale +// should be revisited. +func TestDatabricksE2E_IdentifierFieldsRejected(t *testing.T) { + skipIfNotConfigured(t) + ctx := t.Context() + + tableName := uniqueTableName("mor_create_e2e") + // In case creation unexpectedly succeeds, don't leave the table behind. + t.Cleanup(func() { dropTable(t, tableName) }) + + router := newRouter(t, *dbxSchema, tableName, true, morRowOp(t)) + // int64 id (via structured message) so the identifier column passes the + // router's own non-floating-point key validation and the CREATE TABLE + // actually reaches Unity Catalog carrying identifier-field-ids. + err := router.Route(ctx, service.MessageBatch{ + opRow("insert", map[string]any{"id": int64(1), "name": "alice"}), + }) + if err == nil { + t.Fatal("Unity Catalog ACCEPTED a CREATE TABLE carrying identifier-field-ids — UC behaviour has changed! " + + "Revisit the copy-on-write no-registration rationale (Router.schemaWithIdentifierFields) and this test.") + } + t.Logf("UC rejected CREATE TABLE with identifier-field-ids as expected. Full error: %v", err) + assert.Contains(t, strings.ToLower(err.Error()), "identifier", + "expected the rejection to mention identifier columns (loose match — UC wording may have drifted, check the logged error)") +} + +// TestDatabricksE2E_MORDeleteFilesDiagnostic is a DIAGNOSTIC, not a gate: it +// settles empirically whether UC rejects a merge-on-read equality-delete +// commit outright, or accepts it and then serves stale (or no) rows. The +// table is pre-created WITHOUT identifier-field-ids (COW-style schema) so +// creation cannot fail, then a merge-on-read router attempts an +// equality-delete commit against it. Every outcome is logged; nothing about +// UC's choice is a hard pass/fail — the test only fails if we learn nothing. +func TestDatabricksE2E_MORDeleteFilesDiagnostic(t *testing.T) { + skipIfNotConfigured(t) + ctx := t.Context() + + tableName := uniqueTableName("mor_diag_e2e") + t.Cleanup(func() { dropTable(t, tableName) }) + + client := newCatalogClient(t, ctx) + _, err := client.CreateTable(ctx, tableName, iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.Int64Type{}}, + iceberg.NestedField{ID: 2, Name: "name", Type: iceberg.StringType{}}, + )) + require.NoError(t, err, "pre-creating the diagnostic table (no identifier-field-ids) must succeed") + + router := newRouter(t, *dbxSchema, tableName, true, morRowOp(t)) + + // Seed with plain inserts — append-only commits, expected to succeed even + // under merge-on-read. + if err := router.Route(ctx, service.MessageBatch{ + opRow("insert", map[string]any{"id": int64(1), "name": "one"}), + opRow("insert", map[string]any{"id": int64(2), "name": "two"}), + }); err != nil { + t.Logf("LEARNED (unexpected): even append-only inserts failed under merge-on-read: %v", err) + return + } + time.Sleep(2 * time.Second) + + // The probe: a merge-on-read delete writes an Iceberg v2 equality-delete + // file and commits it. Does UC reject the commit, or accept it? + deleteErr := router.Route(ctx, service.MessageBatch{ + opRow("delete", map[string]any{"id": int64(2)}), + }) + if deleteErr != nil { + t.Logf("LEARNED: UC REJECTED the equality-delete commit outright (error, not silent staleness): %v", deleteErr) + return + } + time.Sleep(2 * time.Second) + t.Log("LEARNED: UC ACCEPTED the equality-delete commit; inspecting what a reader now sees...") + + if loaded, lerr := client.LoadTable(ctx, tableName); lerr == nil { + dataManifests, deleteManifests := countManifestsByContent(t, ctx, loaded) + t.Logf("catalog view after commit: %d data manifests, %d delete manifests", dataManifests, deleteManifests) + } else { + t.Logf("could not load table back through the catalog: %v", lerr) + } + + rows, qerr := runSQL(ctx, fmt.Sprintf("SELECT id, name FROM %s ORDER BY id", fqTable(tableName))) + switch { + case qerr != nil: + t.Logf("LEARNED: warehouse read-back FAILED after the accepted delete commit (UC likely refuses tables with delete files): %v", qerr) + case len(rows) == 2: + t.Logf("LEARNED: warehouse serves STALE rows — the equality delete was silently ignored by the reader: %v", rows) + case len(rows) == 1 && rows[0]["id"] == "1": + t.Logf("LEARNED: warehouse APPLIED the equality delete — UC merge-on-read reading works here: %v", rows) + default: + t.Logf("LEARNED: unexpected read-back state after the accepted delete commit: %v", rows) + } +} + +// TestDatabricksE2E_CommitLatencyBench measures copy-on-write upsert commit +// latency against the real UC at three batch sizes. Gated behind +// -databricks.bench because it burns (a tiny amount of) warehouse and API +// time. Per size: seed the table, then time 3 full-batch upsert commits — +// the worst case, every data file rewritten. +func TestDatabricksE2E_CommitLatencyBench(t *testing.T) { + skipIfNotConfigured(t) + if !*dbxBench { + t.Skip("set -databricks.bench to run the commit latency bench") + } + ctx := t.Context() + + batch := func(op string, size, gen int) service.MessageBatch { + msgs := make(service.MessageBatch, size) + for i := range msgs { + msgs[i] = opRow(op, map[string]any{ + "id": int64(i), + "name": fmt.Sprintf("user_%d", i), + "value": int64(gen), + }) + } + return msgs + } + + for _, size := range []int{100, 1000, 5000} { + t.Run(fmt.Sprintf("rows_%d", size), func(t *testing.T) { + tableName := uniqueTableName(fmt.Sprintf("cow_bench_%d", size)) + t.Cleanup(func() { dropTable(t, tableName) }) + + router := newRouter(t, *dbxSchema, tableName, true, cowRowOp(t)) + produce(t, ctx, router, batch("insert", size, 0)) // seed (auto-creates the table) + + for commit := 1; commit <= 3; commit++ { + start := time.Now() + require.NoError(t, router.Route(ctx, batch("upsert", size, commit))) + elapsed := time.Since(start) + t.Logf("size=%d commit=%d: %v (%.0f rows/s)", size, commit, elapsed, float64(size)/elapsed.Seconds()) + } + }) + } +} diff --git a/internal/impl/iceberg/e2e/databricks/terraform/main.tf b/internal/impl/iceberg/e2e/databricks/terraform/main.tf new file mode 100644 index 0000000000..5c7337fbdc --- /dev/null +++ b/internal/impl/iceberg/e2e/databricks/terraform/main.tf @@ -0,0 +1,134 @@ +terraform { + required_providers { + databricks = { + source = "databricks/databricks" + version = "~> 1.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + null = { + source = "hashicorp/null" + version = "~> 3.0" + } + } + required_version = ">= 1.2" +} + +# Authentication comes exclusively from the environment: DATABRICKS_HOST and +# DATABRICKS_TOKEN. The token is deliberately NOT a terraform variable so it +# can never appear in plan output, outputs, or state. +provider "databricks" {} + +data "databricks_current_user" "me" {} + +resource "random_id" "suffix" { + byte_length = 4 +} + +locals { + # Bare workspace host (no scheme, no trailing slash), whatever form the + # workspace_host variable arrived in. + workspace_host = trimsuffix(trimprefix(trimprefix(var.workspace_host, "https://"), "http://"), "/") + catalog_name = "${var.prefix}_e2e_${random_id.suffix.hex}" +} + +# --- Isolated Unity Catalog environment --- + +resource "databricks_catalog" "e2e" { + name = local.catalog_name + comment = "Redpanda Connect iceberg copy-on-write e2e (disposable)" + force_destroy = true # cascades: schemas and test tables go with the catalog + + # storage_root is REQUIRED when the metastore has no default storage root + # (common on auto-provisioned metastores). Check with: + # databricks metastores summary + # and set the storage_root variable to e.g. s3://bucket/prefix if empty. + storage_root = var.storage_root != "" ? var.storage_root : null +} + +# The Iceberg namespace used by the tests. Pre-created here because +# client-side namespace creation through UC's Iceberg REST endpoint is +# unverified — the tests never call CreateNamespace. +resource "databricks_schema" "e2e" { + catalog_name = databricks_catalog.e2e.name + name = var.schema_name + comment = "Namespace for Redpanda Connect iceberg e2e tables" + force_destroy = true +} + +# Serverless SQL warehouse for reading written data back via the SQL +# Statement Execution API. 2X-Small, single cluster, auto-stops after a +# minute idle — statement submission auto-restarts it, so cost stays minimal. +resource "databricks_sql_endpoint" "e2e" { + name = "${var.prefix}-e2e-${random_id.suffix.hex}" + cluster_size = "2X-Small" + min_num_clusters = 1 + max_num_clusters = 1 + auto_stop_mins = 1 + enable_serverless_compute = true + warehouse_type = "PRO" +} + +# --- Grants --- +# +# The Iceberg REST principal needs EXTERNAL USE SCHEMA on top of the usual +# privileges. EXTERNAL USE SCHEMA is NOT included in ALL PRIVILEGES and only +# the catalog owner can grant it — terraform's principal creates the catalog +# and is therefore its owner, so granting itself here should work. +# +# UNVERIFIED-WITHOUT-LIVE-ACCESS: the exact provider privilege string for +# EXTERNAL USE SCHEMA ("EXTERNAL_USE_SCHEMA") and whether UC allows a +# redundant self-grant to the owner must be confirmed on the first live run. +resource "databricks_grants" "catalog" { + catalog = databricks_catalog.e2e.name + + grant { + principal = data.databricks_current_user.me.user_name + privileges = [ + "USE_CATALOG", + "USE_SCHEMA", + "CREATE_TABLE", + "MODIFY", + "SELECT", + "EXTERNAL_USE_SCHEMA", + ] + } +} + +# --- Metastore external access (OPTIONAL, off by default) --- +# +# The Iceberg REST endpoint only works when the metastore has +# external_access_enabled = true. Flipping it needs METASTORE ADMIN, so this +# is usually a one-time manual/admin action: +# +# databricks metastores update --json '{"external_access_enabled": true}' +# +# Set manage_external_access = true (plus metastore_id) to have terraform run +# that CLI call for you. Implemented as a null_resource local-exec because +# adopting the whole metastore into state via the databricks_metastore +# resource (which does expose external_access_enabled) would be far more +# invasive than a disposable e2e environment warrants. +# +# UNVERIFIED-WITHOUT-LIVE-ACCESS: the CLI invocation below is written per the +# official docs and needs confirming on the first live run. Note it is not +# reverted on destroy. +resource "null_resource" "enable_external_access" { + count = var.manage_external_access ? 1 : 0 + + triggers = { + metastore_id = var.metastore_id + } + + lifecycle { + precondition { + condition = var.metastore_id != "" + error_message = "metastore_id must be set when manage_external_access = true (find it with: databricks metastores summary)." + } + } + + provisioner "local-exec" { + command = "databricks metastores update ${var.metastore_id} --json '{\"external_access_enabled\": true}'" + } +} diff --git a/internal/impl/iceberg/e2e/databricks/terraform/outputs.tf b/internal/impl/iceberg/e2e/databricks/terraform/outputs.tf new file mode 100644 index 0000000000..be7c528af6 --- /dev/null +++ b/internal/impl/iceberg/e2e/databricks/terraform/outputs.tf @@ -0,0 +1,19 @@ +output "catalog_name" { + value = databricks_catalog.e2e.name +} + +output "schema_name" { + value = databricks_schema.e2e.name +} + +output "warehouse_id" { + value = databricks_sql_endpoint.e2e.id +} + +output "workspace_host" { + value = local.workspace_host +} + +output "iceberg_rest_url" { + value = "https://${local.workspace_host}/api/2.1/unity-catalog/iceberg-rest" +} diff --git a/internal/impl/iceberg/e2e/databricks/terraform/terraform.yml b/internal/impl/iceberg/e2e/databricks/terraform/terraform.yml new file mode 100644 index 0000000000..e5b3c47a1d --- /dev/null +++ b/internal/impl/iceberg/e2e/databricks/terraform/terraform.yml @@ -0,0 +1,22 @@ +version: '3' + +tasks: + init: + desc: Initialize Terraform + cmds: + - terraform init + + plan: + desc: Plan infrastructure changes + cmds: + - terraform plan + + apply: + desc: Provision infrastructure + cmds: + - terraform apply -auto-approve + + destroy: + desc: Tear down infrastructure + cmds: + - terraform destroy -auto-approve diff --git a/internal/impl/iceberg/e2e/databricks/terraform/variables.tf b/internal/impl/iceberg/e2e/databricks/terraform/variables.tf new file mode 100644 index 0000000000..10ef4bbab2 --- /dev/null +++ b/internal/impl/iceberg/e2e/databricks/terraform/variables.tf @@ -0,0 +1,34 @@ +variable "prefix" { + description = "Resource name prefix" + type = string + default = "rpcn" +} + +variable "workspace_host" { + description = "Databricks workspace host, e.g. dbc-abc123.cloud.databricks.com (scheme optional). Usually: export TF_VAR_workspace_host=\"$DATABRICKS_HOST\"" + type = string +} + +variable "schema_name" { + description = "Unity Catalog schema (Iceberg namespace) to pre-create for the tests" + type = string + default = "e2e" +} + +variable "storage_root" { + description = "Managed storage root for the e2e catalog (e.g. s3://bucket/prefix). REQUIRED when the metastore has no default storage root — check with `databricks metastores summary`. Empty means inherit the metastore root." + type = string + default = "" +} + +variable "manage_external_access" { + description = "Have terraform enable external_access_enabled on the metastore via the Databricks CLI. Needs METASTORE ADMIN; usually a one-time manual action instead — see main.tf." + type = bool + default = false +} + +variable "metastore_id" { + description = "Metastore ID, only used when manage_external_access = true (find it with `databricks metastores summary`)" + type = string + default = "" +} From 11642889c6c50f2e34abc52e6dd0a517dba510e6 Mon Sep 17 00:00:00 2001 From: Ashley Jeffs Date: Mon, 3 Aug 2026 14:13:32 +0100 Subject: [PATCH 07/12] iceberg: add customer-owned storage to the Databricks e2e harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Databricks workspaces on default (Databricks-managed) storage cannot vend credentials to external Iceberg REST clients, so the harness's tables must live in a catalog backed by customer-owned S3 — which is also the fix when a metastore has no storage root. A new opt-in `create_storage` variable provisions the full chain: S3 bucket, an IAM role built from the provider's Unity Catalog assume-role-policy data sources (the documented way around the credential/role trust-policy circular dependency), a storage credential, an external location, and the catalog's storage root resolving to it (an explicit storage_root variable still wins; otherwise the metastore root is inherited). The AWS provider is only truly configured when storage creation is enabled, so the default path still needs no AWS setup at all. The README gains a trial-account quick-start covering the express-trial signup, the metastore external-access toggle, and the default-storage limitation that makes customer-owned storage necessary; the provider lock file is now tracked, matching the sibling harnesses. --- .../impl/iceberg/e2e/databricks/README.md | 79 ++++++++++- .../databricks/terraform/.terraform.lock.hcl | 102 ++++++++++++++ .../iceberg/e2e/databricks/terraform/main.tf | 50 ++++++- .../e2e/databricks/terraform/outputs.tf | 5 + .../e2e/databricks/terraform/storage.tf | 125 ++++++++++++++++++ .../e2e/databricks/terraform/terraform.yml | 11 +- .../e2e/databricks/terraform/variables.tf | 19 ++- 7 files changed, 382 insertions(+), 9 deletions(-) create mode 100644 internal/impl/iceberg/e2e/databricks/terraform/.terraform.lock.hcl create mode 100644 internal/impl/iceberg/e2e/databricks/terraform/storage.tf diff --git a/internal/impl/iceberg/e2e/databricks/README.md b/internal/impl/iceberg/e2e/databricks/README.md index a6e7d2f92d..2a33a87993 100644 --- a/internal/impl/iceberg/e2e/databricks/README.md +++ b/internal/impl/iceberg/e2e/databricks/README.md @@ -22,8 +22,11 @@ a serverless SQL warehouse via the SQL Statement Execution API. terraform runs that CLI call for you. - **Storage root check**: if `databricks metastores summary` shows no default storage root (common on auto-provisioned metastores), catalog creation needs - an explicit one — set `TF_VAR_storage_root=s3://bucket/prefix` (or the - `storage_root` variable) before applying. + an explicit one — either set `TF_VAR_storage_root=s3://bucket/prefix` (or + the `storage_root` variable) before applying, or set `create_storage=true` + to have terraform provision a bucket + external location itself (see the + trial quick-start below; works for company accounts too). An explicit + `storage_root` always wins over `create_storage`. ## Auth @@ -44,6 +47,78 @@ OAuth2 (M2M service principal) also works for terraform, but the tests use the PAT bearer token for the Iceberg REST client deliberately: community reports intermittent 500s using OAuth2 tokens against the UC IRC endpoint. +## Trial account quick-start (no company workspace needed) + +A Databricks 14-day express trial can run this whole suite — with one twist: +trial workspaces use [default storage](https://docs.databricks.com/aws/en/storage/default-storage), +which does **not** support credential vending for external clients ("such as +when external systems connect to the Unity REST API or Iceberg REST catalog", +per that doc). A catalog on default storage therefore **cannot** work for +these tests, no matter the grants. Serverless trial workspaces *do* support +catalogs on customer-owned S3, which is exactly what `create_storage=true` +provisions (bucket + IAM role + UC storage credential + external location, +all disposable). You need an AWS account for the bucket; the ~$400 trial +credit covers the Databricks side. + +Known trial constraints: + +- **Free Edition cannot work at all** — no external data access. Use the + trial from [databricks.com/try-databricks](https://www.databricks.com/try-databricks). +- Sign up with a **business email** (no card needed, ~$400 of credits over + 14 days). Personal-email trials are capped at a single SQL warehouse, which + bites the moment anything else holds one — business email avoids that. +- Trial workspace assets are deleted **60 days after the trial expires** — + nothing here is worth keeping anyway, but don't park anything you love in + it. + +Steps: + +1. Sign up, open the workspace, and grab a PAT (User Settings → Developer → + Access tokens). +2. Enable external data access on the metastore (as the trial's only user you + are the account admin; if the call is refused, make yourself metastore + admin first in the account console under Catalog → your metastore): + + ```sh + databricks metastores summary # note the metastore id + databricks metastores update --json '{"external_access_enabled": true}' + ``` + +3. Export Databricks and AWS credentials, plus the storage variables — use + `TF_VAR_*` env vars (not one-off `-var` flags) so `terraform destroy` later + sees the same values: + + ```sh + export DATABRICKS_HOST="https://dbc-abc123.cloud.databricks.com" + export DATABRICKS_TOKEN="dapi..." + export TF_VAR_workspace_host="$DATABRICKS_HOST" + export AWS_PROFILE=... # or AWS_ACCESS_KEY_ID etc. + export TF_VAR_create_storage=true + export TF_VAR_aws_region=us-east-1 # bucket region + ``` + +4. Apply and test as usual: + + ```sh + task terraform:apply + task test + ``` + + (`task terraform:apply -- -var create_storage=true -var aws_region=us-east-1` + also works — args after `--` pass through — but then destroy needs the + same flags, hence the env-var recommendation.) + +The company-account path is unchanged: `create_storage` defaults to `false` +and nothing AWS-side is touched. + +**If the first apply fails at the external location**: Unity Catalog storage +credentials have a chicken-and-egg with the IAM role (the role's trust policy +needs the credential's external ID), handled with the databricks provider's +documented pattern plus a 30s wait for IAM propagation. IAM is eventually +consistent, so a slow region can still occasionally fail the external +location's validation on the first try — just re-run `task terraform:apply`; +it picks up where it left off. + ## Running ```sh diff --git a/internal/impl/iceberg/e2e/databricks/terraform/.terraform.lock.hcl b/internal/impl/iceberg/e2e/databricks/terraform/.terraform.lock.hcl new file mode 100644 index 0000000000..15c6140aee --- /dev/null +++ b/internal/impl/iceberg/e2e/databricks/terraform/.terraform.lock.hcl @@ -0,0 +1,102 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/databricks/databricks" { + version = "1.122.0" + constraints = "~> 1.0" + hashes = [ + "h1:fZ5LA+TKILntJUgGK7uMyqbpFSHkkIqsDb21DtIPuoI=", + "zh:1ad7f43187f9dfb2aa409b6345e4a040657601ed6a05b6bccfd6c9d52d043620", + "zh:6c01baf771cec6c4a49449146e07040a74dcddb5b9a0cbd5b7697c1f1eba1e6d", + "zh:e13b929972e45db613ce9fafbe98cb1e6e7e33036b008d28713b00031e0e2bf9", + "zh:e336b8c68b4bf44279947cb3324d5fc2a26cdb8d4c3c407602c9933b4d4503e4", + "zh:eed311ce5313f31dedbf3b17d74d9f950de75e24639cf5ad8970a4fb3d2d0052", + "zh:f42d516c74024123b99534b2080813755e2536699be10a313fd97d024cf6f34b", + ] +} + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.100.0" + constraints = "~> 5.0" + hashes = [ + "h1:Ijt7pOlB7Tr7maGQIqtsLFbl7pSMIj06TVdkoSBcYOw=", + "zh:054b8dd49f0549c9a7cc27d159e45327b7b65cf404da5e5a20da154b90b8a644", + "zh:0b97bf8d5e03d15d83cc40b0530a1f84b459354939ba6f135a0086c20ebbe6b2", + "zh:1589a2266af699cbd5d80737a0fe02e54ec9cf2ca54e7e00ac51c7359056f274", + "zh:6330766f1d85f01ae6ea90d1b214b8b74cc8c1badc4696b165b36ddd4cc15f7b", + "zh:7c8c2e30d8e55291b86fcb64bdf6c25489d538688545eb48fd74ad622e5d3862", + "zh:99b1003bd9bd32ee323544da897148f46a527f622dc3971af63ea3e251596342", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:9f8b909d3ec50ade83c8062290378b1ec553edef6a447c56dadc01a99f4eaa93", + "zh:aaef921ff9aabaf8b1869a86d692ebd24fbd4e12c21205034bb679b9caf883a2", + "zh:ac882313207aba00dd5a76dbd572a0ddc818bb9cbf5c9d61b28fe30efaec951e", + "zh:bb64e8aff37becab373a1a0cc1080990785304141af42ed6aa3dd4913b000421", + "zh:dfe495f6621df5540d9c92ad40b8067376350b005c637ea6efac5dc15028add4", + "zh:f0ddf0eaf052766cfe09dea8200a946519f653c384ab4336e2a4a64fdd6310e9", + "zh:f1b7e684f4c7ae1eed272b6de7d2049bb87a0275cb04dbb7cda6636f600699c9", + "zh:ff461571e3f233699bf690db319dfe46aec75e58726636a0d97dd9ac6e32fb70", + ] +} + +provider "registry.terraform.io/hashicorp/null" { + version = "3.3.0" + constraints = "~> 3.0" + hashes = [ + "h1:a14TKo7Xvg4W8+H1VA6p+oLZTLxVQnYUD8LOaOs14A8=", + "zh:021748b5ea3b5f6956f2e75c42c5cdc113b391fb98ac71364a4965d23b37000f", + "zh:3b27956f8541d46704fda234e0d535c2ae2a4b33411848b1ee262a1ec03568b0", + "zh:3de4ed47d6d0f4d8edba4a5092c7c9799950eda63989d8d0d2586e6afcb0aa20", + "zh:57ed8935c7d56dbc91cf2673534582cacfaab7a2f105f51d9f797e99df0c0c47", + "zh:58e176ba1d142827089e30e0711e007309a9f2726e8881986da5026e9778fdf4", + "zh:5949c4a3d4a93f841f155cdb7e991c087e637145c1630572e21948224f8f4923", + "zh:76d60f366b743003c1b085afa769b45b2198ee919927e45807d7d44fb42c067d", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:79cd1bab1261a07f84e917191d7ddc4340ac5f5524283767256f7ffd7f87caf0", + "zh:8ec9083038cf710b30e319eaa467c9df7fa52bbd9969b61053a35bc2cdd2e0a6", + "zh:a6e502cb579685ab7aeb886c2bb11ddd9cfed74b41008592d57cbc3351a9218b", + "zh:acb74d6b4f66ff6acfcda315df802a7432170ef3955c9b432cb4580767004006", + "zh:f0ce55d8d9ffdb33dab612b1246f9bab060a9d54fc32ce2b4a038646155660af", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.9.0" + constraints = "~> 3.0" + hashes = [ + "h1:OO+IuvQJSPmWdN8AyyIEvPJbLvDQpgX/zbktoa9KsJE=", + "zh:161ad0bd9a75768c82f53fb6e7172a9d8be2d4889b012645a34795031aaf1bf1", + "zh:19dc9a5b17729725ccfc4f45b0500af0ee5bc6b6b160c7adb8f2bf617d2c80ea", + "zh:269eda8fe42daa7974d5a34d166c3ba9defe80cde86c01e4dadcfdf2e1f05e5f", + "zh:373f7c65566f8f2cc7f45d698654feb9d988996957e1266a69ca00c52d6d16d0", + "zh:5599d16804c41c83009ec621b6d6b6f74e102f5827678a4750f8809055546b61", + "zh:583be0440469a22bff70dcfa56593b01566860b29607437264adb51060cf46fc", + "zh:5f211d8ec3f2e1f414870d9584bfe26e6995560ef81c748f8447a48164767398", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:7b547fd16216761ef86efc3ed516ac5ac0c5c42b7c7eb24a08cef2d93f69ed5e", + "zh:7e7c0679daf2a382151d05068c8c3f0dae6b7b7dccf818827b73dd08638df2ef", + "zh:8089dec888a8038b9b4fb23b3df7e1057293dbc5b60b42cc47ff690d69d4b61b", + "zh:c51f15a031edfd6f23ce8ced3446ca7f8d8d647e2499890d7d5d10d5016d7257", + "zh:c94784f005708890dc6895afd53636ec00ec1e430b15d41e5aebfb1d4b39bd04", + ] +} + +provider "registry.terraform.io/hashicorp/time" { + version = "0.14.0" + constraints = "~> 0.9" + hashes = [ + "h1:/hlxsUpuN/lvPTNL9+NyVGsOyRsK5NsxwFMsj5CdOp4=", + "zh:12abfd6b800e4d7fa6db7310dec8ffd440b31993861ef188c7ed5260b3073937", + "zh:23005521e800bb19e1597bf755c5f70d675d30b685d4255001ed5fa47d9df3f1", + "zh:2fea249b582ae97cd1cc10385187ea50993bb47c28cc5df0305e57ceaabf0a10", + "zh:322018d3b987b7aad08697178029a2bb667bed699e88328f0c89c52a2fd41341", + "zh:32a08e98fce2d273cb9b2c89d6c54727cc9f0a32e15bfd896be4e02cc6b48f95", + "zh:3db89aabd0e619616bd4b0f8b373a7586dfe60feffcea12a84a0bdbc445714b3", + "zh:7488f56c81d742dc020f29063626c8f07ca188aa97be61e7307e8d62397020a2", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:7cb4067f2e7559b13f7562ef722f948950901eb37834873e98360ab28f66e9d7", + "zh:9d552c8345f61e1b7db8e725144981345f18ac1014d58d6f5ddf0928a195fffb", + "zh:a8e69fb6b97fc9d86fb19a9f4d42abe33c4a68e700b15387ce2e17d2b9934bed", + "zh:aeeb900eb8dd0f790c60ea5c0e0c8d42bd6e4a54f391681d4decca15b544394b", + "zh:c239c619101a8c95e1f14061eb973c57a8d15fa0e68878ced5bbd76858ee5b79", + ] +} diff --git a/internal/impl/iceberg/e2e/databricks/terraform/main.tf b/internal/impl/iceberg/e2e/databricks/terraform/main.tf index 5c7337fbdc..a4dfe858b1 100644 --- a/internal/impl/iceberg/e2e/databricks/terraform/main.tf +++ b/internal/impl/iceberg/e2e/databricks/terraform/main.tf @@ -12,8 +12,18 @@ terraform { source = "hashicorp/null" version = "~> 3.0" } + # aws + time are only exercised when create_storage = true (see storage.tf). + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + time = { + source = "hashicorp/time" + version = "~> 0.9" + } } - required_version = ">= 1.2" + # >= 1.9 for the cross-variable validation on create_storage in variables.tf. + required_version = ">= 1.9" } # Authentication comes exclusively from the environment: DATABRICKS_HOST and @@ -21,6 +31,26 @@ terraform { # can never appear in plan output, outputs, or state. provider "databricks" {} +# Only used when create_storage = true (variable validation enforces that +# aws_region is set in that case). Terraform configures every declared +# provider even when all of its resources have count = 0, and the AWS +# provider hard-fails configuration when its credential chain resolves +# nothing — which would break the default create_storage = false path on +# machines with no AWS setup at all. So when the AWS side is inactive this +# block pins placeholder static credentials + region and skips every +# configure-time check (no AWS API call can ever happen: all aws_* resources +# have count = 0). When create_storage = true the placeholders are null and +# credentials come from the usual environment/shared-config chain. +provider "aws" { + region = var.aws_region != "" ? var.aws_region : "us-east-1" + access_key = var.create_storage ? null : "mock-unused-access-key" + secret_key = var.create_storage ? null : "mock-unused-secret-key" + + skip_credentials_validation = true + skip_requesting_account_id = true + skip_metadata_api_check = true +} + data "databricks_current_user" "me" {} resource "random_id" "suffix" { @@ -32,6 +62,18 @@ locals { # workspace_host variable arrived in. workspace_host = trimsuffix(trimprefix(trimprefix(var.workspace_host, "https://"), "http://"), "/") catalog_name = "${var.prefix}_e2e_${random_id.suffix.hex}" + + # Managed storage root for the e2e catalog. Precedence: + # 1. an explicit storage_root variable always wins; + # 2. else, when create_storage = true, the external location provisioned + # in storage.tf (a per-apply subpath so re-created catalogs never + # collide on a previously-used root); + # 3. else null — inherit the metastore's default storage root. + catalog_storage_root = ( + var.storage_root != "" ? var.storage_root : + var.create_storage ? "${databricks_external_location.e2e[0].url}/${local.catalog_name}" : + null + ) } # --- Isolated Unity Catalog environment --- @@ -44,8 +86,10 @@ resource "databricks_catalog" "e2e" { # storage_root is REQUIRED when the metastore has no default storage root # (common on auto-provisioned metastores). Check with: # databricks metastores summary - # and set the storage_root variable to e.g. s3://bucket/prefix if empty. - storage_root = var.storage_root != "" ? var.storage_root : null + # and either set the storage_root variable to e.g. s3://bucket/prefix, or + # set create_storage = true to have this config provision a bucket + + # external location itself (see storage.tf and local.catalog_storage_root). + storage_root = local.catalog_storage_root } # The Iceberg namespace used by the tests. Pre-created here because diff --git a/internal/impl/iceberg/e2e/databricks/terraform/outputs.tf b/internal/impl/iceberg/e2e/databricks/terraform/outputs.tf index be7c528af6..6a93b9465a 100644 --- a/internal/impl/iceberg/e2e/databricks/terraform/outputs.tf +++ b/internal/impl/iceberg/e2e/databricks/terraform/outputs.tf @@ -17,3 +17,8 @@ output "workspace_host" { output "iceberg_rest_url" { value = "https://${local.workspace_host}/api/2.1/unity-catalog/iceberg-rest" } + +output "storage_bucket" { + description = "S3 bucket provisioned for catalog storage (null unless create_storage = true)" + value = one(aws_s3_bucket.e2e[*].bucket) +} diff --git a/internal/impl/iceberg/e2e/databricks/terraform/storage.tf b/internal/impl/iceberg/e2e/databricks/terraform/storage.tf new file mode 100644 index 0000000000..079a80738b --- /dev/null +++ b/internal/impl/iceberg/e2e/databricks/terraform/storage.tf @@ -0,0 +1,125 @@ +# --- BYO catalog storage (OPTIONAL, off by default) --- +# +# Everything in this file is gated behind create_storage = true. It provisions +# an S3 bucket plus the Unity Catalog plumbing (storage credential → external +# location) so the e2e catalog can live on customer-owned storage. Needed +# when the metastore's own storage can't back the catalog: +# +# * Databricks express-setup trials use "default storage", which does NOT +# support credential vending for external Iceberg REST clients +# (https://docs.databricks.com/aws/en/storage/default-storage), so the +# tests can never talk to a catalog created on it; +# * metastores with no storage_root at all can't create a catalog without +# an explicit managed location. +# +# The resource shape follows the databricks provider's Unity Catalog guide +# verbatim (docs/guides/unity-catalog.md): UC storage credentials have a +# chicken-and-egg with the IAM role — the role's trust policy needs the +# credential's external ID, which only exists after the credential is +# created. The documented break: create the credential FIRST, pointing at the +# role ARN as a *constructed string* (never a resource reference, which would +# be a cycle), then build the role's trust policy from the credential's +# external_id via the databricks_aws_unity_catalog_assume_role_policy data +# source. The companion databricks_aws_unity_catalog_policy data source +# generates the S3 access policy (Get/Put/DeleteObject, ListBucket, +# GetBucketLocation + the self-assume statement UC requires). + +data "aws_caller_identity" "current" { + count = var.create_storage ? 1 : 0 +} + +locals { + # IAM role and storage-credential name (they match, per the UC guide). + uc_role_name = "${var.prefix}-databricks-e2e-uc-${random_id.suffix.hex}" +} + +# --- AWS side --- + +resource "aws_s3_bucket" "e2e" { + count = var.create_storage ? 1 : 0 + + bucket = "${var.prefix}-databricks-e2e-${random_id.suffix.hex}" + force_destroy = true # disposable: destroy removes objects too +} + +data "databricks_aws_unity_catalog_assume_role_policy" "e2e" { + count = var.create_storage ? 1 : 0 + + aws_account_id = data.aws_caller_identity.current[0].account_id + role_name = local.uc_role_name + external_id = databricks_storage_credential.e2e[0].aws_iam_role[0].external_id +} + +data "databricks_aws_unity_catalog_policy" "e2e" { + count = var.create_storage ? 1 : 0 + + aws_account_id = data.aws_caller_identity.current[0].account_id + bucket_name = aws_s3_bucket.e2e[0].id + role_name = local.uc_role_name +} + +resource "aws_iam_role" "uc_access" { + count = var.create_storage ? 1 : 0 + + name = local.uc_role_name + assume_role_policy = data.databricks_aws_unity_catalog_assume_role_policy.e2e[0].json +} + +resource "aws_iam_role_policy" "uc_access" { + count = var.create_storage ? 1 : 0 + + name = "s3-access" + role = aws_iam_role.uc_access[0].id + policy = data.databricks_aws_unity_catalog_policy.e2e[0].json +} + +# --- Databricks side --- + +# Created BEFORE the IAM role exists (see the chicken-and-egg note above), so +# validation must be skipped at create time; the external location below +# validates the whole chain once the role is in place. +resource "databricks_storage_credential" "e2e" { + count = var.create_storage ? 1 : 0 + + name = local.uc_role_name + comment = "Redpanda Connect iceberg e2e (disposable)" + aws_iam_role { + # Constructed string on purpose — referencing aws_iam_role.uc_access here + # would create a dependency cycle. + role_arn = "arn:aws:iam::${data.aws_caller_identity.current[0].account_id}:role/${local.uc_role_name}" + } + skip_validation = true + force_destroy = true +} + +# IAM is eventually consistent: creating the external location immediately +# after the role/policy routinely fails validation with an assume-role error. +# Same workaround the provider docs use for the cross-account workspace role +# (docs/guides/aws-workspace.md). If a live apply still trips on propagation, +# just re-run `terraform apply` — everything here is idempotent. +resource "time_sleep" "uc_role_propagation" { + count = var.create_storage ? 1 : 0 + + create_duration = "30s" + depends_on = [ + aws_iam_role.uc_access, + aws_iam_role_policy.uc_access, + ] +} + +resource "databricks_external_location" "e2e" { + count = var.create_storage ? 1 : 0 + + name = "${var.prefix}-databricks-e2e-${random_id.suffix.hex}" + url = "s3://${aws_s3_bucket.e2e[0].bucket}/e2e" + credential_name = databricks_storage_credential.e2e[0].id + comment = "Redpanda Connect iceberg e2e (disposable)" + force_destroy = true + + depends_on = [time_sleep.uc_role_propagation] +} + +# No explicit grant is needed for the catalog to use this location as its +# storage_root: terraform's principal creates (and therefore owns) the +# external location, and UC owners hold all privileges on their securables, +# including CREATE MANAGED STORAGE. diff --git a/internal/impl/iceberg/e2e/databricks/terraform/terraform.yml b/internal/impl/iceberg/e2e/databricks/terraform/terraform.yml index e5b3c47a1d..00631b2699 100644 --- a/internal/impl/iceberg/e2e/databricks/terraform/terraform.yml +++ b/internal/impl/iceberg/e2e/databricks/terraform/terraform.yml @@ -6,17 +6,22 @@ tasks: cmds: - terraform init + # Extra terraform arguments pass through after `--`, e.g. + # task terraform:apply -- -var create_storage=true -var aws_region=us-east-1 + # Prefer TF_VAR_* environment variables for anything destroy needs to see + # again (see README). + plan: desc: Plan infrastructure changes cmds: - - terraform plan + - terraform plan {{.CLI_ARGS}} apply: desc: Provision infrastructure cmds: - - terraform apply -auto-approve + - terraform apply -auto-approve {{.CLI_ARGS}} destroy: desc: Tear down infrastructure cmds: - - terraform destroy -auto-approve + - terraform destroy -auto-approve {{.CLI_ARGS}} diff --git a/internal/impl/iceberg/e2e/databricks/terraform/variables.tf b/internal/impl/iceberg/e2e/databricks/terraform/variables.tf index 10ef4bbab2..d9c8a8fe8e 100644 --- a/internal/impl/iceberg/e2e/databricks/terraform/variables.tf +++ b/internal/impl/iceberg/e2e/databricks/terraform/variables.tf @@ -16,7 +16,24 @@ variable "schema_name" { } variable "storage_root" { - description = "Managed storage root for the e2e catalog (e.g. s3://bucket/prefix). REQUIRED when the metastore has no default storage root — check with `databricks metastores summary`. Empty means inherit the metastore root." + description = "Managed storage root for the e2e catalog (e.g. s3://bucket/prefix). REQUIRED when the metastore has no default storage root — check with `databricks metastores summary`. Empty means inherit the metastore root (or, with create_storage = true, use the provisioned bucket). An explicit value always wins over create_storage." + type = string + default = "" +} + +variable "create_storage" { + description = "Provision catalog storage too: an S3 bucket + IAM role on the AWS side, and a Unity Catalog storage credential + external location on the Databricks side, used as the e2e catalog's storage_root. Needed on express-trial workspaces (default storage can't serve external Iceberg REST clients) and on metastores with no storage_root. Requires aws_region and AWS credentials in the environment." + type = bool + default = false + + validation { + condition = !var.create_storage || var.aws_region != "" + error_message = "aws_region must be set when create_storage = true." + } +} + +variable "aws_region" { + description = "AWS region for the provisioned catalog storage bucket (e.g. us-east-1). Only used — and required — when create_storage = true." type = string default = "" } From 567486afd8ef7d61dc6e688b5359bf13f08e33b1 Mon Sep 17 00:00:00 2001 From: Ashley Jeffs Date: Mon, 3 Aug 2026 15:06:25 +0100 Subject: [PATCH 08/12] iceberg: strip catalog-prohibited table properties from commits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unity Catalog rejects commits whose property updates carry keys it manages itself — iceberg-go's copy-on-write path defensively sets schema.name-mapping.default on tables without a name mapping, which UC prohibits external clients from writing, so every mutating copy-on-write commit against a UC table failed with "Table properties contain prohibited keys". The committer now learns prohibited keys from the catalog's own rejection (parsing the keys named in the error), warns once per key, and retries with those keys stripped from set-properties updates — via a CatalogIO wrapper that filters only property updates and passes every other update through untouched. Stripping the name mapping is safe: it exists as a read fallback for data files without parquet field IDs, which the Iceberg spec requires and this output always writes. The connector's own redpanda-connect.* properties are never stripped — a catalog prohibiting those fails loudly instead, since silently dropping the timestamp-encoding pin would change write semantics. NewCommitter now takes the table's catalog explicitly (the table type exposes no accessor for it), provided by catalogx so reloaded tables are rebound onto the wrapper too. Verified live against a Databricks Unity Catalog trial workspace: the first mutating commit is rejected, the key is learned and stripped, and the retried commit lands; the copy-on-write round-trip then reads back correctly through Databricks SQL with zero delete files. --- internal/impl/iceberg/catalogx/catalog.go | 21 +- internal/impl/iceberg/committer.go | 106 +++++- internal/impl/iceberg/committer_test.go | 14 +- internal/impl/iceberg/cow_concurrency_test.go | 2 +- .../iceberg/cow_merge_key_roundtrip_test.go | 4 +- .../cow_schema_evolution_disabled_test.go | 2 +- .../cow_temporal_data_roundtrip_test.go | 6 +- internal/impl/iceberg/cow_test.go | 34 +- .../impl/iceberg/cow_type_roundtrip_test.go | 2 +- .../impl/iceberg/prohibited_properties.go | 213 +++++++++++ .../iceberg/prohibited_properties_test.go | 354 ++++++++++++++++++ internal/impl/iceberg/router.go | 2 +- .../impl/iceberg/row_operation_commit_test.go | 32 +- .../impl/iceberg/timestamp_encoding_test.go | 4 +- 14 files changed, 738 insertions(+), 58 deletions(-) create mode 100644 internal/impl/iceberg/prohibited_properties.go create mode 100644 internal/impl/iceberg/prohibited_properties_test.go diff --git a/internal/impl/iceberg/catalogx/catalog.go b/internal/impl/iceberg/catalogx/catalog.go index b168f10327..8b95e7a968 100644 --- a/internal/impl/iceberg/catalogx/catalog.go +++ b/internal/impl/iceberg/catalogx/catalog.go @@ -1,4 +1,4 @@ -// Copyright 2025 Redpanda Data, Inc. +// Copyright 2026 Redpanda Data, Inc. // // Licensed as a Redpanda Enterprise file under the Redpanda Community // License (the "License"); you may not use this file except in compliance with @@ -319,6 +319,25 @@ func (c *Client) loadCatalog() catalog.Catalog { return c.catalog.Load() } +// TableIO returns a table.CatalogIO view of the client — the interface a +// *table.Table binds its commits and refreshes to. It resolves the client's +// current underlying REST catalog on every call, so the returned value stays +// valid across the auth-driven catalog refreshes the client performs +// internally. +func (c *Client) TableIO() table.CatalogIO { + return clientTableIO{c} +} + +type clientTableIO struct{ c *Client } + +func (t clientTableIO) LoadTable(ctx context.Context, ident table.Identifier) (*table.Table, error) { + return t.c.loadCatalog().LoadTable(ctx, ident) +} + +func (t clientTableIO) CommitTable(ctx context.Context, ident table.Identifier, reqs []table.Requirement, updates []table.Update) (table.Metadata, string, error) { + return t.c.loadCatalog().CommitTable(ctx, ident, reqs, updates) +} + // isNamespaceAlreadyExists checks if the error indicates the namespace already exists. func isNamespaceAlreadyExists(err error) bool { return errors.Is(err, catalog.ErrNamespaceAlreadyExists) diff --git a/internal/impl/iceberg/committer.go b/internal/impl/iceberg/committer.go index 08f1d5e319..bf8a7fbea9 100644 --- a/internal/impl/iceberg/committer.go +++ b/internal/impl/iceberg/committer.go @@ -104,6 +104,14 @@ type committer struct { cfg CommitConfig reloadTable func(ctx context.Context) (*table.Table, error) batcher *asyncroutine.Batcher[CommitInput, struct{}] + // stripper wraps the table's catalog at the commit boundary so that + // property keys a catalog rejects as prohibited (learned from the + // rejection error in commitLocked) are filtered from later attempts. It + // is installed unconditionally and is a pass-through until a key is + // learned. Every table the committer retains (initial, reloaded, and + // post-commit — the latter inherits the binding from its transaction) is + // bound to it; see NewCommitter for the rebinding choke points. + stripper *propertyStrippingCatalog // commitMu serializes all commits and guards c.table. The batcher's // doCommit and the direct commitRowDelta path both take it. commitMu sync.Mutex @@ -112,8 +120,15 @@ type committer struct { logger *service.Logger } -// NewCommitter creates a new committer for a specific table. -func NewCommitter(tbl *table.Table, cfg CommitConfig, reloadTable func(ctx context.Context) (*table.Table, error), logger *service.Logger) (*committer, error) { +// NewCommitter creates a new committer for a specific table. cat must be the +// catalog tbl was loaded from (the table.CatalogIO its commits go to); the +// committer rebinds tbl — and every table reloadTable returns — onto a +// wrapper of cat so prohibited property keys can be stripped at the commit +// boundary (see propertyStrippingCatalog). +func NewCommitter(tbl *table.Table, cat table.CatalogIO, cfg CommitConfig, reloadTable func(ctx context.Context) (*table.Table, error), logger *service.Logger) (*committer, error) { + if cat == nil { + return nil, errors.New("creating committer: catalog must not be nil") + } // Defensively clamp MaxRetries to at least 1: commitLocked's retry loop is // `for range cfg.MaxRetries`, so a zero or negative value would never run a // single attempt and return a "committing transaction after 0 attempts" @@ -122,11 +137,24 @@ func NewCommitter(tbl *table.Table, cfg CommitConfig, reloadTable func(ctx conte if cfg.MaxRetries < 1 { cfg.MaxRetries = 1 } + stripper := newPropertyStrippingCatalog(cat) c := &committer{ - table: tbl, - cfg: cfg, - reloadTable: reloadTable, - logger: logger, + table: rebindTable(tbl, stripper), + cfg: cfg, + stripper: stripper, + logger: logger, + } + if reloadTable != nil { + // Single choke point for reloaded tables: every table handle the + // committer adopts after a reload is rebound onto the stripper, so + // retried commits keep flowing through the prohibited-key filter. + c.reloadTable = func(ctx context.Context) (*table.Table, error) { + fresh, err := reloadTable(ctx) + if err != nil { + return nil, err + } + return rebindTable(fresh, stripper), nil + } } batcher, err := asyncroutine.NewBatcher(100, c.doCommit) @@ -493,6 +521,30 @@ func (c *committer) commitLocked(ctx context.Context, commitID string, retryOnUn return attempt > 1, err } tbl, err := txn.Commit(ctx) + // Some engine-backed catalogs (Databricks Unity Catalog) reject a + // commit whose set-properties updates touch reserved keys, naming the + // offending keys in the error (e.g. "Table properties contain + // prohibited keys: schema.name-mapping.default"). Learn those keys, + // arm the stripper, and retry: the rejection is a clean 400 (nothing + // landed), so re-staging from the same base is safe, and the next + // attempt commits with the keys filtered out. Keys under + // reservedTablePropertyPrefix are never stripped — they carry + // connector semantics (e.g. the timestamp-encoding pin) — so a + // catalog prohibiting them fails the commit loudly instead. + if err != nil { + if retry, fatalErr := c.noteProhibitedKeys(attempt, err); fatalErr != nil { + // Reload so the next call uses fresh metadata, mirroring the + // non-retryable branch below. + if reloaded, reloadErr := c.reloadTable(ctx); reloadErr == nil { + c.table = reloaded + } + c.incrCommitFailure() + return attempt > 1, fatalErr + } else if retry { + commitErr = err + continue + } + } // ErrCommitFailed is a clean conflict (our commit did not land), so a // reload-and-retry re-adds our files exactly once. ErrCommitStateUnknown // means the commit may have landed; retrying is only safe when stage @@ -534,6 +586,48 @@ func (c *committer) commitLocked(ctx context.Context, commitID string, retryOnUn return attempt > 1, fmt.Errorf("committing transaction after %d attempts: %w", attempt, commitErr) } +// noteProhibitedKeys inspects a failed commit's error for a catalog +// prohibited-table-property rejection and updates the stripper accordingly. +// It returns retry=true when at least one new (non-reserved) key was learned — +// the caller should count the attempt and re-stage, letting the stripper +// filter the keys on the next commit. It returns a non-nil fatalErr when the +// catalog named a key under reservedTablePropertyPrefix: those keys carry +// connector semantics (the commit-id idempotency token, the +// timestamp-encoding pin) that stripping would silently break, so the commit +// must fail loudly instead. Both zero values mean the error is not a +// prohibited-keys rejection — or it names only keys that are already being +// stripped, in which case retrying would loop futilely — and the caller's +// standard error handling applies. +func (c *committer) noteProhibitedKeys(attempt int, err error) (retry bool, fatalErr error) { + keys := parseProhibitedPropertyKeys(err) + if len(keys) == 0 { + return false, nil + } + var learned, reserved []string + for _, k := range keys { + if strings.HasPrefix(k, reservedTablePropertyPrefix) { + reserved = append(reserved, k) + } else if c.stripper.addProhibitedKey(k) { + learned = append(learned, k) + } + } + if len(reserved) > 0 { + return false, fmt.Errorf( + "catalog prohibits table properties %v, which this connector depends on (%s* keys pin semantics such as the table's timestamp encoding) and refuses to strip: %w", + reserved, reservedTablePropertyPrefix, err) + } + if len(learned) == 0 { + return false, nil + } + // One-time warning per key: addProhibitedKey only reports a key the first + // time it is learned. + for _, k := range learned { + c.logger.Warnf("Catalog prohibits table property %q; stripping it from commits — safe because our data files carry Iceberg field IDs, so the property only duplicates optional metadata (e.g. the name-mapping fallback for ID-less files) that readers of this table never need", k) + } + c.logger.Warnf("Commit attempt %d/%d rejected for prohibited table properties %v; retrying with them stripped", attempt, c.cfg.MaxRetries, learned) + return true, nil +} + // dropAlreadyCommitted returns the subset of files whose paths are not already // referenced by the current snapshot of c.table, which the caller must have just // reloaded. It exists because commit retries re-add the same DataFile objects diff --git a/internal/impl/iceberg/committer_test.go b/internal/impl/iceberg/committer_test.go index dbd1f7b37d..a4572e0982 100644 --- a/internal/impl/iceberg/committer_test.go +++ b/internal/impl/iceberg/committer_test.go @@ -135,7 +135,7 @@ func TestCommitterSkipsDuplicateCheck(t *testing.T) { tbl = seedTable(t, ctx, tbl, 1) logger := service.MockResources().Logger() - c, err := NewCommitter(tbl, CommitConfig{ + c, err := NewCommitter(tbl, cat, CommitConfig{ ManifestMergeEnabled: false, MaxRetries: 1, }, func(context.Context) (*table.Table, error) { return cat.snapshot(), nil }, logger) @@ -264,7 +264,7 @@ func newScriptedCommitter(tb testing.TB, outcomes ...commitOutcome) (*committer, _, mem := newTestTable(tb) cat := &scriptedCatalog{memCatalog: mem, outcomes: outcomes} logger := service.MockResources().Logger() - c, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, + c, err := NewCommitter(cat.snapshot(), cat, CommitConfig{MaxRetries: 3}, func(context.Context) (*table.Table, error) { return cat.snapshot(), nil }, logger) require.NoError(tb, err) tb.Cleanup(c.Close) @@ -353,7 +353,7 @@ func TestStaleSchemaErrorOnAllEntryPoints(t *testing.T) { t.Run("doCommit (append path)", func(t *testing.T) { tbl, cat := newTestTable(t) - c, err := NewCommitter(tbl, CommitConfig{MaxRetries: 1}, reloadFn(cat), service.MockResources().Logger()) + c, err := NewCommitter(tbl, cat, CommitConfig{MaxRetries: 1}, reloadFn(cat), service.MockResources().Logger()) require.NoError(t, err) defer c.Close() cur := c.currentSchemaID() @@ -363,7 +363,7 @@ func TestStaleSchemaErrorOnAllEntryPoints(t *testing.T) { t.Run("commitRowDelta (merge-on-read path)", func(t *testing.T) { tbl, cat := newTestTable(t) - c, err := NewCommitter(tbl, CommitConfig{MaxRetries: 1}, reloadFn(cat), service.MockResources().Logger()) + c, err := NewCommitter(tbl, cat, CommitConfig{MaxRetries: 1}, reloadFn(cat), service.MockResources().Logger()) require.NoError(t, err) defer c.Close() cur := c.currentSchemaID() @@ -376,7 +376,7 @@ func TestStaleSchemaErrorOnAllEntryPoints(t *testing.T) { t.Run("commitOverwrite (copy-on-write path)", func(t *testing.T) { tbl, cat := newTestTable(t) - c, err := NewCommitter(tbl, CommitConfig{MaxRetries: 1}, reloadFn(cat), service.MockResources().Logger()) + c, err := NewCommitter(tbl, cat, CommitConfig{MaxRetries: 1}, reloadFn(cat), service.MockResources().Logger()) require.NoError(t, err) defer c.Close() cur := c.currentSchemaID() @@ -398,7 +398,7 @@ func TestCommitStampsMaxSnapshotAge(t *testing.T) { ctx := t.Context() tbl, cat := newTestTable(t) const age = 48 * time.Hour - c, err := NewCommitter(tbl, CommitConfig{MaxRetries: 1, MaxSnapshotAge: age}, reloadFn(cat), service.MockResources().Logger()) + c, err := NewCommitter(tbl, cat, CommitConfig{MaxRetries: 1, MaxSnapshotAge: age}, reloadFn(cat), service.MockResources().Logger()) require.NoError(t, err) defer c.Close() @@ -420,7 +420,7 @@ func TestNewCommitterClampsMaxRetries(t *testing.T) { for _, n := range []int{0, -3} { t.Run(fmt.Sprintf("max_retries_%d", n), func(t *testing.T) { tbl, cat := newTestTable(t) - c, err := NewCommitter(tbl, CommitConfig{MaxRetries: n}, reloadFn(cat), service.MockResources().Logger()) + c, err := NewCommitter(tbl, cat, CommitConfig{MaxRetries: n}, reloadFn(cat), service.MockResources().Logger()) require.NoError(t, err) defer c.Close() assert.Equal(t, 1, c.cfg.MaxRetries, "MaxRetries must be clamped to at least 1") diff --git a/internal/impl/iceberg/cow_concurrency_test.go b/internal/impl/iceberg/cow_concurrency_test.go index afa1f5a1cc..ec4bf9c323 100644 --- a/internal/impl/iceberg/cow_concurrency_test.go +++ b/internal/impl/iceberg/cow_concurrency_test.go @@ -141,7 +141,7 @@ func TestCOWConcurrentCommittersConverge(t *testing.T) { // MaxRetries lets the loser of the race reload and re-stage. mkWriter := func(t *testing.T, occ *occCatalog) *writer { t.Helper() - comm, err := NewCommitter(occ.snapshot(), CommitConfig{MaxRetries: 10}, + comm, err := NewCommitter(occ.snapshot(), occ, CommitConfig{MaxRetries: 10}, func(context.Context) (*table.Table, error) { return occ.snapshot(), nil }, logger) require.NoError(t, err) t.Cleanup(comm.Close) diff --git a/internal/impl/iceberg/cow_merge_key_roundtrip_test.go b/internal/impl/iceberg/cow_merge_key_roundtrip_test.go index 3dd97bb86e..c013399426 100644 --- a/internal/impl/iceberg/cow_merge_key_roundtrip_test.go +++ b/internal/impl/iceberg/cow_merge_key_roundtrip_test.go @@ -172,7 +172,7 @@ func TestCOWMergeKeyRoundTrip(t *testing.T) { seedByPay := invertByPayload(t, seedMap) // Drive a real copy-on-write upsert(k2)+delete(k3) batch. - comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) + comm, err := NewCommitter(cat.snapshot(), cat, CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) require.NoError(t, err) defer comm.Close() w := cowWriter(t, cat.snapshot(), "k") @@ -216,7 +216,7 @@ func driveCOWKeyed(t testing.TB, ctx context.Context, sc *iceberg.Schema, seed [ tbl, cat := newCOWTable(t, sc) _ = seedMergeKeyRows(t, ctx, tbl, cat, seed) - comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) + comm, err := NewCommitter(cat.snapshot(), cat, CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) require.NoError(t, err) defer comm.Close() w := cowWriter(t, cat.snapshot(), "k") diff --git a/internal/impl/iceberg/cow_schema_evolution_disabled_test.go b/internal/impl/iceberg/cow_schema_evolution_disabled_test.go index 0512b94e2f..7c3f2bd2b1 100644 --- a/internal/impl/iceberg/cow_schema_evolution_disabled_test.go +++ b/internal/impl/iceberg/cow_schema_evolution_disabled_test.go @@ -59,7 +59,7 @@ func TestCOWUpsertUnknownColumnSchemaEvolutionDisabled(t *testing.T) { // A real committer over the in-memory catalog — so if the write erroneously // reached the overwrite commit, it would land a new snapshot we could detect. - comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3, SkipFormatUpgrade: true}, reloadFn(cat), logger) + comm, err := NewCommitter(cat.snapshot(), cat, CommitConfig{MaxRetries: 3, SkipFormatUpgrade: true}, reloadFn(cat), logger) require.NoError(t, err) defer comm.Close() w := cowWriter(t, cat.snapshot(), "id") diff --git a/internal/impl/iceberg/cow_temporal_data_roundtrip_test.go b/internal/impl/iceberg/cow_temporal_data_roundtrip_test.go index 1de0ee8334..69c22aeb2a 100644 --- a/internal/impl/iceberg/cow_temporal_data_roundtrip_test.go +++ b/internal/impl/iceberg/cow_temporal_data_roundtrip_test.go @@ -149,7 +149,7 @@ func TestCOWNumericEpochTimestampDataColumnUpsert(t *testing.T) { {"id": int64(2), "ts": sentinel}, }) - comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) + comm, err := NewCommitter(cat.snapshot(), cat, CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) require.NoError(t, err) defer comm.Close() w := cowWriterWithResolver(t, cat.snapshot(), false, "id") @@ -201,7 +201,7 @@ func TestCOWNumericEpochTimestampRequireSchemaMetadata(t *testing.T) { sentinel := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) _ = seedCOWRows(t, ctx, seedTbl, cat, "id", []map[string]any{{"id": int64(1), "ts": sentinel}}) - comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) + comm, err := NewCommitter(cat.snapshot(), cat, CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) require.NoError(t, err) defer comm.Close() w := cowWriterWithResolver(t, cat.snapshot(), true, "id") @@ -220,7 +220,7 @@ func TestCOWNumericEpochTimestampRequireSchemaMetadata(t *testing.T) { sentinel := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) _ = seedCOWRows(t, ctx, seedTbl, cat, "id", []map[string]any{{"id": int64(1), "ts": sentinel}}) - comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) + comm, err := NewCommitter(cat.snapshot(), cat, CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) require.NoError(t, err) defer comm.Close() w := cowWriterWithResolver(t, cat.snapshot(), true, "id") diff --git a/internal/impl/iceberg/cow_test.go b/internal/impl/iceberg/cow_test.go index ec773057a9..a368edeb5b 100644 --- a/internal/impl/iceberg/cow_test.go +++ b/internal/impl/iceberg/cow_test.go @@ -407,7 +407,7 @@ func TestCommitOverwriteCleansUpOrphansOnFailure(t *testing.T) { // A non-retryable failure guarantees the mutation's commit does not land, so // the files the overwrite wrote are genuine orphans. fc := &flakyCatalog{memCatalog: cat, failuresLeft: 1 << 30, failErr: errors.New("storage unavailable")} - comm, err := NewCommitter(fc.snapshot(), CommitConfig{MaxRetries: 2}, func(context.Context) (*table.Table, error) { return fc.snapshot(), nil }, logger) + comm, err := NewCommitter(fc.snapshot(), fc, CommitConfig{MaxRetries: 2}, func(context.Context) (*table.Table, error) { return fc.snapshot(), nil }, logger) require.NoError(t, err) defer comm.Close() w := cowWriter(t, fc.snapshot(), "id") @@ -441,7 +441,7 @@ func TestCommitOverwriteIdempotentOnUnknownState(t *testing.T) { seedTbl, mem := newCOWTable(t, sc) _ = appendCOWRows(t, ctx, seedTbl, map[int64]string{1: "one", 2: "two", 3: "three"}) cat := &scriptedCatalog{memCatalog: mem, outcomes: []commitOutcome{outcome}} - comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, func(context.Context) (*table.Table, error) { return cat.snapshot(), nil }, logger) + comm, err := NewCommitter(cat.snapshot(), cat, CommitConfig{MaxRetries: 3}, func(context.Context) (*table.Table, error) { return cat.snapshot(), nil }, logger) require.NoError(t, err) t.Cleanup(comm.Close) w := cowWriter(t, cat.snapshot(), "id") @@ -546,7 +546,7 @@ func TestCOWv1TableStaysV1(t *testing.T) { tbl = appendCOWRows(t, ctx, tbl, map[int64]string{1: "one", 2: "two", 3: "three"}) require.EqualValues(t, 1, cat.snapshot().Metadata().Version(), "seeding must not upgrade the table") - comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3, SkipFormatUpgrade: true}, reloadFn(cat), service.MockResources().Logger()) + comm, err := NewCommitter(cat.snapshot(), cat, CommitConfig{MaxRetries: 3, SkipFormatUpgrade: true}, reloadFn(cat), service.MockResources().Logger()) require.NoError(t, err) defer comm.Close() w := cowWriter(t, tbl, "id") @@ -581,7 +581,7 @@ func TestCOWUpsertDeleteRoundTrip(t *testing.T) { seedTbl = appendCOWRows(t, ctx, seedTbl, map[int64]string{1: "one", 2: "two", 3: "three"}) // Build a writer whose committer shares the catalog. - comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) + comm, err := NewCommitter(cat.snapshot(), cat, CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) require.NoError(t, err) defer comm.Close() w := cowWriter(t, seedTbl, "id") @@ -616,7 +616,7 @@ func TestCOWOnlyDeletesFastPath(t *testing.T) { seedTbl, cat := newCOWTable(t, sc) seedTbl = appendCOWRows(t, ctx, seedTbl, map[int64]string{1: "one", 2: "two"}) - comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) + comm, err := NewCommitter(cat.snapshot(), cat, CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) require.NoError(t, err) defer comm.Close() w := cowWriter(t, seedTbl, "id") @@ -640,7 +640,7 @@ func TestCOWOnlyInsertsUsesAppend(t *testing.T) { ) seedTbl, cat := newCOWTable(t, sc) - comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) + comm, err := NewCommitter(cat.snapshot(), cat, CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) require.NoError(t, err) defer comm.Close() w := cowWriter(t, seedTbl, "id") @@ -767,7 +767,7 @@ func TestCOWPartitionedUpsertDeleteRoundTrip(t *testing.T) { {"id": 4, "region": "apac", "payload": "four"}, }) - comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) + comm, err := NewCommitter(cat.snapshot(), cat, CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) require.NoError(t, err) defer comm.Close() w := cowWriter(t, seedTbl, "id") @@ -819,7 +819,7 @@ func TestCOWPartitionKeyChangeRoundTrip(t *testing.T) { {"id": 1, "region": "us", "payload": "one"}, }) - comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) + comm, err := NewCommitter(cat.snapshot(), cat, CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) require.NoError(t, err) defer comm.Close() w := cowWriter(t, seedTbl, "id") @@ -863,7 +863,7 @@ func TestCOWBucketPartitionRoundTrip(t *testing.T) { {"id": 3, "region": "apac", "payload": "three"}, }) - comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) + comm, err := NewCommitter(cat.snapshot(), cat, CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) require.NoError(t, err) defer comm.Close() w := cowWriter(t, seedTbl, "id") @@ -916,7 +916,7 @@ func TestCOWCaseInsensitiveUpsert(t *testing.T) { _, err = tx.Commit(ctx) require.NoError(t, err) - comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) + comm, err := NewCommitter(cat.snapshot(), cat, CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) require.NoError(t, err) defer comm.Close() w := cowWriterCI(t, cat.snapshot(), "Id") @@ -1076,7 +1076,7 @@ func TestCOWInsertPlusUpsertSameKeyDuplicates(t *testing.T) { ) seedTbl, cat := newCOWTable(t, sc) - comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) + comm, err := NewCommitter(cat.snapshot(), cat, CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) require.NoError(t, err) defer comm.Close() w := cowWriter(t, seedTbl, "id") @@ -1193,7 +1193,7 @@ func TestCommitOverwriteNoLeakOnConflictThenSuccess(t *testing.T) { // attempt 1 = clean conflict (nothing lands), attempt 2 = success. cat := &scriptedCatalog{memCatalog: mem, outcomes: []commitOutcome{commitConflict}} - comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, + comm, err := NewCommitter(cat.snapshot(), cat, CommitConfig{MaxRetries: 3}, func(context.Context) (*table.Table, error) { return cat.snapshot(), nil }, service.MockResources().Logger()) require.NoError(t, err) defer comm.Close() @@ -1242,7 +1242,7 @@ func TestCommitOverwritePreservesFilesOnTerminalUnknown(t *testing.T) { outcomes[i] = commitUnknownNoLand } cat := &scriptedCatalog{memCatalog: mem, outcomes: outcomes} - comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: maxRetries}, + comm, err := NewCommitter(cat.snapshot(), cat, CommitConfig{MaxRetries: maxRetries}, func(context.Context) (*table.Table, error) { return cat.snapshot(), nil }, service.MockResources().Logger()) require.NoError(t, err) defer comm.Close() @@ -1286,7 +1286,7 @@ func TestCommitOverwriteResumesAfterReloadFailures(t *testing.T) { } return cat.snapshot(), nil } - comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 5}, reload, service.MockResources().Logger()) + comm, err := NewCommitter(cat.snapshot(), cat, CommitConfig{MaxRetries: 5}, reload, service.MockResources().Logger()) require.NoError(t, err) defer comm.Close() w := cowWriter(t, cat.snapshot(), "id") @@ -1333,7 +1333,7 @@ func TestCommitOverwriteGracefulWithoutListableFS(t *testing.T) { fc := &flakyCatalog{memCatalog: mem, failuresLeft: 1 << 30, failErr: errors.New("storage unavailable")} nlSnap := table.New(fc.ident, fc.meta, fc.metadataLocation, func(context.Context) (iceio.IO, error) { return nonListableFS{}, nil }, fc) - comm, err := NewCommitter(nlSnap, CommitConfig{MaxRetries: 2}, + comm, err := NewCommitter(nlSnap, fc, CommitConfig{MaxRetries: 2}, func(context.Context) (*table.Table, error) { return nlSnap, nil }, service.MockResources().Logger()) require.NoError(t, err) defer comm.Close() @@ -1362,7 +1362,7 @@ func TestCleanupOverwriteReferenceGuard(t *testing.T) { seedTbl, cat := newCOWTable(t, sc) seedTbl = appendCOWRows(t, ctx, seedTbl, map[int64]string{1: "one", 2: "two"}) - comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 1}, reloadFn(cat), service.MockResources().Logger()) + comm, err := NewCommitter(cat.snapshot(), cat, CommitConfig{MaxRetries: 1}, reloadFn(cat), service.MockResources().Logger()) require.NoError(t, err) defer comm.Close() @@ -1398,7 +1398,7 @@ func TestCommitOverwriteReturnsNewReaderError(t *testing.T) { seedTbl = appendCOWRows(t, ctx, seedTbl, map[int64]string{1: "one", 2: "two"}) seedCount := countParquetFiles(t, seedTbl.Location()) - comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) + comm, err := NewCommitter(cat.snapshot(), cat, CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) require.NoError(t, err) defer comm.Close() diff --git a/internal/impl/iceberg/cow_type_roundtrip_test.go b/internal/impl/iceberg/cow_type_roundtrip_test.go index 04c9ff2eb7..0e546cfb4b 100644 --- a/internal/impl/iceberg/cow_type_roundtrip_test.go +++ b/internal/impl/iceberg/cow_type_roundtrip_test.go @@ -63,7 +63,7 @@ func cowMutateDirect(t testing.TB, ctx context.Context, sc *iceberg.Schema, seed } } - comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) + comm, err := NewCommitter(cat.snapshot(), cat, CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) require.NoError(t, err) defer comm.Close() w = cowWriter(t, cat.snapshot(), "id") diff --git a/internal/impl/iceberg/prohibited_properties.go b/internal/impl/iceberg/prohibited_properties.go new file mode 100644 index 0000000000..abf963d8c5 --- /dev/null +++ b/internal/impl/iceberg/prohibited_properties.go @@ -0,0 +1,213 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md + +package iceberg + +import ( + "context" + "encoding/json" + "fmt" + "regexp" + "strings" + "sync" + + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/table" +) + +// reservedTablePropertyPrefix namespaces the table properties this connector +// itself depends on (the commit-id idempotency token in snapshot summaries and +// the timestamp-encoding pin, see commitIDProp and +// icebergx.TimestampEncodingProperty). Keys under this prefix must NEVER be +// stripped from commits: e.g. the timestamp-encoding pin is what guarantees a +// table's parquet files all carry one timestamp annotation, so silently +// dropping it would let later writers flip the encoding mid-table. If a +// catalog prohibits one of these keys the commit fails loudly instead. +const reservedTablePropertyPrefix = "redpanda-connect." + +// prohibitedKeysRe matches a catalog rejection that names the table property +// keys it refuses, e.g. Databricks Unity Catalog's +// +// BadRequestException: Malformed request: INVALID_PARAMETER_VALUE: +// Table properties contain prohibited keys: schema.name-mapping.default +// +// The match is case-insensitive on the "prohibited keys" marker, tolerates any +// prefix text, an optional colon, and captures the remainder of the message +// for tokenising in parseProhibitedPropertyKeys. +var prohibitedKeysRe = regexp.MustCompile(`(?i)prohibited\s+keys?\s*:?\s*(.+)`) + +// parseProhibitedPropertyKeys extracts the property keys named by a catalog's +// prohibited-table-property rejection. It returns nil when err does not look +// like such a rejection. The parse is deliberately tolerant: surrounding text, +// case differences on the marker, quotes/brackets around the list, trailing +// prose after a key, and sentence-terminating punctuation are all accepted. +func parseProhibitedPropertyKeys(err error) []string { + if err == nil { + return nil + } + m := prohibitedKeysRe.FindStringSubmatch(err.Error()) + if m == nil { + return nil + } + var keys []string + for tok := range strings.SplitSeq(m[1], ",") { + tok = strings.Trim(strings.TrimSpace(tok), "\"'`[]() ") + // A property key is a run of [A-Za-z0-9._-]; cut the token at the + // first character outside that set so trailing prose ("a.b (remove + // them)") does not leak into the key. + if i := strings.IndexFunc(tok, func(r rune) bool { return !isPropertyKeyRune(r) }); i >= 0 { + tok = tok[:i] + } + // Keys never start or end with a dot; a trailing one is sentence + // punctuation ("... keys: a.b."). + tok = strings.Trim(tok, ".") + if tok != "" { + keys = append(keys, tok) + } + } + return keys +} + +func isPropertyKeyRune(r rune) bool { + return r == '.' || r == '-' || r == '_' || + (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') +} + +// propertyStrippingCatalog is a table.CatalogIO wrapper that filters +// catalog-prohibited property keys out of set-properties updates at the commit +// boundary. Some engine-backed catalogs (Databricks Unity Catalog at least) +// reject commits from external clients that set specific reserved table +// properties — e.g. schema.name-mapping.default, which iceberg-go's +// copy-on-write and merge-on-read deletion paths stage defensively whenever a +// table has no name mapping. The committer learns the prohibited keys from the +// catalog's own rejection (see commitLocked) and records them here; the next +// commit attempt then goes through with those keys removed. +// +// Stripping schema.name-mapping.default is safe: every data file this +// connector (and iceberg-go) writes carries Iceberg field IDs in its parquet +// schema, and a name mapping is only a read-time fallback for resolving files +// WITHOUT field IDs — so a table whose files all have IDs never consults it. +// Similarly write.delete.mode is only a writer-side default; the committer +// enforces copy-on-write behaviour in-process regardless of whether the +// property persists (commitOverwrite stages it into each transaction, where it +// steers txn.Delete before the update is stripped at this boundary). +// +// Only set-properties updates are ever filtered — every other update type +// (add-snapshot, set-snapshot-ref, ...) passes through untouched — and keys +// under reservedTablePropertyPrefix are never accepted into the strip set (see +// addProhibitedKey). The wrapper is installed unconditionally at committer +// construction and is a pure pass-through until a key is learned. +type propertyStrippingCatalog struct { + inner table.CatalogIO + + mu sync.RWMutex + strip map[string]struct{} +} + +func newPropertyStrippingCatalog(inner table.CatalogIO) *propertyStrippingCatalog { + return &propertyStrippingCatalog{inner: inner, strip: map[string]struct{}{}} +} + +// addProhibitedKey records key for stripping from future commits, reporting +// whether it was newly added. Keys under reservedTablePropertyPrefix are +// refused (returning false): those carry connector semantics that must not be +// silently dropped — the caller is expected to fail loudly instead. +func (p *propertyStrippingCatalog) addProhibitedKey(key string) bool { + if strings.HasPrefix(key, reservedTablePropertyPrefix) { + return false + } + p.mu.Lock() + defer p.mu.Unlock() + if _, ok := p.strip[key]; ok { + return false + } + p.strip[key] = struct{}{} + return true +} + +// LoadTable delegates to the wrapped catalog. The returned table keeps its +// binding to the inner catalog, which is fine for the two callers that exist: +// iceberg-go's refresh-and-replay retry only reads the fresh table's metadata, +// and the committer rebinds every table it retains (see NewCommitter). +func (p *propertyStrippingCatalog) LoadTable(ctx context.Context, ident table.Identifier) (*table.Table, error) { + return p.inner.LoadTable(ctx, ident) +} + +// CommitTable filters learned prohibited keys out of set-properties updates, +// then delegates to the wrapped catalog. With an empty strip set the updates +// slice is forwarded untouched. +func (p *propertyStrippingCatalog) CommitTable(ctx context.Context, ident table.Identifier, reqs []table.Requirement, updates []table.Update) (table.Metadata, string, error) { + filtered, err := p.filterUpdates(updates) + if err != nil { + return nil, "", fmt.Errorf("stripping prohibited table properties from commit updates: %w", err) + } + return p.inner.CommitTable(ctx, ident, reqs, filtered) +} + +// filterUpdates returns updates with the strip set's keys removed from every +// set-properties update. A set-properties update left with no keys is dropped +// entirely; updates of any other type, and set-properties updates naming no +// stripped key, pass through as the original values. iceberg-go's concrete +// update structs are unexported JSON-tagged action structs, so the property +// map is extracted via a JSON round-trip and rebuilt with the exported +// table.NewSetPropertiesUpdate constructor (same action, same no-op +// PostCommit). +func (p *propertyStrippingCatalog) filterUpdates(updates []table.Update) ([]table.Update, error) { + p.mu.RLock() + n := len(p.strip) + strip := make([]string, 0, n) + for k := range p.strip { + strip = append(strip, k) + } + p.mu.RUnlock() + if n == 0 { + return updates, nil + } + + out := make([]table.Update, 0, len(updates)) + for _, u := range updates { + if u.Action() != table.UpdateSetProperties { + out = append(out, u) + continue + } + raw, err := json.Marshal(u) + if err != nil { + return nil, fmt.Errorf("marshaling set-properties update: %w", err) + } + var payload struct { + Updates iceberg.Properties `json:"updates"` + } + if err := json.Unmarshal(raw, &payload); err != nil { + return nil, fmt.Errorf("unmarshaling set-properties update: %w", err) + } + removed := false + for _, k := range strip { + if _, ok := payload.Updates[k]; ok { + delete(payload.Updates, k) + removed = true + } + } + switch { + case !removed: + out = append(out, u) + case len(payload.Updates) == 0: + // The update only set prohibited keys; drop it outright. + default: + out = append(out, table.NewSetPropertiesUpdate(payload.Updates)) + } + } + return out, nil +} + +// rebindTable returns a table identical to tbl but bound to cat, so every +// transaction commit and refresh issued through the returned handle flows +// through cat. This mirrors the table.New rebinding pattern iceberg-go's own +// tests use; tbl.FS is the table's FSysF as a method value. +func rebindTable(tbl *table.Table, cat table.CatalogIO) *table.Table { + return table.New(tbl.Identifier(), tbl.Metadata(), tbl.MetadataLocation(), tbl.FS, cat) +} diff --git a/internal/impl/iceberg/prohibited_properties_test.go b/internal/impl/iceberg/prohibited_properties_test.go new file mode 100644 index 0000000000..a25a375ab7 --- /dev/null +++ b/internal/impl/iceberg/prohibited_properties_test.go @@ -0,0 +1,354 @@ +// Copyright 2026 Redpanda Data, Inc. +// +// Licensed as a Redpanda Enterprise file under the Redpanda Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md + +package iceberg + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "strings" + "sync" + "testing" + + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/table" + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/redpanda-data/benthos/v4/public/service" +) + +// These tests pin the error-driven prohibited-property stripping added for +// engine-backed catalogs (Databricks Unity Catalog) that reject commits whose +// set-properties updates touch reserved keys. The live failure this guards +// against: a copy-on-write mutation fails because iceberg-go defensively sets +// schema.name-mapping.default when the table has no name mapping, and UC +// prohibits external clients writing that key. + +// liveUCProhibitedKeysError reproduces the exact error a live Databricks Unity +// Catalog run returned for a copy-on-write commit (modulo the offending key +// list, which UC populates with the keys it saw). +func liveUCProhibitedKeysError(keys ...string) error { + return fmt.Errorf("BadRequestException: Malformed request: INVALID_PARAMETER_VALUE: Table properties contain prohibited keys: %s", strings.Join(keys, ", ")) +} + +// updatePropertyKeys extracts a set-properties update's key/value map via the +// same JSON round-trip the production filter uses (the concrete update struct +// is unexported in iceberg-go). +func updatePropertyKeys(u table.Update) map[string]string { + raw, err := json.Marshal(u) + if err != nil { + return nil + } + var payload struct { + Updates map[string]string `json:"updates"` + } + if err := json.Unmarshal(raw, &payload); err != nil { + return nil + } + return payload.Updates +} + +// prohibitingCatalog is a table.CatalogIO that models Unity Catalog's +// prohibited-key enforcement: any CommitTable whose set-properties updates +// contain a prohibited key is rejected with the exact live UC error naming the +// offending keys; every other commit is applied through the embedded +// memCatalog. +type prohibitingCatalog struct { + *memCatalog + prohibited []string + commits int + rejections int +} + +func (p *prohibitingCatalog) CommitTable(ctx context.Context, ident table.Identifier, reqs []table.Requirement, updates []table.Update) (table.Metadata, string, error) { + p.commits++ + var offending []string + for _, u := range updates { + if u.Action() != table.UpdateSetProperties { + continue + } + props := updatePropertyKeys(u) + for _, k := range p.prohibited { + if _, ok := props[k]; ok { + offending = append(offending, k) + } + } + } + if len(offending) > 0 { + p.rejections++ + return nil, "", liveUCProhibitedKeysError(offending...) + } + return p.memCatalog.CommitTable(ctx, ident, reqs, updates) +} + +func (p *prohibitingCatalog) snapshot() *table.Table { + return rebindTable(p.memCatalog.snapshot(), p) +} + +// lockedBuffer is a goroutine-safe bytes.Buffer for capturing committer logs +// under -race. +type lockedBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *lockedBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *lockedBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +func capturedLogger(buf *lockedBuffer) *service.Logger { + return service.NewLoggerFromSlog(slog.New(slog.NewTextHandler(buf, &slog.HandlerOptions{ + Level: slog.LevelDebug, + }))) +} + +// TestCOWOverwriteStripsProhibitedKeys drives a real copy-on-write overwrite +// against a catalog that prohibits reserved property keys, reproducing the +// live Unity Catalog failure. The commit must succeed on the retry with the +// prohibited keys stripped, apply the mutation exactly once, keep the +// prohibited keys out of the final metadata, and log the one-time warning. +func TestCOWOverwriteStripsProhibitedKeys(t *testing.T) { + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}, + iceberg.NestedField{ID: 2, Name: "payload", Type: iceberg.PrimitiveTypes.String, Required: false}, + ) + + // setup seeds id=1,2,3 through the plain in-memory catalog (Transaction. + // Append does not auto-set a name mapping, mirroring a UC-created table), + // then wraps it in the prohibiting catalog so only the mutation under test + // is subject to key enforcement. + setup := func(t *testing.T, prohibited ...string) (*prohibitingCatalog, *writer, *lockedBuffer) { + ctx := t.Context() + seedTbl, mem := newCOWTable(t, sc) + _ = appendCOWRows(t, ctx, seedTbl, map[int64]string{1: "one", 2: "two", 3: "three"}) + require.NotContains(t, mem.snapshot().Properties(), "schema.name-mapping.default", + "precondition: the seeded table must have no name mapping, so the COW path stages one") + + cat := &prohibitingCatalog{memCatalog: mem, prohibited: prohibited} + var buf lockedBuffer + comm, err := NewCommitter(cat.snapshot(), cat, CommitConfig{MaxRetries: 3}, + func(context.Context) (*table.Table, error) { return cat.snapshot(), nil }, capturedLogger(&buf)) + require.NoError(t, err) + t.Cleanup(comm.Close) + w := cowWriter(t, cat.snapshot(), "id") + w.committer = comm + return cat, w, &buf + } + + want := map[int64]string{1: "one", 2: "TWO", 3: "three"} + upsert := func() service.MessageBatch { + return service.MessageBatch{cowMsg(t, "upsert", map[string]any{"id": 2, "payload": "TWO"})} + } + + t.Run("name mapping stripped after live UC rejection", func(t *testing.T) { + ctx := t.Context() + cat, w, buf := setup(t, "schema.name-mapping.default") + + require.NoError(t, w.Write(ctx, upsert()), "the overwrite must succeed once the prohibited key is stripped") + + assert.Equal(t, 1, cat.rejections, "exactly one attempt is rejected before the strip set is armed") + assert.Equal(t, 2, cat.commits, "reject once, then succeed on the retry") + assert.Equal(t, 1, countSnapshotsWithCommitID(cat.snapshot()), "mutation applied exactly once") + assert.Equal(t, want, scanRows(t, ctx, cat.snapshot())) + + props := cat.snapshot().Properties() + assert.NotContains(t, props, "schema.name-mapping.default", + "the prohibited key must not reach the committed metadata") + assert.Equal(t, table.WriteModeCopyOnWrite, props[table.WriteDeleteModeKey], + "non-prohibited keys in the same commit must still land") + + logs := buf.String() + assert.Contains(t, logs, "prohibits table property", "the strip warning must be logged") + assert.Contains(t, logs, "schema.name-mapping.default", "the strip warning must name the key") + assert.Contains(t, logs, "stripping it from commits", "the strip warning must state the action") + }) + + // Unity Catalog may also prohibit other reserved keys such as + // write.delete.mode, which our own commitOverwrite sets defensively. A + // multi-key rejection must strip every named key in one retry. + t.Run("multi-key rejection stripped in one retry", func(t *testing.T) { + ctx := t.Context() + cat, w, _ := setup(t, "write.delete.mode", "schema.name-mapping.default") + + require.NoError(t, w.Write(ctx, upsert())) + + assert.Equal(t, 1, cat.rejections) + assert.Equal(t, 2, cat.commits, "both keys are learned from a single rejection") + assert.Equal(t, 1, countSnapshotsWithCommitID(cat.snapshot()), "mutation applied exactly once") + assert.Equal(t, want, scanRows(t, ctx, cat.snapshot())) + + props := cat.snapshot().Properties() + assert.NotContains(t, props, "schema.name-mapping.default") + assert.NotContains(t, props, table.WriteDeleteModeKey) + }) +} + +// alwaysProhibitingCatalog rejects every commit with the live UC error naming +// a fixed key list, regardless of the updates' content. Used to prove that a +// catalog prohibiting a reserved redpanda-connect.* key fails loudly rather +// than being stripped. +type alwaysProhibitingCatalog struct { + *memCatalog + keys []string + commits int +} + +func (p *alwaysProhibitingCatalog) CommitTable(context.Context, table.Identifier, []table.Requirement, []table.Update) (table.Metadata, string, error) { + p.commits++ + return nil, "", liveUCProhibitedKeysError(p.keys...) +} + +func (p *alwaysProhibitingCatalog) snapshot() *table.Table { + return rebindTable(p.memCatalog.snapshot(), p) +} + +// TestProhibitedReservedKeyFailsLoudly pins the safety guard: keys under +// redpanda-connect.* carry connector semantics (e.g. the timestamp-encoding +// pin), so a catalog that prohibits them must fail the commit with a clear +// error instead of silently stripping them — and must not burn retries doing +// so. +func TestProhibitedReservedKeyFailsLoudly(t *testing.T) { + ctx := t.Context() + _, mem := newTestTable(t) + cat := &alwaysProhibitingCatalog{memCatalog: mem, keys: []string{"redpanda-connect.timestamp-encoding"}} + + c, err := NewCommitter(cat.snapshot(), cat, CommitConfig{MaxRetries: 3}, + func(context.Context) (*table.Table, error) { return cat.snapshot(), nil }, service.MockResources().Logger()) + require.NoError(t, err) + defer c.Close() + + df := synthDataFile(t, cat.snapshot().Spec(), fmt.Sprintf("%s/data/reserved-%s.parquet", cat.location, uuid.New())) + err = c.Commit(ctx, CommitInput{Files: []iceberg.DataFile{df}, SchemaID: c.currentSchemaID()}) + require.Error(t, err) + assert.Contains(t, err.Error(), "redpanda-connect.timestamp-encoding") + assert.Contains(t, err.Error(), "refuses to strip") + assert.Equal(t, 1, cat.commits, "a reserved-key rejection must fail on the first attempt, not retry") +} + +// TestParseProhibitedPropertyKeys pins the rejection-parsing grammar: a +// case-insensitive "prohibited key(s)" marker with arbitrary prefix text, an +// optional colon, and a comma-separated key list that tolerates quotes, +// brackets, trailing prose, and sentence punctuation. +func TestParseProhibitedPropertyKeys(t *testing.T) { + for _, tc := range []struct { + name string + err error + want []string + }{ + { + name: "exact live Unity Catalog error", + err: errors.New("committing copy-on-write overwrite: committing transaction: BadRequestException: Malformed request: INVALID_PARAMETER_VALUE: Table properties contain prohibited keys: schema.name-mapping.default"), + want: []string{"schema.name-mapping.default"}, + }, + { + name: "multiple keys", + err: errors.New("Table properties contain prohibited keys: a.b, c.d"), + want: []string{"a.b", "c.d"}, + }, + { + name: "prefix text and case-insensitive marker", + err: errors.New("rpc error: SOME WRAPPER: Prohibited Keys: write.delete.mode"), + want: []string{"write.delete.mode"}, + }, + { + name: "singular key marker", + err: errors.New("prohibited key: schema.name-mapping.default"), + want: []string{"schema.name-mapping.default"}, + }, + { + name: "quoted and bracketed list", + err: errors.New(`prohibited keys: ["a.b", "c-d.e_f"]`), + want: []string{"a.b", "c-d.e_f"}, + }, + { + name: "trailing prose after a key", + err: errors.New("prohibited keys: a.b (remove them and retry)"), + want: []string{"a.b"}, + }, + { + name: "sentence-terminating period", + err: errors.New("prohibited keys: schema.name-mapping.default."), + want: []string{"schema.name-mapping.default"}, + }, + { + name: "unrelated error", + err: errors.New("commit failed, refresh and try again"), + want: nil, + }, + { + name: "nil error", + err: nil, + want: nil, + }, + } { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, parseProhibitedPropertyKeys(tc.err)) + }) + } +} + +// TestStripperFiltersOnlySetPropertiesUpdates pins the filtering guarantees: +// with stripping active, non-property updates pass through as the SAME values +// (never rewritten), set-properties updates lose only the stripped keys, an +// update left empty is dropped, and untouched set-properties updates keep +// their original value. It also pins the strip-set rules: dedupe on re-add and +// refusal of reserved redpanda-connect.* keys. +func TestStripperFiltersOnlySetPropertiesUpdates(t *testing.T) { + s := newPropertyStrippingCatalog(nil) + + // Pass-through while the strip set is empty: same slice, no copies. + unfilteredProps := table.NewSetPropertiesUpdate(iceberg.Properties{"schema.name-mapping.default": "m"}) + out, err := s.filterUpdates([]table.Update{unfilteredProps}) + require.NoError(t, err) + require.Len(t, out, 1) + assert.Same(t, table.Update(unfilteredProps), out[0], "an empty strip set must not touch any update") + + require.True(t, s.addProhibitedKey("schema.name-mapping.default")) + assert.False(t, s.addProhibitedKey("schema.name-mapping.default"), "re-learning a key must report not-new (one-time warning)") + assert.False(t, s.addProhibitedKey("redpanda-connect.timestamp-encoding"), "reserved keys must never enter the strip set") + + snapUpd := table.NewAddSnapshotUpdate(&table.Snapshot{ + SnapshotID: 42, + TimestampMs: 1, + ManifestList: "s3://bucket/manifest-list.avro", + Summary: &table.Summary{Operation: table.OpAppend}, + }) + fmtUpd := table.NewUpgradeFormatVersionUpdate(2) + mixedProps := table.NewSetPropertiesUpdate(iceberg.Properties{ + "schema.name-mapping.default": "m", + "write.delete.mode": "copy-on-write", + }) + onlyStripped := table.NewSetPropertiesUpdate(iceberg.Properties{"schema.name-mapping.default": "m"}) + cleanProps := table.NewSetPropertiesUpdate(iceberg.Properties{"redpanda-connect.timestamp-encoding": "spec"}) + + out, err = s.filterUpdates([]table.Update{snapUpd, mixedProps, fmtUpd, onlyStripped, cleanProps}) + require.NoError(t, err) + require.Len(t, out, 4, "the update that only set stripped keys must be dropped") + + assert.Same(t, table.Update(snapUpd), out[0], "an add-snapshot update must pass through untouched") + assert.Same(t, table.Update(fmtUpd), out[2], "a non-property update must pass through untouched") + assert.Same(t, table.Update(cleanProps), out[3], "a set-properties update naming no stripped key must pass through untouched") + + require.Equal(t, table.UpdateSetProperties, out[1].Action()) + assert.Equal(t, map[string]string{"write.delete.mode": "copy-on-write"}, updatePropertyKeys(out[1]), + "only the stripped key may be removed from a mixed set-properties update") +} diff --git a/internal/impl/iceberg/router.go b/internal/impl/iceberg/router.go index ce5f8644ab..982ec74c40 100644 --- a/internal/impl/iceberg/router.go +++ b/internal/impl/iceberg/router.go @@ -774,7 +774,7 @@ func (r *Router) createWriter(ctx context.Context, key tableKey, entry *tableEnt // irreversible v1->v2 upgrade the merge-on-read path needs. commitCfg := r.commitCfg commitCfg.SkipFormatUpgrade = r.rowOpCfg.MergeStrategy == mergeStrategyCOW - comm, err := NewCommitter(committerTbl, commitCfg, reloadTable, r.logger) + comm, err := NewCommitter(committerTbl, client.TableIO(), commitCfg, reloadTable, r.logger) if err != nil { return nil, fmt.Errorf("creating committer: %w", err) } diff --git a/internal/impl/iceberg/row_operation_commit_test.go b/internal/impl/iceberg/row_operation_commit_test.go index ed1177282c..e8ae7b8a6f 100644 --- a/internal/impl/iceberg/row_operation_commit_test.go +++ b/internal/impl/iceberg/row_operation_commit_test.go @@ -86,7 +86,7 @@ func TestCommitDeleteOnlyProducesDeleteSnapshot(t *testing.T) { deleteFiles, err := w.writeEqualityDeletes(ctx, service.MessageBatch{structuredMsg(t, map[string]any{"id": 2})}) require.NoError(t, err) - c, err := NewCommitter(tbl, CommitConfig{MaxRetries: 1}, reloadFn(cat), service.MockResources().Logger()) + c, err := NewCommitter(tbl, cat, CommitConfig{MaxRetries: 1}, reloadFn(cat), service.MockResources().Logger()) require.NoError(t, err) defer c.Close() @@ -110,7 +110,7 @@ func TestCommitUpsertProducesOverwriteSnapshot(t *testing.T) { // the shape an upsert produces. dataFile := synthDataFile(t, tbl.Spec(), fmt.Sprintf("%s/data/new-%s.parquet", tbl.Location(), uuid.New())) - c, err := NewCommitter(tbl, CommitConfig{MaxRetries: 1}, reloadFn(cat), service.MockResources().Logger()) + c, err := NewCommitter(tbl, cat, CommitConfig{MaxRetries: 1}, reloadFn(cat), service.MockResources().Logger()) require.NoError(t, err) defer c.Close() @@ -132,7 +132,7 @@ func TestCommitInsertOnlyStaysAppend(t *testing.T) { dataFile := synthDataFile(t, tbl.Spec(), fmt.Sprintf("%s/data/ins-%s.parquet", tbl.Location(), uuid.New())) - c, err := NewCommitter(tbl, CommitConfig{MaxRetries: 1}, reloadFn(cat), service.MockResources().Logger()) + c, err := NewCommitter(tbl, cat, CommitConfig{MaxRetries: 1}, reloadFn(cat), service.MockResources().Logger()) require.NoError(t, err) defer c.Close() @@ -211,7 +211,7 @@ func TestCommitRowDeltaConcurrentNotCoalesced(t *testing.T) { ctx := t.Context() tbl, cat := newTestTable(t) logger := service.MockResources().Logger() - c, err := NewCommitter(tbl, CommitConfig{MaxRetries: 5}, reloadFn(cat), logger) + c, err := NewCommitter(tbl, cat, CommitConfig{MaxRetries: 5}, reloadFn(cat), logger) require.NoError(t, err) defer c.Close() @@ -325,7 +325,7 @@ func TestCommitUpgradesFormatVersionV1ToV2(t *testing.T) { tbl, cat := newTestTableV1(t) require.EqualValues(t, 1, tbl.Metadata().Version(), "precondition: table starts at v1") - c, err := NewCommitter(tbl, CommitConfig{MaxRetries: 1}, reloadFn(cat), service.MockResources().Logger()) + c, err := NewCommitter(tbl, cat, CommitConfig{MaxRetries: 1}, reloadFn(cat), service.MockResources().Logger()) require.NoError(t, err) defer c.Close() @@ -352,7 +352,7 @@ func TestCommitRetriesOnConflict(t *testing.T) { _, plain := newTestTable(t) fc := &flakyCatalog{memCatalog: plain, failuresLeft: 2, failErr: rest.ErrCommitFailed} ftbl := fc.snapshot() - c, err := NewCommitter(ftbl, CommitConfig{MaxRetries: 5}, func(context.Context) (*table.Table, error) { return fc.snapshot(), nil }, logger) + c, err := NewCommitter(ftbl, fc, CommitConfig{MaxRetries: 5}, func(context.Context) (*table.Table, error) { return fc.snapshot(), nil }, logger) require.NoError(t, err) defer c.Close() @@ -366,7 +366,7 @@ func TestCommitRetriesOnConflict(t *testing.T) { _, plain := newTestTable(t) fc := &flakyCatalog{memCatalog: plain, failuresLeft: 1 << 30, failErr: rest.ErrCommitFailed} ftbl := fc.snapshot() - c, err := NewCommitter(ftbl, CommitConfig{MaxRetries: 2}, func(context.Context) (*table.Table, error) { return fc.snapshot(), nil }, logger) + c, err := NewCommitter(ftbl, fc, CommitConfig{MaxRetries: 2}, func(context.Context) (*table.Table, error) { return fc.snapshot(), nil }, logger) require.NoError(t, err) defer c.Close() @@ -390,7 +390,7 @@ func TestWriteCleansUpFilesOnCommitFailure(t *testing.T) { // Control: a healthy committer leaves the written parquet file in place. t.Run("control writes a file", func(t *testing.T) { tbl, cat := newTestTable(t) - c, err := NewCommitter(tbl, CommitConfig{MaxRetries: 1}, reloadFn(cat), logger) + c, err := NewCommitter(tbl, cat, CommitConfig{MaxRetries: 1}, reloadFn(cat), logger) require.NoError(t, err) defer c.Close() require.NoError(t, os.MkdirAll(filepath.Join(tbl.Location(), "data"), 0o755)) @@ -403,7 +403,7 @@ func TestWriteCleansUpFilesOnCommitFailure(t *testing.T) { _, plain := newTestTable(t) fc := &flakyCatalog{memCatalog: plain, failuresLeft: 1 << 30, failErr: errors.New("storage unavailable")} ftbl := fc.snapshot() - c, err := NewCommitter(ftbl, CommitConfig{MaxRetries: 2}, func(context.Context) (*table.Table, error) { return fc.snapshot(), nil }, logger) + c, err := NewCommitter(ftbl, fc, CommitConfig{MaxRetries: 2}, func(context.Context) (*table.Table, error) { return fc.snapshot(), nil }, logger) require.NoError(t, err) defer c.Close() @@ -445,7 +445,7 @@ func TestCommitRowDeltaIdempotentOnUnknownState(t *testing.T) { ctx := t.Context() _, mem := newTestTable(t) cat := &scriptedCatalog{memCatalog: mem, outcomes: []commitOutcome{commitLandThenUnknown}} - c, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, func(context.Context) (*table.Table, error) { return cat.snapshot(), nil }, logger) + c, err := NewCommitter(cat.snapshot(), cat, CommitConfig{MaxRetries: 3}, func(context.Context) (*table.Table, error) { return cat.snapshot(), nil }, logger) require.NoError(t, err) defer c.Close() @@ -463,7 +463,7 @@ func TestCommitRowDeltaIdempotentOnUnknownState(t *testing.T) { ctx := t.Context() _, mem := newTestTable(t) cat := &scriptedCatalog{memCatalog: mem, outcomes: []commitOutcome{commitUnknownNoLand}} - c, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, func(context.Context) (*table.Table, error) { return cat.snapshot(), nil }, logger) + c, err := NewCommitter(cat.snapshot(), cat, CommitConfig{MaxRetries: 3}, func(context.Context) (*table.Table, error) { return cat.snapshot(), nil }, logger) require.NoError(t, err) defer c.Close() @@ -481,7 +481,7 @@ func TestCommitRowDeltaIdempotentOnUnknownState(t *testing.T) { ctx := t.Context() _, mem := newTestTable(t) cat := &scriptedCatalog{memCatalog: mem, outcomes: []commitOutcome{commitConflict}} - c, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, func(context.Context) (*table.Table, error) { return cat.snapshot(), nil }, logger) + c, err := NewCommitter(cat.snapshot(), cat, CommitConfig{MaxRetries: 3}, func(context.Context) (*table.Table, error) { return cat.snapshot(), nil }, logger) require.NoError(t, err) defer c.Close() @@ -501,7 +501,7 @@ func TestCommitRowDeltaIdempotentOnUnknownState(t *testing.T) { ctx := t.Context() _, mem := newTestTable(t) cat := &scriptedCatalog{memCatalog: mem, outcomes: []commitOutcome{commitLandThenFail}} - c, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, func(context.Context) (*table.Table, error) { return cat.snapshot(), nil }, logger) + c, err := NewCommitter(cat.snapshot(), cat, CommitConfig{MaxRetries: 3}, func(context.Context) (*table.Table, error) { return cat.snapshot(), nil }, logger) require.NoError(t, err) defer c.Close() @@ -520,7 +520,7 @@ func TestCommitRowDeltaIdempotentOnUnknownState(t *testing.T) { func TestCommitRowDeltaWritesCommitIDToSummary(t *testing.T) { ctx := t.Context() tbl, cat := newTestTable(t) - c, err := NewCommitter(tbl, CommitConfig{MaxRetries: 1}, reloadFn(cat), service.MockResources().Logger()) + c, err := NewCommitter(tbl, cat, CommitConfig{MaxRetries: 1}, reloadFn(cat), service.MockResources().Logger()) require.NoError(t, err) defer c.Close() @@ -547,7 +547,7 @@ func BenchmarkCommitterAppend(b *testing.B) { for i := 0; i < b.N; i++ { b.StopTimer() tbl, cat := newTestTable(b) - c, err := NewCommitter(tbl, CommitConfig{MaxRetries: 1}, reloadFn(cat), logger) + c, err := NewCommitter(tbl, cat, CommitConfig{MaxRetries: 1}, reloadFn(cat), logger) require.NoError(b, err) df := synthDataFile(b, tbl.Spec(), fmt.Sprintf("%s/data/bench-%d.parquet", tbl.Location(), i)) b.StartTimer() @@ -577,7 +577,7 @@ func BenchmarkCommitterRowDelta(b *testing.B) { deleteFiles, err := w.writeEqualityDeletes(ctx, service.MessageBatch{structuredMsg(b, map[string]any{"id": i})}) require.NoError(b, err) dataFile := synthDataFile(b, tbl.Spec(), fmt.Sprintf("%s/data/bench-%d.parquet", tbl.Location(), i)) - c, err := NewCommitter(tbl, CommitConfig{MaxRetries: 1}, reloadFn(cat), logger) + c, err := NewCommitter(tbl, cat, CommitConfig{MaxRetries: 1}, reloadFn(cat), logger) require.NoError(b, err) b.StartTimer() diff --git a/internal/impl/iceberg/timestamp_encoding_test.go b/internal/impl/iceberg/timestamp_encoding_test.go index f7a8211de7..25b47dbf33 100644 --- a/internal/impl/iceberg/timestamp_encoding_test.go +++ b/internal/impl/iceberg/timestamp_encoding_test.go @@ -346,7 +346,7 @@ func TestCOWLegacyTimestampGuard(t *testing.T) { newCOWEncWriter := func(t *testing.T, cat *memCatalog, enc icebergx.TimestampEncoding) *writer { t.Helper() - comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3, SkipFormatUpgrade: true}, reloadFn(cat), service.MockResources().Logger()) + comm, err := NewCommitter(cat.snapshot(), cat, CommitConfig{MaxRetries: 3, SkipFormatUpgrade: true}, reloadFn(cat), service.MockResources().Logger()) require.NoError(t, err) t.Cleanup(comm.Close) w := cowWriter(t, cat.snapshot(), "id") @@ -439,7 +439,7 @@ func TestCOWLegacyTimestampGuard(t *testing.T) { _, cat := newEncTable(t, encTestSchema(), nil) seedEncTimestampFile(t, ctx, cat, icebergx.TimestampEncodingLegacy) - comm, err := NewCommitter(cat.snapshot(), CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) + comm, err := NewCommitter(cat.snapshot(), cat, CommitConfig{MaxRetries: 3}, reloadFn(cat), service.MockResources().Logger()) require.NoError(t, err) defer comm.Close() w := &writer{ From 5ad8891d044fdbee396c1d161adc62b4fde85ad4 Mon Sep 17 00:00:00 2001 From: Ashley Jeffs Date: Mon, 3 Aug 2026 16:42:54 +0100 Subject: [PATCH 09/12] iceberg: add short description to merge_strategy Matches the short-description convention the other row-operation fields gained upstream. --- internal/impl/iceberg/config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/impl/iceberg/config.go b/internal/impl/iceberg/config.go index 199394b450..797c6c4eb6 100644 --- a/internal/impl/iceberg/config.go +++ b/internal/impl/iceberg/config.go @@ -327,6 +327,7 @@ array:list service.NewStringEnumField(ioFieldMergeStrategy, string(mergeStrategyMOR), string(mergeStrategyCOW)). Description("How `upsert` and `delete` are materialised on disk.\n\n* `merge-on-read` (the default) writes Iceberg v2 equality-delete files. Deletes are applied at read time, so writes stay cheap and streaming-friendly, but only catalog-native / Flink-world engines can read the result — engine-backed catalogs such as Snowflake and the Databricks Unity Catalog cannot read equality deletes.\n* `copy-on-write` rewrites whole data files so the table only ever contains plain data files (no delete files), which every engine can read — including Snowflake and Databricks Unity Catalog. It works on version-1 or version-2 tables and never forces the irreversible v1->v2 upgrade. The trade-off is heavy write amplification: each mutating batch rewrites every data file that contains a touched key, so it is a batch / moderate-throughput mode. Sort the table by the identifier key and use large batches so each rewrite touches as few files as possible.\n\nSee the <> section above for the full decision guide, copy-on-write support matrix (column and merge-key types, partitioning, table format), and maintenance guidance."). + ShortDescription("How upsert and delete are materialised: merge-on-read (equality deletes) or copy-on-write (plain data files every engine can read)."). Default(string(mergeStrategyMOR)). Advanced(), From 8466c837410eae2d7ee568db2bd039bb0a49d253 Mon Sep 17 00:00:00 2001 From: Ashley Jeffs Date: Tue, 4 Aug 2026 16:25:54 +0100 Subject: [PATCH 10/12] Add changelog entries for the copy-on-write merge strategy (#4666) --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index da4aa7f33f..3d54060537 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,18 @@ Changelog All notable changes to this project will be documented in this file. +## 4.104.0 - 2026-08-04 + +### Added + +- iceberg: Add an opt-in `merge_strategy: copy-on-write` so `upsert` and `delete` land as plain data files readable by engine-backed catalogs such as Snowflake and the Databricks Unity Catalog. ([@Jeffail](https://github.com/Jeffail), [#4666](https://github.com/redpanda-data/connect/pull/4666)) + +### Fixed + +- iceberg: No-timezone `timestamp` columns are now written to parquet with the spec-correct `isAdjustedToUTC=false` annotation; the encoding is pinned per table via the `redpanda-connect.timestamp-encoding` property so existing tables never change or mix encodings. ([@Jeffail](https://github.com/Jeffail), [#4666](https://github.com/redpanda-data/connect/pull/4666)) +- iceberg: Row-mutation commits are now idempotent across ambiguous catalog responses, preventing duplicate rows on redelivery after a commit whose response was lost; mutation snapshots carry a `redpanda-connect.commit-id` summary property. ([@Jeffail](https://github.com/Jeffail), [#4666](https://github.com/redpanda-data/connect/pull/4666)) +- iceberg: Table properties a catalog refuses to accept (for example Unity Catalog's managed keys) are now learned from the rejection and stripped from subsequent commits instead of failing every mutation. ([@Jeffail](https://github.com/Jeffail), [#4666](https://github.com/redpanda-data/connect/pull/4666)) + ## 4.103.1 - 2026-07-31 ### Added From 421266041780a186ce3d16902208fa02dc9818ee Mon Sep 17 00:00:00 2001 From: Ashley Jeffs Date: Tue, 4 Aug 2026 16:40:43 +0100 Subject: [PATCH 11/12] iceberg: give the integration package a CI budget matching its suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change roughly triples the package's integration test count, so the runner's default five-minute go test timeout no longer fits — the run was killed mid-test in CI. Grant the package fifteen minutes, in line with the other larger suites. Also rewords a few code comments to describe their hazards directly rather than by internal reference. --- cmd/tools/integration/packages.json | 2 +- internal/impl/iceberg/cow.go | 4 ++-- internal/impl/iceberg/cow_merge_key_roundtrip_test.go | 4 ++-- internal/impl/iceberg/cow_polish_test.go | 2 +- internal/impl/iceberg/cow_test.go | 2 +- .../integration/cow_row_operation_types_integration_test.go | 2 +- internal/impl/iceberg/output_iceberg.go | 2 +- internal/impl/iceberg/writer.go | 2 +- 8 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cmd/tools/integration/packages.json b/cmd/tools/integration/packages.json index 6bcdf7deb0..b94f79d1c3 100644 --- a/cmd/tools/integration/packages.json +++ b/cmd/tools/integration/packages.json @@ -25,7 +25,7 @@ {"path":"./internal/impl/gcp/enterprise/changestreams"}, {"path":"./internal/impl/gcp/enterprise/changestreams/metadata"}, {"path":"./internal/impl/hdfs"}, - {"path":"./internal/impl/iceberg/integration"}, + {"path":"./internal/impl/iceberg/integration","timeout":"15m"}, {"path":"./internal/impl/influxdb"}, {"path":"./internal/impl/jira"}, {"path":"./internal/impl/kafka","timeout":"10m"}, diff --git a/internal/impl/iceberg/cow.go b/internal/impl/iceberg/cow.go index 818bd99232..6c5f226a8a 100644 --- a/internal/impl/iceberg/cow.go +++ b/internal/impl/iceberg/cow.go @@ -357,7 +357,7 @@ func (w *writer) lookupKeyValue(msg *service.Message, field iceberg.NestedField, // // The overriding invariant is that the literal's encoding MUST match how // buildCOWRecordFactory stores the same value, or the overwrite filter selects -// no rows and the upsert/delete silently becomes a no-op (the CON-490 hazard). +// no rows and the upsert/delete silently becomes a no-op (a silent no-op instead of a mutation). // The rewrite stores every value by running it through deleteKeyJSONValue and // then array.RecordFromJSON, so this function derives each literal from that // same canonicalisation: @@ -368,7 +368,7 @@ func (w *writer) lookupKeyValue(msg *service.Message, field iceberg.NestedField, // StringLiteral.To — so filter and storage share an encoding by construction // (date days, microsecond time-of-day, uuid bytes). // - timestamp/timestamptz: deleteKeyJSONValue requires a time.Time and rejects -// a bare number (a numeric timestamp is ambiguous — the exact CON-490 silent +// a bare number (a numeric timestamp is ambiguous — the exact silent // no-match), so it is reused for that validation. The literal is then built // directly from the time.Time as UnixMicro, because StringLiteral.To's // timestamp parser does not accept the RFC3339 form the data path stores; diff --git a/internal/impl/iceberg/cow_merge_key_roundtrip_test.go b/internal/impl/iceberg/cow_merge_key_roundtrip_test.go index c013399426..20e28451ab 100644 --- a/internal/impl/iceberg/cow_merge_key_roundtrip_test.go +++ b/internal/impl/iceberg/cow_merge_key_roundtrip_test.go @@ -34,7 +34,7 @@ import ( // (see TestCOWDecimalMergeKeyGated) because iceberg-go's overwrite filter panics // on a decimal literal. // -// The load-bearing guard is against the CON-490 silent-no-match bug: if the +// The load-bearing guard is against the silent-no-match bug: if the // filter literal's encoding disagreed with the stored value, the overwrite // would match nothing, leaving a duplicate of the upserted key and failing to // delete — which these assertions (exact final row set, keyed identity) @@ -368,7 +368,7 @@ func TestCOWBooleanMergeKeyGated(t *testing.T) { // // Consequence (documented, not a bug): two instants differing only below the // microsecond collapse to the same key. That is inherent to a microsecond column -// and is consistent between filter and storage, so it never causes the CON-490 +// and is consistent between filter and storage, so it never causes the // silent-no-match — it only means sub-microsecond precision is not part of the // key identity. func TestCOWSubMicrosecondTimestampKeyTruncation(t *testing.T) { diff --git a/internal/impl/iceberg/cow_polish_test.go b/internal/impl/iceberg/cow_polish_test.go index 24560bbb23..356bb36668 100644 --- a/internal/impl/iceberg/cow_polish_test.go +++ b/internal/impl/iceberg/cow_polish_test.go @@ -78,7 +78,7 @@ func TestCOWAmplificationWarning(t *testing.T) { // TestSplitByOperationCOWCountsFeedMetrics pins item 3.2 at the reachable seam: // the per-operation counts that drive iceberg_row_operations_total{operation=...} // are computed by splitByOperation. Because the emitted counter *values* are not -// readable from a unit test (see the CON-490 note in output_iceberg.go), this +// readable from a unit test (see the metrics note in output_iceberg.go), this // guards the numbers that would be handed to incrInserted/incrUpserted/ // incrDeleted instead — including the last-writer-wins per-key collapse, so a // counter can never over-count a repeatedly-mutated key. diff --git a/internal/impl/iceberg/cow_test.go b/internal/impl/iceberg/cow_test.go index a368edeb5b..35598d8daa 100644 --- a/internal/impl/iceberg/cow_test.go +++ b/internal/impl/iceberg/cow_test.go @@ -316,7 +316,7 @@ func TestBuildCOWFilterUnsupportedKeyType(t *testing.T) { assert.Contains(t, err.Error(), "does not support merge key column") } -// TestBuildCOWFilterBareNumberTemporalKeyRejected pins the CON-490 guard on the +// TestBuildCOWFilterBareNumberTemporalKeyRejected pins the silent-no-match guard on the // merge-key path: a temporal key given as a bare number is ambiguous (the data // path cannot reproduce how a number would be interpreted), so it must be // rejected loudly rather than silently building a literal that matches nothing. diff --git a/internal/impl/iceberg/integration/cow_row_operation_types_integration_test.go b/internal/impl/iceberg/integration/cow_row_operation_types_integration_test.go index 4638b0df3f..bae5c98ed4 100644 --- a/internal/impl/iceberg/integration/cow_row_operation_types_integration_test.go +++ b/internal/impl/iceberg/integration/cow_row_operation_types_integration_test.go @@ -54,7 +54,7 @@ func assertCOWSnapshot(t *testing.T, ctx context.Context, infra *testInfrastruct // literal (cowKeyLiteral) and the rewrite's re-encoding of the surviving key // (cowMassage -> deleteKeyJSONValue -> Arrow) are the riskiest code: a wrong // encoding makes the overwrite filter select no rows, so the delete/upsert -// silently becomes a no-op (the CON-490 hazard) or the rewritten key is +// silently becomes a no-op (a silent no-op instead of a mutation) or the rewritten key is // corrupted. All existing unit round-trips read back through iceberg-go's own // Arrow scan, so a self-consistent-but-wrong encoding would pass. DuckDB reads // the parquet + manifests itself, so a wrong encoding shows up here as a wrong diff --git a/internal/impl/iceberg/output_iceberg.go b/internal/impl/iceberg/output_iceberg.go index f2d35b2f07..a5f9c397df 100644 --- a/internal/impl/iceberg/output_iceberg.go +++ b/internal/impl/iceberg/output_iceberg.go @@ -85,7 +85,7 @@ func newOpMetrics(m *service.Metrics) *opMetrics { } } -// NOTE(CON-490, item 3.2): there is no importable seam here to assert emitted +// NOTE: there is no importable seam here to assert emitted // counter *values* in a unit test. service.MockResources() backs its Metrics // with metrics.Noop() (which discards writes), and the only readable in-memory // implementation (metrics.NewLocal) lives in benthos-internal diff --git a/internal/impl/iceberg/writer.go b/internal/impl/iceberg/writer.go index beaed229fa..b44193feec 100644 --- a/internal/impl/iceberg/writer.go +++ b/internal/impl/iceberg/writer.go @@ -522,7 +522,7 @@ func deleteKeyJSONValue(t iceberg.Type, v any) (any, error) { // (its unit — seconds/millis/micros — cannot be recovered without schema // metadata), so accepting one blindly would silently mismatch what the // insert path wrote. Merge-key callers keep this strict on purpose (an - // unambiguous key must round-trip exactly, the CON-490 guarantee); the + // unambiguous key must round-trip exactly, the silent-no-match guarantee); the // copy-on-write data-column path (cowMassage) resolves a numeric temporal to // a time.Time via the shredder's unit-aware conversion BEFORE calling this, // so a numeric only reaches here for a merge key or a genuinely From 2f24fbbb2e19db9723bf0d5d219a025146ff8369 Mon Sep 17 00:00:00 2001 From: Ashley Jeffs Date: Tue, 4 Aug 2026 20:57:03 +0100 Subject: [PATCH 12/12] iceberg: harden prohibited-key learning and encoding-pin edges MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-ups on the newest machinery: - The prohibited-keys error parse now requires the colon-delimited list form, stops at the first token that isn't shaped like a property key, and only learns keys the failed commit actually sent — so prose or unrelated errors mentioning the phrase can no longer poison the strip set. The reserved-prefix refusal is case-insensitive. - Learned prohibited keys now live on the router's table entry and seed each new committer, so writer recreation (schema evolution, transient errors) no longer costs a rejected commit per generation to re-learn them — which also removes a latent no-progress loop against catalogs that name one key per rejection. - An ambiguous commit response is handled by the idempotency check even when its text mentions prohibited keys, so a landed-but-unreported commit can never be retried without the duplicate guard. - NewCommitter rejects a nil reload function instead of panicking later. - The copy-on-write legacy-encoding guard and docs now tell operators to stop connector writers around an encoding migration, and comments on the stamp path describe the REST-catalog race semantics honestly. - Adds the previously-unexercised probe test for a table whose timestamptz column sorts ahead of its timestamp column. --- .../components/pages/outputs/iceberg.adoc | 2 +- internal/impl/iceberg/committer.go | 69 ++++--- internal/impl/iceberg/committer_test.go | 11 ++ internal/impl/iceberg/config.go | 2 +- internal/impl/iceberg/cow.go | 4 +- .../impl/iceberg/prohibited_properties.go | 183 +++++++++++++++--- .../iceberg/prohibited_properties_test.go | 183 +++++++++++++++++- internal/impl/iceberg/router.go | 16 ++ internal/impl/iceberg/timestamp_encoding.go | 26 ++- .../impl/iceberg/timestamp_encoding_test.go | 39 ++++ 10 files changed, 472 insertions(+), 63 deletions(-) diff --git a/docs/modules/components/pages/outputs/iceberg.adoc b/docs/modules/components/pages/outputs/iceberg.adoc index 30ce09775d..b36e2081c8 100644 --- a/docs/modules/components/pages/outputs/iceberg.adoc +++ b/docs/modules/components/pages/outputs/iceberg.adoc @@ -291,7 +291,7 @@ To guarantee an existing table never ends up with a mix of the two annotations, * For an existing table without the property, the output resolves the encoding automatically on first contact and stamps the result onto the table: if the schema has no no-timezone `timestamp` column, or the table has no data files, it resolves `spec`; otherwise the output inspects one data file's parquet footer and adopts whatever that file already contains (`legacy` for `isAdjustedToUTC=true`). A table that cannot be probed (unreadable file) fails the write rather than risk mixing annotations. * Once stamped, the property is authoritative and the probe never runs again. An unrecognised property value is a hard error. -A table pinned `legacy` keeps receiving the legacy annotation on every new file — byte-identical to what previous releases wrote — so appends and `merge-on-read` continue working unchanged forever. The one restriction is mutating `copy-on-write` (`upsert`/`delete`): it must rewrite existing files, which the legacy annotation prevents, so such writes fail upfront with an actionable error (pure `insert` batches still work). To migrate a legacy table to the spec encoding: rewrite/compact the table's data files with an engine that writes the spec annotation (e.g. Spark's `rewrite_data_files`), then set the table property `redpanda-connect.timestamp-encoding` to `spec`. Alternatively, keep the table on `merge-on-read`. +A table pinned `legacy` keeps receiving the legacy annotation on every new file — byte-identical to what previous releases wrote — so appends and `merge-on-read` continue working unchanged forever. The one restriction is mutating `copy-on-write` (`upsert`/`delete`): it must rewrite existing files, which the legacy annotation prevents, so such writes fail upfront with an actionable error (pure `insert` batches still work). To migrate a legacy table to the spec encoding: rewrite/compact the table's data files with an engine that writes the spec annotation (e.g. Spark's `rewrite_data_files`), then set the table property `redpanda-connect.timestamp-encoding` to `spec`, keeping any running instances of this output that write to the table stopped (or restarting them) around the migration — a live writer only re-reads the property when its writer is recreated. Alternatively, keep the table on `merge-on-read`. == Performance diff --git a/internal/impl/iceberg/committer.go b/internal/impl/iceberg/committer.go index bf8a7fbea9..79a7f79f37 100644 --- a/internal/impl/iceberg/committer.go +++ b/internal/impl/iceberg/committer.go @@ -83,6 +83,13 @@ type CommitConfig struct { // unnecessary, irreversible v1->v2 upgrade. Merge-on-read/append leave this // false: their equality-delete path requires v2. SkipFormatUpgrade bool + // ProhibitedKeys, when non-nil, is the shared set of catalog-prohibited + // property keys the committer's stripper reads and learns into. The + // router owns one per tableEntry so keys learned from one committer's + // rejection persist across writer recreation (writeWithRetry closes the + // writer on every failure) instead of costing a rejected commit per + // generation. Nil gets a fresh, private set. + ProhibitedKeys *prohibitedKeySet } // StaleSchemaError is returned when data was written with a schema @@ -129,6 +136,13 @@ func NewCommitter(tbl *table.Table, cat table.CatalogIO, cfg CommitConfig, reloa if cat == nil { return nil, errors.New("creating committer: catalog must not be nil") } + // commitLocked dereferences reloadTable on every failure branch, so a nil + // one would panic mid-commit rather than fail construction; reject it here + // with a clear error. The production caller (Router.createWriter) always + // supplies one. + if reloadTable == nil { + return nil, errors.New("creating committer: reloadTable must not be nil") + } // Defensively clamp MaxRetries to at least 1: commitLocked's retry loop is // `for range cfg.MaxRetries`, so a zero or negative value would never run a // single attempt and return a "committing transaction after 0 attempts" @@ -137,24 +151,22 @@ func NewCommitter(tbl *table.Table, cat table.CatalogIO, cfg CommitConfig, reloa if cfg.MaxRetries < 1 { cfg.MaxRetries = 1 } - stripper := newPropertyStrippingCatalog(cat) + stripper := newPropertyStrippingCatalog(cat, cfg.ProhibitedKeys) c := &committer{ table: rebindTable(tbl, stripper), cfg: cfg, stripper: stripper, logger: logger, } - if reloadTable != nil { - // Single choke point for reloaded tables: every table handle the - // committer adopts after a reload is rebound onto the stripper, so - // retried commits keep flowing through the prohibited-key filter. - c.reloadTable = func(ctx context.Context) (*table.Table, error) { - fresh, err := reloadTable(ctx) - if err != nil { - return nil, err - } - return rebindTable(fresh, stripper), nil + // Single choke point for reloaded tables: every table handle the + // committer adopts after a reload is rebound onto the stripper, so + // retried commits keep flowing through the prohibited-key filter. + c.reloadTable = func(ctx context.Context) (*table.Table, error) { + fresh, err := reloadTable(ctx) + if err != nil { + return nil, err } + return rebindTable(fresh, stripper), nil } batcher, err := asyncroutine.NewBatcher(100, c.doCommit) @@ -531,7 +543,13 @@ func (c *committer) commitLocked(ctx context.Context, commitID string, retryOnUn // reservedTablePropertyPrefix are never stripped — they carry // connector semantics (e.g. the timestamp-encoding pin) — so a // catalog prohibiting them fails the commit loudly instead. - if err != nil { + // + // An ErrCommitStateUnknown is never treated as a prohibited-keys + // rejection, even if its text mentions them: the commit may have + // landed server-side, and the prohibited-keys retry re-stages WITHOUT + // the reload + commit-id idempotency check below, so it could apply + // the mutation twice. Unknown-state takes precedence. + if err != nil && !errors.Is(err, rest.ErrCommitStateUnknown) { if retry, fatalErr := c.noteProhibitedKeys(attempt, err); fatalErr != nil { // Reload so the next call uses fresh metadata, mirroring the // non-retryable branch below. @@ -590,14 +608,18 @@ func (c *committer) commitLocked(ctx context.Context, commitID string, retryOnUn // prohibited-table-property rejection and updates the stripper accordingly. // It returns retry=true when at least one new (non-reserved) key was learned — // the caller should count the attempt and re-stage, letting the stripper -// filter the keys on the next commit. It returns a non-nil fatalErr when the -// catalog named a key under reservedTablePropertyPrefix: those keys carry -// connector semantics (the commit-id idempotency token, the -// timestamp-encoding pin) that stripping would silently break, so the commit -// must fail loudly instead. Both zero values mean the error is not a -// prohibited-keys rejection — or it names only keys that are already being -// stripped, in which case retrying would loop futilely — and the caller's -// standard error handling applies. +// filter the keys on the next commit. Only keys the failed commit actually +// sent (in its set-properties updates, tracked by the stripper) can be +// learned: a named key we never sent cannot be the cause of THIS rejection, +// so learning it would let arbitrary error text poison the strip set — it is +// logged at debug and skipped instead. It returns a non-nil fatalErr when the +// catalog named a key under reservedTablePropertyPrefix (matched +// case-insensitively): those keys carry connector semantics (the commit-id +// idempotency token, the timestamp-encoding pin) that stripping would +// silently break, so the commit must fail loudly instead. Both zero values +// mean the error is not a prohibited-keys rejection — or it names only keys +// that are already being stripped or were never sent, in which case retrying +// would loop futilely — and the caller's standard error handling applies. func (c *committer) noteProhibitedKeys(attempt int, err error) (retry bool, fatalErr error) { keys := parseProhibitedPropertyKeys(err) if len(keys) == 0 { @@ -605,9 +627,12 @@ func (c *committer) noteProhibitedKeys(attempt int, err error) (retry bool, fata } var learned, reserved []string for _, k := range keys { - if strings.HasPrefix(k, reservedTablePropertyPrefix) { + switch { + case hasReservedPrefix(k): reserved = append(reserved, k) - } else if c.stripper.addProhibitedKey(k) { + case !c.stripper.sentPropertyKey(k): + c.logger.Debugf("Catalog rejection named prohibited table property %q, which this commit never set; not learning it", k) + case c.stripper.addProhibitedKey(k): learned = append(learned, k) } } diff --git a/internal/impl/iceberg/committer_test.go b/internal/impl/iceberg/committer_test.go index a4572e0982..1ab467dda9 100644 --- a/internal/impl/iceberg/committer_test.go +++ b/internal/impl/iceberg/committer_test.go @@ -151,6 +151,17 @@ func TestCommitterSkipsDuplicateCheck(t *testing.T) { require.NoError(t, c.Commit(ctx, CommitInput{Files: []iceberg.DataFile{df2}, SchemaID: c.currentSchemaID()})) } +// TestNewCommitterRejectsNilReloadTable pins the constructor guard: +// commitLocked dereferences reloadTable on every failure branch, so a nil one +// must be rejected at construction with a clear error instead of panicking +// mid-commit. +func TestNewCommitterRejectsNilReloadTable(t *testing.T) { + tbl, cat := newTestTable(t) + _, err := NewCommitter(tbl, cat, CommitConfig{MaxRetries: 1}, nil, service.MockResources().Logger()) + require.Error(t, err) + assert.Contains(t, err.Error(), "reloadTable must not be nil") +} + // commitOutcome scripts how scriptedCatalog handles a single CommitTable call. type commitOutcome int diff --git a/internal/impl/iceberg/config.go b/internal/impl/iceberg/config.go index 797c6c4eb6..afa11ec057 100644 --- a/internal/impl/iceberg/config.go +++ b/internal/impl/iceberg/config.go @@ -177,7 +177,7 @@ const rowOperationDocs = "\n" + "* For an existing table without the property, the output resolves the encoding automatically on first contact and stamps the result onto the table: if the schema has no no-timezone `timestamp` column, or the table has no data files, it resolves `spec`; otherwise the output inspects one data file's parquet footer and adopts whatever that file already contains (`legacy` for `isAdjustedToUTC=true`). A table that cannot be probed (unreadable file) fails the write rather than risk mixing annotations.\n" + "* Once stamped, the property is authoritative and the probe never runs again. An unrecognised property value is a hard error.\n" + "\n" + - "A table pinned `legacy` keeps receiving the legacy annotation on every new file — byte-identical to what previous releases wrote — so appends and `merge-on-read` continue working unchanged forever. The one restriction is mutating `copy-on-write` (`upsert`/`delete`): it must rewrite existing files, which the legacy annotation prevents, so such writes fail upfront with an actionable error (pure `insert` batches still work). To migrate a legacy table to the spec encoding: rewrite/compact the table's data files with an engine that writes the spec annotation (e.g. Spark's `rewrite_data_files`), then set the table property `redpanda-connect.timestamp-encoding` to `spec`. Alternatively, keep the table on `merge-on-read`.\n" + "A table pinned `legacy` keeps receiving the legacy annotation on every new file — byte-identical to what previous releases wrote — so appends and `merge-on-read` continue working unchanged forever. The one restriction is mutating `copy-on-write` (`upsert`/`delete`): it must rewrite existing files, which the legacy annotation prevents, so such writes fail upfront with an actionable error (pure `insert` batches still work). To migrate a legacy table to the spec encoding: rewrite/compact the table's data files with an engine that writes the spec annotation (e.g. Spark's `rewrite_data_files`), then set the table property `redpanda-connect.timestamp-encoding` to `spec`, keeping any running instances of this output that write to the table stopped (or restarting them) around the migration — a live writer only re-reads the property when its writer is recreated. Alternatively, keep the table on `merge-on-read`.\n" // icebergOutputConfig returns the configuration spec for the Iceberg output. func icebergOutputConfig() *service.ConfigSpec { diff --git a/internal/impl/iceberg/cow.go b/internal/impl/iceberg/cow.go index 6c5f226a8a..72a99df87f 100644 --- a/internal/impl/iceberg/cow.go +++ b/internal/impl/iceberg/cow.go @@ -163,7 +163,9 @@ func (w *writer) checkCOWTimestampEncoding() error { } return fmt.Errorf( "table %s uses the legacy UTC-adjusted parquet encoding for its `timestamp` columns (table property %s=legacy), which copy-on-write cannot rewrite; "+ - "compact/rewrite the table's data files with an engine that writes the spec encoding and set the table property %s=spec, or use merge_strategy: merge-on-read", + "compact/rewrite the table's data files with an engine that writes the spec encoding and set the table property %s=spec "+ + "(stop or restart connector writers to the table around the migration — a running writer only re-reads the property when its writer is recreated), "+ + "or use merge_strategy: merge-on-read", strings.Join(w.table.Identifier(), "."), icebergx.TimestampEncodingProperty, icebergx.TimestampEncodingProperty, ) } diff --git a/internal/impl/iceberg/prohibited_properties.go b/internal/impl/iceberg/prohibited_properties.go index abf963d8c5..f6ae7ec122 100644 --- a/internal/impl/iceberg/prohibited_properties.go +++ b/internal/impl/iceberg/prohibited_properties.go @@ -36,16 +36,22 @@ const reservedTablePropertyPrefix = "redpanda-connect." // BadRequestException: Malformed request: INVALID_PARAMETER_VALUE: // Table properties contain prohibited keys: schema.name-mapping.default // -// The match is case-insensitive on the "prohibited keys" marker, tolerates any -// prefix text, an optional colon, and captures the remainder of the message -// for tokenising in parseProhibitedPropertyKeys. -var prohibitedKeysRe = regexp.MustCompile(`(?i)prohibited\s+keys?\s*:?\s*(.+)`) +// The match is case-insensitive on the "prohibited keys" marker and tolerates +// any prefix text, but REQUIRES a colon introducing the key list — a bare +// "prohibited keys" phrase, or prose like "prohibited keys detected in the +// request", names no keys and must not match (the \b stops the ? from +// backtracking "keys" into "key"+"s"). The captured remainder is tokenised in +// parseProhibitedPropertyKeys. +var prohibitedKeysRe = regexp.MustCompile(`(?i)prohibited\s+keys?\b\s*:\s*(.+)`) // parseProhibitedPropertyKeys extracts the property keys named by a catalog's // prohibited-table-property rejection. It returns nil when err does not look -// like such a rejection. The parse is deliberately tolerant: surrounding text, -// case differences on the marker, quotes/brackets around the list, trailing -// prose after a key, and sentence-terminating punctuation are all accepted. +// like such a rejection. The parse is tolerant of surrounding text, case +// differences on the marker, quotes/brackets around the list, trailing prose +// after a key, and sentence-terminating punctuation — but each token must +// still look like a property key (see looksLikePropertyKey), and tokenising +// stops at the first that doesn't, so sentence prose after the list ("Remove +// them, then retry.") is never learned as keys. func parseProhibitedPropertyKeys(err error) []string { if err == nil { return nil @@ -66,9 +72,13 @@ func parseProhibitedPropertyKeys(err error) []string { // Keys never start or end with a dot; a trailing one is sentence // punctuation ("... keys: a.b."). tok = strings.Trim(tok, ".") - if tok != "" { - keys = append(keys, tok) + if !looksLikePropertyKey(tok) { + // The comma-separated list has run into sentence prose ("a.b. + // Remove them, then retry." yields "then"); stop rather than + // learn prose words as keys. + break } + keys = append(keys, tok) } return keys } @@ -78,6 +88,71 @@ func isPropertyKeyRune(r rune) bool { (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') } +// looksLikePropertyKey reports whether tok plausibly names a table property: +// it must start with a letter and contain at least one dot. Every key this +// stripper exists for (schema.name-mapping.default, write.delete.mode, ...) +// is dotted; requiring the dot is what keeps prose words out of the strip set. +func looksLikePropertyKey(tok string) bool { + if tok == "" { + return false + } + if c := tok[0]; (c < 'a' || c > 'z') && (c < 'A' || c > 'Z') { + return false + } + return strings.Contains(tok, ".") +} + +// hasReservedPrefix reports whether key falls under +// reservedTablePropertyPrefix, comparing case-insensitively so a catalog that +// case-folds key names in its rejection (e.g. "Redpanda-Connect.…") still +// triggers the loud refuses-to-strip diagnostic instead of a futile retry. +func hasReservedPrefix(key string) bool { + return len(key) >= len(reservedTablePropertyPrefix) && + strings.EqualFold(key[:len(reservedTablePropertyPrefix)], reservedTablePropertyPrefix) +} + +// prohibitedKeySet is a concurrency-safe set of catalog-prohibited property +// keys, guarded by its own mutex so it can be SHARED: the router owns one per +// tableEntry and seeds every committer created for that table with it (see +// CommitConfig.ProhibitedKeys and Router.createWriter). Sharing matters +// because writeWithRetry closes the writer on every failure — without it each +// writer generation would start with an empty strip set and burn one rejected +// commit re-learning the same keys (and a catalog naming one key per +// rejection could livelock recreation forever). +type prohibitedKeySet struct { + mu sync.RWMutex + keys map[string]struct{} +} + +func newProhibitedKeySet() *prohibitedKeySet { + return &prohibitedKeySet{keys: map[string]struct{}{}} +} + +// add records key, reporting whether it was newly added. +func (s *prohibitedKeySet) add(key string) bool { + s.mu.Lock() + defer s.mu.Unlock() + if _, ok := s.keys[key]; ok { + return false + } + s.keys[key] = struct{}{} + return true +} + +// snapshot returns the current keys; nil when the set is empty. +func (s *prohibitedKeySet) snapshot() []string { + s.mu.RLock() + defer s.mu.RUnlock() + if len(s.keys) == 0 { + return nil + } + out := make([]string, 0, len(s.keys)) + for k := range s.keys { + out = append(out, k) + } + return out +} + // propertyStrippingCatalog is a table.CatalogIO wrapper that filters // catalog-prohibited property keys out of set-properties updates at the commit // boundary. Some engine-backed catalogs (Databricks Unity Catalog at least) @@ -105,29 +180,78 @@ func isPropertyKeyRune(r rune) bool { type propertyStrippingCatalog struct { inner table.CatalogIO - mu sync.RWMutex - strip map[string]struct{} + // strip carries its own lock and may be shared across committers for the + // same table (see prohibitedKeySet), so learned keys survive writer + // recreation. + strip *prohibitedKeySet + + // sentMu guards lastSent: the property keys the most recent CommitTable + // actually forwarded to the inner catalog. noteProhibitedKeys consults it + // so only keys we genuinely sent can be learned from a rejection — + // commits through a committer are serialized (commitMu), so the last call + // is always the one whose error is being inspected. + sentMu sync.Mutex + lastSent map[string]struct{} } -func newPropertyStrippingCatalog(inner table.CatalogIO) *propertyStrippingCatalog { - return &propertyStrippingCatalog{inner: inner, strip: map[string]struct{}{}} +// newPropertyStrippingCatalog wraps inner with prohibited-key filtering. A +// nil strip gets a fresh, private key set; passing a shared set persists +// learned keys across committer generations for the same table. +func newPropertyStrippingCatalog(inner table.CatalogIO, strip *prohibitedKeySet) *propertyStrippingCatalog { + if strip == nil { + strip = newProhibitedKeySet() + } + return &propertyStrippingCatalog{inner: inner, strip: strip} } // addProhibitedKey records key for stripping from future commits, reporting -// whether it was newly added. Keys under reservedTablePropertyPrefix are -// refused (returning false): those carry connector semantics that must not be -// silently dropped — the caller is expected to fail loudly instead. +// whether it was newly added. Keys under reservedTablePropertyPrefix +// (compared case-insensitively) are refused (returning false): those carry +// connector semantics that must not be silently dropped — the caller is +// expected to fail loudly instead. func (p *propertyStrippingCatalog) addProhibitedKey(key string) bool { - if strings.HasPrefix(key, reservedTablePropertyPrefix) { + if hasReservedPrefix(key) { return false } - p.mu.Lock() - defer p.mu.Unlock() - if _, ok := p.strip[key]; ok { - return false + return p.strip.add(key) +} + +// sentPropertyKey reports whether the most recent CommitTable through this +// wrapper forwarded a set-properties update containing key. +func (p *propertyStrippingCatalog) sentPropertyKey(key string) bool { + p.sentMu.Lock() + defer p.sentMu.Unlock() + _, ok := p.lastSent[key] + return ok +} + +// recordSentPropertyKeys stores the union of property keys in updates' +// set-properties updates as the most recent commit's sent set. Best-effort: +// an update that cannot be introspected is skipped (the same JSON round-trip +// in filterUpdates would have failed the commit first anyway). +func (p *propertyStrippingCatalog) recordSentPropertyKeys(updates []table.Update) { + sent := map[string]struct{}{} + for _, u := range updates { + if u.Action() != table.UpdateSetProperties { + continue + } + raw, err := json.Marshal(u) + if err != nil { + continue + } + var payload struct { + Updates iceberg.Properties `json:"updates"` + } + if err := json.Unmarshal(raw, &payload); err != nil { + continue + } + for k := range payload.Updates { + sent[k] = struct{}{} + } } - p.strip[key] = struct{}{} - return true + p.sentMu.Lock() + p.lastSent = sent + p.sentMu.Unlock() } // LoadTable delegates to the wrapped catalog. The returned table keeps its @@ -146,6 +270,9 @@ func (p *propertyStrippingCatalog) CommitTable(ctx context.Context, ident table. if err != nil { return nil, "", fmt.Errorf("stripping prohibited table properties from commit updates: %w", err) } + // Remember which property keys this commit actually carries, so a + // rejection naming a key we never sent is not learned (noteProhibitedKeys). + p.recordSentPropertyKeys(filtered) return p.inner.CommitTable(ctx, ident, reqs, filtered) } @@ -158,14 +285,8 @@ func (p *propertyStrippingCatalog) CommitTable(ctx context.Context, ident table. // table.NewSetPropertiesUpdate constructor (same action, same no-op // PostCommit). func (p *propertyStrippingCatalog) filterUpdates(updates []table.Update) ([]table.Update, error) { - p.mu.RLock() - n := len(p.strip) - strip := make([]string, 0, n) - for k := range p.strip { - strip = append(strip, k) - } - p.mu.RUnlock() - if n == 0 { + strip := p.strip.snapshot() + if len(strip) == 0 { return updates, nil } diff --git a/internal/impl/iceberg/prohibited_properties_test.go b/internal/impl/iceberg/prohibited_properties_test.go index a25a375ab7..39355cf6d3 100644 --- a/internal/impl/iceberg/prohibited_properties_test.go +++ b/internal/impl/iceberg/prohibited_properties_test.go @@ -20,6 +20,7 @@ import ( "testing" "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/catalog/rest" "github.com/apache/iceberg-go/table" "github.com/google/uuid" "github.com/stretchr/testify/assert" @@ -244,6 +245,150 @@ func TestProhibitedReservedKeyFailsLoudly(t *testing.T) { assert.Equal(t, 1, cat.commits, "a reserved-key rejection must fail on the first attempt, not retry") } +// TestProhibitedReservedKeyCaseInsensitive pins that the reserved-prefix guard +// is case-insensitive: a catalog that case-folds key names in its rejection +// ("Redpanda-Connect.…") must still trigger the loud refuses-to-strip +// diagnostic — with the property surviving outside the strip set — rather than +// burning a futile stripped retry and surfacing the raw catalog error. +func TestProhibitedReservedKeyCaseInsensitive(t *testing.T) { + ctx := t.Context() + _, mem := newTestTable(t) + cat := &alwaysProhibitingCatalog{memCatalog: mem, keys: []string{"Redpanda-Connect.timestamp-encoding"}} + + c, err := NewCommitter(cat.snapshot(), cat, CommitConfig{MaxRetries: 3}, + func(context.Context) (*table.Table, error) { return cat.snapshot(), nil }, service.MockResources().Logger()) + require.NoError(t, err) + defer c.Close() + + df := synthDataFile(t, cat.snapshot().Spec(), fmt.Sprintf("%s/data/reserved-fold-%s.parquet", cat.location, uuid.New())) + err = c.Commit(ctx, CommitInput{Files: []iceberg.DataFile{df}, SchemaID: c.currentSchemaID()}) + require.Error(t, err) + assert.Contains(t, err.Error(), "Redpanda-Connect.timestamp-encoding") + assert.Contains(t, err.Error(), "refuses to strip", "a case-variant reserved key must hit the loud diagnostic, not the raw error") + assert.Equal(t, 1, cat.commits, "a reserved-key rejection must fail on the first attempt, not retry") + assert.Empty(t, c.stripper.strip.snapshot(), "the reserved key must never enter the strip set") +} + +// TestProhibitedKeyNeverSentIsNotLearned pins the strongest parser guard: a +// rejection naming a property key the failed commit never actually sent (the +// append path stages no set-properties updates at all) must not be learned — +// otherwise arbitrary error text could poison the strip set. +func TestProhibitedKeyNeverSentIsNotLearned(t *testing.T) { + ctx := t.Context() + _, mem := newTestTable(t) + cat := &alwaysProhibitingCatalog{memCatalog: mem, keys: []string{"some.innocent.key"}} + + c, err := NewCommitter(cat.snapshot(), cat, CommitConfig{MaxRetries: 3}, + func(context.Context) (*table.Table, error) { return cat.snapshot(), nil }, service.MockResources().Logger()) + require.NoError(t, err) + defer c.Close() + + df := synthDataFile(t, cat.snapshot().Spec(), fmt.Sprintf("%s/data/unsent-%s.parquet", cat.location, uuid.New())) + err = c.Commit(ctx, CommitInput{Files: []iceberg.DataFile{df}, SchemaID: c.currentSchemaID()}) + require.Error(t, err, "with nothing to strip, the rejection is not retryable and must surface") + assert.Empty(t, c.stripper.strip.snapshot(), "a key the commit never sent must not be learned") + assert.Equal(t, 1, cat.commits, "no stripped retry may be attempted for a key we never sent") +} + +// TestProhibitedKeysPersistAcrossCommitters pins the strip-set persistence the +// router provides via tableEntry.prohibitedProps: writeWithRetry closes the +// writer (and its committer) on every failure, so a NEW committer seeded with +// the SAME shared prohibitedKeySet — exactly what Router.createWriter does — +// must strip already-learned keys on its FIRST attempt instead of burning one +// rejected commit per writer generation. +func TestProhibitedKeysPersistAcrossCommitters(t *testing.T) { + ctx := t.Context() + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}, + iceberg.NestedField{ID: 2, Name: "payload", Type: iceberg.PrimitiveTypes.String, Required: false}, + ) + seedTbl, mem := newCOWTable(t, sc) + _ = appendCOWRows(t, ctx, seedTbl, map[int64]string{1: "one", 2: "two", 3: "three"}) + cat := &prohibitingCatalog{memCatalog: mem, prohibited: []string{"schema.name-mapping.default"}} + + // One shared set for the table, as Router.createWriter seeds from + // tableEntry.prohibitedProps. + shared := newProhibitedKeySet() + newGenWriter := func() *writer { + comm, err := NewCommitter(cat.snapshot(), cat, CommitConfig{MaxRetries: 3, ProhibitedKeys: shared}, + func(context.Context) (*table.Table, error) { return cat.snapshot(), nil }, service.MockResources().Logger()) + require.NoError(t, err) + t.Cleanup(comm.Close) + w := cowWriter(t, cat.snapshot(), "id") + w.committer = comm + return w + } + + // First committer generation pays one rejection to learn the key. + w1 := newGenWriter() + require.NoError(t, w1.Write(ctx, service.MessageBatch{cowMsg(t, "upsert", map[string]any{"id": 2, "payload": "TWO"})})) + require.Equal(t, 1, cat.rejections) + require.Equal(t, 2, cat.commits, "generation one: one rejection, one stripped success") + + // Second generation (fresh committer, same shared set — the writer was + // recreated): the mutation must commit FIRST-TRY, no rejection burned. + w2 := newGenWriter() + require.NoError(t, w2.Write(ctx, service.MessageBatch{cowMsg(t, "upsert", map[string]any{"id": 3, "payload": "THREE"})})) + assert.Equal(t, 1, cat.rejections, "a committer seeded with the shared set must not re-learn the key via a rejection") + assert.Equal(t, 3, cat.commits, "generation two must land its commit on the first attempt") + assert.Equal(t, map[int64]string{1: "one", 2: "TWO", 3: "THREE"}, scanRows(t, ctx, cat.snapshot())) +} + +// unknownStateProhibitedTextCatalog applies the FIRST commit server-side but +// reports an ErrCommitStateUnknown whose text also names a prohibited key that +// the commit really sent. It models a 5xx from a catalog whose error body +// happens to mention prohibited keys: the ambiguity handling (reload + +// commit-id idempotency) must take precedence over prohibited-key learning. +type unknownStateProhibitedTextCatalog struct { + *memCatalog + calls int +} + +func (c *unknownStateProhibitedTextCatalog) CommitTable(ctx context.Context, ident table.Identifier, reqs []table.Requirement, updates []table.Update) (table.Metadata, string, error) { + c.calls++ + if c.calls == 1 { + if _, _, err := c.memCatalog.CommitTable(ctx, ident, reqs, updates); err != nil { + return nil, "", err + } + return nil, "", fmt.Errorf("500 Internal Server Error: Table properties contain prohibited keys: schema.name-mapping.default: %w", rest.ErrCommitStateUnknown) + } + return c.memCatalog.CommitTable(ctx, ident, reqs, updates) +} + +func (c *unknownStateProhibitedTextCatalog) snapshot() *table.Table { + return rebindTable(c.memCatalog.snapshot(), c) +} + +// TestUnknownStateTakesPrecedenceOverProhibitedKeys pins the precedence fix: a +// commit that LANDED but returned ErrCommitStateUnknown must go through the +// reload + commit-id idempotency check — treating it as a prohibited-keys +// rejection would re-stage and apply the mutation a second time. Exactly one +// snapshot may carry the commit id, and no key may be learned. +func TestUnknownStateTakesPrecedenceOverProhibitedKeys(t *testing.T) { + ctx := t.Context() + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}, + iceberg.NestedField{ID: 2, Name: "payload", Type: iceberg.PrimitiveTypes.String, Required: false}, + ) + seedTbl, mem := newCOWTable(t, sc) + _ = appendCOWRows(t, ctx, seedTbl, map[int64]string{1: "one", 2: "two"}) + cat := &unknownStateProhibitedTextCatalog{memCatalog: mem} + + comm, err := NewCommitter(cat.snapshot(), cat, CommitConfig{MaxRetries: 3}, + func(context.Context) (*table.Table, error) { return cat.snapshot(), nil }, service.MockResources().Logger()) + require.NoError(t, err) + t.Cleanup(comm.Close) + w := cowWriter(t, cat.snapshot(), "id") + w.committer = comm + + require.NoError(t, w.Write(ctx, service.MessageBatch{cowMsg(t, "upsert", map[string]any{"id": 2, "payload": "TWO"})})) + + assert.Equal(t, 1, cat.calls, "the landed commit must be detected via the commit-id, not re-sent") + assert.Equal(t, 1, countSnapshotsWithCommitID(cat.snapshot()), "the mutation must be applied exactly once") + assert.Equal(t, map[int64]string{1: "one", 2: "TWO"}, scanRows(t, ctx, cat.snapshot())) + assert.Empty(t, comm.stripper.strip.snapshot(), "an unknown-state error must never teach the stripper") +} + // TestParseProhibitedPropertyKeys pins the rejection-parsing grammar: a // case-insensitive "prohibited key(s)" marker with arbitrary prefix text, an // optional colon, and a comma-separated key list that tolerates quotes, @@ -289,6 +434,42 @@ func TestParseProhibitedPropertyKeys(t *testing.T) { err: errors.New("prohibited keys: schema.name-mapping.default."), want: []string{"schema.name-mapping.default"}, }, + { + // Regression: the old lazy-colon regex backtracked "keys" into + // "key" + list "s" and learned the key "s". + name: "bare phrase without a list learns nothing", + err: errors.New("prohibited keys"), + want: nil, + }, + { + // Regression: without the required colon this learned "detected". + name: "prose after the marker without a colon learns nothing", + err: errors.New("prohibited keys detected in the request"), + want: nil, + }, + { + // Regression: the prose after the sentence break used to yield a + // bogus second key "then". + name: "sentence prose after the key list is not learned", + err: errors.New("prohibited keys: a.b. Remove them, then retry."), + want: []string{"a.b"}, + }, + { + name: "prose tail after multiple keys keeps only the real keys", + err: errors.New("prohibited keys: a.b, c.d. Please remove them, then retry."), + want: []string{"a.b", "c.d"}, + }, + { + name: "quoted comma-separated keys", + err: errors.New("prohibited keys: 'x.y', 'z.w'"), + want: []string{"x.y", "z.w"}, + }, + { + // Real property keys are dotted; a bare word is prose, not a key. + name: "dotless token is not a property key", + err: errors.New("prohibited keys: forbidden"), + want: nil, + }, { name: "unrelated error", err: errors.New("commit failed, refresh and try again"), @@ -313,7 +494,7 @@ func TestParseProhibitedPropertyKeys(t *testing.T) { // their original value. It also pins the strip-set rules: dedupe on re-add and // refusal of reserved redpanda-connect.* keys. func TestStripperFiltersOnlySetPropertiesUpdates(t *testing.T) { - s := newPropertyStrippingCatalog(nil) + s := newPropertyStrippingCatalog(nil, nil) // Pass-through while the strip set is empty: same slice, no copies. unfilteredProps := table.NewSetPropertiesUpdate(iceberg.Properties{"schema.name-mapping.default": "m"}) diff --git a/internal/impl/iceberg/router.go b/internal/impl/iceberg/router.go index 982ec74c40..25b0d689fe 100644 --- a/internal/impl/iceberg/router.go +++ b/internal/impl/iceberg/router.go @@ -74,6 +74,14 @@ type tableEntry struct { // error recovery) reuses it. Guarded by mu; valid when tsEncodingResolved. tsEncoding icebergx.TimestampEncoding tsEncodingResolved bool + + // prohibitedProps persists the catalog-prohibited property keys learned + // by this table's committers (mirroring the tsEncoding cache pattern): + // writeWithRetry closes the writer on every failure, so without it each + // recreated committer would re-learn the keys at the cost of one rejected + // commit per generation. Created lazily in createWriter under mu; the set + // carries its own lock, so committers share it race-safely. + prohibitedProps *prohibitedKeySet } // Router routes message batches to per-table writers. @@ -774,6 +782,14 @@ func (r *Router) createWriter(ctx context.Context, key tableKey, entry *tableEnt // irreversible v1->v2 upgrade the merge-on-read path needs. commitCfg := r.commitCfg commitCfg.SkipFormatUpgrade = r.rowOpCfg.MergeStrategy == mergeStrategyCOW + // Seed the committer with the entry's persistent prohibited-key set (we + // hold entry.mu), so property keys a catalog rejection taught a previous + // committer stay stripped after writer recreation instead of costing one + // rejected commit per writer generation. + if entry.prohibitedProps == nil { + entry.prohibitedProps = newProhibitedKeySet() + } + commitCfg.ProhibitedKeys = entry.prohibitedProps comm, err := NewCommitter(committerTbl, client.TableIO(), commitCfg, reloadTable, r.logger) if err != nil { return nil, fmt.Errorf("creating committer: %w", err) diff --git a/internal/impl/iceberg/timestamp_encoding.go b/internal/impl/iceberg/timestamp_encoding.go index e305b47829..e61f7ed8e6 100644 --- a/internal/impl/iceberg/timestamp_encoding.go +++ b/internal/impl/iceberg/timestamp_encoding.go @@ -167,12 +167,26 @@ func probeParquetFooterEncoding(fsys iceio.IO, path string, schema *iceberg.Sche // stampTimestampEncoding commits the resolved encoding onto the table as the // redpanda-connect.timestamp-encoding property, making the bootstrap decision -// permanent and visible. Two writers may race to stamp the same table: on a -// commit failure the table is reloaded and, if the property appeared -// meanwhile with our value, the race is benign and the reloaded table is -// used. A property that appeared with a DIFFERENT value is a hard error — -// both writers probed the same files, so a disagreement means something is -// wrong and writing could mix annotations. +// permanent and visible. Two writers may race to stamp the same table; the +// invariant that keeps that safe is that two new-version writers probing the +// same snapshot always resolve the SAME encoding, so whichever stamp lands, +// every writer agrees with it. The failure-path check below (reload and +// compare a concurrently-appeared value) is a best-effort guard for +// CAS-style catalogs whose commits actually fail on concurrent writes: on +// REST catalogs a SetProperties-only commit carries only an AssertTableUUID +// requirement, so it rarely fails and the last writer simply wins — benign, +// because agreeing writers write identical values. A DIFFERENT value can +// therefore only be detected when the commit does fail; it means something +// is genuinely wrong (e.g. mixed connector versions probing differently) and +// writing could mix annotations, so it is a hard error. +// +// This commit goes through the table's own catalog binding, NOT through a +// committer's propertyStrippingCatalog: the transaction stages only the +// single redpanda-connect.* property, which the stripper refuses to strip +// anyway (reservedTablePropertyPrefix), so prohibited-key stripping +// intentionally does not apply here — a catalog that prohibits +// redpanda-connect.* keys fails this path with the raw catalog error by +// design. func stampTimestampEncoding(ctx context.Context, tbl *table.Table, enc icebergx.TimestampEncoding, reload func(context.Context) (*table.Table, error)) (*table.Table, error) { txn := tbl.NewTransaction() if err := txn.SetProperties(iceberg.Properties{icebergx.TimestampEncodingProperty: enc.String()}); err != nil { diff --git a/internal/impl/iceberg/timestamp_encoding_test.go b/internal/impl/iceberg/timestamp_encoding_test.go index 25b47dbf33..0c8f98053a 100644 --- a/internal/impl/iceberg/timestamp_encoding_test.go +++ b/internal/impl/iceberg/timestamp_encoding_test.go @@ -289,6 +289,45 @@ func TestResolveTimestampEncodingBootstrap(t *testing.T) { }) } +// TestProbeSkipsTimestamptzLeaf pins the probe's timestamptz-leaf skip. The +// schema orders a `timestamptz` column FIRST, so the footer's first +// timestamp-annotated leaf is the tstz one — which carries isAdjustedToUTC=true +// in BOTH encodings and so says nothing about the table's encoding. The probe +// must skip it and decide from the no-tz `timestamp` leaf: a spec-encoded file +// must pin `spec`, not a legacy mispin read off the tstz leaf. +func TestProbeSkipsTimestamptzLeaf(t *testing.T) { + ctx := t.Context() + sc := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "a_tstz", Type: iceberg.PrimitiveTypes.TimestampTz, Required: false}, + iceberg.NestedField{ID: 2, Name: "b_ts", Type: iceberg.PrimitiveTypes.Timestamp, Required: false}, + ) + _, cat := newEncTable(t, sc, nil) + tbl := cat.snapshot() + require.NoError(t, os.MkdirAll(filepath.Join(tbl.Location(), "data"), 0o755)) + w := &writer{table: tbl, caseSensitive: true, tsEncoding: icebergx.TimestampEncodingSpec, logger: service.MockResources().Logger()} + files, err := w.writeDataFiles(ctx, service.MessageBatch{structuredMsg(t, map[string]any{ + "a_tstz": encSeedTime, "b_ts": encSeedTime, + })}) + require.NoError(t, err) + require.Len(t, files, 1) + tx := tbl.NewTransaction() + require.NoError(t, tx.AddDataFiles(ctx, files, nil, table.WithoutAutoNameMapping(), table.WithoutDuplicateCheck())) + _, err = tx.Commit(ctx) + require.NoError(t, err) + + // Precondition: the file's leaves follow the schema order (tstz leaf + // first) and the tstz leaf is UTC-adjusted — a probe that read the first + // timestamp-annotated leaf naively would mispin legacy. + ann := footerTimestampAdjusted(t, files[0].FilePath()) + require.Equal(t, map[int]bool{1: true, 2: false}, ann, + "precondition: tstz leaf UTC-adjusted, no-tz spec leaf not") + + enc, err := probeTimestampEncoding(ctx, cat.snapshot()) + require.NoError(t, err) + assert.Equal(t, icebergx.TimestampEncodingSpec, enc, + "the probe must skip the timestamptz leaf and pin spec from the no-tz timestamp leaf") +} + // --- stamping race ------------------------------------------------------------ // TestStampTimestampEncodingRace covers two writers bootstrapping the same