fix(s3): add delimiter option support for CSV files (supabase#561)

* fix(s3): add delimiter option support for CSV files

* feat(s3): add csv delimiter table option

---------

Co-authored-by: Bo Lu <[email protected]>

Author: 7ttp

docs/catalog/s3.md

@@ -159,6 +159,7 @@ The following options are available when creating S3 foreign tables:
 - `format` - File format, required. `csv`, `jsonl`, or `parquet`
 - `has_header` - If the CSV file has header, optional. `true` or `false`, default is `false`
 - `compress` - Compression algorithm, optional. One of `gzip`, `bzip2`, `xz`, `zlib`, default is no compression
+- `delimiter` - Field delimiter for CSV files, optional. Single character like `,`, `;`, `|`, or escaped sequence like `E'\t'` for tab, default is `,`
 
 ## Entities
 
@@ -192,6 +193,25 @@ create foreign table s3.table_csv (
   );
 ```
 
+Using custom delimiter:
+
+```sql
+create foreign table s3.table_tsv (
+  name text,
+  sex text,
+  age text,
+  height text,
+  weight text
+)
+  server s3_server
+  options (
+    uri 's3://bucket/s3_table.tsv',
+    format 'csv',
+    delimiter E'\t',  -- Tab-separated values
+    has_header 'false'
+  );
+```
+
 #### Notes
 
 - All columns must be defined in the foreign table

wrappers/dockerfiles/s3/test_data/test_data.tsv

@@ -0,0 +1,4 @@
+"name"	"sex"	"age"	"height"	"weight"
+"Bert"	"M"	42	68	166
+"Alex"	"M"	41	74	170
+"Carl"	"M"	32		155

wrappers/src/fdw/s3_fdw/README.md

@@ -10,6 +10,7 @@ This is a foreign data wrapper for [AWS S3](https://aws.amazon.com/s3/). It is d
 
 | Version | Date       | Notes                                                |
 | ------- | ---------- | ---------------------------------------------------- |
+| 0.1.6   | 2026-01-21 | Added csv delimiter foreign table option             |
 | 0.1.5   | 2025-07-25 | Fixed parquet file reading position issue            |
 | 0.1.4   | 2024-08-20 | Added `path_style_url` server option                 |
 | 0.1.2   | 2023-07-13 | Added fdw stats collection                           |

wrappers/src/fdw/s3_fdw/mod.rs

@@ -20,6 +20,9 @@ enum S3FdwError {
     #[error("invalid format option: '{0}', it can only be 'csv', 'jsonl' or 'parquet'")]
     InvalidFormatOption(String),
 
+    #[error("invalid delimiter option: '{0}', it must be exactly one character")]
+    InvalidDelimiterOption(String),
+
     #[error("invalid compression option: {0}")]
     InvalidCompressOption(String),
 

wrappers/src/fdw/s3_fdw/s3_fdw.rs

@@ -25,7 +25,7 @@ enum Parser {
 }
 
 #[wrappers_fdw(
-    version = "0.1.5",
+    version = "0.1.6",
     author = "Supabase",
     website = "https://github.com/supabase/wrappers/tree/main/wrappers/src/fdw/s3_fdw",
     error_type = "S3FdwError"
@@ -37,6 +37,7 @@ pub(crate) struct S3Fdw {
     parser: Parser,
     tgt_cols: Vec<Column>,
     rows_out: i64,
+    csv_delimiter: u8,
 
     // local string buffer for CSV and JSONL
     buf: String,
@@ -86,6 +87,7 @@ impl S3Fdw {
                 buf.extend(self.buf.as_bytes());
                 *rdr = csv::ReaderBuilder::new()
                     .has_headers(false)
+                    .delimiter(self.csv_delimiter)
                     .from_reader(Cursor::new(buf));
             }
             Parser::JsonLine(records) => {
@@ -124,6 +126,7 @@ impl ForeignDataWrapper<S3FdwError> for S3Fdw {
             parser: Parser::JsonLine(VecDeque::new()),
             tgt_cols: Vec::new(),
             rows_out: 0,
+            csv_delimiter: b',',
             buf: String::new(),
         };
 
@@ -225,6 +228,14 @@ impl ForeignDataWrapper<S3FdwError> for S3Fdw {
 
         let has_header: bool = options.get("has_header") == Some(&"true".to_string());
 
+        if let Some(delimiter) = options.get("delimiter") {
+            if delimiter.len() == 1 {
+                self.csv_delimiter = delimiter.as_bytes()[0];
+            } else {
+                return Err(S3FdwError::InvalidDelimiterOption(delimiter.to_string()));
+            }
+        }
+
         self.tgt_cols = columns.to_vec();
 
         if let Some(client) = &self.client {

wrappers/src/fdw/s3_fdw/tests.rs

@@ -41,7 +41,8 @@ mod tests {
                 OPTIONS (
                     uri 's3://warehouse/test_data.csv',
                     format 'csv',
-                    has_header 'true'
+                    has_header 'true',
+                    delimiter ','
                   )
              "#,
                 None,
@@ -178,6 +179,28 @@ mod tests {
             )
             .unwrap();
 
+            c.update(
+                r#"
+                CREATE FOREIGN TABLE s3_test_table_tsv (
+                  name text,
+                  sex text,
+                  age text,
+                  height text,
+                  weight text
+                )
+                SERVER s3_server
+                OPTIONS (
+                    uri 's3://warehouse/test_data.tsv',
+                    format 'csv',
+                    has_header 'true',
+                    delimiter E'\t'
+                  )
+             "#,
+                None,
+                &[],
+            )
+            .unwrap();
+
             let check_test_table = |table| {
                 let sql = format!("SELECT * FROM {table} ORDER BY name LIMIT 1");
                 let results = c
@@ -197,6 +220,7 @@ mod tests {
             check_test_table("s3_test_table_csv_gz");
             check_test_table("s3_test_table_jsonl");
             check_test_table("s3_test_table_jsonl_bz");
+            check_test_table("s3_test_table_tsv");
 
             let check_parquet_table = |table| {
                 let sql = format!("SELECT * FROM {table} ORDER BY id LIMIT 1");