fix: preserve BOM in text tool round-trips (#23797)

Author: Hona

packages/opencode/src/format/index.ts

@@ -25,7 +25,7 @@ export type Status = z.infer<typeof Status>
 export interface Interface {
   readonly init: () => Effect.Effect<void>
   readonly status: () => Effect.Effect<Status[]>
-  readonly file: (filepath: string) => Effect.Effect<void>
+  readonly file: (filepath: string) => Effect.Effect<boolean>
 }
 
 export class Service extends Context.Service<Service, Interface>()("@opencode/Format") {}
@@ -70,16 +70,19 @@ export const layer = Layer.effect(
               }
             }),
           )
-          return checks.filter((x) => x.cmd).map((x) => ({ item: x.item, cmd: x.cmd! }))
+          return checks
+            .filter((x): x is { item: Formatter.Info; cmd: string[] } => x.cmd !== false)
+            .map((x) => ({ item: x.item, cmd: x.cmd }))
         }
 
         function formatFile(filepath: string) {
           return Effect.gen(function* () {
             log.info("formatting", { file: filepath })
-            const ext = path.extname(filepath)
+            const formatters = yield* Effect.promise(() => getFormatter(path.extname(filepath)))
 
-            for (const { item, cmd } of yield* Effect.promise(() => getFormatter(ext))) {
-              if (cmd === false) continue
+            if (!formatters.length) return false
+
+            for (const { item, cmd } of formatters) {
               log.info("running", { command: cmd })
               const replaced = cmd.map((x) => x.replace("$FILE", filepath))
               const dir = yield* InstanceState.directory
@@ -113,6 +116,8 @@ export const layer = Layer.effect(
                 })
               }
             }
+
+            return true
           })
         }
 
@@ -188,7 +193,7 @@ export const layer = Layer.effect(
 
     const file = Effect.fn("Format.file")(function* (filepath: string) {
       const { formatFile } = yield* InstanceState.get(state)
-      yield* formatFile(filepath)
+      return yield* formatFile(filepath)
     })
 
     return Service.of({ init, status, file })

packages/opencode/src/patch/index.ts

@@ -3,6 +3,7 @@ import * as path from "path"
 import * as fs from "fs/promises"
 import { readFileSync } from "fs"
 import { Log } from "../util"
+import * as Bom from "../util/bom"
 
 const log = Log.create({ service: "patch" })
 
@@ -305,18 +306,19 @@ export function maybeParseApplyPatch(
 interface ApplyPatchFileUpdate {
   unified_diff: string
   content: string
+  bom: boolean
 }
 
 export function deriveNewContentsFromChunks(filePath: string, chunks: UpdateFileChunk[]): ApplyPatchFileUpdate {
   // Read original file content
-  let originalContent: string
+  let originalContent: ReturnType<typeof Bom.split>
   try {
-    originalContent = readFileSync(filePath, "utf-8")
+    originalContent = Bom.split(readFileSync(filePath, "utf-8"))
   } catch (error) {
     throw new Error(`Failed to read file ${filePath}: ${error}`, { cause: error })
   }
 
-  let originalLines = originalContent.split("\n")
+  let originalLines = originalContent.text.split("\n")
 
   // Drop trailing empty element for consistent line counting
   if (originalLines.length > 0 && originalLines[originalLines.length - 1] === "") {
@@ -331,14 +333,16 @@ export function deriveNewContentsFromChunks(filePath: string, chunks: UpdateFile
     newLines.push("")
   }
 
-  const newContent = newLines.join("\n")
+  const next = Bom.split(newLines.join("\n"))
+  const newContent = next.text
 
   // Generate unified diff
-  const unifiedDiff = generateUnifiedDiff(originalContent, newContent)
+  const unifiedDiff = generateUnifiedDiff(originalContent.text, newContent)
 
   return {
     unified_diff: unifiedDiff,
     content: newContent,
+    bom: originalContent.bom || next.bom,
   }
 }
 
@@ -553,13 +557,13 @@ export async function applyHunksToFiles(hunks: Hunk[]): Promise<AffectedPaths> {
             await fs.mkdir(moveDir, { recursive: true })
           }
 
-          await fs.writeFile(hunk.move_path, fileUpdate.content, "utf-8")
+          await fs.writeFile(hunk.move_path, Bom.join(fileUpdate.content, fileUpdate.bom), "utf-8")
           await fs.unlink(hunk.path)
           modified.push(hunk.move_path)
           log.info(`Moved file: ${hunk.path} -> ${hunk.move_path}`)
         } else {
           // Regular update
-          await fs.writeFile(hunk.path, fileUpdate.content, "utf-8")
+          await fs.writeFile(hunk.path, Bom.join(fileUpdate.content, fileUpdate.bom), "utf-8")
           modified.push(hunk.path)
           log.info(`Updated file: ${hunk.path}`)
         }

packages/opencode/src/tool/apply_patch.ts

@@ -14,6 +14,7 @@ import { AppFileSystem } from "@opencode-ai/shared/filesystem"
 import DESCRIPTION from "./apply_patch.txt"
 import { File } from "../file"
 import { Format } from "../format"
+import * as Bom from "@/util/bom"
 
 const PatchParams = z.object({
   patchText: z.string().describe("The full patch text that describes all changes to be made"),
@@ -59,6 +60,7 @@ export const ApplyPatchTool = Tool.define(
         diff: string
         additions: number
         deletions: number
+        bom: boolean
       }> = []
 
       let totalDiff = ""
@@ -72,23 +74,25 @@ export const ApplyPatchTool = Tool.define(
             const oldContent = ""
             const newContent =
               hunk.contents.length === 0 || hunk.contents.endsWith("\n") ? hunk.contents : `${hunk.contents}\n`
-            const diff = trimDiff(createTwoFilesPatch(filePath, filePath, oldContent, newContent))
+            const next = Bom.split(newContent)
+            const diff = trimDiff(createTwoFilesPatch(filePath, filePath, oldContent, next.text))
 
             let additions = 0
             let deletions = 0
-            for (const change of diffLines(oldContent, newContent)) {
+            for (const change of diffLines(oldContent, next.text)) {
               if (change.added) additions += change.count || 0
               if (change.removed) deletions += change.count || 0
             }
 
             fileChanges.push({
               filePath,
               oldContent,
-              newContent,
+              newContent: next.text,
               type: "add",
               diff,
               additions,
               deletions,
+              bom: next.bom,
             })
 
             totalDiff += diff + "\n"
@@ -104,13 +108,16 @@ export const ApplyPatchTool = Tool.define(
               )
             }
 
-            const oldContent = yield* afs.readFileString(filePath)
+            const source = yield* Bom.readFile(afs, filePath)
+            const oldContent = source.text
             let newContent = oldContent
+            let bom = source.bom
 
             // Apply the update chunks to get new content
             try {
               const fileUpdate = Patch.deriveNewContentsFromChunks(filePath, hunk.chunks)
               newContent = fileUpdate.content
+              bom = fileUpdate.bom
             } catch (error) {
               return yield* Effect.fail(new Error(`apply_patch verification failed: ${error}`))
             }
@@ -136,15 +143,16 @@ export const ApplyPatchTool = Tool.define(
               diff,
               additions,
               deletions,
+              bom,
             })
 
             totalDiff += diff + "\n"
             break
           }
 
           case "delete": {
-            const contentToDelete = yield* afs
-              .readFileString(filePath)
+            const source = yield* Bom
+              .readFile(afs, filePath)
               .pipe(
                 Effect.catch((error) =>
                   Effect.fail(
@@ -154,6 +162,7 @@ export const ApplyPatchTool = Tool.define(
                   ),
                 ),
               )
+            const contentToDelete = source.text
             const deleteDiff = trimDiff(createTwoFilesPatch(filePath, filePath, contentToDelete, ""))
 
             const deletions = contentToDelete.split("\n").length
@@ -166,6 +175,7 @@ export const ApplyPatchTool = Tool.define(
               diff: deleteDiff,
               additions: 0,
               deletions,
+              bom: source.bom,
             })
 
             totalDiff += deleteDiff + "\n"
@@ -207,20 +217,20 @@ export const ApplyPatchTool = Tool.define(
           case "add":
             // Create parent directories (recursive: true is safe on existing/root dirs)
 
-            yield* afs.writeWithDirs(change.filePath, change.newContent)
+            yield* afs.writeWithDirs(change.filePath, Bom.join(change.newContent, change.bom))
             updates.push({ file: change.filePath, event: "add" })
             break
 
           case "update":
-            yield* afs.writeWithDirs(change.filePath, change.newContent)
+            yield* afs.writeWithDirs(change.filePath, Bom.join(change.newContent, change.bom))
             updates.push({ file: change.filePath, event: "change" })
             break
 
           case "move":
             if (change.movePath) {
               // Create parent directories (recursive: true is safe on existing/root dirs)
 
-              yield* afs.writeWithDirs(change.movePath!, change.newContent)
+              yield* afs.writeWithDirs(change.movePath!, Bom.join(change.newContent, change.bom))
               yield* afs.remove(change.filePath)
               updates.push({ file: change.filePath, event: "unlink" })
               updates.push({ file: change.movePath, event: "add" })
@@ -234,7 +244,9 @@ export const ApplyPatchTool = Tool.define(
         }
 
         if (edited) {
-          yield* format.file(edited)
+          if (yield* format.file(edited)) {
+            yield* Bom.syncFile(afs, edited, change.bom)
+          }
           yield* bus.publish(File.Event.Edited, { file: edited })
         }
       }

packages/opencode/src/tool/edit.ts

@@ -18,6 +18,7 @@ import { Instance } from "../project/instance"
 import { Snapshot } from "@/snapshot"
 import { assertExternalDirectoryEffect } from "./external-directory"
 import { AppFileSystem } from "@opencode-ai/shared/filesystem"
+import * as Bom from "@/util/bom"
 
 function normalizeLineEndings(text: string): string {
   return text.replaceAll("\r\n", "\n")
@@ -84,7 +85,11 @@ export const EditTool = Tool.define(
             Effect.gen(function* () {
               if (params.oldString === "") {
                 const existed = yield* afs.existsSafe(filePath)
-                contentNew = params.newString
+                const source = existed ? yield* Bom.readFile(afs, filePath) : { bom: false, text: "" }
+                const next = Bom.split(params.newString)
+                const desiredBom = source.bom || next.bom
+                contentOld = source.text
+                contentNew = next.text
                 diff = trimDiff(createTwoFilesPatch(filePath, filePath, contentOld, contentNew))
                 yield* ctx.ask({
                   permission: "edit",
@@ -95,8 +100,10 @@ export const EditTool = Tool.define(
                     diff,
                   },
                 })
-                yield* afs.writeWithDirs(filePath, params.newString)
-                yield* format.file(filePath)
+                yield* afs.writeWithDirs(filePath, Bom.join(contentNew, desiredBom))
+                if (yield* format.file(filePath)) {
+                  contentNew = yield* Bom.syncFile(afs, filePath, desiredBom)
+                }
                 yield* bus.publish(File.Event.Edited, { file: filePath })
                 yield* bus.publish(FileWatcher.Event.Updated, {
                   file: filePath,
@@ -108,13 +115,16 @@ export const EditTool = Tool.define(
               const info = yield* afs.stat(filePath).pipe(Effect.catch(() => Effect.succeed(undefined)))
               if (!info) throw new Error(`File ${filePath} not found`)
               if (info.type === "Directory") throw new Error(`Path is a directory, not a file: ${filePath}`)
-              contentOld = yield* afs.readFileString(filePath)
+              const source = yield* Bom.readFile(afs, filePath)
+              contentOld = source.text
 
               const ending = detectLineEnding(contentOld)
               const old = convertToLineEnding(normalizeLineEndings(params.oldString), ending)
-              const next = convertToLineEnding(normalizeLineEndings(params.newString), ending)
+              const replacement = convertToLineEnding(normalizeLineEndings(params.newString), ending)
 
-              contentNew = replace(contentOld, old, next, params.replaceAll)
+              const next = Bom.split(replace(contentOld, old, replacement, params.replaceAll))
+              const desiredBom = source.bom || next.bom
+              contentNew = next.text
 
               diff = trimDiff(
                 createTwoFilesPatch(
@@ -134,14 +144,15 @@ export const EditTool = Tool.define(
                 },
               })
 
-              yield* afs.writeWithDirs(filePath, contentNew)
-              yield* format.file(filePath)
+              yield* afs.writeWithDirs(filePath, Bom.join(contentNew, desiredBom))
+              if (yield* format.file(filePath)) {
+                contentNew = yield* Bom.syncFile(afs, filePath, desiredBom)
+              }
               yield* bus.publish(File.Event.Edited, { file: filePath })
               yield* bus.publish(FileWatcher.Event.Updated, {
                 file: filePath,
                 event: "change",
               })
-              contentNew = yield* afs.readFileString(filePath)
               diff = trimDiff(
                 createTwoFilesPatch(
                   filePath,

packages/opencode/src/tool/write.ts

@@ -13,6 +13,7 @@ import { AppFileSystem } from "@opencode-ai/shared/filesystem"
 import { Instance } from "../project/instance"
 import { trimDiff } from "./edit"
 import { assertExternalDirectoryEffect } from "./external-directory"
+import * as Bom from "@/util/bom"
 
 const MAX_PROJECT_DIAGNOSTICS_FILES = 5
 
@@ -38,9 +39,13 @@ export const WriteTool = Tool.define(
           yield* assertExternalDirectoryEffect(ctx, filepath)
 
           const exists = yield* fs.existsSafe(filepath)
-          const contentOld = exists ? yield* fs.readFileString(filepath) : ""
+          const source = exists ? yield* Bom.readFile(fs, filepath) : { bom: false, text: "" }
+          const next = Bom.split(params.content)
+          const desiredBom = source.bom || next.bom
+          const contentOld = source.text
+          const contentNew = next.text
 
-          const diff = trimDiff(createTwoFilesPatch(filepath, filepath, contentOld, params.content))
+          const diff = trimDiff(createTwoFilesPatch(filepath, filepath, contentOld, contentNew))
           yield* ctx.ask({
             permission: "edit",
             patterns: [path.relative(Instance.worktree, filepath)],
@@ -51,8 +56,10 @@ export const WriteTool = Tool.define(
             },
           })
 
-          yield* fs.writeWithDirs(filepath, params.content)
-          yield* format.file(filepath)
+          yield* fs.writeWithDirs(filepath, Bom.join(contentNew, desiredBom))
+          if (yield* format.file(filepath)) {
+            yield* Bom.syncFile(fs, filepath, desiredBom)
+          }
           yield* bus.publish(File.Event.Edited, { file: filepath })
           yield* bus.publish(FileWatcher.Event.Updated, {
             file: filepath,

packages/opencode/src/util/bom.ts

@@ -0,0 +1,31 @@
+import { Effect } from "effect"
+import { AppFileSystem } from "@opencode-ai/shared/filesystem"
+
+const BOM_CODE = 0xfeff
+const BOM = String.fromCharCode(BOM_CODE)
+
+export function split(text: string) {
+  if (text.charCodeAt(0) !== BOM_CODE) return { bom: false, text }
+  return { bom: true, text: text.slice(1) }
+}
+
+export function join(text: string, bom: boolean) {
+  const stripped = split(text).text
+  if (!bom) return stripped
+  return BOM + stripped
+}
+
+export const readFile = Effect.fn("Bom.readFile")(function* (fs: AppFileSystem.Interface, filePath: string) {
+  return split(new TextDecoder("utf-8", { ignoreBOM: true }).decode(yield* fs.readFile(filePath)))
+})
+
+export const syncFile = Effect.fn("Bom.syncFile")(function* (
+  fs: AppFileSystem.Interface,
+  filePath: string,
+  bom: boolean,
+) {
+  const current = yield* readFile(fs, filePath)
+  if (current.bom === bom) return current.text
+  yield* fs.writeWithDirs(filePath, join(current.text, bom))
+  return current.text
+})