From 00d80544317079dc977c2f252fbbf63e74e3a20f Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 15 Jul 2026 03:09:41 +0800 Subject: [PATCH 01/24] feat(codex): serve provider-relative image requests Mount the image generation and edit paths Codex appends to its configured model-provider base. Reuse the public generation handler and translate Codex's JSON multi-image edit request into the shared multipart image-edit execution path. Preserve original request bytes for dump capture, validate the Codex edit envelope and base64 image URLs, and document the namespaced route surface with pinned Codex sources. --- README.md | 4 + .../gateway/src/data-plane/codex/images.ts | 99 +++++++++++++ .../src/data-plane/codex/images_test.ts | 140 ++++++++++++++++++ .../gateway/src/data-plane/codex/routes.ts | 8 +- .../gateway/src/data-plane/images/serve.ts | 40 ++--- 5 files changed, 272 insertions(+), 19 deletions(-) create mode 100644 packages/gateway/src/data-plane/codex/images.ts create mode 100644 packages/gateway/src/data-plane/codex/images_test.ts diff --git a/README.md b/README.md index 7218d31d1..18d38c5f5 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,10 @@ target ships in the same repo for self-hosting on a long-lived process. | OpenAI Models | `GET /v1/models` | | Google Gemini (generate / count tokens) | `POST /v1beta/models/...` | +The dashboard's Codex configuration uses `/azure-api.codex` as its provider +base. That namespace also exposes provider-relative `images/generations` and +JSON `images/edits` routes for Codex's client-owned image extension. + For each public model, Floway picks the first (provider, model) pair that can serve the request, translating between source and target protocols when the upstream speaks a different shape. `/v1/completions` is forwarded to upstreams that diff --git a/packages/gateway/src/data-plane/codex/images.ts b/packages/gateway/src/data-plane/codex/images.ts new file mode 100644 index 000000000..fd5c1baab --- /dev/null +++ b/packages/gateway/src/data-plane/codex/images.ts @@ -0,0 +1,99 @@ +import type { Context } from 'hono'; + +import { backgroundSchedulerFromContext } from '../../runtime/background.ts'; +import { createGatewayCtxFromHono, finalizeGatewayResponse } from '../chat/shared/gateway-ctx.ts'; +import { readRequestBody, takeRequestBody, type RequestBody } from '../chat/shared/request-body.ts'; +import { serveImagesEditForm } from '../images/serve.ts'; +import { passthroughApiError } from '../shared/passthrough-serve.ts'; + +interface CodexImageEditRequest { + images?: unknown; + model?: unknown; + prompt?: unknown; + [key: string]: unknown; +} + +type PreparedCodexEdit = + | { type: 'ok'; form: FormData } + | { type: 'invalid'; message: string }; + +const invalidRequest = (c: Context, requestBody: RequestBody, message: string): Response => { + const ctx = createGatewayCtxFromHono(c, { + wantsStream: false, + requestBody: takeRequestBody(requestBody), + backgroundScheduler: backgroundSchedulerFromContext(c), + }); + ctx.dump?.error('gateway'); + return finalizeGatewayResponse(ctx, passthroughApiError(c, message, 400)); +}; + +const imageFile = (value: unknown, index: number): File | string => { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + return `Codex image edits images[${index}] must be an object.`; + } + const imageUrl = (value as { image_url?: unknown }).image_url; + if (typeof imageUrl !== 'string') { + return `Codex image edits images[${index}].image_url must be a string.`; + } + const match = /^data:(image\/[a-z0-9.+-]+);base64,(.*)$/isu.exec(imageUrl); + if (!match) { + return `Codex image edits images[${index}].image_url must be a base64 image data URL.`; + } + let bytes: Uint8Array; + try { + bytes = Uint8Array.from(atob(match[2]!), character => character.charCodeAt(0)); + } catch { + return `Codex image edits images[${index}].image_url must contain valid base64 image data.`; + } + const mimeType = match[1]!.toLowerCase(); + const extension = mimeType === 'image/jpeg' ? 'jpg' : mimeType.slice('image/'.length).replace(/[^a-z0-9]+/gu, '-'); + return new File([bytes], `image-${index + 1}.${extension}`, { type: mimeType }); +}; + +const prepareCodexImageEdit = (bytes: Uint8Array): PreparedCodexEdit => { + let request: CodexImageEditRequest; + try { + const parsed = JSON.parse(new TextDecoder().decode(bytes)) as unknown; + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + return { type: 'invalid', message: 'Codex image edits request body must be an object.' }; + } + request = parsed as CodexImageEditRequest; + } catch { + return { type: 'invalid', message: 'Codex image edits request body must be valid JSON.' }; + } + if (typeof request.model !== 'string' || request.model.length === 0) { + return { type: 'invalid', message: 'Codex image edits request body must include a model string.' }; + } + if (typeof request.prompt !== 'string' || request.prompt.length === 0) { + return { type: 'invalid', message: 'Codex image edits request body must include a prompt string.' }; + } + if (!Array.isArray(request.images) || request.images.length === 0) { + return { type: 'invalid', message: 'Codex image edits request body must include at least one image.' }; + } + + const form = new FormData(); + for (const [name, value] of Object.entries(request)) { + if (name === 'images') continue; + if (typeof value !== 'string' && typeof value !== 'number' && typeof value !== 'boolean') { + return { type: 'invalid', message: `Codex image edits ${name} must be a string, number, or boolean.` }; + } + form.append(name, String(value)); + } + for (const [index, image] of request.images.entries()) { + const file = imageFile(image, index); + if (typeof file === 'string') return { type: 'invalid', message: file }; + form.append('image[]', file); + } + return { type: 'ok', form }; +}; + +// Codex's client-owned image extension posts JSON to provider-relative paths; +// the public OpenAI edits route remains multipart/form-data. +// https://github.com/openai/codex/blob/f90e7deea6a715bbd153044af6f475eefa749177/codex-rs/codex-api/src/endpoint/images.rs#L33-L68 +// https://github.com/openai/codex/blob/f90e7deea6a715bbd153044af6f475eefa749177/codex-rs/codex-api/src/images.rs#L4-L31 +export const codexImagesEdits = async (c: Context): Promise => { + const requestBody = await readRequestBody(c); + const prepared = prepareCodexImageEdit(requestBody.bytes); + if (prepared.type === 'invalid') return invalidRequest(c, requestBody, prepared.message); + return await serveImagesEditForm(c, requestBody, prepared.form); +}; diff --git a/packages/gateway/src/data-plane/codex/images_test.ts b/packages/gateway/src/data-plane/codex/images_test.ts new file mode 100644 index 000000000..2af47070c --- /dev/null +++ b/packages/gateway/src/data-plane/codex/images_test.ts @@ -0,0 +1,140 @@ +import { test } from 'vitest'; + +import { InMemoryRepo } from '../../repo/memory.ts'; +import { copilotModels, requestApp, setupAppTest } from '../../test-helpers.ts'; +import { assertEquals, assertExists, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; + +const saveAzureImages = async (repo: InMemoryRepo): Promise => { + await repo.upstreams.save({ + id: 'az-image', + kind: 'azure', + name: 'azure-images', + enabled: true, + sortOrder: 1, + createdAt: '2026-05-25T00:00:00Z', + updatedAt: '2026-05-25T00:00:00Z', + flagOverrides: {}, + disabledPublicModelIds: [], + proxyFallbackList: [], + modelPrefix: null, + color: null, + config: { + endpoint: 'https://example.openai.azure.com/openai/v1', + apiKey: 'azkey', + models: [{ + upstreamModelId: 'gpt-image-2', + endpoints: { imagesGenerations: {}, imagesEdits: {} }, + }], + }, + state: null, + }); +}; + +const controlPlaneFetch = (request: Request): Response | undefined => { + const url = new URL(request.url); + if (url.hostname === 'update.code.visualstudio.com') return jsonResponse(['1.110.1']); + if (url.pathname === '/copilot_internal/v2/token') { + return jsonResponse({ token: 'copilot-access-token', expires_at: 4102444800, refresh_in: 3600, endpoints: { api: 'https://api.individual.githubcopilot.com' } }); + } + if (url.hostname === 'api.individual.githubcopilot.com' && url.pathname === '/models') { + return jsonResponse(copilotModels([{ id: 'copilot-chat', supported_endpoints: ['/chat/completions'] }])); + } + return undefined; +}; + +test('Codex provider-relative image generation reuses the public JSON route', async () => { + const { apiKey, repo } = await setupAppTest(); + await saveAzureImages(repo); + let observedUrl: string | undefined; + let observedBody: Record | undefined; + + await withMockedFetch( + async request => { + const control = controlPlaneFetch(request); + if (control) return control; + const url = new URL(request.url); + if (url.hostname === 'example.openai.azure.com') { + observedUrl = request.url; + observedBody = await request.json() as Record; + return jsonResponse({ data: [{ b64_json: 'aGk=' }] }); + } + throw new Error(`Unhandled fetch ${request.url}`); + }, + async () => { + const response = await requestApp('/azure-api.codex/images/generations', { + method: 'POST', + headers: { authorization: `Bearer ${apiKey.key}`, 'content-type': 'application/json' }, + body: JSON.stringify({ model: 'gpt-image-2', prompt: 'a fox in space', quality: 'high' }), + }); + assertEquals(response.status, 200); + assertEquals(await response.json(), { data: [{ b64_json: 'aGk=' }] }); + }, + ); + + assertEquals(observedUrl?.endsWith('/images/generations?api-version=preview'), true); + assertExists(observedBody); + assertEquals(observedBody.prompt, 'a fox in space'); + assertEquals(observedBody.quality, 'high'); +}); + +test('Codex provider-relative image edits convert JSON data URLs to multipart images', async () => { + const { apiKey, repo } = await setupAppTest(); + await saveAzureImages(repo); + let observedUrl: string | undefined; + let observedForm: FormData | undefined; + + await withMockedFetch( + async request => { + const control = controlPlaneFetch(request); + if (control) return control; + const url = new URL(request.url); + if (url.hostname === 'example.openai.azure.com') { + observedUrl = request.url; + observedForm = await request.formData(); + return jsonResponse({ data: [{ b64_json: 'ZWRpdA==' }] }); + } + throw new Error(`Unhandled fetch ${request.url}`); + }, + async () => { + const response = await requestApp('/azure-api.codex/images/edits', { + method: 'POST', + headers: { authorization: `Bearer ${apiKey.key}`, 'content-type': 'application/json' }, + body: JSON.stringify({ + model: 'gpt-image-2', + prompt: 'add a red hat', + quality: 'high', + images: [ + { image_url: 'data:image/png;base64,AQID' }, + { image_url: 'data:image/jpeg;base64,BAUG' }, + ], + }), + }); + assertEquals(response.status, 200); + assertEquals(await response.json(), { data: [{ b64_json: 'ZWRpdA==' }] }); + }, + ); + + assertEquals(observedUrl?.endsWith('/images/edits?api-version=preview'), true); + assertExists(observedForm); + assertEquals(observedForm.get('model'), 'gpt-image-2'); + assertEquals(observedForm.get('prompt'), 'add a red hat'); + assertEquals(observedForm.get('quality'), 'high'); + const images = observedForm.getAll('image[]') as File[]; + assertEquals(images.map(image => image.type), ['image/png', 'image/jpeg']); + assertEquals(await Promise.all(images.map(image => image.arrayBuffer().then(buffer => [...new Uint8Array(buffer)]))), [[1, 2, 3], [4, 5, 6]]); +}); + +test.each([ + ['not-json', 'Codex image edits request body must be valid JSON.'], + [JSON.stringify({ model: 'gpt-image-2', prompt: 'edit', images: [{ image_url: 'https://example.com/image.png' }] }), 'Codex image edits images[0].image_url must be a base64 image data URL.'], + [JSON.stringify({ model: 'gpt-image-2', prompt: 'edit', images: [] }), 'Codex image edits request body must include at least one image.'], +])('Codex image edits reject malformed input %#', async (body, message) => { + const { apiKey } = await setupAppTest(); + const response = await requestApp('/azure-api.codex/images/edits', { + method: 'POST', + headers: { authorization: `Bearer ${apiKey.key}`, 'content-type': 'application/json' }, + body, + }); + assertEquals(response.status, 400); + assertEquals(await response.json(), { error: { message, type: 'invalid_request_error', param: null, code: 'invalid_request_error' } }); +}); diff --git a/packages/gateway/src/data-plane/codex/routes.ts b/packages/gateway/src/data-plane/codex/routes.ts index 02d35da11..0eb2119de 100644 --- a/packages/gateway/src/data-plane/codex/routes.ts +++ b/packages/gateway/src/data-plane/codex/routes.ts @@ -6,10 +6,12 @@ // model_providers.x.base_url — models, Responses HTTP/WS, remote compaction // // The dashboard points both at this namespace. Codex appends `models`, -// `responses`, and `responses/compact` directly to the model-provider base: +// `responses`, `responses/compact`, and `images/*` directly to the +// model-provider base: // https://github.com/openai/codex/blob/44918ea10c0f99151c6710411b4322c2f5c96bea/codex-rs/codex-api/src/endpoint/models.rs#L31-L43 // https://github.com/openai/codex/blob/44918ea10c0f99151c6710411b4322c2f5c96bea/codex-rs/codex-api/src/endpoint/responses.rs#L100-L102 // https://github.com/openai/codex/blob/44918ea10c0f99151c6710411b4322c2f5c96bea/codex-rs/codex-api/src/endpoint/compact.rs#L35-L57 +// https://github.com/openai/codex/blob/f90e7deea6a715bbd153044af6f475eefa749177/codex-rs/codex-api/src/endpoint/images.rs#L33-L68 // GET on `/responses` is the WebSocket upgrade entry; POST uses the generic // Responses handler, and `/responses/compact` uses its compaction counterpart. // Mounting the generic WS handler here preserves its session-local item chain. @@ -46,9 +48,11 @@ import { codexWhamAgentIdentitiesJwks, } from './chatgpt-backend.ts'; import { codexModels } from './models.ts'; +import { codexImagesEdits } from './images.ts'; import type { AuthVars } from '../../middleware/auth.ts'; import { responsesHttp } from '../chat/responses/http.ts'; import { responsesWebSocket } from '../chat/responses/websocket.ts'; +import { imagesGenerations } from '../images/serve.ts'; const CODEX_BASE_PATH = '/azure-api.codex'; @@ -56,6 +60,8 @@ export const mountCodexRoutes = (app: Hono<{ Variables: AuthVars }>) => { app.post(`${CODEX_BASE_PATH}/responses`, responsesHttp.generate); app.post(`${CODEX_BASE_PATH}/responses/compact`, responsesHttp.compact); app.get(`${CODEX_BASE_PATH}/responses`, responsesWebSocket); + app.post(`${CODEX_BASE_PATH}/images/generations`, imagesGenerations); + app.post(`${CODEX_BASE_PATH}/images/edits`, codexImagesEdits); app.get(`${CODEX_BASE_PATH}/models`, codexModels); app.post(`${CODEX_BASE_PATH}/codex/analytics-events/events`, codexAnalyticsEventsEvents); diff --git a/packages/gateway/src/data-plane/images/serve.ts b/packages/gateway/src/data-plane/images/serve.ts index f7c9cce4e..40bd1ca22 100644 --- a/packages/gateway/src/data-plane/images/serve.ts +++ b/packages/gateway/src/data-plane/images/serve.ts @@ -12,7 +12,7 @@ import type { Context } from 'hono'; import { backgroundSchedulerFromContext } from '../../runtime/background.ts'; import { createGatewayCtxFromHono, finalizeGatewayResponse } from '../chat/shared/gateway-ctx.ts'; -import { readRequestBody, takeRequestBody } from '../chat/shared/request-body.ts'; +import { readRequestBody, takeRequestBody, type RequestBody } from '../chat/shared/request-body.ts'; import { passthroughApiError, passthroughServe } from '../shared/passthrough-serve.ts'; import { tokenUsageFromImagesBody } from '../shared/telemetry/usage.ts'; @@ -70,23 +70,7 @@ export const imagesGenerations = async (c: Context): Promise => { return finalizeGatewayResponse(ctx, response); }; -export const imagesEdits = async (c: Context): Promise => { - // Buffer the multipart body once. Hono's formData() helper would consume - // c.req.raw.body internally; re-parsing from the captured bytes via a fresh - // Response keeps the dump capture honest without a second read on the wire. - const requestBody = await readRequestBody(c); - let form: FormData; - try { - form = await new Response(requestBody.bytes as BodyInit, { headers: { 'content-type': c.req.header('content-type') ?? '' } }).formData(); - } catch { - // Match the embeddings serve stance: do not surface the underlying - // parser's error text. The wording is enough for a client to know - // they sent the wrong content type or a malformed body. - const errorCtx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) }); - errorCtx.dump?.error('gateway'); - return finalizeGatewayResponse(errorCtx, passthroughApiError(c, 'Image edits request body must be a valid multipart/form-data payload.', 400)); - } - +export const serveImagesEditForm = async (c: Context, requestBody: RequestBody, form: FormData): Promise => { const ctx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) }); const modelRaw = form.get('model'); @@ -121,3 +105,23 @@ export const imagesEdits = async (c: Context): Promise => { }); return finalizeGatewayResponse(ctx, response); }; + +export const imagesEdits = async (c: Context): Promise => { + // Buffer the multipart body once. Hono's formData() helper would consume + // c.req.raw.body internally; re-parsing from the captured bytes via a fresh + // Response keeps the dump capture honest without a second read on the wire. + const requestBody = await readRequestBody(c); + let form: FormData; + try { + form = await new Response(requestBody.bytes as BodyInit, { headers: { 'content-type': c.req.header('content-type') ?? '' } }).formData(); + } catch { + // Match the embeddings serve stance: do not surface the underlying + // parser's error text. The wording is enough for a client to know + // they sent the wrong content type or a malformed body. + const errorCtx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) }); + errorCtx.dump?.error('gateway'); + return finalizeGatewayResponse(errorCtx, passthroughApiError(c, 'Image edits request body must be a valid multipart/form-data payload.', 400)); + } + + return await serveImagesEditForm(c, requestBody, form); +}; From b198c20ca909fc9a8271431b75ac6aa34b7fac5a Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 15 Jul 2026 03:11:46 +0800 Subject: [PATCH 02/24] fix(codex): type image edit bytes and error fixtures --- packages/gateway/src/data-plane/codex/images.ts | 9 ++++++--- packages/gateway/src/data-plane/codex/images_test.ts | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/gateway/src/data-plane/codex/images.ts b/packages/gateway/src/data-plane/codex/images.ts index fd5c1baab..c68da688e 100644 --- a/packages/gateway/src/data-plane/codex/images.ts +++ b/packages/gateway/src/data-plane/codex/images.ts @@ -39,15 +39,18 @@ const imageFile = (value: unknown, index: number): File | string => { if (!match) { return `Codex image edits images[${index}].image_url must be a base64 image data URL.`; } - let bytes: Uint8Array; + let data: ArrayBuffer; try { - bytes = Uint8Array.from(atob(match[2]!), character => character.charCodeAt(0)); + const decoded = atob(match[2]!); + data = new ArrayBuffer(decoded.length); + const bytes = new Uint8Array(data); + for (let i = 0; i < decoded.length; i++) bytes[i] = decoded.charCodeAt(i); } catch { return `Codex image edits images[${index}].image_url must contain valid base64 image data.`; } const mimeType = match[1]!.toLowerCase(); const extension = mimeType === 'image/jpeg' ? 'jpg' : mimeType.slice('image/'.length).replace(/[^a-z0-9]+/gu, '-'); - return new File([bytes], `image-${index + 1}.${extension}`, { type: mimeType }); + return new File([data], `image-${index + 1}.${extension}`, { type: mimeType }); }; const prepareCodexImageEdit = (bytes: Uint8Array): PreparedCodexEdit => { diff --git a/packages/gateway/src/data-plane/codex/images_test.ts b/packages/gateway/src/data-plane/codex/images_test.ts index 2af47070c..a6dc9d04c 100644 --- a/packages/gateway/src/data-plane/codex/images_test.ts +++ b/packages/gateway/src/data-plane/codex/images_test.ts @@ -136,5 +136,5 @@ test.each([ body, }); assertEquals(response.status, 400); - assertEquals(await response.json(), { error: { message, type: 'invalid_request_error', param: null, code: 'invalid_request_error' } }); + assertEquals(await response.json(), { error: { message, type: 'api_error' } }); }); From 6c981dd3c00fa47a5d782d21d6b487ab8a86621a Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 15 Jul 2026 03:26:52 +0800 Subject: [PATCH 03/24] refactor(images): share edit source preparation Accept Codex edit inputs from inline or remote image URLs, validate their actual raster format, and reuse the hosted image shim's edit transcoding path. This keeps standalone and hosted image edits aligned while preserving the provider-relative JSON adapter. --- .../server-tools/image-generation.ts | 87 +------------ .../server-tools/image-generation_test.ts | 6 +- .../gateway/src/data-plane/codex/images.ts | 117 +++++++++++++----- .../src/data-plane/codex/images_test.ts | 36 +++++- .../gateway/src/data-plane/codex/routes.ts | 3 +- .../src/data-plane/images/edit-source.ts | 75 +++++++++++ 6 files changed, 197 insertions(+), 127 deletions(-) create mode 100644 packages/gateway/src/data-plane/images/edit-source.ts diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts index f02eac566..09026c75f 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts @@ -5,8 +5,9 @@ import { recordPerformance, type PerformanceTelemetryContext } from '../../../.. import { recordTokenUsage, tokenUsageFromImagesBody } from '../../../../shared/telemetry/usage.ts'; import { createExternalImageFetcher, type ExternalImageFetchResult } from '../../../shared/external-image-loader.ts'; import { stampUpstreamCallStart, type AttemptState } from '../../../shared/gateway-ctx.ts'; +import { editSupportedMime, prepareImageEditSources, supportedImageMimeFromBytes, type ImageEditSource } from '../../../../images/edit-source.ts'; import type { ServerToolLifecycleEvent, ServerToolOutputItem, ServerToolRegistration, ServerToolTerminal } from '../server-tool-shim.ts'; -import { dimensionsFromBytes, getImageProcessor, type BackgroundScheduler } from '@floway-dev/platform'; +import type { BackgroundScheduler } from '@floway-dev/platform'; import { parseSSEStream } from '@floway-dev/protocols/common'; import type { ResponsesFunctionCallOutputItem, @@ -48,25 +49,6 @@ const ALLOWED_MODERATIONS = new Set(['auto', 'low']); const ALLOWED_ACTIONS = new Set(['generate', 'edit', 'auto']); const ALLOWED_INPUT_FIDELITY = new Set(['high', 'low']); -// gpt-image-* `/images/edits` accepts only these input image mimetypes; a live -// Azure probe confirmed png/jpeg/webp succeed while gif is rejected with -// `unsupported_file_mimetype`. Native Responses accepts the same GIF and -// re-encodes it before editing, so the shim mirrors that behavior through the -// platform image processor. Common aliases are folded onto the backend form. -const EDIT_MIME_ALIASES: Record = { - 'image/jpg': 'image/jpeg', - 'image/pjpeg': 'image/jpeg', - 'image/x-png': 'image/png', -}; -const EDIT_SUPPORTED_MIMES = new Set(['image/png', 'image/jpeg', 'image/webp']); - -// The canonical edit-supported mimetype for a source, or null when the -// standalone endpoint requires local WebP transcoding first. -const editSupportedMime = (mime: string): string | null => { - const canonical = EDIT_MIME_ALIASES[mime] ?? mime; - return EDIT_SUPPORTED_MIMES.has(canonical) ? canonical : null; -}; - const editFileExt = (mime: string): string => mime === 'image/jpeg' ? 'jpg' : mime === 'image/webp' ? 'webp' : 'png'; @@ -95,10 +77,7 @@ export const canonicalizeImageGenerationTool = (raw: ResponsesTool): ResponsesHo // A base64-data-URL or bare-base64 image source bound for an edit call. // Bytes are held in a concrete ArrayBuffer so they can be wrapped in a Blob. -interface ImageSource { - bytes: ArrayBuffer; - mimeType: string; -} +type ImageSource = ImageEditSource; interface RemoteImageSource { wireUrl: string; @@ -111,38 +90,6 @@ type ImageSourceReference = ImageSource | RemoteImageSource; const isRemoteImageSource = (source: ImageSourceReference): source is RemoteImageSource => 'wireUrl' in source; -export const prepareEditSources = async (sources: readonly ImageSource[]): Promise => { - const keyBySource = new Map>(); - const preparedByContent = new Map>(); - return await Promise.all(sources.map(async source => { - if (editSupportedMime(source.mimeType) !== null) return source; - - let keyPromise = keyBySource.get(source); - if (keyPromise === undefined) { - keyPromise = crypto.subtle.digest('SHA-256', source.bytes).then(buffer => { - const digest = [...new Uint8Array(buffer)].map(byte => byte.toString(16).padStart(2, '0')).join(''); - return `${source.mimeType}\u0000${digest}`; - }); - keyBySource.set(source, keyPromise); - } - const key = await keyPromise; - - let prepared = preparedByContent.get(key); - if (prepared === undefined) { - // Native Responses accepts formats such as GIF through its multimodal - // preprocessing, while the standalone edits endpoint accepts only - // PNG/JPEG/WebP. Re-encode locally to preserve the hosted-tool behavior. - // https://github.com/openai/openai-node/blob/ec2f57fd0d66e94782656b986d7b3eb03225369c/src/resources/images.ts#L560-L572 - prepared = getImageProcessor().compressToWebp(new Uint8Array(source.bytes), null).then(encoded => { - const bytes = encoded.buffer.slice(encoded.byteOffset, encoded.byteOffset + encoded.byteLength) as ArrayBuffer; - return { bytes, mimeType: 'image/webp' }; - }); - preparedByContent.set(key, prepared); - } - return await prepared; - })); -}; - const base64ToArrayBuffer = (b64: string): ArrayBuffer => { const binary = atob(b64); const buffer = new ArrayBuffer(binary.length); @@ -264,7 +211,7 @@ const prepareEditRequest = async ( } const originals = [...sources]; if (config.mask !== undefined && !originals.includes(config.mask)) originals.push(config.mask); - const prepared = await prepareEditSources(originals); + const prepared = await prepareImageEditSources(originals); const bySource = new Map(); for (const [index, source] of originals.entries()) { const wireSource = prepared[index]; @@ -572,32 +519,6 @@ const remoteInputError = (source: RemoteImageSource, failure: RemoteImageFailure // https://platform.openai.com/docs/guides/images-vision#image-input-requirements const MAX_REMOTE_IMAGE_TOTAL_BYTES = 50 * 1024 * 1024; -const supportedImageMimeFromBytes = (bytes: Uint8Array): string | null => { - if (dimensionsFromBytes(bytes) === null) return null; - if ( - bytes.length >= 8 - && bytes[0] === 0x89 - && bytes[1] === 0x50 - && bytes[2] === 0x4e - && bytes[3] === 0x47 - && bytes[4] === 0x0d - && bytes[5] === 0x0a - && bytes[6] === 0x1a - && bytes[7] === 0x0a - ) return 'image/png'; - if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) return 'image/jpeg'; - if (bytes.length >= 6) { - const signature = String.fromCharCode(...bytes.subarray(0, 6)); - if (signature === 'GIF87a' || signature === 'GIF89a') return 'image/gif'; - } - if ( - bytes.length >= 12 - && String.fromCharCode(...bytes.subarray(0, 4)) === 'RIFF' - && String.fromCharCode(...bytes.subarray(8, 12)) === 'WEBP' - ) return 'image/webp'; - return null; -}; - const createRemoteImageMaterializer = (requestSignal: AbortSignal | undefined) => { const fetchImage = createExternalImageFetcher(requestSignal); const materialized = new Map(); diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation_test.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation_test.ts index 6bf80b562..5eb75b2fb 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation_test.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation_test.ts @@ -13,13 +13,13 @@ import { isHostedImageGenerationTool, parseImageStreamEvent, parseRetryAfterMs, - prepareEditSources, prepareImageGenerationConfig, resolveImageOperation, SHIM_TOOL_NAME, synthesizeImageGenerationCallId, transformInputItemsForImageGeneration, } from './image-generation.ts'; +import { prepareImageEditSources } from '../../../../images/edit-source.ts'; import { initRepo } from '../../../../../repo/index.ts'; import { InMemoryRepo } from '../../../../../repo/memory.ts'; import { mockChatGatewayCtx } from '../../../../../test-helpers/gateway-ctx.ts'; @@ -352,7 +352,7 @@ test('inspectImageSources reads tool-result images and preserves forward order', assertEquals(third.mimeType, 'image/webp'); }); -test('prepareEditSources transcodes formats accepted by native Responses but rejected by images edits', async () => { +test('prepareImageEditSources transcodes formats accepted by native Responses but rejected by images edits', async () => { let observedTarget: unknown = undefined; let processorCalls = 0; initImageProcessor({ @@ -367,7 +367,7 @@ test('prepareEditSources transcodes formats accepted by native Responses but rej mimeType: 'image/gif', }; const sameBytes = { bytes: Uint8Array.of(4, 5, 6).buffer, mimeType: 'image/gif' }; - const prepared = await prepareEditSources([first, first, sameBytes]); + const prepared = await prepareImageEditSources([first, first, sameBytes]); assertEquals(processorCalls, 1); assert(prepared[0] === prepared[1]); assert(prepared[0] === prepared[2]); diff --git a/packages/gateway/src/data-plane/codex/images.ts b/packages/gateway/src/data-plane/codex/images.ts index c68da688e..95edf0723 100644 --- a/packages/gateway/src/data-plane/codex/images.ts +++ b/packages/gateway/src/data-plane/codex/images.ts @@ -1,8 +1,14 @@ import type { Context } from 'hono'; import { backgroundSchedulerFromContext } from '../../runtime/background.ts'; +import { createExternalImageFetcher, type ExternalImageFetchResult } from '../chat/shared/external-image-loader.ts'; import { createGatewayCtxFromHono, finalizeGatewayResponse } from '../chat/shared/gateway-ctx.ts'; import { readRequestBody, takeRequestBody, type RequestBody } from '../chat/shared/request-body.ts'; +import { + prepareImageEditSources, + supportedImageMimeFromBytes, + type ImageEditSource, +} from '../images/edit-source.ts'; import { serveImagesEditForm } from '../images/serve.ts'; import { passthroughApiError } from '../shared/passthrough-serve.ts'; @@ -17,43 +23,73 @@ type PreparedCodexEdit = | { type: 'ok'; form: FormData } | { type: 'invalid'; message: string }; -const invalidRequest = (c: Context, requestBody: RequestBody, message: string): Response => { - const ctx = createGatewayCtxFromHono(c, { - wantsStream: false, - requestBody: takeRequestBody(requestBody), - backgroundScheduler: backgroundSchedulerFromContext(c), - }); - ctx.dump?.error('gateway'); - return finalizeGatewayResponse(ctx, passthroughApiError(c, message, 400)); +type LoadedImage = + | { type: 'ok'; source: ImageEditSource } + | { type: 'invalid'; message: string }; + +const imageBytes = (data: Uint8Array): ArrayBuffer => + data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength) as ArrayBuffer; + +const decodeDataUrl = (value: string, index: number): LoadedImage | null => { + const match = /^data:image\/[a-z0-9.+-]+;base64,(.*)$/isu.exec(value); + if (match === null) return null; + + let bytes: Uint8Array; + try { + const decoded = atob(match[1]!); + bytes = Uint8Array.from(decoded, character => character.charCodeAt(0)); + } catch { + return { type: 'invalid', message: `Codex image edits images[${index}].image_url must contain valid base64 image data.` }; + } + const mimeType = supportedImageMimeFromBytes(bytes); + return mimeType === null + ? { type: 'invalid', message: `Codex image edits images[${index}].image_url must contain a supported raster image.` } + : { type: 'ok', source: { bytes: imageBytes(bytes), mimeType } }; +}; + +const externalImageError = (result: Exclude, index: number): string => { + const path = `Codex image edits images[${index}].image_url`; + switch (result.type) { + case 'invalid-url': return `${path} must be an HTTP(S) URL or a base64 image data URL.`; + case 'invalid-redirect': return `${path} has an invalid redirect (${result.reason}).`; + case 'http-error': return `${path} returned HTTP ${result.status}.`; + case 'too-large': return `${path} exceeds the ${result.limitBytes}-byte image limit.`; + case 'empty-body': return `${path} returned an empty response.`; + case 'timeout': return `${path} timed out.`; + case 'transport-error': return `${path} could not be downloaded.`; + } }; -const imageFile = (value: unknown, index: number): File | string => { +const loadImage = async ( + value: unknown, + index: number, + fetchExternalImage: ReturnType, +): Promise => { if (!value || typeof value !== 'object' || Array.isArray(value)) { - return `Codex image edits images[${index}] must be an object.`; + return { type: 'invalid', message: `Codex image edits images[${index}] must be an object.` }; } const imageUrl = (value as { image_url?: unknown }).image_url; if (typeof imageUrl !== 'string') { - return `Codex image edits images[${index}].image_url must be a string.`; + return { type: 'invalid', message: `Codex image edits images[${index}].image_url must be a string.` }; } - const match = /^data:(image\/[a-z0-9.+-]+);base64,(.*)$/isu.exec(imageUrl); - if (!match) { - return `Codex image edits images[${index}].image_url must be a base64 image data URL.`; - } - let data: ArrayBuffer; - try { - const decoded = atob(match[2]!); - data = new ArrayBuffer(decoded.length); - const bytes = new Uint8Array(data); - for (let i = 0; i < decoded.length; i++) bytes[i] = decoded.charCodeAt(i); - } catch { - return `Codex image edits images[${index}].image_url must contain valid base64 image data.`; - } - const mimeType = match[1]!.toLowerCase(); - const extension = mimeType === 'image/jpeg' ? 'jpg' : mimeType.slice('image/'.length).replace(/[^a-z0-9]+/gu, '-'); - return new File([data], `image-${index + 1}.${extension}`, { type: mimeType }); + + const inline = decodeDataUrl(imageUrl, index); + if (inline !== null) return inline; + + const result = await fetchExternalImage(imageUrl); + if (result.type !== 'success') return { type: 'invalid', message: externalImageError(result, index) }; + const mimeType = supportedImageMimeFromBytes(result.data); + return mimeType === null + ? { type: 'invalid', message: `Codex image edits images[${index}].image_url must resolve to a supported raster image.` } + : { type: 'ok', source: { bytes: imageBytes(result.data), mimeType } }; +}; + +const imageFile = (source: ImageEditSource, index: number): File => { + const extension = source.mimeType === 'image/jpeg' ? 'jpg' : source.mimeType.slice('image/'.length); + return new File([source.bytes], `image-${index + 1}.${extension}`, { type: source.mimeType }); }; -const prepareCodexImageEdit = (bytes: Uint8Array): PreparedCodexEdit => { +const prepareCodexImageEdit = async (bytes: Uint8Array, signal: AbortSignal): Promise => { let request: CodexImageEditRequest; try { const parsed = JSON.parse(new TextDecoder().decode(bytes)) as unknown; @@ -67,7 +103,7 @@ const prepareCodexImageEdit = (bytes: Uint8Array): PreparedCodexEdit => { if (typeof request.model !== 'string' || request.model.length === 0) { return { type: 'invalid', message: 'Codex image edits request body must include a model string.' }; } - if (typeof request.prompt !== 'string' || request.prompt.length === 0) { + if (typeof request.prompt !== 'string') { return { type: 'invalid', message: 'Codex image edits request body must include a prompt string.' }; } if (!Array.isArray(request.images) || request.images.length === 0) { @@ -82,11 +118,16 @@ const prepareCodexImageEdit = (bytes: Uint8Array): PreparedCodexEdit => { } form.append(name, String(value)); } + + const fetchExternalImage = createExternalImageFetcher(signal); + const sources: ImageEditSource[] = []; for (const [index, image] of request.images.entries()) { - const file = imageFile(image, index); - if (typeof file === 'string') return { type: 'invalid', message: file }; - form.append('image[]', file); + const loaded = await loadImage(image, index, fetchExternalImage); + if (loaded.type === 'invalid') return loaded; + sources.push(loaded.source); } + const prepared = await prepareImageEditSources(sources); + prepared.forEach((source, index) => form.append('image[]', imageFile(source, index))); return { type: 'ok', form }; }; @@ -96,7 +137,15 @@ const prepareCodexImageEdit = (bytes: Uint8Array): PreparedCodexEdit => { // https://github.com/openai/codex/blob/f90e7deea6a715bbd153044af6f475eefa749177/codex-rs/codex-api/src/images.rs#L4-L31 export const codexImagesEdits = async (c: Context): Promise => { const requestBody = await readRequestBody(c); - const prepared = prepareCodexImageEdit(requestBody.bytes); - if (prepared.type === 'invalid') return invalidRequest(c, requestBody, prepared.message); + const prepared = await prepareCodexImageEdit(requestBody.bytes, c.req.raw.signal); + if (prepared.type === 'invalid') { + const ctx = createGatewayCtxFromHono(c, { + wantsStream: false, + requestBody: takeRequestBody(requestBody), + backgroundScheduler: backgroundSchedulerFromContext(c), + }); + ctx.dump?.error('gateway'); + return finalizeGatewayResponse(ctx, passthroughApiError(c, prepared.message, 400)); + } return await serveImagesEditForm(c, requestBody, prepared.form); }; diff --git a/packages/gateway/src/data-plane/codex/images_test.ts b/packages/gateway/src/data-plane/codex/images_test.ts index a6dc9d04c..8f081f012 100644 --- a/packages/gateway/src/data-plane/codex/images_test.ts +++ b/packages/gateway/src/data-plane/codex/images_test.ts @@ -2,8 +2,13 @@ import { test } from 'vitest'; import { InMemoryRepo } from '../../repo/memory.ts'; import { copilotModels, requestApp, setupAppTest } from '../../test-helpers.ts'; +import { initExternalResourceFetcher, initImageProcessor } from '@floway-dev/platform'; import { assertEquals, assertExists, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; +const PNG_B64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M8AAAMBAQDJ/wEAAAAASUVORK5CYII='; +const GIF_B64 = 'R0lGODlhAQABAIAAAAAAAP///ywAAAAAAQABAAACAUwAOw=='; +const bytesFromBase64 = (value: string): Uint8Array => Uint8Array.from(atob(value), character => character.charCodeAt(0)); + const saveAzureImages = async (repo: InMemoryRepo): Promise => { await repo.upstreams.save({ id: 'az-image', @@ -77,11 +82,25 @@ test('Codex provider-relative image generation reuses the public JSON route', as assertEquals(observedBody.quality, 'high'); }); -test('Codex provider-relative image edits convert JSON data URLs to multipart images', async () => { +test('Codex provider-relative image edits materialize remote images and transcode unsupported raster formats', async () => { const { apiKey, repo } = await setupAppTest(); await saveAzureImages(repo); let observedUrl: string | undefined; let observedForm: FormData | undefined; + let observedImageSourceUrl: string | undefined; + const webpBytes = Uint8Array.of(0x52, 0x49, 0x46, 0x46, 0, 0, 0, 0, 0x57, 0x45, 0x42, 0x50); + initExternalResourceFetcher(url => { + observedImageSourceUrl = url.href; + return Promise.resolve(new Response(bytesFromBase64(PNG_B64).buffer as ArrayBuffer, { + headers: { 'content-type': 'image/png' }, + })); + }); + initImageProcessor({ + compressToWebp: input => { + assertEquals([...input.subarray(0, 6)], [...new TextEncoder().encode('GIF89a')]); + return Promise.resolve(webpBytes); + }, + }); await withMockedFetch( async request => { @@ -104,8 +123,8 @@ test('Codex provider-relative image edits convert JSON data URLs to multipart im prompt: 'add a red hat', quality: 'high', images: [ - { image_url: 'data:image/png;base64,AQID' }, - { image_url: 'data:image/jpeg;base64,BAUG' }, + { image_url: 'https://assets.example/image.png' }, + { image_url: `data:image/gif;base64,${GIF_B64}` }, ], }), }); @@ -114,19 +133,24 @@ test('Codex provider-relative image edits convert JSON data URLs to multipart im }, ); + assertEquals(observedImageSourceUrl, 'https://assets.example/image.png'); assertEquals(observedUrl?.endsWith('/images/edits?api-version=preview'), true); assertExists(observedForm); assertEquals(observedForm.get('model'), 'gpt-image-2'); assertEquals(observedForm.get('prompt'), 'add a red hat'); assertEquals(observedForm.get('quality'), 'high'); const images = observedForm.getAll('image[]') as File[]; - assertEquals(images.map(image => image.type), ['image/png', 'image/jpeg']); - assertEquals(await Promise.all(images.map(image => image.arrayBuffer().then(buffer => [...new Uint8Array(buffer)]))), [[1, 2, 3], [4, 5, 6]]); + assertEquals(images.map(image => image.type), ['image/png', 'image/webp']); + assertEquals( + await Promise.all(images.map(image => image.arrayBuffer().then(buffer => [...new Uint8Array(buffer)]))), + [[...bytesFromBase64(PNG_B64)], [...webpBytes]], + ); }); test.each([ ['not-json', 'Codex image edits request body must be valid JSON.'], - [JSON.stringify({ model: 'gpt-image-2', prompt: 'edit', images: [{ image_url: 'https://example.com/image.png' }] }), 'Codex image edits images[0].image_url must be a base64 image data URL.'], + [JSON.stringify({ model: 'gpt-image-2', prompt: 'edit', images: [{ image_url: 'ftp://example.com/image.png' }] }), 'Codex image edits images[0].image_url must be an HTTP(S) URL or a base64 image data URL.'], + [JSON.stringify({ model: 'gpt-image-2', prompt: '', images: [{ image_url: 'data:image/png;base64,AQID' }] }), 'Codex image edits images[0].image_url must contain a supported raster image.'], [JSON.stringify({ model: 'gpt-image-2', prompt: 'edit', images: [] }), 'Codex image edits request body must include at least one image.'], ])('Codex image edits reject malformed input %#', async (body, message) => { const { apiKey } = await setupAppTest(); diff --git a/packages/gateway/src/data-plane/codex/routes.ts b/packages/gateway/src/data-plane/codex/routes.ts index 0eb2119de..f40a491a1 100644 --- a/packages/gateway/src/data-plane/codex/routes.ts +++ b/packages/gateway/src/data-plane/codex/routes.ts @@ -6,7 +6,8 @@ // model_providers.x.base_url — models, Responses HTTP/WS, remote compaction // // The dashboard points both at this namespace. Codex appends `models`, -// `responses`, `responses/compact`, and `images/*` directly to the +// `responses`, `responses/compact`, `images/generations`, and `images/edits` +// directly to the // model-provider base: // https://github.com/openai/codex/blob/44918ea10c0f99151c6710411b4322c2f5c96bea/codex-rs/codex-api/src/endpoint/models.rs#L31-L43 // https://github.com/openai/codex/blob/44918ea10c0f99151c6710411b4322c2f5c96bea/codex-rs/codex-api/src/endpoint/responses.rs#L100-L102 diff --git a/packages/gateway/src/data-plane/images/edit-source.ts b/packages/gateway/src/data-plane/images/edit-source.ts new file mode 100644 index 000000000..0b2e0d9b0 --- /dev/null +++ b/packages/gateway/src/data-plane/images/edit-source.ts @@ -0,0 +1,75 @@ +import { dimensionsFromBytes, getImageProcessor } from '@floway-dev/platform'; + +export interface ImageEditSource { + bytes: ArrayBuffer; + mimeType: string; +} + +const EDIT_MIME_ALIASES: Record = { + 'image/jpg': 'image/jpeg', + 'image/pjpeg': 'image/jpeg', + 'image/x-png': 'image/png', +}; +const EDIT_SUPPORTED_MIMES = new Set(['image/png', 'image/jpeg', 'image/webp']); + +export const editSupportedMime = (mime: string): string | null => { + const canonical = EDIT_MIME_ALIASES[mime] ?? mime; + return EDIT_SUPPORTED_MIMES.has(canonical) ? canonical : null; +}; + +export const supportedImageMimeFromBytes = (bytes: Uint8Array): string | null => { + if (dimensionsFromBytes(bytes) === null) return null; + if ( + bytes.length >= 8 + && bytes[0] === 0x89 + && bytes[1] === 0x50 + && bytes[2] === 0x4e + && bytes[3] === 0x47 + && bytes[4] === 0x0d + && bytes[5] === 0x0a + && bytes[6] === 0x1a + && bytes[7] === 0x0a + ) return 'image/png'; + if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) return 'image/jpeg'; + if (bytes.length >= 6) { + const signature = String.fromCharCode(...bytes.subarray(0, 6)); + if (signature === 'GIF87a' || signature === 'GIF89a') return 'image/gif'; + } + if ( + bytes.length >= 12 + && String.fromCharCode(...bytes.subarray(0, 4)) === 'RIFF' + && String.fromCharCode(...bytes.subarray(8, 12)) === 'WEBP' + ) return 'image/webp'; + return null; +}; + +export const prepareImageEditSources = async (sources: readonly ImageEditSource[]): Promise => { + const keyBySource = new Map>(); + const preparedByContent = new Map>(); + return await Promise.all(sources.map(async source => { + if (editSupportedMime(source.mimeType) !== null) return source; + + let keyPromise = keyBySource.get(source); + if (keyPromise === undefined) { + keyPromise = crypto.subtle.digest('SHA-256', source.bytes).then(buffer => { + const digest = [...new Uint8Array(buffer)].map(byte => byte.toString(16).padStart(2, '0')).join(''); + return `${source.mimeType}\u0000${digest}`; + }); + keyBySource.set(source, keyPromise); + } + const key = await keyPromise; + + let prepared = preparedByContent.get(key); + if (prepared === undefined) { + // The standalone edits endpoint accepts PNG/JPEG/WebP, so encode other + // raster formats before dispatch. + // https://github.com/openai/openai-node/blob/ec2f57fd0d66e94782656b986d7b3eb03225369c/src/resources/images.ts#L560-L572 + prepared = getImageProcessor().compressToWebp(new Uint8Array(source.bytes), null).then(encoded => { + const bytes = encoded.buffer.slice(encoded.byteOffset, encoded.byteOffset + encoded.byteLength) as ArrayBuffer; + return { bytes, mimeType: 'image/webp' }; + }); + preparedByContent.set(key, prepared); + } + return await prepared; + })); +}; From 077c9b98dfc5b0aeba97810d5188413ef5db86f3 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 15 Jul 2026 03:29:34 +0800 Subject: [PATCH 04/24] style(images): satisfy workspace lint rules --- .../server-tools/image-generation.ts | 2 +- .../server-tools/image-generation_test.ts | 2 +- packages/gateway/src/data-plane/codex/images.ts | 16 ++++++++-------- .../gateway/src/data-plane/codex/images_test.ts | 2 +- packages/gateway/src/data-plane/codex/routes.ts | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts index 09026c75f..81c5fa7b0 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts @@ -1,11 +1,11 @@ import { sleep } from '../../../../../shared/sleep.ts'; +import { editSupportedMime, prepareImageEditSources, supportedImageMimeFromBytes, type ImageEditSource } from '../../../../images/edit-source.ts'; import { enumerateModelCandidates } from '../../../../providers/registry.ts'; import { appendFailedUpstreams } from '../../../../shared/failed-upstreams.ts'; import { recordPerformance, type PerformanceTelemetryContext } from '../../../../shared/telemetry/performance.ts'; import { recordTokenUsage, tokenUsageFromImagesBody } from '../../../../shared/telemetry/usage.ts'; import { createExternalImageFetcher, type ExternalImageFetchResult } from '../../../shared/external-image-loader.ts'; import { stampUpstreamCallStart, type AttemptState } from '../../../shared/gateway-ctx.ts'; -import { editSupportedMime, prepareImageEditSources, supportedImageMimeFromBytes, type ImageEditSource } from '../../../../images/edit-source.ts'; import type { ServerToolLifecycleEvent, ServerToolOutputItem, ServerToolRegistration, ServerToolTerminal } from '../server-tool-shim.ts'; import type { BackgroundScheduler } from '@floway-dev/platform'; import { parseSSEStream } from '@floway-dev/protocols/common'; diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation_test.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation_test.ts index 5eb75b2fb..680d14e75 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation_test.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation_test.ts @@ -19,10 +19,10 @@ import { synthesizeImageGenerationCallId, transformInputItemsForImageGeneration, } from './image-generation.ts'; -import { prepareImageEditSources } from '../../../../images/edit-source.ts'; import { initRepo } from '../../../../../repo/index.ts'; import { InMemoryRepo } from '../../../../../repo/memory.ts'; import { mockChatGatewayCtx } from '../../../../../test-helpers/gateway-ctx.ts'; +import { prepareImageEditSources } from '../../../../images/edit-source.ts'; import type { ResponsesInvocation } from '../types.ts'; import { initExternalResourceFetcher, initImageProcessor } from '@floway-dev/platform'; import type { CanonicalResponsesPayload, ResponsesInputImage, ResponsesInputItem, ResponsesPayload, ResponsesTool } from '@floway-dev/protocols/responses'; diff --git a/packages/gateway/src/data-plane/codex/images.ts b/packages/gateway/src/data-plane/codex/images.ts index 95edf0723..731980ea3 100644 --- a/packages/gateway/src/data-plane/codex/images.ts +++ b/packages/gateway/src/data-plane/codex/images.ts @@ -3,7 +3,7 @@ import type { Context } from 'hono'; import { backgroundSchedulerFromContext } from '../../runtime/background.ts'; import { createExternalImageFetcher, type ExternalImageFetchResult } from '../chat/shared/external-image-loader.ts'; import { createGatewayCtxFromHono, finalizeGatewayResponse } from '../chat/shared/gateway-ctx.ts'; -import { readRequestBody, takeRequestBody, type RequestBody } from '../chat/shared/request-body.ts'; +import { readRequestBody, takeRequestBody } from '../chat/shared/request-body.ts'; import { prepareImageEditSources, supportedImageMimeFromBytes, @@ -50,13 +50,13 @@ const decodeDataUrl = (value: string, index: number): LoadedImage | null => { const externalImageError = (result: Exclude, index: number): string => { const path = `Codex image edits images[${index}].image_url`; switch (result.type) { - case 'invalid-url': return `${path} must be an HTTP(S) URL or a base64 image data URL.`; - case 'invalid-redirect': return `${path} has an invalid redirect (${result.reason}).`; - case 'http-error': return `${path} returned HTTP ${result.status}.`; - case 'too-large': return `${path} exceeds the ${result.limitBytes}-byte image limit.`; - case 'empty-body': return `${path} returned an empty response.`; - case 'timeout': return `${path} timed out.`; - case 'transport-error': return `${path} could not be downloaded.`; + case 'invalid-url': return `${path} must be an HTTP(S) URL or a base64 image data URL.`; + case 'invalid-redirect': return `${path} has an invalid redirect (${result.reason}).`; + case 'http-error': return `${path} returned HTTP ${result.status}.`; + case 'too-large': return `${path} exceeds the ${result.limitBytes}-byte image limit.`; + case 'empty-body': return `${path} returned an empty response.`; + case 'timeout': return `${path} timed out.`; + case 'transport-error': return `${path} could not be downloaded.`; } }; diff --git a/packages/gateway/src/data-plane/codex/images_test.ts b/packages/gateway/src/data-plane/codex/images_test.ts index 8f081f012..989835794 100644 --- a/packages/gateway/src/data-plane/codex/images_test.ts +++ b/packages/gateway/src/data-plane/codex/images_test.ts @@ -1,6 +1,6 @@ import { test } from 'vitest'; -import { InMemoryRepo } from '../../repo/memory.ts'; +import type { InMemoryRepo } from '../../repo/memory.ts'; import { copilotModels, requestApp, setupAppTest } from '../../test-helpers.ts'; import { initExternalResourceFetcher, initImageProcessor } from '@floway-dev/platform'; import { assertEquals, assertExists, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; diff --git a/packages/gateway/src/data-plane/codex/routes.ts b/packages/gateway/src/data-plane/codex/routes.ts index f40a491a1..81191e593 100644 --- a/packages/gateway/src/data-plane/codex/routes.ts +++ b/packages/gateway/src/data-plane/codex/routes.ts @@ -48,8 +48,8 @@ import { codexPsPluginsList, codexWhamAgentIdentitiesJwks, } from './chatgpt-backend.ts'; -import { codexModels } from './models.ts'; import { codexImagesEdits } from './images.ts'; +import { codexModels } from './models.ts'; import type { AuthVars } from '../../middleware/auth.ts'; import { responsesHttp } from '../chat/responses/http.ts'; import { responsesWebSocket } from '../chat/responses/websocket.ts'; From 113be1854ebe113d19bcc5a6e3643960e8c42fcd Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 15 Jul 2026 03:35:22 +0800 Subject: [PATCH 05/24] refactor(codex): reuse materialized image buffers Avoid copying complete image buffers in the provider-relative edit adapter and tighten the route contract descriptions. --- packages/gateway/src/data-plane/codex/images.ts | 7 +++++-- packages/gateway/src/data-plane/codex/images_test.ts | 2 +- packages/gateway/src/data-plane/codex/routes.ts | 5 ++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/gateway/src/data-plane/codex/images.ts b/packages/gateway/src/data-plane/codex/images.ts index 731980ea3..411a0283b 100644 --- a/packages/gateway/src/data-plane/codex/images.ts +++ b/packages/gateway/src/data-plane/codex/images.ts @@ -27,8 +27,11 @@ type LoadedImage = | { type: 'ok'; source: ImageEditSource } | { type: 'invalid'; message: string }; -const imageBytes = (data: Uint8Array): ArrayBuffer => - data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength) as ArrayBuffer; +const imageBytes = (data: Uint8Array): ArrayBuffer => data.byteOffset === 0 + && data.buffer instanceof ArrayBuffer + && data.byteLength === data.buffer.byteLength + ? data.buffer + : Uint8Array.from(data).buffer; const decodeDataUrl = (value: string, index: number): LoadedImage | null => { const match = /^data:image\/[a-z0-9.+-]+;base64,(.*)$/isu.exec(value); diff --git a/packages/gateway/src/data-plane/codex/images_test.ts b/packages/gateway/src/data-plane/codex/images_test.ts index 989835794..95c92e255 100644 --- a/packages/gateway/src/data-plane/codex/images_test.ts +++ b/packages/gateway/src/data-plane/codex/images_test.ts @@ -47,7 +47,7 @@ const controlPlaneFetch = (request: Request): Response | undefined => { return undefined; }; -test('Codex provider-relative image generation reuses the public JSON route', async () => { +test('Codex provider-relative image generation reuses the public image-generation handler', async () => { const { apiKey, repo } = await setupAppTest(); await saveAzureImages(repo); let observedUrl: string | undefined; diff --git a/packages/gateway/src/data-plane/codex/routes.ts b/packages/gateway/src/data-plane/codex/routes.ts index 81191e593..2cfeb066d 100644 --- a/packages/gateway/src/data-plane/codex/routes.ts +++ b/packages/gateway/src/data-plane/codex/routes.ts @@ -3,12 +3,11 @@ // OAuth ("ChatGPT") mode has two independently configured bases: // // chatgpt_base_url — analytics, plugins, agent identity, Apps MCP -// model_providers.x.base_url — models, Responses HTTP/WS, remote compaction +// model_providers.x.base_url — models, Responses, compaction, image requests // // The dashboard points both at this namespace. Codex appends `models`, // `responses`, `responses/compact`, `images/generations`, and `images/edits` -// directly to the -// model-provider base: +// directly to the model-provider base: // https://github.com/openai/codex/blob/44918ea10c0f99151c6710411b4322c2f5c96bea/codex-rs/codex-api/src/endpoint/models.rs#L31-L43 // https://github.com/openai/codex/blob/44918ea10c0f99151c6710411b4322c2f5c96bea/codex-rs/codex-api/src/endpoint/responses.rs#L100-L102 // https://github.com/openai/codex/blob/44918ea10c0f99151c6710411b4322c2f5c96bea/codex-rs/codex-api/src/endpoint/compact.rs#L35-L57 From d80a4aa181c516c00b5e134adede3d0cba4c1c82 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 15 Jul 2026 03:38:01 +0800 Subject: [PATCH 06/24] refactor(images): type prepared edit payloads Canonicalize supported MIME aliases during edit preparation and carry only prepared PNG, JPEG, or WebP sources into multipart construction. Model generation and edit calls as distinct requests so raw image sources cannot reach the edit wire builder. --- .../server-tools/image-generation.ts | 79 ++++++++++++------- .../src/data-plane/images/edit-source.ts | 24 ++++-- 2 files changed, 65 insertions(+), 38 deletions(-) diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts index 81c5fa7b0..36f31dda8 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts @@ -1,5 +1,11 @@ import { sleep } from '../../../../../shared/sleep.ts'; -import { editSupportedMime, prepareImageEditSources, supportedImageMimeFromBytes, type ImageEditSource } from '../../../../images/edit-source.ts'; +import { + prepareImageEditSources, + supportedImageMimeFromBytes, + type ImageEditMime, + type ImageEditSource, + type PreparedImageEditSource, +} from '../../../../images/edit-source.ts'; import { enumerateModelCandidates } from '../../../../providers/registry.ts'; import { appendFailedUpstreams } from '../../../../shared/failed-upstreams.ts'; import { recordPerformance, type PerformanceTelemetryContext } from '../../../../shared/telemetry/performance.ts'; @@ -49,7 +55,7 @@ const ALLOWED_MODERATIONS = new Set(['auto', 'low']); const ALLOWED_ACTIONS = new Set(['generate', 'edit', 'auto']); const ALLOWED_INPUT_FIDELITY = new Set(['high', 'low']); -const editFileExt = (mime: string): string => +const editFileExt = (mime: ImageEditMime): string => mime === 'image/jpeg' ? 'jpg' : mime === 'image/webp' ? 'webp' : 'png'; // The public `image_generation` tool-config surface. Azure rejects any other @@ -205,14 +211,14 @@ export interface ImageGenerationConfig { const prepareEditRequest = async ( sources: readonly ImageSource[], config: ImageGenerationConfig, -): Promise<{ sources: readonly ImageSource[]; config: ImageGenerationConfig }> => { +): Promise<{ sources: readonly PreparedImageEditSource[]; mask?: PreparedImageEditSource }> => { if (config.mask !== undefined && isRemoteImageSource(config.mask)) { throw new Error('Remote image mask reached edit dispatch before materialization'); } const originals = [...sources]; if (config.mask !== undefined && !originals.includes(config.mask)) originals.push(config.mask); const prepared = await prepareImageEditSources(originals); - const bySource = new Map(); + const bySource = new Map(); for (const [index, source] of originals.entries()) { const wireSource = prepared[index]; if (wireSource === undefined) throw new Error('Missing prepared image edit source'); @@ -223,10 +229,10 @@ const prepareEditRequest = async ( if (wireSource === undefined) throw new Error('Missing prepared image edit source'); return wireSource; }); - if (config.mask === undefined) return { sources: wireSources, config }; + if (config.mask === undefined) return { sources: wireSources }; const mask = bySource.get(config.mask); if (mask === undefined) throw new Error('Missing prepared image edit mask'); - return { sources: wireSources, config: { ...config, mask } }; + return { sources: wireSources, mask }; }; interface PrepareConfigError { @@ -852,7 +858,13 @@ export const buildGenerationsBody = (prompt: string, config: ImageGenerationConf ...(stream ? { stream: true, partial_images: config.partial_images } : {}), }); -const buildEditsForm = (prompt: string, config: ImageGenerationConfig, sources: readonly ImageSource[], stream: boolean): FormData => { +const buildEditsForm = ( + prompt: string, + config: ImageGenerationConfig, + sources: readonly PreparedImageEditSource[], + mask: PreparedImageEditSource | undefined, + stream: boolean, +): FormData => { const form = new FormData(); form.append('prompt', prompt); form.append('n', '1'); @@ -868,24 +880,25 @@ const buildEditsForm = (prompt: string, config: ImageGenerationConfig, sources: form.append('partial_images', String(config.partial_images)); } for (const [i, source] of sources.entries()) { - // Forward the canonical supported mime; an unsupported source is rejected - // before dispatch, so the fallback only ever carries an already-supported - // generated image. Sending the raw mime (rather than relabeling as png) - // keeps a stray unsupported byte stream failing loud at the backend. - const mime = editSupportedMime(source.mimeType) ?? source.mimeType; // `image[]` repeated parts: gpt-image accepts multiple, resolving "the // Nth image" against attach order (see `inspectImageSources`). - form.append('image[]', new Blob([source.bytes], { type: mime }), `image_${i}.${editFileExt(mime)}`); + form.append('image[]', new Blob([source.bytes], { type: source.mimeType }), `image_${i}.${editFileExt(source.mimeType)}`); } - if (config.mask !== undefined) { - if (isRemoteImageSource(config.mask)) throw new Error('Remote image mask reached form encoding before materialization'); - const mask = config.mask; - const mime = editSupportedMime(mask.mimeType) ?? mask.mimeType; - form.append('mask', new Blob([mask.bytes], { type: mime }), `mask.${editFileExt(mime)}`); + if (mask !== undefined) { + form.append('mask', new Blob([mask.bytes], { type: mask.mimeType }), `mask.${editFileExt(mask.mimeType)}`); } return form; }; +type ImageCallRequest = + | { type: 'generation'; config: ImageGenerationConfig } + | { + type: 'edit'; + config: ImageGenerationConfig; + sources: readonly PreparedImageEditSource[]; + mask?: PreparedImageEditSource; + }; + const serverError = (e: unknown): ImageError => ({ type: 'image_generation_error', message: e instanceof Error ? e.message : String(e), @@ -991,9 +1004,7 @@ const issueImageCall = async ( model: ProviderModel, fetcher: Fetcher, prompt: string, - isEdit: boolean, - sources: readonly ImageSource[], - config: ImageGenerationConfig, + request: ImageCallRequest, state: ShimState, stream: boolean, attempt: AttemptState, @@ -1010,9 +1021,19 @@ const issueImageCall = async ( // retry so it reflects the dispatch that actually returned. wrapUpstreamCall: stampUpstreamCallStart(attempt), }; - const { response, modelKey } = await (isEdit - ? provider.instance.callImagesEdits(model, buildEditsForm(prompt, config, sources, stream), state.downstreamAbortSignal, opts) - : provider.instance.callImagesGenerations(model, buildGenerationsBody(prompt, config, stream), state.downstreamAbortSignal, opts)); + const { response, modelKey } = await (request.type === 'edit' + ? provider.instance.callImagesEdits( + model, + buildEditsForm(prompt, request.config, request.sources, request.mask, stream), + state.downstreamAbortSignal, + opts, + ) + : provider.instance.callImagesGenerations( + model, + buildGenerationsBody(prompt, request.config, stream), + state.downstreamAbortSignal, + opts, + )); if (response.status !== 429 || retry >= MAX_RATE_LIMIT_RETRIES) return { response, modelKey }; // 25% jitter desynchronizes parallel callers so a burst of orchestrator @@ -1178,17 +1199,15 @@ const streamImageGeneration = ( let response: Response; let modelKey: string; try { - const prepared = isEdit - ? await prepareEditRequest(sources, state.config) - : { sources, config: state.config }; + const request: ImageCallRequest = isEdit + ? { type: 'edit', config: state.config, ...await prepareEditRequest(sources, state.config) } + : { type: 'generation', config: state.config }; ({ response, modelKey } = await issueImageCall( provider, model, fetcher, prompt, - isEdit, - prepared.sources, - prepared.config, + request, state, wantsPartials, attempt, diff --git a/packages/gateway/src/data-plane/images/edit-source.ts b/packages/gateway/src/data-plane/images/edit-source.ts index 0b2e0d9b0..0bdbc9a17 100644 --- a/packages/gateway/src/data-plane/images/edit-source.ts +++ b/packages/gateway/src/data-plane/images/edit-source.ts @@ -5,16 +5,23 @@ export interface ImageEditSource { mimeType: string; } -const EDIT_MIME_ALIASES: Record = { +export type ImageEditMime = 'image/png' | 'image/jpeg' | 'image/webp'; + +export interface PreparedImageEditSource extends ImageEditSource { + mimeType: ImageEditMime; +} + +const EDIT_MIME_ALIASES: Record = { 'image/jpg': 'image/jpeg', 'image/pjpeg': 'image/jpeg', 'image/x-png': 'image/png', }; -const EDIT_SUPPORTED_MIMES = new Set(['image/png', 'image/jpeg', 'image/webp']); -export const editSupportedMime = (mime: string): string | null => { +export const editSupportedMime = (mime: string): ImageEditMime | null => { const canonical = EDIT_MIME_ALIASES[mime] ?? mime; - return EDIT_SUPPORTED_MIMES.has(canonical) ? canonical : null; + return canonical === 'image/png' || canonical === 'image/jpeg' || canonical === 'image/webp' + ? canonical + : null; }; export const supportedImageMimeFromBytes = (bytes: Uint8Array): string | null => { @@ -43,11 +50,12 @@ export const supportedImageMimeFromBytes = (bytes: Uint8Array): string | null => return null; }; -export const prepareImageEditSources = async (sources: readonly ImageEditSource[]): Promise => { +export const prepareImageEditSources = async (sources: readonly ImageEditSource[]): Promise => { const keyBySource = new Map>(); - const preparedByContent = new Map>(); + const preparedByContent = new Map>(); return await Promise.all(sources.map(async source => { - if (editSupportedMime(source.mimeType) !== null) return source; + const mimeType = editSupportedMime(source.mimeType); + if (mimeType !== null) return { bytes: source.bytes, mimeType }; let keyPromise = keyBySource.get(source); if (keyPromise === undefined) { @@ -66,7 +74,7 @@ export const prepareImageEditSources = async (sources: readonly ImageEditSource[ // https://github.com/openai/openai-node/blob/ec2f57fd0d66e94782656b986d7b3eb03225369c/src/resources/images.ts#L560-L572 prepared = getImageProcessor().compressToWebp(new Uint8Array(source.bytes), null).then(encoded => { const bytes = encoded.buffer.slice(encoded.byteOffset, encoded.byteOffset + encoded.byteLength) as ArrayBuffer; - return { bytes, mimeType: 'image/webp' }; + return { bytes, mimeType: 'image/webp' } satisfies PreparedImageEditSource; }); preparedByContent.set(key, prepared); } From c912fae75d22bf139be5e79767b9fa0d957c8585 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 15 Jul 2026 03:40:20 +0800 Subject: [PATCH 07/24] refactor(codex): require prepared image files --- packages/gateway/src/data-plane/codex/images.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/gateway/src/data-plane/codex/images.ts b/packages/gateway/src/data-plane/codex/images.ts index 411a0283b..25ab2eaad 100644 --- a/packages/gateway/src/data-plane/codex/images.ts +++ b/packages/gateway/src/data-plane/codex/images.ts @@ -8,6 +8,7 @@ import { prepareImageEditSources, supportedImageMimeFromBytes, type ImageEditSource, + type PreparedImageEditSource, } from '../images/edit-source.ts'; import { serveImagesEditForm } from '../images/serve.ts'; import { passthroughApiError } from '../shared/passthrough-serve.ts'; @@ -87,7 +88,7 @@ const loadImage = async ( : { type: 'ok', source: { bytes: imageBytes(result.data), mimeType } }; }; -const imageFile = (source: ImageEditSource, index: number): File => { +const imageFile = (source: PreparedImageEditSource, index: number): File => { const extension = source.mimeType === 'image/jpeg' ? 'jpg' : source.mimeType.slice('image/'.length); return new File([source.bytes], `image-${index + 1}.${extension}`, { type: source.mimeType }); }; From 7a167bb0011c11f3440178d8d1100503b70348f9 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 15 Jul 2026 03:45:18 +0800 Subject: [PATCH 08/24] test(images): colocate edit source coverage Keep MIME canonicalization and transcoding-deduplication tests with the shared edit-source module, and keep its MIME helper private to that module. --- .../server-tools/image-generation_test.ts | 27 +------------- .../src/data-plane/images/edit-source.ts | 2 +- .../src/data-plane/images/edit-source_test.ts | 37 +++++++++++++++++++ 3 files changed, 39 insertions(+), 27 deletions(-) create mode 100644 packages/gateway/src/data-plane/images/edit-source_test.ts diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation_test.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation_test.ts index 680d14e75..260544ed3 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation_test.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation_test.ts @@ -22,9 +22,8 @@ import { import { initRepo } from '../../../../../repo/index.ts'; import { InMemoryRepo } from '../../../../../repo/memory.ts'; import { mockChatGatewayCtx } from '../../../../../test-helpers/gateway-ctx.ts'; -import { prepareImageEditSources } from '../../../../images/edit-source.ts'; import type { ResponsesInvocation } from '../types.ts'; -import { initExternalResourceFetcher, initImageProcessor } from '@floway-dev/platform'; +import { initExternalResourceFetcher } from '@floway-dev/platform'; import type { CanonicalResponsesPayload, ResponsesInputImage, ResponsesInputItem, ResponsesPayload, ResponsesTool } from '@floway-dev/protocols/responses'; import { assert, assertEquals, assertFalse, assertStringIncludes, assertThrows, stubModelCandidate } from '@floway-dev/test-utils'; @@ -352,30 +351,6 @@ test('inspectImageSources reads tool-result images and preserves forward order', assertEquals(third.mimeType, 'image/webp'); }); -test('prepareImageEditSources transcodes formats accepted by native Responses but rejected by images edits', async () => { - let observedTarget: unknown = undefined; - let processorCalls = 0; - initImageProcessor({ - compressToWebp: (_input, target) => { - processorCalls += 1; - observedTarget = target; - return Promise.resolve(Uint8Array.of(1, 2, 3)); - }, - }); - const first = { - bytes: Uint8Array.of(4, 5, 6).buffer, - mimeType: 'image/gif', - }; - const sameBytes = { bytes: Uint8Array.of(4, 5, 6).buffer, mimeType: 'image/gif' }; - const prepared = await prepareImageEditSources([first, first, sameBytes]); - assertEquals(processorCalls, 1); - assert(prepared[0] === prepared[1]); - assert(prepared[0] === prepared[2]); - assertEquals(prepared[0].mimeType, 'image/webp'); - assertEquals([...new Uint8Array(prepared[0].bytes)], [1, 2, 3]); - assertEquals(observedTarget, null); -}); - test('resolveImageOperation exposes a malformed replayed result as an invariant failure', () => { const inspection = inspectImageSources([ { type: 'image_generation_call', id: 'ig_bad', status: 'completed', result: '%%%' }, diff --git a/packages/gateway/src/data-plane/images/edit-source.ts b/packages/gateway/src/data-plane/images/edit-source.ts index 0bdbc9a17..96b00604f 100644 --- a/packages/gateway/src/data-plane/images/edit-source.ts +++ b/packages/gateway/src/data-plane/images/edit-source.ts @@ -17,7 +17,7 @@ const EDIT_MIME_ALIASES: Record = { 'image/x-png': 'image/png', }; -export const editSupportedMime = (mime: string): ImageEditMime | null => { +const editSupportedMime = (mime: string): ImageEditMime | null => { const canonical = EDIT_MIME_ALIASES[mime] ?? mime; return canonical === 'image/png' || canonical === 'image/jpeg' || canonical === 'image/webp' ? canonical diff --git a/packages/gateway/src/data-plane/images/edit-source_test.ts b/packages/gateway/src/data-plane/images/edit-source_test.ts new file mode 100644 index 000000000..92e9fca4e --- /dev/null +++ b/packages/gateway/src/data-plane/images/edit-source_test.ts @@ -0,0 +1,37 @@ +import { test } from 'vitest'; + +import { prepareImageEditSources } from './edit-source.ts'; +import { initImageProcessor } from '@floway-dev/platform'; +import { assert, assertEquals } from '@floway-dev/test-utils'; + +test('prepareImageEditSources canonicalizes MIME aliases without copying bytes', async () => { + const bytes = Uint8Array.of(1, 2, 3).buffer; + const [prepared] = await prepareImageEditSources([{ bytes, mimeType: 'image/jpg' }]); + assert(prepared !== undefined); + assert(prepared.bytes === bytes); + assertEquals(prepared.mimeType, 'image/jpeg'); +}); + +test('prepareImageEditSources deduplicates transcoding for unsupported raster formats', async () => { + let observedTarget: unknown = undefined; + let processorCalls = 0; + initImageProcessor({ + compressToWebp: (_input, target) => { + processorCalls += 1; + observedTarget = target; + return Promise.resolve(Uint8Array.of(1, 2, 3)); + }, + }); + const first = { + bytes: Uint8Array.of(4, 5, 6).buffer, + mimeType: 'image/gif', + }; + const sameBytes = { bytes: Uint8Array.of(4, 5, 6).buffer, mimeType: 'image/gif' }; + const prepared = await prepareImageEditSources([first, first, sameBytes]); + assertEquals(processorCalls, 1); + assert(prepared[0] === prepared[1]); + assert(prepared[0] === prepared[2]); + assertEquals(prepared[0].mimeType, 'image/webp'); + assertEquals([...new Uint8Array(prepared[0].bytes)], [1, 2, 3]); + assertEquals(observedTarget, null); +}); From f7c23f3d53ef31adc70b2731f50c58f7558f8357 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 15 Jul 2026 11:40:13 +0800 Subject: [PATCH 09/24] refactor(images): unify JSON edit handling Treat the current application/json Images Edits request as a first-class public wire shape alongside multipart uploads. Preserve JSON image_url, file_id, and mask references through provider dispatch, and mount the same generation and edit handlers under the Codex provider base. --- README.md | 6 +- .../image-generation-integration_test.ts | 5 +- .../server-tools/image-generation.ts | 164 ++++++++++++------ .../server-tools/image-generation_test.ts | 27 ++- .../gateway/src/data-plane/codex/images.ts | 155 ----------------- .../src/data-plane/codex/images_test.ts | 58 +------ .../gateway/src/data-plane/codex/routes.ts | 5 +- .../src/data-plane/images/edit-source.ts | 83 --------- .../src/data-plane/images/edit-source_test.ts | 37 ---- .../gateway/src/data-plane/images/serve.ts | 109 +++++++----- .../src/data-plane/images/serve_test.ts | 77 +++++++- packages/protocols/src/images/index.ts | 30 +++- packages/provider-azure/src/provider.ts | 3 + packages/provider-custom/src/provider.ts | 3 + packages/provider/src/index.ts | 1 + packages/provider/src/provider.ts | 10 +- 16 files changed, 331 insertions(+), 442 deletions(-) delete mode 100644 packages/gateway/src/data-plane/codex/images.ts delete mode 100644 packages/gateway/src/data-plane/images/edit-source.ts delete mode 100644 packages/gateway/src/data-plane/images/edit-source_test.ts diff --git a/README.md b/README.md index 18d38c5f5..d521f60a9 100644 --- a/README.md +++ b/README.md @@ -24,9 +24,9 @@ target ships in the same repo for self-hosting on a long-lived process. | OpenAI Models | `GET /v1/models` | | Google Gemini (generate / count tokens) | `POST /v1beta/models/...` | -The dashboard's Codex configuration uses `/azure-api.codex` as its provider -base. That namespace also exposes provider-relative `images/generations` and -JSON `images/edits` routes for Codex's client-owned image extension. +`POST /v1/images/edits` accepts multipart image uploads and JSON `images` +references. The dashboard's Codex provider base, `/azure-api.codex`, exposes +the same generation and edit handlers at their provider-relative paths. For each public model, Floway picks the first (provider, model) pair that can serve the request, translating between source and target protocols when the diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation-integration_test.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation-integration_test.ts index 8465c3544..6c87e3778 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation-integration_test.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation-integration_test.ts @@ -52,8 +52,9 @@ const defaultCandidates = vi.hoisted(() => () => [{ if (response === undefined) throw new Error('test did not enqueue a generations response'); return { response, modelKey: 'gpt-image-2' }; }, - callImagesEdits: async (_model: unknown, form: FormData) => { - stub.editsForms.push(form); + callImagesEdits: async (_model: unknown, body: FormData | Record) => { + if (!(body instanceof FormData)) throw new Error('hosted image edit must use multipart'); + stub.editsForms.push(body); const response = stub.nextEdits.shift(); if (response === undefined) throw new Error('test did not enqueue an edits response'); return { response, modelKey: 'gpt-image-2' }; diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts index 36f31dda8..f02eac566 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts @@ -1,11 +1,4 @@ import { sleep } from '../../../../../shared/sleep.ts'; -import { - prepareImageEditSources, - supportedImageMimeFromBytes, - type ImageEditMime, - type ImageEditSource, - type PreparedImageEditSource, -} from '../../../../images/edit-source.ts'; import { enumerateModelCandidates } from '../../../../providers/registry.ts'; import { appendFailedUpstreams } from '../../../../shared/failed-upstreams.ts'; import { recordPerformance, type PerformanceTelemetryContext } from '../../../../shared/telemetry/performance.ts'; @@ -13,7 +6,7 @@ import { recordTokenUsage, tokenUsageFromImagesBody } from '../../../../shared/t import { createExternalImageFetcher, type ExternalImageFetchResult } from '../../../shared/external-image-loader.ts'; import { stampUpstreamCallStart, type AttemptState } from '../../../shared/gateway-ctx.ts'; import type { ServerToolLifecycleEvent, ServerToolOutputItem, ServerToolRegistration, ServerToolTerminal } from '../server-tool-shim.ts'; -import type { BackgroundScheduler } from '@floway-dev/platform'; +import { dimensionsFromBytes, getImageProcessor, type BackgroundScheduler } from '@floway-dev/platform'; import { parseSSEStream } from '@floway-dev/protocols/common'; import type { ResponsesFunctionCallOutputItem, @@ -55,7 +48,26 @@ const ALLOWED_MODERATIONS = new Set(['auto', 'low']); const ALLOWED_ACTIONS = new Set(['generate', 'edit', 'auto']); const ALLOWED_INPUT_FIDELITY = new Set(['high', 'low']); -const editFileExt = (mime: ImageEditMime): string => +// gpt-image-* `/images/edits` accepts only these input image mimetypes; a live +// Azure probe confirmed png/jpeg/webp succeed while gif is rejected with +// `unsupported_file_mimetype`. Native Responses accepts the same GIF and +// re-encodes it before editing, so the shim mirrors that behavior through the +// platform image processor. Common aliases are folded onto the backend form. +const EDIT_MIME_ALIASES: Record = { + 'image/jpg': 'image/jpeg', + 'image/pjpeg': 'image/jpeg', + 'image/x-png': 'image/png', +}; +const EDIT_SUPPORTED_MIMES = new Set(['image/png', 'image/jpeg', 'image/webp']); + +// The canonical edit-supported mimetype for a source, or null when the +// standalone endpoint requires local WebP transcoding first. +const editSupportedMime = (mime: string): string | null => { + const canonical = EDIT_MIME_ALIASES[mime] ?? mime; + return EDIT_SUPPORTED_MIMES.has(canonical) ? canonical : null; +}; + +const editFileExt = (mime: string): string => mime === 'image/jpeg' ? 'jpg' : mime === 'image/webp' ? 'webp' : 'png'; // The public `image_generation` tool-config surface. Azure rejects any other @@ -83,7 +95,10 @@ export const canonicalizeImageGenerationTool = (raw: ResponsesTool): ResponsesHo // A base64-data-URL or bare-base64 image source bound for an edit call. // Bytes are held in a concrete ArrayBuffer so they can be wrapped in a Blob. -type ImageSource = ImageEditSource; +interface ImageSource { + bytes: ArrayBuffer; + mimeType: string; +} interface RemoteImageSource { wireUrl: string; @@ -96,6 +111,38 @@ type ImageSourceReference = ImageSource | RemoteImageSource; const isRemoteImageSource = (source: ImageSourceReference): source is RemoteImageSource => 'wireUrl' in source; +export const prepareEditSources = async (sources: readonly ImageSource[]): Promise => { + const keyBySource = new Map>(); + const preparedByContent = new Map>(); + return await Promise.all(sources.map(async source => { + if (editSupportedMime(source.mimeType) !== null) return source; + + let keyPromise = keyBySource.get(source); + if (keyPromise === undefined) { + keyPromise = crypto.subtle.digest('SHA-256', source.bytes).then(buffer => { + const digest = [...new Uint8Array(buffer)].map(byte => byte.toString(16).padStart(2, '0')).join(''); + return `${source.mimeType}\u0000${digest}`; + }); + keyBySource.set(source, keyPromise); + } + const key = await keyPromise; + + let prepared = preparedByContent.get(key); + if (prepared === undefined) { + // Native Responses accepts formats such as GIF through its multimodal + // preprocessing, while the standalone edits endpoint accepts only + // PNG/JPEG/WebP. Re-encode locally to preserve the hosted-tool behavior. + // https://github.com/openai/openai-node/blob/ec2f57fd0d66e94782656b986d7b3eb03225369c/src/resources/images.ts#L560-L572 + prepared = getImageProcessor().compressToWebp(new Uint8Array(source.bytes), null).then(encoded => { + const bytes = encoded.buffer.slice(encoded.byteOffset, encoded.byteOffset + encoded.byteLength) as ArrayBuffer; + return { bytes, mimeType: 'image/webp' }; + }); + preparedByContent.set(key, prepared); + } + return await prepared; + })); +}; + const base64ToArrayBuffer = (b64: string): ArrayBuffer => { const binary = atob(b64); const buffer = new ArrayBuffer(binary.length); @@ -211,14 +258,14 @@ export interface ImageGenerationConfig { const prepareEditRequest = async ( sources: readonly ImageSource[], config: ImageGenerationConfig, -): Promise<{ sources: readonly PreparedImageEditSource[]; mask?: PreparedImageEditSource }> => { +): Promise<{ sources: readonly ImageSource[]; config: ImageGenerationConfig }> => { if (config.mask !== undefined && isRemoteImageSource(config.mask)) { throw new Error('Remote image mask reached edit dispatch before materialization'); } const originals = [...sources]; if (config.mask !== undefined && !originals.includes(config.mask)) originals.push(config.mask); - const prepared = await prepareImageEditSources(originals); - const bySource = new Map(); + const prepared = await prepareEditSources(originals); + const bySource = new Map(); for (const [index, source] of originals.entries()) { const wireSource = prepared[index]; if (wireSource === undefined) throw new Error('Missing prepared image edit source'); @@ -229,10 +276,10 @@ const prepareEditRequest = async ( if (wireSource === undefined) throw new Error('Missing prepared image edit source'); return wireSource; }); - if (config.mask === undefined) return { sources: wireSources }; + if (config.mask === undefined) return { sources: wireSources, config }; const mask = bySource.get(config.mask); if (mask === undefined) throw new Error('Missing prepared image edit mask'); - return { sources: wireSources, mask }; + return { sources: wireSources, config: { ...config, mask } }; }; interface PrepareConfigError { @@ -525,6 +572,32 @@ const remoteInputError = (source: RemoteImageSource, failure: RemoteImageFailure // https://platform.openai.com/docs/guides/images-vision#image-input-requirements const MAX_REMOTE_IMAGE_TOTAL_BYTES = 50 * 1024 * 1024; +const supportedImageMimeFromBytes = (bytes: Uint8Array): string | null => { + if (dimensionsFromBytes(bytes) === null) return null; + if ( + bytes.length >= 8 + && bytes[0] === 0x89 + && bytes[1] === 0x50 + && bytes[2] === 0x4e + && bytes[3] === 0x47 + && bytes[4] === 0x0d + && bytes[5] === 0x0a + && bytes[6] === 0x1a + && bytes[7] === 0x0a + ) return 'image/png'; + if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) return 'image/jpeg'; + if (bytes.length >= 6) { + const signature = String.fromCharCode(...bytes.subarray(0, 6)); + if (signature === 'GIF87a' || signature === 'GIF89a') return 'image/gif'; + } + if ( + bytes.length >= 12 + && String.fromCharCode(...bytes.subarray(0, 4)) === 'RIFF' + && String.fromCharCode(...bytes.subarray(8, 12)) === 'WEBP' + ) return 'image/webp'; + return null; +}; + const createRemoteImageMaterializer = (requestSignal: AbortSignal | undefined) => { const fetchImage = createExternalImageFetcher(requestSignal); const materialized = new Map(); @@ -858,13 +931,7 @@ export const buildGenerationsBody = (prompt: string, config: ImageGenerationConf ...(stream ? { stream: true, partial_images: config.partial_images } : {}), }); -const buildEditsForm = ( - prompt: string, - config: ImageGenerationConfig, - sources: readonly PreparedImageEditSource[], - mask: PreparedImageEditSource | undefined, - stream: boolean, -): FormData => { +const buildEditsForm = (prompt: string, config: ImageGenerationConfig, sources: readonly ImageSource[], stream: boolean): FormData => { const form = new FormData(); form.append('prompt', prompt); form.append('n', '1'); @@ -880,25 +947,24 @@ const buildEditsForm = ( form.append('partial_images', String(config.partial_images)); } for (const [i, source] of sources.entries()) { + // Forward the canonical supported mime; an unsupported source is rejected + // before dispatch, so the fallback only ever carries an already-supported + // generated image. Sending the raw mime (rather than relabeling as png) + // keeps a stray unsupported byte stream failing loud at the backend. + const mime = editSupportedMime(source.mimeType) ?? source.mimeType; // `image[]` repeated parts: gpt-image accepts multiple, resolving "the // Nth image" against attach order (see `inspectImageSources`). - form.append('image[]', new Blob([source.bytes], { type: source.mimeType }), `image_${i}.${editFileExt(source.mimeType)}`); + form.append('image[]', new Blob([source.bytes], { type: mime }), `image_${i}.${editFileExt(mime)}`); } - if (mask !== undefined) { - form.append('mask', new Blob([mask.bytes], { type: mask.mimeType }), `mask.${editFileExt(mask.mimeType)}`); + if (config.mask !== undefined) { + if (isRemoteImageSource(config.mask)) throw new Error('Remote image mask reached form encoding before materialization'); + const mask = config.mask; + const mime = editSupportedMime(mask.mimeType) ?? mask.mimeType; + form.append('mask', new Blob([mask.bytes], { type: mime }), `mask.${editFileExt(mime)}`); } return form; }; -type ImageCallRequest = - | { type: 'generation'; config: ImageGenerationConfig } - | { - type: 'edit'; - config: ImageGenerationConfig; - sources: readonly PreparedImageEditSource[]; - mask?: PreparedImageEditSource; - }; - const serverError = (e: unknown): ImageError => ({ type: 'image_generation_error', message: e instanceof Error ? e.message : String(e), @@ -1004,7 +1070,9 @@ const issueImageCall = async ( model: ProviderModel, fetcher: Fetcher, prompt: string, - request: ImageCallRequest, + isEdit: boolean, + sources: readonly ImageSource[], + config: ImageGenerationConfig, state: ShimState, stream: boolean, attempt: AttemptState, @@ -1021,19 +1089,9 @@ const issueImageCall = async ( // retry so it reflects the dispatch that actually returned. wrapUpstreamCall: stampUpstreamCallStart(attempt), }; - const { response, modelKey } = await (request.type === 'edit' - ? provider.instance.callImagesEdits( - model, - buildEditsForm(prompt, request.config, request.sources, request.mask, stream), - state.downstreamAbortSignal, - opts, - ) - : provider.instance.callImagesGenerations( - model, - buildGenerationsBody(prompt, request.config, stream), - state.downstreamAbortSignal, - opts, - )); + const { response, modelKey } = await (isEdit + ? provider.instance.callImagesEdits(model, buildEditsForm(prompt, config, sources, stream), state.downstreamAbortSignal, opts) + : provider.instance.callImagesGenerations(model, buildGenerationsBody(prompt, config, stream), state.downstreamAbortSignal, opts)); if (response.status !== 429 || retry >= MAX_RATE_LIMIT_RETRIES) return { response, modelKey }; // 25% jitter desynchronizes parallel callers so a burst of orchestrator @@ -1199,15 +1257,17 @@ const streamImageGeneration = ( let response: Response; let modelKey: string; try { - const request: ImageCallRequest = isEdit - ? { type: 'edit', config: state.config, ...await prepareEditRequest(sources, state.config) } - : { type: 'generation', config: state.config }; + const prepared = isEdit + ? await prepareEditRequest(sources, state.config) + : { sources, config: state.config }; ({ response, modelKey } = await issueImageCall( provider, model, fetcher, prompt, - request, + isEdit, + prepared.sources, + prepared.config, state, wantsPartials, attempt, diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation_test.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation_test.ts index 260544ed3..6bf80b562 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation_test.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation_test.ts @@ -13,6 +13,7 @@ import { isHostedImageGenerationTool, parseImageStreamEvent, parseRetryAfterMs, + prepareEditSources, prepareImageGenerationConfig, resolveImageOperation, SHIM_TOOL_NAME, @@ -23,7 +24,7 @@ import { initRepo } from '../../../../../repo/index.ts'; import { InMemoryRepo } from '../../../../../repo/memory.ts'; import { mockChatGatewayCtx } from '../../../../../test-helpers/gateway-ctx.ts'; import type { ResponsesInvocation } from '../types.ts'; -import { initExternalResourceFetcher } from '@floway-dev/platform'; +import { initExternalResourceFetcher, initImageProcessor } from '@floway-dev/platform'; import type { CanonicalResponsesPayload, ResponsesInputImage, ResponsesInputItem, ResponsesPayload, ResponsesTool } from '@floway-dev/protocols/responses'; import { assert, assertEquals, assertFalse, assertStringIncludes, assertThrows, stubModelCandidate } from '@floway-dev/test-utils'; @@ -351,6 +352,30 @@ test('inspectImageSources reads tool-result images and preserves forward order', assertEquals(third.mimeType, 'image/webp'); }); +test('prepareEditSources transcodes formats accepted by native Responses but rejected by images edits', async () => { + let observedTarget: unknown = undefined; + let processorCalls = 0; + initImageProcessor({ + compressToWebp: (_input, target) => { + processorCalls += 1; + observedTarget = target; + return Promise.resolve(Uint8Array.of(1, 2, 3)); + }, + }); + const first = { + bytes: Uint8Array.of(4, 5, 6).buffer, + mimeType: 'image/gif', + }; + const sameBytes = { bytes: Uint8Array.of(4, 5, 6).buffer, mimeType: 'image/gif' }; + const prepared = await prepareEditSources([first, first, sameBytes]); + assertEquals(processorCalls, 1); + assert(prepared[0] === prepared[1]); + assert(prepared[0] === prepared[2]); + assertEquals(prepared[0].mimeType, 'image/webp'); + assertEquals([...new Uint8Array(prepared[0].bytes)], [1, 2, 3]); + assertEquals(observedTarget, null); +}); + test('resolveImageOperation exposes a malformed replayed result as an invariant failure', () => { const inspection = inspectImageSources([ { type: 'image_generation_call', id: 'ig_bad', status: 'completed', result: '%%%' }, diff --git a/packages/gateway/src/data-plane/codex/images.ts b/packages/gateway/src/data-plane/codex/images.ts deleted file mode 100644 index 25ab2eaad..000000000 --- a/packages/gateway/src/data-plane/codex/images.ts +++ /dev/null @@ -1,155 +0,0 @@ -import type { Context } from 'hono'; - -import { backgroundSchedulerFromContext } from '../../runtime/background.ts'; -import { createExternalImageFetcher, type ExternalImageFetchResult } from '../chat/shared/external-image-loader.ts'; -import { createGatewayCtxFromHono, finalizeGatewayResponse } from '../chat/shared/gateway-ctx.ts'; -import { readRequestBody, takeRequestBody } from '../chat/shared/request-body.ts'; -import { - prepareImageEditSources, - supportedImageMimeFromBytes, - type ImageEditSource, - type PreparedImageEditSource, -} from '../images/edit-source.ts'; -import { serveImagesEditForm } from '../images/serve.ts'; -import { passthroughApiError } from '../shared/passthrough-serve.ts'; - -interface CodexImageEditRequest { - images?: unknown; - model?: unknown; - prompt?: unknown; - [key: string]: unknown; -} - -type PreparedCodexEdit = - | { type: 'ok'; form: FormData } - | { type: 'invalid'; message: string }; - -type LoadedImage = - | { type: 'ok'; source: ImageEditSource } - | { type: 'invalid'; message: string }; - -const imageBytes = (data: Uint8Array): ArrayBuffer => data.byteOffset === 0 - && data.buffer instanceof ArrayBuffer - && data.byteLength === data.buffer.byteLength - ? data.buffer - : Uint8Array.from(data).buffer; - -const decodeDataUrl = (value: string, index: number): LoadedImage | null => { - const match = /^data:image\/[a-z0-9.+-]+;base64,(.*)$/isu.exec(value); - if (match === null) return null; - - let bytes: Uint8Array; - try { - const decoded = atob(match[1]!); - bytes = Uint8Array.from(decoded, character => character.charCodeAt(0)); - } catch { - return { type: 'invalid', message: `Codex image edits images[${index}].image_url must contain valid base64 image data.` }; - } - const mimeType = supportedImageMimeFromBytes(bytes); - return mimeType === null - ? { type: 'invalid', message: `Codex image edits images[${index}].image_url must contain a supported raster image.` } - : { type: 'ok', source: { bytes: imageBytes(bytes), mimeType } }; -}; - -const externalImageError = (result: Exclude, index: number): string => { - const path = `Codex image edits images[${index}].image_url`; - switch (result.type) { - case 'invalid-url': return `${path} must be an HTTP(S) URL or a base64 image data URL.`; - case 'invalid-redirect': return `${path} has an invalid redirect (${result.reason}).`; - case 'http-error': return `${path} returned HTTP ${result.status}.`; - case 'too-large': return `${path} exceeds the ${result.limitBytes}-byte image limit.`; - case 'empty-body': return `${path} returned an empty response.`; - case 'timeout': return `${path} timed out.`; - case 'transport-error': return `${path} could not be downloaded.`; - } -}; - -const loadImage = async ( - value: unknown, - index: number, - fetchExternalImage: ReturnType, -): Promise => { - if (!value || typeof value !== 'object' || Array.isArray(value)) { - return { type: 'invalid', message: `Codex image edits images[${index}] must be an object.` }; - } - const imageUrl = (value as { image_url?: unknown }).image_url; - if (typeof imageUrl !== 'string') { - return { type: 'invalid', message: `Codex image edits images[${index}].image_url must be a string.` }; - } - - const inline = decodeDataUrl(imageUrl, index); - if (inline !== null) return inline; - - const result = await fetchExternalImage(imageUrl); - if (result.type !== 'success') return { type: 'invalid', message: externalImageError(result, index) }; - const mimeType = supportedImageMimeFromBytes(result.data); - return mimeType === null - ? { type: 'invalid', message: `Codex image edits images[${index}].image_url must resolve to a supported raster image.` } - : { type: 'ok', source: { bytes: imageBytes(result.data), mimeType } }; -}; - -const imageFile = (source: PreparedImageEditSource, index: number): File => { - const extension = source.mimeType === 'image/jpeg' ? 'jpg' : source.mimeType.slice('image/'.length); - return new File([source.bytes], `image-${index + 1}.${extension}`, { type: source.mimeType }); -}; - -const prepareCodexImageEdit = async (bytes: Uint8Array, signal: AbortSignal): Promise => { - let request: CodexImageEditRequest; - try { - const parsed = JSON.parse(new TextDecoder().decode(bytes)) as unknown; - if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { - return { type: 'invalid', message: 'Codex image edits request body must be an object.' }; - } - request = parsed as CodexImageEditRequest; - } catch { - return { type: 'invalid', message: 'Codex image edits request body must be valid JSON.' }; - } - if (typeof request.model !== 'string' || request.model.length === 0) { - return { type: 'invalid', message: 'Codex image edits request body must include a model string.' }; - } - if (typeof request.prompt !== 'string') { - return { type: 'invalid', message: 'Codex image edits request body must include a prompt string.' }; - } - if (!Array.isArray(request.images) || request.images.length === 0) { - return { type: 'invalid', message: 'Codex image edits request body must include at least one image.' }; - } - - const form = new FormData(); - for (const [name, value] of Object.entries(request)) { - if (name === 'images') continue; - if (typeof value !== 'string' && typeof value !== 'number' && typeof value !== 'boolean') { - return { type: 'invalid', message: `Codex image edits ${name} must be a string, number, or boolean.` }; - } - form.append(name, String(value)); - } - - const fetchExternalImage = createExternalImageFetcher(signal); - const sources: ImageEditSource[] = []; - for (const [index, image] of request.images.entries()) { - const loaded = await loadImage(image, index, fetchExternalImage); - if (loaded.type === 'invalid') return loaded; - sources.push(loaded.source); - } - const prepared = await prepareImageEditSources(sources); - prepared.forEach((source, index) => form.append('image[]', imageFile(source, index))); - return { type: 'ok', form }; -}; - -// Codex's client-owned image extension posts JSON to provider-relative paths; -// the public OpenAI edits route remains multipart/form-data. -// https://github.com/openai/codex/blob/f90e7deea6a715bbd153044af6f475eefa749177/codex-rs/codex-api/src/endpoint/images.rs#L33-L68 -// https://github.com/openai/codex/blob/f90e7deea6a715bbd153044af6f475eefa749177/codex-rs/codex-api/src/images.rs#L4-L31 -export const codexImagesEdits = async (c: Context): Promise => { - const requestBody = await readRequestBody(c); - const prepared = await prepareCodexImageEdit(requestBody.bytes, c.req.raw.signal); - if (prepared.type === 'invalid') { - const ctx = createGatewayCtxFromHono(c, { - wantsStream: false, - requestBody: takeRequestBody(requestBody), - backgroundScheduler: backgroundSchedulerFromContext(c), - }); - ctx.dump?.error('gateway'); - return finalizeGatewayResponse(ctx, passthroughApiError(c, prepared.message, 400)); - } - return await serveImagesEditForm(c, requestBody, prepared.form); -}; diff --git a/packages/gateway/src/data-plane/codex/images_test.ts b/packages/gateway/src/data-plane/codex/images_test.ts index 95c92e255..95e0285ea 100644 --- a/packages/gateway/src/data-plane/codex/images_test.ts +++ b/packages/gateway/src/data-plane/codex/images_test.ts @@ -2,13 +2,8 @@ import { test } from 'vitest'; import type { InMemoryRepo } from '../../repo/memory.ts'; import { copilotModels, requestApp, setupAppTest } from '../../test-helpers.ts'; -import { initExternalResourceFetcher, initImageProcessor } from '@floway-dev/platform'; import { assertEquals, assertExists, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; -const PNG_B64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M8AAAMBAQDJ/wEAAAAASUVORK5CYII='; -const GIF_B64 = 'R0lGODlhAQABAIAAAAAAAP///ywAAAAAAQABAAACAUwAOw=='; -const bytesFromBase64 = (value: string): Uint8Array => Uint8Array.from(atob(value), character => character.charCodeAt(0)); - const saveAzureImages = async (repo: InMemoryRepo): Promise => { await repo.upstreams.save({ id: 'az-image', @@ -82,25 +77,11 @@ test('Codex provider-relative image generation reuses the public image-generatio assertEquals(observedBody.quality, 'high'); }); -test('Codex provider-relative image edits materialize remote images and transcode unsupported raster formats', async () => { +test('Codex provider-relative image edits reuse the public JSON handler', async () => { const { apiKey, repo } = await setupAppTest(); await saveAzureImages(repo); let observedUrl: string | undefined; - let observedForm: FormData | undefined; - let observedImageSourceUrl: string | undefined; - const webpBytes = Uint8Array.of(0x52, 0x49, 0x46, 0x46, 0, 0, 0, 0, 0x57, 0x45, 0x42, 0x50); - initExternalResourceFetcher(url => { - observedImageSourceUrl = url.href; - return Promise.resolve(new Response(bytesFromBase64(PNG_B64).buffer as ArrayBuffer, { - headers: { 'content-type': 'image/png' }, - })); - }); - initImageProcessor({ - compressToWebp: input => { - assertEquals([...input.subarray(0, 6)], [...new TextEncoder().encode('GIF89a')]); - return Promise.resolve(webpBytes); - }, - }); + let observedBody: Record | undefined; await withMockedFetch( async request => { @@ -109,7 +90,7 @@ test('Codex provider-relative image edits materialize remote images and transcod const url = new URL(request.url); if (url.hostname === 'example.openai.azure.com') { observedUrl = request.url; - observedForm = await request.formData(); + observedBody = await request.json() as Record; return jsonResponse({ data: [{ b64_json: 'ZWRpdA==' }] }); } throw new Error(`Unhandled fetch ${request.url}`); @@ -124,7 +105,6 @@ test('Codex provider-relative image edits materialize remote images and transcod quality: 'high', images: [ { image_url: 'https://assets.example/image.png' }, - { image_url: `data:image/gif;base64,${GIF_B64}` }, ], }), }); @@ -133,32 +113,12 @@ test('Codex provider-relative image edits materialize remote images and transcod }, ); - assertEquals(observedImageSourceUrl, 'https://assets.example/image.png'); assertEquals(observedUrl?.endsWith('/images/edits?api-version=preview'), true); - assertExists(observedForm); - assertEquals(observedForm.get('model'), 'gpt-image-2'); - assertEquals(observedForm.get('prompt'), 'add a red hat'); - assertEquals(observedForm.get('quality'), 'high'); - const images = observedForm.getAll('image[]') as File[]; - assertEquals(images.map(image => image.type), ['image/png', 'image/webp']); - assertEquals( - await Promise.all(images.map(image => image.arrayBuffer().then(buffer => [...new Uint8Array(buffer)]))), - [[...bytesFromBase64(PNG_B64)], [...webpBytes]], - ); -}); - -test.each([ - ['not-json', 'Codex image edits request body must be valid JSON.'], - [JSON.stringify({ model: 'gpt-image-2', prompt: 'edit', images: [{ image_url: 'ftp://example.com/image.png' }] }), 'Codex image edits images[0].image_url must be an HTTP(S) URL or a base64 image data URL.'], - [JSON.stringify({ model: 'gpt-image-2', prompt: '', images: [{ image_url: 'data:image/png;base64,AQID' }] }), 'Codex image edits images[0].image_url must contain a supported raster image.'], - [JSON.stringify({ model: 'gpt-image-2', prompt: 'edit', images: [] }), 'Codex image edits request body must include at least one image.'], -])('Codex image edits reject malformed input %#', async (body, message) => { - const { apiKey } = await setupAppTest(); - const response = await requestApp('/azure-api.codex/images/edits', { - method: 'POST', - headers: { authorization: `Bearer ${apiKey.key}`, 'content-type': 'application/json' }, - body, + assertExists(observedBody); + assertEquals(observedBody, { + model: 'gpt-image-2', + prompt: 'add a red hat', + quality: 'high', + images: [{ image_url: 'https://assets.example/image.png' }], }); - assertEquals(response.status, 400); - assertEquals(await response.json(), { error: { message, type: 'api_error' } }); }); diff --git a/packages/gateway/src/data-plane/codex/routes.ts b/packages/gateway/src/data-plane/codex/routes.ts index 2cfeb066d..67fdb9786 100644 --- a/packages/gateway/src/data-plane/codex/routes.ts +++ b/packages/gateway/src/data-plane/codex/routes.ts @@ -47,12 +47,11 @@ import { codexPsPluginsList, codexWhamAgentIdentitiesJwks, } from './chatgpt-backend.ts'; -import { codexImagesEdits } from './images.ts'; import { codexModels } from './models.ts'; import type { AuthVars } from '../../middleware/auth.ts'; import { responsesHttp } from '../chat/responses/http.ts'; import { responsesWebSocket } from '../chat/responses/websocket.ts'; -import { imagesGenerations } from '../images/serve.ts'; +import { imagesEdits, imagesGenerations } from '../images/serve.ts'; const CODEX_BASE_PATH = '/azure-api.codex'; @@ -61,7 +60,7 @@ export const mountCodexRoutes = (app: Hono<{ Variables: AuthVars }>) => { app.post(`${CODEX_BASE_PATH}/responses/compact`, responsesHttp.compact); app.get(`${CODEX_BASE_PATH}/responses`, responsesWebSocket); app.post(`${CODEX_BASE_PATH}/images/generations`, imagesGenerations); - app.post(`${CODEX_BASE_PATH}/images/edits`, codexImagesEdits); + app.post(`${CODEX_BASE_PATH}/images/edits`, imagesEdits); app.get(`${CODEX_BASE_PATH}/models`, codexModels); app.post(`${CODEX_BASE_PATH}/codex/analytics-events/events`, codexAnalyticsEventsEvents); diff --git a/packages/gateway/src/data-plane/images/edit-source.ts b/packages/gateway/src/data-plane/images/edit-source.ts deleted file mode 100644 index 96b00604f..000000000 --- a/packages/gateway/src/data-plane/images/edit-source.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { dimensionsFromBytes, getImageProcessor } from '@floway-dev/platform'; - -export interface ImageEditSource { - bytes: ArrayBuffer; - mimeType: string; -} - -export type ImageEditMime = 'image/png' | 'image/jpeg' | 'image/webp'; - -export interface PreparedImageEditSource extends ImageEditSource { - mimeType: ImageEditMime; -} - -const EDIT_MIME_ALIASES: Record = { - 'image/jpg': 'image/jpeg', - 'image/pjpeg': 'image/jpeg', - 'image/x-png': 'image/png', -}; - -const editSupportedMime = (mime: string): ImageEditMime | null => { - const canonical = EDIT_MIME_ALIASES[mime] ?? mime; - return canonical === 'image/png' || canonical === 'image/jpeg' || canonical === 'image/webp' - ? canonical - : null; -}; - -export const supportedImageMimeFromBytes = (bytes: Uint8Array): string | null => { - if (dimensionsFromBytes(bytes) === null) return null; - if ( - bytes.length >= 8 - && bytes[0] === 0x89 - && bytes[1] === 0x50 - && bytes[2] === 0x4e - && bytes[3] === 0x47 - && bytes[4] === 0x0d - && bytes[5] === 0x0a - && bytes[6] === 0x1a - && bytes[7] === 0x0a - ) return 'image/png'; - if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) return 'image/jpeg'; - if (bytes.length >= 6) { - const signature = String.fromCharCode(...bytes.subarray(0, 6)); - if (signature === 'GIF87a' || signature === 'GIF89a') return 'image/gif'; - } - if ( - bytes.length >= 12 - && String.fromCharCode(...bytes.subarray(0, 4)) === 'RIFF' - && String.fromCharCode(...bytes.subarray(8, 12)) === 'WEBP' - ) return 'image/webp'; - return null; -}; - -export const prepareImageEditSources = async (sources: readonly ImageEditSource[]): Promise => { - const keyBySource = new Map>(); - const preparedByContent = new Map>(); - return await Promise.all(sources.map(async source => { - const mimeType = editSupportedMime(source.mimeType); - if (mimeType !== null) return { bytes: source.bytes, mimeType }; - - let keyPromise = keyBySource.get(source); - if (keyPromise === undefined) { - keyPromise = crypto.subtle.digest('SHA-256', source.bytes).then(buffer => { - const digest = [...new Uint8Array(buffer)].map(byte => byte.toString(16).padStart(2, '0')).join(''); - return `${source.mimeType}\u0000${digest}`; - }); - keyBySource.set(source, keyPromise); - } - const key = await keyPromise; - - let prepared = preparedByContent.get(key); - if (prepared === undefined) { - // The standalone edits endpoint accepts PNG/JPEG/WebP, so encode other - // raster formats before dispatch. - // https://github.com/openai/openai-node/blob/ec2f57fd0d66e94782656b986d7b3eb03225369c/src/resources/images.ts#L560-L572 - prepared = getImageProcessor().compressToWebp(new Uint8Array(source.bytes), null).then(encoded => { - const bytes = encoded.buffer.slice(encoded.byteOffset, encoded.byteOffset + encoded.byteLength) as ArrayBuffer; - return { bytes, mimeType: 'image/webp' } satisfies PreparedImageEditSource; - }); - preparedByContent.set(key, prepared); - } - return await prepared; - })); -}; diff --git a/packages/gateway/src/data-plane/images/edit-source_test.ts b/packages/gateway/src/data-plane/images/edit-source_test.ts deleted file mode 100644 index 92e9fca4e..000000000 --- a/packages/gateway/src/data-plane/images/edit-source_test.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { test } from 'vitest'; - -import { prepareImageEditSources } from './edit-source.ts'; -import { initImageProcessor } from '@floway-dev/platform'; -import { assert, assertEquals } from '@floway-dev/test-utils'; - -test('prepareImageEditSources canonicalizes MIME aliases without copying bytes', async () => { - const bytes = Uint8Array.of(1, 2, 3).buffer; - const [prepared] = await prepareImageEditSources([{ bytes, mimeType: 'image/jpg' }]); - assert(prepared !== undefined); - assert(prepared.bytes === bytes); - assertEquals(prepared.mimeType, 'image/jpeg'); -}); - -test('prepareImageEditSources deduplicates transcoding for unsupported raster formats', async () => { - let observedTarget: unknown = undefined; - let processorCalls = 0; - initImageProcessor({ - compressToWebp: (_input, target) => { - processorCalls += 1; - observedTarget = target; - return Promise.resolve(Uint8Array.of(1, 2, 3)); - }, - }); - const first = { - bytes: Uint8Array.of(4, 5, 6).buffer, - mimeType: 'image/gif', - }; - const sameBytes = { bytes: Uint8Array.of(4, 5, 6).buffer, mimeType: 'image/gif' }; - const prepared = await prepareImageEditSources([first, first, sameBytes]); - assertEquals(processorCalls, 1); - assert(prepared[0] === prepared[1]); - assert(prepared[0] === prepared[2]); - assertEquals(prepared[0].mimeType, 'image/webp'); - assertEquals([...new Uint8Array(prepared[0].bytes)], [1, 2, 3]); - assertEquals(observedTarget, null); -}); diff --git a/packages/gateway/src/data-plane/images/serve.ts b/packages/gateway/src/data-plane/images/serve.ts index 40bd1ca22..fe9d561ea 100644 --- a/packages/gateway/src/data-plane/images/serve.ts +++ b/packages/gateway/src/data-plane/images/serve.ts @@ -2,11 +2,12 @@ // requests to the provider that declares the requested model and the // matching image endpoint capability. // -// Edits multipart bodies are loaded into memory via `request.formData()`; -// this caps the per-request body size at the Workers heap (~128 MB). -// Sufficient for the gpt-image-2 single-image edit case (≤50 MB image + -// ≤50 MB mask). Multi-image edits with the gpt-image-1 `image[]` array -// may exceed the heap — a streaming multipart parser is a follow-up. +// Edits accepts multipart uploads and JSON `images` references. Both shapes +// are buffered once for dump capture; JSON remains JSON through provider +// dispatch, while multipart is parsed and re-encoded with a fresh boundary. +// https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L12558-L12620 +// Multipart size is therefore capped by the Workers heap (~128 MB); a +// streaming parser is required before larger multi-image uploads are viable. import type { Context } from 'hono'; @@ -15,37 +16,38 @@ import { createGatewayCtxFromHono, finalizeGatewayResponse } from '../chat/share import { readRequestBody, takeRequestBody, type RequestBody } from '../chat/shared/request-body.ts'; import { passthroughApiError, passthroughServe } from '../shared/passthrough-serve.ts'; import { tokenUsageFromImagesBody } from '../shared/telemetry/usage.ts'; +import type { ImagesEditsJsonPayload } from '@floway-dev/protocols/images'; +import type { ImagesEditsBody } from '@floway-dev/provider'; -interface ImagesGenerationsRequestBody { +interface JsonModelRequestBody { model?: unknown; - prompt?: unknown; [key: string]: unknown; } -type PreparedRequest = +type PreparedJsonRequest = | { type: 'ok'; body: Record; model: string } | { type: 'invalid'; message: string }; -const prepareImagesGenerationsRequest = (bytes: Uint8Array): PreparedRequest => { - let request: ImagesGenerationsRequestBody; +const prepareJsonModelRequest = (bytes: Uint8Array, requestName: string): PreparedJsonRequest => { + let request: JsonModelRequestBody; try { const parsed = JSON.parse(new TextDecoder().decode(bytes)) as unknown; if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { - return { type: 'invalid', message: 'Images generations request body must be an object.' }; + return { type: 'invalid', message: `${requestName} request body must be an object.` }; } - request = parsed as ImagesGenerationsRequestBody; + request = parsed as JsonModelRequestBody; } catch { - return { type: 'invalid', message: 'Images generations request body must be valid JSON.' }; + return { type: 'invalid', message: `${requestName} request body must be valid JSON.` }; } if (typeof request.model !== 'string' || request.model.length === 0) { - return { type: 'invalid', message: 'Images generations request body must include a model string.' }; + return { type: 'invalid', message: `${requestName} request body must include a model string.` }; } return { type: 'ok', body: request as Record, model: request.model }; }; export const imagesGenerations = async (c: Context): Promise => { const requestBody = await readRequestBody(c); - const request = prepareImagesGenerationsRequest(requestBody.bytes); + const request = prepareJsonModelRequest(requestBody.bytes, 'Images generations'); const ctx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) }); if (request.type === 'invalid') { ctx.dump?.error('gateway'); @@ -70,36 +72,33 @@ export const imagesGenerations = async (c: Context): Promise => { return finalizeGatewayResponse(ctx, response); }; -export const serveImagesEditForm = async (c: Context, requestBody: RequestBody, form: FormData): Promise => { +const serveImagesEditBody = async ( + c: Context, + requestBody: RequestBody, + model: string, + body: ImagesEditsBody, +): Promise => { const ctx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) }); - - const modelRaw = form.get('model'); - if (typeof modelRaw !== 'string' || modelRaw.length === 0) { - ctx.dump?.error('gateway'); - return finalizeGatewayResponse(ctx, passthroughApiError(c, 'Image edits request body must include a model field.', 400)); - } - - ctx.dump?.requestedModel(modelRaw); + ctx.dump?.requestedModel(model); const response = await passthroughServe({ c, ctx, sourceApi: '/images/edits', operation: 'image_edit', - model: modelRaw, + model, kind: 'image', modelServesEndpoint: model => model.endpoints.imagesEdits !== undefined, call: (provider, model, opts) => { - // ProviderInstance.callImagesEdits takes ownership of the FormData and - // appends the upstream-specific model/deployment id; allocate a fresh - // copy per candidate so the contract holds even if cross-candidate - // fallback is ever extended to try a second match. File-blob entries - // are passed by reference so no buffer copy happens. - const passthrough = new FormData(); - for (const [name, value] of form.entries()) { - if (name === 'model') continue; - passthrough.append(name, value); + if (!(body instanceof FormData)) { + return provider.instance.callImagesEdits(model, body, undefined, opts); } - return provider.instance.callImagesEdits(model, passthrough, undefined, opts); + // The provider may append its model id to multipart bodies, so allocate + // a fresh FormData per candidate. Blob values remain shared. + const candidateBody = new FormData(); + for (const [name, value] of body.entries()) { + if (name !== 'model') candidateBody.append(name, value); + } + return provider.instance.callImagesEdits(model, candidateBody, undefined, opts); }, response: { format: 'json', extractBilling: tokenUsageFromImagesBody }, }); @@ -107,21 +106,37 @@ export const serveImagesEditForm = async (c: Context, requestBody: RequestBody, }; export const imagesEdits = async (c: Context): Promise => { - // Buffer the multipart body once. Hono's formData() helper would consume - // c.req.raw.body internally; re-parsing from the captured bytes via a fresh - // Response keeps the dump capture honest without a second read on the wire. const requestBody = await readRequestBody(c); - let form: FormData; - try { - form = await new Response(requestBody.bytes as BodyInit, { headers: { 'content-type': c.req.header('content-type') ?? '' } }).formData(); - } catch { - // Match the embeddings serve stance: do not surface the underlying - // parser's error text. The wording is enough for a client to know - // they sent the wrong content type or a malformed body. + const invalid = (message: string): Response => { const errorCtx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) }); errorCtx.dump?.error('gateway'); - return finalizeGatewayResponse(errorCtx, passthroughApiError(c, 'Image edits request body must be a valid multipart/form-data payload.', 400)); + return finalizeGatewayResponse(errorCtx, passthroughApiError(c, message, 400)); + }; + + const contentType = c.req.header('content-type'); + if (contentType === undefined) { + return invalid('Image edits request body must use application/json or multipart/form-data.'); + } + const mediaType = contentType.replace(/;.*$/u, '').trim().toLowerCase(); + if (mediaType === 'application/json') { + const request = prepareJsonModelRequest(requestBody.bytes, 'Image edits'); + if (request.type === 'invalid') return invalid(request.message); + const { model: _model, ...body } = request.body as ImagesEditsJsonPayload; + return await serveImagesEditBody(c, requestBody, request.model, body); } - return await serveImagesEditForm(c, requestBody, form); + if (mediaType !== 'multipart/form-data') { + return invalid('Image edits request body must use application/json or multipart/form-data.'); + } + let form: FormData; + try { + form = await new Response(requestBody.bytes as BodyInit, { headers: { 'content-type': contentType } }).formData(); + } catch { + return invalid('Image edits request body must be valid multipart/form-data.'); + } + const model = form.get('model'); + if (typeof model !== 'string' || model.length === 0) { + return invalid('Image edits request body must include a model field.'); + } + return await serveImagesEditBody(c, requestBody, model, form); }; diff --git a/packages/gateway/src/data-plane/images/serve_test.ts b/packages/gateway/src/data-plane/images/serve_test.ts index 437020d36..d17ff3aab 100644 --- a/packages/gateway/src/data-plane/images/serve_test.ts +++ b/packages/gateway/src/data-plane/images/serve_test.ts @@ -42,7 +42,7 @@ test('/v1/images/generations 404s when no upstream provides the model', async () async () => { const response = await requestApp('/v1/images/generations', { method: 'POST', - headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, + headers: { 'content-type': 'application/json; charset=utf-8', 'x-api-key': apiKey.key }, body: JSON.stringify({ model: 'no-such-model', prompt: 'hi' }), }); assertEquals(response.status, 404); @@ -50,12 +50,22 @@ test('/v1/images/generations 404s when no upstream provides the model', async () ); }); -test('/v1/images/edits rejects non-multipart body with 400', async () => { +test('/v1/images/edits rejects malformed JSON with 400', async () => { + const { apiKey } = await setupAppTest(); + const response = await requestApp('/v1/images/edits', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, + body: 'not json', + }); + assertEquals(response.status, 400); +}); + +test('/v1/images/edits rejects JSON without a model with 400', async () => { const { apiKey } = await setupAppTest(); const response = await requestApp('/v1/images/edits', { method: 'POST', headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, - body: '{}', + body: JSON.stringify({ prompt: 'hi', images: [{ file_id: 'file-image' }] }), }); assertEquals(response.status, 400); }); @@ -235,3 +245,64 @@ test('/v1/images/edits forwards a multipart request through an Azure model and r const usageRows = await repo.usage.listAll(); assertEquals(usageRows.some(row => row.model === 'gpt-image-2' && row.tokens.input === 7 && row.tokens.output === 11), true); }); + +test('/v1/images/edits forwards JSON image references through a custom provider', async () => { + const { apiKey, repo } = await setupAppTest(); + clearInProcessCopilotTokenCache(); + await repo.upstreams.save(buildCustomUpstreamRecord({ + id: 'up_image_edits_json', + name: 'Custom Image Provider', + sortOrder: 100, + config: { + baseUrl: 'https://images.example.com', + authStyle: 'bearer', + apiKey: 'sk-images', + endpoints: {}, + }, + })); + + let forwarded: Record | undefined; + await withMockedFetch( + async request => { + const url = new URL(request.url); + if (url.hostname === 'update.code.visualstudio.com') return jsonResponse(['1.110.1']); + if (url.pathname === '/copilot_internal/v2/token') { + return jsonResponse({ token: 'copilot-access-token', expires_at: 4102444800, refresh_in: 3600, endpoints: { api: 'https://api.individual.githubcopilot.com' } }); + } + if (url.hostname === 'api.individual.githubcopilot.com' && url.pathname === '/models') { + return jsonResponse(copilotModels([{ id: 'copilot-chat', supported_endpoints: ['/chat/completions'] }])); + } + if (url.hostname === 'images.example.com' && url.pathname === '/v1/models') { + return jsonResponse({ data: [{ id: 'gpt-image-2' }] }); + } + if (url.hostname === 'images.example.com' && url.pathname === '/v1/images/edits') { + forwarded = await request.json() as Record; + return jsonResponse({ data: [{ b64_json: 'ZWRpdA==' }] }); + } + throw new Error(`Unhandled fetch ${request.url}`); + }, + async () => { + const response = await requestApp('/v1/images/edits', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, + body: JSON.stringify({ + model: 'gpt-image-2', + prompt: 'replace the background', + images: [{ file_id: 'file-source' }], + mask: { file_id: 'file-mask' }, + quality: 'high', + }), + }); + assertEquals(response.status, 200); + assertEquals(await response.json(), { data: [{ b64_json: 'ZWRpdA==' }] }); + }, + ); + + assertEquals(forwarded, { + model: 'gpt-image-2', + prompt: 'replace the background', + images: [{ file_id: 'file-source' }], + mask: { file_id: 'file-mask' }, + quality: 'high', + }); +}); diff --git a/packages/protocols/src/images/index.ts b/packages/protocols/src/images/index.ts index dfb67aa95..c65ab39f8 100644 --- a/packages/protocols/src/images/index.ts +++ b/packages/protocols/src/images/index.ts @@ -5,8 +5,6 @@ // without a gateway-side reject while named fields keep their narrow // types when accessed directly — the `T & Record` // intersection form would widen every typed field to `unknown` on read. -// /v1/images/edits is multipart and has no JSON payload type — its source -// serve handles `FormData` directly. export interface ImagesGenerationsPayload { model: string; prompt: string; @@ -23,3 +21,31 @@ export interface ImagesGenerationsPayload { user?: string; [key: string]: unknown; } + +export type ImageEditReference = + | { image_url: string; file_id?: never } + | { file_id: string; image_url?: never }; + +// POST /v1/images/edits accepts either multipart uploads or this JSON shape. +// JSON uses `images` rather than the multipart `image` field, and references +// may point at a URL/data URL or an uploaded file. +// https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L12558-L12620 +// https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L47542-L47673 +export interface ImagesEditsJsonPayload { + model: string; + prompt: string; + images: ImageEditReference[]; + mask?: ImageEditReference; + n?: number; + quality?: string; + input_fidelity?: string; + size?: string; + user?: string; + output_format?: string; + output_compression?: number; + moderation?: string; + background?: string; + stream?: boolean; + partial_images?: number; + [key: string]: unknown; +} diff --git a/packages/provider-azure/src/provider.ts b/packages/provider-azure/src/provider.ts index ea6c7f56c..3757ae311 100644 --- a/packages/provider-azure/src/provider.ts +++ b/packages/provider-azure/src/provider.ts @@ -91,6 +91,9 @@ export const createAzureProvider = (record: UpstreamRecord): Provider => { callEmbeddings: (model, body, signal, opts) => callNonStreaming(azureFetchEmbeddings, model, body, signal, opts.headers, opts), callImagesGenerations: (model, body, signal, opts) => callNonStreaming(azureFetchImagesGenerations, model, body, signal, opts.headers, opts), callImagesEdits: async (model, body, signal, opts) => { + if (!(body instanceof FormData)) { + return callNonStreaming(azureFetchImagesEdits, model, body, signal, opts.headers, opts); + } // Azure routes by upstream model id in the multipart `model` field; the // runtime re-encodes the FormData with a fresh boundary and sets // Content-Type itself. diff --git a/packages/provider-custom/src/provider.ts b/packages/provider-custom/src/provider.ts index abc349338..5d7a2094b 100644 --- a/packages/provider-custom/src/provider.ts +++ b/packages/provider-custom/src/provider.ts @@ -179,6 +179,9 @@ export const createCustomProvider = (record: UpstreamRecord): Provider => { callEmbeddings: (model, body, signal, opts) => call(customFetchEmbeddings, model, body, signal, opts.headers, opts), callImagesGenerations: (model, body, signal, opts) => call(customFetchImagesGenerations, model, body, signal, opts.headers, opts), callImagesEdits: async (model, body, signal, opts) => { + if (!(body instanceof FormData)) { + return call(customFetchImagesEdits, model, body, signal, opts.headers, opts); + } // The runtime auto-encodes the FormData with a fresh boundary and sets // Content-Type itself. const rawModelId = rawModelIdOf(model); diff --git a/packages/provider/src/index.ts b/packages/provider/src/index.ts index 6d83c0ec5..a76c651b7 100644 --- a/packages/provider/src/index.ts +++ b/packages/provider/src/index.ts @@ -47,6 +47,7 @@ export type { AddressableForm, ModelPrefixConfig } from './model-prefix.ts'; export { MODEL_PREFIX_MAX_LENGTH, MODEL_PREFIX_REGEX, normalizeModelPrefix } from './model-prefix.ts'; export type { + ImagesEditsBody, Provider, ProviderInstance, ProviderCallResult, diff --git a/packages/provider/src/provider.ts b/packages/provider/src/provider.ts index 9f44fc629..fa0cca696 100644 --- a/packages/provider/src/provider.ts +++ b/packages/provider/src/provider.ts @@ -6,7 +6,7 @@ import type { ChatCompletionsPayload, ChatCompletionsStreamEvent } from '@floway import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { CompletionsPayload } from '@floway-dev/protocols/completions'; import type { EmbeddingsPayload } from '@floway-dev/protocols/embeddings'; -import type { ImagesGenerationsPayload } from '@floway-dev/protocols/images'; +import type { ImagesEditsJsonPayload, ImagesGenerationsPayload } from '@floway-dev/protocols/images'; import type { MessagesPayload, MessagesStreamEvent } from '@floway-dev/protocols/messages'; import type { CanonicalResponsesPayload, ResponsesResult, ResponsesStreamEvent } from '@floway-dev/protocols/responses'; @@ -19,6 +19,7 @@ import type { CanonicalResponsesPayload, ResponsesResult, ResponsesStreamEvent } // into 'generate' so the inner upstream call runs an ordinary summarization // turn against the SUMMARIZATION_PROMPT). export type ResponsesAction = 'generate' | 'compact'; +export type ImagesEditsBody = FormData | Omit; export interface Provider { upstream: string; @@ -129,10 +130,9 @@ export interface ProviderInstance { callMessagesCountTokens(model: ProviderModel, body: Omit, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise; callEmbeddings(model: ProviderModel, body: Omit, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise; callImagesGenerations(model: ProviderModel, body: Omit, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise; - // The provider takes ownership of `body` and may mutate it (e.g. append - // the upstream-specific model/deployment id). Callers must allocate a - // fresh FormData per call. - callImagesEdits(model: ProviderModel, body: FormData, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise; + // Providers insert their upstream-specific model id. Multipart bodies are + // owned by the provider and may be mutated; JSON bodies remain immutable. + callImagesEdits(model: ProviderModel, body: ImagesEditsBody, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise; } // Static, module-shaped surface each provider package exports. The gateway From 874544ecdd5d73120b06eab2718c413d7269307e Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 15 Jul 2026 11:41:11 +0800 Subject: [PATCH 10/24] style(images): await JSON provider dispatch --- packages/provider-azure/src/provider.ts | 2 +- packages/provider-custom/src/provider.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/provider-azure/src/provider.ts b/packages/provider-azure/src/provider.ts index 3757ae311..0ae998035 100644 --- a/packages/provider-azure/src/provider.ts +++ b/packages/provider-azure/src/provider.ts @@ -92,7 +92,7 @@ export const createAzureProvider = (record: UpstreamRecord): Provider => { callImagesGenerations: (model, body, signal, opts) => callNonStreaming(azureFetchImagesGenerations, model, body, signal, opts.headers, opts), callImagesEdits: async (model, body, signal, opts) => { if (!(body instanceof FormData)) { - return callNonStreaming(azureFetchImagesEdits, model, body, signal, opts.headers, opts); + return await callNonStreaming(azureFetchImagesEdits, model, body, signal, opts.headers, opts); } // Azure routes by upstream model id in the multipart `model` field; the // runtime re-encodes the FormData with a fresh boundary and sets diff --git a/packages/provider-custom/src/provider.ts b/packages/provider-custom/src/provider.ts index 5d7a2094b..e69c170d3 100644 --- a/packages/provider-custom/src/provider.ts +++ b/packages/provider-custom/src/provider.ts @@ -180,7 +180,7 @@ export const createCustomProvider = (record: UpstreamRecord): Provider => { callImagesGenerations: (model, body, signal, opts) => call(customFetchImagesGenerations, model, body, signal, opts.headers, opts), callImagesEdits: async (model, body, signal, opts) => { if (!(body instanceof FormData)) { - return call(customFetchImagesEdits, model, body, signal, opts.headers, opts); + return await call(customFetchImagesEdits, model, body, signal, opts.headers, opts); } // The runtime auto-encodes the FormData with a fresh boundary and sets // Content-Type itself. From dd55033f8ff2411d7e23eb9aff3ec156f24ddb06 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 15 Jul 2026 11:49:02 +0800 Subject: [PATCH 11/24] docs(images): clarify dual-format buffering --- packages/gateway/src/data-plane/images/serve.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/gateway/src/data-plane/images/serve.ts b/packages/gateway/src/data-plane/images/serve.ts index fe9d561ea..5c2213d2d 100644 --- a/packages/gateway/src/data-plane/images/serve.ts +++ b/packages/gateway/src/data-plane/images/serve.ts @@ -2,12 +2,11 @@ // requests to the provider that declares the requested model and the // matching image endpoint capability. // -// Edits accepts multipart uploads and JSON `images` references. Both shapes -// are buffered once for dump capture; JSON remains JSON through provider -// dispatch, while multipart is parsed and re-encoded with a fresh boundary. +// The edits handler accepts multipart uploads and JSON `images` references. +// Both shapes are buffered once for dump capture; JSON remains JSON through +// provider dispatch, while multipart is re-encoded with a fresh boundary. // https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L12558-L12620 -// Multipart size is therefore capped by the Workers heap (~128 MB); a -// streaming parser is required before larger multi-image uploads are viable. +// A streaming parser is required before unbounded multipart uploads are viable. import type { Context } from 'hono'; From 4bbc044fc5e3a3d2fc6922b87718358297c6ef86 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 15 Jul 2026 12:00:47 +0800 Subject: [PATCH 12/24] docs(images): keep buffering rationale concise --- packages/gateway/src/data-plane/images/serve.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/gateway/src/data-plane/images/serve.ts b/packages/gateway/src/data-plane/images/serve.ts index 5c2213d2d..36367e1c5 100644 --- a/packages/gateway/src/data-plane/images/serve.ts +++ b/packages/gateway/src/data-plane/images/serve.ts @@ -6,7 +6,6 @@ // Both shapes are buffered once for dump capture; JSON remains JSON through // provider dispatch, while multipart is re-encoded with a fresh boundary. // https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L12558-L12620 -// A streaming parser is required before unbounded multipart uploads are viable. import type { Context } from 'hono'; From a54e28d71fe3dff9caa02a038d25888d690404fe Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 15 Jul 2026 13:50:30 +0800 Subject: [PATCH 13/24] refactor(images): dispatch edits through semantic requests Normalize JSON references and multipart uploads into a homogeneous Images Edits request. Providers now choose JSON for image_url/file_id references and multipart for uploaded files, preserving Codex wire behavior without downloading URLs or base64-encoding binary uploads. --- .../image-generation-integration_test.ts | 49 +++++----- .../server-tools/image-generation.ts | 50 +++++----- .../gateway/src/data-plane/images/serve.ts | 97 ++++++++++++++----- .../providers/azure/provider_test.ts | 9 +- .../providers/custom/provider_test.ts | 9 +- packages/provider-azure/src/provider.ts | 13 +-- packages/provider-custom/src/provider.ts | 12 +-- packages/provider/src/images.ts | 42 ++++++++ packages/provider/src/index.ts | 3 +- packages/provider/src/provider.ts | 8 +- 10 files changed, 192 insertions(+), 100 deletions(-) create mode 100644 packages/provider/src/images.ts diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation-integration_test.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation-integration_test.ts index 6c87e3778..845b6961d 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation-integration_test.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation-integration_test.ts @@ -8,7 +8,7 @@ import { createInMemoryImageProcessor, initExternalResourceFetcher, initImagePro import { eventFrame } from '@floway-dev/protocols/common'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { ResponsesResult, ResponsesStreamEvent, ResponsesTool, ResponsesToolChoice } from '@floway-dev/protocols/responses'; -import { type EventResult, type ExecuteResult, type FlagId } from '@floway-dev/provider'; +import { type EventResult, type ExecuteResult, type FlagId, type ImagesEditsRequest } from '@floway-dev/provider'; import { assert, assertEquals, assertStringIncludes, stubModelCandidate } from '@floway-dev/test-utils'; // Dirty integration harness: mock the model registry so the image backend is a @@ -20,7 +20,7 @@ import { assert, assertEquals, assertStringIncludes, stubModelCandidate } from ' interface BackendStub { generationsCalls: Record[]; - editsForms: FormData[]; + editsRequests: ImagesEditsRequest[]; nextGenerations: Response[]; nextEdits: Response[]; // When set, the next `enumerateModelCandidates` call returns this @@ -31,7 +31,7 @@ interface BackendStub { // Hoisted so the vi.mock factory below can close over it; tests mutate the // `next*` queues and read back the recorded calls. -const stub = vi.hoisted((): BackendStub => ({ generationsCalls: [], editsForms: [], nextGenerations: [], nextEdits: [], nextResolutionOverride: null })); +const stub = vi.hoisted((): BackendStub => ({ generationsCalls: [], editsRequests: [], nextGenerations: [], nextEdits: [], nextResolutionOverride: null })); // Assigned per test in beforeEach and captured so the perf-attribution test // can read `repo.performance.listAll()` after the shim completes. @@ -52,9 +52,8 @@ const defaultCandidates = vi.hoisted(() => () => [{ if (response === undefined) throw new Error('test did not enqueue a generations response'); return { response, modelKey: 'gpt-image-2' }; }, - callImagesEdits: async (_model: unknown, body: FormData | Record) => { - if (!(body instanceof FormData)) throw new Error('hosted image edit must use multipart'); - stub.editsForms.push(body); + callImagesEdits: async (_model: unknown, request: ImagesEditsRequest) => { + stub.editsRequests.push(request); const response = stub.nextEdits.shift(); if (response === undefined) throw new Error('test did not enqueue an edits response'); return { response, modelKey: 'gpt-image-2' }; @@ -222,7 +221,7 @@ beforeEach(async () => { initRepo(repo); initImageProcessor(createInMemoryImageProcessor()); stub.generationsCalls = []; - stub.editsForms = []; + stub.editsRequests = []; stub.nextGenerations = []; stub.nextEdits = []; stub.nextResolutionOverride = null; @@ -237,7 +236,7 @@ test('generates an image end-to-end and emits the native lifecycle', async () => const events = await drain(result); assertEquals(stub.generationsCalls.length, 1); - assertEquals(stub.editsForms.length, 0); + assertEquals(stub.editsRequests.length, 0); const igcDone = events.find(e => e.type === 'response.output_item.done' && (e as { item: { type: string } }).item.type === 'image_generation_call'); assert(igcDone !== undefined); const item = (igcDone as { item: { status: string; result: string; action: string } }).item; @@ -299,10 +298,11 @@ test('an image generated in turn 1 is re-collected as an edit source in turn 2', await drain(result); assertEquals(stub.generationsCalls.length, 1); - assertEquals(stub.editsForms.length, 1); - const images = stub.editsForms[0].getAll('image[]'); - assertEquals(images.length, 1); - const bytes = await (images[0] as Blob).text(); + assertEquals(stub.editsRequests.length, 1); + const request = stub.editsRequests[0]; + assert(request.type === 'uploads'); + assertEquals(request.images.length, 1); + const bytes = await request.images[0].text(); assertEquals(bytes, 'AAAA'); }); @@ -338,8 +338,10 @@ test('a prefetched remote edit source remains visible to orchestration and is re assertEquals(fetched, ['https://example.com/source.png']); assertEquals(orchestratorImageUrl, 'https://example.com/source.png'); - assertEquals(stub.editsForms.length, 1); - const image = stub.editsForms[0].getAll('image[]')[0] as Blob; + assertEquals(stub.editsRequests.length, 1); + const request = stub.editsRequests[0]; + assert(request.type === 'uploads'); + const image = request.images[0]; assertEquals(new Uint8Array(await image.arrayBuffer()), Uint8Array.from(atob(REMOTE_PNG_B64), c => c.charCodeAt(0))); }); @@ -362,10 +364,12 @@ test('mask-only GIF edit transcodes one shared image and mask to WebP', async () await drain(result); assertEquals(processorCalls, 1); - assertEquals(stub.editsForms.length, 1); - const form = stub.editsForms[0]; - const image = form.getAll('image[]')[0] as Blob; - const mask = form.get('mask') as Blob; + assertEquals(stub.editsRequests.length, 1); + const request = stub.editsRequests[0]; + assert(request.type === 'uploads'); + const image = request.images[0]; + const mask = request.mask; + assert(mask !== undefined); assertEquals(image.type, 'image/webp'); assertEquals(mask.type, 'image/webp'); assertEquals(await image.text(), 'WEBP'); @@ -394,9 +398,10 @@ test('identical GIF source and mask share one transcode', async () => { await drain(result); assertEquals(processorCalls, 1); - const form = stub.editsForms[0]; - assertEquals(await (form.getAll('image[]')[0] as Blob).text(), 'WEBP'); - assertEquals(await (form.get('mask') as Blob).text(), 'WEBP'); + const request = stub.editsRequests[0]; + assert(request.type === 'uploads'); + assertEquals(await request.images[0].text(), 'WEBP'); + assertEquals(await request.mask?.text(), 'WEBP'); }); test('image transcoding failure becomes a terminal image tool failure', async () => { @@ -413,7 +418,7 @@ test('image transcoding failure becomes a terminal image tool failure', async () ])); const events = await drain(result); - assertEquals(stub.editsForms.length, 0); + assertEquals(stub.editsRequests.length, 0); const done = events.find(event => event.type === 'response.output_item.done' && (event as { item: { type: string } }).item.type === 'image_generation_call'); assert(done !== undefined); diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts index f02eac566..b8b55d9cc 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts @@ -19,7 +19,7 @@ import type { ResponsesOutputImageGenerationCall, ResponsesTool, } from '@floway-dev/protocols/responses'; -import { providerModelOf, type Fetcher, type Provider, type ModelCandidate, type ProviderModel } from '@floway-dev/provider'; +import { providerModelOf, type Fetcher, type ImagesEditsRequest, type Provider, type ModelCandidate, type ProviderModel } from '@floway-dev/provider'; export const SHIM_TOOL_NAME = 'image_generation'; @@ -931,38 +931,34 @@ export const buildGenerationsBody = (prompt: string, config: ImageGenerationConf ...(stream ? { stream: true, partial_images: config.partial_images } : {}), }); -const buildEditsForm = (prompt: string, config: ImageGenerationConfig, sources: readonly ImageSource[], stream: boolean): FormData => { - const form = new FormData(); - form.append('prompt', prompt); - form.append('n', '1'); - if (config.size !== undefined) form.append('size', config.size); - if (config.quality !== undefined) form.append('quality', config.quality); - if (config.output_format !== undefined) form.append('output_format', config.output_format); - if (config.background !== undefined) form.append('background', config.background); - if (config.moderation !== undefined) form.append('moderation', config.moderation); - if (config.output_compression !== undefined) form.append('output_compression', String(config.output_compression)); - if (config.input_fidelity !== undefined) form.append('input_fidelity', config.input_fidelity); - if (stream) { - form.append('stream', 'true'); - form.append('partial_images', String(config.partial_images)); - } - for (const [i, source] of sources.entries()) { +const buildEditsRequest = (prompt: string, config: ImageGenerationConfig, sources: readonly ImageSource[], stream: boolean): ImagesEditsRequest => { + const parameters: Record = { + prompt, + n: 1, + ...(config.size === undefined ? {} : { size: config.size }), + ...(config.quality === undefined ? {} : { quality: config.quality }), + ...(config.output_format === undefined ? {} : { output_format: config.output_format }), + ...(config.background === undefined ? {} : { background: config.background }), + ...(config.moderation === undefined ? {} : { moderation: config.moderation }), + ...(config.output_compression === undefined ? {} : { output_compression: config.output_compression }), + ...(config.input_fidelity === undefined ? {} : { input_fidelity: config.input_fidelity }), + ...(stream ? { stream: true, partial_images: config.partial_images } : {}), + }; + const images = sources.map((source, index) => { // Forward the canonical supported mime; an unsupported source is rejected // before dispatch, so the fallback only ever carries an already-supported // generated image. Sending the raw mime (rather than relabeling as png) // keeps a stray unsupported byte stream failing loud at the backend. const mime = editSupportedMime(source.mimeType) ?? source.mimeType; - // `image[]` repeated parts: gpt-image accepts multiple, resolving "the - // Nth image" against attach order (see `inspectImageSources`). - form.append('image[]', new Blob([source.bytes], { type: mime }), `image_${i}.${editFileExt(mime)}`); - } + return new File([source.bytes], `image_${index}.${editFileExt(mime)}`, { type: mime }); + }); + let mask: File | undefined; if (config.mask !== undefined) { - if (isRemoteImageSource(config.mask)) throw new Error('Remote image mask reached form encoding before materialization'); - const mask = config.mask; - const mime = editSupportedMime(mask.mimeType) ?? mask.mimeType; - form.append('mask', new Blob([mask.bytes], { type: mime }), `mask.${editFileExt(mime)}`); + if (isRemoteImageSource(config.mask)) throw new Error('Remote image mask reached edit request construction before materialization'); + const mime = editSupportedMime(config.mask.mimeType) ?? config.mask.mimeType; + mask = new File([config.mask.bytes], `mask.${editFileExt(mime)}`, { type: mime }); } - return form; + return { type: 'uploads', images, ...(mask === undefined ? {} : { mask }), parameters }; }; const serverError = (e: unknown): ImageError => ({ @@ -1090,7 +1086,7 @@ const issueImageCall = async ( wrapUpstreamCall: stampUpstreamCallStart(attempt), }; const { response, modelKey } = await (isEdit - ? provider.instance.callImagesEdits(model, buildEditsForm(prompt, config, sources, stream), state.downstreamAbortSignal, opts) + ? provider.instance.callImagesEdits(model, buildEditsRequest(prompt, config, sources, stream), state.downstreamAbortSignal, opts) : provider.instance.callImagesGenerations(model, buildGenerationsBody(prompt, config, stream), state.downstreamAbortSignal, opts)); if (response.status !== 429 || retry >= MAX_RATE_LIMIT_RETRIES) return { response, modelKey }; diff --git a/packages/gateway/src/data-plane/images/serve.ts b/packages/gateway/src/data-plane/images/serve.ts index 36367e1c5..5df97d157 100644 --- a/packages/gateway/src/data-plane/images/serve.ts +++ b/packages/gateway/src/data-plane/images/serve.ts @@ -3,8 +3,8 @@ // matching image endpoint capability. // // The edits handler accepts multipart uploads and JSON `images` references. -// Both shapes are buffered once for dump capture; JSON remains JSON through -// provider dispatch, while multipart is re-encoded with a fresh boundary. +// Both are buffered once for dump capture and normalized into a semantic +// request; each provider owns the final JSON or multipart serialization. // https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L12558-L12620 import type { Context } from 'hono'; @@ -14,8 +14,8 @@ import { createGatewayCtxFromHono, finalizeGatewayResponse } from '../chat/share import { readRequestBody, takeRequestBody, type RequestBody } from '../chat/shared/request-body.ts'; import { passthroughApiError, passthroughServe } from '../shared/passthrough-serve.ts'; import { tokenUsageFromImagesBody } from '../shared/telemetry/usage.ts'; -import type { ImagesEditsJsonPayload } from '@floway-dev/protocols/images'; -import type { ImagesEditsBody } from '@floway-dev/provider'; +import type { ImageEditReference } from '@floway-dev/protocols/images'; +import type { ImagesEditsRequest } from '@floway-dev/provider'; interface JsonModelRequestBody { model?: unknown; @@ -43,6 +43,48 @@ const prepareJsonModelRequest = (bytes: Uint8Array, requestName: string): Prepar return { type: 'ok', body: request as Record, model: request.model }; }; +type PreparedReferencedImagesEdit = + | { type: 'ok'; request: ImagesEditsRequest } + | { type: 'invalid'; message: string }; + +const imageEditReference = (value: unknown, path: string): ImageEditReference | string => { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + return `${path} must be an object.`; + } + const { image_url: imageUrl, file_id: fileId } = value as { image_url?: unknown; file_id?: unknown }; + if (typeof imageUrl === 'string' && fileId === undefined) return { image_url: imageUrl }; + if (typeof fileId === 'string' && imageUrl === undefined) return { file_id: fileId }; + return `${path} must contain exactly one string field: image_url or file_id.`; +}; + +const prepareReferencedImagesEdit = (body: Record): PreparedReferencedImagesEdit => { + if (!Array.isArray(body.images)) { + return { type: 'invalid', message: 'Image edits request body must include an images array.' }; + } + const images: ImageEditReference[] = []; + for (const [index, value] of body.images.entries()) { + const reference = imageEditReference(value, `Image edits images[${index}]`); + if (typeof reference === 'string') return { type: 'invalid', message: reference }; + images.push(reference); + } + let mask: ImageEditReference | undefined; + if (body.mask !== undefined) { + const reference = imageEditReference(body.mask, 'Image edits mask'); + if (typeof reference === 'string') return { type: 'invalid', message: reference }; + mask = reference; + } + const { model: _model, images: _images, mask: _mask, ...parameters } = body; + return { + type: 'ok', + request: { + type: 'references', + images, + ...(mask === undefined ? {} : { mask }), + parameters, + }, + }; +}; + export const imagesGenerations = async (c: Context): Promise => { const requestBody = await readRequestBody(c); const request = prepareJsonModelRequest(requestBody.bytes, 'Images generations'); @@ -70,11 +112,11 @@ export const imagesGenerations = async (c: Context): Promise => { return finalizeGatewayResponse(ctx, response); }; -const serveImagesEditBody = async ( +const serveImagesEditRequest = async ( c: Context, requestBody: RequestBody, model: string, - body: ImagesEditsBody, + request: ImagesEditsRequest, ): Promise => { const ctx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) }); ctx.dump?.requestedModel(model); @@ -86,18 +128,7 @@ const serveImagesEditBody = async ( model, kind: 'image', modelServesEndpoint: model => model.endpoints.imagesEdits !== undefined, - call: (provider, model, opts) => { - if (!(body instanceof FormData)) { - return provider.instance.callImagesEdits(model, body, undefined, opts); - } - // The provider may append its model id to multipart bodies, so allocate - // a fresh FormData per candidate. Blob values remain shared. - const candidateBody = new FormData(); - for (const [name, value] of body.entries()) { - if (name !== 'model') candidateBody.append(name, value); - } - return provider.instance.callImagesEdits(model, candidateBody, undefined, opts); - }, + call: (provider, model, opts) => provider.instance.callImagesEdits(model, request, undefined, opts), response: { format: 'json', extractBilling: tokenUsageFromImagesBody }, }); return finalizeGatewayResponse(ctx, response); @@ -117,10 +148,11 @@ export const imagesEdits = async (c: Context): Promise => { } const mediaType = contentType.replace(/;.*$/u, '').trim().toLowerCase(); if (mediaType === 'application/json') { - const request = prepareJsonModelRequest(requestBody.bytes, 'Image edits'); + const body = prepareJsonModelRequest(requestBody.bytes, 'Image edits'); + if (body.type === 'invalid') return invalid(body.message); + const request = prepareReferencedImagesEdit(body.body); if (request.type === 'invalid') return invalid(request.message); - const { model: _model, ...body } = request.body as ImagesEditsJsonPayload; - return await serveImagesEditBody(c, requestBody, request.model, body); + return await serveImagesEditRequest(c, requestBody, body.model, request.request); } if (mediaType !== 'multipart/form-data') { @@ -136,5 +168,26 @@ export const imagesEdits = async (c: Context): Promise => { if (typeof model !== 'string' || model.length === 0) { return invalid('Image edits request body must include a model field.'); } - return await serveImagesEditBody(c, requestBody, model, form); + const images: File[] = []; + let mask: File | undefined; + const parameters: Record = {}; + for (const [name, value] of form.entries()) { + if (name === 'model') continue; + if (name === 'image' || name === 'image[]') { + if (!(value instanceof File)) return invalid(`Image edits ${name} fields must be files.`); + images.push(value); + } else if (name === 'mask') { + if (!(value instanceof File)) return invalid('Image edits mask field must be a file.'); + mask = value; + } else { + if (typeof value !== 'string') return invalid(`Image edits ${name} field must be text.`); + parameters[name] = value; + } + } + return await serveImagesEditRequest(c, requestBody, model, { + type: 'uploads', + images, + ...(mask === undefined ? {} : { mask }), + parameters, + }); }; diff --git a/packages/gateway/src/data-plane/providers/azure/provider_test.ts b/packages/gateway/src/data-plane/providers/azure/provider_test.ts index 9facb4544..5a0fc3cb2 100644 --- a/packages/gateway/src/data-plane/providers/azure/provider_test.ts +++ b/packages/gateway/src/data-plane/providers/azure/provider_test.ts @@ -432,10 +432,11 @@ test('createAzureProvider callImagesEdits posts multipart with model replaced by async () => { const provider = createAzureProvider(record); const models = await provider.instance.getProvidedModels(directFetcher); - const form = new FormData(); - form.append('prompt', 'replace sky'); - form.append('image', new Blob([new Uint8Array([1, 2, 3])], { type: 'image/png' }), 'photo.png'); - const result = await provider.instance.callImagesEdits(models[0], form, undefined, noopUpstreamCallOptions()); + const result = await provider.instance.callImagesEdits(models[0], { + type: 'uploads', + parameters: { prompt: 'replace sky' }, + images: [new File([new Uint8Array([1, 2, 3])], 'photo.png', { type: 'image/png' })], + }, undefined, noopUpstreamCallOptions()); assertEquals(result.modelKey, 'gpt-image-2'); assertEquals(result.response.status, 200); }, diff --git a/packages/gateway/src/data-plane/providers/custom/provider_test.ts b/packages/gateway/src/data-plane/providers/custom/provider_test.ts index ffe4d969a..a5ba32679 100644 --- a/packages/gateway/src/data-plane/providers/custom/provider_test.ts +++ b/packages/gateway/src/data-plane/providers/custom/provider_test.ts @@ -225,11 +225,12 @@ test('Custom provider callImagesEdits forwards multipart body with model field a async () => { const provider = createCustomProvider(record); const models = await provider.instance.getProvidedModels(directFetcher); - const form = new FormData(); - form.append('prompt', 'add a kite'); - form.append('image', new Blob([new Uint8Array([1, 2, 3])], { type: 'image/png' }), 'photo.png'); const model = models[0]; const opts = noopUpstreamCallOptions(); - const result = await provider.instance.callImagesEdits(model, form, undefined, opts); + const result = await provider.instance.callImagesEdits(model, { + type: 'uploads', + parameters: { prompt: 'add a kite' }, + images: [new File([new Uint8Array([1, 2, 3])], 'photo.png', { type: 'image/png' })], + }, undefined, opts); assertEquals(result.modelKey, 'gpt-image-2'); assertEquals(result.response.status, 200); }, diff --git a/packages/provider-azure/src/provider.ts b/packages/provider-azure/src/provider.ts index 0ae998035..1f8843411 100644 --- a/packages/provider-azure/src/provider.ts +++ b/packages/provider-azure/src/provider.ts @@ -5,7 +5,7 @@ import { parseChatCompletionsStream } from '@floway-dev/protocols/chat-completio import { kindForEndpoints } from '@floway-dev/protocols/common'; import { parseMessagesStream } from '@floway-dev/protocols/messages'; import { parseResponsesStream, type ResponsesResult, toCompactPayloadShape } from '@floway-dev/protocols/responses'; -import { type ProviderInstance, type Provider, type ProviderModel, type ProviderStreamParser, type UpstreamCallOptions, type UpstreamFetchOptions, type UpstreamRecord, publicModelId, resolveEffectiveFlags, streamingProviderCall } from '@floway-dev/provider'; +import { imagesEditsJsonBody, imagesEditsMultipartBody, type ProviderInstance, type Provider, type ProviderModel, type ProviderStreamParser, type UpstreamCallOptions, type UpstreamFetchOptions, type UpstreamRecord, publicModelId, resolveEffectiveFlags, streamingProviderCall } from '@floway-dev/provider'; const upstreamModelIdOf = (model: ProviderModel): string => (model.providerData as { upstreamModelId: string }).upstreamModelId; @@ -90,15 +90,12 @@ export const createAzureProvider = (record: UpstreamRecord): Provider => { callMessagesCountTokens: (model, body, signal, opts) => callNonStreaming(azureFetchMessagesCountTokens, model, body, signal, opts.headers, opts), callEmbeddings: (model, body, signal, opts) => callNonStreaming(azureFetchEmbeddings, model, body, signal, opts.headers, opts), callImagesGenerations: (model, body, signal, opts) => callNonStreaming(azureFetchImagesGenerations, model, body, signal, opts.headers, opts), - callImagesEdits: async (model, body, signal, opts) => { - if (!(body instanceof FormData)) { - return await callNonStreaming(azureFetchImagesEdits, model, body, signal, opts.headers, opts); + callImagesEdits: async (model, request, signal, opts) => { + if (request.type === 'references') { + return await callNonStreaming(azureFetchImagesEdits, model, imagesEditsJsonBody(request), signal, opts.headers, opts); } - // Azure routes by upstream model id in the multipart `model` field; the - // runtime re-encodes the FormData with a fresh boundary and sets - // Content-Type itself. const upstreamModelId = upstreamModelIdOf(model); - body.append('model', upstreamModelId); + const body = imagesEditsMultipartBody(request, upstreamModelId); const response = await azureFetchImagesEdits(azure.config, { method: 'POST', body, signal }, { extraHeaders: opts.headers, fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }); return { response, modelKey: upstreamModelId }; }, diff --git a/packages/provider-custom/src/provider.ts b/packages/provider-custom/src/provider.ts index e69c170d3..9cc7b871d 100644 --- a/packages/provider-custom/src/provider.ts +++ b/packages/provider-custom/src/provider.ts @@ -7,7 +7,7 @@ import { parseChatCompletionsStream } from '@floway-dev/protocols/chat-completio import { type ModelEndpoints, kindForEndpoints } from '@floway-dev/protocols/common'; import { parseMessagesStream } from '@floway-dev/protocols/messages'; import { parseResponsesStream, type ResponsesResult, toCompactPayloadShape } from '@floway-dev/protocols/responses'; -import { publicModelId, resolveEffectiveFlags, streamingProviderCall, type FlagId, type ProviderInstance, type Provider, type ProviderCallResult, type ProviderModel, type ProviderStreamParser, type UpstreamCallOptions, type UpstreamFetchOptions, type UpstreamRecord } from '@floway-dev/provider'; +import { imagesEditsJsonBody, imagesEditsMultipartBody, publicModelId, resolveEffectiveFlags, streamingProviderCall, type FlagId, type ProviderInstance, type Provider, type ProviderCallResult, type ProviderModel, type ProviderStreamParser, type UpstreamCallOptions, type UpstreamFetchOptions, type UpstreamRecord } from '@floway-dev/provider'; const rawModelIdOf = (model: ProviderModel): string => model.providerData as string; @@ -178,14 +178,12 @@ export const createCustomProvider = (record: UpstreamRecord): Provider => { callMessagesCountTokens: (model, body, signal, opts) => call(customFetchMessagesCountTokens, model, body, signal, opts.headers, opts), callEmbeddings: (model, body, signal, opts) => call(customFetchEmbeddings, model, body, signal, opts.headers, opts), callImagesGenerations: (model, body, signal, opts) => call(customFetchImagesGenerations, model, body, signal, opts.headers, opts), - callImagesEdits: async (model, body, signal, opts) => { - if (!(body instanceof FormData)) { - return await call(customFetchImagesEdits, model, body, signal, opts.headers, opts); + callImagesEdits: async (model, request, signal, opts) => { + if (request.type === 'references') { + return await call(customFetchImagesEdits, model, imagesEditsJsonBody(request), signal, opts.headers, opts); } - // The runtime auto-encodes the FormData with a fresh boundary and sets - // Content-Type itself. const rawModelId = rawModelIdOf(model); - body.append('model', rawModelId); + const body = imagesEditsMultipartBody(request, rawModelId); const response = await customFetchImagesEdits(config, { method: 'POST', body, signal }, { extraHeaders: opts.headers, fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }); return { response, modelKey: rawModelId }; }, diff --git a/packages/provider/src/images.ts b/packages/provider/src/images.ts new file mode 100644 index 000000000..90f815d2e --- /dev/null +++ b/packages/provider/src/images.ts @@ -0,0 +1,42 @@ +import type { ImageEditReference } from '@floway-dev/protocols/images'; + +// The gateway normalizes both public wire formats into one homogeneous source +// kind. Providers serialize references as JSON and uploads as multipart +// without downloading URLs or base64-encoding uploaded bytes. +interface ImagesEditsRequestBase { + parameters: Record; +} + +export interface UploadedImagesEditsRequest extends ImagesEditsRequestBase { + type: 'uploads'; + images: File[]; + mask?: File; +} + +export interface ReferencedImagesEditsRequest extends ImagesEditsRequestBase { + type: 'references'; + images: ImageEditReference[]; + mask?: ImageEditReference; +} + +export type ImagesEditsRequest = UploadedImagesEditsRequest | ReferencedImagesEditsRequest; + +export const imagesEditsJsonBody = (request: ReferencedImagesEditsRequest): Record => ({ + ...request.parameters, + images: request.images, + ...(request.mask === undefined ? {} : { mask: request.mask }), +}); + +export const imagesEditsMultipartBody = (request: UploadedImagesEditsRequest, model: string): FormData => { + const form = new FormData(); + for (const [name, value] of Object.entries(request.parameters)) { + if (typeof value !== 'string' && typeof value !== 'number' && typeof value !== 'boolean') { + throw new Error(`Multipart image edits parameter ${name} must be a string, number, or boolean`); + } + form.append(name, String(value)); + } + for (const image of request.images) form.append('image[]', image); + if (request.mask !== undefined) form.append('mask', request.mask); + form.append('model', model); + return form; +}; diff --git a/packages/provider/src/index.ts b/packages/provider/src/index.ts index a76c651b7..45df2cc15 100644 --- a/packages/provider/src/index.ts +++ b/packages/provider/src/index.ts @@ -47,7 +47,6 @@ export type { AddressableForm, ModelPrefixConfig } from './model-prefix.ts'; export { MODEL_PREFIX_MAX_LENGTH, MODEL_PREFIX_REGEX, normalizeModelPrefix } from './model-prefix.ts'; export type { - ImagesEditsBody, Provider, ProviderInstance, ProviderCallResult, @@ -57,6 +56,8 @@ export type { ResponsesAction, UpstreamCallOptions, } from './provider.ts'; +export type { ImagesEditsRequest, ReferencedImagesEditsRequest, UploadedImagesEditsRequest } from './images.ts'; +export { imagesEditsJsonBody, imagesEditsMultipartBody } from './images.ts'; export type { ProviderStreamParser } from './streaming.ts'; export { streamingProviderCall } from './streaming.ts'; diff --git a/packages/provider/src/provider.ts b/packages/provider/src/provider.ts index fa0cca696..ae47bedd4 100644 --- a/packages/provider/src/provider.ts +++ b/packages/provider/src/provider.ts @@ -6,9 +6,10 @@ import type { ChatCompletionsPayload, ChatCompletionsStreamEvent } from '@floway import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { CompletionsPayload } from '@floway-dev/protocols/completions'; import type { EmbeddingsPayload } from '@floway-dev/protocols/embeddings'; -import type { ImagesEditsJsonPayload, ImagesGenerationsPayload } from '@floway-dev/protocols/images'; +import type { ImagesGenerationsPayload } from '@floway-dev/protocols/images'; import type { MessagesPayload, MessagesStreamEvent } from '@floway-dev/protocols/messages'; import type { CanonicalResponsesPayload, ResponsesResult, ResponsesStreamEvent } from '@floway-dev/protocols/responses'; +import type { ImagesEditsRequest } from './images.ts'; // Action tag threaded through the Responses pipeline. `generate` is a normal // streaming /responses turn; `compact` is the summarize-and-replace-history @@ -19,7 +20,6 @@ import type { CanonicalResponsesPayload, ResponsesResult, ResponsesStreamEvent } // into 'generate' so the inner upstream call runs an ordinary summarization // turn against the SUMMARIZATION_PROMPT). export type ResponsesAction = 'generate' | 'compact'; -export type ImagesEditsBody = FormData | Omit; export interface Provider { upstream: string; @@ -130,9 +130,7 @@ export interface ProviderInstance { callMessagesCountTokens(model: ProviderModel, body: Omit, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise; callEmbeddings(model: ProviderModel, body: Omit, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise; callImagesGenerations(model: ProviderModel, body: Omit, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise; - // Providers insert their upstream-specific model id. Multipart bodies are - // owned by the provider and may be mutated; JSON bodies remain immutable. - callImagesEdits(model: ProviderModel, body: ImagesEditsBody, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise; + callImagesEdits(model: ProviderModel, request: ImagesEditsRequest, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise; } // Static, module-shaped surface each provider package exports. The gateway From 9664407a1a00fd44ef6c5dd7b318bb877bbfe144 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 15 Jul 2026 13:53:29 +0800 Subject: [PATCH 14/24] style(provider): order image request import --- packages/provider/src/provider.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/provider/src/provider.ts b/packages/provider/src/provider.ts index ae47bedd4..c1761524d 100644 --- a/packages/provider/src/provider.ts +++ b/packages/provider/src/provider.ts @@ -1,4 +1,5 @@ import type { FlagDefaults } from './flags.ts'; +import type { ImagesEditsRequest } from './images.ts'; import type { ModelPrefixConfig } from './model-prefix.ts'; import type { ProviderModel, UpstreamProviderKind, UpstreamRecord } from './model.ts'; import type { Fetcher } from './options.ts'; @@ -9,7 +10,6 @@ import type { EmbeddingsPayload } from '@floway-dev/protocols/embeddings'; import type { ImagesGenerationsPayload } from '@floway-dev/protocols/images'; import type { MessagesPayload, MessagesStreamEvent } from '@floway-dev/protocols/messages'; import type { CanonicalResponsesPayload, ResponsesResult, ResponsesStreamEvent } from '@floway-dev/protocols/responses'; -import type { ImagesEditsRequest } from './images.ts'; // Action tag threaded through the Responses pipeline. `generate` is a normal // streaming /responses turn; `compact` is the summarize-and-replace-history From 446225f305426818fd990fc3d86632487f80345d Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 15 Jul 2026 13:55:25 +0800 Subject: [PATCH 15/24] test(images): assert canonical upload field --- .../gateway/src/data-plane/providers/custom/provider_test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gateway/src/data-plane/providers/custom/provider_test.ts b/packages/gateway/src/data-plane/providers/custom/provider_test.ts index a5ba32679..940c349fd 100644 --- a/packages/gateway/src/data-plane/providers/custom/provider_test.ts +++ b/packages/gateway/src/data-plane/providers/custom/provider_test.ts @@ -238,7 +238,7 @@ test('Custom provider callImagesEdits forwards multipart body with model field a assertExists(forwarded); assertEquals(forwarded.form.get('model'), 'gpt-image-2'); assertEquals(forwarded.form.get('prompt'), 'add a kite'); - assertEquals(forwarded.form.get('image') instanceof File, true); + assertEquals(forwarded.form.get('image[]') instanceof File, true); }); test('Custom provider with modelsFetch disabled serves only manual models and never fetches', async () => { From d6f30f592aa7387a3d19039a557ea657b1779d62 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 15 Jul 2026 14:08:17 +0800 Subject: [PATCH 16/24] refactor(images): enforce prepared edit invariants Preserve unknown JSON-reference fields, model upload parameters as scalar values, and type hosted edit sources after MIME preparation. Keep request builders private and verify their observable provider requests through integration tests. --- .../image-generation-integration_test.ts | 7 +- .../server-tools/image-generation.ts | 90 ++++++++++--------- .../server-tools/image-generation_test.ts | 50 +---------- .../gateway/src/data-plane/images/serve.ts | 9 +- packages/protocols/src/images/index.ts | 24 ++--- packages/provider/src/images.ts | 13 +-- 6 files changed, 75 insertions(+), 118 deletions(-) diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation-integration_test.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation-integration_test.ts index 845b6961d..c0348958f 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation-integration_test.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation-integration_test.ts @@ -229,13 +229,17 @@ beforeEach(async () => { test('generates an image end-to-end and emits the native lifecycle', async () => { stub.nextGenerations = [jsonResponse('R0VO')]; // "GEN" - const result = await shim(makeCtx([{ type: 'message', role: 'user', content: 'draw a cat' }]), gatewayCtx(), scriptedRun([ + const result = await shim(makeCtx([{ type: 'message', role: 'user', content: 'draw a cat' }], 'auto', { + size: '1024x1024', + quality: 'low', + }), gatewayCtx(), scriptedRun([ callTurn(0, 'call_1', 'a cat'), messageTurn('here it is'), ])); const events = await drain(result); assertEquals(stub.generationsCalls.length, 1); + assertEquals(stub.generationsCalls[0], { prompt: 'a cat', n: 1, size: '1024x1024', quality: 'low' }); assertEquals(stub.editsRequests.length, 0); const igcDone = events.find(e => e.type === 'response.output_item.done' && (e as { item: { type: string } }).item.type === 'image_generation_call'); assert(igcDone !== undefined); @@ -275,6 +279,7 @@ test('relays real partial_image frames when partial_images > 0', async () => { ])); const events = await drain(result); + assertEquals(stub.generationsCalls[0], { prompt: 'a cat', n: 1, stream: true, partial_images: 2 }); const partials = events.filter(e => e.type === 'response.image_generation_call.partial_image'); assertEquals(partials.length, 2); assertEquals((partials[0] as { partial_image_b64: string }).partial_image_b64, 'UDA='); diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts index b8b55d9cc..604602d1a 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts @@ -53,21 +53,23 @@ const ALLOWED_INPUT_FIDELITY = new Set(['high', 'low']); // `unsupported_file_mimetype`. Native Responses accepts the same GIF and // re-encodes it before editing, so the shim mirrors that behavior through the // platform image processor. Common aliases are folded onto the backend form. -const EDIT_MIME_ALIASES: Record = { +type EditMime = 'image/png' | 'image/jpeg' | 'image/webp'; + +const EDIT_MIME_ALIASES: Record = { 'image/jpg': 'image/jpeg', 'image/pjpeg': 'image/jpeg', 'image/x-png': 'image/png', }; -const EDIT_SUPPORTED_MIMES = new Set(['image/png', 'image/jpeg', 'image/webp']); - // The canonical edit-supported mimetype for a source, or null when the // standalone endpoint requires local WebP transcoding first. -const editSupportedMime = (mime: string): string | null => { +const editSupportedMime = (mime: string): EditMime | null => { const canonical = EDIT_MIME_ALIASES[mime] ?? mime; - return EDIT_SUPPORTED_MIMES.has(canonical) ? canonical : null; + return canonical === 'image/png' || canonical === 'image/jpeg' || canonical === 'image/webp' + ? canonical + : null; }; -const editFileExt = (mime: string): string => +const editFileExt = (mime: EditMime): string => mime === 'image/jpeg' ? 'jpg' : mime === 'image/webp' ? 'webp' : 'png'; // The public `image_generation` tool-config surface. Azure rejects any other @@ -100,6 +102,10 @@ interface ImageSource { mimeType: string; } +interface PreparedImageSource extends ImageSource { + mimeType: EditMime; +} + interface RemoteImageSource { wireUrl: string; invalidUrlParam: string; @@ -111,11 +117,12 @@ type ImageSourceReference = ImageSource | RemoteImageSource; const isRemoteImageSource = (source: ImageSourceReference): source is RemoteImageSource => 'wireUrl' in source; -export const prepareEditSources = async (sources: readonly ImageSource[]): Promise => { +const prepareEditSources = async (sources: readonly ImageSource[]): Promise => { const keyBySource = new Map>(); - const preparedByContent = new Map>(); + const preparedByContent = new Map>(); return await Promise.all(sources.map(async source => { - if (editSupportedMime(source.mimeType) !== null) return source; + const mimeType = editSupportedMime(source.mimeType); + if (mimeType !== null) return { bytes: source.bytes, mimeType }; let keyPromise = keyBySource.get(source); if (keyPromise === undefined) { @@ -135,7 +142,7 @@ export const prepareEditSources = async (sources: readonly ImageSource[]): Promi // https://github.com/openai/openai-node/blob/ec2f57fd0d66e94782656b986d7b3eb03225369c/src/resources/images.ts#L560-L572 prepared = getImageProcessor().compressToWebp(new Uint8Array(source.bytes), null).then(encoded => { const bytes = encoded.buffer.slice(encoded.byteOffset, encoded.byteOffset + encoded.byteLength) as ArrayBuffer; - return { bytes, mimeType: 'image/webp' }; + return { bytes, mimeType: 'image/webp' } satisfies PreparedImageSource; }); preparedByContent.set(key, prepared); } @@ -258,14 +265,14 @@ export interface ImageGenerationConfig { const prepareEditRequest = async ( sources: readonly ImageSource[], config: ImageGenerationConfig, -): Promise<{ sources: readonly ImageSource[]; config: ImageGenerationConfig }> => { +): Promise<{ sources: readonly PreparedImageSource[]; mask?: PreparedImageSource }> => { if (config.mask !== undefined && isRemoteImageSource(config.mask)) { throw new Error('Remote image mask reached edit dispatch before materialization'); } const originals = [...sources]; if (config.mask !== undefined && !originals.includes(config.mask)) originals.push(config.mask); const prepared = await prepareEditSources(originals); - const bySource = new Map(); + const bySource = new Map(); for (const [index, source] of originals.entries()) { const wireSource = prepared[index]; if (wireSource === undefined) throw new Error('Missing prepared image edit source'); @@ -276,10 +283,10 @@ const prepareEditRequest = async ( if (wireSource === undefined) throw new Error('Missing prepared image edit source'); return wireSource; }); - if (config.mask === undefined) return { sources: wireSources, config }; + if (config.mask === undefined) return { sources: wireSources }; const mask = bySource.get(config.mask); if (mask === undefined) throw new Error('Missing prepared image edit mask'); - return { sources: wireSources, config: { ...config, mask } }; + return { sources: wireSources, mask }; }; interface PrepareConfigError { @@ -913,7 +920,7 @@ const recordImageUsage = (state: ShimState, provider: Provider, model: ProviderM state.backgroundScheduler(promise); }; -export const buildGenerationsBody = (prompt: string, config: ImageGenerationConfig, stream: boolean): Record => ({ +const buildGenerationsBody = (prompt: string, config: ImageGenerationConfig, stream: boolean): Record => ({ prompt, // Public Responses tool config forbids `n`, but the private standalone // backend call always requests a single image, mirroring Azure's @@ -931,8 +938,14 @@ export const buildGenerationsBody = (prompt: string, config: ImageGenerationConf ...(stream ? { stream: true, partial_images: config.partial_images } : {}), }); -const buildEditsRequest = (prompt: string, config: ImageGenerationConfig, sources: readonly ImageSource[], stream: boolean): ImagesEditsRequest => { - const parameters: Record = { +const buildEditsRequest = ( + prompt: string, + config: ImageGenerationConfig, + sources: readonly PreparedImageSource[], + mask: PreparedImageSource | undefined, + stream: boolean, +): ImagesEditsRequest => { + const parameters: Record = { prompt, n: 1, ...(config.size === undefined ? {} : { size: config.size }), @@ -944,21 +957,12 @@ const buildEditsRequest = (prompt: string, config: ImageGenerationConfig, source ...(config.input_fidelity === undefined ? {} : { input_fidelity: config.input_fidelity }), ...(stream ? { stream: true, partial_images: config.partial_images } : {}), }; - const images = sources.map((source, index) => { - // Forward the canonical supported mime; an unsupported source is rejected - // before dispatch, so the fallback only ever carries an already-supported - // generated image. Sending the raw mime (rather than relabeling as png) - // keeps a stray unsupported byte stream failing loud at the backend. - const mime = editSupportedMime(source.mimeType) ?? source.mimeType; - return new File([source.bytes], `image_${index}.${editFileExt(mime)}`, { type: mime }); - }); - let mask: File | undefined; - if (config.mask !== undefined) { - if (isRemoteImageSource(config.mask)) throw new Error('Remote image mask reached edit request construction before materialization'); - const mime = editSupportedMime(config.mask.mimeType) ?? config.mask.mimeType; - mask = new File([config.mask.bytes], `mask.${editFileExt(mime)}`, { type: mime }); - } - return { type: 'uploads', images, ...(mask === undefined ? {} : { mask }), parameters }; + const images = sources.map((source, index) => + new File([source.bytes], `image_${index}.${editFileExt(source.mimeType)}`, { type: source.mimeType })); + const maskFile = mask === undefined + ? undefined + : new File([mask.bytes], `mask.${editFileExt(mask.mimeType)}`, { type: mask.mimeType }); + return { type: 'uploads', images, ...(maskFile === undefined ? {} : { mask: maskFile }), parameters }; }; const serverError = (e: unknown): ImageError => ({ @@ -1066,8 +1070,7 @@ const issueImageCall = async ( model: ProviderModel, fetcher: Fetcher, prompt: string, - isEdit: boolean, - sources: readonly ImageSource[], + editRequest: ImagesEditsRequest | null, config: ImageGenerationConfig, state: ShimState, stream: boolean, @@ -1085,9 +1088,9 @@ const issueImageCall = async ( // retry so it reflects the dispatch that actually returned. wrapUpstreamCall: stampUpstreamCallStart(attempt), }; - const { response, modelKey } = await (isEdit - ? provider.instance.callImagesEdits(model, buildEditsRequest(prompt, config, sources, stream), state.downstreamAbortSignal, opts) - : provider.instance.callImagesGenerations(model, buildGenerationsBody(prompt, config, stream), state.downstreamAbortSignal, opts)); + const { response, modelKey } = await (editRequest === null + ? provider.instance.callImagesGenerations(model, buildGenerationsBody(prompt, config, stream), state.downstreamAbortSignal, opts) + : provider.instance.callImagesEdits(model, editRequest, state.downstreamAbortSignal, opts)); if (response.status !== 429 || retry >= MAX_RATE_LIMIT_RETRIES) return { response, modelKey }; // 25% jitter desynchronizes parallel callers so a burst of orchestrator @@ -1253,17 +1256,18 @@ const streamImageGeneration = ( let response: Response; let modelKey: string; try { - const prepared = isEdit - ? await prepareEditRequest(sources, state.config) - : { sources, config: state.config }; + let editRequest: ImagesEditsRequest | null = null; + if (isEdit) { + const prepared = await prepareEditRequest(sources, state.config); + editRequest = buildEditsRequest(prompt, state.config, prepared.sources, prepared.mask, wantsPartials); + } ({ response, modelKey } = await issueImageCall( provider, model, fetcher, prompt, - isEdit, - prepared.sources, - prepared.config, + editRequest, + state.config, state, wantsPartials, attempt, diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation_test.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation_test.ts index 6bf80b562..c0f7bb3f6 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation_test.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation_test.ts @@ -1,7 +1,6 @@ import { beforeEach, test } from 'vitest'; import { - buildGenerationsBody, buildImageGenerationFunctionTool, createImageSourceInspector, inspectImageSources, @@ -13,7 +12,6 @@ import { isHostedImageGenerationTool, parseImageStreamEvent, parseRetryAfterMs, - prepareEditSources, prepareImageGenerationConfig, resolveImageOperation, SHIM_TOOL_NAME, @@ -24,7 +22,7 @@ import { initRepo } from '../../../../../repo/index.ts'; import { InMemoryRepo } from '../../../../../repo/memory.ts'; import { mockChatGatewayCtx } from '../../../../../test-helpers/gateway-ctx.ts'; import type { ResponsesInvocation } from '../types.ts'; -import { initExternalResourceFetcher, initImageProcessor } from '@floway-dev/platform'; +import { initExternalResourceFetcher } from '@floway-dev/platform'; import type { CanonicalResponsesPayload, ResponsesInputImage, ResponsesInputItem, ResponsesPayload, ResponsesTool } from '@floway-dev/protocols/responses'; import { assert, assertEquals, assertFalse, assertStringIncludes, assertThrows, stubModelCandidate } from '@floway-dev/test-utils'; @@ -352,30 +350,6 @@ test('inspectImageSources reads tool-result images and preserves forward order', assertEquals(third.mimeType, 'image/webp'); }); -test('prepareEditSources transcodes formats accepted by native Responses but rejected by images edits', async () => { - let observedTarget: unknown = undefined; - let processorCalls = 0; - initImageProcessor({ - compressToWebp: (_input, target) => { - processorCalls += 1; - observedTarget = target; - return Promise.resolve(Uint8Array.of(1, 2, 3)); - }, - }); - const first = { - bytes: Uint8Array.of(4, 5, 6).buffer, - mimeType: 'image/gif', - }; - const sameBytes = { bytes: Uint8Array.of(4, 5, 6).buffer, mimeType: 'image/gif' }; - const prepared = await prepareEditSources([first, first, sameBytes]); - assertEquals(processorCalls, 1); - assert(prepared[0] === prepared[1]); - assert(prepared[0] === prepared[2]); - assertEquals(prepared[0].mimeType, 'image/webp'); - assertEquals([...new Uint8Array(prepared[0].bytes)], [1, 2, 3]); - assertEquals(observedTarget, null); -}); - test('resolveImageOperation exposes a malformed replayed result as an invariant failure', () => { const inspection = inspectImageSources([ { type: 'image_generation_call', id: 'ig_bad', status: 'completed', result: '%%%' }, @@ -471,28 +445,6 @@ test('transformInputItemsForImageGeneration passes non-image items through untou assertEquals(out[0], message); }); -// ── buildGenerationsBody ── - -test('buildGenerationsBody always sends n:1 and maps config, omitting undefined', () => { - const config: ImageGenerationConfig = { model: 'gpt-image-2', size: '1024x1024', quality: 'low', action: 'generate' }; - const body = buildGenerationsBody('a cat', config, false); - assertEquals(body.prompt, 'a cat'); - assertEquals(body.n, 1); - assertEquals(body.size, '1024x1024'); - assertEquals(body.quality, 'low'); - assertFalse('background' in body); - assertFalse('output_format' in body); - assertFalse('stream' in body); - assertFalse('partial_images' in body); -}); - -test('buildGenerationsBody adds stream and partial_images when streaming', () => { - const config: ImageGenerationConfig = { model: 'gpt-image-2', partial_images: 2, action: 'generate' }; - const body = buildGenerationsBody('a cat', config, true); - assertEquals(body.stream, true); - assertEquals(body.partial_images, 2); -}); - // ── imageTerminal ── test('imageTerminal on success echoes the backend-resolved fields and closes with a single completed event', () => { diff --git a/packages/gateway/src/data-plane/images/serve.ts b/packages/gateway/src/data-plane/images/serve.ts index 5df97d157..f151de5cd 100644 --- a/packages/gateway/src/data-plane/images/serve.ts +++ b/packages/gateway/src/data-plane/images/serve.ts @@ -51,9 +51,10 @@ const imageEditReference = (value: unknown, path: string): ImageEditReference | if (value === null || typeof value !== 'object' || Array.isArray(value)) { return `${path} must be an object.`; } - const { image_url: imageUrl, file_id: fileId } = value as { image_url?: unknown; file_id?: unknown }; - if (typeof imageUrl === 'string' && fileId === undefined) return { image_url: imageUrl }; - if (typeof fileId === 'string' && imageUrl === undefined) return { file_id: fileId }; + const reference = value as { image_url?: unknown; file_id?: unknown }; + const { image_url: imageUrl, file_id: fileId } = reference; + if (typeof imageUrl === 'string' && fileId === undefined) return reference as ImageEditReference; + if (typeof fileId === 'string' && imageUrl === undefined) return reference as ImageEditReference; return `${path} must contain exactly one string field: image_url or file_id.`; }; @@ -170,7 +171,7 @@ export const imagesEdits = async (c: Context): Promise => { } const images: File[] = []; let mask: File | undefined; - const parameters: Record = {}; + const parameters: Record = {}; for (const [name, value] of form.entries()) { if (name === 'model') continue; if (name === 'image' || name === 'image[]') { diff --git a/packages/protocols/src/images/index.ts b/packages/protocols/src/images/index.ts index c65ab39f8..512f802c7 100644 --- a/packages/protocols/src/images/index.ts +++ b/packages/protocols/src/images/index.ts @@ -23,8 +23,8 @@ export interface ImagesGenerationsPayload { } export type ImageEditReference = - | { image_url: string; file_id?: never } - | { file_id: string; image_url?: never }; + | { image_url: string; file_id?: never; [key: string]: unknown } + | { file_id: string; image_url?: never; [key: string]: unknown }; // POST /v1/images/edits accepts either multipart uploads or this JSON shape. // JSON uses `images` rather than the multipart `image` field, and references @@ -36,16 +36,16 @@ export interface ImagesEditsJsonPayload { prompt: string; images: ImageEditReference[]; mask?: ImageEditReference; - n?: number; - quality?: string; - input_fidelity?: string; - size?: string; + n?: number | null; + quality?: string | null; + input_fidelity?: string | null; + size?: string | null; user?: string; - output_format?: string; - output_compression?: number; - moderation?: string; - background?: string; - stream?: boolean; - partial_images?: number; + output_format?: string | null; + output_compression?: number | null; + moderation?: string | null; + background?: string | null; + stream?: boolean | null; + partial_images?: number | null; [key: string]: unknown; } diff --git a/packages/provider/src/images.ts b/packages/provider/src/images.ts index 90f815d2e..e2af4a279 100644 --- a/packages/provider/src/images.ts +++ b/packages/provider/src/images.ts @@ -3,20 +3,18 @@ import type { ImageEditReference } from '@floway-dev/protocols/images'; // The gateway normalizes both public wire formats into one homogeneous source // kind. Providers serialize references as JSON and uploads as multipart // without downloading URLs or base64-encoding uploaded bytes. -interface ImagesEditsRequestBase { - parameters: Record; -} - -export interface UploadedImagesEditsRequest extends ImagesEditsRequestBase { +export interface UploadedImagesEditsRequest { type: 'uploads'; images: File[]; mask?: File; + parameters: Record; } -export interface ReferencedImagesEditsRequest extends ImagesEditsRequestBase { +export interface ReferencedImagesEditsRequest { type: 'references'; images: ImageEditReference[]; mask?: ImageEditReference; + parameters: Record; } export type ImagesEditsRequest = UploadedImagesEditsRequest | ReferencedImagesEditsRequest; @@ -30,9 +28,6 @@ export const imagesEditsJsonBody = (request: ReferencedImagesEditsRequest): Reco export const imagesEditsMultipartBody = (request: UploadedImagesEditsRequest, model: string): FormData => { const form = new FormData(); for (const [name, value] of Object.entries(request.parameters)) { - if (typeof value !== 'string' && typeof value !== 'number' && typeof value !== 'boolean') { - throw new Error(`Multipart image edits parameter ${name} must be a string, number, or boolean`); - } form.append(name, String(value)); } for (const image of request.images) form.append('image[]', image); From 277f93627d10a825ca74043f5acf11be5dc26fba Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 15 Jul 2026 14:20:50 +0800 Subject: [PATCH 17/24] refactor(images): complete edit request invariants Separate scalar upload parameters from opaque JSON parameters, preserve unknown reference fields and nullable wire values, and keep post-materialization hosted-tool state typed. Serialize single uploads with the canonical image field while retaining image[] for multiple uploads. --- .../server-tools/image-generation.ts | 65 +++++++++++-------- .../providers/custom/provider_test.ts | 2 +- packages/provider/src/images.ts | 9 +-- packages/provider/src/images_test.ts | 40 ++++++++++++ 4 files changed, 83 insertions(+), 33 deletions(-) create mode 100644 packages/provider/src/images_test.ts diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts index 604602d1a..1850d93cc 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts @@ -262,13 +262,12 @@ export interface ImageGenerationConfig { action: 'generate' | 'edit' | 'auto'; } +type MaterializedImageGenerationConfig = Omit & { mask?: ImageSource }; + const prepareEditRequest = async ( sources: readonly ImageSource[], - config: ImageGenerationConfig, + config: MaterializedImageGenerationConfig, ): Promise<{ sources: readonly PreparedImageSource[]; mask?: PreparedImageSource }> => { - if (config.mask !== undefined && isRemoteImageSource(config.mask)) { - throw new Error('Remote image mask reached edit dispatch before materialization'); - } const originals = [...sources]; if (config.mask !== undefined && !originals.includes(config.mask)) originals.push(config.mask); const prepared = await prepareEditSources(originals); @@ -897,7 +896,7 @@ const errorFromBody = (body: string, status: number): { type?: string; code: str // in a later turn. `imageDispatchCount` bounds how many real backend image // calls one response may issue. interface ShimState { - config: ImageGenerationConfig; + config: MaterializedImageGenerationConfig; apiKeyId: string; upstreamIds: readonly string[] | null; backgroundScheduler: BackgroundScheduler; @@ -1410,7 +1409,7 @@ export const imageGenerationServerTool: ServerToolRegistration = async (invocati if (!prepared.ok) { return { type: 'invalid-request', message: prepared.error.message, param: prepared.error.param, code: prepared.error.code }; } - let config = prepared.config; + const config = prepared.config; const inspectSources = createImageSourceInspector(); const initialInspection = inspectSources(invocation.payload.input); const initialOperation = resolveImageOperation(config, initialInspection); @@ -1436,31 +1435,41 @@ export const imageGenerationServerTool: ServerToolRegistration = async (invocati code: materializedInputs.error.code, }; } - if (config.mask !== undefined && isRemoteImageSource(config.mask)) { - const remoteMask = config.mask; - const materializedMask = await materializer.mask(remoteMask); - if (!materializedMask.ok) { - return { - type: 'invalid-request', - message: materializedMask.error.message, - param: materializedMask.error.param, - errorType: materializedMask.error.errorType, - code: materializedMask.error.code, - }; - } - if (remoteMask.afterMaterializationError !== undefined) { - return { - type: 'invalid-request', - message: remoteMask.afterMaterializationError.message, - param: remoteMask.afterMaterializationError.param, - code: remoteMask.afterMaterializationError.code, - }; + let mask: ImageSource | undefined; + if (config.mask !== undefined) { + if (isRemoteImageSource(config.mask)) { + const remoteMask = config.mask; + const materializedMask = await materializer.mask(remoteMask); + if (!materializedMask.ok) { + return { + type: 'invalid-request', + message: materializedMask.error.message, + param: materializedMask.error.param, + errorType: materializedMask.error.errorType, + code: materializedMask.error.code, + }; + } + if (remoteMask.afterMaterializationError !== undefined) { + return { + type: 'invalid-request', + message: remoteMask.afterMaterializationError.message, + param: remoteMask.afterMaterializationError.param, + code: remoteMask.afterMaterializationError.code, + }; + } + mask = materializedMask.source; + } else { + mask = config.mask; } - config = { ...config, mask: materializedMask.source }; } + const { mask: _unmaterializedMask, ...configWithoutMask } = config; + const materializedConfig: MaterializedImageGenerationConfig = { + ...configWithoutMask, + ...(mask === undefined ? {} : { mask }), + }; const state: ShimState = { - config, + config: materializedConfig, apiKeyId: gatewayCtx.apiKeyId, upstreamIds: gatewayCtx.upstreamIds, backgroundScheduler: gatewayCtx.backgroundScheduler, @@ -1487,7 +1496,7 @@ export const imageGenerationServerTool: ServerToolRegistration = async (invocati // to editing without bypassing the same source policy used at ingress. // The per-request inspector cache reuses bytes already decoded during // registration and earlier turns. - const operation = resolveImageOperation(config, inspectSources(invocation.payload.input)); + const operation = resolveImageOperation(materializedConfig, inspectSources(invocation.payload.input)); if (!operation.ok) { throw new Error(`image_generation live source invariant violated after request validation: ${operation.error.message}`); } diff --git a/packages/gateway/src/data-plane/providers/custom/provider_test.ts b/packages/gateway/src/data-plane/providers/custom/provider_test.ts index 940c349fd..a5ba32679 100644 --- a/packages/gateway/src/data-plane/providers/custom/provider_test.ts +++ b/packages/gateway/src/data-plane/providers/custom/provider_test.ts @@ -238,7 +238,7 @@ test('Custom provider callImagesEdits forwards multipart body with model field a assertExists(forwarded); assertEquals(forwarded.form.get('model'), 'gpt-image-2'); assertEquals(forwarded.form.get('prompt'), 'add a kite'); - assertEquals(forwarded.form.get('image[]') instanceof File, true); + assertEquals(forwarded.form.get('image') instanceof File, true); }); test('Custom provider with modelsFetch disabled serves only manual models and never fetches', async () => { diff --git a/packages/provider/src/images.ts b/packages/provider/src/images.ts index e2af4a279..55d56b152 100644 --- a/packages/provider/src/images.ts +++ b/packages/provider/src/images.ts @@ -1,8 +1,8 @@ import type { ImageEditReference } from '@floway-dev/protocols/images'; -// The gateway normalizes both public wire formats into one homogeneous source -// kind. Providers serialize references as JSON and uploads as multipart -// without downloading URLs or base64-encoding uploaded bytes. +// The gateway normalizes each public wire format into one of two semantic +// source kinds. Providers serialize references as JSON and uploads as +// multipart without downloading URLs or base64-encoding uploaded bytes. export interface UploadedImagesEditsRequest { type: 'uploads'; images: File[]; @@ -30,7 +30,8 @@ export const imagesEditsMultipartBody = (request: UploadedImagesEditsRequest, mo for (const [name, value] of Object.entries(request.parameters)) { form.append(name, String(value)); } - for (const image of request.images) form.append('image[]', image); + const imageField = request.images.length === 1 ? 'image' : 'image[]'; + for (const image of request.images) form.append(imageField, image); if (request.mask !== undefined) form.append('mask', request.mask); form.append('model', model); return form; diff --git a/packages/provider/src/images_test.ts b/packages/provider/src/images_test.ts new file mode 100644 index 000000000..396f88152 --- /dev/null +++ b/packages/provider/src/images_test.ts @@ -0,0 +1,40 @@ +import { test } from 'vitest'; + +import { imagesEditsJsonBody, imagesEditsMultipartBody } from './images.ts'; +import { assertEquals } from '@floway-dev/test-utils'; + +test('imagesEditsJsonBody preserves reference fields and nullable parameters', () => { + const body = imagesEditsJsonBody({ + type: 'references', + images: [{ image_url: 'https://example.test/image.png', detail: 'future-field' }], + mask: { file_id: 'file-mask' }, + parameters: { prompt: 'edit', background: null }, + }); + assertEquals(body, { + prompt: 'edit', + background: null, + images: [{ image_url: 'https://example.test/image.png', detail: 'future-field' }], + mask: { file_id: 'file-mask' }, + }); +}); + +test('imagesEditsMultipartBody uses the singular field for one upload and the array field for many', () => { + const first = new File(['first'], 'first.png', { type: 'image/png' }); + const second = new File(['second'], 'second.png', { type: 'image/png' }); + const single = imagesEditsMultipartBody({ + type: 'uploads', + images: [first], + parameters: { prompt: 'single' }, + }, 'gpt-image'); + assertEquals(single.get('image'), first); + assertEquals(single.getAll('image[]'), []); + + const multiple = imagesEditsMultipartBody({ + type: 'uploads', + images: [first, second], + parameters: { prompt: 'multiple' }, + }, 'gpt-image'); + assertEquals(multiple.getAll('image[]'), [first, second]); + assertEquals(multiple.get('image'), null); + assertEquals(multiple.get('model'), 'gpt-image'); +}); From 0b8cdd447ee92fb2f59dfc5c8b51150b5f457e57 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 15 Jul 2026 15:02:07 +0800 Subject: [PATCH 18/24] refactor(images): classify inline data as uploads Decode base64 data URLs into the upload source variant and keep one semantic truth per image. Providers prefer multipart for upload-only requests, use JSON for references, and encode uploads from bytes only when a mixed request requires JSON. --- .../image-generation-integration_test.ts | 29 +++---- .../server-tools/image-generation.ts | 12 ++- .../src/data-plane/codex/images_test.ts | 37 +++++++++ .../gateway/src/data-plane/images/serve.ts | 51 +++++++----- .../src/data-plane/images/serve_test.ts | 12 ++- .../providers/azure/provider_test.ts | 6 +- .../providers/custom/provider_test.ts | 6 +- packages/provider-azure/src/provider.ts | 18 +++-- packages/provider-custom/src/provider.ts | 18 +++-- packages/provider/src/images.ts | 78 +++++++++++++------ packages/provider/src/images_test.ts | 23 +++--- packages/provider/src/index.ts | 4 +- 12 files changed, 207 insertions(+), 87 deletions(-) diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation-integration_test.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation-integration_test.ts index c0348958f..82776e6b6 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation-integration_test.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation-integration_test.ts @@ -305,9 +305,10 @@ test('an image generated in turn 1 is re-collected as an edit source in turn 2', assertEquals(stub.generationsCalls.length, 1); assertEquals(stub.editsRequests.length, 1); const request = stub.editsRequests[0]; - assert(request.type === 'uploads'); assertEquals(request.images.length, 1); - const bytes = await request.images[0].text(); + const image = request.images[0]; + assert(image.type === 'upload'); + const bytes = await image.file.text(); assertEquals(bytes, 'AAAA'); }); @@ -345,9 +346,9 @@ test('a prefetched remote edit source remains visible to orchestration and is re assertEquals(orchestratorImageUrl, 'https://example.com/source.png'); assertEquals(stub.editsRequests.length, 1); const request = stub.editsRequests[0]; - assert(request.type === 'uploads'); const image = request.images[0]; - assertEquals(new Uint8Array(await image.arrayBuffer()), Uint8Array.from(atob(REMOTE_PNG_B64), c => c.charCodeAt(0))); + assert(image.type === 'upload'); + assertEquals(new Uint8Array(await image.file.arrayBuffer()), Uint8Array.from(atob(REMOTE_PNG_B64), c => c.charCodeAt(0))); }); test('mask-only GIF edit transcodes one shared image and mask to WebP', async () => { @@ -371,14 +372,14 @@ test('mask-only GIF edit transcodes one shared image and mask to WebP', async () assertEquals(processorCalls, 1); assertEquals(stub.editsRequests.length, 1); const request = stub.editsRequests[0]; - assert(request.type === 'uploads'); const image = request.images[0]; const mask = request.mask; - assert(mask !== undefined); - assertEquals(image.type, 'image/webp'); - assertEquals(mask.type, 'image/webp'); - assertEquals(await image.text(), 'WEBP'); - assertEquals(await mask.text(), 'WEBP'); + assert(image.type === 'upload'); + assert(mask?.type === 'upload'); + assertEquals(image.file.type, 'image/webp'); + assertEquals(mask.file.type, 'image/webp'); + assertEquals(await image.file.text(), 'WEBP'); + assertEquals(await mask.file.text(), 'WEBP'); }); test('identical GIF source and mask share one transcode', async () => { @@ -404,9 +405,11 @@ test('identical GIF source and mask share one transcode', async () => { assertEquals(processorCalls, 1); const request = stub.editsRequests[0]; - assert(request.type === 'uploads'); - assertEquals(await request.images[0].text(), 'WEBP'); - assertEquals(await request.mask?.text(), 'WEBP'); + const image = request.images[0]; + assert(image.type === 'upload'); + assert(request.mask?.type === 'upload'); + assertEquals(await image.file.text(), 'WEBP'); + assertEquals(await request.mask.file.text(), 'WEBP'); }); test('image transcoding failure becomes a terminal image tool failure', async () => { diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts index 1850d93cc..e9d6fa322 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts @@ -956,12 +956,18 @@ const buildEditsRequest = ( ...(config.input_fidelity === undefined ? {} : { input_fidelity: config.input_fidelity }), ...(stream ? { stream: true, partial_images: config.partial_images } : {}), }; - const images = sources.map((source, index) => - new File([source.bytes], `image_${index}.${editFileExt(source.mimeType)}`, { type: source.mimeType })); + const images = sources.map((source, index) => ({ + type: 'upload' as const, + file: new File([source.bytes], `image_${index}.${editFileExt(source.mimeType)}`, { type: source.mimeType }), + })); const maskFile = mask === undefined ? undefined : new File([mask.bytes], `mask.${editFileExt(mask.mimeType)}`, { type: mask.mimeType }); - return { type: 'uploads', images, ...(maskFile === undefined ? {} : { mask: maskFile }), parameters }; + return { + images, + ...(maskFile === undefined ? {} : { mask: { type: 'upload' as const, file: maskFile } }), + parameters, + }; }; const serverError = (e: unknown): ImageError => ({ diff --git a/packages/gateway/src/data-plane/codex/images_test.ts b/packages/gateway/src/data-plane/codex/images_test.ts index 95e0285ea..e1d257261 100644 --- a/packages/gateway/src/data-plane/codex/images_test.ts +++ b/packages/gateway/src/data-plane/codex/images_test.ts @@ -4,6 +4,8 @@ import type { InMemoryRepo } from '../../repo/memory.ts'; import { copilotModels, requestApp, setupAppTest } from '../../test-helpers.ts'; import { assertEquals, assertExists, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; +const PNG_B64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M8AAAMBAQDJ/wEAAAAASUVORK5CYII='; + const saveAzureImages = async (repo: InMemoryRepo): Promise => { await repo.upstreams.save({ id: 'az-image', @@ -122,3 +124,38 @@ test('Codex provider-relative image edits reuse the public JSON handler', async images: [{ image_url: 'https://assets.example/image.png' }], }); }); + +test('Codex inline data URL edits egress as multipart uploads', async () => { + const { apiKey, repo } = await setupAppTest(); + await saveAzureImages(repo); + let observedForm: FormData | undefined; + + await withMockedFetch( + async request => { + const control = controlPlaneFetch(request); + if (control) return control; + if (new URL(request.url).hostname === 'example.openai.azure.com') { + observedForm = await request.formData(); + return jsonResponse({ data: [{ b64_json: 'ZWRpdA==' }] }); + } + throw new Error(`Unhandled fetch ${request.url}`); + }, + async () => { + const response = await requestApp('/azure-api.codex/images/edits', { + method: 'POST', + headers: { authorization: `Bearer ${apiKey.key}`, 'content-type': 'application/json' }, + body: JSON.stringify({ + model: 'gpt-image-2', + prompt: 'add a red hat', + images: [{ image_url: `data:image/png;base64,${PNG_B64}` }], + }), + }); + assertEquals(response.status, 200); + }, + ); + + assertExists(observedForm); + const image = observedForm.get('image'); + assertEquals(image instanceof File, true); + assertEquals((image as File).type, 'image/png'); +}); diff --git a/packages/gateway/src/data-plane/images/serve.ts b/packages/gateway/src/data-plane/images/serve.ts index f151de5cd..18c8350e1 100644 --- a/packages/gateway/src/data-plane/images/serve.ts +++ b/packages/gateway/src/data-plane/images/serve.ts @@ -15,7 +15,7 @@ import { readRequestBody, takeRequestBody, type RequestBody } from '../chat/shar import { passthroughApiError, passthroughServe } from '../shared/passthrough-serve.ts'; import { tokenUsageFromImagesBody } from '../shared/telemetry/usage.ts'; import type { ImageEditReference } from '@floway-dev/protocols/images'; -import type { ImagesEditsRequest } from '@floway-dev/provider'; +import type { ImagesEditsRequest, ImagesEditsSource } from '@floway-dev/provider'; interface JsonModelRequestBody { model?: unknown; @@ -43,42 +43,56 @@ const prepareJsonModelRequest = (bytes: Uint8Array, requestName: string): Prepar return { type: 'ok', body: request as Record, model: request.model }; }; -type PreparedReferencedImagesEdit = +type PreparedImagesEdit = | { type: 'ok'; request: ImagesEditsRequest } | { type: 'invalid'; message: string }; -const imageEditReference = (value: unknown, path: string): ImageEditReference | string => { +const imageEditSource = (value: unknown, path: string): ImagesEditsSource | string => { if (value === null || typeof value !== 'object' || Array.isArray(value)) { return `${path} must be an object.`; } const reference = value as { image_url?: unknown; file_id?: unknown }; const { image_url: imageUrl, file_id: fileId } = reference; - if (typeof imageUrl === 'string' && fileId === undefined) return reference as ImageEditReference; - if (typeof fileId === 'string' && imageUrl === undefined) return reference as ImageEditReference; + if (typeof imageUrl === 'string' && fileId === undefined) { + if (!imageUrl.startsWith('data:')) { + return { type: 'reference', reference: reference as ImageEditReference }; + } + const match = /^data:([^;,]+);base64,(.*)$/su.exec(imageUrl); + if (match === null || !match[1]!.startsWith('image/')) return `${path}.image_url must be a base64 image data URL.`; + try { + const bytes = Uint8Array.from(atob(match[2]!), character => character.charCodeAt(0)); + const extension = match[1]!.slice('image/'.length).replace(/[^a-z0-9]+/giu, '-'); + return { type: 'upload', file: new File([bytes], `${path}.${extension}`, { type: match[1] }) }; + } catch { + return `${path}.image_url must contain valid base64 image data.`; + } + } + if (typeof fileId === 'string' && imageUrl === undefined) { + return { type: 'reference', reference: reference as ImageEditReference }; + } return `${path} must contain exactly one string field: image_url or file_id.`; }; -const prepareReferencedImagesEdit = (body: Record): PreparedReferencedImagesEdit => { +const prepareJsonImagesEdit = (body: Record): PreparedImagesEdit => { if (!Array.isArray(body.images)) { return { type: 'invalid', message: 'Image edits request body must include an images array.' }; } - const images: ImageEditReference[] = []; + const images: ImagesEditsSource[] = []; for (const [index, value] of body.images.entries()) { - const reference = imageEditReference(value, `Image edits images[${index}]`); - if (typeof reference === 'string') return { type: 'invalid', message: reference }; - images.push(reference); + const source = imageEditSource(value, `Image edits images[${index}]`); + if (typeof source === 'string') return { type: 'invalid', message: source }; + images.push(source); } - let mask: ImageEditReference | undefined; + let mask: ImagesEditsSource | undefined; if (body.mask !== undefined) { - const reference = imageEditReference(body.mask, 'Image edits mask'); - if (typeof reference === 'string') return { type: 'invalid', message: reference }; - mask = reference; + const source = imageEditSource(body.mask, 'Image edits mask'); + if (typeof source === 'string') return { type: 'invalid', message: source }; + mask = source; } const { model: _model, images: _images, mask: _mask, ...parameters } = body; return { type: 'ok', request: { - type: 'references', images, ...(mask === undefined ? {} : { mask }), parameters, @@ -151,7 +165,7 @@ export const imagesEdits = async (c: Context): Promise => { if (mediaType === 'application/json') { const body = prepareJsonModelRequest(requestBody.bytes, 'Image edits'); if (body.type === 'invalid') return invalid(body.message); - const request = prepareReferencedImagesEdit(body.body); + const request = prepareJsonImagesEdit(body.body); if (request.type === 'invalid') return invalid(request.message); return await serveImagesEditRequest(c, requestBody, body.model, request.request); } @@ -186,9 +200,8 @@ export const imagesEdits = async (c: Context): Promise => { } } return await serveImagesEditRequest(c, requestBody, model, { - type: 'uploads', - images, - ...(mask === undefined ? {} : { mask }), + images: images.map(file => ({ type: 'upload', file })), + ...(mask === undefined ? {} : { mask: { type: 'upload' as const, file: mask } }), parameters, }); }; diff --git a/packages/gateway/src/data-plane/images/serve_test.ts b/packages/gateway/src/data-plane/images/serve_test.ts index d17ff3aab..2898099ae 100644 --- a/packages/gateway/src/data-plane/images/serve_test.ts +++ b/packages/gateway/src/data-plane/images/serve_test.ts @@ -4,6 +4,8 @@ import { buildCustomUpstreamRecord, copilotModels, flushAsyncWork, requestApp, s import { clearInProcessCopilotTokenCache } from '@floway-dev/provider-copilot'; import { jsonResponse, withMockedFetch, assertEquals, assertExists } from '@floway-dev/test-utils'; +const PNG_B64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M8AAAMBAQDJ/wEAAAAASUVORK5CYII='; + test('/v1/images/generations rejects malformed JSON body with 400', async () => { const { apiKey } = await setupAppTest(); const response = await requestApp('/v1/images/generations', { @@ -288,7 +290,10 @@ test('/v1/images/edits forwards JSON image references through a custom provider' body: JSON.stringify({ model: 'gpt-image-2', prompt: 'replace the background', - images: [{ file_id: 'file-source' }], + images: [ + { image_url: `data:image/png;base64,${PNG_B64}` }, + { file_id: 'file-source' }, + ], mask: { file_id: 'file-mask' }, quality: 'high', }), @@ -301,7 +306,10 @@ test('/v1/images/edits forwards JSON image references through a custom provider' assertEquals(forwarded, { model: 'gpt-image-2', prompt: 'replace the background', - images: [{ file_id: 'file-source' }], + images: [ + { image_url: `data:image/png;base64,${PNG_B64}` }, + { file_id: 'file-source' }, + ], mask: { file_id: 'file-mask' }, quality: 'high', }); diff --git a/packages/gateway/src/data-plane/providers/azure/provider_test.ts b/packages/gateway/src/data-plane/providers/azure/provider_test.ts index 5a0fc3cb2..0014e0851 100644 --- a/packages/gateway/src/data-plane/providers/azure/provider_test.ts +++ b/packages/gateway/src/data-plane/providers/azure/provider_test.ts @@ -433,9 +433,11 @@ test('createAzureProvider callImagesEdits posts multipart with model replaced by const provider = createAzureProvider(record); const models = await provider.instance.getProvidedModels(directFetcher); const result = await provider.instance.callImagesEdits(models[0], { - type: 'uploads', parameters: { prompt: 'replace sky' }, - images: [new File([new Uint8Array([1, 2, 3])], 'photo.png', { type: 'image/png' })], + images: [{ + type: 'upload', + file: new File([new Uint8Array([1, 2, 3])], 'photo.png', { type: 'image/png' }), + }], }, undefined, noopUpstreamCallOptions()); assertEquals(result.modelKey, 'gpt-image-2'); assertEquals(result.response.status, 200); diff --git a/packages/gateway/src/data-plane/providers/custom/provider_test.ts b/packages/gateway/src/data-plane/providers/custom/provider_test.ts index a5ba32679..e4f2172fe 100644 --- a/packages/gateway/src/data-plane/providers/custom/provider_test.ts +++ b/packages/gateway/src/data-plane/providers/custom/provider_test.ts @@ -227,9 +227,11 @@ test('Custom provider callImagesEdits forwards multipart body with model field a const models = await provider.instance.getProvidedModels(directFetcher); const model = models[0]; const opts = noopUpstreamCallOptions(); const result = await provider.instance.callImagesEdits(model, { - type: 'uploads', parameters: { prompt: 'add a kite' }, - images: [new File([new Uint8Array([1, 2, 3])], 'photo.png', { type: 'image/png' })], + images: [{ + type: 'upload', + file: new File([new Uint8Array([1, 2, 3])], 'photo.png', { type: 'image/png' }), + }], }, undefined, opts); assertEquals(result.modelKey, 'gpt-image-2'); assertEquals(result.response.status, 200); diff --git a/packages/provider-azure/src/provider.ts b/packages/provider-azure/src/provider.ts index 1f8843411..b08cad0a3 100644 --- a/packages/provider-azure/src/provider.ts +++ b/packages/provider-azure/src/provider.ts @@ -5,7 +5,7 @@ import { parseChatCompletionsStream } from '@floway-dev/protocols/chat-completio import { kindForEndpoints } from '@floway-dev/protocols/common'; import { parseMessagesStream } from '@floway-dev/protocols/messages'; import { parseResponsesStream, type ResponsesResult, toCompactPayloadShape } from '@floway-dev/protocols/responses'; -import { imagesEditsJsonBody, imagesEditsMultipartBody, type ProviderInstance, type Provider, type ProviderModel, type ProviderStreamParser, type UpstreamCallOptions, type UpstreamFetchOptions, type UpstreamRecord, publicModelId, resolveEffectiveFlags, streamingProviderCall } from '@floway-dev/provider'; +import { imagesEditsCanUseMultipart, imagesEditsJsonBody, imagesEditsMultipartBody, type ProviderInstance, type Provider, type ProviderModel, type ProviderStreamParser, type UpstreamCallOptions, type UpstreamFetchOptions, type UpstreamRecord, publicModelId, resolveEffectiveFlags, streamingProviderCall } from '@floway-dev/provider'; const upstreamModelIdOf = (model: ProviderModel): string => (model.providerData as { upstreamModelId: string }).upstreamModelId; @@ -91,12 +91,18 @@ export const createAzureProvider = (record: UpstreamRecord): Provider => { callEmbeddings: (model, body, signal, opts) => callNonStreaming(azureFetchEmbeddings, model, body, signal, opts.headers, opts), callImagesGenerations: (model, body, signal, opts) => callNonStreaming(azureFetchImagesGenerations, model, body, signal, opts.headers, opts), callImagesEdits: async (model, request, signal, opts) => { - if (request.type === 'references') { - return await callNonStreaming(azureFetchImagesEdits, model, imagesEditsJsonBody(request), signal, opts.headers, opts); - } const upstreamModelId = upstreamModelIdOf(model); - const body = imagesEditsMultipartBody(request, upstreamModelId); - const response = await azureFetchImagesEdits(azure.config, { method: 'POST', body, signal }, { extraHeaders: opts.headers, fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }); + if (imagesEditsCanUseMultipart(request)) { + const body = imagesEditsMultipartBody(request, upstreamModelId); + const response = await azureFetchImagesEdits(azure.config, { method: 'POST', body, signal }, { extraHeaders: opts.headers, fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }); + return { response, modelKey: upstreamModelId }; + } + const body = await imagesEditsJsonBody(request); + const response = await azureFetchImagesEdits( + azure.config, + { method: 'POST', body: JSON.stringify({ ...body, model: upstreamModelId }), signal }, + { extraHeaders: opts.headers, fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }, + ); return { response, modelKey: upstreamModelId }; }, }; diff --git a/packages/provider-custom/src/provider.ts b/packages/provider-custom/src/provider.ts index 9cc7b871d..68897c644 100644 --- a/packages/provider-custom/src/provider.ts +++ b/packages/provider-custom/src/provider.ts @@ -7,7 +7,7 @@ import { parseChatCompletionsStream } from '@floway-dev/protocols/chat-completio import { type ModelEndpoints, kindForEndpoints } from '@floway-dev/protocols/common'; import { parseMessagesStream } from '@floway-dev/protocols/messages'; import { parseResponsesStream, type ResponsesResult, toCompactPayloadShape } from '@floway-dev/protocols/responses'; -import { imagesEditsJsonBody, imagesEditsMultipartBody, publicModelId, resolveEffectiveFlags, streamingProviderCall, type FlagId, type ProviderInstance, type Provider, type ProviderCallResult, type ProviderModel, type ProviderStreamParser, type UpstreamCallOptions, type UpstreamFetchOptions, type UpstreamRecord } from '@floway-dev/provider'; +import { imagesEditsCanUseMultipart, imagesEditsJsonBody, imagesEditsMultipartBody, publicModelId, resolveEffectiveFlags, streamingProviderCall, type FlagId, type ProviderInstance, type Provider, type ProviderCallResult, type ProviderModel, type ProviderStreamParser, type UpstreamCallOptions, type UpstreamFetchOptions, type UpstreamRecord } from '@floway-dev/provider'; const rawModelIdOf = (model: ProviderModel): string => model.providerData as string; @@ -179,12 +179,18 @@ export const createCustomProvider = (record: UpstreamRecord): Provider => { callEmbeddings: (model, body, signal, opts) => call(customFetchEmbeddings, model, body, signal, opts.headers, opts), callImagesGenerations: (model, body, signal, opts) => call(customFetchImagesGenerations, model, body, signal, opts.headers, opts), callImagesEdits: async (model, request, signal, opts) => { - if (request.type === 'references') { - return await call(customFetchImagesEdits, model, imagesEditsJsonBody(request), signal, opts.headers, opts); - } const rawModelId = rawModelIdOf(model); - const body = imagesEditsMultipartBody(request, rawModelId); - const response = await customFetchImagesEdits(config, { method: 'POST', body, signal }, { extraHeaders: opts.headers, fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }); + if (imagesEditsCanUseMultipart(request)) { + const body = imagesEditsMultipartBody(request, rawModelId); + const response = await customFetchImagesEdits(config, { method: 'POST', body, signal }, { extraHeaders: opts.headers, fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }); + return { response, modelKey: rawModelId }; + } + const body = await imagesEditsJsonBody(request); + const response = await customFetchImagesEdits( + config, + { method: 'POST', body: JSON.stringify({ ...body, model: rawModelId }), signal }, + { extraHeaders: opts.headers, fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }, + ); return { response, modelKey: rawModelId }; }, }; diff --git a/packages/provider/src/images.ts b/packages/provider/src/images.ts index 55d56b152..2c77ea244 100644 --- a/packages/provider/src/images.ts +++ b/packages/provider/src/images.ts @@ -1,38 +1,72 @@ import type { ImageEditReference } from '@floway-dev/protocols/images'; -// The gateway normalizes each public wire format into one of two semantic -// source kinds. Providers serialize references as JSON and uploads as -// multipart without downloading URLs or base64-encoding uploaded bytes. -export interface UploadedImagesEditsRequest { - type: 'uploads'; - images: File[]; - mask?: File; - parameters: Record; +// Inline data URLs are decoded into uploads before reaching this boundary. +// Providers prefer multipart when every source is an upload and use JSON for +// external references or mixed requests. +interface UploadedImagesEditsSource { + type: 'upload'; + file: File; +} + +interface ReferencedImagesEditsSource { + type: 'reference'; + reference: ImageEditReference; } -export interface ReferencedImagesEditsRequest { - type: 'references'; - images: ImageEditReference[]; - mask?: ImageEditReference; +export type ImagesEditsSource = UploadedImagesEditsSource | ReferencedImagesEditsSource; + +export interface ImagesEditsRequest { + images: ImagesEditsSource[]; + mask?: ImagesEditsSource; parameters: Record; } -export type ImagesEditsRequest = UploadedImagesEditsRequest | ReferencedImagesEditsRequest; +interface MultipartImagesEditsRequest extends ImagesEditsRequest { + images: UploadedImagesEditsSource[]; + mask?: UploadedImagesEditsSource; + parameters: Record; +} + +export const imagesEditsCanUseMultipart = (request: ImagesEditsRequest): request is MultipartImagesEditsRequest => { + const sources = [...request.images, ...(request.mask === undefined ? [] : [request.mask])]; + return sources.every(source => source.type === 'upload') + && Object.values(request.parameters).every(value => + typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean'); +}; + +const bytesToBase64 = (bytes: Uint8Array): string => { + const chunks: string[] = []; + for (let offset = 0; offset < bytes.length; offset += 32_768) { + chunks.push(String.fromCharCode(...bytes.subarray(offset, offset + 32_768))); + } + return btoa(chunks.join('')); +}; -export const imagesEditsJsonBody = (request: ReferencedImagesEditsRequest): Record => ({ - ...request.parameters, - images: request.images, - ...(request.mask === undefined ? {} : { mask: request.mask }), -}); +const jsonReference = async (source: ImagesEditsSource): Promise => { + if (source.type === 'reference') return source.reference; + const bytes = new Uint8Array(await source.file.arrayBuffer()); + return { image_url: `data:${source.file.type};base64,${bytesToBase64(bytes)}` }; +}; + +export const imagesEditsJsonBody = async (request: ImagesEditsRequest): Promise> => { + const images = await Promise.all(request.images.map(jsonReference)); + const mask = request.mask === undefined ? undefined : await jsonReference(request.mask); + return { + ...request.parameters, + images, + ...(mask === undefined ? {} : { mask }), + }; +}; -export const imagesEditsMultipartBody = (request: UploadedImagesEditsRequest, model: string): FormData => { +export const imagesEditsMultipartBody = (request: MultipartImagesEditsRequest, model: string): FormData => { const form = new FormData(); for (const [name, value] of Object.entries(request.parameters)) { form.append(name, String(value)); } - const imageField = request.images.length === 1 ? 'image' : 'image[]'; - for (const image of request.images) form.append(imageField, image); - if (request.mask !== undefined) form.append('mask', request.mask); + const uploads = request.images.map(source => source.file); + const imageField = uploads.length === 1 ? 'image' : 'image[]'; + for (const image of uploads) form.append(imageField, image); + if (request.mask !== undefined) form.append('mask', request.mask.file); form.append('model', model); return form; }; diff --git a/packages/provider/src/images_test.ts b/packages/provider/src/images_test.ts index 396f88152..9ec337c92 100644 --- a/packages/provider/src/images_test.ts +++ b/packages/provider/src/images_test.ts @@ -3,17 +3,22 @@ import { test } from 'vitest'; import { imagesEditsJsonBody, imagesEditsMultipartBody } from './images.ts'; import { assertEquals } from '@floway-dev/test-utils'; -test('imagesEditsJsonBody preserves reference fields and nullable parameters', () => { - const body = imagesEditsJsonBody({ - type: 'references', - images: [{ image_url: 'https://example.test/image.png', detail: 'future-field' }], - mask: { file_id: 'file-mask' }, +test('imagesEditsJsonBody preserves reference fields and encodes mixed uploads from their bytes', async () => { + const body = await imagesEditsJsonBody({ + images: [ + { type: 'reference', reference: { image_url: 'https://example.test/image.png', detail: 'future-field' } }, + { type: 'upload', file: new File(['inline'], 'inline.png', { type: 'image/png' }) }, + ], + mask: { type: 'reference', reference: { file_id: 'file-mask' } }, parameters: { prompt: 'edit', background: null }, }); assertEquals(body, { prompt: 'edit', background: null, - images: [{ image_url: 'https://example.test/image.png', detail: 'future-field' }], + images: [ + { image_url: 'https://example.test/image.png', detail: 'future-field' }, + { image_url: 'data:image/png;base64,aW5saW5l' }, + ], mask: { file_id: 'file-mask' }, }); }); @@ -22,16 +27,14 @@ test('imagesEditsMultipartBody uses the singular field for one upload and the ar const first = new File(['first'], 'first.png', { type: 'image/png' }); const second = new File(['second'], 'second.png', { type: 'image/png' }); const single = imagesEditsMultipartBody({ - type: 'uploads', - images: [first], + images: [{ type: 'upload', file: first }], parameters: { prompt: 'single' }, }, 'gpt-image'); assertEquals(single.get('image'), first); assertEquals(single.getAll('image[]'), []); const multiple = imagesEditsMultipartBody({ - type: 'uploads', - images: [first, second], + images: [{ type: 'upload', file: first }, { type: 'upload', file: second }], parameters: { prompt: 'multiple' }, }, 'gpt-image'); assertEquals(multiple.getAll('image[]'), [first, second]); diff --git a/packages/provider/src/index.ts b/packages/provider/src/index.ts index 45df2cc15..57be85f41 100644 --- a/packages/provider/src/index.ts +++ b/packages/provider/src/index.ts @@ -56,8 +56,8 @@ export type { ResponsesAction, UpstreamCallOptions, } from './provider.ts'; -export type { ImagesEditsRequest, ReferencedImagesEditsRequest, UploadedImagesEditsRequest } from './images.ts'; -export { imagesEditsJsonBody, imagesEditsMultipartBody } from './images.ts'; +export type { ImagesEditsRequest, ImagesEditsSource } from './images.ts'; +export { imagesEditsCanUseMultipart, imagesEditsJsonBody, imagesEditsMultipartBody } from './images.ts'; export type { ProviderStreamParser } from './streaming.ts'; export { streamingProviderCall } from './streaming.ts'; From cc98076344d79875cc4f2a1cd04c015e7c17bf2c Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 15 Jul 2026 15:04:50 +0800 Subject: [PATCH 19/24] style(images): narrow inline data URL captures --- packages/gateway/src/data-plane/images/serve.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/gateway/src/data-plane/images/serve.ts b/packages/gateway/src/data-plane/images/serve.ts index 18c8350e1..f10273f7d 100644 --- a/packages/gateway/src/data-plane/images/serve.ts +++ b/packages/gateway/src/data-plane/images/serve.ts @@ -58,11 +58,13 @@ const imageEditSource = (value: unknown, path: string): ImagesEditsSource | stri return { type: 'reference', reference: reference as ImageEditReference }; } const match = /^data:([^;,]+);base64,(.*)$/su.exec(imageUrl); - if (match === null || !match[1]!.startsWith('image/')) return `${path}.image_url must be a base64 image data URL.`; + const mimeType = match?.[1]; + const payload = match?.[2]; + if (!mimeType?.startsWith('image/') || payload === undefined) return `${path}.image_url must be a base64 image data URL.`; try { - const bytes = Uint8Array.from(atob(match[2]!), character => character.charCodeAt(0)); - const extension = match[1]!.slice('image/'.length).replace(/[^a-z0-9]+/giu, '-'); - return { type: 'upload', file: new File([bytes], `${path}.${extension}`, { type: match[1] }) }; + const bytes = Uint8Array.from(atob(payload), character => character.charCodeAt(0)); + const extension = mimeType.slice('image/'.length).replace(/[^a-z0-9]+/giu, '-'); + return { type: 'upload', file: new File([bytes], `${path}.${extension}`, { type: mimeType }) }; } catch { return `${path}.image_url must contain valid base64 image data.`; } From d1488eb65507905966452054f754a869403626b4 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 15 Jul 2026 15:34:07 +0800 Subject: [PATCH 20/24] refactor(images): model inline sources without duplication Represent uploaded files, inline data URLs, and external references as separate single-truth IR variants. The shared OpenAI serializer emits multipart for upload-like scalar requests and JSON for reference or mixed requests, decoding or encoding only at the provider wire boundary. --- .../gateway/src/data-plane/images/serve.ts | 20 ++--- packages/provider-azure/src/provider.ts | 15 +--- packages/provider-custom/src/provider.ts | 15 +--- packages/provider/src/image-helpers.ts | 21 +++-- packages/provider/src/images.ts | 78 +++++++++++-------- packages/provider/src/images_test.ts | 47 ++++++++--- packages/provider/src/index.ts | 2 +- 7 files changed, 107 insertions(+), 91 deletions(-) diff --git a/packages/gateway/src/data-plane/images/serve.ts b/packages/gateway/src/data-plane/images/serve.ts index f10273f7d..9195424b4 100644 --- a/packages/gateway/src/data-plane/images/serve.ts +++ b/packages/gateway/src/data-plane/images/serve.ts @@ -15,7 +15,7 @@ import { readRequestBody, takeRequestBody, type RequestBody } from '../chat/shar import { passthroughApiError, passthroughServe } from '../shared/passthrough-serve.ts'; import { tokenUsageFromImagesBody } from '../shared/telemetry/usage.ts'; import type { ImageEditReference } from '@floway-dev/protocols/images'; -import type { ImagesEditsRequest, ImagesEditsSource } from '@floway-dev/provider'; +import { isBase64ImageDataUrl, type ImagesEditsRequest, type ImagesEditsSource } from '@floway-dev/provider'; interface JsonModelRequestBody { model?: unknown; @@ -54,20 +54,10 @@ const imageEditSource = (value: unknown, path: string): ImagesEditsSource | stri const reference = value as { image_url?: unknown; file_id?: unknown }; const { image_url: imageUrl, file_id: fileId } = reference; if (typeof imageUrl === 'string' && fileId === undefined) { - if (!imageUrl.startsWith('data:')) { - return { type: 'reference', reference: reference as ImageEditReference }; - } - const match = /^data:([^;,]+);base64,(.*)$/su.exec(imageUrl); - const mimeType = match?.[1]; - const payload = match?.[2]; - if (!mimeType?.startsWith('image/') || payload === undefined) return `${path}.image_url must be a base64 image data URL.`; - try { - const bytes = Uint8Array.from(atob(payload), character => character.charCodeAt(0)); - const extension = mimeType.slice('image/'.length).replace(/[^a-z0-9]+/giu, '-'); - return { type: 'upload', file: new File([bytes], `${path}.${extension}`, { type: mimeType }) }; - } catch { - return `${path}.image_url must contain valid base64 image data.`; - } + const imageReference = reference as ImageEditReference & { image_url: string }; + return isBase64ImageDataUrl(imageUrl) + ? { type: 'inline', reference: imageReference } + : { type: 'reference', reference: imageReference }; } if (typeof fileId === 'string' && imageUrl === undefined) { return { type: 'reference', reference: reference as ImageEditReference }; diff --git a/packages/provider-azure/src/provider.ts b/packages/provider-azure/src/provider.ts index b08cad0a3..209a0b5ba 100644 --- a/packages/provider-azure/src/provider.ts +++ b/packages/provider-azure/src/provider.ts @@ -5,7 +5,7 @@ import { parseChatCompletionsStream } from '@floway-dev/protocols/chat-completio import { kindForEndpoints } from '@floway-dev/protocols/common'; import { parseMessagesStream } from '@floway-dev/protocols/messages'; import { parseResponsesStream, type ResponsesResult, toCompactPayloadShape } from '@floway-dev/protocols/responses'; -import { imagesEditsCanUseMultipart, imagesEditsJsonBody, imagesEditsMultipartBody, type ProviderInstance, type Provider, type ProviderModel, type ProviderStreamParser, type UpstreamCallOptions, type UpstreamFetchOptions, type UpstreamRecord, publicModelId, resolveEffectiveFlags, streamingProviderCall } from '@floway-dev/provider'; +import { serializeOpenAIImagesEditsRequest, type ProviderInstance, type Provider, type ProviderModel, type ProviderStreamParser, type UpstreamCallOptions, type UpstreamFetchOptions, type UpstreamRecord, publicModelId, resolveEffectiveFlags, streamingProviderCall } from '@floway-dev/provider'; const upstreamModelIdOf = (model: ProviderModel): string => (model.providerData as { upstreamModelId: string }).upstreamModelId; @@ -92,17 +92,8 @@ export const createAzureProvider = (record: UpstreamRecord): Provider => { callImagesGenerations: (model, body, signal, opts) => callNonStreaming(azureFetchImagesGenerations, model, body, signal, opts.headers, opts), callImagesEdits: async (model, request, signal, opts) => { const upstreamModelId = upstreamModelIdOf(model); - if (imagesEditsCanUseMultipart(request)) { - const body = imagesEditsMultipartBody(request, upstreamModelId); - const response = await azureFetchImagesEdits(azure.config, { method: 'POST', body, signal }, { extraHeaders: opts.headers, fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }); - return { response, modelKey: upstreamModelId }; - } - const body = await imagesEditsJsonBody(request); - const response = await azureFetchImagesEdits( - azure.config, - { method: 'POST', body: JSON.stringify({ ...body, model: upstreamModelId }), signal }, - { extraHeaders: opts.headers, fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }, - ); + const body = await serializeOpenAIImagesEditsRequest(request, upstreamModelId); + const response = await azureFetchImagesEdits(azure.config, { method: 'POST', body, signal }, { extraHeaders: opts.headers, fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }); return { response, modelKey: upstreamModelId }; }, }; diff --git a/packages/provider-custom/src/provider.ts b/packages/provider-custom/src/provider.ts index 68897c644..576894229 100644 --- a/packages/provider-custom/src/provider.ts +++ b/packages/provider-custom/src/provider.ts @@ -7,7 +7,7 @@ import { parseChatCompletionsStream } from '@floway-dev/protocols/chat-completio import { type ModelEndpoints, kindForEndpoints } from '@floway-dev/protocols/common'; import { parseMessagesStream } from '@floway-dev/protocols/messages'; import { parseResponsesStream, type ResponsesResult, toCompactPayloadShape } from '@floway-dev/protocols/responses'; -import { imagesEditsCanUseMultipart, imagesEditsJsonBody, imagesEditsMultipartBody, publicModelId, resolveEffectiveFlags, streamingProviderCall, type FlagId, type ProviderInstance, type Provider, type ProviderCallResult, type ProviderModel, type ProviderStreamParser, type UpstreamCallOptions, type UpstreamFetchOptions, type UpstreamRecord } from '@floway-dev/provider'; +import { serializeOpenAIImagesEditsRequest, publicModelId, resolveEffectiveFlags, streamingProviderCall, type FlagId, type ProviderInstance, type Provider, type ProviderCallResult, type ProviderModel, type ProviderStreamParser, type UpstreamCallOptions, type UpstreamFetchOptions, type UpstreamRecord } from '@floway-dev/provider'; const rawModelIdOf = (model: ProviderModel): string => model.providerData as string; @@ -180,17 +180,8 @@ export const createCustomProvider = (record: UpstreamRecord): Provider => { callImagesGenerations: (model, body, signal, opts) => call(customFetchImagesGenerations, model, body, signal, opts.headers, opts), callImagesEdits: async (model, request, signal, opts) => { const rawModelId = rawModelIdOf(model); - if (imagesEditsCanUseMultipart(request)) { - const body = imagesEditsMultipartBody(request, rawModelId); - const response = await customFetchImagesEdits(config, { method: 'POST', body, signal }, { extraHeaders: opts.headers, fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }); - return { response, modelKey: rawModelId }; - } - const body = await imagesEditsJsonBody(request); - const response = await customFetchImagesEdits( - config, - { method: 'POST', body: JSON.stringify({ ...body, model: rawModelId }), signal }, - { extraHeaders: opts.headers, fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }, - ); + const body = await serializeOpenAIImagesEditsRequest(request, rawModelId); + const response = await customFetchImagesEdits(config, { method: 'POST', body, signal }, { extraHeaders: opts.headers, fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }); return { response, modelKey: rawModelId }; }, }; diff --git a/packages/provider/src/image-helpers.ts b/packages/provider/src/image-helpers.ts index c92434dd2..368e1cfd6 100644 --- a/packages/provider/src/image-helpers.ts +++ b/packages/provider/src/image-helpers.ts @@ -2,14 +2,14 @@ import { compressBytesToWebp, type ImageSizeCalculator } from '@floway-dev/platf const BASE64_CHUNK = 0x8000; -const base64ToBytes = (base64: string): Uint8Array => { +export const base64ToBytes = (base64: string): Uint8Array => { const binary = atob(base64); const bytes = new Uint8Array(binary.length); for (let i = 0; i < binary.length; i += 1) bytes[i] = binary.charCodeAt(i); return bytes; }; -const bytesToBase64 = (bytes: Uint8Array): string => { +export const bytesToBase64 = (bytes: Uint8Array): string => { let binary = ''; for (let offset = 0; offset < bytes.length; offset += BASE64_CHUNK) { binary += String.fromCharCode(...bytes.subarray(offset, offset + BASE64_CHUNK)); @@ -17,7 +17,14 @@ const bytesToBase64 = (bytes: Uint8Array): string => { return btoa(binary); }; -const BASE64_DATA_URL = /^data:([^;,]+);base64,(.*)$/s; +const BASE64_DATA_URL = /^data:([^;,]+)(?:;[^,;]*)*;base64,(.*)$/is; + +export const parseBase64ImageDataUrl = (url: string): { mimeType: string; base64: string } | null => { + const match = BASE64_DATA_URL.exec(url); + const mimeType = match?.[1]; + const base64 = match?.[2]; + return mimeType?.startsWith('image/') && base64 !== undefined ? { mimeType, base64 } : null; +}; const compressBase64ImageToWebp = async ( base64: string, @@ -34,14 +41,14 @@ const compressImageDataUrlToWebp = async ( url: string, calculator: ImageSizeCalculator, ): Promise => { - const match = BASE64_DATA_URL.exec(url); - if (!match?.[1].startsWith('image/')) return url; - const webp = await compressBase64ImageToWebp(match[2], calculator); + const parsed = parseBase64ImageDataUrl(url); + if (parsed === null) return url; + const webp = await compressBase64ImageToWebp(parsed.base64, calculator); return `data:image/webp;base64,${webp}`; }; export const isBase64ImageDataUrl = (url: string): boolean => - BASE64_DATA_URL.exec(url)?.[1].startsWith('image/') ?? false; + parseBase64ImageDataUrl(url) !== null; // Per-request memoizing wrappers around the compress helpers above. A single // agentic request often replays the same screenshot across many turns, so the diff --git a/packages/provider/src/images.ts b/packages/provider/src/images.ts index 2c77ea244..897fd8766 100644 --- a/packages/provider/src/images.ts +++ b/packages/provider/src/images.ts @@ -1,19 +1,26 @@ +import { base64ToBytes, bytesToBase64, parseBase64ImageDataUrl } from './image-helpers.ts'; import type { ImageEditReference } from '@floway-dev/protocols/images'; -// Inline data URLs are decoded into uploads before reaching this boundary. -// Providers prefer multipart when every source is an upload and use JSON for -// external references or mixed requests. +// Each source stores one authoritative representation. Serialization prefers +// multipart when every source is upload-like and every parameter is scalar; +// requests containing external references use JSON, encoding File uploads only +// when they are mixed with references. interface UploadedImagesEditsSource { type: 'upload'; file: File; } +interface InlineImagesEditsSource { + type: 'inline'; + reference: ImageEditReference & { image_url: string }; +} + interface ReferencedImagesEditsSource { type: 'reference'; reference: ImageEditReference; } -export type ImagesEditsSource = UploadedImagesEditsSource | ReferencedImagesEditsSource; +export type ImagesEditsSource = UploadedImagesEditsSource | InlineImagesEditsSource | ReferencedImagesEditsSource; export interface ImagesEditsRequest { images: ImagesEditsSource[]; @@ -21,34 +28,27 @@ export interface ImagesEditsRequest { parameters: Record; } -interface MultipartImagesEditsRequest extends ImagesEditsRequest { - images: UploadedImagesEditsSource[]; - mask?: UploadedImagesEditsSource; - parameters: Record; -} - -export const imagesEditsCanUseMultipart = (request: ImagesEditsRequest): request is MultipartImagesEditsRequest => { - const sources = [...request.images, ...(request.mask === undefined ? [] : [request.mask])]; - return sources.every(source => source.type === 'upload') - && Object.values(request.parameters).every(value => - typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean'); -}; - -const bytesToBase64 = (bytes: Uint8Array): string => { - const chunks: string[] = []; - for (let offset = 0; offset < bytes.length; offset += 32_768) { - chunks.push(String.fromCharCode(...bytes.subarray(offset, offset + 32_768))); +const uploadedFile = (source: ImagesEditsSource, index: number): File | null => { + if (source.type === 'upload') return source.file; + if (source.type === 'reference') return null; + const parsed = parseBase64ImageDataUrl(source.reference.image_url); + if (parsed === null) return null; + let bytes: Uint8Array; + try { + bytes = base64ToBytes(parsed.base64); + } catch { + return null; } - return btoa(chunks.join('')); + return new File([bytes], `image-${index}`, { type: parsed.mimeType }); }; const jsonReference = async (source: ImagesEditsSource): Promise => { - if (source.type === 'reference') return source.reference; + if (source.type === 'inline' || source.type === 'reference') return source.reference; const bytes = new Uint8Array(await source.file.arrayBuffer()); return { image_url: `data:${source.file.type};base64,${bytesToBase64(bytes)}` }; }; -export const imagesEditsJsonBody = async (request: ImagesEditsRequest): Promise> => { +const jsonBody = async (request: ImagesEditsRequest): Promise> => { const images = await Promise.all(request.images.map(jsonReference)); const mask = request.mask === undefined ? undefined : await jsonReference(request.mask); return { @@ -58,15 +58,29 @@ export const imagesEditsJsonBody = async (request: ImagesEditsRequest): Promise< }; }; -export const imagesEditsMultipartBody = (request: MultipartImagesEditsRequest, model: string): FormData => { - const form = new FormData(); - for (const [name, value] of Object.entries(request.parameters)) { - form.append(name, String(value)); +const multipartBody = (request: ImagesEditsRequest, model: string): FormData | null => { + if (!Object.values(request.parameters).every(value => + typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean')) return null; + const images: File[] = []; + for (const [index, source] of request.images.entries()) { + const file = uploadedFile(source, index); + if (file === null) return null; + images.push(file); } - const uploads = request.images.map(source => source.file); - const imageField = uploads.length === 1 ? 'image' : 'image[]'; - for (const image of uploads) form.append(imageField, image); - if (request.mask !== undefined) form.append('mask', request.mask.file); + const mask = request.mask === undefined ? undefined : uploadedFile(request.mask, images.length); + if (mask === null) return null; + + const form = new FormData(); + for (const [name, value] of Object.entries(request.parameters)) form.append(name, String(value)); + const imageField = images.length === 1 ? 'image' : 'image[]'; + for (const image of images) form.append(imageField, image); + if (mask !== undefined) form.append('mask', mask); form.append('model', model); return form; }; + +export const serializeOpenAIImagesEditsRequest = async (request: ImagesEditsRequest, model: string): Promise => { + const multipart = multipartBody(request, model); + if (multipart !== null) return multipart; + return JSON.stringify({ ...await jsonBody(request), model }); +}; diff --git a/packages/provider/src/images_test.ts b/packages/provider/src/images_test.ts index 9ec337c92..c7ae10dd4 100644 --- a/packages/provider/src/images_test.ts +++ b/packages/provider/src/images_test.ts @@ -1,17 +1,19 @@ import { test } from 'vitest'; -import { imagesEditsJsonBody, imagesEditsMultipartBody } from './images.ts'; +import { serializeOpenAIImagesEditsRequest } from './images.ts'; import { assertEquals } from '@floway-dev/test-utils'; -test('imagesEditsJsonBody preserves reference fields and encodes mixed uploads from their bytes', async () => { - const body = await imagesEditsJsonBody({ +test('serializeOpenAIImagesEditsRequest preserves reference fields and encodes mixed uploads from their bytes', async () => { + const serialized = await serializeOpenAIImagesEditsRequest({ images: [ { type: 'reference', reference: { image_url: 'https://example.test/image.png', detail: 'future-field' } }, { type: 'upload', file: new File(['inline'], 'inline.png', { type: 'image/png' }) }, ], mask: { type: 'reference', reference: { file_id: 'file-mask' } }, parameters: { prompt: 'edit', background: null }, - }); + }, 'gpt-image'); + assertEquals(typeof serialized, 'string'); + const body = JSON.parse(serialized as string) as Record; assertEquals(body, { prompt: 'edit', background: null, @@ -20,24 +22,45 @@ test('imagesEditsJsonBody preserves reference fields and encodes mixed uploads f { image_url: 'data:image/png;base64,aW5saW5l' }, ], mask: { file_id: 'file-mask' }, + model: 'gpt-image', }); }); -test('imagesEditsMultipartBody uses the singular field for one upload and the array field for many', () => { +test('serializeOpenAIImagesEditsRequest uses the singular field for one upload and the array field for many', async () => { const first = new File(['first'], 'first.png', { type: 'image/png' }); const second = new File(['second'], 'second.png', { type: 'image/png' }); - const single = imagesEditsMultipartBody({ + const single = await serializeOpenAIImagesEditsRequest({ images: [{ type: 'upload', file: first }], parameters: { prompt: 'single' }, }, 'gpt-image'); - assertEquals(single.get('image'), first); - assertEquals(single.getAll('image[]'), []); + assertEquals(single instanceof FormData, true); + const singleForm = single as FormData; + assertEquals(singleForm.get('image'), first); + assertEquals(singleForm.getAll('image[]'), []); - const multiple = imagesEditsMultipartBody({ + const multiple = await serializeOpenAIImagesEditsRequest({ images: [{ type: 'upload', file: first }, { type: 'upload', file: second }], parameters: { prompt: 'multiple' }, }, 'gpt-image'); - assertEquals(multiple.getAll('image[]'), [first, second]); - assertEquals(multiple.get('image'), null); - assertEquals(multiple.get('model'), 'gpt-image'); + assertEquals(multiple instanceof FormData, true); + const multipleForm = multiple as FormData; + assertEquals(multipleForm.getAll('image[]'), [first, second]); + assertEquals(multipleForm.get('image'), null); + assertEquals(multipleForm.get('model'), 'gpt-image'); +}); + +test('serializeOpenAIImagesEditsRequest leaves malformed inline data for upstream JSON validation', async () => { + const serialized = await serializeOpenAIImagesEditsRequest({ + images: [{ + type: 'inline', + reference: { image_url: 'data:image/png;base64,%%%' }, + }], + parameters: { prompt: 'edit' }, + }, 'gpt-image'); + assertEquals(typeof serialized, 'string'); + assertEquals(JSON.parse(serialized as string), { + prompt: 'edit', + images: [{ image_url: 'data:image/png;base64,%%%' }], + model: 'gpt-image', + }); }); diff --git a/packages/provider/src/index.ts b/packages/provider/src/index.ts index 57be85f41..9a0b4d540 100644 --- a/packages/provider/src/index.ts +++ b/packages/provider/src/index.ts @@ -57,7 +57,7 @@ export type { UpstreamCallOptions, } from './provider.ts'; export type { ImagesEditsRequest, ImagesEditsSource } from './images.ts'; -export { imagesEditsCanUseMultipart, imagesEditsJsonBody, imagesEditsMultipartBody } from './images.ts'; +export { serializeOpenAIImagesEditsRequest } from './images.ts'; export type { ProviderStreamParser } from './streaming.ts'; export { streamingProviderCall } from './streaming.ts'; From 1d35c7241974e25eff54a49d31d2ef65840296e5 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 15 Jul 2026 15:35:52 +0800 Subject: [PATCH 21/24] fix(images): type decoded bytes as array buffers --- packages/provider/src/image-helpers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/provider/src/image-helpers.ts b/packages/provider/src/image-helpers.ts index 368e1cfd6..494bb6f1b 100644 --- a/packages/provider/src/image-helpers.ts +++ b/packages/provider/src/image-helpers.ts @@ -2,7 +2,7 @@ import { compressBytesToWebp, type ImageSizeCalculator } from '@floway-dev/platf const BASE64_CHUNK = 0x8000; -export const base64ToBytes = (base64: string): Uint8Array => { +export const base64ToBytes = (base64: string): Uint8Array => { const binary = atob(base64); const bytes = new Uint8Array(binary.length); for (let i = 0; i < binary.length; i += 1) bytes[i] = binary.charCodeAt(i); From 3a2880e8231935b5bbd9efa01f37640022513adf Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 15 Jul 2026 15:37:28 +0800 Subject: [PATCH 22/24] fix(images): preserve decoded buffer ownership --- packages/provider/src/images.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/provider/src/images.ts b/packages/provider/src/images.ts index 897fd8766..a335131f6 100644 --- a/packages/provider/src/images.ts +++ b/packages/provider/src/images.ts @@ -33,7 +33,7 @@ const uploadedFile = (source: ImagesEditsSource, index: number): File | null => if (source.type === 'reference') return null; const parsed = parseBase64ImageDataUrl(source.reference.image_url); if (parsed === null) return null; - let bytes: Uint8Array; + let bytes: Uint8Array; try { bytes = base64ToBytes(parsed.base64); } catch { From 0725cb3f5683c9329652a1a3362d87079850d51d Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 15 Jul 2026 15:43:18 +0800 Subject: [PATCH 23/24] refactor(images): preflight edit serialization Choose multipart only after every source and parameter is known to be form-safe. Preserve inline reference extensions through JSON and defer base64 decoding until the complete request qualifies for multipart. --- packages/provider/src/image-helpers.ts | 2 +- packages/provider/src/images.ts | 24 ++++++++++++++++-------- packages/provider/src/images_test.ts | 16 ++++++++++++++++ 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/packages/provider/src/image-helpers.ts b/packages/provider/src/image-helpers.ts index 494bb6f1b..1540006fc 100644 --- a/packages/provider/src/image-helpers.ts +++ b/packages/provider/src/image-helpers.ts @@ -23,7 +23,7 @@ export const parseBase64ImageDataUrl = (url: string): { mimeType: string; base64 const match = BASE64_DATA_URL.exec(url); const mimeType = match?.[1]; const base64 = match?.[2]; - return mimeType?.startsWith('image/') && base64 !== undefined ? { mimeType, base64 } : null; + return mimeType?.toLowerCase().startsWith('image/') && base64 !== undefined ? { mimeType, base64 } : null; }; const compressBase64ImageToWebp = async ( diff --git a/packages/provider/src/images.ts b/packages/provider/src/images.ts index a335131f6..420ef0bf9 100644 --- a/packages/provider/src/images.ts +++ b/packages/provider/src/images.ts @@ -1,10 +1,9 @@ import { base64ToBytes, bytesToBase64, parseBase64ImageDataUrl } from './image-helpers.ts'; import type { ImageEditReference } from '@floway-dev/protocols/images'; -// Each source stores one authoritative representation. Serialization prefers -// multipart when every source is upload-like and every parameter is scalar; -// requests containing external references use JSON, encoding File uploads only -// when they are mixed with references. +// Each source stores one authoritative representation. Multipart requires +// upload-like sources with no extra reference fields plus scalar parameters; +// every other request uses JSON and encodes File uploads as data URLs. interface UploadedImagesEditsSource { type: 'upload'; file: File; @@ -42,6 +41,15 @@ const uploadedFile = (source: ImagesEditsSource, index: number): File | null => return new File([bytes], `image-${index}`, { type: parsed.mimeType }); }; +const canUseMultipart = (request: ImagesEditsRequest): boolean => { + const sources = [...request.images, ...(request.mask === undefined ? [] : [request.mask])]; + return sources.every(source => + source.type === 'upload' + || (source.type === 'inline' && Object.keys(source.reference).every(key => key === 'image_url'))) + && Object.values(request.parameters).every(value => + typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean'); +}; + const jsonReference = async (source: ImagesEditsSource): Promise => { if (source.type === 'inline' || source.type === 'reference') return source.reference; const bytes = new Uint8Array(await source.file.arrayBuffer()); @@ -59,8 +67,6 @@ const jsonBody = async (request: ImagesEditsRequest): Promise { - if (!Object.values(request.parameters).every(value => - typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean')) return null; const images: File[] = []; for (const [index, source] of request.images.entries()) { const file = uploadedFile(source, index); @@ -80,7 +86,9 @@ const multipartBody = (request: ImagesEditsRequest, model: string): FormData | n }; export const serializeOpenAIImagesEditsRequest = async (request: ImagesEditsRequest, model: string): Promise => { - const multipart = multipartBody(request, model); - if (multipart !== null) return multipart; + if (canUseMultipart(request)) { + const multipart = multipartBody(request, model); + if (multipart !== null) return multipart; + } return JSON.stringify({ ...await jsonBody(request), model }); }; diff --git a/packages/provider/src/images_test.ts b/packages/provider/src/images_test.ts index c7ae10dd4..4c5a2af5e 100644 --- a/packages/provider/src/images_test.ts +++ b/packages/provider/src/images_test.ts @@ -64,3 +64,19 @@ test('serializeOpenAIImagesEditsRequest leaves malformed inline data for upstrea model: 'gpt-image', }); }); + +test('serializeOpenAIImagesEditsRequest preserves extra inline reference fields through JSON', async () => { + const serialized = await serializeOpenAIImagesEditsRequest({ + images: [{ + type: 'inline', + reference: { image_url: 'data:image/png;base64,aW1hZ2U=', future_field: 'keep' }, + }], + parameters: { prompt: 'edit' }, + }, 'gpt-image'); + assertEquals(typeof serialized, 'string'); + assertEquals(JSON.parse(serialized as string), { + prompt: 'edit', + images: [{ image_url: 'data:image/png;base64,aW1hZ2U=', future_field: 'keep' }], + model: 'gpt-image', + }); +}); From 854bc6b2031183270a72d9c237088797a89c320d Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 15 Jul 2026 15:53:35 +0800 Subject: [PATCH 24/24] refactor(images): centralize multipart selection --- packages/provider/src/images.ts | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/packages/provider/src/images.ts b/packages/provider/src/images.ts index 420ef0bf9..a0ef376cf 100644 --- a/packages/provider/src/images.ts +++ b/packages/provider/src/images.ts @@ -41,15 +41,6 @@ const uploadedFile = (source: ImagesEditsSource, index: number): File | null => return new File([bytes], `image-${index}`, { type: parsed.mimeType }); }; -const canUseMultipart = (request: ImagesEditsRequest): boolean => { - const sources = [...request.images, ...(request.mask === undefined ? [] : [request.mask])]; - return sources.every(source => - source.type === 'upload' - || (source.type === 'inline' && Object.keys(source.reference).every(key => key === 'image_url'))) - && Object.values(request.parameters).every(value => - typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean'); -}; - const jsonReference = async (source: ImagesEditsSource): Promise => { if (source.type === 'inline' || source.type === 'reference') return source.reference; const bytes = new Uint8Array(await source.file.arrayBuffer()); @@ -67,6 +58,14 @@ const jsonBody = async (request: ImagesEditsRequest): Promise { + const sources = [...request.images, ...(request.mask === undefined ? [] : [request.mask])]; + const compatibleSources = sources.every(source => + source.type === 'upload' + || (source.type === 'inline' && Object.keys(source.reference).every(key => key === 'image_url'))); + const compatibleParameters = Object.values(request.parameters).every(value => + typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean'); + if (!compatibleSources || !compatibleParameters) return null; + const images: File[] = []; for (const [index, source] of request.images.entries()) { const file = uploadedFile(source, index); @@ -86,9 +85,7 @@ const multipartBody = (request: ImagesEditsRequest, model: string): FormData | n }; export const serializeOpenAIImagesEditsRequest = async (request: ImagesEditsRequest, model: string): Promise => { - if (canUseMultipart(request)) { - const multipart = multipartBody(request, model); - if (multipart !== null) return multipart; - } + const multipart = multipartBody(request, model); + if (multipart !== null) return multipart; return JSON.stringify({ ...await jsonBody(request), model }); };