diff --git a/README.md b/README.md index 7218d31d1..d521f60a9 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,10 @@ target ships in the same repo for self-hosting on a long-lived process. | OpenAI Models | `GET /v1/models` | | Google Gemini (generate / count tokens) | `POST /v1beta/models/...` | +`POST /v1/images/edits` accepts multipart image uploads and JSON `images` +references. The dashboard's Codex provider base, `/azure-api.codex`, exposes +the same generation and edit handlers at their provider-relative paths. + For each public model, Floway picks the first (provider, model) pair that can serve the request, translating between source and target protocols when the upstream speaks a different shape. `/v1/completions` is forwarded to upstreams that diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation-integration_test.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation-integration_test.ts index 8465c3544..82776e6b6 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation-integration_test.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation-integration_test.ts @@ -8,7 +8,7 @@ import { createInMemoryImageProcessor, initExternalResourceFetcher, initImagePro import { eventFrame } from '@floway-dev/protocols/common'; import type { ProtocolFrame } from '@floway-dev/protocols/common'; import type { ResponsesResult, ResponsesStreamEvent, ResponsesTool, ResponsesToolChoice } from '@floway-dev/protocols/responses'; -import { type EventResult, type ExecuteResult, type FlagId } from '@floway-dev/provider'; +import { type EventResult, type ExecuteResult, type FlagId, type ImagesEditsRequest } from '@floway-dev/provider'; import { assert, assertEquals, assertStringIncludes, stubModelCandidate } from '@floway-dev/test-utils'; // Dirty integration harness: mock the model registry so the image backend is a @@ -20,7 +20,7 @@ import { assert, assertEquals, assertStringIncludes, stubModelCandidate } from ' interface BackendStub { generationsCalls: Record[]; - editsForms: FormData[]; + editsRequests: ImagesEditsRequest[]; nextGenerations: Response[]; nextEdits: Response[]; // When set, the next `enumerateModelCandidates` call returns this @@ -31,7 +31,7 @@ interface BackendStub { // Hoisted so the vi.mock factory below can close over it; tests mutate the // `next*` queues and read back the recorded calls. -const stub = vi.hoisted((): BackendStub => ({ generationsCalls: [], editsForms: [], nextGenerations: [], nextEdits: [], nextResolutionOverride: null })); +const stub = vi.hoisted((): BackendStub => ({ generationsCalls: [], editsRequests: [], nextGenerations: [], nextEdits: [], nextResolutionOverride: null })); // Assigned per test in beforeEach and captured so the perf-attribution test // can read `repo.performance.listAll()` after the shim completes. @@ -52,8 +52,8 @@ const defaultCandidates = vi.hoisted(() => () => [{ if (response === undefined) throw new Error('test did not enqueue a generations response'); return { response, modelKey: 'gpt-image-2' }; }, - callImagesEdits: async (_model: unknown, form: FormData) => { - stub.editsForms.push(form); + callImagesEdits: async (_model: unknown, request: ImagesEditsRequest) => { + stub.editsRequests.push(request); const response = stub.nextEdits.shift(); if (response === undefined) throw new Error('test did not enqueue an edits response'); return { response, modelKey: 'gpt-image-2' }; @@ -221,7 +221,7 @@ beforeEach(async () => { initRepo(repo); initImageProcessor(createInMemoryImageProcessor()); stub.generationsCalls = []; - stub.editsForms = []; + stub.editsRequests = []; stub.nextGenerations = []; stub.nextEdits = []; stub.nextResolutionOverride = null; @@ -229,14 +229,18 @@ beforeEach(async () => { test('generates an image end-to-end and emits the native lifecycle', async () => { stub.nextGenerations = [jsonResponse('R0VO')]; // "GEN" - const result = await shim(makeCtx([{ type: 'message', role: 'user', content: 'draw a cat' }]), gatewayCtx(), scriptedRun([ + const result = await shim(makeCtx([{ type: 'message', role: 'user', content: 'draw a cat' }], 'auto', { + size: '1024x1024', + quality: 'low', + }), gatewayCtx(), scriptedRun([ callTurn(0, 'call_1', 'a cat'), messageTurn('here it is'), ])); const events = await drain(result); assertEquals(stub.generationsCalls.length, 1); - assertEquals(stub.editsForms.length, 0); + assertEquals(stub.generationsCalls[0], { prompt: 'a cat', n: 1, size: '1024x1024', quality: 'low' }); + assertEquals(stub.editsRequests.length, 0); const igcDone = events.find(e => e.type === 'response.output_item.done' && (e as { item: { type: string } }).item.type === 'image_generation_call'); assert(igcDone !== undefined); const item = (igcDone as { item: { status: string; result: string; action: string } }).item; @@ -275,6 +279,7 @@ test('relays real partial_image frames when partial_images > 0', async () => { ])); const events = await drain(result); + assertEquals(stub.generationsCalls[0], { prompt: 'a cat', n: 1, stream: true, partial_images: 2 }); const partials = events.filter(e => e.type === 'response.image_generation_call.partial_image'); assertEquals(partials.length, 2); assertEquals((partials[0] as { partial_image_b64: string }).partial_image_b64, 'UDA='); @@ -298,10 +303,12 @@ test('an image generated in turn 1 is re-collected as an edit source in turn 2', await drain(result); assertEquals(stub.generationsCalls.length, 1); - assertEquals(stub.editsForms.length, 1); - const images = stub.editsForms[0].getAll('image[]'); - assertEquals(images.length, 1); - const bytes = await (images[0] as Blob).text(); + assertEquals(stub.editsRequests.length, 1); + const request = stub.editsRequests[0]; + assertEquals(request.images.length, 1); + const image = request.images[0]; + assert(image.type === 'upload'); + const bytes = await image.file.text(); assertEquals(bytes, 'AAAA'); }); @@ -337,9 +344,11 @@ test('a prefetched remote edit source remains visible to orchestration and is re assertEquals(fetched, ['https://example.com/source.png']); assertEquals(orchestratorImageUrl, 'https://example.com/source.png'); - assertEquals(stub.editsForms.length, 1); - const image = stub.editsForms[0].getAll('image[]')[0] as Blob; - assertEquals(new Uint8Array(await image.arrayBuffer()), Uint8Array.from(atob(REMOTE_PNG_B64), c => c.charCodeAt(0))); + assertEquals(stub.editsRequests.length, 1); + const request = stub.editsRequests[0]; + const image = request.images[0]; + assert(image.type === 'upload'); + assertEquals(new Uint8Array(await image.file.arrayBuffer()), Uint8Array.from(atob(REMOTE_PNG_B64), c => c.charCodeAt(0))); }); test('mask-only GIF edit transcodes one shared image and mask to WebP', async () => { @@ -361,14 +370,16 @@ test('mask-only GIF edit transcodes one shared image and mask to WebP', async () await drain(result); assertEquals(processorCalls, 1); - assertEquals(stub.editsForms.length, 1); - const form = stub.editsForms[0]; - const image = form.getAll('image[]')[0] as Blob; - const mask = form.get('mask') as Blob; - assertEquals(image.type, 'image/webp'); - assertEquals(mask.type, 'image/webp'); - assertEquals(await image.text(), 'WEBP'); - assertEquals(await mask.text(), 'WEBP'); + assertEquals(stub.editsRequests.length, 1); + const request = stub.editsRequests[0]; + const image = request.images[0]; + const mask = request.mask; + assert(image.type === 'upload'); + assert(mask?.type === 'upload'); + assertEquals(image.file.type, 'image/webp'); + assertEquals(mask.file.type, 'image/webp'); + assertEquals(await image.file.text(), 'WEBP'); + assertEquals(await mask.file.text(), 'WEBP'); }); test('identical GIF source and mask share one transcode', async () => { @@ -393,9 +404,12 @@ test('identical GIF source and mask share one transcode', async () => { await drain(result); assertEquals(processorCalls, 1); - const form = stub.editsForms[0]; - assertEquals(await (form.getAll('image[]')[0] as Blob).text(), 'WEBP'); - assertEquals(await (form.get('mask') as Blob).text(), 'WEBP'); + const request = stub.editsRequests[0]; + const image = request.images[0]; + assert(image.type === 'upload'); + assert(request.mask?.type === 'upload'); + assertEquals(await image.file.text(), 'WEBP'); + assertEquals(await request.mask.file.text(), 'WEBP'); }); test('image transcoding failure becomes a terminal image tool failure', async () => { @@ -412,7 +426,7 @@ test('image transcoding failure becomes a terminal image tool failure', async () ])); const events = await drain(result); - assertEquals(stub.editsForms.length, 0); + assertEquals(stub.editsRequests.length, 0); const done = events.find(event => event.type === 'response.output_item.done' && (event as { item: { type: string } }).item.type === 'image_generation_call'); assert(done !== undefined); diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts index f02eac566..e9d6fa322 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation.ts @@ -19,7 +19,7 @@ import type { ResponsesOutputImageGenerationCall, ResponsesTool, } from '@floway-dev/protocols/responses'; -import { providerModelOf, type Fetcher, type Provider, type ModelCandidate, type ProviderModel } from '@floway-dev/provider'; +import { providerModelOf, type Fetcher, type ImagesEditsRequest, type Provider, type ModelCandidate, type ProviderModel } from '@floway-dev/provider'; export const SHIM_TOOL_NAME = 'image_generation'; @@ -53,21 +53,23 @@ const ALLOWED_INPUT_FIDELITY = new Set(['high', 'low']); // `unsupported_file_mimetype`. Native Responses accepts the same GIF and // re-encodes it before editing, so the shim mirrors that behavior through the // platform image processor. Common aliases are folded onto the backend form. -const EDIT_MIME_ALIASES: Record = { +type EditMime = 'image/png' | 'image/jpeg' | 'image/webp'; + +const EDIT_MIME_ALIASES: Record = { 'image/jpg': 'image/jpeg', 'image/pjpeg': 'image/jpeg', 'image/x-png': 'image/png', }; -const EDIT_SUPPORTED_MIMES = new Set(['image/png', 'image/jpeg', 'image/webp']); - // The canonical edit-supported mimetype for a source, or null when the // standalone endpoint requires local WebP transcoding first. -const editSupportedMime = (mime: string): string | null => { +const editSupportedMime = (mime: string): EditMime | null => { const canonical = EDIT_MIME_ALIASES[mime] ?? mime; - return EDIT_SUPPORTED_MIMES.has(canonical) ? canonical : null; + return canonical === 'image/png' || canonical === 'image/jpeg' || canonical === 'image/webp' + ? canonical + : null; }; -const editFileExt = (mime: string): string => +const editFileExt = (mime: EditMime): string => mime === 'image/jpeg' ? 'jpg' : mime === 'image/webp' ? 'webp' : 'png'; // The public `image_generation` tool-config surface. Azure rejects any other @@ -100,6 +102,10 @@ interface ImageSource { mimeType: string; } +interface PreparedImageSource extends ImageSource { + mimeType: EditMime; +} + interface RemoteImageSource { wireUrl: string; invalidUrlParam: string; @@ -111,11 +117,12 @@ type ImageSourceReference = ImageSource | RemoteImageSource; const isRemoteImageSource = (source: ImageSourceReference): source is RemoteImageSource => 'wireUrl' in source; -export const prepareEditSources = async (sources: readonly ImageSource[]): Promise => { +const prepareEditSources = async (sources: readonly ImageSource[]): Promise => { const keyBySource = new Map>(); - const preparedByContent = new Map>(); + const preparedByContent = new Map>(); return await Promise.all(sources.map(async source => { - if (editSupportedMime(source.mimeType) !== null) return source; + const mimeType = editSupportedMime(source.mimeType); + if (mimeType !== null) return { bytes: source.bytes, mimeType }; let keyPromise = keyBySource.get(source); if (keyPromise === undefined) { @@ -135,7 +142,7 @@ export const prepareEditSources = async (sources: readonly ImageSource[]): Promi // https://github.com/openai/openai-node/blob/ec2f57fd0d66e94782656b986d7b3eb03225369c/src/resources/images.ts#L560-L572 prepared = getImageProcessor().compressToWebp(new Uint8Array(source.bytes), null).then(encoded => { const bytes = encoded.buffer.slice(encoded.byteOffset, encoded.byteOffset + encoded.byteLength) as ArrayBuffer; - return { bytes, mimeType: 'image/webp' }; + return { bytes, mimeType: 'image/webp' } satisfies PreparedImageSource; }); preparedByContent.set(key, prepared); } @@ -255,17 +262,16 @@ export interface ImageGenerationConfig { action: 'generate' | 'edit' | 'auto'; } +type MaterializedImageGenerationConfig = Omit & { mask?: ImageSource }; + const prepareEditRequest = async ( sources: readonly ImageSource[], - config: ImageGenerationConfig, -): Promise<{ sources: readonly ImageSource[]; config: ImageGenerationConfig }> => { - if (config.mask !== undefined && isRemoteImageSource(config.mask)) { - throw new Error('Remote image mask reached edit dispatch before materialization'); - } + config: MaterializedImageGenerationConfig, +): Promise<{ sources: readonly PreparedImageSource[]; mask?: PreparedImageSource }> => { const originals = [...sources]; if (config.mask !== undefined && !originals.includes(config.mask)) originals.push(config.mask); const prepared = await prepareEditSources(originals); - const bySource = new Map(); + const bySource = new Map(); for (const [index, source] of originals.entries()) { const wireSource = prepared[index]; if (wireSource === undefined) throw new Error('Missing prepared image edit source'); @@ -276,10 +282,10 @@ const prepareEditRequest = async ( if (wireSource === undefined) throw new Error('Missing prepared image edit source'); return wireSource; }); - if (config.mask === undefined) return { sources: wireSources, config }; + if (config.mask === undefined) return { sources: wireSources }; const mask = bySource.get(config.mask); if (mask === undefined) throw new Error('Missing prepared image edit mask'); - return { sources: wireSources, config: { ...config, mask } }; + return { sources: wireSources, mask }; }; interface PrepareConfigError { @@ -890,7 +896,7 @@ const errorFromBody = (body: string, status: number): { type?: string; code: str // in a later turn. `imageDispatchCount` bounds how many real backend image // calls one response may issue. interface ShimState { - config: ImageGenerationConfig; + config: MaterializedImageGenerationConfig; apiKeyId: string; upstreamIds: readonly string[] | null; backgroundScheduler: BackgroundScheduler; @@ -913,7 +919,7 @@ const recordImageUsage = (state: ShimState, provider: Provider, model: ProviderM state.backgroundScheduler(promise); }; -export const buildGenerationsBody = (prompt: string, config: ImageGenerationConfig, stream: boolean): Record => ({ +const buildGenerationsBody = (prompt: string, config: ImageGenerationConfig, stream: boolean): Record => ({ prompt, // Public Responses tool config forbids `n`, but the private standalone // backend call always requests a single image, mirroring Azure's @@ -931,38 +937,37 @@ export const buildGenerationsBody = (prompt: string, config: ImageGenerationConf ...(stream ? { stream: true, partial_images: config.partial_images } : {}), }); -const buildEditsForm = (prompt: string, config: ImageGenerationConfig, sources: readonly ImageSource[], stream: boolean): FormData => { - const form = new FormData(); - form.append('prompt', prompt); - form.append('n', '1'); - if (config.size !== undefined) form.append('size', config.size); - if (config.quality !== undefined) form.append('quality', config.quality); - if (config.output_format !== undefined) form.append('output_format', config.output_format); - if (config.background !== undefined) form.append('background', config.background); - if (config.moderation !== undefined) form.append('moderation', config.moderation); - if (config.output_compression !== undefined) form.append('output_compression', String(config.output_compression)); - if (config.input_fidelity !== undefined) form.append('input_fidelity', config.input_fidelity); - if (stream) { - form.append('stream', 'true'); - form.append('partial_images', String(config.partial_images)); - } - for (const [i, source] of sources.entries()) { - // Forward the canonical supported mime; an unsupported source is rejected - // before dispatch, so the fallback only ever carries an already-supported - // generated image. Sending the raw mime (rather than relabeling as png) - // keeps a stray unsupported byte stream failing loud at the backend. - const mime = editSupportedMime(source.mimeType) ?? source.mimeType; - // `image[]` repeated parts: gpt-image accepts multiple, resolving "the - // Nth image" against attach order (see `inspectImageSources`). - form.append('image[]', new Blob([source.bytes], { type: mime }), `image_${i}.${editFileExt(mime)}`); - } - if (config.mask !== undefined) { - if (isRemoteImageSource(config.mask)) throw new Error('Remote image mask reached form encoding before materialization'); - const mask = config.mask; - const mime = editSupportedMime(mask.mimeType) ?? mask.mimeType; - form.append('mask', new Blob([mask.bytes], { type: mime }), `mask.${editFileExt(mime)}`); - } - return form; +const buildEditsRequest = ( + prompt: string, + config: ImageGenerationConfig, + sources: readonly PreparedImageSource[], + mask: PreparedImageSource | undefined, + stream: boolean, +): ImagesEditsRequest => { + const parameters: Record = { + prompt, + n: 1, + ...(config.size === undefined ? {} : { size: config.size }), + ...(config.quality === undefined ? {} : { quality: config.quality }), + ...(config.output_format === undefined ? {} : { output_format: config.output_format }), + ...(config.background === undefined ? {} : { background: config.background }), + ...(config.moderation === undefined ? {} : { moderation: config.moderation }), + ...(config.output_compression === undefined ? {} : { output_compression: config.output_compression }), + ...(config.input_fidelity === undefined ? {} : { input_fidelity: config.input_fidelity }), + ...(stream ? { stream: true, partial_images: config.partial_images } : {}), + }; + const images = sources.map((source, index) => ({ + type: 'upload' as const, + file: new File([source.bytes], `image_${index}.${editFileExt(source.mimeType)}`, { type: source.mimeType }), + })); + const maskFile = mask === undefined + ? undefined + : new File([mask.bytes], `mask.${editFileExt(mask.mimeType)}`, { type: mask.mimeType }); + return { + images, + ...(maskFile === undefined ? {} : { mask: { type: 'upload' as const, file: maskFile } }), + parameters, + }; }; const serverError = (e: unknown): ImageError => ({ @@ -1070,8 +1075,7 @@ const issueImageCall = async ( model: ProviderModel, fetcher: Fetcher, prompt: string, - isEdit: boolean, - sources: readonly ImageSource[], + editRequest: ImagesEditsRequest | null, config: ImageGenerationConfig, state: ShimState, stream: boolean, @@ -1089,9 +1093,9 @@ const issueImageCall = async ( // retry so it reflects the dispatch that actually returned. wrapUpstreamCall: stampUpstreamCallStart(attempt), }; - const { response, modelKey } = await (isEdit - ? provider.instance.callImagesEdits(model, buildEditsForm(prompt, config, sources, stream), state.downstreamAbortSignal, opts) - : provider.instance.callImagesGenerations(model, buildGenerationsBody(prompt, config, stream), state.downstreamAbortSignal, opts)); + const { response, modelKey } = await (editRequest === null + ? provider.instance.callImagesGenerations(model, buildGenerationsBody(prompt, config, stream), state.downstreamAbortSignal, opts) + : provider.instance.callImagesEdits(model, editRequest, state.downstreamAbortSignal, opts)); if (response.status !== 429 || retry >= MAX_RATE_LIMIT_RETRIES) return { response, modelKey }; // 25% jitter desynchronizes parallel callers so a burst of orchestrator @@ -1257,17 +1261,18 @@ const streamImageGeneration = ( let response: Response; let modelKey: string; try { - const prepared = isEdit - ? await prepareEditRequest(sources, state.config) - : { sources, config: state.config }; + let editRequest: ImagesEditsRequest | null = null; + if (isEdit) { + const prepared = await prepareEditRequest(sources, state.config); + editRequest = buildEditsRequest(prompt, state.config, prepared.sources, prepared.mask, wantsPartials); + } ({ response, modelKey } = await issueImageCall( provider, model, fetcher, prompt, - isEdit, - prepared.sources, - prepared.config, + editRequest, + state.config, state, wantsPartials, attempt, @@ -1410,7 +1415,7 @@ export const imageGenerationServerTool: ServerToolRegistration = async (invocati if (!prepared.ok) { return { type: 'invalid-request', message: prepared.error.message, param: prepared.error.param, code: prepared.error.code }; } - let config = prepared.config; + const config = prepared.config; const inspectSources = createImageSourceInspector(); const initialInspection = inspectSources(invocation.payload.input); const initialOperation = resolveImageOperation(config, initialInspection); @@ -1436,31 +1441,41 @@ export const imageGenerationServerTool: ServerToolRegistration = async (invocati code: materializedInputs.error.code, }; } - if (config.mask !== undefined && isRemoteImageSource(config.mask)) { - const remoteMask = config.mask; - const materializedMask = await materializer.mask(remoteMask); - if (!materializedMask.ok) { - return { - type: 'invalid-request', - message: materializedMask.error.message, - param: materializedMask.error.param, - errorType: materializedMask.error.errorType, - code: materializedMask.error.code, - }; - } - if (remoteMask.afterMaterializationError !== undefined) { - return { - type: 'invalid-request', - message: remoteMask.afterMaterializationError.message, - param: remoteMask.afterMaterializationError.param, - code: remoteMask.afterMaterializationError.code, - }; + let mask: ImageSource | undefined; + if (config.mask !== undefined) { + if (isRemoteImageSource(config.mask)) { + const remoteMask = config.mask; + const materializedMask = await materializer.mask(remoteMask); + if (!materializedMask.ok) { + return { + type: 'invalid-request', + message: materializedMask.error.message, + param: materializedMask.error.param, + errorType: materializedMask.error.errorType, + code: materializedMask.error.code, + }; + } + if (remoteMask.afterMaterializationError !== undefined) { + return { + type: 'invalid-request', + message: remoteMask.afterMaterializationError.message, + param: remoteMask.afterMaterializationError.param, + code: remoteMask.afterMaterializationError.code, + }; + } + mask = materializedMask.source; + } else { + mask = config.mask; } - config = { ...config, mask: materializedMask.source }; } + const { mask: _unmaterializedMask, ...configWithoutMask } = config; + const materializedConfig: MaterializedImageGenerationConfig = { + ...configWithoutMask, + ...(mask === undefined ? {} : { mask }), + }; const state: ShimState = { - config, + config: materializedConfig, apiKeyId: gatewayCtx.apiKeyId, upstreamIds: gatewayCtx.upstreamIds, backgroundScheduler: gatewayCtx.backgroundScheduler, @@ -1487,7 +1502,7 @@ export const imageGenerationServerTool: ServerToolRegistration = async (invocati // to editing without bypassing the same source policy used at ingress. // The per-request inspector cache reuses bytes already decoded during // registration and earlier turns. - const operation = resolveImageOperation(config, inspectSources(invocation.payload.input)); + const operation = resolveImageOperation(materializedConfig, inspectSources(invocation.payload.input)); if (!operation.ok) { throw new Error(`image_generation live source invariant violated after request validation: ${operation.error.message}`); } diff --git a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation_test.ts b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation_test.ts index 6bf80b562..c0f7bb3f6 100644 --- a/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation_test.ts +++ b/packages/gateway/src/data-plane/chat/responses/interceptors/server-tools/image-generation_test.ts @@ -1,7 +1,6 @@ import { beforeEach, test } from 'vitest'; import { - buildGenerationsBody, buildImageGenerationFunctionTool, createImageSourceInspector, inspectImageSources, @@ -13,7 +12,6 @@ import { isHostedImageGenerationTool, parseImageStreamEvent, parseRetryAfterMs, - prepareEditSources, prepareImageGenerationConfig, resolveImageOperation, SHIM_TOOL_NAME, @@ -24,7 +22,7 @@ import { initRepo } from '../../../../../repo/index.ts'; import { InMemoryRepo } from '../../../../../repo/memory.ts'; import { mockChatGatewayCtx } from '../../../../../test-helpers/gateway-ctx.ts'; import type { ResponsesInvocation } from '../types.ts'; -import { initExternalResourceFetcher, initImageProcessor } from '@floway-dev/platform'; +import { initExternalResourceFetcher } from '@floway-dev/platform'; import type { CanonicalResponsesPayload, ResponsesInputImage, ResponsesInputItem, ResponsesPayload, ResponsesTool } from '@floway-dev/protocols/responses'; import { assert, assertEquals, assertFalse, assertStringIncludes, assertThrows, stubModelCandidate } from '@floway-dev/test-utils'; @@ -352,30 +350,6 @@ test('inspectImageSources reads tool-result images and preserves forward order', assertEquals(third.mimeType, 'image/webp'); }); -test('prepareEditSources transcodes formats accepted by native Responses but rejected by images edits', async () => { - let observedTarget: unknown = undefined; - let processorCalls = 0; - initImageProcessor({ - compressToWebp: (_input, target) => { - processorCalls += 1; - observedTarget = target; - return Promise.resolve(Uint8Array.of(1, 2, 3)); - }, - }); - const first = { - bytes: Uint8Array.of(4, 5, 6).buffer, - mimeType: 'image/gif', - }; - const sameBytes = { bytes: Uint8Array.of(4, 5, 6).buffer, mimeType: 'image/gif' }; - const prepared = await prepareEditSources([first, first, sameBytes]); - assertEquals(processorCalls, 1); - assert(prepared[0] === prepared[1]); - assert(prepared[0] === prepared[2]); - assertEquals(prepared[0].mimeType, 'image/webp'); - assertEquals([...new Uint8Array(prepared[0].bytes)], [1, 2, 3]); - assertEquals(observedTarget, null); -}); - test('resolveImageOperation exposes a malformed replayed result as an invariant failure', () => { const inspection = inspectImageSources([ { type: 'image_generation_call', id: 'ig_bad', status: 'completed', result: '%%%' }, @@ -471,28 +445,6 @@ test('transformInputItemsForImageGeneration passes non-image items through untou assertEquals(out[0], message); }); -// ── buildGenerationsBody ── - -test('buildGenerationsBody always sends n:1 and maps config, omitting undefined', () => { - const config: ImageGenerationConfig = { model: 'gpt-image-2', size: '1024x1024', quality: 'low', action: 'generate' }; - const body = buildGenerationsBody('a cat', config, false); - assertEquals(body.prompt, 'a cat'); - assertEquals(body.n, 1); - assertEquals(body.size, '1024x1024'); - assertEquals(body.quality, 'low'); - assertFalse('background' in body); - assertFalse('output_format' in body); - assertFalse('stream' in body); - assertFalse('partial_images' in body); -}); - -test('buildGenerationsBody adds stream and partial_images when streaming', () => { - const config: ImageGenerationConfig = { model: 'gpt-image-2', partial_images: 2, action: 'generate' }; - const body = buildGenerationsBody('a cat', config, true); - assertEquals(body.stream, true); - assertEquals(body.partial_images, 2); -}); - // ── imageTerminal ── test('imageTerminal on success echoes the backend-resolved fields and closes with a single completed event', () => { diff --git a/packages/gateway/src/data-plane/codex/images_test.ts b/packages/gateway/src/data-plane/codex/images_test.ts new file mode 100644 index 000000000..e1d257261 --- /dev/null +++ b/packages/gateway/src/data-plane/codex/images_test.ts @@ -0,0 +1,161 @@ +import { test } from 'vitest'; + +import type { InMemoryRepo } from '../../repo/memory.ts'; +import { copilotModels, requestApp, setupAppTest } from '../../test-helpers.ts'; +import { assertEquals, assertExists, jsonResponse, withMockedFetch } from '@floway-dev/test-utils'; + +const PNG_B64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M8AAAMBAQDJ/wEAAAAASUVORK5CYII='; + +const saveAzureImages = async (repo: InMemoryRepo): Promise => { + await repo.upstreams.save({ + id: 'az-image', + kind: 'azure', + name: 'azure-images', + enabled: true, + sortOrder: 1, + createdAt: '2026-05-25T00:00:00Z', + updatedAt: '2026-05-25T00:00:00Z', + flagOverrides: {}, + disabledPublicModelIds: [], + proxyFallbackList: [], + modelPrefix: null, + color: null, + config: { + endpoint: 'https://example.openai.azure.com/openai/v1', + apiKey: 'azkey', + models: [{ + upstreamModelId: 'gpt-image-2', + endpoints: { imagesGenerations: {}, imagesEdits: {} }, + }], + }, + state: null, + }); +}; + +const controlPlaneFetch = (request: Request): Response | undefined => { + const url = new URL(request.url); + if (url.hostname === 'update.code.visualstudio.com') return jsonResponse(['1.110.1']); + if (url.pathname === '/copilot_internal/v2/token') { + return jsonResponse({ token: 'copilot-access-token', expires_at: 4102444800, refresh_in: 3600, endpoints: { api: 'https://api.individual.githubcopilot.com' } }); + } + if (url.hostname === 'api.individual.githubcopilot.com' && url.pathname === '/models') { + return jsonResponse(copilotModels([{ id: 'copilot-chat', supported_endpoints: ['/chat/completions'] }])); + } + return undefined; +}; + +test('Codex provider-relative image generation reuses the public image-generation handler', async () => { + const { apiKey, repo } = await setupAppTest(); + await saveAzureImages(repo); + let observedUrl: string | undefined; + let observedBody: Record | undefined; + + await withMockedFetch( + async request => { + const control = controlPlaneFetch(request); + if (control) return control; + const url = new URL(request.url); + if (url.hostname === 'example.openai.azure.com') { + observedUrl = request.url; + observedBody = await request.json() as Record; + return jsonResponse({ data: [{ b64_json: 'aGk=' }] }); + } + throw new Error(`Unhandled fetch ${request.url}`); + }, + async () => { + const response = await requestApp('/azure-api.codex/images/generations', { + method: 'POST', + headers: { authorization: `Bearer ${apiKey.key}`, 'content-type': 'application/json' }, + body: JSON.stringify({ model: 'gpt-image-2', prompt: 'a fox in space', quality: 'high' }), + }); + assertEquals(response.status, 200); + assertEquals(await response.json(), { data: [{ b64_json: 'aGk=' }] }); + }, + ); + + assertEquals(observedUrl?.endsWith('/images/generations?api-version=preview'), true); + assertExists(observedBody); + assertEquals(observedBody.prompt, 'a fox in space'); + assertEquals(observedBody.quality, 'high'); +}); + +test('Codex provider-relative image edits reuse the public JSON handler', async () => { + const { apiKey, repo } = await setupAppTest(); + await saveAzureImages(repo); + let observedUrl: string | undefined; + let observedBody: Record | undefined; + + await withMockedFetch( + async request => { + const control = controlPlaneFetch(request); + if (control) return control; + const url = new URL(request.url); + if (url.hostname === 'example.openai.azure.com') { + observedUrl = request.url; + observedBody = await request.json() as Record; + return jsonResponse({ data: [{ b64_json: 'ZWRpdA==' }] }); + } + throw new Error(`Unhandled fetch ${request.url}`); + }, + async () => { + const response = await requestApp('/azure-api.codex/images/edits', { + method: 'POST', + headers: { authorization: `Bearer ${apiKey.key}`, 'content-type': 'application/json' }, + body: JSON.stringify({ + model: 'gpt-image-2', + prompt: 'add a red hat', + quality: 'high', + images: [ + { image_url: 'https://assets.example/image.png' }, + ], + }), + }); + assertEquals(response.status, 200); + assertEquals(await response.json(), { data: [{ b64_json: 'ZWRpdA==' }] }); + }, + ); + + assertEquals(observedUrl?.endsWith('/images/edits?api-version=preview'), true); + assertExists(observedBody); + assertEquals(observedBody, { + model: 'gpt-image-2', + prompt: 'add a red hat', + quality: 'high', + images: [{ image_url: 'https://assets.example/image.png' }], + }); +}); + +test('Codex inline data URL edits egress as multipart uploads', async () => { + const { apiKey, repo } = await setupAppTest(); + await saveAzureImages(repo); + let observedForm: FormData | undefined; + + await withMockedFetch( + async request => { + const control = controlPlaneFetch(request); + if (control) return control; + if (new URL(request.url).hostname === 'example.openai.azure.com') { + observedForm = await request.formData(); + return jsonResponse({ data: [{ b64_json: 'ZWRpdA==' }] }); + } + throw new Error(`Unhandled fetch ${request.url}`); + }, + async () => { + const response = await requestApp('/azure-api.codex/images/edits', { + method: 'POST', + headers: { authorization: `Bearer ${apiKey.key}`, 'content-type': 'application/json' }, + body: JSON.stringify({ + model: 'gpt-image-2', + prompt: 'add a red hat', + images: [{ image_url: `data:image/png;base64,${PNG_B64}` }], + }), + }); + assertEquals(response.status, 200); + }, + ); + + assertExists(observedForm); + const image = observedForm.get('image'); + assertEquals(image instanceof File, true); + assertEquals((image as File).type, 'image/png'); +}); diff --git a/packages/gateway/src/data-plane/codex/routes.ts b/packages/gateway/src/data-plane/codex/routes.ts index 02d35da11..67fdb9786 100644 --- a/packages/gateway/src/data-plane/codex/routes.ts +++ b/packages/gateway/src/data-plane/codex/routes.ts @@ -3,13 +3,15 @@ // OAuth ("ChatGPT") mode has two independently configured bases: // // chatgpt_base_url — analytics, plugins, agent identity, Apps MCP -// model_providers.x.base_url — models, Responses HTTP/WS, remote compaction +// model_providers.x.base_url — models, Responses, compaction, image requests // // The dashboard points both at this namespace. Codex appends `models`, -// `responses`, and `responses/compact` directly to the model-provider base: +// `responses`, `responses/compact`, `images/generations`, and `images/edits` +// directly to the model-provider base: // https://github.com/openai/codex/blob/44918ea10c0f99151c6710411b4322c2f5c96bea/codex-rs/codex-api/src/endpoint/models.rs#L31-L43 // https://github.com/openai/codex/blob/44918ea10c0f99151c6710411b4322c2f5c96bea/codex-rs/codex-api/src/endpoint/responses.rs#L100-L102 // https://github.com/openai/codex/blob/44918ea10c0f99151c6710411b4322c2f5c96bea/codex-rs/codex-api/src/endpoint/compact.rs#L35-L57 +// https://github.com/openai/codex/blob/f90e7deea6a715bbd153044af6f475eefa749177/codex-rs/codex-api/src/endpoint/images.rs#L33-L68 // GET on `/responses` is the WebSocket upgrade entry; POST uses the generic // Responses handler, and `/responses/compact` uses its compaction counterpart. // Mounting the generic WS handler here preserves its session-local item chain. @@ -49,6 +51,7 @@ import { codexModels } from './models.ts'; import type { AuthVars } from '../../middleware/auth.ts'; import { responsesHttp } from '../chat/responses/http.ts'; import { responsesWebSocket } from '../chat/responses/websocket.ts'; +import { imagesEdits, imagesGenerations } from '../images/serve.ts'; const CODEX_BASE_PATH = '/azure-api.codex'; @@ -56,6 +59,8 @@ export const mountCodexRoutes = (app: Hono<{ Variables: AuthVars }>) => { app.post(`${CODEX_BASE_PATH}/responses`, responsesHttp.generate); app.post(`${CODEX_BASE_PATH}/responses/compact`, responsesHttp.compact); app.get(`${CODEX_BASE_PATH}/responses`, responsesWebSocket); + app.post(`${CODEX_BASE_PATH}/images/generations`, imagesGenerations); + app.post(`${CODEX_BASE_PATH}/images/edits`, imagesEdits); app.get(`${CODEX_BASE_PATH}/models`, codexModels); app.post(`${CODEX_BASE_PATH}/codex/analytics-events/events`, codexAnalyticsEventsEvents); diff --git a/packages/gateway/src/data-plane/images/serve.ts b/packages/gateway/src/data-plane/images/serve.ts index f7c9cce4e..9195424b4 100644 --- a/packages/gateway/src/data-plane/images/serve.ts +++ b/packages/gateway/src/data-plane/images/serve.ts @@ -2,50 +2,99 @@ // requests to the provider that declares the requested model and the // matching image endpoint capability. // -// Edits multipart bodies are loaded into memory via `request.formData()`; -// this caps the per-request body size at the Workers heap (~128 MB). -// Sufficient for the gpt-image-2 single-image edit case (≤50 MB image + -// ≤50 MB mask). Multi-image edits with the gpt-image-1 `image[]` array -// may exceed the heap — a streaming multipart parser is a follow-up. +// The edits handler accepts multipart uploads and JSON `images` references. +// Both are buffered once for dump capture and normalized into a semantic +// request; each provider owns the final JSON or multipart serialization. +// https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L12558-L12620 import type { Context } from 'hono'; import { backgroundSchedulerFromContext } from '../../runtime/background.ts'; import { createGatewayCtxFromHono, finalizeGatewayResponse } from '../chat/shared/gateway-ctx.ts'; -import { readRequestBody, takeRequestBody } from '../chat/shared/request-body.ts'; +import { readRequestBody, takeRequestBody, type RequestBody } from '../chat/shared/request-body.ts'; import { passthroughApiError, passthroughServe } from '../shared/passthrough-serve.ts'; import { tokenUsageFromImagesBody } from '../shared/telemetry/usage.ts'; +import type { ImageEditReference } from '@floway-dev/protocols/images'; +import { isBase64ImageDataUrl, type ImagesEditsRequest, type ImagesEditsSource } from '@floway-dev/provider'; -interface ImagesGenerationsRequestBody { +interface JsonModelRequestBody { model?: unknown; - prompt?: unknown; [key: string]: unknown; } -type PreparedRequest = +type PreparedJsonRequest = | { type: 'ok'; body: Record; model: string } | { type: 'invalid'; message: string }; -const prepareImagesGenerationsRequest = (bytes: Uint8Array): PreparedRequest => { - let request: ImagesGenerationsRequestBody; +const prepareJsonModelRequest = (bytes: Uint8Array, requestName: string): PreparedJsonRequest => { + let request: JsonModelRequestBody; try { const parsed = JSON.parse(new TextDecoder().decode(bytes)) as unknown; if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { - return { type: 'invalid', message: 'Images generations request body must be an object.' }; + return { type: 'invalid', message: `${requestName} request body must be an object.` }; } - request = parsed as ImagesGenerationsRequestBody; + request = parsed as JsonModelRequestBody; } catch { - return { type: 'invalid', message: 'Images generations request body must be valid JSON.' }; + return { type: 'invalid', message: `${requestName} request body must be valid JSON.` }; } if (typeof request.model !== 'string' || request.model.length === 0) { - return { type: 'invalid', message: 'Images generations request body must include a model string.' }; + return { type: 'invalid', message: `${requestName} request body must include a model string.` }; } return { type: 'ok', body: request as Record, model: request.model }; }; +type PreparedImagesEdit = + | { type: 'ok'; request: ImagesEditsRequest } + | { type: 'invalid'; message: string }; + +const imageEditSource = (value: unknown, path: string): ImagesEditsSource | string => { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + return `${path} must be an object.`; + } + const reference = value as { image_url?: unknown; file_id?: unknown }; + const { image_url: imageUrl, file_id: fileId } = reference; + if (typeof imageUrl === 'string' && fileId === undefined) { + const imageReference = reference as ImageEditReference & { image_url: string }; + return isBase64ImageDataUrl(imageUrl) + ? { type: 'inline', reference: imageReference } + : { type: 'reference', reference: imageReference }; + } + if (typeof fileId === 'string' && imageUrl === undefined) { + return { type: 'reference', reference: reference as ImageEditReference }; + } + return `${path} must contain exactly one string field: image_url or file_id.`; +}; + +const prepareJsonImagesEdit = (body: Record): PreparedImagesEdit => { + if (!Array.isArray(body.images)) { + return { type: 'invalid', message: 'Image edits request body must include an images array.' }; + } + const images: ImagesEditsSource[] = []; + for (const [index, value] of body.images.entries()) { + const source = imageEditSource(value, `Image edits images[${index}]`); + if (typeof source === 'string') return { type: 'invalid', message: source }; + images.push(source); + } + let mask: ImagesEditsSource | undefined; + if (body.mask !== undefined) { + const source = imageEditSource(body.mask, 'Image edits mask'); + if (typeof source === 'string') return { type: 'invalid', message: source }; + mask = source; + } + const { model: _model, images: _images, mask: _mask, ...parameters } = body; + return { + type: 'ok', + request: { + images, + ...(mask === undefined ? {} : { mask }), + parameters, + }, + }; +}; + export const imagesGenerations = async (c: Context): Promise => { const requestBody = await readRequestBody(c); - const request = prepareImagesGenerationsRequest(requestBody.bytes); + const request = prepareJsonModelRequest(requestBody.bytes, 'Images generations'); const ctx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) }); if (request.type === 'invalid') { ctx.dump?.error('gateway'); @@ -70,54 +119,81 @@ export const imagesGenerations = async (c: Context): Promise => { return finalizeGatewayResponse(ctx, response); }; -export const imagesEdits = async (c: Context): Promise => { - // Buffer the multipart body once. Hono's formData() helper would consume - // c.req.raw.body internally; re-parsing from the captured bytes via a fresh - // Response keeps the dump capture honest without a second read on the wire. - const requestBody = await readRequestBody(c); - let form: FormData; - try { - form = await new Response(requestBody.bytes as BodyInit, { headers: { 'content-type': c.req.header('content-type') ?? '' } }).formData(); - } catch { - // Match the embeddings serve stance: do not surface the underlying - // parser's error text. The wording is enough for a client to know - // they sent the wrong content type or a malformed body. - const errorCtx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) }); - errorCtx.dump?.error('gateway'); - return finalizeGatewayResponse(errorCtx, passthroughApiError(c, 'Image edits request body must be a valid multipart/form-data payload.', 400)); - } - +const serveImagesEditRequest = async ( + c: Context, + requestBody: RequestBody, + model: string, + request: ImagesEditsRequest, +): Promise => { const ctx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) }); - - const modelRaw = form.get('model'); - if (typeof modelRaw !== 'string' || modelRaw.length === 0) { - ctx.dump?.error('gateway'); - return finalizeGatewayResponse(ctx, passthroughApiError(c, 'Image edits request body must include a model field.', 400)); - } - - ctx.dump?.requestedModel(modelRaw); + ctx.dump?.requestedModel(model); const response = await passthroughServe({ c, ctx, sourceApi: '/images/edits', operation: 'image_edit', - model: modelRaw, + model, kind: 'image', modelServesEndpoint: model => model.endpoints.imagesEdits !== undefined, - call: (provider, model, opts) => { - // ProviderInstance.callImagesEdits takes ownership of the FormData and - // appends the upstream-specific model/deployment id; allocate a fresh - // copy per candidate so the contract holds even if cross-candidate - // fallback is ever extended to try a second match. File-blob entries - // are passed by reference so no buffer copy happens. - const passthrough = new FormData(); - for (const [name, value] of form.entries()) { - if (name === 'model') continue; - passthrough.append(name, value); - } - return provider.instance.callImagesEdits(model, passthrough, undefined, opts); - }, + call: (provider, model, opts) => provider.instance.callImagesEdits(model, request, undefined, opts), response: { format: 'json', extractBilling: tokenUsageFromImagesBody }, }); return finalizeGatewayResponse(ctx, response); }; + +export const imagesEdits = async (c: Context): Promise => { + const requestBody = await readRequestBody(c); + const invalid = (message: string): Response => { + const errorCtx = createGatewayCtxFromHono(c, { wantsStream: false, requestBody: takeRequestBody(requestBody), backgroundScheduler: backgroundSchedulerFromContext(c) }); + errorCtx.dump?.error('gateway'); + return finalizeGatewayResponse(errorCtx, passthroughApiError(c, message, 400)); + }; + + const contentType = c.req.header('content-type'); + if (contentType === undefined) { + return invalid('Image edits request body must use application/json or multipart/form-data.'); + } + const mediaType = contentType.replace(/;.*$/u, '').trim().toLowerCase(); + if (mediaType === 'application/json') { + const body = prepareJsonModelRequest(requestBody.bytes, 'Image edits'); + if (body.type === 'invalid') return invalid(body.message); + const request = prepareJsonImagesEdit(body.body); + if (request.type === 'invalid') return invalid(request.message); + return await serveImagesEditRequest(c, requestBody, body.model, request.request); + } + + if (mediaType !== 'multipart/form-data') { + return invalid('Image edits request body must use application/json or multipart/form-data.'); + } + let form: FormData; + try { + form = await new Response(requestBody.bytes as BodyInit, { headers: { 'content-type': contentType } }).formData(); + } catch { + return invalid('Image edits request body must be valid multipart/form-data.'); + } + const model = form.get('model'); + if (typeof model !== 'string' || model.length === 0) { + return invalid('Image edits request body must include a model field.'); + } + const images: File[] = []; + let mask: File | undefined; + const parameters: Record = {}; + for (const [name, value] of form.entries()) { + if (name === 'model') continue; + if (name === 'image' || name === 'image[]') { + if (!(value instanceof File)) return invalid(`Image edits ${name} fields must be files.`); + images.push(value); + } else if (name === 'mask') { + if (!(value instanceof File)) return invalid('Image edits mask field must be a file.'); + mask = value; + } else { + if (typeof value !== 'string') return invalid(`Image edits ${name} field must be text.`); + parameters[name] = value; + } + } + return await serveImagesEditRequest(c, requestBody, model, { + images: images.map(file => ({ type: 'upload', file })), + ...(mask === undefined ? {} : { mask: { type: 'upload' as const, file: mask } }), + parameters, + }); +}; diff --git a/packages/gateway/src/data-plane/images/serve_test.ts b/packages/gateway/src/data-plane/images/serve_test.ts index 437020d36..2898099ae 100644 --- a/packages/gateway/src/data-plane/images/serve_test.ts +++ b/packages/gateway/src/data-plane/images/serve_test.ts @@ -4,6 +4,8 @@ import { buildCustomUpstreamRecord, copilotModels, flushAsyncWork, requestApp, s import { clearInProcessCopilotTokenCache } from '@floway-dev/provider-copilot'; import { jsonResponse, withMockedFetch, assertEquals, assertExists } from '@floway-dev/test-utils'; +const PNG_B64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M8AAAMBAQDJ/wEAAAAASUVORK5CYII='; + test('/v1/images/generations rejects malformed JSON body with 400', async () => { const { apiKey } = await setupAppTest(); const response = await requestApp('/v1/images/generations', { @@ -42,7 +44,7 @@ test('/v1/images/generations 404s when no upstream provides the model', async () async () => { const response = await requestApp('/v1/images/generations', { method: 'POST', - headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, + headers: { 'content-type': 'application/json; charset=utf-8', 'x-api-key': apiKey.key }, body: JSON.stringify({ model: 'no-such-model', prompt: 'hi' }), }); assertEquals(response.status, 404); @@ -50,12 +52,22 @@ test('/v1/images/generations 404s when no upstream provides the model', async () ); }); -test('/v1/images/edits rejects non-multipart body with 400', async () => { +test('/v1/images/edits rejects malformed JSON with 400', async () => { + const { apiKey } = await setupAppTest(); + const response = await requestApp('/v1/images/edits', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, + body: 'not json', + }); + assertEquals(response.status, 400); +}); + +test('/v1/images/edits rejects JSON without a model with 400', async () => { const { apiKey } = await setupAppTest(); const response = await requestApp('/v1/images/edits', { method: 'POST', headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, - body: '{}', + body: JSON.stringify({ prompt: 'hi', images: [{ file_id: 'file-image' }] }), }); assertEquals(response.status, 400); }); @@ -235,3 +247,70 @@ test('/v1/images/edits forwards a multipart request through an Azure model and r const usageRows = await repo.usage.listAll(); assertEquals(usageRows.some(row => row.model === 'gpt-image-2' && row.tokens.input === 7 && row.tokens.output === 11), true); }); + +test('/v1/images/edits forwards JSON image references through a custom provider', async () => { + const { apiKey, repo } = await setupAppTest(); + clearInProcessCopilotTokenCache(); + await repo.upstreams.save(buildCustomUpstreamRecord({ + id: 'up_image_edits_json', + name: 'Custom Image Provider', + sortOrder: 100, + config: { + baseUrl: 'https://images.example.com', + authStyle: 'bearer', + apiKey: 'sk-images', + endpoints: {}, + }, + })); + + let forwarded: Record | undefined; + await withMockedFetch( + async request => { + const url = new URL(request.url); + if (url.hostname === 'update.code.visualstudio.com') return jsonResponse(['1.110.1']); + if (url.pathname === '/copilot_internal/v2/token') { + return jsonResponse({ token: 'copilot-access-token', expires_at: 4102444800, refresh_in: 3600, endpoints: { api: 'https://api.individual.githubcopilot.com' } }); + } + if (url.hostname === 'api.individual.githubcopilot.com' && url.pathname === '/models') { + return jsonResponse(copilotModels([{ id: 'copilot-chat', supported_endpoints: ['/chat/completions'] }])); + } + if (url.hostname === 'images.example.com' && url.pathname === '/v1/models') { + return jsonResponse({ data: [{ id: 'gpt-image-2' }] }); + } + if (url.hostname === 'images.example.com' && url.pathname === '/v1/images/edits') { + forwarded = await request.json() as Record; + return jsonResponse({ data: [{ b64_json: 'ZWRpdA==' }] }); + } + throw new Error(`Unhandled fetch ${request.url}`); + }, + async () => { + const response = await requestApp('/v1/images/edits', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-api-key': apiKey.key }, + body: JSON.stringify({ + model: 'gpt-image-2', + prompt: 'replace the background', + images: [ + { image_url: `data:image/png;base64,${PNG_B64}` }, + { file_id: 'file-source' }, + ], + mask: { file_id: 'file-mask' }, + quality: 'high', + }), + }); + assertEquals(response.status, 200); + assertEquals(await response.json(), { data: [{ b64_json: 'ZWRpdA==' }] }); + }, + ); + + assertEquals(forwarded, { + model: 'gpt-image-2', + prompt: 'replace the background', + images: [ + { image_url: `data:image/png;base64,${PNG_B64}` }, + { file_id: 'file-source' }, + ], + mask: { file_id: 'file-mask' }, + quality: 'high', + }); +}); diff --git a/packages/gateway/src/data-plane/providers/azure/provider_test.ts b/packages/gateway/src/data-plane/providers/azure/provider_test.ts index 9facb4544..0014e0851 100644 --- a/packages/gateway/src/data-plane/providers/azure/provider_test.ts +++ b/packages/gateway/src/data-plane/providers/azure/provider_test.ts @@ -432,10 +432,13 @@ test('createAzureProvider callImagesEdits posts multipart with model replaced by async () => { const provider = createAzureProvider(record); const models = await provider.instance.getProvidedModels(directFetcher); - const form = new FormData(); - form.append('prompt', 'replace sky'); - form.append('image', new Blob([new Uint8Array([1, 2, 3])], { type: 'image/png' }), 'photo.png'); - const result = await provider.instance.callImagesEdits(models[0], form, undefined, noopUpstreamCallOptions()); + const result = await provider.instance.callImagesEdits(models[0], { + parameters: { prompt: 'replace sky' }, + images: [{ + type: 'upload', + file: new File([new Uint8Array([1, 2, 3])], 'photo.png', { type: 'image/png' }), + }], + }, undefined, noopUpstreamCallOptions()); assertEquals(result.modelKey, 'gpt-image-2'); assertEquals(result.response.status, 200); }, diff --git a/packages/gateway/src/data-plane/providers/custom/provider_test.ts b/packages/gateway/src/data-plane/providers/custom/provider_test.ts index ffe4d969a..e4f2172fe 100644 --- a/packages/gateway/src/data-plane/providers/custom/provider_test.ts +++ b/packages/gateway/src/data-plane/providers/custom/provider_test.ts @@ -225,11 +225,14 @@ test('Custom provider callImagesEdits forwards multipart body with model field a async () => { const provider = createCustomProvider(record); const models = await provider.instance.getProvidedModels(directFetcher); - const form = new FormData(); - form.append('prompt', 'add a kite'); - form.append('image', new Blob([new Uint8Array([1, 2, 3])], { type: 'image/png' }), 'photo.png'); const model = models[0]; const opts = noopUpstreamCallOptions(); - const result = await provider.instance.callImagesEdits(model, form, undefined, opts); + const result = await provider.instance.callImagesEdits(model, { + parameters: { prompt: 'add a kite' }, + images: [{ + type: 'upload', + file: new File([new Uint8Array([1, 2, 3])], 'photo.png', { type: 'image/png' }), + }], + }, undefined, opts); assertEquals(result.modelKey, 'gpt-image-2'); assertEquals(result.response.status, 200); }, diff --git a/packages/protocols/src/images/index.ts b/packages/protocols/src/images/index.ts index dfb67aa95..512f802c7 100644 --- a/packages/protocols/src/images/index.ts +++ b/packages/protocols/src/images/index.ts @@ -5,8 +5,6 @@ // without a gateway-side reject while named fields keep their narrow // types when accessed directly — the `T & Record` // intersection form would widen every typed field to `unknown` on read. -// /v1/images/edits is multipart and has no JSON payload type — its source -// serve handles `FormData` directly. export interface ImagesGenerationsPayload { model: string; prompt: string; @@ -23,3 +21,31 @@ export interface ImagesGenerationsPayload { user?: string; [key: string]: unknown; } + +export type ImageEditReference = + | { image_url: string; file_id?: never; [key: string]: unknown } + | { file_id: string; image_url?: never; [key: string]: unknown }; + +// POST /v1/images/edits accepts either multipart uploads or this JSON shape. +// JSON uses `images` rather than the multipart `image` field, and references +// may point at a URL/data URL or an uploaded file. +// https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L12558-L12620 +// https://github.com/openai/openai-openapi/blob/a3276900e58b8b2a92e0cb087cd2e6e005f58458/openapi.yaml#L47542-L47673 +export interface ImagesEditsJsonPayload { + model: string; + prompt: string; + images: ImageEditReference[]; + mask?: ImageEditReference; + n?: number | null; + quality?: string | null; + input_fidelity?: string | null; + size?: string | null; + user?: string; + output_format?: string | null; + output_compression?: number | null; + moderation?: string | null; + background?: string | null; + stream?: boolean | null; + partial_images?: number | null; + [key: string]: unknown; +} diff --git a/packages/provider-azure/src/provider.ts b/packages/provider-azure/src/provider.ts index ea6c7f56c..209a0b5ba 100644 --- a/packages/provider-azure/src/provider.ts +++ b/packages/provider-azure/src/provider.ts @@ -5,7 +5,7 @@ import { parseChatCompletionsStream } from '@floway-dev/protocols/chat-completio import { kindForEndpoints } from '@floway-dev/protocols/common'; import { parseMessagesStream } from '@floway-dev/protocols/messages'; import { parseResponsesStream, type ResponsesResult, toCompactPayloadShape } from '@floway-dev/protocols/responses'; -import { type ProviderInstance, type Provider, type ProviderModel, type ProviderStreamParser, type UpstreamCallOptions, type UpstreamFetchOptions, type UpstreamRecord, publicModelId, resolveEffectiveFlags, streamingProviderCall } from '@floway-dev/provider'; +import { serializeOpenAIImagesEditsRequest, type ProviderInstance, type Provider, type ProviderModel, type ProviderStreamParser, type UpstreamCallOptions, type UpstreamFetchOptions, type UpstreamRecord, publicModelId, resolveEffectiveFlags, streamingProviderCall } from '@floway-dev/provider'; const upstreamModelIdOf = (model: ProviderModel): string => (model.providerData as { upstreamModelId: string }).upstreamModelId; @@ -90,12 +90,9 @@ export const createAzureProvider = (record: UpstreamRecord): Provider => { callMessagesCountTokens: (model, body, signal, opts) => callNonStreaming(azureFetchMessagesCountTokens, model, body, signal, opts.headers, opts), callEmbeddings: (model, body, signal, opts) => callNonStreaming(azureFetchEmbeddings, model, body, signal, opts.headers, opts), callImagesGenerations: (model, body, signal, opts) => callNonStreaming(azureFetchImagesGenerations, model, body, signal, opts.headers, opts), - callImagesEdits: async (model, body, signal, opts) => { - // Azure routes by upstream model id in the multipart `model` field; the - // runtime re-encodes the FormData with a fresh boundary and sets - // Content-Type itself. + callImagesEdits: async (model, request, signal, opts) => { const upstreamModelId = upstreamModelIdOf(model); - body.append('model', upstreamModelId); + const body = await serializeOpenAIImagesEditsRequest(request, upstreamModelId); const response = await azureFetchImagesEdits(azure.config, { method: 'POST', body, signal }, { extraHeaders: opts.headers, fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }); return { response, modelKey: upstreamModelId }; }, diff --git a/packages/provider-custom/src/provider.ts b/packages/provider-custom/src/provider.ts index abc349338..576894229 100644 --- a/packages/provider-custom/src/provider.ts +++ b/packages/provider-custom/src/provider.ts @@ -7,7 +7,7 @@ import { parseChatCompletionsStream } from '@floway-dev/protocols/chat-completio import { type ModelEndpoints, kindForEndpoints } from '@floway-dev/protocols/common'; import { parseMessagesStream } from '@floway-dev/protocols/messages'; import { parseResponsesStream, type ResponsesResult, toCompactPayloadShape } from '@floway-dev/protocols/responses'; -import { publicModelId, resolveEffectiveFlags, streamingProviderCall, type FlagId, type ProviderInstance, type Provider, type ProviderCallResult, type ProviderModel, type ProviderStreamParser, type UpstreamCallOptions, type UpstreamFetchOptions, type UpstreamRecord } from '@floway-dev/provider'; +import { serializeOpenAIImagesEditsRequest, publicModelId, resolveEffectiveFlags, streamingProviderCall, type FlagId, type ProviderInstance, type Provider, type ProviderCallResult, type ProviderModel, type ProviderStreamParser, type UpstreamCallOptions, type UpstreamFetchOptions, type UpstreamRecord } from '@floway-dev/provider'; const rawModelIdOf = (model: ProviderModel): string => model.providerData as string; @@ -178,11 +178,9 @@ export const createCustomProvider = (record: UpstreamRecord): Provider => { callMessagesCountTokens: (model, body, signal, opts) => call(customFetchMessagesCountTokens, model, body, signal, opts.headers, opts), callEmbeddings: (model, body, signal, opts) => call(customFetchEmbeddings, model, body, signal, opts.headers, opts), callImagesGenerations: (model, body, signal, opts) => call(customFetchImagesGenerations, model, body, signal, opts.headers, opts), - callImagesEdits: async (model, body, signal, opts) => { - // The runtime auto-encodes the FormData with a fresh boundary and sets - // Content-Type itself. + callImagesEdits: async (model, request, signal, opts) => { const rawModelId = rawModelIdOf(model); - body.append('model', rawModelId); + const body = await serializeOpenAIImagesEditsRequest(request, rawModelId); const response = await customFetchImagesEdits(config, { method: 'POST', body, signal }, { extraHeaders: opts.headers, fetcher: opts.fetcher, wrapUpstreamCall: opts.wrapUpstreamCall }); return { response, modelKey: rawModelId }; }, diff --git a/packages/provider/src/image-helpers.ts b/packages/provider/src/image-helpers.ts index c92434dd2..1540006fc 100644 --- a/packages/provider/src/image-helpers.ts +++ b/packages/provider/src/image-helpers.ts @@ -2,14 +2,14 @@ import { compressBytesToWebp, type ImageSizeCalculator } from '@floway-dev/platf const BASE64_CHUNK = 0x8000; -const base64ToBytes = (base64: string): Uint8Array => { +export const base64ToBytes = (base64: string): Uint8Array => { const binary = atob(base64); const bytes = new Uint8Array(binary.length); for (let i = 0; i < binary.length; i += 1) bytes[i] = binary.charCodeAt(i); return bytes; }; -const bytesToBase64 = (bytes: Uint8Array): string => { +export const bytesToBase64 = (bytes: Uint8Array): string => { let binary = ''; for (let offset = 0; offset < bytes.length; offset += BASE64_CHUNK) { binary += String.fromCharCode(...bytes.subarray(offset, offset + BASE64_CHUNK)); @@ -17,7 +17,14 @@ const bytesToBase64 = (bytes: Uint8Array): string => { return btoa(binary); }; -const BASE64_DATA_URL = /^data:([^;,]+);base64,(.*)$/s; +const BASE64_DATA_URL = /^data:([^;,]+)(?:;[^,;]*)*;base64,(.*)$/is; + +export const parseBase64ImageDataUrl = (url: string): { mimeType: string; base64: string } | null => { + const match = BASE64_DATA_URL.exec(url); + const mimeType = match?.[1]; + const base64 = match?.[2]; + return mimeType?.toLowerCase().startsWith('image/') && base64 !== undefined ? { mimeType, base64 } : null; +}; const compressBase64ImageToWebp = async ( base64: string, @@ -34,14 +41,14 @@ const compressImageDataUrlToWebp = async ( url: string, calculator: ImageSizeCalculator, ): Promise => { - const match = BASE64_DATA_URL.exec(url); - if (!match?.[1].startsWith('image/')) return url; - const webp = await compressBase64ImageToWebp(match[2], calculator); + const parsed = parseBase64ImageDataUrl(url); + if (parsed === null) return url; + const webp = await compressBase64ImageToWebp(parsed.base64, calculator); return `data:image/webp;base64,${webp}`; }; export const isBase64ImageDataUrl = (url: string): boolean => - BASE64_DATA_URL.exec(url)?.[1].startsWith('image/') ?? false; + parseBase64ImageDataUrl(url) !== null; // Per-request memoizing wrappers around the compress helpers above. A single // agentic request often replays the same screenshot across many turns, so the diff --git a/packages/provider/src/images.ts b/packages/provider/src/images.ts new file mode 100644 index 000000000..a0ef376cf --- /dev/null +++ b/packages/provider/src/images.ts @@ -0,0 +1,91 @@ +import { base64ToBytes, bytesToBase64, parseBase64ImageDataUrl } from './image-helpers.ts'; +import type { ImageEditReference } from '@floway-dev/protocols/images'; + +// Each source stores one authoritative representation. Multipart requires +// upload-like sources with no extra reference fields plus scalar parameters; +// every other request uses JSON and encodes File uploads as data URLs. +interface UploadedImagesEditsSource { + type: 'upload'; + file: File; +} + +interface InlineImagesEditsSource { + type: 'inline'; + reference: ImageEditReference & { image_url: string }; +} + +interface ReferencedImagesEditsSource { + type: 'reference'; + reference: ImageEditReference; +} + +export type ImagesEditsSource = UploadedImagesEditsSource | InlineImagesEditsSource | ReferencedImagesEditsSource; + +export interface ImagesEditsRequest { + images: ImagesEditsSource[]; + mask?: ImagesEditsSource; + parameters: Record; +} + +const uploadedFile = (source: ImagesEditsSource, index: number): File | null => { + if (source.type === 'upload') return source.file; + if (source.type === 'reference') return null; + const parsed = parseBase64ImageDataUrl(source.reference.image_url); + if (parsed === null) return null; + let bytes: Uint8Array; + try { + bytes = base64ToBytes(parsed.base64); + } catch { + return null; + } + return new File([bytes], `image-${index}`, { type: parsed.mimeType }); +}; + +const jsonReference = async (source: ImagesEditsSource): Promise => { + if (source.type === 'inline' || source.type === 'reference') return source.reference; + const bytes = new Uint8Array(await source.file.arrayBuffer()); + return { image_url: `data:${source.file.type};base64,${bytesToBase64(bytes)}` }; +}; + +const jsonBody = async (request: ImagesEditsRequest): Promise> => { + const images = await Promise.all(request.images.map(jsonReference)); + const mask = request.mask === undefined ? undefined : await jsonReference(request.mask); + return { + ...request.parameters, + images, + ...(mask === undefined ? {} : { mask }), + }; +}; + +const multipartBody = (request: ImagesEditsRequest, model: string): FormData | null => { + const sources = [...request.images, ...(request.mask === undefined ? [] : [request.mask])]; + const compatibleSources = sources.every(source => + source.type === 'upload' + || (source.type === 'inline' && Object.keys(source.reference).every(key => key === 'image_url'))); + const compatibleParameters = Object.values(request.parameters).every(value => + typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean'); + if (!compatibleSources || !compatibleParameters) return null; + + const images: File[] = []; + for (const [index, source] of request.images.entries()) { + const file = uploadedFile(source, index); + if (file === null) return null; + images.push(file); + } + const mask = request.mask === undefined ? undefined : uploadedFile(request.mask, images.length); + if (mask === null) return null; + + const form = new FormData(); + for (const [name, value] of Object.entries(request.parameters)) form.append(name, String(value)); + const imageField = images.length === 1 ? 'image' : 'image[]'; + for (const image of images) form.append(imageField, image); + if (mask !== undefined) form.append('mask', mask); + form.append('model', model); + return form; +}; + +export const serializeOpenAIImagesEditsRequest = async (request: ImagesEditsRequest, model: string): Promise => { + const multipart = multipartBody(request, model); + if (multipart !== null) return multipart; + return JSON.stringify({ ...await jsonBody(request), model }); +}; diff --git a/packages/provider/src/images_test.ts b/packages/provider/src/images_test.ts new file mode 100644 index 000000000..4c5a2af5e --- /dev/null +++ b/packages/provider/src/images_test.ts @@ -0,0 +1,82 @@ +import { test } from 'vitest'; + +import { serializeOpenAIImagesEditsRequest } from './images.ts'; +import { assertEquals } from '@floway-dev/test-utils'; + +test('serializeOpenAIImagesEditsRequest preserves reference fields and encodes mixed uploads from their bytes', async () => { + const serialized = await serializeOpenAIImagesEditsRequest({ + images: [ + { type: 'reference', reference: { image_url: 'https://example.test/image.png', detail: 'future-field' } }, + { type: 'upload', file: new File(['inline'], 'inline.png', { type: 'image/png' }) }, + ], + mask: { type: 'reference', reference: { file_id: 'file-mask' } }, + parameters: { prompt: 'edit', background: null }, + }, 'gpt-image'); + assertEquals(typeof serialized, 'string'); + const body = JSON.parse(serialized as string) as Record; + assertEquals(body, { + prompt: 'edit', + background: null, + images: [ + { image_url: 'https://example.test/image.png', detail: 'future-field' }, + { image_url: 'data:image/png;base64,aW5saW5l' }, + ], + mask: { file_id: 'file-mask' }, + model: 'gpt-image', + }); +}); + +test('serializeOpenAIImagesEditsRequest uses the singular field for one upload and the array field for many', async () => { + const first = new File(['first'], 'first.png', { type: 'image/png' }); + const second = new File(['second'], 'second.png', { type: 'image/png' }); + const single = await serializeOpenAIImagesEditsRequest({ + images: [{ type: 'upload', file: first }], + parameters: { prompt: 'single' }, + }, 'gpt-image'); + assertEquals(single instanceof FormData, true); + const singleForm = single as FormData; + assertEquals(singleForm.get('image'), first); + assertEquals(singleForm.getAll('image[]'), []); + + const multiple = await serializeOpenAIImagesEditsRequest({ + images: [{ type: 'upload', file: first }, { type: 'upload', file: second }], + parameters: { prompt: 'multiple' }, + }, 'gpt-image'); + assertEquals(multiple instanceof FormData, true); + const multipleForm = multiple as FormData; + assertEquals(multipleForm.getAll('image[]'), [first, second]); + assertEquals(multipleForm.get('image'), null); + assertEquals(multipleForm.get('model'), 'gpt-image'); +}); + +test('serializeOpenAIImagesEditsRequest leaves malformed inline data for upstream JSON validation', async () => { + const serialized = await serializeOpenAIImagesEditsRequest({ + images: [{ + type: 'inline', + reference: { image_url: 'data:image/png;base64,%%%' }, + }], + parameters: { prompt: 'edit' }, + }, 'gpt-image'); + assertEquals(typeof serialized, 'string'); + assertEquals(JSON.parse(serialized as string), { + prompt: 'edit', + images: [{ image_url: 'data:image/png;base64,%%%' }], + model: 'gpt-image', + }); +}); + +test('serializeOpenAIImagesEditsRequest preserves extra inline reference fields through JSON', async () => { + const serialized = await serializeOpenAIImagesEditsRequest({ + images: [{ + type: 'inline', + reference: { image_url: 'data:image/png;base64,aW1hZ2U=', future_field: 'keep' }, + }], + parameters: { prompt: 'edit' }, + }, 'gpt-image'); + assertEquals(typeof serialized, 'string'); + assertEquals(JSON.parse(serialized as string), { + prompt: 'edit', + images: [{ image_url: 'data:image/png;base64,aW1hZ2U=', future_field: 'keep' }], + model: 'gpt-image', + }); +}); diff --git a/packages/provider/src/index.ts b/packages/provider/src/index.ts index 6d83c0ec5..9a0b4d540 100644 --- a/packages/provider/src/index.ts +++ b/packages/provider/src/index.ts @@ -56,6 +56,8 @@ export type { ResponsesAction, UpstreamCallOptions, } from './provider.ts'; +export type { ImagesEditsRequest, ImagesEditsSource } from './images.ts'; +export { serializeOpenAIImagesEditsRequest } from './images.ts'; export type { ProviderStreamParser } from './streaming.ts'; export { streamingProviderCall } from './streaming.ts'; diff --git a/packages/provider/src/provider.ts b/packages/provider/src/provider.ts index 9f44fc629..c1761524d 100644 --- a/packages/provider/src/provider.ts +++ b/packages/provider/src/provider.ts @@ -1,4 +1,5 @@ import type { FlagDefaults } from './flags.ts'; +import type { ImagesEditsRequest } from './images.ts'; import type { ModelPrefixConfig } from './model-prefix.ts'; import type { ProviderModel, UpstreamProviderKind, UpstreamRecord } from './model.ts'; import type { Fetcher } from './options.ts'; @@ -129,10 +130,7 @@ export interface ProviderInstance { callMessagesCountTokens(model: ProviderModel, body: Omit, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise; callEmbeddings(model: ProviderModel, body: Omit, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise; callImagesGenerations(model: ProviderModel, body: Omit, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise; - // The provider takes ownership of `body` and may mutate it (e.g. append - // the upstream-specific model/deployment id). Callers must allocate a - // fresh FormData per call. - callImagesEdits(model: ProviderModel, body: FormData, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise; + callImagesEdits(model: ProviderModel, request: ImagesEditsRequest, signal: AbortSignal | undefined, opts: UpstreamCallOptions): Promise; } // Static, module-shaped surface each provider package exports. The gateway