From 2c29eb6678f1567e8e45b6aae571228fbfca6d3e Mon Sep 17 00:00:00 2001 From: bealqiu Date: Mon, 20 Jul 2026 11:19:26 +0800 Subject: [PATCH 1/2] fix: stabilize mobile import auto vectorization --- .../src/lib/rag/auto-vectorize-book.ts | 30 +++-- .../src/lib/rag/auto-vectorize-service.ts | 22 +++- .../screens/library/useVectorizationQueue.ts | 16 ++- packages/app-expo/src/stores/library-store.ts | 114 +++++------------- 4 files changed, 73 insertions(+), 109 deletions(-) diff --git a/packages/app-expo/src/lib/rag/auto-vectorize-book.ts b/packages/app-expo/src/lib/rag/auto-vectorize-book.ts index f798a8abc..5521f5d84 100644 --- a/packages/app-expo/src/lib/rag/auto-vectorize-book.ts +++ b/packages/app-expo/src/lib/rag/auto-vectorize-book.ts @@ -3,6 +3,8 @@ import type { Book } from "@readany/core/types"; import * as FileSystem from "expo-file-system/legacy"; import { queueBook as queueAutoVectorize } from "./auto-vectorize-service"; +export const MOBILE_AUTO_VECTORIZE_MAX_BYTES = 32 * 1024 * 1024; + const MIME_TYPES: Record = { epub: "application/epub+zip", pdf: "application/pdf", @@ -16,18 +18,6 @@ export function getMobileVectorizeMimeType(format: string | undefined): string | return MIME_TYPES[normalized] ?? null; } -function bytesToBase64(bytes: Uint8Array): string { - const chunkSize = 0x8000; - let binary = ""; - - for (let i = 0; i < bytes.length; i += chunkSize) { - const chunk = bytes.subarray(i, i + chunkSize); - binary += String.fromCharCode(...chunk); - } - - return btoa(binary); -} - export async function resolveMobileBookPath(filePath: string): Promise { if ( filePath.startsWith("/") || @@ -54,7 +44,7 @@ export async function inspectMobileBookForVectorize(book: Book): Promise<{ mimeType: string | null; size: number | null; canVectorize: boolean; - reason?: "unsupported-format" | "missing-file"; + reason?: "unsupported-format" | "missing-file" | "file-too-large"; }> { const absPath = await resolveMobileBookPath(book.filePath); const mimeType = getMobileVectorizeMimeType(book.format); @@ -66,12 +56,14 @@ export async function inspectMobileBookForVectorize(book: Book): Promise<{ if (size == null) { return { absPath, mimeType, size, canVectorize: false, reason: "missing-file" }; } + if (size > MOBILE_AUTO_VECTORIZE_MAX_BYTES) { + return { absPath, mimeType, size, canVectorize: false, reason: "file-too-large" }; + } return { absPath, mimeType, size, canVectorize: true }; } export async function queueBookForAutoVectorize(book: Book): Promise { - const platform = getPlatformService(); const info = await inspectMobileBookForVectorize(book); if (!info.canVectorize || !info.mimeType) { console.warn( @@ -80,7 +72,13 @@ export async function queueBookForAutoVectorize(book: Book): Promise { return false; } - const bytes = await platform.readFile(info.absPath); - queueAutoVectorize(book, bytesToBase64(bytes), info.mimeType); + queueAutoVectorize( + book, + () => + FileSystem.readAsStringAsync(info.absPath, { + encoding: FileSystem.EncodingType.Base64, + }), + info.mimeType, + ); return true; } diff --git a/packages/app-expo/src/lib/rag/auto-vectorize-service.ts b/packages/app-expo/src/lib/rag/auto-vectorize-service.ts index 4d27ec229..a2005e2ea 100644 --- a/packages/app-expo/src/lib/rag/auto-vectorize-service.ts +++ b/packages/app-expo/src/lib/rag/auto-vectorize-service.ts @@ -1,21 +1,26 @@ import type { ChapterData } from "@readany/core/rag"; import type { Book } from "@readany/core/types"; -export type AutoVectorizeCallback = (bookId: string, progress: { status: string; progress: number }) => void; +export type AutoVectorizeCallback = ( + bookId: string, + progress: { status: string; progress: number }, +) => void; interface ExtractorRef { extractChapters: (base64BookData: string, mimeType?: string) => Promise; } +export type AutoVectorizeDataLoader = () => Promise; + interface QueueItem { book: Book; - base64Data: string; + loadBase64Data: AutoVectorizeDataLoader; mimeType: string; } let extractorRef: ExtractorRef | null = null; let callback: AutoVectorizeCallback | null = null; -let queue: QueueItem[] = []; +const queue: QueueItem[] = []; let processing = false; export function setExtractorRef(ref: ExtractorRef | null) { @@ -34,8 +39,12 @@ export function getQueueLength() { return queue.length; } -export async function queueBook(book: Book, base64Data: string, mimeType: string) { - queue.push({ book, base64Data, mimeType }); +export async function queueBook( + book: Book, + loadBase64Data: AutoVectorizeDataLoader, + mimeType: string, +) { + queue.push({ book, loadBase64Data, mimeType }); if (!processing) { processQueue(); } @@ -51,7 +60,7 @@ async function processQueue() { const item = queue.shift(); if (!item) break; - const { book, base64Data, mimeType } = item; + const { book, loadBase64Data, mimeType } = item; try { callback?.(book.id, { status: "extracting", progress: 0 }); @@ -61,6 +70,7 @@ async function processQueue() { continue; } + const base64Data = await loadBase64Data(); const chapters = await extractorRef.extractChapters(base64Data, mimeType); if (!chapters || chapters.length === 0) { console.warn(`[AutoVectorize] No chapters for ${book.meta.title}`); diff --git a/packages/app-expo/src/screens/library/useVectorizationQueue.ts b/packages/app-expo/src/screens/library/useVectorizationQueue.ts index fa280e992..e6c9a535b 100644 --- a/packages/app-expo/src/screens/library/useVectorizationQueue.ts +++ b/packages/app-expo/src/screens/library/useVectorizationQueue.ts @@ -1,5 +1,8 @@ import type { ExtractorRef } from "@/components/rag/ExtractorWebView"; -import { inspectMobileBookForVectorize } from "@/lib/rag/auto-vectorize-book"; +import { + MOBILE_AUTO_VECTORIZE_MAX_BYTES, + inspectMobileBookForVectorize, +} from "@/lib/rag/auto-vectorize-book"; import { triggerVectorizeBook } from "@/lib/rag/vectorize-trigger"; import type { RootStackParamList } from "@/navigation/RootNavigator"; import { useVectorModelStore } from "@/stores/vector-model-store"; @@ -127,6 +130,17 @@ export function useVectorizationQueue({ extractorRef, nav }: UseVectorizationQue ); return; } + if (info.reason === "file-too-large") { + const maxMb = Math.round(MOBILE_AUTO_VECTORIZE_MAX_BYTES / 1024 / 1024); + Alert.alert( + t("vectorize.tooLargeTitle", "文件过大"), + t("vectorize.tooLargeDesc", { + maxMb, + defaultValue: `移动端暂不处理超过 ${maxMb}MB 的书籍向量化，请在桌面端处理或换用更小的文件。`, + }), + ); + return; + } const alreadyQueued = vectorQueueRef.current.some((b) => b.id === book.id); if (alreadyQueued || vectorizingBookId === book.id) return; diff --git a/packages/app-expo/src/stores/library-store.ts b/packages/app-expo/src/stores/library-store.ts index d9014c14c..fc0dadf63 100644 --- a/packages/app-expo/src/stores/library-store.ts +++ b/packages/app-expo/src/stores/library-store.ts @@ -3,7 +3,7 @@ import { extractBookMetadata, extractBookMetadataFromFile, } from "@/lib/book/metadata-extractor"; -import { queueBook as queueAutoVectorize } from "@/lib/rag/auto-vectorize-service"; +import { queueBookForAutoVectorize } from "@/lib/rag/auto-vectorize-book"; import { type ImportBooksResult, createEmptyImportBooksResult, @@ -143,18 +143,6 @@ async function saveCoverToAppData(bookId: string, coverBlob: Blob): Promise { @@ -204,6 +192,30 @@ function shouldAutoVectorizeMobile(format: Book["format"]): boolean { return format === "epub" || format === "txt" || format === "umd"; } +async function maybeQueueImportedBookForAutoVectorize(book: Book, fileName: string): Promise { + try { + const vmState = useVectorModelStore.getState(); + if ( + !vmState.autoVectorizeOnImport || + !vmState.vectorModelEnabled || + !vmState.hasVectorCapability() + ) { + return; + } + + if (!shouldAutoVectorizeMobile(book.format)) { + console.warn( + `[importBooks] Skip auto-vectorize for unsupported mobile import: ${fileName} (format=${book.format})`, + ); + return; + } + + await queueBookForAutoVectorize(book); + } catch (autoVectorizeErr) { + console.warn(`[importBooks] Auto-vectorize enqueue failed for ${fileName}:`, autoVectorizeErr); + } +} + /** * Ensure raw bytes are UTF-8 encoded. Hermes (React Native) only supports * UTF-8 in TextDecoder — GBK/GB18030/Shift-JIS etc. are NOT supported. @@ -991,25 +1003,7 @@ export const useLibraryStore = create((set, get) => ({ } console.log(`[importBooks] TXT imported as EPUB: ${title}`); - // Auto-vectorize if enabled. Keep failures isolated so a - // successful import doesn't get reported as a failed import. - try { - const vmState = useVectorModelStore.getState(); - if ( - vmState.autoVectorizeOnImport && - vmState.vectorModelEnabled && - vmState.hasVectorCapability() && - shouldAutoVectorizeMobile("txt") - ) { - const base64 = bytesToBase64(conversion.epubBytes); - queueAutoVectorize(book, base64, "application/epub+zip"); - } - } catch (autoVectorizeErr) { - console.warn( - `[importBooks] Auto-vectorize enqueue failed for ${fileName}:`, - autoVectorizeErr, - ); - } + await maybeQueueImportedBookForAutoVectorize(book, fileName); continue; } catch (convErr) { console.error("[importBooks] TXT conversion failed:", convErr); @@ -1114,23 +1108,7 @@ export const useLibraryStore = create((set, get) => ({ } console.log(`[importBooks] UMD imported as EPUB: ${title}`); - try { - const vmState = useVectorModelStore.getState(); - if ( - vmState.autoVectorizeOnImport && - vmState.vectorModelEnabled && - vmState.hasVectorCapability() && - shouldAutoVectorizeMobile("umd") - ) { - const base64 = bytesToBase64(conversion.epubBytes); - queueAutoVectorize(book, base64, "application/epub+zip"); - } - } catch (autoVectorizeErr) { - console.warn( - `[importBooks] Auto-vectorize enqueue failed for ${fileName}:`, - autoVectorizeErr, - ); - } + await maybeQueueImportedBookForAutoVectorize(book, fileName); continue; } catch (convErr) { console.error("[importBooks] UMD conversion failed:", convErr); @@ -1231,43 +1209,7 @@ export const useLibraryStore = create((set, get) => ({ duplicateIndex.byHash.set(fileHash, book); } - // Auto-vectorize if enabled. Keep failures isolated so a - // successful import doesn't get reported as a failed import. - try { - const vmState = useVectorModelStore.getState(); - if ( - vmState.autoVectorizeOnImport && - vmState.vectorModelEnabled && - vmState.hasVectorCapability() && - shouldAutoVectorizeMobile(format) - ) { - const sourceBytes = await platform.readFile(filePath); - const base64 = bytesToBase64(sourceBytes); - const mimeTypes: Record = { - epub: "application/epub+zip", - pdf: "application/pdf", - mobi: "application/x-mobipocket-ebook", - azw: "application/vnd.amazon.ebook", - azw3: "application/vnd.amazon.ebook", - cbz: "application/vnd.comicbook+zip", - cbr: "application/vnd.comicbook+zip", - fb2: "application/x-fictionbook+xml", - fbz: "application/x-zip-compressed-fb2", - txt: "text/plain", - }; - const mimeType = mimeTypes[format] || "application/epub+zip"; - queueAutoVectorize(book, base64, mimeType); - } else if (vmState.autoVectorizeOnImport && vmState.vectorModelEnabled) { - console.warn( - `[importBooks] Skip auto-vectorize for unsupported mobile import: ${fileName} (${fileSize} bytes, format=${format})`, - ); - } - } catch (autoVectorizeErr) { - console.warn( - `[importBooks] Auto-vectorize enqueue failed for ${fileName}:`, - autoVectorizeErr, - ); - } + await maybeQueueImportedBookForAutoVectorize(book, fileName); } catch (err) { console.error(`Failed to import ${fileInfo.uri}:`, err); result.failures.push({ From 20a8914358ec885234092cc8e051701f7934c708 Mon Sep 17 00:00:00 2001 From: bealqiu Date: Mon, 20 Jul 2026 11:39:41 +0800 Subject: [PATCH 2/2] fix: allow manual vectorization of large books --- .../src/lib/rag/auto-vectorize-book.ts | 11 ++++-- .../screens/library/useVectorizationQueue.ts | 36 +++++++++++++++---- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/packages/app-expo/src/lib/rag/auto-vectorize-book.ts b/packages/app-expo/src/lib/rag/auto-vectorize-book.ts index 5521f5d84..2470e35bd 100644 --- a/packages/app-expo/src/lib/rag/auto-vectorize-book.ts +++ b/packages/app-expo/src/lib/rag/auto-vectorize-book.ts @@ -39,7 +39,10 @@ export async function getMobileBookFileSize(filePath: string): Promise MOBILE_AUTO_VECTORIZE_MAX_BYTES) { + if (options?.maxBytes != null && size > options.maxBytes) { return { absPath, mimeType, size, canVectorize: false, reason: "file-too-large" }; } @@ -64,7 +67,9 @@ export async function inspectMobileBookForVectorize(book: Book): Promise<{ } export async function queueBookForAutoVectorize(book: Book): Promise { - const info = await inspectMobileBookForVectorize(book); + const info = await inspectMobileBookForVectorize(book, { + maxBytes: MOBILE_AUTO_VECTORIZE_MAX_BYTES, + }); if (!info.canVectorize || !info.mimeType) { console.warn( `[AutoVectorize] Skip mobile book: ${book.meta.title} (${info.reason}, size=${info.size ?? "unknown"}, format=${book.format})`, diff --git a/packages/app-expo/src/screens/library/useVectorizationQueue.ts b/packages/app-expo/src/screens/library/useVectorizationQueue.ts index e6c9a535b..10b1bb3f3 100644 --- a/packages/app-expo/src/screens/library/useVectorizationQueue.ts +++ b/packages/app-expo/src/screens/library/useVectorizationQueue.ts @@ -109,7 +109,9 @@ export function useVectorizationQueue({ extractorRef, nav }: UseVectorizationQue const handleVectorize = useCallback( (book: Book) => { const prepareAndQueue = async () => { - const info = await inspectMobileBookForVectorize(book); + const info = await inspectMobileBookForVectorize(book, { + maxBytes: MOBILE_AUTO_VECTORIZE_MAX_BYTES, + }); if (info.reason === "unsupported-format") { Alert.alert( t("vectorize.unsupportedFormatTitle", "Unsupported format"), @@ -132,12 +134,34 @@ export function useVectorizationQueue({ extractorRef, nav }: UseVectorizationQue } if (info.reason === "file-too-large") { const maxMb = Math.round(MOBILE_AUTO_VECTORIZE_MAX_BYTES / 1024 / 1024); + const shouldContinue = await new Promise((resolve) => { + Alert.alert( + t("vectorize.largeBookTitle", "文件较大"), + t("vectorize.largeBookDesc", { + maxMb, + defaultValue: `这本书超过 ${maxMb}MB，移动端向量化可能较慢并占用较多内存。仍要继续吗？`, + }), + [ + { + text: t("common.cancel", "取消"), + style: "cancel", + onPress: () => resolve(false), + }, + { + text: t("common.continue", "继续"), + onPress: () => resolve(true), + }, + ], + { cancelable: true, onDismiss: () => resolve(false) }, + ); + }); + if (!shouldContinue) { + return; + } + } else if (!info.canVectorize) { Alert.alert( - t("vectorize.tooLargeTitle", "文件过大"), - t("vectorize.tooLargeDesc", { - maxMb, - defaultValue: `移动端暂不处理超过 ${maxMb}MB 的书籍向量化，请在桌面端处理或换用更小的文件。`, - }), + t("common.error", "Error"), + t("vectorize.prepareFailed", "Failed to prepare vectorization."), ); return; }