diff --git a/ONNX_MIGRATION_README.md b/ONNX_MIGRATION_README.md new file mode 100644 index 0000000..63af780 --- /dev/null +++ b/ONNX_MIGRATION_README.md @@ -0,0 +1,208 @@ +# BaseweightSnap ONNX Migration Status + +## Project Overview +Migration of BaseweightSnap Android app from llama.cpp to ONNX Runtime for running SmolVLM2-500M-Video-Instruct multimodal vision-language model inference. + +## Current Status: Vision Encoder Working, Decoder Generation Loop Needs Past Key Values + +The vision encoder is now working correctly, but the decoder is failing because it expects `past_key_values.0.key` inputs that we're not providing in the generation loop. + +### Latest Error +``` +Error: Non-zero status code returned while running Shape node. Name:'/model/attn_mask_reformat/past_key_subgraph/Shape' Status Message: Missing Input: past_key_values.0.key +``` + +## Working Components + +### ✅ ONNX Runtime Infrastructure +- **Status**: WORKING +- ONNX Runtime 2.0.0-rc.10 with XNNPack execution provider +- Android NDK cross-compilation for arm64-v8a architecture +- JNI integration between Rust and Kotlin +- Model loading (3 second load time, all models load successfully) + +### ✅ Vision Encoder +- **Status**: WORKING +- Successfully processes images and outputs vision features +- Uses working image processor from `/home/bowserj/rust_onnx_explore/smolvlm/src/image_processor.rs` +- Provides both `pixel_values` and `pixel_attention_mask` inputs +- Handles 5D tensor shapes `[1, 17, 3, 512, 512]` (17 patches = 4x4 grid + original) +- Converts attention mask to boolean: `pixel_attention_mask_bool = attention_mask.map(|&x| x != 0)` + +### ✅ Text Embedder +- **Status**: WORKING +- Successfully converts text tokens to embeddings +- Properly handles multimodal embedding fusion (vision + text) + +### ❌ Decoder Generation Loop +- **Status**: BROKEN - Missing past_key_values +- The decoder expects `past_key_values.{layer}.key` and `past_key_values.{layer}.value` inputs +- We need to implement the full generation loop from the working reference implementation + +## Architecture + +### Models +- `vision_encoder_q4.onnx` - Processes images into vision features +- `embed_tokens_q4.onnx` - Converts text tokens to embeddings +- `decoder_model_merged_q4.onnx` - Generates text tokens autoregressively + +### File Structure +``` +/home/bowserj/baseweight_download/BaseweightSnap/ +├── smolvlm_rs_lib/ # Rust library +│ ├── src/ +│ │ ├── lib_simple.rs # Main SmolVLM implementation (NEEDS FIXING) +│ │ └── image_processor.rs # Working image processor (copied from reference) +│ └── Cargo.toml +├── app/src/main/ +│ ├── java/ai/baseweight/baseweightsnap/ +│ │ ├── MainActivity.kt # Android UI +│ │ ├── SmolVLMAndroid.kt # JNI wrapper +│ │ └── ModelManager.kt # Model management +│ └── jniLibs/arm64-v8a/ +│ └── libsmolvlm_snap.so # Rust library +└── app/build.gradle.kts # Android build config +``` + +### Reference Implementation +Working desktop implementation at `/home/bowserj/rust_onnx_explore/smolvlm/src/main.rs` that we should copy from. + +## Key Technical Details + +### Vision Encoder Inputs +```rust +inputs.insert("pixel_values", Value::from_array(processed_images.clone())?.into()); +let pixel_attention_mask_bool = attention_mask.map(|&x| x != 0); +inputs.insert("pixel_attention_mask", Value::from_array(pixel_attention_mask_bool)?.into()); +``` + +### Image Processing +- Uses 512x512 patch size (hardcoded throughout image processor) +- Creates 4x4 grid split + original image = 17 total patches +- Returns `(Array5, Array4)` for pixel_values and attention_mask + +### Threading +- `processImage()` and `generateResponse()` both run on the same `runLoop` thread +- Both are suspend functions in Kotlin + +## Next Steps to Fix + +### 1. Fix Decoder Generation Loop +The working reference implementation (`/home/bowserj/rust_onnx_explore/smolvlm/src/main.rs`) shows how to: + +1. **Auto-detect model configuration** from decoder inputs: +```rust +let decoder_inputs = &decoder_session.inputs; +let mut max_layer = 0; +for input in decoder_inputs { + if input.name.starts_with("past_key_values.") { + if let Some(layer_str) = input.name.split('.').nth(1) { + if let Ok(layer_num) = layer_str.parse::() { + max_layer = max_layer.max(layer_num); + } + } + } +} +let num_hidden_layers = max_layer + 1; +``` + +2. **Initialize past_key_values** before generation loop: +```rust +let mut past_key_values: HashMap> = HashMap::new(); +for layer in 0..num_hidden_layers { + let key_array = Array::zeros((batch_size, num_key_value_heads, 0, head_dim)).into_dyn(); + let value_array = Array::zeros((batch_size, num_key_value_heads, 0, head_dim)).into_dyn(); + past_key_values.insert(format!("past_key_values.{}.key", layer), key_array); + past_key_values.insert(format!("past_key_values.{}.value", layer), value_array); +} +``` + +3. **Add past_key_values to each decoder call**: +```rust +for (key, value) in &past_key_values { + decoder_inputs.insert(key, Value::from_array(value.clone())?.into()); +} +``` + +4. **Update past_key_values after each decoder call**: +```rust +for i in 0..num_hidden_layers { + let key = format!("past_key_values.{}.key", i); + let value = format!("past_key_values.{}.value", i); + + if let Some(past_key) = past_key_values.get_mut(&key) { + if i * 2 + 1 < decoder_outputs.len() { + let present_key = decoder_outputs[i * 2 + 1].try_extract_tensor::()?.to_owned(); + *past_key = present_key.into_dyn(); + } + } + // Similar for values... +} +``` + +### 2. Build and Deploy Process +```bash +# 1. Build Rust library +cd /home/bowserj/baseweight_download/BaseweightSnap/smolvlm_rs_lib +ORT_LIB_LOCATION=/home/bowserj/ort/onnxruntime/build/Android/Debug cargo ndk -t arm64-v8a build --release + +# 2. Copy to Android +cp target/aarch64-linux-android/release/libsmolvlm_snap.so ../app/src/main/jniLibs/arm64-v8a/ + +# 3. Build APK +cd /home/bowserj/baseweight_download/BaseweightSnap +./gradlew clean assembleDebug + +# 4. Install +adb install app/build/outputs/apk/debug/app-debug.apk +``` + +### 3. Testing +```bash +# Launch app +adb shell monkey -p ai.baseweight.baseweightsnap -c android.intent.category.LAUNCHER 1 + +# Monitor logs +adb logcat | grep "SmolVLM" +``` + +## Model Configuration (Detected) +- SmolVLM2-500M-Video-Instruct +- Approximately 18+ layers (based on past_key_values inputs) +- 3 key-value heads +- 64 head dimension +- Vocabulary size: ~32000+ tokens + +## Known Working Values +- Batch size: 1 +- Image tensor shape: `[1, 17, 3, 512, 512]` +- Attention mask shape: `[1, 17, 512, 512]` +- Vision encoder outputs: image_features tensor +- EOS token: 2 + +## Dependencies +- ONNX Runtime 2.0.0-rc.10 +- Android NDK +- Rust with `cargo-ndk` +- Android SDK API 28+ + +## Important: Copy Working Implementation +Don't reinvent the wheel - copy the working generation loop from `/home/bowserj/rust_onnx_explore/smolvlm/src/main.rs` lines 285-410 which handles past_key_values correctly. + +## Debugging History + +### Problems Solved +1. **Threading Issue**: Image processing happened on main thread while text generation happened on `runLoop` thread, causing image embeddings to be unavailable. Fixed by making `processImage()` a suspend function. + +2. **Vision Encoder Input Rank Mismatch**: "Invalid rank for input: pixel_values Got: 4 Expected: 5". Fixed by copying working image processor with proper 5D tensor shapes. + +3. **Vision Encoder Dimension Mismatch**: "Got: 384 Expected: 512" for dimensions 3 and 4. Fixed by hardcoding 512x512 patch sizes throughout the image processor. + +4. **Missing Attention Mask Input**: "Missing Input: pixel_attention_mask". Fixed by using working implementation with boolean attention masks. + +5. **Wrong Output Key Name**: Fixed by taking first output instead of looking for specific key name. + +6. **Decoder Input Name Error**: "Invalid input name: hidden_states". Fixed by using correct input names: "inputs_embeds", "attention_mask", and "position_ids". + +### Current Problem +7. **Missing Past Key Values**: "Missing Input: past_key_values.0.key" - The decoder generation loop needs to be completely rewritten to match the working reference implementation. \ No newline at end of file diff --git a/app/build.gradle.kts b/app/build.gradle.kts index e212bff..65e568a 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -11,20 +11,14 @@ android { applicationId = "ai.baseweight.baseweightsnap" minSdk = 28 targetSdk = 35 - versionCode = 6 - versionName = "1.3.0" + versionCode = 7 + versionName = "1.4.0" testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" - externalNativeBuild { - cmake { - arguments("-DANDROID_STL=c++_shared") - } - // Don't build 32 bit libraries in 2025 - ndk { - abiFilters.add("arm64-v8a") - abiFilters.add("x86_64") - } + // Removed CMake configuration - using pure Rust dylib like AnimeGanRust + ndk { + abiFilters.add("arm64-v8a") } } @@ -45,12 +39,7 @@ android { kotlinOptions { jvmTarget = "11" } - externalNativeBuild { - cmake { - path = file("src/main/cpp/CMakeLists.txt") - version = "3.22.1" - } - } + // Removed CMake - using pure Rust dylib approach buildFeatures { viewBinding = true } diff --git a/app/src/main/java/ai/baseweight/baseweightsnap/MainActivity.kt b/app/src/main/java/ai/baseweight/baseweightsnap/MainActivity.kt index 57c9921..5ddf134 100644 --- a/app/src/main/java/ai/baseweight/baseweightsnap/MainActivity.kt +++ b/app/src/main/java/ai/baseweight/baseweightsnap/MainActivity.kt @@ -52,7 +52,7 @@ class MainActivity : AppCompatActivity() { private var currentCameraSelector = CameraSelector.DEFAULT_BACK_CAMERA private val scope = CoroutineScope(Dispatchers.Main) private var generationJob: Job? = null - private val vlmRunner: MTMD_Android = MTMD_Android.instance() + private val vlmRunner: SmolVLMAndroid = SmolVLMAndroid.instance() override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) @@ -353,10 +353,11 @@ class MainActivity : AppCompatActivity() { binding.btnDismissResponse.visibility = View.GONE try { - vlmRunner.processImage(bitmap) + val imageProcessed = vlmRunner.processImage(bitmap) + Log.d("MainActivity", "Image processed: $imageProcessed") - Log.d("MainActivity", "Generating response for prompt: $prompt") - generationJob = scope.launch { + if (imageProcessed) { + Log.d("MainActivity", "Generating response for prompt: $prompt") vlmRunner.generateResponse(prompt, 2048).collect { text -> Log.d("MainActivity", "Received text: $text") withContext(Dispatchers.Main) { @@ -384,6 +385,9 @@ class MainActivity : AppCompatActivity() { } } } + } else { + progressDialog.dismiss() + showResponseText("Failed to process image. Please try again.") } } catch (e: Exception) { Log.e("MainActivity", "Error generating response", e) @@ -475,7 +479,7 @@ class MainActivity : AppCompatActivity() { // Used to load the 'baseweightsnap' library on application startup. init { - System.loadLibrary("baseweightsnap") + System.loadLibrary("smolvlm_snap") } } } \ No newline at end of file diff --git a/app/src/main/java/ai/baseweight/baseweightsnap/ModelManager.kt b/app/src/main/java/ai/baseweight/baseweightsnap/ModelManager.kt index 7a3b016..e37d35a 100644 --- a/app/src/main/java/ai/baseweight/baseweightsnap/ModelManager.kt +++ b/app/src/main/java/ai/baseweight/baseweightsnap/ModelManager.kt @@ -17,17 +17,25 @@ data class Model( val name: String, val size: Long, // in bytes val isDefault: Boolean = false, - val isLanguage: Boolean = false + val modelType: ModelType = ModelType.VISION_ENCODER ) -// Container class for MTMD Model pairs -data class MTMDModel( +enum class ModelType { + VISION_ENCODER, EMBED_TOKENS, DECODER_MODEL, TOKENIZER +} + +// Container class for SmolVLM Model set +data class SmolVLMModelSet( val name: String, - val language: Model, - val vision: Model + val visionEncoder: Model, + val embedTokens: Model, + val decoderModel: Model, + val tokenizer: Model ) { - val languageId: String get() = language.id - val visionId: String get() = vision.id + val visionEncoderId: String get() = visionEncoder.id + val embedTokensId: String get() = embedTokens.id + val decoderModelId: String get() = decoderModel.id + val tokenizerId: String get() = tokenizer.id } data class DownloadProgress( @@ -49,28 +57,44 @@ class ModelManager(private val context: Context) { private const val MODELS_DIR = "models" const val DEFAULT_MODEL_NAME = "SmolVLM2-256M-VidInstruct" - // HuggingFace URLs for the models - private const val LANGUAGE_MODEL_URL = "https://huggingface.co/ggml-org/SmolVLM2-256M-Video-Instruct-GGUF/resolve/main/SmolVLM2-256M-Video-Instruct-Q8_0.gguf" - private const val VISION_MODEL_URL = "https://huggingface.co/ggml-org/SmolVLM2-256M-Video-Instruct-GGUF/resolve/main/mmproj-SmolVLM2-256M-Video-Instruct-Q8_0.gguf" + // HuggingFace URLs for the ONNX models - using uint8 for better XNNPack support + private const val VISION_ENCODER_URL = "https://huggingface.co/HuggingFaceTB/SmolVLM2-500M-Video-Instruct/resolve/main/onnx/vision_encoder_uint8.onnx" + private const val EMBED_TOKENS_URL = "https://huggingface.co/HuggingFaceTB/SmolVLM2-500M-Video-Instruct/resolve/main/onnx/embed_tokens_uint8.onnx" + private const val DECODER_MODEL_URL = "https://huggingface.co/HuggingFaceTB/SmolVLM2-500M-Video-Instruct/resolve/main/onnx/decoder_model_merged_uint8.onnx" + private const val TOKENIZER_URL = "https://huggingface.co/HuggingFaceTB/SmolVLM2-500M-Video-Instruct/raw/main/tokenizer.json" } - // List of available models - simplified to just one model pair + // List of available model sets val availableModels = listOf( - MTMDModel( + SmolVLMModelSet( name = DEFAULT_MODEL_NAME, - language = Model( - id = "smolvlm2-256m-language", - name = "SmolVLM", - size = 175_000_000L, // Approximate size in bytes + visionEncoder = Model( + id = "vision_encoder_uint8", + name = "Vision Encoder UInt8", + size = 133_400_000L, // ~133 MB (uint8 is ~2x larger than q4) + isDefault = true, + modelType = ModelType.VISION_ENCODER + ), + embedTokens = Model( + id = "embed_tokens_uint8", + name = "Embed Tokens UInt8", + size = 378_000_000L, // ~378 MB (uint8 is ~2x larger than q4) + isDefault = true, + modelType = ModelType.EMBED_TOKENS + ), + decoderModel = Model( + id = "decoder_model_merged_uint8", + name = "Decoder Model UInt8", + size = 458_000_000L, // ~458 MB (uint8 is ~2x larger than q4) isDefault = true, - isLanguage = true + modelType = ModelType.DECODER_MODEL ), - vision = Model( - id = "smolvlm2-256m-vision", - name = "SmolVLM", - size = 104_000_000L, // Approximate size in bytes + tokenizer = Model( + id = "tokenizer", + name = "Tokenizer", + size = 2_400_000L, // ~2.4 MB isDefault = true, - isLanguage = false + modelType = ModelType.TOKENIZER ) ) ) @@ -84,31 +108,42 @@ class ModelManager(private val context: Context) { return File(context.getExternalFilesDir(null), MODELS_DIR) } - // Get MTMD model pair by name - fun getMTMDModel(name: String): MTMDModel? { + // Get SmolVLM model set by name + fun getModelSet(name: String): SmolVLMModelSet? { return availableModels.find { it.name == name } } // Get model by ID fun getModel(modelId: String): Model? { return availableModels - .flatMap { listOf(it.language, it.vision) } + .flatMap { listOf(it.visionEncoder, it.embedTokens, it.decoderModel, it.tokenizer) } .find { it.id == modelId } } - // Check if a model pair is downloaded - fun isModelPairDownloaded(modelName: String): Boolean { - val model = getMTMDModel(modelName) ?: return false - val languageFile = File(getModelPath(model.languageId)) - val visionFile = File(getModelPath(model.visionId)) - return languageFile.exists() && languageFile.length() > 0 && - visionFile.exists() && visionFile.length() > 0 + // Check if a complete model set is downloaded + fun isModelSetDownloaded(modelName: String): Boolean { + val modelSet = getModelSet(modelName) ?: return false + val visionFile = File(getModelPath(modelSet.visionEncoderId)) + val embedFile = File(getModelPath(modelSet.embedTokensId)) + val decoderFile = File(getModelPath(modelSet.decoderModelId)) + val tokenizerFile = File(getTokenizerPath(modelSet.tokenizerId)) + + return visionFile.exists() && visionFile.length() > 0 && + embedFile.exists() && embedFile.length() > 0 && + decoderFile.exists() && decoderFile.length() > 0 && + tokenizerFile.exists() && tokenizerFile.length() > 0 } // Get the local path for a model fun getModelPath(modelId: String): String { val model = getModel(modelId) ?: throw IllegalArgumentException("Invalid model ID: $modelId") - return "${getModelsDirectory().absolutePath}/${modelId}.gguf" + val extension = if (model.modelType == ModelType.TOKENIZER) ".json" else ".onnx" + return "${getModelsDirectory().absolutePath}/${modelId}${extension}" + } + + // Get the local path for a tokenizer specifically + fun getTokenizerPath(modelId: String): String { + return "${getModelsDirectory().absolutePath}/${modelId}.json" } // Get the local path for a model object @@ -116,42 +151,57 @@ class ModelManager(private val context: Context) { return getModelPath(model.id) } - // Download both language and vision models for a model pair - fun downloadModelPair(modelName: String): Flow = flow { - val mtmdModel = getMTMDModel(modelName) ?: throw IllegalArgumentException("Invalid model name: $modelName") - - // Download language model first - downloadModel(mtmdModel.languageId, LANGUAGE_MODEL_URL).collect { progress -> - // Progress is already emitted by downloadModel + // Download complete model set (4 files) + fun downloadModelSet(modelName: String): Flow = flow { + val modelSet = getModelSet(modelName) ?: throw IllegalArgumentException("Invalid model name: $modelName") + + // Download all models sequentially + downloadModel(modelSet.visionEncoderId, VISION_ENCODER_URL).collect { progress -> + emit(progress.copy(message = "Downloading vision encoder...")) } - - // Download vision model - downloadModel(mtmdModel.visionId, VISION_MODEL_URL).collect { progress -> - // Progress is already emitted by downloadModel + + downloadModel(modelSet.embedTokensId, EMBED_TOKENS_URL).collect { progress -> + emit(progress.copy(message = "Downloading embed tokens...")) } - - // Verify both files are present before marking as COMPLETED - val languageFile = File(getModelPath(mtmdModel.languageId)) - val visionFile = File(getModelPath(mtmdModel.visionId)) - - val languagePresent = languageFile.exists() + + downloadModel(modelSet.decoderModelId, DECODER_MODEL_URL).collect { progress -> + emit(progress.copy(message = "Downloading decoder model...")) + } + + downloadModel(modelSet.tokenizerId, TOKENIZER_URL).collect { progress -> + emit(progress.copy(message = "Downloading tokenizer...")) + } + + // Verify all files are present before marking as COMPLETED + val visionFile = File(getModelPath(modelSet.visionEncoderId)) + val embedFile = File(getModelPath(modelSet.embedTokensId)) + val decoderFile = File(getModelPath(modelSet.decoderModelId)) + val tokenizerFile = File(getTokenizerPath(modelSet.tokenizerId)) + val visionPresent = visionFile.exists() - - val totalSize = mtmdModel.language.size + mtmdModel.vision.size - if (languagePresent && visionPresent) { + val embedPresent = embedFile.exists() + val decoderPresent = decoderFile.exists() + val tokenizerPresent = tokenizerFile.exists() + + val totalSize = modelSet.visionEncoder.size + modelSet.embedTokens.size + + modelSet.decoderModel.size + modelSet.tokenizer.size + + if (visionPresent && embedPresent && decoderPresent && tokenizerPresent) { emit(DownloadProgress( progress = 100, bytesDownloaded = totalSize, totalBytes = totalSize, status = DownloadStatus.COMPLETED, - message = "Both models downloaded successfully" + message = "All models downloaded successfully" )) - Log.d(TAG, "Both language and vision models downloaded successfully for: $modelName") + Log.d(TAG, "All models downloaded successfully for: $modelName") } else { - // If either file is missing, clean up and mark as error - if (!languagePresent) languageFile.delete() + // Clean up any incomplete downloads if (!visionPresent) visionFile.delete() - + if (!embedPresent) embedFile.delete() + if (!decoderPresent) decoderFile.delete() + if (!tokenizerPresent) tokenizerFile.delete() + emit(DownloadProgress( progress = 100, bytesDownloaded = totalSize, @@ -159,7 +209,7 @@ class ModelManager(private val context: Context) { status = DownloadStatus.ERROR, message = "Failed to verify downloaded files" )) - Log.e(TAG, "Failed to verify downloaded files for: $modelName. Language: $languagePresent, Vision: $visionPresent") + Log.e(TAG, "Failed to verify downloaded files for: $modelName") } }.flowOn(Dispatchers.IO) @@ -173,17 +223,22 @@ class ModelManager(private val context: Context) { try { emit(DownloadProgress(0, 0, model.size, DownloadStatus.PENDING, "Connecting to download server...")) - + + Log.d(TAG, "Downloading from URL: $downloadUrl") val downloadConnection = URL(downloadUrl).openConnection() as HttpURLConnection + downloadConnection.instanceFollowRedirects = true + downloadConnection.setRequestProperty("User-Agent", "BaseweightSnap/1.0") downloadConnection.connect() - + val statusCode = downloadConnection.responseCode + Log.d(TAG, "HTTP Status: $statusCode, Final URL: ${downloadConnection.url}") + if (statusCode != HttpURLConnection.HTTP_OK) { val errorStream = downloadConnection.errorStream val errorMessage = errorStream?.bufferedReader()?.use { it.readText() } ?: "No error message available" val errorDetails = "HTTP ${statusCode} - ${downloadConnection.responseMessage}\n${errorMessage}" Log.e(TAG, "Download Error: $errorDetails") - + emit(DownloadProgress( progress = 0, bytesDownloaded = 0, @@ -191,48 +246,62 @@ class ModelManager(private val context: Context) { status = DownloadStatus.ERROR, message = "Download failed: $errorDetails" )) - + throw Exception("Download failed with HTTP ${statusCode}: ${downloadConnection.responseMessage}") } - + val totalSize = downloadConnection.contentLength.toLong() - if (totalSize <= 0) { - Log.e(TAG, "Invalid content length received: $totalSize") - throw Exception("Invalid content length received from server") + Log.d(TAG, "Content-Length header: ${downloadConnection.contentLength}") + Log.d(TAG, "Content-Length as long: $totalSize") + + // For tokenizer files and other small files, content-length might not be available + val effectiveSize = if (totalSize <= 0) { + Log.w(TAG, "Content-Length not available, using model size estimate: ${model.size}") + model.size + } else { + totalSize } - Log.d(TAG, "Download size: $totalSize bytes") - + Log.d(TAG, "Download size: $effectiveSize bytes") + val inputStream = downloadConnection.inputStream val outputStream = FileOutputStream(modelFile) val buffer = ByteArray(8192) var bytesRead: Int var downloadedSize: Long = 0 - - emit(DownloadProgress(0, 0, totalSize, DownloadStatus.DOWNLOADING)) - + + emit(DownloadProgress(0, 0, effectiveSize, DownloadStatus.DOWNLOADING)) + while (true) { bytesRead = inputStream.read(buffer) if (bytesRead == -1) break - + outputStream.write(buffer, 0, bytesRead) downloadedSize += bytesRead - - val progress = (downloadedSize * 100 / totalSize).toInt() - emit(DownloadProgress(progress, downloadedSize, totalSize, DownloadStatus.DOWNLOADING)) + + // Calculate progress, but cap at 100% and handle case where actual size exceeds estimate + val progress = if (effectiveSize > 0) { + minOf(100, (downloadedSize * 100 / effectiveSize).toInt()) + } else { + 50 // Default progress for unknown size + } + emit(DownloadProgress(progress, downloadedSize, effectiveSize, DownloadStatus.DOWNLOADING)) } outputStream.flush() outputStream.close() inputStream.close() - + val finalFileSize = modelFile.length() - if (finalFileSize != totalSize) { + // Only verify file size if we had a reliable content-length + if (totalSize > 0 && finalFileSize != totalSize) { Log.e(TAG, "Downloaded file size mismatch: Expected $totalSize, got $finalFileSize") modelFile.delete() throw Exception("Download failed: File size mismatch") + } else { + Log.d(TAG, "Downloaded file size: $finalFileSize bytes") } - - emit(DownloadProgress(100, totalSize, totalSize, DownloadStatus.COMPLETED)) + + emit(DownloadProgress(100, finalFileSize, maxOf(finalFileSize, effectiveSize), DownloadStatus.COMPLETED)) } catch (e: Exception) { Log.e(TAG, "Error downloading model: ${e.message}", e) diff --git a/app/src/main/java/ai/baseweight/baseweightsnap/SmolVLMAndroid.kt b/app/src/main/java/ai/baseweight/baseweightsnap/SmolVLMAndroid.kt new file mode 100644 index 0000000..78674ed --- /dev/null +++ b/app/src/main/java/ai/baseweight/baseweightsnap/SmolVLMAndroid.kt @@ -0,0 +1,116 @@ +package ai.baseweight.baseweightsnap + +import android.graphics.Bitmap +import android.util.Log +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.asCoroutineDispatcher +import kotlinx.coroutines.cancel +import kotlinx.coroutines.channels.awaitClose +import kotlinx.coroutines.channels.onFailure +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.callbackFlow +import kotlinx.coroutines.withContext +import java.nio.ByteBuffer +import java.util.concurrent.Executors +import kotlin.concurrent.thread + +class SmolVLMAndroid { + private val tag: String? = this::class.simpleName + + private val runLoop: CoroutineDispatcher = Executors.newSingleThreadExecutor { + thread(start = false, name = "SmolVLM-RunLoop") { + Log.d(tag, "Dedicated thread for ONNX Runtime: ${Thread.currentThread().name}") + + // Load the native library directly (pure Rust, no CMake wrapper) + System.loadLibrary("smolvlm_snap") + + Log.d(tag, "SmolVLM ONNX Runtime initialized") + + it.run() + }.apply { + uncaughtExceptionHandler = Thread.UncaughtExceptionHandler { _, exception: Throwable -> + Log.e(tag, "Unhandled exception", exception) + } + } + }.asCoroutineDispatcher() + + // Native method declarations + private external fun nativeLoadModels(modelDir: String, tokenizerPath: String): Boolean + private external fun processImageFromBuffer(buffer: ByteBuffer, width: Int, height: Int): Boolean + private external fun nativeGenerateResponse(prompt: String, maxTokens: Int): String + + suspend fun loadModels(modelDir: String, tokenizerPath: String): Boolean { + return withContext(runLoop) { + nativeLoadModels(modelDir, tokenizerPath) + } + } + + suspend fun processImage(bitmap: Bitmap): Boolean { + return withContext(runLoop) { + // Ensure bitmap is in ARGB_8888 format + val config = if (bitmap.config == Bitmap.Config.ARGB_8888) bitmap else bitmap.copy(Bitmap.Config.ARGB_8888, false) + + val byteBuffer = ByteBuffer.allocateDirect(config.byteCount) + config.copyPixelsToBuffer(byteBuffer) + byteBuffer.rewind() + processImageFromBuffer(byteBuffer, config.width, config.height) + } + } + + fun stopGeneration() { + // For now, this is a no-op since we don't have streaming generation implemented + // In a full implementation, this would cancel ongoing generation + Log.d(tag, "stopGeneration called") + } + + fun generateResponse(prompt: String, maxTokens: Int): Flow = callbackFlow { + withContext(runLoop) { + val callback = object : TextGenerationCallback { + override fun onTextGenerated(text: String) { + trySend(text).onFailure { exception: Throwable? -> + Log.e(tag, "Failed to send text to flow", exception) + close(exception) + } + } + + override fun onGenerationComplete() { + close() + } + + override fun onGenerationError(error: String) { + cancel("Generation error: $error", null) + } + + override fun onProgressUpdate(phase: String, progress: Int) { + trySend("PROGRESS:$phase:$progress").onFailure { exception: Throwable? -> + Log.e(tag, "Failed to send progress update", exception) + } + } + } + + try { + val result = nativeGenerateResponse(prompt, maxTokens) + // For now, just send the complete result + // In a more sophisticated implementation, we'd use streaming + trySend(result).onFailure { exception: Throwable? -> + Log.e(tag, "Failed to send result", exception) + close(exception) + } + close() + } catch (e: Exception) { + Log.e(tag, "Exception in generateResponse", e) + cancel("Error: ${e.message}", null) + } + } + + awaitClose { + // Cleanup if needed + } + } + + companion object { + // Enforce only one instance of SmolVLMAndroid + private val _instance: SmolVLMAndroid = SmolVLMAndroid() + fun instance(): SmolVLMAndroid = _instance + } +} \ No newline at end of file diff --git a/app/src/main/java/ai/baseweight/baseweightsnap/SplashActivity.kt b/app/src/main/java/ai/baseweight/baseweightsnap/SplashActivity.kt index 235569b..fb66ab4 100644 --- a/app/src/main/java/ai/baseweight/baseweightsnap/SplashActivity.kt +++ b/app/src/main/java/ai/baseweight/baseweightsnap/SplashActivity.kt @@ -23,7 +23,7 @@ import java.io.File class SplashActivity : AppCompatActivity() { private lateinit var binding: ActivitySplashBinding private val scope = CoroutineScope(Dispatchers.Main) - private val vlmRunner: MTMD_Android = MTMD_Android.instance() + private val vlmRunner: SmolVLMAndroid = SmolVLMAndroid.instance() private val modelManager: ModelManager by lazy { ModelManager(this) } private val DEFAULT_ERROR_MESSAGE = "An unexpected error occurred. Please try reinstalling the app." @@ -50,18 +50,24 @@ class SplashActivity : AppCompatActivity() { try { // Check if models are already downloaded val defaultModelName = ModelManager.DEFAULT_MODEL_NAME - val modelsDownloaded = modelManager.isModelPairDownloaded(defaultModelName) + Log.d(TAG, "Checking if models are downloaded for: $defaultModelName") + val modelsDownloaded = modelManager.isModelSetDownloaded(defaultModelName) + Log.d(TAG, "Models downloaded: $modelsDownloaded") if (modelsDownloaded) { + Log.d(TAG, "Models are downloaded, attempting to load...") binding.splashStatus.text = "Loading models..." if (loadModels()) { + Log.d(TAG, "Models loaded successfully, starting main activity") delay(500) startMainActivity() } else { + Log.e(TAG, "Failed to load models into memory") binding.splashStatus.text = "Failed to load models. Please try again." binding.splashProgress.visibility = View.GONE } } else { + Log.d(TAG, "Models not downloaded, checking WiFi...") // Check if on WiFi val isOnWifi = isOnWiFi() if (isOnWifi) { @@ -80,25 +86,35 @@ class SplashActivity : AppCompatActivity() { private suspend fun downloadAndLoadModels() { binding.splashStatus.text = "Downloading models..." try { - modelManager.downloadModelPair(ModelManager.DEFAULT_MODEL_NAME) + modelManager.downloadModelSet(ModelManager.DEFAULT_MODEL_NAME) .collect { progress -> - binding.splashStatus.text = "Downloading: ${progress.progress}%" - if (progress.status == DownloadStatus.COMPLETED) { - // Download is complete, load the models - val model = modelManager.getMTMDModel(ModelManager.DEFAULT_MODEL_NAME)!! - val languagePath = modelManager.getModelPath(model.languageId) - val visionPath = modelManager.getModelPath(model.visionId) - - if (File(languagePath).exists() && File(visionPath).exists()) { + // Update progress display + binding.splashStatus.text = progress.message + + // Only load models when ALL downloads are complete (this is the final COMPLETED status) + if (progress.status == DownloadStatus.COMPLETED && progress.message.contains("All models downloaded")) { + Log.d(TAG, "All models downloaded successfully, loading into memory...") + binding.splashStatus.text = "Loading models..." + + val modelSet = modelManager.getModelSet(ModelManager.DEFAULT_MODEL_NAME)!! + val modelDir = File(modelManager.getModelPath(modelSet.visionEncoderId)).parent!! + val tokenizerPath = modelManager.getTokenizerPath(modelSet.tokenizerId) + + if (File(modelDir).exists() && File(tokenizerPath).exists()) { if (loadModels()) { delay(500) startMainActivity() } else { + Log.e(TAG, "Failed to load models from: $modelDir and $tokenizerPath") showErrorAndExit("Failed to load models after download. Please try reinstalling the app.") } } else { + Log.e(TAG, "Model files not found: $modelDir or $tokenizerPath") showErrorAndExit("Download failed: Model files not found. Please try reinstalling the app.") } + } else if (progress.status == DownloadStatus.ERROR) { + Log.e(TAG, "Download error: ${progress.message}") + showErrorAndExit("Download failed: ${progress.message}") } } } catch (e: Exception) { @@ -153,21 +169,30 @@ class SplashActivity : AppCompatActivity() { private suspend fun loadModels(): Boolean { return withContext(Dispatchers.IO) { try { - val model = modelManager.getMTMDModel(ModelManager.DEFAULT_MODEL_NAME)!! - val modelPath = modelManager.getModelPath(model.languageId) - val visionPath = modelManager.getModelPath(model.visionId) - + val modelSet = modelManager.getModelSet(ModelManager.DEFAULT_MODEL_NAME)!! + val modelDir = File(modelManager.getModelPath(modelSet.visionEncoderId)).parent!! + val tokenizerPath = modelManager.getTokenizerPath(modelSet.tokenizerId) + // Verify files exist before loading - if (!File(modelPath).exists() || !File(visionPath).exists()) { - Log.e(TAG, "Model files not found: $modelPath or $visionPath") + if (!File(modelDir).exists() || !File(tokenizerPath).exists()) { + Log.e(TAG, "Model files not found: $modelDir or $tokenizerPath") return@withContext false } - + + Log.d(TAG, "Model paths - modelDir: $modelDir, tokenizerPath: $tokenizerPath") + + // Verify files exist before loading + val modelDirExists = File(modelDir).exists() + val tokenizerExists = File(tokenizerPath).exists() + Log.d(TAG, "File existence - modelDir: $modelDirExists, tokenizer: $tokenizerExists") + // Load the models - val success = vlmRunner.loadModels(modelPath, visionPath) - + Log.d(TAG, "Calling vlmRunner.loadModels()...") + val success = vlmRunner.loadModels(modelDir, tokenizerPath) + Log.d(TAG, "vlmRunner.loadModels() returned: $success") + if (!success) { - Log.e(TAG, "Failed to load models from: $modelPath and $visionPath") + Log.e(TAG, "Failed to load models from: $modelDir and $tokenizerPath") } success diff --git a/build_static.sh b/build_static.sh new file mode 100755 index 0000000..10d040d --- /dev/null +++ b/build_static.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Build script for static linking with ONNX Runtime +# This script sets the proper environment variables for static linking + +# Path to ONNX Runtime static libraries (MinSizeRel build) +export ORT_LIB_LOCATION="/home/bowserj/ort/onnxruntime/build/Android/MinSizeRel" +export ORT_LIB_PROFILE="MinSizeRel" + +# Set the strategy to use static linking (instead of dynamic linking) +export ORT_STRATEGY="static" + +# Android NDK environment +export ANDROID_NDK="/home/bowserj/Android/Sdk/ndk/28.0.12916984" + +echo "Building with static ONNX Runtime linking..." +echo "ORT_LIB_LOCATION: $ORT_LIB_LOCATION" +echo "ORT_LIB_PROFILE: $ORT_LIB_PROFILE" +echo "ORT_STRATEGY: $ORT_STRATEGY" + +cd smolvlm_rs_lib + +# Build for Android ARM64 +cargo ndk -t arm64-v8a build --release + +echo "Build completed!" \ No newline at end of file diff --git a/setup_dynamic_env.sh b/setup_dynamic_env.sh new file mode 100755 index 0000000..3af93bd --- /dev/null +++ b/setup_dynamic_env.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Environment setup for dynamic linking with ONNX Runtime 2.0.0-rc.10 +# Source this file before building: source setup_dynamic_env.sh + +echo "Setting up environment for dynamic ONNX Runtime linking (ort 2.0.0-rc.10)..." + +# Path to ONNX Runtime shared libraries (MinSizeRel build) +export ORT_LIB_LOCATION="/home/bowserj/ort/onnxruntime/build/Android/MinSizeRel" +export ORT_LIB_PROFILE="MinSizeRel" + +# Set the strategy to use dynamic linking +export ORT_STRATEGY="load-dynamic" + +# Android NDK environment +export ANDROID_NDK="/home/bowserj/Android/Sdk/ndk/28.0.12916984" + +echo "✓ ORT_LIB_LOCATION: $ORT_LIB_LOCATION" +echo "✓ ORT_LIB_PROFILE: $ORT_LIB_PROFILE" +echo "✓ ORT_STRATEGY: $ORT_STRATEGY" +echo "✓ Environment ready for dynamic linking build" +echo "" +echo "Now you can build with:" +echo " cd smolvlm_rs_lib && cargo ndk -t arm64-v8a build --release" \ No newline at end of file diff --git a/setup_static_env.sh b/setup_static_env.sh new file mode 100644 index 0000000..f07861c --- /dev/null +++ b/setup_static_env.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Environment setup for static linking with ONNX Runtime 2.0.0-rc.10 +# Source this file before building: source setup_static_env.sh + +echo "Setting up environment for static ONNX Runtime linking (ort 2.0.0-rc.10)..." + +# Path to ONNX Runtime static libraries (MinSizeRel build) +export ORT_LIB_LOCATION="/home/bowserj/ort/onnxruntime/build/Android/MinSizeRel" +export ORT_LIB_PROFILE="MinSizeRel" + +# Set the strategy to use static linking +export ORT_STRATEGY="static" + +# Android NDK environment +export ANDROID_NDK="/home/bowserj/Android/Sdk/ndk/28.0.12916984" + +echo "✓ ORT_LIB_LOCATION: $ORT_LIB_LOCATION" +echo "✓ ORT_LIB_PROFILE: $ORT_LIB_PROFILE" +echo "✓ ORT_STRATEGY: $ORT_STRATEGY" +echo "✓ Environment ready for static linking build" +echo "" +echo "Now you can build with:" +echo " cd smolvlm_rs_lib && cargo ndk -t arm64-v8a build --release" \ No newline at end of file diff --git a/smolvlm_rs_lib/Cargo.lock b/smolvlm_rs_lib/Cargo.lock new file mode 100644 index 0000000..5e0fa2c --- /dev/null +++ b/smolvlm_rs_lib/Cargo.lock @@ -0,0 +1,1926 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom", + "once_cell", + "serde", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "android_log-sys" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84521a3cf562bc62942e294181d9eef17eb38ceb8c68677bc49f144e4c3d4f8d" + +[[package]] +name = "android_logger" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c494134f746c14dc653a35a4ea5aca24ac368529da5370ecf41fe0341c35772f" +dependencies = [ + "android_log-sys", + "env_logger", + "log", + "once_cell", +] + +[[package]] +name = "anyhow" +version = "1.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba" + +[[package]] +name = "bit_field" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e4b40c7323adcfc0a41c4b88143ed58346ff65a288fc144329c5c45e05d70c6" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" + +[[package]] +name = "bytemuck" +version = "1.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3995eaeebcdf32f91f980d360f78732ddc061097ab4e39991ae7a6ace9194677" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + +[[package]] +name = "cc" +version = "1.2.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1354349954c6fc9cb0deab020f27f783cf0b604e8bb754dc4658ecf0d29c35f" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + +[[package]] +name = "cfg-if" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" + +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "compact_str" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "serde", + "static_assertions", +] + +[[package]] +name = "console" +version = "0.15.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "unicode-width", + "windows-sys 0.59.0", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "dary_heap" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06d2e3287df1c007e74221c49ca10a95d557349e54b3a75dc2fb14712c751f04" +dependencies = [ + "serde", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + +[[package]] +name = "env_logger" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.1", +] + +[[package]] +name = "esaxx-rs" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" +dependencies = [ + "cc", +] + +[[package]] +name = "exr" +version = "1.73.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83197f59927b46c04a183a619b7c29df34e63e63c7869320862268c0ef687e0" +dependencies = [ + "bit_field", + "half", + "lebe", + "miniz_oxide", + "rayon-core", + "smallvec 1.15.1", + "zune-inflate", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "filetime" +version = "0.2.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc0505cd1b6fa6580283f6bdf70a73fcf4aba1184038c90902b92b3dd0df63ed" +dependencies = [ + "cfg-if", + "libc", + "libredox", + "windows-sys 0.60.2", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ced73b1dacfc750a6db6c0a0c3a3853c8b41997e2e2c563dc90804ae6867959" + +[[package]] +name = "flate2" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi", +] + +[[package]] +name = "gif" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae047235e33e2829703574b54fdec96bfbad892062d97fed2f76022287de61b" +dependencies = [ + "color_quant", + "weezl", +] + +[[package]] +name = "half" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" +dependencies = [ + "cfg-if", + "crunchy", +] + +[[package]] +name = "http" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "image" +version = "0.24.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5690139d2f55868e080017335e4b94cb7414274c74f1669c84fb5feba2c9f69d" +dependencies = [ + "bytemuck", + "byteorder", + "color_quant", + "exr", + "gif", + "jpeg-decoder", + "num-traits", + "png", + "qoi", + "tiff", +] + +[[package]] +name = "indicatif" +version = "0.17.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" +dependencies = [ + "console", + "number_prefix", + "portable-atomic", + "unicode-width", + "web-time", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "jni" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "039022cdf4d7b1cf548d31f60ae783138e5fd42013f6271049d7df7afadef96c" +dependencies = [ + "cesu8", + "combine", + "jni-sys", + "log", + "thiserror 1.0.69", + "walkdir", +] + +[[package]] +name = "jni-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" + +[[package]] +name = "jpeg-decoder" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00810f1d8b74be64b13dbf3db89ac67740615d6c891f0e7b6179326533011a07" +dependencies = [ + "rayon", +] + +[[package]] +name = "js-sys" +version = "0.3.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec48937a97411dcb524a265206ccd4c90bb711fca92b2792c407f268825b9305" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lebe" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a79a3332a6609480d7d0c9eab957bca6b455b91bb84e66d19f5ff66294b85b8" + +[[package]] +name = "libc" +version = "0.2.176" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58f929b4d672ea937a23a1ab494143d968337a5f47e56d0815df1e0890ddf174" + +[[package]] +name = "libredox" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "416f7e718bdb06000964960ffa43b4335ad4012ae8b99060261aa4a8088d5ccb" +dependencies = [ + "bitflags 2.9.4", + "libc", + "redox_syscall", +] + +[[package]] +name = "linux-raw-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" + +[[package]] +name = "log" +version = "0.4.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" + +[[package]] +name = "macro_rules_attribute" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65049d7923698040cd0b1ddcced9b0eb14dd22c5f86ae59c3740eab64a676520" +dependencies = [ + "macro_rules_attribute-proc_macro", + "paste", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30" + +[[package]] +name = "matrixmultiply" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08" +dependencies = [ + "autocfg", + "rawpointer", +] + +[[package]] +name = "memchr" +version = "2.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "monostate" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a2901b7478a273256ce419c446289eb3c883a790d0bf5ed2f363c0cc3988012" +dependencies = [ + "monostate-impl", + "serde", + "serde_core", +] + +[[package]] +name = "monostate-impl" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "328f7435b7f54322b33832f1e981ff192781f0d12473f12d062705a55015de8d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "native-tls" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "ndarray" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "onig" +version = "6.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0" +dependencies = [ + "bitflags 2.9.4", + "libc", + "once_cell", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "openssl" +version = "0.10.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8505734d46c8ab1e19a1dce3aef597ad87dcb4c37e7188231769bd6bd51cebf8" +dependencies = [ + "bitflags 2.9.4", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-sys" +version = "0.9.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90096e2e47630d78b7d1c20952dc621f957103f8bc2c8359ec81290d75238571" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "ort" +version = "2.0.0-rc.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa7e49bd669d32d7bc2a15ec540a527e7764aec722a45467814005725bcd721" +dependencies = [ + "ndarray", + "ort-sys", + "smallvec 2.0.0-alpha.10", + "tracing", +] + +[[package]] +name = "ort-sys" +version = "2.0.0-rc.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2aba9f5c7c479925205799216e7e5d07cc1d4fa76ea8058c60a9a30f6a4e890" +dependencies = [ + "flate2", + "pkg-config", + "sha2", + "tar", + "ureq", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "png" +version = "0.17.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82151a2fc869e011c153adc57cf2789ccb8d9906ce52c0b39a6b5697749d7526" +dependencies = [ + "bitflags 1.3.2", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + +[[package]] +name = "portable-atomic" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" + +[[package]] +name = "portable-atomic-util" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "qoi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6d64c71eb498fe9eae14ce4ec935c555749aef511cca85b5568910d6e48001" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-cond" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f" +dependencies = [ + "either", + "itertools", + "rayon", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redox_syscall" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" +dependencies = [ + "bitflags 2.9.4", +] + +[[package]] +name = "regex" +version = "1.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b5288124840bee7b386bc413c487869b360b2b4ec421ea56425128692f2a82c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "833eb9ce86d40ef33cb1306d8accf7bc8ec2bfea4355cbdebb3df68b40925cad" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" + +[[package]] +name = "rustix" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" +dependencies = [ + "bitflags 2.9.4", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.1", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +dependencies = [ + "windows-sys 0.61.1", +] + +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags 2.9.4", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.145" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", + "serde_core", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "smallvec" +version = "2.0.0-alpha.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d44cfb396c3caf6fbfd0ab422af02631b69ddd96d2eff0b0f0724f9024051b" + +[[package]] +name = "smolvlm_rs_lib" +version = "0.1.0" +dependencies = [ + "android_logger", + "anyhow", + "image", + "jni", + "log", + "ndarray", + "ort", + "serde", + "serde_json", + "tokenizers", +] + +[[package]] +name = "socks" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b" +dependencies = [ + "byteorder", + "libc", + "winapi", +] + +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64 0.13.1", + "nom", + "serde", + "unicode-segmentation", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tar" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "tempfile" +version = "3.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys 0.61.1", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" +dependencies = [ + "thiserror-impl 2.0.16", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tiff" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba1310fcea54c6a9a4fd1aad794ecc02c31682f6bfbecdf460bf19533eed1e3e" +dependencies = [ + "flate2", + "jpeg-decoder", + "weezl", +] + +[[package]] +name = "tokenizers" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a620b996116a59e184c2fa2dfd8251ea34a36d0a514758c6f966386bd2e03476" +dependencies = [ + "ahash", + "aho-corasick", + "compact_str", + "dary_heap", + "derive_builder", + "esaxx-rs", + "getrandom", + "indicatif", + "itertools", + "log", + "macro_rules_attribute", + "monostate", + "onig", + "paste", + "rand", + "rayon", + "rayon-cond", + "regex", + "regex-syntax", + "serde", + "serde_json", + "spm_precompiled", + "thiserror 2.0.16", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + +[[package]] +name = "tracing" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +dependencies = [ + "once_cell", +] + +[[package]] +name = "typenum" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" + +[[package]] +name = "unicode-ident" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d" + +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec 1.15.1", +] + +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + +[[package]] +name = "unicode-width" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "ureq" +version = "3.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99ba1025f18a4a3fc3e9b48c868e9beb4f24f4b4b1a325bada26bd4119f46537" +dependencies = [ + "base64 0.22.1", + "der", + "log", + "native-tls", + "percent-encoding", + "rustls-pemfile", + "rustls-pki-types", + "socks", + "ureq-proto", + "utf-8", + "webpki-root-certs", +] + +[[package]] +name = "ureq-proto" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b4531c118335662134346048ddb0e54cc86bd7e81866757873055f0e38f5d2" +dependencies = [ + "base64 0.22.1", + "http", + "httparse", + "log", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.14.7+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" +dependencies = [ + "wasip2", +] + +[[package]] +name = "wasip2" +version = "1.0.1+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1da10c01ae9f1ae40cbfac0bac3b1e724b320abfcf52229f80b547c0d250e2d" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "671c9a5a66f49d8a47345ab942e2cb93c7d1d0339065d4f8139c486121b43b19" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ca60477e4c59f5f2986c50191cd972e3a50d8a95603bc9434501cf156a9a119" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f07d2f20d4da7b26400c9f4a0511e6e0345b040694e8a75bd41d578fa4421d7" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad67dc8b2a1a6e5448428adec4c3e84c43e561d8c9ee8a9e5aabeb193ec41d1" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e4ffd8df1c57e87c325000a3d6ef93db75279dc3a231125aac571650f22b12a" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "weezl" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a751b3277700db47d3e574514de2eced5e54dc8a5436a3bf7a0b248b2cee16f3" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.1", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45e46c0661abb7180e7b9c281db115305d49ca1709ab8242adf09666d2173c65" + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.4", +] + +[[package]] +name = "windows-sys" +version = "0.61.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f109e41dd4a3c848907eb83d5a42ea98b3769495597450cf6d153507b166f0f" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d42b7b7f66d2a06854650af09cfdf8713e427a439c97ad65a6375318033ac4b" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + +[[package]] +name = "wit-bindgen" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" + +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + +[[package]] +name = "zerocopy" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + +[[package]] +name = "zune-inflate" +version = "0.2.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab332fe2f6680068f3582b16a24f90ad7096d5d39b974d1c0aff0125116f02" +dependencies = [ + "simd-adler32", +] diff --git a/smolvlm_rs_lib/Cargo.toml b/smolvlm_rs_lib/Cargo.toml new file mode 100644 index 0000000..6ce6e32 --- /dev/null +++ b/smolvlm_rs_lib/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "smolvlm_rs_lib" +version = "0.1.0" +edition = "2024" + +[dependencies] +ort = "=2.0.0-rc.10" +image = "0.24" +anyhow = "1.0" +ndarray = "0.16.1" +jni = "0.20.0" +tokenizers = { version = "0.21.1" } +serde_json = "1.0" +serde = { version = "1.0", features = ["derive"] } +android_logger = "0.13" +log = "0.4" + +[lib] +name = "smolvlm_snap" +path = "src/lib_simple.rs" +crate-type = ["dylib"] \ No newline at end of file diff --git a/smolvlm_rs_lib/src/image_processor.rs b/smolvlm_rs_lib/src/image_processor.rs new file mode 100644 index 0000000..d16cfb1 --- /dev/null +++ b/smolvlm_rs_lib/src/image_processor.rs @@ -0,0 +1,289 @@ +use anyhow::Result; +use image::{DynamicImage, GenericImageView}; +use ndarray::{Array4, Array5}; +use std::collections::HashMap; + +const MAX_IMAGE_SIZE: u32 = 4096; // 4k resolution as absolute maximum +const SMOLVLM_MEAN: [f32; 3] = [0.5, 0.5, 0.5]; +const SMOLVLM_STD: [f32; 3] = [0.5, 0.5, 0.5]; + +#[derive(Debug, Clone)] +pub struct SmolVLMImageProcessor { + do_convert_rgb: bool, + do_resize: bool, + size: HashMap, + do_image_splitting: bool, + max_image_size: HashMap, + do_rescale: bool, + rescale_factor: f32, + do_normalize: bool, + image_mean: [f32; 3], + image_std: [f32; 3], + do_pad: bool, +} + +impl Default for SmolVLMImageProcessor { + fn default() -> Self { + Self { + do_convert_rgb: true, + do_resize: true, + size: HashMap::from([("longest_edge".to_string(), 2048)]), + do_image_splitting: true, + max_image_size: HashMap::from([("longest_edge".to_string(), 512)]), + do_rescale: true, + rescale_factor: 1.0 / 255.0, + do_normalize: true, + image_mean: SMOLVLM_MEAN, + image_std: SMOLVLM_STD, + do_pad: true, + } + } +} + +impl SmolVLMImageProcessor { + pub fn new() -> Self { + Self::default() + } + + fn resize_output_size_rescale_to_max_len( + height: u32, + width: u32, + min_len: Option, + max_len: Option, + ) -> (u32, u32) { + let max_len = max_len.unwrap_or_else(|| height.max(width)); + let aspect_ratio = width as f32 / height as f32; + + let (mut width, mut height) = if width >= height { + let width = max_len; + let height = (width as f32 / aspect_ratio).round() as u32; + (width, height) + } else { + let height = max_len; + let width = (height as f32 * aspect_ratio).round() as u32; + (width, height) + }; + + // Avoid resizing to a size smaller than min_len + let min_len = min_len.unwrap_or(1); + height = height.max(min_len); + width = width.max(min_len); + + (height, width) + } + + fn resize_output_size_scale_below_upper_bound( + height: u32, + width: u32, + max_len: Option, + ) -> (u32, u32) { + let max_len = max_len.unwrap_or_else(|| height.max(width)); + let aspect_ratio = width as f32 / height as f32; + + let (mut width, mut height) = if width >= height && width > max_len { + let width = max_len; + let height = (width as f32 / aspect_ratio).round() as u32; + (width, height) + } else if height > width && height > max_len { + let height = max_len; + let width = (height as f32 * aspect_ratio).round() as u32; + (width, height) + } else { + (width, height) + }; + + // Avoid resizing to a size smaller than 1 + height = height.max(1); + width = width.max(1); + + (height, width) + } + + fn get_resize_output_image_size( + image: &DynamicImage, + resolution_max_side: u32, + ) -> (u32, u32) { + let (height, width) = (image.height(), image.width()); + + // Only resize if the image is larger than resolution_max_side + if height <= resolution_max_side && width <= resolution_max_side { + return (height, width); + } + + // Find the output size, when rescaling the longest edge to max_len and preserving the aspect ratio + let (height, width) = Self::resize_output_size_rescale_to_max_len(height, width, None, Some(resolution_max_side)); + + // Find the output size when scaling the image to be below the MAX_IMAGE_SIZE + let (height, width) = Self::resize_output_size_scale_below_upper_bound(height, width, Some(MAX_IMAGE_SIZE)); + + (height, width) + } + + fn convert_to_rgb(&self, image: DynamicImage) -> DynamicImage { + image.to_rgb8().into() + } + + fn resize(&self, image: DynamicImage, size: HashMap) -> Result { + let (height, width) = if let Some(longest_edge) = size.get("longest_edge") { + Self::get_resize_output_image_size(&image, *longest_edge) + } else if let (Some(height), Some(width)) = (size.get("height"), size.get("width")) { + (*height, *width) + } else { + return Err(anyhow::anyhow!("size must be a dictionary with key 'longest_edge' or 'height' and 'width'")); + }; + + Ok(image.resize_exact(width, height, image::imageops::FilterType::Lanczos3)) + } + + fn split_image( + &self, + image: DynamicImage, + max_image_size: &HashMap, + ) -> Result<(Vec, u32, u32)> { + let (height, width) = (image.height(), image.width()); + let max_size = max_image_size.get("longest_edge").unwrap_or(&512); + + let mut frames = Vec::new(); + + // Always do a 4x4 split + let num_splits_h = 4; + let num_splits_w = 4; + + // Calculate optimal split sizes + let optimal_height = (height as f32 / num_splits_h as f32).ceil() as u32; + let optimal_width = (width as f32 / num_splits_w as f32).ceil() as u32; + + for r in 0..num_splits_h { + for c in 0..num_splits_w { + let start_x = c * optimal_width; + let start_y = r * optimal_height; + let end_x = (start_x + optimal_width).min(width); + let end_y = (start_y + optimal_height).min(height); + + let cropped = image.crop_imm(start_x, start_y, end_x - start_x, end_y - start_y); + // Resize each cropped frame to 512x512 (hardcoded for vision encoder requirement) + let resized = self.resize( + cropped, + HashMap::from([ + ("height".to_string(), 512u32), + ("width".to_string(), 512u32), + ]), + )?; + frames.push(resized); + } + } + + // Add the original image, resized to 512x512 (hardcoded for vision encoder requirement) + let resized = self.resize( + image, + HashMap::from([ + ("height".to_string(), 512u32), + ("width".to_string(), 512u32), + ]), + )?; + frames.push(resized); + + Ok((frames, num_splits_h, num_splits_w)) + } + + pub fn preprocess(&self, image: DynamicImage) -> Result<(Array5, Array4)> { + let mut image = image; + + // Convert to RGB if needed + if self.do_convert_rgb { + image = self.convert_to_rgb(image); + } + + // Resize if needed - first resize to size["longest_edge"] while preserving aspect ratio + if self.do_resize { + image = self.resize(image, self.size.clone())?; + } + + // Split image if needed + let (frames, num_splits_h, num_splits_w) = if self.do_image_splitting { + // First resize to be multiples of max_image_size while preserving aspect ratio + let max_size = self.max_image_size.get("longest_edge").unwrap_or(&512); + let (height, width) = (image.height(), image.width()); + let aspect_ratio = width as f32 / height as f32; + + let (new_width, new_height) = if width >= height { + let new_width = ((width as f32 / *max_size as f32).ceil() * *max_size as f32) as u32; + let new_height = (new_width as f32 / aspect_ratio).round() as u32; + let new_height = ((new_height as f32 / *max_size as f32).ceil() * *max_size as f32) as u32; + (new_width, new_height) + } else { + let new_height = ((height as f32 / *max_size as f32).ceil() * *max_size as f32) as u32; + let new_width = (new_height as f32 * aspect_ratio).round() as u32; + let new_width = ((new_width as f32 / *max_size as f32).ceil() * *max_size as f32) as u32; + (new_width, new_height) + }; + + let resized = self.resize( + image, + HashMap::from([ + ("height".to_string(), new_height), + ("width".to_string(), new_width), + ]), + )?; + + self.split_image(resized, &self.max_image_size)? + } else { + // If not splitting, just resize to 512x512 (hardcoded for vision encoder requirement) + let resized = self.resize( + image, + HashMap::from([ + ("height".to_string(), 512u32), + ("width".to_string(), 512u32), + ]), + )?; + (vec![resized], 0, 0) + }; + + // Convert frames to arrays and normalize + let mut processed_frames = Vec::new(); + for frame in frames { + let mut array = Array5::::zeros((1, 1, 3, frame.height() as usize, frame.width() as usize)); + + // Convert to RGB if needed - image crate returns RGBA, we need RGB + let rgb_frame = frame.to_rgb8(); + + for y in 0..rgb_frame.height() { + for x in 0..rgb_frame.width() { + let pixel = rgb_frame.get_pixel(x, y); + // pixel[0] = R, pixel[1] = G, pixel[2] = B (correct RGB order) + for c in 0..3 { + let val = pixel[c] as f32; + let val = if self.do_rescale { + val * self.rescale_factor + } else { + val + }; + let val = if self.do_normalize { + (val - self.image_mean[c]) / self.image_std[c] + } else { + val + }; + array[[0, 0, c, y as usize, x as usize]] = val; + } + } + } + processed_frames.push(array); + } + + // Stack frames into a batch with shape (batch, num_frames, channels, height, width) + let batch = Array5::from_shape_fn( + (1, processed_frames.len(), 3, processed_frames[0].shape()[3], processed_frames[0].shape()[4]), + |(_, i, c, y, x)| { + let frame = &processed_frames[i]; + frame[[0, 0, c, y, x]] + }, + ); + + // Create attention mask with shape (batch, num_frames, height, width) + let height = processed_frames[0].shape()[3]; + let width = processed_frames[0].shape()[4]; + let attention_mask = Array4::ones((1, processed_frames.len(), height, width)); + + Ok((batch, attention_mask)) + } +} \ No newline at end of file diff --git a/smolvlm_rs_lib/src/lib.rs b/smolvlm_rs_lib/src/lib.rs new file mode 100644 index 0000000..251160b --- /dev/null +++ b/smolvlm_rs_lib/src/lib.rs @@ -0,0 +1,292 @@ +#[cfg(target_os = "android")] +#[allow(non_snake_case)] +pub mod smolvlm_snap { + extern crate jni; + + use ort::session::{builder::GraphOptimizationLevel, Session}; + use ort::value::Tensor; + use image::{ImageBuffer, Rgb, RgbImage, DynamicImage, GenericImageView}; + use anyhow::{Result as AnyhowResult, Context}; + use ndarray::{Array, Array4}; + use jni::JNIEnv; + use jni::objects::{JClass, JObject, JString}; + use jni::sys::{jstring, jint, jboolean}; + use tokenizers::Tokenizer; + use std::sync::Mutex; + use std::collections::HashMap; + + static mut SMOLVLM_INSTANCE: Option> = None; + + pub struct SmolVLMProcessor { + vision_session: Option, + embed_session: Option, + decoder_session: Option, + tokenizer: Option, + model_paths: HashMap, + } + + impl SmolVLMProcessor { + fn new() -> AnyhowResult { + Ok(SmolVLMProcessor { + vision_session: None, + embed_session: None, + decoder_session: None, + tokenizer: None, + model_paths: HashMap::new(), + }) + } + + fn load_models(&mut self, model_dir: &str, tokenizer_path: &str) -> AnyhowResult { + // Load tokenizer + let tokenizer = Tokenizer::from_file(tokenizer_path) + .map_err(|e| anyhow::anyhow!("Failed to load tokenizer: {}", e))?; + self.tokenizer = Some(tokenizer); + + // Load vision encoder + let vision_path = format!("{}/vision_encoder_q4.onnx", model_dir); + let vision_session = Session::builder()? + .with_optimization_level(GraphOptimizationLevel::Level3)? + .commit_from_file(&vision_path) + .context("Failed to load vision encoder")?; + self.vision_session = Some(vision_session); + + // Load embed tokens + let embed_path = format!("{}/embed_tokens_q4.onnx", model_dir); + let embed_session = Session::builder()? + .with_optimization_level(GraphOptimizationLevel::Level3)? + .commit_from_file(&embed_path) + .context("Failed to load embed tokens")?; + self.embed_session = Some(embed_session); + + // Load decoder + let decoder_path = format!("{}/decoder_model_merged_q4.onnx", model_dir); + let decoder_session = Session::builder()? + .with_optimization_level(GraphOptimizationLevel::Level3)? + .commit_from_file(&decoder_path) + .context("Failed to load decoder")?; + self.decoder_session = Some(decoder_session); + + Ok(true) + } + + fn process_image_from_buffer(&self, buffer: &[u8], width: i32, height: i32) -> AnyhowResult> { + // Convert ARGB8888 buffer to RGB image + let mut rgb_data = Vec::with_capacity((height * width * 3) as usize); + for i in 0..(height * width) as usize { + let idx = i * 4; + // ARGB8888 format: [B, G, R, A] in memory + rgb_data.push(buffer[idx + 2]); // R + rgb_data.push(buffer[idx + 1]); // G + rgb_data.push(buffer[idx]); // B + } + + let img = RgbImage::from_raw( + width as u32, + height as u32, + rgb_data + ).ok_or_else(|| anyhow::anyhow!("Failed to create image from buffer"))?; + + self.preprocess_image(DynamicImage::ImageRgb8(img)) + } + + fn preprocess_image(&self, image: DynamicImage) -> AnyhowResult> { + // Resize to model input size (typically 384x384 for SmolVLM) + let resized = image.resize_exact(384, 384, image::imageops::FilterType::CatmullRom); + let rgb = resized.to_rgb8(); + + // Convert to CHW format and normalize + let mut data = Array4::::zeros((1, 3, 384, 384)); + + for c in 0..3 { + for y in 0..384 { + for x in 0..384 { + let pixel = rgb.get_pixel(x, y); + // Normalize to [0, 1] and then to model expected range + let normalized = (pixel[c] as f32 / 255.0 - 0.5) / 0.5; + data[[0, c, y as usize, x as usize]] = normalized; + } + } + } + + Ok(data) + } + + fn generate_response(&self, prompt: &str, max_tokens: i32, callback: Box) -> AnyhowResult { + let tokenizer = self.tokenizer.as_ref() + .ok_or_else(|| anyhow::anyhow!("Tokenizer not loaded"))?; + + // Tokenize prompt + let encoding = tokenizer.encode(prompt, false) + .map_err(|e| anyhow::anyhow!("Tokenization failed: {}", e))?; + let input_ids = encoding.get_ids(); + + // Convert to tensor format + let input_tensor = Array::from_shape_vec( + (1, input_ids.len()), + input_ids.iter().map(|&x| x as i64).collect() + )?; + + // Get embeddings + let embed_session = self.embed_session.as_ref() + .ok_or_else(|| anyhow::anyhow!("Embed session not loaded"))?; + + let embed_input = Tensor::from_array(input_tensor)?; + let embed_outputs = embed_session.run(ort::inputs!["input_ids" => embed_input])?; + let embeddings = embed_outputs[0].try_extract_tensor::()?; + + // Generation loop + let mut generated_tokens = Vec::new(); + let embeddings_array = Array::from_shape_vec(embeddings.0.dims()?, embeddings.1.to_vec())?; + let mut current_embeddings = embeddings_array; + + for _ in 0..max_tokens { + // Run decoder + let decoder_session = self.decoder_session.as_ref() + .ok_or_else(|| anyhow::anyhow!("Decoder session not loaded"))?; + + let decoder_input = Tensor::from_array(current_embeddings.view())?; + let decoder_outputs = decoder_session.run(ort::inputs!["inputs_embeds" => decoder_input])?; + let logits = decoder_outputs[0].try_extract_tensor::()?; + + // Get next token (argmax for simplicity) + let logits_array = Array::from_shape_vec(logits.0.dims()?, logits.1.to_vec())?; + let last_logits = logits_array.slice(ndarray::s![0, -1, ..]); + let next_token = last_logits.iter() + .enumerate() + .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap()) + .map(|(idx, _)| idx as u32) + .unwrap(); + + generated_tokens.push(next_token); + + // Check for EOS token (assuming 2 is EOS) + if next_token == 2 { + break; + } + + // Decode partial result and call callback + if let Ok(decoded) = tokenizer.decode(&generated_tokens, false) { + callback(&decoded); + } + + // Update embeddings for next iteration + let next_token_array = Array::from_shape_vec((1, 1), vec![next_token as i64])?; + let next_embed_input = Tensor::from_array(next_token_array)?; + let next_embed_outputs = embed_session.run(ort::inputs!["input_ids" => next_embed_input])?; + let next_embeddings = next_embed_outputs[0].try_extract_tensor::()?; + + // Concatenate embeddings + current_embeddings = ndarray::concatenate![ + ndarray::Axis(1), + current_embeddings.view(), + next_embeddings.view() + ]; + } + + // Final decode + tokenizer.decode(&generated_tokens, false) + .map_err(|e| anyhow::anyhow!("Final decode failed: {}", e)) + } + } + + fn get_or_create_instance() -> &'static Mutex { + unsafe { + if SMOLVLM_INSTANCE.is_none() { + SMOLVLM_INSTANCE = Some(Mutex::new( + SmolVLMProcessor::new().expect("Failed to create SmolVLM processor") + )); + } + SMOLVLM_INSTANCE.as_ref().unwrap() + } + } + + #[no_mangle] + pub extern "C" fn Java_ai_baseweight_baseweightsnap_SmolVLMAndroid_loadModels( + env: JNIEnv, + _class: JClass, + model_dir: JString, + tokenizer_path: JString, + ) -> jboolean { + let model_dir_str = match env.get_string(model_dir) { + Ok(s) => s.to_string_lossy().to_string(), + Err(_) => return false as jboolean, + }; + + let tokenizer_path_str = match env.get_string(tokenizer_path) { + Ok(s) => s.to_string_lossy().to_string(), + Err(_) => return false as jboolean, + }; + + let instance = get_or_create_instance(); + match instance.lock() { + Ok(mut processor) => { + match processor.load_models(&model_dir_str, &tokenizer_path_str) { + Ok(success) => success as jboolean, + Err(_) => false as jboolean, + } + } + Err(_) => false as jboolean, + } + } + + #[no_mangle] + pub extern "C" fn Java_ai_baseweight_baseweightsnap_SmolVLMAndroid_processImageFromBuffer( + env: JNIEnv, + _class: JClass, + buffer: JObject, + width: jint, + height: jint, + ) -> jboolean { + let buffer_ptr = match env.get_direct_buffer_address(buffer.into()) { + Ok(ptr) => ptr, + Err(_) => return false as jboolean, + }; + + let buffer_slice = unsafe { + std::slice::from_raw_parts( + buffer_ptr as *const u8, + (height * width * 4) as usize + ) + }; + + let instance = get_or_create_instance(); + match instance.lock() { + Ok(processor) => { + match processor.process_image_from_buffer(buffer_slice, width, height) { + Ok(_) => true as jboolean, + Err(_) => false as jboolean, + } + } + Err(_) => false as jboolean, + } + } + + #[no_mangle] + pub extern "C" fn Java_ai_baseweight_baseweightsnap_SmolVLMAndroid_generateResponse( + env: JNIEnv, + _class: JClass, + prompt: JString, + max_tokens: jint, + ) -> jstring { + let prompt_str = match env.get_string(prompt) { + Ok(s) => s.to_string_lossy().to_string(), + Err(_) => return env.new_string("Error: Failed to get prompt").unwrap().into_raw(), + }; + + let instance = get_or_create_instance(); + match instance.lock() { + Ok(processor) => { + let callback = Box::new(|_text: &str| { + // For now, we'll just accumulate and return the final result + // In a more sophisticated implementation, we'd use a callback mechanism + }); + + match processor.generate_response(&prompt_str, max_tokens, callback) { + Ok(result) => env.new_string(&result).unwrap().into_raw(), + Err(e) => env.new_string(&format!("Error: {}", e)).unwrap().into_raw(), + } + } + Err(_) => env.new_string("Error: Failed to lock processor").unwrap().into_raw(), + } + } +} \ No newline at end of file diff --git a/smolvlm_rs_lib/src/lib_simple.rs b/smolvlm_rs_lib/src/lib_simple.rs new file mode 100644 index 0000000..faf746e --- /dev/null +++ b/smolvlm_rs_lib/src/lib_simple.rs @@ -0,0 +1,633 @@ +mod image_processor; + +#[cfg(target_os = "android")] +#[allow(non_snake_case)] +pub mod smolvlm_snap { + extern crate jni; + + use log::{info, error}; + use android_logger::Config; + use std::sync::Once; + + static INIT: Once = Once::new(); + + fn init_logger() { + INIT.call_once(|| { + android_logger::init_once( + Config::default() + .with_max_level(log::LevelFilter::Info) + .with_tag("SmolVLM") + ); + }); + } + + use ort::session::{builder::GraphOptimizationLevel, Session}; + use ort::value::Value; + use ort::execution_providers::{XNNPACKExecutionProvider, ExecutionProvider}; + use image::{RgbImage, DynamicImage}; + use anyhow::{Result as AnyhowResult, Context}; + use ndarray::{Array2, Array3}; + use jni::JNIEnv; + use jni::objects::{JClass, JObject, JString}; + use jni::sys::{jstring, jint, jboolean}; + use std::sync::Mutex; + use std::collections::HashMap; + use tokenizers::Tokenizer; + use crate::image_processor::SmolVLMImageProcessor; + + use std::sync::OnceLock; + static SMOLVLM_INSTANCE: OnceLock> = OnceLock::new(); + + pub struct SmolVLMProcessor { + vision_session: Option, + embed_session: Option, + decoder_session: Option, + tokenizer: Option, + image_processor: SmolVLMImageProcessor, + image_embeddings: Option>, + initialized: bool, + } + + impl SmolVLMProcessor { + fn new() -> AnyhowResult { + Ok(SmolVLMProcessor { + vision_session: None, + embed_session: None, + decoder_session: None, + tokenizer: None, + image_processor: SmolVLMImageProcessor::new(), + image_embeddings: None, + initialized: false, + }) + } + + fn load_models(&mut self, model_dir: &str, tokenizer_path: &str) -> AnyhowResult { + init_logger(); + info!("SmolVLM: Starting model loading from {} with tokenizer {}", model_dir, tokenizer_path); + + // Load tokenizer + info!("SmolVLM: Loading tokenizer from {}", tokenizer_path); + let tokenizer = Tokenizer::from_file(tokenizer_path) + .map_err(|e| anyhow::anyhow!("Failed to load tokenizer: {}", e))?; + info!("SmolVLM: Tokenizer loaded successfully"); + + // Configure session builder with XNNPack + info!("SmolVLM: Configuring session builder with XNNPack"); + + // Check if XNNPack is available + match XNNPACKExecutionProvider::default().is_available() { + Ok(true) => info!("SmolVLM: XNNPack execution provider is AVAILABLE"), + Ok(false) => error!("SmolVLM: XNNPack execution provider is NOT AVAILABLE - will fall back to CPU!"), + Err(e) => error!("SmolVLM: Error checking XNNPack availability: {}", e), + } + + // Try to enable XNNPack with explicit configuration + let xnnpack = XNNPACKExecutionProvider::default() + .with_intra_op_num_threads(std::num::NonZeroUsize::new(4).unwrap()) + .build(); + + let session_builder = Session::builder()? + .with_optimization_level(GraphOptimizationLevel::Level3)? + .with_intra_threads(4)? + .with_execution_providers([xnnpack])?; + info!("SmolVLM: Session builder configured with XNNPack"); + + // Load vision encoder - use uint8 for better XNNPack support + let vision_path = format!("{}/vision_encoder_uint8.onnx", model_dir); + info!("SmolVLM: Loading vision encoder (uint8) from {}", vision_path); + let vision_session = session_builder + .clone() + .commit_from_file(&vision_path) + .context("Failed to load vision encoder")?; + + info!("SmolVLM: Vision encoder loaded successfully"); + + // Load embed tokens model - use uint8 for better XNNPack support + let embed_path = format!("{}/embed_tokens_uint8.onnx", model_dir); + info!("SmolVLM: Loading embed tokens (uint8) from {}", embed_path); + let embed_session = session_builder + .clone() + .commit_from_file(&embed_path) + .context("Failed to load embed tokens model")?; + info!("SmolVLM: Embed tokens loaded successfully"); + + // Load decoder model - use uint8 for better XNNPack support + let decoder_path = format!("{}/decoder_model_merged_uint8.onnx", model_dir); + info!("SmolVLM: Loading decoder (uint8) from {}", decoder_path); + let decoder_session = session_builder + .commit_from_file(&decoder_path) + .context("Failed to load decoder model")?; + info!("SmolVLM: Decoder loaded successfully"); + + self.vision_session = Some(vision_session); + self.embed_session = Some(embed_session); + self.decoder_session = Some(decoder_session); + self.tokenizer = Some(tokenizer); + self.initialized = true; + info!("SmolVLM: All models loaded successfully, processor initialized"); + Ok(true) + } + + fn process_image_from_buffer(&mut self, buffer: &[u8], width: i32, height: i32) -> AnyhowResult { + if !self.initialized { + info!("SmolVLM: Cannot process image - models not initialized"); + return Ok(false); + } + + info!("SmolVLM: Processing image {}x{}", width, height); + + // Convert ARGB8888 buffer to RGB image + // Android's ARGB_8888 with copyPixelsToBuffer gives us RGBA in memory + let mut rgb_data = Vec::with_capacity((height * width * 3) as usize); + for i in 0..(height * width) as usize { + let idx = i * 4; + // RGBA format: [R, G, B, A] in memory + rgb_data.push(buffer[idx]); // R + rgb_data.push(buffer[idx + 1]); // G + rgb_data.push(buffer[idx + 2]); // B + // Skip alpha channel (idx + 3) + } + + let img = RgbImage::from_raw( + width as u32, + height as u32, + rgb_data + ).ok_or_else(|| anyhow::anyhow!("Failed to create image from buffer"))?; + + let dynamic_img = DynamicImage::ImageRgb8(img); + + // Use the SmolVLM image processor for proper preprocessing + info!("SmolVLM: Preprocessing image with SmolVLM processor"); + let (processed_images, attention_mask) = self.image_processor.preprocess(dynamic_img)?; + + // Run vision encoder + if let Some(ref mut session) = self.vision_session { + info!("SmolVLM: Running vision encoder"); + let vision_start = std::time::Instant::now(); + let mut inputs: HashMap<&str, Value> = HashMap::new(); + + // For SmolVLM2, we process the full image tensor at once + let shape = processed_images.shape(); + info!("SmolVLM: Image tensor shape: {:?}", shape); + + // Keep the 5D shape (batch_size, num_patches, channels, height, width) as expected by the vision encoder + inputs.insert("pixel_values", Value::from_array(processed_images.clone())?.into()); + + // Convert attention mask to boolean while maintaining shape (batch, num_frames, height, width) + let pixel_attention_mask_bool = attention_mask.map(|&x| x != 0); + inputs.insert("pixel_attention_mask", Value::from_array(pixel_attention_mask_bool)?.into()); + + info!("SmolVLM: About to run vision encoder with pixel_values and pixel_attention_mask inputs"); + let outputs = match session.run(inputs) { + Ok(outputs) => { + let vision_elapsed = vision_start.elapsed(); + info!("SmolVLM: Vision encoder ran successfully in {:.2}ms, got {} outputs", vision_elapsed.as_millis(), outputs.len()); + // Log all available output keys + let output_keys: Vec = outputs.keys().map(|k| k.to_string()).collect(); + info!("SmolVLM: Vision encoder output keys: {:?}", output_keys); + outputs + } + Err(e) => { + error!("SmolVLM: Vision encoder failed: {}", e); + return Ok(false); + } + }; + + // Extract image embeddings from vision encoder output (take first output) + if let Some((_, embeddings_value)) = outputs.into_iter().next() { + let embeddings_array = embeddings_value.try_extract_tensor::()?; + let (shape, data) = embeddings_array; + info!("SmolVLM: Vision encoder output shape: {:?}", shape); + + // Convert to ndarray for easier manipulation + let total_len = data.len(); + let last_dim = shape[shape.len() - 1] as usize; + let seq_len = total_len / last_dim; + + let embeddings_2d = ndarray::Array2::from_shape_vec((seq_len, last_dim), data.to_vec())?; + self.image_embeddings = Some(embeddings_2d); + info!("SmolVLM: Image embeddings extracted, shape: {:?}", (seq_len, last_dim)); + return Ok(true); + } else { + error!("SmolVLM: Vision encoder returned no outputs"); + return Ok(false); + } + } else { + error!("SmolVLM: Vision session not available"); + return Ok(false); + } + } + + fn generate_response(&mut self, prompt: &str, max_tokens: i32) -> AnyhowResult { + if !self.initialized { + info!("SmolVLM: Cannot generate response - models not initialized"); + return Ok("Error: Models not loaded".to_string()); + } + + let image_embeddings = match &self.image_embeddings { + Some(embeddings) => embeddings, + None => { + info!("SmolVLM: No image processed yet"); + return Ok("Please capture and process an image first.".to_string()); + } + }; + + info!("SmolVLM: Starting text generation for prompt: {}", prompt); + + let tokenizer = self.tokenizer.as_ref() + .ok_or_else(|| anyhow::anyhow!("Tokenizer not loaded"))?; + + // Get image token ID + let image_token = ""; + let image_token_id = tokenizer.token_to_id(image_token) + .ok_or_else(|| anyhow::anyhow!("Failed to get image token ID"))?; + + // SmolVLM2 uses 4x4 grid with 64 tokens per patch + let image_rows = 4; + let image_cols = 4; + let image_seq_len = 64; + + // Generate the full image prompt string with proper structure + let mut image_prompt_str = String::new(); + for n_h in 0..image_rows { + for n_w in 0..image_cols { + image_prompt_str.push_str(&format!( + "{}", + n_h + 1, + n_w + 1, + image_token.repeat(image_seq_len) + )); + } + image_prompt_str.push('\n'); + } + image_prompt_str.push_str(&format!( + "\n{}", + image_token.repeat(image_seq_len) + )); + image_prompt_str.push_str(""); + + // Create the full prompt with expanded image tokens + let full_prompt = format!("<|im_start|>User:{}\n{}\nAssistant:", + image_prompt_str, prompt); + + info!("SmolVLM: Full prompt length: {}", full_prompt.len()); + + // Encode the expanded prompt + let encoding = tokenizer.encode(full_prompt.as_str(), true) + .map_err(|e| anyhow::anyhow!("Failed to encode prompt: {}", e))?; + let input_ids = encoding.get_ids(); + let attention_mask = encoding.get_attention_mask(); + + info!("SmolVLM: Encoded prompt to {} tokens", input_ids.len()); + + // Convert input IDs to ndarray + let input_ids_array = ndarray::Array::from_vec(input_ids.iter().map(|&x| x as i64).collect()) + .into_shape_with_order((1, input_ids.len()))?; + let attention_mask_array = ndarray::Array::from_vec(attention_mask.iter().map(|&x| x as i64).collect()) + .into_shape_with_order((1, attention_mask.len()))?; + + // Get embeddings for all tokens + let mut input_embeds = { + let mut embed_inputs: HashMap<&str, Value> = HashMap::new(); + embed_inputs.insert("input_ids", Value::from_array(input_ids_array.clone())?.into()); + + let embed_outputs = if let Some(ref mut embed_session) = self.embed_session { + embed_session.run(embed_inputs)? + } else { + return Ok("Error: Embedding model not loaded".to_string()); + }; + + let embeddings_value = embed_outputs.values().next() + .ok_or_else(|| anyhow::anyhow!("No embedding output"))?; + let tensor_data = embeddings_value.try_extract_tensor::()?; + let (shape, data) = tensor_data; + ndarray::Array3::from_shape_vec( + (shape[0] as usize, shape[1] as usize, shape[2] as usize), + data.to_vec() + )? + }; + + info!("SmolVLM: Input embeddings shape: {:?}", input_embeds.shape()); + + // Replace image token embeddings with actual image features + let mut feature_idx = 0; + for i in 0..input_ids_array.shape()[1] { + if input_ids_array[[0, i]] == image_token_id as i64 { + if feature_idx < image_embeddings.shape()[0] { + for j in 0..image_embeddings.shape()[1] { + input_embeds[[0, i, j]] = image_embeddings[[feature_idx, j]]; + } + feature_idx += 1; + } + } + } + + info!("SmolVLM: Replaced {} image tokens with vision features", feature_idx); + + let combined_embeddings = input_embeds; + + // Generate tokens using decoder model with proper past key values + if let Some(ref mut decoder_session) = self.decoder_session { + info!("SmolVLM: Starting generation with proper past key values"); + + // Auto-detect model configuration from decoder inputs + let decoder_inputs = &decoder_session.inputs; + let mut max_layer = 0; + let detected_kv_heads = 5; // Based on error messages - updated from 3 to 5 + + for input in decoder_inputs { + if input.name.starts_with("past_key_values.") { + if let Some(layer_str) = input.name.split('.').nth(1) { + if let Ok(layer_num) = layer_str.parse::() { + max_layer = max_layer.max(layer_num); + } + } + } + } + + let num_hidden_layers = max_layer + 1; + let head_dim = 64; // Standard head dimension for SmolVLM + info!("SmolVLM: Auto-detected {} layers, {} kv heads", num_hidden_layers, detected_kv_heads); + + // Initialize past key values + let batch_size = 1; + let mut past_key_values: HashMap> = HashMap::new(); + for layer in 0..num_hidden_layers { + let key_array = ndarray::Array::zeros((batch_size, detected_kv_heads, 0, head_dim)).into_dyn(); + let value_array = ndarray::Array::zeros((batch_size, detected_kv_heads, 0, head_dim)).into_dyn(); + past_key_values.insert(format!("past_key_values.{}.key", layer), key_array); + past_key_values.insert(format!("past_key_values.{}.value", layer), value_array); + } + + // Use the properly tokenized input_ids for the initial prompt + let mut input_ids = input_ids_array.clone(); + + // Use the attention mask from tokenization (already matches the sequence) + let mut attention_mask = attention_mask_array.clone(); + + // Calculate cumulative position_ids + let mut position_ids_vec = Vec::new(); + let mut cumsum = 0i64; + for &mask_val in attention_mask.iter() { + cumsum += mask_val; + position_ids_vec.push(cumsum); + } + let mut position_ids = ndarray::Array2::from_shape_vec((1, attention_mask.len()), position_ids_vec)?; + + let mut generated_tokens = Vec::new(); + let max_new_tokens = max_tokens.min(50); + + for step in 0..max_new_tokens { + // Get input embeddings + let input_embeds = if step == 0 { + // Use the pre-computed combined embeddings for first step + combined_embeddings.clone() + } else { + // Get embeddings for the new token + if let Some(ref mut embed_session) = self.embed_session { + let mut embed_inputs: HashMap<&str, Value> = HashMap::new(); + embed_inputs.insert("input_ids", Value::from_array(input_ids.clone())?.into()); + let embed_outputs = embed_session.run(embed_inputs)?; + + if let Some(embeddings_value) = embed_outputs.values().next() { + let tensor_data = embeddings_value.try_extract_tensor::()?; + let (shape, data) = tensor_data; + // Convert to ndarray + Array3::from_shape_vec( + (shape[0] as usize, shape[1] as usize, shape[2] as usize), + data.to_vec(), + )? + } else { + return Err(anyhow::anyhow!("No embeddings output")); + } + } else { + return Err(anyhow::anyhow!("Embed session not available")); + } + }; + + // Prepare decoder inputs + let mut decoder_inputs: HashMap<&str, Value> = HashMap::new(); + decoder_inputs.insert("inputs_embeds", Value::from_array(input_embeds.clone())?.into()); + decoder_inputs.insert("attention_mask", Value::from_array(attention_mask.clone())?.into()); + decoder_inputs.insert("position_ids", Value::from_array(position_ids.clone())?.into()); + + // Add past key values + for (key, value) in &past_key_values { + decoder_inputs.insert(key, Value::from_array(value.clone())?.into()); + } + + info!("SmolVLM: Running decoder step {}", step); + let decoder_outputs = decoder_session.run(decoder_inputs)?; + + // Get logits (first output) + let logits_tensor = decoder_outputs[0].try_extract_tensor::()?; + let (logits_shape, logits_data) = logits_tensor; + let logits = Array3::from_shape_vec( + (logits_shape[0] as usize, logits_shape[1] as usize, logits_shape[2] as usize), + logits_data.to_vec(), + )?; + info!("SmolVLM: Decoder logits shape: {:?}", logits.shape()); + + // Get next token from last position + let last_idx = logits.shape()[1] - 1; + let logits_slice = logits.slice(ndarray::s![0, last_idx, ..]); + + // Use argmax to get the most likely token + let next_token = logits_slice + .iter() + .enumerate() + .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)) + .map(|(i, _)| i as i64) + .unwrap(); + + generated_tokens.push(next_token as u32); + + // Check for end-of-sequence token + if next_token == 2 { + info!("SmolVLM: End-of-sequence token reached at step {}", step); + break; + } + + info!("SmolVLM: Generated token {} at step {}", next_token, step); + + // Update inputs for next iteration (following working reference implementation) + input_ids = ndarray::Array::from_vec(vec![next_token]).into_shape_with_order((1, 1))?; + + // Concatenate new attention mask (following working reference implementation) + let new_attention = ndarray::Array::::ones((1, 1)); + let total_length = attention_mask.len() + new_attention.len(); + let mut attention_vec = attention_mask.iter().copied().collect::>(); + attention_vec.extend(new_attention.iter().copied()); + attention_mask = ndarray::Array::from_vec(attention_vec).into_shape_with_order((1, total_length))?; + + // Update position_ids (following working reference implementation) + let current_pos = position_ids[[0, position_ids.shape()[1] - 1]] + 1; + position_ids = ndarray::Array::from_vec(vec![current_pos]).into_shape_with_order((1, 1))?; + + // Update past key values from decoder outputs + for i in 0..num_hidden_layers { + let key = format!("past_key_values.{}.key", i); + let value = format!("past_key_values.{}.value", i); + + if let Some(past_key) = past_key_values.get_mut(&key) { + if i * 2 + 1 < decoder_outputs.len() { + let present_key_tensor = decoder_outputs[i * 2 + 1].try_extract_tensor::()?; + let (shape, data) = present_key_tensor; + let present_key = ndarray::ArrayD::from_shape_vec( + shape.iter().map(|&s| s as usize).collect::>(), + data.to_vec(), + )?; + *past_key = present_key; + } + } + + if let Some(past_value) = past_key_values.get_mut(&value) { + if i * 2 + 2 < decoder_outputs.len() { + let present_value_tensor = decoder_outputs[i * 2 + 2].try_extract_tensor::()?; + let (shape, data) = present_value_tensor; + let present_value = ndarray::ArrayD::from_shape_vec( + shape.iter().map(|&s| s as usize).collect::>(), + data.to_vec(), + )?; + *past_value = present_value; + } + } + } + + if step >= 10 { // Generate at least 10 tokens for a meaningful response + break; + } + } + + // Decode generated tokens back to text + if !generated_tokens.is_empty() { + match tokenizer.decode(&generated_tokens, true) { + Ok(generated_text) => { + info!("SmolVLM: Generated text: {}", generated_text); + Ok(generated_text) + } + Err(e) => { + error!("SmolVLM: Failed to decode tokens: {}", e); + Ok(format!("Generated {} tokens but failed to decode", generated_tokens.len())) + } + } + } else { + Ok("No tokens generated.".to_string()) + } + } else { + Ok("Error: Decoder model not loaded".to_string()) + } + } + } + + fn get_or_create_instance() -> &'static Mutex { + SMOLVLM_INSTANCE.get_or_init(|| { + Mutex::new(SmolVLMProcessor::new().expect("Failed to create SmolVLM processor")) + }) + } + + #[unsafe(no_mangle)] + pub extern "C" fn Java_ai_baseweight_baseweightsnap_SmolVLMAndroid_nativeLoadModels( + env: JNIEnv, + _class: JClass, + model_dir: JString, + tokenizer_path: JString, + ) -> jboolean { + init_logger(); + info!("SmolVLM: nativeLoadModels called from JNI"); + + let model_dir_str = match env.get_string(model_dir) { + Ok(s) => s.to_string_lossy().to_string(), + Err(e) => { + error!("SmolVLM: Failed to get model_dir string: {:?}", e); + return false as jboolean; + } + }; + + let tokenizer_path_str = match env.get_string(tokenizer_path) { + Ok(s) => s.to_string_lossy().to_string(), + Err(e) => { + error!("SmolVLM: Failed to get tokenizer_path string: {:?}", e); + return false as jboolean; + } + }; + + info!("SmolVLM: JNI parameters - model_dir: {}, tokenizer_path: {}", model_dir_str, tokenizer_path_str); + + let instance = get_or_create_instance(); + match instance.lock() { + Ok(mut processor) => { + match processor.load_models(&model_dir_str, &tokenizer_path_str) { + Ok(success) => { + info!("SmolVLM: load_models returned: {}", success); + success as jboolean + } + Err(e) => { + error!("SmolVLM: load_models failed with error: {}", e); + false as jboolean + } + } + } + Err(e) => { + error!("SmolVLM: Failed to lock processor: {:?}", e); + false as jboolean + } + } + } + + #[unsafe(no_mangle)] + pub extern "C" fn Java_ai_baseweight_baseweightsnap_SmolVLMAndroid_processImageFromBuffer( + env: JNIEnv, + _class: JClass, + buffer: JObject, + width: jint, + height: jint, + ) -> jboolean { + let buffer_ptr = match env.get_direct_buffer_address(buffer.into()) { + Ok(ptr) => ptr, + Err(_) => return false as jboolean, + }; + + let buffer_slice = unsafe { + std::slice::from_raw_parts( + buffer_ptr as *const u8, + (height * width * 4) as usize + ) + }; + + let instance = get_or_create_instance(); + match instance.lock() { + Ok(mut processor) => { + match processor.process_image_from_buffer(buffer_slice, width, height) { + Ok(success) => success as jboolean, + Err(_) => false as jboolean, + } + } + Err(_) => false as jboolean, + } + } + + #[unsafe(no_mangle)] + pub extern "C" fn Java_ai_baseweight_baseweightsnap_SmolVLMAndroid_nativeGenerateResponse( + env: JNIEnv, + _class: JClass, + prompt: JString, + max_tokens: jint, + ) -> jstring { + let prompt_str = match env.get_string(prompt) { + Ok(s) => s.to_string_lossy().to_string(), + Err(_) => return env.new_string("Error: Failed to get prompt").unwrap().into_raw(), + }; + + let instance = get_or_create_instance(); + match instance.lock() { + Ok(mut processor) => { + match processor.generate_response(&prompt_str, max_tokens) { + Ok(result) => env.new_string(&result).unwrap().into_raw(), + Err(e) => env.new_string(&format!("Error: {}", e)).unwrap().into_raw(), + } + } + Err(_) => env.new_string("Error: Failed to lock processor").unwrap().into_raw(), + } + } +} \ No newline at end of file