diff --git a/Cargo.lock b/Cargo.lock index 7a3eeea..a9563d3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3402,6 +3402,7 @@ dependencies = [ "tauri", "tauri-build", "tauri-plugin-autostart", + "tauri-plugin-dialog", "tauri-plugin-opener", "tauri-plugin-process", "tauri-plugin-sentry", @@ -3906,6 +3907,30 @@ dependencies = [ "web-sys", ] +[[package]] +name = "rfd" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a15ad77d9e70a92437d8f74c35d99b4e4691128df018833e99f90bcd36152672" +dependencies = [ + "block2", + "dispatch2", + "glib-sys", + "gobject-sys", + "gtk-sys", + "js-sys", + "log", + "objc2", + "objc2-app-kit", + "objc2-core-foundation", + "objc2-foundation", + "raw-window-handle", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "windows-sys 0.60.2", +] + [[package]] name = "ring" version = "0.17.14" @@ -4968,6 +4993,48 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "tauri-plugin-dialog" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65981abb771e74e571a38196c3baa11c459379164791eba0e67abc1a5fac9884" +dependencies = [ + "log", + "raw-window-handle", + "rfd", + "serde", + "serde_json", + "tauri", + "tauri-plugin", + "tauri-plugin-fs", + "thiserror 2.0.18", + "url", +] + +[[package]] +name = "tauri-plugin-fs" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7ecc274121aca0c036a2b42d1cbe83d368d348f54e0bb8a735c2b1548e8f371" +dependencies = [ + "anyhow", + "dunce", + "glob", + "log", + "objc2-foundation", + "percent-encoding", + "schemars 0.8.22", + "serde", + "serde_json", + "serde_repr", + "tauri", + "tauri-plugin", + "tauri-utils", + "thiserror 2.0.18", + "toml 1.1.2+spec-1.1.0", + "url", +] + [[package]] name = "tauri-plugin-opener" version = "2.5.4" diff --git a/docs/releases/UNRELEASED.md b/docs/releases/UNRELEASED.md index 441b571..e7b28cd 100644 --- a/docs/releases/UNRELEASED.md +++ b/docs/releases/UNRELEASED.md @@ -6,7 +6,13 @@ then reset this file. ## User-facing changes -- None yet. +- Transcribe a file: drop an audio or video file onto the app (or browse from + the dashboard) to get a transcript from the local whisper.cpp engine, with + live progress and cancel. Results land in History marked with the source + file and export as TXT, SRT, or VTT. Fully local; the optional AI cleanup + step is off by default for files. Requires `ffmpeg` (new doctor check). +- File transcriptions are excluded from the dictation metrics (sessions, + minutes saved). ## Internal/release changes @@ -21,13 +27,17 @@ then reset this file. ### Tested -- None yet. +- `cargo test --workspace` (79), `bun run check`, `bun run test:coverage` + (ratchet green) on the file-transcription PRs. +- ffmpeg conversion flags verified live against WAV/MP3/MP4 samples; whisper + `--output-json`/progress format verified against the installed whisper-cli. ### Not tested yet - App build. - Installer or updater flow. - Platform smoke checks. +- Interactive drag-drop/dialog smoke in a real desktop session. ### Release blockers diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 7f6cc36..26734dd 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -25,6 +25,7 @@ tauri = { version = "2", features = ["tray-icon", "image-png"] } tauri-plugin-opener = "2" tauri-plugin-single-instance = "2" tauri-plugin-autostart = "2" +tauri-plugin-dialog = "2" tauri-plugin-updater = "2" tauri-plugin-process = "2" tauri-plugin-sentry = "0.5" diff --git a/src-tauri/capabilities/default.json b/src-tauri/capabilities/default.json index 8803a1b..af63e21 100644 --- a/src-tauri/capabilities/default.json +++ b/src-tauri/capabilities/default.json @@ -13,6 +13,7 @@ "core:window:allow-set-position", "core:window:allow-outer-position", "sentry:default", + "dialog:default", "autostart:allow-enable", "autostart:allow-disable", "autostart:allow-is-enabled", diff --git a/src-tauri/src/engine.rs b/src-tauri/src/engine.rs index ed1cf87..0d72507 100644 --- a/src-tauri/src/engine.rs +++ b/src-tauri/src/engine.rs @@ -18,6 +18,8 @@ use pickscribe::platform; use serde::Serialize; use tauri::{AppHandle, Emitter, Manager}; +use crate::file_job::FileJobControl; + pub const EVENT_STATE: &str = "pickscribe://state"; pub const EVENT_LEVEL: &str = "pickscribe://level"; pub const EVENT_HISTORY: &str = "pickscribe://history"; @@ -103,6 +105,7 @@ struct SessionControl { pub struct Engine { recording: Mutex>, active_session: Mutex>, + pub(crate) file_job: Mutex>, state: Mutex, levels_running: Arc, /// Paste chord requested by the triggering invocation (e.g. the legacy @@ -124,6 +127,7 @@ impl Engine { Ok(Self { recording: Mutex::new(None), active_session: Mutex::new(None), + file_job: Mutex::new(None), state: Mutex::new(StatePayload { stage: Stage::Idle, recording_started_ms: None, diff --git a/src-tauri/src/file_job.rs b/src-tauri/src/file_job.rs new file mode 100644 index 0000000..0b1ba85 --- /dev/null +++ b/src-tauri/src/file_job.rs @@ -0,0 +1,335 @@ +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::atomic::{AtomicU8, Ordering}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use anyhow::{Context, Result, bail}; +use pickscribe::config::AppConfig; +use pickscribe::engine::{ + cleanup, + incremental::CancelToken, + media::{self, MEDIA_EXTENSIONS}, + recorder, stt, transcript, +}; +use pickscribe::history::NewEntry; +use serde::Serialize; +use tauri::{AppHandle, Emitter}; + +use crate::engine::{EVENT_HISTORY, Engine}; + +pub const EVENT_FILE: &str = "pickscribe://file"; + +#[derive(Debug, Clone, Copy, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum FileStage { + Converting, + Transcribing, + Cleaning, + Done, + Error, + Cancelled, +} + +#[derive(Debug, Clone, Serialize)] +pub struct FileJobState { + pub stage: FileStage, + pub progress: u8, + pub source_file: String, + pub error: Option, + pub entry_id: Option, +} + +pub(crate) struct FileJobControl { + pub(crate) cancel_token: CancelToken, +} + +pub fn start( + engine: Arc, + app: AppHandle, + source_file: String, + cleanup_requested: bool, +) -> Result<()> { + validate_input_path(Path::new(&source_file))?; + + let cancel_token = CancelToken::new(); + { + let mut file_job = engine.file_job.lock().unwrap(); + if file_job.is_some() { + bail!("a file transcription is already running"); + } + *file_job = Some(FileJobControl { + cancel_token: cancel_token.clone(), + }); + } + + std::thread::spawn(move || { + run_file_job(engine, app, source_file, cleanup_requested, cancel_token) + }); + Ok(()) +} + +struct FileJobGuard { + engine: Arc, + temp_dir: Option, +} + +impl FileJobGuard { + fn new(engine: Arc) -> Self { + Self { + engine, + temp_dir: None, + } + } + + fn set_temp_dir(&mut self, temp_dir: PathBuf) { + self.temp_dir = Some(temp_dir); + } +} + +impl Drop for FileJobGuard { + fn drop(&mut self) { + if let Some(temp_dir) = self.temp_dir.take() { + let _ = fs::remove_dir_all(temp_dir); + } + match self.engine.file_job.lock() { + Ok(mut file_job) => *file_job = None, + Err(poisoned) => { + *poisoned.into_inner() = None; + self.engine.file_job.clear_poison(); + } + } + } +} + +pub fn cancel(engine: &Engine) { + if let Some(file_job) = engine.file_job.lock().unwrap().as_ref() { + file_job.cancel_token.cancel(); + } +} + +fn validate_input_path(path: &Path) -> Result<()> { + if !path.is_file() { + bail!("media file not found: {}", path.display()); + } + let extension = path + .extension() + .and_then(|extension| extension.to_str()) + .map(str::to_ascii_lowercase) + .context("media file has no extension")?; + if !MEDIA_EXTENSIONS.contains(&extension.as_str()) { + bail!("unsupported media file extension: .{extension}"); + } + Ok(()) +} + +fn run_file_job( + engine: Arc, + app: AppHandle, + source_file: String, + cleanup_requested: bool, + cancel_token: CancelToken, +) { + let progress = Arc::new(AtomicU8::new(0)); + let mut guard = FileJobGuard::new(Arc::clone(&engine)); + let result = (|| -> Result { + let dir = create_temp_dir()?; + guard.set_temp_dir(dir.clone()); + let wav = dir.join("audio.wav"); + let cfg = AppConfig::load(); + + if cancel_token.is_cancelled() { + bail!("file transcription cancelled"); + } + emit_state(&app, FileStage::Converting, 0, &source_file, None, None); + media::convert_to_wav_16k_mono(Path::new(&source_file), &wav, || { + cancel_token.is_cancelled() + })?; + if cancel_token.is_cancelled() { + bail!("file transcription cancelled"); + } + + let duration_ms = media::wav_duration_ms(&wav)?; + emit_state(&app, FileStage::Transcribing, 0, &source_file, None, None); + let progress_for_callback = Arc::clone(&progress); + let app_for_callback = app.clone(); + let source_for_callback = source_file.clone(); + let segments = stt::transcribe_file_with_cancel( + &cfg.stt, + &wav, + || cancel_token.is_cancelled(), + move |percentage| { + if progress_for_callback.swap(percentage, Ordering::Relaxed) != percentage { + emit_state( + &app_for_callback, + FileStage::Transcribing, + percentage, + &source_for_callback, + None, + None, + ); + } + }, + )?; + if cancel_token.is_cancelled() { + bail!("file transcription cancelled"); + } + + let raw_text = transcript::to_txt(&segments); + if raw_text.is_empty() { + bail!("no speech detected in the file"); + } + let (cleaned_text, provider, model, cleanup_error) = if cleanup_requested { + emit_state( + &app, + FileStage::Cleaning, + progress.load(Ordering::Relaxed), + &source_file, + None, + None, + ); + let outcome = cleanup::clean(&cfg, &raw_text); + if cancel_token.is_cancelled() { + bail!("file transcription cancelled"); + } + ( + (outcome.text != raw_text).then_some(outcome.text), + outcome.provider, + outcome.model, + outcome + .error + .map(|error| format!("cleanup skipped: {error}")), + ) + } else { + (None, "none".to_string(), String::new(), None) + }; + let segments_json = if segments.is_empty() { + None + } else { + Some(serde_json::to_string(&segments).context("serializing file segments")?) + }; + if cancel_token.is_cancelled() { + bail!("file transcription cancelled"); + } + + let entry_id = engine.history.insert(&NewEntry { + duration_ms, + raw_text, + cleaned_text, + provider, + model, + language: cfg.stt.language, + source_file: Some(source_file.clone()), + segments_json, + })?; + let _ = app.emit(EVENT_HISTORY, ()); + Ok(FileJobComplete { + entry_id, + cleanup_error, + }) + })(); + + match result { + Ok(result) => emit_state( + &app, + FileStage::Done, + progress.load(Ordering::Relaxed), + &source_file, + result.cleanup_error, + Some(result.entry_id), + ), + Err(_) if cancel_token.is_cancelled() => emit_state( + &app, + FileStage::Cancelled, + progress.load(Ordering::Relaxed), + &source_file, + None, + None, + ), + Err(err) => emit_state( + &app, + FileStage::Error, + progress.load(Ordering::Relaxed), + &source_file, + Some(format!("{err:#}")), + None, + ), + } +} + +struct FileJobComplete { + entry_id: i64, + cleanup_error: Option, +} + +fn create_temp_dir() -> Result { + let parent = recorder::state_dir(); + fs::create_dir_all(&parent).with_context(|| format!("creating {}", parent.display()))?; + let stamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_nanos()) + .unwrap_or(0); + let dir = parent.join(format!("file-{}-{stamp}", std::process::id())); + fs::create_dir(&dir).with_context(|| format!("creating {}", dir.display()))?; + Ok(dir) +} + +fn emit_state( + app: &AppHandle, + stage: FileStage, + progress: u8, + source_file: &str, + error: Option, + entry_id: Option, +) { + let _ = app.emit( + EVENT_FILE, + FileJobState { + stage, + progress, + source_file: source_file.to_string(), + error, + entry_id, + }, + ); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn temp_dir() -> Result { + let stamp = SystemTime::now().duration_since(UNIX_EPOCH)?.as_nanos(); + let dir = std::env::temp_dir().join(format!( + "pickscribe-file-job-{}-{stamp}", + std::process::id() + )); + fs::create_dir(&dir)?; + Ok(dir) + } + + #[test] + fn validates_supported_media_extensions_case_insensitively() -> Result<()> { + let dir = temp_dir()?; + let path = dir.join("recording.MP4"); + fs::write(&path, [])?; + + validate_input_path(&path)?; + + fs::remove_dir_all(dir)?; + Ok(()) + } + + #[test] + fn rejects_unsupported_media_extensions() -> Result<()> { + let dir = temp_dir()?; + let path = dir.join("recording.txt"); + fs::write(&path, [])?; + + let error = validate_input_path(&path).unwrap_err(); + + assert!(error.to_string().contains("unsupported media file extension")); + fs::remove_dir_all(dir)?; + Ok(()) + } +} diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 344b0ea..385989d 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -1,19 +1,23 @@ mod engine; +mod file_job; mod kwin; mod tray; +use std::fs; +use std::path::Path; use std::sync::Arc; use std::sync::Mutex; use std::sync::OnceLock; use std::sync::atomic::{AtomicBool, Ordering}; use pickscribe::config::AppConfig; -use pickscribe::engine::{command_exists, stt}; +use pickscribe::engine::{command_exists, media, stt, transcript}; use pickscribe::history::{HistoryEntry, Metrics}; use pickscribe::platform::{self, PlatformSupport}; use serde::Serialize; use tauri::{AppHandle, Manager, State, WebviewUrl, WebviewWindowBuilder, WindowEvent}; use tauri_plugin_autostart::MacosLauncher; +use tauri_plugin_dialog::DialogExt; use engine::{Engine, StatePayload}; @@ -188,6 +192,57 @@ mod tests { other => panic!("expected wasm debug image, got {other:?}"), } } + + fn history_entry() -> HistoryEntry { + HistoryEntry { + id: 42, + created_at: 0, + duration_ms: 1_000, + raw_text: "raw transcript".into(), + cleaned_text: Some("clean transcript".into()), + provider: "none".into(), + model: String::new(), + language: "en".into(), + source_file: Some("/home/me/meeting.mp4".into()), + segments_json: Some( + r#"[{"start_ms":0,"end_ms":1000,"text":"First sentence."}]"#.into(), + ), + word_count: 2, + } + } + + #[test] + fn export_content_uses_cleaned_text_and_timestamped_segments() { + let entry = history_entry(); + + assert_eq!(export_content(&entry, "txt").unwrap(), "clean transcript"); + assert_eq!( + export_content(&entry, "srt").unwrap(), + "1\n00:00:00,000 --> 00:00:01,000\nFirst sentence.\n" + ); + assert_eq!(export_file_name(&entry, "vtt"), "meeting.vtt"); + } + + #[test] + fn timestamped_exports_require_segments() { + let mut entry = history_entry(); + entry.segments_json = None; + + assert_eq!( + export_content(&entry, "vtt").unwrap_err(), + "no timestamped segments for this entry" + ); + assert_eq!( + export_file_name( + &HistoryEntry { + source_file: None, + ..entry + }, + "txt" + ), + "transcript-42.txt" + ); + } } #[tauri::command] @@ -260,6 +315,110 @@ fn clear_history(engine: State<'_, Arc>) -> CommandResult<()> { engine.history.clear().map_err(err_string) } +#[tauri::command] +fn transcribe_media_file( + app: AppHandle, + engine: State<'_, Arc>, + path: String, + cleanup: bool, +) -> CommandResult<()> { + file_job::start(Arc::clone(engine.inner()), app, path, cleanup).map_err(err_string) +} + +#[tauri::command] +fn cancel_file_transcription(engine: State<'_, Arc>) -> CommandResult<()> { + file_job::cancel(engine.inner()); + Ok(()) +} + +#[tauri::command] +async fn pick_media_file(app: AppHandle) -> CommandResult> { + tauri::async_runtime::spawn_blocking(move || { + app.dialog() + .file() + .add_filter("Media", media::MEDIA_EXTENSIONS) + .blocking_pick_file() + .map(|path| { + path.into_path() + .map(|path| path.display().to_string()) + .map_err(err_string) + }) + .transpose() + }) + .await + .map_err(err_string)? +} + +#[tauri::command] +async fn export_history_entry( + app: AppHandle, + engine: State<'_, Arc>, + id: i64, + format: String, +) -> CommandResult> { + let entry = engine + .history + .get(id) + .map_err(err_string)? + .ok_or_else(|| format!("history entry not found: {id}"))?; + let content = export_content(&entry, &format)?; + let file_name = export_file_name(&entry, &format); + + tauri::async_runtime::spawn_blocking(move || { + let Some(path) = app + .dialog() + .file() + .add_filter(format.to_ascii_uppercase(), &[format.as_str()]) + .set_file_name(file_name) + .blocking_save_file() + else { + return Ok(None); + }; + let path = path.into_path().map_err(err_string)?; + fs::write(&path, content).map_err(err_string)?; + Ok(Some(path.display().to_string())) + }) + .await + .map_err(err_string)? +} + +fn export_content(entry: &HistoryEntry, format: &str) -> CommandResult { + match format { + "txt" => Ok(entry + .cleaned_text + .as_deref() + .filter(|text| !text.trim().is_empty()) + .unwrap_or(&entry.raw_text) + .to_string()), + "srt" | "vtt" => { + let segments_json = entry + .segments_json + .as_deref() + .ok_or_else(|| "no timestamped segments for this entry".to_string())?; + let segments: Vec = + serde_json::from_str(segments_json).map_err(err_string)?; + Ok(if format == "srt" { + transcript::to_srt(&segments) + } else { + transcript::to_vtt(&segments) + }) + } + _ => Err(format!("unsupported export format: {format}")), + } +} + +fn export_file_name(entry: &HistoryEntry, extension: &str) -> String { + let source_name = entry.source_file.as_deref().and_then(|source| { + Path::new(source) + .file_stem() + .and_then(|name| name.to_str()) + }); + match source_name.filter(|name| !name.is_empty()) { + Some(name) => format!("{name}.{extension}"), + None => format!("transcript-{}.{}", entry.id, extension), + } +} + #[tauri::command] fn get_metrics(engine: State<'_, Arc>) -> CommandResult { let cfg = AppConfig::load(); @@ -294,6 +453,10 @@ fn run_doctor() -> Vec { support.dictation_supported, support.summary.clone(), ); + match media::resolve_ffmpeg() { + Ok(path) => push("ffmpeg", true, path.display().to_string()), + Err(err) => push("ffmpeg", false, format!("{err:#}")), + } for blocker in &support.blockers { push(&blocker.name, false, blocker.detail.clone()); } @@ -674,6 +837,7 @@ pub fn run() { .manage(engine) .plugin(sentry_plugin) .plugin(tauri_plugin_opener::init()) + .plugin(tauri_plugin_dialog::init()) .plugin(tauri_plugin_updater::Builder::new().build()) .plugin(tauri_plugin_process::init()) .plugin(tauri_plugin_autostart::init( @@ -699,6 +863,10 @@ pub fn run() { list_history, delete_history_entry, clear_history, + transcribe_media_file, + cancel_file_transcription, + pick_media_file, + export_history_entry, get_metrics, run_doctor, list_models, diff --git a/src/App.svelte b/src/App.svelte index 7fe3fe6..f8c6678 100644 --- a/src/App.svelte +++ b/src/App.svelte @@ -13,6 +13,7 @@ EVENT_STATE, type StatePayload, } from "./lib/api"; + import FileTranscribe from "./lib/components/FileTranscribe.svelte"; import ResizeHandles from "./lib/components/ResizeHandles.svelte"; import Titlebar from "./lib/components/Titlebar.svelte"; import Dashboard from "./lib/views/Dashboard.svelte"; @@ -38,6 +39,8 @@ }); let levels = $state(Array(LEVEL_BARS).fill(0)); let historyVersion = $state(0); + let fileBusy = $state(false); + let fileActions: { browse: () => void } | null = null; const navItems: { id: View; label: string; icon: typeof Microphone }[] = [ { id: "dashboard", label: "Dictate", icon: Microphone }, @@ -171,7 +174,13 @@
{#if view === "dashboard"} - + fileActions?.browse()} + /> {:else if view === "history"} {:else} @@ -191,6 +200,12 @@ + (fileActions = actions)} + onViewHistory={() => navigate("history")} + onBusyChange={(busy) => (fileBusy = busy)} +/> + {#if pendingView}