From a8fe532d6991aeb9299324a9c371a62da1a7463a Mon Sep 17 00:00:00 2001 From: xlgzsgf <51521689+hiqiancheng@users.noreply.github.com> Date: Wed, 3 Jun 2026 01:06:43 +0800 Subject: [PATCH 1/4] feat(agent-service): add read-only desktop context capsules Related to #281 --- .../src-tauri/src/commands/desktop_context.rs | 25 + apps/desktop/src-tauri/src/commands/mod.rs | 3 + .../src/core/system/desktop_context.rs | 433 ++++++++++++++++++ apps/desktop/src-tauri/src/core/system/mod.rs | 1 + .../src/core/window/search/surface.rs | 25 + apps/desktop/src-tauri/src/lib.rs | 1 + .../desktop/src/composables/agent/useAgent.ts | 8 +- .../database/artifacts/import/chat_merge.sql | 26 ++ .../artifacts/import/full_postlude.sql | 2 + .../src/database/artifacts/runtime/seed.sql | 6 + .../0003_session_turn_context_artifacts.sql | 20 + .../src/database/drizzle/meta/_journal.json | 7 + apps/desktop/src/database/queries/index.ts | 1 + .../queries/sessionTurnContextArtifacts.ts | 77 ++++ apps/desktop/src/database/schema.ts | 42 ++ apps/desktop/src/database/types/index.ts | 27 ++ apps/desktop/src/i18n/messages.ts | 6 + .../AgentService/execution/executor.ts | 5 + .../AgentService/execution/runtime.ts | 14 + .../AgentService/outputs/persistence.ts | 60 +++ .../services/AgentService/prompt/composer.ts | 15 +- .../src/services/AgentService/prompt/types.ts | 3 + .../services/AgentService/session/history.ts | 84 +++- .../services/AgentService/session/manager.ts | 9 +- .../src/services/AgentService/task/center.ts | 1 + .../src/services/AgentService/task/types.ts | 1 + .../services/BuiltInToolService/registry.ts | 2 + .../services/BuiltInToolService/service.ts | 3 + .../tools/desktopContext/constants.ts | 56 +++ .../tools/desktopContext/index.ts | 81 ++++ .../src/services/BuiltInToolService/types.ts | 3 + .../services/DesktopContextService/index.ts | 44 ++ .../DesktopContextService/toolPayload.ts | 218 +++++++++ .../services/DesktopContextService/types.ts | 137 ++++++ .../src/services/EventService/types.ts | 1 + .../services/NativeService/desktopContext.ts | 22 + .../src/services/NativeService/index.ts | 3 + apps/desktop/src/types/session.ts | 2 + .../components/UserMessage.vue | 49 ++ .../SearchView/composables/useSearchPage.ts | 13 + .../composables/useSearchRequest.ts | 20 +- apps/desktop/src/views/SearchView/index.vue | 5 + apps/desktop/src/views/SearchView/types.ts | 1 + .../components/BuiltInTools/types.ts | 5 + .../AgentService/session-history-i18n.test.ts | 72 +++ .../tool-payload.test.ts | 99 ++++ 46 files changed, 1726 insertions(+), 12 deletions(-) create mode 100644 apps/desktop/src-tauri/src/commands/desktop_context.rs create mode 100644 apps/desktop/src-tauri/src/core/system/desktop_context.rs create mode 100644 apps/desktop/src/database/drizzle/0003_session_turn_context_artifacts.sql create mode 100644 apps/desktop/src/database/queries/sessionTurnContextArtifacts.ts create mode 100644 apps/desktop/src/services/BuiltInToolService/tools/desktopContext/constants.ts create mode 100644 apps/desktop/src/services/BuiltInToolService/tools/desktopContext/index.ts create mode 100644 apps/desktop/src/services/DesktopContextService/index.ts create mode 100644 apps/desktop/src/services/DesktopContextService/toolPayload.ts create mode 100644 apps/desktop/src/services/DesktopContextService/types.ts create mode 100644 apps/desktop/src/services/NativeService/desktopContext.ts create mode 100644 apps/desktop/tests/services/DesktopContextService/tool-payload.test.ts diff --git a/apps/desktop/src-tauri/src/commands/desktop_context.rs b/apps/desktop/src-tauri/src/commands/desktop_context.rs new file mode 100644 index 00000000..e588de99 --- /dev/null +++ b/apps/desktop/src-tauri/src/commands/desktop_context.rs @@ -0,0 +1,25 @@ +// Copyright (c) 2026. Qian Cheng. Licensed under GPL v3. + +//! Desktop context commands. + +use tauri::State; + +use crate::core::system::desktop_context::{ + BoundDesktopContext, DesktopContextCapsule, DesktopContextRuntime, +}; + +#[tauri::command] +pub fn desktop_context_get_capsule( + runtime: State<'_, DesktopContextRuntime>, + capsule_id: String, +) -> Result, String> { + runtime.get_capsule(&capsule_id) +} + +#[tauri::command] +pub fn desktop_context_bind_capsule( + runtime: State<'_, DesktopContextRuntime>, + capsule_id: String, +) -> Result, String> { + runtime.bind_capsule(&capsule_id) +} diff --git a/apps/desktop/src-tauri/src/commands/mod.rs b/apps/desktop/src-tauri/src/commands/mod.rs index 987d6465..503d4ccc 100644 --- a/apps/desktop/src-tauri/src/commands/mod.rs +++ b/apps/desktop/src-tauri/src/commands/mod.rs @@ -5,6 +5,7 @@ pub mod autostart; pub mod built_in_tools; pub mod clipboard; pub mod database; +pub mod desktop_context; pub mod mcp; pub mod paths; pub mod quick_search; @@ -42,6 +43,8 @@ pub fn invoke_handler( clipboard::read_clipboard_payload, clipboard::consume_shortcut_auto_paste_payload, clipboard::write_clipboard_text, + desktop_context::desktop_context_get_capsule, + desktop_context::desktop_context_bind_capsule, autostart::enable_autostart, autostart::disable_autostart, autostart::is_autostart_enabled, diff --git a/apps/desktop/src-tauri/src/core/system/desktop_context.rs b/apps/desktop/src-tauri/src/core/system/desktop_context.rs new file mode 100644 index 00000000..8453a2bf --- /dev/null +++ b/apps/desktop/src-tauri/src/core/system/desktop_context.rs @@ -0,0 +1,433 @@ +// Copyright (c) 2026. Qian Cheng. Licensed under GPL v3. + +//! Read-only desktop context capsules captured at TouchAI invocation time. + +use std::{ + collections::HashMap, + sync::{ + atomic::{AtomicU64, Ordering}, + Mutex, + }, +}; + +use serde::Serialize; +use tauri::{AppHandle, Manager, Runtime}; + +use crate::core::system::clipboard::{ClipboardPayload, ClipboardRuntime}; + +const CLIPBOARD_SUMMARY_LIMIT: usize = 500; + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DesktopContextCapability { + pub id: String, + pub supported: bool, + pub method: String, + pub reason: Option, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DesktopContextBounds { + pub x: i32, + pub y: i32, + pub width: i32, + pub height: i32, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DesktopContextActiveWindow { + pub title: Option, + pub app_name: Option, + pub process_name: Option, + pub process_id: Option, + pub window_handle: Option, + pub bounds: Option, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DesktopContextSelectedText { + pub available: bool, + pub source: Option, + pub text: Option, + pub text_length: usize, + pub truncated: bool, + pub reason: Option, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DesktopContextClipboard { + pub available: bool, + pub snapshot_id: Option, + pub observed_at: Option, + pub text: Option, + pub text_summary: Option, + pub text_length: usize, + pub image_count: usize, + pub file_count: usize, + pub reason: Option, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DesktopContextScreenshot { + pub available: bool, + pub path: Option, + pub mime_type: Option, + pub width: Option, + pub height: Option, + pub target: String, + pub persisted: bool, + pub captured_at: Option, + pub reason: Option, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DesktopContextRedaction { + pub field: String, + pub reason: String, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DesktopContextCapsule { + pub id: String, + pub sequence: u64, + pub captured_at: String, + pub invocation_source: String, + pub platform: String, + pub summary: String, + pub active_window: Option, + pub selected_text: DesktopContextSelectedText, + pub clipboard: DesktopContextClipboard, + pub screenshot: DesktopContextScreenshot, + pub capabilities: Vec, + pub redactions: Vec, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct BoundDesktopContext { + #[serde(flatten)] + pub capsule: DesktopContextCapsule, + pub bound_at: String, +} + +#[derive(Default)] +pub struct DesktopContextRuntime { + next_sequence: AtomicU64, + capsules: Mutex>, +} + +impl DesktopContextRuntime { + pub fn new() -> Self { + Self::default() + } + + pub fn capture_invocation( + &self, + app_handle: &AppHandle, + source: &'static str, + ) -> Result { + let sequence = self.next_sequence.fetch_add(1, Ordering::Relaxed) + 1; + let captured_at = now_rfc3339_millis(); + let active_window = capture_active_window(); + let clipboard = capture_clipboard(app_handle); + let selected_text = unsupported_selected_text(); + let screenshot = unsupported_screenshot(&captured_at); + let capabilities = build_capabilities(&active_window, &clipboard, &screenshot); + let redactions = vec![DesktopContextRedaction { + field: "clipboard.fullText".to_string(), + reason: "Only returned when the model explicitly asks for clipboard.full_text." + .to_string(), + }]; + let summary = build_summary( + active_window.as_ref(), + &selected_text, + &clipboard, + &screenshot, + ); + + let capsule = DesktopContextCapsule { + id: format!("desktop-context-{sequence}"), + sequence, + captured_at, + invocation_source: source.to_string(), + platform: std::env::consts::OS.to_string(), + summary, + active_window, + selected_text, + clipboard, + screenshot, + capabilities, + redactions, + }; + + let mut capsules = self + .capsules + .lock() + .map_err(|error| format!("Desktop context state is poisoned: {error}"))?; + capsules.insert(capsule.id.clone(), capsule.clone()); + Ok(capsule) + } + + pub fn get_capsule(&self, capsule_id: &str) -> Result, String> { + let capsules = self + .capsules + .lock() + .map_err(|error| format!("Desktop context state is poisoned: {error}"))?; + Ok(capsules.get(capsule_id).cloned()) + } + + pub fn bind_capsule(&self, capsule_id: &str) -> Result, String> { + Ok(self + .get_capsule(capsule_id)? + .map(|capsule| BoundDesktopContext { + capsule, + bound_at: now_rfc3339_millis(), + })) + } +} + +fn clipboard_from_payload(payload: ClipboardPayload) -> DesktopContextClipboard { + let text = payload.text; + let text_length = text + .as_ref() + .map(|value| value.chars().count()) + .unwrap_or(0); + let text_summary = text + .as_ref() + .map(|value| value.chars().take(CLIPBOARD_SUMMARY_LIMIT).collect()); + + DesktopContextClipboard { + available: true, + snapshot_id: Some(payload.snapshot_id), + observed_at: Some(payload.observed_at), + text, + text_summary, + text_length, + image_count: payload.image_paths.len(), + file_count: payload.file_paths.len(), + reason: None, + } +} + +fn capture_clipboard(app_handle: &AppHandle) -> DesktopContextClipboard { + let Some(runtime) = app_handle.try_state::() else { + return DesktopContextClipboard { + available: false, + snapshot_id: None, + observed_at: None, + text: None, + text_summary: None, + text_length: 0, + image_count: 0, + file_count: 0, + reason: Some("Clipboard runtime is not initialized.".to_string()), + }; + }; + + match runtime.read_clipboard_payload() { + Ok(Some(payload)) => clipboard_from_payload(payload), + Ok(None) => DesktopContextClipboard { + available: false, + snapshot_id: None, + observed_at: None, + text: None, + text_summary: None, + text_length: 0, + image_count: 0, + file_count: 0, + reason: Some("Clipboard is empty or unsupported.".to_string()), + }, + Err(error) => DesktopContextClipboard { + available: false, + snapshot_id: None, + observed_at: None, + text: None, + text_summary: None, + text_length: 0, + image_count: 0, + file_count: 0, + reason: Some(error), + }, + } +} + +fn unsupported_selected_text() -> DesktopContextSelectedText { + DesktopContextSelectedText { + available: false, + source: None, + text: None, + text_length: 0, + truncated: false, + reason: Some( + "Native selected-text extraction is not enabled in this first desktop-context slice." + .to_string(), + ), + } +} + +fn unsupported_screenshot(captured_at: &str) -> DesktopContextScreenshot { + DesktopContextScreenshot { + available: false, + path: None, + mime_type: None, + width: None, + height: None, + target: "active_display".to_string(), + persisted: false, + captured_at: Some(captured_at.to_string()), + reason: Some( + "Native screenshot persistence is reserved for the follow-up capture backend." + .to_string(), + ), + } +} + +fn build_capabilities( + active_window: &Option, + clipboard: &DesktopContextClipboard, + screenshot: &DesktopContextScreenshot, +) -> Vec { + vec![ + DesktopContextCapability { + id: "active_window".to_string(), + supported: active_window.is_some(), + method: if cfg!(target_os = "windows") { + "win32-foreground-window" + } else { + "unsupported-platform" + } + .to_string(), + reason: active_window + .is_none() + .then(|| "Foreground window metadata is only implemented on Windows.".to_string()), + }, + DesktopContextCapability { + id: "clipboard".to_string(), + supported: clipboard.available, + method: "clipboard-runtime-snapshot".to_string(), + reason: clipboard.reason.clone(), + }, + DesktopContextCapability { + id: "selected_text".to_string(), + supported: false, + method: "pending-native-backend".to_string(), + reason: Some( + "Selected-text extraction requires a dedicated native backend.".to_string(), + ), + }, + DesktopContextCapability { + id: "screenshot".to_string(), + supported: screenshot.available, + method: "pending-capture-backend".to_string(), + reason: screenshot.reason.clone(), + }, + ] +} + +fn build_summary( + active_window: Option<&DesktopContextActiveWindow>, + selected_text: &DesktopContextSelectedText, + clipboard: &DesktopContextClipboard, + screenshot: &DesktopContextScreenshot, +) -> String { + let mut parts = Vec::new(); + if let Some(title) = active_window.and_then(|window| window.title.as_deref()) { + if !title.trim().is_empty() { + parts.push(format!("Active window: {title}")); + } + } + if selected_text.available { + parts.push(format!( + "Selected text length: {}", + selected_text.text_length + )); + } + if clipboard.available { + parts.push(format!( + "Clipboard: {} chars, {} images, {} files", + clipboard.text_length, clipboard.image_count, clipboard.file_count + )); + } + if screenshot.available { + parts.push("Screenshot persisted".to_string()); + } + + if parts.is_empty() { + "Desktop context capsule captured with no supported desktop signals.".to_string() + } else { + parts.join("; ") + } +} + +fn now_rfc3339_millis() -> String { + let now = time::OffsetDateTime::now_utc(); + format!( + "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}.{:03}Z", + now.year(), + u8::from(now.month()), + now.day(), + now.hour(), + now.minute(), + now.second(), + now.millisecond() + ) +} + +#[cfg(target_os = "windows")] +fn capture_active_window() -> Option { + use windows::Win32::{ + Foundation::RECT, + UI::WindowsAndMessaging::{ + GetForegroundWindow, GetWindowRect, GetWindowTextW, GetWindowThreadProcessId, + }, + }; + + unsafe { + let hwnd = GetForegroundWindow(); + if hwnd.0.is_null() { + return None; + } + + let mut title_buffer = [0u16; 512]; + let title_len = GetWindowTextW(hwnd, &mut title_buffer); + let title = (title_len > 0).then(|| { + String::from_utf16_lossy(&title_buffer[..title_len as usize]) + .trim() + .to_string() + }); + + let mut process_id = 0u32; + GetWindowThreadProcessId(hwnd, Some(&mut process_id)); + + let mut rect = RECT::default(); + let bounds = GetWindowRect(hwnd, &mut rect) + .ok() + .map(|_| DesktopContextBounds { + x: rect.left, + y: rect.top, + width: rect.right.saturating_sub(rect.left), + height: rect.bottom.saturating_sub(rect.top), + }); + + Some(DesktopContextActiveWindow { + title, + app_name: None, + process_name: None, + process_id: (process_id > 0).then_some(process_id), + window_handle: Some(format!("0x{:x}", hwnd.0 as usize)), + bounds, + }) + } +} + +#[cfg(not(target_os = "windows"))] +fn capture_active_window() -> Option { + None +} diff --git a/apps/desktop/src-tauri/src/core/system/mod.rs b/apps/desktop/src-tauri/src/core/system/mod.rs index beaf01d8..cb5d21db 100644 --- a/apps/desktop/src-tauri/src/core/system/mod.rs +++ b/apps/desktop/src-tauri/src/core/system/mod.rs @@ -8,6 +8,7 @@ pub mod assets; pub mod autostart; pub mod bundled; pub mod clipboard; +pub mod desktop_context; pub mod logging; pub mod paths; pub mod runtime; diff --git a/apps/desktop/src-tauri/src/core/window/search/surface.rs b/apps/desktop/src-tauri/src/core/window/search/surface.rs index e3e1a9a9..bc290269 100644 --- a/apps/desktop/src-tauri/src/core/window/search/surface.rs +++ b/apps/desktop/src-tauri/src/core/window/search/surface.rs @@ -24,6 +24,14 @@ impl SearchSurfaceShowSource { Self::Notification => "notification", } } + + /// 转成桌面上下文胶囊的调用来源。 + fn as_desktop_context_source(&self) -> &'static str { + match self { + Self::Shortcut => "shortcut", + Self::Notification => "notification", + } + } } /// 搜索窗口组隐藏原因。 @@ -49,6 +57,7 @@ impl SearchSurfaceHideReason { struct SearchSurfaceShownPayload { source: &'static str, sequence: u64, + context_capsule_id: Option, } #[derive(Clone, serde::Serialize)] @@ -318,6 +327,21 @@ pub fn show_surface( let window = app_handle .get_webview_window("main") .ok_or_else(|| "Failed to get main window".to_string())?; + let context_capsule_id = if let Some(desktop_context_runtime) = + app_handle.try_state::() + { + match desktop_context_runtime + .capture_invocation(app_handle, source.as_desktop_context_source()) + { + Ok(capsule) => Some(capsule.id), + Err(error) => { + log::warn!("Failed to capture desktop context capsule: {}", error); + None + } + } + } else { + None + }; super::show_and_activate_search_window(&window)?; app_handle @@ -326,6 +350,7 @@ pub fn show_surface( SearchSurfaceShownPayload { source: source.as_str(), sequence: runtime.next_sequence(), + context_capsule_id, }, ) .map_err(|error| format!("Failed to emit search surface shown event: {}", error)) diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index abc5a880..318bf1f9 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -107,6 +107,7 @@ pub fn run() { ) .manage(PopupRegistry::new()) .manage(core::window::search::surface::SearchSurfaceRuntime::new()) + .manage(core::system::desktop_context::DesktopContextRuntime::new()) .manage(core::window::status_reminder::SessionStatusReminderNotificationRuntime::new()) .manage(core::window::tray::TrayStatusRuntime::new()) .manage(BuiltInProcessExecutionRegistry::new()) diff --git a/apps/desktop/src/composables/agent/useAgent.ts b/apps/desktop/src/composables/agent/useAgent.ts index bb5cb378..9571febb 100644 --- a/apps/desktop/src/composables/agent/useAgent.ts +++ b/apps/desktop/src/composables/agent/useAgent.ts @@ -130,11 +130,13 @@ export function useAgent(options: UseAiRequestOptions = {}) { * 从持久化数据恢复指定会话。 */ async function loadPersistedSession(sessionId: number): Promise { - const { session, messages, turns, attempts, model } = await getSessionData(sessionId); + const { session, messages, turns, attempts, contextArtifacts, model } = + await getSessionData(sessionId); sessionHistory.value = await buildSessionHistoryFromData({ messages, turns, attempts, + contextArtifacts, }); currentSessionId.value = session.id; resetTransientState(); @@ -194,7 +196,8 @@ export function useAgent(options: UseAiRequestOptions = {}) { attachments: Index[] = [], inputSnapshot?: InputHistorySnapshot, modelId?: string, - providerId?: number + providerId?: number, + desktopContextCapsuleId?: string | null ) { if (!prompt.trim() && attachments.length === 0) { error.value = new Error(t('search.error.promptEmpty')); @@ -219,6 +222,7 @@ export function useAgent(options: UseAiRequestOptions = {}) { providerId, attachments, inputSnapshot, + desktopContextCapsuleId, executionMode: 'foreground', signal: startAbortController.signal, }); diff --git a/apps/desktop/src/database/artifacts/import/chat_merge.sql b/apps/desktop/src/database/artifacts/import/chat_merge.sql index a8f48d47..6a4e53c2 100644 --- a/apps/desktop/src/database/artifacts/import/chat_merge.sql +++ b/apps/desktop/src/database/artifacts/import/chat_merge.sql @@ -374,3 +374,29 @@ WHERE NOT EXISTS ( WHERE existing_attempts.turn_id = turn_map.target_turn_id AND existing_attempts.attempt_index = source_attempts.attempt_index ); + +INSERT INTO main.session_turn_context_artifacts ( + turn_id, capsule_id, artifact_kind, artifact_path, mime_type, + width, height, captured_at, metadata_json, created_at +) +SELECT + turn_map.target_turn_id, + source_artifacts.capsule_id, + source_artifacts.artifact_kind, + source_artifacts.artifact_path, + source_artifacts.mime_type, + source_artifacts.width, + source_artifacts.height, + source_artifacts.captured_at, + source_artifacts.metadata_json, + source_artifacts.created_at +FROM imported.session_turn_context_artifacts AS source_artifacts +INNER JOIN temp_turn_map AS turn_map + ON turn_map.source_turn_id = source_artifacts.turn_id +WHERE NOT EXISTS ( + SELECT 1 + FROM main.session_turn_context_artifacts AS existing_artifacts + WHERE existing_artifacts.turn_id = turn_map.target_turn_id + AND existing_artifacts.capsule_id = source_artifacts.capsule_id + AND existing_artifacts.artifact_kind = source_artifacts.artifact_kind +); diff --git a/apps/desktop/src/database/artifacts/import/full_postlude.sql b/apps/desktop/src/database/artifacts/import/full_postlude.sql index 848503b6..11d0a6ee 100644 --- a/apps/desktop/src/database/artifacts/import/full_postlude.sql +++ b/apps/desktop/src/database/artifacts/import/full_postlude.sql @@ -8,6 +8,7 @@ WHERE name IN ( 'message_attachments', 'session_turns', 'session_turn_attempts', + 'session_turn_context_artifacts', 'settings', 'statistics', 'llm_metadata' @@ -21,6 +22,7 @@ INSERT INTO main.sqlite_sequence (name, seq) SELECT 'attachments', COALESCE(MAX( INSERT INTO main.sqlite_sequence (name, seq) SELECT 'message_attachments', COALESCE(MAX(id), 0) FROM main.message_attachments; INSERT INTO main.sqlite_sequence (name, seq) SELECT 'session_turns', COALESCE(MAX(id), 0) FROM main.session_turns; INSERT INTO main.sqlite_sequence (name, seq) SELECT 'session_turn_attempts', COALESCE(MAX(id), 0) FROM main.session_turn_attempts; +INSERT INTO main.sqlite_sequence (name, seq) SELECT 'session_turn_context_artifacts', COALESCE(MAX(id), 0) FROM main.session_turn_context_artifacts; INSERT INTO main.sqlite_sequence (name, seq) SELECT 'settings', COALESCE(MAX(id), 0) FROM main.settings; INSERT INTO main.sqlite_sequence (name, seq) SELECT 'statistics', COALESCE(MAX(id), 0) FROM main.statistics; INSERT INTO main.sqlite_sequence (name, seq) SELECT 'llm_metadata', COALESCE(MAX(id), 0) FROM main.llm_metadata; diff --git a/apps/desktop/src/database/artifacts/runtime/seed.sql b/apps/desktop/src/database/artifacts/runtime/seed.sql index 3fe89d1a..641f7a4a 100644 --- a/apps/desktop/src/database/artifacts/runtime/seed.sql +++ b/apps/desktop/src/database/artifacts/runtime/seed.sql @@ -135,6 +135,12 @@ INSERT INTO built_in_tools ( SELECT 'web_fetch', 'WebFetch', '抓取网页并提取易读文本', 1, 'low', NULL WHERE NOT EXISTS (SELECT 1 FROM built_in_tools WHERE tool_id = 'web_fetch'); +INSERT INTO built_in_tools ( + tool_id, display_name, description, enabled, risk_level, config_json +) +SELECT 'get_desktop_context', 'GetDesktopContext', '读取本轮绑定的只读桌面上下文', 1, 'low', NULL +WHERE NOT EXISTS (SELECT 1 FROM built_in_tools WHERE tool_id = 'get_desktop_context'); + INSERT INTO built_in_tools ( tool_id, display_name, description, enabled, risk_level, config_json ) diff --git a/apps/desktop/src/database/drizzle/0003_session_turn_context_artifacts.sql b/apps/desktop/src/database/drizzle/0003_session_turn_context_artifacts.sql new file mode 100644 index 00000000..4cd27e93 --- /dev/null +++ b/apps/desktop/src/database/drizzle/0003_session_turn_context_artifacts.sql @@ -0,0 +1,20 @@ +CREATE TABLE `session_turn_context_artifacts` ( + `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + `turn_id` integer NOT NULL, + `capsule_id` text NOT NULL, + `artifact_kind` text NOT NULL, + `artifact_path` text, + `mime_type` text, + `width` integer, + `height` integer, + `captured_at` text NOT NULL, + `metadata_json` text, + `created_at` text DEFAULT (datetime('now')) NOT NULL, + FOREIGN KEY (`turn_id`) REFERENCES `session_turns`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE INDEX `session_turn_context_artifacts_turn_id_idx` ON `session_turn_context_artifacts` (`turn_id`); +--> statement-breakpoint +CREATE INDEX `session_turn_context_artifacts_capsule_id_idx` ON `session_turn_context_artifacts` (`capsule_id`); +--> statement-breakpoint +CREATE UNIQUE INDEX `session_turn_context_artifacts_turn_capsule_kind_unique` ON `session_turn_context_artifacts` (`turn_id`,`capsule_id`,`artifact_kind`); diff --git a/apps/desktop/src/database/drizzle/meta/_journal.json b/apps/desktop/src/database/drizzle/meta/_journal.json index aebe4c54..edec3b05 100644 --- a/apps/desktop/src/database/drizzle/meta/_journal.json +++ b/apps/desktop/src/database/drizzle/meta/_journal.json @@ -22,6 +22,13 @@ "when": 1780185600000, "tag": "0002_quick_search_click_stats_unique", "breakpoints": true + }, + { + "idx": 3, + "version": "7", + "when": 1780358400000, + "tag": "0003_session_turn_context_artifacts", + "breakpoints": true } ] } diff --git a/apps/desktop/src/database/queries/index.ts b/apps/desktop/src/database/queries/index.ts index 15814a4f..89402119 100644 --- a/apps/desktop/src/database/queries/index.ts +++ b/apps/desktop/src/database/queries/index.ts @@ -14,6 +14,7 @@ export * from './providers'; export * from './quickSearchClicks'; export * from './sessions'; export * from './sessionTurnAttempts'; +export * from './sessionTurnContextArtifacts'; export * from './sessionTurns'; export * from './settings'; export * from './statistics'; diff --git a/apps/desktop/src/database/queries/sessionTurnContextArtifacts.ts b/apps/desktop/src/database/queries/sessionTurnContextArtifacts.ts new file mode 100644 index 00000000..9e727ad0 --- /dev/null +++ b/apps/desktop/src/database/queries/sessionTurnContextArtifacts.ts @@ -0,0 +1,77 @@ +// Copyright (c) 2026. Qian Cheng. Licensed under GPL v3 + +import { and, asc, eq } from 'drizzle-orm'; + +import { type DatabaseExecutor, db } from '../index'; +import { sessionTurnContextArtifacts, sessionTurns } from '../schema'; +import type { + SessionTurnContextArtifactCreateData, + SessionTurnContextArtifactEntity, +} from '../types'; + +export interface SessionTurnContextArtifactHistoryRow + extends SessionTurnContextArtifactEntity { + prompt_message_id: number | null; +} + +export async function createSessionTurnContextArtifact( + data: SessionTurnContextArtifactCreateData, + database: DatabaseExecutor = db +): Promise { + const row = await database + .insert(sessionTurnContextArtifacts) + .values(data) + .onConflictDoNothing() + .returning() + .get(); + + if (row && row.id !== undefined) { + return row; + } + + const existing = await database + .select() + .from(sessionTurnContextArtifacts) + .where( + and( + eq(sessionTurnContextArtifacts.turn_id, data.turn_id), + eq(sessionTurnContextArtifacts.capsule_id, data.capsule_id), + eq(sessionTurnContextArtifacts.artifact_kind, data.artifact_kind) + ) + ) + .get(); + + if (!existing) { + throw new Error('Failed to create session turn context artifact'); + } + + return existing; +} + +export async function findSessionTurnContextArtifactsBySessionId( + sessionId: number +): Promise { + return db + .select({ + id: sessionTurnContextArtifacts.id, + turn_id: sessionTurnContextArtifacts.turn_id, + prompt_message_id: sessionTurns.prompt_message_id, + capsule_id: sessionTurnContextArtifacts.capsule_id, + artifact_kind: sessionTurnContextArtifacts.artifact_kind, + artifact_path: sessionTurnContextArtifacts.artifact_path, + mime_type: sessionTurnContextArtifacts.mime_type, + width: sessionTurnContextArtifacts.width, + height: sessionTurnContextArtifacts.height, + captured_at: sessionTurnContextArtifacts.captured_at, + metadata_json: sessionTurnContextArtifacts.metadata_json, + created_at: sessionTurnContextArtifacts.created_at, + }) + .from(sessionTurnContextArtifacts) + .innerJoin(sessionTurns, eq(sessionTurns.id, sessionTurnContextArtifacts.turn_id)) + .where(eq(sessionTurns.session_id, sessionId)) + .orderBy( + asc(sessionTurnContextArtifacts.created_at), + asc(sessionTurnContextArtifacts.id) + ) + .all(); +} diff --git a/apps/desktop/src/database/schema.ts b/apps/desktop/src/database/schema.ts index 5f94c259..6ffed55e 100644 --- a/apps/desktop/src/database/schema.ts +++ b/apps/desktop/src/database/schema.ts @@ -370,6 +370,44 @@ export const sessionTurnAttempts = sqliteTable( ] ); +/** + * 会话轮次绑定的桌面上下文工件。 + * + * 这是 read-only desktop context 的专用 provenance 关系，不复用 message attachments， + * 避免把自动捕获的 invocation 截图误表示为用户手动上传附件。 + */ +export const sessionTurnContextArtifacts = sqliteTable( + 'session_turn_context_artifacts', + { + id: integer('id').primaryKey({ autoIncrement: true }), + turn_id: integer('turn_id') + .notNull() + .references(() => sessionTurns.id, { onDelete: 'cascade' }), + capsule_id: text('capsule_id').notNull(), + artifact_kind: text('artifact_kind', { + enum: ['screenshot', 'metadata'], + }).notNull(), + artifact_path: text('artifact_path'), + mime_type: text('mime_type'), + width: integer('width'), + height: integer('height'), + captured_at: text('captured_at').notNull(), + metadata_json: text('metadata_json'), + created_at: text('created_at') + .notNull() + .default(sql`(datetime('now'))`), + }, + (table) => [ + index('session_turn_context_artifacts_turn_id_idx').on(table.turn_id), + index('session_turn_context_artifacts_capsule_id_idx').on(table.capsule_id), + uniqueIndex('session_turn_context_artifacts_turn_capsule_kind_unique').on( + table.turn_id, + table.capsule_id, + table.artifact_kind + ), + ] +); + /** * LLM 元数据表 */ @@ -601,6 +639,10 @@ export type SessionTurnAttempt = typeof sessionTurnAttempts.$inferSelect; export type NewSessionTurnAttempt = typeof sessionTurnAttempts.$inferInsert; export type SessionTurnAttemptUpdate = Partial; +export type SessionTurnContextArtifact = typeof sessionTurnContextArtifacts.$inferSelect; +export type NewSessionTurnContextArtifact = typeof sessionTurnContextArtifacts.$inferInsert; +export type SessionTurnContextArtifactUpdate = Partial; + export type LlmMetadata = typeof llmMetadata.$inferSelect; export type NewLlmMetadata = typeof llmMetadata.$inferInsert; export type LlmMetadataUpdate = Partial; diff --git a/apps/desktop/src/database/types/index.ts b/apps/desktop/src/database/types/index.ts index a8299e8f..eca81d23 100644 --- a/apps/desktop/src/database/types/index.ts +++ b/apps/desktop/src/database/types/index.ts @@ -360,6 +360,33 @@ export interface SessionTurnAttemptCreateData { export type SessionTurnAttemptUpdateData = Partial; +export interface SessionTurnContextArtifactEntity { + id: number; + turn_id: number; + capsule_id: string; + artifact_kind: 'screenshot' | 'metadata'; + artifact_path: string | null; + mime_type: string | null; + width: number | null; + height: number | null; + captured_at: string; + metadata_json: string | null; + created_at: string; +} + +export interface SessionTurnContextArtifactCreateData { + turn_id: number; + capsule_id: string; + artifact_kind: 'screenshot' | 'metadata'; + artifact_path?: string | null; + mime_type?: string | null; + width?: number | null; + height?: number | null; + captured_at: string; + metadata_json?: string | null; + created_at?: string; +} + // ==================== LLM 元数据 ==================== export interface LlmMetadataEntity { diff --git a/apps/desktop/src/i18n/messages.ts b/apps/desktop/src/i18n/messages.ts index ce9698ff..16d6a783 100644 --- a/apps/desktop/src/i18n/messages.ts +++ b/apps/desktop/src/i18n/messages.ts @@ -355,6 +355,7 @@ const zhCNMessages = { 'settings.builtInTools.summary.read': '读取本地文件或目录，支持图片与 PDF', 'settings.builtInTools.summary.setting': '读取和修改应用设置', 'settings.builtInTools.summary.webFetch': '抓取网页并提取易读文本', + 'settings.builtInTools.summary.desktopContext': '读取本轮绑定的只读桌面上下文', 'settings.builtInTools.summary.upgradeModel': '升级当前请求模型', 'settings.builtInTools.summary.showWidget': '聊天内联可交互可视化', 'settings.builtInTools.summary.visualizeReadMe': '读取 ShowWidget 规范', @@ -562,6 +563,8 @@ const zhCNMessages = { 'conversation.attachment.unsupportedFileSuffix': '（当前模型不支持文件）', 'conversation.attachment.unnamedImage': '未命名图片', 'conversation.attachment.unnamedFile': '未命名文件', + 'conversation.desktopContext.title': '桌面上下文', + 'conversation.desktopContext.screenshotAlt': '呼出时桌面截图', 'conversation.approval.defaultTitle': '需要确认', 'conversation.approval.commandExecutionTitle': '命令执行确认', 'conversation.approval.readLocalContentTitle': '读取本地内容确认', @@ -1117,6 +1120,7 @@ const enUSMessages: Record = { 'settings.builtInTools.summary.read': 'Read local files or folders, including images and PDFs', 'settings.builtInTools.summary.setting': 'Read and modify application settings', 'settings.builtInTools.summary.webFetch': 'Fetch web pages and extract readable text', + 'settings.builtInTools.summary.desktopContext': 'Read the bound read-only desktop context', 'settings.builtInTools.summary.upgradeModel': 'Upgrade the current request model', 'settings.builtInTools.summary.showWidget': 'Inline interactive visualization in chat', 'settings.builtInTools.summary.visualizeReadMe': 'Read the ShowWidget specification', @@ -1330,6 +1334,8 @@ const enUSMessages: Record = { 'conversation.attachment.unsupportedFileSuffix': ' (current model does not support files)', 'conversation.attachment.unnamedImage': 'Image', 'conversation.attachment.unnamedFile': 'File', + 'conversation.desktopContext.title': 'Desktop context', + 'conversation.desktopContext.screenshotAlt': 'Desktop screenshot at invocation', 'conversation.approval.defaultTitle': 'Confirmation required', 'conversation.approval.commandExecutionTitle': 'Confirm command execution', 'conversation.approval.readLocalContentTitle': 'Confirm local content read', diff --git a/apps/desktop/src/services/AgentService/execution/executor.ts b/apps/desktop/src/services/AgentService/execution/executor.ts index 33d8ac23..e1727eda 100644 --- a/apps/desktop/src/services/AgentService/execution/executor.ts +++ b/apps/desktop/src/services/AgentService/execution/executor.ts @@ -10,6 +10,7 @@ import { type BuiltInToolId, builtInToolService, } from '@/services/BuiltInToolService'; +import type { BoundDesktopContext } from '@/services/DesktopContextService/types'; import { z } from '@/utils/zod'; import { createProviderForModel, getModel, resolveToolDefinitions } from '../catalog'; @@ -72,6 +73,7 @@ export interface RequestExecutionCallbacks { callId: string, questions: AskUserQuestion[] ) => Promise; + desktopContext?: BoundDesktopContext | null; } export interface AttemptCheckpoint { @@ -724,6 +726,7 @@ export class AiRequestExecutor { sessionId: options.persister.getSessionId(), requestToolApproval: options.requestToolApproval, requestUserQuestions: options.requestUserQuestions, + desktopContext: options.desktopContext ?? null, emitToolEvent: (toolEvent) => this.emitToolEvent(options.onChunk, toolEvent), }); @@ -777,6 +780,7 @@ export class AiRequestExecutor { onChunk: options.onChunk, requestToolApproval: options.requestToolApproval, requestUserQuestions: options.requestUserQuestions, + desktopContext: options.desktopContext ?? null, }) ) ); @@ -888,6 +892,7 @@ export class AiRequestExecutor { onChunk: options.onChunk, requestToolApproval: options.requestToolApproval, requestUserQuestions: options.requestUserQuestions, + desktopContext: options.desktopContext ?? null, }); runtime.iteration += 1; diff --git a/apps/desktop/src/services/AgentService/execution/runtime.ts b/apps/desktop/src/services/AgentService/execution/runtime.ts index 412b74d6..f868cc1e 100644 --- a/apps/desktop/src/services/AgentService/execution/runtime.ts +++ b/apps/desktop/src/services/AgentService/execution/runtime.ts @@ -6,6 +6,11 @@ import type { SessionTurnEntity } from '@database/types'; import { t } from '@/i18n'; import type { AttachmentIndex } from '@/services/AgentService/infrastructure/attachments'; import { ensurePersistedAttachmentIndex } from '@/services/AgentService/infrastructure/attachments'; +import { + buildDesktopContextPromptMetadata, + desktopContextService, + type BoundDesktopContext, +} from '@/services/DesktopContextService'; import type { InputHistorySnapshot } from '@/types/session'; import { AiError, AiErrorCode } from '../contracts/errors'; @@ -136,6 +141,7 @@ export interface ExecuteRequestOptions extends RequestExecutionCallbacks { inputSnapshot?: InputHistorySnapshot; executionMode?: TaskExecutionMode; promptSnapshot?: PromptSnapshot; + desktopContextCapsuleId?: string | null; environment?: ConversationRuntimeEnvironment; } @@ -150,6 +156,7 @@ interface RuntimeContext { initialCheckpoint: AttemptCheckpoint; persister: PersistenceProjector; requestStartRecordPromise: Promise; + desktopContext: BoundDesktopContext | null; } /** @@ -225,6 +232,9 @@ export class AiConversationRuntime { if (attachments.length > 0) { await this.prepareAttachmentsForTransport(attachments); } + const desktopContext = await desktopContextService.bindCapsule( + this.options.desktopContextCapsuleId + ); // prompt 快照在整个 turn 生命周期内只生成一次。 // 后续 retry、tool iteration、checkpoint resume 都必须复用它。 const promptSnapshot = @@ -234,6 +244,7 @@ export class AiConversationRuntime { attachments, executionMode: this.options.executionMode ?? 'foreground', inputSnapshot: this.options.inputSnapshot, + desktopContext: buildDesktopContextPromptMetadata(desktopContext), })); const modelLanguageContext = promptSnapshot.modelLanguageContext ?? getCurrentModelLanguageContext(); @@ -265,6 +276,7 @@ export class AiConversationRuntime { taskId: this.options.taskId ?? crypto.randomUUID(), executionMode: this.options.executionMode ?? 'foreground', promptSnapshot, + desktopContext, maxRetries: MAX_REQUEST_RETRIES, buildSessionTitle, }); @@ -298,6 +310,7 @@ export class AiConversationRuntime { initialCheckpoint, persister, requestStartRecordPromise, + desktopContext, }; } @@ -420,6 +433,7 @@ export class AiConversationRuntime { onTurnEvent: this.options.onTurnEvent, requestToolApproval: this.options.requestToolApproval, requestUserQuestions: this.options.requestUserQuestions, + desktopContext: context.desktopContext, }); await context.requestStartRecordPromise; diff --git a/apps/desktop/src/services/AgentService/outputs/persistence.ts b/apps/desktop/src/services/AgentService/outputs/persistence.ts index 659f413e..cefcbd4a 100644 --- a/apps/desktop/src/services/AgentService/outputs/persistence.ts +++ b/apps/desktop/src/services/AgentService/outputs/persistence.ts @@ -7,6 +7,7 @@ import { createSession, createSessionTurn, createSessionTurnAttempt, + createSessionTurnContextArtifact, refreshSessionMetadata, updateSession, updateSessionTurn, @@ -27,6 +28,7 @@ import { serializeAttachmentDeliveryManifest, upsertAttachmentDeliveryManifestRequest, } from '@/services/AgentService/infrastructure/attachments'; +import type { BoundDesktopContext } from '@/services/DesktopContextService/types'; import { toDbTimestamp } from '@/utils/date'; import type { @@ -68,6 +70,7 @@ interface PersistenceProjectorOptions { taskId: string; executionMode: TaskExecutionMode; promptSnapshot: PromptSnapshot; + desktopContext?: BoundDesktopContext | null; deliveryManifest?: AttachmentDeliveryManifest; maxRetries: number; buildSessionTitle: (prompt: string) => string; @@ -230,6 +233,7 @@ export class PersistenceProjector { private readonly taskId: string; private readonly executionMode: TaskExecutionMode; private readonly promptSnapshot: PromptSnapshot; + private readonly desktopContext: BoundDesktopContext | null; private deliveryManifest: AttachmentDeliveryManifest; private readonly maxRetries: number; private readonly buildSessionTitle: (prompt: string) => string; @@ -250,6 +254,7 @@ export class PersistenceProjector { this.taskId = options.taskId; this.executionMode = options.executionMode; this.promptSnapshot = options.promptSnapshot; + this.desktopContext = options.desktopContext ?? null; this.deliveryManifest = options.deliveryManifest ?? createEmptyAttachmentDeliveryManifest(); this.maxRetries = options.maxRetries; this.buildSessionTitle = options.buildSessionTitle; @@ -293,6 +298,7 @@ export class PersistenceProjector { sessionId )); const turn = await this.ensureTurnRecord(tx, sessionId, userMessageId); + await this.persistDesktopContextArtifacts(turn.id, tx); const attemptState = await this.ensureAttemptRecord(1, initialCheckpoint, tx, turn.id); return { sessionId, userMessageId, turn, attemptState }; @@ -729,6 +735,60 @@ export class PersistenceProjector { return turn; } + private async persistDesktopContextArtifacts( + turnId: number, + database: DatabaseExecutor = db + ): Promise { + const context = this.desktopContext; + if (!context) { + return; + } + + await createSessionTurnContextArtifact( + { + turn_id: turnId, + capsule_id: context.id, + artifact_kind: 'metadata', + captured_at: context.capturedAt, + metadata_json: JSON.stringify({ + capsuleId: context.id, + capturedAt: context.capturedAt, + boundAt: context.boundAt, + summary: context.summary, + activeWindowTitle: context.activeWindow?.title ?? null, + activeWindowAppName: context.activeWindow?.appName ?? null, + selectedTextLength: context.selectedText.textLength, + clipboardTextLength: context.clipboard.textLength, + capabilities: context.capabilities, + redactions: context.redactions, + }), + }, + database + ); + + if (!context.screenshot.available || !context.screenshot.path) { + return; + } + + await createSessionTurnContextArtifact( + { + turn_id: turnId, + capsule_id: context.id, + artifact_kind: 'screenshot', + artifact_path: context.screenshot.path, + mime_type: context.screenshot.mimeType, + width: context.screenshot.width, + height: context.screenshot.height, + captured_at: context.screenshot.capturedAt ?? context.capturedAt, + metadata_json: JSON.stringify({ + target: context.screenshot.target, + persisted: context.screenshot.persisted, + }), + }, + database + ); + } + private async ensureAttemptRecord( attemptIndex: number, checkpoint: AttemptCheckpoint, diff --git a/apps/desktop/src/services/AgentService/prompt/composer.ts b/apps/desktop/src/services/AgentService/prompt/composer.ts index a1f90014..d9ae80b4 100644 --- a/apps/desktop/src/services/AgentService/prompt/composer.ts +++ b/apps/desktop/src/services/AgentService/prompt/composer.ts @@ -4,6 +4,7 @@ import { type AttachmentIndex, inspectAttachments, } from '@/services/AgentService/infrastructure/attachments'; +import type { DesktopContextPromptMetadata } from '@/services/DesktopContextService/types'; import type { InputHistorySnapshot } from '@/types/session'; import { @@ -26,6 +27,12 @@ const BACKGROUND_MODE_PROMPT = [ '除非用户主动取消，或工具审批明确被拒绝，否则应继续完成原始任务。', ].join('\n'); +const DESKTOP_CONTEXT_TOOL_PROMPT = [ + '本轮请求绑定了一份只读桌面上下文胶囊。', + '如果用户的问题需要理解呼出 TouchAI 前的桌面、前台窗口、选中文本、剪贴板或截图，请调用 `builtin__get_desktop_context` 读取。', + '不要假设桌面上下文已经完整出现在 prompt 中；不要执行点击、输入、滚动、聚焦或控制外部应用等 computer use 行为。', +].join('\n'); + const PROMPT_SOURCE_ORDER: PromptFragmentSource[] = [ 'override', 'platform', @@ -50,6 +57,7 @@ interface ComposePromptSnapshotOptions { mode?: string[]; feature?: string[]; userAppend?: string[]; + desktopContext?: DesktopContextPromptMetadata; } function buildFragments(source: PromptFragmentSource, contents: string[]): PromptFragment[] { @@ -101,7 +109,10 @@ async function buildPromptAssembly(options: ComposePromptSnapshotOptions): Promi 'mode', options.mode ?? (executionMode === 'background' ? [BACKGROUND_MODE_PROMPT] : []) ), - feature: buildFragments('feature', options.feature ?? []), + feature: buildFragments('feature', [ + ...(options.feature ?? []), + ...(options.desktopContext ? [DESKTOP_CONTEXT_TOOL_PROMPT] : []), + ]), user_append: buildFragments('user_append', options.userAppend ?? []), }; @@ -111,6 +122,7 @@ async function buildPromptAssembly(options: ComposePromptSnapshotOptions): Promi fragments: PROMPT_SOURCE_ORDER.flatMap((source) => fragmentsBySource[source]), userPrompt: options.prompt, attachments: await summarizeAttachments(options.attachments ?? []), + ...(options.desktopContext ? { desktopContext: options.desktopContext } : {}), }; } @@ -130,6 +142,7 @@ export async function composePromptSnapshot( fragments: assembly.fragments, userPrompt: assembly.userPrompt, attachments: assembly.attachments, + ...(assembly.desktopContext ? { desktopContext: assembly.desktopContext } : {}), systemPrompt: assembly.fragments .map((fragment) => fragment.content) .join('\n\n') diff --git a/apps/desktop/src/services/AgentService/prompt/types.ts b/apps/desktop/src/services/AgentService/prompt/types.ts index fbb52451..ad935494 100644 --- a/apps/desktop/src/services/AgentService/prompt/types.ts +++ b/apps/desktop/src/services/AgentService/prompt/types.ts @@ -3,6 +3,7 @@ import type { JSONContent } from '@tiptap/core'; import type { AttachmentIndex } from '@/services/AgentService/infrastructure/attachments'; +import type { DesktopContextPromptMetadata } from '@/services/DesktopContextService/types'; import type { AttachmentDerivedKind, AttachmentSemanticIntent } from '../contracts/protocol'; import type { ModelLanguageContext } from '../languageContext'; @@ -60,6 +61,7 @@ export interface PromptAssembly { fragments: PromptFragment[]; userPrompt: string; attachments: PromptSnapshotAttachmentSummary[]; + desktopContext?: DesktopContextPromptMetadata; } export interface PromptInputSnapshot { @@ -77,4 +79,5 @@ export interface PromptSnapshot extends PromptAssembly { createdAt: string; systemPrompt: string; inputSnapshot?: PromptInputSnapshot; + desktopContext?: DesktopContextPromptMetadata; } diff --git a/apps/desktop/src/services/AgentService/session/history.ts b/apps/desktop/src/services/AgentService/session/history.ts index 770cab25..6cae2d72 100644 --- a/apps/desktop/src/services/AgentService/session/history.ts +++ b/apps/desktop/src/services/AgentService/session/history.ts @@ -2,6 +2,7 @@ import { findAllMcpServers } from '@database/queries/mcpServers'; import type { MessageRow } from '@database/queries/messages'; +import type { SessionTurnContextArtifactHistoryRow } from '@database/queries/sessionTurnContextArtifacts'; import type { SessionTurnAttemptHistoryRow } from '@database/queries/sessionTurnAttempts'; import type { SessionTurnHistoryRow } from '@database/queries/sessionTurns'; import type { PersistedToolLogStatus } from '@database/schema'; @@ -16,6 +17,7 @@ import { SHOW_WIDGET_TOOL_NAME, type ShowWidgetPayload, } from '@/services/BuiltInToolService/tools/widgetTool'; +import type { UserMessageDesktopContext } from '@/services/DesktopContextService/types'; import { createInputHistorySnapshot, type SessionMessage, @@ -66,10 +68,21 @@ export interface BuildSessionHistoryOptions { messages: MessageRow[]; turns: SessionTurnHistoryRow[]; attempts: SessionTurnAttemptHistoryRow[]; + contextArtifacts?: SessionTurnContextArtifactHistoryRow[]; resolveServerName: (serverId: number | null) => string; } -type SessionHistorySourceData = Pick; +type SessionHistorySourceData = Pick< + SessionData, + 'messages' | 'turns' | 'attempts' | 'contextArtifacts' +>; + +interface DesktopContextMetadataArtifact { + capsuleId?: string; + capturedAt?: string; + summary?: string; + activeWindowTitle?: string | null; +} function normalizeDisplayName(namespacedName: string): string { const match = namespacedName.match(/^mcp__\d+__(.+)$/); @@ -133,6 +146,61 @@ function isCancellationStatusText(text?: string | null): boolean { normalized.includes('request cancelled') ); } + +function parseDesktopContextMetadata( + metadataJson: string | null +): DesktopContextMetadataArtifact { + if (!metadataJson) { + return {}; + } + + try { + const parsed = JSON.parse(metadataJson) as DesktopContextMetadataArtifact; + return parsed && typeof parsed === 'object' ? parsed : {}; + } catch (error) { + console.error('[SessionHistory] Failed to parse desktop context metadata JSON:', error); + return {}; + } +} + +function buildDesktopContextsByPromptMessageId( + artifacts: SessionTurnContextArtifactHistoryRow[] = [] +): Map { + const byPromptMessageId = new Map(); + for (const artifact of artifacts) { + if (artifact.prompt_message_id === null) { + continue; + } + + const existing = byPromptMessageId.get(artifact.prompt_message_id) ?? []; + existing.push(artifact); + byPromptMessageId.set(artifact.prompt_message_id, existing); + } + + const contexts = new Map(); + for (const [promptMessageId, rows] of byPromptMessageId.entries()) { + const metadataRow = rows.find((row) => row.artifact_kind === 'metadata'); + const screenshotRow = rows.find((row) => row.artifact_kind === 'screenshot'); + const metadata = parseDesktopContextMetadata(metadataRow?.metadata_json ?? null); + const fallbackRow = metadataRow ?? screenshotRow; + if (!fallbackRow) { + continue; + } + + contexts.set(promptMessageId, { + capsuleId: metadata.capsuleId ?? fallbackRow.capsule_id, + capturedAt: metadata.capturedAt ?? fallbackRow.captured_at, + summary: metadata.summary ?? tt('已绑定桌面上下文'), + activeWindowTitle: metadata.activeWindowTitle ?? null, + screenshotPath: screenshotRow?.artifact_path ?? null, + screenshotMimeType: screenshotRow?.mime_type ?? null, + screenshotWidth: screenshotRow?.width ?? null, + screenshotHeight: screenshotRow?.height ?? null, + }); + } + + return contexts; +} function assertUnreachablePersistedToolStatus(value: never): never { throw new Error(`Unexpected persisted tool status: ${String(value)}`); } @@ -294,7 +362,8 @@ async function buildPersistedEntries( */ function convertEntriesToSessionHistory( entries: PersistedHistoryEntry[], - promptSnapshotsByMessageId: Map + promptSnapshotsByMessageId: Map, + desktopContextsByMessageId: Map ): SessionHistoryBuildResult { const history: SessionMessage[] = []; const historyIndexByPersistedMessageId = new Map(); @@ -475,6 +544,7 @@ function convertEntriesToSessionHistory( editorDoc: promptSnapshot?.inputSnapshot?.editorDoc, excludeFromHistory: promptSnapshot?.inputSnapshot?.excludeFromHistory, }), + desktopContext: desktopContextsByMessageId.get(entry.id), parts: [], timestamp: parseDbDateTimestamp(entry.created_at), }); @@ -860,8 +930,9 @@ function injectDerivedRequestStatuses( export async function buildSessionHistory( options: BuildSessionHistoryOptions ): Promise { - const { messages, turns, attempts, resolveServerName } = options; + const { messages, turns, attempts, contextArtifacts, resolveServerName } = options; const promptSnapshotsByMessageId = new Map(); + const desktopContextsByMessageId = buildDesktopContextsByPromptMessageId(contextArtifacts); for (const turn of turns) { if (turn.prompt_message_id === null) { continue; @@ -878,7 +949,8 @@ export async function buildSessionHistory( return injectDerivedRequestStatuses( convertEntriesToSessionHistory( await buildPersistedEntries(messages, resolveServerName), - promptSnapshotsByMessageId + promptSnapshotsByMessageId, + desktopContextsByMessageId ), turns, attempts @@ -898,6 +970,7 @@ export async function buildSessionHistoryFromData( messages: data.messages, turns: data.turns, attempts: data.attempts, + contextArtifacts: data.contextArtifacts, resolveServerName: (serverId) => { if (serverId === null) { return ''; @@ -912,10 +985,11 @@ export async function buildSessionHistoryFromData( * 按会话主键直接加载页面历史。 */ export async function loadSessionHistory(sessionId: number): Promise { - const { messages, turns, attempts } = await getSessionData(sessionId); + const { messages, turns, attempts, contextArtifacts } = await getSessionData(sessionId); return buildSessionHistoryFromData({ messages, turns, attempts, + contextArtifacts, }); } diff --git a/apps/desktop/src/services/AgentService/session/manager.ts b/apps/desktop/src/services/AgentService/session/manager.ts index 65fb7f7f..0dcf87c1 100644 --- a/apps/desktop/src/services/AgentService/session/manager.ts +++ b/apps/desktop/src/services/AgentService/session/manager.ts @@ -14,6 +14,10 @@ import { findSessionTurnAttemptsBySessionId, type SessionTurnAttemptHistoryRow, } from '@database/queries/sessionTurnAttempts'; +import { + findSessionTurnContextArtifactsBySessionId, + type SessionTurnContextArtifactHistoryRow, +} from '@database/queries/sessionTurnContextArtifacts'; import { findLatestModelBySessionId, findSessionTurnsBySessionId, @@ -26,6 +30,7 @@ export interface SessionData { messages: MessageRow[]; turns: SessionTurnHistoryRow[]; attempts: SessionTurnAttemptHistoryRow[]; + contextArtifacts: SessionTurnContextArtifactHistoryRow[]; model: ModelWithProvider | null; } @@ -70,10 +75,11 @@ export async function getSessionData(sessionId: number): Promise { throw new Error(`Session ${sessionId} not found`); } - const [messages, turns, attempts, model] = await Promise.all([ + const [messages, turns, attempts, contextArtifacts, model] = await Promise.all([ findMessagesBySessionId(sessionId), findSessionTurnsBySessionId(sessionId), findSessionTurnAttemptsBySessionId(sessionId), + findSessionTurnContextArtifactsBySessionId(sessionId), findLatestModelBySessionId({ sessionId }), ]); @@ -82,6 +88,7 @@ export async function getSessionData(sessionId: number): Promise { messages, turns, attempts, + contextArtifacts, model, }; } diff --git a/apps/desktop/src/services/AgentService/task/center.ts b/apps/desktop/src/services/AgentService/task/center.ts index 5bedc233..9ddf1541 100644 --- a/apps/desktop/src/services/AgentService/task/center.ts +++ b/apps/desktop/src/services/AgentService/task/center.ts @@ -329,6 +329,7 @@ class SessionTaskCenter { providerId: options.providerId, attachments: options.attachments, inputSnapshot: options.inputSnapshot, + desktopContextCapsuleId: options.desktopContextCapsuleId, executionMode: options.executionMode ?? 'foreground', environment: runtimeEnvironmentResult.value, signal: abortController.signal, diff --git a/apps/desktop/src/services/AgentService/task/types.ts b/apps/desktop/src/services/AgentService/task/types.ts index f1565c8c..fb112401 100644 --- a/apps/desktop/src/services/AgentService/task/types.ts +++ b/apps/desktop/src/services/AgentService/task/types.ts @@ -56,6 +56,7 @@ export interface StartSessionTaskOptions { providerId?: number; attachments?: AttachmentIndex[]; inputSnapshot?: InputHistorySnapshot; + desktopContextCapsuleId?: string | null; executionMode?: TaskExecutionMode; signal?: AbortSignal; } diff --git a/apps/desktop/src/services/BuiltInToolService/registry.ts b/apps/desktop/src/services/BuiltInToolService/registry.ts index efd4eb2d..21a2a437 100644 --- a/apps/desktop/src/services/BuiltInToolService/registry.ts +++ b/apps/desktop/src/services/BuiltInToolService/registry.ts @@ -2,6 +2,7 @@ import { builtInTools as askUserTools } from './tools/askUser'; import { builtInTools as bashTools } from './tools/bash'; +import { builtInTools as desktopContextTools } from './tools/desktopContext'; import { builtInTools as fileSearchTools } from './tools/fileSearch'; import { builtInTools as readTools } from './tools/read'; import { builtInTools as settingTools } from './tools/setting'; @@ -55,6 +56,7 @@ export const builtInToolRegistry = new BuiltInToolRegistry(); builtInToolRegistry.register(askUserTools); builtInToolRegistry.register(bashTools); +builtInToolRegistry.register(desktopContextTools); builtInToolRegistry.register(fileSearchTools); builtInToolRegistry.register(readTools); builtInToolRegistry.register(settingTools); diff --git a/apps/desktop/src/services/BuiltInToolService/service.ts b/apps/desktop/src/services/BuiltInToolService/service.ts index 0d0a0995..ffbf75b3 100644 --- a/apps/desktop/src/services/BuiltInToolService/service.ts +++ b/apps/desktop/src/services/BuiltInToolService/service.ts @@ -21,6 +21,7 @@ import type { ToolApprovalRequest, ToolEvent, } from '@/services/AgentService/contracts/tooling'; +import type { BoundDesktopContext } from '@/services/DesktopContextService/types'; import { builtInToolRegistry } from './registry'; import type { @@ -40,6 +41,7 @@ interface BuiltInToolExecutionOptions { iteration: number; currentModel: ModelWithProvider; hasExecutedBuiltInTool: (toolId: BuiltInToolId) => boolean; + desktopContext?: BoundDesktopContext | null; signal?: AbortSignal; sessionId: number | null; toolCallMessageId: number | null; @@ -256,6 +258,7 @@ class BuiltInToolService { signal: options.signal, emitToolEvent: options.emitToolEvent, hasExecutedBuiltInTool: options.hasExecutedBuiltInTool, + desktopContext: options.desktopContext ?? null, requestUserQuestions: options.requestUserQuestions, }; diff --git a/apps/desktop/src/services/BuiltInToolService/tools/desktopContext/constants.ts b/apps/desktop/src/services/BuiltInToolService/tools/desktopContext/constants.ts new file mode 100644 index 00000000..49369f1c --- /dev/null +++ b/apps/desktop/src/services/BuiltInToolService/tools/desktopContext/constants.ts @@ -0,0 +1,56 @@ +// Copyright (c) 2026. Qian Cheng. Licensed under GPL v3 + +import type { AiToolDefinition } from '@/services/AgentService/contracts/tooling'; + +export const DESKTOP_CONTEXT_TOOL_NAME = 'GetDesktopContext'; + +export const DESKTOP_CONTEXT_TOOL_DESCRIPTION = [ + 'Read the immutable desktop context capsule bound to the current user turn.', + 'This is read-only context, not computer use: the tool cannot click, type, focus, scroll, or control apps.', + 'Use include as an extensible array. Sensitive fields such as clipboard full text and screenshot image path are returned only when explicitly requested.', +].join(' '); + +export const DESKTOP_CONTEXT_INCLUDE_VALUES = [ + 'summary', + 'active_window', + 'selected_text.summary', + 'selected_text.full_text', + 'clipboard.summary', + 'clipboard.full_text', + 'screenshot.metadata', + 'screenshot.image', + 'capabilities', + 'redactions', +] as const; + +export const DESKTOP_CONTEXT_TOOL_INPUT_SCHEMA: AiToolDefinition['input_schema'] = { + type: 'object', + properties: { + scope: { + type: 'string', + enum: ['current', 'previous', 'recent', 'diff'], + description: + 'Desktop context scope. Phase 1 supports the current bound turn capsule; other scopes return current-compatible metadata until recent capsule history is enabled.', + }, + limit: { + type: 'number', + description: 'Maximum number of capsules to return for recent scope. Reserved for later.', + }, + include: { + type: 'array', + items: { + type: 'string', + enum: [...DESKTOP_CONTEXT_INCLUDE_VALUES], + }, + description: + 'Fields to include. Defaults to summary, active_window, selected_text.full_text, clipboard.summary, capabilities, and redactions.', + }, + screenshotTarget: { + type: 'string', + enum: ['capsule_default', 'active_window', 'active_display', 'all_displays'], + description: + 'Requested screenshot target. Phase 1 returns the capsule default screenshot metadata or path.', + }, + }, + additionalProperties: false, +}; diff --git a/apps/desktop/src/services/BuiltInToolService/tools/desktopContext/index.ts b/apps/desktop/src/services/BuiltInToolService/tools/desktopContext/index.ts new file mode 100644 index 00000000..8ac9a9e4 --- /dev/null +++ b/apps/desktop/src/services/BuiltInToolService/tools/desktopContext/index.ts @@ -0,0 +1,81 @@ +// Copyright (c) 2026. Qian Cheng. Licensed under GPL v3 + +import { + buildDesktopContextToolPayload, +} from '@/services/DesktopContextService/toolPayload'; +import type { + DesktopContextInclude, + DesktopContextToolRequest, +} from '@/services/DesktopContextService/types'; + +import { + type BaseBuiltInToolExecutionContext, + BuiltInTool, + type BuiltInToolConversationSemantic, + type BuiltInToolExecutionResult, + type BuiltInToolGroup, +} from '../../types'; +import { + DESKTOP_CONTEXT_INCLUDE_VALUES, + DESKTOP_CONTEXT_TOOL_DESCRIPTION, + DESKTOP_CONTEXT_TOOL_INPUT_SCHEMA, + DESKTOP_CONTEXT_TOOL_NAME, +} from './constants'; + +const includeValues = new Set(DESKTOP_CONTEXT_INCLUDE_VALUES); + +function parseDesktopContextRequest(args: Record): DesktopContextToolRequest { + const include = Array.isArray(args.include) + ? args.include.filter((item): item is DesktopContextInclude => { + return typeof item === 'string' && includeValues.has(item); + }) + : undefined; + + return { + ...(include ? { include } : {}), + ...(typeof args.scope === 'string' + ? { scope: args.scope as DesktopContextToolRequest['scope'] } + : {}), + ...(typeof args.limit === 'number' ? { limit: args.limit } : {}), + ...(typeof args.screenshotTarget === 'string' + ? { + screenshotTarget: + args.screenshotTarget as DesktopContextToolRequest['screenshotTarget'], + } + : {}), + }; +} + +class DesktopContextTool extends BuiltInTool> { + readonly id = 'get_desktop_context' as const; + readonly displayName = DESKTOP_CONTEXT_TOOL_NAME; + readonly description = DESKTOP_CONTEXT_TOOL_DESCRIPTION; + readonly inputSchema = DESKTOP_CONTEXT_TOOL_INPUT_SCHEMA; + readonly defaultConfig = {}; + + override buildConversationSemantic(): BuiltInToolConversationSemantic { + return { + action: 'review', + target: '桌面上下文', + }; + } + + override async execute( + args: Record, + _config: Record, + context: BaseBuiltInToolExecutionContext + ): Promise { + const request = parseDesktopContextRequest(args); + const payload = buildDesktopContextToolPayload(context.desktopContext, request); + + return { + result: JSON.stringify(payload, null, 2), + isError: !payload.available, + status: payload.available ? 'success' : 'error', + errorMessage: payload.available ? undefined : payload.reason, + }; + } +} + +export const desktopContextTool = new DesktopContextTool(); +export const builtInTools: BuiltInToolGroup = [desktopContextTool]; diff --git a/apps/desktop/src/services/BuiltInToolService/types.ts b/apps/desktop/src/services/BuiltInToolService/types.ts index 68f1572e..7de71da3 100644 --- a/apps/desktop/src/services/BuiltInToolService/types.ts +++ b/apps/desktop/src/services/BuiltInToolService/types.ts @@ -13,6 +13,7 @@ import type { ToolEventBuiltInConversationSemanticAction, } from '@/services/AgentService/contracts/tooling'; import type { AttachmentIndex } from '@/services/AgentService/infrastructure/attachments'; +import type { BoundDesktopContext } from '@/services/DesktopContextService/types'; /** * 当前内置工具体系允许暴露给模型的稳定工具标识。 @@ -23,6 +24,7 @@ export type BuiltInToolId = | 'read' | 'setting' | 'web_fetch' + | 'get_desktop_context' | 'upgrade_model' | 'show_widget' | 'visualize_read_me' @@ -38,6 +40,7 @@ export interface BaseBuiltInToolExecutionContext { iteration: number; emitToolEvent?: (toolEvent: ToolEvent) => void; hasExecutedBuiltInTool: (toolId: BuiltInToolId) => boolean; + desktopContext?: BoundDesktopContext | null; requestUserQuestions?: ( callId: string, questions: AskUserQuestion[] diff --git a/apps/desktop/src/services/DesktopContextService/index.ts b/apps/desktop/src/services/DesktopContextService/index.ts new file mode 100644 index 00000000..84de3eae --- /dev/null +++ b/apps/desktop/src/services/DesktopContextService/index.ts @@ -0,0 +1,44 @@ +// Copyright (c) 2026. Qian Cheng. Licensed under GPL v3 + +import { native } from '@/services/NativeService'; + +import type { BoundDesktopContext, DesktopContextCapsule } from './types'; + +class DesktopContextService { + async bindCapsule(capsuleId: string | null | undefined): Promise { + const normalizedCapsuleId = capsuleId?.trim(); + if (!normalizedCapsuleId) { + return null; + } + + try { + return await native.desktopContext.bindCapsule(normalizedCapsuleId); + } catch (error) { + console.warn('[DesktopContextService] Failed to bind desktop context capsule:', error); + return null; + } + } + + async getCapsule(capsuleId: string): Promise { + return await native.desktopContext.getCapsule(capsuleId); + } +} + +export const desktopContextService = new DesktopContextService(); + +export type { + BoundDesktopContext, + DesktopContextActiveWindow, + DesktopContextCapability, + DesktopContextCapsule, + DesktopContextClipboard, + DesktopContextInclude, + DesktopContextPromptMetadata, + DesktopContextRedaction, + DesktopContextScreenshot, + DesktopContextSelectedText, + DesktopContextToolRequest, + DesktopContextTurnArtifact, + UserMessageDesktopContext, +} from './types'; +export { buildDesktopContextPromptMetadata, buildDesktopContextToolPayload } from './toolPayload'; diff --git a/apps/desktop/src/services/DesktopContextService/toolPayload.ts b/apps/desktop/src/services/DesktopContextService/toolPayload.ts new file mode 100644 index 00000000..d9d80296 --- /dev/null +++ b/apps/desktop/src/services/DesktopContextService/toolPayload.ts @@ -0,0 +1,218 @@ +// Copyright (c) 2026. Qian Cheng. Licensed under GPL v3 + +import type { + BoundDesktopContext, + DesktopContextActiveWindow, + DesktopContextCapability, + DesktopContextInclude, + DesktopContextPromptMetadata, + DesktopContextRedaction, + DesktopContextToolRequest, +} from './types'; + +interface DesktopContextToolUnavailablePayload { + available: false; + reason: string; +} + +interface DesktopContextToolSelectedTextPayload { + available: boolean; + source: string | null; + textLength: number; + truncated: boolean; + reason: string | null; + fullText?: string | null; +} + +interface DesktopContextToolClipboardPayload { + available: boolean; + snapshotId: string | null; + observedAt: number | null; + textSummary: string | null; + textLength: number; + imageCount: number; + fileCount: number; + reason: string | null; + fullText?: string | null; +} + +interface DesktopContextToolScreenshotPayload { + available: boolean; + mimeType: string | null; + width: number | null; + height: number | null; + target: string; + persisted: boolean; + capturedAt: string | null; + reason: string | null; + path?: string | null; +} + +interface DesktopContextToolAvailablePayload { + available: true; + capsuleId: string; + scope: string; + capturedAt: string; + boundAt: string; + summary?: string; + activeWindow?: DesktopContextActiveWindow | null; + selectedText?: DesktopContextToolSelectedTextPayload; + clipboard?: DesktopContextToolClipboardPayload; + screenshot?: DesktopContextToolScreenshotPayload; + capabilities?: DesktopContextCapability[]; + redactions?: DesktopContextRedaction[]; +} + +export type DesktopContextToolPayload = + | DesktopContextToolUnavailablePayload + | DesktopContextToolAvailablePayload; + +const DEFAULT_INCLUDE: DesktopContextInclude[] = [ + 'summary', + 'active_window', + 'selected_text.full_text', + 'clipboard.summary', + 'capabilities', + 'redactions', +]; + +function normalizeInclude(include: unknown): Set { + if (!Array.isArray(include) || include.length === 0) { + return new Set(DEFAULT_INCLUDE); + } + + return new Set( + include.filter((item): item is DesktopContextInclude => typeof item === 'string') + ); +} + +function has(include: Set, key: DesktopContextInclude): boolean { + return include.has(key); +} + +function selectedTextPayload( + context: BoundDesktopContext, + include: Set +): DesktopContextToolSelectedTextPayload | undefined { + if (!has(include, 'selected_text.summary') && !has(include, 'selected_text.full_text')) { + return undefined; + } + + const selectedText = context.selectedText; + return { + available: selectedText.available, + source: selectedText.source, + textLength: selectedText.textLength, + truncated: selectedText.truncated, + reason: selectedText.reason ?? null, + ...(has(include, 'selected_text.full_text') ? { fullText: selectedText.text } : {}), + }; +} + +function clipboardPayload( + context: BoundDesktopContext, + include: Set +): DesktopContextToolClipboardPayload | undefined { + if (!has(include, 'clipboard.summary') && !has(include, 'clipboard.full_text')) { + return undefined; + } + + const clipboard = context.clipboard; + return { + available: clipboard.available, + snapshotId: clipboard.snapshotId, + observedAt: clipboard.observedAt, + textSummary: clipboard.textSummary, + textLength: clipboard.textLength, + imageCount: clipboard.imageCount, + fileCount: clipboard.fileCount, + reason: clipboard.reason ?? null, + ...(has(include, 'clipboard.full_text') ? { fullText: clipboard.text } : {}), + }; +} + +function screenshotPayload( + context: BoundDesktopContext, + include: Set +): DesktopContextToolScreenshotPayload | undefined { + if (!has(include, 'screenshot.metadata') && !has(include, 'screenshot.image')) { + return undefined; + } + + const screenshot = context.screenshot; + return { + available: screenshot.available, + mimeType: screenshot.mimeType, + width: screenshot.width, + height: screenshot.height, + target: screenshot.target, + persisted: screenshot.persisted, + capturedAt: screenshot.capturedAt, + reason: screenshot.reason ?? null, + ...(has(include, 'screenshot.image') ? { path: screenshot.path } : {}), + }; +} + +export function buildDesktopContextToolPayload( + context: BoundDesktopContext, + request?: DesktopContextToolRequest +): DesktopContextToolAvailablePayload; +export function buildDesktopContextToolPayload( + context: null | undefined, + request?: DesktopContextToolRequest +): DesktopContextToolUnavailablePayload; +export function buildDesktopContextToolPayload( + context: BoundDesktopContext | null | undefined, + request?: DesktopContextToolRequest +): DesktopContextToolPayload; +export function buildDesktopContextToolPayload( + context: BoundDesktopContext | null | undefined, + request: DesktopContextToolRequest = {} +): DesktopContextToolPayload { + if (!context) { + return { + available: false, + reason: 'No desktop context capsule is bound to this turn.', + }; + } + + const include = normalizeInclude(request.include); + const selectedText = selectedTextPayload(context, include); + const clipboard = clipboardPayload(context, include); + const screenshot = screenshotPayload(context, include); + + return { + available: true, + capsuleId: context.id, + scope: request.scope ?? 'current', + capturedAt: context.capturedAt, + boundAt: context.boundAt, + ...(has(include, 'summary') ? { summary: context.summary } : {}), + ...(has(include, 'active_window') ? { activeWindow: context.activeWindow } : {}), + ...(selectedText ? { selectedText } : {}), + ...(clipboard ? { clipboard } : {}), + ...(screenshot ? { screenshot } : {}), + ...(has(include, 'capabilities') ? { capabilities: context.capabilities } : {}), + ...(has(include, 'redactions') ? { redactions: context.redactions } : {}), + }; +} + +export function buildDesktopContextPromptMetadata( + context: BoundDesktopContext | null | undefined +): DesktopContextPromptMetadata | undefined { + if (!context) { + return undefined; + } + + return { + capsuleId: context.id, + capturedAt: context.capturedAt, + boundAt: context.boundAt, + summary: context.summary, + activeWindowTitle: context.activeWindow?.title ?? null, + selectedTextLength: context.selectedText.textLength, + clipboardTextLength: context.clipboard.textLength, + screenshotPersisted: context.screenshot.persisted, + capabilities: context.capabilities, + }; +} diff --git a/apps/desktop/src/services/DesktopContextService/types.ts b/apps/desktop/src/services/DesktopContextService/types.ts new file mode 100644 index 00000000..67af0c92 --- /dev/null +++ b/apps/desktop/src/services/DesktopContextService/types.ts @@ -0,0 +1,137 @@ +// Copyright (c) 2026. Qian Cheng. Licensed under GPL v3 + +export type DesktopContextInclude = + | 'summary' + | 'active_window' + | 'selected_text.summary' + | 'selected_text.full_text' + | 'clipboard.summary' + | 'clipboard.full_text' + | 'screenshot.metadata' + | 'screenshot.image' + | 'capabilities' + | 'redactions'; + +export type DesktopContextScope = 'current' | 'previous' | 'recent' | 'diff'; + +export interface DesktopContextCapability { + id: string; + supported: boolean; + method: string; + reason?: string | null; +} + +export interface DesktopContextActiveWindow { + title: string | null; + appName: string | null; + processName: string | null; + processId: number | null; + windowHandle: string | null; + bounds: { + x: number; + y: number; + width: number; + height: number; + } | null; +} + +export interface DesktopContextSelectedText { + available: boolean; + source: string | null; + text: string | null; + textLength: number; + truncated: boolean; + reason?: string | null; +} + +export interface DesktopContextClipboard { + available: boolean; + snapshotId: string | null; + observedAt: number | null; + text: string | null; + textSummary: string | null; + textLength: number; + imageCount: number; + fileCount: number; + reason?: string | null; +} + +export interface DesktopContextScreenshot { + available: boolean; + path: string | null; + mimeType: string | null; + width: number | null; + height: number | null; + target: 'active_display' | 'active_window' | 'all_displays' | 'unknown'; + persisted: boolean; + capturedAt: string | null; + reason?: string | null; +} + +export interface DesktopContextRedaction { + field: string; + reason: string; +} + +export interface DesktopContextCapsule { + id: string; + sequence: number; + capturedAt: string; + invocationSource: 'shortcut' | 'notification' | 'manual' | 'unknown'; + platform: string; + summary: string; + activeWindow: DesktopContextActiveWindow | null; + selectedText: DesktopContextSelectedText; + clipboard: DesktopContextClipboard; + screenshot: DesktopContextScreenshot; + capabilities: DesktopContextCapability[]; + redactions: DesktopContextRedaction[]; +} + +export interface BoundDesktopContext extends DesktopContextCapsule { + boundAt: string; +} + +export interface DesktopContextToolRequest { + include?: DesktopContextInclude[]; + scope?: DesktopContextScope; + limit?: number; + screenshotTarget?: 'capsule_default' | 'active_window' | 'active_display' | 'all_displays'; +} + +export interface DesktopContextPromptMetadata { + capsuleId: string; + capturedAt: string; + boundAt: string; + summary: string; + activeWindowTitle: string | null; + selectedTextLength: number; + clipboardTextLength: number; + screenshotPersisted: boolean; + capabilities: DesktopContextCapability[]; +} + +export interface DesktopContextTurnArtifact { + id: number; + turn_id: number; + capsule_id: string; + artifact_kind: 'screenshot' | 'metadata'; + artifact_path: string | null; + mime_type: string | null; + width: number | null; + height: number | null; + captured_at: string; + metadata_json: string | null; + created_at: string; +} + +export interface UserMessageDesktopContext { + capsuleId: string; + capturedAt: string; + summary: string; + activeWindowTitle: string | null; + screenshotPath: string | null; + screenshotMimeType: string | null; + screenshotWidth: number | null; + screenshotHeight: number | null; +} diff --git a/apps/desktop/src/services/EventService/types.ts b/apps/desktop/src/services/EventService/types.ts index 280dfd93..884dcd45 100644 --- a/apps/desktop/src/services/EventService/types.ts +++ b/apps/desktop/src/services/EventService/types.ts @@ -118,6 +118,7 @@ export interface WindowResizeEvent { export interface SearchSurfaceShownEvent { source: 'shortcut' | 'notification'; sequence?: number; + contextCapsuleId?: string | null; } export interface SearchSurfaceHiddenEvent { diff --git a/apps/desktop/src/services/NativeService/desktopContext.ts b/apps/desktop/src/services/NativeService/desktopContext.ts new file mode 100644 index 00000000..1146a8b2 --- /dev/null +++ b/apps/desktop/src/services/NativeService/desktopContext.ts @@ -0,0 +1,22 @@ +// Copyright (c) 2026. Qian Cheng. Licensed under GPL v3 + +import { invoke } from '@tauri-apps/api/core'; + +import type { + BoundDesktopContext, + DesktopContextCapsule, +} from '@/services/DesktopContextService/types'; + +export const desktopContext = { + getCapsule(capsuleId: string): Promise { + return invoke('desktop_context_get_capsule', { + capsuleId, + }); + }, + + bindCapsule(capsuleId: string): Promise { + return invoke('desktop_context_bind_capsule', { + capsuleId, + }); + }, +} as const; diff --git a/apps/desktop/src/services/NativeService/index.ts b/apps/desktop/src/services/NativeService/index.ts index b6b6f7fb..3832b12d 100644 --- a/apps/desktop/src/services/NativeService/index.ts +++ b/apps/desktop/src/services/NativeService/index.ts @@ -2,6 +2,7 @@ import { autostart } from './autostart'; import { builtInTools } from './builtInTools'; import { clipboard } from './clipboard'; import { database } from './database'; +import { desktopContext } from './desktopContext'; import { log } from './log'; import * as mcp from './mcp'; import { paths } from './paths'; @@ -45,6 +46,7 @@ export { builtInTools, clipboard, database, + desktopContext, log, mcp, paths, @@ -63,6 +65,7 @@ export const native = { builtInTools, log, database, + desktopContext, paths, mcp, quickSearch, diff --git a/apps/desktop/src/types/session.ts b/apps/desktop/src/types/session.ts index 1b5361a8..0cb9221a 100644 --- a/apps/desktop/src/types/session.ts +++ b/apps/desktop/src/types/session.ts @@ -9,6 +9,7 @@ import type { BuiltInToolConversationSemantic, } from '@/services/BuiltInToolService'; import type { ShowWidgetPayload } from '@/services/BuiltInToolService/tools/widgetTool'; +import type { UserMessageDesktopContext } from '@/services/DesktopContextService/types'; /** * SearchView 与 agent 运行时共享的会话展示模型。 @@ -233,6 +234,7 @@ export interface SessionMessage { statusText?: string; attachments?: Index[]; inputSnapshot?: InputHistorySnapshot; + desktopContext?: UserMessageDesktopContext; toolCalls?: ToolCallInfo[]; approvals?: ToolApprovalInfo[]; widgets?: WidgetInfo[]; diff --git a/apps/desktop/src/views/SearchView/components/ConversationPanel/components/UserMessage.vue b/apps/desktop/src/views/SearchView/components/ConversationPanel/components/UserMessage.vue index 70732191..52ccef8c 100644 --- a/apps/desktop/src/views/SearchView/components/ConversationPanel/components/UserMessage.vue +++ b/apps/desktop/src/views/SearchView/components/ConversationPanel/components/UserMessage.vue @@ -43,6 +43,48 @@ + +
+
+ + {{ t('conversation.desktopContext.title') }} +
+
+ {{ message.desktopContext.summary }} +
+
+ {{ message.desktopContext.activeWindowTitle }} +
+ + + +
+ {{ message.desktopContext.screenshotPath }} +
+
@@ -61,6 +103,8 @@ import ActionButton from '@components/ActionButton.vue'; import AppIcon from '@components/AppIcon.vue'; import { notify } from '@services/NotificationService'; + import { convertFileSrc } from '@tauri-apps/api/core'; + import { computed } from 'vue'; import { t } from '@/i18n'; import { clipboardService } from '@/services/ClipboardService'; @@ -72,6 +116,11 @@ const props = defineProps(); + const desktopContextScreenshotUrl = computed(() => { + const screenshotPath = props.message.desktopContext?.screenshotPath; + return screenshotPath ? convertFileSrc(screenshotPath) : null; + }); + async function handleCopy() { try { await clipboardService.writeText(props.message.content); diff --git a/apps/desktop/src/views/SearchView/composables/useSearchPage.ts b/apps/desktop/src/views/SearchView/composables/useSearchPage.ts index dfaf5aa3..2fda3b10 100644 --- a/apps/desktop/src/views/SearchView/composables/useSearchPage.ts +++ b/apps/desktop/src/views/SearchView/composables/useSearchPage.ts @@ -392,6 +392,10 @@ interface UseSearchPageLifecycleOptions { ) => void | Promise; handleAiModelsUpdated?: () => void | Promise; handleShortcutAutoPaste?: () => void | Promise; + handleSearchSurfaceShown?: (payload: { + contextCapsuleId?: string | null; + sequence?: number; + }) => void | Promise; } export function useSearchPageLifecycle(options: UseSearchPageLifecycleOptions) { @@ -411,6 +415,7 @@ export function useSearchPageLifecycle(options: UseSearchPageLifecycleOptions) { handleSessionStatusReminderAction, handleAiModelsUpdated, handleShortcutAutoPaste, + handleSearchSurfaceShown, } = options; const settingsStore = useSettingsStore(); @@ -633,6 +638,14 @@ export function useSearchPageLifecycle(options: UseSearchPageLifecycleOptions) { } latestSurfaceSequence = Math.max(latestSurfaceSequence, sequence); + await Promise.resolve( + handleSearchSurfaceShown?.({ + contextCapsuleId: payload.contextCapsuleId ?? null, + sequence, + }) + ).catch((error) => { + console.error('[SearchView] Failed to record desktop context capsule:', error); + }); interactionContext.markWindowVisible(); handleReminderSurfaceVisible(); await handleSearchWindowActivated(payload.source ?? 'shortcut'); diff --git a/apps/desktop/src/views/SearchView/composables/useSearchRequest.ts b/apps/desktop/src/views/SearchView/composables/useSearchRequest.ts index 36e54f3e..e21ec041 100644 --- a/apps/desktop/src/views/SearchView/composables/useSearchRequest.ts +++ b/apps/desktop/src/views/SearchView/composables/useSearchRequest.ts @@ -29,6 +29,7 @@ interface UseSearchRequestFlowOptions { getSupportedAttachments: () => Index[]; getUnsupportedAttachmentMessage: () => string | null; getCurrentInputSnapshot: (query: string) => InputHistorySnapshot; + getCurrentDesktopContextCapsuleId?: () => string | null; } const SESSION_LIST_LIMIT = 40; @@ -65,6 +66,7 @@ export function useSearchRequestFlow(options: UseSearchRequestFlowOptions) { getSupportedAttachments, getUnsupportedAttachmentMessage, getCurrentInputSnapshot, + getCurrentDesktopContextCapsuleId, } = options; const pendingRequest = ref(null); @@ -152,12 +154,20 @@ export function useSearchRequestFlow(options: UseSearchRequestFlowOptions) { inputSnapshot, modelId, providerId, + desktopContextCapsuleId, } = pendingRequest.value; clearPendingRequestState(); clearDraft({ preserveModelTag: true }); - await sendRequest(query, pendingAttachments, inputSnapshot, modelId, providerId); + await sendRequest( + query, + pendingAttachments, + inputSnapshot, + modelId, + providerId, + desktopContextCapsuleId + ); }, onError: () => { invalidateSessionListCache({ @@ -338,6 +348,7 @@ export function useSearchRequestFlow(options: UseSearchRequestFlowOptions) { } const inputSnapshot = inputSnapshotOverride ?? getCurrentInputSnapshot(query); + const desktopContextCapsuleId = getCurrentDesktopContextCapsuleId?.() ?? null; if (isLoading.value) { if (pendingRequest.value) { @@ -352,6 +363,7 @@ export function useSearchRequestFlow(options: UseSearchRequestFlowOptions) { inputSnapshot, modelId: selectedModelId ?? undefined, providerId: selectedProviderId ?? undefined, + desktopContextCapsuleId, }; isWaitingForCompletion.value = true; return; @@ -368,7 +380,8 @@ export function useSearchRequestFlow(options: UseSearchRequestFlowOptions) { supportedAttachments, inputSnapshot, selectedModelId ?? undefined, - selectedProviderId ?? undefined + selectedProviderId ?? undefined, + desktopContextCapsuleId ); } @@ -440,7 +453,8 @@ export function useSearchRequestFlow(options: UseSearchRequestFlowOptions) { supportedAttachments, inputSnapshot, selectedModelId ?? undefined, - selectedProviderId ?? undefined + selectedProviderId ?? undefined, + getCurrentDesktopContextCapsuleId?.() ?? null ); } diff --git a/apps/desktop/src/views/SearchView/index.vue b/apps/desktop/src/views/SearchView/index.vue index 1cf2a4d6..7f9a9b7c 100644 --- a/apps/desktop/src/views/SearchView/index.vue +++ b/apps/desktop/src/views/SearchView/index.vue @@ -105,6 +105,7 @@ const historyAnchorElement = ref(null); const pageContainer = ref(null); const approvalAttentionToken = ref(0); + const latestDesktopContextCapsuleId = ref(null); const isDragging = ref(false); const latestContentVisible = ref(true); const latestSeenCompletedMessageMarker = ref(null); @@ -249,6 +250,7 @@ getSupportedAttachments, getUnsupportedAttachmentMessage, getCurrentInputSnapshot: buildCurrentInputHistorySnapshot, + getCurrentDesktopContextCapsuleId: () => latestDesktopContextCapsuleId.value, }); const { @@ -418,6 +420,9 @@ handleSessionStatusReminderAction, handleAiModelsUpdated, handleShortcutAutoPaste: tryShortcutAutoPaste, + handleSearchSurfaceShown: ({ contextCapsuleId }) => { + latestDesktopContextCapsuleId.value = contextCapsuleId ?? null; + }, }); function getSessionHistoryPopupData(): SessionHistoryData { diff --git a/apps/desktop/src/views/SearchView/types.ts b/apps/desktop/src/views/SearchView/types.ts index 9cb104f5..5b1f29c5 100644 --- a/apps/desktop/src/views/SearchView/types.ts +++ b/apps/desktop/src/views/SearchView/types.ts @@ -100,6 +100,7 @@ export interface PendingRequest { inputSnapshot?: InputHistorySnapshot; modelId?: string; providerId?: number; + desktopContextCapsuleId?: string | null; } export interface SearchModelCapabilities { diff --git a/apps/desktop/src/views/SettingsView/components/BuiltInTools/types.ts b/apps/desktop/src/views/SettingsView/components/BuiltInTools/types.ts index 0a76a275..928e55ac 100644 --- a/apps/desktop/src/views/SettingsView/components/BuiltInTools/types.ts +++ b/apps/desktop/src/views/SettingsView/components/BuiltInTools/types.ts @@ -78,6 +78,7 @@ const BUILT_IN_TOOL_EMPTY_CONFIG_IDS = new Set([ 'read', 'setting', 'web_fetch', + 'get_desktop_context', 'show_widget', 'visualize_read_me', ]); @@ -105,6 +106,10 @@ export function getBuiltInToolSummary(toolId: string, description?: string | nul return t('settings.builtInTools.summary.webFetch'); } + if (toolId === 'get_desktop_context') { + return t('settings.builtInTools.summary.desktopContext'); + } + if (toolId === 'upgrade_model') { return t('settings.builtInTools.summary.upgradeModel'); } diff --git a/apps/desktop/tests/services/AgentService/session-history-i18n.test.ts b/apps/desktop/tests/services/AgentService/session-history-i18n.test.ts index bf9a44b4..68b84531 100644 --- a/apps/desktop/tests/services/AgentService/session-history-i18n.test.ts +++ b/apps/desktop/tests/services/AgentService/session-history-i18n.test.ts @@ -1,4 +1,5 @@ import type { MessageRow } from '@database/queries/messages'; +import type { SessionTurnContextArtifactHistoryRow } from '@database/queries/sessionTurnContextArtifacts'; import type { SessionTurnAttemptHistoryRow } from '@database/queries/sessionTurnAttempts'; import type { SessionTurnHistoryRow } from '@database/queries/sessionTurns'; import { beforeEach, describe, expect, it, vi } from 'vitest'; @@ -95,6 +96,26 @@ function createAttempt( }; } +function createContextArtifact( + overrides: Partial +): SessionTurnContextArtifactHistoryRow { + return { + id: 1, + turn_id: 1, + prompt_message_id: 10, + capsule_id: 'ctx-1', + artifact_kind: 'metadata', + artifact_path: null, + mime_type: null, + width: null, + height: null, + captured_at: BASE_TIME, + metadata_json: null, + created_at: BASE_TIME, + ...overrides, + }; +} + describe('AgentService session history i18n', () => { beforeEach(() => { setLocale('zh-CN'); @@ -207,4 +228,55 @@ describe('AgentService session history i18n', () => { content: 'Write-Output 设置', }); }); + + it('restores persisted desktop context artifacts onto the prompt message', async () => { + const history = await buildSessionHistory({ + messages: [ + createMessageRow({ + id: 10, + role: 'user', + content: '解释我刚才选中的内容', + }), + ], + turns: [ + createTurn({ + id: 1, + prompt_message_id: 10, + }), + ], + attempts: [], + contextArtifacts: [ + createContextArtifact({ + artifact_kind: 'metadata', + metadata_json: JSON.stringify({ + capsuleId: 'ctx-1', + capturedAt: BASE_TIME, + summary: 'Visual Studio Code focused with selected Rust error text.', + activeWindowTitle: 'main.rs - TouchAI', + }), + }), + createContextArtifact({ + id: 2, + artifact_kind: 'screenshot', + artifact_path: 'E:/TouchAI/data/session-context/ctx-1.png', + mime_type: 'image/png', + width: 1200, + height: 800, + }), + ], + resolveServerName: () => '', + }); + + expect(history[0]).toMatchObject({ + role: 'user', + desktopContext: { + capsuleId: 'ctx-1', + summary: 'Visual Studio Code focused with selected Rust error text.', + activeWindowTitle: 'main.rs - TouchAI', + screenshotPath: 'E:/TouchAI/data/session-context/ctx-1.png', + screenshotWidth: 1200, + screenshotHeight: 800, + }, + }); + }); }); diff --git a/apps/desktop/tests/services/DesktopContextService/tool-payload.test.ts b/apps/desktop/tests/services/DesktopContextService/tool-payload.test.ts new file mode 100644 index 00000000..faf17b7e --- /dev/null +++ b/apps/desktop/tests/services/DesktopContextService/tool-payload.test.ts @@ -0,0 +1,99 @@ +// Copyright (c) 2026. Qian Cheng. Licensed under GPL v3 + +import { describe, expect, it } from 'vitest'; + +import type { BoundDesktopContext } from '@/services/DesktopContextService/types'; +import { buildDesktopContextToolPayload } from '@/services/DesktopContextService/toolPayload'; + +const context: BoundDesktopContext = { + id: 'ctx-1', + sequence: 7, + capturedAt: '2026-06-02T12:00:00.000Z', + boundAt: '2026-06-02T12:00:05.000Z', + invocationSource: 'shortcut', + platform: 'windows', + summary: 'Visual Studio Code focused with selected Rust error text.', + activeWindow: { + title: 'main.rs - TouchAI', + appName: 'Visual Studio Code', + processName: 'Code.exe', + processId: 123, + windowHandle: '0x123', + bounds: { x: 0, y: 0, width: 1200, height: 800 }, + }, + selectedText: { + available: true, + source: 'win32-edit-control', + text: 'selected compiler error', + textLength: 23, + truncated: false, + }, + clipboard: { + available: true, + snapshotId: 'clip-1', + observedAt: 1770000000000, + text: 'clipboard secret', + textSummary: 'clipboard secret', + textLength: 16, + imageCount: 0, + fileCount: 0, + }, + screenshot: { + available: true, + path: 'E:/TouchAI/data/session-context/ctx-1.png', + mimeType: 'image/png', + width: 1200, + height: 800, + target: 'active_display', + persisted: true, + capturedAt: '2026-06-02T12:00:00.000Z', + }, + capabilities: [{ id: 'screenshot', supported: true, method: 'gdi' }], + redactions: [{ field: 'clipboard.fullText', reason: 'not requested by default' }], +}; + +describe('buildDesktopContextToolPayload', () => { + it('returns unavailable when no capsule is bound to the turn', () => { + expect(buildDesktopContextToolPayload(null)).toEqual({ + available: false, + reason: 'No desktop context capsule is bound to this turn.', + }); + }); + + it('uses a safe default include set without clipboard full text or screenshot path', () => { + const payload = buildDesktopContextToolPayload(context); + + expect(payload).toMatchObject({ + available: true, + capsuleId: 'ctx-1', + summary: context.summary, + activeWindow: context.activeWindow, + selectedText: { + fullText: 'selected compiler error', + }, + clipboard: { + textSummary: 'clipboard secret', + textLength: 16, + }, + capabilities: context.capabilities, + redactions: context.redactions, + }); + expect(payload).not.toHaveProperty('screenshot'); + expect(payload.clipboard).not.toHaveProperty('fullText'); + }); + + it('returns sensitive fields only when the include array asks for them', () => { + const payload = buildDesktopContextToolPayload(context, { + include: ['clipboard.full_text', 'screenshot.image'], + }); + + expect(payload).not.toHaveProperty('summary'); + expect(payload.clipboard).toMatchObject({ + fullText: 'clipboard secret', + }); + expect(payload.screenshot).toMatchObject({ + path: 'E:/TouchAI/data/session-context/ctx-1.png', + persisted: true, + }); + }); +}); From 367b9f7d915f0224fc1f7189d0557f99a4a73f5f Mon Sep 17 00:00:00 2001 From: xlgzsgf <51521689+hiqiancheng@users.noreply.github.com> Date: Wed, 3 Jun 2026 05:52:52 +0800 Subject: [PATCH 2/4] feat(desktop-context): require approval for sensitive context Related to #281 --- apps/desktop/src-tauri/Cargo.lock | 665 +++++++++- apps/desktop/src-tauri/Cargo.toml | 2 + .../src-tauri/src/commands/desktop_context.rs | 13 +- apps/desktop/src-tauri/src/commands/mod.rs | 1 + .../src/core/system/desktop_context.rs | 1179 ++++++++++++++++- .../src-tauri/src/core/window/search/mod.rs | 10 + .../queries/sessionTurnContextArtifacts.ts | 24 +- apps/desktop/src/i18n/textMap.ts | 8 + .../AgentService/execution/executor.ts | 7 + .../AgentService/execution/runtime.ts | 2 +- .../AgentService/outputs/persistence.ts | 21 +- .../services/AgentService/prompt/composer.ts | 26 +- .../services/AgentService/session/history.ts | 6 +- .../services/BuiltInToolService/service.ts | 2 + .../tools/desktopContext/constants.ts | 10 +- .../tools/desktopContext/index.ts | 140 +- .../src/services/BuiltInToolService/types.ts | 1 + .../services/DesktopContextService/index.ts | 2 +- .../DesktopContextService/toolPayload.ts | 21 +- .../services/DesktopContextService/types.ts | 3 + .../services/NativeService/desktopContext.ts | 14 + .../SearchView/useSearchRequest.test.ts | 7 +- .../prompt/language-context.test.ts | 33 + .../AgentService/session-history-i18n.test.ts | 2 +- .../tools/desktopContext/index.test.ts | 233 ++++ .../tool-payload.test.ts | 72 +- 26 files changed, 2398 insertions(+), 106 deletions(-) create mode 100644 apps/desktop/tests/services/BuiltInToolService/tools/desktopContext/index.test.ts diff --git a/apps/desktop/src-tauri/Cargo.lock b/apps/desktop/src-tauri/Cargo.lock index 9b8c8550..0fc03af1 100644 --- a/apps/desktop/src-tauri/Cargo.lock +++ b/apps/desktop/src-tauri/Cargo.lock @@ -93,6 +93,22 @@ dependencies = [ "libc", ] +[[package]] +name = "annotate-snippets" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "710e8eae58854cdc1790fcb56cca04d712a17be849eeb81da2a724bf4bae2bc4" +dependencies = [ + "anstyle", + "unicode-width", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + [[package]] name = "anyhow" version = "1.0.100" @@ -185,7 +201,7 @@ dependencies = [ "futures-lite", "parking", "polling", - "rustix", + "rustix 1.1.2", "slab", "windows-sys 0.61.2", ] @@ -216,7 +232,7 @@ dependencies = [ "cfg-if", "event-listener", "futures-lite", - "rustix", + "rustix 1.1.2", ] [[package]] @@ -242,7 +258,7 @@ dependencies = [ "cfg-if", "futures-core", "futures-io", - "rustix", + "rustix 1.1.2", "signal-hook-registry", "slab", "windows-sys 0.61.2", @@ -349,7 +365,7 @@ dependencies = [ "anyhow", "arrayvec", "log", - "nom", + "nom 8.0.0", "num-rational", "v_frame", ] @@ -397,6 +413,25 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "annotate-snippets", + "bitflags 2.11.1", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.111", +] + [[package]] name = "bit-set" version = "0.8.0" @@ -598,6 +633,20 @@ name = "bytemuck" version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] [[package]] name = "byteorder" @@ -705,6 +754,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom 7.1.3", +] + [[package]] name = "cfb" version = "0.7.3" @@ -771,6 +829,17 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading 0.8.9", +] + [[package]] name = "clipboard-rs" version = "0.3.4" @@ -873,6 +942,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" +[[package]] +name = "convert_case" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baaaa0ecca5b51987b9423ccdc971514dd8b0bb7b4060b983d3664dad3f1f89f" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "cookie" version = "0.18.1" @@ -884,6 +962,12 @@ dependencies = [ "version_check", ] +[[package]] +name = "cookie-factory" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9885fa71e26b8ab7855e2ec7cae6e9b380edff76cd052e07c683a0319d51b3a2" + [[package]] name = "cookie_store" version = "0.22.1" @@ -1251,7 +1335,7 @@ version = "0.99.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6edb4b64a43d977b8e99788fe3a04d483834fba1215a7e02caa415b626497f7f" dependencies = [ - "convert_case", + "convert_case 0.4.0", "proc-macro2", "quote", "rustc_version", @@ -1343,9 +1427,9 @@ dependencies = [ [[package]] name = "dispatch2" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89a09f22a6c6069a18470eb92d2298acf25463f14256d24778e1230d789a2aec" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" dependencies = [ "bitflags 2.11.1", "block2 0.6.2", @@ -1364,6 +1448,15 @@ dependencies = [ "syn 2.0.111", ] +[[package]] +name = "dlib" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab8ecd87370524b461f8557c119c405552c396ed91fc0a8eec68679eab26f94a" +dependencies = [ + "libloading 0.7.4", +] + [[package]] name = "dlopen2" version = "0.8.2" @@ -1426,6 +1519,12 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "downcast-rs" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" + [[package]] name = "dpi" version = "0.1.2" @@ -1435,6 +1534,46 @@ dependencies = [ "serde", ] +[[package]] +name = "drm" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80bc8c5c6c2941f70a55c15f8d9f00f9710ebda3ffda98075f996a0e6c92756f" +dependencies = [ + "bitflags 2.11.1", + "bytemuck", + "drm-ffi", + "drm-fourcc", + "libc", + "rustix 0.38.44", +] + +[[package]] +name = "drm-ffi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51a91c9b32ac4e8105dec255e849e0d66e27d7c34d184364fb93e469db08f690" +dependencies = [ + "drm-sys", + "rustix 1.1.2", +] + +[[package]] +name = "drm-fourcc" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aafbcdb8afc29c1a7ee5fbe53b5d62f4565b35a042a662ca9fecd0b54dae6f4" + +[[package]] +name = "drm-sys" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8e1361066d91f5ffccff060a3c3be9c3ecde15be2959c1937595f7a82a9f8" +dependencies = [ + "libc", + "linux-raw-sys 0.9.4", +] + [[package]] name = "dtoa" version = "1.0.10" @@ -1947,6 +2086,30 @@ dependencies = [ "byteorder", ] +[[package]] +name = "gbm" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce852e998d3ca5e4a97014fb31c940dc5ef344ec7d364984525fd11e8a547e6a" +dependencies = [ + "bitflags 2.11.1", + "drm", + "drm-fourcc", + "gbm-sys", + "libc", + "wayland-backend", + "wayland-server", +] + +[[package]] +name = "gbm-sys" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13a5f2acc785d8fb6bf6b7ab6bfb0ef5dad4f4d97e8e70bb8e470722312f76f" +dependencies = [ + "libc", +] + [[package]] name = "gdk" version = "0.18.2" @@ -2062,7 +2225,7 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" dependencies = [ - "rustix", + "rustix 1.1.2", "windows-link 0.2.1", ] @@ -2190,6 +2353,26 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "gl" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a94edab108827d67608095e269cf862e60d920f144a5026d3dbcfd8b877fb404" +dependencies = [ + "gl_generator", +] + +[[package]] +name = "gl_generator" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a95dfc23a2b4a9a2f5ab41d194f8bfda3cabec42af4e39f08c339eb2a0c124d" +dependencies = [ + "khronos_api", + "log", + "xml-rs", +] + [[package]] name = "glib" version = "0.18.5" @@ -2887,6 +3070,15 @@ dependencies = [ "once_cell", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.14.0" @@ -3051,6 +3243,22 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "khronos-egl" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6aae1df220ece3c0ada96b8153459b67eebe9ae9212258bb0134ae60416fdf76" +dependencies = [ + "libc", + "pkg-config", +] + +[[package]] +name = "khronos_api" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2db585e1d738fc771bf08a151420d3ed193d9d895a36df7f6f8a9456b911ddc" + [[package]] name = "kuchikiki" version = "0.8.8-speedreader" @@ -3101,7 +3309,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e9ec52138abedcc58dc17a7c6c0c00a2bdb4f3427c7f63fa97fd0d859155caf" dependencies = [ "gtk-sys", - "libloading", + "libloading 0.7.4", "once_cell", ] @@ -3140,6 +3348,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link 0.2.1", +] + [[package]] name = "libredox" version = "0.1.10" @@ -3151,6 +3369,34 @@ dependencies = [ "redox_syscall", ] +[[package]] +name = "libspa" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6b8cfa2a7656627b4c92c6b9ef929433acd673d5ab3708cda1b18478ac00df4" +dependencies = [ + "bitflags 2.11.1", + "cc", + "convert_case 0.8.0", + "cookie-factory", + "libc", + "libspa-sys", + "nix 0.30.1", + "nom 8.0.0", + "system-deps 7.0.7", +] + +[[package]] +name = "libspa-sys" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901049455d2eb6decf9058235d745237952f4804bc584c5fcb41412e6adcc6e0" +dependencies = [ + "bindgen", + "cc", + "system-deps 7.0.7", +] + [[package]] name = "libsqlite3-sys" version = "0.30.1" @@ -3162,6 +3408,27 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libwayshot-xcap" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "558a3a7ca16a17a14adf8f051b3adcd7766d397532f5f6d6a48034db11e54c22" +dependencies = [ + "drm", + "gbm", + "gl", + "image", + "khronos-egl", + "memmap2", + "rustix 1.1.2", + "thiserror 2.0.17", + "tracing", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "wayland-protocols-wlr", +] + [[package]] name = "libz-rs-sys" version = "0.5.5" @@ -3171,6 +3438,18 @@ dependencies = [ "zlib-rs", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "linux-raw-sys" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" + [[package]] name = "linux-raw-sys" version = "0.11.0" @@ -3298,6 +3577,15 @@ version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +[[package]] +name = "memmap2" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +dependencies = [ + "libc", +] + [[package]] name = "memoffset" version = "0.9.1" @@ -3313,6 +3601,12 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -3426,6 +3720,16 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nom" version = "8.0.0" @@ -3614,6 +3918,38 @@ dependencies = [ "objc2-quartz-core 0.3.2", ] +[[package]] +name = "objc2-av-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478ae33fcac9df0a18db8302387c666b8ef08a3e2d62b510ca4fc278a384b6c0" +dependencies = [ + "bitflags 2.11.1", + "block2 0.6.2", + "dispatch2", + "objc2 0.6.4", + "objc2-avf-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-core-image 0.3.2", + "objc2-core-video", + "objc2-foundation 0.3.2", + "objc2-image-io", + "objc2-media-toolbox", + "objc2-quartz-core 0.3.2", +] + +[[package]] +name = "objc2-avf-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13a380031deed8e99db00065c45937da434ca987c034e13b87e4441f9e4090be" +dependencies = [ + "objc2 0.6.4", + "objc2-foundation 0.3.2", +] + [[package]] name = "objc2-cloud-kit" version = "0.3.2" @@ -3625,6 +3961,28 @@ dependencies = [ "objc2-foundation 0.3.2", ] +[[package]] +name = "objc2-core-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1eebcea8b0dbff5f7c8504f3107c68fc061a3eb44932051c8cf8a68d969c3b2" +dependencies = [ + "dispatch2", + "objc2 0.6.4", + "objc2-core-audio-types", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-core-audio-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a89f2ec274a0cf4a32642b2991e8b351a404d290da87bb6a9a9d8632490bd1c" +dependencies = [ + "bitflags 2.11.1", + "objc2 0.6.4", +] + [[package]] name = "objc2-core-data" version = "0.2.2" @@ -3655,7 +4013,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ "bitflags 2.11.1", + "block2 0.6.2", "dispatch2", + "libc", "objc2 0.6.4", ] @@ -3666,10 +4026,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e022c9d066895efa1345f8e33e584b9f958da2fd4cd116792e15e07e4720a807" dependencies = [ "bitflags 2.11.1", + "block2 0.6.2", "dispatch2", + "libc", "objc2 0.6.4", "objc2-core-foundation", "objc2-io-surface", + "objc2-metal 0.3.2", ] [[package]] @@ -3681,7 +4044,7 @@ dependencies = [ "block2 0.5.1", "objc2 0.5.2", "objc2-foundation 0.2.2", - "objc2-metal", + "objc2-metal 0.2.2", ] [[package]] @@ -3704,6 +4067,22 @@ dependencies = [ "objc2-foundation 0.3.2", ] +[[package]] +name = "objc2-core-media" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ec576860167a15dd9fce7fbee7512beb4e31f532159d3482d1f9c6caedf31d" +dependencies = [ + "bitflags 2.11.1", + "block2 0.6.2", + "dispatch2", + "objc2 0.6.4", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-core-video", +] + [[package]] name = "objc2-core-text" version = "0.3.2" @@ -3723,10 +4102,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d425caf1df73233f29fd8a5c3e5edbc30d2d4307870f802d18f00d83dc5141a6" dependencies = [ "bitflags 2.11.1", + "block2 0.6.2", "objc2 0.6.4", "objc2-core-foundation", "objc2-core-graphics", "objc2-io-surface", + "objc2-metal 0.3.2", ] [[package]] @@ -3769,6 +4150,17 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "objc2-image-io" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b0446e98cf4a784cc7a0177715ff317eeaa8463841c616cfc78aa4f953c4ea" +dependencies = [ + "objc2 0.6.4", + "objc2-core-foundation", + "objc2-core-graphics", +] + [[package]] name = "objc2-io-surface" version = "0.3.2" @@ -3780,6 +4172,18 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "objc2-media-toolbox" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd9fdde720df3da7046bb9097811000c1e7ab5cd579fa89d96b27d56781fb30" +dependencies = [ + "objc2 0.6.4", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-core-media", +] + [[package]] name = "objc2-metal" version = "0.2.2" @@ -3792,6 +4196,17 @@ dependencies = [ "objc2-foundation 0.2.2", ] +[[package]] +name = "objc2-metal" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0125f776a10d00af4152d74616409f0d4a2053a6f57fa5b7d6aa2854ac04794" +dependencies = [ + "bitflags 2.11.1", + "objc2 0.6.4", + "objc2-foundation 0.3.2", +] + [[package]] name = "objc2-quartz-core" version = "0.2.2" @@ -3802,7 +4217,7 @@ dependencies = [ "block2 0.5.1", "objc2 0.5.2", "objc2-foundation 0.2.2", - "objc2-metal", + "objc2-metal 0.2.2", ] [[package]] @@ -4205,6 +4620,34 @@ dependencies = [ "futures-io", ] +[[package]] +name = "pipewire" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9688b89abf11d756499f7c6190711d6dbe5a3acdb30c8fbf001d6596d06a8d44" +dependencies = [ + "anyhow", + "bitflags 2.11.1", + "libc", + "libspa", + "libspa-sys", + "nix 0.30.1", + "once_cell", + "pipewire-sys", + "thiserror 2.0.17", +] + +[[package]] +name = "pipewire-sys" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb028afee0d6ca17020b090e3b8fa2d7de23305aef975c7e5192a5050246ea36" +dependencies = [ + "bindgen", + "libspa-sys", + "system-deps 7.0.7", +] + [[package]] name = "pkg-config" version = "0.3.32" @@ -4260,7 +4703,7 @@ dependencies = [ "concurrent-queue", "hermit-abi", "pin-project-lite", - "rustix", + "rustix 1.1.2", "windows-sys 0.61.2", ] @@ -4465,6 +4908,15 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" +[[package]] +name = "quick-xml" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eff6510e86862b57b210fd8cbe8ed3f0d7d600b9c2863cd4549a2e033c66e956" +dependencies = [ + "memchr", +] + [[package]] name = "quick-xml" version = "0.37.5" @@ -4483,6 +4935,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "quick-xml" +version = "0.39.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" +dependencies = [ + "memchr", +] + [[package]] name = "quinn" version = "0.11.9" @@ -4709,7 +5170,7 @@ dependencies = [ "built", "cfg-if", "interpolate_name", - "itertools", + "itertools 0.14.0", "libc", "libfuzzer-sys", "log", @@ -5099,6 +5560,19 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.11.1", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + [[package]] name = "rustix" version = "1.1.2" @@ -5108,7 +5582,7 @@ dependencies = [ "bitflags 2.11.1", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.11.0", "windows-sys 0.61.2", ] @@ -5284,6 +5758,12 @@ dependencies = [ "syn 2.0.111", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.2.0" @@ -6614,7 +7094,7 @@ dependencies = [ "fastrand", "getrandom 0.3.4", "once_cell", - "rustix", + "rustix 1.1.2", "windows-sys 0.61.2", ] @@ -6994,6 +7474,7 @@ dependencies = [ "webview2-com", "windows 0.58.0", "windows-core 0.61.2", + "xcap", "zip 2.4.2", ] @@ -7184,6 +7665,12 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + [[package]] name = "unicode-xid" version = "0.2.6" @@ -7404,7 +7891,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18aa3ce681e189f125c4c1e1388c03285e2fd434ef52c7203084012ac29c5e4a" dependencies = [ - "rustix", + "rustix 1.1.2", "windows-sys 0.59.0", ] @@ -7559,6 +8046,94 @@ dependencies = [ "semver", ] +[[package]] +name = "wayland-backend" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2857dd20b54e916ec7253b3d6b4d5c4d7d4ca2c33c2e11c6c76a99bd8744755d" +dependencies = [ + "cc", + "downcast-rs", + "rustix 1.1.2", + "scoped-tls", + "smallvec", + "wayland-sys", +] + +[[package]] +name = "wayland-client" +version = "0.31.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645c7c96bb74690c3189b5c9cb4ca1627062bb23693a4fad9d8c3de958260144" +dependencies = [ + "bitflags 2.11.1", + "rustix 1.1.2", + "wayland-backend", + "wayland-scanner", +] + +[[package]] +name = "wayland-protocols" +version = "0.32.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "563a85523cade2429938e790815fd7319062103b9f4a2dc806e9b53b95982d8f" +dependencies = [ + "bitflags 2.11.1", + "wayland-backend", + "wayland-client", + "wayland-scanner", +] + +[[package]] +name = "wayland-protocols-wlr" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb04e52f7836d7c7976c78ca0250d61e33873c34156a2a1fc9474828ec268234" +dependencies = [ + "bitflags 2.11.1", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "wayland-scanner", +] + +[[package]] +name = "wayland-scanner" +version = "0.31.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c324a910fd86ebdc364a3e61ec1f11737d3b1d6c273c0239ee8ff4bc0d24b4a" +dependencies = [ + "proc-macro2", + "quick-xml 0.39.4", + "quote", +] + +[[package]] +name = "wayland-server" +version = "0.31.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc1846eb04c49182e04f4a099e2a830a2b745610bbc1d61246e206f29c7000a0" +dependencies = [ + "bitflags 2.11.1", + "downcast-rs", + "rustix 1.1.2", + "wayland-backend", + "wayland-scanner", +] + +[[package]] +name = "wayland-sys" +version = "0.31.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8eab23fefc9e41f8e841df4a9c707e8a8c4ed26e944ef69297184de2785e3be" +dependencies = [ + "dlib", + "libc", + "log", + "memoffset", + "pkg-config", +] + [[package]] name = "web-sys" version = "0.3.98" @@ -7704,6 +8279,12 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + [[package]] name = "winapi" version = "0.3.9" @@ -8551,7 +9132,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9993aa5be5a26815fe2c3eacfc1fde061fc1a1f094bf1ad2a18bf9c495dd7414" dependencies = [ "gethostname", - "rustix", + "rustix 1.1.2", "x11rb-protocol", ] @@ -8568,7 +9149,49 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" dependencies = [ "libc", - "rustix", + "rustix 1.1.2", +] + +[[package]] +name = "xcap" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6ad471d5ba232bc276382d26a9d3b837d6853b7df389058b5bb1e94dcdd248c" +dependencies = [ + "dispatch2", + "image", + "libwayshot-xcap", + "log", + "objc2 0.6.4", + "objc2-app-kit 0.3.2", + "objc2-av-foundation", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-core-media", + "objc2-core-video", + "objc2-foundation 0.3.2", + "percent-encoding", + "pipewire", + "rand 0.9.2", + "scopeguard", + "serde", + "thiserror 2.0.17", + "url", + "widestring", + "windows 0.62.2", + "xcb", + "zbus", +] + +[[package]] +name = "xcb" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee4c580d8205abb0a5cf4eb7e927bd664e425b6c3263f9c5310583da96970cf6" +dependencies = [ + "bitflags 1.3.2", + "libc", + "quick-xml 0.30.0", ] [[package]] @@ -8583,6 +9206,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "636f85e5ca6488e96401b61eb7de54f4e44755c988af0f52cf90230c312a1a89" +[[package]] +name = "xml-rs" +version = "0.8.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae8337f8a065cfc972643663ea4279e04e7256de865aa66fe25cec5fb912d3f" + [[package]] name = "y4m" version = "0.8.0" diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.toml index 58bbb5ce..c8b8c7fe 100644 --- a/apps/desktop/src-tauri/Cargo.toml +++ b/apps/desktop/src-tauri/Cargo.toml @@ -67,6 +67,7 @@ clipboard-rs = "0.3.4" html5gum = { version = "0.8.3", default-features = false } sha2 = "0.10" velopack = { version = "=0.0.1589-ga2c5a97", features = ["public-utils"] } +xcap = "0.9.6" [dev-dependencies] tempfile = "3" @@ -109,6 +110,7 @@ windows = { version = "0.58", features = [ "Win32_System_Com", "Win32_System_Threading", "Win32_UI_Controls_Dialogs", + "Win32_UI_Accessibility", "Win32_UI_Input_KeyboardAndMouse", "Win32_UI_Shell", "Win32_UI_Shell_Common", diff --git a/apps/desktop/src-tauri/src/commands/desktop_context.rs b/apps/desktop/src-tauri/src/commands/desktop_context.rs index e588de99..4d2469cd 100644 --- a/apps/desktop/src-tauri/src/commands/desktop_context.rs +++ b/apps/desktop/src-tauri/src/commands/desktop_context.rs @@ -2,7 +2,7 @@ //! Desktop context commands. -use tauri::State; +use tauri::{AppHandle, Runtime, State}; use crate::core::system::desktop_context::{ BoundDesktopContext, DesktopContextCapsule, DesktopContextRuntime, @@ -23,3 +23,14 @@ pub fn desktop_context_bind_capsule( ) -> Result, String> { runtime.bind_capsule(&capsule_id) } + +#[tauri::command] +pub fn desktop_context_capture_sensitive( + app: AppHandle, + runtime: State<'_, DesktopContextRuntime>, + capsule_id: String, + include: Vec, + screenshot_target: Option, +) -> Result, String> { + runtime.capture_sensitive(&app, &capsule_id, &include, screenshot_target.as_deref()) +} diff --git a/apps/desktop/src-tauri/src/commands/mod.rs b/apps/desktop/src-tauri/src/commands/mod.rs index 503d4ccc..edd71a37 100644 --- a/apps/desktop/src-tauri/src/commands/mod.rs +++ b/apps/desktop/src-tauri/src/commands/mod.rs @@ -45,6 +45,7 @@ pub fn invoke_handler( clipboard::write_clipboard_text, desktop_context::desktop_context_get_capsule, desktop_context::desktop_context_bind_capsule, + desktop_context::desktop_context_capture_sensitive, autostart::enable_autostart, autostart::disable_autostart, autostart::is_autostart_enabled, diff --git a/apps/desktop/src-tauri/src/core/system/desktop_context.rs b/apps/desktop/src-tauri/src/core/system/desktop_context.rs index 8453a2bf..e9545676 100644 --- a/apps/desktop/src-tauri/src/core/system/desktop_context.rs +++ b/apps/desktop/src-tauri/src/core/system/desktop_context.rs @@ -4,18 +4,30 @@ use std::{ collections::HashMap, + fs, + path::PathBuf, sync::{ atomic::{AtomicU64, Ordering}, Mutex, }, }; +use image::{imageops, RgbaImage}; use serde::Serialize; use tauri::{AppHandle, Manager, Runtime}; +use xcap::{Monitor, Window}; -use crate::core::system::clipboard::{ClipboardPayload, ClipboardRuntime}; +use crate::core::system::{ + clipboard::{ClipboardPayload, ClipboardRuntime}, + paths::{app_directory_path, AppDirectory}, +}; const CLIPBOARD_SUMMARY_LIMIT: usize = 500; +const SELECTED_TEXT_LIMIT: usize = 20_000; +const UIA_TEXT_PATTERN_DESCENDANT_LIMIT: i32 = 500; +const NATIVE_TEXT_BUFFER_LIMIT: usize = 1_000_000; +const SENSITIVE_ACCESS_REQUIRES_APPROVAL: &str = + "Requires user approval before reading this desktop context signal."; #[derive(Clone, Debug, Serialize)] #[serde(rename_all = "camelCase")] @@ -123,6 +135,12 @@ pub struct DesktopContextRuntime { capsules: Mutex>, } +struct CaptureOutcome { + value: Option, + method: &'static str, + reason: Option, +} + impl DesktopContextRuntime { pub fn new() -> Self { Self::default() @@ -134,17 +152,15 @@ impl DesktopContextRuntime { source: &'static str, ) -> Result { let sequence = self.next_sequence.fetch_add(1, Ordering::Relaxed) + 1; + let capsule_id = format!("desktop-context-{sequence}"); let captured_at = now_rfc3339_millis(); - let active_window = capture_active_window(); - let clipboard = capture_clipboard(app_handle); - let selected_text = unsupported_selected_text(); - let screenshot = unsupported_screenshot(&captured_at); - let capabilities = build_capabilities(&active_window, &clipboard, &screenshot); - let redactions = vec![DesktopContextRedaction { - field: "clipboard.fullText".to_string(), - reason: "Only returned when the model explicitly asks for clipboard.full_text." - .to_string(), - }]; + let active_window_capture = capture_active_window(); + let active_window = active_window_capture.value.clone(); + let clipboard = pending_clipboard(app_handle); + let selected_text = pending_selected_text(); + let screenshot = pending_screenshot(); + let capabilities = build_initial_capabilities(&active_window_capture, &clipboard); + let redactions = sensitive_redactions(); let summary = build_summary( active_window.as_ref(), &selected_text, @@ -153,7 +169,7 @@ impl DesktopContextRuntime { ); let capsule = DesktopContextCapsule { - id: format!("desktop-context-{sequence}"), + id: capsule_id, sequence, captured_at, invocation_source: source.to_string(), @@ -175,6 +191,56 @@ impl DesktopContextRuntime { Ok(capsule) } + pub fn capture_sensitive( + &self, + app_handle: &AppHandle, + capsule_id: &str, + include: &[String], + screenshot_target: Option<&str>, + ) -> Result, String> { + let mut capsules = self + .capsules + .lock() + .map_err(|error| format!("Desktop context state is poisoned: {error}"))?; + let Some(existing) = capsules.get(capsule_id).cloned() else { + return Ok(None); + }; + + let mut capsule = existing; + if include_requests_clipboard(include) { + capsule.clipboard = capture_clipboard(app_handle); + } + if include_requests_selected_text(include) { + capsule.selected_text = capture_selected_text(capsule.active_window.as_ref()); + } + if include_requests_screenshot(include) { + let captured_at = now_rfc3339_millis(); + capsule.screenshot = capture_screenshot( + &capsule.id, + &captured_at, + capsule.active_window.as_ref(), + screenshot_target, + ); + } + + capsule.capabilities = build_sensitive_capabilities( + capsule.active_window.as_ref(), + &capsule.selected_text, + &capsule.clipboard, + &capsule.screenshot, + ); + capsule.redactions = sensitive_redactions(); + capsule.summary = build_summary( + capsule.active_window.as_ref(), + &capsule.selected_text, + &capsule.clipboard, + &capsule.screenshot, + ); + + capsules.insert(capsule.id.clone(), capsule.clone()); + Ok(Some(capsule)) + } + pub fn get_capsule(&self, capsule_id: &str) -> Result, String> { let capsules = self .capsules @@ -258,21 +324,561 @@ fn capture_clipboard(app_handle: &AppHandle) -> DesktopContextCli } } -fn unsupported_selected_text() -> DesktopContextSelectedText { +fn pending_clipboard(app_handle: &AppHandle) -> DesktopContextClipboard { + if app_handle.try_state::().is_none() { + return DesktopContextClipboard { + available: false, + snapshot_id: None, + observed_at: None, + text: None, + text_summary: None, + text_length: 0, + image_count: 0, + file_count: 0, + reason: Some("Clipboard runtime is not initialized.".to_string()), + }; + } + + DesktopContextClipboard { + available: false, + snapshot_id: None, + observed_at: None, + text: None, + text_summary: None, + text_length: 0, + image_count: 0, + file_count: 0, + reason: Some(SENSITIVE_ACCESS_REQUIRES_APPROVAL.to_string()), + } +} + +fn unavailable_selected_text( + source: Option<&'static str>, + reason: impl Into, +) -> DesktopContextSelectedText { DesktopContextSelectedText { available: false, - source: None, + source: source.map(str::to_string), text: None, text_length: 0, truncated: false, - reason: Some( - "Native selected-text extraction is not enabled in this first desktop-context slice." + reason: Some(reason.into()), + } +} + +fn selected_text_from_text(source: &'static str, text: String) -> DesktopContextSelectedText { + let text_length = text.chars().count(); + if text_length == 0 { + return unavailable_selected_text( + Some(source), + "The focused element reported no selected text.", + ); + } + + let truncated = text_length > SELECTED_TEXT_LIMIT; + let stored_text = if truncated { + text.chars().take(SELECTED_TEXT_LIMIT).collect() + } else { + text + }; + + DesktopContextSelectedText { + available: true, + source: Some(source.to_string()), + text: Some(stored_text), + text_length, + truncated, + reason: None, + } +} + +fn normalize_text_selection_range(start: usize, end: usize) -> Option<(usize, usize)> { + if start == end { + return None; + } + + Some((start.min(end), start.max(end))) +} + +fn selected_text_from_utf16_range( + source: &'static str, + text: &[u16], + start: usize, + end: usize, +) -> Result { + let Some((start, end)) = normalize_text_selection_range(start, end) else { + return Err("The native text control reported no selected text.".to_string()); + }; + if end > text.len() { + return Err("The native text control selection is outside the captured text.".to_string()); + } + + Ok(selected_text_from_text( + source, + String::from_utf16_lossy(&text[start..end]), + )) +} + +fn pending_selected_text() -> DesktopContextSelectedText { + unavailable_selected_text( + Some(selected_text_provider_method()), + SENSITIVE_ACCESS_REQUIRES_APPROVAL, + ) +} + +fn capture_selected_text( + active_window: Option<&DesktopContextActiveWindow>, +) -> DesktopContextSelectedText { + match capture_selected_text_result(active_window) { + Ok(selected_text) => selected_text, + Err(error) => unavailable_selected_text(Some(selected_text_provider_method()), error), + } +} + +fn selected_text_provider_method() -> &'static str { + if cfg!(target_os = "windows") { + "windows-uia-textpattern" + } else if cfg!(target_os = "macos") { + "macos-accessibility-pending" + } else if cfg!(target_os = "linux") { + "linux-selection-unsupported" + } else { + "unsupported-platform" + } +} + +#[cfg(target_os = "windows")] +fn capture_selected_text_result( + active_window: Option<&DesktopContextActiveWindow>, +) -> Result { + use windows::{ + core::{HRESULT, VARIANT}, + Win32::{ + Foundation::{BOOL, HWND, LPARAM, RPC_E_CHANGED_MODE, S_FALSE, S_OK, WPARAM}, + System::Com::{ + CoCreateInstance, CoInitializeEx, CoUninitialize, CLSCTX_INPROC_SERVER, + COINIT_APARTMENTTHREADED, + }, + UI::Accessibility::{ + CUIAutomation, IUIAutomation, IUIAutomationElement, IUIAutomationTextPattern, + TreeScope_Descendants, UIA_IsTextPatternAvailablePropertyId, UIA_TextPatternId, + }, + UI::WindowsAndMessaging::{ + EnumChildWindows, GetClassNameW, SendMessageTimeoutW, SMTO_ABORTIFHUNG, WM_GETTEXT, + WM_GETTEXTLENGTH, + }, + }, + }; + + const EM_GETSEL: u32 = 0x00B0; + const NATIVE_TEXT_CHILD_SCAN_LIMIT: usize = 512; + const TEXT_MESSAGE_TIMEOUT_MS: u32 = 80; + + struct CoInitializeGuard { + should_uninitialize: bool, + } + + impl Drop for CoInitializeGuard { + fn drop(&mut self) { + if self.should_uninitialize { + unsafe { + CoUninitialize(); + } + } + } + } + + fn coinitialize_guard() -> Result { + let result = unsafe { CoInitializeEx(None, COINIT_APARTMENTTHREADED) }; + if result == S_OK || result == S_FALSE { + return Ok(CoInitializeGuard { + should_uninitialize: true, + }); + } + if result == RPC_E_CHANGED_MODE { + return Ok(CoInitializeGuard { + should_uninitialize: false, + }); + } + + Err(format!( + "Failed to initialize COM for UI Automation: {result:?}" + )) + } + + fn stringify_hresult(error: windows::core::Error) -> String { + let code: HRESULT = error.code(); + format!("{error} ({code:?})") + } + + fn selected_text_from_element( + element: &IUIAutomationElement, + source: &'static str, + ) -> Result { + let text_pattern: IUIAutomationTextPattern = + unsafe { element.GetCurrentPatternAs(UIA_TextPatternId) }.map_err(|error| { + format!( + "Element does not expose UI Automation TextPattern: {}", + stringify_hresult(error) + ) + })?; + selected_text_from_pattern(&text_pattern, source) + } + + fn selected_text_from_pattern( + text_pattern: &IUIAutomationTextPattern, + source: &'static str, + ) -> Result { + let selection = unsafe { text_pattern.GetSelection() } + .map_err(|error| format!("Failed to read UI Automation text selection: {error}"))?; + let selection_count = unsafe { selection.Length() } + .map_err(|error| format!("Failed to count selected text ranges: {error}"))?; + if selection_count <= 0 { + return Err("The element reported no selected text ranges.".to_string()); + } + + let mut ranges = Vec::new(); + for index in 0..selection_count { + let range = unsafe { selection.GetElement(index) } + .map_err(|error| format!("Failed to read selected text range {index}: {error}"))?; + let text = unsafe { range.GetText((SELECTED_TEXT_LIMIT + 1) as i32) } + .map_err(|error| format!("Failed to read selected text range {index}: {error}"))? + .to_string(); + if !text.is_empty() { + ranges.push(text); + } + } + + if ranges.is_empty() { + return Err("The element reported empty selected text ranges.".to_string()); + } + + Ok(selected_text_from_text(source, ranges.join("\n"))) + } + + fn hwnd_is_null(hwnd: HWND) -> bool { + hwnd.0.is_null() + } + + fn parse_hwnd(window_handle: Option<&str>) -> Option { + let value = window_handle?; + let trimmed = value.trim().strip_prefix("0x").unwrap_or(value.trim()); + usize::from_str_radix(trimmed, 16) + .ok() + .filter(|value| *value != 0) + .map(|value| HWND(value as _)) + } + + fn send_text_message( + hwnd: HWND, + message: u32, + wparam: usize, + lparam: isize, + ) -> Result { + let mut result = 0usize; + let status = unsafe { + SendMessageTimeoutW( + hwnd, + message, + WPARAM(wparam), + LPARAM(lparam), + SMTO_ABORTIFHUNG, + TEXT_MESSAGE_TIMEOUT_MS, + Some(&mut result), + ) + }; + if status.0 == 0 { + return Err(format!( + "Native text control message 0x{message:x} timed out or failed." + )); + } + + Ok(result) + } + + fn native_text_selection_range(hwnd: HWND) -> Result<(usize, usize), String> { + let mut start = 0u32; + let mut end = 0u32; + send_text_message( + hwnd, + EM_GETSEL, + &mut start as *mut u32 as usize, + &mut end as *mut u32 as isize, + )?; + + Ok((start as usize, end as usize)) + } + + fn read_native_text_prefix(hwnd: HWND, end: usize) -> Result, String> { + if end > NATIVE_TEXT_BUFFER_LIMIT { + return Err(format!( + "Native text control selection exceeds the {NATIVE_TEXT_BUFFER_LIMIT} UTF-16 unit read limit." + )); + } + + let reported_length = send_text_message(hwnd, WM_GETTEXTLENGTH, 0, 0)?; + if reported_length == 0 { + return Err("Native text control reported empty text.".to_string()); + } + + let read_length = reported_length.min(end).min(NATIVE_TEXT_BUFFER_LIMIT); + if read_length < end { + return Err("Native text control selection is outside the readable text.".to_string()); + } + + let mut buffer = vec![0u16; read_length + 1]; + let copied = + send_text_message(hwnd, WM_GETTEXT, buffer.len(), buffer.as_mut_ptr() as isize)? + .min(read_length); + buffer.truncate(copied); + Ok(buffer) + } + + fn selected_text_from_native_text_control( + hwnd: HWND, + ) -> Result { + let (start, end) = native_text_selection_range(hwnd)?; + let Some((_, normalized_end)) = normalize_text_selection_range(start, end) else { + return Err("Native text control reported no selected text.".to_string()); + }; + let text = read_native_text_prefix(hwnd, normalized_end)?; + + selected_text_from_utf16_range("windows-win32-native-text-control", &text, start, end) + } + + unsafe extern "system" fn collect_child_hwnd(hwnd: HWND, lparam: LPARAM) -> BOOL { + let handles = &mut *(lparam.0 as *mut Vec); + if handles.len() >= NATIVE_TEXT_CHILD_SCAN_LIMIT { + return BOOL(0); + } + + handles.push(hwnd); + BOOL(1) + } + + fn child_hwnds(root: HWND) -> Vec { + let mut handles = Vec::new(); + unsafe { + let _ = EnumChildWindows( + root, + Some(collect_child_hwnd), + LPARAM(&mut handles as *mut Vec as isize), + ); + } + handles + } + + fn window_class_name(hwnd: HWND) -> Option { + let mut buffer = [0u16; 256]; + let length = unsafe { GetClassNameW(hwnd, &mut buffer) }; + (length > 0).then(|| String::from_utf16_lossy(&buffer[..length as usize])) + } + + fn is_native_text_class(class_name: &str) -> bool { + let normalized = class_name.to_ascii_lowercase(); + normalized == "edit" || normalized.contains("richedit") + } + + fn selected_text_from_native_window(hwnd: HWND) -> Result { + if hwnd_is_null(hwnd) { + return Err("Invocation active window handle is empty.".to_string()); + } + + let mut errors = Vec::new(); + let mut scanned = 0usize; + for candidate in std::iter::once(hwnd).chain(child_hwnds(hwnd).into_iter()) { + let Some(class_name) = window_class_name(candidate) else { + continue; + }; + if !is_native_text_class(&class_name) { + continue; + } + + scanned += 1; + match selected_text_from_native_text_control(candidate) { + Ok(selected_text) => return Ok(selected_text), + Err(error) => errors.push(format!("{class_name}: {error}")), + } + } + + if scanned == 0 { + Err("No native Edit/RichEdit text controls were found under the invocation active window.".to_string()) + } else { + Err(format!( + "No selected text was found in {scanned} native text controls: {}", + errors.join("; ") + )) + } + } + + fn selected_text_from_descendants( + automation: &IUIAutomation, + root: &IUIAutomationElement, + ) -> Result { + let is_text_pattern_available = VARIANT::from(true); + let condition = unsafe { + automation.CreatePropertyCondition( + UIA_IsTextPatternAvailablePropertyId, + &is_text_pattern_available, + ) + } + .map_err(|error| { + format!("Failed to create UI Automation TextPattern search condition: {error}") + })?; + let descendants = unsafe { root.FindAll(TreeScope_Descendants, &condition) } + .map_err(|error| format!("Failed to enumerate UI Automation descendants: {error}"))?; + let descendant_count = unsafe { descendants.Length() } + .map_err(|error| format!("Failed to count UI Automation descendants: {error}"))?; + let limit = descendant_count.min(UIA_TEXT_PATTERN_DESCENDANT_LIMIT); + + for index in 0..limit { + let Ok(element) = (unsafe { descendants.GetElement(index) }) else { + continue; + }; + if let Ok(selected_text) = + selected_text_from_element(&element, "windows-uia-descendant-textpattern") + { + return Ok(selected_text); + } + } + + Err(format!( + "No selected text was found in the first {limit} UI Automation descendants." + )) + } + + fn element_process_id(element: &IUIAutomationElement) -> Option { + unsafe { element.CurrentProcessId() } + .ok() + .and_then(|process_id| u32::try_from(process_id).ok()) + } + + fn focused_element_matches_invocation_window( + focused: &IUIAutomationElement, + active_window: Option<&DesktopContextActiveWindow>, + ) -> bool { + let Some(active_window) = active_window else { + return true; + }; + let Some(invocation_process_id) = active_window.process_id else { + return false; + }; + + element_process_id(focused) == Some(invocation_process_id) + } + + let _guard = coinitialize_guard()?; + let automation: IUIAutomation = + unsafe { CoCreateInstance(&CUIAutomation, None, CLSCTX_INPROC_SERVER) } + .map_err(|error| format!("Failed to create UI Automation client: {error}"))?; + + let mut invocation_window_error: Option = None; + if let Some(hwnd) = parse_hwnd(active_window.and_then(|window| window.window_handle.as_deref())) + { + invocation_window_error = Some(match unsafe { automation.ElementFromHandle(hwnd) } { + Ok(element) => { + let direct_error = + match selected_text_from_element(&element, "windows-uia-window-textpattern") { + Ok(selected_text) => return Ok(selected_text), + Err(error) => error, + }; + match selected_text_from_descendants(&automation, &element) { + Ok(selected_text) => return Ok(selected_text), + Err(error) => match selected_text_from_native_window(hwnd) { + Ok(selected_text) => return Ok(selected_text), + Err(native_error) => { + format!("{direct_error}; {error}; {native_error}") + } + }, + } + } + Err(error) => { + log::warn!( + "Failed to read UI Automation element from active window handle: {}", + error + ); + match selected_text_from_native_window(hwnd) { + Ok(selected_text) => return Ok(selected_text), + Err(native_error) => format!( + "Failed to read UI Automation element from invocation active window handle: {error}; {native_error}" + ), + } + } + }); + } + + let focused = unsafe { automation.GetFocusedElement() } + .map_err(|error| format!("Failed to get focused UI Automation element: {error}"))?; + if !focused_element_matches_invocation_window(&focused, active_window) { + return Err(invocation_window_error.unwrap_or_else(|| { + "Focused element no longer belongs to the invocation active window; selected text was not read to avoid capturing TouchAI approval UI focus.".to_string() + })); + } + + match selected_text_from_element(&focused, "windows-uia-focused-textpattern") { + Ok(selected_text) => Ok(selected_text), + Err(focused_error) => { + let native_error = match unsafe { focused.CurrentNativeWindowHandle() } { + Ok(hwnd) => match selected_text_from_native_window(hwnd) { + Ok(selected_text) => return Ok(selected_text), + Err(error) => Some(error), + }, + Err(error) => Some(format!( + "Focused element does not expose a native window handle: {error}" + )), + }; + if let Some(window_error) = invocation_window_error { + Err(format!( + "{window_error}; focused fallback failed: {focused_error}; {}", + native_error.unwrap_or_else(|| "native focused fallback failed".to_string()) + )) + } else { + Err(native_error + .map(|error| format!("{focused_error}; {error}")) + .unwrap_or(focused_error)) + } + } + } +} + +#[cfg(not(target_os = "windows"))] +fn capture_selected_text_result( + _active_window: Option<&DesktopContextActiveWindow>, +) -> Result { + if cfg!(target_os = "macos") { + Err( + "macOS selected-text extraction requires an Accessibility provider and user permission." + .to_string(), + ) + } else if cfg!(target_os = "linux") { + Err( + "Linux selected-text extraction has no stable cross-desktop read-only API; Wayland support is compositor dependent." .to_string(), - ), + ) + } else { + Err("Selected-text extraction is not supported on this platform.".to_string()) } } -fn unsupported_screenshot(captured_at: &str) -> DesktopContextScreenshot { +fn unsupported_screenshot_for_target( + captured_at: &str, + target: &str, + reason: String, +) -> DesktopContextScreenshot { + DesktopContextScreenshot { + available: false, + path: None, + mime_type: None, + width: None, + height: None, + target: target.to_string(), + persisted: false, + captured_at: Some(captured_at.to_string()), + reason: Some(reason), + } +} + +fn pending_screenshot() -> DesktopContextScreenshot { DesktopContextScreenshot { available: false, path: None, @@ -281,32 +887,319 @@ fn unsupported_screenshot(captured_at: &str) -> DesktopContextScreenshot { height: None, target: "active_display".to_string(), persisted: false, + captured_at: None, + reason: Some(SENSITIVE_ACCESS_REQUIRES_APPROVAL.to_string()), + } +} + +fn sanitize_artifact_file_stem(value: &str) -> String { + let sanitized = value + .chars() + .map(|ch| { + if ch.is_ascii_alphanumeric() || ch == '-' || ch == '_' { + ch + } else { + '_' + } + }) + .collect::(); + + if sanitized.trim_matches('_').is_empty() { + "desktop-context".to_string() + } else { + sanitized + } +} + +fn screenshot_artifact_path(data_root: PathBuf, capsule_id: &str) -> PathBuf { + data_root + .join("desktop-context") + .join("screenshots") + .join(format!("{}.png", sanitize_artifact_file_stem(capsule_id))) +} + +fn include_requests_key(include: &[String], key: &str) -> bool { + include.iter().any(|item| item == key) +} + +fn include_requests_selected_text(include: &[String]) -> bool { + include_requests_key(include, "selected_text.summary") + || include_requests_key(include, "selected_text.full_text") +} + +fn include_requests_clipboard(include: &[String]) -> bool { + include_requests_key(include, "clipboard.summary") + || include_requests_key(include, "clipboard.full_text") +} + +fn include_requests_screenshot(include: &[String]) -> bool { + include_requests_key(include, "screenshot.metadata") + || include_requests_key(include, "screenshot.image") +} + +fn normalize_screenshot_target(target: Option<&str>) -> &'static str { + match target { + Some("active_window") => "active_window", + Some("all_displays") => "all_displays", + Some("active_display") | Some("capsule_default") | None => "active_display", + Some(_) => "active_display", + } +} + +fn active_window_center(active_window: Option<&DesktopContextActiveWindow>) -> Option<(i32, i32)> { + let bounds = active_window?.bounds.as_ref()?; + if bounds.width <= 0 || bounds.height <= 0 { + return None; + } + + Some(( + bounds.x.saturating_add(bounds.width / 2), + bounds.y.saturating_add(bounds.height / 2), + )) +} + +fn select_capture_monitor( + active_window: Option<&DesktopContextActiveWindow>, +) -> Result { + if let Some((x, y)) = active_window_center(active_window) { + if let Ok(monitor) = Monitor::from_point(x, y) { + return Ok(monitor); + } + } + + let monitors = + Monitor::all().map_err(|error| format!("Failed to enumerate monitors: {error}"))?; + monitors + .iter() + .find_map(|monitor| match monitor.is_primary() { + Ok(true) => Some(monitor.clone()), + _ => None, + }) + .or_else(|| monitors.into_iter().next()) + .ok_or_else(|| "No monitor is available for screenshot capture.".to_string()) +} + +fn capture_screenshot( + capsule_id: &str, + captured_at: &str, + active_window: Option<&DesktopContextActiveWindow>, + screenshot_target: Option<&str>, +) -> DesktopContextScreenshot { + let target = normalize_screenshot_target(screenshot_target); + match capture_screenshot_result(capsule_id, captured_at, active_window, target) { + Ok(screenshot) => screenshot, + Err(error) => unsupported_screenshot_for_target(captured_at, target, error), + } +} + +fn capture_screenshot_result( + capsule_id: &str, + captured_at: &str, + active_window: Option<&DesktopContextActiveWindow>, + target: &'static str, +) -> Result { + let data_root = app_directory_path(AppDirectory::Data)?; + let target_path = screenshot_artifact_path(data_root, capsule_id); + if let Some(parent) = target_path.parent() { + fs::create_dir_all(parent).map_err(|error| { + format!("Failed to create desktop context screenshot directory: {error}") + })?; + } + + let image = match target { + "active_window" => capture_active_window_image(active_window)?, + "all_displays" => capture_all_displays_image()?, + _ => capture_active_display_image(active_window)?, + }; + let width = image.width(); + let height = image.height(); + image + .save(&target_path) + .map_err(|error| format!("Failed to save desktop context screenshot: {error}"))?; + + Ok(DesktopContextScreenshot { + available: true, + path: Some(target_path.to_string_lossy().into_owned()), + mime_type: Some("image/png".to_string()), + width: Some(width), + height: Some(height), + target: target.to_string(), + persisted: true, captured_at: Some(captured_at.to_string()), - reason: Some( - "Native screenshot persistence is reserved for the follow-up capture backend." - .to_string(), - ), + reason: None, + }) +} + +fn capture_active_display_image( + active_window: Option<&DesktopContextActiveWindow>, +) -> Result { + let monitor = select_capture_monitor(active_window)?; + monitor + .capture_image() + .map_err(|error| format!("Failed to capture active display screenshot: {error}")) +} + +fn parse_window_provider_id(handle: Option<&str>) -> Option { + let value = handle?.trim(); + if let Some(hex) = value.strip_prefix("0x") { + return u32::from_str_radix(hex, 16).ok(); + } + if let Some(decimal) = value.strip_prefix("xcap-window-") { + return decimal.parse::().ok(); + } + value.parse::().ok() +} + +fn find_xcap_window_for_active_window( + active_window: &DesktopContextActiveWindow, +) -> Result { + let target_id = parse_window_provider_id(active_window.window_handle.as_deref()); + let windows = Window::all().map_err(|error| format!("Failed to enumerate windows: {error}"))?; + + if let Some(target_id) = target_id { + if let Some(window) = windows + .iter() + .find(|window| window.id().ok() == Some(target_id)) + { + return Ok(window.clone()); + } + } + + let expected_pid = active_window.process_id; + let expected_title = active_window + .title + .as_deref() + .filter(|title| !title.is_empty()); + windows + .into_iter() + .find(|window| { + expected_pid.is_some_and(|pid| window.pid().ok() == Some(pid)) + && expected_title.is_none_or(|title| window.title().ok().as_deref() == Some(title)) + }) + .ok_or_else(|| { + "Failed to find the invocation active window for screenshot capture.".to_string() + }) +} + +fn capture_active_window_region_fallback( + active_window: &DesktopContextActiveWindow, +) -> Result { + let bounds = active_window + .bounds + .as_ref() + .ok_or_else(|| "Active window bounds are unavailable.".to_string())?; + if bounds.width <= 0 || bounds.height <= 0 { + return Err("Active window bounds are empty.".to_string()); } + + let monitor = select_capture_monitor(Some(active_window))?; + let monitor_x = monitor + .x() + .map_err(|error| format!("Failed to read monitor x coordinate: {error}"))?; + let monitor_y = monitor + .y() + .map_err(|error| format!("Failed to read monitor y coordinate: {error}"))?; + let relative_x = bounds.x.saturating_sub(monitor_x); + let relative_y = bounds.y.saturating_sub(monitor_y); + if relative_x < 0 || relative_y < 0 { + return Err("Active window is outside the selected monitor capture bounds.".to_string()); + } + + monitor + .capture_region( + relative_x as u32, + relative_y as u32, + bounds.width as u32, + bounds.height as u32, + ) + .map_err(|error| format!("Failed to capture active window screenshot region: {error}")) +} + +fn capture_active_window_image( + active_window: Option<&DesktopContextActiveWindow>, +) -> Result { + let active_window = active_window.ok_or_else(|| { + "No invocation active window is available for screenshot capture.".to_string() + })?; + if let Ok(window) = find_xcap_window_for_active_window(active_window) { + if let Ok(image) = window.capture_image() { + return Ok(image); + } + } + + capture_active_window_region_fallback(active_window) +} + +fn capture_all_displays_image() -> Result { + struct CapturedDisplay { + x: i32, + y: i32, + image: RgbaImage, + } + + let monitors = + Monitor::all().map_err(|error| format!("Failed to enumerate monitors: {error}"))?; + let mut captures = Vec::new(); + for monitor in monitors { + let x = monitor + .x() + .map_err(|error| format!("Failed to read monitor x coordinate: {error}"))?; + let y = monitor + .y() + .map_err(|error| format!("Failed to read monitor y coordinate: {error}"))?; + let image = monitor + .capture_image() + .map_err(|error| format!("Failed to capture monitor screenshot: {error}"))?; + captures.push(CapturedDisplay { x, y, image }); + } + + if captures.is_empty() { + return Err("No monitor is available for screenshot capture.".to_string()); + } + + let min_x = captures.iter().map(|capture| capture.x).min().unwrap_or(0); + let min_y = captures.iter().map(|capture| capture.y).min().unwrap_or(0); + let max_x = captures + .iter() + .map(|capture| capture.x.saturating_add(capture.image.width() as i32)) + .max() + .unwrap_or(0); + let max_y = captures + .iter() + .map(|capture| capture.y.saturating_add(capture.image.height() as i32)) + .max() + .unwrap_or(0); + let width = max_x.saturating_sub(min_x) as u32; + let height = max_y.saturating_sub(min_y) as u32; + if width == 0 || height == 0 { + return Err("Combined display screenshot bounds are empty.".to_string()); + } + + let mut canvas = RgbaImage::new(width, height); + for capture in captures { + imageops::overlay( + &mut canvas, + &capture.image, + i64::from(capture.x.saturating_sub(min_x)), + i64::from(capture.y.saturating_sub(min_y)), + ); + } + + Ok(canvas) } fn build_capabilities( - active_window: &Option, + active_window: &CaptureOutcome, + selected_text: &DesktopContextSelectedText, clipboard: &DesktopContextClipboard, screenshot: &DesktopContextScreenshot, ) -> Vec { vec![ DesktopContextCapability { id: "active_window".to_string(), - supported: active_window.is_some(), - method: if cfg!(target_os = "windows") { - "win32-foreground-window" - } else { - "unsupported-platform" - } - .to_string(), - reason: active_window - .is_none() - .then(|| "Foreground window metadata is only implemented on Windows.".to_string()), + supported: active_window.value.is_some(), + method: active_window.method.to_string(), + reason: active_window.reason.clone(), }, DesktopContextCapability { id: "clipboard".to_string(), @@ -316,21 +1209,73 @@ fn build_capabilities( }, DesktopContextCapability { id: "selected_text".to_string(), - supported: false, - method: "pending-native-backend".to_string(), - reason: Some( - "Selected-text extraction requires a dedicated native backend.".to_string(), - ), + supported: selected_text.available, + method: selected_text + .source + .clone() + .unwrap_or_else(|| selected_text_provider_method().to_string()), + reason: selected_text.reason.clone(), }, DesktopContextCapability { id: "screenshot".to_string(), supported: screenshot.available, - method: "pending-capture-backend".to_string(), + method: "xcap-monitor-capture".to_string(), reason: screenshot.reason.clone(), }, ] } +fn build_initial_capabilities( + active_window: &CaptureOutcome, + clipboard: &DesktopContextClipboard, +) -> Vec { + build_capabilities( + active_window, + &pending_selected_text(), + clipboard, + &pending_screenshot(), + ) +} + +fn build_sensitive_capabilities( + active_window: Option<&DesktopContextActiveWindow>, + selected_text: &DesktopContextSelectedText, + clipboard: &DesktopContextClipboard, + screenshot: &DesktopContextScreenshot, +) -> Vec { + build_capabilities( + &CaptureOutcome { + value: active_window.cloned(), + method: "captured-invocation-active-window", + reason: active_window + .is_none() + .then(|| "No active window metadata was captured at invocation time.".to_string()), + }, + selected_text, + clipboard, + screenshot, + ) +} + +fn sensitive_redactions() -> Vec { + vec![ + DesktopContextRedaction { + field: "selectedText.fullText".to_string(), + reason: "Only read after explicit user approval for selected_text.full_text." + .to_string(), + }, + DesktopContextRedaction { + field: "clipboard.fullText".to_string(), + reason: "Only read after explicit user approval for clipboard.full_text.".to_string(), + }, + DesktopContextRedaction { + field: "screenshot.path".to_string(), + reason: "Only captured and returned after explicit user approval for screenshot." + .to_string(), + }, + ] +} + fn build_summary( active_window: Option<&DesktopContextActiveWindow>, selected_text: &DesktopContextSelectedText, @@ -380,8 +1325,107 @@ fn now_rfc3339_millis() -> String { ) } +fn capture_active_window() -> CaptureOutcome { + match capture_active_window_with_xcap() { + Ok(active_window) => CaptureOutcome { + value: Some(active_window), + method: "xcap-focused-window", + reason: None, + }, + Err(error) => capture_active_window_fallback(error), + } +} + +fn capture_active_window_with_xcap() -> Result { + let windows = Window::all().map_err(|error| format!("Failed to enumerate windows: {error}"))?; + let focused_window = windows + .into_iter() + .find_map(|window| match window.is_focused() { + Ok(true) => Some(window), + _ => None, + }) + .ok_or_else(|| "No focused window was reported by the window provider.".to_string())?; + + let title = focused_window + .title() + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()); + let app_name = focused_window + .app_name() + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()); + let width = focused_window + .width() + .ok() + .and_then(|value| i32::try_from(value).ok()); + let height = focused_window + .height() + .ok() + .and_then(|value| i32::try_from(value).ok()); + let bounds = match ( + focused_window.x().ok(), + focused_window.y().ok(), + width, + height, + ) { + (Some(x), Some(y), Some(width), Some(height)) => Some(DesktopContextBounds { + x, + y, + width, + height, + }), + _ => None, + }; + + Ok(DesktopContextActiveWindow { + title, + app_name, + process_name: None, + process_id: focused_window.pid().ok(), + window_handle: focused_window.id().ok().map(format_xcap_window_handle), + bounds, + }) +} + #[cfg(target_os = "windows")] -fn capture_active_window() -> Option { +fn format_xcap_window_handle(id: u32) -> String { + format!("0x{id:x}") +} + +#[cfg(not(target_os = "windows"))] +fn format_xcap_window_handle(id: u32) -> String { + format!("xcap-window-{id}") +} + +#[cfg(target_os = "windows")] +fn capture_active_window_fallback(reason: String) -> CaptureOutcome { + match capture_active_window_with_win32() { + Some(active_window) => CaptureOutcome { + value: Some(active_window), + method: "win32-foreground-window", + reason: None, + }, + None => CaptureOutcome { + value: None, + method: "xcap-focused-window", + reason: Some(reason), + }, + } +} + +#[cfg(not(target_os = "windows"))] +fn capture_active_window_fallback(reason: String) -> CaptureOutcome { + CaptureOutcome { + value: None, + method: "xcap-focused-window", + reason: Some(reason), + } +} + +#[cfg(target_os = "windows")] +fn capture_active_window_with_win32() -> Option { use windows::Win32::{ Foundation::RECT, UI::WindowsAndMessaging::{ @@ -427,7 +1471,56 @@ fn capture_active_window() -> Option { } } -#[cfg(not(target_os = "windows"))] -fn capture_active_window() -> Option { - None +#[cfg(test)] +mod tests { + use super::{ + screenshot_artifact_path, selected_text_from_text, selected_text_from_utf16_range, + SELECTED_TEXT_LIMIT, + }; + use std::path::PathBuf; + + #[test] + fn screenshot_artifact_path_stays_under_desktop_context_directory() { + let path = + screenshot_artifact_path(PathBuf::from("E:/TouchAI/data"), "../desktop-context-7"); + + assert_eq!( + path, + PathBuf::from("E:/TouchAI/data") + .join("desktop-context") + .join("screenshots") + .join("___desktop-context-7.png") + ); + } + + #[test] + fn selected_text_payload_truncates_large_selections() { + let text = "a".repeat(SELECTED_TEXT_LIMIT + 7); + + let selected_text = selected_text_from_text("test-provider", text); + + assert!(selected_text.available); + assert!(selected_text.truncated); + assert_eq!(selected_text.text_length, SELECTED_TEXT_LIMIT + 7); + assert_eq!( + selected_text.text.unwrap().chars().count(), + SELECTED_TEXT_LIMIT + ); + } + + #[test] + fn selected_text_from_utf16_range_reads_native_control_offsets() { + let text = "before selected text after" + .encode_utf16() + .collect::>(); + let start = "before ".encode_utf16().count(); + let end = start + "selected text".encode_utf16().count(); + + let selected_text = + selected_text_from_utf16_range("test-provider", &text, start, end).unwrap(); + + assert!(selected_text.available); + assert_eq!(selected_text.text.as_deref(), Some("selected text")); + assert_eq!(selected_text.text_length, 13); + } } diff --git a/apps/desktop/src-tauri/src/core/window/search/mod.rs b/apps/desktop/src-tauri/src/core/window/search/mod.rs index db5be12e..23ab97d6 100644 --- a/apps/desktop/src-tauri/src/core/window/search/mod.rs +++ b/apps/desktop/src-tauri/src/core/window/search/mod.rs @@ -107,11 +107,21 @@ pub(super) fn show_and_activate_search_window( ) -> Result<(), String> { let _ = window.unminimize(); crate::core::window::rounded_corners::sync_window_corner_style(window)?; + protect_search_window_from_screen_capture(window); window.show().map_err(|e| e.to_string())?; crate::core::window::rounded_corners::sync_window_corner_style(window)?; reactivate_search_window_input(window) } +fn protect_search_window_from_screen_capture(window: &WebviewWindow) { + if let Err(error) = window.set_content_protected(true) { + log::warn!( + "Failed to protect search window from screen capture: {}", + error + ); + } +} + /// 让主搜索窗口重新成为键盘输入目标，并唤醒 WebView2 accelerator 管线。 pub(super) fn reactivate_search_window_input( window: &WebviewWindow, diff --git a/apps/desktop/src/database/queries/sessionTurnContextArtifacts.ts b/apps/desktop/src/database/queries/sessionTurnContextArtifacts.ts index 9e727ad0..b005943d 100644 --- a/apps/desktop/src/database/queries/sessionTurnContextArtifacts.ts +++ b/apps/desktop/src/database/queries/sessionTurnContextArtifacts.ts @@ -9,8 +9,7 @@ import type { SessionTurnContextArtifactEntity, } from '../types'; -export interface SessionTurnContextArtifactHistoryRow - extends SessionTurnContextArtifactEntity { +export interface SessionTurnContextArtifactHistoryRow extends SessionTurnContextArtifactEntity { prompt_message_id: number | null; } @@ -21,7 +20,21 @@ export async function createSessionTurnContextArtifact( const row = await database .insert(sessionTurnContextArtifacts) .values(data) - .onConflictDoNothing() + .onConflictDoUpdate({ + target: [ + sessionTurnContextArtifacts.turn_id, + sessionTurnContextArtifacts.capsule_id, + sessionTurnContextArtifacts.artifact_kind, + ], + set: { + artifact_path: data.artifact_path ?? null, + mime_type: data.mime_type ?? null, + width: data.width ?? null, + height: data.height ?? null, + captured_at: data.captured_at, + metadata_json: data.metadata_json ?? null, + }, + }) .returning() .get(); @@ -69,9 +82,6 @@ export async function findSessionTurnContextArtifactsBySessionId( .from(sessionTurnContextArtifacts) .innerJoin(sessionTurns, eq(sessionTurns.id, sessionTurnContextArtifacts.turn_id)) .where(eq(sessionTurns.session_id, sessionId)) - .orderBy( - asc(sessionTurnContextArtifacts.created_at), - asc(sessionTurnContextArtifacts.id) - ) + .orderBy(asc(sessionTurnContextArtifacts.created_at), asc(sessionTurnContextArtifacts.id)) .all(); } diff --git a/apps/desktop/src/i18n/textMap.ts b/apps/desktop/src/i18n/textMap.ts index 5501b966..26974f1f 100644 --- a/apps/desktop/src/i18n/textMap.ts +++ b/apps/desktop/src/i18n/textMap.ts @@ -533,7 +533,14 @@ export const zhToEnTextMap = { 'Failed to save retry checkpoint, but this turn will continue retrying.', 命令执行确认: 'Confirm command execution', 读取本地内容确认: 'Confirm local content read', + 桌面上下文读取确认: 'Confirm desktop context read', 模型切换确认: 'Confirm model switch', + 选中文本: 'selected text', + 剪贴板: 'clipboard', + 屏幕截图: 'screenshot', + 敏感桌面上下文: 'sensitive desktop context', + '模型请求读取 {targets}。批准后，TouchAI 会读取或捕获这些桌面上下文，并将结果发送给模型。': + 'The model requested {targets}. After approval, TouchAI will read or capture this desktop context and send the result to the model.', '允许从 {currentModel} 切换到 {targetModel}': 'Allow switching from {currentModel} to {targetModel}', 命令执行需要确认: 'Command execution needs confirmation', @@ -733,6 +740,7 @@ export const zhToEnTextMap = { 任务正在等待用户回复: 'Task is waiting for a user response', '回复 TouchAI': 'Reply to TouchAI', 回复: 'Reply', + 已绑定桌面上下文: 'Desktop context attached', } as const; export type SourceText = keyof typeof zhToEnTextMap; diff --git a/apps/desktop/src/services/AgentService/execution/executor.ts b/apps/desktop/src/services/AgentService/execution/executor.ts index e1727eda..dbb062f0 100644 --- a/apps/desktop/src/services/AgentService/execution/executor.ts +++ b/apps/desktop/src/services/AgentService/execution/executor.ts @@ -109,6 +109,7 @@ interface ToolExecutionResult { toolLogId: number | null; toolLogKind: ToolLogKind | null; attachments?: AttachmentIndex[]; + desktopContextArtifact?: BoundDesktopContext | null; builtInToolId?: BuiltInToolId; controlSignal?: BuiltInToolControlSignal; } @@ -438,6 +439,7 @@ export class AiRequestExecutor { toolLogId: number | null; toolLogKind: ToolLogKind | null; attachments?: AttachmentIndex[]; + desktopContextArtifact?: undefined; builtInToolId?: undefined; controlSignal?: undefined; }> { @@ -794,6 +796,7 @@ export class AiRequestExecutor { toolLogId, toolLogKind, attachments, + desktopContextArtifact, controlSignal, } of toolResults) { runtime.messages.push({ @@ -811,6 +814,10 @@ export class AiRequestExecutor { attachments ); + if (desktopContextArtifact) { + await options.persister.persistDesktopContextArtifact(desktopContextArtifact); + } + if (builtInToolId && !isError) { runtime.executedBuiltInTools.add(builtInToolId); } diff --git a/apps/desktop/src/services/AgentService/execution/runtime.ts b/apps/desktop/src/services/AgentService/execution/runtime.ts index f868cc1e..24c19c0d 100644 --- a/apps/desktop/src/services/AgentService/execution/runtime.ts +++ b/apps/desktop/src/services/AgentService/execution/runtime.ts @@ -7,9 +7,9 @@ import { t } from '@/i18n'; import type { AttachmentIndex } from '@/services/AgentService/infrastructure/attachments'; import { ensurePersistedAttachmentIndex } from '@/services/AgentService/infrastructure/attachments'; import { + type BoundDesktopContext, buildDesktopContextPromptMetadata, desktopContextService, - type BoundDesktopContext, } from '@/services/DesktopContextService'; import type { InputHistorySnapshot } from '@/types/session'; diff --git a/apps/desktop/src/services/AgentService/outputs/persistence.ts b/apps/desktop/src/services/AgentService/outputs/persistence.ts index cefcbd4a..cde76f2b 100644 --- a/apps/desktop/src/services/AgentService/outputs/persistence.ts +++ b/apps/desktop/src/services/AgentService/outputs/persistence.ts @@ -569,6 +569,16 @@ export class PersistenceProjector { ); } + async persistDesktopContextArtifact(context: BoundDesktopContext): Promise { + if (!this.turn) { + await this.ensureTurnRecord(); + } + + await db.transaction(async (tx) => { + await this.persistDesktopContextArtifacts(this.getTurnId(), tx, context); + }); + } + async persistCheckpoint(checkpoint: AttemptCheckpoint): Promise { if (!this.attemptState) { return; @@ -737,9 +747,9 @@ export class PersistenceProjector { private async persistDesktopContextArtifacts( turnId: number, - database: DatabaseExecutor = db + database: DatabaseExecutor = db, + context: BoundDesktopContext | null = this.desktopContext ): Promise { - const context = this.desktopContext; if (!context) { return; } @@ -759,6 +769,13 @@ export class PersistenceProjector { activeWindowAppName: context.activeWindow?.appName ?? null, selectedTextLength: context.selectedText.textLength, clipboardTextLength: context.clipboard.textLength, + screenshotAvailable: context.screenshot.available, + screenshotPersisted: context.screenshot.persisted, + screenshotWidth: context.screenshot.width, + screenshotHeight: context.screenshot.height, + screenshotCapturedAt: context.screenshot.capturedAt, + screenshotTarget: context.screenshot.target, + screenshotReason: context.screenshot.reason ?? null, capabilities: context.capabilities, redactions: context.redactions, }), diff --git a/apps/desktop/src/services/AgentService/prompt/composer.ts b/apps/desktop/src/services/AgentService/prompt/composer.ts index d9ae80b4..2aef3832 100644 --- a/apps/desktop/src/services/AgentService/prompt/composer.ts +++ b/apps/desktop/src/services/AgentService/prompt/composer.ts @@ -27,11 +27,23 @@ const BACKGROUND_MODE_PROMPT = [ '除非用户主动取消，或工具审批明确被拒绝，否则应继续完成原始任务。', ].join('\n'); -const DESKTOP_CONTEXT_TOOL_PROMPT = [ - '本轮请求绑定了一份只读桌面上下文胶囊。', - '如果用户的问题需要理解呼出 TouchAI 前的桌面、前台窗口、选中文本、剪贴板或截图，请调用 `builtin__get_desktop_context` 读取。', - '不要假设桌面上下文已经完整出现在 prompt 中；不要执行点击、输入、滚动、聚焦或控制外部应用等 computer use 行为。', -].join('\n'); +function buildDesktopContextToolPrompt(context: DesktopContextPromptMetadata): string { + const screenshotHint = context.screenshotAvailable + ? `本轮已有一张经用户批准后捕获的桌面截图${context.screenshotPersisted ? '并已持久化' : ''}${ + context.screenshotWidth && context.screenshotHeight + ? `，尺寸 ${context.screenshotWidth}x${context.screenshotHeight}` + : '' + }。如需图片文件路径，调用 \`builtin__get_desktop_context\` 并传入 \`include: ['screenshot.image']\`。` + : "如果需要屏幕截图，调用 `builtin__get_desktop_context` 并传入 `include: ['screenshot.image']`；TouchAI 会先请求用户批准，批准后再捕获并持久化截图。"; + + return [ + `本轮请求绑定了一份只读桌面上下文胶囊：${context.capsuleId}。`, + `上下文摘要：${context.summary}`, + screenshotHint, + '如果用户的问题需要理解呼出 TouchAI 前的桌面、前台窗口、选中文本、剪贴板或截图，请调用 `builtin__get_desktop_context` 读取；选中文本、剪贴板和截图字段都需要用户批准后才会读取或捕获。', + '不要假设桌面上下文已经完整出现在 prompt 中；不要执行点击、输入、滚动、聚焦或控制外部应用等 computer use 行为。', + ].join('\n'); +} const PROMPT_SOURCE_ORDER: PromptFragmentSource[] = [ 'override', @@ -111,7 +123,9 @@ async function buildPromptAssembly(options: ComposePromptSnapshotOptions): Promi ), feature: buildFragments('feature', [ ...(options.feature ?? []), - ...(options.desktopContext ? [DESKTOP_CONTEXT_TOOL_PROMPT] : []), + ...(options.desktopContext + ? [buildDesktopContextToolPrompt(options.desktopContext)] + : []), ]), user_append: buildFragments('user_append', options.userAppend ?? []), }; diff --git a/apps/desktop/src/services/AgentService/session/history.ts b/apps/desktop/src/services/AgentService/session/history.ts index 6cae2d72..474f8a52 100644 --- a/apps/desktop/src/services/AgentService/session/history.ts +++ b/apps/desktop/src/services/AgentService/session/history.ts @@ -2,8 +2,8 @@ import { findAllMcpServers } from '@database/queries/mcpServers'; import type { MessageRow } from '@database/queries/messages'; -import type { SessionTurnContextArtifactHistoryRow } from '@database/queries/sessionTurnContextArtifacts'; import type { SessionTurnAttemptHistoryRow } from '@database/queries/sessionTurnAttempts'; +import type { SessionTurnContextArtifactHistoryRow } from '@database/queries/sessionTurnContextArtifacts'; import type { SessionTurnHistoryRow } from '@database/queries/sessionTurns'; import type { PersistedToolLogStatus } from '@database/schema'; @@ -147,9 +147,7 @@ function isCancellationStatusText(text?: string | null): boolean { ); } -function parseDesktopContextMetadata( - metadataJson: string | null -): DesktopContextMetadataArtifact { +function parseDesktopContextMetadata(metadataJson: string | null): DesktopContextMetadataArtifact { if (!metadataJson) { return {}; } diff --git a/apps/desktop/src/services/BuiltInToolService/service.ts b/apps/desktop/src/services/BuiltInToolService/service.ts index ffbf75b3..69ff8118 100644 --- a/apps/desktop/src/services/BuiltInToolService/service.ts +++ b/apps/desktop/src/services/BuiltInToolService/service.ts @@ -61,6 +61,7 @@ interface BuiltInToolExecutionResponse { toolLogId: number | null; toolLogKind: ToolLogKind; attachments?: BuiltInToolExecutionResult['attachments']; + desktopContextArtifact?: BuiltInToolExecutionResult['desktopContextArtifact']; controlSignal?: BuiltInToolControlSignal; } @@ -476,6 +477,7 @@ class BuiltInToolService { toolLogId, toolLogKind: 'builtin', attachments: toolResult.attachments, + desktopContextArtifact: toolResult.desktopContextArtifact, controlSignal: toolResult.controlSignal, }; } diff --git a/apps/desktop/src/services/BuiltInToolService/tools/desktopContext/constants.ts b/apps/desktop/src/services/BuiltInToolService/tools/desktopContext/constants.ts index 49369f1c..b5afdc0c 100644 --- a/apps/desktop/src/services/BuiltInToolService/tools/desktopContext/constants.ts +++ b/apps/desktop/src/services/BuiltInToolService/tools/desktopContext/constants.ts @@ -7,7 +7,8 @@ export const DESKTOP_CONTEXT_TOOL_NAME = 'GetDesktopContext'; export const DESKTOP_CONTEXT_TOOL_DESCRIPTION = [ 'Read the immutable desktop context capsule bound to the current user turn.', 'This is read-only context, not computer use: the tool cannot click, type, focus, scroll, or control apps.', - 'Use include as an extensible array. Sensitive fields such as clipboard full text and screenshot image path are returned only when explicitly requested.', + 'Use include as an extensible array. By default only safe metadata is returned: summary, active_window, capabilities, and redactions.', + 'Sensitive fields such as selected text, clipboard content, and screenshots require explicit include values and user approval before TouchAI reads or captures them.', ].join(' '); export const DESKTOP_CONTEXT_INCLUDE_VALUES = [ @@ -34,7 +35,8 @@ export const DESKTOP_CONTEXT_TOOL_INPUT_SCHEMA: AiToolDefinition['input_schema'] }, limit: { type: 'number', - description: 'Maximum number of capsules to return for recent scope. Reserved for later.', + description: + 'Maximum number of capsules to return for recent scope. Reserved for later.', }, include: { type: 'array', @@ -43,13 +45,13 @@ export const DESKTOP_CONTEXT_TOOL_INPUT_SCHEMA: AiToolDefinition['input_schema'] enum: [...DESKTOP_CONTEXT_INCLUDE_VALUES], }, description: - 'Fields to include. Defaults to summary, active_window, selected_text.full_text, clipboard.summary, capabilities, and redactions.', + 'Fields to include. Defaults to safe fields only: summary, active_window, capabilities, and redactions. Sensitive values selected_text.*, clipboard.*, and screenshot.* require user approval and are read or captured only after approval.', }, screenshotTarget: { type: 'string', enum: ['capsule_default', 'active_window', 'active_display', 'all_displays'], description: - 'Requested screenshot target. Phase 1 returns the capsule default screenshot metadata or path.', + 'Requested screenshot target when screenshot.metadata or screenshot.image is included and approved.', }, }, additionalProperties: false, diff --git a/apps/desktop/src/services/BuiltInToolService/tools/desktopContext/index.ts b/apps/desktop/src/services/BuiltInToolService/tools/desktopContext/index.ts index 8ac9a9e4..f6683d37 100644 --- a/apps/desktop/src/services/BuiltInToolService/tools/desktopContext/index.ts +++ b/apps/desktop/src/services/BuiltInToolService/tools/desktopContext/index.ts @@ -1,12 +1,15 @@ // Copyright (c) 2026. Qian Cheng. Licensed under GPL v3 -import { - buildDesktopContextToolPayload, -} from '@/services/DesktopContextService/toolPayload'; +import { tt } from '@/i18n'; +import { createAttachment } from '@/services/AgentService/infrastructure/attachments'; +import { buildDesktopContextToolPayload } from '@/services/DesktopContextService/toolPayload'; import type { + BoundDesktopContext, + DesktopContextCapsule, DesktopContextInclude, DesktopContextToolRequest, } from '@/services/DesktopContextService/types'; +import { native } from '@/services/NativeService'; import { type BaseBuiltInToolExecutionContext, @@ -23,6 +26,14 @@ import { } from './constants'; const includeValues = new Set(DESKTOP_CONTEXT_INCLUDE_VALUES); +const sensitiveIncludeValues = new Set([ + 'selected_text.summary', + 'selected_text.full_text', + 'clipboard.summary', + 'clipboard.full_text', + 'screenshot.metadata', + 'screenshot.image', +]); function parseDesktopContextRequest(args: Record): DesktopContextToolRequest { const include = Array.isArray(args.include) @@ -46,6 +57,86 @@ function parseDesktopContextRequest(args: Record): DesktopConte }; } +function hasSensitiveInclude(request: DesktopContextToolRequest): boolean { + return request.include?.some((item) => sensitiveIncludeValues.has(item)) ?? false; +} + +function hasScreenshotInclude(request: DesktopContextToolRequest): boolean { + return ( + request.include?.some( + (item) => item === 'screenshot.metadata' || item === 'screenshot.image' + ) ?? false + ); +} + +function describeSensitiveIncludes(request: DesktopContextToolRequest): string { + const labels = new Set(); + for (const item of request.include ?? []) { + if (item.startsWith('selected_text.')) { + labels.add(tt('选中文本')); + } else if (item.startsWith('clipboard.')) { + labels.add(tt('剪贴板')); + } else if (item.startsWith('screenshot.')) { + labels.add(tt('屏幕截图')); + } + } + + return [...labels].join(', ') || tt('敏感桌面上下文'); +} + +function bindCapturedCapsule( + capsule: DesktopContextCapsule, + currentContext: BoundDesktopContext +): BoundDesktopContext { + return { + ...capsule, + boundAt: currentContext.boundAt, + }; +} + +async function captureSensitiveContext( + request: DesktopContextToolRequest, + context: BoundDesktopContext | null | undefined +): Promise<{ + context: BoundDesktopContext | null | undefined; + captured: boolean; +}> { + if (!context || !hasSensitiveInclude(request)) { + return { context, captured: false }; + } + + const capsule = await native.desktopContext.captureSensitive( + context.id, + request.include ?? [], + request.screenshotTarget + ); + + return capsule + ? { context: bindCapturedCapsule(capsule, context), captured: true } + : { context, captured: false }; +} + +async function buildScreenshotAttachments( + request: DesktopContextToolRequest, + context: BoundDesktopContext | null | undefined +): Promise { + if (!request.include?.includes('screenshot.image')) { + return undefined; + } + + const screenshot = context?.screenshot; + if (!screenshot?.available || !screenshot.path) { + return undefined; + } + + try { + return [await createAttachment('image', screenshot.path)]; + } catch (error) { + console.warn('[DesktopContextTool] Failed to attach desktop screenshot:', error); + return undefined; + } +} + class DesktopContextTool extends BuiltInTool> { readonly id = 'get_desktop_context' as const; readonly displayName = DESKTOP_CONTEXT_TOOL_NAME; @@ -60,19 +151,60 @@ class DesktopContextTool extends BuiltInTool> { }; } + override buildApprovalRequest( + args: Record, + _config: Record, + _namespacedName: string, + context: BaseBuiltInToolExecutionContext + ) { + const request = parseDesktopContextRequest(args); + if (!context.desktopContext || !hasSensitiveInclude(request)) { + return null; + } + + const sensitiveTargets = describeSensitiveIncludes(request); + return { + title: tt('桌面上下文读取确认'), + description: sensitiveTargets, + command: request.include?.join(', ') ?? '', + riskLabel: '', + reason: tt( + '模型请求读取 {targets}。批准后，TouchAI 会读取或捕获这些桌面上下文，并将结果发送给模型。', + { targets: sensitiveTargets } + ), + commandLabel: '', + approveLabel: tt('批准'), + rejectLabel: tt('拒绝'), + enterHint: 'Enter', + escHint: 'Esc', + keyboardApproveDelayMs: 450, + }; + } + override async execute( args: Record, _config: Record, context: BaseBuiltInToolExecutionContext ): Promise { const request = parseDesktopContextRequest(args); - const payload = buildDesktopContextToolPayload(context.desktopContext, request); + const { context: desktopContext, captured } = await captureSensitiveContext( + request, + context.desktopContext + ); + const payload = buildDesktopContextToolPayload(desktopContext, request); + const attachments = await buildScreenshotAttachments(request, desktopContext); + const desktopContextArtifact = + captured && hasScreenshotInclude(request) && desktopContext?.screenshot.available + ? desktopContext + : undefined; return { result: JSON.stringify(payload, null, 2), isError: !payload.available, status: payload.available ? 'success' : 'error', errorMessage: payload.available ? undefined : payload.reason, + attachments, + desktopContextArtifact, }; } } diff --git a/apps/desktop/src/services/BuiltInToolService/types.ts b/apps/desktop/src/services/BuiltInToolService/types.ts index 7de71da3..8acaa7f3 100644 --- a/apps/desktop/src/services/BuiltInToolService/types.ts +++ b/apps/desktop/src/services/BuiltInToolService/types.ts @@ -71,6 +71,7 @@ export interface BuiltInToolExecutionResult { errorMessage?: string | null; approvalSummary?: string | null; attachments?: AttachmentIndex[]; + desktopContextArtifact?: BoundDesktopContext | null; controlSignal?: BuiltInToolControlSignal; } diff --git a/apps/desktop/src/services/DesktopContextService/index.ts b/apps/desktop/src/services/DesktopContextService/index.ts index 84de3eae..499597c7 100644 --- a/apps/desktop/src/services/DesktopContextService/index.ts +++ b/apps/desktop/src/services/DesktopContextService/index.ts @@ -26,6 +26,7 @@ class DesktopContextService { export const desktopContextService = new DesktopContextService(); +export { buildDesktopContextPromptMetadata, buildDesktopContextToolPayload } from './toolPayload'; export type { BoundDesktopContext, DesktopContextActiveWindow, @@ -41,4 +42,3 @@ export type { DesktopContextTurnArtifact, UserMessageDesktopContext, } from './types'; -export { buildDesktopContextPromptMetadata, buildDesktopContextToolPayload } from './toolPayload'; diff --git a/apps/desktop/src/services/DesktopContextService/toolPayload.ts b/apps/desktop/src/services/DesktopContextService/toolPayload.ts index d9d80296..2838acc0 100644 --- a/apps/desktop/src/services/DesktopContextService/toolPayload.ts +++ b/apps/desktop/src/services/DesktopContextService/toolPayload.ts @@ -18,6 +18,7 @@ interface DesktopContextToolUnavailablePayload { interface DesktopContextToolSelectedTextPayload { available: boolean; source: string | null; + textSummary: string | null; textLength: number; truncated: boolean; reason: string | null; @@ -70,11 +71,10 @@ export type DesktopContextToolPayload = const DEFAULT_INCLUDE: DesktopContextInclude[] = [ 'summary', 'active_window', - 'selected_text.full_text', - 'clipboard.summary', 'capabilities', 'redactions', ]; +const SELECTED_TEXT_SUMMARY_LIMIT = 500; function normalizeInclude(include: unknown): Set { if (!Array.isArray(include) || include.length === 0) { @@ -90,6 +90,19 @@ function has(include: Set, key: DesktopContextInclude): b return include.has(key); } +function summarizeSelectedText(text: string | null): string | null { + if (!text) { + return null; + } + + const chars = Array.from(text); + if (chars.length <= SELECTED_TEXT_SUMMARY_LIMIT) { + return text; + } + + return `${chars.slice(0, SELECTED_TEXT_SUMMARY_LIMIT).join('')}...`; +} + function selectedTextPayload( context: BoundDesktopContext, include: Set @@ -102,6 +115,7 @@ function selectedTextPayload( return { available: selectedText.available, source: selectedText.source, + textSummary: summarizeSelectedText(selectedText.text), textLength: selectedText.textLength, truncated: selectedText.truncated, reason: selectedText.reason ?? null, @@ -212,7 +226,10 @@ export function buildDesktopContextPromptMetadata( activeWindowTitle: context.activeWindow?.title ?? null, selectedTextLength: context.selectedText.textLength, clipboardTextLength: context.clipboard.textLength, + screenshotAvailable: context.screenshot.available, screenshotPersisted: context.screenshot.persisted, + screenshotWidth: context.screenshot.width, + screenshotHeight: context.screenshot.height, capabilities: context.capabilities, }; } diff --git a/apps/desktop/src/services/DesktopContextService/types.ts b/apps/desktop/src/services/DesktopContextService/types.ts index 67af0c92..97ea3da3 100644 --- a/apps/desktop/src/services/DesktopContextService/types.ts +++ b/apps/desktop/src/services/DesktopContextService/types.ts @@ -107,7 +107,10 @@ export interface DesktopContextPromptMetadata { activeWindowTitle: string | null; selectedTextLength: number; clipboardTextLength: number; + screenshotAvailable: boolean; screenshotPersisted: boolean; + screenshotWidth: number | null; + screenshotHeight: number | null; capabilities: DesktopContextCapability[]; } diff --git a/apps/desktop/src/services/NativeService/desktopContext.ts b/apps/desktop/src/services/NativeService/desktopContext.ts index 1146a8b2..820c7236 100644 --- a/apps/desktop/src/services/NativeService/desktopContext.ts +++ b/apps/desktop/src/services/NativeService/desktopContext.ts @@ -5,6 +5,8 @@ import { invoke } from '@tauri-apps/api/core'; import type { BoundDesktopContext, DesktopContextCapsule, + DesktopContextInclude, + DesktopContextToolRequest, } from '@/services/DesktopContextService/types'; export const desktopContext = { @@ -19,4 +21,16 @@ export const desktopContext = { capsuleId, }); }, + + captureSensitive( + capsuleId: string, + include: DesktopContextInclude[], + screenshotTarget?: DesktopContextToolRequest['screenshotTarget'] + ): Promise { + return invoke('desktop_context_capture_sensitive', { + capsuleId, + include, + screenshotTarget, + }); + }, } as const; diff --git a/apps/desktop/tests/composables/SearchView/useSearchRequest.test.ts b/apps/desktop/tests/composables/SearchView/useSearchRequest.test.ts index 27ec1ba1..2ace0653 100644 --- a/apps/desktop/tests/composables/SearchView/useSearchRequest.test.ts +++ b/apps/desktop/tests/composables/SearchView/useSearchRequest.test.ts @@ -211,6 +211,7 @@ describe('useSearchRequestFlow', () => { inputSnapshot: queuedSnapshot, modelId: 'queued-model', providerId: 3, + desktopContextCapsuleId: null, }); expect(mounted.result.isWaitingForCompletion.value).toBe(true); expect(agentState.sendRequest).not.toHaveBeenCalled(); @@ -226,7 +227,8 @@ describe('useSearchRequestFlow', () => { [supportedAttachment], queuedSnapshot, 'queued-model', - 3 + 3, + null ); expect(mounted.result.pendingRequest.value).toBeNull(); expect(mounted.result.isWaitingForCompletion.value).toBe(false); @@ -398,7 +400,8 @@ describe('useSearchRequestFlow', () => { excludeFromHistory: true, }), 'regen-model', - 11 + 11, + null ); mounted.unmount(); diff --git a/apps/desktop/tests/services/AgentService/prompt/language-context.test.ts b/apps/desktop/tests/services/AgentService/prompt/language-context.test.ts index af70a2f7..ebb91b9c 100644 --- a/apps/desktop/tests/services/AgentService/prompt/language-context.test.ts +++ b/apps/desktop/tests/services/AgentService/prompt/language-context.test.ts @@ -52,6 +52,39 @@ describe('prompt current language context', () => { ); }); + it('tells the model how to request a persisted desktop screenshot path', async () => { + const snapshot = await composePromptSnapshot({ + prompt: '看看我桌面上是什么。', + desktopContext: { + capsuleId: 'ctx-1', + capturedAt: '2026-06-02T12:00:00.000Z', + boundAt: '2026-06-02T12:00:05.000Z', + summary: 'Visual Studio Code focused with selected Rust error text.', + activeWindowTitle: 'main.rs - TouchAI', + selectedTextLength: 23, + clipboardTextLength: 16, + screenshotAvailable: true, + screenshotPersisted: true, + screenshotWidth: 1200, + screenshotHeight: 800, + capabilities: [ + { + id: 'screenshot', + supported: true, + method: 'xcap-monitor-capture', + }, + ], + }, + }); + + expect(snapshot.systemPrompt).toContain('本轮请求绑定了一份只读桌面上下文胶囊：ctx-1'); + expect(snapshot.systemPrompt).toContain( + '本轮已有一张经用户批准后捕获的桌面截图并已持久化，尺寸 1200x800' + ); + expect(snapshot.systemPrompt).toContain("include: ['screenshot.image']"); + expect(snapshot.systemPrompt).toContain('builtin__get_desktop_context'); + }); + it('builds the current system and tool language context from the active locale', () => { setLocale('en-US'); diff --git a/apps/desktop/tests/services/AgentService/session-history-i18n.test.ts b/apps/desktop/tests/services/AgentService/session-history-i18n.test.ts index 68b84531..5e223dd1 100644 --- a/apps/desktop/tests/services/AgentService/session-history-i18n.test.ts +++ b/apps/desktop/tests/services/AgentService/session-history-i18n.test.ts @@ -1,6 +1,6 @@ import type { MessageRow } from '@database/queries/messages'; -import type { SessionTurnContextArtifactHistoryRow } from '@database/queries/sessionTurnContextArtifacts'; import type { SessionTurnAttemptHistoryRow } from '@database/queries/sessionTurnAttempts'; +import type { SessionTurnContextArtifactHistoryRow } from '@database/queries/sessionTurnContextArtifacts'; import type { SessionTurnHistoryRow } from '@database/queries/sessionTurns'; import { beforeEach, describe, expect, it, vi } from 'vitest'; diff --git a/apps/desktop/tests/services/BuiltInToolService/tools/desktopContext/index.test.ts b/apps/desktop/tests/services/BuiltInToolService/tools/desktopContext/index.test.ts new file mode 100644 index 00000000..dc1ec626 --- /dev/null +++ b/apps/desktop/tests/services/BuiltInToolService/tools/desktopContext/index.test.ts @@ -0,0 +1,233 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { desktopContextTool } from '@/services/BuiltInToolService/tools/desktopContext'; +import type { BaseBuiltInToolExecutionContext } from '@/services/BuiltInToolService/types'; +import type { BoundDesktopContext } from '@/services/DesktopContextService/types'; + +const { captureSensitiveMock, createAttachmentMock } = vi.hoisted(() => ({ + captureSensitiveMock: vi.fn(), + createAttachmentMock: vi.fn(), +})); + +vi.mock('@/services/AgentService/infrastructure/attachments', () => ({ + createAttachment: createAttachmentMock, +})); + +vi.mock('@/services/NativeService', () => ({ + native: { + desktopContext: { + captureSensitive: captureSensitiveMock, + }, + }, +})); + +const desktopContext: BoundDesktopContext = { + id: 'ctx-1', + sequence: 1, + capturedAt: '2026-06-02T12:00:00.000Z', + boundAt: '2026-06-02T12:00:05.000Z', + invocationSource: 'shortcut', + platform: 'windows', + summary: 'Visual Studio Code focused with a persisted desktop screenshot.', + activeWindow: null, + selectedText: { + available: false, + source: 'windows-uia-textpattern', + text: null, + textLength: 0, + truncated: false, + reason: 'No selected text.', + }, + clipboard: { + available: false, + snapshotId: null, + observedAt: null, + text: null, + textSummary: null, + textLength: 0, + imageCount: 0, + fileCount: 0, + reason: 'Clipboard is empty.', + }, + screenshot: { + available: true, + path: 'E:/TouchAI/data/desktop-context/screenshots/ctx-1.png', + mimeType: 'image/png', + width: 1200, + height: 800, + target: 'active_display', + persisted: true, + capturedAt: '2026-06-02T12:00:00.000Z', + }, + capabilities: [{ id: 'screenshot', supported: true, method: 'xcap-monitor-capture' }], + redactions: [], +}; + +const approvedDesktopContext: BoundDesktopContext = { + ...desktopContext, + selectedText: { + available: true, + source: 'windows-uia-focused-textpattern', + text: 'approved selected text', + textLength: 22, + truncated: false, + }, + clipboard: { + available: true, + snapshotId: 'clip-approved', + observedAt: 1770000000001, + text: 'approved clipboard text', + textSummary: 'approved clipboard text', + textLength: 23, + imageCount: 0, + fileCount: 0, + }, + screenshot: { + available: true, + path: 'E:/TouchAI/data/desktop-context/screenshots/ctx-1-approved.png', + mimeType: 'image/png', + width: 1600, + height: 900, + target: 'active_window', + persisted: true, + capturedAt: '2026-06-02T12:01:00.000Z', + }, +}; + +function createContext( + overrides: Partial = {} +): BaseBuiltInToolExecutionContext { + return { + callId: 'call-1', + iteration: 0, + hasExecutedBuiltInTool: () => false, + desktopContext, + ...overrides, + }; +} + +describe('desktopContextTool', () => { + beforeEach(() => { + captureSensitiveMock.mockReset(); + createAttachmentMock.mockReset(); + captureSensitiveMock.mockResolvedValue(approvedDesktopContext); + createAttachmentMock.mockResolvedValue({ + id: 'attachment-1', + type: 'image', + path: approvedDesktopContext.screenshot.path, + originPath: approvedDesktopContext.screenshot.path, + name: 'ctx-1-approved.png', + mimeType: 'image/png', + supportStatus: 'supported', + }); + }); + + it('does not attach the screenshot for the safe default include set', async () => { + const result = await desktopContextTool.execute({}, {}, createContext()); + + expect(result.attachments).toBeUndefined(); + expect(createAttachmentMock).not.toHaveBeenCalled(); + expect(captureSensitiveMock).not.toHaveBeenCalled(); + const payload = JSON.parse(result.result); + expect(payload).toMatchObject({ + available: true, + summary: desktopContext.summary, + }); + expect(payload).not.toHaveProperty('selectedText'); + expect(payload).not.toHaveProperty('clipboard'); + expect(payload).not.toHaveProperty('screenshot'); + }); + + it('does not require approval for the safe default include set', async () => { + expect( + desktopContextTool.buildApprovalRequest( + {}, + {}, + 'builtin__get_desktop_context', + createContext() + ) + ).toBeNull(); + }); + + it.each([ + ['selected text', ['selected_text.summary']], + ['clipboard', ['clipboard.summary']], + ['screenshot', ['screenshot.metadata']], + ])('requires approval before reading %s context', async (_label, include) => { + const approval = await desktopContextTool.buildApprovalRequest( + { include }, + {}, + 'builtin__get_desktop_context', + createContext() + ); + + expect(approval).toMatchObject({ + title: expect.stringContaining('桌面上下文'), + reason: expect.stringContaining('发送给模型'), + }); + }); + + it('captures approved sensitive context before building the tool payload', async () => { + const result = await desktopContextTool.execute( + { + include: ['selected_text.full_text', 'clipboard.full_text', 'screenshot.image'], + screenshotTarget: 'active_window', + }, + {}, + createContext() + ); + + expect(captureSensitiveMock).toHaveBeenCalledWith( + 'ctx-1', + ['selected_text.full_text', 'clipboard.full_text', 'screenshot.image'], + 'active_window' + ); + expect(createAttachmentMock).toHaveBeenCalledWith( + 'image', + 'E:/TouchAI/data/desktop-context/screenshots/ctx-1-approved.png' + ); + expect(result.attachments).toEqual([ + expect.objectContaining({ + id: 'attachment-1', + type: 'image', + path: 'E:/TouchAI/data/desktop-context/screenshots/ctx-1-approved.png', + }), + ]); + expect(JSON.parse(result.result)).toMatchObject({ + selectedText: { + fullText: 'approved selected text', + }, + clipboard: { + fullText: 'approved clipboard text', + }, + screenshot: { + path: 'E:/TouchAI/data/desktop-context/screenshots/ctx-1-approved.png', + }, + }); + expect(result.desktopContextArtifact).toMatchObject({ + id: 'ctx-1', + boundAt: '2026-06-02T12:00:05.000Z', + screenshot: { + path: 'E:/TouchAI/data/desktop-context/screenshots/ctx-1-approved.png', + }, + }); + }); + + it('returns the stale capsule reason when sensitive capture is unavailable', async () => { + captureSensitiveMock.mockResolvedValue(null); + + const result = await desktopContextTool.execute( + { include: ['selected_text.full_text'] }, + {}, + createContext() + ); + + expect(JSON.parse(result.result)).toMatchObject({ + selectedText: { + available: false, + reason: 'No selected text.', + }, + }); + expect(result.desktopContextArtifact).toBeUndefined(); + }); +}); diff --git a/apps/desktop/tests/services/DesktopContextService/tool-payload.test.ts b/apps/desktop/tests/services/DesktopContextService/tool-payload.test.ts index faf17b7e..66df533c 100644 --- a/apps/desktop/tests/services/DesktopContextService/tool-payload.test.ts +++ b/apps/desktop/tests/services/DesktopContextService/tool-payload.test.ts @@ -2,8 +2,11 @@ import { describe, expect, it } from 'vitest'; +import { + buildDesktopContextPromptMetadata, + buildDesktopContextToolPayload, +} from '@/services/DesktopContextService/toolPayload'; import type { BoundDesktopContext } from '@/services/DesktopContextService/types'; -import { buildDesktopContextToolPayload } from '@/services/DesktopContextService/toolPayload'; const context: BoundDesktopContext = { id: 'ctx-1', @@ -60,7 +63,7 @@ describe('buildDesktopContextToolPayload', () => { }); }); - it('uses a safe default include set without clipboard full text or screenshot path', () => { + it('uses a safe default include set without selected text, clipboard, or screenshot contents', () => { const payload = buildDesktopContextToolPayload(context); expect(payload).toMatchObject({ @@ -68,26 +71,64 @@ describe('buildDesktopContextToolPayload', () => { capsuleId: 'ctx-1', summary: context.summary, activeWindow: context.activeWindow, - selectedText: { - fullText: 'selected compiler error', - }, - clipboard: { - textSummary: 'clipboard secret', - textLength: 16, - }, capabilities: context.capabilities, redactions: context.redactions, }); + expect(payload).not.toHaveProperty('selectedText'); + expect(payload).not.toHaveProperty('clipboard'); expect(payload).not.toHaveProperty('screenshot'); + }); + + it('returns selected text metadata only when explicitly requested', () => { + const payload = buildDesktopContextToolPayload(context, { + include: ['selected_text.summary'], + }); + + expect(payload.selectedText).toMatchObject({ + available: true, + source: 'win32-edit-control', + textSummary: 'selected compiler error', + textLength: 23, + truncated: false, + }); + expect(payload.selectedText).not.toHaveProperty('fullText'); + }); + + it('returns clipboard metadata only when explicitly requested', () => { + const payload = buildDesktopContextToolPayload(context, { + include: ['clipboard.summary'], + }); + + expect(payload.clipboard).toMatchObject({ + textSummary: 'clipboard secret', + textLength: 16, + }); expect(payload.clipboard).not.toHaveProperty('fullText'); }); + it('returns screenshot metadata only when explicitly requested', () => { + const payload = buildDesktopContextToolPayload(context, { + include: ['screenshot.metadata'], + }); + + expect(payload.screenshot).toMatchObject({ + available: true, + width: 1200, + height: 800, + persisted: true, + }); + expect(payload.screenshot).not.toHaveProperty('path'); + }); + it('returns sensitive fields only when the include array asks for them', () => { const payload = buildDesktopContextToolPayload(context, { - include: ['clipboard.full_text', 'screenshot.image'], + include: ['selected_text.full_text', 'clipboard.full_text', 'screenshot.image'], }); expect(payload).not.toHaveProperty('summary'); + expect(payload.selectedText).toMatchObject({ + fullText: 'selected compiler error', + }); expect(payload.clipboard).toMatchObject({ fullText: 'clipboard secret', }); @@ -96,4 +137,15 @@ describe('buildDesktopContextToolPayload', () => { persisted: true, }); }); + + it('exposes screenshot availability in prompt metadata without leaking the image path', () => { + expect(buildDesktopContextPromptMetadata(context)).toMatchObject({ + capsuleId: 'ctx-1', + summary: 'Visual Studio Code focused with selected Rust error text.', + screenshotAvailable: true, + screenshotPersisted: true, + screenshotWidth: 1200, + screenshotHeight: 800, + }); + }); }); From cd731e77f2b354c3b38235470d1d1acb5a445137 Mon Sep 17 00:00:00 2001 From: xlgzsgf <51521689+hiqiancheng@users.noreply.github.com> Date: Wed, 3 Jun 2026 10:20:01 +0800 Subject: [PATCH 3/4] fix(desktop-context): avoid persistent screenshot protection Refs #281 --- .../src/core/system/desktop_context.rs | 148 +++++++++++++++++- .../src-tauri/src/core/window/search/mod.rs | 10 -- .../src/core/window/search/surface.rs | 41 +++++ 3 files changed, 181 insertions(+), 18 deletions(-) diff --git a/apps/desktop/src-tauri/src/core/system/desktop_context.rs b/apps/desktop/src-tauri/src/core/system/desktop_context.rs index e9545676..e6558299 100644 --- a/apps/desktop/src-tauri/src/core/system/desktop_context.rs +++ b/apps/desktop/src-tauri/src/core/system/desktop_context.rs @@ -10,11 +10,13 @@ use std::{ atomic::{AtomicU64, Ordering}, Mutex, }, + thread, + time::Duration, }; use image::{imageops, RgbaImage}; use serde::Serialize; -use tauri::{AppHandle, Manager, Runtime}; +use tauri::{AppHandle, Manager, Runtime, WebviewWindow}; use xcap::{Monitor, Window}; use crate::core::system::{ @@ -216,6 +218,7 @@ impl DesktopContextRuntime { if include_requests_screenshot(include) { let captured_at = now_rfc3339_millis(); capsule.screenshot = capture_screenshot( + app_handle, &capsule.id, &captured_at, capsule.active_window.as_ref(), @@ -980,19 +983,133 @@ fn select_capture_monitor( } fn capture_screenshot( + app_handle: &AppHandle, capsule_id: &str, captured_at: &str, active_window: Option<&DesktopContextActiveWindow>, screenshot_target: Option<&str>, ) -> DesktopContextScreenshot { let target = normalize_screenshot_target(screenshot_target); - match capture_screenshot_result(capsule_id, captured_at, active_window, target) { + match capture_screenshot_result(app_handle, capsule_id, captured_at, active_window, target) { Ok(screenshot) => screenshot, Err(error) => unsupported_screenshot_for_target(captured_at, target, error), } } -fn capture_screenshot_result( +fn should_hide_window_for_context_screenshot(label: &str) -> bool { + label == "main" || label == "tray-menu" || label.starts_with("popup-") +} + +fn context_screenshot_window_hide_priority(label: &str) -> u8 { + if label == "main" { + 1 + } else { + 0 + } +} + +struct HiddenContextScreenshotWindow { + label: String, + window: WebviewWindow, + was_focused: bool, +} + +struct ContextScreenshotWindowGuard { + app_handle: AppHandle, + hidden_windows: Vec>, + entered_context_screenshot_window_hide: bool, +} + +impl ContextScreenshotWindowGuard { + fn hide_visible_touchai_windows(app_handle: &AppHandle) -> Self { + let entered_context_screenshot_window_hide = app_handle + .try_state::() + .map(|runtime| { + runtime.enter_context_screenshot_window_hide(); + }) + .is_some(); + + let mut windows = app_handle + .webview_windows() + .into_iter() + .filter(|(label, _)| should_hide_window_for_context_screenshot(label)) + .collect::>(); + windows.sort_by(|(left_label, _), (right_label, _)| { + context_screenshot_window_hide_priority(left_label) + .cmp(&context_screenshot_window_hide_priority(right_label)) + .then_with(|| left_label.cmp(right_label)) + }); + + let mut hidden_windows = Vec::new(); + for (label, window) in windows { + if !window.is_visible().unwrap_or(false) { + continue; + } + + let was_focused = window.is_focused().unwrap_or(false); + match window.hide() { + Ok(()) => hidden_windows.push(HiddenContextScreenshotWindow { + label, + window, + was_focused, + }), + Err(error) => log::warn!( + "Failed to temporarily hide TouchAI window for context screenshot: {}", + error + ), + } + } + + if !hidden_windows.is_empty() { + thread::sleep(Duration::from_millis(80)); + } + + Self { + app_handle: app_handle.clone(), + hidden_windows, + entered_context_screenshot_window_hide, + } + } +} + +impl Drop for ContextScreenshotWindowGuard { + fn drop(&mut self) { + for hidden_window in self.hidden_windows.iter().rev() { + if let Err(error) = hidden_window.window.show() { + log::warn!( + "Failed to restore TouchAI window '{}' after context screenshot: {}", + hidden_window.label, + error + ); + continue; + } + if hidden_window.was_focused { + if let Err(error) = hidden_window.window.set_focus() { + log::warn!( + "Failed to restore TouchAI window '{}' focus after context screenshot: {}", + hidden_window.label, + error + ); + } + } + } + + if self.entered_context_screenshot_window_hide { + if !self.hidden_windows.is_empty() { + thread::sleep(Duration::from_millis(80)); + } + if let Some(runtime) = + self.app_handle + .try_state::() + { + runtime.leave_context_screenshot_window_hide(); + } + } + } +} + +fn capture_screenshot_result( + app_handle: &AppHandle, capsule_id: &str, captured_at: &str, active_window: Option<&DesktopContextActiveWindow>, @@ -1006,10 +1123,14 @@ fn capture_screenshot_result( })?; } - let image = match target { - "active_window" => capture_active_window_image(active_window)?, - "all_displays" => capture_all_displays_image()?, - _ => capture_active_display_image(active_window)?, + let image = { + let _touchai_window_guard = + ContextScreenshotWindowGuard::hide_visible_touchai_windows(app_handle); + match target { + "active_window" => capture_active_window_image(active_window)?, + "all_displays" => capture_all_displays_image()?, + _ => capture_active_display_image(active_window)?, + } }; let width = image.width(); let height = image.height(); @@ -1475,10 +1596,21 @@ fn capture_active_window_with_win32() -> Option { mod tests { use super::{ screenshot_artifact_path, selected_text_from_text, selected_text_from_utf16_range, - SELECTED_TEXT_LIMIT, + should_hide_window_for_context_screenshot, SELECTED_TEXT_LIMIT, }; use std::path::PathBuf; + #[test] + fn context_screenshot_hides_only_transient_touchai_windows() { + assert!(should_hide_window_for_context_screenshot("main")); + assert!(should_hide_window_for_context_screenshot("tray-menu")); + assert!(should_hide_window_for_context_screenshot( + "popup-model-dropdown-popup" + )); + assert!(!should_hide_window_for_context_screenshot("settings")); + assert!(!should_hide_window_for_context_screenshot("assistant-log")); + } + #[test] fn screenshot_artifact_path_stays_under_desktop_context_directory() { let path = diff --git a/apps/desktop/src-tauri/src/core/window/search/mod.rs b/apps/desktop/src-tauri/src/core/window/search/mod.rs index 23ab97d6..db5be12e 100644 --- a/apps/desktop/src-tauri/src/core/window/search/mod.rs +++ b/apps/desktop/src-tauri/src/core/window/search/mod.rs @@ -107,21 +107,11 @@ pub(super) fn show_and_activate_search_window( ) -> Result<(), String> { let _ = window.unminimize(); crate::core::window::rounded_corners::sync_window_corner_style(window)?; - protect_search_window_from_screen_capture(window); window.show().map_err(|e| e.to_string())?; crate::core::window::rounded_corners::sync_window_corner_style(window)?; reactivate_search_window_input(window) } -fn protect_search_window_from_screen_capture(window: &WebviewWindow) { - if let Err(error) = window.set_content_protected(true) { - log::warn!( - "Failed to protect search window from screen capture: {}", - error - ); - } -} - /// 让主搜索窗口重新成为键盘输入目标，并唤醒 WebView2 accelerator 管线。 pub(super) fn reactivate_search_window_input( window: &WebviewWindow, diff --git a/apps/desktop/src-tauri/src/core/window/search/surface.rs b/apps/desktop/src-tauri/src/core/window/search/surface.rs index bc290269..e137325d 100644 --- a/apps/desktop/src-tauri/src/core/window/search/surface.rs +++ b/apps/desktop/src-tauri/src/core/window/search/surface.rs @@ -82,6 +82,7 @@ pub struct PopupSurfaceSession { pub struct SearchWindowRuntime { hide_on_app_blur: AtomicBool, allow_height_override: AtomicBool, + context_screenshot_window_hide_depth: AtomicU64, sequence: AtomicU64, popup_sessions: Mutex>, window_state: SearchWindowState, @@ -97,6 +98,7 @@ struct FocusLostDecisionInput<'a> { main_always_on_top: bool, app_focused: bool, has_popup_sessions: bool, + context_screenshot_window_hide_active: bool, } pub type SearchSurfaceRuntime = SearchWindowRuntime; @@ -107,6 +109,7 @@ impl SearchWindowRuntime { Self { hide_on_app_blur: AtomicBool::new(true), allow_height_override: AtomicBool::new(false), + context_screenshot_window_hide_depth: AtomicU64::new(0), sequence: AtomicU64::new(0), popup_sessions: Mutex::new(HashMap::new()), window_state: SearchWindowState::default(), @@ -138,6 +141,28 @@ impl SearchWindowRuntime { self.allow_height_override.load(Ordering::Relaxed) } + /// 标记原生层正在为上下文截图临时隐藏 TouchAI surface。 + pub fn enter_context_screenshot_window_hide(&self) { + self.context_screenshot_window_hide_depth + .fetch_add(1, Ordering::Relaxed); + } + + /// 结束上下文截图临时隐藏 TouchAI surface 的标记。 + pub fn leave_context_screenshot_window_hide(&self) { + let _ = self.context_screenshot_window_hide_depth.fetch_update( + Ordering::Relaxed, + Ordering::Relaxed, + |depth| Some(depth.saturating_sub(1)), + ); + } + + /// 判断是否需要忽略由上下文截图 hide/show 造成的原生失焦事件。 + pub fn is_context_screenshot_window_hide_active(&self) -> bool { + self.context_screenshot_window_hide_depth + .load(Ordering::Relaxed) + > 0 + } + /// 记录一个 popup 原生窗口会话。 pub fn register_popup_session(&self, session: PopupSurfaceSession) { let mut sessions = self @@ -215,6 +240,10 @@ pub fn update_window_defaults( /// 判断某次原生失焦是否应该隐藏完整搜索窗口组。 fn should_hide_on_focus_lost(input: FocusLostDecisionInput<'_>) -> bool { + if input.context_screenshot_window_hide_active { + return false; + } + if !input.hide_on_app_blur || !input.main_visible || input.main_always_on_top { return false; } @@ -401,6 +430,7 @@ pub fn handle_focus_lost( main_always_on_top: main_window.is_always_on_top().unwrap_or(false), app_focused: crate::core::window::popup::is_app_focused(app_handle.clone())?, has_popup_sessions: runtime.has_popup_sessions(), + context_screenshot_window_hide_active: runtime.is_context_screenshot_window_hide_active(), }; let should_hide = should_hide_on_focus_lost(decision_input); if !should_hide { @@ -467,6 +497,7 @@ mod tests { main_always_on_top: false, app_focused: false, has_popup_sessions: false, + context_screenshot_window_hide_active: false, } } @@ -506,4 +537,14 @@ mod tests { assert!(!should_hide); } + + #[test] + fn blur_does_not_hide_surface_during_context_screenshot_window_hide() { + let should_hide = should_hide_on_focus_lost(FocusLostDecisionInput { + context_screenshot_window_hide_active: true, + ..decision_input("main") + }); + + assert!(!should_hide); + } } From aa37d4906f176f9c9ad7f7ca584a3e808722d966 Mon Sep 17 00:00:00 2001 From: xlgzsgf <51521689+hiqiancheng@users.noreply.github.com> Date: Wed, 3 Jun 2026 19:13:55 +0800 Subject: [PATCH 4/4] feat(desktop-context): surface desktop context in tool calls --- .../src/core/system/desktop_context.rs | 18 +- apps/desktop/src/i18n/textMap.ts | 1 + .../services/AgentService/prompt/composer.ts | 9 +- .../tools/desktopContext/constants.ts | 6 +- .../tools/desktopContext/index.ts | 3 +- .../DesktopContextService/toolPayload.ts | 54 ++- .../services/DesktopContextService/types.ts | 1 + .../BuiltInDesktopContextToolCallItem.vue | 418 ++++++++++++++++++ .../components/ToolCallItem.vue | 5 +- .../components/UserMessage.vue | 49 -- .../DesktopContextToolCallItem.test.ts | 135 ++++++ .../prompt/language-context.test.ts | 5 + .../tools/desktopContext/index.test.ts | 19 +- .../tool-payload.test.ts | 36 +- 14 files changed, 673 insertions(+), 86 deletions(-) create mode 100644 apps/desktop/src/views/SearchView/components/ConversationPanel/components/BuiltInDesktopContextToolCallItem.vue create mode 100644 apps/desktop/tests/components/DesktopContextToolCallItem.test.ts diff --git a/apps/desktop/src-tauri/src/core/system/desktop_context.rs b/apps/desktop/src-tauri/src/core/system/desktop_context.rs index e6558299..467e89a0 100644 --- a/apps/desktop/src-tauri/src/core/system/desktop_context.rs +++ b/apps/desktop/src-tauri/src/core/system/desktop_context.rs @@ -159,9 +159,10 @@ impl DesktopContextRuntime { let active_window_capture = capture_active_window(); let active_window = active_window_capture.value.clone(); let clipboard = pending_clipboard(app_handle); - let selected_text = pending_selected_text(); + let selected_text = capture_selected_text(active_window.as_ref()); let screenshot = pending_screenshot(); - let capabilities = build_initial_capabilities(&active_window_capture, &clipboard); + let capabilities = + build_initial_capabilities(&active_window_capture, &selected_text, &clipboard); let redactions = sensitive_redactions(); let summary = build_summary( active_window.as_ref(), @@ -422,13 +423,6 @@ fn selected_text_from_utf16_range( )) } -fn pending_selected_text() -> DesktopContextSelectedText { - unavailable_selected_text( - Some(selected_text_provider_method()), - SENSITIVE_ACCESS_REQUIRES_APPROVAL, - ) -} - fn capture_selected_text( active_window: Option<&DesktopContextActiveWindow>, ) -> DesktopContextSelectedText { @@ -1348,11 +1342,12 @@ fn build_capabilities( fn build_initial_capabilities( active_window: &CaptureOutcome, + selected_text: &DesktopContextSelectedText, clipboard: &DesktopContextClipboard, ) -> Vec { build_capabilities( active_window, - &pending_selected_text(), + selected_text, clipboard, &pending_screenshot(), ) @@ -1382,8 +1377,7 @@ fn sensitive_redactions() -> Vec { vec![ DesktopContextRedaction { field: "selectedText.fullText".to_string(), - reason: "Only read after explicit user approval for selected_text.full_text." - .to_string(), + reason: "Selected text is captured at invocation, but raw full text is only returned after explicit user approval for selected_text.full_text.".to_string(), }, DesktopContextRedaction { field: "clipboard.fullText".to_string(), diff --git a/apps/desktop/src/i18n/textMap.ts b/apps/desktop/src/i18n/textMap.ts index 26974f1f..71599e30 100644 --- a/apps/desktop/src/i18n/textMap.ts +++ b/apps/desktop/src/i18n/textMap.ts @@ -536,6 +536,7 @@ export const zhToEnTextMap = { 桌面上下文读取确认: 'Confirm desktop context read', 模型切换确认: 'Confirm model switch', 选中文本: 'selected text', + 选中文本原文: 'selected text full text', 剪贴板: 'clipboard', 屏幕截图: 'screenshot', 敏感桌面上下文: 'sensitive desktop context', diff --git a/apps/desktop/src/services/AgentService/prompt/composer.ts b/apps/desktop/src/services/AgentService/prompt/composer.ts index 2aef3832..0d8977a2 100644 --- a/apps/desktop/src/services/AgentService/prompt/composer.ts +++ b/apps/desktop/src/services/AgentService/prompt/composer.ts @@ -35,12 +35,19 @@ function buildDesktopContextToolPrompt(context: DesktopContextPromptMetadata): s : '' }。如需图片文件路径，调用 \`builtin__get_desktop_context\` 并传入 \`include: ['screenshot.image']\`。` : "如果需要屏幕截图，调用 `builtin__get_desktop_context` 并传入 `include: ['screenshot.image']`；TouchAI 会先请求用户批准，批准后再捕获并持久化截图。"; + const selectedTextHint = context.selectedTextSummary?.trim() + ? [ + `已默认注入脱敏后的选中文本摘要：${context.selectedTextSummary}`, + "如需选中文本原文，调用 `builtin__get_desktop_context` 并传入 `include: ['selected_text.full_text']`；TouchAI 会先请求用户批准，批准后再返回原始完整文本。", + ].join('\n') + : "本轮没有可默认注入的选中文本摘要。如用户问题需要原始选中文本，可调用 `builtin__get_desktop_context` 并传入 `include: ['selected_text.full_text']`。"; return [ `本轮请求绑定了一份只读桌面上下文胶囊：${context.capsuleId}。`, `上下文摘要：${context.summary}`, + selectedTextHint, screenshotHint, - '如果用户的问题需要理解呼出 TouchAI 前的桌面、前台窗口、选中文本、剪贴板或截图，请调用 `builtin__get_desktop_context` 读取；选中文本、剪贴板和截图字段都需要用户批准后才会读取或捕获。', + '如果用户的问题需要理解呼出 TouchAI 前的桌面、前台窗口、剪贴板或截图，请调用 `builtin__get_desktop_context` 读取；剪贴板、截图和选中文本原文字段都需要用户批准后才会读取或捕获。', '不要假设桌面上下文已经完整出现在 prompt 中；不要执行点击、输入、滚动、聚焦或控制外部应用等 computer use 行为。', ].join('\n'); } diff --git a/apps/desktop/src/services/BuiltInToolService/tools/desktopContext/constants.ts b/apps/desktop/src/services/BuiltInToolService/tools/desktopContext/constants.ts index b5afdc0c..c28f388a 100644 --- a/apps/desktop/src/services/BuiltInToolService/tools/desktopContext/constants.ts +++ b/apps/desktop/src/services/BuiltInToolService/tools/desktopContext/constants.ts @@ -7,8 +7,8 @@ export const DESKTOP_CONTEXT_TOOL_NAME = 'GetDesktopContext'; export const DESKTOP_CONTEXT_TOOL_DESCRIPTION = [ 'Read the immutable desktop context capsule bound to the current user turn.', 'This is read-only context, not computer use: the tool cannot click, type, focus, scroll, or control apps.', - 'Use include as an extensible array. By default only safe metadata is returned: summary, active_window, capabilities, and redactions.', - 'Sensitive fields such as selected text, clipboard content, and screenshots require explicit include values and user approval before TouchAI reads or captures them.', + 'Use include as an extensible array. By default safe context is returned: summary, active_window, redacted selected_text.summary, capabilities, and redactions.', + 'Sensitive fields such as selected_text.full_text, clipboard content, and screenshots require explicit include values and user approval before TouchAI reads or captures them.', ].join(' '); export const DESKTOP_CONTEXT_INCLUDE_VALUES = [ @@ -45,7 +45,7 @@ export const DESKTOP_CONTEXT_TOOL_INPUT_SCHEMA: AiToolDefinition['input_schema'] enum: [...DESKTOP_CONTEXT_INCLUDE_VALUES], }, description: - 'Fields to include. Defaults to safe fields only: summary, active_window, capabilities, and redactions. Sensitive values selected_text.*, clipboard.*, and screenshot.* require user approval and are read or captured only after approval.', + 'Fields to include. Defaults to safe fields only: summary, active_window, redacted selected_text.summary, capabilities, and redactions. Sensitive values selected_text.full_text, clipboard.*, and screenshot.* require user approval and are read or captured only after approval.', }, screenshotTarget: { type: 'string', diff --git a/apps/desktop/src/services/BuiltInToolService/tools/desktopContext/index.ts b/apps/desktop/src/services/BuiltInToolService/tools/desktopContext/index.ts index f6683d37..ca3727c2 100644 --- a/apps/desktop/src/services/BuiltInToolService/tools/desktopContext/index.ts +++ b/apps/desktop/src/services/BuiltInToolService/tools/desktopContext/index.ts @@ -27,7 +27,6 @@ import { const includeValues = new Set(DESKTOP_CONTEXT_INCLUDE_VALUES); const sensitiveIncludeValues = new Set([ - 'selected_text.summary', 'selected_text.full_text', 'clipboard.summary', 'clipboard.full_text', @@ -73,7 +72,7 @@ function describeSensitiveIncludes(request: DesktopContextToolRequest): string { const labels = new Set(); for (const item of request.include ?? []) { if (item.startsWith('selected_text.')) { - labels.add(tt('选中文本')); + labels.add(tt('选中文本原文')); } else if (item.startsWith('clipboard.')) { labels.add(tt('剪贴板')); } else if (item.startsWith('screenshot.')) { diff --git a/apps/desktop/src/services/DesktopContextService/toolPayload.ts b/apps/desktop/src/services/DesktopContextService/toolPayload.ts index 2838acc0..3c4bc59d 100644 --- a/apps/desktop/src/services/DesktopContextService/toolPayload.ts +++ b/apps/desktop/src/services/DesktopContextService/toolPayload.ts @@ -71,10 +71,22 @@ export type DesktopContextToolPayload = const DEFAULT_INCLUDE: DesktopContextInclude[] = [ 'summary', 'active_window', + 'selected_text.summary', 'capabilities', 'redactions', ]; const SELECTED_TEXT_SUMMARY_LIMIT = 500; +const SELECTED_TEXT_REDACTION: DesktopContextRedaction = { + field: 'selectedText.textSummary', + reason: 'Sensitive-looking selected text was redacted before default prompt/tool exposure.', +}; + +const SECRET_PATTERNS: RegExp[] = [ + /\b(authorization\s*:\s*)(?:bearer\s+)?[^\s"',;]+/gi, + /\b(api[_-]?key|access[_-]?token|refresh[_-]?token|client[_-]?secret|token|secret|password|passwd)\b(\s*[:=]\s*)["']?[^"'\s,;]+["']?/gi, + /\b(?:sk|pk|rk|xox[baprs]|gh[pousr]|github_pat|ta_live|tp)-[A-Za-z0-9_-]{10,}\b/g, + /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/g, +]; function normalizeInclude(include: unknown): Set { if (!Array.isArray(include) || include.length === 0) { @@ -103,6 +115,36 @@ function summarizeSelectedText(text: string | null): string | null { return `${chars.slice(0, SELECTED_TEXT_SUMMARY_LIMIT).join('')}...`; } +function redactSelectedTextSummary(summary: string | null): string | null { + if (!summary) { + return summary; + } + + return SECRET_PATTERNS.reduce((redacted, pattern) => { + pattern.lastIndex = 0; + return redacted.replace(pattern, (_match, prefix, separator) => { + if (typeof prefix === 'string' && typeof separator === 'string') { + return `${prefix}${separator}[REDACTED:secret]`; + } + + if (typeof prefix === 'string' && prefix.toLowerCase().startsWith('authorization')) { + return `${prefix}[REDACTED:secret]`; + } + + return '[REDACTED:secret]'; + }); + }, summary); +} + +function selectedTextSummary(context: BoundDesktopContext): string | null { + return redactSelectedTextSummary(summarizeSelectedText(context.selectedText.text)); +} + +function selectedTextWasRedacted(context: BoundDesktopContext): boolean { + const summary = summarizeSelectedText(context.selectedText.text); + return summary !== null && redactSelectedTextSummary(summary) !== summary; +} + function selectedTextPayload( context: BoundDesktopContext, include: Set @@ -112,10 +154,11 @@ function selectedTextPayload( } const selectedText = context.selectedText; + const redactedSummary = selectedTextSummary(context); return { available: selectedText.available, source: selectedText.source, - textSummary: summarizeSelectedText(selectedText.text), + textSummary: redactedSummary, textLength: selectedText.textLength, truncated: selectedText.truncated, reason: selectedText.reason ?? null, @@ -194,6 +237,12 @@ export function buildDesktopContextToolPayload( const selectedText = selectedTextPayload(context, include); const clipboard = clipboardPayload(context, include); const screenshot = screenshotPayload(context, include); + const redactions = has(include, 'redactions') + ? [ + ...context.redactions, + ...(selectedTextWasRedacted(context) ? [SELECTED_TEXT_REDACTION] : []), + ] + : undefined; return { available: true, @@ -207,7 +256,7 @@ export function buildDesktopContextToolPayload( ...(clipboard ? { clipboard } : {}), ...(screenshot ? { screenshot } : {}), ...(has(include, 'capabilities') ? { capabilities: context.capabilities } : {}), - ...(has(include, 'redactions') ? { redactions: context.redactions } : {}), + ...(redactions ? { redactions } : {}), }; } @@ -224,6 +273,7 @@ export function buildDesktopContextPromptMetadata( boundAt: context.boundAt, summary: context.summary, activeWindowTitle: context.activeWindow?.title ?? null, + selectedTextSummary: selectedTextSummary(context), selectedTextLength: context.selectedText.textLength, clipboardTextLength: context.clipboard.textLength, screenshotAvailable: context.screenshot.available, diff --git a/apps/desktop/src/services/DesktopContextService/types.ts b/apps/desktop/src/services/DesktopContextService/types.ts index 97ea3da3..17990501 100644 --- a/apps/desktop/src/services/DesktopContextService/types.ts +++ b/apps/desktop/src/services/DesktopContextService/types.ts @@ -105,6 +105,7 @@ export interface DesktopContextPromptMetadata { boundAt: string; summary: string; activeWindowTitle: string | null; + selectedTextSummary: string | null; selectedTextLength: number; clipboardTextLength: number; screenshotAvailable: boolean; diff --git a/apps/desktop/src/views/SearchView/components/ConversationPanel/components/BuiltInDesktopContextToolCallItem.vue b/apps/desktop/src/views/SearchView/components/ConversationPanel/components/BuiltInDesktopContextToolCallItem.vue new file mode 100644 index 00000000..b6889c50 --- /dev/null +++ b/apps/desktop/src/views/SearchView/components/ConversationPanel/components/BuiltInDesktopContextToolCallItem.vue @@ -0,0 +1,418 @@ + + + + + + + diff --git a/apps/desktop/src/views/SearchView/components/ConversationPanel/components/ToolCallItem.vue b/apps/desktop/src/views/SearchView/components/ConversationPanel/components/ToolCallItem.vue index 732073f9..5273f4db 100644 --- a/apps/desktop/src/views/SearchView/components/ConversationPanel/components/ToolCallItem.vue +++ b/apps/desktop/src/views/SearchView/components/ConversationPanel/components/ToolCallItem.vue @@ -1,7 +1,8 @@