AI service: Add Apple on-device OCR/Translation (#18671)

Author: warmenhoven

accessibility.h

@@ -82,6 +82,20 @@ bool is_narrator_running(bool accessibility_enable);
 */
 bool run_translation_service(settings_t *settings, bool paused);
 
+/**
+ * config_get_ai_service_backend_options:
+ *
+ * Get available AI service backends separated by '|'.
+ **/
+const char *config_get_ai_service_backend_options(void);
+
+/**
+ * config_get_default_ai_service_backend:
+ *
+ * Get the default AI service backend identifier.
+ **/
+const char *config_get_default_ai_service_backend(void);
+
 bool accessibility_speak_priority(
       bool accessibility_enable,
       unsigned accessibility_narrator_speech_speed,

configuration.c

@@ -53,6 +53,7 @@
 
 #include "tasks/task_content.h"
 #include "tasks/tasks_internal.h"
+#include "accessibility.h"
 
 #include "list_special.h"
 
@@ -1556,6 +1557,7 @@ static struct config_array_setting *populate_settings_array(
    SETTING_ARRAY("midi_driver",                  settings->arrays.midi_driver, false, NULL, true);
    SETTING_ARRAY("midi_input",                   settings->arrays.midi_input, true, DEFAULT_MIDI_INPUT, true);
    SETTING_ARRAY("midi_output",                  settings->arrays.midi_output, true, DEFAULT_MIDI_OUTPUT, true);
+   SETTING_ARRAY("ai_service_backend",           settings->arrays.ai_service_backend, false, NULL, true);
 
    SETTING_ARRAY("video_driver",                 settings->arrays.video_driver, false, NULL, true);
    SETTING_ARRAY("video_context_driver",         settings->arrays.video_context_driver, false, NULL, true);
@@ -2855,6 +2857,9 @@ void config_set_defaults(void *data)
    const char *def_location         = config_get_default_location();
    const char *def_record           = config_get_default_record();
    const char *def_midi             = config_get_default_midi();
+#ifdef HAVE_TRANSLATE
+   const char *def_ai_service_backend = config_get_default_ai_service_backend();
+#endif
    const char *def_mitm             = DEFAULT_NETPLAY_MITM_SERVER;
    struct video_viewport *custom_vp = &settings->video_vp_custom;
    struct config_float_setting      *float_settings = populate_settings_float (settings, &float_settings_size);
@@ -2938,6 +2943,12 @@ void config_set_defaults(void *data)
       configuration_set_string(settings,
             settings->arrays.cloud_sync_driver,
             def_cloudsync);
+#ifdef HAVE_TRANSLATE
+   if (def_ai_service_backend)
+      configuration_set_string(settings,
+            settings->arrays.ai_service_backend,
+            def_ai_service_backend);
+#endif
    if (def_location)
       configuration_set_string(settings,
             settings->arrays.location_driver,

configuration.h

@@ -512,6 +512,7 @@ typedef struct settings
       char midi_driver[32];
       char midi_input[32];
       char midi_output[32];
+      char ai_service_backend[32];
 #ifdef HAVE_LAKKA
       char cpu_main_gov[32];
       char cpu_menu_gov[32];

intl/msg_hash_us.h

@@ -7376,6 +7376,14 @@ MSG_HASH(
    MENU_ENUM_SUBLABEL_AI_SERVICE_MODE,
    "Show translation as a text overlay (Image Mode), play as Text-To-Speech (Speech), or use a system narrator like NVDA (Narrator)."
    )
+MSG_HASH(
+   MENU_ENUM_LABEL_VALUE_AI_SERVICE_BACKEND,
+   "AI Service Backend"
+   )
+MSG_HASH(
+   MENU_ENUM_SUBLABEL_AI_SERVICE_BACKEND,
+   "Select which translation backend to use. HTTP uses a remote server at the configured URL. Apple uses on-device OCR and translation (macOS/iOS)."
+   )
 MSG_HASH(
    MENU_ENUM_LABEL_VALUE_AI_SERVICE_URL,
    "AI Service URL"

menu/cbs/menu_cbs_sublabel.c

@@ -277,6 +277,7 @@ DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_user_interface_settings_list,  MENU_
 DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_ai_service_settings_list,  MENU_ENUM_SUBLABEL_AI_SERVICE_SETTINGS)
 DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_accessibility_settings_list,  MENU_ENUM_SUBLABEL_ACCESSIBILITY_SETTINGS)
 DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_ai_service_mode,  MENU_ENUM_SUBLABEL_AI_SERVICE_MODE)
+DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_ai_service_backend,  MENU_ENUM_SUBLABEL_AI_SERVICE_BACKEND)
 DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_ai_service_pause,  MENU_ENUM_SUBLABEL_AI_SERVICE_PAUSE)
 DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_ai_service_target_lang,  MENU_ENUM_SUBLABEL_AI_SERVICE_TARGET_LANG)
 DEFAULT_SUBLABEL_MACRO(action_bind_sublabel_ai_service_source_lang,  MENU_ENUM_SUBLABEL_AI_SERVICE_SOURCE_LANG)
@@ -5437,6 +5438,9 @@ int menu_cbs_init_bind_sublabel(menu_file_list_cbs_t *cbs,
          case MENU_ENUM_LABEL_AI_SERVICE_MODE:
             BIND_ACTION_SUBLABEL(cbs, action_bind_sublabel_ai_service_mode);
             break;
+         case MENU_ENUM_LABEL_AI_SERVICE_BACKEND:
+            BIND_ACTION_SUBLABEL(cbs, action_bind_sublabel_ai_service_backend);
+            break;
          case MENU_ENUM_LABEL_AI_SERVICE_PAUSE:
             BIND_ACTION_SUBLABEL(cbs, action_bind_sublabel_ai_service_pause);
             break;

menu/menu_displaylist.c

@@ -8515,6 +8515,7 @@ unsigned menu_displaylist_build_list(
                {MENU_ENUM_LABEL_AI_SERVICE_ENABLE,        PARSE_ONLY_BOOL,   true},
                {MENU_ENUM_LABEL_AI_SERVICE_MODE,          PARSE_ONLY_UINT,   false},
                {MENU_ENUM_LABEL_AI_SERVICE_URL,           PARSE_ONLY_STRING, false},
+               {MENU_ENUM_LABEL_AI_SERVICE_BACKEND,       PARSE_ONLY_STRING_OPTIONS, false},
                {MENU_ENUM_LABEL_AI_SERVICE_PAUSE,         PARSE_ONLY_BOOL,   false},
                {MENU_ENUM_LABEL_AI_SERVICE_SOURCE_LANG,   PARSE_ONLY_UINT,   false},
                {MENU_ENUM_LABEL_AI_SERVICE_TARGET_LANG,   PARSE_ONLY_UINT,   false},
@@ -8525,6 +8526,7 @@ unsigned menu_displaylist_build_list(
                switch (build_list[i].enum_idx)
                {
                   case MENU_ENUM_LABEL_AI_SERVICE_MODE:
+                  case MENU_ENUM_LABEL_AI_SERVICE_BACKEND:
                   case MENU_ENUM_LABEL_AI_SERVICE_URL:
                   case MENU_ENUM_LABEL_AI_SERVICE_PAUSE:
                   case MENU_ENUM_LABEL_AI_SERVICE_SOURCE_LANG:

menu/menu_setting.c

@@ -99,6 +99,7 @@
 #include "../network/cloud_sync_driver.h"
 #include "../record/record_driver.h"
 #include "../tasks/tasks_internal.h"
+#include "../accessibility.h"
 #include "../config.def.h"
 #include "../ui/ui_companion_driver.h"
 #include "../performance_counters.h"
@@ -21100,6 +21101,23 @@ static bool setting_append_list(
          (*list)[list_info->index - 1].action_ok     = &setting_action_ok_uint;
          menu_settings_list_current_add_range(list, list_info, 0, 2, 1, true, true);
 
+         CONFIG_STRING_OPTIONS(
+               list, list_info,
+               settings->arrays.ai_service_backend,
+               sizeof(settings->arrays.ai_service_backend),
+               MENU_ENUM_LABEL_AI_SERVICE_BACKEND,
+               MENU_ENUM_LABEL_VALUE_AI_SERVICE_BACKEND,
+               config_get_default_ai_service_backend(),
+               strdup(config_get_ai_service_backend_options()),
+               &group_info,
+               &subgroup_info,
+               parent_group,
+               general_write_handler,
+               general_read_handler);
+         (*list)[list_info->index - 1].action_ok    = setting_action_ok_uint;
+         (*list)[list_info->index - 1].action_left  = setting_string_action_left_driver;
+         (*list)[list_info->index - 1].action_right = setting_string_action_right_driver;
+
          CONFIG_STRING(
                list, list_info,
                settings->arrays.ai_service_url,

msg_hash.h

@@ -2997,6 +2997,7 @@ enum msg_hash_enums
    MENU_LABEL(ACCESSIBILITY_NARRATOR_SPEECH_SPEED),
    MENU_LABEL(AI_SERVICE_SETTINGS),
    MENU_LABEL(AI_SERVICE_MODE),
+   MENU_LABEL(AI_SERVICE_BACKEND),
    MENU_LABEL(AI_SERVICE_TARGET_LANG),
    MENU_LABEL(AI_SERVICE_SOURCE_LANG),
    MENU_LABEL(AI_SERVICE_URL),

pkg/apple/BaseConfig.xcconfig

@@ -81,6 +81,7 @@ OTHER_CFLAGS = $(inherited) -DHAVE_SWIFT
 OTHER_CFLAGS = $(inherited) -DHAVE_THREADS
 OTHER_CFLAGS = $(inherited) -DHAVE_THREAD_STORAGE
 OTHER_CFLAGS = $(inherited) -DHAVE_TRANSLATE
+OTHER_CFLAGS = $(inherited) -DHAVE_TRANSLATE_APPLE
 OTHER_CFLAGS = $(inherited) -DHAVE_UPDATE_ASSETS
 OTHER_CFLAGS = $(inherited) -DHAVE_UPDATE_CORE_INFO
 OTHER_CFLAGS = $(inherited) -DHAVE_VIDEO_FILTER

pkg/apple/RetroArch_Metal.xcodeproj/project.pbxproj

@@ -94,6 +94,10 @@
 		07B9B9802E49A7A3007E419E /* RetroArchAppShortcuts.swift in Sources */ = {isa = PBXBuildFile; fileRef = 07B9B97D2E49A7A3007E419E /* RetroArchAppShortcuts.swift */; };
 		07B9B9812E49A7A3007E419E /* RetroArchAppShortcuts.swift in Sources */ = {isa = PBXBuildFile; fileRef = 07B9B97D2E49A7A3007E419E /* RetroArchAppShortcuts.swift */; };
 		07B9B9822E49A7A3007E419E /* RetroArchAppShortcuts.swift in Sources */ = {isa = PBXBuildFile; fileRef = 07B9B97D2E49A7A3007E419E /* RetroArchAppShortcuts.swift */; };
+		07ATRANS032025010700000003 /* AppleTranslateBackend.swift in Sources */ = {isa = PBXBuildFile; fileRef = 07ATRANS012025010700000001 /* AppleTranslateBackend.swift */; };
+		07ATRANS042025010700000004 /* AppleTranslateBackend.swift in Sources */ = {isa = PBXBuildFile; fileRef = 07ATRANS012025010700000001 /* AppleTranslateBackend.swift */; };
+		07ATRANS052025010700000005 /* AppleTranslateBackend.swift in Sources */ = {isa = PBXBuildFile; fileRef = 07ATRANS012025010700000001 /* AppleTranslateBackend.swift */; };
+		07ATRANS062025010700000006 /* AppleTranslateBackend.swift in Sources */ = {isa = PBXBuildFile; fileRef = 07ATRANS012025010700000001 /* AppleTranslateBackend.swift */; };
 		07D851CD2CAF06E2005097EA /* CloudKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 075650242C488918004C5E7E /* CloudKit.framework */; };
 		07F2BBC62BE83A4700FD1295 /* InfoPlist.strings in Resources */ = {isa = PBXBuildFile; fileRef = 089C165CFE840E0CC02AAC07 /* InfoPlist.strings */; };
 		07F2BBC82BE83A4700FD1295 /* assets.zip in Resources */ = {isa = PBXBuildFile; fileRef = 9254B33025FA0BA300A1E0DA /* assets.zip */; };
@@ -267,6 +271,8 @@
 		05A8C5BA20DB72F000FF7857 /* ui_cocoa.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = ui_cocoa.m; sourceTree = "<group>"; };
 		05A8C5BD20DB72F000FF7857 /* cocoa_common.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = cocoa_common.m; sourceTree = "<group>"; };
 		05A8C5C220DB72F000FF7857 /* cocoa_common.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = cocoa_common.h; sourceTree = "<group>"; };
+		07ATRANS012025010700000001 /* AppleTranslateBackend.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppleTranslateBackend.swift; sourceTree = "<group>"; };
+		07ATRANS022025010700000002 /* translation_driver_apple.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = translation_driver_apple.h; sourceTree = "<group>"; };
 		05A8C5C820DB72F000FF7857 /* ui_qt.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = ui_qt.h; sourceTree = "<group>"; };
 		05A8C5CE20DB72F000FF7857 /* ui_cocoa.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = ui_cocoa.h; sourceTree = "<group>"; };
 		05A8C5CF20DB72F000FF7857 /* ui_cocoatouch.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = ui_cocoatouch.m; sourceTree = "<group>"; };
@@ -854,8 +860,10 @@
 		05A8C5BB20DB72F000FF7857 /* cocoa */ = {
 			isa = PBXGroup;
 			children = (
+				07ATRANS012025010700000001 /* AppleTranslateBackend.swift */,
 				05A8C5C220DB72F000FF7857 /* cocoa_common.h */,
 				05A8C5BD20DB72F000FF7857 /* cocoa_common.m */,
+				07ATRANS022025010700000002 /* translation_driver_apple.h */,
 			);
 			path = cocoa;
 			sourceTree = "<group>";
@@ -1681,6 +1689,7 @@
 			isa = PBXSourcesBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
+				07ATRANS032025010700000003 /* AppleTranslateBackend.swift in Sources */,
 				07B9B97F2E49A7A3007E419E /* RetroArchAppShortcuts.swift in Sources */,
 				05422E432140C8DB00F09961 /* griffin_glslang.cpp in Sources */,
 				05422E442140C8DB00F09961 /* menu_pipeline.metal in Sources */,
@@ -1696,6 +1705,7 @@
 			isa = PBXSourcesBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
+				07ATRANS042025010700000004 /* AppleTranslateBackend.swift in Sources */,
 				07B9B9812E49A7A3007E419E /* RetroArchAppShortcuts.swift in Sources */,
 				0720994429B1258C001642BB /* griffin_glslang.cpp in Sources */,
 				0720994529B1258C001642BB /* menu_pipeline.metal in Sources */,
@@ -1711,6 +1721,7 @@
 			isa = PBXSourcesBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
+				07ATRANS052025010700000005 /* AppleTranslateBackend.swift in Sources */,
 				07B9B9802E49A7A3007E419E /* RetroArchAppShortcuts.swift in Sources */,
 				07F2BBCC2BE83A4700FD1295 /* griffin_glslang.cpp in Sources */,
 				07F2BBCD2BE83A4700FD1295 /* menu_pipeline.metal in Sources */,
@@ -1726,6 +1737,7 @@
 			isa = PBXSourcesBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
+				07ATRANS062025010700000006 /* AppleTranslateBackend.swift in Sources */,
 				07B9B9822E49A7A3007E419E /* RetroArchAppShortcuts.swift in Sources */,
 				05D7753720A567A700646447 /* griffin_glslang.cpp in Sources */,
 				05770B9920E805160013DABC /* menu_pipeline.metal in Sources */,