Fix mouse grab behavior on Android (#16203)

PatrickStankard · schellingb · web-flow · commit 5452999b2a95 · 2024-03-19T05:33:02.000-07:00
* Add grab_mouse interface for Android
Makes mouse grabbing and 'Game Focus' work on Android with a real mouse
Properly handle relative mouse motion events on Android (SDK 28 and newer)

* Enable workflow_dispatch on CI Android

* Update android_mouse_calculate_deltas callsites

* Add RETRO_DEVICE_MOUSE to android_input_get_capabilities

* Use Handler to trigger UI events (toggle mouse, immersive mode) with 300ms delay

* Enable input_auto_mouse_grab by default for Android

* Handle RARCH_DEVICE_MOUSE_SCREEN in Android input driver

* Add android.hardware.type.pc to manifest

* Don't attempt to set pointer speed via scaling in android_mouse_calculate_deltas

* Keep x/y values within viewport resolution for screen mouse

* Use video_driver_get_size to get width/height

---------

Co-authored-by: Bernhard Schelling &lt;14200249+schellingb@users.noreply.github.com&gt;
diff --git a/.github/workflows/Android.yml b/.github/workflows/Android.yml
@@ -3,8 +3,10 @@ name: CI Android
 on:
   push:
   pull_request:
+  workflow_dispatch:
   repository_dispatch:
     types: [run_build]
+  
 
 permissions:
   contents: read
diff --git a/config.def.h b/config.def.h
@@ -1552,6 +1552,14 @@
 #define DEFAULT_TURBO_DEFAULT_BTN RETRO_DEVICE_ID_JOYPAD_B
 #define DEFAULT_ALLOW_TURBO_DPAD false
 
+/* Enable automatic mouse grab by default
+ * only on Android */
+#if defined(ANDROID)
+#define DEFAULT_INPUT_AUTO_MOUSE_GRAB true
+#else
+#define DEFAULT_INPUT_AUTO_MOUSE_GRAB false
+#endif
+
 #if TARGET_OS_IPHONE
 #define DEFAULT_INPUT_KEYBOARD_GAMEPAD_ENABLE false
 #else
diff --git a/configuration.c b/configuration.c
@@ -2086,7 +2086,7 @@ static struct config_bool_setting *populate_settings_bool(
    SETTING_BOOL("keyboard_gamepad_enable",       &settings->bools.input_keyboard_gamepad_enable, true, DEFAULT_INPUT_KEYBOARD_GAMEPAD_ENABLE, false);
    SETTING_BOOL("input_autodetect_enable",       &settings->bools.input_autodetect_enable, true, DEFAULT_INPUT_AUTODETECT_ENABLE, false);
    SETTING_BOOL("input_allow_turbo_dpad",        &settings->bools.input_allow_turbo_dpad, true, DEFAULT_ALLOW_TURBO_DPAD, false);
-   SETTING_BOOL("input_auto_mouse_grab",         &settings->bools.input_auto_mouse_grab, true, false, false);
+   SETTING_BOOL("input_auto_mouse_grab",         &settings->bools.input_auto_mouse_grab, true, DEFAULT_INPUT_AUTO_MOUSE_GRAB, false);
    SETTING_BOOL("input_remap_binds_enable",      &settings->bools.input_remap_binds_enable, true, true, false);
    SETTING_BOOL("input_hotkey_device_merge",     &settings->bools.input_hotkey_device_merge, true, DEFAULT_INPUT_HOTKEY_DEVICE_MERGE, false);
    SETTING_BOOL("all_users_control_menu",        &settings->bools.input_all_users_control_menu, true, DEFAULT_ALL_USERS_CONTROL_MENU, false);
diff --git a/frontend/drivers/platform_unix.c b/frontend/drivers/platform_unix.c
@@ -2090,6 +2090,8 @@ static void frontend_unix_init(void *data)
          "getVolumeCount", "()I");
    GET_METHOD_ID(env, android_app->getVolumePath, class,
          "getVolumePath", "(Ljava/lang/String;)Ljava/lang/String;");
+   GET_METHOD_ID(env, android_app->inputGrabMouse, class,
+         "inputGrabMouse", "(Z)V");
 
    GET_OBJECT_CLASS(env, class, obj);
    GET_METHOD_ID(env, android_app->getStringExtra, class,
diff --git a/frontend/drivers/platform_unix.h b/frontend/drivers/platform_unix.h
@@ -178,6 +178,7 @@ struct android_app
 
    jmethodID getVolumeCount;
    jmethodID getVolumePath;
+   jmethodID inputGrabMouse;
 
    struct
    {
diff --git a/input/drivers/android_input.c b/input/drivers/android_input.c
@@ -61,11 +61,17 @@ enum {
     AMOTION_EVENT_BUTTON_FORWARD = 1 << 4,
     AMOTION_EVENT_AXIS_VSCROLL = 9,
     AMOTION_EVENT_ACTION_HOVER_MOVE = 7,
-    AINPUT_SOURCE_STYLUS = 0x00004002,
+    AINPUT_SOURCE_STYLUS = 0x00004000 | AINPUT_SOURCE_CLASS_POINTER,
     AMOTION_EVENT_BUTTON_STYLUS_PRIMARY = 1 << 5,
     AMOTION_EVENT_BUTTON_STYLUS_SECONDARY = 1 << 6
 };
 #endif
+/* If using an NDK lower than 16b then add missing definition */
+#ifndef __ANDROID_API_O_MR1__
+enum {
+   AINPUT_SOURCE_MOUSE_RELATIVE = 0x00020000 | AINPUT_SOURCE_CLASS_NAVIGATION
+};
+#endif
 
 /* If using an SDK lower than 24 then add missing relative axis codes */
 #ifndef AMOTION_EVENT_AXIS_RELATIVE_X
@@ -144,6 +150,7 @@ typedef struct android_input
 {
    int64_t quick_tap_time;
    state_device_t pad_states[MAX_USERS];        /* int alignment */
+   int mouse_x, mouse_y;
    int mouse_x_delta, mouse_y_delta;
    int mouse_l, mouse_r, mouse_m, mouse_wu, mouse_wd;
    unsigned pads_connected;
@@ -638,53 +645,77 @@ static int android_check_quick_tap(android_input_t *android)
 }
 
 static INLINE void android_mouse_calculate_deltas(android_input_t *android,
-      AInputEvent *event,size_t motion_ptr)
+      AInputEvent *event,size_t motion_ptr,int source)
 {
-   /* Adjust mouse speed based on ratio
-    * between core resolution and system resolution */
-   float x = 0, y = 0;
-   float                        x_scale      = 1;
-   float                        y_scale      = 1;
-   settings_t *settings                      = config_get_ptr();
-   video_driver_state_t *video_st            = video_state_get_ptr();
-   struct retro_system_av_info *av_info      = &video_st->av_info;
-
-   if (av_info)
+   unsigned video_width, video_height;
+   video_driver_get_size(&video_width, &video_height);
+
+   float x       = 0;
+   float x_delta = 0;
+   float x_min   = 0;
+   float x_max   = (float)video_width;
+
+   float y       = 0;
+   float y_delta = 0;
+   float y_min   = 0;
+   float y_max   = (float)video_height;
+
+   /* AINPUT_SOURCE_MOUSE_RELATIVE is available on Oreo (SDK 26) and newer,
+    * it passes the relative coordinates in the regular X and Y parts.
+    * NOTE: AINPUT_SOURCE_* defines have multiple bits set so do full check */
+   if ((source & AINPUT_SOURCE_MOUSE_RELATIVE) == AINPUT_SOURCE_MOUSE_RELATIVE)
    {
-      video_viewport_t *custom_vp            = &settings->video_viewport_custom;
-      const struct retro_game_geometry *geom = (const struct retro_game_geometry*)&av_info->geometry;
-      x_scale = 2 * (float)geom->base_width  / (float)custom_vp->width;
-      y_scale = 2 * (float)geom->base_height / (float)custom_vp->height;
+      x_delta = AMotionEvent_getX(event, motion_ptr);
+      y_delta = AMotionEvent_getY(event, motion_ptr);
    }
-
-   /* This axis is only available on Android Nougat and on 
-    * Android devices with NVIDIA extensions */
-   if (p_AMotionEvent_getAxisValue)
+   else
    {
-      x = AMotionEvent_getAxisValue(event,AMOTION_EVENT_AXIS_RELATIVE_X,
-            motion_ptr);
-      y = AMotionEvent_getAxisValue(event,AMOTION_EVENT_AXIS_RELATIVE_Y,
-            motion_ptr);
-   }
+      /* This axis is only available on Android Nougat or on
+      * Android devices with NVIDIA extensions */
+      if (p_AMotionEvent_getAxisValue)
+      {
+         x_delta = AMotionEvent_getAxisValue(event,AMOTION_EVENT_AXIS_RELATIVE_X,
+               motion_ptr);
+         y_delta = AMotionEvent_getAxisValue(event,AMOTION_EVENT_AXIS_RELATIVE_Y,
+               motion_ptr);
+      }
 
-   /* If AXIS_RELATIVE had 0 values it might be because we're not 
-    * running Android Nougat or on a device
-    * with NVIDIA extension, so re-calculate deltas based on 
-    * AXIS_X and AXIS_Y. This has limitations
-    * compared to AXIS_RELATIVE because once the Android mouse cursor 
-    * hits the edge of the screen it is
-    * not possible to move the in-game mouse any further in that direction.
-    */
-   if (!x && !y)
-   {
-      x = (AMotionEvent_getX(event, motion_ptr) - android->mouse_x_prev);
-      y = (AMotionEvent_getY(event, motion_ptr) - android->mouse_y_prev);
-      android->mouse_x_prev = AMotionEvent_getX(event, motion_ptr);
-      android->mouse_y_prev = AMotionEvent_getY(event, motion_ptr);
+      /* If AXIS_RELATIVE had 0 values it might be because we're not
+      * running Android Nougat or on a device
+      * with NVIDIA extension, so re-calculate deltas based on
+      * AXIS_X and AXIS_Y. This has limitations
+      * compared to AXIS_RELATIVE because once the Android mouse cursor
+      * hits the edge of the screen it is
+      * not possible to move the in-game mouse any further in that direction.
+      */
+      if (!x_delta && !y_delta)
+      {
+         x = AMotionEvent_getX(event, motion_ptr);
+         y = AMotionEvent_getY(event, motion_ptr);
+
+         x_delta = (x_delta - android->mouse_x_prev);
+         y_delta = (y_delta - android->mouse_y_prev);
+
+         android->mouse_x_prev = x;
+         android->mouse_y_prev = y;
+      }
    }
 
-   android->mouse_x_delta = ceil(x) * x_scale;
-   android->mouse_y_delta = ceil(y) * y_scale;
+   android->mouse_x_delta = x_delta;
+   android->mouse_y_delta = y_delta;
+
+   if (!x) x = android->mouse_x + android->mouse_x_delta;
+   if (!y) y = android->mouse_y + android->mouse_y_delta;
+
+   /* x and y are used for the screen mouse, so we want
+    * to avoid values outside of the viewport resolution */
+   if (x < x_min) x = x_min;
+   else if (x > x_max) x = x_max;
+   if (y < y_min) y = y_min;
+   else if (y > y_max) y = y_max;
+
+   android->mouse_x = x;
+   android->mouse_y = y;
 }
 
 static INLINE void android_input_poll_event_type_motion(
@@ -697,13 +728,13 @@ static INLINE void android_input_poll_event_type_motion(
    bool keyup        = (
             action == AMOTION_EVENT_ACTION_UP
          || action == AMOTION_EVENT_ACTION_CANCEL
-         || action == AMOTION_EVENT_ACTION_POINTER_UP)
-         || (source == AINPUT_SOURCE_MOUSE &&
-             action != AMOTION_EVENT_ACTION_DOWN);
+         || action == AMOTION_EVENT_ACTION_POINTER_UP);
 
    /* If source is mouse then calculate button state
-    * and mouse deltas and don't process as touchscreen event */
-   if (source == AINPUT_SOURCE_MOUSE)
+    * and mouse deltas and don't process as touchscreen event.
+    * NOTE: AINPUT_SOURCE_* defines have multiple bits set so do full check */
+   if (    (source & AINPUT_SOURCE_MOUSE) == AINPUT_SOURCE_MOUSE
+        || (source & AINPUT_SOURCE_MOUSE_RELATIVE) == AINPUT_SOURCE_MOUSE_RELATIVE)
    {
       /* getButtonState requires API level 14 */
       if (p_AMotionEvent_getButtonState)
@@ -732,7 +763,7 @@ static INLINE void android_input_poll_event_type_motion(
             android->mouse_l = 0;
       }
 
-      android_mouse_calculate_deltas(android,event,motion_ptr);
+      android_mouse_calculate_deltas(android,event,motion_ptr,source);
 
       return;
    }
@@ -785,7 +816,7 @@ static INLINE void android_input_poll_event_type_motion(
       if ((       action == AMOTION_EVENT_ACTION_MOVE 
                || action == AMOTION_EVENT_ACTION_HOVER_MOVE) 
             && ENABLE_TOUCH_SCREEN_MOUSE)
-         android_mouse_calculate_deltas(android,event,motion_ptr);
+         android_mouse_calculate_deltas(android,event,motion_ptr,source);
 
       for (motion_ptr = 0; motion_ptr < pointer_max; motion_ptr++)
       {
@@ -850,7 +881,7 @@ static INLINE void android_input_poll_event_type_motion_stylus(
             android->mouse_l = 0;
          }
 
-         android_mouse_calculate_deltas(android,event,motion_ptr);
+         android_mouse_calculate_deltas(android,event,motion_ptr,source);
       }
 
       if (action == AMOTION_EVENT_ACTION_MOVE) {
@@ -893,7 +924,7 @@ static INLINE void android_input_poll_event_type_motion_stylus(
       {
          android->mouse_l        = 0;
 
-         android_mouse_calculate_deltas(android,event,motion_ptr);
+         android_mouse_calculate_deltas(android,event,motion_ptr,source);
       }
 
       // pointer was already released during AMOTION_EVENT_ACTION_HOVER_MOVE
@@ -967,7 +998,7 @@ static int android_input_get_id_port(android_input_t *android, int id,
    unsigned i;
    int ret = -1;
    if (source & (AINPUT_SOURCE_TOUCHSCREEN | AINPUT_SOURCE_MOUSE |
-            AINPUT_SOURCE_TOUCHPAD))
+            AINPUT_SOURCE_MOUSE_RELATIVE | AINPUT_SOURCE_TOUCHPAD))
          ret = 0; /* touch overlay is always user 1 */
 
    for (i = 0; i < android->pads_connected; i++)
@@ -1565,7 +1596,10 @@ static void android_input_poll_input_default(android_input_t *android)
                else if ((source & AINPUT_SOURCE_STYLUS) == AINPUT_SOURCE_STYLUS)
                   android_input_poll_event_type_motion_stylus(android, event,
                         port, source);
-               else if ((source & (AINPUT_SOURCE_TOUCHSCREEN | AINPUT_SOURCE_MOUSE)))
+               /* Only handle events from a touchscreen or mouse */
+               else if ((source & (AINPUT_SOURCE_TOUCHSCREEN 
+                           | AINPUT_SOURCE_MOUSE
+                           | AINPUT_SOURCE_MOUSE_RELATIVE)))
                   android_input_poll_event_type_motion(android, event,
                         port, source);
                else
@@ -1774,6 +1808,7 @@ static int16_t android_input_state(
       case RETRO_DEVICE_KEYBOARD:
          return (id && id < RETROK_LAST) && BIT_GET(android_key_state[ANDROID_KEYBOARD_PORT], rarch_keysym_lut[id]);
       case RETRO_DEVICE_MOUSE:
+      case RARCH_DEVICE_MOUSE_SCREEN:
          {
             int val = 0;
             if (port > 0)
@@ -1788,11 +1823,17 @@ static int16_t android_input_state(
                case RETRO_DEVICE_ID_MOUSE_MIDDLE:
                   return android->mouse_m;
                case RETRO_DEVICE_ID_MOUSE_X:
+                  if (device == RARCH_DEVICE_MOUSE_SCREEN)
+                     return android->mouse_x;
+
                   val = android->mouse_x_delta;
                   android->mouse_x_delta = 0;
                   /* flush delta after it has been read */
                   return val;
                case RETRO_DEVICE_ID_MOUSE_Y:
+                  if (device == RARCH_DEVICE_MOUSE_SCREEN)
+                     return android->mouse_y;
+
                   val = android->mouse_y_delta;
                   android->mouse_y_delta = 0;
                   /* flush delta after it has been read */
@@ -1907,6 +1948,7 @@ static uint64_t android_input_get_capabilities(void *data)
    return
         (1 << RETRO_DEVICE_JOYPAD)
       | (1 << RETRO_DEVICE_POINTER)
+      | (1 << RETRO_DEVICE_MOUSE)
       | (1 << RETRO_DEVICE_KEYBOARD)
       | (1 << RETRO_DEVICE_LIGHTGUN)
       | (1 << RETRO_DEVICE_ANALOG);
@@ -2056,6 +2098,18 @@ static float android_input_get_sensor_input(void *data,
    return 0.0f;
 }
 
+static void android_input_grab_mouse(void *data, bool state)
+{
+   JNIEnv *env = jni_thread_getenv();
+
+   if (!env || !g_android)
+      return;
+
+   if (g_android->inputGrabMouse)
+      CALL_VOID_METHOD_PARAM(env, g_android->activity->clazz,
+            g_android->inputGrabMouse, state);
+}
+
 static void android_input_keypress_vibrate()
 {
    static const int keyboard_press = 3;
@@ -2077,8 +2131,7 @@ input_driver_t input_android = {
    android_input_get_sensor_input,
    android_input_get_capabilities,
    "android",
-
-   NULL,                            /* grab_mouse */
+   android_input_grab_mouse,
    NULL,
    android_input_keypress_vibrate
 };
diff --git a/menu/menu_setting.c b/menu/menu_setting.c
@@ -15272,7 +15272,7 @@ static bool setting_append_list(
                   &settings->bools.input_auto_mouse_grab,
                   MENU_ENUM_LABEL_INPUT_AUTO_MOUSE_GRAB,
                   MENU_ENUM_LABEL_VALUE_INPUT_AUTO_MOUSE_GRAB,
-                  false,
+                  DEFAULT_INPUT_AUTO_MOUSE_GRAB,
                   MENU_ENUM_LABEL_VALUE_OFF,
                   MENU_ENUM_LABEL_VALUE_ON,
                   &group_info,
diff --git a/pkg/android/phoenix/AndroidManifest.xml b/pkg/android/phoenix/AndroidManifest.xml
@@ -6,6 +6,7 @@
     android:versionName="1.17.0"
     android:installLocation="internalOnly">
     <uses-feature android:glEsVersion="0x00020000" />
+    <uses-feature android:name="android.hardware.type.pc" android:required="false"/>
     <uses-feature android:name="android.hardware.touchscreen" android:required="false"/>
     <uses-feature android:name="android.software.leanback" android:required="false" />
     <uses-feature android:name="android.hardware.gamepad" android:required="false"/>
diff --git a/pkg/android/phoenix/src/com/retroarch/browser/retroactivity/RetroActivityFuture.java b/pkg/android/phoenix/src/com/retroarch/browser/retroactivity/RetroActivityFuture.java

Original file line number	Diff line number	Diff line change
`@@ -178,6 +178,7 @@ struct android_app`
`178`	`178`
`179`	`179`	`jmethodID getVolumeCount;`
`180`	`180`	`jmethodID getVolumePath;`
	`181`	`+ jmethodID inputGrabMouse;`
`181`	`182`
`182`	`183`	`struct`
`183`	`184`	`{`