gfx/drivers: scattered OOM and NULL-return fixes across drm / switch_nx / gx2 / mali_fbdev

Four unrelated OOM/NULL-deref bugs found in a sweep across the
gfx/drivers tree.  Lumped into one commit because each is small
on its own and they all fall under the same 'unchecked allocation
in a gfx driver' theme.

=== gfx/drivers/drm_gfx.c: get_plane_prop_id ===

    props      = drmModeObjectGetProperties(drm.fd,
          plane->plane_id, DRM_MODE_OBJECT_PLANE);
    props_info = malloc(props->count_props * sizeof *props_info);

    for (j = 0; j < props->count_props; ++j)
       props_info[j] = drmModeGetProperty(drm.fd, props->props[j]);

Two NULL-deref sites stacked:
  - drmModeObjectGetProperties returns NULL on kernel/driver
    error or when the plane exposes no properties; accessing
    props->count_props NULL-derefs.
  - malloc is unchecked; props_info[j] = ... below NULL-derefs
    on OOM.

Fix: NULL-check both; 'continue' to the next plane on either
failure.  If no plane produces the requested property, the
function already falls through to 'return 0' (the 'not found'
signal) at the end.

NOTE: this function has pre-existing resource leaks
(plane_resources, plane, props, props_info are all libdrm-
allocated and never freed, including on the early 'return
props_info[j]->prop_id' path inside the loop).  Those are out
of scope for this fix - plugging them needs a goto-cleanup
rewrite.

=== gfx/drivers/switch_nx_gfx.c: switch_set_texture_frame ===

    if (sw->menu_texture.pixels)
       sw->menu_texture.pixels = realloc(sw->menu_texture.pixels, sz);
    else
       sw->menu_texture.pixels = malloc(sz);
    if (!sw->menu_texture.pixels)
       return;

Classic realloc-assign-self leak: on OOM realloc returns NULL
but the old menu-texture buffer is still valid; the self-assign
overwrites the only pointer to it and the subsequent NULL-check
catches the crash but not the leak.  Every size-change on the
menu texture (common during menu navigation - icon/thumbnail
scaling) leaks the previous buffer.

Fix: realloc-to-tmp, NULL-check, only commit the new pointer on
success.

=== gfx/drivers/gx2_gfx.c: gx2_set_shader ===

    wiiu->shader_preset = calloc(1, sizeof(*wiiu->shader_preset));

    if (!video_shader_load_preset_into_shader(path, wiiu->shader_preset))

calloc is unchecked.  video_shader_load_preset_into_shader (in
gfx/video_shader_parse.c:2294) does not NULL-check its 'shader'
argument either - it walks through to config-file parsing and
writes into shader->... fields, NULL-derefing on the first
write.

Fix: NULL-check the calloc, bail out of set_shader before the
call on OOM.

=== gfx/drivers_context/mali_fbdev_ctx.c: gfx_ctx_mali_fbdev_clear_screen ===

    int fd                = open("/dev/fb0", O_RDWR);
    ioctl (fd, FBIOGET_VSCREENINFO, &vinfo);
    buffer_size           = vinfo.xres * vinfo.yres * vinfo.bits_per_pixel / 8;
    buffer                = calloc(1, buffer_size);
    write(fd,buffer,buffer_size);
    free(buffer);
    close(fd);

Three cascading failures:
  - open() can return -1 (fb0 missing, permission denied,
    exclusive-use by another process).  ioctl(-1, ...) returns
    EBADF but doesn't crash; vinfo is left uninitialised stack
    garbage.
  - buffer_size is derived from that garbage - could be 0, could
    be multi-GB.  calloc(1, absurd) returns NULL on the big
    case.
  - write(fd, NULL, buffer_size) on NULL buffer is EFAULT on
    Linux but undefined per POSIX.

Fix: guard each step.  Early-return on open() failure; close()
and return on ioctl() failure; skip the write() on calloc
failure.  The whole function is a cosmetic teardown step
(clearing the framebuffer on exit), so silent no-op on error is
the right policy.

=== Thread-safety ===

All four run on the main / driver-init thread.  No lock changes.

=== Reachability ===

  - drm_gfx: get_plane_prop_id runs once at video-driver init on
    bare-metal DRM targets; hit on every startup.
  - switch_nx_gfx: set_texture_frame hits every menu-texture
    size change (menu navigation).
  - gx2_gfx: set_shader hits on shader-preset load (menu action).
  - mali_fbdev_ctx: clear_screen runs at process exit on Mali
    framebuffer targets.

All realistic on the handheld/embedded hardware these drivers
target, where OOM on 128-512MB systems is a real concern.

Author: LibretroAdmin

gfx/drivers/drm_gfx.c

@@ -360,7 +360,23 @@ static uint32_t get_plane_prop_id(uint32_t obj_id, const char *name)
        * This implementation must be improved. */
       props      = drmModeObjectGetProperties(drm.fd,
             plane->plane_id, DRM_MODE_OBJECT_PLANE);
+      /* drmModeObjectGetProperties returns NULL on kernel/driver
+       * error or if the plane has no properties; previously
+       * 'props->count_props' NULL-deref'd in that case.  Also
+       * malloc on the next line was unchecked and props_info[j]
+       * below would NULL-deref on OOM.  On either failure skip
+       * this plane and continue; the caller falls through to
+       * 'return 0' (not-found) if no plane yields the prop.
+       *
+       * NOTE: pre-existing leaks in this function (plane_resources,
+       * plane, props, props_info are all libdrm-allocated and
+       * never freed even on the success path that 'return's from
+       * inside the loop) are out of scope for this fix. */
+      if (!props)
+         continue;
       props_info = malloc(props->count_props * sizeof *props_info);
+      if (!props_info)
+         continue;
 
       for (j = 0; j < props->count_props; ++j)
          props_info[j] =	drmModeGetProperty(drm.fd, props->props[j]);

gfx/drivers/gx2_gfx.c

@@ -247,6 +247,13 @@ static bool gx2_set_shader(void *data,
 
       wiiu->shader_preset = calloc(1, sizeof(*wiiu->shader_preset));
 
+      /* NULL-check the calloc before passing the pointer to
+       * video_shader_load_preset_into_shader, which does not
+       * NULL-check its 'shader' parameter and will NULL-deref
+       * on the first field write. */
+      if (!wiiu->shader_preset)
+         return false;
+
       if (!video_shader_load_preset_into_shader(path, wiiu->shader_preset))
       {
          free(wiiu->shader_preset);

gfx/drivers/switch_nx_gfx.c

@@ -825,8 +825,19 @@ static void switch_set_texture_frame(
         int xsf, ysf, sf;
         struct scaler_ctx *sctx = NULL;
 
+        /* realloc-to-tmp to avoid the classic realloc-assign-self
+         * leak: the pre-patch form 'pixels = realloc(pixels, sz)'
+         * overwrites the only pointer to the old menu-texture
+         * buffer with NULL on OOM, leaking it.  The subsequent
+         * 'if (!pixels) return' catches the crash but not the
+         * leak. */
         if (sw->menu_texture.pixels)
-            sw->menu_texture.pixels = realloc(sw->menu_texture.pixels, sz);
+        {
+            void *tmp = realloc(sw->menu_texture.pixels, sz);
+            if (!tmp)
+                return;
+            sw->menu_texture.pixels = tmp;
+        }
         else
             sw->menu_texture.pixels = malloc(sz);
 

gfx/drivers_context/mali_fbdev_ctx.c

@@ -144,11 +144,32 @@ static void gfx_ctx_mali_fbdev_clear_screen(void)
    struct fb_var_screeninfo vinfo;
    void *buffer          = NULL;
    int fd                = open("/dev/fb0", O_RDWR);
-   ioctl (fd, FBIOGET_VSCREENINFO, &vinfo);
+   /* open() returns -1 on failure (fb0 missing, permission denied,
+    * exclusive use, etc.).  The subsequent ioctl / write on an
+    * invalid fd would not crash but would read garbage out of
+    * 'vinfo' (uninitialised stack), produce nonsense buffer_size,
+    * and write(-1, NULL, garbage_size) below NULL-derefs inside
+    * the write() syscall boundary when buffer is also NULL from
+    * the calloc failure.  Just skip the framebuffer clear on
+    * error - it's a cosmetic teardown step, not load-bearing. */
+   if (fd < 0)
+      return;
+   if (ioctl (fd, FBIOGET_VSCREENINFO, &vinfo) < 0)
+   {
+      close(fd);
+      return;
+   }
    buffer_size           = vinfo.xres * vinfo.yres * vinfo.bits_per_pixel / 8;
    buffer                = calloc(1, buffer_size);
-   write(fd,buffer,buffer_size);
-   free(buffer);
+   /* NULL-check the calloc: write(fd, NULL, buffer_size) is
+    * undefined (POSIX leaves write-from-NULL as EFAULT-or-crash
+    * territory, and Linux returns EFAULT but some implementations
+    * don't). */
+   if (buffer)
+   {
+      write(fd,buffer,buffer_size);
+      free(buffer);
+   }
    close(fd);
 
    /* Clear framebuffer and set cursor on again */