libretro-common/file: NULL-check mallocs and unwind resources on OOM

Four OOM-deref sites across the file-I/O subsystem:

nbio_linux.c (nbio_linux_open):
    handle       = (struct nbio_linux_t*)malloc(sizeof(struct nbio_linux_t));
    handle->fd   = fd;         /* NULL-deref on OOM */
    handle->ctx  = ctx;
    handle->len  = lseek(fd, 0, SEEK_END);
    handle->ptr  = malloc(handle->len);
    handle->busy = false;

Two back-to-back mallocs with no NULL checks between the deref'ing
field writes.  Also leaks the open()'d fd and the io_setup()'d
aio context on any OOM.  The sibling nbio_stdio_open (same file
family, line 121) already does this correctly with fclose-on-OOM
and the nested malloc NULL-check; _linux_open was the lone holdout.

nbio_unixmmap.c (nbio_mmap_unix_open):
    handle            = malloc(sizeof(struct nbio_mmap_unix_t));
    handle->fd        = fd;    /* NULL-deref on OOM */
    handle->map_flags = map_flags[mode];
    handle->len       = _len;
    handle->ptr       = ptr;

Same unchecked-handle-malloc pattern; on OOM leaks the fd and the
mmap'd region.

nbio_windowsmmap.c (nbio_mmap_win32_open):
    handle           = (struct nbio_mmap_win32_t*)malloc(sizeof(struct nbio_mmap_win32_t));
    handle->file     = file;   /* NULL-deref on OOM */
    handle->is_write = is_write;
    handle->len      = len.QuadPart;
    handle->ptr      = ptr;

Same, Win32 variant; on OOM leaks the HANDLE and the MapViewOfFile
mapping.

archive_file_7z.c (sevenzip_file_read):
    *buf = malloc((size_t)(outsize + 1));
    ((char*)(*buf))[outsize] = '\0';   /* NULL-deref on OOM */
    memcpy(*buf, output + offset, outsize);

Only one site in 7z extraction; on OOM NULL-derefs on the
NUL-terminator write.  Unlike the nbio sites, this one sits
inside a for-loop that already has an error label; added
res = SZ_ERROR_MEM / outsize = -1 / break so the existing
'!(file_found && res == SZ_OK)' cleanup branch runs.

Fixes in all four cases: NULL-check each malloc, unwind everything
we already acquired above it, and return NULL / -1 as the
respective function's error signal.  free(NULL) / close(-1) are
defined as safe no-ops where relevant but nothing here relies on
that - each unwind is explicit about what it's releasing.

Thread-safety: unchanged.  nbio handles are owned by whichever
thread opened them (typically a task queue thread); the 7z
extractor runs on the main thread during content scans.

Reachability: nbio_*_open is the entry point for every async
file load in the engine - most relevant for menu thumbnails,
where hundreds of handles can be in flight simultaneously on
memory-constrained handhelds.  7z extraction runs whenever a
7z-packed core / content is loaded.

Author: LibretroAdmin

libretro-common/file/archive_file_7z.c

@@ -269,7 +269,18 @@ static int64_t sevenzip_file_read(
                 * We would however need to realloc anyways, because RetroArch
                 * expects a \0 at the end, therefore we allocate new,
                 * copy and free the old one. */
-               *buf = malloc((size_t)(outsize + 1));
+               /* NULL-check the malloc before the NUL-termination
+                * write and the memcpy below dereference '*buf'.  On
+                * OOM mark the extraction failed and break out of the
+                * file-search loop; the '!(file_found && res == SZ_OK)'
+                * branch further down then forces outsize = -1 and
+                * the teardown runs. */
+               if (!(*buf = malloc((size_t)(outsize + 1))))
+               {
+                  res     = SZ_ERROR_MEM;
+                  outsize = -1;
+                  break;
+               }
                ((char*)(*buf))[outsize] = '\0';
                memcpy(*buf, output + offset, outsize);
             }

libretro-common/file/nbio/nbio_linux.c

@@ -113,10 +113,32 @@ static void *nbio_linux_open(const char * filename, unsigned mode)
    }
 
    handle       = (struct nbio_linux_t*)malloc(sizeof(struct nbio_linux_t));
+   /* NULL-check the handle malloc: the next five lines dereferenced
+    * 'handle' unconditionally, so OOM segfaulted.  On failure we
+    * also have to unwind the fd we open()'d above and the aio
+    * context we io_setup()'d - both are resources that would be
+    * owned by the handle had it been allocated. */
+   if (!handle)
+   {
+      io_destroy(ctx);
+      close(fd);
+      return NULL;
+   }
    handle->fd   = fd;
    handle->ctx  = ctx;
    handle->len  = lseek(fd, 0, SEEK_END);
    handle->ptr  = malloc(handle->len);
+   /* Same pattern for the data buffer.  Subsequent begin_read /
+    * begin_write / iterate paths all assume handle->ptr is a valid
+    * buffer of handle->len bytes; passing a NULL through them
+    * would crash in the iocb setup.  Unwind everything we got. */
+   if (!handle->ptr)
+   {
+      free(handle);
+      io_destroy(ctx);
+      close(fd);
+      return NULL;
+   }
    handle->busy = false;
 
    return handle;

libretro-common/file/nbio/nbio_unixmmap.c

@@ -84,6 +84,17 @@ static void *nbio_mmap_unix_open(const char * filename, unsigned mode)
    }
 
    handle            = malloc(sizeof(struct nbio_mmap_unix_t));
+   /* NULL-check the handle malloc: the four field writes below
+    * would segfault on OOM.  On failure munmap the region and
+    * close the fd we acquired above - both were destined to be
+    * owned by the handle and will leak otherwise. */
+   if (!handle)
+   {
+      if (_len != 0)
+         munmap(ptr, _len);
+      close(fd);
+      return NULL;
+   }
    handle->fd        = fd;
    handle->map_flags = map_flags[mode];
    handle->len       = _len;

libretro-common/file/nbio/nbio_windowsmmap.c

@@ -112,6 +112,17 @@ static void *nbio_mmap_win32_open(const char * filename, unsigned mode)
    CloseHandle(mem);
 
    handle           = (struct nbio_mmap_win32_t*)malloc(sizeof(struct nbio_mmap_win32_t));
+   /* NULL-check the handle malloc: the field writes below would
+    * segfault on OOM.  On failure unmap the view and close the
+    * file handle - both are resources destined to be owned by
+    * the handle and will leak otherwise. */
+   if (!handle)
+   {
+      if (ptr)
+         UnmapViewOfFile(ptr);
+      CloseHandle(file);
+      return NULL;
+   }
 
    handle->file     = file;
    handle->is_write = is_write;