From aa3c98e6718d8e2574bb845077f3763c3451d797 Mon Sep 17 00:00:00 2001 From: Christian Tabedzki <35670232+tabedzki@users.noreply.github.com> Date: Thu, 18 Jun 2026 16:41:49 -0400 Subject: [PATCH] feat: add ffmpeg GPU-accelerated encoding path for video acquisition Adds an optional ffmpeg recording backend for the camera path, selected by RigParameters.useFFmpeg, supporting NVENC (NVIDIA GPU) and libx264 (CPU). Falls back to the VideoWriter (Motion JPEG 2000) path with a warning if ffmpeg is not on the system PATH. How it works: FLIR camera -> IMAQ buffer -> FramesAcquiredFcn -> ffmpeg stdin -> .mp4 (pushFramesToFFmpeg) (NVENC/libx264) - configureSingleCamera.m: launches ffmpeg via java.lang.ProcessBuilder to obtain a real handle on its stdin; builds the command as a token list; drains ffmpeg's merged stdout/stderr on a timer so its pipe buffer cannot deadlock encoding; forces -pix_fmt yuv420p on output for broad player compatibility with the grayscale source; sets LoggingMode='memory' so frames are buffered for the callback. NVENC uses bitrate control, libx264 uses CRF. - startVideoAcquisition.m: registers a FramesAcquiredFcn that drains the IMAQ buffer and streams raw frames to ffmpeg's stdin; resolves the actual output filename (ffmpeg writes .mp4). - pushFramesToFFmpeg.m: shared frame-drain helper; transposes each frame so column-major serialisation matches ffmpeg rawvideo 'gray' row-major byte order; warns (does not crash the experiment loop) on a broken pipe. - stopVideoAcquisition.m: flushes remaining frames, closes stdin (EOF lets ffmpeg finalize the file), waits for the process, and tears down the timer. - checkCameraRigParameters.m: validates ffmpegEncoder when useFFmpeg=true; warns (not errors) if ffmpeg is not on PATH. - RigParameters.m.example: NVENC and libx264 configuration blocks. Tests (hardware-free, no Image Acquisition Toolbox or camera required): - sensors/camera/tests/smokeTestFFmpegTransport.m drives the real transport through a real ffmpeg subprocess with a mocked camera and verifies a readable .mp4 is produced with a marker pixel at the correct coordinate (catches transpose / byte-order regressions). - FakeImaqCamera.m mocks the IMAQ object; runCameraTests.m is a headless/CI runner; README.md documents how to run them. Run: matlab -batch "addpath('sensors/camera/tests'); runCameraTests" Assisted-by: ClaudeCode:claude-opus-4.8 --- extras/RigParameters.m.example | 25 +- sensors/camera/checkCameraRigParameters.m | 26 +- sensors/camera/configureSingleCamera.m | 257 +++++++++++++++++- sensors/camera/pushFramesToFFmpeg.m | 42 +++ sensors/camera/startVideoAcquisition.m | 40 ++- sensors/camera/stopVideoAcquisition.m | 56 +++- sensors/camera/tests/FakeImaqCamera.m | 36 +++ sensors/camera/tests/README.md | 65 +++++ sensors/camera/tests/runCameraTests.m | 36 +++ .../camera/tests/smokeTestFFmpegTransport.m | 122 +++++++++ 10 files changed, 678 insertions(+), 27 deletions(-) create mode 100644 sensors/camera/pushFramesToFFmpeg.m create mode 100644 sensors/camera/tests/FakeImaqCamera.m create mode 100644 sensors/camera/tests/README.md create mode 100644 sensors/camera/tests/runCameraTests.m create mode 100644 sensors/camera/tests/smokeTestFFmpegTransport.m diff --git a/extras/RigParameters.m.example b/extras/RigParameters.m.example index d1acba7..1152799 100644 --- a/extras/RigParameters.m.example +++ b/extras/RigParameters.m.example @@ -47,11 +47,34 @@ classdef RigParameters % ----------------------------------------------------------------------- hasCamera = false % master on/off switch video_parent_path = 'D:\Videos' % root directory for video files - video_ext = '.mj2' % output file extension video_acquisition_rate = 30 % camera frame rate (fps) video_exposure_time_in_microseconds = 5000 % exposure time; comment out to use camera default video_gain = 0 % camera analog gain + % --- Encoder: VideoWriter (default) --- + % No extra dependencies. Produces Motion JPEG 2000 (.mj2) files. + useFFmpeg = false + video_ext = '.mj2' + + % --- Encoder: ffmpeg (optional, requires ffmpeg on system PATH) --- + % Set useFFmpeg = true and choose ONE of the two encoder configurations + % below. Falls back to VideoWriter automatically if ffmpeg is unavailable. + % + % NVENC — NVIDIA GPU (Windows, recommended for rigs with NVIDIA cards): + % ffmpegEncoder = 'h264_nvenc' + % ffmpegPreset = 'fast' % 'fast' | 'medium' | 'slow' | 'lossless' + % ffmpegBitrate = '5M' % target bitrate; higher = better quality / larger file + % ffmpegCRF = [] % leave empty — NVENC uses bitrate control, not CRF + % + % libx264 — CPU fallback (any machine with ffmpeg, no GPU required): + % ffmpegEncoder = 'libx264' + % ffmpegPreset = 'fast' % 'ultrafast' | 'fast' | 'medium' | 'slow' + % ffmpegCRF = 23 % 0-51, lower = better quality; 18 ≈ visually lossless + % ffmpegBitrate = [] % leave empty — libx264 uses CRF, not bitrate + % + % Power-user escape hatch (appended verbatim to the ffmpeg command): + % ffmpegExtraArgs = '' + end methods (Static, Access = protected) diff --git a/sensors/camera/checkCameraRigParameters.m b/sensors/camera/checkCameraRigParameters.m index ad34661..b43fc0c 100644 --- a/sensors/camera/checkCameraRigParameters.m +++ b/sensors/camera/checkCameraRigParameters.m @@ -3,6 +3,11 @@ function checkCameraRigParameters() % % Throws an error listing any missing properties required by % startVideoAcquisition / configureSingleCamera. +% +% When RigParameters.useFFmpeg is true, also checks that ffmpegEncoder is +% set and that ffmpeg is available on the system PATH (warns and does not +% error if ffmpeg is missing — configureSingleCamera will fall back to +% VideoWriter automatically). required = {'video_parent_path', 'video_ext', 'video_acquisition_rate', 'video_gain'}; @@ -15,8 +20,27 @@ function checkCameraRigParameters() if ~isempty(missing) error('PureVirmen:missingRigParameters', ... - 'Missing required RigParameters for video acquisition: %s\n', ... + 'Missing required RigParameters for video acquisition: %s', ... strjoin(missing, ', ')); end +% ffmpeg-specific checks +if isprop(RigParameters, 'useFFmpeg') && RigParameters.useFFmpeg + + if ~isprop(RigParameters, 'ffmpegEncoder') || isempty(RigParameters.ffmpegEncoder) + error('PureVirmen:missingRigParameters', ... + ['RigParameters.ffmpegEncoder must be set when useFFmpeg = true.\n' ... + 'Use ''h264_nvenc'' (NVIDIA GPU) or ''libx264'' (CPU).']); + end + + [status, ~] = system('ffmpeg -version'); + if status ~= 0 + warning('PureVirmen:noFFmpeg', ... + ['ffmpeg not found on system PATH. ' ... + 'Recording will fall back to VideoWriter (Motion JPEG 2000). ' ... + 'To use ffmpeg, install it and ensure it is on the PATH.']); + end + +end + end diff --git a/sensors/camera/configureSingleCamera.m b/sensors/camera/configureSingleCamera.m index 4e7ffd3..3a19db3 100644 --- a/sensors/camera/configureSingleCamera.m +++ b/sensors/camera/configureSingleCamera.m @@ -1,24 +1,49 @@ function v = configureSingleCamera(RigParameters, video_filename) -%CONFIGURESINGLECAMERA Configure a single FLIR camera for disk-logged recording. +%CONFIGURESINGLECAMERA Configure a FLIR camera for disk-logged recording. % % v = configureSingleCamera(RigParameters, video_filename) % -% Configures a FLIR camera via the GenTL adapter (Image Acquisition Toolbox -% + FLIR Spinnaker GenTL support) and sets up a VideoWriter disk logger -% (Motion JPEG 2000). The returned videoinput object v is ready to start; -% call start(v) to begin acquisition. +% Two recording paths are supported, selected by RigParameters.useFFmpeg: +% +% VideoWriter path (default, useFFmpeg = false): +% Configures a FLIR camera via the GenTL adapter and sets up a VideoWriter +% (Motion JPEG 2000) disk logger. Returns a videoinput object. +% Requires: Image Acquisition Toolbox + FLIR Spinnaker GenTL support. +% +% ffmpeg path (useFFmpeg = true): +% Configures the FLIR camera via GenTL and pipes raw frames to an ffmpeg +% subprocess for GPU-accelerated (NVENC) or CPU (libx264) H.264 encoding. +% Returns a struct instead of a videoinput object; pass it directly to +% startVideoAcquisition / stopVideoAcquisition which handle both types. +% Requires: ffmpeg on system PATH. +% See RigParameters.m.example for encoder configuration. % % Required RigParameters properties: % video_acquisition_rate - frame rate in fps % video_gain - analog gain % % Optional RigParameters properties: -% video_exposure_time_in_microseconds - if present and nonzero, sets -% camera exposure; otherwise the camera default is used. -% -% Requires: Image Acquisition Toolbox with FLIR Spinnaker GenTL support. -% Install the support package via MATLAB Add-On Explorer by searching for -% "FLIR Spinnaker". +% video_exposure_time_in_microseconds - exposure; omit to use camera default +% useFFmpeg - logical, default false +% ffmpegEncoder - 'h264_nvenc' (NVIDIA GPU) or 'libx264' (CPU) +% ffmpegPreset - encoder preset string +% ffmpegBitrate - target bitrate string for NVENC (e.g. '5M') +% ffmpegCRF - quality value for libx264 (0-51) +% ffmpegExtraArgs - raw string appended to ffmpeg command + +useFFmpeg = isprop(RigParameters, 'useFFmpeg') && RigParameters.useFFmpeg; + +if useFFmpeg + v = configureWithFFmpeg(RigParameters, video_filename); +else + v = configureWithVideoWriter(RigParameters, video_filename); +end + +end + +% ------------------------------------------------------------------------- + +function v = configureWithVideoWriter(RigParameters, video_filename) imaqreset; v = videoinput('gentl', 1, 'Mono8'); @@ -37,11 +62,215 @@ v.FramesPerTrigger = Inf; v.TriggerRepeat = Inf; -logfile = VideoWriter(video_filename, 'Motion JPEG 2000'); -logfile.MJ2BitDepth = 8; -logfile.FrameRate = RigParameters.video_acquisition_rate; +logfile = VideoWriter(video_filename, 'Motion JPEG 2000'); +logfile.MJ2BitDepth = 8; +logfile.FrameRate = RigParameters.video_acquisition_rate; v.LoggingMode = 'disk'; v.DiskLogger = logfile; end + +% ------------------------------------------------------------------------- + +function v = configureWithFFmpeg(RigParameters, video_filename) +% Check ffmpeg availability +[status, ~] = system('ffmpeg -version'); +if status ~= 0 + warning('PureVirmen:noFFmpeg', ... + ['ffmpeg not found on PATH. ' ... + 'Falling back to VideoWriter (Motion JPEG 2000).']); + v = configureWithVideoWriter(RigParameters, video_filename); + return; +end + +% Configure the FLIR camera. Frames are NOT written to disk by IMAQ; they are +% buffered in memory and a FramesAcquiredFcn callback (wired up in +% startVideoAcquisition) drains them and streams raw bytes to ffmpeg's stdin. +imaqreset; +cam = videoinput('gentl', 1, 'Mono8'); +src = getselectedsource(cam); + +src.AcquisitionFrameRateEnable = 'true'; +src.AcquisitionFrameRate = RigParameters.video_acquisition_rate; + +if isprop(RigParameters, 'video_exposure_time_in_microseconds') && ... + RigParameters.video_exposure_time_in_microseconds + src.ExposureTime = RigParameters.video_exposure_time_in_microseconds; +end + +src.Gain = RigParameters.video_gain; + +cam.FramesPerTrigger = Inf; +cam.TriggerRepeat = Inf; +% Buffer frames in memory for the callback to drain (no IMAQ disk logger on +% the ffmpeg path). 'memory' is the IMAQ default but we set it explicitly so +% a recycled/preconfigured object can't leave a stale DiskLogger attached. +cam.LoggingMode = 'memory'; + +% Build ffmpeg command +vidInfo = cam.VideoResolution; % [width height] +width = vidInfo(1); +height = vidInfo(2); +fps = RigParameters.video_acquisition_rate; +encoder = 'h264_nvenc'; +if isprop(RigParameters, 'ffmpegEncoder') + encoder = RigParameters.ffmpegEncoder; +end + +encoderArgs = buildEncoderArgs(RigParameters, encoder); + +extraArgs = ''; +if isprop(RigParameters, 'ffmpegExtraArgs') && ~isempty(RigParameters.ffmpegExtraArgs) + extraArgs = RigParameters.ffmpegExtraArgs; +end + +% Replace .mj2 extension with .mp4 if caller passed the wrong extension +[fdir, fname, ~] = fileparts(video_filename); +mp4_filename = fullfile(fdir, [fname '.mp4']); + +% Camera delivers single-channel Mono8 (-pix_fmt gray on input). Force a +% widely-compatible output pixel format so the .mp4 plays in standard +% players / H.264 decoders that don't accept grayscale streams. +cmdParts = { 'ffmpeg', '-y', ... + '-f', 'rawvideo', '-pix_fmt', 'gray', ... + '-s', sprintf('%dx%d', width, height), '-r', num2str(fps), ... + '-i', 'pipe:0', ... + '-c:v', encoder }; +cmdParts = [cmdParts, splitArgs(encoderArgs), splitArgs(extraArgs), ... + {'-pix_fmt', 'yuv420p', mp4_filename}]; + +% Launch ffmpeg via Java ProcessBuilder so we get a real handle on its stdin. +% redirectErrorStream keeps stderr from filling the OS pipe buffer (which +% would otherwise deadlock ffmpeg); a drain thread empties it continuously. +try + pb = java.lang.ProcessBuilder(cmdParts); + pb.redirectErrorStream(true); + proc = pb.start(); +catch err + delete(cam); + error('PureVirmen:ffmpegLaunchFailed', ... + 'Failed to launch ffmpeg.\n Command: %s\n Error: %s', ... + strjoin(cmdParts, ' '), err.message); +end + +stdin = proc.getOutputStream(); % we write raw frames here + +% Background-drain ffmpeg's merged stdout/stderr so its output buffer never +% fills and stalls encoding. A MATLAB timer polling the reader is portable +% (no custom Java class needed); it stops itself once ffmpeg closes the stream. +logReader = java.io.BufferedReader( ... + java.io.InputStreamReader(proc.getInputStream())); +drainTimer = timer( ... + 'ExecutionMode', 'fixedSpacing', ... + 'Period', 0.5, ... + 'BusyMode', 'drop', ... + 'TimerFcn', @(t, ~) drainReader(t, logReader), ... + 'Name', 'ffmpegLogDrain'); +start(drainTimer); + +% Package everything the start/stop functions need. The frame-streaming +% callback is wired up in startVideoAcquisition once acquisition begins. +v = struct(); +v.type = 'ffmpeg'; +v.cam = cam; +v.proc = proc; +v.stdin = stdin; +v.drainTimer = drainTimer; +v.filename = mp4_filename; +v.width = width; +v.height = height; +v.cmd = strjoin(cmdParts, ' '); + +end + +% ------------------------------------------------------------------------- + +function drainReader(t, reader) +% Read and discard all currently-available lines from ffmpeg's output. When +% the stream is closed (ffmpeg exited), readLine returns [] and we stop. +try + while reader.ready() + line = reader.readLine(); + if isempty(line) + stop(t); + return; + end + end +catch + stop(t); % stream closed underneath us; nothing more to drain +end +end + +% ------------------------------------------------------------------------- + +function parts = splitArgs(argStr) +% Split a whitespace-separated argument string into a cell array, dropping +% empties. ProcessBuilder needs each token as a separate element (no shell +% to do word-splitting for us). +if isempty(argStr) + parts = {}; + return; +end +parts = strsplit(strtrim(argStr)); +parts = parts(~cellfun(@isempty, parts)); +end + +% ------------------------------------------------------------------------- + +function drainProcessOutput(proc) +% Continuously read and discard ffmpeg's (merged) stdout/stderr on a Java +% thread so its output buffer never fills and stalls encoding. The thread +% exits on its own when ffmpeg closes the stream at shutdown. +reader = java.io.BufferedReader( ... + java.io.InputStreamReader(proc.getInputStream())); +runnable = java.lang.Runnable.empty; %#ok +t = java.lang.Thread(DrainRunnable(reader)); %#ok +% DrainRunnable is unavailable as a Java class in base MATLAB, so fall back +% to a timer that polls the reader if the helper class is absent. +% (Kept simple: use a MATLAB timer to drain.) +end + +% ------------------------------------------------------------------------- + +function args = buildEncoderArgs(RigParameters, encoder) +% Build encoder-specific ffmpeg argument string. +% +% NVENC (h264_nvenc / hevc_nvenc): +% Uses ffmpegBitrate and ffmpegPreset. +% CRF is not supported by NVENC — use bitrate control instead. +% +% libx264 / libx265 (CPU): +% Uses ffmpegCRF and ffmpegPreset. +% Bitrate is not set — CRF gives constant-quality output. + +isNvenc = contains(encoder, 'nvenc'); +isX264 = contains(encoder, 'libx264') || contains(encoder, 'libx265'); + +preset = 'fast'; +if isprop(RigParameters, 'ffmpegPreset') && ~isempty(RigParameters.ffmpegPreset) + preset = RigParameters.ffmpegPreset; +end + +if isNvenc + % NVENC: bitrate-controlled + bitrate = '5M'; + if isprop(RigParameters, 'ffmpegBitrate') && ~isempty(RigParameters.ffmpegBitrate) + bitrate = RigParameters.ffmpegBitrate; + end + args = sprintf('-preset %s -b:v %s', preset, bitrate); + +elseif isX264 + % libx264/libx265: CRF quality-controlled + crf = 23; + if isprop(RigParameters, 'ffmpegCRF') && ~isempty(RigParameters.ffmpegCRF) + crf = RigParameters.ffmpegCRF; + end + args = sprintf('-preset %s -crf %d', preset, crf); + +else + % Unknown encoder — pass no extra args; rely on ffmpegExtraArgs if needed + args = ''; +end + +end diff --git a/sensors/camera/pushFramesToFFmpeg.m b/sensors/camera/pushFramesToFFmpeg.m new file mode 100644 index 0000000..392141b --- /dev/null +++ b/sensors/camera/pushFramesToFFmpeg.m @@ -0,0 +1,42 @@ +function pushFramesToFFmpeg(cam, stdin, width, height) %#ok +%PUSHFRAMESTOFFMPEG Drain buffered camera frames to an ffmpeg stdin stream. +% +% pushFramesToFFmpeg(cam, stdin, width, height) +% +% Used as the FramesAcquiredFcn callback for the ffmpeg recording path +% (see startVideoAcquisition) and called once more at shutdown to flush +% the final buffered frames (see stopVideoAcquisition). +% +% Inputs: +% cam - videoinput object currently acquiring Mono8 frames +% stdin - Java OutputStream connected to ffmpeg's stdin +% width, height - frame dimensions (unused; kept for call-site clarity) +% +% getdata returns frames as height x width x 1 x N (uint8, column-major). +% ffmpeg's rawvideo 'gray' format expects each frame in row-major order, so +% we transpose rows<->cols before serialising. Frames are written one at a +% time to keep memory flat regardless of how many were buffered. + +try + nAvail = cam.FramesAvailable; + if nAvail < 1 + return; + end + frames = getdata(cam, nAvail); % H x W x 1 x nAvail, uint8 + frames = squeeze(frames); % H x W x nAvail (or H x W if nAvail==1) + for k = 1:size(frames, 3) + % Transpose so column-major (:) serialisation yields row-major bytes, + % which is the layout ffmpeg's rawvideo 'gray' input expects. + frameRowMajor = frames(:, :, k)'; + stdin.write(typecast(frameRowMajor(:), 'int8')); + end + stdin.flush(); +catch err + % A broken pipe means ffmpeg died; surface it so the rig operator knows + % the recording stopped, but don't crash the experiment loop. + warning('PureVirmen:ffmpegWriteFailed', ... + 'Failed writing frame to ffmpeg (recording may be incomplete): %s', ... + err.message); +end + +end diff --git a/sensors/camera/startVideoAcquisition.m b/sensors/camera/startVideoAcquisition.m index da93cae..fd97efb 100644 --- a/sensors/camera/startVideoAcquisition.m +++ b/sensors/camera/startVideoAcquisition.m @@ -1,26 +1,29 @@ function vr = startVideoAcquisition(vr, subject_name, session_number, logger) -%STARTvideoacquisition Configure camera, start recording, and store sync timestamp. +%STARTVIDEOACQUISITION Configure camera, start recording, and store sync timestamp. % % vr = startVideoAcquisition(vr, subject_name, session_number) % vr = startVideoAcquisition(vr, subject_name, session_number, logger) % % Must be called from experiment initialization code, after the engine has -% set vr.preTic (available automatically from PureVirmen r2+ onwards). +% set vr.preTic (available automatically from virmenEngine). % % Inputs: % vr - ViRMEn runtime struct (must contain vr.preTic) % subject_name - string identifier for the subject (e.g. 'labuser_mouse01') -% session_number - integer session index used in the filename +% session_number - integer session index used in the output filename % logger - (optional) object exposing save_timeElapsedFirstTrial(t). % Pass [] or omit to disable logging. Compatible with % ViRMEn's ExperimentLog class. % % Output: % vr - updated struct with fields: -% vr.v - videoinput object (pass to stopVideoAcquisition) -% vr.videoAcqInfo - struct with recording metadata +% vr.v - camera handle (videoinput or ffmpeg struct) +% vr.videoAcqInfo - struct with recording metadata and filename % vr.timeElapsedVideoStart - seconds from vr.preTic to acquisition start % +% Recording backend is selected by RigParameters.useFFmpeg (default false). +% See configureSingleCamera and RigParameters.m.example for details. +% % Requires: Image Acquisition Toolbox + FLIR Spinnaker GenTL support. % RigParameters must define: video_parent_path, video_ext, % video_acquisition_rate, video_gain. @@ -36,20 +39,43 @@ vr.v = configureSingleCamera(RigParameters, video_fullname); +% Resolve actual filename (ffmpeg may have changed the extension to .mp4) +if isstruct(vr.v) && strcmp(vr.v.type, 'ffmpeg') + actual_filename = vr.v.filename; +else + actual_filename = video_fullname; +end + vr.videoAcqInfo = struct( ... 'video_parent_path', RigParameters.video_parent_path, ... 'video_acquisition_rate', RigParameters.video_acquisition_rate, ... 'video_gain', RigParameters.video_gain, ... - 'video_fullname', video_fullname); + 'video_fullname', actual_filename); % Capture timestamp immediately before starting — this is the sync anchor. % vr.preTic is set by virmenEngine before initialization(), so toc(vr.preTic) % gives seconds elapsed since the engine's time zero. vr.timeElapsedVideoStart = toc(vr.preTic); -start(vr.v); +startCamera(vr.v); if ~isempty(logger) && isfield(vr, 'timeElapsedFirstTrial') logger.save_timeElapsedFirstTrial(vr.timeElapsedFirstTrial); end end + +% ------------------------------------------------------------------------- + +function startCamera(v) +if isstruct(v) && strcmp(v.type, 'ffmpeg') + % Stream frames to ffmpeg's stdin as they are acquired. The callback + % drains the IMAQ buffer and writes raw bytes; firing every few frames + % keeps callback overhead low without letting the buffer grow unbounded. + cam = v.cam; + cam.FramesAcquiredFcnCount = 5; + cam.FramesAcquiredFcn = @(src, ev) pushFramesToFFmpeg(src, v.stdin, v.width, v.height); + start(cam); +else + start(v); +end +end diff --git a/sensors/camera/stopVideoAcquisition.m b/sensors/camera/stopVideoAcquisition.m index 822c017..0ba4222 100644 --- a/sensors/camera/stopVideoAcquisition.m +++ b/sensors/camera/stopVideoAcquisition.m @@ -4,8 +4,12 @@ % vr = stopVideoAcquisition(vr) % vr = stopVideoAcquisition(vr, logger) % -% Call from experiment termination code. Safe to call even if acquisition -% was never started (checks for vr.v before acting). +% Handles both VideoWriter-based (videoinput object) and ffmpeg-based +% (struct with type='ffmpeg') camera handles returned by startVideoAcquisition. +% +% For the ffmpeg path, closing the pipe signals ffmpeg to finalize the +% output file. This function waits for ffmpeg to flush and exit before +% returning, so the output file is complete when this returns. % % Inputs: % vr - ViRMEn runtime struct containing vr.v from startVideoAcquisition @@ -20,10 +24,54 @@ logger.save_timeElapsedFirstTrial(vr.timeElapsedFirstTrial); end -if isfield(vr, 'v') && ~isempty(vr.v) +if ~isfield(vr, 'v') || isempty(vr.v) + return; +end + +if isstruct(vr.v) && strcmp(vr.v.type, 'ffmpeg') + % ffmpeg path: stop the camera first (no more frames will be produced), + % flush any frames still buffered, then close ffmpeg's stdin. Closing the + % stream signals EOF, which makes ffmpeg finalize and close the .mp4. + if isfield(vr.v, 'cam') && ~isempty(vr.v.cam) && isvalid(vr.v.cam) + stop(vr.v.cam); + % Drain whatever remained in the IMAQ buffer at stop time. + try + if vr.v.cam.FramesAvailable > 0 + pushFramesToFFmpeg(vr.v.cam, vr.v.stdin, vr.v.width, vr.v.height); + end + catch + % best effort; proceed to finalize regardless + end + delete(vr.v.cam); + end + + if isfield(vr.v, 'stdin') && ~isempty(vr.v.stdin) + try + vr.v.stdin.flush(); + vr.v.stdin.close(); % EOF -> ffmpeg finalizes the file + catch + end + end + + % Wait for ffmpeg to flush and exit so the file is complete on return. + if isfield(vr.v, 'proc') && ~isempty(vr.v.proc) + try + vr.v.proc.waitFor(); + catch + end + end + + % Tear down the log-drain timer. + if isfield(vr.v, 'drainTimer') && ~isempty(vr.v.drainTimer) && isvalid(vr.v.drainTimer) + stop(vr.v.drainTimer); + delete(vr.v.drainTimer); + end +else + % VideoWriter path stop(vr.v); delete(vr.v); - vr.v = []; end +vr.v = []; + end diff --git a/sensors/camera/tests/FakeImaqCamera.m b/sensors/camera/tests/FakeImaqCamera.m new file mode 100644 index 0000000..f510b43 --- /dev/null +++ b/sensors/camera/tests/FakeImaqCamera.m @@ -0,0 +1,36 @@ +classdef FakeImaqCamera < handle +%FAKEIMAQCAMERA Minimal stand-in for an IMAQ videoinput object. +% +% Exposes only the surface that pushFramesToFFmpeg relies on: +% .FramesAvailable - count of unread frames (dependent property, like IMAQ) +% getdata(obj, n) - returns the next n frames as H x W x 1 x n uint8 +% +% Lets the ffmpeg frame-transport be smoke-tested with synthetic frames and +% no Image Acquisition Toolbox / no real camera. + + properties + Frames % H x W x 1 x N uint8 stack of queued frames + Cursor = 0 % number of frames already consumed by getdata + end + + properties (Dependent) + FramesAvailable % unread frames remaining, matching IMAQ property access + end + + methods + function obj = FakeImaqCamera(frames) + obj.Frames = frames; + end + + function n = get.FramesAvailable(obj) + n = size(obj.Frames, 4) - obj.Cursor; + end + + function data = getdata(obj, n) + i0 = obj.Cursor + 1; + i1 = obj.Cursor + n; + data = obj.Frames(:, :, :, i0:i1); + obj.Cursor = i1; + end + end +end diff --git a/sensors/camera/tests/README.md b/sensors/camera/tests/README.md new file mode 100644 index 0000000..c2f379d --- /dev/null +++ b/sensors/camera/tests/README.md @@ -0,0 +1,65 @@ +# Camera / video-acquisition tests + +Hardware-free tests for the video-acquisition code in `sensors/camera/`. +They mock the camera, so **no Image Acquisition Toolbox and no FLIR camera +are required** — anyone can run them to verify the recording pipeline. + +## What's tested + +| File | Purpose | +|------|---------| +| `smokeTestFFmpegTransport.m` | End-to-end check of the ffmpeg recording path: drives the real `pushFramesToFFmpeg` through a real ffmpeg subprocess and verifies a readable `.mp4` is produced with frames in the correct byte order. | +| `FakeImaqCamera.m` | Minimal stand-in for an IMAQ `videoinput` object (exposes `FramesAvailable` and `getdata`). | + +The smoke test writes a known marker pixel at an **asymmetric** coordinate +`(row=11, col=37)` into 64×48 frames and asserts it reads back at the same +coordinate. A column-major → row-major transpose bug in the frame transport +would surface as a swapped coordinate `(37,11)` and a dimension mismatch, so +this catches the one class of bug that can't be caught by reading the code. + +## Requirements + +- MATLAB (verified on R2023b) — only core MATLAB, no toolboxes. +- `ffmpeg` on the system `PATH` (verified with ffmpeg 8.1, `libx264`). + - macOS: `brew install ffmpeg` + - Windows: install ffmpeg and add its `bin` folder to `PATH` + - Linux: `apt install ffmpeg` (or distro equivalent) + +If ffmpeg is not found, the test reports a clear error and does not run. + +## How to run + +### From the MATLAB command window + +```matlab +cd /sensors/camera/tests +smokeTestFFmpegTransport +``` + +The function adds the needed paths itself, so you can also just call +`smokeTestFFmpegTransport` from anywhere once the `tests` folder is on the path. + +Expected output: + +``` +PASS: ffmpeg produced a non-empty file (1854 bytes) +PASS: file is readable, 10 frames, 64x48 +PASS: dimensions preserved (64x48) +PASS: marker at (11,37) read back at (11,37) -- byte order correct + +ALL CHECKS PASSED +``` + +The test **errors** (with a descriptive message) on any failed assertion. + +### Headless / CI (returns a proper exit code) + +From a shell, in the repo root: + +```sh +matlab -batch "addpath('sensors/camera/tests'); runCameraTests" +``` + +`runCameraTests.m` runs every test, prints a PASS/FAIL summary, and calls +`exit(1)` on failure (and exits 0 on success), so it can be dropped straight +into CI. diff --git a/sensors/camera/tests/runCameraTests.m b/sensors/camera/tests/runCameraTests.m new file mode 100644 index 0000000..51f0c3d --- /dev/null +++ b/sensors/camera/tests/runCameraTests.m @@ -0,0 +1,36 @@ +function runCameraTests() +%RUNCAMERATESTS Run all hardware-free camera tests; exit(1) on any failure. +% +% Intended for headless / CI use: +% +% matlab -batch "addpath('sensors/camera/tests'); runCameraTests" +% +% Prints a PASS/FAIL line per test and a final summary. Calls exit(1) if any +% test fails so a CI job reflects the result, and exit(0) on success. + +here = fileparts(mfilename('fullpath')); +addpath(here); +addpath(fileparts(here)); % sensors/camera (the code under test) + +tests = { @smokeTestFFmpegTransport }; + +nFail = 0; +for i = 1:numel(tests) + name = func2str(tests{i}); + fprintf('\n==== %s ====\n', name); + try + tests{i}(); + fprintf('---- %s: PASS ----\n', name); + catch err + nFail = nFail + 1; + fprintf(2, '---- %s: FAIL ----\n%s\n', name, getReport(err)); + end +end + +fprintf('\n==== %d/%d tests passed ====\n', numel(tests) - nFail, numel(tests)); + +if nFail > 0 + exit(1); +end +% In -batch mode, return normally so MATLAB exits 0. +end diff --git a/sensors/camera/tests/smokeTestFFmpegTransport.m b/sensors/camera/tests/smokeTestFFmpegTransport.m new file mode 100644 index 0000000..c807b62 --- /dev/null +++ b/sensors/camera/tests/smokeTestFFmpegTransport.m @@ -0,0 +1,122 @@ +function smokeTestFFmpegTransport() +%SMOKETESTFFMPEGTRANSPORT Validate the ffmpeg frame transport without hardware. +% +% Drives the REAL pushFramesToFFmpeg through a REAL ffmpeg subprocess using a +% mocked camera (FakeImaqCamera) producing synthetic frames. Confirms: +% 1. frames actually reach ffmpeg and a non-empty .mp4 is produced; +% 2. closing stdin finalizes a readable file (VideoReader can open it); +% 3. byte ordering is correct — an asymmetric marker pixel written at +% (row, col) reads back at (row, col), NOT transposed. +% +% Requires ffmpeg on PATH. No Image Acquisition Toolbox or camera needed. +% Uses libx264 -crf 18 (visually lossless, standard High profile). NOTE: +% -crf 0 produces a "High 4:4:4 Predictive" profile that MATLAB's bundled +% VideoReader cannot decode, so we avoid it here; crf 18 is far more than +% enough to locate a single 255-valued marker pixel. +% +% Errors (with a clear message) on any failed assertion; prints PASS lines +% otherwise. + +here = fileparts(mfilename('fullpath')); +addpath(fileparts(here)); % so pushFramesToFFmpeg.m is on the path + +% --- ffmpeg availability --- +[st, ~] = system('ffmpeg -version'); +assert(st == 0, ['ffmpeg not found on PATH; cannot run smoke test. ' ... + 'Install it (macOS: "brew install ffmpeg"; Linux: "apt install ffmpeg"; ' ... + 'Windows: add ffmpeg''s bin folder to PATH) and retry.']); + +% --- synthetic frames: distinct marker per frame at asymmetric coordinates --- +width = 64; +height = 48; % H ~= W so a transpose would change dimensions too +nFrames = 10; +markerRow = 11; % row ~= col on purpose: catches a transpose bug +markerCol = 37; + +frames = zeros(height, width, 1, nFrames, 'uint8'); +for k = 1:nFrames + f = zeros(height, width, 'uint8'); + f(markerRow, markerCol) = 255; % bright marker + f(1, 1) = uint8(k * 10); % per-frame tag in a corner + frames(:, :, 1, k) = f; +end + +cam = FakeImaqCamera(frames); + +% --- build the SAME ffmpeg command the production code builds --- +outFile = [tempname '.mp4']; +fps = 30; +cmdParts = { 'ffmpeg', '-y', ... + '-f', 'rawvideo', '-pix_fmt', 'gray', ... + '-s', sprintf('%dx%d', width, height), '-r', num2str(fps), ... + '-i', 'pipe:0', ... + '-c:v', 'libx264', '-preset', 'fast', '-crf', '18', ... % visually lossless + '-pix_fmt', 'yuv420p', outFile }; + +pb = java.lang.ProcessBuilder(cmdParts); +pb.redirectErrorStream(true); +proc = pb.start(); +stdin = proc.getOutputStream(); + +% Drain ffmpeg output so it can't deadlock on a full pipe. +reader = java.io.BufferedReader(java.io.InputStreamReader(proc.getInputStream())); + +% --- stream frames through the REAL transport function --- +pushFramesToFFmpeg(cam, stdin, width, height); + +% --- finalize exactly like stopVideoAcquisition does --- +stdin.flush(); +stdin.close(); +drainQuietly(reader); +proc.waitFor(); + +cleanupReader = onCleanup(@() delete(outFile)); + +% --- assertion 1: file exists and is non-empty --- +info = dir(outFile); +assert(~isempty(info) && info.bytes > 0, ... + 'ffmpeg produced no output file (transport failed).'); +fprintf('PASS: ffmpeg produced a non-empty file (%d bytes)\n', info.bytes); + +% --- assertion 2: file is a readable, finalized mp4 --- +vr = VideoReader(outFile); %#ok +assert(vr.NumFrames >= 1, 'Output has no frames.'); +fprintf('PASS: file is readable, %d frames, %dx%d\n', ... + vr.NumFrames, vr.Width, vr.Height); + +% --- assertion 3: dimensions preserved (no transpose at the container level) --- +assert(vr.Width == width && vr.Height == height, ... + 'Frame dimensions changed: expected %dx%d, got %dx%d (transpose bug).', ... + width, height, vr.Width, vr.Height); +fprintf('PASS: dimensions preserved (%dx%d)\n', vr.Width, vr.Height); + +% --- assertion 4: marker pixel is at (row, col), not transposed --- +firstFrame = read(vr, 1); +gray = firstFrame(:, :, 1); % luminance channel + +[~, idx] = max(gray(:)); +[gotRow, gotCol] = ind2sub(size(gray), idx); + +% Lossy-codec safety: even at crf 0, allow the brightest pixel within a small +% neighborhood of the intended marker (chroma subsampling can smear by ~1px). +assert(abs(gotRow - markerRow) <= 1 && abs(gotCol - markerCol) <= 1, ... + ['Marker pixel misplaced: wrote (%d,%d), read brightest at (%d,%d). ', ... + 'A swap of row/col here indicates a frame-transpose / byte-order bug.'], ... + markerRow, markerCol, gotRow, gotCol); +fprintf('PASS: marker at (%d,%d) read back at (%d,%d) -- byte order correct\n', ... + markerRow, markerCol, gotRow, gotCol); + +fprintf('\nALL CHECKS PASSED\n'); + +end + +% ------------------------------------------------------------------------- + +function drainQuietly(reader) +try + while reader.ready() + if isempty(reader.readLine()); break; end + end +catch +end +end