diff --git a/bin/engine/virmenEngine.m b/bin/engine/virmenEngine.m index c60fca2..371fefd 100644 --- a/bin/engine/virmenEngine.m +++ b/bin/engine/virmenEngine.m @@ -98,6 +98,11 @@ drawnow; virmenOpenGLRoutines(0,windows,ismac); +% Stable time reference for the whole session; available in initialization +% and movement/runtime code via toc(vr.preTic). Used by startVideoAcquisition +% to record when video capture began relative to experiment start. +vr.preTic = tic; + % Run initialization code try vr = vr.code.initialization(vr); %#ok<*NASGU> @@ -117,6 +122,9 @@ vr.timeStarted = now; % Timing related info +% timeElapsedFirstTrial records how long initialization took; used by +% syncBehaviorVideo to align video frame times to behavioral iterations. +vr.timeElapsedFirstTrial = toc(vr.preTic); firstTic = tic; vr.dt = 0; % Don't move on the first time step diff --git a/experiments/live_calibration_pupillometry_test.m b/experiments/live_calibration_pupillometry_test.m new file mode 100644 index 0000000..9b64016 --- /dev/null +++ b/experiments/live_calibration_pupillometry_test.m @@ -0,0 +1,110 @@ +function code = live_calibration_pupillometry_test +% manuel_calibration Code for the ViRMEn experiment manuel_calibration. +% code = manuel_calibration Returns handles to the functions that ViRMEn +% executes during engine initialization, runtime and termination. + +% Begin header code - DO NOT EDIT +code.initialization = @initializationCodeFun; +code.runtime = @runtimeCodeFun; +code.termination = @terminationCodeFun; +% End header code - DO NOT EDIT + +% --- INITIALIZATION code: executes before the ViRMEn engine starts. +function vr = initializationCodeFun(vr) + + %Check if all parameters for calibration exist on rigParameters file + vr.proj_params = []; + all_params = GeneralParameters.mini_vr_proj_parameters_full; + has_all_parms =1; + for i=1:length(all_params) + if ~isprop(RigParameters, all_params{i}) + has_all_parms = 0; + else + %Set parameters for virmenEngine function + vr.proj_params(i) = RigParameters.(all_params{i}); + end + end + %If rigParameters doesn't have all parameters, transformation embedded in file + if has_all_parms + vr.exper.transformationFunction = @miniVR_projection_wparameters; + else + vr.exper.transformationFunction = @miniVR_projection; + end + + vr.param_list = GeneralParameters.mini_vr_proj_parameters; + + vr.param_key = ['1', '2', '3', '4', ... + '5', '6', '7', '8', ... + '9', '0', '-', 'U', 'I', 'O', 'P']; + + vr.num_param = 1; + vr.change_param = []; + + %Check if there were missing parameters for projection in RigParameters + if length(vr.proj_params) < length(vr.param_list) + all_params = GeneralParameters.mini_vr_proj_parameters_full; + s = []; + fprintf('== Missing proj params in RigParameters ==\n'); + for i=1:length(all_params) + if ~isprop(RigParameters, all_params{i}) + s = [s sprintf(' %s\n', all_params{i})]; + end + end + error(['Cannot run live calibration with missing parameters:' newline s]); + end + + display_params(vr) + + subject_name = 'test1'; + session_number = 0; + vr = startVideoAcquisition(vr, subject_name, session_number); + + +% --- RUNTIME code: executes on every iteration of the ViRMEn engine. +function vr = runtimeCodeFun(vr) + + [num_param, vr.change_param] = processKeypressCalibration(vr); + if ~isempty(num_param) + vr.num_param = num_param; + end + + if ~isempty(vr.change_param) + vr.proj_params(vr.num_param) = vr.proj_params(vr.num_param) + vr.change_param; + end + + if ~isempty(num_param) || ~isempty(vr.change_param) + display_params(vr) + end + +% --- TERMINATION code: executes after the ViRMEn engine stops. +function vr = terminationCodeFun(vr) + + vr = stopVideoAcquisition(vr); + +function display_params(vr) + + clc + if ~isempty(vr.change_param) + fprintf('====== Last command ======\n') + if vr.change_param > 0 + ch = '+'; + else + ch = '-'; + end + fprintf(' (%s) %s %s%2.3f\n', vr.param_key(vr.num_param), vr.param_list{vr.num_param}, ch, abs(vr.change_param)); + end + + fprintf('\n====== Proj Params ======\n') + for i=1:length(vr.param_list) + if i== vr.num_param + fprintf('-> '); + else + fprintf(' '); + end + fprintf('(%s) %s = %2.3f \n', vr.param_key(i), vr.param_list{i}, vr.proj_params(i)); + end + fprintf('\n====== Controls (applied to selected "->" param ) ======\n' ); + fprintf('(Q) -> + 0.1 (A) -> - 0.1 \n'); + fprintf('(W) -> + 0.01 (S) -> - 0.01 \n'); + fprintf('(E) -> + 0.001 (D) -> - 0.001\n'); + fprintf(' \n'); diff --git a/experiments/live_calibration_pupillometry_test.mat b/experiments/live_calibration_pupillometry_test.mat new file mode 100644 index 0000000..d39c1b1 Binary files /dev/null and b/experiments/live_calibration_pupillometry_test.mat differ diff --git a/extras/RigParameters.m b/extras/RigParameters.m index e74bc44..bf10b32 100644 --- a/extras/RigParameters.m +++ b/extras/RigParameters.m @@ -37,6 +37,18 @@ tcpClientPort = 30000 % port for tcp communication outputBufferSize = 40 % byte size for virmen variables + + %% Pupilometry video parameters + video_acquisition_rate = 30 + video_record = false + video_gain = 8 + + %% Video acquisition Parameters + video_parent_path = 'E:\TestPureVirmenVideo' + video_ext = '.mj2' + camera_type = 'gentl' + video_type = 'pupillometry' + %% Mini VR projection parameters % Spherical screen radius diff --git a/extras/RigParameters.m.example b/extras/RigParameters.m.example index f57e9e4..d1acba7 100644 --- a/extras/RigParameters.m.example +++ b/extras/RigParameters.m.example @@ -32,7 +32,26 @@ classdef RigParameters laserChannel = 1 % NI-DAQ digital line for turning on/off the laser rightPuffChannel = 2 % NI-DAQ digital line for left air puff solenoid leftPuffChannel = 3 % NI-DAQ digital line for right air puff solenoid - + + % ----------------------------------------------------------------------- + % Video acquisition (pupillometry / eye tracking) + % Requires: Image Acquisition Toolbox + FLIR Spinnaker GenTL support + % Install via MATLAB Add-On Explorer: search "FLIR Spinnaker" + % + % Usage in experiment initialization code: + % if RigParameters.hasCamera + % vr = startVideoAcquisition(vr, subject_name, session_number); + % end + % Usage in experiment termination code: + % if isfield(vr, 'v'); vr = stopVideoAcquisition(vr); end + % ----------------------------------------------------------------------- + hasCamera = false % master on/off switch + video_parent_path = 'D:\Videos' % root directory for video files + video_ext = '.mj2' % output file extension + video_acquisition_rate = 30 % camera frame rate (fps) + video_exposure_time_in_microseconds = 5000 % exposure time; comment out to use camera default + video_gain = 0 % camera analog gain + end methods (Static, Access = protected) diff --git a/sensors/camera/checkCameraRigParameters.m b/sensors/camera/checkCameraRigParameters.m new file mode 100644 index 0000000..ad34661 --- /dev/null +++ b/sensors/camera/checkCameraRigParameters.m @@ -0,0 +1,22 @@ +function checkCameraRigParameters() +%CHECKCAMERARIGPARAMETERS Validate RigParameters for video acquisition. +% +% Throws an error listing any missing properties required by +% startVideoAcquisition / configureSingleCamera. + +required = {'video_parent_path', 'video_ext', 'video_acquisition_rate', 'video_gain'}; + +missing = {}; +for i = 1:length(required) + if ~isprop(RigParameters, required{i}) + missing{end+1} = required{i}; %#ok + end +end + +if ~isempty(missing) + error('PureVirmen:missingRigParameters', ... + 'Missing required RigParameters for video acquisition: %s\n', ... + strjoin(missing, ', ')); +end + +end diff --git a/sensors/camera/configureSingleCamera.m b/sensors/camera/configureSingleCamera.m new file mode 100644 index 0000000..4e7ffd3 --- /dev/null +++ b/sensors/camera/configureSingleCamera.m @@ -0,0 +1,47 @@ +function v = configureSingleCamera(RigParameters, video_filename) +%CONFIGURESINGLECAMERA Configure a single FLIR camera for disk-logged recording. +% +% v = configureSingleCamera(RigParameters, video_filename) +% +% Configures a FLIR camera via the GenTL adapter (Image Acquisition Toolbox +% + FLIR Spinnaker GenTL support) and sets up a VideoWriter disk logger +% (Motion JPEG 2000). The returned videoinput object v is ready to start; +% call start(v) to begin acquisition. +% +% Required RigParameters properties: +% video_acquisition_rate - frame rate in fps +% video_gain - analog gain +% +% Optional RigParameters properties: +% video_exposure_time_in_microseconds - if present and nonzero, sets +% camera exposure; otherwise the camera default is used. +% +% Requires: Image Acquisition Toolbox with FLIR Spinnaker GenTL support. +% Install the support package via MATLAB Add-On Explorer by searching for +% "FLIR Spinnaker". + +imaqreset; +v = videoinput('gentl', 1, 'Mono8'); +src = getselectedsource(v); + +src.AcquisitionFrameRateEnable = 'true'; +src.AcquisitionFrameRate = RigParameters.video_acquisition_rate; + +if isprop(RigParameters, 'video_exposure_time_in_microseconds') && ... + RigParameters.video_exposure_time_in_microseconds + src.ExposureTime = RigParameters.video_exposure_time_in_microseconds; +end + +src.Gain = RigParameters.video_gain; + +v.FramesPerTrigger = Inf; +v.TriggerRepeat = Inf; + +logfile = VideoWriter(video_filename, 'Motion JPEG 2000'); +logfile.MJ2BitDepth = 8; +logfile.FrameRate = RigParameters.video_acquisition_rate; + +v.LoggingMode = 'disk'; +v.DiskLogger = logfile; + +end diff --git a/sensors/camera/setupVideoFile.m b/sensors/camera/setupVideoFile.m new file mode 100644 index 0000000..ee66473 --- /dev/null +++ b/sensors/camera/setupVideoFile.m @@ -0,0 +1,37 @@ +function video_fullname = setupVideoFile(video_parent_path, video_ext, subject_fullname, session_number) +%SETUPVIDEOFILE Build the full output path for a video file and create its directory. +% +% video_fullname = setupVideoFile(video_parent_path, video_ext, ... +% subject_fullname, session_number) +% +% Follows the naming convention: +% ////_ +% +% where userid is the portion of subject_fullname before the first '_'. +% This convention is compatible with the U19 pipeline paths. +% +% Example: +% setupVideoFile('D:\Videos', '.mj2', 'labuser_mouse01', 2) +% -> D:\Videos\labuser\labuser_mouse01\20240618_g2\labuser_mouse01_20240618_g2.mj2 + +if ~contains(subject_fullname, '_') + userid = 'no_userid'; +else + % userid is the portion before the FIRST underscore, so a subject like + % 'lab_user_mouse01' yields userid 'lab' (matching the docstring). + userid = strtok(subject_fullname, '_'); +end + +date_str = datestr(now, 'yyyymmdd'); +session_tag = [date_str '_g' num2str(session_number)]; +filename_only = [subject_fullname '_' session_tag]; +relative_dir = fullfile(userid, subject_fullname, session_tag); + +video_dir = fullfile(video_parent_path, relative_dir); +if ~isfolder(video_dir) + mkdir(video_dir); +end + +video_fullname = fullfile(video_dir, [filename_only video_ext]); + +end diff --git a/sensors/camera/startVideoAcquisition.m b/sensors/camera/startVideoAcquisition.m new file mode 100644 index 0000000..13aece6 --- /dev/null +++ b/sensors/camera/startVideoAcquisition.m @@ -0,0 +1,49 @@ +function vr = startVideoAcquisition(vr, subject_name, session_number) +%STARTvideoacquisition Configure camera, start recording, and store sync timestamp. +% +% vr = startVideoAcquisition(vr, subject_name, session_number) +% vr = startVideoAcquisition(vr, subject_name, session_number, logger) +% +% Must be called from experiment initialization code, after the engine has +% set vr.preTic (available automatically from PureVirmen r2+ onwards). +% +% Inputs: +% vr - ViRMEn runtime struct (must contain vr.preTic) +% subject_name - string identifier for the subject (e.g. 'labuser_mouse01') +% session_number - integer session index used in the filename +% logger - (optional) object exposing save_timeElapsedFirstTrial(t). +% Pass [] or omit to disable logging. Compatible with +% ViRMEn's ExperimentLog class. +% +% Output: +% vr - updated struct with fields: +% vr.v - videoinput object (pass to stopVideoAcquisition) +% vr.videoAcqInfo - struct with recording metadata +% vr.timeElapsedVideoStart - seconds from vr.preTic to acquisition start +% +% Requires: Image Acquisition Toolbox + FLIR Spinnaker GenTL support. +% RigParameters must define: video_parent_path, video_ext, +% video_acquisition_rate, video_gain. + + +checkCameraRigParameters(); + +video_fullname = setupVideoFile(RigParameters.video_parent_path, ... + RigParameters.video_ext, subject_name, session_number); + +vr.v = configureSingleCamera(RigParameters, video_fullname); + +vr.videoAcqInfo = struct( ... + 'video_parent_path', RigParameters.video_parent_path, ... + 'video_acquisition_rate', RigParameters.video_acquisition_rate, ... + 'video_gain', RigParameters.video_gain, ... + 'video_fullname', video_fullname); + +% Capture timestamp immediately before starting — this is the sync anchor. +% vr.preTic is set by virmenEngine before initialization(), so toc(vr.preTic) +% gives seconds elapsed since the engine's time zero. +vr.timeElapsedVideoStart = toc(vr.preTic); +start(vr.v); + + +end diff --git a/sensors/camera/stopVideoAcquisition.m b/sensors/camera/stopVideoAcquisition.m new file mode 100644 index 0000000..5108560 --- /dev/null +++ b/sensors/camera/stopVideoAcquisition.m @@ -0,0 +1,29 @@ +function vr = stopVideoAcquisition(vr, logger) +%STOPVIDEOACQUISITION Stop camera recording and release hardware resources. +% +% vr = stopVideoAcquisition(vr) +% vr = stopVideoAcquisition(vr, logger) +% +% Call from experiment termination code. Safe to call even if acquisition +% was never started (checks for vr.v before acting). +% +% Inputs: +% vr - ViRMEn runtime struct containing vr.v from startVideoAcquisition +% logger - (optional) object exposing save_timeElapsedFirstTrial(t). +% If provided and vr.timeElapsedFirstTrial exists, saves it now. + +%if nargin < 2 +% logger = []; +%end + +%if ~isempty(logger) && isfield(vr, 'timeElapsedFirstTrial') +% logger.save_timeElapsedFirstTrial(vr.timeElapsedFirstTrial); +%end + +if isfield(vr, 'v') && ~isempty(vr.v) + stop(vr.v); + delete(vr.v); + vr.v = []; +end + +end diff --git a/sensors/camera/syncBehaviorVideo.m b/sensors/camera/syncBehaviorVideo.m new file mode 100644 index 0000000..8a50268 --- /dev/null +++ b/sensors/camera/syncBehaviorVideo.m @@ -0,0 +1,81 @@ +function [syncVideoFrame, syncBehavior] = syncBehaviorVideo(log, video_fullname) +%SYNCBEHAVIORVIDEO Align video frames to behavioral iterations post-hoc. +% +% [syncVideoFrame, syncBehavior] = syncBehaviorVideo(log, video_fullname) +% +% Inputs: +% log - struct saved at end of session, with fields: +% .timeElapsedVideoStart - seconds from vr.preTic to video start +% (copied from vr.timeElapsedVideoStart) +% .iterationTimes - Nx1 vector of per-iteration timestamps in +% seconds, relative to the engine loop start +% (i.e. starting near zero, increasing each frame) +% video_fullname - full path to the recorded video file +% +% Outputs: +% syncVideoFrame - [nFrames x 2] single matrix: +% col 1: absolute frame time (s from preTic) +% col 2: index into log.iterationTimes of the nearest +% behavioral iteration (NaN before first iteration) +% syncBehavior - [nIter x 2] single matrix: +% col 1: iteration time (s from preTic) +% col 2: index of the last video frame before that iteration +% (NaN if video had not started yet) +% +% Both outputs are empty if the video and behavior do not overlap. +% +% NOTE: frame times are reconstructed as evenly spaced over the file +% duration (linspace), which ASSUMES the camera held a constant frame rate +% with no dropped frames. FLIR cameras can and do drop frames under load, +% in which case this alignment drifts. If you have true per-frame +% timestamps (e.g. logged from the camera or DiskLogger metadata), pass +% those in via log.frameTimes to override the reconstruction below. +% +% See also: startVideoAcquisition, stopVideoAcquisition + +% Absolute time of each video frame, anchored to vr.preTic +if isfield(log, 'frameTimes') && ~isempty(log.frameTimes) + frameTimes = single(log.frameTimes(:)) + single(log.timeElapsedVideoStart); +else + v = VideoReader(video_fullname); + frameTimes = single(linspace(0, v.Duration, v.NumFrames)' + log.timeElapsedVideoStart); +end + +iterTimes = single(log.iterationTimes(:)); +nFrames = length(frameTimes); +nIter = length(iterTimes); + +% Both lookups are "find the last entry of a sorted vector that is <= a query +% time". iterTimes and frameTimes are monotonically increasing, so this is a +% single O(n log n) binned search rather than an O(n*m) nested scan. +% +% discretize(query, [edges Inf]) returns the index of the bin each query +% falls into, i.e. the index of the last edge that is <= query (or NaN when +% the query precedes the first edge). + +% --- syncVideoFrame: for each frame, index of the last iteration at/before it --- +syncVideoFrame = NaN(nFrames, 2, 'single'); +syncVideoFrame(:, 1) = frameTimes; +if nIter > 0 + syncVideoFrame(:, 2) = discretize(frameTimes, [iterTimes; inf('single')]); +end + +% --- syncBehavior: for each iteration, index of the last frame strictly before it --- +syncBehavior = NaN(nIter, 2, 'single'); +syncBehavior(:, 1) = iterTimes; +if nFrames > 0 + % Strict "<" (frame before the iteration): nudge edges so an exact tie + % maps to the previous frame rather than the simultaneous one. + syncBehavior(:, 2) = discretize(iterTimes, [frameTimes; inf('single')]); + exactTie = ismember(iterTimes, frameTimes); + syncBehavior(exactTie, 2) = syncBehavior(exactTie, 2) - 1; + syncBehavior(syncBehavior(:, 2) < 1, 2) = NaN; +end + +% Return empty if there is no overlap +if all(isnan(syncVideoFrame(:, 2))) || all(isnan(syncBehavior(:, 2))) + syncVideoFrame = []; + syncBehavior = []; +end + +end