From 884131d59742b20d47a0ca7bfcdf81a6d44d7777 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 11 Jun 2026 18:24:25 +0000 Subject: [PATCH 1/6] Extract shared rig SfM pipeline from run_pano_sfm Move feature extraction, rig configuration, sequential matching, and mapping into run_rig_sfm_pipeline so pipelines that render virtual perspective views from other sources (e.g. dual fisheye streams) can reuse the same SfM machinery. No behavior change for the equirectangular path. https://claude.ai/code/session_01MdiAmjGY3SEAQLHsxKBVac --- vid2scene_core/pano_sfm.py | 223 +++++++++++++++++++++---------------- 1 file changed, 130 insertions(+), 93 deletions(-) diff --git a/vid2scene_core/pano_sfm.py b/vid2scene_core/pano_sfm.py index e1248de..113ba57 100644 --- a/vid2scene_core/pano_sfm.py +++ b/vid2scene_core/pano_sfm.py @@ -527,120 +527,54 @@ def render_perspective_images( return processor.rig_config -def run_pano_sfm( - input_image_path: Path, +def run_rig_sfm_pipeline( output_path: Path, + image_dir: Path, + rig_config: pycolmap.RigConfig, + mask_dir: Path | None = None, mapper: str = "glomap", - generate_masks: bool = True, - detect_static: bool = True, kill_check=None, ) -> Path | None: """ - Run the full panorama SfM pipeline. - - This function: - 1. (Optional) Detects static objects like tripods using temporal variance - 2. Renders perspective images from equirectangular panoramas - 3. Extracts features using pycolmap - 4. Applies rig configuration for multi-camera constraint - 5. Matches features with COLMAP sequential matching - 6. Runs reconstruction using GLOMAP (global) or COLMAP (incremental) - + Run rig-constrained SfM on pre-rendered virtual perspective images. + + This covers feature extraction, rig configuration, sequential matching, + and mapping. Shared by the equirectangular pipeline (run_pano_sfm) and + the direct dual-fisheye pipeline (fisheye_sfm.run_insv_sfm). + Args: - input_image_path: Directory containing equirectangular panorama images - output_path: Output directory for rendered images and SfM results + output_path: Output directory for the database and sparse reconstruction + image_dir: Directory containing the rendered perspective images + rig_config: Rig configuration describing the virtual camera geometry + mask_dir: Optional directory with feature-extraction masks mapper: Reconstruction method ("glomap" for global, "colmap" for incremental) - generate_masks: Whether to generate per-pixel masks for feature extraction - detect_static: Whether to detect and mask static objects (tripod, car hood) kill_check: Optional callback to check if processing should abort - + Returns: Path to sparse reconstruction directory, or None if aborted/failed """ from run_command import run_command - - pycolmap.set_random_seed(0) - render_options = PANO_RENDER_OPTIONS - - # Setup directories - image_dir = output_path / "images" - mask_dir = output_path / "masks" if generate_masks else None + database_path = output_path / "database.db" sparse_path = output_path / "sparse" - - image_dir.mkdir(exist_ok=True, parents=True) sparse_path.mkdir(exist_ok=True, parents=True) - if mask_dir: - mask_dir.mkdir(exist_ok=True, parents=True) - + # Clean up old database if exists if database_path.exists(): database_path.unlink() - # Find input panoramas - pano_image_names = sorted( - p.relative_to(input_image_path).as_posix() - for p in input_image_path.rglob("*") - if not p.is_dir() and p.suffix.lower() in ('.jpg', '.jpeg', '.png', '.tiff', '.tif') - ) - logger.info(f"Found {len(pano_image_names)} panorama images in {input_image_path}") - - if not pano_image_names: - logger.error("No panorama images found!") - return None - - if kill_check and kill_check(): - logger.info("Job cancelled before rendering") - return None - - # ========== Step 0 (Optional): Detect ego-object using SAM3 ========== - ego_masks_dict = {} # {pano_name: mask} - training_mask_dir = None - if detect_static and generate_masks: - pano_paths = [input_image_path / name for name in pano_image_names] - - # Use SAM3-based detection (text prompts like "car", "tripod", "person") - ego_masks_dict = detect_ego_object_sam3(pano_paths) - - if ego_masks_dict: - # Create training mask directory for gsplat - training_mask_dir = output_path / "training_masks" - training_mask_dir.mkdir(exist_ok=True, parents=True) - - # Save first mask for debugging - first_mask = next(iter(ego_masks_dict.values())) - static_mask_path = output_path / "static_mask.png" - cv2.imwrite(str(static_mask_path), first_mask) - logger.info(f"Saved sample ego mask to {static_mask_path}") - - # ========== Step 1: Render perspective images from panoramas ========== - rig_config = render_perspective_images( - pano_image_names, - input_image_path, - image_dir, - render_options, - mask_dir, - generate_masks, - ego_masks_dict, - training_mask_dir, - ) - - if kill_check and kill_check(): - logger.info("Job cancelled after rendering") - return None - # ========== Step 2: Feature extraction ========== logger.info("Extracting features...") reader_options = pycolmap.ImageReaderOptions() if mask_dir and mask_dir.exists(): reader_options.mask_path = str(mask_dir) - + # Feature extraction options for denser feature extraction extraction_options = pycolmap.FeatureExtractionOptions() extraction_options.sift.max_num_features = 32768 # Default is 8192, very high for dense points extraction_options.sift.first_octave = -1 # Default is 0, -1 extracts more features at finest scale extraction_options.sift.peak_threshold = 0.003 # Default is 0.0067, lower = more features - + pycolmap.extract_features( str(database_path), str(image_dir), @@ -669,7 +603,7 @@ def run_pano_sfm( # with some versions of pycolmap/glomap. Disable for now. # matching_options.rig_verification = True # matching_options.skip_image_pairs_in_same_frame = True - + seq_options = pycolmap.SequentialPairingOptions() seq_options.overlap = 30 # Default is ~10, increase for more matches seq_options.loop_detection = False @@ -687,7 +621,7 @@ def run_pano_sfm( if mapper == "glomap": # GLOMAP global mapping with rig support logger.info("Running GLOMAP global mapping with rig support...") - + success = run_command([ "glomap", "mapper", "--database_path", str(database_path), @@ -699,11 +633,11 @@ def run_pano_sfm( "--ba_iteration_num", "5", "--skip_pruning", "0", ], kill_check=kill_check) - + if not success: logger.error("GLOMAP mapping failed!") return None - + # Re-register images that may have been pruned zero_dir = sparse_path / "0" if zero_dir.exists(): @@ -723,21 +657,124 @@ def run_pano_sfm( ba_refine_principal_point=False, ba_refine_extra_params=False, ) - + recs = pycolmap.incremental_mapping( str(database_path), str(image_dir), str(sparse_path), opts ) - + for idx, rec in recs.items(): logger.info(f"Reconstruction #{idx}: {rec.summary()}") - + if not recs: logger.error("No reconstructions created!") return None - + return sparse_path +def run_pano_sfm( + input_image_path: Path, + output_path: Path, + mapper: str = "glomap", + generate_masks: bool = True, + detect_static: bool = True, + kill_check=None, +) -> Path | None: + """ + Run the full panorama SfM pipeline. + + This function: + 1. (Optional) Detects static objects like tripods using temporal variance + 2. Renders perspective images from equirectangular panoramas + 3. Extracts features using pycolmap + 4. Applies rig configuration for multi-camera constraint + 5. Matches features with COLMAP sequential matching + 6. Runs reconstruction using GLOMAP (global) or COLMAP (incremental) + + Args: + input_image_path: Directory containing equirectangular panorama images + output_path: Output directory for rendered images and SfM results + mapper: Reconstruction method ("glomap" for global, "colmap" for incremental) + generate_masks: Whether to generate per-pixel masks for feature extraction + detect_static: Whether to detect and mask static objects (tripod, car hood) + kill_check: Optional callback to check if processing should abort + + Returns: + Path to sparse reconstruction directory, or None if aborted/failed + """ + pycolmap.set_random_seed(0) + render_options = PANO_RENDER_OPTIONS + + # Setup directories + image_dir = output_path / "images" + mask_dir = output_path / "masks" if generate_masks else None + image_dir.mkdir(exist_ok=True, parents=True) + if mask_dir: + mask_dir.mkdir(exist_ok=True, parents=True) + + # Find input panoramas + pano_image_names = sorted( + p.relative_to(input_image_path).as_posix() + for p in input_image_path.rglob("*") + if not p.is_dir() and p.suffix.lower() in ('.jpg', '.jpeg', '.png', '.tiff', '.tif') + ) + logger.info(f"Found {len(pano_image_names)} panorama images in {input_image_path}") + + if not pano_image_names: + logger.error("No panorama images found!") + return None + + if kill_check and kill_check(): + logger.info("Job cancelled before rendering") + return None + + # ========== Step 0 (Optional): Detect ego-object using SAM3 ========== + ego_masks_dict = {} # {pano_name: mask} + training_mask_dir = None + if detect_static and generate_masks: + pano_paths = [input_image_path / name for name in pano_image_names] + + # Use SAM3-based detection (text prompts like "car", "tripod", "person") + ego_masks_dict = detect_ego_object_sam3(pano_paths) + + if ego_masks_dict: + # Create training mask directory for gsplat + training_mask_dir = output_path / "training_masks" + training_mask_dir.mkdir(exist_ok=True, parents=True) + + # Save first mask for debugging + first_mask = next(iter(ego_masks_dict.values())) + static_mask_path = output_path / "static_mask.png" + cv2.imwrite(str(static_mask_path), first_mask) + logger.info(f"Saved sample ego mask to {static_mask_path}") + + # ========== Step 1: Render perspective images from panoramas ========== + rig_config = render_perspective_images( + pano_image_names, + input_image_path, + image_dir, + render_options, + mask_dir, + generate_masks, + ego_masks_dict, + training_mask_dir, + ) + + if kill_check and kill_check(): + logger.info("Job cancelled after rendering") + return None + + # ========== Steps 2-5: Features, rig config, matching, mapping ========== + return run_rig_sfm_pipeline( + output_path=output_path, + image_dir=image_dir, + rig_config=rig_config, + mask_dir=mask_dir, + mapper=mapper, + kill_check=kill_check, + ) + + if __name__ == "__main__": logging.basicConfig(level=logging.INFO) From 374c227152af4d23bc078ae33c3c28091a3ab6d2 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 11 Jun 2026 18:24:25 +0000 Subject: [PATCH 2/6] Add direct .insv dual-fisheye to pinhole rig ingestion Process Insta360 .insv recordings straight from their two raw fisheye sensor streams instead of requiring a pre-stitched equirectangular video. The equirectangular intermediate stretches the poles of the sphere, and the ER virtual rig's lowest pitch (-35 deg, 90 deg FOV) never sees below -80 deg, so the ground under the camera gets zero direct, full-resolution observations. Cropping pinhole views directly from each fisheye keeps native sensor pixels and the default view grid (3x3 at +/-60 deg per lens, 75 deg crops, 18 views per frame pair) covers the nadir with up to 6 views per frame pair. - fisheye_projection.py: equidistant fisheye model with optional Kannala-Brandt distortion terms, lens-local view grids, and cv2.remap grid construction (pure numpy/scipy, unit-tested) - insv_extract.py: ffmpeg demuxing of the three known .insv layouts (dual-stream, two-file _00_/_10_ pairs, side-by-side single stream) plus best-effort trailer metadata parsing for logging - fisheye_sfm.py: renders the virtual views with validity and closest-view partition masks, builds the two-lens rig config, and runs the shared rig SfM pipeline - vid2scene.py: --insv_fisheye / --insv_lens_fov / --insv_calibration flags, auto-enabled for .insv inputs - tests: 29 unit tests for the projection math and container parsing Lens intrinsics default to an idealized model; per-unit calibration can be supplied as JSON (docs/insv_fisheye.md). Factory calibration and IMU parsing from the trailer are follow-ups. https://claude.ai/code/session_01MdiAmjGY3SEAQLHsxKBVac --- README.md | 2 +- docs/insv_fisheye.md | 118 +++++ vid2scene_core/fisheye_projection.py | 218 ++++++++ vid2scene_core/fisheye_sfm.py | 471 ++++++++++++++++++ vid2scene_core/insv_extract.py | 316 ++++++++++++ vid2scene_core/tests/conftest.py | 5 + .../tests/test_fisheye_projection.py | 164 ++++++ vid2scene_core/tests/test_insv_extract.py | 118 +++++ vid2scene_core/vid2scene.py | 87 +++- 9 files changed, 1497 insertions(+), 2 deletions(-) create mode 100644 docs/insv_fisheye.md create mode 100644 vid2scene_core/fisheye_projection.py create mode 100644 vid2scene_core/fisheye_sfm.py create mode 100644 vid2scene_core/insv_extract.py create mode 100644 vid2scene_core/tests/conftest.py create mode 100644 vid2scene_core/tests/test_fisheye_projection.py create mode 100644 vid2scene_core/tests/test_insv_extract.py diff --git a/README.md b/README.md index 87d246c..403cbf6 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ HTTP │ │ direct SAS (browser ⇄ blob storage) **How a job flows:** the browser uploads the video straight to blob storage via a SAS URL, then asks `web` to create a job. `web` writes a row to Postgres and pushes a task onto the django-rq queue in Redis. The `worker` pops it, runs the pipeline on the GPU, writes the resulting scene files back to blob storage, and updates the job in Postgres. The browser polls status and finally loads the finished scene straight from blob storage. -The pipeline itself lives in [`vid2scene_core/`](vid2scene_core/) and is orchestrated by the worker. SfM defaults to GLOMAP (no model weights needed). VGGT and SAM3-based panorama background removal are opt-in and require a Hugging Face token (see below). +The pipeline itself lives in [`vid2scene_core/`](vid2scene_core/) and is orchestrated by the worker. SfM defaults to GLOMAP (no model weights needed). VGGT and SAM3-based panorama background removal are opt-in and require a Hugging Face token (see below). 360° content is supported two ways: stitched equirectangular video (`--equirectangular`), or Insta360 `.insv` recordings processed directly from their dual fisheye streams for better ground/sky detail — see [docs/insv_fisheye.md](docs/insv_fisheye.md). On the hosted vid2scene deployment each box in the diagram was its own dedicated piece of infrastructure — the web app on one server, one or more GPU workers each on their own server (the django-rq queue lets you scale out by simply adding more workers that pull from it), plus managed Redis, managed PostgreSQL, and real Azure Blob Storage. This self-host stack collapses all of them into a single `docker compose` project on one machine, swapping the managed cloud services for drop-in equivalents (plain Redis/Postgres containers, and **Azurite** standing in for Azure Blob Storage) so it runs with no cloud account. diff --git a/docs/insv_fisheye.md b/docs/insv_fisheye.md new file mode 100644 index 0000000..9adce22 --- /dev/null +++ b/docs/insv_fisheye.md @@ -0,0 +1,118 @@ +# Direct .insv dual-fisheye processing + +vid2scene can process Insta360 `.insv` recordings directly from their two raw +fisheye sensor streams, without first stitching them to an equirectangular +video in Insta360 Studio. + +## Why + +The equirectangular (ER) path loses exactly the part of the scene users +notice most: the ground. + +1. **Stitching resamples the sphere.** An ER image stretches the poles + (straight down and straight up) across the entire bottom/top row of + pixels. The ground directly under the camera — already the noisiest + region optically — is smeared before SfM ever sees it. +2. **The ER virtual rig never looks down.** `pano_sfm.py` renders 12 virtual + views at pitches `(-35°, 0°, +35°)` with 90° FOV, so nothing below + −80° pitch is covered at all. The nadir gets zero direct observations. + +The fisheye-direct path (`fisheye_sfm.py`) crops virtual pinhole views +straight out of each fisheye stream: + +- Native sensor pixels everywhere. An equidistant fisheye has roughly + uniform angular resolution; the nadir sits ~90° off each lens axis, well + inside a ~200° lens, at full resolution. +- The default view grid per lens is 3 yaws × 3 pitches at ±60° with 75° + square crops (9 views per lens, **18 per frame pair**). The down-pitched + views of both lenses each put the nadir ~30° from their optical axis, so + the ground under the camera is covered by **up to 6 views per frame pair** + instead of 0. +- All 18 views are registered as a single COLMAP rig (shared orientation, + zero translation), same machinery as the ER path; the SfM steps + (`pano_sfm.run_rig_sfm_pipeline`) are shared. + +## Usage + +```bash +# Auto-detected from the .insv extension: +python vid2scene_core/vid2scene.py output_dir --video_path VID_20260611_120000_00_001.insv + +# Explicit, with a lens FOV override: +python vid2scene_core/vid2scene.py output_dir --video_path capture.insv \ + --insv_fisheye --insv_lens_fov 195 + +# Run only the SfM stage: +python vid2scene_core/fisheye_sfm.py --insv_path capture.insv --output_path out/ +``` + +Supported `.insv` layouts (demuxed with ffmpeg): + +| Layout | Cameras | Handling | +|---|---|---| +| Two video streams in one file | X3 / X4 / X5 single-file recordings | streams `0:v:0` and `0:v:1` | +| Two files `..._00_...insv` + `..._10_...insv` | older cameras, some high-res modes | pass either file; the companion is found automatically | +| One 2:1 stream, two fisheyes side by side | some older models | split in half; guarded by a dark-corner check so ER videos aren't mis-split | + +## Lens model and calibration + +By default each lens is modeled as an idealized equidistant fisheye: +principal point at the frame center, image circle inscribed in the frame, +and a nominal 200° FOV mapped linearly onto the circle +(`fisheye_projection.FisheyeLensModel`). Real lenses deviate; if the +reconstruction shows systematic warp, supply a calibration JSON via +`--insv_calibration`: + +```json +{ + "fov_deg": 197.5, + "k1": -0.012, "k2": 0.0021, "k3": 0.0, "k4": 0.0, + "lenses": [ + {"cx": 1441.2, "cy": 1439.1, "focal_px_per_rad": 824.5}, + {"cx": 1438.9, "cy": 1442.3, "focal_px_per_rad": 826.1} + ], + "lens1_yaw_deg": 180.0, + "lens1_roll_deg": 0.0 +} +``` + +Top-level keys apply to both lenses; entries in `lenses` override per lens. +`k1..k4` are odd-polynomial (Kannala-Brandt) distortion terms on +`r = focal * (θ + k1·θ³ + k2·θ⁵ + k3·θ⁷ + k4·θ⁹)`. `lens1_yaw_deg` / +`lens1_roll_deg` describe how the rear lens is mounted relative to the front +one. + +## Masks + +For every rendered view two masks are produced: + +- **SfM masks** (`masks/`): valid-fisheye-pixel ∩ closest-view partition, + so overlapping views don't generate duplicate SIFT features of the same + physical point. COLMAP requires a mask for every image once `mask_path` + is set; one is written per rendered image. +- **Training masks** (`training_masks/`): valid fisheye pixels only, so + gsplat doesn't train on the black corners of views that extend past the + lens FOV. + +## Current limitations / follow-ups + +- **No per-unit factory calibration.** Insta360 ships MEI-model calibration + (protobuf, in the trailer or an `.insv.pb` sidecar on X5); parsing it + would remove the idealized-lens assumption. The + [telemetry-parser](https://github.com/AdrianEddy/telemetry-parser) crate + (used by [insv-stitch](https://github.com/BenjaminHenriksson/insv-stitch)) + already reads these. +- **No IMU use.** The trailer's gyro record (`0x300`) could provide + horizon leveling and rolling-shutter correction. +- **Lens baseline ignored.** The two lens centers are ~2–3 cm apart but the + rig assumes a shared center, like the ER path (and Insta360's own + stitching). Only matters for subjects very close to the camera. +- **No ego-object masking.** The ER path can mask the operator/tripod with + SAM3; that step is not yet wired into the fisheye path. +- **Trailer parsing is best-effort.** Based on community reverse + engineering (ExifTool, Sub-Etha Software); used only for logging camera + metadata. Failures are silent and harmless. +- **Server/API wiring.** The web upload flow only exposes the + `equirectangular` flag; `.insv` uploads through the server need a model + field + form pass-through (auto-detection by extension already works at + the pipeline level). diff --git a/vid2scene_core/fisheye_projection.py b/vid2scene_core/fisheye_projection.py new file mode 100644 index 0000000..b75a8c4 --- /dev/null +++ b/vid2scene_core/fisheye_projection.py @@ -0,0 +1,218 @@ +""" +Fisheye projection math for direct dual-fisheye (.insv) processing. + +Pure numpy/scipy module (no pycolmap dependency) so the projection math is +independently unit-testable. Used by fisheye_sfm.py to render virtual pinhole +views directly from Insta360 fisheye sensor frames, skipping the +equirectangular intermediate that resamples (and badly stretches) the image +near the poles — i.e. exactly at the ground and sky. + +Conventions match pano_sfm.py / COLMAP: +- Camera frame: x right, y down, z forward (optical axis). +- Continuous pixel coordinates place the center of the top-left pixel at + (0.5, 0.5). cv2.remap maps are shifted by -0.5 accordingly. +- Rotations are "cam_from_X" matrices applied to row vectors as + ``ray_in_X = ray_in_cam @ cam_from_X_r``. +""" + +from collections.abc import Sequence +from dataclasses import dataclass + +import numpy as np +import numpy.typing as npt +from scipy.spatial.transform import Rotation + + +@dataclass +class FisheyeLensModel: + """Equidistant fisheye model with optional Kannala-Brandt distortion terms. + + The projection is r = focal * (theta + k1*theta^3 + k2*theta^5 + + k3*theta^7 + k4*theta^9) where theta is the angle from the optical axis + and r is the radial distance from the principal point in pixels. + + Defaults model an idealized Insta360-style lens: principal point at the + image center, image circle inscribed in the (square) frame, and the + nominal full lens FOV mapped linearly onto the image circle. Real lenses + deviate from this; per-unit calibration can be supplied via the k terms, + focal, and principal point (see docs/insv_fisheye.md). + """ + + width: int + height: int + fov_deg: float = 200.0 + focal_px_per_rad: float | None = None + cx: float | None = None + cy: float | None = None + k1: float = 0.0 + k2: float = 0.0 + k3: float = 0.0 + k4: float = 0.0 + image_circle_radius: float | None = None + + def __post_init__(self): + if self.cx is None: + self.cx = self.width / 2 + if self.cy is None: + self.cy = self.height / 2 + if self.image_circle_radius is None: + self.image_circle_radius = min(self.width, self.height) / 2 + if self.focal_px_per_rad is None: + self.focal_px_per_rad = self.image_circle_radius / np.deg2rad(self.fov_deg / 2) + + def theta_to_radius(self, theta: npt.NDArray[np.floating]) -> npt.NDArray[np.floating]: + """Radial distance in pixels for angle theta (radians) from the optical axis.""" + t2 = theta * theta + poly = 1.0 + t2 * (self.k1 + t2 * (self.k2 + t2 * (self.k3 + t2 * self.k4))) + return self.focal_px_per_rad * theta * poly + + def radius_to_theta(self, radius: npt.NDArray[np.floating]) -> npt.NDArray[np.floating]: + """Invert theta_to_radius with Newton iterations (exact when k1..k4 are 0).""" + radius = np.asarray(radius, dtype=np.float64) + theta = radius / self.focal_px_per_rad + for _ in range(20): + t2 = theta * theta + poly = 1.0 + t2 * (self.k1 + t2 * (self.k2 + t2 * (self.k3 + t2 * self.k4))) + d_poly = 1.0 + t2 * (3 * self.k1 + t2 * (5 * self.k2 + t2 * (7 * self.k3 + t2 * 9 * self.k4))) + residual = self.focal_px_per_rad * theta * poly - radius + theta = theta - residual / (self.focal_px_per_rad * d_poly) + return theta + + def project_rays( + self, rays: npt.NDArray[np.floating] + ) -> tuple[npt.NDArray[np.floating], npt.NDArray[np.bool_]]: + """Project rays (N, 3) in the lens frame to pixel coordinates. + + Returns (uv, valid) where uv is (N, 2) in COLMAP pixel convention + (top-left pixel center at (0.5, 0.5)) and valid marks rays inside + the lens FOV, the image circle, and the image bounds. + """ + x, y, z = np.moveaxis(np.asarray(rays, dtype=np.float64), -1, 0) + r_xy = np.hypot(x, y) + theta = np.arctan2(r_xy, z) + phi = np.arctan2(y, x) + radius = self.theta_to_radius(theta) + u = self.cx + radius * np.cos(phi) + v = self.cy + radius * np.sin(phi) + valid = ( + (theta <= np.deg2rad(self.fov_deg / 2)) + & (radius <= self.image_circle_radius) + & (u >= 0.0) + & (u <= self.width) + & (v >= 0.0) + & (v <= self.height) + ) + return np.stack([u, v], axis=-1), valid + + def unproject_points(self, uv: npt.NDArray[np.floating]) -> npt.NDArray[np.floating]: + """Unproject pixel coordinates (N, 2) to unit rays (N, 3) in the lens frame.""" + uv = np.asarray(uv, dtype=np.float64) + dx = uv[..., 0] - self.cx + dy = uv[..., 1] - self.cy + radius = np.hypot(dx, dy) + theta = self.radius_to_theta(radius) + phi = np.arctan2(dy, dx) + sin_theta = np.sin(theta) + rays = np.stack( + [sin_theta * np.cos(phi), sin_theta * np.sin(phi), np.cos(theta)], axis=-1 + ) + return rays + + +@dataclass +class FisheyeRigRenderOptions: + """Virtual pinhole view layout rendered from each fisheye lens. + + Angles are lens-local: yaw/pitch (0, 0) looks along the lens optical + axis. The default 3x3 grid at +/-60 degrees with 75 degree square crops + covers each lens out to ~97 degrees off-axis, so the down-pitched views + of both lenses see the nadir (ground directly under the camera) at full + sensor resolution — the region the equirectangular path samples worst. + """ + + yaws_deg: Sequence[float] = (-60.0, 0.0, 60.0) + pitches_deg: Sequence[float] = (-60.0, 0.0, 60.0) + crop_fov_deg: float = 75.0 + crop_size: int | None = None # None = derived from source resolution + max_crop_size: int = 1600 + + @property + def num_views_per_lens(self) -> int: + return len(self.yaws_deg) * len(self.pitches_deg) + + +def pinhole_focal(image_size: int, fov_deg: float) -> float: + """Focal length in pixels for a square pinhole image with the given FOV.""" + return image_size / (2 * np.tan(np.deg2rad(fov_deg) / 2)) + + +def pinhole_camera_rays(image_size: int, fov_deg: float) -> npt.NDArray[np.floating]: + """Unit ray directions (image_size**2, 3) for a square pinhole camera. + + Rays are ordered row-major (y, then x) so a reshape to + (image_size, image_size) is directly indexable as [row, col]. + """ + focal = pinhole_focal(image_size, fov_deg) + coords = (np.arange(image_size, dtype=np.float64) + 0.5 - image_size / 2) / focal + xx, yy = np.meshgrid(coords, coords) + rays = np.stack([xx, yy, np.ones_like(xx)], axis=-1).reshape(-1, 3) + rays /= np.linalg.norm(rays, axis=-1, keepdims=True) + return rays + + +def get_lens_local_rotations( + yaws_deg: Sequence[float], pitches_deg: Sequence[float] +) -> list[npt.NDArray[np.floating]]: + """cam_from_lens rotation matrices for a lens-local yaw/pitch view grid. + + Same Euler convention as pano_sfm.get_virtual_rotations: positive pitch + looks up, positive yaw looks right (with the y-down camera frame). + """ + rotations = [] + for pitch_deg in pitches_deg: + for yaw_deg in yaws_deg: + rotations.append( + Rotation.from_euler("XY", [-pitch_deg, -yaw_deg], degrees=True).as_matrix() + ) + return rotations + + +def derive_crop_size(lens_model: FisheyeLensModel, crop_fov_deg: float, max_crop_size: int) -> int: + """Crop size whose center angular resolution matches the fisheye source.""" + size = lens_model.focal_px_per_rad * 2 * np.tan(np.deg2rad(crop_fov_deg) / 2) + size = int(np.clip(round(size), 256, max_crop_size)) + return size + (size % 2) + + +def build_remap_grid( + lens_model: FisheyeLensModel, + cam_from_lens_r: npt.NDArray[np.floating], + crop_fov_deg: float, + crop_size: int, +) -> tuple[npt.NDArray[np.float32], npt.NDArray[np.float32], npt.NDArray[np.bool_]]: + """Build cv2.remap maps rendering a pinhole view out of a fisheye image. + + Returns (map_x, map_y, valid), each (crop_size, crop_size). Invalid + pixels (outside the lens FOV / image circle) map to (-1, -1) so + cv2.remap with BORDER_CONSTANT renders them black. + """ + rays_in_cam = pinhole_camera_rays(crop_size, crop_fov_deg) + rays_in_lens = rays_in_cam @ cam_from_lens_r + uv, valid = lens_model.project_rays(rays_in_lens) + uv = uv - 0.5 # COLMAP to OpenCV pixel origin + uv[~valid] = -1.0 + map_x = uv[:, 0].astype(np.float32).reshape(crop_size, crop_size) + map_y = uv[:, 1].astype(np.float32).reshape(crop_size, crop_size) + return map_x, map_y, valid.reshape(crop_size, crop_size) + + +def closest_view_partition( + rays: npt.NDArray[np.floating], view_axes: npt.NDArray[np.floating] +) -> npt.NDArray[np.intp]: + """Assign each ray (N, 3) to the view axis (V, 3) it is most aligned with. + + Used to build feature-extraction masks so overlapping virtual views don't + contribute duplicate SIFT features of the same physical point, mirroring + the closest-camera partition in pano_sfm.PanoProcessor. + """ + return np.argmax(rays @ np.asarray(view_axes).T, axis=-1) diff --git a/vid2scene_core/fisheye_sfm.py b/vid2scene_core/fisheye_sfm.py new file mode 100644 index 0000000..eae477e --- /dev/null +++ b/vid2scene_core/fisheye_sfm.py @@ -0,0 +1,471 @@ +""" +Direct dual-fisheye (.insv) SfM pipeline for vid2scene. + +Renders virtual pinhole views straight out of the two Insta360 fisheye sensor +streams and reconstructs them with the same rig-constrained SfM used for +equirectangular panoramas (pano_sfm.run_rig_sfm_pipeline). + +Why: the equirectangular path (Insta360 Studio stitch -> pano_sfm) resamples +the sphere into a 2:1 image whose poles — the ground under the camera and the +sky — are maximally stretched, and the virtual rig only reaches -35 degrees of +pitch. Cropping pinhole views directly from the fisheye sources keeps native +sensor pixels everywhere, and the default view grid points views down past the +nadir, so the ground is covered by up to 6 views per frame pair instead of 0. + +The lens model is an idealized equidistant fisheye by default (see +fisheye_projection.FisheyeLensModel); per-unit calibration can be supplied as +a JSON file. Factory calibration / IMU parsing from the .insv trailer is a +follow-up (see docs/insv_fisheye.md). +""" + +import argparse +import json +import logging +import os +import shutil +import tempfile +from collections.abc import Sequence +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path +from threading import Lock +from typing import cast + +import cv2 +import numpy as np +import numpy.typing as npt +from scipy.spatial.transform import Rotation +from tqdm import tqdm + +import pycolmap + +import insv_extract +import pano_sfm +from fisheye_projection import ( + FisheyeLensModel, + FisheyeRigRenderOptions, + build_remap_grid, + closest_view_partition, + derive_crop_size, + get_lens_local_rotations, + pinhole_camera_rays, + pinhole_focal, +) + +logger = logging.getLogger(__name__) + +INSV_RENDER_OPTIONS = FisheyeRigRenderOptions() + + +def get_lens_from_rig_rotations( + lens1_yaw_deg: float = 180.0, lens1_roll_deg: float = 0.0 +) -> list[npt.NDArray[np.floating]]: + """lens_from_rig rotations for the two back-to-back lenses. + + Lens 0 defines the rig frame; lens 1 nominally points the opposite way + (180 degree yaw). The optional roll accounts for sensors mounted rotated + relative to each other. + """ + lens0_from_rig = np.eye(3) + lens1_from_rig = Rotation.from_euler( + "ZY", [lens1_roll_deg, lens1_yaw_deg], degrees=True + ).as_matrix() + return [lens0_from_rig, lens1_from_rig] + + +def create_fisheye_rig_config( + image_prefixes: Sequence[str], + cams_from_rig_rotation: Sequence[npt.NDArray[np.floating]], + ref_idx: int = 0, +) -> pycolmap.RigConfig: + """Create a RigConfig over all virtual views of both lenses.""" + rig_cameras = [] + zero_translation = np.zeros((3, 1), dtype=np.float64) + for idx, (prefix, cam_from_rig_rotation) in enumerate( + zip(image_prefixes, cams_from_rig_rotation) + ): + if idx == ref_idx: + cam_from_rig = None + else: + cam_from_ref_rotation = ( + cam_from_rig_rotation @ cams_from_rig_rotation[ref_idx].T + ) + cam_from_rig = pycolmap.Rigid3d( + pycolmap.Rotation3d(cam_from_ref_rotation), + cast(np.ndarray, zero_translation), + ) + rig_cameras.append( + pycolmap.RigConfigCamera( + ref_sensor=idx == ref_idx, + image_prefix=prefix, + cam_from_rig=cam_from_rig, + ) + ) + return pycolmap.RigConfig(cameras=rig_cameras) + + +class FisheyeFrameProcessor: + """Renders paired fisheye frames into virtual pinhole views with masks.""" + + def __init__( + self, + lens_frame_dirs: Sequence[Path], + output_image_dir: Path, + render_options: FisheyeRigRenderOptions, + mask_dir: Path | None = None, + training_mask_dir: Path | None = None, + lens_model_overrides: Sequence[dict] | None = None, + lens1_yaw_deg: float = 180.0, + lens1_roll_deg: float = 0.0, + ): + self.lens_frame_dirs = [Path(d) for d in lens_frame_dirs] + self.output_image_dir = Path(output_image_dir) + self.render_options = render_options + self.mask_dir = Path(mask_dir) if mask_dir else None + self.training_mask_dir = Path(training_mask_dir) if training_mask_dir else None + self.lens_model_overrides = list(lens_model_overrides or [{}, {}]) + + num_lenses = len(self.lens_frame_dirs) + if num_lenses != 2: + raise ValueError(f"Expected 2 lens directories, got {num_lenses}") + + self.cams_from_lens = get_lens_local_rotations( + render_options.yaws_deg, render_options.pitches_deg + ) + self.lens_from_rig = get_lens_from_rig_rotations(lens1_yaw_deg, lens1_roll_deg) + + # One virtual view per (lens, lens-local rotation) + self.view_lens_idx: list[int] = [] + self.view_prefixes: list[str] = [] + self.views_cam_from_lens: list[npt.NDArray[np.floating]] = [] + self.views_cam_from_rig: list[npt.NDArray[np.floating]] = [] + for lens_idx in range(num_lenses): + for cam_idx, cam_from_lens in enumerate(self.cams_from_lens): + self.view_lens_idx.append(lens_idx) + self.view_prefixes.append(f"lens{lens_idx}_cam{cam_idx}/") + self.views_cam_from_lens.append(cam_from_lens) + self.views_cam_from_rig.append(cam_from_lens @ self.lens_from_rig[lens_idx]) + + self.rig_config = create_fisheye_rig_config( + self.view_prefixes, self.views_cam_from_rig + ) + + self._lock = Lock() + self._initialized = False + self._camera: pycolmap.Camera | None = None + self.lens_models: list[FisheyeLensModel] | None = None + self.crop_size: int | None = None + self._maps: list[tuple[np.ndarray, np.ndarray]] = [] + self._sfm_mask_png: list[bytes] = [] + self._training_mask_png: list[bytes] = [] + + def _initialize(self, lens_images: Sequence[np.ndarray]) -> None: + """Build lens models, remap grids, and static masks from the first frame pair.""" + self.lens_models = [] + for lens_idx, image in enumerate(lens_images): + height, width = image.shape[:2] + overrides = dict(self.lens_model_overrides[lens_idx]) + self.lens_models.append(FisheyeLensModel(width=width, height=height, **overrides)) + logger.info( + f"Lens {lens_idx}: {width}x{height}, fov={self.lens_models[lens_idx].fov_deg} deg, " + f"focal={self.lens_models[lens_idx].focal_px_per_rad:.1f} px/rad" + ) + + options = self.render_options + self.crop_size = options.crop_size or derive_crop_size( + self.lens_models[0], options.crop_fov_deg, options.max_crop_size + ) + logger.info( + f"Rendering {len(self.view_prefixes)} virtual views per frame pair " + f"({self.crop_size}x{self.crop_size}, {options.crop_fov_deg} deg FOV)" + ) + + self._camera = pycolmap.Camera.create( + 0, + pycolmap.CameraModelId.SIMPLE_PINHOLE, + pinhole_focal(self.crop_size, options.crop_fov_deg), + self.crop_size, + self.crop_size, + ) + for rig_camera in self.rig_config.cameras: + rig_camera.camera = self._camera + + # View axes in the rig frame, for the closest-view feature partition + view_axes_in_rig = np.stack([r[2, :] for r in self.views_cam_from_rig]) + rays_in_cam = pinhole_camera_rays(self.crop_size, options.crop_fov_deg) + + for view_idx, lens_idx in enumerate(self.view_lens_idx): + map_x, map_y, valid = build_remap_grid( + self.lens_models[lens_idx], + self.views_cam_from_lens[view_idx], + options.crop_fov_deg, + self.crop_size, + ) + self._maps.append((map_x, map_y)) + + # SfM mask: valid fisheye pixels owned by this view in the partition + rays_in_rig = rays_in_cam @ self.views_cam_from_rig[view_idx] + closest_view = closest_view_partition(rays_in_rig, view_axes_in_rig) + owned = (closest_view == view_idx).reshape(self.crop_size, self.crop_size) + sfm_mask = ((owned & valid) * np.uint8(255)) + self._sfm_mask_png.append(_encode_png(sfm_mask)) + + # Training mask: every valid pixel (gsplat may supervise overlaps) + training_mask = (valid * np.uint8(255)) + self._training_mask_png.append(_encode_png(training_mask)) + + self._initialized = True + + def process(self, frame_name: str) -> None: + """Render one paired fisheye frame into all virtual views.""" + lens_images = [] + for lens_dir in self.lens_frame_dirs: + image = cv2.imread(str(lens_dir / frame_name), cv2.IMREAD_COLOR) + if image is None: + logger.warning(f"Skipping frame {frame_name}: unreadable in {lens_dir}") + return + lens_images.append(image) + + with self._lock: + if not self._initialized: + self._initialize(lens_images) + + for lens_idx, image in enumerate(lens_images): + model = self.lens_models[lens_idx] + if image.shape[1] != model.width or image.shape[0] != model.height: + logger.warning( + f"Skipping frame {frame_name}: lens {lens_idx} size mismatch " + f"({image.shape[1]}x{image.shape[0]} vs {model.width}x{model.height})" + ) + return + + for view_idx, prefix in enumerate(self.view_prefixes): + map_x, map_y = self._maps[view_idx] + view_image = cv2.remap( + lens_images[self.view_lens_idx[view_idx]], + map_x, + map_y, + cv2.INTER_LINEAR, + borderMode=cv2.BORDER_CONSTANT, + ) + image_path = self.output_image_dir / prefix / frame_name + image_path.parent.mkdir(exist_ok=True, parents=True) + cv2.imwrite(str(image_path), view_image) + + if self.mask_dir is not None: + # COLMAP looks for masks named ".png" + mask_path = self.mask_dir / prefix / f"{frame_name}.png" + mask_path.parent.mkdir(exist_ok=True, parents=True) + mask_path.write_bytes(self._sfm_mask_png[view_idx]) + + if self.training_mask_dir is not None: + training_mask_path = self.training_mask_dir / prefix / frame_name + training_mask_path.parent.mkdir(exist_ok=True, parents=True) + training_mask_path.write_bytes(self._training_mask_png[view_idx]) + + +def _encode_png(mask: np.ndarray) -> bytes: + success, buffer = cv2.imencode(".png", mask) + if not success: + raise RuntimeError("Failed to encode mask PNG") + return buffer.tobytes() + + +def render_fisheye_views( + processor: FisheyeFrameProcessor, + frame_names: Sequence[str], + max_workers: int | None = None, +) -> None: + """Render all paired frames in parallel.""" + if max_workers is None: + max_workers = min(32, (os.cpu_count() or 2) - 1) + + # Process the first frame alone so initialization (remap grids, masks) + # happens once instead of racing in every worker. + processor.process(frame_names[0]) + + with tqdm(total=len(frame_names), desc="Rendering fisheye views") as pbar: + pbar.update(1) + with ThreadPoolExecutor(max_workers=max_workers) as thread_pool: + futures = [ + thread_pool.submit(processor.process, frame_name) + for frame_name in frame_names[1:] + ] + for future in as_completed(futures): + future.result() + pbar.update(1) + + +def load_lens_calibration(calibration_path) -> dict: + """Load the optional lens calibration JSON (see docs/insv_fisheye.md).""" + with open(calibration_path) as f: + calibration = json.load(f) + logger.info(f"Loaded .insv lens calibration from {calibration_path}") + return calibration + + +def build_lens_model_overrides( + calibration: dict | None, lens_fov_deg: float | None +) -> list[dict]: + """Merge global and per-lens calibration into FisheyeLensModel kwargs.""" + calibration = calibration or {} + shared_keys = ( + "fov_deg", "focal_px_per_rad", "cx", "cy", + "k1", "k2", "k3", "k4", "image_circle_radius", + ) + shared = {k: calibration[k] for k in shared_keys if k in calibration} + if lens_fov_deg is not None: + shared["fov_deg"] = lens_fov_deg + + overrides = [] + per_lens = calibration.get("lenses", [{}, {}]) + for lens_idx in range(2): + lens_overrides = dict(shared) + if lens_idx < len(per_lens): + lens_overrides.update( + {k: per_lens[lens_idx][k] for k in shared_keys if k in per_lens[lens_idx]} + ) + overrides.append(lens_overrides) + return overrides + + +def run_insv_sfm( + insv_path, + output_path: Path, + target_framecount: int = 90, + mapper: str = "colmap", + generate_masks: bool = True, + render_options: FisheyeRigRenderOptions = INSV_RENDER_OPTIONS, + lens_fov_deg: float | None = None, + calibration_path=None, + kill_check=None, +) -> Path | None: + """ + Run the direct .insv dual-fisheye SfM pipeline. + + This function: + 1. Demuxes the .insv into paired per-lens fisheye frames (ffmpeg) + 2. Renders virtual pinhole views directly from each fisheye frame + 3. Runs the shared rig-constrained SfM pipeline (pano_sfm.run_rig_sfm_pipeline) + + Args: + insv_path: Path to the .insv recording (either file of a two-file pair) + output_path: Output directory for rendered images and SfM results + target_framecount: Number of paired fisheye frames to extract + mapper: Reconstruction method ("glomap" or "colmap") + generate_masks: Generate feature-extraction masks (validity + view partition) + render_options: Virtual view layout configuration + lens_fov_deg: Override the assumed lens FOV (default 200) + calibration_path: Optional lens calibration JSON + kill_check: Optional callback to check if processing should abort + + Returns: + Path to sparse reconstruction directory, or None if aborted/failed + """ + pycolmap.set_random_seed(0) + output_path = Path(output_path) + image_dir = output_path / "images" + mask_dir = output_path / "masks" if generate_masks else None + training_mask_dir = output_path / "training_masks" if generate_masks else None + image_dir.mkdir(exist_ok=True, parents=True) + if mask_dir: + mask_dir.mkdir(exist_ok=True, parents=True) + if training_mask_dir: + training_mask_dir.mkdir(exist_ok=True, parents=True) + + metadata = insv_extract.summarize_insv_metadata( + insv_extract.read_insv_trailer_records(insv_path) + ) + if metadata.get("record_ids"): + logger.info(f".insv trailer records: {metadata['record_ids']}") + if metadata.get("file_info_strings"): + logger.info(f".insv file info: {metadata['file_info_strings']}") + + calibration = load_lens_calibration(calibration_path) if calibration_path else None + lens_model_overrides = build_lens_model_overrides(calibration, lens_fov_deg) + lens1_yaw_deg = (calibration or {}).get("lens1_yaw_deg", 180.0) + lens1_roll_deg = (calibration or {}).get("lens1_roll_deg", 0.0) + + if kill_check and kill_check(): + logger.info("Job cancelled before .insv extraction") + return None + + # ========== Step 1: Demux .insv into paired fisheye frames ========== + frames_temp_dir = tempfile.mkdtemp(prefix="insv_fisheye_") + try: + lens_dirs = insv_extract.extract_dual_fisheye_frames( + insv_path, frames_temp_dir, target_framecount + ) + + if kill_check and kill_check(): + logger.info("Job cancelled after .insv extraction") + return None + + frame_names = sorted( + set(os.listdir(lens_dirs[0])) & set(os.listdir(lens_dirs[1])) + ) + if not frame_names: + logger.error("No paired fisheye frames extracted!") + return None + + # ========== Step 2: Render virtual pinhole views ========== + processor = FisheyeFrameProcessor( + lens_frame_dirs=[Path(d) for d in lens_dirs], + output_image_dir=image_dir, + render_options=render_options, + mask_dir=mask_dir, + training_mask_dir=training_mask_dir, + lens_model_overrides=lens_model_overrides, + lens1_yaw_deg=lens1_yaw_deg, + lens1_roll_deg=lens1_roll_deg, + ) + render_fisheye_views(processor, frame_names) + finally: + shutil.rmtree(frames_temp_dir, ignore_errors=True) + + if kill_check and kill_check(): + logger.info("Job cancelled after rendering fisheye views") + return None + + # ========== Step 3: Shared rig SfM (features, rig, matching, mapping) ========== + return pano_sfm.run_rig_sfm_pipeline( + output_path=output_path, + image_dir=image_dir, + rig_config=processor.rig_config, + mask_dir=mask_dir, + mapper=mapper, + kill_check=kill_check, + ) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + + parser = argparse.ArgumentParser(description="Direct .insv dual-fisheye SfM pipeline") + parser.add_argument("--insv_path", type=Path, required=True, + help="Path to the .insv recording") + parser.add_argument("--output_path", type=Path, required=True, + help="Output directory for results") + parser.add_argument("--target_framecount", type=int, default=90, + help="Number of paired fisheye frames to extract") + parser.add_argument("--mapper", choices=["glomap", "colmap"], default="colmap", + help="Reconstruction method") + parser.add_argument("--lens_fov", type=float, default=None, + help="Assumed full lens FOV in degrees (default 200)") + parser.add_argument("--calibration", type=Path, default=None, + help="Optional lens calibration JSON") + + args = parser.parse_args() + + result = run_insv_sfm( + insv_path=args.insv_path, + output_path=args.output_path, + target_framecount=args.target_framecount, + mapper=args.mapper, + lens_fov_deg=args.lens_fov, + calibration_path=args.calibration, + ) + + if result: + logger.info(f"Success! Reconstruction saved to {result}") + else: + logger.error("Pipeline failed") diff --git a/vid2scene_core/insv_extract.py b/vid2scene_core/insv_extract.py new file mode 100644 index 0000000..57bf9d9 --- /dev/null +++ b/vid2scene_core/insv_extract.py @@ -0,0 +1,316 @@ +""" +Insta360 .insv ingestion: demux the dual fisheye video streams to paired +per-lens frame directories, and best-effort parse the proprietary metadata +trailer. + +An .insv file is an ISO-BMFF (MP4-like) container that ffmpeg can read +directly. Depending on the camera generation the two fisheye eyes are stored +as: + - two separate video streams in one file (X3/X4/X5 single-file recordings), + - two separate files, ``..._00_...insv`` and ``..._10_...insv`` (older + cameras, and high-resolution modes on some models), or + - one video stream with both fisheyes side by side (some older models). + +After the MP4 body, Insta360 appends a proprietary trailer holding IMU +samples, exposure data, GPS, and file info. The format was reverse engineered +by the community (ExifTool's QuickTimeStream.pl and +https://subethasoftware.com/2022/06/08/insta360-one-x2-insv-file-format/): +the file ends with a 32-byte ASCII magic, and records are walked backwards +from EOF-78, each record being a (uint16 id, uint32 size) descriptor preceded +by ``size`` bytes of payload. Trailer parsing here is purely informational +(camera model detection / logging); every failure degrades to an empty result. +""" + +import json +import logging +import os +import re +import shutil +import struct +import subprocess +from math import ceil +from pathlib import Path + +logger = logging.getLogger(__name__) + +INSV_TRAILER_MAGIC = b"8db42d694ccc418790edff439fe026bf" +INSV_TRAILER_FOOTER_SIZE = 78 + +# Known trailer record ids (per ExifTool QuickTimeStream.pl). Only 0x101 is +# interpreted here; the rest are listed for logging/debugging. +INSV_RECORD_NAMES = { + 0x101: "file_info", + 0x200: "preview_image", + 0x300: "gyro", + 0x400: "exposure", + 0x600: "timestamps", + 0x700: "gps", +} + + +def read_insv_trailer_records(path, max_records: int = 64) -> dict[int, bytes]: + """Best-effort parse of the .insv trailer into {record_id: payload_bytes}. + + Returns an empty dict if the trailer is missing or doesn't parse; callers + must treat all metadata as optional. + """ + try: + file_size = os.path.getsize(path) + if file_size < INSV_TRAILER_FOOTER_SIZE: + return {} + with open(path, "rb") as f: + f.seek(file_size - INSV_TRAILER_FOOTER_SIZE) + footer = f.read(INSV_TRAILER_FOOTER_SIZE) + if footer[-len(INSV_TRAILER_MAGIC):] != INSV_TRAILER_MAGIC: + return {} + + records: dict[int, bytes] = {} + # The first record descriptor is the start of the 78-byte footer; + # each record's payload sits immediately before its descriptor, + # and the previous record's descriptor before that payload. + pos = file_size - INSV_TRAILER_FOOTER_SIZE + for _ in range(max_records): + if pos < 6: + break + f.seek(pos) + record_id, record_size = struct.unpack(" pos: + break + f.seek(pos - record_size) + records[record_id] = f.read(record_size) + pos = pos - record_size - 6 + return records + except (OSError, struct.error) as e: + logger.warning(f"Could not parse .insv trailer of {path}: {e}") + return {} + + +def summarize_insv_metadata(records: dict[int, bytes]) -> dict: + """Extract loggable info from trailer records (camera model, serial, ...). + + Record 0x101 is a small protobuf message whose first three string fields + are serial number, camera model, and firmware version. Rather than pull in + a protobuf dependency, printable ASCII runs are extracted, which yields + the same strings. + """ + info: dict = { + "record_ids": { + f"0x{record_id:x}": INSV_RECORD_NAMES.get(record_id, "unknown") + for record_id in sorted(records) + } + } + file_info = records.get(0x101) + if file_info: + strings = [s.decode("ascii") for s in re.findall(rb"[ -~]{4,}", file_info)] + info["file_info_strings"] = strings + return info + + +def find_companion_lens_file(insv_path) -> Path | None: + """Find the second-eye file of a two-file recording (``_00_`` <-> ``_10_``).""" + path = Path(insv_path) + name = path.name + for this_tag, other_tag in (("_00_", "_10_"), ("_10_", "_00_")): + if this_tag in name: + companion = path.with_name(name.replace(this_tag, other_tag, 1)) + if companion.exists(): + return companion + return None + return None + + +def probe_video_streams(path) -> list[dict]: + """Return [{width, height}] for each video stream, in ffmpeg ordinal order.""" + command = [ + "ffprobe", + "-v", "error", + "-select_streams", "v", + "-show_entries", "stream=width,height", + "-print_format", "json", + str(path), + ] + output = subprocess.check_output(command) + streams = json.loads(output).get("streams", []) + return [{"width": int(s["width"]), "height": int(s["height"])} for s in streams] + + +def count_stream_frames(path, stream_ordinal: int) -> int | None: + """Count frames (packets) of video stream ``v:stream_ordinal``.""" + command = [ + "ffprobe", + "-v", "error", + "-select_streams", f"v:{stream_ordinal}", + "-count_packets", + "-show_entries", "stream=nb_read_packets", + "-of", "default=nokey=1:noprint_wrappers=1", + str(path), + ] + try: + return int(subprocess.check_output(command).strip()) + except (subprocess.CalledProcessError, ValueError): + logger.error(f"Unable to count frames of stream v:{stream_ordinal} in {path}") + return None + + +def looks_like_fisheye_frame(image) -> bool: + """Heuristic: fisheye frames have a dark border outside the image circle. + + Used to distinguish a side-by-side dual-fisheye stream from an already + stitched equirectangular video (both are 2:1). + """ + import numpy as np + + h, w = image.shape[:2] + margin = max(2, min(h, w) // 50) + corners = [ + image[:margin, :margin], + image[:margin, -margin:], + image[-margin:, :margin], + image[-margin:, -margin:], + ] + mean_corner = float(np.mean([np.mean(c) for c in corners])) + return mean_corner < 20.0 + + +def _extract_stream_frames(path, stream_ordinal: int, output_dir, interval: int, crop_filter: str | None = None): + """Extract every ``interval``-th frame of one video stream as PNGs.""" + os.makedirs(output_dir, exist_ok=True) + video_filter = f"select=not(mod(n\\,{interval}))" + if crop_filter: + video_filter += f",{crop_filter}" + output_pattern = os.path.join(output_dir, "frame_%05d.png") + command = [ + "ffmpeg", + "-loglevel", "error", + "-i", str(path), + "-map", f"0:v:{stream_ordinal}", + "-vf", video_filter, + "-vsync", "vfr", + "-start_number", "1", + output_pattern, + ] + subprocess.run(command, check=True) + + +def _trim_to_paired_frames(lens_dirs: list[str]) -> int: + """Delete trailing frames so both lens directories have the same count.""" + frame_lists = [sorted(os.listdir(d)) for d in lens_dirs] + num_pairs = min(len(frames) for frames in frame_lists) + for lens_dir, frames in zip(lens_dirs, frame_lists): + for extra in frames[num_pairs:]: + os.remove(os.path.join(lens_dir, extra)) + return num_pairs + + +def extract_dual_fisheye_frames(insv_path, output_dir, target_framecount: int) -> list[str]: + """Demux an .insv recording into two paired per-lens frame directories. + + Returns [lens0_dir, lens1_dir] where each directory holds PNG frames named + frame_%05d.png; frame N in lens0 was captured (approximately) simultaneously + with frame N in lens1. + """ + os.makedirs(output_dir, exist_ok=True) + lens_dirs = [os.path.join(output_dir, "lens0"), os.path.join(output_dir, "lens1")] + + streams = probe_video_streams(insv_path) + if not streams: + raise ValueError(f"No video streams found in {insv_path}") + companion = find_companion_lens_file(insv_path) + + if len(streams) >= 2: + # Modern single-file recording: one video stream per lens. Streams + # beyond the first two (e.g. low-res preview tracks) are ignored. + if streams[0]["width"] != streams[1]["width"] or streams[0]["height"] != streams[1]["height"]: + raise ValueError( + f"First two video streams of {insv_path} differ in size " + f"({streams[0]} vs {streams[1]}); not a dual-fisheye recording?" + ) + total_frames = min( + count_stream_frames(insv_path, 0) or 0, + count_stream_frames(insv_path, 1) or 0, + ) + interval = _frame_interval(total_frames, target_framecount) + logger.info( + f"Demuxing 2 fisheye streams ({streams[0]['width']}x{streams[0]['height']}) " + f"from {insv_path}, every {interval}th of {total_frames} frames" + ) + for lens_idx in (0, 1): + _extract_stream_frames(insv_path, lens_idx, lens_dirs[lens_idx], interval) + elif companion is not None: + # Two-file recording: each file holds one lens. Which file is which + # physical lens only flips the reconstruction's global yaw, so the + # assignment doesn't need to be exact. + paths = sorted([str(insv_path), str(companion)]) + total_frames = min(count_stream_frames(p, 0) or 0 for p in paths) + interval = _frame_interval(total_frames, target_framecount) + logger.info( + f"Demuxing two-file recording {paths[0]} + {paths[1]}, " + f"every {interval}th of {total_frames} frames" + ) + for lens_idx, path in enumerate(paths): + _extract_stream_frames(path, 0, lens_dirs[lens_idx], interval) + elif streams[0]["width"] == 2 * streams[0]["height"]: + # Single stream with both fisheyes side by side. Guard against + # equirectangular input, which has the same 2:1 aspect ratio. + _check_side_by_side_is_fisheye(insv_path) + total_frames = count_stream_frames(insv_path, 0) or 0 + interval = _frame_interval(total_frames, target_framecount) + logger.info( + f"Splitting side-by-side dual fisheye stream of {insv_path}, " + f"every {interval}th of {total_frames} frames" + ) + _extract_stream_frames(insv_path, 0, lens_dirs[0], interval, crop_filter="crop=iw/2:ih:0:0") + _extract_stream_frames(insv_path, 0, lens_dirs[1], interval, crop_filter="crop=iw/2:ih:iw/2:0") + else: + raise ValueError( + f"Unsupported .insv layout in {insv_path}: single " + f"{streams[0]['width']}x{streams[0]['height']} video stream and no " + f"companion _00_/_10_ file" + ) + + num_pairs = _trim_to_paired_frames(lens_dirs) + if num_pairs == 0: + raise ValueError(f"No frames could be extracted from {insv_path}") + logger.info(f"Extracted {num_pairs} paired fisheye frames to {output_dir}") + return lens_dirs + + +def _frame_interval(total_frames: int, target_framecount: int) -> int: + if total_frames <= 0: + raise ValueError("Could not determine the video frame count") + if target_framecount and target_framecount < total_frames: + return max(1, ceil(total_frames / target_framecount)) + return 1 + + +def _check_side_by_side_is_fisheye(insv_path): + """Extract one frame and verify both halves look like fisheye circles.""" + import tempfile + + import cv2 + + probe_dir = tempfile.mkdtemp(prefix="insv_probe_") + try: + probe_frame = os.path.join(probe_dir, "frame_%05d.png") + subprocess.run( + [ + "ffmpeg", "-loglevel", "error", + "-i", str(insv_path), + "-map", "0:v:0", "-frames:v", "1", + probe_frame, + ], + check=True, + ) + image = cv2.imread(os.path.join(probe_dir, "frame_00001.png")) + if image is None: + return # Can't verify; let the pipeline proceed and fail later if wrong. + half_width = image.shape[1] // 2 + halves = [image[:, :half_width], image[:, half_width:]] + if not all(looks_like_fisheye_frame(h) for h in halves): + raise ValueError( + f"{insv_path} has a single 2:1 video stream that does not look like " + "side-by-side fisheye footage (no dark image-circle borders). If this " + "is an equirectangular video, use --equirectangular instead." + ) + finally: + shutil.rmtree(probe_dir, ignore_errors=True) diff --git a/vid2scene_core/tests/conftest.py b/vid2scene_core/tests/conftest.py new file mode 100644 index 0000000..5ffcfe1 --- /dev/null +++ b/vid2scene_core/tests/conftest.py @@ -0,0 +1,5 @@ +import os +import sys + +# vid2scene_core modules are flat scripts imported by name (e.g. `import pano_sfm`) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) diff --git a/vid2scene_core/tests/test_fisheye_projection.py b/vid2scene_core/tests/test_fisheye_projection.py new file mode 100644 index 0000000..08b902e --- /dev/null +++ b/vid2scene_core/tests/test_fisheye_projection.py @@ -0,0 +1,164 @@ +import cv2 +import numpy as np +import pytest + +from fisheye_projection import ( + FisheyeLensModel, + build_remap_grid, + closest_view_partition, + derive_crop_size, + get_lens_local_rotations, + pinhole_camera_rays, + pinhole_focal, +) + + +def make_model(**kwargs) -> FisheyeLensModel: + defaults = dict(width=400, height=400, fov_deg=200.0) + defaults.update(kwargs) + return FisheyeLensModel(**defaults) + + +class TestFisheyeLensModel: + def test_center_ray_projects_to_principal_point(self): + model = make_model() + uv, valid = model.project_rays(np.array([[0.0, 0.0, 1.0]])) + assert valid[0] + np.testing.assert_allclose(uv[0], [200.0, 200.0], atol=1e-9) + + def test_90deg_off_axis_radius(self): + # Equidistant model: r = focal * theta, so a 90 degree ray lands at + # radius (90 / 100) * image_circle_radius for a 200 degree lens. + model = make_model() + uv, valid = model.project_rays(np.array([[1.0, 0.0, 0.0]])) + assert valid[0] + expected_radius = 200.0 * (90.0 / 100.0) + np.testing.assert_allclose(uv[0], [200.0 + expected_radius, 200.0], atol=1e-9) + + def test_rays_beyond_fov_are_invalid(self): + model = make_model(fov_deg=190.0) + # 100 degrees off-axis is outside a 190 degree (95 half-angle) lens + theta = np.deg2rad(100.0) + ray = np.array([[np.sin(theta), 0.0, np.cos(theta)]]) + _, valid = model.project_rays(ray) + assert not valid[0] + + def test_backward_ray_is_invalid(self): + model = make_model() + _, valid = model.project_rays(np.array([[0.0, 0.0, -1.0]])) + assert not valid[0] + + @pytest.mark.parametrize( + "distortion", + [dict(), dict(k1=-0.02, k2=0.004, k3=-0.0008, k4=0.0001)], + ) + def test_project_unproject_roundtrip(self, distortion): + model = make_model(**distortion) + rng = np.random.default_rng(0) + # Random rays within the lens FOV + theta = rng.uniform(0.0, np.deg2rad(95.0), size=500) + phi = rng.uniform(-np.pi, np.pi, size=500) + rays = np.stack( + [np.sin(theta) * np.cos(phi), np.sin(theta) * np.sin(phi), np.cos(theta)], + axis=-1, + ) + uv, valid = model.project_rays(rays) + assert valid.all() + rays_back = model.unproject_points(uv) + np.testing.assert_allclose(rays_back, rays, atol=1e-8) + + def test_defaults_resolve_from_frame_size(self): + model = FisheyeLensModel(width=2880, height=2880, fov_deg=200.0) + assert model.cx == 1440.0 + assert model.cy == 1440.0 + assert model.image_circle_radius == 1440.0 + np.testing.assert_allclose( + model.focal_px_per_rad, 1440.0 / np.deg2rad(100.0) + ) + + +class TestPinholeRays: + def test_center_pixel_points_forward(self): + rays = pinhole_camera_rays(100, 75.0).reshape(100, 100, 3) + center = (rays[49, 49] + rays[50, 50] + rays[49, 50] + rays[50, 49]) / 4 + center /= np.linalg.norm(center) + np.testing.assert_allclose(center, [0.0, 0.0, 1.0], atol=1e-9) + + def test_horizontal_fov_matches(self): + size, fov = 200, 75.0 + rays = pinhole_camera_rays(size, fov).reshape(size, size, 3) + left = rays[size // 2, 0] + right = rays[size // 2, -1] + angle = np.rad2deg(np.arccos(np.clip(np.dot(left, right), -1, 1))) + # Edge pixel centers sit half a pixel inside the exact FOV + assert abs(angle - fov) < 1.0 + + def test_pinhole_focal(self): + np.testing.assert_allclose(pinhole_focal(100, 90.0), 50.0) + + +class TestViewGrid: + def test_default_grid_covers_nadir_and_zenith(self): + # The headline fix over the equirectangular path: down-pitched views + # must actually contain the nadir (straight down, +y in the y-down + # camera convention). + rotations = get_lens_local_rotations((-60.0, 0.0, 60.0), (-60.0, 0.0, 60.0)) + axes = np.stack([r[2, :] for r in rotations]) # view axes in lens frame + crop_half_fov = 75.0 / 2 + for pole in ([0.0, 1.0, 0.0], [0.0, -1.0, 0.0]): + angles = np.rad2deg(np.arccos(np.clip(axes @ pole, -1, 1))) + assert angles.min() < crop_half_fov - 5.0 + # The nadir is 90 degrees off the lens axis: inside a 200 degree lens + model = make_model() + _, valid = model.project_rays(np.array([[0.0, 1.0, 0.0]])) + assert valid[0] + + def test_identity_rotation_looks_along_axis(self): + rotations = get_lens_local_rotations((0.0,), (0.0,)) + np.testing.assert_allclose(rotations[0], np.eye(3), atol=1e-12) + + def test_negative_pitch_looks_down(self): + rotations = get_lens_local_rotations((0.0,), (-60.0,)) + axis = rotations[0][2, :] + assert axis[1] > 0.5 # +y is down + + +class TestRemapGrid: + def test_center_view_fully_valid_and_samples_circle(self): + model = make_model() + # Green inside the image circle, black outside + fisheye = np.zeros((400, 400, 3), dtype=np.uint8) + cv2.circle(fisheye, (200, 200), 200, (0, 255, 0), -1) + + map_x, map_y, valid = build_remap_grid(model, np.eye(3), 75.0, 128) + assert valid.all() + crop = cv2.remap(fisheye, map_x, map_y, cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT) + assert (crop[:, :, 1] == 255).all() + + def test_view_past_lens_fov_is_partially_invalid_and_black(self): + model = make_model() + fisheye = np.full((400, 400, 3), 255, dtype=np.uint8) + # Axis 90 degrees off: bottom of a 75 degree crop reaches ~127 degrees, + # well past the 100 degree half-FOV of the lens. + rotation = get_lens_local_rotations((0.0,), (-90.0,))[0] + map_x, map_y, valid = build_remap_grid(model, rotation, 75.0, 128) + assert valid.any() and not valid.all() + crop = cv2.remap(fisheye, map_x, map_y, cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT) + assert (crop[~valid] == 0).all() + + def test_derive_crop_size(self): + model = make_model(width=2880, height=2880) + size = derive_crop_size(model, 75.0, max_crop_size=1600) + # focal ~825 px/rad => ~1266 px for a 75 degree crop + assert 1200 < size < 1350 + assert size % 2 == 0 + assert derive_crop_size(model, 75.0, max_crop_size=1024) == 1024 + + +class TestPartition: + def test_rays_assigned_to_most_aligned_axis(self): + axes = np.array([[0.0, 0.0, 1.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0]]) + rays = np.array( + [[0.1, 0.0, 0.9], [0.9, 0.1, 0.1], [0.1, 0.9, 0.0]] + ) + np.testing.assert_array_equal(closest_view_partition(rays, axes), [0, 1, 2]) diff --git a/vid2scene_core/tests/test_insv_extract.py b/vid2scene_core/tests/test_insv_extract.py new file mode 100644 index 0000000..541e1a6 --- /dev/null +++ b/vid2scene_core/tests/test_insv_extract.py @@ -0,0 +1,118 @@ +import os +import struct + +import numpy as np + +import insv_extract +from insv_extract import ( + INSV_TRAILER_MAGIC, + find_companion_lens_file, + looks_like_fisheye_frame, + read_insv_trailer_records, + summarize_insv_metadata, + _frame_interval, + _trim_to_paired_frames, +) + + +def build_insv_trailer(records: list[tuple[int, bytes]]) -> bytes: + """Build a synthetic trailer per the reverse-engineered layout. + + Records are laid out first-to-last; each record is its payload followed by + a (uint16 id, uint32 size) descriptor. The final record's descriptor is + the start of the 78-byte footer that ends with the 32-byte magic. + """ + blob = b"" + for record_id, payload in records[:-1]: + blob += payload + struct.pack(" Date: Thu, 11 Jun 2026 16:32:25 -0700 Subject: [PATCH 3/6] Parse Insta360 factory lens calibration (MEI model) from .insv recordings Replace the idealized-lens assumption with Insta360's per-unit factory calibration whenever it can be read from the recording itself, removing the main registration risk flagged in the original PR. Every Insta360 camera embeds an MEI (unified omnidirectional) camera model per lens: mirror parameter xi, fx/fy/cx/cy, radial k1..k4, tangential p1/p2 and thin-prism s1..s4 distortion, plus sub-degree per-lens mounting corrections. Two sources are read, in order of preference: - the .insv.pb sidecar (X5; 27 fields per lens incl. k4 and thin-prism terms), layout validated by insv-stitch against in-camera stitching - the trailer metadata record's offset_v3 string (20 fields per lens), layout per telemetry-parser's prost definitions Implementation: - insv_calibration.py: minimal protobuf wire-format walker (no protobuf dependency), offset_v3 / .pb sidecar parsers, and reference-resolution to stream-resolution conversion (window_crop_info-aware, centered aspect-fit fallback). Best-effort throughout: any failure falls back to the idealized model. - fisheye_projection.MeiLensModel: the MEI forward projection with the same project_rays interface as the equidistant model; the FOV cone bounds validity since the projection itself accepts nearly all directions for xi >= 1. get_lens_from_rig_rotations moved here from fisheye_sfm and extended with per-lens mounting corrections (keeps tests pycolmap-free). - fisheye_sfm.py: precedence explicit JSON > factory > idealized; factory mounting corrections feed the rig config. - --insv_no_factory_calibration / --no_factory_calibration escape hatch in vid2scene.py and fisheye_sfm.py. - 22 new unit tests (51 total): wire decoding, both calibration string layouts, full-frame cx normalization, X4/X5 scaling cases, MEI projection math, rig corrections. Pending real-footage validation (documented in docs/insv_fisheye.md): reference scaling on models other than X4/X5, and mounting-correction sign conventions (sub-degree, so low risk either way). https://claude.ai/code/session_01MdiAmjGY3SEAQLHsxKBVac Co-Authored-By: Claude Fable 5 --- docs/insv_fisheye.md | 73 +++- vid2scene_core/fisheye_projection.py | 121 ++++++ vid2scene_core/fisheye_sfm.py | 103 +++-- vid2scene_core/insv_calibration.py | 393 ++++++++++++++++++ .../tests/test_fisheye_projection.py | 78 ++++ vid2scene_core/tests/test_insv_calibration.py | 284 +++++++++++++ vid2scene_core/vid2scene.py | 17 +- 7 files changed, 1020 insertions(+), 49 deletions(-) create mode 100644 vid2scene_core/insv_calibration.py create mode 100644 vid2scene_core/tests/test_insv_calibration.py diff --git a/docs/insv_fisheye.md b/docs/insv_fisheye.md index 9adce22..95b30d5 100644 --- a/docs/insv_fisheye.md +++ b/docs/insv_fisheye.md @@ -56,12 +56,52 @@ Supported `.insv` layouts (demuxed with ffmpeg): ## Lens model and calibration -By default each lens is modeled as an idealized equidistant fisheye: -principal point at the frame center, image circle inscribed in the frame, -and a nominal 200° FOV mapped linearly onto the circle -(`fisheye_projection.FisheyeLensModel`). Real lenses deviate; if the -reconstruction shows systematic warp, supply a calibration JSON via -`--insv_calibration`: +Lens intrinsics are resolved in this order: + +1. **Explicit calibration JSON** (`--insv_calibration`, schema below) — + always wins when provided. +2. **Insta360's per-unit factory calibration**, parsed from the recording + itself (`insv_calibration.py`). Disable with + `--insv_no_factory_calibration`. +3. **Idealized equidistant fisheye**: principal point at the frame center, + image circle inscribed in the frame, and a nominal 200° FOV mapped + linearly onto the circle (`fisheye_projection.FisheyeLensModel`). + +### Factory calibration + +Every Insta360 camera is calibrated at the factory and embeds the result in +each recording as an MEI (unified omnidirectional) camera model — the same +model `cv2.omnidir` uses: unit-sphere projection with mirror parameter `xi`, +pinhole-like `fx/fy/cx/cy`, radial `k1..k4`, tangential `p1/p2` and +thin-prism `s1..s4` distortion (`fisheye_projection.MeiLensModel`). It is +read from two places, in order of preference: + +- the **`.insv.pb` sidecar** (X5; `MISC/Camera01/.insv.pb` on the SD + card, or next to the video) — 27 fields per lens including k4 and + thin-prism terms; +- the **trailer's `offset_v3` string** (protobuf metadata record of the + `.insv` itself) — 20 fields per lens. + +Field layouts follow [telemetry-parser](https://github.com/AdrianEddy/telemetry-parser) +(`src/insta360/extra_info.rs`, `mod.rs`) and +[insv-stitch](https://github.com/BenjaminHenriksson/insv-stitch), which +validated the `.pb` interpretation against in-camera stitching. The factory +values also provide per-lens mounting corrections (sub-degree yaw/pitch/roll +deviations from the nominal back-to-back geometry), which are applied to the +rig configuration. + +Calibration values are stored at a per-model reference resolution (5376 px +per lens on the X5, 8000×6000 on the X4) and are rescaled to the demuxed +stream resolution using the `window_crop_info` metadata when present, or a +centered aspect-fit otherwise. Pending validation on real footage: the +scaling rule for models other than X4/X5, and the sign convention of the +mounting corrections (sub-degree, so low risk). Parsing is best-effort — +any failure falls back to the idealized model with a log line. + +### Calibration JSON + +If the reconstruction shows systematic warp (or to override the factory +values), supply a calibration JSON via `--insv_calibration`: ```json { @@ -96,22 +136,25 @@ For every rendered view two masks are produced: ## Current limitations / follow-ups -- **No per-unit factory calibration.** Insta360 ships MEI-model calibration - (protobuf, in the trailer or an `.insv.pb` sidecar on X5); parsing it - would remove the idealized-lens assumption. The - [telemetry-parser](https://github.com/AdrianEddy/telemetry-parser) crate - (used by [insv-stitch](https://github.com/BenjaminHenriksson/insv-stitch)) - already reads these. +- **Factory calibration not yet validated on real footage.** The parsing is + exercised against synthetic data matching community-documented layouts; + real X4/X5 recordings should confirm the reference-resolution scaling and + the mounting-correction signs. `--insv_no_factory_calibration` provides + an immediate fallback for A/B comparison. - **No IMU use.** The trailer's gyro record (`0x300`) could provide horizon leveling and rolling-shutter correction. - **Lens baseline ignored.** The two lens centers are ~2–3 cm apart but the rig assumes a shared center, like the ER path (and Insta360's own - stitching). Only matters for subjects very close to the camera. + stitching). The factory calibration's per-lens translation is parsed but + not applied (observed as zeros in community dumps; using a metric value + would also pin the reconstruction scale, which needs deliberate handling). + Only matters for subjects very close to the camera. - **No ego-object masking.** The ER path can mask the operator/tripod with SAM3; that step is not yet wired into the fisheye path. - **Trailer parsing is best-effort.** Based on community reverse - engineering (ExifTool, Sub-Etha Software); used only for logging camera - metadata. Failures are silent and harmless. + engineering (ExifTool, Sub-Etha Software, telemetry-parser); used for + factory calibration and camera-metadata logging. Failures degrade to the + idealized lens model. - **Server/API wiring.** The web upload flow only exposes the `equirectangular` flag; `.insv` uploads through the server need a model field + form pass-through (auto-detection by extension already works at diff --git a/vid2scene_core/fisheye_projection.py b/vid2scene_core/fisheye_projection.py index b75a8c4..ce69b32 100644 --- a/vid2scene_core/fisheye_projection.py +++ b/vid2scene_core/fisheye_projection.py @@ -119,6 +119,98 @@ def unproject_points(self, uv: npt.NDArray[np.floating]) -> npt.NDArray[np.float return rays +@dataclass +class MeiLensModel: + """Unified omnidirectional (MEI/UCM) camera model with extended distortion. + + This is the model Insta360 uses for its per-unit factory calibration + (parsed from the .insv trailer / .pb sidecar by insv_calibration.py). A + ray is normalized onto the unit sphere, perspective-projected from a + point ``xi`` above the sphere center onto the normalized plane + ``m = (x, y) / (z + xi)``, distorted with radial (k1..k4 on r^2..r^8), + tangential (p1, p2) and thin-prism (s1..s4) terms, and scaled by the + pinhole-like intrinsics fx/fy/cx/cy. + + All parameters are expected in actual stream resolution and COLMAP pixel + convention (top-left pixel center at (0.5, 0.5)). ``fov_deg`` bounds the + valid cone: with xi >= 1 the projection itself accepts rays from (nearly) + every direction, so validity must come from the known lens coverage. + """ + + width: int + height: int + xi: float + fx: float + fy: float + cx: float + cy: float + k1: float = 0.0 + k2: float = 0.0 + k3: float = 0.0 + k4: float = 0.0 + p1: float = 0.0 + p2: float = 0.0 + s1: float = 0.0 + s2: float = 0.0 + s3: float = 0.0 + s4: float = 0.0 + fov_deg: float = 200.0 + + @property + def focal_px_per_rad(self) -> float: + """Angular resolution at the image center, for crop-size derivation. + + Near the optical axis r = fx * sin(theta) / (cos(theta) + xi) + ~= fx * theta / (1 + xi). + """ + return self.fx / (1.0 + self.xi) + + def project_rays( + self, rays: npt.NDArray[np.floating] + ) -> tuple[npt.NDArray[np.floating], npt.NDArray[np.bool_]]: + """Project rays (N, 3) in the lens frame to pixel coordinates. + + Same interface as FisheyeLensModel.project_rays: returns (uv, valid) + with uv in COLMAP pixel convention and valid marking rays inside the + lens FOV cone and the image bounds. + """ + rays = np.asarray(rays, dtype=np.float64) + norm = np.linalg.norm(rays, axis=-1) + norm = np.maximum(norm, 1e-12) + x, y, z = np.moveaxis(rays, -1, 0) / norm + + denom = z + self.xi + theta = np.arctan2(np.hypot(x, y), z) + valid = (denom > 1e-6) & (theta <= np.deg2rad(self.fov_deg / 2)) + + safe_denom = np.where(valid, denom, 1.0) + mx = x / safe_denom + my = y / safe_denom + + r2 = mx * mx + my * my + r4 = r2 * r2 + radial = 1.0 + self.k1 * r2 + self.k2 * r4 + self.k3 * r2 * r4 + self.k4 * r4 * r4 + xd = ( + mx * radial + + 2 * self.p1 * mx * my + + self.p2 * (r2 + 2 * mx * mx) + + self.s1 * r2 + + self.s2 * r4 + ) + yd = ( + my * radial + + self.p1 * (r2 + 2 * my * my) + + 2 * self.p2 * mx * my + + self.s3 * r2 + + self.s4 * r4 + ) + + u = self.fx * xd + self.cx + v = self.fy * yd + self.cy + valid &= (u >= 0.0) & (u <= self.width) & (v >= 0.0) & (v <= self.height) + return np.stack([u, v], axis=-1), valid + + @dataclass class FisheyeRigRenderOptions: """Virtual pinhole view layout rendered from each fisheye lens. @@ -177,6 +269,35 @@ def get_lens_local_rotations( return rotations +def get_lens_from_rig_rotations( + lens1_yaw_deg: float = 180.0, + lens1_roll_deg: float = 0.0, + mount_corrections: Sequence[tuple[float, float, float]] | None = None, +) -> list[npt.NDArray[np.floating]]: + """lens_from_rig rotations for the two back-to-back lenses. + + Lens 0 defines the rig frame; lens 1 nominally points the opposite way + (180 degree yaw). The optional roll accounts for sensors mounted rotated + relative to each other. mount_corrections are per-lens factory-measured + (yaw, pitch, roll) deviations in degrees, applied in each lens' local + frame on top of its nominal orientation (matching how insv-stitch + composes the Insta360 factory extrinsics). + """ + corrections = mount_corrections or ((0.0, 0.0, 0.0), (0.0, 0.0, 0.0)) + # Column-vector lens->rig corrections; transposed below into this file's + # row-vector convention (ray_in_rig = ray_in_lens @ lens_from_rig). + correction_cols = [ + Rotation.from_euler("YXZ", list(correction), degrees=True).as_matrix() + for correction in corrections + ] + lens1_base = Rotation.from_euler( + "ZY", [lens1_roll_deg, lens1_yaw_deg], degrees=True + ).as_matrix() + lens0_from_rig = correction_cols[0].T + lens1_from_rig = correction_cols[1].T @ lens1_base + return [lens0_from_rig, lens1_from_rig] + + def derive_crop_size(lens_model: FisheyeLensModel, crop_fov_deg: float, max_crop_size: int) -> int: """Crop size whose center angular resolution matches the fisheye source.""" size = lens_model.focal_px_per_rad * 2 * np.tan(np.deg2rad(crop_fov_deg) / 2) diff --git a/vid2scene_core/fisheye_sfm.py b/vid2scene_core/fisheye_sfm.py index eae477e..f420ef2 100644 --- a/vid2scene_core/fisheye_sfm.py +++ b/vid2scene_core/fisheye_sfm.py @@ -12,10 +12,11 @@ sensor pixels everywhere, and the default view grid points views down past the nadir, so the ground is covered by up to 6 views per frame pair instead of 0. -The lens model is an idealized equidistant fisheye by default (see -fisheye_projection.FisheyeLensModel); per-unit calibration can be supplied as -a JSON file. Factory calibration / IMU parsing from the .insv trailer is a -follow-up (see docs/insv_fisheye.md). +Lens intrinsics come from Insta360's per-unit factory calibration when it can +be parsed out of the recording (insv_calibration.py: the trailer's offset_v3 +or the .insv.pb sidecar, both MEI camera models). An explicit calibration +JSON (--insv_calibration) overrides it; with neither, an idealized +equidistant fisheye is assumed (see docs/insv_fisheye.md). """ import argparse @@ -33,19 +34,21 @@ import cv2 import numpy as np import numpy.typing as npt -from scipy.spatial.transform import Rotation from tqdm import tqdm import pycolmap +import insv_calibration import insv_extract import pano_sfm from fisheye_projection import ( FisheyeLensModel, FisheyeRigRenderOptions, + MeiLensModel, build_remap_grid, closest_view_partition, derive_crop_size, + get_lens_from_rig_rotations, get_lens_local_rotations, pinhole_camera_rays, pinhole_focal, @@ -56,22 +59,6 @@ INSV_RENDER_OPTIONS = FisheyeRigRenderOptions() -def get_lens_from_rig_rotations( - lens1_yaw_deg: float = 180.0, lens1_roll_deg: float = 0.0 -) -> list[npt.NDArray[np.floating]]: - """lens_from_rig rotations for the two back-to-back lenses. - - Lens 0 defines the rig frame; lens 1 nominally points the opposite way - (180 degree yaw). The optional roll accounts for sensors mounted rotated - relative to each other. - """ - lens0_from_rig = np.eye(3) - lens1_from_rig = Rotation.from_euler( - "ZY", [lens1_roll_deg, lens1_yaw_deg], degrees=True - ).as_matrix() - return [lens0_from_rig, lens1_from_rig] - - def create_fisheye_rig_config( image_prefixes: Sequence[str], cams_from_rig_rotation: Sequence[npt.NDArray[np.floating]], @@ -116,6 +103,7 @@ def __init__( lens_model_overrides: Sequence[dict] | None = None, lens1_yaw_deg: float = 180.0, lens1_roll_deg: float = 0.0, + factory_lenses: Sequence[dict] | None = None, ): self.lens_frame_dirs = [Path(d) for d in lens_frame_dirs] self.output_image_dir = Path(output_image_dir) @@ -123,15 +111,27 @@ def __init__( self.mask_dir = Path(mask_dir) if mask_dir else None self.training_mask_dir = Path(training_mask_dir) if training_mask_dir else None self.lens_model_overrides = list(lens_model_overrides or [{}, {}]) + self.factory_lenses = list(factory_lenses) if factory_lenses else None num_lenses = len(self.lens_frame_dirs) if num_lenses != 2: raise ValueError(f"Expected 2 lens directories, got {num_lenses}") + mount_corrections = None + if self.factory_lenses: + mount_corrections = insv_calibration.get_mount_corrections(self.factory_lenses) + logger.info( + "Factory lens mounting corrections (yaw, pitch, roll deg): " + + ", ".join(f"lens{i}=({y:.3f}, {p:.3f}, {r:.3f})" + for i, (y, p, r) in enumerate(mount_corrections)) + ) + self.cams_from_lens = get_lens_local_rotations( render_options.yaws_deg, render_options.pitches_deg ) - self.lens_from_rig = get_lens_from_rig_rotations(lens1_yaw_deg, lens1_roll_deg) + self.lens_from_rig = get_lens_from_rig_rotations( + lens1_yaw_deg, lens1_roll_deg, mount_corrections + ) # One virtual view per (lens, lens-local rotation) self.view_lens_idx: list[int] = [] @@ -152,7 +152,7 @@ def __init__( self._lock = Lock() self._initialized = False self._camera: pycolmap.Camera | None = None - self.lens_models: list[FisheyeLensModel] | None = None + self.lens_models: list[FisheyeLensModel | MeiLensModel] | None = None self.crop_size: int | None = None self._maps: list[tuple[np.ndarray, np.ndarray]] = [] self._sfm_mask_png: list[bytes] = [] @@ -163,11 +163,22 @@ def _initialize(self, lens_images: Sequence[np.ndarray]) -> None: self.lens_models = [] for lens_idx, image in enumerate(lens_images): height, width = image.shape[:2] - overrides = dict(self.lens_model_overrides[lens_idx]) - self.lens_models.append(FisheyeLensModel(width=width, height=height, **overrides)) + if self.factory_lenses: + kwargs = insv_calibration.mei_model_kwargs_for_stream( + self.factory_lenses[lens_idx], width, height + ) + model = MeiLensModel(width=width, height=height, **kwargs) + logger.info( + f"Lens {lens_idx}: factory MEI calibration, xi={model.xi:.3f}, " + f"f=({model.fx:.1f}, {model.fy:.1f}), c=({model.cx:.1f}, {model.cy:.1f})" + ) + else: + overrides = dict(self.lens_model_overrides[lens_idx]) + model = FisheyeLensModel(width=width, height=height, **overrides) + self.lens_models.append(model) logger.info( - f"Lens {lens_idx}: {width}x{height}, fov={self.lens_models[lens_idx].fov_deg} deg, " - f"focal={self.lens_models[lens_idx].focal_px_per_rad:.1f} px/rad" + f"Lens {lens_idx}: {width}x{height}, fov={model.fov_deg} deg, " + f"focal={model.focal_px_per_rad:.1f} px/rad" ) options = self.render_options @@ -337,6 +348,7 @@ def run_insv_sfm( render_options: FisheyeRigRenderOptions = INSV_RENDER_OPTIONS, lens_fov_deg: float | None = None, calibration_path=None, + use_factory_calibration: bool = True, kill_check=None, ) -> Path | None: """ @@ -355,7 +367,9 @@ def run_insv_sfm( generate_masks: Generate feature-extraction masks (validity + view partition) render_options: Virtual view layout configuration lens_fov_deg: Override the assumed lens FOV (default 200) - calibration_path: Optional lens calibration JSON + calibration_path: Optional lens calibration JSON (overrides factory) + use_factory_calibration: Use Insta360's per-unit factory calibration + when it can be parsed from the recording (default True) kill_check: Optional callback to check if processing should abort Returns: @@ -372,9 +386,8 @@ def run_insv_sfm( if training_mask_dir: training_mask_dir.mkdir(exist_ok=True, parents=True) - metadata = insv_extract.summarize_insv_metadata( - insv_extract.read_insv_trailer_records(insv_path) - ) + trailer_records = insv_extract.read_insv_trailer_records(insv_path) + metadata = insv_extract.summarize_insv_metadata(trailer_records) if metadata.get("record_ids"): logger.info(f".insv trailer records: {metadata['record_ids']}") if metadata.get("file_info_strings"): @@ -385,6 +398,27 @@ def run_insv_sfm( lens1_yaw_deg = (calibration or {}).get("lens1_yaw_deg", 180.0) lens1_roll_deg = (calibration or {}).get("lens1_roll_deg", 0.0) + factory_lenses = None + if calibration is not None: + logger.info("Explicit calibration JSON provided; factory calibration not used") + elif use_factory_calibration: + factory = insv_calibration.load_factory_calibration(insv_path, trailer_records) + if factory: + factory_lenses = factory["lenses"] + if lens_fov_deg is not None: + for lens_spec in factory_lenses: + lens_spec["fov_deg"] = lens_fov_deg + camera = factory.get("camera_type") or "unknown camera" + logger.info( + f"Using Insta360 factory lens calibration for {camera} " + f"from {factory['source']}" + ) + else: + logger.info( + "No factory lens calibration found in the recording; " + "using the idealized lens model (see docs/insv_fisheye.md)" + ) + if kill_check and kill_check(): logger.info("Job cancelled before .insv extraction") return None @@ -417,6 +451,7 @@ def run_insv_sfm( lens_model_overrides=lens_model_overrides, lens1_yaw_deg=lens1_yaw_deg, lens1_roll_deg=lens1_roll_deg, + factory_lenses=factory_lenses, ) render_fisheye_views(processor, frame_names) finally: @@ -452,7 +487,10 @@ def run_insv_sfm( parser.add_argument("--lens_fov", type=float, default=None, help="Assumed full lens FOV in degrees (default 200)") parser.add_argument("--calibration", type=Path, default=None, - help="Optional lens calibration JSON") + help="Optional lens calibration JSON (overrides factory calibration)") + parser.add_argument("--no_factory_calibration", action="store_true", + help="Ignore Insta360's factory lens calibration and use the " + "idealized lens model") args = parser.parse_args() @@ -463,6 +501,7 @@ def run_insv_sfm( mapper=args.mapper, lens_fov_deg=args.lens_fov, calibration_path=args.calibration, + use_factory_calibration=not args.no_factory_calibration, ) if result: diff --git a/vid2scene_core/insv_calibration.py b/vid2scene_core/insv_calibration.py new file mode 100644 index 0000000..a9b7aea --- /dev/null +++ b/vid2scene_core/insv_calibration.py @@ -0,0 +1,393 @@ +""" +Insta360 factory lens calibration parsing for .insv recordings. + +Every Insta360 camera is calibrated per-unit at the factory and the result is +embedded in the recording itself, in two places: + +- The metadata record of the .insv trailer (record 0x101, a protobuf message) + holds ``offset_v3``: an underscore-separated float string with one MEI + (unified omnidirectional) lens block per lens. Layout per the prost + definitions in AdrianEddy/telemetry-parser (src/insta360/extra_info.rs) and + the field comment in src/insta360/mod.rs: + ``num, then per lens: xi fx fy cx cy yaw pitch roll tx ty tz + k1 k2 k3 p1 p2 width height lensType flag``. +- Newer cameras (X5) additionally ship a ``.insv.pb`` sidecar (under + MISC/Camera01 on the SD card) whose base64 payload holds an extended + calibration with 27 fields per lens, adding k4, thin-prism (s1..s4) and + sensor-tilt terms. Layout reverse engineered and validated against + in-camera stitching by BenjaminHenriksson/insv-stitch. + +Calibration values are stored at a per-model reference resolution (e.g. +5376px per lens on the X5, 8000x6000 on the X4) while the demuxed video +streams are smaller center crops (window_crop_info in the same protobuf gives +the crop when present); this module converts them to actual stream resolution. + +Everything here is best effort: any parse failure returns None and the +caller falls back to the idealized lens model. Pure stdlib (a ~40-line +protobuf wire-format walker avoids a protobuf dependency for nine fields). + +Caveats, pending validation on real footage (see docs/insv_fisheye.md): +- The yaw/pitch mounting-correction sign convention is taken from + insv-stitch; values are fractions of a degree, so a wrong sign is a small + systematic error rather than a failure. +- Per-lens field 7 of the .pb block is ambiguous in community sources + (roll vs half-FOV; observed ~89.8) and is not used. +- The lens translation (tx, ty, tz; the ~2-3cm baseline) is parsed but not + applied, matching the zero-baseline rig assumption of the ER path. +""" + +import base64 +import logging +import re +import struct +from pathlib import Path + +logger = logging.getLogger(__name__) + +# Record key in insv_extract.read_insv_trailer_records: (format | id << 8) +# with format=1 (protobuf) and id=1 (metadata). +METADATA_RECORD_KEY = 0x101 + +_METADATA_STRING_FIELDS = { + 1: "serial_number", + 2: "camera_type", + 3: "fw_version", + 5: "offset", + 17: "original_offset", + 53: "offset_v2", + 54: "offset_v3", + 55: "original_offset_v2", + 56: "original_offset_v3", +} +_WINDOW_CROP_INFO_FIELD = 27 # submessage: src_width, src_height, dst_width, dst_height + +_OFFSET_V3_LENS_FIELDS = ( + "xi", "fx", "fy", "cx", "cy", + "yaw_deg", "pitch_deg", "roll_deg", + "tx", "ty", "tz", + "k1", "k2", "k3", "p1", "p2", + "ref_width", "ref_height", "lens_type", "flag", +) + +_PB_LENS_BLOCK_SIZE = 27 + + +def _read_varint(data: bytes, pos: int) -> tuple[int, int]: + result = 0 + shift = 0 + while True: + if pos >= len(data) or shift > 63: + raise ValueError("Truncated or oversized varint") + byte = data[pos] + result |= (byte & 0x7F) << shift + pos += 1 + if not byte & 0x80: + return result, pos + shift += 7 + + +def iter_protobuf_fields(data: bytes): + """Yield (field_number, wire_type, raw_value) for a protobuf message. + + Varints yield ints; length-delimited fields yield bytes; fixed32/fixed64 + yield their raw bytes. Raises ValueError on malformed input. + """ + pos = 0 + while pos < len(data): + key, pos = _read_varint(data, pos) + field_number, wire_type = key >> 3, key & 7 + if wire_type == 0: + value, pos = _read_varint(data, pos) + elif wire_type == 1: + value, pos = data[pos:pos + 8], pos + 8 + elif wire_type == 2: + length, pos = _read_varint(data, pos) + if pos + length > len(data): + raise ValueError("Truncated length-delimited field") + value, pos = data[pos:pos + length], pos + length + elif wire_type == 5: + value, pos = data[pos:pos + 4], pos + 4 + else: + raise ValueError(f"Unsupported wire type {wire_type}") + if pos > len(data): + raise ValueError("Truncated field payload") + yield field_number, wire_type, value + + +def parse_metadata_record(payload: bytes) -> dict: + """Decode the interesting fields of the trailer's protobuf metadata record. + + Returns {} if the payload doesn't parse as a protobuf message. + """ + info: dict = {} + try: + for field_number, wire_type, value in iter_protobuf_fields(payload): + name = _METADATA_STRING_FIELDS.get(field_number) + if name is not None and wire_type == 2: + try: + info[name] = value.decode("utf-8") + except UnicodeDecodeError: + continue + elif field_number == _WINDOW_CROP_INFO_FIELD and wire_type == 2: + crop = [v for n, w, v in iter_protobuf_fields(value) if w == 0] + if len(crop) >= 4: + info["window_crop"] = { + "src_width": crop[0], "src_height": crop[1], + "dst_width": crop[2], "dst_height": crop[3], + } + except ValueError as e: + logger.debug(f"Unparseable .insv metadata record: {e}") + return {} + return info + + +def _parse_float_list(text: str) -> list[float] | None: + try: + return [float(token) for token in text.split("_")] + except (ValueError, AttributeError): + return None + + +def parse_offset_v3(text: str) -> list[dict] | None: + """Parse the trailer's offset_v3 string into per-lens MEI lens specs.""" + values = _parse_float_list(text) + if not values: + return None + num_lenses = int(values[0]) + block = len(_OFFSET_V3_LENS_FIELDS) + if num_lenses < 1 or len(values) < 1 + num_lenses * block: + return None + lenses = [] + for lens_idx in range(num_lenses): + fields = values[1 + lens_idx * block: 1 + (lens_idx + 1) * block] + lenses.append(dict(zip(_OFFSET_V3_LENS_FIELDS, fields))) + return lenses + + +def parse_pb_calibration(data: bytes) -> list[dict] | None: + """Parse the extended calibration out of an .insv.pb sidecar file. + + The sidecar embeds a base64 blob containing an underscore-separated + calibration string of 1 + 27 * num_lenses values. Field layout within + each lens block (insv-stitch, validated against in-camera stitching): + 0=xi 1=fx 2=fy 3=cx 4=cy 5=yaw 6=pitch 7=(ambiguous, unused) + 8-10=tx,ty,tz 11-14=k1..k4 15=zero 16=p1 17=p2 18-21=s1..s4 + 22-23=tauX,tauY 24=ref_width(full dual frame) 25=ref_height 26=type + """ + text = data.decode("latin-1") + for blob_match in re.finditer(r"[A-Za-z0-9+/=]{100,}", text): + blob = blob_match.group() + # Surrounding bytes that happen to be base64-alphabet characters can + # get glued onto the blob and shift its framing; try all four + # alignments, truncating the tail to a decodable length. + for start in range(4): + candidate = blob[start:] + candidate = candidate[: len(candidate) - len(candidate) % 4] + try: + decoded = base64.b64decode(candidate).decode("latin-1") + except (ValueError, UnicodeDecodeError): + continue + lenses = _find_pb_calibration_string(decoded) + if lenses: + return lenses + return None + + +def _find_pb_calibration_string(decoded: str) -> list[dict] | None: + for match in re.finditer(r"2_\d+\.\d+_", decoded): + values = [] + for token in decoded[match.start():].split("_"): + # The last number can have non-numeric bytes glued straight onto + # it, so accept a numeric prefix and stop there. + numeric = re.match(r"[-+]?\d+(?:\.\d+)?", token) + if not numeric: + break + values.append(float(numeric.group())) + if numeric.end() != len(token): + break + num_lenses = int(values[0]) if values else 0 + if num_lenses < 1 or len(values) < 1 + num_lenses * _PB_LENS_BLOCK_SIZE: + continue + lenses = [] + for lens_idx in range(num_lenses): + b = values[1 + lens_idx * _PB_LENS_BLOCK_SIZE:] + lenses.append({ + "xi": b[0], "fx": b[1], "fy": b[2], "cx": b[3], "cy": b[4], + "yaw_deg": b[5], "pitch_deg": b[6], "roll_deg": 0.0, + "tx": b[8], "ty": b[9], "tz": b[10], + "k1": b[11], "k2": b[12], "k3": b[13], "k4": b[14], + "p1": b[16], "p2": b[17], + "s1": b[18], "s2": b[19], "s3": b[20], "s4": b[21], + "ref_width": b[24], "ref_height": b[25], "lens_type": b[26], + }) + return lenses + return None + + +def find_pb_sidecar(insv_path) -> Path | None: + """Locate the .pb calibration sidecar of an .insv recording. + + On the SD card it lives in MISC/Camera01/.insv.pb beside + DCIM/Camera01/.insv; users copying files often place it next to + the video instead, so both locations are checked. + """ + insv_path = Path(insv_path) + candidates = [ + insv_path.with_name(insv_path.name + ".pb"), + insv_path.parent.parent.parent / "MISC" / "Camera01" / (insv_path.name + ".pb"), + ] + for candidate in candidates: + try: + if candidate.exists(): + return candidate + except OSError: + continue + return None + + +def _normalize_lens_spec(spec: dict, lens_idx: int) -> dict: + """Normalize a per-lens spec to a single-lens reference frame. + + Reference dims may describe the full side-by-side dual-fisheye frame + (e.g. 16000x6000 on the X4, 10752x5376 on the X5) and cx may be stored + in those full-frame coordinates; reduce both to one lens. + """ + spec = dict(spec) + ref_width = spec.get("ref_width", 0.0) + ref_height = spec.get("ref_height", 0.0) + if ref_width <= 0 or ref_height <= 0: + return spec + if ref_width >= 2 * ref_height: + ref_width /= 2 + spec["ref_width"] = ref_width + if spec["cx"] >= ref_width: + offset = ref_width * (spec["cx"] // ref_width) + logger.info( + f"Lens {lens_idx}: principal point cx={spec['cx']:.1f} is in " + f"full-frame coordinates, shifting by -{offset:.0f}" + ) + spec["cx"] -= offset + return spec + + +def load_factory_calibration(insv_path, trailer_records: dict[int, bytes]) -> dict | None: + """Load the factory lens calibration of an .insv recording, best effort. + + Prefers the .pb sidecar (more distortion terms) over the trailer's + offset_v3. Returns {"source", "camera_type", "lenses": [spec, spec]} + where each spec is in single-lens reference resolution (see + mei_model_kwargs_for_stream), or None when no calibration is found. + """ + metadata = parse_metadata_record(trailer_records.get(METADATA_RECORD_KEY, b"")) + + lenses = None + source = None + pb_path = find_pb_sidecar(insv_path) + if pb_path is not None: + try: + lenses = parse_pb_calibration(pb_path.read_bytes()) + except OSError as e: + logger.warning(f"Could not read calibration sidecar {pb_path}: {e}") + if lenses: + source = str(pb_path) + else: + logger.warning(f"No calibration block found in sidecar {pb_path}") + + if not lenses and metadata.get("offset_v3"): + lenses = parse_offset_v3(metadata["offset_v3"]) + if lenses: + source = ".insv trailer offset_v3" + + if not lenses: + return None + if len(lenses) != 2: + logger.warning( + f"Factory calibration describes {len(lenses)} lenses, expected 2; ignoring" + ) + return None + + window_crop = metadata.get("window_crop") + normalized = [] + for lens_idx, spec in enumerate(lenses): + spec = _normalize_lens_spec(spec, lens_idx) + if window_crop: + spec["crop_width"] = window_crop["dst_width"] + spec["crop_height"] = window_crop["dst_height"] + normalized.append(spec) + + return { + "source": source, + "camera_type": metadata.get("camera_type"), + "lenses": normalized, + } + + +def mei_model_kwargs_for_stream(spec: dict, stream_width: int, stream_height: int) -> dict: + """Convert a reference-resolution lens spec to MeiLensModel kwargs. + + The demuxed stream is assumed to be a centered crop of the reference + frame, uniformly scaled. The crop region comes from window_crop_info when + present (e.g. 5312px of the X5's 5376px reference) and otherwise from + aspect-fitting the stream into the reference frame (e.g. 6000x6000 out + of the X4's 8000x6000). 0.5 converts the calibration's OpenCV pixel + origin to the COLMAP convention used by the render pipeline. + """ + ref_width = spec.get("ref_width", 0.0) or stream_width + ref_height = spec.get("ref_height", 0.0) or stream_height + + crop_width = spec.get("crop_width", 0.0) + crop_height = spec.get("crop_height", 0.0) + crop_usable = ( + 0 < crop_width <= ref_width + and 0 < crop_height <= ref_height + and abs(crop_width / crop_height - stream_width / stream_height) < 0.02 + ) + if not crop_usable: + fit_scale = max(stream_width / ref_width, stream_height / ref_height) + crop_width = stream_width / fit_scale + crop_height = stream_height / fit_scale + + crop_x = (ref_width - crop_width) / 2 + crop_y = (ref_height - crop_height) / 2 + scale_x = stream_width / crop_width + scale_y = stream_height / crop_height + + kwargs = { + "xi": spec["xi"], + "fx": spec["fx"] * scale_x, + "fy": spec["fy"] * scale_y, + "cx": (spec["cx"] - crop_x) * scale_x + 0.5, + "cy": (spec["cy"] - crop_y) * scale_y + 0.5, + } + for key in ("k1", "k2", "k3", "k4", "p1", "p2", "s1", "s2", "s3", "s4"): + kwargs[key] = spec.get(key, 0.0) + if "fov_deg" in spec: + kwargs["fov_deg"] = spec["fov_deg"] + + center_offset = max( + abs(kwargs["cx"] - stream_width / 2) / stream_width, + abs(kwargs["cy"] - stream_height / 2) / stream_height, + ) + if center_offset > 0.1: + logger.warning( + f"Factory principal point ({kwargs['cx']:.1f}, {kwargs['cy']:.1f}) is " + f"far from the {stream_width}x{stream_height} stream center; the " + "reference-to-stream scaling may be wrong for this camera model" + ) + return kwargs + + +def get_mount_corrections(lenses: list[dict]) -> list[tuple[float, float, float]]: + """Per-lens (yaw, pitch, roll) mounting corrections in degrees. + + These are the factory-measured sub-degree deviations of each lens from + its nominal orientation (lens 1 nominally 180 deg yaw from lens 0). + """ + return [ + ( + spec.get("yaw_deg", 0.0), + spec.get("pitch_deg", 0.0), + spec.get("roll_deg", 0.0), + ) + for spec in lenses + ] diff --git a/vid2scene_core/tests/test_fisheye_projection.py b/vid2scene_core/tests/test_fisheye_projection.py index 08b902e..82e5c24 100644 --- a/vid2scene_core/tests/test_fisheye_projection.py +++ b/vid2scene_core/tests/test_fisheye_projection.py @@ -4,6 +4,7 @@ from fisheye_projection import ( FisheyeLensModel, + MeiLensModel, build_remap_grid, closest_view_partition, derive_crop_size, @@ -77,6 +78,83 @@ def test_defaults_resolve_from_frame_size(self): ) +def make_mei_model(**kwargs) -> MeiLensModel: + defaults = dict( + width=3840, height=3840, xi=2.0, + fx=3087.0, fy=3087.0, cx=1920.0, cy=1920.0, + ) + defaults.update(kwargs) + return MeiLensModel(**defaults) + + +class TestMeiLensModel: + def test_center_ray_projects_to_principal_point(self): + model = make_mei_model() + uv, valid = model.project_rays(np.array([[0.0, 0.0, 1.0]])) + assert valid[0] + np.testing.assert_allclose(uv[0], [1920.0, 1920.0], atol=1e-9) + + def test_undistorted_radius_matches_unified_sphere_formula(self): + # r = fx * sin(theta) / (cos(theta) + xi) without distortion terms + model = make_mei_model() + for theta_deg in (10.0, 45.0, 90.0): + theta = np.deg2rad(theta_deg) + uv, valid = model.project_rays( + np.array([[np.sin(theta), 0.0, np.cos(theta)]]) + ) + assert valid[0] + expected = model.fx * np.sin(theta) / (np.cos(theta) + model.xi) + np.testing.assert_allclose(uv[0], [model.cx + expected, model.cy], atol=1e-9) + + def test_focal_px_per_rad_matches_small_angle_slope(self): + model = make_mei_model() + theta = 1e-6 + uv, _ = model.project_rays(np.array([[np.sin(theta), 0.0, np.cos(theta)]])) + slope = (uv[0, 0] - model.cx) / theta + np.testing.assert_allclose(model.focal_px_per_rad, slope, rtol=1e-6) + np.testing.assert_allclose(model.focal_px_per_rad, model.fx / (1.0 + model.xi)) + + def test_radial_distortion_is_applied(self): + theta = np.deg2rad(60.0) + ray = np.array([[np.sin(theta), 0.0, np.cos(theta)]]) + plain, _ = make_mei_model().project_rays(ray) + distorted, _ = make_mei_model(k1=0.22).project_rays(ray) + m = np.sin(theta) / (np.cos(theta) + 2.0) + expected_shift = 3087.0 * 0.22 * m**3 + np.testing.assert_allclose( + distorted[0, 0] - plain[0, 0], expected_shift, rtol=1e-9 + ) + + def test_rays_beyond_fov_are_invalid(self): + # With xi >= 1 the projection accepts almost any direction, so the + # FOV cone must bound validity. + model = make_mei_model(fov_deg=200.0) + theta = np.deg2rad(105.0) + _, valid = model.project_rays(np.array([[np.sin(theta), 0.0, np.cos(theta)]])) + assert not valid[0] + theta = np.deg2rad(95.0) + _, valid = model.project_rays(np.array([[np.sin(theta), 0.0, np.cos(theta)]])) + assert valid[0] + + def test_unnormalized_rays_project_identically(self): + model = make_mei_model() + ray = np.array([[0.3, -0.2, 0.9]]) + uv1, _ = model.project_rays(ray) + uv2, _ = model.project_rays(ray * 7.5) + np.testing.assert_allclose(uv1, uv2, atol=1e-9) + + def test_works_with_remap_grid_and_crop_size(self): + model = make_mei_model() + size = derive_crop_size(model, 75.0, max_crop_size=1600) + assert size % 2 == 0 and size >= 256 + map_x, map_y, valid = build_remap_grid(model, np.eye(3), 75.0, 64) + assert valid.all() + center = np.array( + [map_x[31:33, 31:33].mean(), map_y[31:33, 31:33].mean()] + ) + np.testing.assert_allclose(center, [model.cx - 0.5, model.cy - 0.5], atol=1.0) + + class TestPinholeRays: def test_center_pixel_points_forward(self): rays = pinhole_camera_rays(100, 75.0).reshape(100, 100, 3) diff --git a/vid2scene_core/tests/test_insv_calibration.py b/vid2scene_core/tests/test_insv_calibration.py new file mode 100644 index 0000000..4a8b5ba --- /dev/null +++ b/vid2scene_core/tests/test_insv_calibration.py @@ -0,0 +1,284 @@ +import base64 + +import numpy as np +import pytest + +import insv_calibration +from insv_calibration import ( + METADATA_RECORD_KEY, + load_factory_calibration, + mei_model_kwargs_for_stream, + parse_metadata_record, + parse_offset_v3, + parse_pb_calibration, +) + + +# --------------------------------------------------------------------------- +# Protobuf encoding helpers (test-side mirror of the wire format) +# --------------------------------------------------------------------------- + +def _varint(value: int) -> bytes: + out = bytearray() + while True: + byte = value & 0x7F + value >>= 7 + if value: + out.append(byte | 0x80) + else: + out.append(byte) + return bytes(out) + + +def _string_field(number: int, text: str) -> bytes: + payload = text.encode("utf-8") + return _varint((number << 3) | 2) + _varint(len(payload)) + payload + + +def _varint_field(number: int, value: int) -> bytes: + return _varint((number << 3) | 0) + _varint(value) + + +def _bytes_field(number: int, payload: bytes) -> bytes: + return _varint((number << 3) | 2) + _varint(len(payload)) + payload + + +def _fixed64_field(number: int, payload: bytes) -> bytes: + return _varint((number << 3) | 1) + payload + + +def make_window_crop(src_w, src_h, dst_w, dst_h) -> bytes: + return b"".join( + _varint_field(i + 1, v) for i, v in enumerate((src_w, src_h, dst_w, dst_h)) + ) + + +# X4-like offset_v3: 1 count + 2 lenses x 20 fields +# (xi fx fy cx cy yaw pitch roll tx ty tz k1 k2 k3 p1 p2 w h type flag) +OFFSET_V3_LENS0 = [1.9, 2870.0, 2871.0, 3987.5, 3012.7, 0.1, -0.2, 0.05, + 0.0, 0.0, 0.0, 0.21, 1.5, -1.3, -0.0005, -0.0013, + 16000.0, 6000.0, 2.0, 1.0] +OFFSET_V3_LENS1 = [1.9, 2868.0, 2869.0, 3990.1, 3010.2, -0.15, 0.3, -0.02, + 0.0, 0.0, 0.0, 0.22, 1.4, -1.2, -0.0004, -0.0011, + 16000.0, 6000.0, 2.0, 1.0] +OFFSET_V3_TEXT = "_".join( + f"{v:.6f}" for v in [2.0] + OFFSET_V3_LENS0 + OFFSET_V3_LENS1 +) + + +def make_metadata_payload(offset_v3: str | None = OFFSET_V3_TEXT, + window_crop: bytes | None = None) -> bytes: + payload = ( + _string_field(1, "XAS1234567890") + + _string_field(2, "Insta360 X5") + + _string_field(3, "v1.0.0") + + _varint_field(7, 1718000000) # creation_time, must be skipped + + _fixed64_field(25, b"\x00" * 8) # rolling_shutter_time, skipped + + _bytes_field(11, b"\x01\x02\x03\xff") # gps blob, skipped + ) + if offset_v3 is not None: + payload += _string_field(54, offset_v3) + if window_crop is not None: + payload += _bytes_field(27, window_crop) + return payload + + +class TestParseMetadataRecord: + def test_extracts_strings_and_window_crop(self): + payload = make_metadata_payload( + window_crop=make_window_crop(5376, 5376, 5312, 5312) + ) + info = parse_metadata_record(payload) + assert info["serial_number"] == "XAS1234567890" + assert info["camera_type"] == "Insta360 X5" + assert info["fw_version"] == "v1.0.0" + assert info["offset_v3"] == OFFSET_V3_TEXT + assert info["window_crop"] == { + "src_width": 5376, "src_height": 5376, + "dst_width": 5312, "dst_height": 5312, + } + + def test_garbage_returns_empty(self): + assert parse_metadata_record(b"\xff\xff\xff\xff") == {} + assert parse_metadata_record(b"") == {} + + +class TestParseOffsetV3: + def test_parses_two_lens_blocks(self): + lenses = parse_offset_v3(OFFSET_V3_TEXT) + assert len(lenses) == 2 + lens0 = lenses[0] + assert lens0["xi"] == pytest.approx(1.9) + assert lens0["fx"] == pytest.approx(2870.0) + assert lens0["cx"] == pytest.approx(3987.5) + assert lens0["yaw_deg"] == pytest.approx(0.1) + assert lens0["roll_deg"] == pytest.approx(0.05) + assert lens0["k1"] == pytest.approx(0.21) + assert lens0["p2"] == pytest.approx(-0.0013) + assert lens0["ref_width"] == pytest.approx(16000.0) + assert lenses[1]["fx"] == pytest.approx(2868.0) + + def test_rejects_truncated_or_junk(self): + assert parse_offset_v3("2_1.0_2.0_3.0") is None + assert parse_offset_v3("not_numbers_at_all") is None + assert parse_offset_v3("") is None + + +def make_pb_lens_block(fx, fy, cx, cy, yaw, pitch) -> list[float]: + # 27 fields: xi fx fy cx cy yaw pitch field7 tx ty tz + # k1 k2 k3 k4 zero p1 p2 s1 s2 s3 s4 tauX tauY ref_w ref_h type + return [2.0, fx, fy, cx, cy, yaw, pitch, 89.816, 0.0, 0.0, 0.0, + 0.2199, 1.6416, -1.4439, -2.6206, 0.0, -0.000536, -0.001321, + -0.00148, 0.0008, 0.00204, 0.00194, 0.02581, 0.00293, + 10752.0, 5376.0, 113.0] + + +def make_pb_sidecar_bytes() -> bytes: + lens_blocks = ( + make_pb_lens_block(4271.09, 4272.20, 2680.96, 2680.49, -0.132, 0.434) + + make_pb_lens_block(4268.30, 4269.10, 5376.0 + 2682.10, 2679.80, 0.090, -0.210) + ) + # Real sidecars store the string as "2__" + # with an integer lens count. + calibration = "2_" + "_".join(f"{v:.6f}" for v in lens_blocks) + blob = base64.b64encode(f"junk-prefix {calibration} junk-suffix".encode("latin-1")) + return b"\x0a\x10binary-protobuf" + blob + b"\x00\x01trailing" + + +class TestParsePbCalibration: + def test_parses_extended_calibration(self): + lenses = parse_pb_calibration(make_pb_sidecar_bytes()) + assert lenses is not None and len(lenses) == 2 + lens0 = lenses[0] + assert lens0["xi"] == pytest.approx(2.0) + assert lens0["fx"] == pytest.approx(4271.09) + assert lens0["cx"] == pytest.approx(2680.96) + assert lens0["yaw_deg"] == pytest.approx(-0.132) + assert lens0["pitch_deg"] == pytest.approx(0.434) + assert lens0["roll_deg"] == 0.0 # ambiguous field 7 is not used as roll + assert lens0["k4"] == pytest.approx(-2.6206) + assert lens0["p1"] == pytest.approx(-0.000536) + assert lens0["s4"] == pytest.approx(0.00194) + assert lens0["ref_width"] == pytest.approx(10752.0) + + def test_no_calibration_block(self): + assert parse_pb_calibration(b"no base64 here") is None + blob = base64.b64encode(b"x" * 200) + assert parse_pb_calibration(blob) is None + + +class TestMeiModelKwargs: + def test_x5_like_with_window_crop(self): + # X5: 5376px per-lens reference, encoded video is a centered 5312px + # crop scaled to 3840. fx scales by 3840/5312, cx shifts by the crop. + spec = { + "xi": 2.0, "fx": 4271.09, "fy": 4272.20, "cx": 2680.96, "cy": 2680.49, + "k1": 0.2199, "ref_width": 5376.0, "ref_height": 5376.0, + "crop_width": 5312, "crop_height": 5312, + } + kwargs = mei_model_kwargs_for_stream(spec, 3840, 3840) + scale = 3840 / 5312 + assert kwargs["fx"] == pytest.approx(4271.09 * scale) + assert kwargs["cx"] == pytest.approx((2680.96 - 32) * scale + 0.5) + assert kwargs["cy"] == pytest.approx((2680.49 - 32) * scale + 0.5) + assert kwargs["k1"] == pytest.approx(0.2199) + assert kwargs["k4"] == 0.0 + # Lands near the stream center, as a principal point must + assert abs(kwargs["cx"] - 1920) < 20 + + def test_x4_like_cover_fit_without_crop_info(self): + # X4: 8000x6000 per-lens reference (4:3 sensor), square 3840 video + # -> centered 6000x6000 crop scaled by 0.64. + spec = { + "xi": 1.9, "fx": 2870.0, "fy": 2871.0, "cx": 3987.5, "cy": 3012.7, + "ref_width": 8000.0, "ref_height": 6000.0, + } + kwargs = mei_model_kwargs_for_stream(spec, 3840, 3840) + assert kwargs["fx"] == pytest.approx(2870.0 * 0.64) + assert kwargs["cx"] == pytest.approx((3987.5 - 1000) * 0.64 + 0.5) + assert kwargs["cy"] == pytest.approx(3012.7 * 0.64 + 0.5) + assert abs(kwargs["cx"] - 1920) < 20 + assert abs(kwargs["cy"] - 1920) < 20 + + def test_mismatched_crop_aspect_falls_back_to_cover_fit(self): + spec = { + "xi": 2.0, "fx": 4000.0, "fy": 4000.0, "cx": 2688.0, "cy": 2688.0, + "ref_width": 5376.0, "ref_height": 5376.0, + "crop_width": 5312, "crop_height": 2656, # 2:1, not the stream's 1:1 + } + kwargs = mei_model_kwargs_for_stream(spec, 3840, 3840) + assert kwargs["fx"] == pytest.approx(4000.0 * 3840 / 5376) + + def test_fov_override_passes_through(self): + spec = { + "xi": 2.0, "fx": 4000.0, "fy": 4000.0, "cx": 2688.0, "cy": 2688.0, + "ref_width": 5376.0, "ref_height": 5376.0, "fov_deg": 195.0, + } + assert mei_model_kwargs_for_stream(spec, 3840, 3840)["fov_deg"] == 195.0 + + +class TestLoadFactoryCalibration: + def test_from_trailer_offset_v3(self, tmp_path): + records = { + METADATA_RECORD_KEY: make_metadata_payload( + window_crop=make_window_crop(6000, 6000, 6000, 6000) + ) + } + factory = load_factory_calibration(tmp_path / "video.insv", records) + assert factory is not None + assert factory["source"] == ".insv trailer offset_v3" + assert factory["camera_type"] == "Insta360 X5" + assert len(factory["lenses"]) == 2 + lens0 = factory["lenses"][0] + # 16000x6000 full dual frame is normalized to one 8000x6000 lens + assert lens0["ref_width"] == pytest.approx(8000.0) + assert lens0["crop_width"] == 6000 + assert lens0["cx"] == pytest.approx(3987.5) + + def test_prefers_pb_sidecar(self, tmp_path): + insv_path = tmp_path / "video.insv" + insv_path.write_bytes(b"") + (tmp_path / "video.insv.pb").write_bytes(make_pb_sidecar_bytes()) + records = {METADATA_RECORD_KEY: make_metadata_payload()} + factory = load_factory_calibration(insv_path, records) + assert factory is not None + assert factory["source"].endswith("video.insv.pb") + lens1 = factory["lenses"][1] + # Full-frame cx of the second lens is shifted into lens-local coords + assert lens1["cx"] == pytest.approx(2682.10) + assert lens1["ref_width"] == pytest.approx(5376.0) + + def test_no_calibration_returns_none(self, tmp_path): + records = {METADATA_RECORD_KEY: make_metadata_payload(offset_v3=None)} + assert load_factory_calibration(tmp_path / "video.insv", records) is None + assert load_factory_calibration(tmp_path / "video.insv", {}) is None + + +class TestMountCorrections: + def test_extracted_from_lens_specs(self): + lenses = parse_offset_v3(OFFSET_V3_TEXT) + corrections = insv_calibration.get_mount_corrections(lenses) + assert corrections[0] == pytest.approx((0.1, -0.2, 0.05)) + assert corrections[1] == pytest.approx((-0.15, 0.3, -0.02)) + + def test_rig_rotations_with_corrections(self): + from fisheye_projection import get_lens_from_rig_rotations + + # Identity corrections reproduce the nominal back-to-back rig + nominal = get_lens_from_rig_rotations() + with_zero = get_lens_from_rig_rotations( + mount_corrections=[(0.0, 0.0, 0.0), (0.0, 0.0, 0.0)] + ) + for a, b in zip(nominal, with_zero): + np.testing.assert_allclose(a, b, atol=1e-12) + + # A small yaw correction tilts the lens optical axis by that angle + corrected = get_lens_from_rig_rotations( + mount_corrections=[(0.5, 0.0, 0.0), (0.0, 0.0, 0.0)] + ) + axis_in_rig = np.array([0.0, 0.0, 1.0]) @ corrected[0] + angle = np.rad2deg(np.arccos(np.clip(axis_in_rig @ [0, 0, 1], -1, 1))) + assert angle == pytest.approx(0.5, abs=1e-9) + # Lens 1 still points backward + back_axis = np.array([0.0, 0.0, 1.0]) @ corrected[1] + assert back_axis[2] == pytest.approx(-1.0) diff --git a/vid2scene_core/vid2scene.py b/vid2scene_core/vid2scene.py index 6cbbdc1..192d414 100644 --- a/vid2scene_core/vid2scene.py +++ b/vid2scene_core/vid2scene.py @@ -93,9 +93,16 @@ def parse_arguments(): parser.add_argument( "--insv_calibration", help="Path to a lens calibration JSON for --insv_fisheye " - "(see docs/insv_fisheye.md).", + "(overrides the factory calibration; see docs/insv_fisheye.md).", default=None, ) + parser.add_argument( + "--insv_no_factory_calibration", + help="Ignore Insta360's per-unit factory lens calibration embedded in " + "the .insv recording and use the idealized lens model instead.", + action="store_true", + default=False, + ) parser.add_argument( "--use_background_sphere", help="Add a fibonacci sphere for the background.", @@ -508,6 +515,7 @@ def process_video_to_scene( insv_fisheye=False, insv_lens_fov=None, insv_calibration=None, + insv_no_factory_calibration=False, use_background_sphere=False, apply_pilgram_filter_name=None, training_max_num_gaussians=1_000_000, @@ -538,7 +546,10 @@ def process_video_to_scene( fisheye streams (skips the equirectangular intermediate). Enabled automatically when video_path ends in .insv. insv_lens_fov: Assumed full fisheye lens FOV in degrees (default 200) - insv_calibration: Path to a lens calibration JSON (see docs/insv_fisheye.md) + insv_calibration: Path to a lens calibration JSON, overriding the factory + calibration (see docs/insv_fisheye.md) + insv_no_factory_calibration: Ignore the per-unit factory calibration + embedded in the recording and use the idealized lens model use_background_sphere: Whether to use a background sphere apply_pilgram_filter_name: Name of Pilgram filter to apply (if any) training_max_num_gaussians: Maximum number of Gaussians to use in Gsplat @@ -620,6 +631,7 @@ def process_video_to_scene( generate_masks=True, lens_fov_deg=insv_lens_fov, calibration_path=insv_calibration, + use_factory_calibration=not insv_no_factory_calibration, kill_check=kill_check, ) @@ -844,6 +856,7 @@ def prune_ply(ply_path, new_ply_path, alpha_prune_threshold=12, kill_check=None) insv_fisheye=args.insv_fisheye, insv_lens_fov=args.insv_lens_fov, insv_calibration=args.insv_calibration, + insv_no_factory_calibration=args.insv_no_factory_calibration, use_background_sphere=args.use_background_sphere, apply_pilgram_filter_name=args.apply_pilgram_filter, training_max_num_gaussians=args.training_max_num_gaussians, From 540cb108a9d678e90c5351f0f2f17666d45f327b Mon Sep 17 00:00:00 2001 From: Kieran Farr Date: Fri, 12 Jun 2026 12:35:41 -0700 Subject: [PATCH 4/6] Parse X5 fw1.9 trailers: v3 index record + 19-field offset_v3 First validation against real X5 footage (fw v1.9.6_build1) found two gaps that silently dropped factory calibration to the idealized fallback: - The trailer chains an id-0 record at the top whose payload is an index table of (uint16 id, uint32 size, uint32 offset) entries, offsets relative to the trailer data start, small ids = legacy >> 8. Records below it no longer follow the strict payload+descriptor chain, so the walk now resolves everything through the table instead of treating id 0 as a terminator. - offset_v3 writes 19 fields per lens (no per-lens flag) plus trailing file-level values, vs the X4-era 20. parse_offset_v3 now tries both layouts and validates the reference-dimension slots, where a misaligned block lands lens_type/flag-scale values. With both fixes the real recording loads end-to-end from the trailer: principal points land within 5 px of the 3840x3840 stream center (full-frame cx shift + 5376->5312 window-crop scaling both verified), fx scales 4280->3094, and mount corrections surface the ~90 deg portrait-sensor roll confirmed by inspecting the demuxed frames - sub-degree-only assumptions would have broken this camera. Co-Authored-By: Claude Fable 5 --- vid2scene_core/insv_calibration.py | 36 +++++++++++++---- vid2scene_core/insv_extract.py | 39 +++++++++++++++++- vid2scene_core/tests/test_insv_calibration.py | 27 +++++++++++++ vid2scene_core/tests/test_insv_extract.py | 40 +++++++++++++++++++ 4 files changed, 133 insertions(+), 9 deletions(-) diff --git a/vid2scene_core/insv_calibration.py b/vid2scene_core/insv_calibration.py index a9b7aea..71f6f53 100644 --- a/vid2scene_core/insv_calibration.py +++ b/vid2scene_core/insv_calibration.py @@ -149,19 +149,39 @@ def _parse_float_list(text: str) -> list[float] | None: def parse_offset_v3(text: str) -> list[dict] | None: - """Parse the trailer's offset_v3 string into per-lens MEI lens specs.""" + """Parse the trailer's offset_v3 string into per-lens MEI lens specs. + + The per-lens field count varies by camera/firmware: X4-era trailers + write 20 fields (with a trailing per-lens flag), X5 fw 1.9 writes 19 + (no flag) followed by file-level trailing values. Both layouts are + tried; the winner must put plausible values in the reference-dimension + slots (a misaligned block lands lens_type/flag-scale values there). + """ values = _parse_float_list(text) if not values: return None num_lenses = int(values[0]) - block = len(_OFFSET_V3_LENS_FIELDS) - if num_lenses < 1 or len(values) < 1 + num_lenses * block: + if num_lenses < 1: return None - lenses = [] - for lens_idx in range(num_lenses): - fields = values[1 + lens_idx * block: 1 + (lens_idx + 1) * block] - lenses.append(dict(zip(_OFFSET_V3_LENS_FIELDS, fields))) - return lenses + for field_names in (_OFFSET_V3_LENS_FIELDS, _OFFSET_V3_LENS_FIELDS[:-1]): + block = len(field_names) + if len(values) < 1 + num_lenses * block: + continue + lenses = [ + dict(zip(field_names, values[1 + i * block: 1 + (i + 1) * block])) + for i in range(num_lenses) + ] + if all(_offset_v3_lens_plausible(spec) for spec in lenses): + return lenses + return None + + +def _offset_v3_lens_plausible(spec: dict) -> bool: + return ( + spec["fx"] > 0 + and spec["fy"] > 0 + and spec["ref_width"] >= spec["ref_height"] >= 1000 + ) def parse_pb_calibration(data: bytes) -> list[dict] | None: diff --git a/vid2scene_core/insv_extract.py b/vid2scene_core/insv_extract.py index 57bf9d9..e16c4b8 100644 --- a/vid2scene_core/insv_extract.py +++ b/vid2scene_core/insv_extract.py @@ -74,7 +74,19 @@ def read_insv_trailer_records(path, max_records: int = 64) -> dict[int, bytes]: break f.seek(pos) record_id, record_size = struct.unpack(" pos: + if record_size == 0 or record_size > pos: + break + if record_id == 0: + # Newer trailers (seen on X5 fw 1.9.x; trailer version 3 in + # the footer) chain an id-0 record at the top whose payload + # is an index table, and records below it no longer follow + # the strict payload+descriptor chain — so the table is + # authoritative for everything else. + f.seek(pos - record_size) + table = f.read(record_size) + records.update( + _parse_trailer_index(f, table, file_size, footer) + ) break f.seek(pos - record_size) records[record_id] = f.read(record_size) @@ -85,6 +97,31 @@ def read_insv_trailer_records(path, max_records: int = 64) -> dict[int, bytes]: return {} +def _parse_trailer_index(f, table: bytes, file_size: int, footer: bytes) -> dict[int, bytes]: + """Resolve a version-3 trailer index table into {record_id: payload}. + + The table is the payload of the id-0 record: 10-byte entries of + (uint16 id, uint32 size, uint32 offset), offset relative to the trailer + data start (= EOF minus the trailer size stored in the footer). Zero or + out-of-range entries are slot padding and skipped. Small ids are the + legacy record ids shifted right by 8 (2 -> 0x200 preview, 3 -> 0x300 + gyro, ...); 0x101 (file_info) keeps its legacy value. + """ + trailer_size = struct.unpack(" trailer_size: + continue + legacy_id = rec_id if rec_id >= 0x100 else rec_id << 8 + f.seek(trailer_start + rec_offset) + records[legacy_id] = f.read(rec_size) + return records + + def summarize_insv_metadata(records: dict[int, bytes]) -> dict: """Extract loggable info from trailer records (camera model, serial, ...). diff --git a/vid2scene_core/tests/test_insv_calibration.py b/vid2scene_core/tests/test_insv_calibration.py index 4a8b5ba..cf75dbc 100644 --- a/vid2scene_core/tests/test_insv_calibration.py +++ b/vid2scene_core/tests/test_insv_calibration.py @@ -65,6 +65,19 @@ def make_window_crop(src_w, src_h, dst_w, dst_h) -> bytes: f"{v:.6f}" for v in [2.0] + OFFSET_V3_LENS0 + OFFSET_V3_LENS1 ) +# X5 fw 1.9 offset_v3: 19 fields per lens (no per-lens flag) plus one +# trailing file-level value. Verbatim from a real X5 recording +# (fw v1.9.6_build1); note the ~90 deg nominal roll of the portrait-mounted +# sensors and lens 1's cx in full dual-frame coordinates. +OFFSET_V3_X5_TEXT = ( + "2_2.000000_4280.730_4280.890_2693.330_2680.260_-0.467_0.351_90.426" + "_0.000000_0.000000_0.000000_0.18062226_2.09071612_-3.32824707" + "_-0.00253176_-0.00075190_10752_5376_113" + "_2.000000_4278.380_4278.480_8069.920_2688.010_0.560_0.096_89.252" + "_-0.001068_0.000384_-0.032249_0.17816089_2.12469268_-3.33294296" + "_-0.00017225_-0.00264012_10752_5376_113_197632" +) + def make_metadata_payload(offset_v3: str | None = OFFSET_V3_TEXT, window_crop: bytes | None = None) -> bytes: @@ -118,6 +131,20 @@ def test_parses_two_lens_blocks(self): assert lens0["ref_width"] == pytest.approx(16000.0) assert lenses[1]["fx"] == pytest.approx(2868.0) + def test_parses_x5_19_field_blocks(self): + lenses = parse_offset_v3(OFFSET_V3_X5_TEXT) + assert len(lenses) == 2 + lens0, lens1 = lenses + assert lens0["xi"] == pytest.approx(2.0) + assert lens0["fx"] == pytest.approx(4280.730) + assert lens0["roll_deg"] == pytest.approx(90.426) + assert lens0["ref_width"] == pytest.approx(10752) + assert lens0["ref_height"] == pytest.approx(5376) + assert "flag" not in lens0 + assert lens1["fx"] == pytest.approx(4278.380) + assert lens1["cx"] == pytest.approx(8069.920) + assert lens1["lens_type"] == pytest.approx(113) + def test_rejects_truncated_or_junk(self): assert parse_offset_v3("2_1.0_2.0_3.0") is None assert parse_offset_v3("not_numbers_at_all") is None diff --git a/vid2scene_core/tests/test_insv_extract.py b/vid2scene_core/tests/test_insv_extract.py index 541e1a6..1f2ca45 100644 --- a/vid2scene_core/tests/test_insv_extract.py +++ b/vid2scene_core/tests/test_insv_extract.py @@ -32,6 +32,33 @@ def build_insv_trailer(records: list[tuple[int, bytes]]) -> bytes: return blob + footer +def build_insv_trailer_v3(records: list[tuple[int, bytes]]) -> bytes: + """Build a synthetic version-3 trailer (seen on X5 fw 1.9). + + Record payloads sit at the front of the trailer data region; an id-0 + record chained at the top holds an index table of 10-byte + (uint16 id, uint32 size, uint32 offset) entries with offsets relative + to the data start, and small table ids are the legacy ids >> 8. The + footer carries the total trailer size and version 3. + """ + data = b"" + entries = b"\x00" * 10 # leading padding slot, as in real trailers + for record_id, payload in records: + table_id = record_id if record_id == 0x101 else record_id >> 8 + entries += struct.pack(" Date: Fri, 12 Jun 2026 12:44:53 -0700 Subject: [PATCH 5/6] Add real X4 fw1.9.21 offset_v3 fixture Validated against a real X4 recording (fw v1.9.21_build5): newer X4 firmware also writes the v3 index trailer and 19-field offset_v3 blocks, with a per-lens landscape 8000x6000 reference (no halving) and lens 1 cx in 16000-wide full-frame coordinates. Principal points land within ~5 px of the 2880x2880 stream center after normalization, and the demuxed frames confirm the same ~90 deg portrait-sensor roll as the X5. Co-Authored-By: Claude Fable 5 --- vid2scene_core/tests/test_insv_calibration.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/vid2scene_core/tests/test_insv_calibration.py b/vid2scene_core/tests/test_insv_calibration.py index cf75dbc..7168616 100644 --- a/vid2scene_core/tests/test_insv_calibration.py +++ b/vid2scene_core/tests/test_insv_calibration.py @@ -65,6 +65,19 @@ def make_window_crop(src_w, src_h, dst_w, dst_h) -> bytes: f"{v:.6f}" for v in [2.0] + OFFSET_V3_LENS0 + OFFSET_V3_LENS1 ) +# X4 fw 1.9.21 offset_v3: also the 19-field layout (the 20-field X4 fixture +# above matches older firmware). Verbatim from a real X4 recording; per-lens +# landscape reference frame (8000x6000, no halving) with lens 1's cx in +# 16000-wide full dual-frame coordinates. +OFFSET_V3_X4_TEXT = ( + "2_1.948170_4613.480_4613.680_4007.080_2999.740_-0.045_0.017_89.764" + "_0.000000_0.000000_0.000000_0.37750375_1.42040467_-4.14384651" + "_0.00027564_-0.00161791_8000_6000_71" + "_1.948170_4632.750_4632.940_12009.280_3005.090_0.015_0.177_89.567" + "_-0.001166_-0.000030_-0.032311_0.39006260_1.31805623_-4.07983112" + "_0.00039543_-0.00112667_8000_6000_71_197632" +) + # X5 fw 1.9 offset_v3: 19 fields per lens (no per-lens flag) plus one # trailing file-level value. Verbatim from a real X5 recording # (fw v1.9.6_build1); note the ~90 deg nominal roll of the portrait-mounted @@ -145,6 +158,17 @@ def test_parses_x5_19_field_blocks(self): assert lens1["cx"] == pytest.approx(8069.920) assert lens1["lens_type"] == pytest.approx(113) + def test_parses_x4_fw19_19_field_blocks(self): + lenses = parse_offset_v3(OFFSET_V3_X4_TEXT) + assert len(lenses) == 2 + assert lenses[0]["xi"] == pytest.approx(1.94817) + assert lenses[0]["fx"] == pytest.approx(4613.480) + assert lenses[0]["roll_deg"] == pytest.approx(89.764) + assert lenses[0]["ref_width"] == pytest.approx(8000) + assert lenses[0]["ref_height"] == pytest.approx(6000) + assert "flag" not in lenses[0] + assert lenses[1]["cx"] == pytest.approx(12009.280) + def test_rejects_truncated_or_junk(self): assert parse_offset_v3("2_1.0_2.0_3.0") is None assert parse_offset_v3("not_numbers_at_all") is None From 253e52c5b1c20fcbdf2fd6327a9d6b9599720529 Mon Sep 17 00:00:00 2001 From: Kieran Farr Date: Fri, 12 Jun 2026 14:10:48 -0700 Subject: [PATCH 6/6] Fail .insv jobs early when factory calibration cannot be parsed The idealized-lens fallback is no longer silent: real X4/X5 sensors are portrait-mounted (~90 deg roll), which the idealized model doesn't know, so a job that silently degraded would burn a full SfM+training run on a rig that can't register. run_insv_sfm now raises FactoryCalibrationError before any heavy work (frame extraction, render, SfM, training) when no calibration parses; the idealized model remains available as an explicit opt-in (--insv_no_factory_calibration) or via a calibration JSON. load_factory_calibration now logs which rung of the ladder broke (no trailer / no file_info record / no offset_v3 / unparseable offset_v3) and, for the unparseable case, the raw offset_v3 string verbatim - that string is everything needed to add support for an unknown layout, as the X4-20-field vs X5-fw1.9-19-field split demonstrated. Co-Authored-By: Claude Fable 5 --- vid2scene_core/fisheye_sfm.py | 28 +++++++++---- vid2scene_core/insv_calibration.py | 39 +++++++++++++++++++ vid2scene_core/tests/test_insv_calibration.py | 30 ++++++++++++++ vid2scene_core/vid2scene.py | 8 +++- 4 files changed, 95 insertions(+), 10 deletions(-) diff --git a/vid2scene_core/fisheye_sfm.py b/vid2scene_core/fisheye_sfm.py index f420ef2..2dfa2d1 100644 --- a/vid2scene_core/fisheye_sfm.py +++ b/vid2scene_core/fisheye_sfm.py @@ -12,11 +12,13 @@ sensor pixels everywhere, and the default view grid points views down past the nadir, so the ground is covered by up to 6 views per frame pair instead of 0. -Lens intrinsics come from Insta360's per-unit factory calibration when it can -be parsed out of the recording (insv_calibration.py: the trailer's offset_v3 -or the .insv.pb sidecar, both MEI camera models). An explicit calibration -JSON (--insv_calibration) overrides it; with neither, an idealized -equidistant fisheye is assumed (see docs/insv_fisheye.md). +Lens intrinsics come from Insta360's per-unit factory calibration parsed out +of the recording (insv_calibration.py: the trailer's offset_v3 or the +.insv.pb sidecar, both MEI camera models). An explicit calibration JSON +(--insv_calibration) overrides it. If neither parses, the job FAILS EARLY +(FactoryCalibrationError, before any heavy work) rather than degrading; the +idealized equidistant fisheye is an explicit opt-in via +--no_factory_calibration (see docs/insv_fisheye.md). """ import argparse @@ -414,9 +416,19 @@ def run_insv_sfm( f"from {factory['source']}" ) else: - logger.info( - "No factory lens calibration found in the recording; " - "using the idealized lens model (see docs/insv_fisheye.md)" + # Fail fast — before frame extraction, rendering, SfM, or + # training spend anything. The idealized model doesn't know about + # the ~90 deg portrait-sensor roll real X4/X5 units have, so a + # silent fallback would burn the whole job on a rig that can't + # register. The insv_calibration warnings logged just above say + # exactly which parsing rung broke. + raise insv_calibration.FactoryCalibrationError( + "No usable Insta360 factory lens calibration in this " + "recording (no .insv.pb sidecar; trailer offset_v3 missing " + "or unparseable — see preceding log lines for the trailer " + "contents). Pass --insv_no_factory_calibration to explicitly " + "reconstruct with the idealized lens model, or provide a " + "calibration JSON (see docs/insv_fisheye.md)." ) if kill_check and kill_check(): diff --git a/vid2scene_core/insv_calibration.py b/vid2scene_core/insv_calibration.py index 71f6f53..d70c198 100644 --- a/vid2scene_core/insv_calibration.py +++ b/vid2scene_core/insv_calibration.py @@ -48,6 +48,19 @@ # with format=1 (protobuf) and id=1 (metadata). METADATA_RECORD_KEY = 0x101 + +class FactoryCalibrationError(RuntimeError): + """No usable factory lens calibration in an .insv recording. + + Raised by the pipeline when factory calibration is required (the default + for .insv jobs) but could not be parsed. Reconstructing with the + idealized lens model instead is an explicit opt-in + (--insv_no_factory_calibration), never a silent fallback: real X4/X5 + sensors are portrait-mounted (~90 deg roll), which the idealized model + does not know about, so a silent fallback wastes a full SfM+training run + on a geometry that cannot register. + """ + _METADATA_STRING_FIELDS = { 1: "serial_number", 2: "camera_type", @@ -317,8 +330,34 @@ def load_factory_calibration(insv_path, trailer_records: dict[int, bytes]) -> di lenses = parse_offset_v3(metadata["offset_v3"]) if lenses: source = ".insv trailer offset_v3" + else: + # Log the raw string verbatim: an unparseable offset_v3 means a + # layout this code doesn't know yet (the X4-era 20-field vs X5 + # fw1.9 19-field split was exactly this), and the string itself + # is everything needed to add support for it. + logger.warning( + "Trailer offset_v3 present but unparseable (unknown field " + f"layout?): {metadata['offset_v3']}" + ) if not lenses: + # Spell out which rung of the calibration ladder broke, so a failed + # job's log pinpoints the fix without needing the recording in hand. + if not trailer_records: + logger.warning( + "No .insv trailer records found (trailer missing or its " + "layout did not parse)" + ) + elif METADATA_RECORD_KEY not in trailer_records: + logger.warning( + "Trailer has no file_info record (0x101); records present: " + f"{[hex(key) for key in sorted(trailer_records)]}" + ) + elif not metadata.get("offset_v3"): + logger.warning( + "file_info record carries no offset_v3 calibration string; " + f"fields present: {sorted(metadata)}" + ) return None if len(lenses) != 2: logger.warning( diff --git a/vid2scene_core/tests/test_insv_calibration.py b/vid2scene_core/tests/test_insv_calibration.py index 7168616..e26b740 100644 --- a/vid2scene_core/tests/test_insv_calibration.py +++ b/vid2scene_core/tests/test_insv_calibration.py @@ -1,4 +1,5 @@ import base64 +import logging import numpy as np import pytest @@ -304,6 +305,35 @@ def test_no_calibration_returns_none(self, tmp_path): assert load_factory_calibration(tmp_path / "video.insv", records) is None assert load_factory_calibration(tmp_path / "video.insv", {}) is None + # The pipeline fails an .insv job hard when calibration cannot be parsed, + # so these warnings are the debugging surface — each rung of the ladder + # must say which step broke and carry the raw data needed to fix it. + def test_logs_which_rung_broke(self, tmp_path, caplog): + path = tmp_path / "video.insv" + with caplog.at_level(logging.WARNING): + load_factory_calibration(path, {}) + assert "No .insv trailer records" in caplog.text + + caplog.clear() + with caplog.at_level(logging.WARNING): + load_factory_calibration(path, {0x300: b"\x00"}) + assert "no file_info record" in caplog.text + assert "0x300" in caplog.text + + caplog.clear() + records = {METADATA_RECORD_KEY: make_metadata_payload(offset_v3=None)} + with caplog.at_level(logging.WARNING): + load_factory_calibration(path, records) + assert "no offset_v3" in caplog.text + + def test_logs_raw_string_when_offset_v3_unparseable(self, tmp_path, caplog): + bad = "2_1.0_2.0_3.0" + records = {METADATA_RECORD_KEY: make_metadata_payload(offset_v3=bad)} + with caplog.at_level(logging.WARNING): + assert load_factory_calibration(tmp_path / "video.insv", records) is None + assert "unparseable" in caplog.text + assert bad in caplog.text # verbatim — it's the fix-enabling data + class TestMountCorrections: def test_extracted_from_lens_specs(self): diff --git a/vid2scene_core/vid2scene.py b/vid2scene_core/vid2scene.py index 192d414..349a817 100644 --- a/vid2scene_core/vid2scene.py +++ b/vid2scene_core/vid2scene.py @@ -99,7 +99,9 @@ def parse_arguments(): parser.add_argument( "--insv_no_factory_calibration", help="Ignore Insta360's per-unit factory lens calibration embedded in " - "the .insv recording and use the idealized lens model instead.", + "the .insv recording and use the idealized lens model instead. " + "Without this flag, an .insv whose calibration cannot be parsed " + "fails early instead of silently degrading.", action="store_true", default=False, ) @@ -549,7 +551,9 @@ def process_video_to_scene( insv_calibration: Path to a lens calibration JSON, overriding the factory calibration (see docs/insv_fisheye.md) insv_no_factory_calibration: Ignore the per-unit factory calibration - embedded in the recording and use the idealized lens model + embedded in the recording and use the idealized lens model. + Without it, unparseable calibration fails the job early + (FactoryCalibrationError) instead of silently degrading use_background_sphere: Whether to use a background sphere apply_pilgram_filter_name: Name of Pilgram filter to apply (if any) training_max_num_gaussians: Maximum number of Gaussians to use in Gsplat