3D scene viewer + overlapping-camera scenes (handoff + assembly line)

README.md

@@ -6,6 +6,11 @@
 the pure-numpy rasterizer, with the object's analytic projected box drawn in every
 view that sees it. Reproduce: `uv run --with pillow python scripts/render_multiview_hero.py`.*
 
+![A 3D world view of a scene: a translucent ground plane with three cameras drawn as red view-cones at their calibrated world positions, each pointing at the scene, and the object's ground-truth 3D trajectory as a blue polyline crossing the space.](docs/assets/scene_3d.png)
+
+*The scene in 3D: every camera drawn as a view-cone at its real calibrated world position, the ground plane, and the object's ground-truth trajectory across the frames.
+One picture of where the cameras are and what they cover — camera coverage and overlap are visible directly, not inferred from per-camera tiles. Works on any manifest; reproduce: `uv run --with matplotlib python scripts/view_scene_3d.py`.*
+
 ![Three MTMC camera stations with non-overlapping fields of view watch one object cross a corridor on a synced timeline; it hands off from camera 0 to 1 to 2, with blind-gap frames where no camera sees it.](docs/assets/hero_grid.gif)
 
 *One object, three cameras with **disjoint** views, one timeline — a green border

scripts/view_scene_3d.py

@@ -0,0 +1,207 @@
+"""Render a scene manifest as one 3D world view: camera locations + trajectory.
+
+A single static 3D figure showing where every camera sits in the world (drawn as
+a small view-cone / frustum pointing along its optical axis), a ground plane, and
+each entity's ground-truth 3D trajectory over the frames. This is the "see the
+space" view -- unlike the per-camera tiles of ``record_multiview.py``, it draws
+the whole scene once so camera coverage and overlap are directly visible.
+
+Geometry follows ``multicam_sim.cameras.Camera``: ``R`` is the world->camera
+rotation with rows ``[right, down, forward]`` (OpenCV, +z forward) and ``t`` is
+the world->camera translation, so the camera centre is ``C = -R^T @ t`` and the
+frustum edges are the back-projected image corners ``R^T @ (K^-1 @ [u, v, 1])``.
+World up is +z; the ground plane is drawn at z = 0.
+
+Works on any manifest with the standard schema (``cameras[*].{K,R,t,width,
+height}`` and ``entities[*].frames[*].points[*].xyz_gt``); defaults to the
+bundled MTMC golden fixture.
+
+``matplotlib`` is imported lazily inside :func:`main` (it is not a package
+dependency), mirroring the lazy-import pattern in ``scripts/record_multiview.py``.
+Run::
+
+    uv run --with matplotlib python scripts/view_scene_3d.py
+    uv run --with matplotlib python scripts/view_scene_3d.py --manifest PATH --out PATH
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+import numpy as np
+
+if TYPE_CHECKING:
+    from numpy.typing import NDArray
+
+_ROOT = Path(__file__).resolve().parent.parent
+_DEFAULT_MANIFEST = _ROOT / "tests" / "fixtures" / "manifest_golden" / "mtmc.json"
+_DEFAULT_OUT = _ROOT / "docs" / "assets" / "scene_3d.png"
+
+# How far in front of each camera to draw the frustum apex (world units).
+_FRUSTUM_DEPTH = 1.2
+
+
+def _camera_centre(
+    rotation: NDArray[np.float64], translation: NDArray[np.float64]
+) -> NDArray[np.float64]:
+    """World-space camera centre ``C = -R^T @ t``."""
+    return -rotation.T @ translation
+
+
+def _frustum_corners(
+    intrinsics: NDArray[np.float64],
+    rotation: NDArray[np.float64],
+    centre: NDArray[np.float64],
+    width: float,
+    height: float,
+    depth: float,
+) -> NDArray[np.float64]:
+    """The four image-corner rays back-projected to ``depth`` in world space.
+
+    A pixel ``[u, v, 1]`` back-projects to the world ray direction
+    ``R^T @ (K^-1 @ [u, v, 1])``; scaling so the forward (+z camera) component
+    equals ``depth`` puts the corner a fixed distance in front of the camera.
+    """
+    inv_k = np.linalg.inv(intrinsics)
+    corners_px = np.array(
+        [[0.0, 0.0], [width, 0.0], [width, height], [0.0, height]],
+        dtype=np.float64,
+    )
+    out = np.empty((4, 3), dtype=np.float64)
+    for i, (u, v) in enumerate(corners_px):
+        cam_ray = inv_k @ np.array([u, v, 1.0], dtype=np.float64)
+        cam_ray = cam_ray / cam_ray[2] * depth  # forward (+z) component == depth
+        out[i] = centre + rotation.T @ cam_ray
+    return out
+
+
+def _load_manifest(path: Path) -> dict[str, Any]:
+    with path.open("r", encoding="utf-8") as fh:
+        data: dict[str, Any] = json.load(fh)
+    return data
+
+
+def _trajectories(manifest: dict[str, Any]) -> list[tuple[str, NDArray[np.float64]]]:
+    """Per-(entity, point) ground-truth 3D polylines over the frames."""
+    out: list[tuple[str, NDArray[np.float64]]] = []
+    for entity in manifest.get("entities", []):
+        by_point: dict[str, list[list[float]]] = {}
+        for frame in entity.get("frames", []):
+            for name, point in frame.get("points", {}).items():
+                xyz = point.get("xyz_gt")
+                if xyz is not None:
+                    by_point.setdefault(name, []).append(list(xyz))
+        for name, coords in by_point.items():
+            label = entity.get("id", "object")
+            label = label if name == "center" else f"{label}:{name}"
+            out.append((str(label), np.asarray(coords, dtype=np.float64)))
+    return out
+
+
+def render(manifest: dict[str, Any], out_path: Path) -> Path:
+    """Render the 3D scene view to ``out_path`` (PNG) and return the path."""
+    import matplotlib
+
+    matplotlib.use("Agg")  # headless-safe: no display needed
+    import matplotlib.pyplot as plt
+
+    fig = plt.figure(figsize=(9, 7))
+    ax = fig.add_subplot(111, projection="3d")
+
+    all_points: list[NDArray[np.float64]] = []
+
+    # Cameras: centre marker + a wireframe frustum to the four image corners.
+    for cam in manifest.get("cameras", []):
+        intrinsics = np.asarray(cam["K"], dtype=np.float64)
+        rotation = np.asarray(cam["R"], dtype=np.float64)
+        translation = np.asarray(cam["t"], dtype=np.float64)
+        centre = _camera_centre(rotation, translation)
+        corners = _frustum_corners(
+            intrinsics,
+            rotation,
+            centre,
+            float(cam["width"]),
+            float(cam["height"]),
+            _FRUSTUM_DEPTH,
+        )
+        ax.scatter(*centre, color="crimson", s=40, depthshade=False)
+        ax.text(*centre, f"  cam {cam['id']}", color="crimson", fontsize=8)
+        for corner in corners:
+            edge = np.vstack([centre, corner])
+            ax.plot(edge[:, 0], edge[:, 1], edge[:, 2], color="crimson", linewidth=0.8)
+        loop = np.vstack([corners, corners[0]])
+        ax.plot(loop[:, 0], loop[:, 1], loop[:, 2], color="crimson", linewidth=0.8)
+        all_points.append(centre[None, :])
+        all_points.append(corners)
+
+    # Trajectories: one polyline per entity point.
+    for label, coords in _trajectories(manifest):
+        ax.plot(
+            coords[:, 0],
+            coords[:, 1],
+            coords[:, 2],
+            marker="o",
+            markersize=3,
+            linewidth=1.5,
+            label=f"trajectory: {label}",
+        )
+        all_points.append(coords)
+
+    if not all_points:
+        raise ValueError("manifest has no cameras or trajectories to draw")
+    pts = np.vstack(all_points)
+
+    # Ground plane at z = 0 spanning the scene's xy extent.
+    pad = 1.0
+    x_min, y_min = pts[:, 0].min() - pad, pts[:, 1].min() - pad
+    x_max, y_max = pts[:, 0].max() + pad, pts[:, 1].max() + pad
+    gx, gy = np.meshgrid(
+        np.linspace(x_min, x_max, 2),
+        np.linspace(y_min, y_max, 2),
+    )
+    ax.plot_surface(gx, gy, np.zeros_like(gx), alpha=0.12, color="steelblue")
+
+    ax.set_xlabel("x")
+    ax.set_ylabel("y")
+    ax.set_zlabel("z (up)")
+    ax.set_title("Scene: camera locations and object trajectory")
+    ax.legend(loc="upper left", fontsize=8)
+
+    # Equal aspect over the combined bounds so directions are not skewed.
+    span = pts.max(axis=0) - pts.min(axis=0)
+    span = np.where(span > 0, span, 1.0)
+    ax.set_box_aspect(tuple(span))
+
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    fig.savefig(out_path, dpi=130, bbox_inches="tight")
+    plt.close(fig)
+    return out_path
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--manifest",
+        type=Path,
+        default=_DEFAULT_MANIFEST,
+        help="scene manifest JSON (default: bundled MTMC golden fixture)",
+    )
+    parser.add_argument(
+        "--out",
+        type=Path,
+        default=_DEFAULT_OUT,
+        help="output PNG path (default: docs/assets/scene_3d.png)",
+    )
+    args = parser.parse_args()
+
+    manifest = _load_manifest(args.manifest)
+    out_path = render(manifest, args.out)
+    size = out_path.stat().st_size
+    print(f"wrote {out_path}  ({size} bytes)")
+
+
+if __name__ == "__main__":
+    main()

src/multicam_sim/__init__.py

@@ -18,6 +18,7 @@
     write_coco,
     write_yolo,
 )
+from .assembly_line import build_assembly_line_scene
 from .cameras import Camera, Intrinsics
 from .dropout import SensorDropout
 from .entities import Entity, EntityFrame
@@ -28,6 +29,7 @@
     compute_group_membership,
     write_group_json,
 )
+from .handoff_ltr import build_handoff_ltr_scene
 from .manifest import (
     AssumedCalibration,
     CameraManifest,
@@ -107,7 +109,9 @@
     "TransitEdge",
     "YoloDataset",
     "YoloLabel",
+    "build_assembly_line_scene",
     "build_group_formation_scene",
+    "build_handoff_ltr_scene",
     "build_manifest",
     "build_mtmc_scene",
     "silhouette_visible_fraction",

src/multicam_sim/assembly_line.py

@@ -0,0 +1,149 @@
+"""Assembly-line scene: an operator walks a line of workstations under overlapping cameras.
+
+A domain-neutral counterpart to :mod:`multicam_sim.mtmc`. Where MTMC places
+cameras at SEPARATED stations with *disjoint* fields of view (an object is in at
+most one view at a time, with a genuine blind gap between them), this scene
+deliberately *overlaps* adjacent cameras' coverage along a straight walkway:
+
+  * five fixed **workstations** (``station_1`` .. ``station_5``) sit evenly
+    spaced along the line;
+  * one **operator** — a single moving point named ``center`` — walks the line
+    ``station_1 -> station_5``, pausing (a **dwell**, the operation) at each
+    station before moving on;
+  * five **cameras** stand back off the line, one framing each station, aimed so
+    that adjacent cameras' fields of view overlap over the walkway *between*
+    stations.
+
+The overlap is graduated, not universal: while the operator dwells at a station
+it is in exactly one camera's view, and while it *transits* between two adjacent
+stations it is in view on BOTH covering cameras at once. That two-camera transit
+interval is the overlap this scene exists to demonstrate (issue #84) — the
+manifest's per-camera ``in_view`` flags record it with no schema change, and a
+consumer can triangulate the operator wherever two cameras cover it.
+
+The operator keeps ONE stable ``entity.id`` throughout, the cross-camera
+ground-truth identity a tracker must preserve down the line. The scene carries a
+:class:`CameraTopology` whose stations name each workstation's covering camera
+and whose directed :class:`TransitEdge` s chain the walk ``station_i ->
+station_{i+1}`` (and back) with the transit time between them.
+
+v1 models the operator as a single point target ("center"), which keeps the
+overlap proof unambiguous (one point in view on two cameras) and the manifest
+compact; a COCO-17 skeleton operator (as in :mod:`examples.assembly_station`) is
+a deliberate future extension, not a gap.
+"""
+
+from __future__ import annotations
+
+from .cameras import Camera
+from .dsl.motion import Path, PathUnion
+from .dsl.rig import CameraRig, StationView
+from .entities import Entity
+from .scene import Scene
+from .topology import CameraTopology, Station, TransitEdge
+
+_WIDTH = 1280
+_HEIGHT_PX = 720
+_FPS = 10.0
+
+_NUM_STATIONS = 5
+#: Spacing between adjacent workstations along the walkway (world units, +x).
+_STATION_STEP = 3.0
+#: Height of the tracked operator point (torso, world +z).
+_OPERATOR_Z = 1.0
+#: Cameras stand back off the line along -y, looking toward the walkway.
+_CAM_Y = -4.0
+_CAM_Z = 2.2
+#: Wide-ish FOV so adjacent cameras' cones overlap over the walkway.
+_CAM_FOV_DEG = 70.0
+
+#: Seconds the operator walks between two adjacent stations.
+_WALK_S = 0.5
+#: Seconds the operator dwells (pauses, the operation) at each station.
+_DWELL_S = 0.4
+
+OPERATOR_ID = "operator-1"
+#: Domain-neutral workstation ids down the line.
+STATION_IDS = [f"station_{i + 1}" for i in range(_NUM_STATIONS)]
+
+
+def _station_x(index: int) -> float:
+    """World ``x`` of workstation ``index`` (0-based) along the walkway."""
+    return index * _STATION_STEP
+
+
+def _station_views() -> list[StationView]:
+    """One camera per workstation, standing back off the line and framing that
+    station. The wide FOV makes adjacent cones overlap over the walkway between
+    stations, so a transiting operator is seen by two cameras at once."""
+    return [
+        StationView(
+            position=(_station_x(i), _CAM_Y, _CAM_Z),
+            look_at=(_station_x(i), 0.0, _OPERATOR_Z),
+            fov_deg=_CAM_FOV_DEG,
+        )
+        for i in range(_NUM_STATIONS)
+    ]
+
+
+def _cameras() -> list[Camera]:
+    return CameraRig.stations(_station_views(), width=_WIDTH, height_px=_HEIGHT_PX)
+
+
+def _operator_path() -> PathUnion:
+    """A timed walk ``station_1 -> station_5`` with a dwell at each station.
+
+    Built from the motion DSL: a zero-length ``linear(p, p).over(dwell)`` segment
+    holds the operator in place for the dwell, chained by ``.then`` with a
+    ``linear(p_i, p_{i+1}).over(walk)`` transit segment for each leg. The dwell
+    segments make the operator's position constant across consecutive frames at
+    each station; the transit segments carry it (and its overlapping two-camera
+    coverage) between them.
+    """
+    points = [(_station_x(i), 0.0, _OPERATOR_Z) for i in range(_NUM_STATIONS)]
+    path: PathUnion = Path.linear(points[0], points[0]).over(_DWELL_S)
+    for i in range(_NUM_STATIONS - 1):
+        walk = Path.linear(points[i], points[i + 1]).over(_WALK_S)
+        dwell = Path.linear(points[i + 1], points[i + 1]).over(_DWELL_S)
+        path = path.then(walk).then(dwell)
+    return path
+
+
+def _num_frames(path: PathUnion) -> int:
+    """Frame count covering the whole timed walk at :data:`_FPS`, inclusive."""
+    return int(round(path.total_duration() * _FPS)) + 1
+
+
+def build_assembly_line_scene() -> Scene:
+    """Construct the deterministic overlapping assembly-line scene.
+
+    Five cameras with overlapping coverage watch one operator walk five
+    workstations, dwelling at each. Returns a :class:`Scene` carrying the
+    per-station :class:`CameraTopology` and no occluders.
+    """
+    cameras = _cameras()
+
+    path = _operator_path()
+    num_frames = _num_frames(path)
+    frames = path.compile_frames(_FPS, num_frames, name="center")
+    operator = Entity(id=OPERATOR_ID, frames=frames)
+
+    # Each workstation is covered by its own camera (same index); the walk chains
+    # station_i <-> station_{i+1} with the transit time between them.
+    transit_s = _WALK_S
+    stations = [Station(id=station_id, camera_ids=[i]) for i, station_id in enumerate(STATION_IDS)]
+    edges: list[TransitEdge] = []
+    for i in range(_NUM_STATIONS - 1):
+        src, dst = STATION_IDS[i], STATION_IDS[i + 1]
+        edges.append(TransitEdge(src=src, dst=dst, transit_time_s=transit_s))
+        edges.append(TransitEdge(src=dst, dst=src, transit_time_s=transit_s))
+    topology = CameraTopology(stations=stations, edges=edges)
+
+    return Scene(
+        fps=_FPS,
+        num_frames=num_frames,
+        cameras=cameras,
+        entities=[operator],
+        occluders=[],
+        topology=topology,
+    )