Skip to content
50 changes: 50 additions & 0 deletions DESIGN.md
Original file line number Diff line number Diff line change
Expand Up @@ -329,6 +329,56 @@ The result is the ordinary `Scene`; `build_manifest(scene)` is unchanged. A DSL
scene that reproduces the smoke setup recovers ground truth for a cam-1-occluded
frame through the real `triangulate_dlt`, exactly like the hand-built smoke.

### Domain randomization (optional, seeded — issue #41)

Synthetic-to-real robustness needs varied scenes. A typed
`RandomizationSpec` (`multicam_sim.randomization`) attaches seeded
randomization knobs to the builder: `builder.randomize(spec, seed=...)` samples
the spec once (`RandomizationSpec.sample(seed)`, a pure function over
`numpy.random.default_rng(seed)` — the builder never owns the randomness) and
applies the concrete result. Every knob is a **closed `(min, max)` interval**,
validated at construction (an inverted `min > max` interval is rejected), and
every knob is **off by default** (`None` / empty), so a scene built without
randomization serialises byte-identically to before.

The three knobs:

- **`background`** (`BackgroundSpec`) — the render-time background colour.
`rgb_min` / `rgb_max` are RGB triples with channels in `[0, 1]` (the
renderer's colour convention), sampled per channel. Default interval
`(0, 0, 0)`–`(0.2, 0.2, 0.2)`: dark, around the default black. Background
was already configurable per renderer through the `PyrenderBackend(bg=...)`
constructor argument; this knob adds **scene-level, randomizable** control —
the sampled value is recorded on the `Scene` and **takes precedence over the
constructor `bg`** wherever the scene carries one.
- **`light`** (`LightSpec`) — the key light. `intensity` is in renderer light
units (today's fixed headlight is `3.0`; default interval `(2.0, 4.0)`).
`azimuth_deg` / `elevation_deg` give the direction **from the scene to the
light** in **degrees** — azimuth in the world XY plane from +X toward +Y
(default `(0, 360)`), elevation above the XY plane (default `(30, 90)`, so
the light stays above the horizon); the light shines along the negated
vector (`Light.direction()`).
- **`distractors`** (`DistractorSpec`) — a count of static, non-target objects.
`count` is an **inclusive** integer interval (default `(1, 3)`); each
distractor's `x` / `y` / `z` (scene units) is drawn uniformly from its
interval (default a 4×4×1 box around the origin at floor level) and added
through the **existing** `SceneBuilder.distractor` entry point as
`rand_distractor_{i}` — so randomized distractors appear in the manifest
exactly like hand-added ones and never touch the primary entities' ground
truth.

Sampling draws in a **fixed order** (background channels; light intensity,
azimuth, elevation; distractor count; then three coordinates per distractor),
so the **reproducibility contract** is: same spec + same seed → byte-identical
sample, and therefore a byte-identical scene. Provenance is recorded
additively: the built `Scene` carries the sampled `background` / `light` values
plus a `randomization` sidecar (`RandomizationRecord`, spec + seed) that
round-trips through scene JSON, so a randomized run regenerates from its own
output via `record.spec.sample(record.seed)`. None of these fields is read by
the manifest builder; the sampled background/light are consumed by the
pyrender backend (the Kubric backend keeps its fixed key light — see the
renderer section).

## Renderer backend (`multicam_sim.dsl.render`) — not the contract

`RendererBackend` is a `Protocol` (Scene + camera + frame → `(H,W,3)` pixels).
Expand Down
18 changes: 18 additions & 0 deletions src/multicam_sim/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,16 @@
PossessionTimeline,
write_possession_json,
)
from .randomization import (
Background,
BackgroundSpec,
DistractorSpec,
Light,
LightSpec,
RandomizationRecord,
RandomizationSample,
RandomizationSpec,
)
from .scene import Scene
from .smoke import build_multi_entity_scene, build_pose_smoke_scene, build_smoke_scene
from .topology import CameraTopology, Station, TransitEdge
Expand All @@ -71,6 +81,8 @@
"ActivityState",
"ActivityTimeline",
"AssumedCalibration",
"Background",
"BackgroundSpec",
"Box",
"CalibrationDrift",
"Cylinder",
Expand All @@ -81,6 +93,7 @@
"CocoCategory",
"CocoDataset",
"CocoImage",
"DistractorSpec",
"Entity",
"EntityFrame",
"EntityManifest",
Expand All @@ -91,6 +104,8 @@
"HandOccluder",
"InteractionEvent",
"Intrinsics",
"Light",
"LightSpec",
"Manifest",
"MeshBackend",
"NoiseModel",
Expand All @@ -103,6 +118,9 @@
"PoseTrajectory",
"PossessionSegment",
"PossessionTimeline",
"RandomizationRecord",
"RandomizationSample",
"RandomizationSpec",
"Scene",
"SensorDropout",
"Skeleton",
Expand Down
36 changes: 35 additions & 1 deletion src/multicam_sim/dsl/builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,10 @@
from ..entities import Entity, EntityFrame
from ..occluders import OccluderUnion
from ..possession import InteractionEvent, PossessionSegment, PossessionTimeline
from ..randomization import Background, Light, RandomizationRecord, RandomizationSpec
from ..scene import Scene
from .behavior import Behavior, PathBehavior
from .motion import PathUnion
from .motion import LinearPath, PathUnion
from .occlusion import HandSweep, Occlusion

Vec3 = tuple[float, float, float]
Expand Down Expand Up @@ -72,6 +73,10 @@ def __init__(self, fps: float, num_frames: int) -> None:
self._attachments: list[_AttachmentSpec] = []
self._interactions: list[InteractionEvent] = []
self._activity_segments: list[ActivitySegment] = []
self._background: Background | None = None
self._light: Light | None = None
self._randomization: RandomizationRecord | None = None
self._rand_distractors = 0

def cameras(self, cameras: list[Camera]) -> SceneBuilder:
"""Set the camera array (e.g. from :class:`multicam_sim.dsl.CameraRig`)."""
Expand Down Expand Up @@ -114,6 +119,32 @@ def distractor(
)
return self

def randomize(self, spec: RandomizationSpec, *, seed: int = 0) -> SceneBuilder:
"""Sample ``spec`` under ``seed`` and apply the result to the scene.

The builder does not own the randomness: :meth:`RandomizationSpec.sample`
draws one concrete sample (deterministic for a given spec + seed), and
this entry point applies it — the sampled ``background`` / ``light`` are
stored on the built :class:`Scene`, each sampled distractor position is
added as a static distractor through the existing :meth:`distractor`
entry point (id ``rand_distractor_{i}``), and a
:class:`~multicam_sim.randomization.RandomizationRecord` (spec + seed)
rides on the scene as provenance. Calling it again re-samples and
appends its distractors on top; ids come from a per-builder counter so
a second call can never collide with the first.
"""
sample = spec.sample(seed)
self._background = sample.background
self._light = sample.light
self._randomization = RandomizationRecord(spec=spec, seed=seed)
for position in sample.distractor_positions:
self.distractor(
f"rand_distractor_{self._rand_distractors}",
LinearPath(a=position, b=position),
)
self._rand_distractors += 1
return self

def occlude(self, occlusion: Occlusion) -> SceneBuilder:
"""Add a declarative occlusion pattern (compiled to real geometry)."""
self._occlusions.append(occlusion)
Expand Down Expand Up @@ -360,4 +391,7 @@ def build(self) -> Scene:
occluders=occluders,
possession=possession,
activity=activity,
background=self._background,
light=self._light,
randomization=self._randomization,
)
31 changes: 29 additions & 2 deletions src/multicam_sim/dsl/render.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,10 @@ def _build_pyrender_scene(
pyrender, trimesh = _import_pyrender("The pyrender backend")

cam = scene.cameras[camera_id]
pr_scene = pyrender.Scene(bg_color=[*bg, 1.0], ambient_light=[0.4, 0.4, 0.4])
# Precedence: a scene-level sampled background wins over the renderer's
# constructor ``bg`` (the more specific value beats the renderer default).
bg_rgb = scene.background.rgb if scene.background is not None else bg
pr_scene = pyrender.Scene(bg_color=[*bg_rgb, 1.0], ambient_light=[0.4, 0.4, 0.4])

# entity points -> small spheres at this frame's ground-truth coords
for entity in scene.entities:
Expand Down Expand Up @@ -92,10 +95,34 @@ def _build_pyrender_scene(
pose[:3, 3] = cam.centre()
pr_scene.add(pr_cam, pose=pose)
if add_light:
pr_scene.add(pyrender.DirectionalLight(intensity=3.0), pose=pose)
# A sampled scene light overrides the fixed headlight (intensity 3.0 at
# the camera pose); without one the render is unchanged.
light = scene.light
light_intensity = light.intensity if light is not None else 3.0
light_pose = _light_pose(pose, light.direction()) if light is not None else pose
pr_scene.add(pyrender.DirectionalLight(intensity=light_intensity), pose=light_pose)
return pr_scene, intr


def _light_pose(camera_pose: FloatArray, direction: tuple[float, float, float]) -> FloatArray:
"""Rotation aligning a directional light's local -Z with ``direction``.

pyrender's ``DirectionalLight`` shines along its pose's local -Z (the same
axis the camera looks along), so the headlight default is just the camera
pose; a sampled world direction needs the pose's third column set to the
negated unit direction, with an orthonormal basis built around it.
"""
z_col = -np.asarray(direction, dtype=np.float64)
z_col /= np.linalg.norm(z_col)
ref = np.array([0.0, 0.0, 1.0]) if abs(z_col[2]) < 0.9 else np.array([1.0, 0.0, 0.0])
x_col = np.cross(ref, z_col)
x_col /= np.linalg.norm(x_col)
y_col = np.cross(z_col, x_col)
pose = np.array(camera_pose, dtype=np.float64)
pose[:3, :3] = np.column_stack([x_col, y_col, z_col])
return pose


@runtime_checkable
class RendererBackend(Protocol):
"""Anything that renders one camera's view of a scene at a frame to pixels.
Expand Down
Loading
Loading