From 61ecb9c3e2e79043db528a25f79d1deaba6d8cba Mon Sep 17 00:00:00 2001 From: Bamdad Dashtban Date: Mon, 3 Aug 2026 13:51:18 +0100 Subject: [PATCH 1/3] Add appearance-descriptor sidecar types and deterministic sample() A seeded, pixel-free, GT-derived appearance descriptor for cross-camera re-id: EntityAppearance (unit-norm descriptor keyed by entity_id) and AppearanceTable with a classmethod sample(entity_ids, dim, separation, seed). Descriptors are drawn deterministically (numpy default_rng, fixed draw order), unit-normalized anchors blended toward their shared mean by (1 - separation) and renormalized, so the separation knob monotonically controls inter-entity cosine similarity (1.0 near-orthogonal/separable, 0.0 collapsed/indistinguishable). The descriptor does NOT encode the GT entity id; it models OSNet-style confusability, and separation is the dose-response axis for re-id eval. Exported from the package top level. --- src/multicam_sim/__init__.py | 4 + src/multicam_sim/appearance.py | 157 +++++++++++++++++++++++++++++++++ 2 files changed, 161 insertions(+) create mode 100644 src/multicam_sim/appearance.py diff --git a/src/multicam_sim/__init__.py b/src/multicam_sim/__init__.py index c4e7dd4..31f6d32 100644 --- a/src/multicam_sim/__init__.py +++ b/src/multicam_sim/__init__.py @@ -27,6 +27,7 @@ write_coco, write_yolo, ) +from .appearance import AppearanceTable, EntityAppearance, write_appearance_json from .cameras import Camera, Intrinsics from .dropout import SensorDropout from .entities import Entity, EntityFrame @@ -92,6 +93,7 @@ "ActivitySegment", "ActivityState", "ActivityTimeline", + "AppearanceTable", "AssumedCalibration", "Background", "BackgroundSpec", @@ -109,6 +111,7 @@ "DipSchedule", "DistractorSpec", "Entity", + "EntityAppearance", "EntityFrame", "EntityManifest", "FrameObs", @@ -160,6 +163,7 @@ "validate_manifest", "write_actions_json", "write_activity_json", + "write_appearance_json", "write_coco", "write_group_json", "write_manifest", diff --git a/src/multicam_sim/appearance.py b/src/multicam_sim/appearance.py new file mode 100644 index 0000000..ac824c9 --- /dev/null +++ b/src/multicam_sim/appearance.py @@ -0,0 +1,157 @@ +"""Appearance-descriptor sidecar: a seeded, pixel-free, GT-derived re-id cue. + +This channel models the *appearance* half of cross-camera re-identification. When +geometry alone cannot disambiguate two subjects — the collinear / ambiguous case +where their epipolar or triangulated positions coincide — a fuser needs a second, +appearance-based signal to tell them apart. This module supplies that signal +*without any pixels*: each entity is assigned a deterministic unit-norm descriptor +vector, and how easily two entities can be told apart is set by a single knob. + +**Honest note — what this does and does NOT encode.** The descriptor does NOT +encode, and cannot be inverted to recover, the ground-truth ``entity_id``. It is +NOT a perfect label handed to the fuser. It models the *confusability* of a real +learned appearance embedding (OSNet-style): distinct identities land at distinct +but not perfectly separated points on the unit sphere, and a downstream matcher +must still cluster them under noise. The :attr:`AppearanceTable.separation` knob +is the dose-response axis for re-id evaluation: it monotonically controls the +inter-entity cosine similarity, from ``1.0`` (near-orthogonal descriptors, +perfectly separable — the easy case) down to ``0.0`` (all descriptors collapse to +a shared mean, indistinguishable — the impossible case). Sweeping it traces how +re-id accuracy degrades as identities become harder to separate. + +Like :mod:`multicam_sim.activity` and :mod:`multicam_sim.possession`, this is a +pure typed-model sidecar. The table rides in a JSON sidecar and attaches to +:class:`~multicam_sim.scene.Scene` via an optional field that the manifest builder +IGNORES, so the byte-golden analytic manifest is unchanged when appearance GT is +absent (and even when it is present). +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import numpy as np +from pydantic import BaseModel, ConfigDict, field_validator + +_DEFAULT_DIM = 512 + + +class EntityAppearance(BaseModel): + """One entity's appearance descriptor: a unit-norm vector keyed by id. + + ``descriptor`` is L2-normalized (unit length). It is a synthetic stand-in for + a learned re-id embedding; it does not encode the ``entity_id``. + """ + + model_config = ConfigDict(frozen=True) + + entity_id: str + descriptor: list[float] + + @field_validator("descriptor") + @classmethod + def _non_empty(cls, value: list[float]) -> list[float]: + if not value: + raise ValueError("descriptor must be non-empty") + return value + + +class AppearanceTable(BaseModel): + """A collection of per-entity appearance descriptors of a fixed ``dim``. + + Built deterministically from ``(entity_ids, dim, separation, seed)`` via + :meth:`sample`. ``separation`` is the dose-response knob (see the module + docstring): ``1.0`` gives near-orthogonal, easily separable descriptors, + ``0.0`` collapses every descriptor onto a shared mean (indistinguishable). + The mean pairwise inter-entity cosine similarity increases monotonically as + ``separation`` falls from ``1.0`` to ``0.0``. + """ + + model_config = ConfigDict(frozen=True) + + dim: int = _DEFAULT_DIM + separation: float + seed: int + entries: list[EntityAppearance] = [] + + @field_validator("dim") + @classmethod + def _positive_dim(cls, value: int) -> int: + if value < 1: + raise ValueError("dim must be >= 1") + return value + + @field_validator("separation") + @classmethod + def _separation_range(cls, value: float) -> float: + if not 0.0 <= value <= 1.0: + raise ValueError("separation must be in [0.0, 1.0]") + return value + + @classmethod + def sample( + cls, + entity_ids: list[str], + *, + dim: int = _DEFAULT_DIM, + separation: float, + seed: int, + ) -> AppearanceTable: + """Deterministically draw one unit-norm descriptor per entity. + + Draw order is fixed: a single ``rng.standard_normal((N, dim))`` call in + the given ``entity_ids`` order. Each raw draw is L2-normalized to a unit + anchor, then blended toward the shared mean of all unit anchors by + ``(1 - separation)`` and renormalized. As ``separation`` falls from + ``1.0`` to ``0.0`` every descriptor moves toward the common mean, so the + inter-entity cosine similarity rises monotonically. ``separation == 0.0`` + collapses all descriptors onto the shared mean (cosine ``1.0``). + + Edge cases: an empty ``entity_ids`` yields an empty table; a single + entity yields its (unchanged, unit-norm) anchor with no mean-blend. + """ + rng = np.random.default_rng(seed) + n = len(entity_ids) + if n == 0: + return cls(dim=dim, separation=separation, seed=seed, entries=[]) + + raw = rng.standard_normal((n, dim)) + # Unit anchors first, so the shared mean is not dominated by the largest + # raw draw (keeps the monotonicity closed form clean). + anchors = raw / np.linalg.norm(raw, axis=1, keepdims=True) + + if n == 1: + vectors = anchors + else: + mean = anchors.mean(axis=0, keepdims=True) + blended = separation * anchors + (1.0 - separation) * mean + norms = np.linalg.norm(blended, axis=1, keepdims=True) + # separation == 0 collapses every row exactly to `mean`; guard the + # (astronomically unlikely) zero-mean degenerate by falling back to + # the anchor so the descriptor stays unit-norm and finite. + safe = norms.squeeze(axis=1) > 0.0 + vectors = np.where(norms > 0.0, blended / np.where(norms > 0.0, norms, 1.0), anchors) + if not safe.all(): + vectors[~safe] = anchors[~safe] + + entries = [ + EntityAppearance(entity_id=entity_id, descriptor=[float(x) for x in vectors[i]]) + for i, entity_id in enumerate(entity_ids) + ] + return cls(dim=dim, separation=separation, seed=seed, entries=entries) + + def to_json(self, *, indent: int | None = 2) -> str: + """Serialise to a JSON string (the ``appearance.json`` sidecar payload).""" + return self.model_dump_json(indent=indent) + + +def write_appearance_json(table: AppearanceTable, path: str | Path) -> dict[str, Any]: + """Write an appearance table to ``path`` as JSON. + + Returns the dumped dict so a caller can assert on it without re-reading. + """ + data: dict[str, Any] = table.model_dump(mode="json") + Path(path).write_text(json.dumps(data, indent=2)) + return data From a4cec48fb53598eaa6a3ac326c599439c0a9d7b8 Mon Sep 17 00:00:00 2001 From: Bamdad Dashtban Date: Mon, 3 Aug 2026 13:51:29 +0100 Subject: [PATCH 2/3] Attach appearance sidecar to Scene as an additive optional field Scene.appearance: AppearanceTable | None = None. Same additive contract as possession/activity: None by default and never read by the manifest builder, so the byte-golden analytic manifest is unchanged whether or not the sidecar is present. --- src/multicam_sim/scene.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/multicam_sim/scene.py b/src/multicam_sim/scene.py index 81952d6..c197a40 100644 --- a/src/multicam_sim/scene.py +++ b/src/multicam_sim/scene.py @@ -7,6 +7,7 @@ from pydantic import BaseModel from .activity import ActivityTimeline +from .appearance import AppearanceTable from .cameras import Camera from .entities import Entity from .occluders import OccluderUnion @@ -31,6 +32,11 @@ class Scene(BaseModel): ``activity`` is an optional activity-state GT sidecar (see :mod:`multicam_sim.activity`) — same additive contract as ``possession``. + ``appearance`` is an optional appearance-descriptor GT sidecar (see + :mod:`multicam_sim.appearance`), a seeded pixel-free re-id cue — same + additive contract: ``None`` by default and never read by the manifest + builder, so the byte-golden analytic manifest is unchanged. + ``background`` / ``light`` are optional render-time environment values and ``randomization`` an optional provenance sidecar (spec + seed, see :mod:`multicam_sim.randomization`). All three are additive — ``None`` by @@ -46,6 +52,7 @@ class Scene(BaseModel): topology: CameraTopology | None = None possession: PossessionTimeline | None = None activity: ActivityTimeline | None = None + appearance: AppearanceTable | None = None background: Background | None = None light: Light | None = None randomization: RandomizationRecord | None = None From 638efde074d9bfb7c7187899224ede1f83d68a03 Mon Sep 17 00:00:00 2001 From: Bamdad Dashtban Date: Mon, 3 Aug 2026 13:51:29 +0100 Subject: [PATCH 3/3] Test the appearance sidecar: determinism, knob monotonicity, byte-golden Covers determinism across 3 seeds (identical descriptors for identical inputs, distinct-but-valid unit vectors across seeds), monotonicity of the separation knob (mean pairwise inter-entity cosine similarity strictly increases as separation falls 1.0 -> 0.0), unit-norm and dim-shape invariants, empty and single-entity edge cases, and that the descriptor does not encode the GT id. Byte-golden: a scene with the sidecar attached produces a manifest byte-identical to the same scene without it, and the manifest never contains appearance GT. --- tests/test_appearance.py | 162 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 tests/test_appearance.py diff --git a/tests/test_appearance.py b/tests/test_appearance.py new file mode 100644 index 0000000..500863a --- /dev/null +++ b/tests/test_appearance.py @@ -0,0 +1,162 @@ +"""Appearance-descriptor sidecar: a seeded, pixel-free, GT-derived re-id cue. + +Covers the contract of :mod:`multicam_sim.appearance`: deterministic sampling +across seeds, unit-norm/shape invariants, edge cases, the monotonic +``separation`` dose-response knob, and — the safety property — that attaching the +sidecar to a :class:`Scene` leaves the byte-golden analytic manifest untouched +(the manifest builder ignores it). + +The descriptor deliberately does NOT encode the ground-truth ``entity_id``; it +models OSNet-style confusability, and ``separation`` is the axis a re-id +evaluation sweeps to trace accuracy vs. identity separability. +""" + +from __future__ import annotations + +import math + +import numpy as np +import pytest + +from multicam_sim import build_manifest, build_smoke_scene +from multicam_sim.appearance import AppearanceTable, EntityAppearance +from multicam_sim.scene import Scene + +_ENTITY_IDS = ["entity_a", "entity_b", "entity_c", "entity_d"] + + +def _is_unit(vec: list[float], *, tol: float = 1e-9) -> bool: + return math.isclose(math.sqrt(sum(x * x for x in vec)), 1.0, abs_tol=tol) + + +def _mean_pairwise_cosine(table: AppearanceTable) -> float: + """Mean cosine similarity over all distinct entity pairs (unit descriptors, so + cosine == dot product).""" + vecs = [np.asarray(e.descriptor) for e in table.entries] + sims = [float(vecs[i] @ vecs[j]) for i in range(len(vecs)) for j in range(i + 1, len(vecs))] + return sum(sims) / len(sims) + + +def test_determinism_same_inputs_identical() -> None: + """Same (entity_ids, dim, separation, seed) -> byte-identical descriptors.""" + a = AppearanceTable.sample(_ENTITY_IDS, dim=512, separation=0.8, seed=7) + b = AppearanceTable.sample(_ENTITY_IDS, dim=512, separation=0.8, seed=7) + assert a == b + assert [e.descriptor for e in a.entries] == [e.descriptor for e in b.entries] + + +@pytest.mark.parametrize("seed", [0, 7, 42]) +def test_seeds_give_distinct_valid_unit_vectors(seed: int) -> None: + """Each seed yields valid unit-norm descriptors; different seeds differ.""" + table = AppearanceTable.sample(_ENTITY_IDS, dim=256, separation=0.9, seed=seed) + assert len(table.entries) == len(_ENTITY_IDS) + for entry in table.entries: + assert len(entry.descriptor) == 256 + assert _is_unit(entry.descriptor) + assert all(math.isfinite(x) for x in entry.descriptor) + + other = AppearanceTable.sample(_ENTITY_IDS, dim=256, separation=0.9, seed=seed + 1) + assert table.entries[0].descriptor != other.entries[0].descriptor + + +def test_descriptor_does_not_encode_entity_id() -> None: + """Renaming entities (same seed/dim/separation) keeps the descriptor vectors + identical: the vector is a function of draw order, not of the id string, so it + cannot be inverted to the GT id.""" + a = AppearanceTable.sample(["x0", "x1", "x2"], dim=128, separation=0.7, seed=3) + b = AppearanceTable.sample(["y0", "y1", "y2"], dim=128, separation=0.7, seed=3) + assert [e.descriptor for e in a.entries] == [e.descriptor for e in b.entries] + + +@pytest.mark.parametrize("seed", [0, 7, 42]) +def test_separation_knob_monotonic(seed: int) -> None: + """Mean pairwise inter-entity cosine similarity increases monotonically as + separation falls 1.0 -> 0.0 (the re-id dose-response axis). Coarse steps clear + the ~1/sqrt(dim) noise floor comfortably.""" + separations = [1.0, 0.75, 0.5, 0.25, 0.0] + sims = [ + _mean_pairwise_cosine(AppearanceTable.sample(_ENTITY_IDS, dim=512, separation=s, seed=seed)) + for s in separations + ] + # separations descend, so similarities must strictly ascend. + for lower, higher in zip(sims, sims[1:], strict=False): + assert higher > lower, f"non-monotone at seed {seed}: {sims}" + + # Endpoints: near-orthogonal (perfectly separable) at 1.0, collapsed at 0.0. + assert sims[0] < 0.2 + assert math.isclose(sims[-1], 1.0, abs_tol=1e-6) + + +def test_unit_norm_and_shape_invariants() -> None: + """Every descriptor is unit-norm and exactly `dim` long, across separations.""" + for separation in (1.0, 0.5, 0.0): + table = AppearanceTable.sample(_ENTITY_IDS, dim=64, separation=separation, seed=11) + assert table.dim == 64 + for entry in table.entries: + assert len(entry.descriptor) == 64 + assert _is_unit(entry.descriptor) + + +def test_empty_entity_ids() -> None: + """No entities -> empty table, no crash (mean never computed).""" + table = AppearanceTable.sample([], dim=512, separation=0.5, seed=1) + assert table.entries == [] + assert table.dim == 512 + + +def test_single_entity() -> None: + """One entity -> its unchanged unit anchor, no mean-blend, no pairwise math.""" + table = AppearanceTable.sample(["solo"], dim=128, separation=0.3, seed=2) + assert len(table.entries) == 1 + assert _is_unit(table.entries[0].descriptor) + # Separation is irrelevant with one entity: any separation gives the same + # anchor for a fixed seed. + other = AppearanceTable.sample(["solo"], dim=128, separation=0.9, seed=2) + assert table.entries[0].descriptor == other.entries[0].descriptor + + +def test_default_dim_is_512() -> None: + table = AppearanceTable.sample(_ENTITY_IDS, separation=0.8, seed=5) + assert table.dim == 512 + assert all(len(e.descriptor) == 512 for e in table.entries) + + +def test_validation_rejects_out_of_range_separation() -> None: + with pytest.raises(ValueError): + AppearanceTable(dim=8, separation=1.5, seed=0, entries=[]) + with pytest.raises(ValueError): + AppearanceTable(dim=0, separation=0.5, seed=0, entries=[]) + with pytest.raises(ValueError): + EntityAppearance(entity_id="e", descriptor=[]) + + +def test_roundtrip_json() -> None: + """Table survives a JSON round-trip unchanged (the sidecar payload).""" + table = AppearanceTable.sample(_ENTITY_IDS, dim=32, separation=0.6, seed=9) + restored = AppearanceTable.model_validate_json(table.to_json()) + assert restored == table + + +def _scene_with_appearance(*, attach: bool) -> Scene: + scene = build_smoke_scene() + if not attach: + return scene + ids = [e.id for e in scene.entities] + table = AppearanceTable.sample(ids, dim=512, separation=0.7, seed=13) + return scene.model_copy(update={"appearance": table}) + + +def test_manifest_byte_identical_with_and_without_sidecar() -> None: + """Attaching the appearance sidecar changes NOTHING in the analytic manifest: + the serialized bytes with and without it are exactly equal (manifest builder + ignores the sidecar; byte-golden default preserved).""" + with_side = build_manifest(_scene_with_appearance(attach=True)).to_json().encode() + without = build_manifest(_scene_with_appearance(attach=False)).to_json().encode() + assert with_side == without + + +def test_manifest_excludes_appearance_sidecar() -> None: + """The manifest never contains appearance GT, even when the scene carries it.""" + manifest_json = build_manifest(_scene_with_appearance(attach=True)).to_json() + assert "appearance" not in manifest_json + assert "descriptor" not in manifest_json