perf: UNet step sub-phase optimizations (5.0-5.6)

src/streamdiffusion/acceleration/tensorrt/engine_manager.py

@@ -138,7 +138,7 @@ def get_engine_path(
             if resolution is not None:
                 prefix = f"controlnet_{model_dir_name}--min_batch-{min_batch_size}--max_batch-{max_batch_size}--res-{resolution[0]}x{resolution[1]}"
             else:
-                prefix = f"controlnet_{model_dir_name}--min_batch-{min_batch_size}--max_batch-{max_batch_size}--dyn-384-1024"
+                prefix = f"controlnet_{model_dir_name}--min_batch-{min_batch_size}--max_batch-{max_batch_size}--dyn-256-1024"
             fp8_suffix = "--fp8" if fp8 else ""
             return self.engine_dir / (prefix + optlvl_suffix + fp8_suffix) / filename
         else:
@@ -277,7 +277,11 @@ def _get_default_controlnet_build_options(
             "build_static_batch": True,
         }
         if build_dynamic_shape:
-            opts["min_image_resolution"] = 384
+            # Match BaseModel/UNet's 256 floor (was 384) so [256, 384) resolutions
+            # don't hard-fail with ControlNet active — see get_input_profile in
+            # controlnet_models.py, which now derives its own floor from the same
+            # BaseModel.min_image_shape instead of a separate hardcoded literal.
+            opts["min_image_resolution"] = 256
             opts["max_image_resolution"] = 1024
         if builder_optimization_level is not None:
             opts["builder_optimization_level"] = builder_optimization_level

src/streamdiffusion/acceleration/tensorrt/models/controlnet_models.py

@@ -65,12 +65,14 @@ def get_input_profile(self, batch_size, image_height, image_width, static_batch,
             min_ctrl_h = max_ctrl_h = opt_ctrl_h = image_height
             min_ctrl_w = max_ctrl_w = opt_ctrl_w = image_width
         else:
-            min_ctrl_h = 384
-            max_ctrl_h = 1024
-            opt_ctrl_h = 704
-            min_ctrl_w = 384
-            max_ctrl_w = 1024
-            opt_ctrl_w = 704
+            # Share the floor/ceiling with BaseModel (min_image_shape/max_image_shape)
+            # instead of a hardcoded 384 — keeps ControlNet's dynamic range aligned with
+            # the UNet's (256-1024), so [256, 384) resolutions no longer hard-fail with
+            # ControlNet active. opt stays 704, clamped into range defensively.
+            min_ctrl_h, max_ctrl_h = self.min_image_shape, self.max_image_shape
+            min_ctrl_w, max_ctrl_w = self.min_image_shape, self.max_image_shape
+            opt_ctrl_h = min(max(704, min_ctrl_h), max_ctrl_h)
+            opt_ctrl_w = min(max(704, min_ctrl_w), max_ctrl_w)
 
         min_latent_h = min_ctrl_h // 8
         max_latent_h = max_ctrl_h // 8

src/streamdiffusion/acceleration/tensorrt/runtime_engines/controlnet_engine.py

@@ -42,6 +42,12 @@ def __init__(
         except Exception:
             self._cs_rank1 = False  # sd15 engines / engines lacking this input
 
+        # Persistent conditioning_scale scalar, allocated lazily on first __call__
+        # (device isn't known until then). fill_() is an async kernel launch, so
+        # reusing this buffer every call avoids the per-frame creation sync that
+        # torch.tensor(...) incurs.
+        self._cs: Optional[torch.Tensor] = None
+
         # Pre-compute model-specific values to eliminate runtime branching
         if self.model_type in ["sdxl", "sdxl_turbo"]:
             self.max_blocks = 9
@@ -119,16 +125,16 @@ def __call__(
         if timestep.dtype != torch.float32:
             timestep = timestep.float()
 
+        if self._cs is None:
+            self._cs = torch.zeros((1,) if self._cs_rank1 else (), dtype=torch.float32, device=sample.device)
+        self._cs.fill_(conditioning_scale)
+
         input_dict = {
             "sample": sample,
             "timestep": timestep,
             "encoder_hidden_states": encoder_hidden_states,
             "controlnet_cond": controlnet_cond,
-            "conditioning_scale": (
-                torch.tensor([conditioning_scale], dtype=torch.float32, device=sample.device)
-                if self._cs_rank1
-                else torch.tensor(conditioning_scale, dtype=torch.float32, device=sample.device)
-            ),
+            "conditioning_scale": self._cs,
         }
 
         if text_embeds is not None:

src/streamdiffusion/acceleration/tensorrt/runtime_engines/unet_engine.py

@@ -54,6 +54,13 @@ def __init__(self, filepath: str, stream: "cuda.Stream", use_cuda_graph: bool =
         self._fio_out_names: List[str] = []
         self._fio_cache_len: int = -1  # -1 = not yet initialized
 
+        # Sub-phase 5.6: kvo/fio cache inputs are persistent, address-stable,
+        # TRT-contiguous tensors (see models/utils.py create_kvo_cache/create_fi_cache),
+        # so Engine.infer() can bind TensorRT directly to them instead of copying into
+        # its own staging buffer every frame. Rebuilt only when the kvo/fio lazy-init
+        # branches below fire (cache length change), never per-frame.
+        self._zero_copy_names: FrozenSet[str] = frozenset()
+
         self._shape_dict: Dict[str, Any] = {}
         self._input_dict: Dict[str, Any] = {}
 
@@ -90,13 +97,15 @@ def __call__(
             self._kvo_in_names = [f"kvo_cache_in_{i}" for i in range(n_kvo)]
             self._kvo_out_names = [f"kvo_cache_out_{i}" for i in range(n_kvo)]
             self._kvo_cache_len = n_kvo
+            self._zero_copy_names = frozenset(self._kvo_in_names + self._fio_in_names)
 
         # Lazy-init FIO key name lists — same pattern as KVO.
         n_fio = len(fio_cache)
         if n_fio != self._fio_cache_len:
             self._fio_in_names = [f"fio_cache_in_{i}" for i in range(n_fio)]
             self._fio_out_names = [f"fio_cache_out_{i}" for i in range(n_fio)]
             self._fio_cache_len = n_fio
+            self._zero_copy_names = frozenset(self._kvo_in_names + self._fio_in_names)
 
         # Update pre-allocated dicts in-place — no new dict objects created per call.
         shape_dict = self._shape_dict
@@ -198,6 +207,7 @@ def __call__(
                 input_dict,
                 self.stream,
                 use_cuda_graph=self.use_cuda_graph,
+                zero_copy_names=self._zero_copy_names,
             )
         except Exception as e:
             logger.exception(f"UNet2DConditionModelEngine.__call__: Engine.infer failed: {e}")
@@ -496,7 +506,13 @@ def __init__(self, filepath: str, stream: "cuda.Stream", use_cuda_graph: bool =
         self.engine.load()
         self.engine.activate()
 
-    def __call__(self, image_tensor: torch.Tensor, threshold: float):
+    def __call__(self, image_tensor: torch.Tensor, prob_pin: torch.Tensor) -> None:
+        """Launch classification and stash the NSFW probability in a pinned host buffer.
+
+        Does not read the result back (no .item()/host sync here) — callers read
+        `prob_pin` on a *later* call, mirroring SimilarImageFilter's 1-frame-delayed
+        pinned readback so this never forces a cudaStreamSynchronize on the hot path.
+        """
         pixel_values = self.image_transforms(image_tensor)
         self.engine.allocate_buffers(
             shape_dict={
@@ -512,8 +528,8 @@ def __call__(self, image_tensor: torch.Tensor, threshold: float):
         )["logits"]
 
         probs = F.softmax(logits, dim=-1)
-        nsfw_prob = 1 - probs[0, 0].item()
-        return nsfw_prob >= threshold
+        nsfw_prob = 1 - probs[0, 0]
+        prob_pin.copy_(nsfw_prob.view(1), non_blocking=True)
 
     def to(self, *args, **kwargs):
         pass

src/streamdiffusion/acceleration/tensorrt/utilities.py

@@ -289,7 +289,7 @@ def _apply_gpu_profile_to_config(
     Args:
         config: TRT IBuilderConfig to modify.
         gpu_profile: Hardware-detected build parameters from detect_gpu_profile().
-        dynamic_shapes: Whether this engine uses dynamic input shapes.
+        dynamic_shapes: Whether this engine has any symbolic dim, incl. batch.
             - True  (default): tiling and l2_limit skipped — TRT confirms these have
               no effect on symbolic-shape graphs and only produce warning spam.
             - False (static): tiling and l2_limit applied for full L2 cache benefit.
@@ -541,6 +541,35 @@ def _median(v):
         return "\n".join(lines)
 
 
+def _staging_action(
+    name: str,
+    zero_copy_names: frozenset,
+    is_contiguous: bool,
+    dtype_match: bool,
+    prev_ptr: Optional[int],
+    cur_ptr: int,
+    graph_exists: bool,
+) -> str:
+    """Decide how Engine.infer() should stage one feed_dict input.
+
+    Pure function (no CUDA calls) so the zero-copy decision table is unit-testable
+    without a real CUDA graph — see Sub-phase 5.6,
+    docs/perf_bestpractices_audit_2026-07-10.md.
+
+    Returns one of:
+      "copy"           - copy into self.tensors[name] (today's behavior; always safe)
+      "bind"           - skip the copy; bind TensorRT directly to the caller's tensor
+      "bind_and_reset" - skip the copy and bind directly, but reset the CUDA graph
+                         first because the caller's address changed while a graph
+                         built from the old address was still live
+    """
+    if name not in zero_copy_names or not is_contiguous or not dtype_match:
+        return "copy"
+    if graph_exists and cur_ptr != prev_ptr:
+        return "bind_and_reset"
+    return "bind"
+
+
 class Engine:
     def __init__(
         self,
@@ -561,6 +590,11 @@ def __init__(
         # Cached ExternalStream wrapping the engine's polygraphy stream; allocated on
         # first infer() call so we avoid constructing a new Python wrapper every frame.
         self._engine_ext_stream = None
+        # Sub-phase 5.6: last-bound caller tensor address per zero-copy input name,
+        # so infer() can detect a pointer change and force a graph reset instead of
+        # silently replaying stale addresses. Only populated for names actually
+        # passed via infer()'s zero_copy_names.
+        self._bound_ptrs: dict[str, int] = {}
 
     def __del__(self):
         # Check if AttributeError: 'Engine' object has no attribute 'buffers'
@@ -1089,7 +1123,7 @@ def reset_cuda_graph(self):
             CUASSERT(cudart.cudaGraphDestroy(self.graph))
             self.graph = None
 
-    def infer(self, feed_dict, stream, use_cuda_graph=False):
+    def infer(self, feed_dict, stream, use_cuda_graph=False, zero_copy_names: frozenset = frozenset()):
         # IProfiler cannot report per-layer times through CUDA graph replay — disable graphs
         # when profiler is attached. This is automatically set when STREAMDIFFUSION_PROFILE_TRT
         # is set in activate(), so callers do not need to change anything.
@@ -1136,14 +1170,49 @@ def infer(self, feed_dict, stream, use_cuda_graph=False):
                     pt_stream,
                     stream.ptr,
                 )
+        # Sub-phase 5.6: for opt-in zero-copy names (kvo/fio UNet cache inputs — already
+        # persistent, address-stable, TRT-contiguous tensors), skip the DtoD copy and
+        # point TensorRT directly at the caller's tensor instead. Every other input
+        # (zero_copy_names defaults to frozenset()) takes the original copy_() path,
+        # so non-UNet engines (VAE/ControlNet/safety) are byte-for-byte unchanged.
+        bind_ptrs: dict[str, int] = {}
+        needs_reset = False
+        graph_exists = self.cuda_graph_instance is not None
         with torch.cuda.stream(self._engine_ext_stream):
             with _gpu_profiler.region("trt.input_staging"):
                 for name, buf in feed_dict.items():
-                    self.tensors[name].copy_(buf)
-
-        for name, tensor in self.tensors.items():
-            if not self.context.set_tensor_address(name, tensor.data_ptr()):
-                raise RuntimeError(f"TensorRT: set_tensor_address failed for '{name}'")
+                    cur_ptr = buf.data_ptr()
+                    action = _staging_action(
+                        name,
+                        zero_copy_names,
+                        buf.is_contiguous(),
+                        buf.dtype == self.tensors[name].dtype,
+                        self._bound_ptrs.get(name),
+                        cur_ptr,
+                        graph_exists,
+                    )
+                    if action == "copy":
+                        self.tensors[name].copy_(buf)
+                    else:
+                        bind_ptrs[name] = cur_ptr
+                        if action == "bind_and_reset":
+                            needs_reset = True
+
+        if needs_reset and self.cuda_graph_instance is not None:
+            self.reset_cuda_graph()
+
+        # In graphed steady state the tensor addresses are baked into the captured graph
+        # (self.tensors[name] are persistent buffers reused via copy_() — see
+        # _can_reuse_buffers/allocate_buffers, which resets cuda_graph_instance to None
+        # whenever a buffer is actually reallocated). Re-binding every frame is then pure
+        # host overhead; only rebind on first capture or after a reset (instance is None).
+        if not (use_cuda_graph and self.cuda_graph_instance is not None):
+            for name, tensor in self.tensors.items():
+                address = bind_ptrs.get(name, tensor.data_ptr())
+                if not self.context.set_tensor_address(name, address):
+                    raise RuntimeError(f"TensorRT: set_tensor_address failed for '{name}'")
+                if name in bind_ptrs:
+                    self._bound_ptrs[name] = address
 
         with _gpu_profiler.region("trt_infer"):
             if use_cuda_graph:
@@ -1366,7 +1435,12 @@ def build_engine(
         workspace_size=max_workspace_size,
         fp8=fp8,
         gpu_profile=gpu_profile,
-        dynamic_shapes=build_dynamic_shape,
+        # Any symbolic dim (resolution OR batch) disqualifies l2tc tiling — see
+        # _apply_gpu_profile_to_config. build_dynamic_shape alone misses the
+        # batch-dynamic/resolution-static case (e.g. default "Flexible" UNet
+        # preset), which previously reached the tiling branch and TRT emitted
+        # "[l2tc] VALIDATE FAIL - Graph contains symbolic shape" as a no-op.
+        dynamic_shapes=build_dynamic_shape or not build_static_batch,
     )
 
     return engine