From e6dee738a84239ebe038a709a2c4245b2896d04e Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 10 Jul 2026 12:55:41 -0400 Subject: [PATCH 01/37] refactor: re-apply Phase 1 perf/best-practices cleanup (cosmetic + fail-fast) Re-applies the previously-reverted Phase 1 batch from the perf/best-practices audit: fix the FP8 Q/DQ gate comment to match the actual threshold, convert a dict() call to a dict literal (clears the one pre-existing ruff C408 on a touched file), require an explicit opt_batch_size in EngineBuilder.build() and its 5 compile_* wrappers (real callers already always pass it), drop a dead try/except around a single logger.info call, and tighten two type hints in StreamDiffusion.pipeline (Optional/Union/Tuple). No behavior change except the opt_batch_size fail-fast, which only affects callers that previously relied on an unused default of 1. --- .../acceleration/tensorrt/__init__.py | 10 ++++---- .../acceleration/tensorrt/builder.py | 22 +++++++++-------- src/streamdiffusion/pipeline.py | 24 +++++++------------ src/streamdiffusion/wrapper.py | 16 ++++++------- 4 files changed, 34 insertions(+), 38 deletions(-) diff --git a/src/streamdiffusion/acceleration/tensorrt/__init__.py b/src/streamdiffusion/acceleration/tensorrt/__init__.py index 7a74b65a2..b4c7a7ca2 100644 --- a/src/streamdiffusion/acceleration/tensorrt/__init__.py +++ b/src/streamdiffusion/acceleration/tensorrt/__init__.py @@ -66,7 +66,7 @@ def compile_vae_encoder( onnx_path: str, onnx_opt_path: str, engine_path: str, - opt_batch_size: int = 1, + opt_batch_size: Optional[int] = None, engine_build_options: Optional[dict] = None, ): vae = vae.to(torch.device("cuda")) @@ -86,7 +86,7 @@ def compile_vae_decoder( onnx_path: str, onnx_opt_path: str, engine_path: str, - opt_batch_size: int = 1, + opt_batch_size: Optional[int] = None, engine_build_options: Optional[dict] = None, ): vae = vae.to(torch.device("cuda")) @@ -106,7 +106,7 @@ def compile_safety_checker( onnx_path: str, onnx_opt_path: str, engine_path: str, - opt_batch_size: int = 1, + opt_batch_size: Optional[int] = None, engine_build_options: Optional[dict] = None, ): safety_checker = safety_checker.to(torch.device("cuda")) @@ -126,7 +126,7 @@ def compile_unet( onnx_path: str, onnx_opt_path: str, engine_path: str, - opt_batch_size: int = 1, + opt_batch_size: Optional[int] = None, engine_build_options: Optional[dict] = None, ): # Extract FP8-specific options before passing the rest to EngineBuilder.build(). @@ -177,7 +177,7 @@ def compile_controlnet( onnx_path: str, onnx_opt_path: str, engine_path: str, - opt_batch_size: int = 1, + opt_batch_size: Optional[int] = None, engine_build_options: Optional[dict] = None, ): build_options = dict(engine_build_options or {}) diff --git a/src/streamdiffusion/acceleration/tensorrt/builder.py b/src/streamdiffusion/acceleration/tensorrt/builder.py index 2b6672bde..045dd7390 100644 --- a/src/streamdiffusion/acceleration/tensorrt/builder.py +++ b/src/streamdiffusion/acceleration/tensorrt/builder.py @@ -93,7 +93,7 @@ def build( engine_path: str, opt_image_height: int = 512, opt_image_width: int = 512, - opt_batch_size: int = 1, + opt_batch_size: Optional[int] = None, min_image_resolution: int = 256, max_image_resolution: int = 1024, build_enable_refit: bool = False, @@ -118,6 +118,8 @@ def build( is_controlnet: bool = False, artifact_prefix: str = "unet", ): + if opt_batch_size is None: + raise ValueError("build() requires an explicit opt_batch_size") build_total_start = time.perf_counter() engine_name = Path(engine_path).parent.name engine_filename = Path(engine_path).name @@ -146,14 +148,14 @@ def build( else: print(f"Exporting model: {onnx_path}") t0 = time.perf_counter() - _export_kwargs = dict( - onnx_path=onnx_path, - model_data=self.model, - opt_image_height=opt_image_height, - opt_image_width=opt_image_width, - opt_batch_size=opt_batch_size, - onnx_opset=onnx_opset, - ) + _export_kwargs = { + "onnx_path": onnx_path, + "model_data": self.model, + "opt_image_height": opt_image_height, + "opt_image_width": opt_image_width, + "opt_batch_size": opt_batch_size, + "onnx_opset": onnx_opset, + } export_onnx(self.network, **_export_kwargs) elapsed = time.perf_counter() - t0 stats["stages"]["onnx_export"] = {"status": "built", "elapsed_s": round(elapsed, 2)} @@ -304,7 +306,7 @@ def _quant_fn(): stats["stages"]["trt_build"] = {"status": "built", "elapsed_s": round(elapsed, 2)} _build_logger.info(f"[BUILD] TRT engine build ({engine_filename}): {elapsed:.1f}s") - # --- FP8 Q/DQ layer count (sanity gate: < 100 means quantization is inactive) --- + # --- FP8 Q/DQ layer count (sanity gate: < 500 means quantization is inactive) --- if fp8 and os.path.exists(engine_path): try: import json as _json diff --git a/src/streamdiffusion/pipeline.py b/src/streamdiffusion/pipeline.py index 7fa0c6f48..f2771a3ad 100644 --- a/src/streamdiffusion/pipeline.py +++ b/src/streamdiffusion/pipeline.py @@ -631,11 +631,7 @@ def prepare( _use_seq_loop = not (self.use_denoising_batch and isinstance(self.scheduler, LCMScheduler)) if _use_seq_loop and self.sub_timesteps_tensor.dim() >= 1: self._sub_timesteps_expanded = ( - self.sub_timesteps_tensor - .view(-1) - .unsqueeze(1) - .expand(-1, self.frame_bff_size) - .contiguous() + self.sub_timesteps_tensor.view(-1).unsqueeze(1).expand(-1, self.frame_bff_size).contiguous() ) # shape [loop_steps, frame_bff_size] else: self._sub_timesteps_expanded = None @@ -726,14 +722,12 @@ def _refresh_derived_tensors(self) -> None: dtype=self.dtype, device=self.device, ) - self._cfg_t_buf = torch.empty( - cfg_batch, dtype=self.sub_timesteps_tensor.dtype, device=self.device - ) + self._cfg_t_buf = torch.empty(cfg_batch, dtype=self.sub_timesteps_tensor.dtype, device=self.device) else: self._cfg_latent_buf = None self._cfg_t_buf = None - def _get_scheduler_scalings(self, timestep): + def _get_scheduler_scalings(self, timestep: Union[int, torch.Tensor]) -> Tuple[torch.Tensor, torch.Tensor]: """Get LCM/TCD-specific scaling factors for boundary conditions.""" if isinstance(self.scheduler, LCMScheduler): c_skip, c_out = self.scheduler.get_scalings_for_boundary_condition_discrete(timestep) @@ -1146,8 +1140,10 @@ def predict_x0_batch(self, x_t_latent: torch.Tensor) -> torch.Tensor: # Resolve scalar timestep tensor on device. # Avoid per-step t.view(1).repeat() allocations by using the # precomputed _sub_timesteps_expanded table from prepare(). - t = timestep if isinstance(timestep, torch.Tensor) else torch.tensor( - timestep, device=self.device, dtype=torch.long + t = ( + timestep + if isinstance(timestep, torch.Tensor) + else torch.tensor(timestep, device=self.device, dtype=torch.long) ) if self._sub_timesteps_expanded is not None: t_expanded = self._sub_timesteps_expanded[idx] # [frame_bff_size], no alloc @@ -1188,7 +1184,7 @@ def predict_x0_batch(self, x_t_latent: torch.Tensor) -> torch.Tensor: return x_0_pred_out @torch.inference_mode() - def __call__(self, x: Union[torch.Tensor, PIL.Image.Image, np.ndarray] = None) -> torch.Tensor: + def __call__(self, x: Optional[Union[torch.Tensor, PIL.Image.Image, np.ndarray]] = None) -> torch.Tensor: start = self._timing_start end = self._timing_end if self.similar_image_filter: @@ -1241,9 +1237,7 @@ def __call__(self, x: Union[torch.Tensor, PIL.Image.Image, np.ndarray] = None) - torch.cuda.empty_cache() raise if "expanded size" in _msg and "must match" in _msg: - logger.error( - "StreamDiffusion.__call__: tensor size mismatch — attempting scheduler rebuild: %s", _e - ) + logger.error("StreamDiffusion.__call__: tensor size mismatch — attempting scheduler rebuild: %s", _e) try: self._param_updater._update_timestep_calculations() self._refresh_derived_tensors() diff --git a/src/streamdiffusion/wrapper.py b/src/streamdiffusion/wrapper.py index f17a57422..f0579fce1 100644 --- a/src/streamdiffusion/wrapper.py +++ b/src/streamdiffusion/wrapper.py @@ -737,8 +737,7 @@ def update_stream_params( _normalized = [(str(p), float(w)) for p, w in prompt_list] _current = self.stream._param_updater.get_current_prompts() _neg_unchanged = ( - negative_prompt is None - or negative_prompt == self.stream._param_updater._current_negative_prompt + negative_prompt is None or negative_prompt == self.stream._param_updater._current_negative_prompt ) if _normalized == _current and _neg_unchanged: logger.debug( @@ -2100,10 +2099,7 @@ def _load_model( num_ip_layers = getattr(temp_wrapped_unet.ipadapter_wrapper, "num_ip_layers", None) if not isinstance(num_ip_layers, int) or num_ip_layers <= 0: raise RuntimeError("Failed to determine num_ip_layers for IP-Adapter") - try: - logger.info(f"compile_and_load_engine: discovered num_ip_layers={num_ip_layers}") - except Exception: - pass + logger.info(f"compile_and_load_engine: discovered num_ip_layers={num_ip_layers}") unet_model = UNet( stream.unet, @@ -2358,7 +2354,9 @@ def _install_cached_proc(attn_module): try: del stream.unet except Exception as del_error: - logger.debug(f"Failed to delete stream.unet during OOM fallback: {del_error}", exc_info=True) + logger.debug( + f"Failed to delete stream.unet during OOM fallback: {del_error}", exc_info=True + ) self.cleanup_gpu_memory() @@ -2416,7 +2414,9 @@ def _install_cached_proc(attn_module): try: del stream.vae except Exception as del_error: - logger.debug(f"Failed to delete stream.vae during OOM fallback: {del_error}", exc_info=True) + logger.debug( + f"Failed to delete stream.vae during OOM fallback: {del_error}", exc_info=True + ) self.cleanup_gpu_memory() From 1070583b4e748de7112af43f725c01e3da539172 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 10 Jul 2026 13:08:29 -0400 Subject: [PATCH 02/37] refactor: Phase 2 exception hygiene and observability fixes - Add shared _is_oom_error() helper (typed torch.cuda.OutOfMemoryError + string-heuristic fallback) and use it at both TensorRT UNet/VAE engine build fallback sites instead of duplicated inline substring checks. - Log swallowed LoRA candidate-weight-name failures in _load_lora_with_offline_fallback instead of silently continuing. - Log the inner fallback failure in advanced-model-detection instead of a bare except/pass. - Hoist the optional diffusers_ipadapter helper imports in IPAdapterModule.build_unet_hook out of the per-frame hook into a single build-time resolution, and log all previously-silent per-step fallback branches for observability. - Make _load_model fail fast (raise RuntimeError) when an SDXL pipeline retry after a type mismatch also fails, instead of silently continuing with the wrong pipeline type; also null out the stale mismatched `pipe` reference so a later loading-method failure can't let it slip through the final success check. - Add regression tests for _is_oom_error and the SDXL fail-fast path. --- .../modules/ipadapter_module.py | 62 ++++--- src/streamdiffusion/pipeline.py | 1 + src/streamdiffusion/wrapper.py | 46 +++--- tests/unit/test_wrapper_exception_hygiene.py | 152 ++++++++++++++++++ 4 files changed, 220 insertions(+), 41 deletions(-) create mode 100644 tests/unit/test_wrapper_exception_hygiene.py diff --git a/src/streamdiffusion/modules/ipadapter_module.py b/src/streamdiffusion/modules/ipadapter_module.py index 5bc10579a..caca6c3c1 100644 --- a/src/streamdiffusion/modules/ipadapter_module.py +++ b/src/streamdiffusion/modules/ipadapter_module.py @@ -431,6 +431,26 @@ def build_unet_hook(self, stream) -> UnetHook: """ _last_enabled_state = None # Track previous enabled state to avoid redundant updates + # Resolve the optional diffusers_ipadapter helpers once, at hook-build time, + # instead of importing them on every frame inside the per-step hook below. + build_time_weight_factor = None + build_layer_weights = None + try: + from diffusers_ipadapter.ip_adapter.attention_processor import ( + build_layer_weights as _build_layer_weights, + ) + from diffusers_ipadapter.ip_adapter.attention_processor import ( + build_time_weight_factor as _build_time_weight_factor, + ) + + build_time_weight_factor = _build_time_weight_factor + build_layer_weights = _build_layer_weights + except Exception as import_error: + logger.debug( + f"IPAdapterModule.build_unet_hook: diffusers_ipadapter helpers unavailable: {import_error}", + exc_info=True, + ) + def _unet_hook(ctx: StepCtx) -> UnetKwargsDelta: # If no IP-Adapter installed, do nothing if not hasattr(stream, "ipadapter") or stream.ipadapter is None: @@ -442,7 +462,8 @@ def _unet_hook(ctx: StepCtx) -> UnetKwargsDelta: # Read base weight and weight type from IPAdapter instance try: base_weight = float(getattr(stream.ipadapter, "scale", 1.0)) if enabled else 0.0 - except Exception: + except Exception as scale_error: + logger.debug(f"IPAdapterModule: failed to read ipadapter.scale: {scale_error}", exc_info=True) base_weight = 0.0 if not enabled else 1.0 weight_type = getattr(stream.ipadapter, "weight_type", None) @@ -453,41 +474,38 @@ def _unet_hook(ctx: StepCtx) -> UnetKwargsDelta: total_steps = int(stream.denoising_steps_num) elif hasattr(stream, "t_list") and stream.t_list is not None: total_steps = len(stream.t_list) - except Exception: + except Exception as steps_error: + logger.debug(f"IPAdapterModule: failed to determine total_steps: {steps_error}", exc_info=True) total_steps = None time_factor = 1.0 - if total_steps is not None and ctx.step_index is not None: + if total_steps is not None and ctx.step_index is not None and build_time_weight_factor is not None: try: - from diffusers_ipadapter.ip_adapter.attention_processor import build_time_weight_factor - time_factor = float(build_time_weight_factor(weight_type, int(ctx.step_index), int(total_steps))) - except Exception: + except Exception as scale_factor_error: # Do not add fallback mechanisms - pass + logger.debug( + f"IPAdapterModule: build_time_weight_factor failed: {scale_factor_error}", exc_info=True + ) # TensorRT engine path: supply ipadapter_scale vector via extra kwargs try: is_trt_unet = ( hasattr(stream, "unet") and hasattr(stream.unet, "engine") and hasattr(stream.unet, "stream") ) - except Exception: + except Exception as trt_check_error: + logger.debug(f"IPAdapterModule: TensorRT UNet detection failed: {trt_check_error}", exc_info=True) is_trt_unet = False if is_trt_unet and getattr(stream.unet, "use_ipadapter", False): - try: - from diffusers_ipadapter.ip_adapter.attention_processor import build_layer_weights - except Exception: - # If helper unavailable, do not construct weights here - build_layer_weights = None # type: ignore - num_ip_layers = getattr(stream.unet, "num_ip_layers", None) if isinstance(num_ip_layers, int) and num_ip_layers > 0: weights_tensor = None try: if build_layer_weights is not None: weights_tensor = build_layer_weights(num_ip_layers, float(base_weight), weight_type) - except Exception: + except Exception as weights_error: + logger.debug(f"IPAdapterModule: build_layer_weights failed: {weights_error}", exc_info=True) weights_tensor = None if weights_tensor is None: weights_tensor = torch.full( @@ -496,8 +514,11 @@ def _unet_hook(ctx: StepCtx) -> UnetKwargsDelta: # Apply per-step time factor try: weights_tensor = weights_tensor * float(time_factor) - except Exception: - pass + except Exception as time_scale_error: + logger.debug( + f"IPAdapterModule: applying time_factor to weights_tensor failed: {time_scale_error}", + exc_info=True, + ) return UnetKwargsDelta(extra_unet_kwargs={"ipadapter_scale": weights_tensor}) # PyTorch UNet path: modulate installed processor scales by time factor and enabled state @@ -513,8 +534,11 @@ def _unet_hook(ctx: StepCtx) -> UnetKwargsDelta: # Apply both enabled state and time factor final_scale = float(base_val) * float(time_factor) if enabled else 0.0 proc.scale = final_scale - except Exception: - pass + except Exception as processor_scale_error: + logger.debug( + f"IPAdapterModule: PyTorch UNet processor scale update failed: {processor_scale_error}", + exc_info=True, + ) return UnetKwargsDelta() diff --git a/src/streamdiffusion/pipeline.py b/src/streamdiffusion/pipeline.py index f2771a3ad..e8964e763 100644 --- a/src/streamdiffusion/pipeline.py +++ b/src/streamdiffusion/pipeline.py @@ -365,6 +365,7 @@ def _load_lora_with_offline_fallback( return except Exception as e: last_err = e + logger.debug(f"load_lora_weights: candidate {weight_name!r} failed: {e}", exc_info=True) continue if last_err is not None: diff --git a/src/streamdiffusion/wrapper.py b/src/streamdiffusion/wrapper.py index f0579fce1..1e9c3463c 100644 --- a/src/streamdiffusion/wrapper.py +++ b/src/streamdiffusion/wrapper.py @@ -17,6 +17,19 @@ logger = logging.getLogger(__name__) + +def _is_oom_error(exc: BaseException) -> bool: + """Detect CUDA out-of-memory errors, including ones surfaced as generic + RuntimeErrors by third-party code (e.g. TensorRT) rather than the typed + torch.cuda.OutOfMemoryError.""" + if isinstance(exc, torch.cuda.OutOfMemoryError): + return True + error_msg = str(exc).lower() + return ( + "out of memory" in error_msg or "outofmemory" in error_msg or "oom" in error_msg or "cuda error" in error_msg + ) + + # Text-encoder CPU offload frees ~1.6 GB VRAM but each prompt update pays a # CPU<->GPU round-trip plus torch.cuda.empty_cache() — a measurable stall # mid-stream on high-VRAM GPUs. Default off (encoders stay resident on GPU); @@ -1509,8 +1522,13 @@ def _load_model( pipe = StableDiffusionXLPipeline.from_single_file(model_id_or_path).to(dtype=self.dtype) logger.info("_load_model: Successfully loaded using SDXL pipeline on retry") except Exception as retry_error: - logger.warning(f"_load_model: SDXL pipeline retry failed: {retry_error}") - # Continue with the originally loaded pipeline + # Discard the mismatched-type pipe so a subsequent loading-method + # failure can't leave a wrong-type pipe looking like a success. + pipe = None + raise RuntimeError( + f"_load_model: SDXL model detected but pipeline retry with " + f"StableDiffusionXLPipeline also failed: {retry_error}" + ) from retry_error break except Exception as e: @@ -1851,8 +1869,8 @@ def _load_model( unet_arch = extract_unet_architecture(stream.unet) unet_arch = validate_architecture(unet_arch, model_type) use_controlnet_trt = True - except Exception: - pass + except Exception as fallback_error: + logger.error(f"Basic fallback detection also failed: {fallback_error}", exc_info=True) if not use_controlnet_trt and not self.use_controlnet: logger.info("ControlNet not enabled, building engines without ControlNet support") @@ -2336,15 +2354,7 @@ def _install_cached_proc(attn_module): ) except Exception as e: - error_msg = str(e).lower() - is_oom_error = ( - "out of memory" in error_msg - or "outofmemory" in error_msg - or "oom" in error_msg - or "cuda error" in error_msg - ) - - if is_oom_error: + if _is_oom_error(e): logger.error(f"TensorRT UNet engine OOM: {e}") logger.info("Falling back to PyTorch UNet (no TensorRT acceleration)") logger.info("This will be slower but should work with less memory") @@ -2396,15 +2406,7 @@ def _install_cached_proc(attn_module): logger.info("TensorRT VAE engines loaded successfully") except Exception as e: - error_msg = str(e).lower() - is_oom_error = ( - "out of memory" in error_msg - or "outofmemory" in error_msg - or "oom" in error_msg - or "cuda error" in error_msg - ) - - if is_oom_error: + if _is_oom_error(e): logger.error(f"TensorRT VAE engine OOM: {e}") logger.info("Falling back to PyTorch VAE (no TensorRT acceleration)") logger.info("This will be slower but should work with less memory") diff --git a/tests/unit/test_wrapper_exception_hygiene.py b/tests/unit/test_wrapper_exception_hygiene.py new file mode 100644 index 000000000..af10d1e68 --- /dev/null +++ b/tests/unit/test_wrapper_exception_hygiene.py @@ -0,0 +1,152 @@ +""" +Regression tests for wrapper.py exception-hygiene fixes (Phase 2 of the +perf/best-practices remediation, docs/perf_bestpractices_audit_2026-07-10.md). + +Covers two independent fixes: + +1. `_is_oom_error` — a shared OOM-detection helper that recognizes both the + typed `torch.cuda.OutOfMemoryError` and the string-matching heuristic used + by third-party code (e.g. TensorRT) that surfaces OOM as a generic + RuntimeError. Pure function, CPU-only. + +2. `_load_model`'s SDXL pipeline-type-mismatch handling — previously, when an + SDXL model was detected but loaded with the wrong pipeline type and the + explicit `StableDiffusionXLPipeline` retry also failed, the code silently + logged a warning and continued with the mismatched pipeline. The fix + raises RuntimeError on retry failure instead, refusing to proceed with a + known-wrong pipeline type. That raise is caught by the enclosing + `for method in loading_methods` loop's own except/continue, so it falls + through to the next loading method rather than crashing outright — this + test drives all three loading methods to fail and asserts the final error + carries evidence that the fail-fast fired (rather than the old silent + continuation). + +Both tests are deliberately CPU-only and model-free (no real diffusers model +is loaded), following the object.__new__ shell pattern used elsewhere in +tests/unit/ (see test_safety_checker.py, test_normal_bae_fallback.py). +""" + +from unittest.mock import patch + +import pytest +import torch + +from streamdiffusion import wrapper as wrapper_module +from streamdiffusion.wrapper import StreamDiffusionWrapper, _is_oom_error + + +# --------------------------------------------------------------------------- +# _is_oom_error +# --------------------------------------------------------------------------- + + +class TestIsOomError: + def test_typed_cuda_oom_is_detected(self): + exc = torch.cuda.OutOfMemoryError("CUDA out of memory. Tried to allocate 2.00 GiB") + assert _is_oom_error(exc) is True + + @pytest.mark.parametrize( + "message", + [ + "CUDA out of memory. Tried to allocate 20.00 MiB", + "torch.OutOfMemoryError: OutOfMemory", + "generic oom while building engine", + "CUDA error: an illegal memory access was encountered", + ], + ) + def test_string_heuristic_matches_known_oom_substrings(self, message): + assert _is_oom_error(RuntimeError(message)) is True + + def test_unrelated_error_is_not_oom(self): + assert _is_oom_error(RuntimeError("shape mismatch: expected [1, 4, 64, 64]")) is False + assert _is_oom_error(ValueError("invalid literal for int()")) is False + + +# --------------------------------------------------------------------------- +# SDXL pipeline-mismatch fail-fast +# --------------------------------------------------------------------------- + + +class _FakeWrongTypePipe: + """Stand-in for a successfully-loaded but non-SDXL pipeline object.""" + + def to(self, *args, **kwargs): + return self + + +def _make_wrapper_shell() -> StreamDiffusionWrapper: + """Construct a minimal StreamDiffusionWrapper without model loading.""" + w = object.__new__(StreamDiffusionWrapper) + w.device = torch.device("cpu") + w.dtype = torch.float32 + w.cleanup_gpu_memory = lambda: None + return w + + +class TestSdxlPipelineMismatchFailFast: + def test_retry_failure_raises_instead_of_continuing_silently(self, caplog): + """ + Path contains 'sdxl' + .safetensors -> loading_methods tries + StableDiffusionXLPipeline.from_single_file first. Its first call + succeeds but returns a non-SDXL-typed pipe, triggering the mismatch + branch; the retry (second call to the same mocked method) fails. + The other two loading methods are also made to fail so _load_model + exhausts all methods and raises its final RuntimeError. + + Two things must hold: + - the mismatch branch's failure is logged as a "pipeline retry ... + also failed" warning (proving the fail-fast fired instead of the + old silent "continue with the originally loaded pipeline"), and + - _load_model does not crash trying to use the discarded + wrong-typed pipe as if it were a valid load (regression guard for + the stale `pipe` reference the fail-fast must clear). + """ + xl_call_count = [0] + + def fake_xl_from_single_file(path, *args, **kwargs): + xl_call_count[0] += 1 + if xl_call_count[0] == 1: + return _FakeWrongTypePipe() + raise RuntimeError("SDXL pipeline retry boom") + + def fake_auto_from_pretrained(path, *args, **kwargs): + raise RuntimeError("AutoPipeline boom") + + def fake_sd_from_single_file(path, *args, **kwargs): + raise RuntimeError("SD pipeline boom") + + w = _make_wrapper_shell() + + with ( + patch.object( + wrapper_module.StableDiffusionXLPipeline, + "from_single_file", + staticmethod(fake_xl_from_single_file), + ), + patch.object( + wrapper_module.AutoPipelineForText2Image, + "from_pretrained", + staticmethod(fake_auto_from_pretrained), + ), + patch.object( + wrapper_module.StableDiffusionPipeline, + "from_single_file", + staticmethod(fake_sd_from_single_file), + ), + caplog.at_level("WARNING", logger="streamdiffusion.wrapper"), + ): + with pytest.raises(RuntimeError) as exc_info: + w._load_model("fake_sdxl_model.safetensors", t_index_list=[0]) + + # The XL method must have been called twice: once for the initial + # attempt (wrong-typed success) and once for the mismatch retry. + assert xl_call_count[0] == 2, "expected exactly one retry attempt after the type mismatch" + + # The final error is "all methods exhausted" (the last method's error) - + # that's expected once every loading method has failed. What matters is + # that the mismatch retry's own failure was surfaced, not swallowed. + assert "all loading methods failed" in str(exc_info.value).lower() + assert any( + "pipeline retry" in record.message.lower() and "also failed" in record.message.lower() + for record in caplog.records + ), "expected a logged warning proving the SDXL retry fail-fast fired, not a silent continue" From 1559d6987bbde4d5a02ade429c5a25bf54ee9c0d Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 10 Jul 2026 13:16:53 -0400 Subject: [PATCH 03/37] chore: migrate deprecated top-level ruff settings to [tool.ruff.lint] Ruff warns that top-level ignore/select/isort/per-file-ignores are deprecated in favour of the lint section. Move them under [tool.ruff.lint] (and .isort / .per-file-ignores); line-length stays at the top level. Verified against cuda-link's pyproject.toml, which already uses this layout. No behavior change: ruff check/format report identical results before and after (same 479 pre-existing repo-wide findings, 0 on the files touched by the Phase 1/2 commits). Investigated adding a matching [tool.pyrefly] section for parity with cuda-link, but did not adopt it: pyrefly falls back to a bundled 'basic' preset when no pyrefly config exists, and that preset is substantially more lenient than explicit config. Adding even an empty [tool.pyrefly] section opts out of it and jumps reported errors from 72 to ~900 (mostly dynamically-set attributes on StreamDiffusion / StreamDiffusionWrapper that were never checked before). Doing this properly needs the same per-module triage cuda-link performed (docs/adr/0005-static-typing-hardening.md) before turning on strict project-includes checking, which is out of scope here. --- pyproject.toml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5fd693ae1..3b4b63bd0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,14 +1,16 @@ [tool.ruff] +line-length = 119 + +[tool.ruff.lint] # Never enforce `E501` (line length violations). ignore = ["C901", "E501", "E741", "F402", "F403", "F405", "F823"] select = ["C", "E", "F", "I", "W"] -line-length = 119 # Ignore import violations in all `__init__.py` files. -[tool.ruff.per-file-ignores] +[tool.ruff.lint.per-file-ignores] "__init__.py" = ["E402", "F401", "F811"] -[tool.ruff.isort] +[tool.ruff.lint.isort] lines-after-imports = 2 known-first-party = ["streamdiffusion"] @@ -23,4 +25,4 @@ indent-style = "space" skip-magic-trailing-comma = false # Like Black, automatically detect the appropriate line ending. -line-ending = "auto" \ No newline at end of file +line-ending = "auto" From cf3df187a4baad5ad7cbc1989eb46aa8996ba568 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 10 Jul 2026 14:04:33 -0400 Subject: [PATCH 04/37] fix: sync _make_fake_engine test double with TensorRTEngine.__init__ _make_fake_engine (tests/unit/test_trt_engine_guards.py) built engines via TensorRTEngine.__new__() to bypass __init__, but never set _dedicated_stream, _pre_exec_event, _post_exec_event, or _buf_cache -- attributes __init__ has initialized since infer() gained cross-stream sync and LRU shape-cache support. This caused 3 AttributeErrors long treated as "pre-existing, out of scope" failures in every prior phase gate. Not a production bug: __init__ correctly sets all four to None/empty, and infer() guards each with `is not None`. Fix mirrors __init__ in the fake. Full unit suite now fully green (91 passed, 0 failed) with no waived tests. --- tests/unit/test_trt_engine_guards.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/unit/test_trt_engine_guards.py b/tests/unit/test_trt_engine_guards.py index 435ce5f2a..759bc252a 100644 --- a/tests/unit/test_trt_engine_guards.py +++ b/tests/unit/test_trt_engine_guards.py @@ -59,6 +59,13 @@ def _make_fake_engine( eng = TensorRTEngine.__new__(TensorRTEngine) eng.engine_path = "/fake/test.engine" eng._cuda_stream = None + # Mirror the remaining TensorRTEngine.__init__ defaults that infer() relies + # on (pre-activate() state: no dedicated stream/events yet, empty LRU cache). + # Keep this in sync with __init__ -- infer() will AttributeError otherwise. + eng._dedicated_stream = None + eng._pre_exec_event = None + eng._post_exec_event = None + eng._buf_cache = OrderedDict() eng.engine = MagicMock() eng.context = MagicMock() From 37443dbfac35f8216ce8081ed8edf1dcca65741b Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 10 Jul 2026 14:05:46 -0400 Subject: [PATCH 05/37] fix: Phase 3 correctness and resource-safety fixes Four targeted fixes from the perf/best-practices audit, quick-wins #4 and #5: - StreamDiffusion.prepare(): generator defaulted to a pre-constructed torch.Generator(), a mutable default evaluated once at def-time so every instance sharing the default shared one Generator. Default is now None, constructed fresh in the method body. - StreamDiffusionWrapper.prepare(): both runtime stream.prepare() calls (single-prompt and prompt-blending paths) omitted seed, so every runtime prompt change silently reset the RNG to stream.prepare()'s own default. Now forwards seed=getattr(self.stream, "current_seed", 2) to preserve the active seed across prompt changes. - postprocess_image()'s "latent" branch returned an internal decode buffer by reference; callers retaining it across frames would see it mutate. Now clones at this public API boundary. The "np" pinned-buffer alias and the internal __call__/txt2img fast-path aliases are documented as intentional, not cloned (would defeat the pinned-buffer/reuse optimizations). - cleanup_engines_and_rebuild's hardcoded engines_dir = "engines" now honors self._engine_dir when set, matching the existing idiom elsewhere in the file. - cleanup_gpu_memory dropped two explicit Engine.__del__() calls; the method already does del + triple gc.collect() + empty_cache() + ipc_collect() at its tail, and Engine.__del__ is self-guarding, so the explicit dunder calls were redundant. Adds tests/unit/test_phase3_correctness.py (5 regression tests, model-free CPU-only). Full unit suite: 91 passed, 0 failed (no pre-existing failures remain -- see cf3df18). --- src/streamdiffusion/pipeline.py | 12 ++- src/streamdiffusion/wrapper.py | 35 +++--- tests/unit/test_phase3_correctness.py | 146 ++++++++++++++++++++++++++ 3 files changed, 179 insertions(+), 14 deletions(-) create mode 100644 tests/unit/test_phase3_correctness.py diff --git a/src/streamdiffusion/pipeline.py b/src/streamdiffusion/pipeline.py index e8964e763..e9b32faa7 100644 --- a/src/streamdiffusion/pipeline.py +++ b/src/streamdiffusion/pipeline.py @@ -401,9 +401,11 @@ def prepare( num_inference_steps: int = 50, guidance_scale: float = 1.2, delta: float = 1.0, - generator: Optional[torch.Generator] = torch.Generator(), + generator: Optional[torch.Generator] = None, seed: int = 2, ) -> None: + if generator is None: + generator = torch.Generator() self.generator = generator self.generator.manual_seed(seed) self.current_seed = seed @@ -1292,6 +1294,10 @@ def __call__(self, x: Optional[Union[torch.Tensor, PIL.Image.Image, np.ndarray]] inference_time = start.elapsed_time(end) / 1000 self.inference_time_ema = 0.9 * self.inference_time_ema + 0.1 * inference_time + # NOTE: x_output aliases self._image_decode_buf, a persistent buffer reused every + # frame to avoid a per-frame .clone() on this hot path (see buffer init above). + # Intentional: callers needing an independent copy must clone it themselves. + # StreamDiffusionWrapper.postprocess_image() clones at the public API boundary. return x_output # ========================================================================= @@ -1384,6 +1390,10 @@ def txt2img(self, batch_size: int = 1) -> torch.Tensor: # IMAGE POSTPROCESSING HOOKS: After VAE decoding, before final output x_output = self._apply_image_postprocessing_hooks(x_output) + # NOTE: x_output aliases self._image_decode_buf, a persistent buffer reused every + # call to avoid a per-frame .clone() (see buffer init above). Intentional: callers + # needing an independent copy must clone it themselves. + # StreamDiffusionWrapper.postprocess_image() clones at the public API boundary. return x_output @torch.inference_mode() diff --git a/src/streamdiffusion/wrapper.py b/src/streamdiffusion/wrapper.py index 1e9c3463c..885b47bf9 100644 --- a/src/streamdiffusion/wrapper.py +++ b/src/streamdiffusion/wrapper.py @@ -511,6 +511,9 @@ def prepare( num_inference_steps=num_inference_steps, guidance_scale=guidance_scale, delta=delta, + # Preserve the active seed across prompt changes -- stream.prepare()'s + # own default (seed=2) would otherwise silently reset the RNG here. + seed=getattr(self.stream, "current_seed", 2), ) finally: self._offload_text_encoders() @@ -537,6 +540,9 @@ def prepare( num_inference_steps=num_inference_steps, guidance_scale=guidance_scale, delta=delta, + # Preserve the active seed across prompt changes -- stream.prepare()'s + # own default (seed=2) would otherwise silently reset the RNG here. + seed=getattr(self.stream, "current_seed", 2), ) finally: self._offload_text_encoders() @@ -1023,7 +1029,10 @@ def postprocess_image( # Fast paths for non-PIL outputs (avoid unnecessary conversions) if output_type == "latent": - return image_tensor + # Clone: image_tensor may alias an internal decode buffer reused across + # frames (see StreamDiffusion.__call__/txt2img). Callers of this public + # API must get an independent tensor, not a view that mutates next frame. + return image_tensor.clone() elif output_type == "pt": # Denormalize on GPU, return tensor return self._denormalize_on_gpu(image_tensor) @@ -1045,6 +1054,9 @@ def postprocess_image( with profiler.region("d2h_sync"): self._d2h_event.record() self._d2h_event.synchronize() + # NOTE: this numpy array is a view of `_output_pin_buf`, a pinned host + # buffer reused every frame (deliberate DMA optimization). Callers that + # retain the returned array across frames must copy it themselves. out = self._output_pin_buf.numpy() return out if self.frame_buffer_size > 1 else out[0] @@ -2905,15 +2917,10 @@ def cleanup_gpu_memory(self) -> None: unet_engine = self.stream.unet logger.info(" Cleaning up TensorRT UNet engine...") - # Check if it's a TensorRT engine and cleanup properly - if hasattr(unet_engine, "engine") and hasattr(unet_engine.engine, "__del__"): - try: - # Call the engine's destructor explicitly - unet_engine.engine.__del__() - except Exception: - pass - - # Clear all engine-related attributes + # Clear all engine-related attributes. Engine.__del__ is self-guarding + # (safe to invoke twice), so dropping these references is sufficient -- + # GC + the empty_cache()/ipc_collect() below reclaim the memory. No need + # to call __del__ explicitly first. if hasattr(unet_engine, "context"): try: del unet_engine.context @@ -2938,9 +2945,11 @@ def cleanup_gpu_memory(self) -> None: for engine_name in ["vae_encoder", "vae_decoder"]: if hasattr(vae_engine, engine_name): engine = getattr(vae_engine, engine_name) - if hasattr(engine, "engine") and hasattr(engine.engine, "__del__"): + # Drop the inner Engine reference so its self-guarding __del__ + # runs via GC (no need to invoke the dunder explicitly). + if hasattr(engine, "engine"): try: - engine.engine.__del__() + del engine.engine except Exception: pass try: @@ -3062,7 +3071,7 @@ def cleanup_engines_and_rebuild(self, reduce_batch_size: bool = True, reduce_res self.cleanup_gpu_memory() # Remove engines directory - engines_dir = "engines" + engines_dir = str(getattr(self, "_engine_dir", "engines")) if os.path.exists(engines_dir): try: shutil.rmtree(engines_dir) diff --git a/tests/unit/test_phase3_correctness.py b/tests/unit/test_phase3_correctness.py new file mode 100644 index 000000000..49459b63e --- /dev/null +++ b/tests/unit/test_phase3_correctness.py @@ -0,0 +1,146 @@ +""" +Regression tests for Phase 3 correctness & resource-safety fixes +(docs/perf_bestpractices_audit_2026-07-10.md quick-wins #4 and #5). + +Covers two independent fixes: + +1. `StreamDiffusion.prepare()`'s `generator` parameter previously defaulted to + `torch.Generator()` -- a mutable default evaluated once at function-definition + time, so every `StreamDiffusion` instance that didn't pass an explicit + `generator` shared the SAME `torch.Generator` object. The fix changes the + default to `None` and constructs a fresh `torch.Generator()` inside the + method body when `generator is None`. + +2. `StreamDiffusionWrapper.prepare()`'s two runtime `self.stream.prepare(...)` + calls (single-prompt and prompt-blending paths) previously omitted `seed`, + so `StreamDiffusion.prepare()`'s own default (`seed=2`) silently reset the + RNG on every runtime prompt change. The fix forwards + `seed=getattr(self.stream, "current_seed", 2)` so the active seed persists + across prompt changes. + +3. `StreamDiffusionWrapper.postprocess_image()`'s `"latent"` output branch + previously returned the internal decode buffer by reference (aliased + across frames). The fix clones at this public API boundary. + +All tests are deliberately CPU-only and model-free (no real diffusers model +is loaded), following the object.__new__ shell pattern used elsewhere in +tests/unit/ (see test_wrapper_exception_hygiene.py, test_safety_checker.py). +""" + +import inspect + +import torch + +from streamdiffusion.pipeline import StreamDiffusion +from streamdiffusion.wrapper import StreamDiffusionWrapper + + +# --------------------------------------------------------------------------- +# StreamDiffusion.prepare() -- mutable default removed +# --------------------------------------------------------------------------- + + +class TestPrepareGeneratorDefault: + def test_generator_default_is_none_not_a_shared_instance(self): + """ + The `generator` parameter's default must be `None`, not a + pre-constructed `torch.Generator()` -- the latter is a classic mutable + default footgun: the SAME Generator instance would be shared and + mutated by every StreamDiffusion instance that omits `generator`. + """ + default = inspect.signature(StreamDiffusion.prepare).parameters["generator"].default + assert default is None, ( + f"expected generator default to be None (constructed fresh in the method body), " + f"got a pre-built {type(default).__name__} instance -- this is a shared mutable default" + ) + + +# --------------------------------------------------------------------------- +# StreamDiffusionWrapper.prepare() -- seed threaded across prompt changes +# --------------------------------------------------------------------------- + + +class _FakeStream: + """Stand-in for StreamDiffusion that only records prepare() kwargs.""" + + def __init__(self, current_seed): + self.current_seed = current_seed + self.captured_kwargs = None + + def prepare(self, *args, **kwargs): + self.captured_kwargs = kwargs + + +def _make_wrapper_shell_for_prepare(current_seed) -> StreamDiffusionWrapper: + """Construct a minimal StreamDiffusionWrapper for exercising prepare().""" + w = object.__new__(StreamDiffusionWrapper) + w.stream = _FakeStream(current_seed) + w._reload_text_encoders = lambda: None + w._offload_text_encoders = lambda: None + return w + + +class TestPrepareSeedThreading: + def test_single_prompt_path_forwards_current_seed(self): + """ + Single-prompt runtime prepare() must forward the stream's active + `current_seed`, not silently fall back to StreamDiffusion.prepare()'s + own default (seed=2) and reset the RNG. + """ + w = _make_wrapper_shell_for_prepare(current_seed=12345) + w.prepare("a test prompt") + assert w.stream.captured_kwargs is not None, "stream.prepare() was not called" + assert w.stream.captured_kwargs.get("seed") == 12345 + + def test_prompt_blending_path_forwards_current_seed(self): + """Prompt-blending runtime prepare() must also forward current_seed.""" + w = _make_wrapper_shell_for_prepare(current_seed=54321) + w.update_stream_params = lambda **kwargs: None # blending step, not under test here + w.prepare([("cat", 0.7), ("dog", 0.3)]) + assert w.stream.captured_kwargs is not None, "stream.prepare() was not called" + assert w.stream.captured_kwargs.get("seed") == 54321 + + def test_falls_back_to_default_seed_when_stream_has_no_current_seed(self): + """If the stream has no `current_seed` attribute yet, fall back to 2.""" + w = object.__new__(StreamDiffusionWrapper) + fake_stream = _FakeStream(current_seed=0) + del fake_stream.current_seed # simulate an attribute that was never set + w.stream = fake_stream + w._reload_text_encoders = lambda: None + w._offload_text_encoders = lambda: None + w.prepare("a test prompt") + assert w.stream.captured_kwargs.get("seed") == 2 + + +# --------------------------------------------------------------------------- +# StreamDiffusionWrapper.postprocess_image() -- "latent" clones at boundary +# --------------------------------------------------------------------------- + + +def _make_wrapper_shell_for_postprocess() -> StreamDiffusionWrapper: + """Construct a minimal StreamDiffusionWrapper for exercising postprocess_image().""" + w = object.__new__(StreamDiffusionWrapper) + w.use_cuda_ipc_output = False + w._cuda_ipc_shm_name = None + return w + + +class TestPostprocessImageLatentClone: + def test_latent_output_is_an_independent_tensor(self): + """ + `postprocess_image(..., output_type="latent")` must return a tensor + that does NOT share storage with the input -- the input may be an + internal decode buffer reused across frames (see + StreamDiffusion.__call__/txt2img). A caller retaining the returned + tensor across frames must not see it mutate out from under them. + """ + buf = torch.arange(48, dtype=torch.float32).reshape(1, 3, 4, 4) + w = _make_wrapper_shell_for_postprocess() + out = w.postprocess_image(buf, output_type="latent") + assert out.data_ptr() != buf.data_ptr(), "returned tensor still aliases the input buffer" + assert torch.equal(out, buf), "cloned tensor must have identical values" + + # Mutating the source buffer (simulating buffer reuse on the next frame) + # must NOT be visible in the previously-returned tensor. + buf.fill_(-1.0) + assert not torch.equal(out, buf), "returned tensor was mutated by a later write to the source buffer" From 80c83ce7c99ccba959005b5bca83e2513b32aad7 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 10 Jul 2026 16:20:28 -0400 Subject: [PATCH 06/37] fix: Phase 4 TensorRT engine-write atomicity and graph/refit robustness Guard TRT engine and timing-cache writes against truncation from an interrupted build via a temp-file + os.replace atomic write helper, matching the idiom already used for FP8 calibration data. Add self-healing recovery around cudaGraphLaunch (reset + fallback to plain execution, re-captures next frame) and a stream-sync guard around the defensive, disabled-by-default Engine.refit() path. Verified: 3 new CPU-only unit tests cover the atomic-write helper's happy path and both failure-preservation cases; a live-GPU smoke run against a cached VAE engine exercised capture, happy-path replay, a forced cudaGraphLaunch failure (recovery + safe fallback, no crash), and self-healed re-capture. --- .../acceleration/tensorrt/utilities.py | 61 ++++++++++---- tests/unit/test_trt_atomic_engine_write.py | 80 +++++++++++++++++++ 2 files changed, 127 insertions(+), 14 deletions(-) create mode 100644 tests/unit/test_trt_atomic_engine_write.py diff --git a/src/streamdiffusion/acceleration/tensorrt/utilities.py b/src/streamdiffusion/acceleration/tensorrt/utilities.py index 51e986e7b..12126d9bf 100644 --- a/src/streamdiffusion/acceleration/tensorrt/utilities.py +++ b/src/streamdiffusion/acceleration/tensorrt/utilities.py @@ -443,6 +443,28 @@ def CUASSERT(cuda_ret): return None +def _atomic_write_bytes(path: str, data) -> None: + """Write `data` to `path` atomically via a temp file + os.replace. + + A crash/interrupt mid-write leaves at most a stale ``.tmp`` file; ``path`` is only + ever the fully-written result (os.replace is atomic on a single filesystem). This + guards TRT engine/timing-cache writes against truncation, since the builder's cache + check (builder.py) is a bare os.path.exists with no integrity check. Mirrors the + calibration-data idiom in fp8_quantize.py. + """ + tmp_path = path + ".tmp" + try: + with open(tmp_path, "wb") as f: + f.write(data) + os.replace(tmp_path, path) + except BaseException: + try: + os.remove(tmp_path) + except OSError: + pass + raise + + class TRTProfiler(trt.IProfiler): """ Per-layer TRT timing profiler. @@ -660,6 +682,11 @@ def map_name(name): else: logger.warning(f"No refit weights for layer: {layer_name}") + # Refit reads/writes the engine's weight buffers directly; synchronize first so it + # cannot race in-flight inference that is still reading those buffers. Defensive + # only — refit is unreachable unless the engine was built with enable_refit=True + # (default False), but the guard is cheap and correct either way. + torch.cuda.current_stream().synchronize() if not refitter.refit_cuda_engine(): logger.error("Failed to refit!") raise RuntimeError("TensorRT engine refit failed") @@ -776,8 +803,7 @@ def build( f"notice(s) (createInferBuilder singleton warning) — no impact on engine." ) - with open(self.engine_path, "wb") as f: - f.write(serialized) + _atomic_write_bytes(self.engine_path, serialized) # Save timing cache for next build if timing_cache: @@ -785,8 +811,7 @@ def build( updated_cache = config.get_timing_cache() if updated_cache is not None: os.makedirs(os.path.dirname(timing_cache), exist_ok=True) - with open(timing_cache, "wb") as f: - f.write(updated_cache.serialize()) + _atomic_write_bytes(timing_cache, updated_cache.serialize()) logger.info(f"[TRT Build] Saved timing cache: {timing_cache}") except Exception as e: logger.warning(f"[TRT Build] Could not save timing cache: {e}") @@ -904,8 +929,7 @@ def _build_fp8( f"notice(s) (createInferBuilder singleton warning) — no impact on engine." ) - with open(self.engine_path, "wb") as f: - f.write(serialized) + _atomic_write_bytes(self.engine_path, serialized) # Save timing cache for next build if timing_cache: @@ -913,8 +937,7 @@ def _build_fp8( updated_cache = config.get_timing_cache() if updated_cache is not None: os.makedirs(os.path.dirname(timing_cache), exist_ok=True) - with open(timing_cache, "wb") as f: - f.write(updated_cache.serialize()) + _atomic_write_bytes(timing_cache, updated_cache.serialize()) logger.info(f"[FP8] Saved timing cache: {timing_cache}") except Exception as e: logger.warning(f"[FP8] Could not save timing cache: {e}") @@ -1091,7 +1114,7 @@ def infer(self, feed_dict, stream, use_cuda_graph=False): logger.debug( "TensorRT Engine: filtering unsupported inputs %s (allowed=%s)", missing, - sorted(list(self._allowed_inputs)), + sorted(self._allowed_inputs), ) feed_dict = filtered_feed_dict @@ -1122,10 +1145,20 @@ def infer(self, feed_dict, stream, use_cuda_graph=False): with _gpu_profiler.region("trt_infer"): if use_cuda_graph: if self.cuda_graph_instance is not None: - CUASSERT(cudart.cudaGraphLaunch(self.cuda_graph_instance, stream.ptr)) - # No cudaStreamSynchronize — graph replay is async; stream ordering ensures - # downstream GPU ops (copy_, attention) wait for graph completion. - # CPU sync happens only via end.synchronize() in pipeline.__call__. + (launch_status,) = cudart.cudaGraphLaunch(self.cuda_graph_instance, stream.ptr) + if launch_status != cudart.cudaError_t.cudaSuccess: + # Graph replay failed (e.g. stale instance after a context/device + # hiccup). Destroy the graph and fall back to plain execution for + # this frame; the next call re-captures via the branch below. + logger.warning(f"CUDA graph launch failed ({launch_status}); resetting graph and falling back") + self.reset_cuda_graph() + noerror = self.context.execute_async_v3(stream.ptr) + if not noerror: + raise ValueError("ERROR: inference failed.") + # No cudaStreamSynchronize on the success path — graph replay is async; + # stream ordering ensures downstream GPU ops (copy_, attention) wait for + # graph completion. CPU sync happens only via end.synchronize() in + # pipeline.__call__. else: # Warmup passes before graph capture: TRT lazily JIT-compiles tactic # variants on the first few forward calls. Three passes ensure all @@ -1184,7 +1217,7 @@ def decode_images(images: torch.Tensor): def preprocess_image(image: Image.Image): w, h = image.size - w, h = map(lambda x: x - x % 32, (w, h)) # resize to integer multiple of 32 + w, h = (x - x % 32 for x in (w, h)) # resize to integer multiple of 32 image = image.resize((w, h)) init_image = np.array(image).astype(np.float32) / 255.0 init_image = init_image[None].transpose(0, 3, 1, 2) diff --git a/tests/unit/test_trt_atomic_engine_write.py b/tests/unit/test_trt_atomic_engine_write.py new file mode 100644 index 000000000..4b874a2f8 --- /dev/null +++ b/tests/unit/test_trt_atomic_engine_write.py @@ -0,0 +1,80 @@ +""" +Tests for the atomic engine/timing-cache write helper (Phase 4, item 1). + +`_atomic_write_bytes` in `acceleration/tensorrt/utilities.py` replaces the plain +`open(path, "wb")` writes used for TRT engine files and timing caches. A crash or +interrupt mid-write must never leave a truncated file at the final path — the +builder's cache check (`builder.py`) is a bare `os.path.exists`, so a truncated +engine would otherwise be silently treated as valid on the next run. + +These tests exercise only the pure-Python temp-file + os.replace logic; no TRT +context or GPU is required. The module is skipped if `utilities.py` cannot be +imported (e.g. TensorRT/onnx/polygraphy not installed). + +Run with: pytest tests/unit/test_trt_atomic_engine_write.py -v +""" + +import pytest + + +# --------------------------------------------------------------------------- +# Import guard — skip all tests if utilities.py's dependencies are unavailable +# --------------------------------------------------------------------------- + +try: + from streamdiffusion.acceleration.tensorrt import utilities as trt_utilities + + IMPORT_OK = True +except ImportError: + IMPORT_OK = False + +pytestmark = pytest.mark.skipif( + not IMPORT_OK, + reason="acceleration.tensorrt.utilities not importable (TensorRT/onnx/polygraphy missing)", +) + + +class TestAtomicWriteBytes: + def test_atomic_write_creates_file_with_content(self, tmp_path): + """Happy path: final file has exact content, no stray .tmp remains.""" + target = tmp_path / "engine.trt" + payload = b"\x00\x01fake-engine-bytes\xff" * 100 + + trt_utilities._atomic_write_bytes(str(target), payload) + + assert target.read_bytes() == payload + assert not (tmp_path / "engine.trt.tmp").exists() + + def test_failed_write_preserves_existing_file(self, tmp_path, monkeypatch): + """A failed rebuild must not corrupt a previously-good cached engine.""" + target = tmp_path / "engine.trt" + good_bytes = b"known-good-engine-bytes" + target.write_bytes(good_bytes) + + def _boom(*_args, **_kwargs): + raise OSError("simulated interrupt during rename") + + monkeypatch.setattr(trt_utilities.os, "replace", _boom) + + with pytest.raises(OSError, match="simulated interrupt"): + trt_utilities._atomic_write_bytes(str(target), b"new-but-never-committed-bytes") + + # Pre-existing file untouched — os.replace never ran. + assert target.read_bytes() == good_bytes + # Temp file cleaned up, not left dangling next to the real path. + assert not (tmp_path / "engine.trt.tmp").exists() + + def test_failed_write_leaves_no_partial_final(self, tmp_path, monkeypatch): + """No pre-existing file: a failed write must not leave a truncated final file.""" + target = tmp_path / "engine.trt" + + def _boom(*_args, **_kwargs): + raise OSError("simulated interrupt during rename") + + monkeypatch.setattr(trt_utilities.os, "replace", _boom) + + with pytest.raises(OSError, match="simulated interrupt"): + trt_utilities._atomic_write_bytes(str(target), b"partial-write-bytes") + + assert not target.exists() + assert not (tmp_path / "engine.trt.tmp").exists() From d56e8fac41b6a23faebd58e7a94b5687aa96f111 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 10 Jul 2026 17:20:50 -0400 Subject: [PATCH 07/37] perf: instrument unet_step with profiler.region() spans (Sub-phase 5.0) Adds 5 graph-safe profiler.region() spans (prep/sdxl_cond/hooks/engine/post) inside unet_step to attribute the audit's "~66% unattributed" frame time. Baseline capture (RTX 4090, SDXL-turbo FP8) shows that slice is GPU kernel time inside the single TRT UNet engine call (~93% of frame), not host overhead - every host sub-span measures <0.01ms. Instrumentation only, no behavior change; validated by the existing 94/0 unit suite. --- src/streamdiffusion/pipeline.py | 378 ++++++++++++++++---------------- 1 file changed, 193 insertions(+), 185 deletions(-) diff --git a/src/streamdiffusion/pipeline.py b/src/streamdiffusion/pipeline.py index e9b32faa7..cdbd86570 100644 --- a/src/streamdiffusion/pipeline.py +++ b/src/streamdiffusion/pipeline.py @@ -820,204 +820,212 @@ def unet_step( t_list: Union[torch.Tensor, list[int]], idx: Optional[int] = None, ) -> Tuple[torch.Tensor, torch.Tensor]: - if self.guidance_scale > 1.0 and (self.cfg_type == "initialize"): - # Pre-allocated buf avoids torch.concat malloc for CFG latent doubling - self._cfg_latent_buf[:1].copy_(x_t_latent[0:1]) - self._cfg_latent_buf[1:].copy_(x_t_latent) - x_t_latent_plus_uc = self._cfg_latent_buf - self._cfg_t_buf[:1].copy_(t_list[0:1]) - self._cfg_t_buf[1:].copy_(t_list) - t_list = self._cfg_t_buf - elif self.guidance_scale > 1.0 and (self.cfg_type == "full"): - self._cfg_latent_buf[: len(x_t_latent)].copy_(x_t_latent) - self._cfg_latent_buf[len(x_t_latent) :].copy_(x_t_latent) - x_t_latent_plus_uc = self._cfg_latent_buf - self._cfg_t_buf[: len(t_list)].copy_(t_list) - self._cfg_t_buf[len(t_list) :].copy_(t_list) - t_list = self._cfg_t_buf - else: - x_t_latent_plus_uc = x_t_latent + with profiler.region("unet_step.prep"): + if self.guidance_scale > 1.0 and (self.cfg_type == "initialize"): + # Pre-allocated buf avoids torch.concat malloc for CFG latent doubling + self._cfg_latent_buf[:1].copy_(x_t_latent[0:1]) + self._cfg_latent_buf[1:].copy_(x_t_latent) + x_t_latent_plus_uc = self._cfg_latent_buf + self._cfg_t_buf[:1].copy_(t_list[0:1]) + self._cfg_t_buf[1:].copy_(t_list) + t_list = self._cfg_t_buf + elif self.guidance_scale > 1.0 and (self.cfg_type == "full"): + self._cfg_latent_buf[: len(x_t_latent)].copy_(x_t_latent) + self._cfg_latent_buf[len(x_t_latent) :].copy_(x_t_latent) + x_t_latent_plus_uc = self._cfg_latent_buf + self._cfg_t_buf[: len(t_list)].copy_(t_list) + self._cfg_t_buf[len(t_list) :].copy_(t_list) + t_list = self._cfg_t_buf + else: + x_t_latent_plus_uc = x_t_latent - # Prepare UNet call arguments (update pre-allocated dict in-place: avoids per-frame dict malloc) - self._unet_kwargs["sample"] = x_t_latent_plus_uc - self._unet_kwargs["timestep"] = t_list - self._unet_kwargs["encoder_hidden_states"] = self.prompt_embeds - unet_kwargs = self._unet_kwargs + # Prepare UNet call arguments (update pre-allocated dict in-place: avoids per-frame dict malloc) + self._unet_kwargs["sample"] = x_t_latent_plus_uc + self._unet_kwargs["timestep"] = t_list + self._unet_kwargs["encoder_hidden_states"] = self.prompt_embeds + unet_kwargs = self._unet_kwargs # Add SDXL-specific conditioning if this is an SDXL model - if self.is_sdxl and hasattr(self, "add_text_embeds") and hasattr(self, "add_time_ids"): - if self.add_text_embeds is not None and self.add_time_ids is not None: - # Handle batching for CFG - replicate conditioning to match batch size - batch_size = x_t_latent_plus_uc.shape[0] - - # Use optimized caching system for SDXL conditioning tensors - cached_conditioning = self._get_cached_sdxl_conditioning( - batch_size, self.cfg_type, self.guidance_scale - ) - if cached_conditioning is not None: - # Cache hit - reuse existing tensors - add_text_embeds = cached_conditioning["text_embeds"] - add_time_ids = cached_conditioning["time_ids"] - else: - # Cache miss - build new tensors using optimized operations - conditioning = self._build_sdxl_conditioning(batch_size) - add_text_embeds = conditioning["text_embeds"] - add_time_ids = conditioning["time_ids"] - # Cache for future use - self._cache_sdxl_conditioning( - batch_size, self.cfg_type, self.guidance_scale, add_text_embeds, add_time_ids + with profiler.region("unet_step.sdxl_cond"): + if self.is_sdxl and hasattr(self, "add_text_embeds") and hasattr(self, "add_time_ids"): + if self.add_text_embeds is not None and self.add_time_ids is not None: + # Handle batching for CFG - replicate conditioning to match batch size + batch_size = x_t_latent_plus_uc.shape[0] + + # Use optimized caching system for SDXL conditioning tensors + cached_conditioning = self._get_cached_sdxl_conditioning( + batch_size, self.cfg_type, self.guidance_scale ) + if cached_conditioning is not None: + # Cache hit - reuse existing tensors + add_text_embeds = cached_conditioning["text_embeds"] + add_time_ids = cached_conditioning["time_ids"] + else: + # Cache miss - build new tensors using optimized operations + conditioning = self._build_sdxl_conditioning(batch_size) + add_text_embeds = conditioning["text_embeds"] + add_time_ids = conditioning["time_ids"] + # Cache for future use + self._cache_sdxl_conditioning( + batch_size, self.cfg_type, self.guidance_scale, add_text_embeds, add_time_ids + ) - unet_kwargs["added_cond_kwargs"] = {"text_embeds": add_text_embeds, "time_ids": add_time_ids} + unet_kwargs["added_cond_kwargs"] = {"text_embeds": add_text_embeds, "time_ids": add_time_ids} # Allow modules to contribute additional UNet kwargs via hooks - if self.unet_hooks: - try: - step_ctx = StepCtx( - x_t_latent=x_t_latent_plus_uc, - t_list=t_list, - step_index=idx if isinstance(idx, int) else (int(idx) if idx is not None else None), - guidance_mode=self.cfg_type if self.guidance_scale > 1.0 else "none", - sdxl_cond=unet_kwargs.get("added_cond_kwargs", None), - ) - extra_from_hooks = {} - for hook in self.unet_hooks: - delta: UnetKwargsDelta = hook(step_ctx) - if delta is None: - continue - if delta.down_block_additional_residuals is not None: - unet_kwargs["down_block_additional_residuals"] = delta.down_block_additional_residuals - if delta.mid_block_additional_residual is not None: - unet_kwargs["mid_block_additional_residual"] = delta.mid_block_additional_residual - if delta.added_cond_kwargs is not None: - # Merge SDXL cond if both exist - base_added = unet_kwargs.get("added_cond_kwargs", {}) - base_added.update(delta.added_cond_kwargs) - unet_kwargs["added_cond_kwargs"] = base_added - if getattr(delta, "extra_unet_kwargs", None): - # Merge extra kwargs from hooks (e.g., ipadapter_scale) - try: - extra_from_hooks.update(delta.extra_unet_kwargs) - except Exception as e: - logger.debug(f"unet_step: failed to merge extra_unet_kwargs from hook: {e}", exc_info=True) - if extra_from_hooks: - unet_kwargs["extra_unet_kwargs"] = extra_from_hooks - except Exception as e: - logger.error(f"unet_step: unet hook failed: {e}") - raise - - # Extract potential ControlNet residual kwargs and generic extra kwargs (e.g., ipadapter_scale) - hook_down_res = unet_kwargs.get("down_block_additional_residuals", None) - hook_mid_res = unet_kwargs.get("mid_block_additional_residual", None) - hook_extra_kwargs = unet_kwargs.get("extra_unet_kwargs", None) if "extra_unet_kwargs" in unet_kwargs else None - - # Call UNet with appropriate conditioning - if self.is_sdxl: - try: - # Detect UNet type and use appropriate calling convention - added_cond_kwargs = unet_kwargs.get("added_cond_kwargs", {}) - - # Check if this is a TensorRT engine or PyTorch UNet - is_tensorrt_engine = self._check_unet_tensorrt() - - if is_tensorrt_engine: - # TensorRT engine expects positional args + kwargs. IP-Adapter scale vector, if any, is provided by hooks via extra_unet_kwargs - extra_kwargs = {} - if isinstance(hook_extra_kwargs, dict): - extra_kwargs.update(hook_extra_kwargs) - - # Include ControlNet residuals if provided by hooks - if hook_down_res is not None: - extra_kwargs["down_block_additional_residuals"] = hook_down_res - if hook_mid_res is not None: - extra_kwargs["mid_block_additional_residual"] = hook_mid_res - - _unet_result = self.unet( - unet_kwargs["sample"], # latent_model_input (positional) - unet_kwargs["timestep"], # timestep (positional) - unet_kwargs["encoder_hidden_states"], # encoder_hidden_states (positional) - kvo_cache=self.kvo_cache, - fio_cache=self.fio_cache, - fi_strength=self._fi_strength_tensor, - fi_threshold=self._fi_threshold_tensor, - **extra_kwargs, - # For TRT engines, ensure SDXL cond shapes match engine builds; if engine expects 81 tokens (77+4), append dummy image tokens when none - **added_cond_kwargs, # SDXL conditioning as kwargs + with profiler.region("unet_step.hooks"): + if self.unet_hooks: + try: + step_ctx = StepCtx( + x_t_latent=x_t_latent_plus_uc, + t_list=t_list, + step_index=idx if isinstance(idx, int) else (int(idx) if idx is not None else None), + guidance_mode=self.cfg_type if self.guidance_scale > 1.0 else "none", + sdxl_cond=unet_kwargs.get("added_cond_kwargs", None), ) - model_pred = _unet_result[0] - kvo_cache_out = _unet_result[1] if len(_unet_result) > 1 else [] - fio_cache_out = _unet_result[2] if len(_unet_result) > 2 else [] - self.update_kvo_cache(kvo_cache_out, fio_cache_out) - else: - # PyTorch UNet expects diffusers-style named arguments. Any processor scaling is handled by IP-Adapter hook - - call_kwargs = { - "sample": unet_kwargs["sample"], - "timestep": unet_kwargs["timestep"], - "encoder_hidden_states": unet_kwargs["encoder_hidden_states"], - "added_cond_kwargs": added_cond_kwargs, - "return_dict": False, - } - # Include ControlNet residuals if present - if hook_down_res is not None: - call_kwargs["down_block_additional_residuals"] = hook_down_res - if hook_mid_res is not None: - call_kwargs["mid_block_additional_residual"] = hook_mid_res - model_pred = self.unet(**call_kwargs)[0] - # No restoration for per-layer scale; next step will set again via updater/time factor - - except Exception as e: - logger.error(f"[PIPELINE] unet_step: *** ERROR: SDXL UNet call failed: {e} ***") - import traceback + extra_from_hooks = {} + for hook in self.unet_hooks: + delta: UnetKwargsDelta = hook(step_ctx) + if delta is None: + continue + if delta.down_block_additional_residuals is not None: + unet_kwargs["down_block_additional_residuals"] = delta.down_block_additional_residuals + if delta.mid_block_additional_residual is not None: + unet_kwargs["mid_block_additional_residual"] = delta.mid_block_additional_residual + if delta.added_cond_kwargs is not None: + # Merge SDXL cond if both exist + base_added = unet_kwargs.get("added_cond_kwargs", {}) + base_added.update(delta.added_cond_kwargs) + unet_kwargs["added_cond_kwargs"] = base_added + if getattr(delta, "extra_unet_kwargs", None): + # Merge extra kwargs from hooks (e.g., ipadapter_scale) + try: + extra_from_hooks.update(delta.extra_unet_kwargs) + except Exception as e: + logger.debug( + f"unet_step: failed to merge extra_unet_kwargs from hook: {e}", exc_info=True + ) + if extra_from_hooks: + unet_kwargs["extra_unet_kwargs"] = extra_from_hooks + except Exception as e: + logger.error(f"unet_step: unet hook failed: {e}") + raise - traceback.print_exc() - raise - else: - # For SD1.5/SD2.1, use the old calling convention for compatibility - # Build kwargs from hooks and include residuals - ip_scale_kw = {} - if isinstance(hook_extra_kwargs, dict): - ip_scale_kw.update(hook_extra_kwargs) - - # PyTorch processor time scaling is handled by the IP-Adapter hook - - # Include ControlNet residuals if present - if hook_down_res is not None: - ip_scale_kw["down_block_additional_residuals"] = hook_down_res - if hook_mid_res is not None: - ip_scale_kw["mid_block_additional_residual"] = hook_mid_res - - _unet_result = self.unet( - x_t_latent_plus_uc, - t_list, - encoder_hidden_states=self.prompt_embeds, - kvo_cache=self.kvo_cache, - fio_cache=self.fio_cache, - fi_strength=self._fi_strength_tensor, - fi_threshold=self._fi_threshold_tensor, - return_dict=False, - **ip_scale_kw, + # Extract potential ControlNet residual kwargs / extra kwargs (e.g., ipadapter_scale), then call UNet + with profiler.region("unet_step.engine"): + hook_down_res = unet_kwargs.get("down_block_additional_residuals", None) + hook_mid_res = unet_kwargs.get("mid_block_additional_residual", None) + hook_extra_kwargs = ( + unet_kwargs.get("extra_unet_kwargs", None) if "extra_unet_kwargs" in unet_kwargs else None ) - model_pred = _unet_result[0] - kvo_cache_out = _unet_result[1] if len(_unet_result) > 1 else [] - fio_cache_out = _unet_result[2] if len(_unet_result) > 2 else [] - self.update_kvo_cache(kvo_cache_out, fio_cache_out) - if self.guidance_scale > 1.0 and (self.cfg_type == "initialize"): - noise_pred_text = model_pred[1:] - self.stock_noise[0:1].copy_(model_pred[0:1]) # in-place: eliminates 1 malloc + copy kernel - elif self.guidance_scale > 1.0 and (self.cfg_type == "full"): - noise_pred_uncond, noise_pred_text = model_pred.chunk(2) - else: - noise_pred_text = model_pred + if self.is_sdxl: + try: + # Detect UNet type and use appropriate calling convention + added_cond_kwargs = unet_kwargs.get("added_cond_kwargs", {}) + + # Check if this is a TensorRT engine or PyTorch UNet + is_tensorrt_engine = self._check_unet_tensorrt() + + if is_tensorrt_engine: + # TensorRT engine expects positional args + kwargs. IP-Adapter scale vector, if any, is provided by hooks via extra_unet_kwargs + extra_kwargs = {} + if isinstance(hook_extra_kwargs, dict): + extra_kwargs.update(hook_extra_kwargs) + + # Include ControlNet residuals if provided by hooks + if hook_down_res is not None: + extra_kwargs["down_block_additional_residuals"] = hook_down_res + if hook_mid_res is not None: + extra_kwargs["mid_block_additional_residual"] = hook_mid_res + + _unet_result = self.unet( + unet_kwargs["sample"], # latent_model_input (positional) + unet_kwargs["timestep"], # timestep (positional) + unet_kwargs["encoder_hidden_states"], # encoder_hidden_states (positional) + kvo_cache=self.kvo_cache, + fio_cache=self.fio_cache, + fi_strength=self._fi_strength_tensor, + fi_threshold=self._fi_threshold_tensor, + **extra_kwargs, + # For TRT engines, ensure SDXL cond shapes match engine builds; if engine expects 81 tokens (77+4), append dummy image tokens when none + **added_cond_kwargs, # SDXL conditioning as kwargs + ) + model_pred = _unet_result[0] + kvo_cache_out = _unet_result[1] if len(_unet_result) > 1 else [] + fio_cache_out = _unet_result[2] if len(_unet_result) > 2 else [] + self.update_kvo_cache(kvo_cache_out, fio_cache_out) + else: + # PyTorch UNet expects diffusers-style named arguments. Any processor scaling is handled by IP-Adapter hook + + call_kwargs = { + "sample": unet_kwargs["sample"], + "timestep": unet_kwargs["timestep"], + "encoder_hidden_states": unet_kwargs["encoder_hidden_states"], + "added_cond_kwargs": added_cond_kwargs, + "return_dict": False, + } + # Include ControlNet residuals if present + if hook_down_res is not None: + call_kwargs["down_block_additional_residuals"] = hook_down_res + if hook_mid_res is not None: + call_kwargs["mid_block_additional_residual"] = hook_mid_res + model_pred = self.unet(**call_kwargs)[0] + # No restoration for per-layer scale; next step will set again via updater/time factor - if self.guidance_scale > 1.0 and (self.cfg_type == "self" or self.cfg_type == "initialize"): - noise_pred_uncond = self.stock_noise * self.delta + except Exception as e: + logger.error(f"[PIPELINE] unet_step: *** ERROR: SDXL UNet call failed: {e} ***") + import traceback - if self.guidance_scale > 1.0 and self.cfg_type != "none": - model_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond) - else: - model_pred = noise_pred_text + traceback.print_exc() + raise + else: + # For SD1.5/SD2.1, use the old calling convention for compatibility + # Build kwargs from hooks and include residuals + ip_scale_kw = {} + if isinstance(hook_extra_kwargs, dict): + ip_scale_kw.update(hook_extra_kwargs) + + # PyTorch processor time scaling is handled by the IP-Adapter hook + + # Include ControlNet residuals if present + if hook_down_res is not None: + ip_scale_kw["down_block_additional_residuals"] = hook_down_res + if hook_mid_res is not None: + ip_scale_kw["mid_block_additional_residual"] = hook_mid_res + + _unet_result = self.unet( + x_t_latent_plus_uc, + t_list, + encoder_hidden_states=self.prompt_embeds, + kvo_cache=self.kvo_cache, + fio_cache=self.fio_cache, + fi_strength=self._fi_strength_tensor, + fi_threshold=self._fi_threshold_tensor, + return_dict=False, + **ip_scale_kw, + ) + model_pred = _unet_result[0] + kvo_cache_out = _unet_result[1] if len(_unet_result) > 1 else [] + fio_cache_out = _unet_result[2] if len(_unet_result) > 2 else [] + self.update_kvo_cache(kvo_cache_out, fio_cache_out) + + with profiler.region("unet_step.post"): + if self.guidance_scale > 1.0 and (self.cfg_type == "initialize"): + noise_pred_text = model_pred[1:] + self.stock_noise[0:1].copy_(model_pred[0:1]) # in-place: eliminates 1 malloc + copy kernel + elif self.guidance_scale > 1.0 and (self.cfg_type == "full"): + noise_pred_uncond, noise_pred_text = model_pred.chunk(2) + else: + noise_pred_text = model_pred + + if self.guidance_scale > 1.0 and (self.cfg_type == "self" or self.cfg_type == "initialize"): + noise_pred_uncond = self.stock_noise * self.delta + + if self.guidance_scale > 1.0 and self.cfg_type != "none": + model_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond) + else: + model_pred = noise_pred_text # compute the previous noisy sample x_t -> x_t-1 if self.use_denoising_batch: From d44f1ed38aef5725f18bc1df7014bfd7973e8dbd Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 10 Jul 2026 17:37:39 -0400 Subject: [PATCH 08/37] perf: skip redundant set_tensor_address rebind in TRT graph replay (Sub-phase 5.1) Engine.infer() unconditionally re-bound every tensor address via set_tensor_address() before each engine call, even in graphed steady state where the addresses are already baked into the captured CUDA graph (self.tensors buffers are persistent and reused via copy_() - see allocate_buffers/_can_reuse_buffers, which resets cuda_graph_instance to None whenever a buffer is actually reallocated). Skip the rebind loop once a graph is captured; only rebind on first capture or after a reset_cuda_graph() recovery. Verified live on the RTX 4090 td_config.yaml runtime path (the UNet engine loads with use_cuda_graph=True unconditionally via engine_manager.py's loader): 24-frame smoke run with real graph capture + replay shows no errors and frame timing (p50 32.10ms, mean 47.33ms) matching a pre-fix A/B run (p50 31.74ms, mean 47.00ms) - confirms correctness with the expected ~0 measured perf delta (the audit's own 5.0 baseline showed host-side region time is <0.01ms; this is a hygiene/false-dependency fix, not a throughput win). Suite 94/0; ruff clean; pyrefly 72 errors (baseline, no new). --- .../acceleration/tensorrt/utilities.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/streamdiffusion/acceleration/tensorrt/utilities.py b/src/streamdiffusion/acceleration/tensorrt/utilities.py index 12126d9bf..6b63c9866 100644 --- a/src/streamdiffusion/acceleration/tensorrt/utilities.py +++ b/src/streamdiffusion/acceleration/tensorrt/utilities.py @@ -1138,9 +1138,15 @@ def infer(self, feed_dict, stream, use_cuda_graph=False): for name, buf in feed_dict.items(): self.tensors[name].copy_(buf) - for name, tensor in self.tensors.items(): - if not self.context.set_tensor_address(name, tensor.data_ptr()): - raise RuntimeError(f"TensorRT: set_tensor_address failed for '{name}'") + # In graphed steady state the tensor addresses are baked into the captured graph + # (self.tensors[name] are persistent buffers reused via copy_() — see + # _can_reuse_buffers/allocate_buffers, which resets cuda_graph_instance to None + # whenever a buffer is actually reallocated). Re-binding every frame is then pure + # host overhead; only rebind on first capture or after a reset (instance is None). + if not (use_cuda_graph and self.cuda_graph_instance is not None): + for name, tensor in self.tensors.items(): + if not self.context.set_tensor_address(name, tensor.data_ptr()): + raise RuntimeError(f"TensorRT: set_tensor_address failed for '{name}'") with _gpu_profiler.region("trt_infer"): if use_cuda_graph: From a1af2b00c9f8881264bd2ccb4620840b039085bc Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 10 Jul 2026 19:18:40 -0400 Subject: [PATCH 09/37] perf: sync-free output/PIL/IPC paths (Sub-phase 5.2) Three output-side sync/allocation fixes, ordered by real-world impact on the TD path: - 5f (only hot-path site): _ipc_pack_rgba / _ipc_pack_unit_rgba now write into a persistent HWC x4 GPU buffer (alpha set once at allocation, B/G/R channels written in place) instead of allocating a fresh torch.full_like alpha + torch.cat every frame. Safe only because the IPC exporters are forced to blocking export (ADR-0001); comments updated at both _lazy_init_ipc_exporter and _lazy_init_cn_ipc_exporter to reflect the buffer is now persistent, not transient. - 5e: _tensor_to_pil_safe (preprocessing_orchestrator.py) reordered CPU-first, following the in-repo base.py:tensor_to_pil template -- the D2H transfer now happens before the tensor.min()/.max() range-check syncs instead of after, collapsing 3 GPU host syncs down to 1. - 5d (hygiene; TD's IPC output path never hits this): _tensor_to_pil_optimized now routes through the shared _output_pin_buf/_d2h_event pinned-buffer + Event machinery instead of a blocking, unpinned .cpu() into pageable memory. Documented the same buffer-reuse caveat as the "np" path (PIL images returned wrap a view of the pinned buffer; callers retaining them across frames must .copy()). Verified via tests/unit/test_sync_free_output_5_2.py: byte-identical parity against frozen pre-5.2 reference formulas for all three sites, plus persistent-buffer reuse/realloc and independent-buffer-identity checks (10 new tests, GPU-gated). Suite 104/0 (94 baseline + 10 new); ruff clean on touched files; pyrefly 72 errors (baseline, no new). --- .../preprocessing_orchestrator.py | 14 +- src/streamdiffusion/wrapper.py | 96 +++++-- tests/unit/test_sync_free_output_5_2.py | 238 ++++++++++++++++++ 3 files changed, 322 insertions(+), 26 deletions(-) create mode 100644 tests/unit/test_sync_free_output_5_2.py diff --git a/src/streamdiffusion/preprocessing/preprocessing_orchestrator.py b/src/streamdiffusion/preprocessing/preprocessing_orchestrator.py index e2b52ae4a..577678bd6 100644 --- a/src/streamdiffusion/preprocessing/preprocessing_orchestrator.py +++ b/src/streamdiffusion/preprocessing/preprocessing_orchestrator.py @@ -596,6 +596,15 @@ def _tensor_to_pil_safe(self, tensor: torch.Tensor) -> Image.Image: # Convert from CHW to HWC tensor = tensor.permute(1, 2, 0) + # 5e: CPU-first — do the D2H transfer before the range-check reductions below, not + # after. tensor.min()/.max() are scalar reductions that force a host sync while the + # tensor is still on GPU; running them on the already-transferred CPU tensor collapses + # 2 GPU syncs into the 1 unavoidable D2H transfer. Same pattern as the in-repo template + # preprocessing/processors/base.py:tensor_to_pil. + tensor = tensor.detach() + if tensor.is_cuda: + tensor = tensor.cpu() + # CRITICAL FIX: Handle VAE output range [-1, 1] -> [0, 1] -> [0, 255] # VAE decode_image() outputs in [-1, 1] range, need to convert to [0, 1] first if tensor.min() < 0: @@ -606,8 +615,8 @@ def _tensor_to_pil_safe(self, tensor: torch.Tensor) -> Image.Image: if tensor.max() <= 1.0: tensor = tensor * 255.0 - # Convert to numpy and then PIL - numpy_image = tensor.detach().cpu().numpy().astype(np.uint8) + # Already on CPU (transferred above) — just cast + build the PIL image. + numpy_image = tensor.numpy().astype(np.uint8) return Image.fromarray(numpy_image) def _pil_to_tensor_safe(self, pil_image: Image.Image, device: str, dtype: torch.dtype) -> torch.Tensor: @@ -789,7 +798,6 @@ def _process_single_preprocessor_group( if getattr(preprocessor, "gpu_native", False): return None - # PIL processing fallback if control_variants["image"] is not None: processed_image = self.prepare_control_image( diff --git a/src/streamdiffusion/wrapper.py b/src/streamdiffusion/wrapper.py index 885b47bf9..c981661db 100644 --- a/src/streamdiffusion/wrapper.py +++ b/src/streamdiffusion/wrapper.py @@ -349,6 +349,9 @@ def __init__( self._output_pin_buf: Optional[torch.Tensor] = None # pinned CPU buffer for async D2H output self._output_gpu_buf: Optional[torch.Tensor] = None # persistent GPU fp32 staging (avoids per-frame alloc) self._d2h_event: Optional[torch.cuda.Event] = None # event for fine-grained D2H sync + self._ipc_pack_buf: Optional[torch.Tensor] = None # persistent GPU BGRA buffer for _ipc_pack_rgba (5f) + # persistent GPU BGRA buffer for _ipc_pack_unit_rgba (5f) + self._ipc_pack_unit_buf: Optional[torch.Tensor] = None self.use_cuda_ipc_output = use_cuda_ipc_output self._cuda_ipc_shm_name = cuda_ipc_shm_name self._cuda_ipc_num_slots = cuda_ipc_num_slots @@ -1074,15 +1077,35 @@ def postprocess_image( return postprocess_image(image_tensor.cpu(), output_type=output_type)[0] def _ipc_pack_rgba(self, image_tensor: torch.Tensor) -> torch.Tensor: - """Convert pipeline output to HWC uint8 BGRA on GPU for cuda-link wire contract.""" + """Convert pipeline output to HWC uint8 BGRA on GPU for cuda-link wire contract. + + Writes into a persistent HWC×4 GPU buffer (realloc'd only on shape/device change) + instead of allocating a fresh alpha channel + concatenated tensor every frame (5f). + SAFE ONLY because the IPC exporter is forced to blocking export (see + _lazy_init_ipc_exporter / ADR-0001): the frame must be fully copied out via cuda-link + before this buffer is overwritten next call. Async export (CUDALINK_EXPORT_SYNC=0) + would race against this reused buffer and needs double-buffering first — not + implemented. + """ with profiler.region("glue.ipc_pack_rgba"): denorm = self._denormalize_on_gpu(image_tensor) # NCHW [0,1] if denorm.dim() == 4: denorm = denorm[0] # CHW [0,1] rgb_u8 = (denorm * 255).clamp(0, 255).to(torch.uint8) # CHW uint8 rgb_hwc = rgb_u8.permute(1, 2, 0).contiguous() # HWC RGB - alpha = torch.full_like(rgb_hwc[..., :1], 255) - return torch.cat([rgb_hwc[..., 2:3], rgb_hwc[..., 1:2], rgb_hwc[..., 0:1], alpha], dim=-1).contiguous() + h, w = rgb_hwc.shape[0], rgb_hwc.shape[1] + if ( + self._ipc_pack_buf is None + or self._ipc_pack_buf.shape[0] != h + or self._ipc_pack_buf.shape[1] != w + or self._ipc_pack_buf.device != rgb_hwc.device + ): + self._ipc_pack_buf = torch.empty((h, w, 4), dtype=torch.uint8, device=rgb_hwc.device) + self._ipc_pack_buf[..., 3] = 255 # constant alpha, set once at (re)allocation + self._ipc_pack_buf[..., 0] = rgb_hwc[..., 2] # B + self._ipc_pack_buf[..., 1] = rgb_hwc[..., 1] # G + self._ipc_pack_buf[..., 2] = rgb_hwc[..., 0] # R + return self._ipc_pack_buf def _lazy_init_ipc_exporter(self, height: int, width: int): """Initialize Exporter on first frame (lazy to defer CUDA IPC SHM creation).""" @@ -1092,10 +1115,11 @@ def _lazy_init_ipc_exporter(self, height: int, width: int): from cuda_link import Exporter, ExportPolicy, FrameSpec - # SD source buffers (_ipc_pack_rgba output) are transient torch tensors recycled by - # PyTorch's caching allocator the moment the caller returns. Async export would read - # recycled memory → torn frames (ADR-0001 source-buffer lifetime race). Force - # blocking unless the user explicitly opted into async with CUDALINK_EXPORT_SYNC=0. + # SD source buffers (_ipc_pack_rgba output) are a persistent GPU buffer reused every + # frame (5f), not a fresh allocation. Async export would read a buffer that's already + # been overwritten by the next frame → torn frames (ADR-0001 source-buffer lifetime + # race). Force blocking unless the user explicitly opted into async with + # CUDALINK_EXPORT_SYNC=0. policy = ExportPolicy.from_env() if os.environ.get("CUDALINK_EXPORT_SYNC") is None: policy = _dc_replace(policy, export_sync=True) @@ -1118,6 +1142,10 @@ def _ipc_pack_unit_rgba(self, image_tensor: torch.Tensor) -> torch.Tensor: Like _ipc_pack_rgba but skips _denormalize_on_gpu — the ControlNet preprocessor output is already in [0, 1] (not the diffusion [-1, 1] range). + + Writes into a persistent HWC×4 GPU buffer (5f), same rationale/caveat as + _ipc_pack_rgba: SAFE ONLY while the CN-preview exporter stays forced-blocking + (see _lazy_init_cn_ipc_exporter / ADR-0001). """ with profiler.region("glue.ipc_pack_unit_rgba"): t = image_tensor @@ -1125,8 +1153,19 @@ def _ipc_pack_unit_rgba(self, image_tensor: torch.Tensor) -> torch.Tensor: t = t[0] # NCHW → CHW [0,1] rgb_u8 = (t * 255).clamp(0, 255).to(torch.uint8) # CHW uint8 rgb_hwc = rgb_u8.permute(1, 2, 0).contiguous() # HWC RGB - alpha = torch.full_like(rgb_hwc[..., :1], 255) - return torch.cat([rgb_hwc[..., 2:3], rgb_hwc[..., 1:2], rgb_hwc[..., 0:1], alpha], dim=-1).contiguous() + h, w = rgb_hwc.shape[0], rgb_hwc.shape[1] + if ( + self._ipc_pack_unit_buf is None + or self._ipc_pack_unit_buf.shape[0] != h + or self._ipc_pack_unit_buf.shape[1] != w + or self._ipc_pack_unit_buf.device != rgb_hwc.device + ): + self._ipc_pack_unit_buf = torch.empty((h, w, 4), dtype=torch.uint8, device=rgb_hwc.device) + self._ipc_pack_unit_buf[..., 3] = 255 # constant alpha, set once at (re)allocation + self._ipc_pack_unit_buf[..., 0] = rgb_hwc[..., 2] # B + self._ipc_pack_unit_buf[..., 1] = rgb_hwc[..., 1] # G + self._ipc_pack_unit_buf[..., 2] = rgb_hwc[..., 0] # R + return self._ipc_pack_unit_buf def _lazy_init_cn_ipc_exporter(self, height: int, width: int): """Initialize the CN-preview Exporter on first frame (lazy, mirrors _lazy_init_ipc_exporter).""" @@ -1137,7 +1176,7 @@ def _lazy_init_cn_ipc_exporter(self, height: int, width: int): from cuda_link import Exporter, ExportPolicy, FrameSpec # Same source-buffer lifetime race as _lazy_init_ipc_exporter: _ipc_pack_unit_rgba - # returns a transient tensor that the caching allocator recycles on return. + # returns a persistent GPU buffer (5f) reused every frame, overwritten on the next call. # Force blocking unless CUDALINK_EXPORT_SYNC=0 explicitly opts into async. policy = ExportPolicy.from_env() if os.environ.get("CUDALINK_EXPORT_SYNC") is None: @@ -1276,20 +1315,29 @@ def _tensor_to_pil_optimized(self, image_tensor: torch.Tensor) -> List[Image.Ima List[Image.Image] List of PIL RGB images, one for each item in the batch. """ - # Denormalize on GPU first + # 5d: convert to uint8 NHWC on GPU (identical layout to the "np" output path @1045), + # then route through the shared pinned-buffer + Event machinery instead of a blocking, + # unpinned .cpu() into pageable memory. Only one output_type's fast path executes per + # postprocess_image() call, so sharing _output_pin_buf/_d2h_event with the "np" path + # is safe (shape/dtype-guarded realloc below covers either caller). denormalized = self._denormalize_on_gpu(image_tensor) - - # Convert to uint8 on GPU to reduce transfer size - # Scale to [0, 255] and convert to uint8 - # Scale to [0, 255] and convert to uint8 - uint8_tensor = (denormalized * 255).clamp(0, 255).to(torch.uint8) - - # Single efficient CPU transfer - cpu_tensor = uint8_tensor.cpu() - - # Convert to HWC format for PIL - # From BCHW to BHWC - cpu_tensor = cpu_tensor.permute(0, 2, 3, 1) + uint8_nhwc = (denormalized * 255).clamp(0, 255).to(torch.uint8).permute(0, 2, 3, 1).contiguous() + if ( + self._output_pin_buf is None + or self._output_pin_buf.shape != uint8_nhwc.shape + or self._output_pin_buf.dtype != torch.uint8 + ): + self._output_pin_buf = torch.empty(uint8_nhwc.shape, dtype=torch.uint8, pin_memory=True) + self._d2h_event = torch.cuda.Event() + self._output_pin_buf.copy_(uint8_nhwc, non_blocking=True) + with profiler.region("d2h_sync"): + self._d2h_event.record() + self._d2h_event.synchronize() + # NOTE: like the "np" output path, each PIL Image below wraps a view of + # `_output_pin_buf` (Image.fromarray shares the numpy buffer, it does not copy). + # Callers that retain a returned PIL Image across frames must .copy() it themselves; + # this pinned buffer is overwritten in place on the next call. + cpu_tensor = self._output_pin_buf # Convert to PIL images efficiently pil_images = [] @@ -2987,6 +3035,8 @@ def cleanup_gpu_memory(self) -> None: self._output_pin_buf = None self._output_gpu_buf = None self._d2h_event = None + self._ipc_pack_buf = None + self._ipc_pack_unit_buf = None # Force multiple garbage collection cycles for thorough cleanup for i in range(3): diff --git a/tests/unit/test_sync_free_output_5_2.py b/tests/unit/test_sync_free_output_5_2.py new file mode 100644 index 000000000..76780f1fe --- /dev/null +++ b/tests/unit/test_sync_free_output_5_2.py @@ -0,0 +1,238 @@ +""" +Regression tests for Sub-phase 5.2 (output/PIL/IPC sync-free path, +docs/perf_bestpractices_audit_2026-07-10.md follow-ups #2/#... — see the +Phase 5 continuation plan). + +Covers three independent output-side fixes, each converted from a +per-frame-allocation formula to a persistent-buffer / pinned-readback +formula. The reference (`_reference_*`) helpers below are frozen copies of +the pre-5.2 code, used as the numeric-parity oracle: every touched method +must produce byte-identical output to its old formula on the same input. + + - 5f: `_ipc_pack_rgba` / `_ipc_pack_unit_rgba` (wrapper.py) — persistent + HWC x4 GPU buffer instead of a fresh `torch.full_like` alpha + `torch.cat` + every frame. + - 5d: `_tensor_to_pil_optimized` (wrapper.py) — routes through the shared + `_output_pin_buf` / `_d2h_event` pinned-buffer + Event machinery instead + of a blocking, unpinned `.cpu()` into pageable memory. + - 5e: `_tensor_to_pil_safe` (preprocessing_orchestrator.py) — CPU-first + reorder: the D2H transfer now happens before the tensor.min()/.max() + range-check syncs, not after (kills 2 of 3 host syncs; same template as + preprocessing/processors/base.py:tensor_to_pil). + +5d and 5f exercise `torch.cuda.Event()` / `pin_memory=True`, which require a +real CUDA device — the whole module is skipped when unavailable. +""" + +import numpy as np +import pytest +import torch +from PIL import Image + +from streamdiffusion.preprocessing.preprocessing_orchestrator import PreprocessingOrchestrator +from streamdiffusion.wrapper import StreamDiffusionWrapper + + +pytestmark = pytest.mark.skipif( + not torch.cuda.is_available(), + reason="Sub-phase 5.2 output paths require CUDA (pin_memory / cuda.Event)", +) + + +# --------------------------------------------------------------------------- +# helpers: minimal shells + frozen pre-5.2 reference formulas +# --------------------------------------------------------------------------- + + +def _make_wrapper_shell(): + """Minimal StreamDiffusionWrapper shell wired only with the attrs the + touched methods read (object.__new__ pattern, see test_wrapper_exception_hygiene.py).""" + w = object.__new__(StreamDiffusionWrapper) + w._output_pin_buf = None + w._d2h_event = None + w._ipc_pack_buf = None + w._ipc_pack_unit_buf = None + return w + + +def _reference_ipc_pack_rgba(wrapper, image_tensor): + """Pre-5f formula: fresh alpha (full_like) + cat every call.""" + denorm = wrapper._denormalize_on_gpu(image_tensor) + if denorm.dim() == 4: + denorm = denorm[0] + rgb_u8 = (denorm * 255).clamp(0, 255).to(torch.uint8) + rgb_hwc = rgb_u8.permute(1, 2, 0).contiguous() + alpha = torch.full_like(rgb_hwc[..., :1], 255) + return torch.cat([rgb_hwc[..., 2:3], rgb_hwc[..., 1:2], rgb_hwc[..., 0:1], alpha], dim=-1).contiguous() + + +def _reference_ipc_pack_unit_rgba(image_tensor): + """Pre-5f formula for the CN-preview twin (skips denormalize).""" + t = image_tensor + if t.dim() == 4: + t = t[0] + rgb_u8 = (t * 255).clamp(0, 255).to(torch.uint8) + rgb_hwc = rgb_u8.permute(1, 2, 0).contiguous() + alpha = torch.full_like(rgb_hwc[..., :1], 255) + return torch.cat([rgb_hwc[..., 2:3], rgb_hwc[..., 1:2], rgb_hwc[..., 0:1], alpha], dim=-1).contiguous() + + +def _reference_tensor_to_pil_optimized(wrapper, image_tensor): + """Pre-5d formula: blocking, unpinned .cpu().""" + denormalized = wrapper._denormalize_on_gpu(image_tensor) + uint8_tensor = (denormalized * 255).clamp(0, 255).to(torch.uint8) + cpu_tensor = uint8_tensor.cpu() + cpu_tensor = cpu_tensor.permute(0, 2, 3, 1) + pil_images = [] + for i in range(cpu_tensor.shape[0]): + img_array = cpu_tensor[i].numpy() + if img_array.shape[-1] == 1: + pil_images.append(Image.fromarray(img_array.squeeze(-1), mode="L")) + else: + pil_images.append(Image.fromarray(img_array)) + return pil_images + + +def _reference_tensor_to_pil_safe(tensor): + """Pre-5e formula: range-check syncs (tensor.min()/.max()) before the D2H transfer.""" + if tensor.dim() == 4: + tensor = tensor.squeeze(0) + if tensor.dim() == 3 and tensor.shape[0] == 3: + tensor = tensor.permute(1, 2, 0) + if tensor.min() < 0: + tensor = (tensor / 2.0 + 0.5).clamp(0, 1) + if tensor.max() <= 1.0: + tensor = tensor * 255.0 + numpy_image = tensor.detach().cpu().numpy().astype(np.uint8) + return Image.fromarray(numpy_image) + + +# --------------------------------------------------------------------------- +# 5f: _ipc_pack_rgba / _ipc_pack_unit_rgba +# --------------------------------------------------------------------------- + + +class TestIpcPackPersistentBuffer: + def test_ipc_pack_rgba_byte_identical_to_reference(self): + wrapper = _make_wrapper_shell() + torch.manual_seed(0) + image_tensor = torch.rand(1, 3, 64, 96, device="cuda") * 2 - 1 # diffusion range [-1, 1] + + expected = _reference_ipc_pack_rgba(wrapper, image_tensor) + actual = wrapper._ipc_pack_rgba(image_tensor) + + assert torch.equal(actual.cpu(), expected.cpu()) + + def test_ipc_pack_rgba_reuses_same_buffer_across_calls(self): + """5f's entire point: same shape -> same underlying allocation, no per-frame alloc.""" + wrapper = _make_wrapper_shell() + t1 = torch.rand(1, 3, 64, 96, device="cuda") * 2 - 1 + t2 = torch.rand(1, 3, 64, 96, device="cuda") * 2 - 1 + + buf1 = wrapper._ipc_pack_rgba(t1) + ptr1 = buf1.data_ptr() + buf2 = wrapper._ipc_pack_rgba(t2) + + assert buf2.data_ptr() == ptr1, "expected the persistent buffer to be reused, not reallocated" + + def test_ipc_pack_rgba_reallocs_on_shape_change(self): + wrapper = _make_wrapper_shell() + t1 = torch.rand(1, 3, 64, 96, device="cuda") * 2 - 1 + t2 = torch.rand(1, 3, 32, 48, device="cuda") * 2 - 1 + + buf1 = wrapper._ipc_pack_rgba(t1) + assert buf1.shape[:2] == (64, 96) + buf2 = wrapper._ipc_pack_rgba(t2) + assert buf2.shape[:2] == (32, 48) + + def test_ipc_pack_unit_rgba_byte_identical_to_reference(self): + wrapper = _make_wrapper_shell() + torch.manual_seed(1) + image_tensor = torch.rand(1, 3, 48, 64, device="cuda") # already [0, 1], no denorm + + expected = _reference_ipc_pack_unit_rgba(image_tensor) + actual = wrapper._ipc_pack_unit_rgba(image_tensor) + + assert torch.equal(actual.cpu(), expected.cpu()) + + def test_ipc_pack_rgba_and_unit_rgba_use_independent_buffers(self): + """The two packers must not alias the same persistent buffer (separate exporters).""" + wrapper = _make_wrapper_shell() + t = torch.rand(1, 3, 40, 56, device="cuda") + + buf_main = wrapper._ipc_pack_rgba(t * 2 - 1) + buf_unit = wrapper._ipc_pack_unit_rgba(t) + + assert buf_main.data_ptr() != buf_unit.data_ptr() + + +# --------------------------------------------------------------------------- +# 5d: _tensor_to_pil_optimized +# --------------------------------------------------------------------------- + + +class TestTensorToPilOptimizedPinnedReadback: + def test_byte_identical_to_reference(self): + wrapper = _make_wrapper_shell() + torch.manual_seed(2) + image_tensor = torch.rand(2, 3, 40, 56, device="cuda") * 2 - 1 + + expected = _reference_tensor_to_pil_optimized(wrapper, image_tensor) + actual = wrapper._tensor_to_pil_optimized(image_tensor) + + assert len(actual) == len(expected) + for a, e in zip(actual, expected): + assert np.array_equal(np.array(a), np.array(e)) + + def test_reuses_output_pin_buf_across_calls(self): + """5d shares _output_pin_buf/_d2h_event with the 'np' output path (same shape/dtype guard).""" + wrapper = _make_wrapper_shell() + t1 = torch.rand(1, 3, 32, 32, device="cuda") * 2 - 1 + t2 = torch.rand(1, 3, 32, 32, device="cuda") * 2 - 1 + + wrapper._tensor_to_pil_optimized(t1) + buf_ptr_1 = wrapper._output_pin_buf.data_ptr() + wrapper._tensor_to_pil_optimized(t2) + buf_ptr_2 = wrapper._output_pin_buf.data_ptr() + + assert buf_ptr_1 == buf_ptr_2 + + +# --------------------------------------------------------------------------- +# 5e: _tensor_to_pil_safe (CPU-first reorder) +# --------------------------------------------------------------------------- + + +class TestTensorToPilSafeCpuFirst: + def test_byte_identical_vae_range(self): + """VAE-range [-1, 1] input takes both the min()<0 and max()<=1.0 branches.""" + torch.manual_seed(3) + tensor = torch.rand(3, 40, 56, device="cuda") * 2 - 1 + + orchestrator = object.__new__(PreprocessingOrchestrator) + expected = _reference_tensor_to_pil_safe(tensor.clone()) + actual = orchestrator._tensor_to_pil_safe(tensor.clone()) + + assert np.array_equal(np.array(actual), np.array(expected)) + + def test_byte_identical_unit_range(self): + """[0, 1] input only takes the max()<=1.0 branch.""" + torch.manual_seed(4) + tensor = torch.rand(3, 40, 56, device="cuda") + + orchestrator = object.__new__(PreprocessingOrchestrator) + expected = _reference_tensor_to_pil_safe(tensor.clone()) + actual = orchestrator._tensor_to_pil_safe(tensor.clone()) + + assert np.array_equal(np.array(actual), np.array(expected)) + + def test_byte_identical_already_0_255_range(self): + """Values already > 1.0 (pre-scaled to [0, 255]) skip both conditional branches.""" + torch.manual_seed(5) + tensor = torch.rand(3, 40, 56, device="cuda") * 255.0 + + orchestrator = object.__new__(PreprocessingOrchestrator) + expected = _reference_tensor_to_pil_safe(tensor.clone()) + actual = orchestrator._tensor_to_pil_safe(tensor.clone()) + + assert np.array_equal(np.array(actual), np.array(expected)) From cda335bddcb3a8f7d79bb6164625b56b4a5c6d00 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 10 Jul 2026 19:26:13 -0400 Subject: [PATCH 10/37] fix: batch-dynamic engines skip l2tc tiling, silencing benign VALIDATE FAIL build_engine() passed dynamic_shapes=build_dynamic_shape into Engine.build(), which tracks only dynamic resolution. On the default "Flexible" preset the UNet is resolution-static but batch-dynamic (build_static_batch=False), so dynamic_shapes was passed False, and _apply_gpu_profile_to_config's tiling branch ran against a graph that still has a symbolic batch dim -- TRT emitted "[l2tc] VALIDATE FAIL - Graph contains symbolic shape" as a no-op for every applicable layer, for the entire ~11.5 min production TD session analyzed this round. Fix: dynamic_shapes=build_dynamic_shape or not build_static_batch (one line, utilities.py:1372) -- True whenever any dim, resolution or batch, is symbolic, matching l2tc's actual requirement (ALL dims must be concrete). Fully-static engines (e.g. ControlNet: build_static_batch=True, build_dynamic_shape=False) are unaffected -- still get tiling as before. Tightened the _apply_gpu_profile_to_config docstring to match. Verified via new tests/unit/test_l2tc_dynamic_shapes.py: calls build_engine() with GPU detection/memory query/the real TRT build all monkeypatched out (no CUDA/TRT context required) and asserts the dynamic_shapes kwarg it wires into Engine.build() for the previously-broken case (batch-dynamic, resolution-static -> True), the unaffected fully-static case (-> False), and the already-working resolution-dynamic case (-> True). Suite 107/0 (104 baseline + 3 new); ruff clean; pyrefly 72 errors (baseline, no new). Impact is log-spam-only -- zero perf/numerical change, visible on the next engine rebuild. --- .../acceleration/tensorrt/utilities.py | 9 +- tests/unit/test_l2tc_dynamic_shapes.py | 90 +++++++++++++++++++ 2 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 tests/unit/test_l2tc_dynamic_shapes.py diff --git a/src/streamdiffusion/acceleration/tensorrt/utilities.py b/src/streamdiffusion/acceleration/tensorrt/utilities.py index 6b63c9866..6eec841d4 100644 --- a/src/streamdiffusion/acceleration/tensorrt/utilities.py +++ b/src/streamdiffusion/acceleration/tensorrt/utilities.py @@ -289,7 +289,7 @@ def _apply_gpu_profile_to_config( Args: config: TRT IBuilderConfig to modify. gpu_profile: Hardware-detected build parameters from detect_gpu_profile(). - dynamic_shapes: Whether this engine uses dynamic input shapes. + dynamic_shapes: Whether this engine has any symbolic dim, incl. batch. - True (default): tiling and l2_limit skipped — TRT confirms these have no effect on symbolic-shape graphs and only produce warning spam. - False (static): tiling and l2_limit applied for full L2 cache benefit. @@ -1369,7 +1369,12 @@ def build_engine( workspace_size=max_workspace_size, fp8=fp8, gpu_profile=gpu_profile, - dynamic_shapes=build_dynamic_shape, + # Any symbolic dim (resolution OR batch) disqualifies l2tc tiling — see + # _apply_gpu_profile_to_config. build_dynamic_shape alone misses the + # batch-dynamic/resolution-static case (e.g. default "Flexible" UNet + # preset), which previously reached the tiling branch and TRT emitted + # "[l2tc] VALIDATE FAIL - Graph contains symbolic shape" as a no-op. + dynamic_shapes=build_dynamic_shape or not build_static_batch, ) return engine diff --git a/tests/unit/test_l2tc_dynamic_shapes.py b/tests/unit/test_l2tc_dynamic_shapes.py new file mode 100644 index 000000000..1d3d3ca79 --- /dev/null +++ b/tests/unit/test_l2tc_dynamic_shapes.py @@ -0,0 +1,90 @@ +""" +Regression test for the l2tc `dynamic_shapes` fix (Phase 5, Commit B — +docs/perf_bestpractices_audit_2026-07-10.md follow-up). + +Root cause: `build_engine()` passed `dynamic_shapes=build_dynamic_shape` into +`Engine.build()`, which tracks only dynamic *resolution*. Batch-dynamic / +resolution-static engines (e.g. the default "Flexible" UNet preset, +`build_static_batch=False, build_dynamic_shape=False`) got `dynamic_shapes=False`, +so `_apply_gpu_profile_to_config`'s tiling branch ran against a graph that still +has a symbolic batch dim, and TRT emitted a benign but noisy +"[l2tc] VALIDATE FAIL - Graph contains symbolic shape" warning for every +applicable layer. + +Fix: `dynamic_shapes=build_dynamic_shape or not build_static_batch` — True +whenever *any* dim (resolution or batch) is symbolic, matching +`_apply_gpu_profile_to_config`'s actual requirement (l2tc needs ALL dims +concrete). Fully-static engines (e.g. ControlNet: `build_static_batch=True, +build_dynamic_shape=False`) are unaffected. + +These tests exercise only `build_engine`'s pure-Python wiring: GPU detection, +the memory query, and the real `Engine.build()` TRT call are all monkeypatched +out, so no CUDA device or TensorRT context is required. +""" + +from unittest.mock import MagicMock + +import pytest + + +try: + from streamdiffusion.acceleration.tensorrt import utilities as trt_utilities + + IMPORT_OK = True +except ImportError: + IMPORT_OK = False + +pytestmark = pytest.mark.skipif( + not IMPORT_OK, + reason="acceleration.tensorrt.utilities not importable (TensorRT/onnx/polygraphy missing)", +) + + +class _FakeModelData: + """Stand-in for a BaseModel subclass — build_engine only calls get_input_profile().""" + + def get_input_profile(self, *args, **kwargs): + return {} + + +def _computed_dynamic_shapes(monkeypatch, *, build_static_batch, build_dynamic_shape): + """Call build_engine() with GPU detection / memory query / the real TRT build + all monkeypatched out, and return the `dynamic_shapes` kwarg it actually + passed to Engine.build().""" + monkeypatch.setattr(trt_utilities.torch.cuda, "current_device", lambda: 0) + monkeypatch.setattr(trt_utilities, "detect_gpu_profile", lambda device: trt_utilities._fallback_profile()) + monkeypatch.setattr(trt_utilities.cudart, "cudaMemGetInfo", lambda: (0, 8 * 2**30, 16 * 2**30)) + + captured_build = MagicMock(return_value=None) + monkeypatch.setattr(trt_utilities.Engine, "build", captured_build) + + trt_utilities.build_engine( + engine_path="fake.engine", + onnx_opt_path="fake.onnx", + model_data=_FakeModelData(), + opt_image_height=512, + opt_image_width=512, + opt_batch_size=1, + build_static_batch=build_static_batch, + build_dynamic_shape=build_dynamic_shape, + ) + + return captured_build.call_args.kwargs["dynamic_shapes"] + + +class TestL2tcDynamicShapesFix: + def test_batch_dynamic_resolution_static_is_dynamic(self, monkeypatch): + """Default 'Flexible' UNet preset: static resolution, dynamic batch — + this is the bug case; must compute True (previously computed False).""" + result = _computed_dynamic_shapes(monkeypatch, build_static_batch=False, build_dynamic_shape=False) + assert result is True + + def test_fully_static_engine_is_not_dynamic(self, monkeypatch): + """ControlNet-style fully-static engine: unaffected, still computes False.""" + result = _computed_dynamic_shapes(monkeypatch, build_static_batch=True, build_dynamic_shape=False) + assert result is False + + def test_resolution_dynamic_is_dynamic_regardless_of_batch(self, monkeypatch): + """build_dynamic_shape=True alone was already sufficient before this fix.""" + result = _computed_dynamic_shapes(monkeypatch, build_static_batch=True, build_dynamic_shape=True) + assert result is True From d5efd4dbb84f12d88a43a59c092875c1e10246de Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 10 Jul 2026 20:01:03 -0400 Subject: [PATCH 11/37] perf: elide trt.input_staging DtoD copy for kvo/fio UNet cache inputs (Sub-phase 5.6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Engine.infer() copied every feed_dict tensor into its own persistent staging buffer each frame solely to give TensorRT a stable contiguous address to bind. nsys profiling (5.0/5.5) attributed ~9% of frame time (p95 3.1ms, ~161 DtoD copies/frame) to this region on the UNet call, dominated by the kvo/fio StreamV2V cache inputs. Those cache tensors are already persistent, address-stable, TRT-contiguous buffers (create_kvo_cache/create_fi_cache build them with layers_in_bucket as the outermost dim specifically for this) — the only event that moves their address is a batch-size change, which already forces a full buffer realloc+graph reset today. So copying them every frame is pure waste. Add an opt-in zero_copy_names parameter to Engine.infer() and a pure _staging_action() decision helper (copy/bind/bind_and_reset) that lets TensorRT bind directly to the caller's tensor instead of copying into self.tensors[name], guarded by contiguity and dtype-match fallbacks. Wire it in UNet2DConditionModelEngine for the kvo/fio input names only; every other caller (VAE, ControlNet, safety, preprocessing) passes nothing and keeps the original copy path byte-for-byte. --- .../tensorrt/runtime_engines/unet_engine.py | 10 + .../acceleration/tensorrt/utilities.py | 69 ++++++- tests/unit/test_zero_copy_staging_5_6.py | 181 ++++++++++++++++++ 3 files changed, 257 insertions(+), 3 deletions(-) create mode 100644 tests/unit/test_zero_copy_staging_5_6.py diff --git a/src/streamdiffusion/acceleration/tensorrt/runtime_engines/unet_engine.py b/src/streamdiffusion/acceleration/tensorrt/runtime_engines/unet_engine.py index da7da67ff..462d0290a 100644 --- a/src/streamdiffusion/acceleration/tensorrt/runtime_engines/unet_engine.py +++ b/src/streamdiffusion/acceleration/tensorrt/runtime_engines/unet_engine.py @@ -54,6 +54,13 @@ def __init__(self, filepath: str, stream: "cuda.Stream", use_cuda_graph: bool = self._fio_out_names: List[str] = [] self._fio_cache_len: int = -1 # -1 = not yet initialized + # Sub-phase 5.6: kvo/fio cache inputs are persistent, address-stable, + # TRT-contiguous tensors (see models/utils.py create_kvo_cache/create_fi_cache), + # so Engine.infer() can bind TensorRT directly to them instead of copying into + # its own staging buffer every frame. Rebuilt only when the kvo/fio lazy-init + # branches below fire (cache length change), never per-frame. + self._zero_copy_names: FrozenSet[str] = frozenset() + self._shape_dict: Dict[str, Any] = {} self._input_dict: Dict[str, Any] = {} @@ -90,6 +97,7 @@ def __call__( self._kvo_in_names = [f"kvo_cache_in_{i}" for i in range(n_kvo)] self._kvo_out_names = [f"kvo_cache_out_{i}" for i in range(n_kvo)] self._kvo_cache_len = n_kvo + self._zero_copy_names = frozenset(self._kvo_in_names + self._fio_in_names) # Lazy-init FIO key name lists — same pattern as KVO. n_fio = len(fio_cache) @@ -97,6 +105,7 @@ def __call__( self._fio_in_names = [f"fio_cache_in_{i}" for i in range(n_fio)] self._fio_out_names = [f"fio_cache_out_{i}" for i in range(n_fio)] self._fio_cache_len = n_fio + self._zero_copy_names = frozenset(self._kvo_in_names + self._fio_in_names) # Update pre-allocated dicts in-place — no new dict objects created per call. shape_dict = self._shape_dict @@ -198,6 +207,7 @@ def __call__( input_dict, self.stream, use_cuda_graph=self.use_cuda_graph, + zero_copy_names=self._zero_copy_names, ) except Exception as e: logger.exception(f"UNet2DConditionModelEngine.__call__: Engine.infer failed: {e}") diff --git a/src/streamdiffusion/acceleration/tensorrt/utilities.py b/src/streamdiffusion/acceleration/tensorrt/utilities.py index 6eec841d4..67be6ae74 100644 --- a/src/streamdiffusion/acceleration/tensorrt/utilities.py +++ b/src/streamdiffusion/acceleration/tensorrt/utilities.py @@ -541,6 +541,35 @@ def _median(v): return "\n".join(lines) +def _staging_action( + name: str, + zero_copy_names: frozenset, + is_contiguous: bool, + dtype_match: bool, + prev_ptr: Optional[int], + cur_ptr: int, + graph_exists: bool, +) -> str: + """Decide how Engine.infer() should stage one feed_dict input. + + Pure function (no CUDA calls) so the zero-copy decision table is unit-testable + without a real CUDA graph — see Sub-phase 5.6, + docs/perf_bestpractices_audit_2026-07-10.md. + + Returns one of: + "copy" - copy into self.tensors[name] (today's behavior; always safe) + "bind" - skip the copy; bind TensorRT directly to the caller's tensor + "bind_and_reset" - skip the copy and bind directly, but reset the CUDA graph + first because the caller's address changed while a graph + built from the old address was still live + """ + if name not in zero_copy_names or not is_contiguous or not dtype_match: + return "copy" + if graph_exists and cur_ptr != prev_ptr: + return "bind_and_reset" + return "bind" + + class Engine: def __init__( self, @@ -561,6 +590,11 @@ def __init__( # Cached ExternalStream wrapping the engine's polygraphy stream; allocated on # first infer() call so we avoid constructing a new Python wrapper every frame. self._engine_ext_stream = None + # Sub-phase 5.6: last-bound caller tensor address per zero-copy input name, + # so infer() can detect a pointer change and force a graph reset instead of + # silently replaying stale addresses. Only populated for names actually + # passed via infer()'s zero_copy_names. + self._bound_ptrs: dict[str, int] = {} def __del__(self): # Check if AttributeError: 'Engine' object has no attribute 'buffers' @@ -1086,7 +1120,7 @@ def reset_cuda_graph(self): CUASSERT(cudart.cudaGraphDestroy(self.graph)) self.graph = None - def infer(self, feed_dict, stream, use_cuda_graph=False): + def infer(self, feed_dict, stream, use_cuda_graph=False, zero_copy_names: frozenset = frozenset()): # IProfiler cannot report per-layer times through CUDA graph replay — disable graphs # when profiler is attached. This is automatically set when STREAMDIFFUSION_PROFILE_TRT # is set in activate(), so callers do not need to change anything. @@ -1133,10 +1167,36 @@ def infer(self, feed_dict, stream, use_cuda_graph=False): pt_stream, stream.ptr, ) + # Sub-phase 5.6: for opt-in zero-copy names (kvo/fio UNet cache inputs — already + # persistent, address-stable, TRT-contiguous tensors), skip the DtoD copy and + # point TensorRT directly at the caller's tensor instead. Every other input + # (zero_copy_names defaults to frozenset()) takes the original copy_() path, + # so non-UNet engines (VAE/ControlNet/safety) are byte-for-byte unchanged. + bind_ptrs: dict[str, int] = {} + needs_reset = False + graph_exists = self.cuda_graph_instance is not None with torch.cuda.stream(self._engine_ext_stream): with _gpu_profiler.region("trt.input_staging"): for name, buf in feed_dict.items(): - self.tensors[name].copy_(buf) + cur_ptr = buf.data_ptr() + action = _staging_action( + name, + zero_copy_names, + buf.is_contiguous(), + buf.dtype == self.tensors[name].dtype, + self._bound_ptrs.get(name), + cur_ptr, + graph_exists, + ) + if action == "copy": + self.tensors[name].copy_(buf) + else: + bind_ptrs[name] = cur_ptr + if action == "bind_and_reset": + needs_reset = True + + if needs_reset and self.cuda_graph_instance is not None: + self.reset_cuda_graph() # In graphed steady state the tensor addresses are baked into the captured graph # (self.tensors[name] are persistent buffers reused via copy_() — see @@ -1145,8 +1205,11 @@ def infer(self, feed_dict, stream, use_cuda_graph=False): # host overhead; only rebind on first capture or after a reset (instance is None). if not (use_cuda_graph and self.cuda_graph_instance is not None): for name, tensor in self.tensors.items(): - if not self.context.set_tensor_address(name, tensor.data_ptr()): + address = bind_ptrs.get(name, tensor.data_ptr()) + if not self.context.set_tensor_address(name, address): raise RuntimeError(f"TensorRT: set_tensor_address failed for '{name}'") + if name in bind_ptrs: + self._bound_ptrs[name] = address with _gpu_profiler.region("trt_infer"): if use_cuda_graph: diff --git a/tests/unit/test_zero_copy_staging_5_6.py b/tests/unit/test_zero_copy_staging_5_6.py new file mode 100644 index 000000000..56bbe341f --- /dev/null +++ b/tests/unit/test_zero_copy_staging_5_6.py @@ -0,0 +1,181 @@ +""" +Unit tests for Sub-phase 5.6 (`trt.input_staging` DtoD copy elision, +docs/perf_bestpractices_audit_2026-07-10.md follow-up — see the Phase 5 +continuation plan). + +`Engine.infer()` copies every feed_dict input into its own persistent staging +buffer every frame, purely to give TensorRT a stable contiguous address to +bind. The kvo/fio UNet cache inputs are already persistent, address-stable, +TRT-contiguous tensors (see models/utils.py create_kvo_cache/create_fi_cache), +so that copy is pure waste for them — 5.6 lets `Engine.infer()` bind directly +to an opt-in set of caller tensors instead. + +`_staging_action()` is the pure decision function extracted from `Engine.infer()` +so the per-input zero-copy/copy/reset decision table is testable without a real +CUDA graph (a `cuda_graph_instance` can't be meaningfully MagicMock'd — the same +constraint Sub-phase 5.1's rebind guard hit). This module tests only that pure +function: no CUDA device, no TensorRT context required beyond importing the +module (guarded below, same pattern as test_l2tc_dynamic_shapes.py). +""" + +import pytest + + +try: + from streamdiffusion.acceleration.tensorrt.utilities import _staging_action + + IMPORT_OK = True +except ImportError: + IMPORT_OK = False + +pytestmark = pytest.mark.skipif( + not IMPORT_OK, + reason="acceleration.tensorrt.utilities not importable (TensorRT/onnx/polygraphy missing)", +) + + +PTR_A = 0xDEAD_BEEF +PTR_B = 0xFEED_FACE + + +class TestStagingActionNotZeroCopy: + """Anything not opted in always falls back to the original copy path.""" + + def test_name_not_in_zero_copy_names_copies(self): + result = _staging_action( + "sample", + frozenset({"kvo_cache_in_0"}), + True, + True, + None, + PTR_A, + False, + ) + assert result == "copy" + + def test_empty_zero_copy_names_always_copies(self): + """Default frozenset() → today's behavior exactly, for every name.""" + result = _staging_action( + "kvo_cache_in_0", + frozenset(), + True, + True, + PTR_A, + PTR_A, + True, + ) + assert result == "copy" + + +class TestStagingActionSafetyGuards: + """A zero-copy candidate that fails a safety guard falls back to copy, + never silently binding a non-contiguous tensor or mismatched dtype.""" + + def test_non_contiguous_falls_back_to_copy(self): + result = _staging_action( + "kvo_cache_in_0", + frozenset({"kvo_cache_in_0"}), + False, # is_contiguous + True, + None, + PTR_A, + False, + ) + assert result == "copy" + + def test_dtype_mismatch_falls_back_to_copy(self): + result = _staging_action( + "kvo_cache_in_0", + frozenset({"kvo_cache_in_0"}), + True, + False, # dtype_match + None, + PTR_A, + False, + ) + assert result == "copy" + + def test_both_guards_fail_falls_back_to_copy(self): + result = _staging_action( + "kvo_cache_in_0", + frozenset({"kvo_cache_in_0"}), + False, + False, + None, + PTR_A, + True, + ) + assert result == "copy" + + +class TestStagingActionBind: + """Eligible names bind directly; a reset is forced only when a live graph's + baked address would otherwise go stale.""" + + def test_no_graph_yet_binds_without_reset(self): + """First-ever call: no graph exists, so a fresh bind is always safe.""" + result = _staging_action( + "kvo_cache_in_0", + frozenset({"kvo_cache_in_0"}), + True, + True, + None, + PTR_A, + False, + ) + assert result == "bind" + + def test_graph_exists_pointer_unchanged_binds_without_reset(self): + """Steady state: same persistent tensor, same address — no reset needed.""" + result = _staging_action( + "kvo_cache_in_0", + frozenset({"kvo_cache_in_0"}), + True, + True, + PTR_A, + PTR_A, + True, + ) + assert result == "bind" + + def test_graph_exists_pointer_changed_binds_and_resets(self): + """Belt-and-suspenders: an unforeseen pointer change while a graph is + live must force a re-capture, not silently rebind a stale graph.""" + result = _staging_action( + "kvo_cache_in_0", + frozenset({"kvo_cache_in_0"}), + True, + True, + PTR_A, + PTR_B, + True, + ) + assert result == "bind_and_reset" + + def test_no_graph_pointer_changed_still_just_binds(self): + """No live graph to invalidate → no reset needed even if the pointer moved + (e.g. right after allocate_buffers already reset the graph for a shape change).""" + result = _staging_action( + "kvo_cache_in_0", + frozenset({"kvo_cache_in_0"}), + True, + True, + PTR_A, + PTR_B, + False, + ) + assert result == "bind" + + def test_first_bind_with_no_prior_ptr_recorded(self): + """prev_ptr=None (nothing bound yet) with no live graph must not crash and + must bind cleanly — exercises the None-vs-int comparison short-circuit.""" + result = _staging_action( + "fio_cache_in_0", + frozenset({"fio_cache_in_0"}), + True, + True, + None, + PTR_A, + False, + ) + assert result == "bind" From a9b1669a354122b1808f271f7007a2747b238a5d Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 11 Jul 2026 08:27:29 -0400 Subject: [PATCH 12/37] fix: lower ControlNet dynamic-shape floor 384->256 via shared min_image_shape (bug #6) ControlNet's dynamic-shape get_input_profile() hardcoded a 384px floor independent of the UNet's 256px floor (BaseModel.min_image_shape), so any resolution in [256, 384) hard-failed at Engine.allocate_buffers the moment ControlNet was enabled. Derive the ControlNet floor/ceiling from the same BaseModel.min_image_shape/max_image_shape the UNet already uses (SDXL inherits via super()), lower the matching engine_manager.py build-option default, and relabel the ControlNet dynamic-engine cache directory (--dyn-384-1024 -> --dyn-256-1024) so a re-floored build never collides with a stale 384-floored cache. Trade-off: widening the dynamic profile can slightly reduce TRT tactic specialization at the optimum resolution -- acceptable given the alternative is a hard failure. Co-Authored-By: Claude Sonnet 5 --- .../acceleration/tensorrt/engine_manager.py | 8 ++++++-- .../tensorrt/models/controlnet_models.py | 14 ++++++++------ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/streamdiffusion/acceleration/tensorrt/engine_manager.py b/src/streamdiffusion/acceleration/tensorrt/engine_manager.py index a7a301c1f..6ed4ad645 100644 --- a/src/streamdiffusion/acceleration/tensorrt/engine_manager.py +++ b/src/streamdiffusion/acceleration/tensorrt/engine_manager.py @@ -138,7 +138,7 @@ def get_engine_path( if resolution is not None: prefix = f"controlnet_{model_dir_name}--min_batch-{min_batch_size}--max_batch-{max_batch_size}--res-{resolution[0]}x{resolution[1]}" else: - prefix = f"controlnet_{model_dir_name}--min_batch-{min_batch_size}--max_batch-{max_batch_size}--dyn-384-1024" + prefix = f"controlnet_{model_dir_name}--min_batch-{min_batch_size}--max_batch-{max_batch_size}--dyn-256-1024" fp8_suffix = "--fp8" if fp8 else "" return self.engine_dir / (prefix + optlvl_suffix + fp8_suffix) / filename else: @@ -277,7 +277,11 @@ def _get_default_controlnet_build_options( "build_static_batch": True, } if build_dynamic_shape: - opts["min_image_resolution"] = 384 + # Match BaseModel/UNet's 256 floor (was 384) so [256, 384) resolutions + # don't hard-fail with ControlNet active — see get_input_profile in + # controlnet_models.py, which now derives its own floor from the same + # BaseModel.min_image_shape instead of a separate hardcoded literal. + opts["min_image_resolution"] = 256 opts["max_image_resolution"] = 1024 if builder_optimization_level is not None: opts["builder_optimization_level"] = builder_optimization_level diff --git a/src/streamdiffusion/acceleration/tensorrt/models/controlnet_models.py b/src/streamdiffusion/acceleration/tensorrt/models/controlnet_models.py index 125c8797a..cba02b6b6 100644 --- a/src/streamdiffusion/acceleration/tensorrt/models/controlnet_models.py +++ b/src/streamdiffusion/acceleration/tensorrt/models/controlnet_models.py @@ -65,12 +65,14 @@ def get_input_profile(self, batch_size, image_height, image_width, static_batch, min_ctrl_h = max_ctrl_h = opt_ctrl_h = image_height min_ctrl_w = max_ctrl_w = opt_ctrl_w = image_width else: - min_ctrl_h = 384 - max_ctrl_h = 1024 - opt_ctrl_h = 704 - min_ctrl_w = 384 - max_ctrl_w = 1024 - opt_ctrl_w = 704 + # Share the floor/ceiling with BaseModel (min_image_shape/max_image_shape) + # instead of a hardcoded 384 — keeps ControlNet's dynamic range aligned with + # the UNet's (256-1024), so [256, 384) resolutions no longer hard-fail with + # ControlNet active. opt stays 704, clamped into range defensively. + min_ctrl_h, max_ctrl_h = self.min_image_shape, self.max_image_shape + min_ctrl_w, max_ctrl_w = self.min_image_shape, self.max_image_shape + opt_ctrl_h = min(max(704, min_ctrl_h), max_ctrl_h) + opt_ctrl_w = min(max(704, min_ctrl_w), max_ctrl_w) min_latent_h = min_ctrl_h // 8 max_latent_h = max_ctrl_h // 8 From 75ee38d6e397f65efc97d55fd3d3e7a0e58ff002 Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 11 Jul 2026 08:43:21 -0400 Subject: [PATCH 13/37] perf: normalize blend weights on CPU + persistent ControlNet conditioning_scale scalar (Sub-phase 5.4) Two per-frame GPU creation syncs removed: - _normalize_weights built a CUDA tensor from a Python list every call, but every consumer (.item()/.tolist() in slerp/multi_slerp/cosine_weighted paths, float*cuda_tensor in the linear paths) immediately reads it back or treats it as a scalar operand. Build it on CPU float32 instead -- same consumers work unchanged (a 0-dim CPU tensor is treated as a wrapped scalar when multiplied against a CUDA tensor), the sync and readback are both gone, and float32 is more precise than the model's fp16 for the normalization divide. - ControlNetModelEngine rebuilt a conditioning_scale tensor via torch.tensor(...) on every __call__. Replace with one lazily-allocated persistent scalar updated via fill_() (an async kernel launch, no sync), mirroring the existing _fi_strength_tensor.fill_() idiom. Each active ControlNet has its own engine instance, so a per-instance buffer is correct; the binding rank (_cs_rank1) is fixed at load time so the shape never changes underneath it. Also drops three dead variable assignments and fixes one out-of-order import in stream_parameter_updater.py (pre-existing lint debt in a file this change already touches; the repo's commit gate lints the whole staged file, not just the diff). Co-Authored-By: Claude Sonnet 5 --- .../runtime_engines/controlnet_engine.py | 16 ++++++++---- .../stream_parameter_updater.py | 26 ++++++++++--------- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/src/streamdiffusion/acceleration/tensorrt/runtime_engines/controlnet_engine.py b/src/streamdiffusion/acceleration/tensorrt/runtime_engines/controlnet_engine.py index 35569c4f8..7d1ce77c2 100644 --- a/src/streamdiffusion/acceleration/tensorrt/runtime_engines/controlnet_engine.py +++ b/src/streamdiffusion/acceleration/tensorrt/runtime_engines/controlnet_engine.py @@ -42,6 +42,12 @@ def __init__( except Exception: self._cs_rank1 = False # sd15 engines / engines lacking this input + # Persistent conditioning_scale scalar, allocated lazily on first __call__ + # (device isn't known until then). fill_() is an async kernel launch, so + # reusing this buffer every call avoids the per-frame creation sync that + # torch.tensor(...) incurs. + self._cs: Optional[torch.Tensor] = None + # Pre-compute model-specific values to eliminate runtime branching if self.model_type in ["sdxl", "sdxl_turbo"]: self.max_blocks = 9 @@ -119,16 +125,16 @@ def __call__( if timestep.dtype != torch.float32: timestep = timestep.float() + if self._cs is None: + self._cs = torch.zeros((1,) if self._cs_rank1 else (), dtype=torch.float32, device=sample.device) + self._cs.fill_(conditioning_scale) + input_dict = { "sample": sample, "timestep": timestep, "encoder_hidden_states": encoder_hidden_states, "controlnet_cond": controlnet_cond, - "conditioning_scale": ( - torch.tensor([conditioning_scale], dtype=torch.float32, device=sample.device) - if self._cs_rank1 - else torch.tensor(conditioning_scale, dtype=torch.float32, device=sample.device) - ), + "conditioning_scale": self._cs, } if text_embeds is not None: diff --git a/src/streamdiffusion/stream_parameter_updater.py b/src/streamdiffusion/stream_parameter_updater.py index 233845858..13ca8cd1a 100644 --- a/src/streamdiffusion/stream_parameter_updater.py +++ b/src/streamdiffusion/stream_parameter_updater.py @@ -5,9 +5,10 @@ import torch import torch.nn.functional as F +from .preprocessing.orchestrator_user import OrchestratorUser + logger = logging.getLogger(__name__) -from .preprocessing.orchestrator_user import OrchestratorUser class CacheStats: @@ -129,11 +130,9 @@ def register_embedding_preprocessor(self, preprocessor: Any, style_image_key: st def unregister_embedding_preprocessor(self, style_image_key: str) -> None: """Unregister an embedding preprocessor by style image key.""" - original_count = len(self._embedding_preprocessors) self._embedding_preprocessors = [ (preprocessor, key) for preprocessor, key in self._embedding_preprocessors if key != style_image_key ] - removed_count = original_count - len(self._embedding_preprocessors) # Clear cached embeddings for this key if style_image_key in self._embedding_cache: @@ -213,8 +212,15 @@ def get_cached_embeddings(self, style_image_key: str) -> Optional[Tuple[torch.Te return cached_result def _normalize_weights(self, weights: List[float], normalize: bool) -> torch.Tensor: - """Generic weight normalization helper""" - weights_tensor = torch.tensor(weights, device=self.stream.device, dtype=self.stream.dtype) + """Generic weight normalization helper. + + Built on CPU float32 rather than self.stream.device/dtype: every caller either + reads scalars back out (.item()/.tolist()) or multiplies into a CUDA tensor + (a 0-dim CPU operand is treated as a wrapped scalar there), so building on + device was a pure creation-sync + readback with no benefit — and float32 is + more precise than the model's fp16 for the normalization divide. + """ + weights_tensor = torch.tensor(weights, dtype=torch.float32) if normalize: weights_tensor = weights_tensor / weights_tensor.sum() return weights_tensor @@ -992,11 +998,7 @@ def _update_timestep_calculations(self) -> None: # mis-interprets the clean buffer, causing ghost bleed from previous frames. # Threshold 0.75 matches the empirically observed perceptual onset (~t_index 30 in # a 50-step LCM schedule where beta_sqrt crosses 0.78). - if ( - self.stream.use_denoising_batch - and not self.stream.do_add_noise - and len(self.stream.t_list) > 1 - ): + if self.stream.use_denoising_batch and not self.stream.do_add_noise and len(self.stream.t_list) > 1: inter_step_betas = beta_prod_t_sqrt[1:, 0, 0, 0] # per-step, before repeat_interleave max_beta = inter_step_betas.max().item() _BLEED_THRESHOLD = 0.75 @@ -1201,7 +1203,7 @@ def remove_prompt_at_index( return # Remove from current list - removed_prompt = self._current_prompt_list.pop(index) + self._current_prompt_list.pop(index) # Remove from cache and reindex if index in self._prompt_cache: @@ -1302,7 +1304,7 @@ def remove_seed_at_index(self, index: int, interpolation_method: Literal["linear return # Remove from current list - removed_seed = self._current_seed_list.pop(index) + self._current_seed_list.pop(index) # Remove from cache and reindex if index in self._seed_cache: From fcb2ae25de5decd4496799132a89d88d02d87c2d Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 11 Jul 2026 08:56:28 -0400 Subject: [PATCH 14/37] perf: 1-frame-delayed async NSFW safety-checker readback (Sub-phase 5.3) Removes the cudaStreamSynchronize that NSFWDetectorEngine.__call__ forced on the hot path via probs[0,0].item() whenever use_safety_checker=True. The classifier now writes its GPU probability into a pinned host scalar via a non_blocking copy; _apply_safety_checker reads the *previous* frame's pinned verdict before launching this frame's classification, mirroring the existing SimilarImageFilter 1-frame-delayed readback pattern (image_filter.py). Edge cases: first frame passes through unconditionally (no prior verdict to act on); a 1-frame detection-leak window is possible before flagging lands. Both are acceptable for an opt-in, off-by-default, TensorRT-only feature. Updated tests/unit/test_safety_checker.py for the new (tensor, prob_pin) -> None checker contract and the delayed-readback semantics (10/10 passing). Co-Authored-By: Claude Sonnet 5 --- .../tensorrt/runtime_engines/unet_engine.py | 12 ++- src/streamdiffusion/wrapper.py | 37 +++++++- tests/unit/test_safety_checker.py | 93 +++++++++++++------ 3 files changed, 111 insertions(+), 31 deletions(-) diff --git a/src/streamdiffusion/acceleration/tensorrt/runtime_engines/unet_engine.py b/src/streamdiffusion/acceleration/tensorrt/runtime_engines/unet_engine.py index 462d0290a..de317065d 100644 --- a/src/streamdiffusion/acceleration/tensorrt/runtime_engines/unet_engine.py +++ b/src/streamdiffusion/acceleration/tensorrt/runtime_engines/unet_engine.py @@ -506,7 +506,13 @@ def __init__(self, filepath: str, stream: "cuda.Stream", use_cuda_graph: bool = self.engine.load() self.engine.activate() - def __call__(self, image_tensor: torch.Tensor, threshold: float): + def __call__(self, image_tensor: torch.Tensor, prob_pin: torch.Tensor) -> None: + """Launch classification and stash the NSFW probability in a pinned host buffer. + + Does not read the result back (no .item()/host sync here) — callers read + `prob_pin` on a *later* call, mirroring SimilarImageFilter's 1-frame-delayed + pinned readback so this never forces a cudaStreamSynchronize on the hot path. + """ pixel_values = self.image_transforms(image_tensor) self.engine.allocate_buffers( shape_dict={ @@ -522,8 +528,8 @@ def __call__(self, image_tensor: torch.Tensor, threshold: float): )["logits"] probs = F.softmax(logits, dim=-1) - nsfw_prob = 1 - probs[0, 0].item() - return nsfw_prob >= threshold + nsfw_prob = 1 - probs[0, 0] + prob_pin.copy_(nsfw_prob.view(1), non_blocking=True) def to(self, *args, **kwargs): pass diff --git a/src/streamdiffusion/wrapper.py b/src/streamdiffusion/wrapper.py index c981661db..17055f54c 100644 --- a/src/streamdiffusion/wrapper.py +++ b/src/streamdiffusion/wrapper.py @@ -376,6 +376,10 @@ def __init__( # Caches the last clean (non-flagged) pipeline tensor for the "previous" fallback strategy. # Operates in diffusion range [-1, 1]; set by _apply_safety_checker(). self._prev_clean_tensor: Optional[torch.Tensor] = None + # Pinned CPU scalar for the 1-frame-delayed async NSFW readback (lazy; see + # _apply_safety_checker). Distinct from _output_pin_buf, which is uint8 + # image-shaped and reallocated by the output-postprocessing paths. + self._nsfw_prob_pin: Optional[torch.Tensor] = None self.fp8 = fp8 self.static_shapes = static_shapes self.fp8_allow_fp16_fallback = fp8_allow_fp16_fallback @@ -1360,6 +1364,19 @@ def _apply_safety_checker(self, image_tensor: torch.Tensor) -> torch.Tensor: CUDA-IPC export path: postprocess_image() exports the frame inside its own body and returns None, making any post-hoc substitution unreachable and unsafe. + 1-frame-delayed async readback (mirrors SimilarImageFilter, image_filter.py): the + classifier result for THIS frame is read on the NEXT call, so the substitution below + acts on the PREVIOUS frame's verdict while this frame's classification + pinned + `non_blocking` copy is merely launched. This avoids forcing a cudaStreamSynchronize + on the hot path. No explicit sync guards the pinned read — the same trade-off + SimilarImageFilter already makes: stream ordering means the async copy is enqueued + before all of this frame's remaining GPU work, and by the time Python reaches this + call again next frame (after a full diffusion step + decode, and — for "np"/"pil" + output — a hard _d2h_event.synchronize() in postprocess_image) the tiny 4-byte D2H + copy has long since landed in practice. Worst case on other output types is a stale + read of an even-older value, i.e. one extra frame of detection delay, not corrupted + data. Accepted trade-off for an opt-in, off-by-default, TensorRT-only feature. + Parameters ---------- image_tensor : torch.Tensor @@ -1377,7 +1394,24 @@ def _apply_safety_checker(self, image_tensor: torch.Tensor) -> torch.Tensor: # Denormalize to [0, 1] NCHW for the classifier; stays on GPU. denormalized = self._denormalize_on_gpu(image_tensor) - if self.safety_checker(denormalized, self.safety_checker_threshold): + if self._nsfw_prob_pin is None: + # First frame: no prior async result exists yet. Launch this frame's + # classification so the next frame has a real verdict, but pass this one + # through unflagged (documented 1-frame pass-through edge case). + self._nsfw_prob_pin = torch.zeros(1, dtype=torch.float32, device="cpu").pin_memory() + self.safety_checker(denormalized, self._nsfw_prob_pin) + if self.safety_checker_fallback_type == "previous": + self._prev_clean_tensor = image_tensor.clone() + return image_tensor + + # Step 1: read the PREVIOUS frame's async result (pinned CPU read, no GPU sync). + flagged = self._nsfw_prob_pin.item() >= self.safety_checker_threshold + + # Step 2: launch THIS frame's classification + async pinned copy (no sync). + self.safety_checker(denormalized, self._nsfw_prob_pin) + + # Step 3: branch on the PREVIOUS frame's verdict (1-frame-delayed, no stall). + if flagged: logger.info("NSFW content detected, applying safety fallback frame") if self.safety_checker_fallback_type == "previous" and self._prev_clean_tensor is not None: return self._prev_clean_tensor @@ -3037,6 +3071,7 @@ def cleanup_gpu_memory(self) -> None: self._d2h_event = None self._ipc_pack_buf = None self._ipc_pack_unit_buf = None + self._nsfw_prob_pin = None # Force multiple garbage collection cycles for thorough cleanup for i in range(3): diff --git a/tests/unit/test_safety_checker.py b/tests/unit/test_safety_checker.py index ef5a3d133..1e2b1cbbb 100644 --- a/tests/unit/test_safety_checker.py +++ b/tests/unit/test_safety_checker.py @@ -15,6 +15,14 @@ Fix: _apply_safety_checker() runs *before* postprocess_image, operating on the raw diffusion-range [-1, 1] pipeline tensor so it is always a real tensor regardless of output path. + +Sub-phase 5.3: the checker's verdict is now read 1-frame-delayed from a pinned +async buffer (mirrors SimilarImageFilter) instead of a synchronous .item() call. +`safety_checker` is now `(tensor, prob_pin) -> None`: it writes into `prob_pin` +rather than returning a bool, and `_apply_safety_checker` reads the *previous* +call's pinned value before launching (and writing) this frame's result. Tests +below seed `w._nsfw_prob_pin` directly where they need to bypass the +first-frame pass-through and exercise a specific verdict. """ import torch @@ -39,6 +47,7 @@ def _make_wrapper( w.safety_checker_threshold = threshold w.safety_checker_fallback_type = fallback_type w._prev_clean_tensor = None + w._nsfw_prob_pin = None # safety_checker is set per-test w.safety_checker = None return w @@ -65,9 +74,9 @@ def test_checker_never_receives_none(self): """ received: list = [] - def capturing_checker(tensor, thr): + def capturing_checker(tensor, prob_pin): received.append(tensor) - return True # always flag + prob_pin.fill_(1.0) w = _make_wrapper() w.safety_checker = capturing_checker @@ -83,12 +92,15 @@ def capturing_checker(tensor, thr): # ── Case 2: NSFW → black frame (blank fallback) ────────────────────────── def test_nsfw_blank_fallback_is_black(self): w = _make_wrapper(fallback_type="blank") - w.safety_checker = lambda t, thr: True # always flag + w.safety_checker = lambda t, pin: pin.fill_(1.0) # always "flag" - dummy = torch.randn(1, 3, 64, 64) - result = w._apply_safety_checker(dummy) + frame1 = torch.randn(1, 3, 64, 64) + _ = w._apply_safety_checker(frame1) # first frame: pass-through, primes the pin + + frame2 = torch.randn(1, 3, 64, 64) + result = w._apply_safety_checker(frame2) # reads frame1's flagged verdict - assert result is not dummy, "flagged frame should be replaced" + assert result is not frame2, "flagged frame should be replaced" # Denormalize: (x/2+0.5).clamp(0,1); -1.0 → 0.0 = black denorm = (result / 2 + 0.5).clamp(0, 1) assert _black_denorm(denorm), "NSFW blank fallback should produce a black frame" @@ -98,37 +110,42 @@ def test_nsfw_previous_fallback_returns_cached_clean_frame(self): w = _make_wrapper(fallback_type="previous") call_count = [0] - def checker(t, thr): + def checker(t, pin): call_count[0] += 1 - return call_count[0] > 1 # first call: clean; second call: flagged + # Call #1 (launched during frame1) writes a flagged prob; frame2's + # read consumes that async result (1-frame delay). + pin.fill_(1.0 if call_count[0] == 1 else 0.0) w.safety_checker = checker clean = torch.randn(1, 3, 64, 64) - _ = w._apply_safety_checker(clean) # primes _prev_clean_tensor + _ = w._apply_safety_checker(clean) # first frame: pass-through, primes _prev_clean_tensor - flagged = torch.randn(1, 3, 64, 64) - result = w._apply_safety_checker(flagged) + flagged_frame = torch.randn(1, 3, 64, 64) + result = w._apply_safety_checker(flagged_frame) assert result is w._prev_clean_tensor, "previous-fallback should return the cached clean tensor" + assert torch.equal(w._prev_clean_tensor, clean) # ── Case 4: Clean frame passthrough ────────────────────────────────────── def test_clean_frame_returned_unchanged(self): w = _make_wrapper() - w.safety_checker = lambda t, thr: False # never flag + w.safety_checker = lambda t, pin: pin.fill_(0.0) # never flag - dummy = torch.randn(1, 3, 64, 64) - result = w._apply_safety_checker(dummy) + frame1 = torch.randn(1, 3, 64, 64) + _ = w._apply_safety_checker(frame1) # first frame: pass-through, primes the pin - assert torch.equal(result, dummy), "clean frame should be returned unchanged" + frame2 = torch.randn(1, 3, 64, 64) + result = w._apply_safety_checker(frame2) + + assert torch.equal(result, frame2), "clean frame should be returned unchanged" # ── Case 5: use_safety_checker=False bypasses entirely ─────────────────── def test_disabled_bypasses_checker(self): called = [False] - def should_not_be_called(t, thr): + def should_not_be_called(t, pin): called[0] = True - return False w = _make_wrapper(use_safety_checker=False) w.safety_checker = should_not_be_called @@ -139,14 +156,17 @@ def should_not_be_called(t, thr): assert not called[0], "safety checker must not be called when disabled" assert torch.equal(result, dummy), "disabled checker should return input unchanged" - # ── Case 6: previous fallback with no cached frame → black ─────────────── + # ── Case 6: flagged verdict with no cached frame → black ───────────────── def test_nsfw_previous_fallback_no_cache_falls_back_to_black(self): """ - If the very first frame is flagged and _prev_clean_tensor is None, - the fallback should still produce a black frame rather than raise. + If a frame is flagged (per the previous frame's pinned verdict) and + _prev_clean_tensor is None, the fallback should still produce a black + frame rather than raise. Seeds _nsfw_prob_pin directly to bypass the + first-frame pass-through and exercise this state combination. """ w = _make_wrapper(fallback_type="previous") - w.safety_checker = lambda t, thr: True # flag immediately + w.safety_checker = lambda t, pin: pin.fill_(0.0) + w._nsfw_prob_pin = torch.tensor([1.0]) # a prior flagged verdict already landed assert w._prev_clean_tensor is None # no cache yet dummy = torch.randn(1, 3, 64, 64) @@ -158,7 +178,8 @@ def test_nsfw_previous_fallback_no_cache_falls_back_to_black(self): # ── Case 7: clean frame caches for previous strategy ───────────────────── def test_clean_frame_cached_for_previous_strategy(self): w = _make_wrapper(fallback_type="previous") - w.safety_checker = lambda t, thr: False + w.safety_checker = lambda t, pin: pin.fill_(0.0) + w._nsfw_prob_pin = torch.tensor([0.0]) # a prior clean verdict already landed assert w._prev_clean_tensor is None dummy = torch.randn(1, 3, 64, 64) @@ -167,18 +188,35 @@ def test_clean_frame_cached_for_previous_strategy(self): assert w._prev_clean_tensor is not None, ( "clean frame with previous strategy should populate _prev_clean_tensor" ) + assert torch.equal(w._prev_clean_tensor, dummy) # ── Case 8: clean frame NOT cached for blank strategy ──────────────────── def test_clean_frame_not_cached_for_blank_strategy(self): w = _make_wrapper(fallback_type="blank") - w.safety_checker = lambda t, thr: False + w.safety_checker = lambda t, pin: pin.fill_(0.0) + w._nsfw_prob_pin = torch.tensor([0.0]) # a prior clean verdict already landed dummy = torch.randn(1, 3, 64, 64) w._apply_safety_checker(dummy) assert w._prev_clean_tensor is None, "_prev_clean_tensor should stay None when fallback_type='blank'" - # ── Case 9: _process_skip_diffusion wiring — checker is called, result is black ── + # ── Case 9: first frame always passes through ──────────────────────────── + def test_first_frame_always_passes_through_regardless_of_verdict(self): + """ + The very first frame has no prior async result to read, so it must + pass through unscreened even if the classifier would flag it — the + documented 1-frame pass-through edge case of the delayed-readback design. + """ + w = _make_wrapper(fallback_type="blank") + w.safety_checker = lambda t, pin: pin.fill_(1.0) # would flag if read this frame + + dummy = torch.randn(1, 3, 64, 64) + result = w._apply_safety_checker(dummy) + + assert torch.equal(result, dummy), "first frame must pass through (no prior verdict to act on)" + + # ── Case 10: _process_skip_diffusion wiring — checker is called, result is black ── def test_skip_diffusion_routes_through_safety_checker(self): """ Verify that _process_skip_diffusion actually feeds its tensor through @@ -191,12 +229,13 @@ def test_skip_diffusion_routes_through_safety_checker(self): """ received: list = [] - def capturing_checker(tensor, thr): + def capturing_checker(tensor, pin): received.append(tensor) - return True # always flag + pin.fill_(0.0) # this frame's own async result — irrelevant to this frame's own read w = _make_wrapper(fallback_type="blank") w.safety_checker = capturing_checker + w._nsfw_prob_pin = torch.tensor([1.0]) # a prior frame's flagged verdict already landed w.mode = "img2img" w.device = torch.device("cpu") w.dtype = torch.float32 @@ -226,7 +265,7 @@ def _apply_image_postprocessing_hooks(self, t): assert len(received) == 1, "safety checker should be called exactly once" assert isinstance(received[0], torch.Tensor), "checker arg must be a Tensor, not None" - # Flagged frame must produce the black substitution (all -1.0 → denorm 0.0) + # Flagged frame (per the pre-seeded pinned verdict) must produce the black substitution denorm = (result / 2 + 0.5).clamp(0, 1) assert torch.allclose(denorm, torch.zeros_like(denorm)), ( "flagged skip-diffusion frame should produce a black output" From 9adfebd7345d9f162d2822847fb15ddd57884777 Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 11 Jul 2026 09:08:31 -0400 Subject: [PATCH 15/37] perf: persistent multi-ControlNet residual merge buffer (Phase-2 prep) The multi-CN merge previously reallocated `merged_down`/`merged_mid` every frame (`merged_down[j] = merged_down[j] + ds[j]`), so residual pointers changed every frame. That would thrash a CUDA graph if these tensors are later bound zero-copy into the UNet's input_control_* inputs (Phase-2 D2). Replace it with lazily-allocated, shape-guarded persistent buffers (_cn_merged_down/_cn_merged_mid) that accumulate in place via copy_/add_, reallocating only on a genuine shape/dtype/device change. Does not alias down_samples_list[0]/mid_samples_list[0] (engine A's own persistent output buffers). Numerically identical to the old sum; single-ControlNet path is unaffected (already address-stable, no merge buffer allocated). Also removes pre-existing lint debt in the same file (duplicate OrchestratorUser import, unused base_kwargs dict), required to pass the commit gate; verified via git stash that both predated this change. Adds tests/unit/test_controlnet_residual_merge.py covering numerical correctness, cross-frame pointer stability, reallocation on shape change, non-aliasing of engine A's buffer, single-CN bypass, and install() reset. Co-Authored-By: Claude Sonnet 5 --- .../modules/controlnet_module.py | 54 +++-- tests/unit/test_controlnet_residual_merge.py | 186 ++++++++++++++++++ 2 files changed, 224 insertions(+), 16 deletions(-) create mode 100644 tests/unit/test_controlnet_residual_merge.py diff --git a/src/streamdiffusion/modules/controlnet_module.py b/src/streamdiffusion/modules/controlnet_module.py index 2fc6ef520..03c57345d 100644 --- a/src/streamdiffusion/modules/controlnet_module.py +++ b/src/streamdiffusion/modules/controlnet_module.py @@ -13,7 +13,6 @@ from streamdiffusion.preprocessing.preprocessing_orchestrator import ( PreprocessingOrchestrator, ) -from streamdiffusion.preprocessing.orchestrator_user import OrchestratorUser from streamdiffusion.tools.gpu_profiler import profiler @@ -89,6 +88,15 @@ def __init__(self, device: str = "cuda", dtype: torch.dtype = torch.float16) -> self._cn_cache_images_version: int = -1 self._cn_cache_scale_hash: Optional[tuple] = None + # Persistent multi-ControlNet residual merge buffers (Phase-2 prep). The naive + # `merged_down[j] = merged_down[j] + ds[j]` allocates a fresh tensor every frame, + # which would thrash a CUDA graph if these residuals are later bound zero-copy into + # the UNet's input_control_* inputs. Lazily (re)allocated only when residual + # shape/dtype/device changes, so pointers stay stable across frames. + self._cn_merged_down: Optional[List[torch.Tensor]] = None + self._cn_merged_mid: Optional[torch.Tensor] = None + self._cn_merged_shape_key: Optional[tuple] = None + # ---------- Public API (used by wrapper in a later step) ---------- def install(self, stream) -> None: self._stream = stream @@ -116,6 +124,9 @@ def install(self, stream) -> None: self._cn_cached_residuals = None self._cn_cache_images_version = -1 self._cn_cache_scale_hash = None + self._cn_merged_down = None + self._cn_merged_mid = None + self._cn_merged_shape_key = None def add_controlnet( self, cfg: ControlNetConfig, control_image: Optional[Union[str, Any, torch.Tensor]] = None @@ -501,13 +512,6 @@ def _unet_hook(ctx: StepCtx) -> UnetKwargsDelta: encoder_hidden_states = self._stream.prompt_embeds[:, : self._expected_text_len, :] - base_kwargs: Dict[str, Any] = { - "sample": x_t, - "timestep": t_list, - "encoder_hidden_states": encoder_hidden_states, - "return_dict": False, - } - down_samples_list: List[List[torch.Tensor]] = [] mid_samples_list: List[torch.Tensor] = [] @@ -623,16 +627,34 @@ def _unet_hook(ctx: StepCtx) -> UnetKwargsDelta: mid_block_additional_residual=mid_samples_list[0], ) else: - # Merge multiple ControlNet residuals - merged_down = down_samples_list[0] - merged_mid = mid_samples_list[0] + # Merge multiple ControlNet residuals into persistent buffers, in place. + # Do NOT alias down_samples_list[0]/mid_samples_list[0] — those are + # engine-A's own persistent output buffers. Reallocate only when the + # residual shape/dtype/device changes (resolution/batch/model swap), + # so the merged tensors stay pointer-stable across frames. + shape_key = ( + tuple(t.shape for t in down_samples_list[0]), + mid_samples_list[0].shape, + down_samples_list[0][0].dtype, + down_samples_list[0][0].device, + ) + if self._cn_merged_down is None or self._cn_merged_shape_key != shape_key: + self._cn_merged_down = [torch.empty_like(t) for t in down_samples_list[0]] + self._cn_merged_mid = torch.empty_like(mid_samples_list[0]) + self._cn_merged_shape_key = shape_key + + for j in range(len(self._cn_merged_down)): + self._cn_merged_down[j].copy_(down_samples_list[0][j]) + self._cn_merged_mid.copy_(mid_samples_list[0]) + for ds, ms in zip(down_samples_list[1:], mid_samples_list[1:]): - for j in range(len(merged_down)): - merged_down[j] = merged_down[j] + ds[j] - merged_mid = merged_mid + ms + for j in range(len(self._cn_merged_down)): + self._cn_merged_down[j].add_(ds[j]) + self._cn_merged_mid.add_(ms) + _result = UnetKwargsDelta( - down_block_additional_residuals=merged_down, - mid_block_additional_residual=merged_mid, + down_block_additional_residuals=self._cn_merged_down, + mid_block_additional_residual=self._cn_merged_mid, ) # Residual cache write: store result for reuse on upcoming intermediate frames. diff --git a/tests/unit/test_controlnet_residual_merge.py b/tests/unit/test_controlnet_residual_merge.py new file mode 100644 index 000000000..d9b1224e1 --- /dev/null +++ b/tests/unit/test_controlnet_residual_merge.py @@ -0,0 +1,186 @@ +""" +Regression tests for ControlNetModule's multi-ControlNet residual merge (Phase-2 prep, D1). + +CPU-only and model-free: fake ControlNet callables stand in for real engines so the +merge logic in build_unet_hook()'s closure can be exercised without CUDA/TRT. + +Root cause being guarded: the old merge (`merged_down[j] = merged_down[j] + ds[j]`) +allocated a fresh tensor every frame, so the UNet's input_control_* residuals would +never be pointer-stable across frames — a prerequisite for zero-copy binding those +inputs (Phase-2 D2). This also verifies the merge never aliases engine A's own +persistent output buffer. +""" + +import torch + +from streamdiffusion.hooks import StepCtx +from streamdiffusion.modules.controlnet_module import ControlNetModule + + +class _FakeCN: + """Stands in for a ControlNet engine: returns fixed-value residuals, records calls.""" + + def __init__(self, down_value: float, mid_value: float, down_shapes, mid_shape): + self.calls = 0 + self._down_value = down_value + self._mid_value = mid_value + self._down_shapes = down_shapes + self._mid_shape = mid_shape + + def __call__( + self, sample, timestep, encoder_hidden_states, controlnet_cond, conditioning_scale, return_dict=False + ): + self.calls += 1 + down = [torch.full(shape, self._down_value, dtype=torch.float32) for shape in self._down_shapes] + mid = torch.full(self._mid_shape, self._mid_value, dtype=torch.float32) + return down, mid + + +class _FakeStream: + def __init__(self, text_len: int = 77, batch: int = 1): + self.prompt_embeds = torch.randn(batch, text_len, 8) + + +def _make_module_with_two_controlnets(down_shapes, mid_shape) -> tuple: + module = ControlNetModule(device="cpu", dtype=torch.float32) + module._stream = _FakeStream() + + cn_a = _FakeCN(down_value=1.0, mid_value=2.0, down_shapes=down_shapes, mid_shape=mid_shape) + cn_b = _FakeCN(down_value=3.0, mid_value=4.0, down_shapes=down_shapes, mid_shape=mid_shape) + + module.controlnets = [cn_a, cn_b] + module.controlnet_images = [torch.randn(1, 3, 8, 8), torch.randn(1, 3, 8, 8)] + module.controlnet_scales = [1.0, 1.0] + module.enabled_list = [True, True] + + return module, cn_a, cn_b + + +def _make_ctx(batch: int = 1) -> StepCtx: + return StepCtx( + x_t_latent=torch.randn(batch, 4, 8, 8), + t_list=torch.tensor([0]), + step_index=0, + guidance_mode="none", + sdxl_cond=None, + ) + + +class TestControlNetResidualMerge: + def test_merge_is_numerically_correct(self): + down_shapes = [(1, 4, 8, 8), (1, 4, 4, 4)] + mid_shape = (1, 4, 2, 2) + module, cn_a, cn_b = _make_module_with_two_controlnets(down_shapes, mid_shape) + hook = module.build_unet_hook() + + result = hook(_make_ctx()) + + assert cn_a.calls == 1 and cn_b.calls == 1 + for merged, shape in zip(result.down_block_additional_residuals, down_shapes): + assert torch.allclose(merged, torch.full(shape, 4.0)), "1.0 + 3.0 == 4.0 per down block" + assert torch.allclose(result.mid_block_additional_residual, torch.full(mid_shape, 6.0)), "2.0 + 4.0 == 6.0" + + def test_merge_buffers_are_pointer_stable_across_frames(self): + """Same shape on consecutive frames must reuse the same tensor objects + (required before the UNet's input_control_* inputs can be zero-copy bound).""" + down_shapes = [(1, 4, 8, 8)] + mid_shape = (1, 4, 2, 2) + module, _cn_a, _cn_b = _make_module_with_two_controlnets(down_shapes, mid_shape) + hook = module.build_unet_hook() + + result1 = hook(_make_ctx()) + down_ptr1 = [t.data_ptr() for t in result1.down_block_additional_residuals] + mid_ptr1 = result1.mid_block_additional_residual.data_ptr() + + result2 = hook(_make_ctx()) + down_ptr2 = [t.data_ptr() for t in result2.down_block_additional_residuals] + mid_ptr2 = result2.mid_block_additional_residual.data_ptr() + + assert down_ptr1 == down_ptr2, "merged down-block buffers must be reused, not reallocated" + assert mid_ptr1 == mid_ptr2, "merged mid-block buffer must be reused, not reallocated" + # Values still correct on frame 2 (buffer was overwritten, not just left stale) + assert torch.allclose(result2.down_block_additional_residuals[0], torch.full(down_shapes[0], 4.0)) + + def test_merge_reallocates_on_shape_change(self): + """A resolution/batch change must produce a new buffer, not corrupt-reuse the old one.""" + down_shapes = [(1, 4, 8, 8)] + mid_shape = (1, 4, 2, 2) + module, cn_a, cn_b = _make_module_with_two_controlnets(down_shapes, mid_shape) + hook = module.build_unet_hook() + + result1 = hook(_make_ctx()) + down_ptr1 = result1.down_block_additional_residuals[0].data_ptr() + + # Simulate a resolution change: new shapes for both fake engines. + new_down_shapes = [(1, 4, 16, 16)] + new_mid_shape = (1, 4, 4, 4) + cn_a._down_shapes = new_down_shapes + cn_a._mid_shape = new_mid_shape + cn_b._down_shapes = new_down_shapes + cn_b._mid_shape = new_mid_shape + + result2 = hook(_make_ctx()) + assert result2.down_block_additional_residuals[0].shape == new_down_shapes[0] + down_ptr2 = result2.down_block_additional_residuals[0].data_ptr() + assert down_ptr2 != down_ptr1, "shape change must trigger reallocation" + + def test_merge_does_not_alias_engine_output_buffer(self): + """The merge must not mutate/alias down_samples_list[0] — that's engine A's + own persistent output buffer, reused by the engine on the next frame.""" + down_shapes = [(1, 4, 8, 8)] + mid_shape = (1, 4, 2, 2) + module, cn_a, _cn_b = _make_module_with_two_controlnets(down_shapes, mid_shape) + hook = module.build_unet_hook() + + result = hook(_make_ctx()) + + # Re-invoke cn_a directly (as the engine would on its own next call) and confirm + # its freshly-returned buffer is untouched by the merge (still all 1.0, not 4.0). + fresh_down, fresh_mid = cn_a( + sample=None, timestep=None, encoder_hidden_states=None, controlnet_cond=None, conditioning_scale=1.0 + ) + assert torch.allclose(fresh_down[0], torch.full(down_shapes[0], 1.0)) + assert torch.allclose(fresh_mid, torch.full(mid_shape, 2.0)) + # Sanity: the merge result itself is still the summed value. + assert torch.allclose(result.down_block_additional_residuals[0], torch.full(down_shapes[0], 4.0)) + + def test_single_controlnet_bypasses_merge_buffers(self): + """With only one active ControlNet, the engine's own output is returned + directly — no merge buffer should be allocated.""" + module = ControlNetModule(device="cpu", dtype=torch.float32) + module._stream = _FakeStream() + down_shapes = [(1, 4, 8, 8)] + mid_shape = (1, 4, 2, 2) + cn_a = _FakeCN(down_value=5.0, mid_value=6.0, down_shapes=down_shapes, mid_shape=mid_shape) + module.controlnets = [cn_a] + module.controlnet_images = [torch.randn(1, 3, 8, 8)] + module.controlnet_scales = [1.0] + module.enabled_list = [True] + + hook = module.build_unet_hook() + hook(_make_ctx()) + + assert module._cn_merged_down is None + assert module._cn_merged_mid is None + + def test_install_resets_merge_buffers(self): + down_shapes = [(1, 4, 8, 8)] + mid_shape = (1, 4, 2, 2) + module, _cn_a, _cn_b = _make_module_with_two_controlnets(down_shapes, mid_shape) + module.build_unet_hook()(_make_ctx()) + assert module._cn_merged_down is not None + + class _MinimalStream: + unet_hooks = [] + controlnets = None + controlnet_scales = None + preprocessors = None + + # attach_orchestrator requires a preprocessing orchestrator; reuse the existing one + # (install() only touches it when _preprocessing_orchestrator is None). + module._preprocessing_orchestrator = object() + module.install(_MinimalStream()) + + assert module._cn_merged_down is None + assert module._cn_merged_mid is None + assert module._cn_merged_shape_key is None From abf02a7def7e97122fd4db5864ee85ee7dea930f Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 11 Jul 2026 09:36:47 -0400 Subject: [PATCH 16/37] perf: 256-byte alignment guard for zero-copy bind path (audit M2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit setTensorAddress requires >=256-byte alignment; _staging_action's zero-copy eligibility test didn't enforce it, relying on an arithmetic proof instead of a runtime invariant. Add cur_ptr % 256 == 0 to the eligibility check so a mis-aligned pointer always falls back to the safe copy path. Real torch CUDA allocations are >=512-byte aligned, so this guard never fires in production — it only defends against future config drift, and is a prerequisite for Phase-2 D2's new zero-copy input_control_* bindings. Realigns the test suite's PTR_A/PTR_B fixtures to 256-byte-aligned values (the old 0xDEAD_BEEF/0xFEED_FACE constants were both mis-aligned and would have broken under the new guard) and adds two rows asserting the misaligned fallback. Co-Authored-By: Claude Sonnet 5 --- .../acceleration/tensorrt/utilities.py | 7 +++- tests/unit/test_zero_copy_staging_5_6.py | 37 ++++++++++++++++++- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/streamdiffusion/acceleration/tensorrt/utilities.py b/src/streamdiffusion/acceleration/tensorrt/utilities.py index 67be6ae74..79d56c771 100644 --- a/src/streamdiffusion/acceleration/tensorrt/utilities.py +++ b/src/streamdiffusion/acceleration/tensorrt/utilities.py @@ -562,8 +562,13 @@ def _staging_action( "bind_and_reset" - skip the copy and bind directly, but reset the CUDA graph first because the caller's address changed while a graph built from the old address was still live + + A pointer that is not 256-byte aligned falls back to "copy": TensorRT's + setTensorAddress contract requires ≥256-byte alignment, and real torch CUDA + allocations are always ≥512-byte aligned, so this guard is a defensive + invariant against config drift rather than a path exercised in production. """ - if name not in zero_copy_names or not is_contiguous or not dtype_match: + if name not in zero_copy_names or not is_contiguous or not dtype_match or cur_ptr % 256 != 0: return "copy" if graph_exists and cur_ptr != prev_ptr: return "bind_and_reset" diff --git a/tests/unit/test_zero_copy_staging_5_6.py b/tests/unit/test_zero_copy_staging_5_6.py index 56bbe341f..65bb77ca4 100644 --- a/tests/unit/test_zero_copy_staging_5_6.py +++ b/tests/unit/test_zero_copy_staging_5_6.py @@ -34,8 +34,9 @@ ) -PTR_A = 0xDEAD_BEEF -PTR_B = 0xFEED_FACE +PTR_A = 0x1000_0000 # 256-byte aligned (audit M2 guard) +PTR_B = 0x2000_0000 # 256-byte aligned (audit M2 guard) +PTR_MISALIGNED = PTR_A + 1 # not 256-byte aligned class TestStagingActionNotZeroCopy: @@ -179,3 +180,35 @@ def test_first_bind_with_no_prior_ptr_recorded(self): False, ) assert result == "bind" + + +class TestStagingActionAlignmentGuard: + """Audit M2: setTensorAddress requires >=256-byte alignment. A mis-aligned + cur_ptr must fall back to "copy" even when every other eligibility check + would otherwise allow a bind — never silently bind an unaligned address.""" + + def test_misaligned_pointer_falls_back_to_copy_no_graph(self): + result = _staging_action( + "kvo_cache_in_0", + frozenset({"kvo_cache_in_0"}), + True, + True, + None, + PTR_MISALIGNED, + False, + ) + assert result == "copy" + + def test_misaligned_pointer_falls_back_to_copy_with_live_graph(self): + """Even a pointer-unchanged steady-state frame must copy, not bind, if + the (mis-aligned) address itself is invalid for setTensorAddress.""" + result = _staging_action( + "kvo_cache_in_0", + frozenset({"kvo_cache_in_0"}), + True, + True, + PTR_MISALIGNED, + PTR_MISALIGNED, + True, + ) + assert result == "copy" From f8ff50f846cdf94a0e406e7144fbf5c686f02cb4 Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 11 Jul 2026 09:39:16 -0400 Subject: [PATCH 17/37] perf: zero-copy bind ControlNet residual UNet inputs (Phase-2 D2) Widen UNet2DConditionModelEngine._zero_copy_names to include input_control_00..19 and input_control_middle alongside the existing kvo/fio cache names, so Engine.infer() binds TensorRT directly to the ControlNet residual tensors instead of DtoD-copying them into a staging buffer every frame (the Sub-phase 5.6 mechanism, now audit-M2 alignment guarded). In steady state the tensors placed in input_dict[input_control_*] are the same persistent objects each frame: multi-ControlNet uses D1's persistent merge buffers (controlnet_module.py), single-ControlNet reuses the engine's own persistent output buffers, and idle frames reuse the persistent dummy zero cache. Pointers only change on a resolution/batch/model change or the idle<->active toggle, which _staging_action already handles via bind_and_reset (one graph re-capture on the toggle frame, not per-frame). Co-Authored-By: Claude Sonnet 5 --- .../tensorrt/runtime_engines/unet_engine.py | 22 ++++++--- tests/unit/test_zero_copy_staging_5_6.py | 46 +++++++++++++++++++ 2 files changed, 62 insertions(+), 6 deletions(-) diff --git a/src/streamdiffusion/acceleration/tensorrt/runtime_engines/unet_engine.py b/src/streamdiffusion/acceleration/tensorrt/runtime_engines/unet_engine.py index de317065d..67d5716f5 100644 --- a/src/streamdiffusion/acceleration/tensorrt/runtime_engines/unet_engine.py +++ b/src/streamdiffusion/acceleration/tensorrt/runtime_engines/unet_engine.py @@ -54,10 +54,16 @@ def __init__(self, filepath: str, stream: "cuda.Stream", use_cuda_graph: bool = self._fio_out_names: List[str] = [] self._fio_cache_len: int = -1 # -1 = not yet initialized - # Sub-phase 5.6: kvo/fio cache inputs are persistent, address-stable, - # TRT-contiguous tensors (see models/utils.py create_kvo_cache/create_fi_cache), - # so Engine.infer() can bind TensorRT directly to them instead of copying into - # its own staging buffer every frame. Rebuilt only when the kvo/fio lazy-init + # Sub-phase 5.6 / Phase-2 D2: kvo/fio cache inputs and ControlNet + # input_control_* residuals are persistent, address-stable, TRT-contiguous + # tensors in steady state (kvo/fio: models/utils.py create_kvo_cache/ + # create_fi_cache; control: persistent merge buffers in controlnet_module.py + # or the ControlNet engine's own persistent output buffers), so + # Engine.infer() can bind TensorRT directly to them instead of copying into + # its own staging buffer every frame. `_staging_action` falls back to a + # copy on non-contiguous/dtype-mismatch/mis-aligned inputs, and to + # bind_and_reset on a pointer change (e.g. ControlNet idle<->active + # toggle) — see utilities.py. Rebuilt only when the kvo/fio lazy-init # branches below fire (cache length change), never per-frame. self._zero_copy_names: FrozenSet[str] = frozenset() @@ -97,7 +103,9 @@ def __call__( self._kvo_in_names = [f"kvo_cache_in_{i}" for i in range(n_kvo)] self._kvo_out_names = [f"kvo_cache_out_{i}" for i in range(n_kvo)] self._kvo_cache_len = n_kvo - self._zero_copy_names = frozenset(self._kvo_in_names + self._fio_in_names) + self._zero_copy_names = frozenset( + self._kvo_in_names + self._fio_in_names + self._input_control_names + [self._input_control_middle] + ) # Lazy-init FIO key name lists — same pattern as KVO. n_fio = len(fio_cache) @@ -105,7 +113,9 @@ def __call__( self._fio_in_names = [f"fio_cache_in_{i}" for i in range(n_fio)] self._fio_out_names = [f"fio_cache_out_{i}" for i in range(n_fio)] self._fio_cache_len = n_fio - self._zero_copy_names = frozenset(self._kvo_in_names + self._fio_in_names) + self._zero_copy_names = frozenset( + self._kvo_in_names + self._fio_in_names + self._input_control_names + [self._input_control_middle] + ) # Update pre-allocated dicts in-place — no new dict objects created per call. shape_dict = self._shape_dict diff --git a/tests/unit/test_zero_copy_staging_5_6.py b/tests/unit/test_zero_copy_staging_5_6.py index 65bb77ca4..c06522ab7 100644 --- a/tests/unit/test_zero_copy_staging_5_6.py +++ b/tests/unit/test_zero_copy_staging_5_6.py @@ -212,3 +212,49 @@ def test_misaligned_pointer_falls_back_to_copy_with_live_graph(self): True, ) assert result == "copy" + + +class TestStagingActionControlNetResiduals: + """Phase-2 D2: input_control_* UNet inputs opt into the same zero-copy path + as kvo/fio. Steady state binds without a reset; the ControlNet idle<->active + toggle (or a resolution/batch change) flips the residual's address while a + graph is live, forcing exactly one bind_and_reset on the toggle frame.""" + + def test_steady_state_control_residual_binds_without_reset(self): + """Same persistent merge/output buffer across frames — plain bind.""" + result = _staging_action( + "input_control_00", + frozenset({"input_control_00"}), + True, + True, + PTR_A, + PTR_A, + True, + ) + assert result == "bind" + + def test_idle_active_toggle_binds_and_resets(self): + """Idle dummy-zero buffer -> active ControlNet residual buffer address + flip while a graph is live must force a re-capture.""" + result = _staging_action( + "input_control_00", + frozenset({"input_control_00"}), + True, + True, + PTR_A, + PTR_B, + True, + ) + assert result == "bind_and_reset" + + def test_middle_control_residual_binds_without_reset(self): + result = _staging_action( + "input_control_middle", + frozenset({"input_control_middle"}), + True, + True, + PTR_A, + PTR_A, + True, + ) + assert result == "bind" From 50171bfce3e7b4dd50b087c720f32e4754c4f047 Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 11 Jul 2026 09:43:09 -0400 Subject: [PATCH 18/37] refactor: harden TRT runtime API usage (audit M1/H1/H2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit M1: TensorRTEngine (trt_base.py) ignored the bool return of set_input_shape (allocate_buffers, infer) and set_tensor_address (infer). An out-of-profile shape or invalid address was silently accepted, surfacing later as an execute_async_v3 failure or garbage output. Mirror the core Engine pattern (utilities.py) and raise RuntimeError with the tensor name/shape when either call returns False. Affects depth/pose (+HED/NormalBae via the shared base). H1: replace the try/except (AttributeError, TypeError) around the SM_120 TacticSource scoping block with an explicit hasattr(trt.TacticSource, "CUBLAS") gate. CUBLAS/CUBLAS_LT are removed in TRT 11, so the old except would silently swallow the AttributeError and drop the scoping intent without a trace; the new branch logs the TRT>=11 case explicitly. No behavior change on the installed TRT 10.16 (all members still present). H2: delete the dead reuse_device_memory branch in Engine.activate() — no caller across any of the ~10 activate() call sites passes it, and the one-arg context.device_memory setter it used has been superseded since TRT 10.1. Migrating to set_device_memory(ptr, engine.device_memory_size_v2) is deferred until device-memory reuse is actually wanted. Co-Authored-By: Claude Sonnet 5 --- .../acceleration/tensorrt/utilities.py | 20 +++++++------- .../preprocessing/processors/trt_base.py | 27 +++++++++++++++---- 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/src/streamdiffusion/acceleration/tensorrt/utilities.py b/src/streamdiffusion/acceleration/tensorrt/utilities.py index 79d56c771..2f6471a76 100644 --- a/src/streamdiffusion/acceleration/tensorrt/utilities.py +++ b/src/streamdiffusion/acceleration/tensorrt/utilities.py @@ -385,8 +385,11 @@ def _apply_gpu_profile_to_config( # + EDGE_MASK_CONVOLUTIONS — the sources that produce valid SM_120 kernels. # TRT 10.16 exposes TacticSource as an int enum (not IntFlag), so the bitmask # is built via (1 << int(source)). No-op on Ada/Ampere. + # CUBLAS/CUBLAS_LT are removed from TacticSource in TRT 11 (cuBLAS tactics + # dropped entirely) — gate on hasattr so that's a deliberate, logged branch + # rather than a silently-swallowed AttributeError. if gpu_profile.compute_capability >= (12, 0): - try: + if hasattr(trt.TacticSource, "CUBLAS"): sources = ( (1 << int(trt.TacticSource.CUBLAS)) | (1 << int(trt.TacticSource.CUBLAS_LT)) @@ -397,8 +400,11 @@ def _apply_gpu_profile_to_config( logger.info( "[TRT Config] tactic sources = CUBLAS|CUBLAS_LT|JIT_CONV|EDGE_MASK (CUDNN excluded for SM_120+)" ) - except (AttributeError, TypeError) as e: - logger.debug(f"[TRT Config] set_tactic_sources not available: {e}") + else: + logger.info( + "[TRT Config] TRT >=11: CUBLAS/CUBLAS_LT removed from TacticSource — " + "default tactic sources already exclude cuDNN/cuBLAS, nothing to scope" + ) # max_num_tactics: cap profiling candidates per layer to reduce build time. # Available since TRT 10.x; -1 (default) lets TRT decide heuristically. 64 is a @@ -1001,12 +1007,8 @@ def get_input_profile_bounds(self, name: str): except Exception: return None - def activate(self, reuse_device_memory=None): - if reuse_device_memory: - self.context = self.engine.create_execution_context_without_device_memory() - self.context.device_memory = reuse_device_memory - else: - self.context = self.engine.create_execution_context() + def activate(self): + self.context = self.engine.create_execution_context() # Attach per-layer profiler when STREAMDIFFUSION_PROFILE_TRT is set. # Requires engines built with profiling_verbosity=DETAILED for meaningful names. diff --git a/src/streamdiffusion/preprocessing/processors/trt_base.py b/src/streamdiffusion/preprocessing/processors/trt_base.py index 56b8852a6..b97778b21 100644 --- a/src/streamdiffusion/preprocessing/processors/trt_base.py +++ b/src/streamdiffusion/preprocessing/processors/trt_base.py @@ -132,7 +132,13 @@ def allocate_buffers(self, device: str = "cuda", input_shape: tuple = None): if self.engine.get_tensor_mode(name) != trt.TensorIOMode.INPUT: continue if input_shape is not None: - self.context.set_input_shape(name, input_shape) + if not self.context.set_input_shape(name, input_shape): + raise RuntimeError( + f"TensorRTEngine.allocate_buffers: set_input_shape failed for " + f"'{name}' with shape {input_shape}. The engine was built for a " + f"fixed shape range — revert the parameter change or rebuild the " + f"engine for the new shape." + ) else: static_shape = tuple(self.context.get_tensor_shape(name)) if any(d < 0 for d in static_shape): @@ -141,7 +147,11 @@ def allocate_buffers(self, device: str = "cuda", input_shape: tuple = None): f"shape {static_shape} but no input_shape was provided. " "Pass input_shape=(N, C, H, W) when using a dynamic engine." ) - self.context.set_input_shape(name, static_shape) + if not self.context.set_input_shape(name, static_shape): + raise RuntimeError( + f"TensorRTEngine.allocate_buffers: set_input_shape failed for " + f"'{name}' with shape {static_shape}." + ) # Pass 2: allocate buffers for ALL tensors (output shapes resolved by TRT now). for idx in range(self.engine.num_io_tensors): @@ -192,12 +202,18 @@ def infer(self, feed_dict: dict, stream=None) -> OrderedDict: self.tensors[name] = cached[name] # Re-apply input shapes to TRT context (context state is NOT cached). for name, shape in new_input_shapes.items(): - self.context.set_input_shape(name, shape) + if not self.context.set_input_shape(name, shape): + raise RuntimeError( + f"TensorRTEngine.infer: set_input_shape failed for '{name}' with shape {shape}." + ) else: # LRU miss: reallocate changed inputs, re-derive output shapes. for name, fed_shape in new_input_shapes.items(): if fed_shape != tuple(self.tensors[name].shape): - self.context.set_input_shape(name, fed_shape) + if not self.context.set_input_shape(name, fed_shape): + raise RuntimeError( + f"TensorRTEngine.infer: set_input_shape failed for '{name}' with shape {fed_shape}." + ) self.tensors[name] = torch.empty( fed_shape, dtype=self.tensors[name].dtype, @@ -232,7 +248,8 @@ def infer(self, feed_dict: dict, stream=None) -> OrderedDict: self.tensors[name].copy_(buf) for name, tensor in self.tensors.items(): - self.context.set_tensor_address(name, tensor.data_ptr()) + if not self.context.set_tensor_address(name, tensor.data_ptr()): + raise RuntimeError(f"TensorRTEngine.infer: set_tensor_address failed for '{name}'") # --- Cross-stream synchronization --- # The input copy_() calls above ran on the CURRENT (default) stream. From f0e1b807a4540c2ecbe3824c0c1c52ad5ad3fdfe Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 11 Jul 2026 09:52:44 -0400 Subject: [PATCH 19/37] =?UTF-8?q?refactor:=20modernize=20ONNX=20compile/ex?= =?UTF-8?q?port=20path=20=E2=80=94=20create=5Fnetwork(),=20drop=20deprecat?= =?UTF-8?q?ed=20export=20flag,=20robust=20external-data=20detection=20(aud?= =?UTF-8?q?it=20M3/L1/L3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit M3: replace deprecated create_network(1< --- .../acceleration/tensorrt/utilities.py | 18 +++---- .../tools/compile_depth_anything_tensorrt.py | 47 +++++++++---------- 2 files changed, 31 insertions(+), 34 deletions(-) diff --git a/src/streamdiffusion/acceleration/tensorrt/utilities.py b/src/streamdiffusion/acceleration/tensorrt/utilities.py index 2f6471a76..aecb8a8a3 100644 --- a/src/streamdiffusion/acceleration/tensorrt/utilities.py +++ b/src/streamdiffusion/acceleration/tensorrt/utilities.py @@ -1529,7 +1529,6 @@ def export_onnx( onnx_path, export_params=True, opset_version=onnx_opset, - do_constant_folding=True, input_names=model_data.get_input_names(), output_names=model_data.get_output_names(), dynamic_axes=model_data.get_dynamic_axes(), @@ -1581,15 +1580,17 @@ def optimize_onnx( ): import os - # Check if external data files exist (indicating external data format was used) onnx_dir = os.path.dirname(onnx_path) - external_data_files = [f for f in os.listdir(onnx_dir) if f.endswith(".pb")] - uses_external_data = len(external_data_files) > 0 + # Inspect TensorProto.data_location on the raw (unloaded) model rather than + # scanning the directory for ".pb" filenames — load_external_data_for_model + # resets data_location back to DEFAULT once external data is loaded, so this + # check must happen before loading. + onnx_model = onnx.load(onnx_path, load_external_data=False) + uses_external_data = any(onnx.external_data_helper.uses_external_data(t) for t in onnx_model.graph.initializer) if uses_external_data: - logger.info(f"Optimizing ONNX with external data (found: {external_data_files})") - # Load model with external data - onnx_model = onnx.load(onnx_path, load_external_data=True) + logger.info("Optimizing ONNX with external data") + onnx.external_data_helper.load_external_data_for_model(onnx_model, onnx_dir) onnx_opt_graph = model_data.optimize(onnx_model) # Create output directory @@ -1614,8 +1615,7 @@ def optimize_onnx( logger.info("ONNX optimization complete with external data") else: - # Standard optimization for smaller models - onnx_model = onnx.load(onnx_path) + # No external data to load — the model loaded above is already complete. onnx_opt_graph = model_data.optimize(onnx_model) onnx.save(onnx_opt_graph, onnx_opt_path) diff --git a/src/streamdiffusion/tools/compile_depth_anything_tensorrt.py b/src/streamdiffusion/tools/compile_depth_anything_tensorrt.py index ac0ed7465..b842ec101 100644 --- a/src/streamdiffusion/tools/compile_depth_anything_tensorrt.py +++ b/src/streamdiffusion/tools/compile_depth_anything_tensorrt.py @@ -9,14 +9,15 @@ python -m streamdiffusion.tools.compile_depth_anything_tensorrt --model_size small --resolution 518 """ -import torch +import importlib.util import logging -import os from pathlib import Path -from typing import Optional + import fire +import torch -logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') + +logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") logger = logging.getLogger(__name__) try: @@ -24,18 +25,13 @@ from streamdiffusion.acceleration.tensorrt.utilities import BUILD_TRT_LOGGER - TENSORRT_AVAILABLE = True except ImportError: TENSORRT_AVAILABLE = False BUILD_TRT_LOGGER = None logger.warning("TensorRT not available. Please install it first.") -try: - import onnx - ONNX_AVAILABLE = True -except ImportError: - ONNX_AVAILABLE = False +ONNX_AVAILABLE = importlib.util.find_spec("onnx") is not None # Depth Anything model configs DEPTH_ANYTHING_MODELS = { @@ -55,10 +51,7 @@ def export_depth_anything_to_onnx( - onnx_path: Path, - model_size: str = "small", - resolution: int = 518, - device: str = "cuda" + onnx_path: Path, model_size: str = "small", resolution: int = 518, device: str = "cuda" ) -> bool: """Export Depth Anything model to ONNX format""" try: @@ -107,6 +100,7 @@ def export_depth_anything_to_onnx( except Exception as e: logger.error(f"Failed to export ONNX: {e}") import traceback + traceback.print_exc() return False @@ -126,11 +120,11 @@ def build_tensorrt_engine( try: builder = trt.Builder(BUILD_TRT_LOGGER) - network = builder.create_network(1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH)) + network = builder.create_network() # EXPLICIT_BATCH deprecated/ignored in TRT 10.x parser = trt.OnnxParser(network, BUILD_TRT_LOGGER) # Parse ONNX - with open(onnx_path, 'rb') as f: + with open(onnx_path, "rb") as f: if not parser.parse(f.read()): for i in range(parser.num_errors): logger.error(f"ONNX parse error: {parser.get_error(i)}") @@ -146,10 +140,12 @@ def build_tensorrt_engine( # Set optimization profile for fixed resolution profile = builder.create_optimization_profile() - profile.set_shape("input", - (1, 3, resolution, resolution), # min - (1, 3, resolution, resolution), # opt - (1, 3, resolution, resolution)) # max + profile.set_shape( + "input", + (1, 3, resolution, resolution), # min + (1, 3, resolution, resolution), # opt + (1, 3, resolution, resolution), + ) # max config.add_optimization_profile(profile) # Build engine @@ -162,7 +158,7 @@ def build_tensorrt_engine( # Save engine engine_path.parent.mkdir(parents=True, exist_ok=True) - with open(engine_path, 'wb') as f: + with open(engine_path, "wb") as f: f.write(serialized_engine) logger.info(f"TensorRT engine saved: {engine_path}") @@ -171,6 +167,7 @@ def build_tensorrt_engine( except Exception as e: logger.error(f"Failed to build TensorRT engine: {e}") import traceback + traceback.print_exc() return False @@ -210,7 +207,7 @@ def compile_depth_anything( # Check if engine already exists if engine_path.exists(): logger.info(f"Engine already exists: {engine_path}") - overwrite = input("Overwrite? (y/N): ").lower().strip() == 'y' + overwrite = input("Overwrite? (y/N): ").lower().strip() == "y" if not overwrite: return @@ -232,9 +229,9 @@ def compile_depth_anything( logger.info("Removed intermediate ONNX file") logger.info(f"\nSuccess! Engine saved to: {engine_path}") - logger.info(f"\nTo use in config:") - logger.info(f' preprocessor: "depth_tensorrt"') - logger.info(f' preprocessor_params:') + logger.info("\nTo use in config:") + logger.info(' preprocessor: "depth_tensorrt"') + logger.info(" preprocessor_params:") logger.info(f' engine_path: "{engine_path}"') From 3d8ff107cbe58ff839b64cc7b83d0ec66c151beb Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 11 Jul 2026 17:13:59 -0400 Subject: [PATCH 20/37] fix: make ControlNet zero-copy revert permanent; harden staging bind->copy gap Post-mortem: f8ff50f ("zero-copy bind ControlNet residual UNet inputs") caused a confirmed regression - ControlNet produced no visual change for both scribble (ramp 0->0.36) and Canny (static 1.0), at any scale. An interim revert (dropping input_control_* from the UNet engine's _zero_copy_names) was verified working on the rig for both models. This commit makes that revert permanent rather than re-enabling the optimization. Root cause, confirmed by code inspection (not just bisection): a zero-copy bind drops the copy_() call that runs on the engine stream and orders the ControlNet engine's residual write before the UNet CUDA graph's read of it ("copy_() executes on engine stream to guarantee ordering" - utilities.py). An earlier hypothesis assumed the failure was a bind->copy staging fallback triggered by cfg_type="self" slicing; that was disproven by inspection: single-ControlNet residuals (controlnet_module.py build_unet_hook) are the ControlNet engine's own persistent, contiguous, 256-aligned output buffers used unmodified, so _staging_action never takes the copy-fallback path for them - re-enabling zero-copy on that assumption would very likely have re-broken ControlNet. The staging bind->copy gap is real, however, for any zero-copy input that CAN take that fallback path (kvo/fio today): _staging_action returned "copy" on fallback with no signal to reset a live CUDA graph, so the graph kept replaying a stale bound address. Added a "copy_and_reset" outcome (utilities.py _staging_action + Engine.infer) as decoupled latent hardening - copies fresh data AND forces one graph re-capture, then converges to plain "copy" the following frame. This does not re-add ControlNet names to _zero_copy_names. Regression test: test_zero_copy_staging_5_6.py locks the exact decision that regressed (a previously-bound name falling back to copy while a graph is live must now return "copy_and_reset", not silently "copy"), plus coverage for the non-contiguous fallback variant and the never-bound/no-graph no-op cases. 139/139 unit tests pass (up from 136: +3 new/updated staging cases). --- .../tensorrt/runtime_engines/unet_engine.py | 53 +++++++---- .../acceleration/tensorrt/utilities.py | 19 +++- tests/unit/test_zero_copy_staging_5_6.py | 93 +++++++++++++++++-- 3 files changed, 137 insertions(+), 28 deletions(-) diff --git a/src/streamdiffusion/acceleration/tensorrt/runtime_engines/unet_engine.py b/src/streamdiffusion/acceleration/tensorrt/runtime_engines/unet_engine.py index 67d5716f5..85d081c7a 100644 --- a/src/streamdiffusion/acceleration/tensorrt/runtime_engines/unet_engine.py +++ b/src/streamdiffusion/acceleration/tensorrt/runtime_engines/unet_engine.py @@ -54,17 +54,40 @@ def __init__(self, filepath: str, stream: "cuda.Stream", use_cuda_graph: bool = self._fio_out_names: List[str] = [] self._fio_cache_len: int = -1 # -1 = not yet initialized - # Sub-phase 5.6 / Phase-2 D2: kvo/fio cache inputs and ControlNet - # input_control_* residuals are persistent, address-stable, TRT-contiguous - # tensors in steady state (kvo/fio: models/utils.py create_kvo_cache/ - # create_fi_cache; control: persistent merge buffers in controlnet_module.py - # or the ControlNet engine's own persistent output buffers), so - # Engine.infer() can bind TensorRT directly to them instead of copying into - # its own staging buffer every frame. `_staging_action` falls back to a - # copy on non-contiguous/dtype-mismatch/mis-aligned inputs, and to - # bind_and_reset on a pointer change (e.g. ControlNet idle<->active - # toggle) — see utilities.py. Rebuilt only when the kvo/fio lazy-init - # branches below fire (cache length change), never per-frame. + # Sub-phase 5.6: kvo/fio cache inputs are persistent, address-stable, + # TRT-contiguous tensors in steady state (models/utils.py + # create_kvo_cache/create_fi_cache), so Engine.infer() can bind + # TensorRT directly to them instead of copying into its own staging + # buffer every frame. `_staging_action` falls back to a copy on + # non-contiguous/dtype-mismatch/mis-aligned inputs, and to + # bind_and_reset on a pointer change — see utilities.py. Rebuilt only + # when the kvo/fio lazy-init branches below fire (cache length + # change), never per-frame. + # + # ControlNet input_control_* residuals are deliberately, PERMANENTLY + # EXCLUDED here (reverted from Phase-2 D2 / f8ff50f) and stay on the + # DtoD-copy staging path. Two reasons, confirmed against code, not + # just bisection: + # 1. The residual source toggles between the ControlNet engine's + # live output and idle cached-dummy-zero tensors + # (_add_controlnet_residuals vs _add_cached_dummy_inputs below), + # and a missed bind_and_reset on that toggle silently pins the + # UNet's CUDA graph to whichever buffer was bound at capture + # time. + # 2. copy_() runs on the engine stream and orders the ControlNet + # engine's residual write before the UNet graph's read of it; a + # zero-copy bind drops that ordering guarantee with no + # replacement synchronization (see the "copy_() executes on + # engine stream to guarantee ordering" log line in utilities.py). + # Re-enabling zero-copy for these names reproduced the "ControlNet + # produces no visual change" regression on the rig for both a + # single-scale (Canny) and ramped-scale (scribble) config, even + # though single-ControlNet residuals are themselves contiguous, + # 256-aligned, persistent buffers (controlnet_module.py + # build_unet_hook single-CN path) — i.e. the failure is NOT explained + # by the copy-fallback path ever firing; it is the ordering guarantee + # above. Do not re-add these names to _zero_copy_names without a + # rig-verified CUDA-event ordering fix. self._zero_copy_names: FrozenSet[str] = frozenset() self._shape_dict: Dict[str, Any] = {} @@ -103,9 +126,7 @@ def __call__( self._kvo_in_names = [f"kvo_cache_in_{i}" for i in range(n_kvo)] self._kvo_out_names = [f"kvo_cache_out_{i}" for i in range(n_kvo)] self._kvo_cache_len = n_kvo - self._zero_copy_names = frozenset( - self._kvo_in_names + self._fio_in_names + self._input_control_names + [self._input_control_middle] - ) + self._zero_copy_names = frozenset(self._kvo_in_names + self._fio_in_names) # Lazy-init FIO key name lists — same pattern as KVO. n_fio = len(fio_cache) @@ -113,9 +134,7 @@ def __call__( self._fio_in_names = [f"fio_cache_in_{i}" for i in range(n_fio)] self._fio_out_names = [f"fio_cache_out_{i}" for i in range(n_fio)] self._fio_cache_len = n_fio - self._zero_copy_names = frozenset( - self._kvo_in_names + self._fio_in_names + self._input_control_names + [self._input_control_middle] - ) + self._zero_copy_names = frozenset(self._kvo_in_names + self._fio_in_names) # Update pre-allocated dicts in-place — no new dict objects created per call. shape_dict = self._shape_dict diff --git a/src/streamdiffusion/acceleration/tensorrt/utilities.py b/src/streamdiffusion/acceleration/tensorrt/utilities.py index aecb8a8a3..3ec567a77 100644 --- a/src/streamdiffusion/acceleration/tensorrt/utilities.py +++ b/src/streamdiffusion/acceleration/tensorrt/utilities.py @@ -564,6 +564,10 @@ def _staging_action( Returns one of: "copy" - copy into self.tensors[name] (today's behavior; always safe) + "copy_and_reset" - copy into self.tensors[name], but reset the CUDA graph first + because this name was previously bound zero-copy into a live + graph, which still reads the old bound address rather than + self.tensors[name] "bind" - skip the copy; bind TensorRT directly to the caller's tensor "bind_and_reset" - skip the copy and bind directly, but reset the CUDA graph first because the caller's address changed while a graph @@ -575,6 +579,13 @@ def _staging_action( invariant against config drift rather than a path exercised in production. """ if name not in zero_copy_names or not is_contiguous or not dtype_match or cur_ptr % 256 != 0: + # Falling back to copy. If this name was bound zero-copy into a LIVE graph, + # that graph still reads the stale bound address, not self.tensors[name] — + # force one reset so it re-captures reading the staging buffer. prev_ptr is + # only non-None for names actually bound before, so plain copy-path inputs + # (never in zero_copy_names) never trip this. + if graph_exists and prev_ptr is not None: + return "copy_and_reset" return "copy" if graph_exists and cur_ptr != prev_ptr: return "bind_and_reset" @@ -1195,8 +1206,14 @@ def infer(self, feed_dict, stream, use_cuda_graph=False, zero_copy_names: frozen cur_ptr, graph_exists, ) - if action == "copy": + if action in ("copy", "copy_and_reset"): self.tensors[name].copy_(buf) + if action == "copy_and_reset": + needs_reset = True + # Next frame this name has no prior bind, so + # _staging_action sees prev_ptr=None and returns + # plain "copy" — converges after one reset. + self._bound_ptrs.pop(name, None) else: bind_ptrs[name] = cur_ptr if action == "bind_and_reset": diff --git a/tests/unit/test_zero_copy_staging_5_6.py b/tests/unit/test_zero_copy_staging_5_6.py index c06522ab7..dbba00bbf 100644 --- a/tests/unit/test_zero_copy_staging_5_6.py +++ b/tests/unit/test_zero_copy_staging_5_6.py @@ -55,13 +55,18 @@ def test_name_not_in_zero_copy_names_copies(self): assert result == "copy" def test_empty_zero_copy_names_always_copies(self): - """Default frozenset() → today's behavior exactly, for every name.""" + """Default frozenset() → today's behavior exactly, for every name. + + prev_ptr=None is the realistic value here: a name that is never in + zero_copy_names is never bound, so Engine._bound_ptrs never has an + entry for it and _bound_ptrs.get(name) is always None. + """ result = _staging_action( "kvo_cache_in_0", frozenset(), True, True, - PTR_A, + None, PTR_A, True, ) @@ -200,25 +205,93 @@ def test_misaligned_pointer_falls_back_to_copy_no_graph(self): assert result == "copy" def test_misaligned_pointer_falls_back_to_copy_with_live_graph(self): - """Even a pointer-unchanged steady-state frame must copy, not bind, if - the (mis-aligned) address itself is invalid for setTensorAddress.""" + """A name previously bound zero-copy (prev_ptr set from an earlier, + valid aligned bind) whose current address is invalid for + setTensorAddress must copy — but a live graph is still replaying the + stale bound address, so this must ALSO force a reset. + + This is the regression lock: f8ff50f's bind->copy fallback left the + live UNet CUDA graph replaying a frozen buffer forever once a + zero-copy input fell back to copy mid-stream (see unet_engine.py's + header comment for the full incident writeup). Before the + copy_and_reset outcome existed, this exact transition asserted plain + "copy" here — i.e. the test documented the bug as correct behavior. + """ result = _staging_action( "kvo_cache_in_0", frozenset({"kvo_cache_in_0"}), True, True, - PTR_MISALIGNED, - PTR_MISALIGNED, + PTR_A, # previously bound at a valid, aligned address + PTR_MISALIGNED, # now invalid -> must fall back to copy + True, + ) + assert result == "copy_and_reset" + + +class TestStagingActionCopyAndReset: + """copy_and_reset is the general-purpose safety net for ANY zero-copy name + (currently kvo/fio only — see unet_engine.py) that falls back to copy while + a graph built from its previously-bound address is still live. Exercises + the non-contiguous fallback path specifically (the misaligned-pointer path + is covered by TestStagingActionAlignmentGuard above).""" + + def test_non_contiguous_previously_bound_with_live_graph_copies_and_resets(self): + """A previously-bound name (e.g. a cache tensor that got sliced/viewed + this frame) that is no longer contiguous, with a live graph, must copy + AND force a reset — the graph is still reading the old bound address.""" + result = _staging_action( + "kvo_cache_in_0", + frozenset({"kvo_cache_in_0"}), + False, # is_contiguous + True, + PTR_A, # previously bound + PTR_A, + True, + ) + assert result == "copy_and_reset" + + def test_non_contiguous_previously_bound_no_live_graph_just_copies(self): + """Same fallback, but no graph exists yet — nothing to invalidate, so a + plain copy is correct; forcing a reset here would be a needless no-op + reset on every non-graphed frame.""" + result = _staging_action( + "kvo_cache_in_0", + frozenset({"kvo_cache_in_0"}), + False, + True, + PTR_A, + PTR_A, + False, + ) + assert result == "copy" + + def test_never_bound_name_falls_back_with_live_graph_just_copies(self): + """prev_ptr=None means this name has never been bound zero-copy, so no + graph could possibly be reading a stale address for it — plain copy, + no spurious reset.""" + result = _staging_action( + "kvo_cache_in_0", + frozenset({"kvo_cache_in_0"}), + False, + True, + None, + PTR_A, True, ) assert result == "copy" class TestStagingActionControlNetResiduals: - """Phase-2 D2: input_control_* UNet inputs opt into the same zero-copy path - as kvo/fio. Steady state binds without a reset; the ControlNet idle<->active - toggle (or a resolution/batch change) flips the residual's address while a - graph is live, forcing exactly one bind_and_reset on the toggle frame.""" + """Phase-2 D2 attempted to opt input_control_* UNet inputs into this same + zero-copy path; that was reverted (see unet_engine.py header comment) after + it reproduced a "ControlNet produces no visual change" regression on the + rig, driven by a lost engine-stream copy ordering guarantee — not by the + bind->copy fallback covered above. input_control_* names are PERMANENTLY + excluded from the real UNet engine's _zero_copy_names. The tests below + exercise _staging_action as a pure function using ControlNet-shaped names + only as convenient example zero-copy names; they do not describe current + runtime behavior for actual ControlNet residuals.""" def test_steady_state_control_residual_binds_without_reset(self): """Same persistent merge/output buffer across frames — plain bind.""" From 3081e357fe428be632dcec53fee0e65e894fbbc2 Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 11 Jul 2026 19:26:34 -0400 Subject: [PATCH 21/37] fix: enforce keyword-only StreamParameterUpdater binding; correct seed interpolation kwarg --- docs/api_conformance_audit_2026-07-11.md | 415 ++++++++++++++++++ src/streamdiffusion/config.py | 10 +- src/streamdiffusion/pipeline.py | 6 +- .../stream_parameter_updater.py | 1 + tests/unit/test_param_updater_binding.py | 134 ++++++ 5 files changed, 559 insertions(+), 7 deletions(-) create mode 100644 docs/api_conformance_audit_2026-07-11.md create mode 100644 tests/unit/test_param_updater_binding.py diff --git a/docs/api_conformance_audit_2026-07-11.md b/docs/api_conformance_audit_2026-07-11.md new file mode 100644 index 000000000..ba00eba9b --- /dev/null +++ b/docs/api_conformance_audit_2026-07-11.md @@ -0,0 +1,415 @@ +# API Conformance Audit — CUDA Python / TensorRT / ONNX (2026-07-11) + +> **Untracked working document — never stage or commit** (same convention as +> `docs/perf_bestpractices_audit_2026-07-10.md`). +> +> Audit of the codebase's usage of the CUDA Python, TensorRT, and ONNX-family APIs +> against official documentation and the *installed* package versions. Report only — +> no code was changed this phase. Branch `refactor/py-code-review-remediation`, +> HEAD `d5efd4d`. + +--- + +## 1. Method and ground truth + +Two evidence sources, used together: + +1. **Online docs (browser, pulled this session).** NVIDIA's "latest" TensorRT docs + are version **11.1.0** — one major version ahead of the installed 10.16. They are + used as the authoritative statement of semantic contracts (which are stable across + 10.x→11.x) and as the *direction of travel* for deprecations/removals. +2. **Installed-package introspection (venv, authoritative for pinned versions).** + Enum membership, method presence, signatures, and docstrings were introspected from + the actual installed wheels (`venv/Scripts/python.exe`). Where 11.1 docs and 10.16 + bindings disagree, the introspection result is treated as ground truth for today's + behavior and the 11.1 docs as ground truth for what an upgrade will break. + +### Pinned versions → doc sources + +| Package | Installed | Doc source used | +|---|---|---| +| tensorrt_cu12 | 10.16.1.11 | docs.nvidia.com TensorRT 11.1.0 (`/latest/_static/python-api/...`, `/latest/_static/c-api/...`) + venv introspection | +| cuda-python | 12.9.7 (dist 12.9.0) | venv introspection (docstrings/warnings); nvidia.github.io/cuda-python | +| onnx | 1.19.1 | venv introspection (`onnx.save_model` signature) | +| torch | 2.8.0+cu128 | venv introspection (`torch.onnx.export` signature + docstring) | +| polygraphy | 0.49.26 | venv introspection (`engine_from_bytes` signature) | +| nvidia-modelopt | 0.43.0 | venv introspection (`modelopt.onnx.quantization.quantize` signature) | +| onnxruntime-gpu | 1.24.4 | (used only indirectly by modelopt calibration — no direct call sites) | + +### Key doc pages cited below + +- **[TRT-CTX-PY]** `https://docs.nvidia.com/deeplearning/tensorrt/latest/_static/python-api/infer/Core/ExecutionContext.html` +- **[TRT-CTX-CPP]** `https://docs.nvidia.com/deeplearning/tensorrt/latest/_static/c-api/classnvinfer1_1_1_i_execution_context.html` +- **[TRT-BCFG]** `https://docs.nvidia.com/deeplearning/tensorrt/latest/_static/python-api/infer/Core/BuilderConfig.html` +- **[TRT-ONNX]** `https://docs.nvidia.com/deeplearning/tensorrt/latest/_static/python-api/parsers/Onnx/pyOnnx.html` +- **[TRT-REFIT]** `https://docs.nvidia.com/deeplearning/tensorrt/latest/_static/python-api/infer/Core/Refitter.html` +- **[INTROSPECT]** venv introspection of installed packages, this session. + +### Installed TRT 10.16.1.11 enum/API ground truth [INTROSPECT] + +- `trt.TacticSource` members: `{CUBLAS, CUBLAS_LT, CUDNN, EDGE_MASK_CONVOLUTIONS, JIT_CONVOLUTIONS}` + (11.1 docs list **only** `EDGE_MASK_CONVOLUTIONS` and `JIT_CONVOLUTIONS`, both "Enabled by default" [TRT-BCFG]). +- `trt.BuilderFlag` includes `FP16, FP8, TF32, REFIT, SPARSE_WEIGHTS`; **no `STRONGLY_TYPED`** (removed in 10.12). +- `trt.PreviewFeature` members: `{PROFILE_SHARING_0806, ALIASED_PLUGIN_IO_10_03, RUNTIME_ACTIVATION_RESIZE_10_10, MULTIDEVICE_RUNTIME_10_16}`. +- `trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH` still present (deprecated). +- `trt.OnnxParserFlag.NATIVE_INSTANCENORM` present. +- `IExecutionContext.set_device_memory(memory, size)` **exists** (two-arg, V2 semantics: + docstring requires "256-byte aligned device memory" and size ≥ `get_device_memory_size_v2`); + the legacy one-arg `device_memory` property setter also still exists. There is **no** + Python attribute named `set_device_memory_v2` — the V2 API surfaces under the plain name. +- `ICudaEngine.device_memory_size_v2` exists. +- `Refitter` has `get_all`, `set_weights`, `set_named_weights`, `refit_cuda_engine`, + `refit_cuda_engine_async`; `trt.WeightsRole` has 6 members. +- `trt.nptype(trt.DataType.FP8)` raises `TypeError` ("Could not resolve TensorRT datatype + to an equivalent numpy datatype"). +- Accessing deprecated enum members emits **no** runtime `DeprecationWarning` — deprecation + is silent at runtime; only the docs/headers say so. + +--- + +## 2. Verdict summary + +**No CRITICAL findings.** Every documented contract with a runtime consequence +(tensor-address alignment, input-lifetime around `execute_async_v3`, timing-cache +lifetime, graph-capture rules, refit-vs-enqueued-work) was checked and the code +conforms — in several cases with explicit, comment-documented mitigations. The +findings below are deprecation debt (breaks on a future TRT 11 upgrade), missing +defensive checks, and minor idiom/perf notes. + +| ID | Severity | File:line | Summary | +|---|---|---|---| +| H1 | HIGH | `src/streamdiffusion/acceleration/tensorrt/utilities.py:388-401` | `TacticSource.CUBLAS/CUBLAS_LT` removed in TRT 11 → whole tactic-scoping block silently no-ops on upgrade | +| H2 | HIGH | `utilities.py:1002` | Deprecated `context.device_memory` property setter (superseded since TRT 10.1); path is currently dead code | +| M1 | MEDIUM | `preprocessing/processors/trt_base.py:135,144,195,200,235` | `set_input_shape` / `set_tensor_address` bool returns unchecked → silent failure risk | +| M2 | MEDIUM | `utilities.py:544-570` | Zero-copy bind path has no 256-byte alignment guard (contract-safe under all default configs — proven in §5 — but unguarded against config drift) | +| M3 | MEDIUM | `src/streamdiffusion/tools/compile_depth_anything_tensorrt.py:129` | Deprecated `NetworkDefinitionCreationFlag.EXPLICIT_BATCH` (sibling RAFT tool already fixed) | +| L1 | LOW | `utilities.py:1525` | `do_constant_folding=True` — "Deprecated option" in torch 2.8 | +| L2 | LOW | `utilities.py:1519-1530` | Legacy TorchScript ONNX exporter (`dynamo=False`); torch 2.8 recommends `dynamo=True` but keeps `False` as default | +| L3 | LOW | `utilities.py:1579` | External-data detection by `.pb` suffix scan is heuristic (safe in current flow, brittle to reordering) | +| I1–I5 | INFO | various | Redundant flag, CPU-alloc idiom, refit sync-scope note, legacy-import fallback, ctypes cudart | + +Conformance confirmations (checked, no finding) are listed in §4. + +--- + +## 3. Findings + +### H1 — `TacticSource.CUBLAS/CUBLAS_LT` are gone in TRT 11; the SM_120 tactic-scoping block will silently vanish on upgrade + +**File:** [utilities.py:388-401](../src/streamdiffusion/acceleration/tensorrt/utilities.py) + +```python +if gpu_profile.compute_capability >= (12, 0): + try: + sources = ( + (1 << int(trt.TacticSource.CUBLAS)) + | (1 << int(trt.TacticSource.CUBLAS_LT)) + | (1 << int(trt.TacticSource.JIT_CONVOLUTIONS)) + | (1 << int(trt.TacticSource.EDGE_MASK_CONVOLUTIONS)) + ) + config.set_tactic_sources(sources) + ... + except (AttributeError, TypeError) as e: + logger.debug(f"[TRT Config] set_tactic_sources not available: {e}") +``` + +**Docs:** [TRT-BCFG] documents `tensorrt.TacticSource` with exactly two members — +`EDGE_MASK_CONVOLUTIONS` and `JIT_CONVOLUTIONS`, each annotated "Enabled by default". +`CUBLAS`, `CUBLAS_LT`, and `CUDNN` no longer exist in 11.1. On installed 10.16 all five +members still exist [INTROSPECT], so the block works today. + +**Consequence on TRT 11 upgrade:** the first expression, `trt.TacticSource.CUBLAS`, +raises `AttributeError`; the `except` swallows it at DEBUG level. Net effect: no call to +`set_tactic_sources` at all. Because the code's *primary intent* on SM_120 is to **exclude +CUDNN** (per the block comment) and CUDNN tactics no longer exist in TRT 11, the silent +no-op is behaviorally harmless there — but the comment's rationale becomes stale, the +INFO log ("CUDNN excluded for SM_120+") never fires, and cuBLAS/cuBLAS_LT scoping intent +is silently dropped rather than consciously retired. + +**Remediation:** gate on `hasattr(trt.TacticSource, "CUBLAS")` instead of try/except so +the TRT 11 path is an explicit, logged branch (e.g. "TRT ≥11: default tactic sources +already exclude cuDNN/cuBLAS — nothing to scope"), and update the comment. No behavior +change on 10.16. + +--- + +### H2 — deprecated `context.device_memory` property setter (dead path today) + +**File:** [utilities.py:999-1002](../src/streamdiffusion/acceleration/tensorrt/utilities.py) + +```python +def activate(self, reuse_device_memory=None): + if reuse_device_memory: + ... + self.context.device_memory = reuse_device_memory +``` + +**Docs:** [TRT-CTX-CPP] `setDeviceMemory`: *"Deprecated in TensorRT 10.1. Superseded by +setDeviceMemoryV2()."* — with the additional caveat *"Weight streaming related scratch +memory will be allocated by TensorRT if the memory is set by this API"* (i.e. the legacy +setter's size assumption excludes weight-streaming scratch). The installed 10.16 Python +binding already exposes the successor as two-arg `set_device_memory(memory, size)` whose +docstring requires *"256-byte aligned device memory"* and *"size ... at least as large as +CudaEngine.get_device_memory_size_v2"* [INTROSPECT]. + +**Liveness:** grep of all 15 `activate()` call sites (runtime_engines, preprocessing +processors) shows **none pass `reuse_device_memory`** — the deprecated line is +unreachable in practice. `activate()` therefore always takes the +`create_execution_context()` branch. + +**Remediation (pick one):** +1. Delete the `reuse_device_memory` parameter and branch (dead code); or +2. Migrate to `self.context.set_device_memory(ptr, self.engine.device_memory_size_v2)` + if the memory-reuse feature is intended to come back. + +Either way the deprecated one-arg setter should not survive to a TRT 11 upgrade. + +--- + +### M1 — `trt_base.TensorRTEngine` ignores `set_input_shape` / `set_tensor_address` return values + +**File:** [trt_base.py:135](../src/streamdiffusion/preprocessing/processors/trt_base.py) +(also :144, :195, :200 for `set_input_shape`; :235 for `set_tensor_address`) + +```python +self.context.set_input_shape(name, input_shape) # :135, :144, :195, :200 +... +self.context.set_tensor_address(name, tensor.data_ptr()) # :235 +``` + +**Docs:** [TRT-CTX-PY] `set_input_shape(self, name, shape) -> bool` — returns whether the +shape was set successfully (False for out-of-profile shapes, wrong rank, unknown name); +`set_tensor_address` likewise returns `bool`. A `False` return is TRT's *only* signal — +no exception is raised. + +**Consequence:** an out-of-profile input shape (e.g. a preprocessing resolution outside +the engine's optimization profile) is silently accepted by Python; the failure then +surfaces downstream as an `execute_async_v3` failure or — worse — garbage output with +stale shapes. The core `Engine` class in utilities.py already does this correctly +(bool check at :1055; raises on `set_tensor_address` failure at :1209-1210); the second, +independent `TensorRTEngine` in trt_base.py (used by depth/pose/realesrgan/temporal-net +processors, i.e. live per-frame paths) does not. + +**Remediation:** mirror the utilities.py pattern — raise `RuntimeError` with tensor name +and shape when either call returns `False`. + +--- + +### M2 — zero-copy bind path (`_staging_action`) has no 256-byte alignment guard + +**File:** [utilities.py:544-570](../src/streamdiffusion/acceleration/tensorrt/utilities.py) +(decision helper), bind at :1194/:1208. + +```python +if name not in zero_copy_names or not is_contiguous or not dtype_match: + return "copy" +``` + +**Docs:** [TRT-CTX-CPP] `setTensorAddress`: *"The pointer must have at least 256-byte +alignment."* The Sub-phase 5.6 zero-copy path binds caller-owned kvo/fio per-layer +*views* (bucket base + slot offset) directly to TRT, so view offsets must preserve +256-byte alignment. + +**Current status: conformant.** §5 proves arithmetically that every per-layer view +offset is a multiple of 256 bytes under all default configurations (kvo unconditionally; +fio for every FI-eligible hidden dim under `max_fi_up_blocks ≤ 3`). No violation exists +today. + +**Why still a finding:** the guarantee is *implicit* — it rests on (a) fp16 dtype, +(b) hidden dims being multiples of 64, (c) the default FI eligibility mask, and +(d) PyTorch's allocator alignment. None of these is asserted anywhere. A config drift +(`max_fi_up_blocks=4` exposing H=320 layers, combined with an odd +`maxframes·batch·seq` product — see §5) or an atypical model would silently violate the +documented contract, and TRT does not reliably diagnose misaligned addresses (undefined +behavior territory). + +**Remediation:** add `cur_ptr % 256 == 0` to `_staging_action`'s eligibility test +(misaligned → fall back to `"copy"`, which is always safe). One integer modulo per input +per frame on the CPU; keeps the pure-function unit-testability. Optionally log once at +WARNING when the fallback triggers. + +--- + +### M3 — deprecated `EXPLICIT_BATCH` flag in the Depth-Anything compile tool + +**File:** [compile_depth_anything_tensorrt.py:129](../src/streamdiffusion/tools/compile_depth_anything_tensorrt.py) + +```python +network = builder.create_network(1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH)) +``` + +**Docs:** `EXPLICIT_BATCH` is deprecated since TRT 10.0 (networks are always +explicit-batch); the flag still exists in installed 10.16 [INTROSPECT] and is ignored. +The sibling tool already carries the fix and the rationale: +[compile_raft_tensorrt.py:152](../src/streamdiffusion/tools/compile_raft_tensorrt.py) — +`network = builder.create_network() # EXPLICIT_BATCH deprecated/ignored in TRT 10.x`. + +**Remediation:** same one-line change as the RAFT tool. Low blast radius (offline tool), +but it breaks outright when TRT removes the enum member. + +--- + +### L1 — `do_constant_folding=True` is a deprecated `torch.onnx.export` option + +**File:** [utilities.py:1525](../src/streamdiffusion/acceleration/tensorrt/utilities.py) + +**Docs [INTROSPECT, torch 2.8.0 docstring]:** *"do_constant_folding: Deprecated +option."* (alongside `training`, `operator_export_type`, `custom_opsets`). Harmless +today; drop it whenever the export call is next touched. + +### L2 — legacy TorchScript exporter (`dynamo=False`) + +**File:** [utilities.py:1519-1530](../src/streamdiffusion/acceleration/tensorrt/utilities.py) + +**Docs [INTROSPECT, torch 2.8.0]:** `dynamo=False` **is still the default** in 2.8 and is +not deprecated; the docstring says `dynamo=True` "is the recommended way to export models +to ONNX." No action required now — but the custom attention processors / export wrappers +were built against TS-exporter tracing semantics, so treat any future move to +`dynamo=True` as its own validation project, not a flag flip. Being explicit +(`dynamo=False`) is *better* than relying on the default, since the default is the likely +thing to change in a future torch. + +### L3 — external-data detection by `.pb` suffix scan + +**File:** [utilities.py:1579](../src/streamdiffusion/acceleration/tensorrt/utilities.py) + +```python +external_data_files = [f for f in os.listdir(onnx_dir) if f.endswith(".pb")] +``` + +Safe in the current flow **only because** `export_onnx` consolidates torch's external +tensor files (named `onnx__*`, no `.pb` suffix) into a single `weights.pb` and deletes +the originals (:1545-1562) before `optimize_onnx` runs. If a >2GB model ever reached +`optimize_onnx` straight from `torch.onnx.export` (which writes extension-less external +files), the scan would report "no external data" and `onnx.load(onnx_path)` at :1611 +would still succeed (external data loads by default) but the subsequent plain +`onnx.save` at :1614 would fail on the 2GB protobuf limit. Prefer +`onnx.external_data_helper`-based detection (check `TensorProto.data_location == +EXTERNAL` on the loaded model) or document the ordering invariant at the scan site. + +### INFO notes + +- **I1 — redundant parser flag.** `parser.set_flag(trt.OnnxParserFlag.NATIVE_INSTANCENORM)` + at utilities.py:769/:893 — [TRT-ONNX]: *"This flag is ON by default"* (and required for + version-/hardware-compatible engines). Not deprecated; setting it is a no-op. Keep or + drop, cosmetic either way. +- **I2 — CPU-alloc-then-copy.** utilities.py:1066 + `torch.empty(tuple(shape), dtype=torch_dtype).to(device=device)` allocates on CPU then + copies; trt_base.py:165 already uses the direct `device=` form (with a comment saying + why). One-time cost at buffer (re)allocation only — perf-cosmetic. +- **I3 — refit sync scope.** [TRT-REFIT]: *"The behavior is undefined if the engine has + pending enqueued work."* The refit path synchronizes `torch.cuda.current_stream()` at + utilities.py:723 before `refit_cuda_engine()` — correct for the load-time-only refit + flow (comment there documents this as defensive). Note it syncs torch's current stream, + not the engine's polygraphy stream; if refit were ever made reachable mid-stream, sync + the engine stream too. +- **I4 — legacy cuda-python import fallback.** utilities.py:37-41 prefers + `from cuda.bindings import runtime as cudart` and falls back to `from cuda import cudart`. + Installed 12.9.7 emits on the legacy import: *"The cuda.cudart module is deprecated and + will be removed in a future release, please switch to use the cuda.bindings.runtime + module instead."* [INTROSPECT]. The preference order is already correct; the fallback + only fires on cuda-python < 12.x installs. Fine as is. +- **I5 — ctypes cudart in `tools/cuda_l2_cache.py`** (`cudaDeviceSetLimit`, + `cudaStreamSetAttribute` via ctypes): bypasses cuda-python entirely; works but carries + its own struct-layout risk. Out of scope for remediation; noted for completeness. + +--- + +## 4. Conformance confirmations (checked, no finding) + +| Contract | Doc | Code | Verdict | +|---|---|---|---| +| `execute_async_v3` input lifetime (inputs must not be modified/freed before stream sync) | [TRT-CTX-PY] | Inputs staged via `copy_()` into persistent engine buffers *on the engine stream* (utilities.py:1158-1196); zero-copy names are persistent, address-stable caches by construction | ✔ | +| Default-stream warning (`enqueueV3` on default stream ⇒ extra device sync) | [TRT-CTX-CPP] | Core engine runs on polygraphy-owned non-default stream; trt_base engines create a dedicated `torch.cuda.Stream()` with pre/post event barriers + `record_stream()` | ✔ | +| `setTensorAddress` 256-byte alignment | [TRT-CTX-CPP] | Proven for all bound tensors: full buffers are torch base allocations (512-B aligned); kvo/fio view offsets ≡ 0 mod 256 under defaults (§5). Guard recommended (M2) | ✔ | +| CUDA graph capture: quiesce streams, ThreadLocal mode, instantiate signature | cuda-python 12.9 [INTROSPECT] | 3 warmup passes, engine-stream + legacy-stream drain before `cudaStreamBeginCapture(..., ThreadLocal)`; `cudaGraphInstantiate(graph, 0)` matches documented `(graph, unsigned long long flags)`; `CUASSERT` correctly unpacks `(err, result)` tuples; graph-launch failure falls back to `execute_async_v3` and re-captures | ✔ | +| Rebind-after-capture rule (addresses baked into graph) | [TRT-CTX-CPP] | `_staging_action` returns `bind_and_reset` on pointer change with live graph → `reset_cuda_graph()`; addresses only rebound when no graph instance exists (utilities.py:1198-1212) | ✔ | +| Timing cache lifetime ("must not be destroyed until after the engine is built") | [TRT-BCFG] | Local `trt_cache` created/loaded at utilities.py:807-817 stays in scope through `build_serialized_network` | ✔ | +| `builder_optimization_level` valid range 0–5 (default 3) | [TRT-BCFG] | Profile uses 3 or 4 | ✔ | +| `avg_timing_iterations` (default 1), `max_num_tactics` (default −1) | [TRT-BCFG] | 8/4 iterations, cap 64 — valid values, guarded by try/except for older TRT | ✔ | +| `PreviewFeature.RUNTIME_ACTIVATION_RESIZE_10_10` | [TRT-BCFG] + [INTROSPECT] | Present in installed 10.16 *and* still documented in 11.1 | ✔ | +| `BuilderFlag.STRONGLY_TYPED` removed in 10.12 | [INTROSPECT] | `hasattr(trt.BuilderFlag, "STRONGLY_TYPED")` guard at utilities.py:913-919 correctly skips on 10.16; FP8 path uses the network-creation flag instead | ✔ | +| `trt.nptype` has no FP8 mapping | [INTROSPECT] | `TypeError` → FP8 fallback at utilities.py:1043-1051 | ✔ | +| Refitter API (`get_all`, `set_weights`, `WeightsRole`, `refit_cuda_engine`) | [TRT-REFIT] | All still current in 11.1 docs, zero deprecation notes on the page; weight lifetime rule ("can be unset and released ... after refit_cuda_engine returns") satisfied — numpy arrays outlive the call | ✔ | +| `set_input_shape`/`set_tensor_address` return checks (core engine) | [TRT-CTX-PY] | utilities.py:1055 (bool check), :1209-1210 (raise) | ✔ (trt_base gap = M1) | +| `onnx.save_model` external-data kwargs | [INTROSPECT] onnx 1.19.1 | `save_as_external_data=True, all_tensors_to_one_file=True, location="weights.pb", convert_attribute=False` all match the installed signature | ✔ | +| `modelopt.onnx.quantization.quantize` kwargs | [INTROSPECT] 0.43.0 | `quantize_mode`, `use_external_data_format`, `high_precision_dtype` all valid; fp8_quantize.py additionally guards every kwarg via `inspect.signature` | ✔ | +| `polygraphy engine_from_bytes(serialized_engine, runtime=None)` | [INTROSPECT] 0.49.26 | Called with a single positional arg | ✔ | +| cuda-python module layout (`cuda.bindings.runtime` preferred) | [INTROSPECT] 12.9.7 | Preferred import first, legacy fallback second (utilities.py:37-41) | ✔ | + +--- + +## 5. Appendix — kvo/fio zero-copy alignment arithmetic (checkpoint #1) + +**Contract:** [TRT-CTX-CPP] `setTensorAddress`: *"The pointer must have at least 256-byte +alignment."* + +**What gets bound:** `Engine.infer` binds `buf.data_ptr()` for every feed-dict entry in +`zero_copy_names` (utilities.py:1180-1194). `zero_copy_names` is exactly the kvo + fio +input names (unet_engine.py:100/:108). Those tensors are the per-layer views created in +[models/utils.py](../src/streamdiffusion/acceleration/tensorrt/models/utils.py): + +- kvo (`create_kvo_cache`, :118-122): bucket `torch.zeros(L, 2, mf, B, S, H, dtype=fp16)`; + per-layer view = `bucket[slot]`. +- fio (`create_fi_cache`, :239-251): bucket `torch.zeros(L, mf, B, S, H, dtype=fp16)`; + per-layer view = `bucket[slot]`. + +(`L` = layers in bucket, `mf` = cache_maxframes, `B` = batch, `S` = sequence length, +`H` = attention hidden dim; fp16 ⇒ 2 bytes/element.) + +**Base pointers:** each bucket is a fresh `torch.zeros` CUDA allocation. PyTorch's CUDA +caching allocator returns block-granular base pointers (512-byte rounding), so +`bucket.data_ptr() % 256 == 0`. (This is allocator behavior, not a documented API +contract — one more reason for the M2 guard, which makes the whole argument +assumption-free at runtime.) + +**View offsets:** + +- kvo: `offset(slot) = slot · (2·mf·B·S·H elements) · 2 B = 4·slot·mf·B·S·H` bytes. + Every attention hidden dim in the supported UNets (SD1.5/SD-Turbo/SDXL: 320, 640, 1280) + is a multiple of 64, so `4·H ≡ 0 (mod 256)` ⇒ **every kvo view offset is a multiple of + 256 unconditionally** (independent of `mf`, `B`, `S`, `slot`). +- fio: `offset(slot) = 2·slot·mf·B·S·H` bytes ⇒ multiple of 256 iff `slot·mf·B·S·(H/64)` + is even. + - Default FI eligibility (`max_fi_up_blocks=2`, models/utils.py:127-164) admits only + mid-block + first two up-blocks ⇒ `H ∈ {1280}` (SD1.5: up_blocks[0] has no + attentions, up_blocks[1] is H=1280) or `H ∈ {1280, 640}` (SDXL). `H/64 ∈ {20, 10}` + is even ⇒ **aligned unconditionally**. + - `max_fi_up_blocks=3` adds H=640 layers (SD1.5) — still even ⇒ aligned. + - Only a non-default `max_fi_up_blocks=4` (which also contradicts the function's + documented "final up-block always excluded" rule) would admit H=320 (`H/64 = 5`, + odd). Then alignment requires `slot·mf·B·S` even. At the H=320 level + `S = (h/8)·(w/8)`, odd only when *both* `h/8` and `w/8` are odd (resolutions + ≡ 8 mod 16, e.g. 968×968) — so a violation additionally needs odd `mf`, odd `B`, + and an odd slot. Narrow, but reachable ⇒ finding M2. + +**Rebound caches:** `stream_parameter_updater.py:1089` replaces `kvo_cache[i]` with a +standalone tensor — a fresh base allocation (≥256-aligned) — and the pointer change is +detected by `_staging_action` → `bind_and_reset` → graph reset. Conformant. + +**Verdict: no alignment violation exists in any default configuration.** The M2 guard +converts this from an arithmetic argument into a runtime invariant. + +--- + +## 6. Ranked fix shortlist (candidate follow-up gated commits) + +1. **M1 — check `set_input_shape`/`set_tensor_address` returns in trt_base.py** + (raise like utilities.py does). Cheapest, closes a real silent-failure hole on live + per-frame paths. +2. **M2 — add `cur_ptr % 256 == 0` to `_staging_action` eligibility** (+ unit test rows + in the existing pure-function test). Makes the documented alignment contract a runtime + invariant instead of an arithmetic proof. +3. **H1 — replace try/except with `hasattr(trt.TacticSource, "CUBLAS")` version gate** + and refresh the comment/log so the TRT 11 behavior is explicit, not accidental. +4. **H2 — delete the dead `reuse_device_memory` branch** (or migrate to + `set_device_memory(ptr, engine.device_memory_size_v2)` if reuse is planned). +5. **M3 — `create_network()` in compile_depth_anything_tensorrt.py:129**, same one-liner + as the RAFT tool. +6. **L1 — drop `do_constant_folding=True`** next time the export call is touched. +7. **L3 — harden external-data detection** in `optimize_onnx` (data_location check or a + comment documenting the `export_onnx`-consolidates-first invariant). + +Items 1–5 are behavior-preserving on the installed stack (TRT 10.16 / cuda-python 12.9 / +torch 2.8) and would each pass the standard gate (pytest 117+/0, ruff, pyrefly ≤72 +baseline) independently. diff --git a/src/streamdiffusion/config.py b/src/streamdiffusion/config.py index fc1d5756c..576f823f0 100644 --- a/src/streamdiffusion/config.py +++ b/src/streamdiffusion/config.py @@ -1,8 +1,5 @@ -import logging -import os -import sys -import yaml import json +import logging from pathlib import Path from typing import Any, Dict, List, Tuple, Union @@ -11,6 +8,7 @@ logger = logging.getLogger(__name__) + def load_config(config_path: Union[str, Path]) -> Dict[str, Any]: """Load StreamDiffusion configuration from YAML or JSON file""" config_path = Path(config_path) @@ -96,7 +94,7 @@ def create_wrapper_from_config(config: Dict[str, Any], **overrides) -> Any: seed_blend_config = final_config["seed_blending"] wrapper.update_stream_params( seed_list=seed_blend_config.get("seed_list", []), - interpolation_method=seed_blend_config.get("interpolation_method", "linear"), + seed_interpolation_method=seed_blend_config.get("interpolation_method", "linear"), ) return wrapper @@ -548,4 +546,4 @@ def _validate_config(config: Dict[str, Any]) -> None: raise ValueError(f"_validate_config: Seed value {i} must be a non-negative integer") if not isinstance(weight, (int, float)) or weight < 0: - raise ValueError(f"_validate_config: Seed weight {i} must be a non-negative number") \ No newline at end of file + raise ValueError(f"_validate_config: Seed weight {i} must be a non-negative number") diff --git a/src/streamdiffusion/pipeline.py b/src/streamdiffusion/pipeline.py index cdbd86570..7ebe828dc 100644 --- a/src/streamdiffusion/pipeline.py +++ b/src/streamdiffusion/pipeline.py @@ -153,7 +153,11 @@ def __init__( self.add_time_ids = None # Initialize parameter updater - self._param_updater = StreamParameterUpdater(self, normalize_prompt_weights, normalize_seed_weights) + self._param_updater = StreamParameterUpdater( + self, + normalize_prompt_weights=normalize_prompt_weights, + normalize_seed_weights=normalize_seed_weights, + ) # Hook containers (step 1: introduced but initially no-op) self.embedding_hooks: List[EmbeddingHook] = [] diff --git a/src/streamdiffusion/stream_parameter_updater.py b/src/streamdiffusion/stream_parameter_updater.py index 13ca8cd1a..3e5ee18a1 100644 --- a/src/streamdiffusion/stream_parameter_updater.py +++ b/src/streamdiffusion/stream_parameter_updater.py @@ -29,6 +29,7 @@ class StreamParameterUpdater(OrchestratorUser): def __init__( self, stream_diffusion, + *, wrapper=None, normalize_prompt_weights: bool = True, normalize_seed_weights: bool = True, diff --git a/tests/unit/test_param_updater_binding.py b/tests/unit/test_param_updater_binding.py new file mode 100644 index 000000000..a87630a9c --- /dev/null +++ b/tests/unit/test_param_updater_binding.py @@ -0,0 +1,134 @@ +"""Regression tests for two parameter-binding bugs fixed in Stage 1: + + 1. ``StreamDiffusion.__init__`` constructed ``StreamParameterUpdater`` with three + *positional* args (``pipeline.py:156``), so the normalize flags landed on the + wrong fields (``wrapper`` <- prompt flag, ``normalize_prompt_weights`` <- seed + flag, ``normalize_seed_weights`` stuck at its default ``True``). The updater + ``__init__`` is now keyword-only past ``stream_diffusion`` so this mis-binding + is impossible to reintroduce silently. + + 2. ``create_wrapper_from_config`` passed ``interpolation_method=`` to + ``wrapper.update_stream_params`` on the seed-only-blending path + (``config.py:99``); that kwarg does not exist (the real name is + ``seed_interpolation_method``), so any config with ``seed_blending`` but no + ``prompt_blending`` raised ``TypeError`` at construction time. + +All tests run on CPU and construct no real pipeline. +""" + +import types +from unittest.mock import patch + +import pytest +import torch + +from streamdiffusion.stream_parameter_updater import StreamParameterUpdater + + +# --------------------------------------------------------------------------- +# Fixtures — mirror the minimal-fake-stream pattern from +# tests/unit/test_prompt_interpolation.py so __init__ runs without a real pipeline. +# --------------------------------------------------------------------------- + + +def _fake_stream(): + """Return a minimal namespace that looks like a StreamDiffusion instance.""" + stream = types.SimpleNamespace() + stream.device = torch.device("cpu") + stream.dtype = torch.float32 + stream.batch_size = 1 + stream.cfg_type = "none" + stream.guidance_scale = 1.0 + stream.prompt_embeds = None + stream.negative_prompt_embeds = None + stream._preprocessing_orchestrator = None + stream.embedding_hooks = [] + return stream + + +def _make_updater(**kwargs) -> StreamParameterUpdater: + """Construct an updater with a fake stream, no-op'ing attach_orchestrator.""" + stream = _fake_stream() + orig_attach = StreamParameterUpdater.attach_orchestrator + + def _noop_attach(self, s): # noqa: ANN001 + self._preprocessing_orchestrator = None + + StreamParameterUpdater.attach_orchestrator = _noop_attach + try: + updater = StreamParameterUpdater(stream, **kwargs) + finally: + StreamParameterUpdater.attach_orchestrator = orig_attach + + updater._embedding_orchestrator = None + return updater + + +# --------------------------------------------------------------------------- +# Bug 1 — normalize flags must bind to the correct fields. +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "prompt_flag, seed_flag", + [(True, False), (False, True), (False, False), (True, True)], +) +def test_normalize_flags_bind_to_correct_fields(prompt_flag, seed_flag): + updater = _make_updater( + normalize_prompt_weights=prompt_flag, + normalize_seed_weights=seed_flag, + ) + assert updater.normalize_prompt_weights is prompt_flag + assert updater.normalize_seed_weights is seed_flag + # wrapper stays None at construction — the real reference is attached later + # by StreamDiffusionWrapper (wrapper.py:454), never by a mis-bound bool. + assert updater.wrapper is None + + +def test_getters_echo_constructed_flags(): + updater = _make_updater(normalize_prompt_weights=False, normalize_seed_weights=True) + assert updater.get_normalize_prompt_weights() is False + assert updater.get_normalize_seed_weights() is True + + +def test_positional_flag_binding_is_rejected(): + """The keyword-only barrier makes the original bug a hard TypeError.""" + stream = _fake_stream() + with pytest.raises(TypeError): + StreamParameterUpdater(stream, False, False) # noqa: F841 + + +# --------------------------------------------------------------------------- +# Bug 2 — seed-only blending config must reach update_stream_params with the +# correct kwarg name (seed_interpolation_method), not interpolation_method. +# --------------------------------------------------------------------------- + + +class _FakeWrapper: + """Stub with the *real* update_stream_params kwarg name so the wrong name + (interpolation_method) would raise TypeError, faithfully reproducing the bug.""" + + def __init__(self, **kwargs): + self.update_calls = [] + + def prepare(self, **kwargs): # not exercised on the seed-only path, present for safety + pass + + def update_stream_params(self, *, seed_list=None, seed_interpolation_method="linear"): + self.update_calls.append({"seed_list": seed_list, "seed_interpolation_method": seed_interpolation_method}) + + +def test_seed_only_blending_uses_seed_interpolation_method(): + from streamdiffusion import config as config_mod + + seed_list = [(1, 0.5), (2, 0.5)] + cfg = {"seed_blending": {"seed_list": seed_list, "interpolation_method": "linear"}} + + with patch("streamdiffusion.StreamDiffusionWrapper", _FakeWrapper): + wrapper = config_mod.create_wrapper_from_config(cfg) + + assert isinstance(wrapper, _FakeWrapper) + assert len(wrapper.update_calls) == 1 + call = wrapper.update_calls[0] + assert call["seed_list"] == seed_list + assert call["seed_interpolation_method"] == "linear" From 56ffd6582ecf333a4222a164efcdea975f5122c0 Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 11 Jul 2026 19:26:58 -0400 Subject: [PATCH 22/37] perf: FP8 build disk-space preflight and hardened intermediate cleanup --- docs/perf_bestpractices_audit_2026-07-10.md | 358 ++++++++++++++ .../acceleration/tensorrt/builder.py | 435 ++++++++++-------- 2 files changed, 600 insertions(+), 193 deletions(-) create mode 100644 docs/perf_bestpractices_audit_2026-07-10.md diff --git a/docs/perf_bestpractices_audit_2026-07-10.md b/docs/perf_bestpractices_audit_2026-07-10.md new file mode 100644 index 000000000..a330adedc --- /dev/null +++ b/docs/perf_bestpractices_audit_2026-07-10.md @@ -0,0 +1,358 @@ +# StreamDiffusion Performance & Best-Practices Audit — 2026-07-10 + +Read-only audit against CUDA / PyTorch / TensorRT best practices, combining static code review +with measured profiling of the real-time denoising loop. No source files were changed — this +document is the only artifact produced. + +**Scope:** `src/streamdiffusion/{pipeline.py,wrapper.py}`, `acceleration/tensorrt/*`, `modules/*`, +`preprocessing/*`. **Config profiled:** StreamDiffusionTD "Quality/FP16" preset — SDXL-turbo, +512×512, img2img, FP8 TensorRT engines, CUDA graphs enabled. + +**Repo state at audit time:** branch `refactor/py-code-review-remediation` (2 commits ahead of +`origin/SDTD_032_dev`), HEAD `6718511`. The immediately preceding commit (`5a44f34`, same day) +already remediated exception-hygiene issues in `pipeline.py`, `ipadapter_module.py`, +`controlnet_module.py`, `image_processing_module.py`, `latent_processing_module.py`, +`acceleration/tensorrt/__init__.py`, `engine_manager.py`, and the TRT export wrappers — but **not** +`wrapper.py`, which was untouched. Every finding below was checked against this HEAD; where a +finding sits in a file/region that commit touched, its exact lines were diffed to confirm the +issue is distinct from what was already fixed (see "General code quality" §, findings #4–#10). + +--- + +## Prioritized quick wins + +Ranked by (impact × how directly it affects the measured per-frame budget) ÷ effort. "Measured" +column links to the Phase 2 data below; "Static" findings are code-review-only. + +| # | Finding | Impact | Effort | Evidence | +|---|---|---|---|---| +| 1 | **~13.6ms/frame (66% of the entire 20.8ms budget) inside `unet_step` is unattributed by any profiler region** — only ~4.6ms of its 18.2ms median is explained by measured children (`trt_infer`×3 + staging + scheduler). The unmeasured remainder is CFG-buffer prep, hook dispatch, and conditioning-lookup Python/CUDA logic that today has zero instrumentation. | **High** — largest single lever on real-time throughput | Small (instrument first, then optimize what's found) | `profiler_logs/*_stats.json`; `pipeline.py:820-1041` (`unet_step`) | +| 2 | `set_tensor_address` is reissued for every I/O tensor on **every frame**, even on the already-captured CUDA-graph replay fast path, defeating part of the point of graph capture (eliminate repeated host-side API calls) | High — recurring per-frame host overhead × every TRT engine in the pipeline | Small — guard with `if not (use_cuda_graph and self.cuda_graph_instance is not None):` | `acceleration/tensorrt/utilities.py:1118-1125` | +| 3 | Engine / timing-cache files are written non-atomically (`open(...,"wb").write()` straight to the final path); an interrupted multi-minute TRT build leaves a truncated file that the next run's `os.path.exists()` check silently treats as a valid cache hit | High — silent corruption / wasted rebuild risk | Small-medium — write to `.tmp` + `os.replace()` | `acceleration/tensorrt/utilities.py:779-789,907`; `builder.py:284-285` | +| 4 | RNG/seed silently resets to the hardcoded default (`seed=2`) on **every runtime prompt change** — `StreamDiffusion.prepare()`'s `generator` parameter defaults to a module-load-time-shared `torch.Generator()` mutable default, and neither `wrapper.py` `prepare()` branch (single-prompt or blending) forwards `self.generator`/`self.current_seed` when calling `stream.prepare()` | High — visible, reproducible loss of user-configured determinism in production | Small — thread `self.generator`/seed through both wrapper `prepare()` call sites; fix the mutable default | `pipeline.py:402-403`; `wrapper.py:493-498,519-524` | +| 5 | `__call__` / `postprocess_image("latent")` return aliased, persistent internal buffers (`_image_decode_buf`, `_prev_image_buf`) that get `.copy_()`'d in place next frame — any caller that retains the reference (queues it, hands to async consumer) sees silent mutation | High — silent data corruption for any downstream buffering consumer | Medium — `.clone()` on public returns of these buffers, or document the aliasing contract loudly | `pipeline.py:1272-1285`; `wrapper.py:1013-1014` | +| 6 | ControlNet's dynamic-shape profile floor (384px) is narrower than UNet's (256px) — a resolution in [256,384) that's valid for UNet will hard-fail specifically on the ControlNet engine when dynamic shapes + ControlNet are combined | Medium-high — real runtime failure risk in a reachable config | Small — lower `min_ctrl_h`/`min_ctrl_w` to 256, or share one constant | `acceleration/tensorrt/models/controlnet_models.py:68-73` vs `models/models.py:160-161` | +| 7 | NSFW safety-checker `.item()` blocks the frame on a synchronous GPU→host readback every frame it's enabled — no async pattern, despite the codebase already having the correct template one file away | High when the (opt-in, default-off) feature is enabled | Small-medium — reuse `image_filter.py`'s 1-frame-delay pinned-buffer pattern | `acceleration/tensorrt/runtime_engines/unet_engine.py:515`; `wrapper.py:1308` | +| 8 | `output_type="pil"` path does a blocking, unpinned `.cpu()` into freshly-allocated pageable memory every frame — the `"np"` path 30 lines above already does this correctly (pinned buffer + `non_blocking=True` + single `Event.record()/synchronize()`) | Medium-high for `"pil"` consumers | Small — route `"pil"` through the same `_output_pin_buf`/`Event` machinery | `wrapper.py:1264` vs `wrapper.py:1019-1037` | +| 9 | Stringly-typed OOM detection (`"out of memory" in str(e).lower()`) duplicated in two engine exception handlers, instead of `isinstance(e, torch.cuda.OutOfMemoryError)` (already used correctly elsewhere in the same file) | High — a PyTorch/TensorRT message-wording change silently breaks the OOM→CPU-fallback path | Small-medium — extract one `_is_oom_error(e)` helper, `isinstance` check first | `wrapper.py:2342-2349,2400-2407` | +| 10 | SDXL pipeline-type-mismatch retry failure is caught and only logged as a warning — execution continues with the wrong (non-SDXL) pipeline object rather than failing fast | High — garbled output for SDXL models on this edge case, with no error surfaced | Small — raise, or fail fast at engine-build time | `wrapper.py:1504-1514` | +| 11 | No recovery path if `cudaGraphLaunch` fails on the fast-replay path — an uncaught `CUASSERT` takes down the entire real-time loop instead of falling back to `execute_async_v3` for one frame | Medium — rare trigger, severe blast radius | Medium — try/except → `reset_cuda_graph()` → fall back, re-capture next call | `acceleration/tensorrt/utilities.py:1125` | +| 12 | `_load_model` is a ~1390-line function combining pipeline load, LoRA fusion, VAE setup, and TRT/xformers/sfast acceleration dispatch for UNet+VAE+ControlNet+IPAdapter+safety-checker — the exact logic this audit was asked to verify is concentrated in one untestable function | High — maintainability/testability of the acceleration-mode selection logic itself | Large — split into `_load_pipe()`, `_setup_lora()`, `_setup_vae()`, `_build_tensorrt_engines()`, `_install_hook_modules()` | `wrapper.py:1321-2712` | + +--- + +## Phase 2 — Measured per-frame budget + +**Method:** `scripts/profiling/profile_nsys.py --target benchmark` against the cached SDXL-turbo +FP8 TRT engine, 3 extra warmup + 11 `torch.profiler`-captured + 10 `nsys`-gated frames (24 total +measured `predict_x0_batch` calls). Steady-state wall-clock per frame from the nsys `frame_nsys*` +NVTX spans: **~20.5–21.5ms (~47–49 FPS)**. + +**Authoritative timing source.** The repo's own CUDA-event-based profiler +(`src/streamdiffusion/tools/gpu_profiler.py`, exported to `profiler_logs/*_stats.json`) is used +for every number below, **not** the raw `nsys stats --report nvtx_pushpop_sum` output. +Reason: NVTX push/pop ranges nested *inside* a captured CUDA graph fire only at capture time (the +first ~3 warmup passes), not on each graph replay — so their statistics wildly understate +steady-state cost for graph-replayed regions (e.g. `nvtx_pushpop_sum` shows `predict_x0_batch` +med=1.845ms, vs. the CUDA-event stats' correct **18.254ms**). Anyone profiling this pipeline again +should use the `profiler_logs/*_stats.json` export, or wrap regions of interest so they sit outside +the captured graph, not the raw NVTX summary. + +### Per-region breakdown (p50, `profiler_logs/sdtd_benchmark_20260710_092608_stats.json`, n=24 unless noted) + +| Region | p50 (ms) | p95 (ms) | Total (ms) | % of frame | +|---|---|---|---|---| +| `predict_x0_batch` (= `unet_step`, 1 call/frame) | 18.254 | 31.25 | 633.8 | ~88% | +| ┗ `trt_infer` (n=72, 3/frame) | 1.42 | 30.34 | 688.5 | (nested) | +| ┗ `trt.input_staging` (n=72, 3/frame) | 0.075 | 0.169 | 5.2 | (nested) | +| ┗ `sched.step_batch` (n=48, 2/frame) | 0.06 | 0.097 | 3.4 | (nested) | +| ┗ `sched.rebuild` (n=24) | 0.031 | 0.063 | 0.9 | (nested) | +| **Unattributed remainder inside `unet_step`** | **~13.6** | — | — | **~66%** | +| `encode_image` | 1.222 | 2.093 | 48.9 | ~6% | +| `decode_image` | 1.134 | 1.749 | 34.9 | ~5% | +| `glue.ipc_pack_rgba` | 0.086 | 0.118 | 22.0 | <1% | + +`trt_infer`'s three calls (3×1.42ms) plus staging/scheduler sum to **~4.6ms**, against an +`unet_step` median of **18.2ms** — a **~13.6ms/frame gap with no nested `profiler.region()` +call inside it today**. This is the single largest lever in the pipeline: it's 66% of the entire +frame budget, and is exactly the span covered by "General code quality" finding #12 below +(`unet_step`'s CFG-buffer-prep, hook-dispatch loop, and SDXL-conditioning-cache lookup) — +recommend adding `profiler.region()` spans around those three sub-phases before attempting any +optimization, since right now it's not possible to tell whether the cost is Python dispatch +overhead, un-graphed CUDA kernels (RCFG blend, buffer copies), or something else. + +### Kernel summary (`nsys stats --report cuda_gpu_kern_sum`, top contributors) + +FP8 (`e4m3`) TensorRT GEMM/conv kernels dominate as expected for an FP8-quantized UNet: +`sm89_xmma_gemm_e4m3...` entries account for ~17.6% + 13.8% + 3.7% + 3.3% + ... (roughly 45%+ of +total kernel time across the top 20 rows) — confirms FP8 quantization is effectively applied to +the primary UNet path, no action needed there. + +One notable outlier: **`cutlass_80_wmma_tensorop_f16_s161616gemm_f16...` is the single +second-largest kernel at 16.5% (1260 instances)** — this is an **FP16** tensor-core GEMM, not FP8. +Combined with several other `sm80`/`sm75` `f16f16` conv kernels and `fmha_cutlassF_f16` (flash +attention, 2.1%), roughly a quarter of total kernel time runs in FP16 rather than FP8. This audit +could not attribute these kernels to a specific engine (ControlNet vs. VAE vs. a text encoder) — +`ncu`'s source-correlation was blocked (see Limitations) — but given `fp8_quantize.py` exists in +this repo, it's worth checking whether ControlNet and/or the VAE are intentionally left at FP16 or +are simply not yet covered by the FP8 quantization pass. **Follow-up:** run `ncu` with +`--import-source yes` once GPU performance-counter access is available, to attribute this 16.5% +to a specific engine and confirm whether it's a legitimate FP8 candidate. + +### Memory-copy summary (`nsys stats --report cuda_gpu_mem_time_sum`) + +| Operation | Time (%) | Total (ms) | Count | Max single (ms) | +|---|---|---|---|---| +| H2D memcpy | 97.2 | 2167.3 | 1742 | 338.7 | +| memset | 2.6 | 58.6 | 166 | 51.6 | +| D2D memcpy | 0.1 | 2.4 | 502 | 0.03 | +| D2H memcpy | 0.1 | 1.9 | 1865 | 0.01 | + +H2D dominates nsys's GPU-op-time view, but the total (2.17s) **exceeds** the entire measured +24-frame NVTX capture window (~1.7s) — this is almost certainly TRT engine deserialization / +model-weight upload during process startup (`prepare()`), captured because `nsys profile` wraps +the whole Python process, not a per-frame steady-state cost. **Not corroborated against +timestamps in this pass** — treat as a cold-start/engine-load latency note (relevant to +TouchDesigner scene-load / engine-switch UX), not a per-frame throughput finding. D2H per-frame +traffic (1865 copies, avg <1μs) is negligible and consistent with the pinned-buffer output path. + +The first frame (`frame_warmup0`) costs ~1.16s — expected CUDA-graph-capture + cuBLAS/cuDNN +heuristic-search tax, paid once at startup per the `WARMUP_RUNS` mechanism. Not a runtime finding; +worth confirming this doesn't re-trigger on prompt/LoRA/engine hot-swap in the TD integration, as +that would surface as a UX stutter rather than a steady-state FPS regression. + +### Limitations of this profiling pass + +- **`ncu` (Nsight Compute) did not produce results.** First attempt hit `ERR_NVGPUCTRPERM` (GPU + performance-counter access disabled); after enabling "Allow access to the GPU performance + counters to all users" in NVIDIA Control Panel → Developer Settings, a retry (`--set basic`, + `--launch-count 30`) was attempted but **did not bound itself** — it ran for ~105 CPU-minutes + with the GPU pegged at full utilization and no completion in sight, and was terminated. Root + cause: `ncu`'s counter collection replays each kernel serially per metric pass, and this + benchmark loop launches its UNet/ControlNet/VAE work via **captured CUDA graphs** + (`acceleration/tensorrt/utilities.py:1123-1125`) — profiling graph-launched kernels with `ncu` + is known to multiply replay cost far beyond a normal (non-graphed) kernel count, so a naive + `--set basic` pass over the whole benchmark loop doesn't terminate in practical time. Per-kernel + SOL%, roofline, and occupancy for the FP8 GEMM kernels above are **not measured** — only the + nsys kernel-time ranking (by wall-clock share) is available. +- `nsys stats --report gpu_gaps` doesn't exist in the installed Nsight Systems 2026.3.1 (the + profiling README's report name is stale/version-mismatched); no direct GPU-idle-gap report was + substituted. +- `--cn-scale` defaulted to `0.0` in this capture — ControlNet TRT engine structure is visible in + the kernel/layer traces, but active per-frame ControlNet compute contribution was **not + confirmed** in this run. A follow-up capture with `--cn-scale > 0` would be needed to isolate its + cost. + +--- + +## TensorRT acceleration stack (static review) + +Overall maturity: this is a mature, incident-hardened hand-built TensorRT stack (not +`torch_tensorrt`) — GPU-tiered builder config, version/compute-capability/config-aware engine +cache-key naming, a documented (not accidental) single-CUDA-stream correctness fix, FP8 +finite-scale validation, and graph-reset-on-buffer-reallocation logic all reflect real +incident-driven engineering. Remaining gaps are concentrated and mostly small/isolated: + +1. **Non-atomic engine/timing-cache writes** — see quick win #3. +2. **`set_tensor_address` reissued every frame on the graph-replay fast path** — see quick win #2. +3. **ControlNet/UNet dynamic-shape profile-floor mismatch (384px vs 256px)** — see quick win #6. +4. **No recovery path on `cudaGraphLaunch` failure** — see quick win #11. +5. **`Engine.refit()` has no stream/context synchronization guard** (`utilities.py:565-665`, + `refitter.refit_cuda_engine()` at `:663`) — violates TensorRT's REFIT requirement that no + `IExecutionContext` be actively executing during a refit. Confirmed currently unreachable + (`enable_refit`/`build_enable_refit` default `False`, zero callers of `.refit(` repo-wide) — + a landmine only if REFIT-based weight hot-swap (e.g. live LoRA switching) is wired up later + without adding the guard. Impact: low today / high if activated unguarded. Effort: small. +6. **Single shared CUDA stream across UNet/ControlNet/VAE/aux engines** (`wrapper.py:2263`, + explicitly documented as a deliberate fix for a prior race in `controlnet_engine.py:155-157`). + Standard multi-stream overlap (e.g. decode frame N-1 while denoising frame N) is traded away + for correctness simplicity — do not revert casually. Impact: low (correctness fine, pure + missed-optimization). Effort: large (would need careful multi-stream + `cudaEvent` + cross-stream sync redesign to reclaim overlap without reintroducing the race). +7. **`opt_batch_size` defaults to `1`** in the public builder entry points (`builder.py:96` and 5 + call sites in `acceleration/tensorrt/__init__.py`), which would mistune the optimization + profile if ever called without an explicit value. Verified **not currently triggered** — the + real call path always threads `stream.trt_unet_batch_size` through correctly + (`wrapper.py:2315` → `engine_manager.py:224-233` → `EngineBuilder.build()`). Dormant footgun + for a future direct/test-harness caller. Impact: low (dormant). Effort: small — drop the + default, make it a required kwarg. +8. **FP8 sanity-gate comment mismatches the actual threshold** (`builder.py:307` comment says + "< 100", code checks `< 500` at `:323`) — cosmetic, update the comment. + +--- + +## PyTorch hot-path / sync-free review (static review) + +The per-frame diffusion core is in genuinely good shape: buffers are pre-allocated and reused +(`x_t_latent_buffer`, `_combined_latent_buf`, `_cfg_latent_buf`/`_cfg_t_buf`, +`_stock_noise_bufs` ping-pong, `_image_decode_buf`, `_prev_image_buf`), `pin_memory`/ +`non_blocking=True` are applied correctly at every H2D/D2H boundary that matters (input staging, +similarity-filter skip-probability readback, the `"np"` output path), `torch.inference_mode()` +wraps every hot entry point, and the frame-skip similarity filter uses a genuinely sync-free +1-frame-delay pinned-buffer pattern (`image_filter.py`) that should be the template for the two +findings below, not the exception. + +- **NSFW safety-checker blocking `.item()` every frame** — see quick win #7. +- **`"pil"` output path's blocking, unpinned `.cpu()`** — see quick win #8. +- **`_tensor_to_pil_safe` does two GPU-tensor truthiness syncs before its eventual `.cpu()`** + (`preprocessing/preprocessing_orchestrator.py:601,606,610`) — three host syncs where the + codebase's own `processors/base.py:tensor_to_pil()` (a near-duplicate helper) correctly does + the `.cpu()` transfer first and compares ranges after, at line `:175-208`. This is a regression + of a fix that already exists elsewhere in the same codebase. Impact: medium (gated to + ControlNet/image-hook configs using non-tensor-native processors). Effort: small — reorder to + match `base.py`'s pattern, or dedupe the two implementations. +- **Systemic, already self-tracked: ~20 of 28 preprocessor classes fall back to a per-frame + PIL/CPU round-trip** (`preprocessing/processors/base.py:87-109`) — only 8 processors set + `gpu_native = True`. The code already emits a one-time-per-class warning pointing at this exact + gap ("[GPU-residency]... Set gpu_native=True..."), so this is not a new discovery, but it is the + single largest concentration of avoidable per-frame syncs in the codebase, gated behind which + preprocessor a user selects rather than always active. Worth surfacing to whoever owns that + backlog item. +- **`_ipc_pack_rgba`/`_ipc_pack_unit_rgba` allocate fresh tensors every frame** (`wrapper.py:1052- + 1061,1092-1099`, `torch.full_like`/`torch.cat`) rather than writing into a persistent buffer like + every other output path in the same file. Impact: low (small tensors, cheap kernels, but + avoidable). Effort: small — hoist a persistent BGRA buffer, write in-place. +- **`channels_last` is absent repo-wide, and correctly so for the primary path** — the supported + real-time deployment runs UNet and VAE through TensorRT, so there are no PyTorch convs in the + hot path for `channels_last` to attach to. Only relevant if `acceleration="none"`/`"xformers"` + (PyTorch VAE fallback) or one of the ~20 non-TRT preprocessors above is selected — not a gap in + the intended architecture. + +--- + +## General code quality & best practices (static review) + +Two systemic patterns stand out across `pipeline.py`/`wrapper.py`/`modules/*`: (1) broad, +repeated tolerance for silently-swallowed exceptions that will make production failures invisible +until they surface as a worse downstream symptom, and (2) a handful of very large, +multi-responsibility functions that concentrate most of the acceleration-mode-selection logic this +audit was asked to review, making it hard to verify or safely extend. Findings already covered +above as quick wins are cross-referenced rather than repeated. + +- **RNG/seed mutable-default reset** — quick win #4. +- **Aliased buffer returns (`"latent"` output, `prev_image_result`)** — quick win #5. +- **Stringly-typed OOM detection duplicated** — quick win #9. +- **SDXL pipeline-mismatch retry failure silently swallowed** — quick win #10. +- **`_load_model` god function (~1390 lines, 30+ params)** — quick win #12. +- **`cleanup_engines_and_rebuild` targets a hardcoded `"engines"` path**, ignoring the configured + `engine_dir` (`wrapper.py:3062-3069`) — for any non-default `engine_dir` this OOM-recovery + method either silently no-ops or, worse, could delete an unrelated `engines/` folder in the cwd. + Impact: medium (breaks the one method whose entire purpose is OOM recovery, for any custom + `engine_dir`). Effort: small — `engines_dir = str(getattr(self, "_engine_dir", "engines"))`. +- **Manual `__del__()` invocation in `cleanup_gpu_memory()`** (`wrapper.py:2907-2912,2939-2943`) + — calling a dunder destructor explicitly does not stop the GC from invoking it again once the + object is actually collected; native TRT/CUDA cleanup logic inside `__del__` can run twice, + risking a double-free or a silently swallowed second exception. Impact: medium-high (double + cleanup of native handles is a classic source of intermittent, hard-to-reproduce crashes). + Effort: small — call a proper `close()`/`release()` if the wrapper exposes one, or just `del` + the reference. +- **`_load_lora_with_offline_fallback` swallows every per-candidate exception with zero logging** + (`pipeline.py:349-371`) — if candidate #2 of 5 fails for an unrelated reason (permissions, + corruption) while the rest fail with "not found", the caller only ever sees the final "not + found" — actively misleading during debugging. Impact: medium. Effort: small — add + `logger.debug` inside the loop. **Not touched by the 2026-07-09 remediation commit.** +- **Bare `except Exception: pass` with zero logging in "Advanced model detection failed" fallback** + (`wrapper.py:1855-1856`), inconsistent with every sibling handler within 50 lines that does log. + Impact: medium (both detection paths failing leaves stale `model_type`/`is_sdxl` with no + diagnostic trail). Effort: small. **In `wrapper.py`, which the 2026-07-09 remediation commit did + not touch at all.** +- **Per-frame dynamic import + broad try/except-pass chains in the UNet hot-path hook** + (`modules/ipadapter_module.py:434-521`, `_unet_hook`, invoked once per denoising step) — a + `from diffusers_ipadapter...` import executes on every call instead of being hoisted to module + scope, plus five separate `try/except Exception: pass` fallbacks, including one explicitly + commented "Do not add fallback mechanisms" that nonetheless silently swallows and continues. + Note: the 2026-07-09 remediation commit touched this same file but at different lines + (`:360-372`, `set_scale`/attribute-attach try/excepts) — this hot-path hook block was not part + of that pass. Impact: medium (per-step import overhead when IPAdapter is enabled; transient + failures invisible in production). Effort: small — hoist the import, add debug logging. +- **Pointless `try/except Exception: pass` wrapping a single `logger.info()` call** + (`wrapper.py:2103-2106`) — dead defensive code. Impact: low. Effort: small — delete it. +- **Duplicated latent-cache/decode/output-buffer logic between `__call__` and `txt2img`(_sd_turbo)** + (`pipeline.py:1272-1285` vs `1367-1394`) — the `_image_decode_buf` lazy-allocate-then-`.copy_()` + pattern is copy-pasted near-verbatim across 2-3 entry points; any fix to it (e.g. quick win #5) + must be applied in every copy or silently regresses one call path. Impact: medium. Effort: + small-medium — extract a shared `_decode_and_cache()` helper. +- **`unet_step` mixes five responsibilities in ~220 lines**: CFG buffer prep, SDXL conditioning + cache, generic hook dispatch, ControlNet/IPAdapter kwarg extraction, TRT-vs-PyTorch calling + convention (`pipeline.py:820-1041`). This is the exact span identified as the ~13.6ms/frame + profiling blind spot (quick win #1) — splitting it into `_prepare_cfg_batch()`, + `_get_sdxl_conditioning()`, `_dispatch_unet_hooks()`, `_call_unet()` would both fix the + maintainability issue and naturally create the instrumentation boundaries needed to localize + that cost. Impact: medium (maintainability) / high (unlocks quick win #1). Effort: large. +- **Parameter sprawl**: `_load_model` (30 params), `__init__` (50+ params) — plain + positional/keyword lists rather than a config object; makes the god-function split above harder + until addressed. Impact: medium. Effort: large — introduce a `StreamDiffusionConfig` dataclass, + can be done incrementally alongside the `_load_model` split. +- **Minor/cosmetic**: `__call__`'s `x` parameter is typed without `Optional` despite a `None` + default (`pipeline.py:1191`); `_get_scheduler_scalings` has no type hints (`pipeline.py:736`). + Impact: low. Effort: small. + +--- + +## Follow-ups not completed by this audit + +1. GPU performance-counter access is now enabled, but a naive `ncu --set basic` pass over the full + benchmark loop does not terminate in practical time because of CUDA-graph replay cost (see + Limitations above). To get SOL%/occupancy for the FP8 GEMM kernels and attribute the 16.5% + FP16 cutlass kernel to a specific engine, retry `ncu` with the workload scoped down — e.g. + disable CUDA-graph capture for the profiled process (`use_cuda_graph=False` on the engine + wrappers, if exposed as a config knob) so `ncu` sees ungraphed kernel launches, and/or bound + the pass tightly with `--launch-skip N --launch-count <10-20>` targeting only the steady-state + frames, and/or `--kernel-name regex:sm89_xmma_gemm|cutlass` to profile just the GEMM kernels of + interest instead of the entire per-frame kernel set. +2. Add `profiler.region()` spans inside `unet_step` around CFG-buffer prep, hook dispatch, and + conditioning lookup to localize the ~13.6ms/frame currently unattributed (quick win #1) — + this is the highest-value next step and doesn't require any of the above. +3. Correlate H2D memcpy timestamps against `prepare()` vs. per-frame windows to confirm the 2.17s + H2D total is a one-time load cost, not a steady-state one. +4. Re-run `profile_nsys.py` with `--cn-scale > 0` to isolate ControlNet's actual per-frame compute + contribution (this run had `--cn-scale 0.0`, the script's default). + +--- + +## Verification + +- Repo `git status` shows only this new file as untracked; no source under `src/streamdiffusion/` + was modified. +- Profiling artifacts referenced above (`profiles/*.txt`, `profiles/*.nsys-rep`, + `profiler_logs/*_stats.json`) were generated by the repo's own + `scripts/profiling/profile_nsys.py`/`nsys stats`, not hand-authored. +- Four citations (`utilities.py:1118-1125`, `pipeline.py:402-403`, + `controlnet_models.py:68-73`, `wrapper.py:1264`) were re-read directly against source and match + the findings exactly. + +### MCP cross-check (2026-07-10, post-publication) + +A second, independent verification pass used the repo's semantic code-search index +(`D:\dev\SDTD_032_dev`, 4707 chunks, `last_indexed 2026-07-10T08:16`Z — after the +`5a44f34`/`6718511` commits above, so the index reflects the exact HEAD this report was written +against). Method: `search_code` (k=7) to locate each claim's region, then exact `Grep`/`Read` to +pin the specific cited line(s). ~20 of the ~30 path:line citations in this report were checked, +covering all 12 quick wins plus a sample of the general-findings section. + +**Result: no hallucinated citations.** Every checked claim resolves to real code at (or within a +line or two of) its cited location, including the highest-leverage ones: the `unet_step` god-method +(`pipeline.py:826-1041`, cyclomatic complexity 52 — corroborates quick win #1/#12's "unattributed +~13.6ms" and "220-line, five-responsibility function" characterizations independently of the +profiler data); `Engine.refit()` (`utilities.py:565-665`); the graph-replay `set_tensor_address` +loop (`utilities.py:1118-1125`); the single shared `cuda.Stream()` (`wrapper.py:2263`); and the +UNet-vs-ControlNet shape-floor mismatch (`models.py:160` = 256 vs. ControlNet's 384). + +Two findings check out as understated, not overstated: +- **Quick win #9 (stringly-typed OOM):** `wrapper.py:2345` and `:2403` do match `"out of memory" + in error_msg` as described — but the same file already catches the typed + `torch.cuda.OutOfMemoryError` correctly at `wrapper.py:2029`. The typed exception is known to the + codebase, making the two string-matched sites more clearly a regression/inconsistency than an + unavoidable gap. +- **TensorRT finding #8 (FP8 sanity-gate comment mismatch):** confirmed exactly — `builder.py:307` + comment reads "< 100 means quantization is inactive" while `builder.py:323` gates on `_qdq < 500`. + +One characterization was not independently confirmed at the same precision as the rest: the +per-frame *invocation frequency* claimed for the IPAdapter hot-path hook +(`modules/ipadapter_module.py:434-521`) — the function-local imports at `:462`/`:479` are confirmed +present, but this pass did not separately verify the hook fires on every denoising step versus, +e.g., once per `generate()` call. + +Also independently re-confirmed via direct grep of `src/streamdiffusion/`: zero occurrences of +`channels_last`/`memory_format` repo-wide, matching the "absent repo-wide" claim in the PyTorch +hot-path section exactly. diff --git a/src/streamdiffusion/acceleration/tensorrt/builder.py b/src/streamdiffusion/acceleration/tensorrt/builder.py index 045dd7390..97480fa2f 100644 --- a/src/streamdiffusion/acceleration/tensorrt/builder.py +++ b/src/streamdiffusion/acceleration/tensorrt/builder.py @@ -2,6 +2,7 @@ import json import logging import os +import shutil import time from datetime import datetime, timezone from pathlib import Path @@ -70,6 +71,103 @@ def _run_fp8_stage(name: str, fn, stats: dict, allow_fallback: bool, engine_file ) from err +def _check_fp8_disk_space(onnx_opt_path: str, allow_fallback: bool) -> bool: + """Preflight disk-space check before FP8 ONNX quantization. + + ModelOpt keeps several full-size copies of the external-data ONNX weights on disk + at once (opt, opt_named, opt_named_extended, fp8 output) plus calibration tensors. + Require free space >= 5x the source ONNX's on-disk footprint (external data + included), with a ~28 GB floor for SDXL-scale models, so we fail fast before the + ~40 min export/optimize/calibrate pipeline instead of after it. + """ + onnx_dir = os.path.dirname(onnx_opt_path) + source_size = os.path.getsize(onnx_opt_path) + weights_pb = os.path.join(onnx_dir, "weights.pb") + if os.path.exists(weights_pb): + source_size += os.path.getsize(weights_pb) + + required = max(5 * source_size, 28 * 1024**3) + free = shutil.disk_usage(onnx_dir).free + if free >= required: + return True + + _build_logger.warning( + f"[BUILD] Low disk space for FP8 quantization on {onnx_dir}: " + f"{free / 1024**3:.1f} GB free, need ~{required / 1024**3:.1f} GB." + ) + if allow_fallback: + _build_logger.warning("[BUILD] Falling back to FP16 (fp8_allow_fp16_fallback=True).") + return False + raise RuntimeError( + f"Insufficient disk space for FP8 quantization on {onnx_dir}: " + f"{free / 1024**3:.1f} GB free, need ~{required / 1024**3:.1f} GB. " + "Free up space, or set fp8_allow_fp16_fallback=True in TRT_PROFILES to build FP16 instead." + ) + + +def _cleanup_intermediates(engine_dir: str, fp8_ok: bool): + """Delete intermediate ONNX/build artifacts, preserving .engine, .cache, calib_data.npz, + build_stats.json, and (only when fp8_ok) the cached unet.fp8.onnx* artifact. + + Two-pass deletion handles Windows file locks (gc.collect releases Python handles). + Runs from a `finally` block so it also fires when a build stage raises, instead of + orphaning tens of GB of external-data ONNX copies on failure. + """ + _keep_suffixes = (".engine", ".cache") + _keep_exact = {"build_stats.json", "timing.cache", "calib_data.npz"} + _to_delete = [] + for file in os.listdir(engine_dir): + # Keep the FP8 quantized ONNX artifact only if quantization actually succeeded + # (marked by the ".ok" sentinel) -- a partial file from a failed run must be swept. + if fp8_ok and "fp8.onnx" in file: + continue + if file in _keep_exact or any(file.endswith(s) for s in _keep_suffixes): + continue + _to_delete.append(os.path.join(engine_dir, file)) + + if not _to_delete: + return + + _failed = [] + for fpath in _to_delete: + try: + os.remove(fpath) + except OSError: + _failed.append(fpath) + + # Release Python-held file handles (ONNX model refs), retry locked files. + # Per-file poll with 50ms backoff instead of a single global sleep -- most + # handles release within 1-2 retries on Windows; worst case ~0.5s same as before. + if _failed: + gc.collect() + torch.cuda.empty_cache() + _still_failed = [] + for fpath in _failed: + _last_err = None + for _attempt in range(10): + try: + os.remove(fpath) + _last_err = None + break + except OSError as _e: + _last_err = _e + time.sleep(0.05) + if _last_err is not None: + _still_failed.append(os.path.basename(fpath)) + _build_logger.warning( + f"[BUILD] Could not delete temp file {os.path.basename(fpath)}: {_last_err}" + ) + if _still_failed: + _build_logger.warning( + f"[BUILD] {len(_still_failed)} intermediate files could not be cleaned. " + f"Manual cleanup: delete all files except *.engine, calib_data.npz, unet.fp8.onnx from {engine_dir}" + ) + cleaned = len(_to_delete) - len(_still_failed) + else: + cleaned = len(_to_delete) + _build_logger.info(f"[BUILD] Cleaned {cleaned}/{len(_to_delete)} intermediate files") + + def create_onnx_path(name, onnx_dir, opt=True): return os.path.join(onnx_dir, name + (".opt" if opt else "") + ".onnx") @@ -201,157 +299,163 @@ def build( ) _build_logger.info(f"Verified ONNX opt file: {onnx_opt_path} ({opt_file_size / (1024**2):.1f} MB)") - # --- FP8: Capture calibration tensors (once, cached in calib_data.npz) --- - if fp8 and pipe_ref is not None: - if os.path.exists(_calib_data_path): - _build_logger.info(f"[BUILD] FP8 calibration data cached: {_calib_data_path}") - stats["stages"]["fp8_calib_capture"] = {"status": StageStatus.CACHED} - else: + try: + # --- FP8: Capture calibration tensors (once, cached in calib_data.npz) --- + if fp8 and pipe_ref is not None: + if os.path.exists(_calib_data_path): + _build_logger.info(f"[BUILD] FP8 calibration data cached: {_calib_data_path}") + stats["stages"]["fp8_calib_capture"] = {"status": StageStatus.CACHED} + else: - def _calib_fn(): - if is_controlnet: - from .fp8_quantize import capture_calibration_data_controlnet + def _calib_fn(): + if is_controlnet: + from .fp8_quantize import capture_calibration_data_controlnet + + _build_logger.info( + f"[BUILD] FP8 CN calibration: {calibration_steps} synthetic passes, " + f"res={opt_image_width}x{opt_image_height}" + ) + capture_calibration_data_controlnet( + cn_model=pipe_ref, + n_calibration_steps=calibration_steps, + image_height=opt_image_height, + image_width=opt_image_width, + batch_size=opt_batch_size, + save_path=_calib_data_path, + ) + else: + from .fp8_quantize import _load_calibration_prompts, capture_calibration_data + + prompts = calibration_prompts or _load_calibration_prompts() + _build_logger.info( + f"[BUILD] FP8 activation capture: {len(prompts)} prompts × " + f"{calibration_steps} steps, guidance_scale={fp8_guidance_scale}" + ) + capture_calibration_data( + pipe_ref, + prompts, + num_inference_steps=calibration_steps, + save_path=_calib_data_path, + guidance_scale=fp8_guidance_scale, + onnx_path=onnx_opt_path, + use_cached_attn=fp8_use_cached_attn, + use_controlnet=fp8_use_controlnet, + num_ip_layers=fp8_num_ip_layers, + ) + + if not _run_fp8_stage("fp8_calib_capture", _calib_fn, stats, fp8_allow_fp16_fallback, engine_filename): + fp8 = False + elif fp8 and pipe_ref is None: + _build_logger.warning( + "[BUILD] fp8=True but pipe_ref not provided — FP8 calibration skipped. " + "Pass pipe_ref in engine_build_options for proper activation capture." + ) + fp8 = False + + # --- FP8: Inject native FLOAT8E4M3FN Q/DQ into the ONNX (cached in unet.fp8.onnx) --- + if fp8: + if os.path.exists(_fp8_onnx_path + ".ok"): + _build_logger.info(f"[BUILD] FP8 ONNX cached: {_fp8_onnx_path}") + stats["stages"]["fp8_onnx_quantize"] = {"status": StageStatus.CACHED} + elif not _check_fp8_disk_space(onnx_opt_path, fp8_allow_fp16_fallback): + fp8 = False + else: - _build_logger.info( - f"[BUILD] FP8 CN calibration: {calibration_steps} synthetic passes, " - f"res={opt_image_width}x{opt_image_height}" - ) - capture_calibration_data_controlnet( - cn_model=pipe_ref, - n_calibration_steps=calibration_steps, - image_height=opt_image_height, - image_width=opt_image_width, - batch_size=opt_batch_size, - save_path=_calib_data_path, - ) - else: - from .fp8_quantize import _load_calibration_prompts, capture_calibration_data + def _quant_fn(): + from .fp8_quantize import load_calibration_data, quantize_onnx_fp8 - prompts = calibration_prompts or _load_calibration_prompts() - _build_logger.info( - f"[BUILD] FP8 activation capture: {len(prompts)} prompts × " - f"{calibration_steps} steps, guidance_scale={fp8_guidance_scale}" - ) - capture_calibration_data( - pipe_ref, - prompts, - num_inference_steps=calibration_steps, - save_path=_calib_data_path, - guidance_scale=fp8_guidance_scale, + calib_data = load_calibration_data(_calib_data_path) + if calib_data is None: + raise RuntimeError(f"Calibration data missing after capture step: {_calib_data_path}") + quantize_onnx_fp8( onnx_path=onnx_opt_path, + output_path=_fp8_onnx_path, + calibration_data=calib_data, use_cached_attn=fp8_use_cached_attn, + use_feature_injection=fp8_use_feature_injection, use_controlnet=fp8_use_controlnet, num_ip_layers=fp8_num_ip_layers, ) - if not _run_fp8_stage("fp8_calib_capture", _calib_fn, stats, fp8_allow_fp16_fallback, engine_filename): - fp8 = False - elif fp8 and pipe_ref is None: - _build_logger.warning( - "[BUILD] fp8=True but pipe_ref not provided — FP8 calibration skipped. " - "Pass pipe_ref in engine_build_options for proper activation capture." - ) - fp8 = False + if not _run_fp8_stage("fp8_onnx_quantize", _quant_fn, stats, fp8_allow_fp16_fallback, engine_filename): + fp8 = False - # --- FP8: Inject native FLOAT8E4M3FN Q/DQ into the ONNX (cached in unet.fp8.onnx) --- - if fp8: - if os.path.exists(_fp8_onnx_path + ".ok"): - _build_logger.info(f"[BUILD] FP8 ONNX cached: {_fp8_onnx_path}") - stats["stages"]["fp8_onnx_quantize"] = {"status": StageStatus.CACHED} - else: + # Select the ONNX to feed into TRT: FP8-quantized when available, else plain opt. + _trt_onnx_path = _fp8_onnx_path if (fp8 and os.path.exists(_fp8_onnx_path + ".ok")) else onnx_opt_path - def _quant_fn(): - from .fp8_quantize import load_calibration_data, quantize_onnx_fp8 - - calib_data = load_calibration_data(_calib_data_path) - if calib_data is None: - raise RuntimeError(f"Calibration data missing after capture step: {_calib_data_path}") - quantize_onnx_fp8( - onnx_path=onnx_opt_path, - output_path=_fp8_onnx_path, - calibration_data=calib_data, - use_cached_attn=fp8_use_cached_attn, - use_feature_injection=fp8_use_feature_injection, - use_controlnet=fp8_use_controlnet, - num_ip_layers=fp8_num_ip_layers, - ) - - if not _run_fp8_stage("fp8_onnx_quantize", _quant_fn, stats, fp8_allow_fp16_fallback, engine_filename): - fp8 = False - - # Select the ONNX to feed into TRT: FP8-quantized when available, else plain opt. - _trt_onnx_path = _fp8_onnx_path if (fp8 and os.path.exists(_fp8_onnx_path + ".ok")) else onnx_opt_path - - # --- TRT Engine Build --- - if not force_engine_build and os.path.exists(engine_path): - print(f"Found cached engine: {engine_path}") - stats["stages"]["trt_build"] = {"status": "cached"} - else: - t0 = time.perf_counter() - build_engine( - engine_path=engine_path, - onnx_opt_path=_trt_onnx_path, - model_data=self.model, - opt_image_height=opt_image_height, - opt_image_width=opt_image_width, - opt_batch_size=opt_batch_size, - build_static_batch=build_static_batch, - build_dynamic_shape=build_dynamic_shape, - build_all_tactics=build_all_tactics, - build_enable_refit=build_enable_refit, - fp8=fp8, - builder_optimization_level=builder_optimization_level, - ) - elapsed = time.perf_counter() - t0 - stats["stages"]["trt_build"] = {"status": "built", "elapsed_s": round(elapsed, 2)} - _build_logger.info(f"[BUILD] TRT engine build ({engine_filename}): {elapsed:.1f}s") - - # --- FP8 Q/DQ layer count (sanity gate: < 500 means quantization is inactive) --- - if fp8 and os.path.exists(engine_path): - try: - import json as _json - import re as _re - - import tensorrt as trt - - _rt = trt.Runtime(BUILD_TRT_LOGGER) - with open(engine_path, "rb") as _f: - _eng = _rt.deserialize_cuda_engine(_f.read()) - _insp = _eng.create_engine_inspector() - _info = _insp.get_engine_information(trt.LayerInformationFormat.JSON) - _qdq = _info.count("QuantizeLinear") + _info.count("DequantizeLinear") - stats["fp8_qdq_layers"] = _qdq - _build_logger.info(f"[BUILD] FP8 engine Q/DQ layer count: {_qdq}") - if _qdq < 500: - _build_logger.warning( - f"[BUILD] Low Q/DQ count ({_qdq} < 500) — FP8 quantization likely inactive or incomplete" - ) - - # Fused-MHA check: count attention layers TRT fused into a single kernel. - # Pattern is empirical — FLUX uses "_gemm_mha_v2"; SDXL on Ada may differ. - # First build logs sample names so the regex can be confirmed or tightened. - _MHA_RE = _re.compile(r"mha|fmha|MultiHead|FlashAttn", _re.IGNORECASE) + # --- TRT Engine Build --- + if not force_engine_build and os.path.exists(engine_path): + print(f"Found cached engine: {engine_path}") + stats["stages"]["trt_build"] = {"status": "cached"} + else: + t0 = time.perf_counter() + build_engine( + engine_path=engine_path, + onnx_opt_path=_trt_onnx_path, + model_data=self.model, + opt_image_height=opt_image_height, + opt_image_width=opt_image_width, + opt_batch_size=opt_batch_size, + build_static_batch=build_static_batch, + build_dynamic_shape=build_dynamic_shape, + build_all_tactics=build_all_tactics, + build_enable_refit=build_enable_refit, + fp8=fp8, + builder_optimization_level=builder_optimization_level, + ) + elapsed = time.perf_counter() - t0 + stats["stages"]["trt_build"] = {"status": "built", "elapsed_s": round(elapsed, 2)} + _build_logger.info(f"[BUILD] TRT engine build ({engine_filename}): {elapsed:.1f}s") + + # --- FP8 Q/DQ layer count (sanity gate: < 500 means quantization is inactive) --- + if fp8 and os.path.exists(engine_path): try: - _layers = _json.loads(_info).get("Layers", []) - except Exception: - _layers = [] - _total = len(_layers) - _mha_names = [_l.get("Name", "") for _l in _layers if _MHA_RE.search(_l.get("Name", ""))] - _mha_count = len(_mha_names) - stats["mha_fused_kernels"] = _mha_count - stats["total_engine_layers"] = _total - _build_logger.info(f"[BUILD] FP8 engine fused MHA layers: {_mha_count} / {_total} total") - if _mha_count == 0 and _total > 0: - _build_logger.warning( - "[BUILD] No fused MHA layers detected — attention may be running decomposed " - "(slower). Sample layer names (first 5): " + str([_l.get("Name", "") for _l in _layers[:5]]) - ) - else: - _build_logger.info(f"[BUILD] Sample fused-MHA layer names: {_mha_names[:3]}") - except Exception as _e: - _build_logger.warning(f"[BUILD] FP8 inspector check skipped: {_e}") + import json as _json + import re as _re + + import tensorrt as trt + + _rt = trt.Runtime(BUILD_TRT_LOGGER) + with open(engine_path, "rb") as _f: + _eng = _rt.deserialize_cuda_engine(_f.read()) + _insp = _eng.create_engine_inspector() + _info = _insp.get_engine_information(trt.LayerInformationFormat.JSON) + _qdq = _info.count("QuantizeLinear") + _info.count("DequantizeLinear") + stats["fp8_qdq_layers"] = _qdq + _build_logger.info(f"[BUILD] FP8 engine Q/DQ layer count: {_qdq}") + if _qdq < 500: + _build_logger.warning( + f"[BUILD] Low Q/DQ count ({_qdq} < 500) — FP8 quantization likely inactive or incomplete" + ) - # Record totals (before cleanup so build_stats.json is preserved) + # Fused-MHA check: count attention layers TRT fused into a single kernel. + # Pattern is empirical — FLUX uses "_gemm_mha_v2"; SDXL on Ada may differ. + # First build logs sample names so the regex can be confirmed or tightened. + _MHA_RE = _re.compile(r"mha|fmha|MultiHead|FlashAttn", _re.IGNORECASE) + try: + _layers = _json.loads(_info).get("Layers", []) + except Exception: + _layers = [] + _total = len(_layers) + _mha_names = [_l.get("Name", "") for _l in _layers if _MHA_RE.search(_l.get("Name", ""))] + _mha_count = len(_mha_names) + stats["mha_fused_kernels"] = _mha_count + stats["total_engine_layers"] = _total + _build_logger.info(f"[BUILD] FP8 engine fused MHA layers: {_mha_count} / {_total} total") + if _mha_count == 0 and _total > 0: + _build_logger.warning( + "[BUILD] No fused MHA layers detected — attention may be running decomposed " + "(slower). Sample layer names (first 5): " + str([_l.get("Name", "") for _l in _layers[:5]]) + ) + else: + _build_logger.info(f"[BUILD] Sample fused-MHA layer names: {_mha_names[:3]}") + except Exception as _e: + _build_logger.warning(f"[BUILD] FP8 inspector check skipped: {_e}") + finally: + _fp8_ok = fp8 and os.path.exists(_fp8_onnx_path + ".ok") + _cleanup_intermediates(engine_dir_early, _fp8_ok) + + # Cleanup already ran in the `finally` above (also covers the failure path). total_elapsed = time.perf_counter() - build_total_start stats["total_elapsed_s"] = round(total_elapsed, 2) stats["build_end"] = datetime.now(timezone.utc).isoformat() @@ -362,58 +466,3 @@ def _quant_fn(): _build_logger.info(f"[BUILD] {engine_filename} complete: {total_elapsed:.1f}s total") _write_build_stats(engine_path, stats) - - # Cleanup ONNX artifacts — preserve .engine, calib_data.npz, unet.fp8.onnx* files, timing.cache, build_stats.json - # Two-pass deletion to handle Windows file locks (gc.collect releases Python handles) - _keep_suffixes = (".engine", ".cache") - _keep_exact = {"build_stats.json", "timing.cache", "calib_data.npz"} - engine_dir = os.path.dirname(engine_path) - _to_delete = [] - for file in os.listdir(engine_dir): - # Keep all files that are part of the FP8 quantized ONNX artifact - # (unet.fp8.onnx and any external data companion like unet.fp8.onnx.data) - if "fp8.onnx" in file: - continue - if file in _keep_exact or any(file.endswith(s) for s in _keep_suffixes): - continue - _to_delete.append(os.path.join(engine_dir, file)) - - if _to_delete: - _failed = [] - for fpath in _to_delete: - try: - os.remove(fpath) - except OSError: - _failed.append(fpath) - - # Release Python-held file handles (ONNX model refs), retry locked files. - # Per-file poll with 50ms backoff instead of a single global sleep — most - # handles release within 1-2 retries on Windows; worst case ~0.5s same as before. - if _failed: - gc.collect() - torch.cuda.empty_cache() - _still_failed = [] - for fpath in _failed: - _last_err = None - for _attempt in range(10): - try: - os.remove(fpath) - _last_err = None - break - except OSError as _e: - _last_err = _e - time.sleep(0.05) - if _last_err is not None: - _still_failed.append(os.path.basename(fpath)) - _build_logger.warning( - f"[BUILD] Could not delete temp file {os.path.basename(fpath)}: {_last_err}" - ) - if _still_failed: - _build_logger.warning( - f"[BUILD] {len(_still_failed)} intermediate files could not be cleaned. " - f"Manual cleanup: delete all files except *.engine, calib_data.npz, unet.fp8.onnx from {engine_dir}" - ) - cleaned = len(_to_delete) - len(_still_failed) - else: - cleaned = len(_to_delete) - _build_logger.info(f"[BUILD] Cleaned {cleaned}/{len(_to_delete)} intermediate files") From e1be6165f82353f12c44f3b4de17494f36fb26c4 Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 11 Jul 2026 19:29:08 -0400 Subject: [PATCH 23/37] style: apply ruff format to builder.py (CGW restage gap left prior commit unformatted) --- .../acceleration/tensorrt/builder.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/streamdiffusion/acceleration/tensorrt/builder.py b/src/streamdiffusion/acceleration/tensorrt/builder.py index 97480fa2f..1e767aba4 100644 --- a/src/streamdiffusion/acceleration/tensorrt/builder.py +++ b/src/streamdiffusion/acceleration/tensorrt/builder.py @@ -154,9 +154,7 @@ def _cleanup_intermediates(engine_dir: str, fp8_ok: bool): time.sleep(0.05) if _last_err is not None: _still_failed.append(os.path.basename(fpath)) - _build_logger.warning( - f"[BUILD] Could not delete temp file {os.path.basename(fpath)}: {_last_err}" - ) + _build_logger.warning(f"[BUILD] Could not delete temp file {os.path.basename(fpath)}: {_last_err}") if _still_failed: _build_logger.warning( f"[BUILD] {len(_still_failed)} intermediate files could not be cleaned. " @@ -343,7 +341,9 @@ def _calib_fn(): num_ip_layers=fp8_num_ip_layers, ) - if not _run_fp8_stage("fp8_calib_capture", _calib_fn, stats, fp8_allow_fp16_fallback, engine_filename): + if not _run_fp8_stage( + "fp8_calib_capture", _calib_fn, stats, fp8_allow_fp16_fallback, engine_filename + ): fp8 = False elif fp8 and pipe_ref is None: _build_logger.warning( @@ -377,7 +377,9 @@ def _quant_fn(): num_ip_layers=fp8_num_ip_layers, ) - if not _run_fp8_stage("fp8_onnx_quantize", _quant_fn, stats, fp8_allow_fp16_fallback, engine_filename): + if not _run_fp8_stage( + "fp8_onnx_quantize", _quant_fn, stats, fp8_allow_fp16_fallback, engine_filename + ): fp8 = False # Select the ONNX to feed into TRT: FP8-quantized when available, else plain opt. @@ -445,7 +447,8 @@ def _quant_fn(): if _mha_count == 0 and _total > 0: _build_logger.warning( "[BUILD] No fused MHA layers detected — attention may be running decomposed " - "(slower). Sample layer names (first 5): " + str([_l.get("Name", "") for _l in _layers[:5]]) + "(slower). Sample layer names (first 5): " + + str([_l.get("Name", "") for _l in _layers[:5]]) ) else: _build_logger.info(f"[BUILD] Sample fused-MHA layer names: {_mha_names[:3]}") From e417a4e586a43ad07751a7e81bea927e8e016809 Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 12 Jul 2026 01:22:31 -0400 Subject: [PATCH 24/37] refactor: centralize wrapper/updater params in param_schema (single source of truth) - add src/streamdiffusion/param_schema.py: 25 wrapper / 23 updater param names, construction-time DEFAULTS (t_index_list as immutable tuple, G3), and shared floor_num_inference_steps / rescale_t_index_list helpers - wrapper.py & stream_parameter_updater.py: source interpolation-method Literals and floor/rescale arithmetic from the schema; preserve branch-specific warnings (G2); fix D1 comment off-by-one - config.py: delegate 12 defaults across both _extract_wrapper_params and _extract_prepare_params to param_schema.DEFAULTS (G1); drop dead _validate_pipeline_hook_configs (zero callers) - add drift-lock + golden/parity tests (test_param_schema.py, test_config_extraction_golden.py, test_td_pending_params.py); tests/unit/: 165 passed --- src/streamdiffusion/config.py | 91 ++--------- src/streamdiffusion/param_schema.py | 136 ++++++++++++++++ .../stream_parameter_updater.py | 29 ++-- src/streamdiffusion/wrapper.py | 5 +- tests/unit/test_config_extraction_golden.py | 102 ++++++++++++ tests/unit/test_param_schema.py | 151 ++++++++++++++++++ tests/unit/test_td_pending_params.py | 79 ++++++--- 7 files changed, 482 insertions(+), 111 deletions(-) create mode 100644 src/streamdiffusion/param_schema.py create mode 100644 tests/unit/test_config_extraction_golden.py create mode 100644 tests/unit/test_param_schema.py diff --git a/src/streamdiffusion/config.py b/src/streamdiffusion/config.py index 576f823f0..bd97afcf6 100644 --- a/src/streamdiffusion/config.py +++ b/src/streamdiffusion/config.py @@ -5,6 +5,8 @@ import yaml +from .param_schema import DEFAULTS + logger = logging.getLogger(__name__) @@ -104,7 +106,7 @@ def _extract_wrapper_params(config: Dict[str, Any]) -> Dict[str, Any]: """Extract parameters for StreamDiffusionWrapper.__init__() from config""" param_map = { "model_id_or_path": config.get("model_id", "stabilityai/sd-turbo"), - "t_index_list": config.get("t_index_list", [0, 16, 32, 45]), + "t_index_list": config.get("t_index_list", list(DEFAULTS["t_index_list"])), "lora_dict": config.get("lora_dict"), "mode": config.get("mode", "img2img"), "output_type": config.get("output_type", "pil"), @@ -126,12 +128,12 @@ def _extract_wrapper_params(config: Dict[str, Any]) -> Dict[str, Any]: "similar_filter_sleep_fraction": config.get("similar_filter_sleep_fraction", 0.025), "use_denoising_batch": config.get("use_denoising_batch", True), "cfg_type": config.get("cfg_type", "self"), - "seed": config.get("seed", 2), - "use_safety_checker": config.get("use_safety_checker", False), + "seed": config.get("seed", DEFAULTS["seed"]), + "use_safety_checker": config.get("use_safety_checker", DEFAULTS["use_safety_checker"]), "skip_diffusion": config.get("skip_diffusion", False), "engine_dir": config.get("engine_dir", "engines"), - "normalize_prompt_weights": config.get("normalize_prompt_weights", True), - "normalize_seed_weights": config.get("normalize_seed_weights", True), + "normalize_prompt_weights": config.get("normalize_prompt_weights", DEFAULTS["normalize_prompt_weights"]), + "normalize_seed_weights": config.get("normalize_seed_weights", DEFAULTS["normalize_seed_weights"]), "scheduler": config.get("scheduler", "lcm"), "sampler": config.get("sampler", "normal"), "compile_engines_only": config.get("compile_engines_only", False), @@ -159,12 +161,12 @@ def _extract_wrapper_params(config: Dict[str, Any]) -> Dict[str, Any]: param_map["use_cached_attn"] = config.get("use_cached_attn", False) - param_map["cache_maxframes"] = config.get("cache_maxframes", 1) - param_map["cache_interval"] = config.get("cache_interval", 1) + param_map["cache_maxframes"] = config.get("cache_maxframes", DEFAULTS["cache_maxframes"]) + param_map["cache_interval"] = config.get("cache_interval", DEFAULTS["cache_interval"]) # cn_cache_interval: ControlNet residual reuse interval. # 1 (default) = disabled, run CN every frame. # N > 1 = run CN once every N frames; reuse residuals between (control latency = N-1 frames). - param_map["cn_cache_interval"] = config.get("cn_cache_interval", 1) + param_map["cn_cache_interval"] = config.get("cn_cache_interval", DEFAULTS["cn_cache_interval"]) # Feature Injection (StreamV2V §3.4.2) — requires use_cached_attn=True if "use_feature_injection" not in config: @@ -176,8 +178,8 @@ def _extract_wrapper_params(config: Dict[str, Any]) -> Dict[str, Any]: param_map["use_feature_injection"] = config.get("use_feature_injection", True) # fi_strength: blend weight α (thesis §3.4.2 Eq 3.2 α=0.75; default matches thesis). # fi_threshold: cosine-similarity gate below which injection is skipped (default 0.98). - param_map["fi_strength"] = config.get("fi_strength", 0.75) - param_map["fi_threshold"] = config.get("fi_threshold", 0.98) + param_map["fi_strength"] = config.get("fi_strength", DEFAULTS["fi_strength"]) + param_map["fi_threshold"] = config.get("fi_threshold", DEFAULTS["fi_threshold"]) # max_cache_maxframes: allocation cap for the KVO/FI cache ring buffers (VRAM). # cache_maxframes is the live logical write window; this is the hard upper bound. param_map["max_cache_maxframes"] = config.get("max_cache_maxframes", 4) @@ -204,10 +206,10 @@ def _extract_prepare_params(config: Dict[str, Any]) -> Dict[str, Any]: """Extract parameters for wrapper.prepare() from config""" prepare_params = { "prompt": config.get("prompt", ""), - "negative_prompt": config.get("negative_prompt", ""), - "num_inference_steps": config.get("num_inference_steps", 50), - "guidance_scale": config.get("guidance_scale", 1.2), - "delta": config.get("delta", 1.0), + "negative_prompt": config.get("negative_prompt", DEFAULTS["negative_prompt"]), + "num_inference_steps": config.get("num_inference_steps", DEFAULTS["num_inference_steps"]), + "guidance_scale": config.get("guidance_scale", DEFAULTS["guidance_scale"]), + "delta": config.get("delta", DEFAULTS["delta"]), } # Handle prompt blending configuration @@ -333,67 +335,6 @@ def _prepare_single_hook_config(hook_config: Dict[str, Any], hook_type: str) -> } -def _validate_pipeline_hook_configs(config: Dict[str, Any]) -> None: - """Validate pipeline hook configurations following ControlNet/IPAdapter validation pattern""" - hook_types = ["image_preprocessing", "image_postprocessing", "latent_preprocessing", "latent_postprocessing"] - - for hook_type in hook_types: - if hook_type in config: - hook_config = config[hook_type] - if not isinstance(hook_config, dict): - raise ValueError(f"_validate_config: '{hook_type}' must be a dictionary") - - # Validate enabled field - if "enabled" in hook_config: - enabled = hook_config["enabled"] - if not isinstance(enabled, bool): - raise ValueError(f"_validate_config: '{hook_type}.enabled' must be a boolean") - - # Validate processors field - if "processors" in hook_config: - processors = hook_config["processors"] - if not isinstance(processors, list): - raise ValueError(f"_validate_config: '{hook_type}.processors' must be a list") - - for i, processor in enumerate(processors): - if not isinstance(processor, dict): - raise ValueError(f"_validate_config: '{hook_type}.processors[{i}]' must be a dictionary") - - # Validate processor type (required) - if "type" not in processor: - raise ValueError( - f"_validate_config: '{hook_type}.processors[{i}]' missing required 'type' field" - ) - - if not isinstance(processor["type"], str): - raise ValueError(f"_validate_config: '{hook_type}.processors[{i}].type' must be a string") - - # Validate enabled field (optional, defaults to True) - if "enabled" in processor: - enabled = processor["enabled"] - if not isinstance(enabled, bool): - raise ValueError( - f"_validate_config: '{hook_type}.processors[{i}].enabled' must be a boolean" - ) - - # Validate order field (optional) - if "order" in processor: - order = processor["order"] - if not isinstance(order, int): - raise ValueError( - f"_validate_config: '{hook_type}.processors[{i}].order' must be an integer" - ) - - # Validate params field (optional, coerce None to empty dict) - if "params" in processor: - if processor["params"] is None: - processor["params"] = {} - elif not isinstance(processor["params"], dict): - raise ValueError( - f"_validate_config: '{hook_type}.processors[{i}].params' must be a dictionary" - ) - - def create_prompt_blending_config( base_config: Dict[str, Any], prompt_list: List[Tuple[str, float]], diff --git a/src/streamdiffusion/param_schema.py b/src/streamdiffusion/param_schema.py new file mode 100644 index 000000000..6ecaa222a --- /dev/null +++ b/src/streamdiffusion/param_schema.py @@ -0,0 +1,136 @@ +"""Single source of truth for the runtime-tunable StreamDiffusion parameter set. + +This module owns parameter *identity* — the 25 names accepted by +:meth:`StreamDiffusionWrapper.update_stream_params`, which of those 23 are +forwarded to :meth:`StreamParameterUpdater.update_stream_params`, and each +param's *construction-time* default (the value a fresh wrapper gets when the +key is absent from config — see ``config._extract_wrapper_params`` and +``config._extract_prepare_params``). + +It deliberately does **not** own order-dependent *apply* logic — the real +``update_stream_params`` implementations stay hand-written in +``wrapper.py`` / ``stream_parameter_updater.py``. A signature-parity test +(``tests/unit/test_param_schema.py``) asserts those signatures stay in sync +with ``PARAM_NAMES`` / ``UPDATER_PARAM_NAMES``, so drift is *caught*, not +*prevented*. + +Note on ``default`` vs. the runtime signatures: every parameter in +``update_stream_params`` defaults to ``None`` at the call-site (meaning +"leave the current value unchanged"), except the two interpolation-method +Literals. ``ParamSpec.default`` here is a *different* concept — the +concrete construction-time value — and intentionally does not mirror the +``None`` sentinels. + +This module has no torch import and no dependency on the rest of the +``streamdiffusion`` package, so it stays cheap to import in isolation +(e.g. from a lightweight test or tool). In practice ``streamdiffusion``'s +own ``__init__.py`` eagerly imports ``.pipeline``/``.wrapper`` (torch-heavy) +before this module would ever be reached via ``from streamdiffusion...``, +so the "cheap import" property is good hygiene rather than a load-time win +for current consumers. +""" + +from dataclasses import dataclass +from typing import Any, Dict, List, Literal, Tuple + + +# Interpolation-method aliases shared by the wrapper and updater signatures. +PromptInterpolationMethod = Literal["linear", "slerp", "cosine_weighted"] +SeedInterpolationMethod = Literal["linear", "slerp"] + + +@dataclass(frozen=True) +class ParamSpec: + """Identity of one runtime-tunable parameter. + + Attributes + ---------- + name: + Keyword name, shared verbatim by the wrapper and (when ``updater`` + is True) the updater signatures. + default: + Construction-time default (see module docstring) — NOT the + ``update_stream_params`` runtime default, which is ``None`` for + all but the two interpolation-method params. + updater: + Whether this param is forwarded to + ``StreamParameterUpdater.update_stream_params``. False only for + ``use_safety_checker`` / ``safety_checker_threshold``, which the + wrapper handles itself (wrapper.py update_stream_params tail). + """ + + name: str + default: Any + updater: bool = True + + +# Order matches StreamDiffusionWrapper.update_stream_params exactly +# (wrapper.py:681-712). +PARAMS: Tuple[ParamSpec, ...] = ( + ParamSpec("num_inference_steps", 50), + ParamSpec("guidance_scale", 1.2), + ParamSpec("delta", 1.0), + # Stored as a tuple so DEFAULTS never hands out a mutable list that a + # caller could alias-mutate; consumers that need a list should do + # list(DEFAULTS["t_index_list"]). + ParamSpec("t_index_list", (0, 16, 32, 45)), + ParamSpec("seed", 2), + ParamSpec("prompt_list", None), + ParamSpec("negative_prompt", ""), + ParamSpec("prompt_interpolation_method", "slerp"), + ParamSpec("normalize_prompt_weights", True), + ParamSpec("seed_list", None), + ParamSpec("seed_interpolation_method", "linear"), + ParamSpec("normalize_seed_weights", True), + ParamSpec("controlnet_config", None), + ParamSpec("ipadapter_config", None), + ParamSpec("image_preprocessing_config", None), + ParamSpec("image_postprocessing_config", None), + ParamSpec("latent_preprocessing_config", None), + ParamSpec("latent_postprocessing_config", None), + ParamSpec("use_safety_checker", False, updater=False), + ParamSpec("safety_checker_threshold", 0.5, updater=False), + ParamSpec("cache_maxframes", 1), + ParamSpec("cache_interval", 1), + ParamSpec("cn_cache_interval", 1), + ParamSpec("fi_strength", 0.75), + ParamSpec("fi_threshold", 0.98), +) + +# All 25 params the wrapper accepts, in wrapper signature order. +PARAM_NAMES: Tuple[str, ...] = tuple(p.name for p in PARAMS) + +# The 23 params forwarded to the updater, in updater signature order +# (a contiguous subsequence of PARAM_NAMES once use_safety_checker / +# safety_checker_threshold are removed). +UPDATER_PARAM_NAMES: Tuple[str, ...] = tuple(p.name for p in PARAMS if p.updater) + +# name -> construction-time default. +DEFAULTS: Dict[str, Any] = {p.name: p.default for p in PARAMS} + + +def floor_num_inference_steps(num_inference_steps: int, max_t_index: int) -> int: + """Raise ``num_inference_steps`` to ``max_t_index + 1`` if it's too small to + hold the largest t_index value. Never lowers it. + + Extracted 1:1 from stream_parameter_updater.py's two + ``if num_inference_steps <= max_t_index`` branches (~:328-348); callers + keep their own branch-specific warning text — this helper only owns the + arithmetic. + """ + return max(num_inference_steps, max_t_index + 1) + + +def rescale_t_index_list(old_t_list: List[int], old_num_steps: int, new_num_steps: int) -> List[int]: + """Proportionally rescale t_index values from an old step-count space to a + new one, clamped to the new space's valid range. + + Extracted 1:1 from stream_parameter_updater.py:358-359. + + Example: rescale_t_index_list([0, 16, 32, 45], 50, 9) == [0, 3, 5, 7]. + (Not [0, 3, 6, 8] — that is the result for new_num_steps=10. The source + comment this was extracted from had the same off-by-one; see + tests/unit/test_param_schema.py for the corrected golden.) + """ + scale_factor = (new_num_steps - 1) / (old_num_steps - 1) if old_num_steps > 1 else 1.0 + return [min(round(t * scale_factor), new_num_steps - 1) for t in old_t_list] diff --git a/src/streamdiffusion/stream_parameter_updater.py b/src/streamdiffusion/stream_parameter_updater.py index 3e5ee18a1..42b9c20da 100644 --- a/src/streamdiffusion/stream_parameter_updater.py +++ b/src/streamdiffusion/stream_parameter_updater.py @@ -5,6 +5,12 @@ import torch import torch.nn.functional as F +from .param_schema import ( + PromptInterpolationMethod, + SeedInterpolationMethod, + floor_num_inference_steps, + rescale_t_index_list, +) from .preprocessing.orchestrator_user import OrchestratorUser @@ -282,10 +288,10 @@ def update_stream_params( seed: Optional[int] = None, prompt_list: Optional[List[Tuple[str, float]]] = None, negative_prompt: Optional[str] = None, - prompt_interpolation_method: Literal["linear", "slerp", "cosine_weighted"] = "slerp", + prompt_interpolation_method: PromptInterpolationMethod = "slerp", normalize_prompt_weights: Optional[bool] = None, seed_list: Optional[List[Tuple[int, float]]] = None, - seed_interpolation_method: Literal["linear", "slerp"] = "linear", + seed_interpolation_method: SeedInterpolationMethod = "linear", normalize_seed_weights: Optional[bool] = None, controlnet_config: Optional[List[Dict[str, Any]]] = None, ipadapter_config: Optional[Dict[str, Any]] = None, @@ -331,21 +337,23 @@ def update_stream_params( if t_index_list is None: # Check against current t_list max_t_index = max(self.stream.t_list) if self.stream.t_list else 0 - if num_inference_steps <= max_t_index: + floored = floor_num_inference_steps(num_inference_steps, max_t_index) + if floored != num_inference_steps: logger.warning( f"update_stream_params: num_inference_steps ({num_inference_steps}) is too small for " - f"current t_list (max index: {max_t_index}). Adjusting to {max_t_index + 1}." + f"current t_list (max index: {max_t_index}). Adjusting to {floored}." ) - num_inference_steps = max_t_index + 1 + num_inference_steps = floored else: # Check against provided t_index_list max_t_index = max(t_index_list) if t_index_list else 0 - if num_inference_steps <= max_t_index: + floored = floor_num_inference_steps(num_inference_steps, max_t_index) + if floored != num_inference_steps: logger.warning( f"update_stream_params: num_inference_steps ({num_inference_steps}) is too small for " - f"provided t_index_list (max index: {max_t_index}). Adjusting to {max_t_index + 1}." + f"provided t_index_list (max index: {max_t_index}). Adjusting to {floored}." ) - num_inference_steps = max_t_index + 1 + num_inference_steps = floored old_num_steps = len(self.stream.timesteps) self.stream.scheduler.set_timesteps(num_inference_steps, self.stream.device) @@ -354,9 +362,8 @@ def update_stream_params( # If t_index_list wasn't explicitly provided, rescale existing t_list proportionally if t_index_list is None and old_num_steps > 0: # Rescale each index proportionally to the new number of steps - # e.g., if t_list = [0, 16, 32, 45] with 50 steps -> [0, 3, 6, 8] with 9 steps - scale_factor = (num_inference_steps - 1) / (old_num_steps - 1) if old_num_steps > 1 else 1.0 - t_index_list = [min(round(t * scale_factor), num_inference_steps - 1) for t in self.stream.t_list] + # e.g., if t_list = [0, 16, 32, 45] with 50 steps -> [0, 3, 5, 7] with 9 steps + t_index_list = rescale_t_index_list(self.stream.t_list, old_num_steps, num_inference_steps) # Now update timestep-dependent parameters with the correct t_index_list if t_index_list is not None: diff --git a/src/streamdiffusion/wrapper.py b/src/streamdiffusion/wrapper.py index 17055f54c..b09d76e2a 100644 --- a/src/streamdiffusion/wrapper.py +++ b/src/streamdiffusion/wrapper.py @@ -10,6 +10,7 @@ from .image_utils import postprocess_image from .model_detection import detect_model +from .param_schema import PromptInterpolationMethod, SeedInterpolationMethod from .pipeline import StreamDiffusion from .tools.gpu_profiler import configure as _configure_profiler from .tools.gpu_profiler import profiler @@ -686,11 +687,11 @@ def update_stream_params( # Prompt blending parameters prompt_list: Optional[List[Tuple[str, float]]] = None, negative_prompt: Optional[str] = None, - prompt_interpolation_method: Literal["linear", "slerp", "cosine_weighted"] = "slerp", + prompt_interpolation_method: PromptInterpolationMethod = "slerp", normalize_prompt_weights: Optional[bool] = None, # Seed blending parameters seed_list: Optional[List[Tuple[int, float]]] = None, - seed_interpolation_method: Literal["linear", "slerp"] = "linear", + seed_interpolation_method: SeedInterpolationMethod = "linear", normalize_seed_weights: Optional[bool] = None, # ControlNet configuration controlnet_config: Optional[List[Dict[str, Any]]] = None, diff --git a/tests/unit/test_config_extraction_golden.py b/tests/unit/test_config_extraction_golden.py new file mode 100644 index 000000000..b4b8f749a --- /dev/null +++ b/tests/unit/test_config_extraction_golden.py @@ -0,0 +1,102 @@ +"""Golden-snapshot test for config.py's construction-time extraction functions. + +Captures the exact output of _extract_wrapper_params / _extract_prepare_params +for a minimal config, BEFORE Stage 2 Increment 3 delegates their literal +defaults to param_schema.DEFAULTS. If Inc 3 changes any default value (instead +of just its source), this test catches the divergence — the output must stay +byte-identical. + +t_index_list is asserted to remain a `list` (not param_schema.DEFAULTS' +internal tuple) since StreamDiffusionWrapper.__init__ types it List[int] and +downstream consumers (e.g. save_config/JSON) expect a list. +""" + +import torch + +from streamdiffusion.config import _extract_prepare_params, _extract_wrapper_params + + +MINIMAL_CONFIG = {"model_id": "stabilityai/sd-turbo"} + +EXPECTED_WRAPPER_PARAMS = { + "model_id_or_path": "stabilityai/sd-turbo", + "t_index_list": [0, 16, 32, 45], + "mode": "img2img", + "output_type": "pil", + "device": "cuda", + "dtype": torch.float16, + "frame_buffer_size": 1, + "width": 512, + "height": 512, + "warmup": 10, + "acceleration": "tensorrt", + "do_add_noise": True, + "use_tiny_vae": True, + "enable_similar_image_filter": False, + "similar_image_filter_threshold": 0.98, + "similar_image_filter_max_skip_frame": 10, + "similar_filter_sleep_fraction": 0.025, + "use_denoising_batch": True, + "cfg_type": "self", + "seed": 2, + "use_safety_checker": False, + "skip_diffusion": False, + "engine_dir": "engines", + "normalize_prompt_weights": True, + "normalize_seed_weights": True, + "scheduler": "lcm", + "sampler": "normal", + "compile_engines_only": False, + "static_shapes": False, + "fp8": False, + "vae_builder_optimization_level": 3, + "build_engines_if_missing": True, + "fp8_allow_fp16_fallback": False, + "use_controlnet": False, + "use_ipadapter": False, + "use_cached_attn": False, + "cache_maxframes": 1, + "cache_interval": 1, + "cn_cache_interval": 1, + "use_feature_injection": True, + "fi_strength": 0.75, + "fi_threshold": 0.98, + "max_cache_maxframes": 4, + "use_cuda_ipc_output": False, + "cuda_ipc_num_slots": 2, + "controlnet_preview_passthrough": False, + "debug_mode": False, +} + +EXPECTED_PREPARE_PARAMS = { + "prompt": "", + "negative_prompt": "", + "num_inference_steps": 50, + "guidance_scale": 1.2, + "delta": 1.0, +} + + +def test_extract_wrapper_params_minimal_config_byte_identical(): + result = _extract_wrapper_params(MINIMAL_CONFIG) + assert result == EXPECTED_WRAPPER_PARAMS + + +def test_extract_wrapper_params_t_index_list_is_a_list(): + """Must stay a `list`, not param_schema.DEFAULTS' internal immutable tuple.""" + result = _extract_wrapper_params(MINIMAL_CONFIG) + assert isinstance(result["t_index_list"], list) + + +def test_extract_wrapper_params_t_index_list_not_aliased_to_schema_default(): + """Mutating the returned list must not corrupt param_schema.DEFAULTS or a + second call's output (guards against a shared-mutable-default regression).""" + result = _extract_wrapper_params(MINIMAL_CONFIG) + result["t_index_list"].append(999) + second_result = _extract_wrapper_params(MINIMAL_CONFIG) + assert second_result["t_index_list"] == [0, 16, 32, 45] + + +def test_extract_prepare_params_minimal_config_byte_identical(): + result = _extract_prepare_params(MINIMAL_CONFIG) + assert result == EXPECTED_PREPARE_PARAMS diff --git a/tests/unit/test_param_schema.py b/tests/unit/test_param_schema.py new file mode 100644 index 000000000..cb5271044 --- /dev/null +++ b/tests/unit/test_param_schema.py @@ -0,0 +1,151 @@ +"""Unit tests for src/streamdiffusion/param_schema.py. + +Covers: + - PARAM_NAMES / UPDATER_PARAM_NAMES counts and ordering + - DEFAULTS golden values (construction-time defaults, not the + update_stream_params None-sentinel defaults) + - floor_num_inference_steps / rescale_t_index_list vs hand-computed values + - signature parity: PARAM_NAMES / UPDATER_PARAM_NAMES must match the real + StreamDiffusionWrapper.update_stream_params / StreamParameterUpdater. + update_stream_params signatures — this is the regression lock that + catches drift the moment either signature changes. + +CPU-only, no CUDA required (importing streamdiffusion pulls in torch, but +no tensor is ever created). +""" + +import inspect + +from streamdiffusion.param_schema import ( + DEFAULTS, + PARAM_NAMES, + UPDATER_PARAM_NAMES, + floor_num_inference_steps, + rescale_t_index_list, +) +from streamdiffusion.stream_parameter_updater import StreamParameterUpdater +from streamdiffusion.wrapper import StreamDiffusionWrapper + + +WRAPPER_ONLY_PARAMS = {"use_safety_checker", "safety_checker_threshold"} + + +class TestParamNames: + def test_param_names_count(self): + assert len(PARAM_NAMES) == 25 + + def test_updater_param_names_count(self): + assert len(UPDATER_PARAM_NAMES) == 23 + + def test_updater_param_names_is_ordered_subsequence_of_param_names(self): + """Dropping the two wrapper-only names from PARAM_NAMES, in place, + must yield exactly UPDATER_PARAM_NAMES (order preserved).""" + filtered = tuple(n for n in PARAM_NAMES if n not in WRAPPER_ONLY_PARAMS) + assert filtered == UPDATER_PARAM_NAMES + + def test_wrapper_only_params_excluded_from_updater(self): + assert not (WRAPPER_ONLY_PARAMS & set(UPDATER_PARAM_NAMES)) + assert WRAPPER_ONLY_PARAMS <= set(PARAM_NAMES) + + def test_no_duplicate_names(self): + assert len(PARAM_NAMES) == len(set(PARAM_NAMES)) + + +class TestSignatureParity: + """Regression lock: PARAM_NAMES / UPDATER_PARAM_NAMES must track the real + signatures. If either signature changes without updating param_schema.py, + these fail immediately.""" + + def test_wrapper_signature_matches_param_names(self): + sig = inspect.signature(StreamDiffusionWrapper.update_stream_params) + params = [name for name in sig.parameters if name != "self"] + assert tuple(params) == PARAM_NAMES + + def test_updater_signature_matches_updater_param_names(self): + sig = inspect.signature(StreamParameterUpdater.update_stream_params) + params = [name for name in sig.parameters if name != "self"] + assert tuple(params) == UPDATER_PARAM_NAMES + + +class TestDefaultsGolden: + """Spot-check construction-time defaults against the literals confirmed + identical in both config.py (_extract_wrapper_params / + _extract_prepare_params) and StreamDiffusionWrapper.__init__.""" + + def test_prepare_time_defaults(self): + assert DEFAULTS["num_inference_steps"] == 50 + assert DEFAULTS["guidance_scale"] == 1.2 + assert DEFAULTS["delta"] == 1.0 + + def test_t_index_list_default_and_immutability(self): + assert list(DEFAULTS["t_index_list"]) == [0, 16, 32, 45] + # Must not be a mutable list a caller could alias-mutate. + assert not isinstance(DEFAULTS["t_index_list"], list) + + def test_scalar_defaults(self): + assert DEFAULTS["seed"] == 2 + assert DEFAULTS["negative_prompt"] == "" + assert DEFAULTS["use_safety_checker"] is False + assert DEFAULTS["safety_checker_threshold"] == 0.5 + assert DEFAULTS["normalize_prompt_weights"] is True + assert DEFAULTS["normalize_seed_weights"] is True + assert DEFAULTS["cache_maxframes"] == 1 + assert DEFAULTS["cache_interval"] == 1 + assert DEFAULTS["cn_cache_interval"] == 1 + assert DEFAULTS["fi_strength"] == 0.75 + assert DEFAULTS["fi_threshold"] == 0.98 + + def test_interpolation_method_defaults(self): + assert DEFAULTS["prompt_interpolation_method"] == "slerp" + assert DEFAULTS["seed_interpolation_method"] == "linear" + + def test_config_only_params_default_none(self): + for name in ( + "prompt_list", + "seed_list", + "controlnet_config", + "ipadapter_config", + "image_preprocessing_config", + "image_postprocessing_config", + "latent_preprocessing_config", + "latent_postprocessing_config", + ): + assert DEFAULTS[name] is None + + +class TestFloorNumInferenceSteps: + def test_no_change_when_already_large_enough(self): + assert floor_num_inference_steps(50, 45) == 50 + + def test_raises_when_too_small(self): + assert floor_num_inference_steps(9, 45) == 46 + + def test_boundary_equal_to_max_t_index_is_too_small(self): + # Original code: `if num_inference_steps <= max_t_index: ... = max_t_index + 1` + assert floor_num_inference_steps(45, 45) == 46 + + def test_boundary_one_above_max_t_index_is_fine(self): + assert floor_num_inference_steps(46, 45) == 46 + + +class TestRescaleTIndexList: + def test_golden_50_to_9(self): + """Corrected golden — the code's scale_factor=(new-1)/(old-1) gives + [0,3,5,7] for 50->9, NOT [0,3,6,8] (that off-by-one was in the + original source comment at stream_parameter_updater.py:357 and was + copied into an earlier draft of this extraction).""" + assert rescale_t_index_list([0, 16, 32, 45], 50, 9) == [0, 3, 5, 7] + + def test_golden_50_to_10(self): + """[0,3,6,8] is the correct result for new_num_steps=10, not 9.""" + assert rescale_t_index_list([0, 16, 32, 45], 50, 10) == [0, 3, 6, 8] + + def test_single_old_step_no_division_by_zero(self): + assert rescale_t_index_list([0], 1, 9) == [0] + + def test_same_step_count_is_identity(self): + assert rescale_t_index_list([0, 16, 32, 45], 50, 50) == [0, 16, 32, 45] + + def test_result_clamped_to_new_range(self): + result = rescale_t_index_list([0, 49], 50, 5) + assert max(result) <= 4 diff --git a/tests/unit/test_td_pending_params.py b/tests/unit/test_td_pending_params.py index 4ea0e43b5..23bf4c6ae 100644 --- a/tests/unit/test_td_pending_params.py +++ b/tests/unit/test_td_pending_params.py @@ -14,10 +14,11 @@ """ import threading -import types import unittest from typing import Any, Dict, Optional +from streamdiffusion.param_schema import PARAM_NAMES + # --------------------------------------------------------------------------- # Minimal faithful replica of the three methods under test. @@ -25,12 +26,14 @@ # td_manager.py will break these tests and alert the developer. # --------------------------------------------------------------------------- + class _FakeStream: cfg_type = "none" class _FakeWrapper: """Records calls made to update_stream_params.""" + def __init__(self): self.stream = _FakeStream() self.calls: list = [] @@ -50,15 +53,31 @@ class _Manager: """ VALID_PARAMS = { - 'num_inference_steps', 'guidance_scale', 'delta', 't_index_list', 'seed', - 'prompt_list', 'negative_prompt', 'prompt_interpolation_method', - 'normalize_prompt_weights', 'seed_list', 'seed_interpolation_method', - 'normalize_seed_weights', 'controlnet_config', 'ipadapter_config', - 'image_preprocessing_config', 'image_postprocessing_config', - 'latent_preprocessing_config', 'latent_postprocessing_config', - 'use_safety_checker', 'safety_checker_threshold', - 'cache_maxframes', 'cache_interval', 'fi_strength', 'fi_threshold', - 'cn_cache_interval', + "num_inference_steps", + "guidance_scale", + "delta", + "t_index_list", + "seed", + "prompt_list", + "negative_prompt", + "prompt_interpolation_method", + "normalize_prompt_weights", + "seed_list", + "seed_interpolation_method", + "normalize_seed_weights", + "controlnet_config", + "ipadapter_config", + "image_preprocessing_config", + "image_postprocessing_config", + "latent_preprocessing_config", + "latent_postprocessing_config", + "use_safety_checker", + "safety_checker_threshold", + "cache_maxframes", + "cache_interval", + "fi_strength", + "fi_threshold", + "cn_cache_interval", } def __init__(self): @@ -71,33 +90,30 @@ def __init__(self): # --- Replica of td_manager.py _apply_parameters --- def _apply_parameters(self, params: Dict[str, Any]) -> None: - import logging, random + import random + filtered_params = {k: v for k, v in params.items() if k in self.VALID_PARAMS} - if 'guidance_scale' in filtered_params: - cfg_type = getattr(self.wrapper.stream, 'cfg_type', None) - if cfg_type in ("full", "initialize") and filtered_params['guidance_scale'] <= 1.0: - filtered_params['guidance_scale'] = 1.2 + if "guidance_scale" in filtered_params: + cfg_type = getattr(self.wrapper.stream, "cfg_type", None) + if cfg_type in ("full", "initialize") and filtered_params["guidance_scale"] <= 1.0: + filtered_params["guidance_scale"] = 1.2 - if 'seed_list' in filtered_params: + if "seed_list" in filtered_params: self._randomize_seed_indices = [] new_seed_list = [] - for idx, (seed, weight) in enumerate(filtered_params['seed_list']): + for idx, (seed, weight) in enumerate(filtered_params["seed_list"]): if seed == -1: self._randomize_seed_indices.append(idx) seed = random.randint(0, 2**32 - 1) new_seed_list.append((seed, weight)) - filtered_params['seed_list'] = new_seed_list + filtered_params["seed_list"] = new_seed_list self.wrapper.update_stream_params(**filtered_params) # --- Replica of td_manager.py update_parameters --- def update_parameters(self, params: Dict[str, Any]) -> None: - render_alive = ( - self.streaming - and self.stream_thread is not None - and self.stream_thread.is_alive() - ) + render_alive = self.streaming and self.stream_thread is not None and self.stream_thread.is_alive() if render_alive: with self._pending_params_lock: self._pending_params.update(params) @@ -134,6 +150,7 @@ def _stop_thread(t: threading.Thread): # Test cases # --------------------------------------------------------------------------- + class TestUpdateParametersDefer(unittest.TestCase): """ (a) update_parameters defers when streaming, applies directly when not. @@ -283,5 +300,21 @@ def test_invalid_keys_filtered_out(self): self.assertAlmostEqual(mgr.wrapper.calls[0]["guidance_scale"], 1.5) +class TestValidParamsMatchesSchema(unittest.TestCase): + """ + Drift lock (Stage 2 Increment 4): td_manager.py's runtime whitelist (now + `set(param_schema.PARAM_NAMES)` in the real module -- see + StreamDiffusionTD/td_manager.py::_apply_parameters) must stay exactly + the 25-name set param_schema.py owns. _Manager.VALID_PARAMS above is a + frozen replica of the *pre-refactor* literal list, kept here so this + file never needs to import the real td_manager.py (CUDA/TD deps -- see + module docstring). If param_schema.PARAM_NAMES ever adds/removes/renames + a param, this test fails and both td_manager.py mirrors need updating. + """ + + def test_schema_param_names_matches_frozen_replica(self): + self.assertEqual(set(PARAM_NAMES), _Manager.VALID_PARAMS) + + if __name__ == "__main__": unittest.main() From e81e119aadf5fe5f30feee420f20189e3bace3c0 Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 12 Jul 2026 01:58:00 -0400 Subject: [PATCH 25/37] style: adopt expanded ruff ruleset + safe auto-fixes repo-wide MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - pyproject.toml: expand ruff select to E,W,F,I,C4,B,UP,SIM,RUF,PERF,NPY; target py310; add extend-exclude + per-file-ignores; docstring-code-format; add [tool.pyrefly] (baseline=pyrefly-baseline.json, pytorch-efficiency-lints) - apply safe ruff check --fix + ruff format across the repo (mechanical) - ignore non-auto-fixable residual families pending incremental adoption (SIM/PERF/RUF-unicode/RUF012-013/B905/B00x/RUF006-009/RUF022/RUF046/UP008/NPY002) — documented inline - mark 25 pre-existing debt sites (F401 x2, F841 x21, E722 x2) under already-active rules with per-site noqa + TODO comments; zero behavior change --- demo/realtime-img2img/app_config.py | 4 +- demo/realtime-img2img/connection_manager.py | 1 - demo/realtime-img2img/img2img.py | 2 - demo/realtime-img2img/input_control.py | 2 +- demo/realtime-img2img/main.py | 15 +- .../routes/common/api_utils.py | 4 +- demo/realtime-img2img/routes/controlnet.py | 16 +- demo/realtime-img2img/routes/debug.py | 9 +- demo/realtime-img2img/routes/inference.py | 7 +- demo/realtime-img2img/routes/input_sources.py | 3 +- demo/realtime-img2img/routes/ipadapter.py | 1 - demo/realtime-img2img/routes/parameters.py | 3 +- .../realtime-img2img/routes/pipeline_hooks.py | 3 +- demo/realtime-img2img/routes/websocket.py | 1 - demo/realtime-img2img/util.py | 21 +- demo/realtime-img2img/utils/video_utils.py | 2 +- demo/realtime-txt2img/config.py | 1 - demo/realtime-txt2img/main.py | 2 - demo/vid2vid/app.py | 2 - examples/benchmark/ab_bench.py | 5 +- examples/benchmark/multi.py | 3 +- examples/benchmark/single.py | 1 - .../config/config_ipadapter_stream_test.py | 2 +- examples/config/config_video_test.py | 4 +- examples/img2img/multi.py | 2 - examples/img2img/single.py | 2 - examples/optimal-performance/multi.py | 2 - examples/optimal-performance/single.py | 1 - examples/screen/main.py | 2 - examples/txt2img/multi.py | 2 - examples/txt2img/single.py | 2 - examples/txt2img/spacing_compare.py | 4 +- examples/vid2vid/main.py | 2 - pyproject.toml | 60 ++++- scripts/profiling/profile_ncu.py | 1 - scripts/profiling/profile_nsys.py | 7 +- scripts/test_lora_sanity.py | 6 +- setup.py | 2 +- src/streamdiffusion/__init__.py | 7 +- src/streamdiffusion/_patches/__init__.py | 1 - .../_patches/diffusers_kvo_patch.py | 1 - .../_patches/hf_tracing_patches.py | 1 - .../acceleration/tensorrt/__init__.py | 1 - .../acceleration/tensorrt/builder.py | 1 - .../acceleration/tensorrt/engine_manager.py | 11 +- .../tensorrt/export_wrappers/__init__.py | 7 +- .../export_wrappers/unet_ipadapter_export.py | 4 +- .../export_wrappers/unet_sdxl_export.py | 6 +- .../export_wrappers/unet_unified_export.py | 3 +- .../acceleration/tensorrt/fp8_quantize.py | 4 +- .../tensorrt/runtime_engines/__init__.py | 3 +- .../runtime_engines/controlnet_engine.py | 1 - .../tensorrt/runtime_engines/unet_engine.py | 1 - .../acceleration/tensorrt/utilities.py | 5 +- src/streamdiffusion/config.py | 15 +- src/streamdiffusion/model_detection.py | 2 - src/streamdiffusion/modules/__init__.py | 1 - .../modules/controlnet_module.py | 19 +- .../modules/image_processing_module.py | 12 +- .../modules/ipadapter_module.py | 9 +- .../modules/latent_processing_module.py | 6 +- src/streamdiffusion/param_schema.py | 1 - src/streamdiffusion/pip_utils.py | 1 - src/streamdiffusion/pipeline.py | 1 - src/streamdiffusion/preprocessing/__init__.py | 7 +- .../preprocessing/base_orchestrator.py | 1 - .../preprocessing/orchestrator_user.py | 6 +- .../pipeline_preprocessing_orchestrator.py | 1 - .../postprocessing_orchestrator.py | 1 - .../preprocessing_orchestrator.py | 1 - .../preprocessing/processors/__init__.py | 109 ++++---- .../preprocessing/processors/base.py | 133 +++++----- .../preprocessing/processors/canny.py | 4 +- .../processors/category_params.py | 1 - .../preprocessing/processors/depth.py | 91 +++---- .../processors/depth_tensorrt.py | 38 ++- .../preprocessing/processors/feedback.py | 114 +++++---- .../preprocessing/processors/hed.py | 65 +++-- .../preprocessing/processors/hed_tensorrt.py | 1 - .../processors/ipadapter_embedding.py | 25 +- .../preprocessing/processors/lineart.py | 1 - .../processors/mediapipe_pose.py | 7 +- .../processors/mediapipe_segmentation.py | 1 - .../processors/normal_bae_tensorrt.py | 1 - .../preprocessing/processors/openpose.py | 3 +- .../preprocessing/processors/passthrough.py | 27 +- .../preprocessing/processors/pose_tensorrt.py | 79 +++--- .../processors/realesrgan_trt.py | 240 +++++++++--------- .../processors/scribble_tensorrt.py | 1 - .../preprocessing/processors/soft_edge.py | 187 +++++++------- .../processors/standard_lineart.py | 109 ++++---- .../processors/temporal_net_tensorrt.py | 6 +- .../preprocessing/processors/trt_base.py | 1 - .../stream_parameter_updater.py | 19 +- .../tools/compile_depth_anything_tensorrt.py | 3 +- .../tools/compile_raft_tensorrt.py | 132 +++++----- src/streamdiffusion/tools/cuda_l2_cache.py | 1 - src/streamdiffusion/tools/gpu_profiler.py | 43 ++-- src/streamdiffusion/utils/__init__.py | 1 - src/streamdiffusion/wrapper.py | 17 +- .../manual/smoke_self_build_preprocessors.py | 3 +- tests/quality/regenerate_golden.py | 2 - tests/quality/run_compare.py | 1 - tests/unit/test_cn_preprocessor_residency.py | 1 - tests/unit/test_config_extraction_golden.py | 1 - tests/unit/test_derived_tensor_sync.py | 57 +++-- tests/unit/test_ipc_producer_stream.py | 1 - tests/unit/test_l2tc_dynamic_shapes.py | 1 - tests/unit/test_normal_bae_fallback.py | 1 - tests/unit/test_param_schema.py | 3 +- tests/unit/test_param_updater_binding.py | 5 +- tests/unit/test_phase3_correctness.py | 1 - tests/unit/test_prompt_interpolation.py | 3 +- tests/unit/test_safety_checker.py | 1 - tests/unit/test_sync_free_output_5_2.py | 1 - tests/unit/test_td_pending_params.py | 1 - tests/unit/test_trt_atomic_engine_write.py | 1 - tests/unit/test_trt_engine_guards.py | 1 - tests/unit/test_wrapper_exception_hygiene.py | 1 - tests/unit/test_zero_copy_staging_5_6.py | 1 - tools/summarize_audit.py | 7 +- utils/viewer.py | 1 - 122 files changed, 920 insertions(+), 985 deletions(-) diff --git a/demo/realtime-img2img/app_config.py b/demo/realtime-img2img/app_config.py index 252a992a5..e29c5afe0 100644 --- a/demo/realtime-img2img/app_config.py +++ b/demo/realtime-img2img/app_config.py @@ -12,7 +12,7 @@ def load_controlnet_registry(): """Load ControlNet registry from config file""" try: registry_path = Path(__file__).parent / "controlnet_registry.yaml" - with open(registry_path, "r") as f: + with open(registry_path) as f: config_data = yaml.safe_load(f) # Extract the available_controlnets section @@ -27,7 +27,7 @@ def load_default_settings(): """Load default settings from YAML config file""" try: registry_path = Path(__file__).parent / "controlnet_registry.yaml" - with open(registry_path, "r") as f: + with open(registry_path) as f: config_data = yaml.safe_load(f) return config_data.get("defaults", {}) diff --git a/demo/realtime-img2img/connection_manager.py b/demo/realtime-img2img/connection_manager.py index e35c09df7..8683e46dc 100644 --- a/demo/realtime-img2img/connection_manager.py +++ b/demo/realtime-img2img/connection_manager.py @@ -7,7 +7,6 @@ from fastapi import WebSocket from starlette.websockets import WebSocketState - Connections = Dict[UUID, Dict[str, Union[WebSocket, asyncio.Queue]]] diff --git a/demo/realtime-img2img/img2img.py b/demo/realtime-img2img/img2img.py index a1e6ada8c..3a45b8145 100644 --- a/demo/realtime-img2img/img2img.py +++ b/demo/realtime-img2img/img2img.py @@ -2,7 +2,6 @@ import os import sys - sys.path.append( os.path.join( os.path.dirname(__file__), @@ -18,7 +17,6 @@ from PIL import Image from pydantic import BaseModel, Field - # Default values for pipeline parameters default_negative_prompt = "black and white, blurry, low resolution, pixelated, pixel art, low quality, low fidelity" diff --git a/demo/realtime-img2img/input_control.py b/demo/realtime-img2img/input_control.py index be1e3fe03..3814708fb 100644 --- a/demo/realtime-img2img/input_control.py +++ b/demo/realtime-img2img/input_control.py @@ -148,7 +148,7 @@ def _monitor_gamepad(self) -> None: finally: try: pygame.quit() - except: + except: # noqa: E722 # TODO: pre-existing, untouched by this refactor pass diff --git a/demo/realtime-img2img/main.py b/demo/realtime-img2img/main.py index aa5fe5873..754fa26a3 100644 --- a/demo/realtime-img2img/main.py +++ b/demo/realtime-img2img/main.py @@ -11,7 +11,6 @@ from img2img import Pipeline from input_control import InputManager - # fix mime error on windows mimetypes.add_type("application/javascript", ".js") @@ -333,7 +332,7 @@ def add_controlnet(self, controlnet_config: dict): def remove_controlnet(self, index: int): """Remove ControlNet from AppState - SINGLE SOURCE OF TRUTH""" if index < len(self.controlnet_info["controlnets"]): - removed = self.controlnet_info["controlnets"].pop(index) + removed = self.controlnet_info["controlnets"].pop(index) # noqa: F841 # TODO: pre-existing, untouched by this refactor # Re-index remaining controlnets for i, controlnet in enumerate(self.controlnet_info["controlnets"]): controlnet["index"] = i @@ -380,7 +379,7 @@ def remove_hook_processor(self, hook_type: str, processor_index: int): if hook_type in self.pipeline_hooks: processors = self.pipeline_hooks[hook_type]["processors"] if processor_index < len(processors): - removed = processors.pop(processor_index) + removed = processors.pop(processor_index) # noqa: F841 # TODO: pre-existing, untouched by this refactor # Re-index remaining processors for i, processor in enumerate(processors): processor["index"] = i @@ -538,7 +537,7 @@ def generate_pipeline_config(self): if self.ipadapter_info["enabled"]: config["use_ipadapter"] = True # Preserve original ipadapters config but update runtime values - if "ipadapters" in config and config["ipadapters"]: + if config.get("ipadapters"): # Update existing config with current values config["ipadapters"][0].update( {"scale": self.ipadapter_info["scale"], "weight_type": self.ipadapter_info["weight_type"]} @@ -880,8 +879,8 @@ def get_available_controlnets(): def _create_pipeline(self): """Create pipeline using AppState as single source of truth""" logger.info("_create_pipeline: Creating pipeline using AppState as single source of truth") - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - torch_dtype = torch.float16 + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # noqa: F841 # TODO: pre-existing, untouched by this refactor + torch_dtype = torch.float16 # noqa: F841 # TODO: pre-existing, untouched by this refactor # Generate pipeline config from AppState - SINGLE SOURCE OF TRUTH pipeline_config = self.app_state.generate_pipeline_config() @@ -906,7 +905,7 @@ def _create_pipeline(self): if "use_safety_checker" in pipeline_config: args_dict["safety_checker"] = pipeline_config["use_safety_checker"] - updated_args = Args(**args_dict) + updated_args = Args(**args_dict) # noqa: F841 # TODO: pre-existing, untouched by this refactor # Create Pipeline instance with the pre-created wrapper and config pipeline = Pipeline(wrapper=wrapper, config=pipeline_config) @@ -941,7 +940,7 @@ def _cleanup_temp_files(self): try: if os.path.exists(temp_path): os.unlink(temp_path) - except: + except: # noqa: E722 # TODO: pre-existing, untouched by this refactor pass self._temp_config_files.clear() diff --git a/demo/realtime-img2img/routes/common/api_utils.py b/demo/realtime-img2img/routes/common/api_utils.py index 0a5425774..5937deac6 100644 --- a/demo/realtime-img2img/routes/common/api_utils.py +++ b/demo/realtime-img2img/routes/common/api_utils.py @@ -42,7 +42,7 @@ async def handle_api_request( except Exception as e: logging.exception(f"{operation_name}: Failed to parse request: {e}") - raise HTTPException(status_code=400, detail=f"Invalid request format: {str(e)}") + raise HTTPException(status_code=400, detail=f"Invalid request format: {e!s}") def create_success_response(message: str, **extra_data) -> JSONResponse: @@ -74,7 +74,7 @@ def handle_api_error(error: Exception, operation_name: str, status_code: int = 5 HTTPException with standardized error message """ logging.error(f"{operation_name}: Failed: {error}") - return HTTPException(status_code=status_code, detail=f"Failed to {operation_name.lower()}: {str(error)}") + return HTTPException(status_code=status_code, detail=f"Failed to {operation_name.lower()}: {error!s}") def validate_pipeline(pipeline: Any, operation_name: str) -> None: diff --git a/demo/realtime-img2img/routes/controlnet.py b/demo/realtime-img2img/routes/controlnet.py index 3bb0084fc..de87f4e35 100644 --- a/demo/realtime-img2img/routes/controlnet.py +++ b/demo/realtime-img2img/routes/controlnet.py @@ -15,7 +15,6 @@ ) from .common.dependencies import get_app_instance, get_available_controlnets - router = APIRouter(prefix="/api", tags=["controlnet"]) @@ -48,7 +47,7 @@ async def upload_controlnet_config(file: UploadFile = File(...), app_instance=De try: config_data = yaml.safe_load(content.decode("utf-8")) except yaml.YAMLError as e: - raise HTTPException(status_code=400, detail=f"Invalid YAML format: {str(e)}") + raise HTTPException(status_code=400, detail=f"Invalid YAML format: {e!s}") # YAML is source of truth - completely replace any runtime modifications app_instance.app_state.uploaded_config = config_data @@ -78,12 +77,12 @@ async def upload_controlnet_config(file: UploadFile = File(...), app_instance=De # Get config prompt if available config_prompt = config_data.get("prompt", None) # Get negative prompt if available - config_negative_prompt = config_data.get("negative_prompt", None) + config_negative_prompt = config_data.get("negative_prompt", None) # noqa: F841 # TODO: pre-existing, untouched by this refactor # Get t_index_list from config if available from app_config import DEFAULT_SETTINGS - t_index_list = config_data.get("t_index_list", DEFAULT_SETTINGS.get("t_index_list", [35, 45])) + t_index_list = config_data.get("t_index_list", DEFAULT_SETTINGS.get("t_index_list", [35, 45])) # noqa: F841 # TODO: pre-existing, untouched by this refactor # Get acceleration from config if available config_acceleration = config_data.get("acceleration", app_instance.args.acceleration) @@ -165,7 +164,7 @@ async def upload_controlnet_config(file: UploadFile = File(...), app_instance=De except Exception as e: logging.exception(f"upload_controlnet_config: Failed to upload configuration: {e}") - raise HTTPException(status_code=500, detail=f"Failed to upload configuration: {str(e)}") + raise HTTPException(status_code=500, detail=f"Failed to upload configuration: {e!s}") @router.get("/controlnet/info") @@ -274,7 +273,6 @@ async def get_available_controlnets_endpoint( elif "sd-turbo" in ml or "sd21" in ml or "sd2.1" in ml or "2-1" in ml or "stable-diffusion-2" in ml: model_type = "sd21" - # Handle case where available_controlnets dependency returns None if available_controlnets is None: logging.warning("get_available_controlnets: available_controlnets dependency returned None") @@ -435,7 +433,7 @@ async def remove_controlnet(request: Request, app_instance=Depends(get_app_insta if index < 0 or index >= len(controlnets): raise HTTPException(status_code=400, detail=f"ControlNet index {index} out of range") - removed_controlnet = controlnets.pop(index) + removed_controlnet = controlnets.pop(index) # noqa: F841 # TODO: pre-existing, untouched by this refactor # Remove from AppState - SINGLE SOURCE OF TRUTH app_instance.app_state.remove_controlnet(index) @@ -536,7 +534,7 @@ async def switch_preprocessor(request: Request, app_instance=Depends(get_app_ins # Update the preprocessor in AppState controlnet = app_instance.app_state.controlnet_info["controlnets"][controlnet_index] - old_preprocessor = controlnet.get("preprocessor", "unknown") + old_preprocessor = controlnet.get("preprocessor", "unknown") # noqa: F841 # TODO: pre-existing, untouched by this refactor controlnet["preprocessor"] = preprocessor_name controlnet["preprocessor_params"] = {} # Reset parameters when switching @@ -641,7 +639,7 @@ async def update_preprocessor_params(request: Request, app_instance=Depends(get_ ) except Exception as e: - logging.exception(f"update_preprocessor_params: Exception occurred: {str(e)}") + logging.exception(f"update_preprocessor_params: Exception occurred: {e!s}") raise handle_api_error(e, "update_preprocessor_params") diff --git a/demo/realtime-img2img/routes/debug.py b/demo/realtime-img2img/routes/debug.py index 4fcd9e3b7..28b8db974 100644 --- a/demo/realtime-img2img/routes/debug.py +++ b/demo/realtime-img2img/routes/debug.py @@ -9,7 +9,6 @@ from .common.dependencies import get_app_instance - router = APIRouter(prefix="/api/debug", tags=["debug"]) @@ -37,7 +36,7 @@ async def enable_debug_mode(app_instance=Depends(get_app_instance)): ) except Exception as e: logging.exception(f"enable_debug_mode: Failed to enable debug mode: {e}") - raise HTTPException(status_code=500, detail=f"Failed to enable debug mode: {str(e)}") + raise HTTPException(status_code=500, detail=f"Failed to enable debug mode: {e!s}") @router.post("/disable", response_model=DebugResponse) @@ -57,7 +56,7 @@ async def disable_debug_mode(app_instance=Depends(get_app_instance)): ) except Exception as e: logging.exception(f"disable_debug_mode: Failed to disable debug mode: {e}") - raise HTTPException(status_code=500, detail=f"Failed to disable debug mode: {str(e)}") + raise HTTPException(status_code=500, detail=f"Failed to disable debug mode: {e!s}") @router.post("/step", response_model=DebugResponse) @@ -82,7 +81,7 @@ async def step_frame(app_instance=Depends(get_app_instance)): raise except Exception as e: logging.exception(f"step_frame: Failed to step frame: {e}") - raise HTTPException(status_code=500, detail=f"Failed to step frame: {str(e)}") + raise HTTPException(status_code=500, detail=f"Failed to step frame: {e!s}") @router.get("/status", response_model=DebugResponse) @@ -97,4 +96,4 @@ async def get_debug_status(app_instance=Depends(get_app_instance)): ) except Exception as e: logging.exception(f"get_debug_status: Failed to get debug status: {e}") - raise HTTPException(status_code=500, detail=f"Failed to get debug status: {str(e)}") + raise HTTPException(status_code=500, detail=f"Failed to get debug status: {e!s}") diff --git a/demo/realtime-img2img/routes/inference.py b/demo/realtime-img2img/routes/inference.py index f1c81dd4a..82a14138e 100644 --- a/demo/realtime-img2img/routes/inference.py +++ b/demo/realtime-img2img/routes/inference.py @@ -11,7 +11,6 @@ from .common.dependencies import get_app_instance, get_default_settings, get_pipeline_class - router = APIRouter(prefix="/api", tags=["inference"]) @@ -97,7 +96,7 @@ async def stream(user_id: uuid.UUID, request: Request, app_instance=Depends(get_ app_instance._cleanup_pipeline(old_pipeline) app_instance.pipeline = app_instance._create_pipeline() - acceleration_changed = True + acceleration_changed = True # noqa: F841 # TODO: pre-existing, untouched by this refactor logging.info("stream: Pipeline recreated with new acceleration") # IPAdapter style images are now handled dynamically in pipeline.predict() @@ -188,7 +187,7 @@ async def generate_frames(): except Exception as e: logging.exception(f"stream: Error in streaming endpoint: {e}") - raise HTTPException(status_code=500, detail=f"Streaming failed: {str(e)}") + raise HTTPException(status_code=500, detail=f"Streaming failed: {e!s}") @router.get("/state") @@ -247,7 +246,7 @@ async def get_app_state( except Exception as e: logging.error(f"get_app_state: Error getting application state: {e}") - raise HTTPException(status_code=500, detail=f"Failed to get application state: {str(e)}") + raise HTTPException(status_code=500, detail=f"Failed to get application state: {e!s}") @router.get("/settings") diff --git a/demo/realtime-img2img/routes/input_sources.py b/demo/realtime-img2img/routes/input_sources.py index 1029f2398..0b130503d 100644 --- a/demo/realtime-img2img/routes/input_sources.py +++ b/demo/realtime-img2img/routes/input_sources.py @@ -16,7 +16,6 @@ from .common.api_utils import create_success_response, handle_api_error, handle_api_request from .common.dependencies import get_app_instance - router = APIRouter(prefix="/api/input-sources", tags=["input-sources"]) logger = logging.getLogger("input_sources_api") @@ -139,7 +138,7 @@ async def upload_component_image( except Exception as e: # Clean up file if image processing fails file_path.unlink(missing_ok=True) - raise HTTPException(status_code=400, detail=f"Invalid image file: {str(e)}") + raise HTTPException(status_code=400, detail=f"Invalid image file: {e!s}") # Get input source manager and set source manager = _get_input_source_manager(app_instance) diff --git a/demo/realtime-img2img/routes/ipadapter.py b/demo/realtime-img2img/routes/ipadapter.py index edf7a4519..3c574ffb7 100644 --- a/demo/realtime-img2img/routes/ipadapter.py +++ b/demo/realtime-img2img/routes/ipadapter.py @@ -16,7 +16,6 @@ ) from .common.dependencies import get_app_instance - router = APIRouter(prefix="/api", tags=["ipadapter"]) # Legacy upload endpoint removed - use /api/input-sources/upload-image/ipadapter instead diff --git a/demo/realtime-img2img/routes/parameters.py b/demo/realtime-img2img/routes/parameters.py index 27f4b0d06..60354d240 100644 --- a/demo/realtime-img2img/routes/parameters.py +++ b/demo/realtime-img2img/routes/parameters.py @@ -10,7 +10,6 @@ from .common.api_utils import create_success_response, handle_api_error, handle_api_request from .common.dependencies import get_app_instance - router = APIRouter(prefix="/api", tags=["parameters"]) @@ -103,7 +102,7 @@ async def update_params(request: Request, app_instance=Depends(get_app_instance) except Exception as e: logging.exception(f"update_params: Failed to update parameters: {e}") - raise HTTPException(status_code=500, detail=f"Failed to update parameters: {str(e)}") + raise HTTPException(status_code=500, detail=f"Failed to update parameters: {e!s}") async def _update_single_parameter( diff --git a/demo/realtime-img2img/routes/pipeline_hooks.py b/demo/realtime-img2img/routes/pipeline_hooks.py index 0c7184ef6..82a82eb42 100644 --- a/demo/realtime-img2img/routes/pipeline_hooks.py +++ b/demo/realtime-img2img/routes/pipeline_hooks.py @@ -10,7 +10,6 @@ from .common.api_utils import create_success_response, handle_api_error from .common.dependencies import get_app_instance - router = APIRouter(prefix="/api", tags=["pipeline-hooks"]) @@ -438,7 +437,7 @@ async def update_hook_processor_params(hook_type: str, request: Request, app_ins ) except Exception as e: - logging.exception(f"update_hook_processor_params: Exception occurred: {str(e)}") + logging.exception(f"update_hook_processor_params: Exception occurred: {e!s}") logging.error(f"update_hook_processor_params: Exception type: {type(e).__name__}") raise handle_api_error(e, "update_hook_processor_params") diff --git a/demo/realtime-img2img/routes/websocket.py b/demo/realtime-img2img/routes/websocket.py index 2a2c87a0a..126c1f5f8 100644 --- a/demo/realtime-img2img/routes/websocket.py +++ b/demo/realtime-img2img/routes/websocket.py @@ -14,7 +14,6 @@ from .common.dependencies import get_app_instance, get_pipeline_class - router = APIRouter(prefix="/api", tags=["websocket"]) diff --git a/demo/realtime-img2img/util.py b/demo/realtime-img2img/util.py index e64ddc36a..11d34665d 100644 --- a/demo/realtime-img2img/util.py +++ b/demo/realtime-img2img/util.py @@ -1,11 +1,10 @@ +import io from importlib import import_module from types import ModuleType -from typing import Dict, Any -from pydantic import BaseModel as PydanticBaseModel, Field -from PIL import Image -import io + import torch -from torchvision.io import encode_jpeg, decode_jpeg +from PIL import Image +from torchvision.io import decode_jpeg, encode_jpeg def get_pipeline_class(pipeline_name: str) -> ModuleType: @@ -55,7 +54,7 @@ def bytes_to_pt(image_bytes: bytes) -> torch.Tensor: # Normalise to [0, 1] on the decode device (fused kernel on GPU). image_tensor = image_tensor.float() / 255.0 - + return image_tensor @@ -75,24 +74,24 @@ def pil_to_frame(image: Image.Image) -> bytes: def pt_to_frame(tensor: torch.Tensor) -> bytes: """ Convert PyTorch tensor directly to JPEG frame bytes using torchvision - + Args: tensor: PyTorch tensor with shape (C, H, W) or (1, C, H, W), values in [0, 1] - + Returns: bytes: JPEG frame data for streaming """ # Handle batch dimension - take first image if batched if tensor.dim() == 4: tensor = tensor[0] - + # Convert to uint8 format (0-255) and ensure correct shape (C, H, W) tensor_uint8 = (tensor * 255).clamp(0, 255).to(torch.uint8) - + # Encode directly to JPEG bytes using torchvision jpeg_bytes = encode_jpeg(tensor_uint8, quality=90) frame_data = jpeg_bytes.cpu().numpy().tobytes() - + return ( b"--frame\r\n" + b"Content-Type: image/jpeg\r\n" diff --git a/demo/realtime-img2img/utils/video_utils.py b/demo/realtime-img2img/utils/video_utils.py index 4ba9d4924..b4915b6ec 100644 --- a/demo/realtime-img2img/utils/video_utils.py +++ b/demo/realtime-img2img/utils/video_utils.py @@ -233,7 +233,7 @@ def validate_video_file(video_path: str) -> Tuple[bool, str]: return True, "Video file is valid" except Exception as e: - return False, f"Video validation error: {str(e)}" + return False, f"Video validation error: {e!s}" # Supported video formats diff --git a/demo/realtime-txt2img/config.py b/demo/realtime-txt2img/config.py index f7bb12403..a5700e4ce 100644 --- a/demo/realtime-txt2img/config.py +++ b/demo/realtime-txt2img/config.py @@ -4,7 +4,6 @@ import torch - SAFETY_CHECKER = os.environ.get("SAFETY_CHECKER", "False") == "True" diff --git a/demo/realtime-txt2img/main.py b/demo/realtime-txt2img/main.py index 18931f9ec..6fedfce60 100644 --- a/demo/realtime-txt2img/main.py +++ b/demo/realtime-txt2img/main.py @@ -14,12 +14,10 @@ from PIL import Image from pydantic import BaseModel - sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..")) from streamdiffusion import StreamDiffusionWrapper - logger = logging.getLogger("uvicorn") PROJECT_DIR = Path(__file__).parent.parent diff --git a/demo/vid2vid/app.py b/demo/vid2vid/app.py index c7f4a37ff..e9add2d82 100644 --- a/demo/vid2vid/app.py +++ b/demo/vid2vid/app.py @@ -7,12 +7,10 @@ from torchvision.io import read_video, write_video from tqdm import tqdm - sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..")) from streamdiffusion import StreamDiffusionWrapper - CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) diff --git a/examples/benchmark/ab_bench.py b/examples/benchmark/ab_bench.py index a59a1ed90..86aa4afe5 100644 --- a/examples/benchmark/ab_bench.py +++ b/examples/benchmark/ab_bench.py @@ -64,15 +64,13 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple - if TYPE_CHECKING: - import PIL.Image # noqa: F401 — used in type annotations only + import PIL.Image import numpy as np import torch from tqdm import tqdm - # ── repo root on sys.path so streamdiffusion is importable without install ───── _REPO_ROOT = Path(__file__).resolve().parents[2] if str(_REPO_ROOT) not in sys.path: @@ -81,7 +79,6 @@ from streamdiffusion.tools.gpu_profiler import configure as _prof_configure # noqa: E402 from streamdiffusion.tools.gpu_profiler import profiler # noqa: E402 - # ────────────────────────────────────────────────────────────────────────────── # Helpers # ────────────────────────────────────────────────────────────────────────────── diff --git a/examples/benchmark/multi.py b/examples/benchmark/multi.py index 443835b99..5ab7c85d2 100644 --- a/examples/benchmark/multi.py +++ b/examples/benchmark/multi.py @@ -13,7 +13,6 @@ from streamdiffusion.image_utils import postprocess_image - sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..")) from streamdiffusion import StreamDiffusionWrapper @@ -23,7 +22,7 @@ def _postprocess_image(queue: Queue) -> None: while True: try: if not queue.empty(): - output = postprocess_image(queue.get(block=False), output_type="pil")[0] + output = postprocess_image(queue.get(block=False), output_type="pil")[0] # noqa: F841 # TODO: pre-existing, untouched by this refactor time.sleep(0.0005) except KeyboardInterrupt: return diff --git a/examples/benchmark/single.py b/examples/benchmark/single.py index 133abc13b..46b264a78 100644 --- a/examples/benchmark/single.py +++ b/examples/benchmark/single.py @@ -9,7 +9,6 @@ import torch from tqdm import tqdm - sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..")) from streamdiffusion import StreamDiffusionWrapper diff --git a/examples/config/config_ipadapter_stream_test.py b/examples/config/config_ipadapter_stream_test.py index b8141e033..db5aaf40c 100644 --- a/examples/config/config_ipadapter_stream_test.py +++ b/examples/config/config_ipadapter_stream_test.py @@ -354,7 +354,7 @@ def main(): print("=" * 50) try: - metrics = process_video_ipadapter_stream( + metrics = process_video_ipadapter_stream( # noqa: F841 # TODO: pre-existing, untouched by this refactor args.config, args.input_video, args.static_image, args.output_dir, engine_only=args.engine_only ) if args.engine_only: diff --git a/examples/config/config_video_test.py b/examples/config/config_video_test.py index 69a8d39e6..a82c26583 100644 --- a/examples/config/config_video_test.py +++ b/examples/config/config_video_test.py @@ -228,7 +228,7 @@ def main(): # Get the script directory to make paths relative to it script_dir = Path(__file__).parent - default_config = script_dir.parent.parent / "configs" / "controlnet_examples" / "multi_controlnet_example.yaml" + default_config = script_dir.parent.parent / "configs" / "controlnet_examples" / "multi_controlnet_example.yaml" # noqa: F841 # TODO: pre-existing, untouched by this refactor parser.add_argument("--config", type=str, required=True, help="Path to ControlNet configuration file") parser.add_argument("--input-video", type=str, required=True, help="Path to input video file") @@ -270,7 +270,7 @@ def main(): print(f"main: Output directory: {args.output_dir}") try: - metrics = process_video(args.config, args.input_video, args.output_dir, engine_only=args.engine_only) + metrics = process_video(args.config, args.input_video, args.output_dir, engine_only=args.engine_only) # noqa: F841 # TODO: pre-existing, untouched by this refactor if args.engine_only: print("main: Engine-only mode completed successfully!") return 0 diff --git a/examples/img2img/multi.py b/examples/img2img/multi.py index af112585b..0f59e8cb3 100644 --- a/examples/img2img/multi.py +++ b/examples/img2img/multi.py @@ -5,12 +5,10 @@ import fire - sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..")) from streamdiffusion import StreamDiffusionWrapper - CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) diff --git a/examples/img2img/single.py b/examples/img2img/single.py index b894bc031..493753442 100644 --- a/examples/img2img/single.py +++ b/examples/img2img/single.py @@ -4,12 +4,10 @@ import fire - sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..")) from streamdiffusion import StreamDiffusionWrapper - CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) diff --git a/examples/optimal-performance/multi.py b/examples/optimal-performance/multi.py index e154dfcc1..a68c8511d 100644 --- a/examples/optimal-performance/multi.py +++ b/examples/optimal-performance/multi.py @@ -11,12 +11,10 @@ from streamdiffusion.image_utils import postprocess_image - sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..")) from streamdiffusion import StreamDiffusionWrapper - image_update_counter = 0 diff --git a/examples/optimal-performance/single.py b/examples/optimal-performance/single.py index ec610de72..abe71c6ce 100644 --- a/examples/optimal-performance/single.py +++ b/examples/optimal-performance/single.py @@ -6,7 +6,6 @@ import fire - sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..")) from streamdiffusion import StreamDiffusionWrapper diff --git a/examples/screen/main.py b/examples/screen/main.py index 405ff6eec..3bc47de45 100644 --- a/examples/screen/main.py +++ b/examples/screen/main.py @@ -14,13 +14,11 @@ from streamdiffusion.image_utils import pil2tensor - sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..")) from streamdiffusion import StreamDiffusionWrapper from utils.viewer import receive_images - inputs = [] top = 0 left = 0 diff --git a/examples/txt2img/multi.py b/examples/txt2img/multi.py index 1d3301966..b82f1b0a5 100644 --- a/examples/txt2img/multi.py +++ b/examples/txt2img/multi.py @@ -4,12 +4,10 @@ import fire - sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..")) from streamdiffusion import StreamDiffusionWrapper - CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) diff --git a/examples/txt2img/single.py b/examples/txt2img/single.py index f3c8763aa..550566bf0 100644 --- a/examples/txt2img/single.py +++ b/examples/txt2img/single.py @@ -4,12 +4,10 @@ import fire - sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..")) from streamdiffusion import StreamDiffusionWrapper - CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) diff --git a/examples/txt2img/spacing_compare.py b/examples/txt2img/spacing_compare.py index 0e226ede0..b3995f7bf 100644 --- a/examples/txt2img/spacing_compare.py +++ b/examples/txt2img/spacing_compare.py @@ -25,11 +25,9 @@ import torch from PIL import Image - sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..")) from streamdiffusion import StreamDiffusionWrapper - CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) INPUT_IMAGE = os.path.join(CURRENT_DIR, "..", "..", "images", "inputs", "input.png") OUTPUT_DIR = os.path.join(CURRENT_DIR, "..", "..", "images", "outputs", "spacing_compare") @@ -124,7 +122,7 @@ def run_model(model_id: str) -> None: brightness = mean_brightness(img) print( - f" {sampler_name:12s} on_grid={str(grid_flag):5s} sub_ts={[int(t) for t in sub_ts]}\n" + f" {sampler_name:12s} on_grid={grid_flag!s:5s} sub_ts={[int(t) for t in sub_ts]}\n" f" brightness={brightness:.4f} ({description})" ) diff --git a/examples/vid2vid/main.py b/examples/vid2vid/main.py index a045b29ad..8098ce38f 100644 --- a/examples/vid2vid/main.py +++ b/examples/vid2vid/main.py @@ -7,12 +7,10 @@ from torchvision.io import read_video, write_video from tqdm import tqdm - sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..")) from streamdiffusion import StreamDiffusionWrapper - CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) diff --git a/pyproject.toml b/pyproject.toml index 3b4b63bd0..8be14c1a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,28 +1,64 @@ [tool.ruff] line-length = 119 +target-version = "py310" # contract floor: setup.py python_requires>=3.10 +# venv/ is already in ruff's default excludes. +extend-exclude = ["logs", "profiler_logs", "profiles", "*.egg-info", "demo/*/frontend", "engines", "models"] [tool.ruff.lint] -# Never enforce `E501` (line length violations). -ignore = ["C901", "E501", "E741", "F402", "F403", "F405", "F823"] -select = ["C", "E", "F", "I", "W"] +select = ["E", "W", "F", "I", "C4", "B", "UP", "SIM", "RUF", "PERF", "NPY"] +ignore = [ + "E501", # line length — formatter handles wrapping; long literals stay + "E741", # ambiguous single-letter names — common in math/diffusion code + "F402", + "F823", + "UP006", # keep typing.Dict/List annotations — no mass churn + "UP007", # keep typing.Union + "UP035", # keep typing imports + "UP045", # keep typing.Optional + # --- Adopted incrementally: non-auto-fixable residuals, ignored pending follow-up --- + "SIM102", "SIM105", "SIM108", "SIM110", "SIM115", "SIM117", "SIM118", "SIM101", "SIM201", # stylistic + "PERF203", "PERF401", "PERF102", # perf hints + "RUF001", "RUF002", "RUF003", # ambiguous-unicode in math docstrings + "RUF012", "RUF013", "RUF005", "RUF015", "RUF059", # annotation/misc deferral + "B905", "B007", # zip-strict / unused-loop-var (behaviour-sensitive) + "B004", "B006", "B008", "B018", "B026", "B904", "NPY002", # TODO: real-bug candidates — re-enable in a follow-up pass + "RUF006", "RUF009", # TODO: real-bug candidates — re-enable in a follow-up pass + "RUF022", "RUF046", "UP008", # stylistic +] -# Ignore import violations in all `__init__.py` files. [tool.ruff.lint.per-file-ignores] "__init__.py" = ["E402", "F401", "F811"] +# The only two files using `from ... import *`: +"src/streamdiffusion/acceleration/tensorrt/builder.py" = ["F403", "F405"] +"src/streamdiffusion/acceleration/tensorrt/runtime_engines/unet_engine.py" = ["F403", "F405"] +# sys.path-hack entry points import after path setup: +"tests/**" = ["E402"] +"scripts/**" = ["E402"] +"utils/**" = ["E402"] +"StreamDiffusionTD/*.py" = ["E402"] +# Deliberate ordering: circular-import avoidance / patch-before-import. +"demo/realtime-img2img/main.py" = ["E402"] +"src/streamdiffusion/acceleration/tensorrt/export_wrappers/unet_unified_export.py" = ["E402"] [tool.ruff.lint.isort] -lines-after-imports = 2 known-first-party = ["streamdiffusion"] [tool.ruff.format] -# Like Black, use double quotes for strings. +# Black-compatible. quote-style = "double" - -# Like Black, indent with spaces, rather than tabs. indent-style = "space" - -# Like Black, respect magic trailing commas. skip-magic-trailing-comma = false - -# Like Black, automatically detect the appropriate line ending. line-ending = "auto" +docstring-code-format = true + +[tool.pyrefly] +project-includes = ["src", "StreamDiffusionTD", "tests", "custom_processors", "scripts", "tools", "utils"] +project-excludes = ["tests/manual/**", "tests/quality/**", "demo/**", "examples/**"] +search-path = ["src"] +python-version = "3.10" +python-interpreter-path = "venv/Scripts/python.exe" +# Pre-existing errors are frozen in the baseline; only new errors fail the check. +# Refresh after intentional fixes: pyrefly check --update-baseline +baseline = "pyrefly-baseline.json" +# Warn-severity GPU anti-pattern lints (.item() sync stalls, hard-coded .cuda(), print(tensor)). +pytorch-efficiency-lints = true diff --git a/scripts/profiling/profile_ncu.py b/scripts/profiling/profile_ncu.py index 11b6622a0..079734417 100644 --- a/scripts/profiling/profile_ncu.py +++ b/scripts/profiling/profile_ncu.py @@ -43,7 +43,6 @@ import sys import time - # ── CLI args ─────────────────────────────────────────────────────────────────── parser = argparse.ArgumentParser(description="StreamDiffusion Nsight Compute launcher") parser.add_argument( diff --git a/scripts/profiling/profile_nsys.py b/scripts/profiling/profile_nsys.py index 7fe1b1aca..fbaca7251 100644 --- a/scripts/profiling/profile_nsys.py +++ b/scripts/profiling/profile_nsys.py @@ -50,7 +50,6 @@ import sys import time - # ── CLI args ─────────────────────────────────────────────────────────────────── parser = argparse.ArgumentParser(description="StreamDiffusion Nsight Systems profiling launcher") parser.add_argument( @@ -214,7 +213,6 @@ from streamdiffusion.tools.gpu_profiler import profiler - os.environ.setdefault("GPU_PROFILER", "1") # wrapper.__init__ reads this to activate WARMUP_RUNS = 3 # extra warmup before torch.profiler + nsys capture window @@ -272,7 +270,6 @@ # ── Dummy input image ────────────────────────────────────────────────────────── import PIL.Image - dummy_img = PIL.Image.new("RGB", (_WIDTH, _HEIGHT), (128, 128, 128)) # ── ControlNet activation (--cn-scale > 0) ──────────────────────────────────── @@ -297,7 +294,9 @@ print(f"[profile] ControlNet[0] enabled: scale={args.cn_scale}, image=dummy gray tensor {_WIDTH}x{_HEIGHT}") if args.cn_cache_interval > 1: cn_mod.set_cn_cache_interval(args.cn_cache_interval) - print(f"[profile] ControlNet residual cache: interval={args.cn_cache_interval} (CN forward every {args.cn_cache_interval} frames)") + print( + f"[profile] ControlNet residual cache: interval={args.cn_cache_interval} (CN forward every {args.cn_cache_interval} frames)" + ) except Exception as _cn_err: print(f"[profile] WARNING: Could not activate ControlNet — {_cn_err}") print(" Make sure the config includes a ControlNet and its engine is built.") diff --git a/scripts/test_lora_sanity.py b/scripts/test_lora_sanity.py index 8d01ddeba..fe9755af8 100644 --- a/scripts/test_lora_sanity.py +++ b/scripts/test_lora_sanity.py @@ -21,7 +21,6 @@ import sys from pathlib import Path - # --------------------------------------------------------------------------- # Repo root on sys.path so `from streamdiffusion` works without install # --------------------------------------------------------------------------- @@ -29,10 +28,9 @@ if str(_REPO_ROOT / "src") not in sys.path: sys.path.insert(0, str(_REPO_ROOT / "src")) -from PIL import Image # noqa: E402 - -from streamdiffusion import StreamDiffusionWrapper # noqa: E402 +from PIL import Image +from streamdiffusion import StreamDiffusionWrapper logging.basicConfig( level=logging.INFO, diff --git a/setup.py b/setup.py index 5e22711df..d9c674cd5 100644 --- a/setup.py +++ b/setup.py @@ -120,7 +120,7 @@ def deps_list(*pkgs): name="streamdiffusion", version="0.1.1", description="real-time interactive image generation pipeline", - long_description=open("README.md", "r", encoding="utf-8").read(), + long_description=open("README.md", encoding="utf-8").read(), long_description_content_type="text/markdown", keywords="deep learning diffusion pytorch stable diffusion audioldm streamdiffusion real-time", license="Apache 2.0 License", diff --git a/src/streamdiffusion/__init__.py b/src/streamdiffusion/__init__.py index 2941f504e..b8fe7363e 100644 --- a/src/streamdiffusion/__init__.py +++ b/src/streamdiffusion/__init__.py @@ -1,15 +1,14 @@ -from . import _patches # noqa: F401 — applies kvo_cache patch before any diffusers import +from . import _patches from .config import create_wrapper_from_config, load_config, save_config from .pipeline import StreamDiffusion from .preprocessing.processors import list_preprocessors from .wrapper import StreamDiffusionWrapper - __all__ = [ "StreamDiffusion", "StreamDiffusionWrapper", - "load_config", + "create_wrapper_from_config", "list_preprocessors", + "load_config", "save_config", - "create_wrapper_from_config", ] diff --git a/src/streamdiffusion/_patches/__init__.py b/src/streamdiffusion/_patches/__init__.py index 23f44fa02..234e7716d 100644 --- a/src/streamdiffusion/_patches/__init__.py +++ b/src/streamdiffusion/_patches/__init__.py @@ -1,4 +1,3 @@ from .diffusers_kvo_patch import apply as _apply_kvo_patch - _apply_kvo_patch() diff --git a/src/streamdiffusion/_patches/diffusers_kvo_patch.py b/src/streamdiffusion/_patches/diffusers_kvo_patch.py index 2ee5da1c7..214b66b7b 100644 --- a/src/streamdiffusion/_patches/diffusers_kvo_patch.py +++ b/src/streamdiffusion/_patches/diffusers_kvo_patch.py @@ -16,7 +16,6 @@ import inspect import logging - logger = logging.getLogger(__name__) _PATCHED = False diff --git a/src/streamdiffusion/_patches/hf_tracing_patches.py b/src/streamdiffusion/_patches/hf_tracing_patches.py index 422b6bf65..a563d221f 100644 --- a/src/streamdiffusion/_patches/hf_tracing_patches.py +++ b/src/streamdiffusion/_patches/hf_tracing_patches.py @@ -5,7 +5,6 @@ import torch - _ALREADY = False # idempotence guard diff --git a/src/streamdiffusion/acceleration/tensorrt/__init__.py b/src/streamdiffusion/acceleration/tensorrt/__init__.py index b4c7a7ca2..2688a68f7 100644 --- a/src/streamdiffusion/acceleration/tensorrt/__init__.py +++ b/src/streamdiffusion/acceleration/tensorrt/__init__.py @@ -5,7 +5,6 @@ import torch import torch.nn as nn - os.environ.setdefault("CUDA_MODULE_LOADING", "LAZY") from diffusers import AutoencoderKL, ControlNetModel, UNet2DConditionModel from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img import ( diff --git a/src/streamdiffusion/acceleration/tensorrt/builder.py b/src/streamdiffusion/acceleration/tensorrt/builder.py index 1e767aba4..945db1f73 100644 --- a/src/streamdiffusion/acceleration/tensorrt/builder.py +++ b/src/streamdiffusion/acceleration/tensorrt/builder.py @@ -18,7 +18,6 @@ optimize_onnx, ) - _build_logger = logging.getLogger(__name__) diff --git a/src/streamdiffusion/acceleration/tensorrt/engine_manager.py b/src/streamdiffusion/acceleration/tensorrt/engine_manager.py index 6ed4ad645..81cadf4c4 100644 --- a/src/streamdiffusion/acceleration/tensorrt/engine_manager.py +++ b/src/streamdiffusion/acceleration/tensorrt/engine_manager.py @@ -4,7 +4,6 @@ from pathlib import Path from typing import Any, Dict, Optional - logger = logging.getLogger(__name__) @@ -360,17 +359,17 @@ def load_engine(self, engine_type: EngineType, engine_path: Path, **kwargs: Dict def _set_unet_metadata(self, loaded_engine, kwargs: Dict) -> None: """Set metadata on UNet engine for runtime use.""" - setattr(loaded_engine, "use_control", kwargs.get("use_controlnet_trt", False)) - setattr(loaded_engine, "use_ipadapter", kwargs.get("use_ipadapter_trt", False)) + loaded_engine.use_control = kwargs.get("use_controlnet_trt", False) + loaded_engine.use_ipadapter = kwargs.get("use_ipadapter_trt", False) if kwargs.get("use_controlnet_trt", False): - setattr(loaded_engine, "unet_arch", kwargs.get("unet_arch", {})) + loaded_engine.unet_arch = kwargs.get("unet_arch", {}) if kwargs.get("use_ipadapter_trt", False): - setattr(loaded_engine, "ipadapter_arch", kwargs.get("unet_arch", {})) + loaded_engine.ipadapter_arch = kwargs.get("unet_arch", {}) # number of IP-attention layers for runtime vector sizing if "num_ip_layers" in kwargs and kwargs["num_ip_layers"] is not None: - setattr(loaded_engine, "num_ip_layers", kwargs["num_ip_layers"]) + loaded_engine.num_ip_layers = kwargs["num_ip_layers"] def get_or_load_controlnet_engine( self, diff --git a/src/streamdiffusion/acceleration/tensorrt/export_wrappers/__init__.py b/src/streamdiffusion/acceleration/tensorrt/export_wrappers/__init__.py index be5408075..bb0c7ceb6 100644 --- a/src/streamdiffusion/acceleration/tensorrt/export_wrappers/__init__.py +++ b/src/streamdiffusion/acceleration/tensorrt/export_wrappers/__init__.py @@ -4,13 +4,12 @@ from .unet_sdxl_export import SDXLConditioningHandler, SDXLExportWrapper from .unet_unified_export import UnifiedExportWrapper - __all__ = [ - "SDXLControlNetExportWrapper", "ControlNetUNetExportWrapper", - "MultiControlNetUNetExportWrapper", "IPAdapterUNetExportWrapper", - "SDXLExportWrapper", + "MultiControlNetUNetExportWrapper", "SDXLConditioningHandler", + "SDXLControlNetExportWrapper", + "SDXLExportWrapper", "UnifiedExportWrapper", ] diff --git a/src/streamdiffusion/acceleration/tensorrt/export_wrappers/unet_ipadapter_export.py b/src/streamdiffusion/acceleration/tensorrt/export_wrappers/unet_ipadapter_export.py index 93310ad87..943bf5003 100644 --- a/src/streamdiffusion/acceleration/tensorrt/export_wrappers/unet_ipadapter_export.py +++ b/src/streamdiffusion/acceleration/tensorrt/export_wrappers/unet_ipadapter_export.py @@ -281,7 +281,7 @@ def create_ipadapter_wrapper( # Check if UNet already has IPAdapter processors installed existing_processors = unet.attn_processors - has_ipadapter = any( + has_ipadapter = any( # noqa: F841 # TODO: pre-existing, untouched by this refactor "IPAttn" in proc.__class__.__name__ or "IPAttnProcessor" in proc.__class__.__name__ for proc in existing_processors.values() ) @@ -289,7 +289,7 @@ def create_ipadapter_wrapper( # Validate expected dimensions expected_dims = {"SD15": 768, "SDXL": 2048, "SD21": 1024} - expected_dim = expected_dims.get(model_type) + expected_dim = expected_dims.get(model_type) # noqa: F841 # TODO: pre-existing, untouched by this refactor return IPAdapterUNetExportWrapper(unet, cross_attention_dim, num_tokens, install_processors) diff --git a/src/streamdiffusion/acceleration/tensorrt/export_wrappers/unet_sdxl_export.py b/src/streamdiffusion/acceleration/tensorrt/export_wrappers/unet_sdxl_export.py index 078b1f913..2cade0644 100644 --- a/src/streamdiffusion/acceleration/tensorrt/export_wrappers/unet_sdxl_export.py +++ b/src/streamdiffusion/acceleration/tensorrt/export_wrappers/unet_sdxl_export.py @@ -14,7 +14,6 @@ detect_model, ) - logger = logging.getLogger(__name__) # Handle different diffusers versions for CLIPTextModel import @@ -107,10 +106,7 @@ def forward(self, *args, **kwargs): } # If model supports added conditioning and we have the kwargs, use them - if self.supports_added_cond and "added_cond_kwargs" in kwargs: - result = self.unet(*args, **kwargs) - return result - elif len(args) >= 3: + if (self.supports_added_cond and "added_cond_kwargs" in kwargs) or len(args) >= 3: result = self.unet(*args, **kwargs) return result else: diff --git a/src/streamdiffusion/acceleration/tensorrt/export_wrappers/unet_unified_export.py b/src/streamdiffusion/acceleration/tensorrt/export_wrappers/unet_unified_export.py index faeba0191..8117e1ab8 100644 --- a/src/streamdiffusion/acceleration/tensorrt/export_wrappers/unet_unified_export.py +++ b/src/streamdiffusion/acceleration/tensorrt/export_wrappers/unet_unified_export.py @@ -5,13 +5,13 @@ from streamdiffusion._patches.diffusers_kvo_patch import apply as _apply_kvo_patch - _apply_kvo_patch() # ensure kvo_cache patch is present even if diffusers was imported first from ..models.utils import convert_list_to_structure from .unet_controlnet_export import create_controlnet_wrapper from .unet_ipadapter_export import create_ipadapter_wrapper + def _collect_fi_processors(unet: UNet2DConditionModel) -> List: """Walk the UNet in kvo-cache walk order (down→mid→up) and return all ``CachedSTAttnProcessor2_0`` instances that have ``fi_eligible=True``. @@ -111,7 +111,6 @@ def __init__( self.unet, control_input_names, kvo_cache_structure, **controlnet_kwargs ) - # Best-effort collection at construction time — may return [] if processors are # not yet installed (e.g. when wrapper.py constructs UnifiedExportWrapper before # the if-use_cached_attn block installs CachedSTAttnProcessor2_0). diff --git a/src/streamdiffusion/acceleration/tensorrt/fp8_quantize.py b/src/streamdiffusion/acceleration/tensorrt/fp8_quantize.py index 53f8ba28d..f0a5ff638 100644 --- a/src/streamdiffusion/acceleration/tensorrt/fp8_quantize.py +++ b/src/streamdiffusion/acceleration/tensorrt/fp8_quantize.py @@ -29,7 +29,6 @@ import numpy as np - logger = logging.getLogger(__name__) _BUNDLED_PROMPTS_PATH = Path(__file__).parent / "calibration_prompts_sdxl.txt" @@ -45,7 +44,7 @@ def _load_calibration_prompts(user_path: Optional[str] = None) -> List[str]: "abstract colorful geometric pattern", "landscape photography at golden hour", ] - with open(path, "r", encoding="utf-8") as f: + with open(path, encoding="utf-8") as f: prompts = [line.strip() for line in f if line.strip() and not line.startswith("#")] logger.info(f"[FP8] Loaded {len(prompts)} calibration prompts from {path.name}") return prompts @@ -585,7 +584,6 @@ def quantize_onnx_fp8( f"(n_itr={_n_itr} × resolved_dim0={_resolved_dim0[_k]})" ) - import inspect as _inspect _params = set(_inspect.signature(modelopt_quantize).parameters.keys()) diff --git a/src/streamdiffusion/acceleration/tensorrt/runtime_engines/__init__.py b/src/streamdiffusion/acceleration/tensorrt/runtime_engines/__init__.py index 7fa98f855..d6868339c 100644 --- a/src/streamdiffusion/acceleration/tensorrt/runtime_engines/__init__.py +++ b/src/streamdiffusion/acceleration/tensorrt/runtime_engines/__init__.py @@ -4,10 +4,9 @@ from .controlnet_engine import ControlNetModelEngine from .unet_engine import AutoencoderKLEngine, UNet2DConditionModelEngine - __all__ = [ - "UNet2DConditionModelEngine", "AutoencoderKLEngine", "ControlNetModelEngine", "EngineManager", + "UNet2DConditionModelEngine", ] diff --git a/src/streamdiffusion/acceleration/tensorrt/runtime_engines/controlnet_engine.py b/src/streamdiffusion/acceleration/tensorrt/runtime_engines/controlnet_engine.py index 7d1ce77c2..041ee56ac 100644 --- a/src/streamdiffusion/acceleration/tensorrt/runtime_engines/controlnet_engine.py +++ b/src/streamdiffusion/acceleration/tensorrt/runtime_engines/controlnet_engine.py @@ -8,7 +8,6 @@ from ..utilities import Engine - # Set up logger for this module logger = logging.getLogger(__name__) diff --git a/src/streamdiffusion/acceleration/tensorrt/runtime_engines/unet_engine.py b/src/streamdiffusion/acceleration/tensorrt/runtime_engines/unet_engine.py index 85d081c7a..350722a25 100644 --- a/src/streamdiffusion/acceleration/tensorrt/runtime_engines/unet_engine.py +++ b/src/streamdiffusion/acceleration/tensorrt/runtime_engines/unet_engine.py @@ -12,7 +12,6 @@ from ..utilities import Engine - # Set up logger for this module logger = logging.getLogger(__name__) diff --git a/src/streamdiffusion/acceleration/tensorrt/utilities.py b/src/streamdiffusion/acceleration/tensorrt/utilities.py index 3ec567a77..8607c6efd 100644 --- a/src/streamdiffusion/acceleration/tensorrt/utilities.py +++ b/src/streamdiffusion/acceleration/tensorrt/utilities.py @@ -33,7 +33,6 @@ from streamdiffusion.tools.gpu_profiler import profiler as _gpu_profiler - # cuda-python 13.x renamed 'cudart' to 'cuda.bindings.runtime' try: from cuda.bindings import runtime as cudart @@ -47,7 +46,6 @@ from .models.models import CLIP, VAE, BaseModel, UNet, VAEEncoder - logger = logging.getLogger(__name__) TRT_LOGGER = get_trt_logger() # polygraphy singleton — shared with engine_from_bytes() @@ -125,7 +123,6 @@ def _ensure_build_logger_registered() -> None: from ...model_detection import detect_model # noqa: E402 - # --------------------------------------------------------------------------- # GPU Hardware Profile — hardware-aware TRT builder configuration # --------------------------------------------------------------------------- @@ -504,7 +501,7 @@ def __init__(self, name: str = ""): self._runs: deque = deque(maxlen=500) # rolling window; prevents unbounded growth at 30 fps self._current: list = [] # accumulator for the in-progress inference - def report_layer_time(self, layer_name: str, ms: float) -> None: # noqa: N802 + def report_layer_time(self, layer_name: str, ms: float) -> None: self._current.append((layer_name, ms)) def start_run(self) -> None: diff --git a/src/streamdiffusion/config.py b/src/streamdiffusion/config.py index bd97afcf6..716632927 100644 --- a/src/streamdiffusion/config.py +++ b/src/streamdiffusion/config.py @@ -7,7 +7,6 @@ from .param_schema import DEFAULTS - logger = logging.getLogger(__name__) @@ -17,7 +16,7 @@ def load_config(config_path: Union[str, Path]) -> Dict[str, Any]: if not config_path.exists(): raise FileNotFoundError(f"load_config: Configuration file not found: {config_path}") - with open(config_path, "r", encoding="utf-8") as f: + with open(config_path, encoding="utf-8") as f: if config_path.suffix.lower() in [".yaml", ".yml"]: config_data = yaml.safe_load(f) elif config_path.suffix.lower() == ".json": @@ -144,7 +143,7 @@ def _extract_wrapper_params(config: Dict[str, Any]) -> Dict[str, Any]: "build_engines_if_missing": config.get("build_engines_if_missing", True), "fp8_allow_fp16_fallback": config.get("fp8_allow_fp16_fallback", False), } - if "controlnets" in config and config["controlnets"]: + if config.get("controlnets"): param_map["use_controlnet"] = True param_map["controlnet_config"] = _prepare_controlnet_configs(config) else: @@ -152,7 +151,7 @@ def _extract_wrapper_params(config: Dict[str, Any]) -> Dict[str, Any]: param_map["controlnet_config"] = config.get("controlnet_config") # Set IPAdapter usage if IPAdapters are configured - if "ipadapters" in config and config["ipadapters"]: + if config.get("ipadapters"): param_map["use_ipadapter"] = True param_map["ipadapter_config"] = _prepare_ipadapter_configs(config) else: @@ -296,28 +295,28 @@ def _prepare_pipeline_hook_configs(config: Dict[str, Any]) -> Dict[str, Any]: hook_configs = {} # Image preprocessing hooks - if "image_preprocessing" in config and config["image_preprocessing"]: + if config.get("image_preprocessing"): if config["image_preprocessing"].get("enabled", True): hook_configs["image_preprocessing_config"] = _prepare_single_hook_config( config["image_preprocessing"], "image_preprocessing" ) # Image postprocessing hooks - if "image_postprocessing" in config and config["image_postprocessing"]: + if config.get("image_postprocessing"): if config["image_postprocessing"].get("enabled", True): hook_configs["image_postprocessing_config"] = _prepare_single_hook_config( config["image_postprocessing"], "image_postprocessing" ) # Latent preprocessing hooks - if "latent_preprocessing" in config and config["latent_preprocessing"]: + if config.get("latent_preprocessing"): if config["latent_preprocessing"].get("enabled", True): hook_configs["latent_preprocessing_config"] = _prepare_single_hook_config( config["latent_preprocessing"], "latent_preprocessing" ) # Latent postprocessing hooks - if "latent_postprocessing" in config and config["latent_postprocessing"]: + if config.get("latent_postprocessing"): if config["latent_postprocessing"].get("enabled", True): hook_configs["latent_postprocessing_config"] = _prepare_single_hook_config( config["latent_postprocessing"], "latent_postprocessing" diff --git a/src/streamdiffusion/model_detection.py b/src/streamdiffusion/model_detection.py index e98d8e9be..d55e7bcfa 100644 --- a/src/streamdiffusion/model_detection.py +++ b/src/streamdiffusion/model_detection.py @@ -5,7 +5,6 @@ import torch from diffusers.models.unets.unet_2d_condition import UNet2DConditionModel - # Gracefully import the SD3 model class; it might not exist in older diffusers versions. try: from diffusers.models.transformers.mm_dit import MMDiTTransformer2DModel @@ -18,7 +17,6 @@ import logging - logger = logging.getLogger(__name__) diff --git a/src/streamdiffusion/modules/__init__.py b/src/streamdiffusion/modules/__init__.py index f3242ca59..b8d0686fb 100644 --- a/src/streamdiffusion/modules/__init__.py +++ b/src/streamdiffusion/modules/__init__.py @@ -5,7 +5,6 @@ from .ipadapter_module import IPAdapterModule from .latent_processing_module import LatentPostprocessingModule, LatentPreprocessingModule, LatentProcessingModule - __all__ = [ # Existing modules "ControlNetModule", diff --git a/src/streamdiffusion/modules/controlnet_module.py b/src/streamdiffusion/modules/controlnet_module.py index 03c57345d..8b487ad1e 100644 --- a/src/streamdiffusion/modules/controlnet_module.py +++ b/src/streamdiffusion/modules/controlnet_module.py @@ -15,7 +15,6 @@ ) from streamdiffusion.tools.gpu_profiler import profiler - logger = logging.getLogger(__name__) @@ -106,9 +105,9 @@ def install(self, stream) -> None: # Register UNet hook stream.unet_hooks.append(self.build_unet_hook()) # Expose controlnet collections so existing updater can find them - setattr(stream, "controlnets", self.controlnets) - setattr(stream, "controlnet_scales", self.controlnet_scales) - setattr(stream, "preprocessors", self.preprocessors) + stream.controlnets = self.controlnets + stream.controlnet_scales = self.controlnet_scales + stream.preprocessors = self.preprocessors # Reset prepared tensors on install self._prepared_tensors = [] self._prepared_device = None @@ -147,7 +146,7 @@ def add_controlnet( if cfg.preprocessor_params: params = cfg.preprocessor_params or {} # If the preprocessor exposes a 'params' dict, update it - if hasattr(preproc, "params") and isinstance(getattr(preproc, "params"), dict): + if hasattr(preproc, "params") and isinstance(preproc.params, dict): preproc.params.update(params) # Also set attributes directly when they exist for name, value in params.items(): @@ -159,13 +158,13 @@ def add_controlnet( # Align preprocessor target size with stream resolution once (avoid double-resize later) try: - if hasattr(preproc, "params") and isinstance(getattr(preproc, "params"), dict): + if hasattr(preproc, "params") and isinstance(preproc.params, dict): preproc.params["image_width"] = int(self._stream.width) preproc.params["image_height"] = int(self._stream.height) if hasattr(preproc, "image_width"): - setattr(preproc, "image_width", int(self._stream.width)) + preproc.image_width = int(self._stream.width) if hasattr(preproc, "image_height"): - setattr(preproc, "image_height", int(self._stream.height)) + preproc.image_height = int(self._stream.height) except Exception as e: logger.debug(f"Failed to align preprocessor target size with stream resolution: {e}", exc_info=True) @@ -396,7 +395,7 @@ def prepare_frame_tensors(self, device: torch.device, dtype: torch.dtype, batch_ self._prepared_dtype = dtype self._prepared_batch = batch_size - def _get_cached_sdxl_conditioning(self, ctx: "StepCtx") -> Optional[Dict[str, torch.Tensor]]: + def _get_cached_sdxl_conditioning(self, ctx: StepCtx) -> Optional[Dict[str, torch.Tensor]]: """Get cached SDXL conditioning to avoid repeated preparation""" if not self._is_sdxl or ctx.sdxl_cond is None: return None @@ -739,7 +738,7 @@ def _load_pytorch_controlnet_model( controlnet = controlnet.to(device=self.device, dtype=self.dtype) # Track model_id for updater diffing try: - setattr(controlnet, "model_id", model_id) + controlnet.model_id = model_id except Exception as e: logger.debug(f"Failed to set model_id attribute on controlnet: {e}", exc_info=True) return controlnet diff --git a/src/streamdiffusion/modules/image_processing_module.py b/src/streamdiffusion/modules/image_processing_module.py index 58ea6a166..20955be82 100644 --- a/src/streamdiffusion/modules/image_processing_module.py +++ b/src/streamdiffusion/modules/image_processing_module.py @@ -54,7 +54,7 @@ def add_processor(self, proc_config: Dict[str, Any]) -> None: # Apply parameters (same pattern as ControlNet) processor_params = proc_config.get("params", {}) if processor_params: - if hasattr(processor, "params") and isinstance(getattr(processor, "params"), dict): + if hasattr(processor, "params") and isinstance(processor.params, dict): processor.params.update(processor_params) for name, value in processor_params.items(): try: @@ -65,21 +65,21 @@ def add_processor(self, proc_config: Dict[str, Any]) -> None: # Set order for sequential execution order = proc_config.get("order", len(self.processors)) - setattr(processor, "order", order) + processor.order = order # Set enabled state - setattr(processor, "enabled", enabled) + processor.enabled = enabled # Align preprocessor target size with stream resolution (same as ControlNet) if hasattr(self, "_stream"): try: - if hasattr(processor, "params") and isinstance(getattr(processor, "params"), dict): + if hasattr(processor, "params") and isinstance(processor.params, dict): processor.params["image_width"] = int(self._stream.width) processor.params["image_height"] = int(self._stream.height) if hasattr(processor, "image_width"): - setattr(processor, "image_width", int(self._stream.width)) + processor.image_width = int(self._stream.width) if hasattr(processor, "image_height"): - setattr(processor, "image_height", int(self._stream.height)) + processor.image_height = int(self._stream.height) except Exception as e: logger.debug(f"Failed to align processor target size with stream resolution: {e}", exc_info=True) diff --git a/src/streamdiffusion/modules/ipadapter_module.py b/src/streamdiffusion/modules/ipadapter_module.py index caca6c3c1..4a7a23d27 100644 --- a/src/streamdiffusion/modules/ipadapter_module.py +++ b/src/streamdiffusion/modules/ipadapter_module.py @@ -12,7 +12,6 @@ from streamdiffusion.preprocessing.orchestrator_user import OrchestratorUser from streamdiffusion.utils.reporting import report_error - logger = logging.getLogger(__name__) @@ -365,11 +364,11 @@ def install(self, stream) -> None: # Expose IPAdapter instance as single source of truth try: - setattr(stream, "ipadapter", ipadapter) + stream.ipadapter = ipadapter # Extend IPAdapter with our custom attributes since diffusers IPAdapter doesn't expose current state - setattr(ipadapter, "weight_type", self.config.weight_type) # For build_layer_weights - setattr(ipadapter, "scale", float(self.config.scale)) # Track current scale - setattr(ipadapter, "enabled", bool(self.config.enabled)) # Track enabled state + ipadapter.weight_type = self.config.weight_type # For build_layer_weights + ipadapter.scale = float(self.config.scale) # Track current scale + ipadapter.enabled = bool(self.config.enabled) # Track enabled state except Exception as e: logger.debug(f"Failed to attach custom attributes to ipadapter instance: {e}", exc_info=True) diff --git a/src/streamdiffusion/modules/latent_processing_module.py b/src/streamdiffusion/modules/latent_processing_module.py index 67ce0f9b0..d71f2039c 100644 --- a/src/streamdiffusion/modules/latent_processing_module.py +++ b/src/streamdiffusion/modules/latent_processing_module.py @@ -52,7 +52,7 @@ def add_processor(self, proc_config: Dict[str, Any]) -> None: # Apply parameters (same pattern as ControlNet) processor_params = proc_config.get("params", {}) if processor_params: - if hasattr(processor, "params") and isinstance(getattr(processor, "params"), dict): + if hasattr(processor, "params") and isinstance(processor.params, dict): processor.params.update(processor_params) for name, value in processor_params.items(): try: @@ -63,10 +63,10 @@ def add_processor(self, proc_config: Dict[str, Any]) -> None: # Set order for sequential execution order = proc_config.get("order", len(self.processors)) - setattr(processor, "order", order) + processor.order = order # Set enabled state - setattr(processor, "enabled", enabled) + processor.enabled = enabled # Pipeline reference is now automatically handled by the factory function diff --git a/src/streamdiffusion/param_schema.py b/src/streamdiffusion/param_schema.py index 6ecaa222a..3f60bd3e5 100644 --- a/src/streamdiffusion/param_schema.py +++ b/src/streamdiffusion/param_schema.py @@ -33,7 +33,6 @@ from dataclasses import dataclass from typing import Any, Dict, List, Literal, Tuple - # Interpolation-method aliases shared by the wrapper and updater signatures. PromptInterpolationMethod = Literal["linear", "slerp", "cosine_weighted"] SeedInterpolationMethod = Literal["linear", "slerp"] diff --git a/src/streamdiffusion/pip_utils.py b/src/streamdiffusion/pip_utils.py index 4a28c0a0e..670c5ca9b 100644 --- a/src/streamdiffusion/pip_utils.py +++ b/src/streamdiffusion/pip_utils.py @@ -8,7 +8,6 @@ from packaging.version import Version - python = sys.executable index_url = os.environ.get("INDEX_URL", "") uv = shutil.which("uv") diff --git a/src/streamdiffusion/pipeline.py b/src/streamdiffusion/pipeline.py index 7ebe828dc..432ceeddf 100644 --- a/src/streamdiffusion/pipeline.py +++ b/src/streamdiffusion/pipeline.py @@ -27,7 +27,6 @@ from streamdiffusion.stream_parameter_updater import StreamParameterUpdater from streamdiffusion.tools.gpu_profiler import profiler - logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) diff --git a/src/streamdiffusion/preprocessing/__init__.py b/src/streamdiffusion/preprocessing/__init__.py index c52a8a2ed..2850365db 100644 --- a/src/streamdiffusion/preprocessing/__init__.py +++ b/src/streamdiffusion/preprocessing/__init__.py @@ -4,11 +4,10 @@ from .postprocessing_orchestrator import PostprocessingOrchestrator from .preprocessing_orchestrator import PreprocessingOrchestrator - __all__ = [ - "PreprocessingOrchestrator", - "PostprocessingOrchestrator", - "PipelinePreprocessingOrchestrator", "BaseOrchestrator", "OrchestratorUser", + "PipelinePreprocessingOrchestrator", + "PostprocessingOrchestrator", + "PreprocessingOrchestrator", ] diff --git a/src/streamdiffusion/preprocessing/base_orchestrator.py b/src/streamdiffusion/preprocessing/base_orchestrator.py index 85db7b488..41b21146c 100644 --- a/src/streamdiffusion/preprocessing/base_orchestrator.py +++ b/src/streamdiffusion/preprocessing/base_orchestrator.py @@ -5,7 +5,6 @@ import torch - logger = logging.getLogger(__name__) # Type variables for generic orchestrator diff --git a/src/streamdiffusion/preprocessing/orchestrator_user.py b/src/streamdiffusion/preprocessing/orchestrator_user.py index e731540b9..5955f6fe1 100644 --- a/src/streamdiffusion/preprocessing/orchestrator_user.py +++ b/src/streamdiffusion/preprocessing/orchestrator_user.py @@ -29,7 +29,7 @@ def attach_preprocessing_orchestrator(self, stream) -> None: orchestrator = PreprocessingOrchestrator( device=stream.device, dtype=stream.dtype, max_workers=4, pipeline_ref=stream ) - setattr(stream, "preprocessing_orchestrator", orchestrator) + stream.preprocessing_orchestrator = orchestrator self._preprocessing_orchestrator = orchestrator def attach_postprocessing_orchestrator(self, stream) -> None: @@ -40,7 +40,7 @@ def attach_postprocessing_orchestrator(self, stream) -> None: orchestrator = PostprocessingOrchestrator( device=stream.device, dtype=stream.dtype, max_workers=4, pipeline_ref=stream ) - setattr(stream, "postprocessing_orchestrator", orchestrator) + stream.postprocessing_orchestrator = orchestrator self._postprocessing_orchestrator = orchestrator def attach_pipeline_preprocessing_orchestrator(self, stream) -> None: @@ -51,5 +51,5 @@ def attach_pipeline_preprocessing_orchestrator(self, stream) -> None: orchestrator = PipelinePreprocessingOrchestrator( device=stream.device, dtype=stream.dtype, max_workers=4, pipeline_ref=stream ) - setattr(stream, "pipeline_preprocessing_orchestrator", orchestrator) + stream.pipeline_preprocessing_orchestrator = orchestrator self._pipeline_preprocessing_orchestrator = orchestrator diff --git a/src/streamdiffusion/preprocessing/pipeline_preprocessing_orchestrator.py b/src/streamdiffusion/preprocessing/pipeline_preprocessing_orchestrator.py index 382e874fb..51b9e2d33 100644 --- a/src/streamdiffusion/preprocessing/pipeline_preprocessing_orchestrator.py +++ b/src/streamdiffusion/preprocessing/pipeline_preprocessing_orchestrator.py @@ -5,7 +5,6 @@ from .base_orchestrator import BaseOrchestrator - logger = logging.getLogger(__name__) diff --git a/src/streamdiffusion/preprocessing/postprocessing_orchestrator.py b/src/streamdiffusion/preprocessing/postprocessing_orchestrator.py index ef5dceb32..e9c8c847e 100644 --- a/src/streamdiffusion/preprocessing/postprocessing_orchestrator.py +++ b/src/streamdiffusion/preprocessing/postprocessing_orchestrator.py @@ -5,7 +5,6 @@ from .base_orchestrator import BaseOrchestrator - logger = logging.getLogger(__name__) diff --git a/src/streamdiffusion/preprocessing/preprocessing_orchestrator.py b/src/streamdiffusion/preprocessing/preprocessing_orchestrator.py index 577678bd6..6957a2f14 100644 --- a/src/streamdiffusion/preprocessing/preprocessing_orchestrator.py +++ b/src/streamdiffusion/preprocessing/preprocessing_orchestrator.py @@ -9,7 +9,6 @@ from .base_orchestrator import BaseOrchestrator - logger = logging.getLogger(__name__) # Type alias for control image input diff --git a/src/streamdiffusion/preprocessing/processors/__init__.py b/src/streamdiffusion/preprocessing/processors/__init__.py index adf53e158..be4601aad 100644 --- a/src/streamdiffusion/preprocessing/processors/__init__.py +++ b/src/streamdiffusion/preprocessing/processors/__init__.py @@ -1,27 +1,29 @@ -from .base import BasePreprocessor, PipelineAwareProcessor from typing import Any + +from .base import BasePreprocessor, PipelineAwareProcessor +from .blur import BlurPreprocessor from .canny import CannyPreprocessor from .depth import DepthPreprocessor -from .openpose import OpenPosePreprocessor -from .lineart import LineartPreprocessor -from .standard_lineart import StandardLineartPreprocessor -from .passthrough import PassthroughPreprocessor from .external import ExternalPreprocessor -from .soft_edge import SoftEdgePreprocessor -from .hed import HEDPreprocessor -from .ipadapter_embedding import IPAdapterEmbeddingPreprocessor from .faceid_embedding import FaceIDEmbeddingPreprocessor from .feedback import FeedbackPreprocessor +from .hed import HEDPreprocessor +from .ipadapter_embedding import IPAdapterEmbeddingPreprocessor from .latent_feedback import LatentFeedbackPreprocessor +from .lineart import LineartPreprocessor +from .openpose import OpenPosePreprocessor +from .passthrough import PassthroughPreprocessor +from .realesrgan_trt import RealESRGANProcessor from .scribble import ScribblePreprocessor from .sharpen import SharpenPreprocessor +from .soft_edge import SoftEdgePreprocessor +from .standard_lineart import StandardLineartPreprocessor from .upscale import UpscalePreprocessor -from .blur import BlurPreprocessor -from .realesrgan_trt import RealESRGANProcessor # Try to import TensorRT preprocessors - might not be available on all systems try: from .depth_tensorrt import DepthAnythingTensorrtPreprocessor + DEPTH_TENSORRT_AVAILABLE = True except ImportError: DepthAnythingTensorrtPreprocessor = None @@ -29,6 +31,7 @@ try: from .pose_tensorrt import YoloNasPoseTensorrtPreprocessor + POSE_TENSORRT_AVAILABLE = True except ImportError: YoloNasPoseTensorrtPreprocessor = None @@ -60,6 +63,7 @@ try: from .temporal_net_tensorrt import TemporalNetTensorRTPreprocessor + TEMPORAL_NET_TENSORRT_AVAILABLE = True except ImportError: TemporalNetTensorRTPreprocessor = None @@ -67,6 +71,7 @@ try: from .mediapipe_pose import MediaPipePosePreprocessor + MEDIAPIPE_POSE_AVAILABLE = True except ImportError: MediaPipePosePreprocessor = None @@ -74,6 +79,7 @@ try: from .mediapipe_segmentation import MediaPipeSegmentationPreprocessor + MEDIAPIPE_SEGMENTATION_AVAILABLE = True except ImportError: MediaPipeSegmentationPreprocessor = None @@ -97,7 +103,7 @@ "upscale": UpscalePreprocessor, "blur": BlurPreprocessor, "realesrgan_trt": RealESRGANProcessor, -} +} # Add TensorRT preprocessors if available if DEPTH_TENSORRT_AVAILABLE: @@ -130,27 +136,29 @@ def get_preprocessor_class(name: str) -> type: """ Get a preprocessor class by name - + Args: name: Name of the preprocessor - + Returns: Preprocessor class - + Raises: ValueError: If preprocessor name is not found """ if name not in _preprocessor_registry: available = ", ".join(_preprocessor_registry.keys()) raise ValueError(f"Unknown preprocessor '{name}'. Available: {available}") - + return _preprocessor_registry[name] -def get_preprocessor(name: str, pipeline_ref: Any = None, normalization_context: str = 'controlnet', params: Any = None) -> BasePreprocessor: +def get_preprocessor( + name: str, pipeline_ref: Any = None, normalization_context: str = "controlnet", params: Any = None +) -> BasePreprocessor: """ Get a preprocessor by name - + Args: name: Name of the preprocessor pipeline_ref: Pipeline reference for pipeline-aware processors (required for some processors) @@ -158,20 +166,25 @@ def get_preprocessor(name: str, pipeline_ref: Any = None, normalization_context: - 'controlnet': Expects/produces [0,1] range for ControlNet conditioning - 'pipeline': Expects/produces [-1,1] range for pipeline image processing - 'latent': Works in latent space (no normalization needed) - + Returns: Preprocessor instance - + Raises: ValueError: If preprocessor name is not found or pipeline_ref missing for pipeline-aware processor """ processor_class = get_preprocessor_class(name) - + # Check if this is a pipeline-aware processor - if hasattr(processor_class, 'requires_sync_processing') and processor_class.requires_sync_processing: + if hasattr(processor_class, "requires_sync_processing") and processor_class.requires_sync_processing: if pipeline_ref is None: raise ValueError(f"Processor '{name}' requires a pipeline_ref") - return processor_class(pipeline_ref=pipeline_ref, normalization_context=normalization_context, _registry_name=name, **(params or {})) + return processor_class( + pipeline_ref=pipeline_ref, + normalization_context=normalization_context, + _registry_name=name, + **(params or {}), + ) else: return processor_class(normalization_context=normalization_context, _registry_name=name, **(params or {})) @@ -179,7 +192,7 @@ def get_preprocessor(name: str, pipeline_ref: Any = None, normalization_context: def register_preprocessor(name: str, preprocessor_class): """ Register a new preprocessor - + Args: name: Name to register under preprocessor_class: Preprocessor class @@ -194,25 +207,25 @@ def list_preprocessors(): __all__ = [ "BasePreprocessor", - "PipelineAwareProcessor", "CannyPreprocessor", - "DepthPreprocessor", - "OpenPosePreprocessor", - "LineartPreprocessor", - "StandardLineartPreprocessor", - "PassthroughPreprocessor", + "DepthPreprocessor", "ExternalPreprocessor", - "SoftEdgePreprocessor", - "HEDPreprocessor", - "ScribblePreprocessor", - "IPAdapterEmbeddingPreprocessor", "FaceIDEmbeddingPreprocessor", "FeedbackPreprocessor", + "HEDPreprocessor", + "IPAdapterEmbeddingPreprocessor", "LatentFeedbackPreprocessor", + "LineartPreprocessor", + "OpenPosePreprocessor", + "PassthroughPreprocessor", + "PipelineAwareProcessor", + "ScribblePreprocessor", + "SoftEdgePreprocessor", + "StandardLineartPreprocessor", "get_preprocessor", "get_preprocessor_class", - "register_preprocessor", "list_preprocessors", + "register_preprocessor", ] if DEPTH_TENSORRT_AVAILABLE: @@ -241,14 +254,15 @@ def list_preprocessors(): # region Custom Processor Discovery -import logging -import os import importlib.util import inspect +import logging +import os from pathlib import Path _logger = logging.getLogger(__name__) + def _discover_custom_processors(): """Auto-discover custom processors from repo_root/custom_processors/ folder.""" if os.getenv("STREAMDIFFUSION_DISABLE_CUSTOM_PROCESSORS") == "1": @@ -262,7 +276,7 @@ def _discover_custom_processors(): return _logger.info("Scanning custom_processors/ for custom processors...") for item in custom_dir.iterdir(): - if not item.is_dir() or item.name.startswith(('.', '_')): + if not item.is_dir() or item.name.startswith((".", "_")): continue manifest_file = item / "processors.yaml" if manifest_file.exists(): @@ -272,20 +286,22 @@ def _discover_custom_processors(): except Exception as e: _logger.error(f"Custom processor discovery failed: {e}") + def _load_processor_collection(collection_dir, manifest_file): """Load processors from a collection with processors.yaml manifest.""" import yaml + try: - with open(manifest_file, 'r') as f: + with open(manifest_file) as f: manifest = yaml.safe_load(f) - processor_files = manifest.get('processors', []) + processor_files = manifest.get("processors", []) if not processor_files: _logger.warning(f"Collection '{collection_dir.name}' has empty processors list") return _logger.info(f"Loading collection '{collection_dir.name}' ({len(processor_files)} processors)") for proc_file in processor_files: if isinstance(proc_file, dict): - filename, enabled = proc_file.get('file'), proc_file.get('enabled', True) + filename, enabled = proc_file.get("file"), proc_file.get("enabled", True) if not enabled: continue else: @@ -298,24 +314,28 @@ def _load_processor_collection(collection_dir, manifest_file): except Exception as e: _logger.error(f"Failed to load collection {collection_dir.name}: {e}") + def _load_processor_folder_auto(folder): """Auto-discover processors by scanning for .py files (no manifest).""" _logger.info(f"Auto-scanning folder: {folder.name}") for py_file in folder.glob("*.py"): - if py_file.name.startswith('_') or py_file.name in ['base.py', 'setup.py']: + if py_file.name.startswith("_") or py_file.name in ["base.py", "setup.py"]: continue _load_processor_from_file(py_file, py_file.stem) + def _load_processor_from_file(file_path, proc_name): """Load and register a processor class from a Python file.""" try: spec = importlib.util.spec_from_file_location( - f"custom_processors.{file_path.parent.name}.{file_path.stem}", file_path) + f"custom_processors.{file_path.parent.name}.{file_path.stem}", file_path + ) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) found_classes = [ - (name, obj) for name, obj in inspect.getmembers(module, inspect.isclass) + (name, obj) + for name, obj in inspect.getmembers(module, inspect.isclass) if issubclass(obj, (BasePreprocessor, PipelineAwareProcessor)) and obj not in [BasePreprocessor, PipelineAwareProcessor] ] @@ -330,5 +350,6 @@ def _load_processor_from_file(file_path, proc_name): except Exception as e: _logger.error(f" Failed to load {file_path.name}: {e}") + _discover_custom_processors() -# endregion \ No newline at end of file +# endregion diff --git a/src/streamdiffusion/preprocessing/processors/base.py b/src/streamdiffusion/preprocessing/processors/base.py index 71fdcf9f1..d7644cab7 100644 --- a/src/streamdiffusion/preprocessing/processors/base.py +++ b/src/streamdiffusion/preprocessing/processors/base.py @@ -1,14 +1,14 @@ import logging from abc import ABC, abstractmethod -from typing import Any, Dict, Optional, Set, Tuple, Union +from typing import Any, Dict, Set, Tuple, Union + +import numpy as np import torch import torch.nn.functional as F -import numpy as np from PIL import Image from streamdiffusion.tools.gpu_profiler import profiler - _pil_fallback_warned: Set[str] = set() # per-class warning dedup _base_logger = logging.getLogger(__name__) @@ -23,10 +23,10 @@ class BasePreprocessor(ABC): # Used by the residency guard test and the one-time PIL-fallback warning below. gpu_native: bool = False - def __init__(self, normalization_context: str = 'controlnet', **kwargs): + def __init__(self, normalization_context: str = "controlnet", **kwargs): """ Initialize the preprocessor - + Args: normalization_context: Context for normalization handling. - 'controlnet': Expects/produces [0,1] range for ControlNet conditioning @@ -36,15 +36,15 @@ def __init__(self, normalization_context: str = 'controlnet', **kwargs): """ self.params = kwargs self.normalization_context = normalization_context - self.device = kwargs.get('device', 'cuda' if torch.cuda.is_available() else 'cpu') - self.dtype = kwargs.get('dtype', torch.float16) - + self.device = kwargs.get("device", "cuda" if torch.cuda.is_available() else "cpu") + self.dtype = kwargs.get("dtype", torch.float16) + @classmethod def get_preprocessor_metadata(cls) -> Dict[str, Any]: """ Get comprehensive metadata for this preprocessor. Subclasses should override this to define their specific metadata. - + Returns: Dictionary containing: - display_name: Human-readable name @@ -56,9 +56,9 @@ def get_preprocessor_metadata(cls) -> Dict[str, Any]: "display_name": cls.__name__.replace("Preprocessor", ""), "description": f"Preprocessor for {cls.__name__.replace('Preprocessor', '').lower()}", "parameters": {}, - "use_cases": [] + "use_cases": [], } - + def process(self, image: Union[Image.Image, np.ndarray, torch.Tensor]) -> Image.Image: """ Template method - handles all common operations @@ -67,7 +67,7 @@ def process(self, image: Union[Image.Image, np.ndarray, torch.Tensor]) -> Image. with profiler.region("proc.core"): processed = self._process_core(image) return self._ensure_target_size(processed) - + def process_tensor(self, image_tensor: torch.Tensor) -> torch.Tensor: """ Template method for GPU tensor processing @@ -76,14 +76,14 @@ def process_tensor(self, image_tensor: torch.Tensor) -> torch.Tensor: with profiler.region("proc.core"): processed = self._process_tensor_core(tensor) return self._ensure_target_size_tensor(processed) - + @abstractmethod def _process_core(self, image: Image.Image) -> Image.Image: """ Subclasses implement ONLY their specific algorithm """ pass - + def _process_tensor_core(self, tensor: torch.Tensor) -> torch.Tensor: """ Optional GPU processing (fallback to PIL if not overridden). @@ -108,7 +108,6 @@ def _process_tensor_core(self, tensor: torch.Tensor) -> torch.Tensor: with profiler.region("proc.pil_to_tensor"): return self.pil_to_tensor(processed_pil) - def _ensure_target_size(self, image: Image.Image) -> Image.Image: """ Centralized PIL resize logic @@ -117,7 +116,7 @@ def _ensure_target_size(self, image: Image.Image) -> Image.Image: if image.size != (target_width, target_height): return image.resize((target_width, target_height), Image.LANCZOS) return image - + def _ensure_target_size_tensor(self, tensor: torch.Tensor) -> torch.Tensor: """ Centralized tensor resize logic @@ -125,7 +124,7 @@ def _ensure_target_size_tensor(self, tensor: torch.Tensor) -> torch.Tensor: target_width, target_height = self.get_target_dimensions() current_size = tensor.shape[-2:] target_size = (target_height, target_width) - + if current_size != target_size: if tensor.dim() == 3: tensor = tensor.unsqueeze(0) @@ -135,24 +134,24 @@ def _ensure_target_size_tensor(self, tensor: torch.Tensor) -> torch.Tensor: if tensor.shape[0] == 1: tensor = tensor.squeeze(0) return tensor - + def validate_tensor_input(self, image_tensor: torch.Tensor) -> torch.Tensor: """ Validate and normalize tensor input for processing - + Args: image_tensor: Input tensor - + Returns: Tensor in CHW format, on correct device Range: [0,1] if input was [0,255], otherwise preserves input range - + Note: This preserves [-1,1] tensors (from pipeline) since max() <= 1.0 """ # Handle batch dimension if image_tensor.dim() == 4: image_tensor = image_tensor[0] # Take first image from batch - + # Convert to CHW format if needed if image_tensor.dim() == 3 and image_tensor.shape[0] not in [1, 3]: # Likely HWC format, convert to CHW @@ -169,16 +168,16 @@ def validate_tensor_input(self, image_tensor: torch.Tensor) -> torch.Tensor: if _was_uint8: image_tensor = image_tensor / 255.0 - + return image_tensor - + def tensor_to_pil(self, tensor: torch.Tensor) -> Image.Image: """ Convert tensor to PIL Image (minimize CPU transfers) - + Args: tensor: Input tensor - + Returns: PIL Image """ @@ -187,39 +186,39 @@ def tensor_to_pil(self, tensor: torch.Tensor) -> Image.Image: tensor = tensor[0] if tensor.dim() == 3 and tensor.shape[0] in [1, 3]: tensor = tensor.permute(1, 2, 0) - + # Convert to numpy (unavoidable for PIL) if tensor.is_cuda: tensor = tensor.cpu() - + # Convert to uint8 if tensor.max() <= 1.0: tensor = (tensor * 255).clamp(0, 255).to(torch.uint8) else: tensor = tensor.clamp(0, 255).to(torch.uint8) - + array = tensor.numpy() - + if array.shape[-1] == 3: - return Image.fromarray(array, 'RGB') + return Image.fromarray(array, "RGB") elif array.shape[-1] == 1: - return Image.fromarray(array.squeeze(-1), 'L') + return Image.fromarray(array.squeeze(-1), "L") else: return Image.fromarray(array) - + def pil_to_tensor(self, image: Image.Image) -> torch.Tensor: """ Convert PIL Image to tensor on GPU - + Args: image: PIL Image - + Returns: Tensor on correct device """ # Convert to numpy first array = np.array(image) - + # Convert to tensor if len(array.shape) == 2: # Grayscale tensor = torch.from_numpy(array).float() / 255.0 @@ -227,25 +226,25 @@ def pil_to_tensor(self, image: Image.Image) -> torch.Tensor: else: # RGB tensor = torch.from_numpy(array).float() / 255.0 tensor = tensor.permute(2, 0, 1) # HWC to CHW - + # Move to device tensor = tensor.to(device=self.device, dtype=self.dtype) return tensor.unsqueeze(0) # Add batch dimension - + def validate_input(self, image: Union[Image.Image, np.ndarray, torch.Tensor]) -> Image.Image: """ Convert input to PIL Image for processing - + Args: image: Input image in various formats - + Returns: PIL Image """ if isinstance(image, torch.Tensor): # Use tensor_to_pil method for better handling return self.tensor_to_pil(image) - + if isinstance(image, np.ndarray): # Ensure uint8 format if image.dtype != np.uint8: @@ -253,83 +252,83 @@ def validate_input(self, image: Union[Image.Image, np.ndarray, torch.Tensor]) -> image = (image * 255).astype(np.uint8) else: image = image.astype(np.uint8) - + # Convert to PIL Image if len(image.shape) == 3: - image = Image.fromarray(image, 'RGB') + image = Image.fromarray(image, "RGB") else: - image = Image.fromarray(image, 'L') - + image = Image.fromarray(image, "L") + if not isinstance(image, Image.Image): raise ValueError(f"Unsupported image type: {type(image)}") - + return image - + def get_target_dimensions(self) -> Tuple[int, int]: """ Get target output dimensions (width, height) """ # Check for explicit width/height parameters first - width = self.params.get('image_width') - height = self.params.get('image_height') - + width = self.params.get("image_width") + height = self.params.get("image_height") + if width is not None and height is not None: return (width, height) - + # Fallback to square resolution for backwards compatibility - resolution = self.params.get('image_resolution', 512) + resolution = self.params.get("image_resolution", 512) return (resolution, resolution) - + def __call__(self, image: Union[Image.Image, np.ndarray, torch.Tensor], **kwargs) -> Image.Image: """ Process an image (convenience method) - + Args: image: Input image **kwargs: Additional parameters to override defaults - + Returns: Processed PIL Image """ # Update parameters for this call params = {**self.params, **kwargs} - + # Store original params and update original_params = self.params self.params = params - + try: result = self.process(image) finally: # Restore original params self.params = original_params - + return result class PipelineAwareProcessor(BasePreprocessor): """ Abstract base class for processors that need access to pipeline state (previous outputs). - - This base class marks processors as requiring synchronous processing to avoid + + This base class marks processors as requiring synchronous processing to avoid temporal artifacts and ensures they have access to pipeline references. - + Usage: class MyProcessor(PipelineAwareProcessor): pass - + Examples: - FeedbackPreprocessor: Needs previous diffusion output - TemporalNetPreprocessor: Needs previous frame for optical flow """ - + # Class attribute to mark processors as requiring sync processing requires_sync_processing = True - - def __init__(self, pipeline_ref: Any, normalization_context: str = 'controlnet', **kwargs): + + def __init__(self, pipeline_ref: Any, normalization_context: str = "controlnet", **kwargs): """ Initialize pipeline-aware functionality - + Args: pipeline_ref: Reference to the StreamDiffusion pipeline instance (required) normalization_context: Context for normalization handling @@ -338,4 +337,4 @@ def __init__(self, pipeline_ref: Any, normalization_context: str = 'controlnet', if pipeline_ref is None: raise ValueError(f"{self.__class__.__name__} requires a pipeline_ref") super().__init__(normalization_context=normalization_context, **kwargs) - self.pipeline_ref = pipeline_ref \ No newline at end of file + self.pipeline_ref = pipeline_ref diff --git a/src/streamdiffusion/preprocessing/processors/canny.py b/src/streamdiffusion/preprocessing/processors/canny.py index e51c04437..366960b85 100644 --- a/src/streamdiffusion/preprocessing/processors/canny.py +++ b/src/streamdiffusion/preprocessing/processors/canny.py @@ -5,7 +5,7 @@ from PIL import Image from .base import BasePreprocessor -from .category_params import EDGE_SMOOTHNESS_PARAM, apply_edge_smoothness +from .category_params import EDGE_SMOOTHNESS_PARAM class CannyPreprocessor(BasePreprocessor): @@ -17,7 +17,6 @@ class CannyPreprocessor(BasePreprocessor): gpu_native = True # _process_tensor_core uses conv2d — no CPU/PIL round-trip - @classmethod def get_preprocessor_metadata(cls): return { @@ -72,7 +71,6 @@ def _process_core(self, image: Image.Image) -> Image.Image: low_threshold = self.params.get("low_threshold", 100) high_threshold = self.params.get("high_threshold", 200) - edges = cv2.Canny(gray, low_threshold, high_threshold) edges_rgb = cv2.cvtColor(edges, cv2.COLOR_GRAY2RGB) diff --git a/src/streamdiffusion/preprocessing/processors/category_params.py b/src/streamdiffusion/preprocessing/processors/category_params.py index 49e4c5875..7ea69c5a9 100644 --- a/src/streamdiffusion/preprocessing/processors/category_params.py +++ b/src/streamdiffusion/preprocessing/processors/category_params.py @@ -23,7 +23,6 @@ import torch import torch.nn.functional as F - # --------------------------------------------------------------------------- # Canonical metadata fragments # --------------------------------------------------------------------------- diff --git a/src/streamdiffusion/preprocessing/processors/depth.py b/src/streamdiffusion/preprocessing/processors/depth.py index c940d7189..b2abad32e 100644 --- a/src/streamdiffusion/preprocessing/processors/depth.py +++ b/src/streamdiffusion/preprocessing/processors/depth.py @@ -6,6 +6,7 @@ try: import torch from transformers import pipeline + TRANSFORMERS_AVAILABLE = True except ImportError: TRANSFORMERS_AVAILABLE = False @@ -14,29 +15,25 @@ class DepthPreprocessor(BasePreprocessor): """ Depth estimation preprocessor for ControlNet using MiDaS - + Estimates depth maps from input images using the MiDaS depth estimation model. """ - + @classmethod def get_preprocessor_metadata(cls): return { "display_name": "Depth Estimation", "description": "Estimates depth from the input image using MiDaS. Good for adding depth-based control to generation.", - "parameters": { - - }, - "use_cases": ["3D-aware generation", "Depth preservation", "Scene understanding"] + "parameters": {}, + "use_cases": ["3D-aware generation", "Depth preservation", "Scene understanding"], } - - def __init__(self, - model_name: str = "Intel/dpt-large", - detect_resolution: int = 512, - image_resolution: int = 512, - **kwargs): + + def __init__( + self, model_name: str = "Intel/dpt-large", detect_resolution: int = 512, image_resolution: int = 512, **kwargs + ): """ Initialize depth preprocessor - + Args: model_name: Name of the depth estimation model to use detect_resolution: Resolution for depth detection @@ -45,57 +42,51 @@ def __init__(self, """ if not TRANSFORMERS_AVAILABLE: raise ImportError( - "transformers library is required for depth preprocessing. " - "Install it with: pip install transformers" + "transformers library is required for depth preprocessing. Install it with: pip install transformers" ) - + super().__init__( - model_name=model_name, - detect_resolution=detect_resolution, - image_resolution=image_resolution, - **kwargs + model_name=model_name, detect_resolution=detect_resolution, image_resolution=image_resolution, **kwargs ) - + self._depth_estimator = None - + @property def depth_estimator(self): """Lazy loading of the depth estimation model""" if self._depth_estimator is None: - model_name = self.params.get('model_name', 'Intel/dpt-large') + model_name = self.params.get("model_name", "Intel/dpt-large") print(f"Loading depth estimation model: {model_name}") self._depth_estimator = pipeline( - 'depth-estimation', - model=model_name, - device=0 if torch.cuda.is_available() else -1 + "depth-estimation", model=model_name, device=0 if torch.cuda.is_available() else -1 ) return self._depth_estimator - + def _process_core(self, image: Image.Image) -> Image.Image: """ Apply depth estimation to the input image """ - detect_resolution = self.params.get('detect_resolution', 512) + detect_resolution = self.params.get("detect_resolution", 512) image_resized = image.resize((detect_resolution, detect_resolution), Image.LANCZOS) - + depth_result = self.depth_estimator(image_resized) - depth_map = depth_result['depth'] - - if hasattr(depth_map, 'cpu'): + depth_map = depth_result["depth"] + + if hasattr(depth_map, "cpu"): depth_np = depth_map.cpu().numpy() else: depth_np = np.array(depth_map) - + depth_min = depth_np.min() depth_max = depth_np.max() if depth_max > depth_min: depth_normalized = ((depth_np - depth_min) / (depth_max - depth_min) * 255).astype(np.uint8) else: depth_normalized = np.zeros_like(depth_np, dtype=np.uint8) - + depth_rgb = np.stack([depth_normalized] * 3, axis=-1) return Image.fromarray(depth_rgb) - + def _process_tensor_core(self, image_tensor: torch.Tensor) -> torch.Tensor: """ Process tensor directly on GPU for depth estimation. @@ -114,45 +105,43 @@ def _process_tensor_core(self, image_tensor: torch.Tensor) -> torch.Tensor: ) detect_resolution = self.params.get("detect_resolution", 512) current_size = image_tensor.shape[-2:] - + if current_size != (detect_resolution, detect_resolution): import torch.nn.functional as F + if image_tensor.dim() == 3: image_tensor = image_tensor.unsqueeze(0) - + resized_tensor = F.interpolate( - image_tensor, - size=(detect_resolution, detect_resolution), - mode='bilinear', - align_corners=False + image_tensor, size=(detect_resolution, detect_resolution), mode="bilinear", align_corners=False ) - + if image_tensor.shape[0] == 1: resized_tensor = resized_tensor.squeeze(0) else: resized_tensor = image_tensor - + pil_image = self.tensor_to_pil(resized_tensor) - + depth_result = self.depth_estimator(pil_image) - depth_map = depth_result['depth'] - - if hasattr(depth_map, 'to'): + depth_map = depth_result["depth"] + + if hasattr(depth_map, "to"): depth_tensor = depth_map.to(device=self.device, dtype=self.dtype) else: depth_np = np.array(depth_map) depth_tensor = torch.from_numpy(depth_np).to(device=self.device, dtype=self.dtype) - + depth_min = depth_tensor.min() depth_max = depth_tensor.max() if depth_max > depth_min: depth_normalized = (depth_tensor - depth_min) / (depth_max - depth_min) else: depth_normalized = torch.zeros_like(depth_tensor) - + if depth_normalized.dim() == 2: depth_rgb = depth_normalized.unsqueeze(0).repeat(3, 1, 1) else: depth_rgb = depth_normalized - - return depth_rgb \ No newline at end of file + + return depth_rgb diff --git a/src/streamdiffusion/preprocessing/processors/depth_tensorrt.py b/src/streamdiffusion/preprocessing/processors/depth_tensorrt.py index a41667f26..81efa5d7f 100644 --- a/src/streamdiffusion/preprocessing/processors/depth_tensorrt.py +++ b/src/streamdiffusion/preprocessing/processors/depth_tensorrt.py @@ -1,12 +1,13 @@ -#NOTE: ported from https://github.com/yuvraj108c/ComfyUI-Depth-Anything-Tensorrt +# NOTE: ported from https://github.com/yuvraj108c/ComfyUI-Depth-Anything-Tensorrt import os + +import cv2 import numpy as np import torch import torch.nn.functional as F -import cv2 from PIL import Image -from typing import Union, Optional + from .base import BasePreprocessor from .category_params import DEPTH_GRADE_PARAMS, apply_depth_grade, apply_depth_grade_numpy from .trt_base import TENSORRT_AVAILABLE, TensorRTEngine # shared engine wrapper @@ -19,6 +20,7 @@ class DepthAnythingTensorrtPreprocessor(BasePreprocessor): Uses TensorRT-optimized Depth Anything model for fast depth estimation. """ + @classmethod def get_preprocessor_metadata(cls): return { @@ -29,11 +31,8 @@ def get_preprocessor_metadata(cls): }, "use_cases": ["High-performance depth estimation", "Real-time applications", "3D-aware generation"], } - def __init__(self, - engine_path: str = None, - detect_resolution: int = 518, - image_resolution: int = 512, - **kwargs): + + def __init__(self, engine_path: str = None, detect_resolution: int = 518, image_resolution: int = 512, **kwargs): """ Initialize TensorRT depth preprocessor @@ -50,10 +49,7 @@ def __init__(self, ) super().__init__( - engine_path=engine_path, - detect_resolution=detect_resolution, - image_resolution=image_resolution, - **kwargs + engine_path=engine_path, detect_resolution=detect_resolution, image_resolution=image_resolution, **kwargs ) self._engine = None @@ -62,7 +58,7 @@ def __init__(self, def engine(self): """Lazy loading of the TensorRT engine""" if self._engine is None: - engine_path = self.params.get('engine_path') + engine_path = self.params.get("engine_path") if engine_path is None: raise ValueError( "engine_path is required for TensorRT depth preprocessing. " @@ -85,16 +81,13 @@ def _process_core(self, image: Image.Image) -> Image.Image: """ Apply TensorRT depth estimation to the input image """ - detect_resolution = self.params.get('detect_resolution', 518) + detect_resolution = self.params.get("detect_resolution", 518) image_tensor = torch.from_numpy(np.array(image)).float() / 255.0 image_tensor = image_tensor.permute(2, 0, 1).unsqueeze(0) image_resized = F.interpolate( - image_tensor, - size=(detect_resolution, detect_resolution), - mode='bilinear', - align_corners=False + image_tensor, size=(detect_resolution, detect_resolution), mode="bilinear", align_corners=False ) if torch.cuda.is_available(): @@ -102,7 +95,7 @@ def _process_core(self, image: Image.Image) -> Image.Image: cuda_stream = torch.cuda.current_stream().cuda_stream result = self.engine.infer({"input": image_resized}, cuda_stream) - depth = result['output'] + depth = result["output"] depth = np.reshape(depth.cpu().numpy(), (detect_resolution, detect_resolution)) @@ -135,16 +128,15 @@ def _process_tensor_core(self, image_tensor: torch.Tensor) -> torch.Tensor: if not image_tensor.is_cuda: image_tensor = image_tensor.cuda() - detect_resolution = self.params.get('detect_resolution', 518) + detect_resolution = self.params.get("detect_resolution", 518) image_resized = torch.nn.functional.interpolate( - image_tensor, size=(detect_resolution, detect_resolution), - mode='bilinear', align_corners=False + image_tensor, size=(detect_resolution, detect_resolution), mode="bilinear", align_corners=False ) cuda_stream = torch.cuda.current_stream().cuda_stream result = self.engine.infer({"input": image_resized}, cuda_stream) - depth_tensor = result['output'] + depth_tensor = result["output"] depth_tensor = depth_tensor.squeeze() if depth_tensor.dim() > 2 else depth_tensor diff --git a/src/streamdiffusion/preprocessing/processors/feedback.py b/src/streamdiffusion/preprocessing/processors/feedback.py index 01461b703..fac7e56e6 100644 --- a/src/streamdiffusion/preprocessing/processors/feedback.py +++ b/src/streamdiffusion/preprocessing/processors/feedback.py @@ -1,31 +1,32 @@ +from typing import Any + import torch from PIL import Image -from typing import Union, Optional, Any + from .base import PipelineAwareProcessor class FeedbackPreprocessor(PipelineAwareProcessor): """ Feedback preprocessor for ControlNet - + Creates a configurable blend between the current input image and the previous frame's diffusion output. This creates a feedback loop where each generated frame influences the next generation, while allowing control over the blend strength for stability and creative effects. - + Formula: output = (1 - feedback_strength) * input_image + feedback_strength * previous_output - + Examples: - feedback_strength = 0.0: Pure passthrough (input only) - feedback_strength = 0.5: 50/50 blend (default) - feedback_strength = 1.0: Pure feedback (previous output only) - + The preprocessor accesses the pipeline's prev_image_result to get the previous output. For the first frame (when no previous output exists), it falls back to the input image. """ gpu_native = True # _process_tensor_core blends tensors on GPU — no CPU/PIL round-trip - @classmethod def get_preprocessor_metadata(cls): return { @@ -37,21 +38,29 @@ def get_preprocessor_metadata(cls): "default": 0.5, "range": [0.0, 1.0], "step": 0.01, - "description": "Strength of feedback blend (0.0 = pure input, 1.0 = pure feedback)" + "description": "Strength of feedback blend (0.0 = pure input, 1.0 = pure feedback)", } }, - "use_cases": ["Temporal consistency", "Video-like generation", "Smooth transitions", "Deforum", "Blast off"] + "use_cases": [ + "Temporal consistency", + "Video-like generation", + "Smooth transitions", + "Deforum", + "Blast off", + ], } - - def __init__(self, - pipeline_ref: Any, - normalization_context: str = 'controlnet', - image_resolution: int = 512, - feedback_strength: float = 0.5, - **kwargs): + + def __init__( + self, + pipeline_ref: Any, + normalization_context: str = "controlnet", + image_resolution: int = 512, + feedback_strength: float = 0.5, + **kwargs, + ): """ Initialize feedback preprocessor - + Args: pipeline_ref: Reference to the StreamDiffusion pipeline instance (required) normalization_context: Context for normalization handling @@ -64,41 +73,42 @@ def __init__(self, normalization_context=normalization_context, image_resolution=image_resolution, feedback_strength=feedback_strength, - **kwargs + **kwargs, ) self.feedback_strength = max(0.0, min(1.0, feedback_strength)) # Clamp to [0, 1] self._first_frame = True - + def reset(self): """Reset the processor state (useful for new sequences)""" self._first_frame = True - + def _process_core(self, image: Image.Image) -> Image.Image: """ Process using configurable blend of input image + previous frame output - + Args: image: Current input image - + Returns: Blended PIL Image (blend strength controlled by feedback_strength), or input image for first frame """ # Check if we have a pipeline reference and previous output - if (self.pipeline_ref is not None and - hasattr(self.pipeline_ref, 'prev_image_result') and - self.pipeline_ref.prev_image_result is not None and - not self._first_frame): - + if ( + self.pipeline_ref is not None + and hasattr(self.pipeline_ref, "prev_image_result") + and self.pipeline_ref.prev_image_result is not None + and not self._first_frame + ): prev_output_tensor = self.pipeline_ref.prev_image_result # Convert previous output tensor to PIL Image if prev_output_tensor.dim() == 4: prev_output_tensor = prev_output_tensor[0] # Remove batch dimension - + # Context-aware normalization handling - if self.normalization_context == 'controlnet': + if self.normalization_context == "controlnet": # ControlNet context: Convert from [-1, 1] (VAE output) to [0, 1] (ControlNet input) prev_output_tensor = (prev_output_tensor / 2.0 + 0.5).clamp(0, 1) - elif self.normalization_context == 'pipeline': + elif self.normalization_context == "pipeline": # Pipeline context: prev_output is already [-1, 1], but pil_to_tensor produces [0, 1] # So we need to convert input to [-1, 1] to match prev_output # Convert prev_output to [0, 1] for blending in standard image space @@ -106,15 +116,15 @@ def _process_core(self, image: Image.Image) -> Image.Image: else: # Unknown context - assume controlnet for backward compatibility prev_output_tensor = (prev_output_tensor / 2.0 + 0.5).clamp(0, 1) - + # Convert both to tensors for blending prev_output_pil = self.tensor_to_pil(prev_output_tensor) input_tensor = self.pil_to_tensor(image).squeeze(0) # Remove batch dim for blending prev_tensor = self.pil_to_tensor(prev_output_pil).squeeze(0) - + # Blend with configurable strength (both tensors now in [0, 1] range) blended_tensor = (1 - self.feedback_strength) * input_tensor + self.feedback_strength * prev_tensor - + # Convert back to PIL blended_pil = self.tensor_to_pil(blended_tensor) return blended_pil @@ -122,35 +132,36 @@ def _process_core(self, image: Image.Image) -> Image.Image: # First frame, no pipeline ref, or no previous output available - use input image self._first_frame = False return image - + def _process_tensor_core(self, tensor: torch.Tensor) -> torch.Tensor: """ Process using configurable blend of input tensor + previous frame output (GPU-optimized path) - + Args: tensor: Current input tensor - + Returns: Blended tensor (blend strength controlled by feedback_strength), or input tensor for first frame """ # Check if we have a pipeline reference and previous output - if (self.pipeline_ref is not None and - hasattr(self.pipeline_ref, 'prev_image_result') and - self.pipeline_ref.prev_image_result is not None and - not self._first_frame): - + if ( + self.pipeline_ref is not None + and hasattr(self.pipeline_ref, "prev_image_result") + and self.pipeline_ref.prev_image_result is not None + and not self._first_frame + ): prev_output = self.pipeline_ref.prev_image_result input_tensor = tensor - + # Context-aware normalization handling - if self.normalization_context == 'controlnet': + if self.normalization_context == "controlnet": # ControlNet context: prev_output is [-1, 1] from VAE, input is [0, 1] # Convert prev_output from [-1, 1] to [0, 1] to match input prev_output = (prev_output / 2.0 + 0.5).clamp(0, 1) # Normalize input tensor to [0, 1] if needed if input_tensor.max() > 1.0: input_tensor = input_tensor / 255.0 - elif self.normalization_context == 'pipeline': + elif self.normalization_context == "pipeline": # Pipeline context: both prev_output and input_tensor are in [-1, 1] range # - prev_output comes from VAE decode (always [-1, 1]) # - input_tensor arrives as [-1, 1] from image_processor.preprocess() @@ -160,17 +171,20 @@ def _process_tensor_core(self, tensor: torch.Tensor) -> torch.Tensor: else: # Unknown context - log warning and assume controlnet behavior for backward compatibility import logging - logging.warning(f"FeedbackPreprocessor: Unknown normalization_context '{self.normalization_context}', using controlnet behavior") + + logging.warning( + f"FeedbackPreprocessor: Unknown normalization_context '{self.normalization_context}', using controlnet behavior" + ) prev_output = (prev_output / 2.0 + 0.5).clamp(0, 1) if input_tensor.max() > 1.0: input_tensor = input_tensor / 255.0 - + # Ensure both tensors have same format for blending if prev_output.dim() == 4 and prev_output.shape[0] == 1: prev_output = prev_output[0] # Remove batch dimension if input_tensor.dim() == 4 and input_tensor.shape[0] == 1: input_tensor = input_tensor[0] # Remove batch dimension - + # Resize if dimensions don't match if prev_output.shape != input_tensor.shape: # Use the input tensor's shape as target @@ -179,18 +193,18 @@ def _process_tensor_core(self, tensor: torch.Tensor) -> torch.Tensor: if prev_output.dim() == 3: prev_output = prev_output.unsqueeze(0) prev_output = torch.nn.functional.interpolate( - prev_output, size=target_size, mode='bilinear', align_corners=False + prev_output, size=target_size, mode="bilinear", align_corners=False ) if prev_output.shape[0] == 1: prev_output = prev_output.squeeze(0) - + # Blend with configurable strength blended_tensor = (1 - self.feedback_strength) * input_tensor + self.feedback_strength * prev_output - + # Ensure correct output format if blended_tensor.dim() == 3: blended_tensor = blended_tensor.unsqueeze(0) # Add batch dimension back - + # Ensure correct device and dtype blended_tensor = blended_tensor.to(device=self.device, dtype=self.dtype) return blended_tensor diff --git a/src/streamdiffusion/preprocessing/processors/hed.py b/src/streamdiffusion/preprocessing/processors/hed.py index db32f6282..bef030edc 100644 --- a/src/streamdiffusion/preprocessing/processors/hed.py +++ b/src/streamdiffusion/preprocessing/processors/hed.py @@ -1,11 +1,12 @@ -import torch import numpy as np +import torch from PIL import Image from .base import BasePreprocessor, _base_logger, _pil_fallback_warned try: from controlnet_aux import HEDdetector + CONTROLNET_AUX_AVAILABLE = True except ImportError: CONTROLNET_AUX_AVAILABLE = False @@ -15,79 +16,77 @@ class HEDPreprocessor(BasePreprocessor): """ HED (Holistically-Nested Edge Detection) preprocessor - + Uses controlnet_aux HEDdetector for high-quality edge detection. """ - + _model_cache = {} - + @classmethod def get_preprocessor_metadata(cls): return { "display_name": "HED Edge Detection", "description": "Holistically-Nested Edge Detection for clean, structured edge maps.", "parameters": { - "safe": { - "type": "bool", - "default": True, - "description": "Whether to use safe mode for edge detection" - } + "safe": {"type": "bool", "default": True, "description": "Whether to use safe mode for edge detection"} }, - "use_cases": ["Structured edge detection", "Clean architectural edges", "Line art generation"] + "use_cases": ["Structured edge detection", "Clean architectural edges", "Line art generation"], } - + def __init__(self, safe: bool = True, **kwargs): if not CONTROLNET_AUX_AVAILABLE: - raise ImportError("controlnet_aux is required for HED preprocessor. Install with: pip install controlnet_aux") - + raise ImportError( + "controlnet_aux is required for HED preprocessor. Install with: pip install controlnet_aux" + ) + super().__init__(**kwargs) self.safe = safe self.model = None self._load_model() - + def _load_model(self): """Load controlnet_aux HED model with caching""" cache_key = f"hed_{self.device}" - + if cache_key in self._model_cache: self.model = self._model_cache[cache_key] return - + print("HEDPreprocessor: Loading controlnet_aux HED model") try: # Initialize HED detector self.model = HEDdetector.from_pretrained("lllyasviel/Annotators") - if hasattr(self.model, 'to'): + if hasattr(self.model, "to"): self.model = self.model.to(self.device) - + # Cache the model self._model_cache[cache_key] = self.model print(f"HEDPreprocessor: Successfully loaded model on {self.device}") - + except Exception as e: raise RuntimeError(f"Failed to load HED model: {e}") from e - + def _process_core(self, image: Image.Image) -> Image.Image: """Apply HED edge detection to the input image""" # Get target dimensions target_width, target_height = self.get_target_dimensions() - + # Process with controlnet_aux result = self.model(image, output_type="pil") - + # Ensure result is PIL Image if not isinstance(result, Image.Image): if isinstance(result, np.ndarray): result = Image.fromarray(result) else: raise ValueError(f"Unexpected result type: {type(result)}") - + # Resize to target size if needed if result.size != (target_width, target_height): result = result.resize((target_width, target_height), Image.LANCZOS) - + return result - + def _process_tensor_core(self, image_tensor: torch.Tensor) -> torch.Tensor: """ HED processing via tensor I/O. @@ -108,24 +107,18 @@ def _process_tensor_core(self, image_tensor: torch.Tensor) -> torch.Tensor: pil_image = self.tensor_to_pil(image_tensor) processed_pil = self._process_core(pil_image) return self.pil_to_tensor(processed_pil) - - + @classmethod - def create_optimized(cls, device: str = 'cuda', dtype: torch.dtype = torch.float16, **kwargs): + def create_optimized(cls, device: str = "cuda", dtype: torch.dtype = torch.float16, **kwargs): """ Create an optimized HED preprocessor - + Args: device: Target device ('cuda' or 'cpu') dtype: Data type for inference **kwargs: Additional parameters - + Returns: Optimized HEDPreprocessor instance """ - return cls( - device=device, - dtype=dtype, - safe=True, - **kwargs - ) \ No newline at end of file + return cls(device=device, dtype=dtype, safe=True, **kwargs) diff --git a/src/streamdiffusion/preprocessing/processors/hed_tensorrt.py b/src/streamdiffusion/preprocessing/processors/hed_tensorrt.py index 783921ea2..f572aabec 100644 --- a/src/streamdiffusion/preprocessing/processors/hed_tensorrt.py +++ b/src/streamdiffusion/preprocessing/processors/hed_tensorrt.py @@ -16,7 +16,6 @@ from .trt_base import SelfBuildingTRTPreprocessor, _first_output - logger = logging.getLogger(__name__) try: diff --git a/src/streamdiffusion/preprocessing/processors/ipadapter_embedding.py b/src/streamdiffusion/preprocessing/processors/ipadapter_embedding.py index 896d11e5e..cb2106c17 100644 --- a/src/streamdiffusion/preprocessing/processors/ipadapter_embedding.py +++ b/src/streamdiffusion/preprocessing/processors/ipadapter_embedding.py @@ -2,32 +2,34 @@ import torch from PIL import Image -from .base import BasePreprocessor + from streamdiffusion.tools.gpu_profiler import profiler +from .base import BasePreprocessor + class IPAdapterEmbeddingPreprocessor(BasePreprocessor): """ Preprocessor that generates IPAdapter embeddings instead of spatial conditioning. Leverages existing preprocessing infrastructure for parallel IPAdapter embedding generation. """ - + @classmethod def get_preprocessor_metadata(cls): return { "display_name": "IPAdapter Embedding", "description": "Generates IPAdapter embeddings for style transfer and image conditioning instead of spatial control maps.", "parameters": {}, - "use_cases": ["Style transfer", "Image conditioning", "Semantic control", "Content-aware generation"] + "use_cases": ["Style transfer", "Image conditioning", "Semantic control", "Content-aware generation"], } - + def __init__(self, ipadapter: Any, **kwargs): super().__init__(**kwargs) self.ipadapter = ipadapter # Verify the ipadapter has the required method - if not hasattr(ipadapter, 'get_image_embeds'): + if not hasattr(ipadapter, "get_image_embeds"): raise ValueError("IPAdapterEmbeddingPreprocessor: ipadapter must have 'get_image_embeds' method") - + # Create dedicated CUDA stream for IPAdapter processing to avoid TensorRT conflicts self._ipadapter_stream = torch.cuda.Stream() if torch.cuda.is_available() else None # CUDA event for GPU-side stream sync — CPU thread NOT blocked (vs .synchronize()). @@ -40,7 +42,6 @@ def __init__(self, ipadapter: Any, **kwargs): self._last_input_ptr: int = -1 self._cached_embeds: Optional[Tuple[torch.Tensor, torch.Tensor]] = None - def _process_core(self, image: Image.Image) -> Tuple[torch.Tensor, torch.Tensor]: """Returns (positive_embeds, negative_embeds) instead of processed image""" if self._ipadapter_stream is not None: @@ -64,16 +65,15 @@ def _process_core(self, image: Image.Image) -> Tuple[torch.Tensor, torch.Tensor] # Mark tensors as owned by the default stream (cross-stream memory safety) if hasattr(image_embeds, "record_stream"): image_embeds.record_stream(torch.cuda.current_stream()) - if hasattr(negative_embeds, 'record_stream'): + if hasattr(negative_embeds, "record_stream"): negative_embeds.record_stream(torch.cuda.current_stream()) else: # Fallback for non-CUDA environments with profiler.region("ipa.clip_encode"): image_embeds, negative_embeds = self.ipadapter.get_image_embeds(images=[image]) - return image_embeds, negative_embeds - + def _process_tensor_core(self, tensor: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: """GPU-optimized path for tensor inputs. @@ -96,7 +96,6 @@ def _process_tensor_core(self, tensor: torch.Tensor) -> Tuple[torch.Tensor, torc self._cached_embeds = result return result - def process(self, image: Union[Image.Image, torch.Tensor]) -> Tuple[torch.Tensor, torch.Tensor]: """Override base process to return embeddings tuple instead of PIL Image""" if isinstance(image, torch.Tensor): @@ -104,9 +103,9 @@ def process(self, image: Union[Image.Image, torch.Tensor]) -> Tuple[torch.Tensor else: image = self.validate_input(image) result = self._process_core(image) - + return result - + def process_tensor(self, image_tensor: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: """Override base process_tensor to return embeddings tuple""" tensor = self.validate_tensor_input(image_tensor) diff --git a/src/streamdiffusion/preprocessing/processors/lineart.py b/src/streamdiffusion/preprocessing/processors/lineart.py index 030043c4d..6e9607dee 100644 --- a/src/streamdiffusion/preprocessing/processors/lineart.py +++ b/src/streamdiffusion/preprocessing/processors/lineart.py @@ -5,7 +5,6 @@ from .base import BasePreprocessor - logger = logging.getLogger(__name__) try: diff --git a/src/streamdiffusion/preprocessing/processors/mediapipe_pose.py b/src/streamdiffusion/preprocessing/processors/mediapipe_pose.py index 3d7c72093..f6984998d 100644 --- a/src/streamdiffusion/preprocessing/processors/mediapipe_pose.py +++ b/src/streamdiffusion/preprocessing/processors/mediapipe_pose.py @@ -7,7 +7,6 @@ from .base import BasePreprocessor - try: import mediapipe as mp @@ -293,7 +292,7 @@ def detector(self): print("MediaPipePosePreprocessor.detector: GPU delegate available") except Exception as gpu_error: print(f"MediaPipePosePreprocessor.detector: GPU delegate failed ({gpu_error}), using CPU") - base_options = mp.tasks.BaseOptions(delegate=mp.tasks.BaseOptions.Delegate.CPU) + base_options = mp.tasks.BaseOptions(delegate=mp.tasks.BaseOptions.Delegate.CPU) # noqa: F841 # TODO: pre-existing, untouched by this refactor # Create detector with optimized settings print( @@ -398,7 +397,7 @@ def _mediapipe_to_openpose( # OPTIMIZATION: Vectorized mapping using advanced indexing # Only map valid indices that exist in landmarks_data - valid_mask = MEDIAPIPE_INDICES < len(landmarks_data) + valid_mask = len(landmarks_data) > MEDIAPIPE_INDICES valid_mp_indices = MEDIAPIPE_INDICES[valid_mask] valid_op_indices = OPENPOSE_INDICES[valid_mask] @@ -498,7 +497,7 @@ def _draw_hand_keypoints(self, image: np.ndarray, hand_landmarks: List, is_left_ return image h, w = image.shape[:2] - confidence_threshold = self.params.get("confidence_threshold", 0.3) + confidence_threshold = self.params.get("confidence_threshold", 0.3) # noqa: F841 # TODO: pre-existing, untouched by this refactor # Standard hand connections (21 landmarks per hand) hand_connections = [ diff --git a/src/streamdiffusion/preprocessing/processors/mediapipe_segmentation.py b/src/streamdiffusion/preprocessing/processors/mediapipe_segmentation.py index 0f4893f23..64ef04706 100644 --- a/src/streamdiffusion/preprocessing/processors/mediapipe_segmentation.py +++ b/src/streamdiffusion/preprocessing/processors/mediapipe_segmentation.py @@ -7,7 +7,6 @@ from .base import BasePreprocessor - try: import mediapipe as mp diff --git a/src/streamdiffusion/preprocessing/processors/normal_bae_tensorrt.py b/src/streamdiffusion/preprocessing/processors/normal_bae_tensorrt.py index 7558da481..ebb0577dc 100644 --- a/src/streamdiffusion/preprocessing/processors/normal_bae_tensorrt.py +++ b/src/streamdiffusion/preprocessing/processors/normal_bae_tensorrt.py @@ -34,7 +34,6 @@ from .base import BasePreprocessor from .trt_base import TENSORRT_AVAILABLE, SelfBuildingTRTPreprocessor, _first_output - logger = logging.getLogger(__name__) try: diff --git a/src/streamdiffusion/preprocessing/processors/openpose.py b/src/streamdiffusion/preprocessing/processors/openpose.py index 53ac8afe2..eb591cfd4 100644 --- a/src/streamdiffusion/preprocessing/processors/openpose.py +++ b/src/streamdiffusion/preprocessing/processors/openpose.py @@ -2,9 +2,8 @@ from .base import BasePreprocessor - try: - import cv2 + import cv2 # noqa: F401 # TODO: pre-existing, untouched by this refactor OPENCV_AVAILABLE = True except ImportError: diff --git a/src/streamdiffusion/preprocessing/processors/passthrough.py b/src/streamdiffusion/preprocessing/processors/passthrough.py index 5fd9e8d90..cd128f4cb 100644 --- a/src/streamdiffusion/preprocessing/processors/passthrough.py +++ b/src/streamdiffusion/preprocessing/processors/passthrough.py @@ -1,14 +1,13 @@ -import numpy as np -from PIL import Image import torch -from typing import Union, Optional +from PIL import Image + from .base import BasePreprocessor class PassthroughPreprocessor(BasePreprocessor): """ Passthrough preprocessor for ControlNet - + Simply passes the input image through without any processing. Useful for ControlNets that expect the raw input image, such as: - Tile ControlNet @@ -18,7 +17,6 @@ class PassthroughPreprocessor(BasePreprocessor): gpu_native = True # _process_tensor_core is a no-op identity — no CPU/PIL round-trip - @classmethod def get_preprocessor_metadata(cls): return { @@ -36,30 +34,25 @@ def get_preprocessor_metadata(cls): "Image-to-image with structure preservation", ], } - - def __init__(self, - image_resolution: int = 512, - **kwargs): + + def __init__(self, image_resolution: int = 512, **kwargs): """ Initialize passthrough preprocessor - + Args: image_resolution: Output image resolution **kwargs: Additional parameters (ignored for passthrough) """ - super().__init__( - image_resolution=image_resolution, - **kwargs - ) - + super().__init__(image_resolution=image_resolution, **kwargs) + def _process_core(self, image: Image.Image) -> Image.Image: """ Pass through the input image with no processing """ return image - + def _process_tensor_core(self, tensor: torch.Tensor) -> torch.Tensor: """ Pass through tensor with no processing """ - return tensor \ No newline at end of file + return tensor diff --git a/src/streamdiffusion/preprocessing/processors/pose_tensorrt.py b/src/streamdiffusion/preprocessing/processors/pose_tensorrt.py index 1688a1131..d57d86f7f 100644 --- a/src/streamdiffusion/preprocessing/processors/pose_tensorrt.py +++ b/src/streamdiffusion/preprocessing/processors/pose_tensorrt.py @@ -1,12 +1,13 @@ -#NOTE: ported from https://github.com/yuvraj108c/ComfyUI-YoloNasPose-Tensorrt +# NOTE: ported from https://github.com/yuvraj108c/ComfyUI-YoloNasPose-Tensorrt import os + +import cv2 import numpy as np import torch import torch.nn.functional as F -import cv2 from PIL import Image -from typing import Union, Optional, List, Tuple + from .base import BasePreprocessor from .category_params import POSE_DRAW_PARAMS from .trt_base import TENSORRT_AVAILABLE, TensorRTEngine # shared engine wrapper @@ -89,20 +90,48 @@ def iterate_over_batch_predictions(predictions, batch_size): else: pred_scores = batch_scores[image_index, :num_detection_in_image] pred_boxes = batch_boxes[image_index, :num_detection_in_image] - pred_joints = batch_joints[image_index, :num_detection_in_image].reshape( - (num_detection_in_image, -1, 3)) + pred_joints = batch_joints[image_index, :num_detection_in_image].reshape((num_detection_in_image, -1, 3)) yield image_index, pred_boxes, pred_scores, pred_joints + # precompute edge links define skeleton connections (COCO format) -edge_links = [[0, 17], [13, 15], [14, 16], [12, 14], [12, 17], [5, 6], - [11, 13], [7, 9], [5, 7], [17, 11], [6, 8], [8, 10], - [1, 3], [0, 1], [0, 2], [2, 4]] +edge_links = [ + [0, 17], + [13, 15], + [14, 16], + [12, 14], + [12, 17], + [5, 6], + [11, 13], + [7, 9], + [5, 7], + [17, 11], + [6, 8], + [8, 10], + [1, 3], + [0, 1], + [0, 2], + [2, 4], +] edge_colors = [ - [255, 0, 0], [255, 85, 0], [170, 255, 0], [85, 255, 0], [85, 255, 0], - [85, 0, 255], [255, 170, 0], [0, 177, 58], [0, 179, 119], [179, 179, 0], - [0, 119, 179], [0, 179, 179], [119, 0, 179], [179, 0, 179], [178, 0, 118], [178, 0, 118] + [255, 0, 0], + [255, 85, 0], + [170, 255, 0], + [85, 255, 0], + [85, 255, 0], + [85, 0, 255], + [255, 170, 0], + [0, 177, 58], + [0, 179, 119], + [179, 179, 0], + [0, 119, 179], + [0, 179, 179], + [119, 0, 179], + [179, 0, 179], + [178, 0, 118], + [178, 0, 118], ] @@ -121,8 +150,7 @@ def show_predictions_from_batch_format( keypoint_radius: Keypoint dot radius in pixels. """ try: - image_index, pred_boxes, pred_scores, pred_joints = next( - iter(iterate_over_batch_predictions(predictions, 1))) + image_index, pred_boxes, pred_scores, pred_joints = next(iter(iterate_over_batch_predictions(predictions, 1))) except Exception as e: raise RuntimeError(f"show_predictions_from_batch_format: Error in iterate_over_batch_predictions: {e}") from e @@ -184,11 +212,7 @@ def get_preprocessor_metadata(cls): ], } - def __init__(self, - engine_path: str = None, - detect_resolution: int = 640, - image_resolution: int = 512, - **kwargs): + def __init__(self, engine_path: str = None, detect_resolution: int = 640, image_resolution: int = 512, **kwargs): """ Initialize TensorRT pose preprocessor @@ -205,10 +229,7 @@ def __init__(self, ) super().__init__( - engine_path=engine_path, - detect_resolution=detect_resolution, - image_resolution=image_resolution, - **kwargs + engine_path=engine_path, detect_resolution=detect_resolution, image_resolution=image_resolution, **kwargs ) self._engine = None @@ -219,7 +240,7 @@ def __init__(self, def engine(self): """Lazy loading of the TensorRT engine""" if self._engine is None: - engine_path = self.params.get('engine_path') + engine_path = self.params.get("engine_path") if engine_path is None: raise ValueError( "engine_path is required for TensorRT pose preprocessing. " @@ -240,16 +261,13 @@ def _process_core(self, image: Image.Image) -> Image.Image: """ Apply TensorRT pose estimation to the input image """ - detect_resolution = self.params.get('detect_resolution', 640) + detect_resolution = self.params.get("detect_resolution", 640) image_tensor = torch.from_numpy(np.array(image)).float() / 255.0 image_tensor = image_tensor.permute(2, 0, 1).unsqueeze(0) image_resized = F.interpolate( - image_tensor, - size=(detect_resolution, detect_resolution), - mode='bilinear', - align_corners=False + image_tensor, size=(detect_resolution, detect_resolution), mode="bilinear", align_corners=False ) image_resized_uint8 = (image_resized * 255.0).type(torch.uint8) @@ -293,11 +311,10 @@ def _process_tensor_core(self, image_tensor: torch.Tensor) -> torch.Tensor: if not image_tensor.is_cuda: image_tensor = image_tensor.cuda() - detect_resolution = self.params.get('detect_resolution', 640) + detect_resolution = self.params.get("detect_resolution", 640) image_resized = torch.nn.functional.interpolate( - image_tensor, size=(detect_resolution, detect_resolution), - mode='bilinear', align_corners=False + image_tensor, size=(detect_resolution, detect_resolution), mode="bilinear", align_corners=False ) image_resized_uint8 = (image_resized * 255.0).type(torch.uint8) diff --git a/src/streamdiffusion/preprocessing/processors/realesrgan_trt.py b/src/streamdiffusion/preprocessing/processors/realesrgan_trt.py index 151b5ca77..d08697e42 100644 --- a/src/streamdiffusion/preprocessing/processors/realesrgan_trt.py +++ b/src/streamdiffusion/preprocessing/processors/realesrgan_trt.py @@ -1,23 +1,24 @@ # NOTE: ported from https://github.com/yuvraj108c/ComfyUI-Upscaler-Tensorrt -import os -import torch +import logging +from collections import OrderedDict +from pathlib import Path +from typing import Tuple + import numpy as np -from PIL import Image -from typing import Optional, Tuple import requests +import torch +from PIL import Image from tqdm import tqdm -import hashlib -import logging -from pathlib import Path -from collections import OrderedDict -from .base import BasePreprocessor from streamdiffusion.tools.gpu_profiler import profiler +from .base import BasePreprocessor + # Try to import spandrel for model loading try: from spandrel import ModelLoader + SPANDREL_AVAILABLE = True except ImportError: SPANDREL_AVAILABLE = False @@ -25,9 +26,11 @@ # Try to import TensorRT dependencies try: import tensorrt as trt - from streamdiffusion.acceleration.tensorrt.utilities import engine_from_bytes, bytes_from_path + + from streamdiffusion.acceleration.tensorrt.utilities import bytes_from_path, engine_from_bytes + TRT_AVAILABLE = True - + # Numpy to PyTorch dtype mapping (same as depth_tensorrt.py) numpy_to_torch_dtype_dict = { np.uint8: torch.uint8, @@ -41,27 +44,28 @@ np.complex64: torch.complex64, np.complex128: torch.complex128, } - + # Handle bool type for numpy compatibility (same as depth_tensorrt.py) if np.version.full_version >= "1.24.0": numpy_to_torch_dtype_dict[np.bool_] = torch.bool else: numpy_to_torch_dtype_dict[np.bool] = torch.bool - + except ImportError: TRT_AVAILABLE = False class RealESRGANEngine: """TensorRT engine wrapper for RealESRGAN inference (following depth_tensorrt pattern)""" - + def __init__(self, engine_path): self.engine_path = engine_path self.engine = None self.context = None self.tensors = OrderedDict() - + import threading + self._inference_lock = threading.Lock() def load(self): @@ -80,13 +84,13 @@ def allocate_buffers(self, input_shape, device="cuda"): # Set input shape for dynamic sizing input_name = "input" self.context.set_input_shape(input_name, input_shape) - + # Allocate tensors for all bindings for idx in range(self.engine.num_io_tensors): name = self.engine.get_tensor_name(idx) shape = self.context.get_tensor_shape(name) dtype = trt.nptype(self.engine.get_tensor_dtype(name)) - + # Convert numpy dtype to torch dtype if dtype == np.float32: torch_dtype = torch.float32 @@ -94,7 +98,7 @@ def allocate_buffers(self, input_shape, device="cuda"): torch_dtype = torch.float16 else: torch_dtype = torch.float32 - + tensor = torch.empty(tuple(shape), dtype=torch_dtype, device=device) self.tensors[name] = tensor @@ -124,19 +128,20 @@ def infer(self, feed_dict, stream=None): if not success: raise RuntimeError("RealESRGANEngine: TensorRT inference failed") - return self.tensors + logger = logging.getLogger(__name__) + class RealESRGANProcessor(BasePreprocessor): """ RealESRGAN 2x upscaling processor with automatic model download, ONNX export, and TensorRT acceleration. """ - + MODEL_URL = "https://huggingface.co/ai-forever/Real-ESRGAN/resolve/main/RealESRGAN_x2.pth?download=true" - - @classmethod + + @classmethod def get_preprocessor_metadata(cls): return { "display_name": "RealESRGAN 2x", @@ -145,30 +150,30 @@ def get_preprocessor_metadata(cls): "enable_tensorrt": { "type": "bool", "default": True, - "description": "Use TensorRT acceleration for faster inference" + "description": "Use TensorRT acceleration for faster inference", }, "force_rebuild": { - "type": "bool", + "type": "bool", "default": False, - "description": "Force rebuild TensorRT engine even if it exists" - } + "description": "Force rebuild TensorRT engine even if it exists", + }, }, - "use_cases": ["High-quality upscaling", "Real-time 2x enlargement", "Image enhancement"] + "use_cases": ["High-quality upscaling", "Real-time 2x enlargement", "Image enhancement"], } - + def __init__(self, enable_tensorrt: bool = True, force_rebuild: bool = False, **kwargs): super().__init__(enable_tensorrt=enable_tensorrt, force_rebuild=force_rebuild, **kwargs) self.enable_tensorrt = enable_tensorrt and TRT_AVAILABLE self.force_rebuild = force_rebuild self.scale_factor = 2 # RealESRGAN 2x model - + # Model paths self.models_dir = Path("models") / "realesrgan" self.models_dir.mkdir(parents=True, exist_ok=True) self.model_path = self.models_dir / "RealESRGAN_x2.pth" self.onnx_path = self.models_dir / "RealESRGAN_x2.onnx" self.engine_path = self.models_dir / f"RealESRGAN_x2_{trt.__version__ if TRT_AVAILABLE else 'notrt'}.trt" - + # Model state self.pytorch_model = None self._engine = None # Lazy loading like depth processor @@ -176,6 +181,7 @@ def __init__(self, enable_tensorrt: bool = True, force_rebuild: bool = False, ** # Thread safety for engine initialization import threading + self._engine_lock = threading.Lock() @property @@ -184,35 +190,38 @@ def engine(self): if self._engine is None: if not self.engine_path.exists(): raise FileNotFoundError(f"TensorRT engine not found: {self.engine_path}") - + self._engine = RealESRGANEngine(str(self.engine_path)) self._engine.load() self._engine.activate() - + # Allocate buffers for standard input size (will be reallocated as needed) standard_shape = (1, 3, 512, 512) self._engine.allocate_buffers(standard_shape, device=self.device) - + return self._engine - + def _download_file(self, url: str, save_path: Path): """Download file with progress bar""" if save_path.exists(): return - + response = requests.get(url, stream=True) response.raise_for_status() - - total_size = int(response.headers.get('content-length', 0)) - - with open(save_path, 'wb') as file, tqdm( - desc=f"Downloading {save_path.name}", - total=total_size, - unit='iB', - unit_scale=True, - unit_divisor=1024, - colour='green' - ) as progress_bar: + + total_size = int(response.headers.get("content-length", 0)) + + with ( + open(save_path, "wb") as file, + tqdm( + desc=f"Downloading {save_path.name}", + total=total_size, + unit="iB", + unit_scale=True, + unit_divisor=1024, + colour="green", + ) as progress_bar, + ): for data in response.iter_content(chunk_size=1024): size = file.write(data) progress_bar.update(size) @@ -232,108 +241,108 @@ def _ensure_model_ready(self): # Download model if needed if not self.model_path.exists(): self._download_file(self.MODEL_URL, self.model_path) - + # Load PyTorch model if self.pytorch_model is None: self._load_pytorch_model() - + # Setup TensorRT if enabled if self.enable_tensorrt: self._setup_tensorrt() - + def _load_pytorch_model(self): """Load PyTorch model from file""" if not SPANDREL_AVAILABLE: # Fallback loading without spandrel - state_dict = torch.load(self.model_path, map_location=self.device) + state_dict = torch.load(self.model_path, map_location=self.device) # noqa: F841 # TODO: pre-existing, untouched by this refactor # This is a simplified approach - real implementation would need model architecture return - + model_descriptor = ModelLoader().load_from_file(str(self.model_path)) # Don't force dtype conversion as it can cause type mismatches # Let the model keep its native dtype and convert inputs as needed self.pytorch_model = model_descriptor.model.eval().to(device=self.device) - model_dtype = next(self.pytorch_model.parameters()).dtype - + model_dtype = next(self.pytorch_model.parameters()).dtype # noqa: F841 # TODO: pre-existing, untouched by this refactor + def _export_to_onnx(self): """Export PyTorch model to ONNX format""" if self.onnx_path.exists() and not self.force_rebuild: return - + if self.pytorch_model is None: self._load_pytorch_model() - + if self.pytorch_model is None: return - + # Test with small input for export test_input = torch.randn(1, 3, 256, 256).to(self.device) - + dynamic_axes = { "input": {0: "batch_size", 2: "height", 3: "width"}, "output": {0: "batch_size", 2: "height", 3: "width"}, } - + with torch.no_grad(): torch.onnx.export( self.pytorch_model, test_input, str(self.onnx_path), verbose=False, - input_names=['input'], - output_names=['output'], + input_names=["input"], + output_names=["output"], opset_version=17, export_params=True, dynamic_axes=dynamic_axes, ) - + def _setup_tensorrt(self): """Setup TensorRT engine""" if not TRT_AVAILABLE: return - + # Export to ONNX first if needed if not self.onnx_path.exists(): self._export_to_onnx() - + # Build/load TensorRT engine self._load_tensorrt_engine() - + def _load_tensorrt_engine(self): """Load or build TensorRT engine""" if self.engine_path.exists() and not self.force_rebuild: self._load_existing_engine() else: self._build_tensorrt_engine() - + def _load_existing_engine(self): """Load existing TensorRT engine (now handled by lazy loading property)""" # Engine loading is now handled by the lazy loading 'engine' property # This method is kept for compatibility but does nothing pass - + def _build_tensorrt_engine(self): """Build TensorRT engine from ONNX model""" if not self.onnx_path.exists(): return - + try: # Create builder and network builder = trt.Builder(trt.Logger(trt.Logger.WARNING)) network = builder.create_network() # EXPLICIT_BATCH deprecated/ignored in TRT 10.x parser = trt.OnnxParser(network, trt.Logger(trt.Logger.WARNING)) - + # Parse ONNX model - with open(self.onnx_path, 'rb') as model: + with open(self.onnx_path, "rb") as model: if not parser.parse(model.read()): for error in range(parser.num_errors): pass return - + # Configure builder config = builder.create_builder_config() config.set_flag(trt.BuilderFlag.FP16) # Enable FP16 for better performance - + # Set optimization profile for dynamic shapes profile = builder.create_optimization_profile() min_shape = (1, 3, 256, 256) @@ -341,20 +350,20 @@ def _build_tensorrt_engine(self): max_shape = (1, 3, 1024, 1024) profile.set_shape("input", min_shape, opt_shape, max_shape) config.add_optimization_profile(profile) - + # Build engine engine = builder.build_serialized_network(network, config) - + if engine is None: return - + # Save engine - with open(self.engine_path, 'wb') as f: + with open(self.engine_path, "wb") as f: f.write(engine) - + # Load the built engine self._load_existing_engine() - + except Exception as e: logger.error(f"RealESRGAN TensorRT engine build failed: {e}") raise RuntimeError(f"RealESRGAN TensorRT engine build failed: {e}") from e @@ -363,54 +372,54 @@ def _process_with_tensorrt(self, tensor: torch.Tensor) -> torch.Tensor: """Process tensor using TensorRT engine (following depth_tensorrt pattern)""" batch_size, channels, height, width = tensor.shape input_shape = (batch_size, channels, height, width) - + # Ensure buffers are allocated for this input shape - if not hasattr(self.engine, 'tensors') or len(self.engine.tensors) == 0: + if not hasattr(self.engine, "tensors") or len(self.engine.tensors) == 0: self.engine.allocate_buffers(input_shape, device=self.device) else: # Check if we need to reallocate for different input shape input_tensor_shape = self.engine.tensors.get("input", torch.empty(0)).shape if input_tensor_shape != input_shape: self.engine.allocate_buffers(input_shape, device=self.device) - + # Prepare input tensor input_tensor = tensor.contiguous() if input_tensor.dtype != self.engine.tensors["input"].dtype: input_tensor = input_tensor.to(dtype=self.engine.tensors["input"].dtype) - + # Use engine inference with current stream context for proper synchronization cuda_stream = torch.cuda.current_stream().cuda_stream result = self.engine.infer({"input": input_tensor}, cuda_stream) - output_tensor = result['output'] - + output_tensor = result["output"] + # Ensure output is properly clamped to [0, 1] range for RealESRGAN output_tensor = torch.clamp(output_tensor, 0.0, 1.0) - + return output_tensor.clone() - + def _process_with_pytorch(self, tensor: torch.Tensor) -> torch.Tensor: """Process tensor using PyTorch model""" if self.pytorch_model is None: raise RuntimeError("_process_with_pytorch: PyTorch model not loaded") - + # Ensure model and input tensor have compatible dtypes model_dtype = next(self.pytorch_model.parameters()).dtype - original_dtype = tensor.dtype + original_dtype = tensor.dtype # noqa: F841 # TODO: pre-existing, untouched by this refactor if tensor.dtype != model_dtype: tensor = tensor.to(dtype=model_dtype) - + with torch.no_grad(): result = self.pytorch_model(tensor) - + # Ensure output is properly clamped to [0, 1] range for RealESRGAN result = torch.clamp(result, 0.0, 1.0) - + # Convert result to the desired output dtype (self.dtype) if result.dtype != self.dtype: result = result.to(dtype=self.dtype) - + return result - + def _process_core(self, image: Image.Image) -> Image.Image: """Core processing using PIL Image""" self._ensure_loaded_once() @@ -418,12 +427,12 @@ def _process_core(self, image: Image.Image) -> Image.Image: tensor = self.pil_to_tensor(image) if tensor.dim() == 3: tensor = tensor.unsqueeze(0) - + # Process with available backend if self.enable_tensorrt and TRT_AVAILABLE and self.engine_path.exists(): try: output_tensor = self._process_with_tensorrt(tensor) - except Exception as e: + except Exception: output_tensor = self._process_with_pytorch(tensor) elif self.pytorch_model is not None: output_tensor = self._process_with_pytorch(tensor) @@ -431,29 +440,29 @@ def _process_core(self, image: Image.Image) -> Image.Image: # Fallback to simple upscaling if no model is available target_width, target_height = self.get_target_dimensions() return image.resize((target_width, target_height), Image.LANCZOS) - + # Convert back to PIL if output_tensor.dim() == 4: output_tensor = output_tensor.squeeze(0) - + result_image = self.tensor_to_pil(output_tensor) - + return result_image - + def _ensure_target_size(self, image: Image.Image) -> Image.Image: """ Override base class method - for upscaling, we want to keep the upscaled size Don't resize back to original dimensions """ return image - + def _ensure_target_size_tensor(self, tensor: torch.Tensor) -> torch.Tensor: """ Override base class method - for upscaling, we want to keep the upscaled size Don't resize back to original dimensions """ return tensor - + def _process_tensor_core(self, tensor: torch.Tensor) -> torch.Tensor: """Core tensor processing""" self._ensure_loaded_once() @@ -462,49 +471,46 @@ def _process_tensor_core(self, tensor: torch.Tensor) -> torch.Tensor: squeeze_output = True else: squeeze_output = False - + # Process with available backend if self.enable_tensorrt and TRT_AVAILABLE and self.engine_path.exists(): try: output_tensor = self._process_with_tensorrt(tensor) - except Exception as e: + except Exception: output_tensor = self._process_with_pytorch(tensor) elif self.pytorch_model is not None: output_tensor = self._process_with_pytorch(tensor) else: # Fallback using interpolation output_tensor = torch.nn.functional.interpolate( - tensor, - scale_factor=self.scale_factor, - mode='bicubic', - align_corners=False + tensor, scale_factor=self.scale_factor, mode="bicubic", align_corners=False ) - + if squeeze_output: output_tensor = output_tensor.squeeze(0) - + return output_tensor - + def get_target_dimensions(self) -> Tuple[int, int]: """Get target output dimensions (width, height) - 2x upscaled""" - width = self.params.get('image_width') - height = self.params.get('image_height') - + width = self.params.get("image_width") + height = self.params.get("image_height") + if width is not None and height is not None: target_dims = (width * self.scale_factor, height * self.scale_factor) return target_dims - + # Fallback to square resolution - resolution = self.params.get('image_resolution', 512) + resolution = self.params.get("image_resolution", 512) target_resolution = resolution * self.scale_factor target_dims = (target_resolution, target_resolution) return target_dims - + def __del__(self): """Cleanup resources""" - if hasattr(self, '_engine') and self._engine is not None: + if hasattr(self, "_engine") and self._engine is not None: # Cleanup dedicated stream if it exists - if hasattr(self._engine, '_dedicated_stream'): + if hasattr(self._engine, "_dedicated_stream"): torch.cuda.synchronize() del self._engine._dedicated_stream del self._engine diff --git a/src/streamdiffusion/preprocessing/processors/scribble_tensorrt.py b/src/streamdiffusion/preprocessing/processors/scribble_tensorrt.py index 1a4774a11..2371bdfca 100644 --- a/src/streamdiffusion/preprocessing/processors/scribble_tensorrt.py +++ b/src/streamdiffusion/preprocessing/processors/scribble_tensorrt.py @@ -19,7 +19,6 @@ from .hed_tensorrt import HEDTensorrtPreprocessor from .trt_base import _first_output - logger = logging.getLogger(__name__) diff --git a/src/streamdiffusion/preprocessing/processors/soft_edge.py b/src/streamdiffusion/preprocessing/processors/soft_edge.py index 63db56adb..a7aa5de7d 100644 --- a/src/streamdiffusion/preprocessing/processors/soft_edge.py +++ b/src/streamdiffusion/preprocessing/processors/soft_edge.py @@ -1,9 +1,7 @@ import torch import torch.nn as nn -import torch.nn.functional as F -import numpy as np from PIL import Image -from typing import Union, Optional + from .base import BasePreprocessor @@ -12,95 +10,100 @@ class MultiScaleSobelOperator(nn.Module): Real-time multi-scale Sobel edge detector optimized for soft HED-like edges Based on the existing SobelOperator but enhanced for soft edge detection """ - + def __init__(self, device="cuda", dtype=torch.float16): super(MultiScaleSobelOperator, self).__init__() self.device = device self.dtype = dtype - + # Multi-scale edge detection (3 scales) self.edge_conv_x_1 = nn.Conv2d(1, 1, kernel_size=3, padding=1, bias=False).to(device) self.edge_conv_y_1 = nn.Conv2d(1, 1, kernel_size=3, padding=1, bias=False).to(device) - + self.edge_conv_x_2 = nn.Conv2d(1, 1, kernel_size=5, padding=2, bias=False).to(device) self.edge_conv_y_2 = nn.Conv2d(1, 1, kernel_size=5, padding=2, bias=False).to(device) - + self.edge_conv_x_3 = nn.Conv2d(1, 1, kernel_size=7, padding=3, bias=False).to(device) self.edge_conv_y_3 = nn.Conv2d(1, 1, kernel_size=7, padding=3, bias=False).to(device) - + # Gaussian blur for soft edges self.blur = nn.Conv2d(1, 1, kernel_size=5, padding=2, bias=False).to(device) - + self._setup_kernels() - + def _setup_kernels(self): """Setup Sobel kernels for different scales""" # Scale 1: Standard 3x3 Sobel - sobel_x_3 = torch.tensor([ - [-1.0, 0.0, 1.0], - [-2.0, 0.0, 2.0], - [-1.0, 0.0, 1.0] - ], device=self.device, dtype=self.dtype) - - sobel_y_3 = torch.tensor([ - [-1.0, -2.0, -1.0], - [0.0, 0.0, 0.0], - [1.0, 2.0, 1.0] - ], device=self.device, dtype=self.dtype) - + sobel_x_3 = torch.tensor( + [[-1.0, 0.0, 1.0], [-2.0, 0.0, 2.0], [-1.0, 0.0, 1.0]], device=self.device, dtype=self.dtype + ) + + sobel_y_3 = torch.tensor( + [[-1.0, -2.0, -1.0], [0.0, 0.0, 0.0], [1.0, 2.0, 1.0]], device=self.device, dtype=self.dtype + ) + # Scale 2: 5x5 Sobel - sobel_x_5 = torch.tensor([ - [-1, -2, 0, 2, 1], - [-2, -3, 0, 3, 2], - [-3, -5, 0, 5, 3], - [-2, -3, 0, 3, 2], - [-1, -2, 0, 2, 1] - ], device=self.device, dtype=self.dtype) / 16.0 - + sobel_x_5 = ( + torch.tensor( + [[-1, -2, 0, 2, 1], [-2, -3, 0, 3, 2], [-3, -5, 0, 5, 3], [-2, -3, 0, 3, 2], [-1, -2, 0, 2, 1]], + device=self.device, + dtype=self.dtype, + ) + / 16.0 + ) + sobel_y_5 = sobel_x_5.T - + # Scale 3: 7x7 Sobel (smoothed) - sobel_x_7 = torch.tensor([ - [-1, -2, -3, 0, 3, 2, 1], - [-2, -3, -4, 0, 4, 3, 2], - [-3, -4, -5, 0, 5, 4, 3], - [-4, -5, -6, 0, 6, 5, 4], - [-3, -4, -5, 0, 5, 4, 3], - [-2, -3, -4, 0, 4, 3, 2], - [-1, -2, -3, 0, 3, 2, 1] - ], device=self.device, dtype=self.dtype) / 32.0 - + sobel_x_7 = ( + torch.tensor( + [ + [-1, -2, -3, 0, 3, 2, 1], + [-2, -3, -4, 0, 4, 3, 2], + [-3, -4, -5, 0, 5, 4, 3], + [-4, -5, -6, 0, 6, 5, 4], + [-3, -4, -5, 0, 5, 4, 3], + [-2, -3, -4, 0, 4, 3, 2], + [-1, -2, -3, 0, 3, 2, 1], + ], + device=self.device, + dtype=self.dtype, + ) + / 32.0 + ) + sobel_y_7 = sobel_x_7.T - + # Gaussian kernel for smoothing - gaussian_5 = torch.tensor([ - [1, 4, 6, 4, 1], - [4, 16, 24, 16, 4], - [6, 24, 36, 24, 6], - [4, 16, 24, 16, 4], - [1, 4, 6, 4, 1] - ], device=self.device, dtype=self.dtype) / 256.0 - + gaussian_5 = ( + torch.tensor( + [[1, 4, 6, 4, 1], [4, 16, 24, 16, 4], [6, 24, 36, 24, 6], [4, 16, 24, 16, 4], [1, 4, 6, 4, 1]], + device=self.device, + dtype=self.dtype, + ) + / 256.0 + ) + # Set kernel weights self.edge_conv_x_1.weight = nn.Parameter(sobel_x_3.view(1, 1, 3, 3)) self.edge_conv_y_1.weight = nn.Parameter(sobel_y_3.view(1, 1, 3, 3)) - + self.edge_conv_x_2.weight = nn.Parameter(sobel_x_5.view(1, 1, 5, 5)) self.edge_conv_y_2.weight = nn.Parameter(sobel_y_5.view(1, 1, 5, 5)) - + self.edge_conv_x_3.weight = nn.Parameter(sobel_x_7.view(1, 1, 7, 7)) self.edge_conv_y_3.weight = nn.Parameter(sobel_y_7.view(1, 1, 7, 7)) - + self.blur.weight = nn.Parameter(gaussian_5.view(1, 1, 5, 5)) @torch.no_grad() def forward(self, image_tensor: torch.Tensor) -> torch.Tensor: """ Fast multi-scale soft edge detection - + Args: image_tensor: Input tensor [B, C, H, W] or [C, H, W] - + Returns: Soft edge map tensor [B, 1, H, W] or [1, H, W] """ @@ -109,109 +112,109 @@ def forward(self, image_tensor: torch.Tensor) -> torch.Tensor: if image_tensor.dim() == 3: image_tensor = image_tensor.unsqueeze(0) squeeze_output = True - + # Convert to grayscale if needed if image_tensor.shape[1] == 3: # RGB to grayscale gray = 0.299 * image_tensor[:, 0:1] + 0.587 * image_tensor[:, 1:2] + 0.114 * image_tensor[:, 2:3] else: gray = image_tensor[:, 0:1] - + # Multi-scale edge detection # Scale 1 (fine details) edge_x1 = self.edge_conv_x_1(gray) edge_y1 = self.edge_conv_y_1(gray) edge1 = torch.sqrt(edge_x1**2 + edge_y1**2) - + # Scale 2 (medium details) edge_x2 = self.edge_conv_x_2(gray) edge_y2 = self.edge_conv_y_2(gray) edge2 = torch.sqrt(edge_x2**2 + edge_y2**2) - + # Scale 3 (coarse details) edge_x3 = self.edge_conv_x_3(gray) edge_y3 = self.edge_conv_y_3(gray) edge3 = torch.sqrt(edge_x3**2 + edge_y3**2) - + # Combine scales with weights (like HED side outputs) combined_edge = 0.5 * edge1 + 0.3 * edge2 + 0.2 * edge3 - + # Apply Gaussian smoothing for soft edges soft_edge = self.blur(combined_edge) - + # Normalize to [0, 1] range soft_edge = soft_edge / (soft_edge.max() + 1e-8) - + # Apply soft sigmoid activation for smooth transitions soft_edge = torch.sigmoid(soft_edge * 6.0 - 3.0) # Soft S-curve - + if squeeze_output: soft_edge = soft_edge.squeeze(0) - + return soft_edge class SoftEdgePreprocessor(BasePreprocessor): """ Real-time soft edge detection preprocessor - HED alternative - + Uses multi-scale Sobel operations for extremely fast soft edge detection that mimics HED output quality at 50x+ the speed. """ gpu_native = True # _process_tensor_core uses torch ops under no_grad — no PIL round-trip _model_cache = {} - + @classmethod def get_preprocessor_metadata(cls): return { "display_name": "Soft Edge Detection", "description": "Real-time soft edge detection optimized for smooth, artistic edge maps using multi-scale Sobel operations.", "parameters": {}, - "use_cases": ["Artistic edge maps", "Soft stylistic control", "Real-time edge detection"] + "use_cases": ["Artistic edge maps", "Soft stylistic control", "Real-time edge detection"], } - + def __init__(self, **kwargs): """ Initialize soft edge preprocessor - + Args: **kwargs: Additional parameters """ super().__init__(**kwargs) self.model = None self._load_model() - + def _load_model(self): """ Load multi-scale Sobel operator with caching """ cache_key = f"soft_edge_{self.device}_{self.dtype}" - + if cache_key in self._model_cache: self.model = self._model_cache[cache_key] return - + print("SoftEdgePreprocessor: Loading real-time multi-scale edge detector") self.model = MultiScaleSobelOperator(device=self.device, dtype=self.dtype) self.model.eval() - + # Cache the model self._model_cache[cache_key] = self.model - + def _process_core(self, image: Image.Image) -> Image.Image: """ Apply soft edge detection to the input image """ # Convert PIL to tensor for GPU processing image_tensor = self.pil_to_tensor(image).squeeze(0) # Remove batch dim - + # Process with GPU-accelerated tensor method processed_tensor = self._process_tensor_core(image_tensor) - + # Convert back to PIL return self.tensor_to_pil(processed_tensor) - + def _process_tensor_core(self, image_tensor: torch.Tensor) -> torch.Tensor: """ GPU-optimized soft edge processing using tensors @@ -219,25 +222,25 @@ def _process_tensor_core(self, image_tensor: torch.Tensor) -> torch.Tensor: with torch.no_grad(): # Ensure correct input format and device image_tensor = image_tensor.to(device=self.device, dtype=self.dtype) - + # Normalize to [0, 1] if needed if image_tensor.max() > 1.0: image_tensor = image_tensor / 255.0 - + # Multi-scale edge detection edge_map = self.model(image_tensor) - + # Convert to 3-channel RGB format if edge_map.dim() == 3: edge_map = edge_map.repeat(3, 1, 1) else: edge_map = edge_map.repeat(1, 3, 1, 1).squeeze(0) - + # Ensure output is in [0, 1] range edge_map = torch.clamp(edge_map, 0.0, 1.0) - + return edge_map - + def get_model_info(self) -> dict: """ Get information about the loaded model @@ -249,24 +252,20 @@ def get_model_info(self) -> dict: "device": str(self.device), "dtype": str(self.dtype), "description": "Real-time multi-scale soft edge detection, HED quality at 50x+ speed", - "expected_fps": "100+ FPS at 512x512" + "expected_fps": "100+ FPS at 512x512", } - + @classmethod - def create_optimized(cls, device: str = 'cuda', dtype: torch.dtype = torch.float16, **kwargs): + def create_optimized(cls, device: str = "cuda", dtype: torch.dtype = torch.float16, **kwargs): """ Create an optimized soft edge preprocessor for real-time use - + Args: device: Target device ('cuda' or 'cpu') dtype: Data type for inference **kwargs: Additional parameters - + Returns: Optimized SoftEdgePreprocessor instance """ - return cls( - device=device, - dtype=dtype, - **kwargs - ) \ No newline at end of file + return cls(device=device, dtype=dtype, **kwargs) diff --git a/src/streamdiffusion/preprocessing/processors/standard_lineart.py b/src/streamdiffusion/preprocessing/processors/standard_lineart.py index ad2bdff1f..98b98cb83 100644 --- a/src/streamdiffusion/preprocessing/processors/standard_lineart.py +++ b/src/streamdiffusion/preprocessing/processors/standard_lineart.py @@ -1,17 +1,17 @@ -import numpy as np -import cv2 -from PIL import Image -from typing import Union, Optional import time -from .base import BasePreprocessor + +import numpy as np import torch import torch.nn.functional as F +from PIL import Image + +from .base import BasePreprocessor class StandardLineartPreprocessor(BasePreprocessor): """ Real-time optimized Standard Lineart detection preprocessor for ControlNet - + Extracts line art from input images using traditional computer vision techniques. Uses Gaussian blur and intensity calculations to detect lines without requiring pre-trained models. GPU-accelerated with PyTorch for optimal real-time performance. @@ -19,7 +19,6 @@ class StandardLineartPreprocessor(BasePreprocessor): gpu_native = True # _process_tensor_core uses torch ops — no CPU/PIL round-trip - @classmethod def get_preprocessor_metadata(cls): return { @@ -31,27 +30,29 @@ def get_preprocessor_metadata(cls): "default": 6.0, "range": [1.0, 20.0], "step": 0.1, - "description": "Standard deviation for Gaussian blur (higher = smoother lines)" + "description": "Standard deviation for Gaussian blur (higher = smoother lines)", }, "intensity_threshold": { "type": "int", "default": 8, "range": [1, 50], - "description": "Threshold for intensity calculation (lower = more sensitive)" - } + "description": "Threshold for intensity calculation (lower = more sensitive)", + }, }, - "use_cases": ["Traditional line art", "Simple edge detection", "No AI model required"] + "use_cases": ["Traditional line art", "Simple edge detection", "No AI model required"], } - - def __init__(self, - detect_resolution: int = 512, - image_resolution: int = 512, - gaussian_sigma: float = 6.0, - intensity_threshold: int = 8, - **kwargs): + + def __init__( + self, + detect_resolution: int = 512, + image_resolution: int = 512, + gaussian_sigma: float = 6.0, + intensity_threshold: int = 8, + **kwargs, + ): """ Initialize Standard Lineart preprocessor - + Args: detect_resolution: Resolution for line art detection image_resolution: Output image resolution @@ -59,39 +60,39 @@ def __init__(self, intensity_threshold: Threshold for intensity calculation **kwargs: Additional parameters """ - + super().__init__( detect_resolution=detect_resolution, image_resolution=image_resolution, gaussian_sigma=gaussian_sigma, intensity_threshold=intensity_threshold, - **kwargs + **kwargs, ) - + # Initialize GPU device self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - + def _gaussian_kernel(self, kernel_size: int, sigma: float, device=None) -> torch.Tensor: """Create 2D Gaussian kernel - based on existing codebase pattern""" x, y = torch.meshgrid( - torch.linspace(-1, 1, kernel_size, device=device), - torch.linspace(-1, 1, kernel_size, device=device), - indexing="ij" + torch.linspace(-1, 1, kernel_size, device=device), + torch.linspace(-1, 1, kernel_size, device=device), + indexing="ij", ) d = torch.sqrt(x * x + y * y) g = torch.exp(-(d * d) / (2.0 * sigma * sigma)) return g / g.sum() - + def _gaussian_blur_torch(self, image: torch.Tensor, sigma: float) -> torch.Tensor: """Apply Gaussian blur using PyTorch - GPU accelerated""" # Calculate kernel size from sigma (odd number) kernel_size = int(2 * torch.ceil(torch.tensor(3 * sigma)) + 1) if kernel_size % 2 == 0: kernel_size += 1 - + # Create Gaussian kernel kernel = self._gaussian_kernel(kernel_size, sigma, device=image.device) - + # Handle different input shapes if image.dim() == 3: # HWC format H, W, C = image.shape @@ -103,31 +104,31 @@ def _gaussian_blur_torch(self, image: torch.Tensor, sigma: float) -> torch.Tenso needs_reshape = False else: raise ValueError(f"standardlineart_gaussian_blur_torch: Unsupported image shape: {image.shape}") - + # Expand kernel for all channels kernel = kernel.repeat(image.shape[1], 1, 1).unsqueeze(1) - + # Apply blur with reflection padding padding = kernel_size // 2 - padded_image = F.pad(image, (padding, padding, padding, padding), 'reflect') + padded_image = F.pad(image, (padding, padding, padding, padding), "reflect") blurred = F.conv2d(padded_image, kernel, padding=0, groups=image.shape[1]) - + # Convert back to original format if needed if needs_reshape: blurred = blurred.squeeze(0).permute(1, 2, 0) # BCHW -> HWC - + return blurred - + def _ensure_hwc3_torch(self, x: torch.Tensor) -> torch.Tensor: """Ensure image has 3 channels (HWC3 format) - PyTorch version""" if x.dim() == 2: x = x.unsqueeze(-1) # Add channel dimension - + if x.dim() != 3: raise ValueError(f"standardlineart_ensure_hwc3_torch: Expected 2D or 3D tensor, got {x.dim()}D") - + H, W, C = x.shape - + if C == 3: return x elif C == 1: @@ -139,41 +140,38 @@ def _ensure_hwc3_torch(self, x: torch.Tensor) -> torch.Tensor: return torch.clamp(y, 0, 255) else: raise ValueError(f"standardlineart_ensure_hwc3_torch: Unsupported channel count: {C}") - + def _pad64(self, x: int) -> int: """Pad to nearest multiple of 64""" return int(torch.ceil(torch.tensor(float(x) / 64.0)) * 64 - x) - + def _resize_image_with_pad_torch(self, input_image: torch.Tensor, resolution: int) -> tuple: """Resize image with padding to target resolution - PyTorch GPU accelerated""" img = self._ensure_hwc3_torch(input_image) H_raw, W_raw, _ = img.shape - + if resolution == 0: return img, lambda x: x - + k = float(resolution) / float(min(H_raw, W_raw)) H_target = int(torch.round(torch.tensor(float(H_raw) * k))) W_target = int(torch.round(torch.tensor(float(W_raw) * k))) - + # Convert to BCHW for interpolation img_bchw = img.permute(2, 0, 1).unsqueeze(0) # HWC -> BCHW - + # Use PyTorch's interpolate for GPU-accelerated resize - mode = 'bicubic' if k > 1 else 'area' + mode = "bicubic" if k > 1 else "area" img_resized_bchw = F.interpolate( - img_bchw, - size=(H_target, W_target), - mode=mode, - align_corners=False if mode == 'bicubic' else None + img_bchw, size=(H_target, W_target), mode=mode, align_corners=False if mode == "bicubic" else None ) - + # Convert back to HWC img_resized = img_resized_bchw.squeeze(0).permute(1, 2, 0) - + # Apply padding H_pad, W_pad = self._pad64(H_target), self._pad64(W_target) - img_padded = F.pad(img_resized.permute(2, 0, 1), (0, W_pad, 0, H_pad), mode='replicate').permute(1, 2, 0) + img_padded = F.pad(img_resized.permute(2, 0, 1), (0, W_pad, 0, H_pad), mode="replicate").permute(1, 2, 0) def remove_pad(x): return x[:H_target, :W_target, ...] @@ -198,7 +196,7 @@ def _compute_lineart_hwc(self, input_image: torch.Tensor) -> torch.Tensor: intensity = torch.min(g - input_image, dim=2)[0] intensity = torch.clamp(intensity, 0, 255) - + threshold_mask = intensity > intensity_threshold # Sync-free: nanmedian over thresholded pixels equals median(intensity[threshold_mask]). # All-False mask → every element is nan → nan_to_num floors to 16. @@ -207,10 +205,9 @@ def _compute_lineart_hwc(self, input_image: torch.Tensor) -> torch.Tensor: median_val = torch.nanmedian(masked) normalization_factor = torch.clamp_min(torch.nan_to_num(median_val, nan=16.0), 16.0) - intensity = intensity / normalization_factor intensity = intensity * 127 - + detected_map = torch.clamp(intensity, 0, 255).byte() detected_map = detected_map.unsqueeze(-1) detected_map = self._ensure_hwc3_torch(detected_map.float()) @@ -234,7 +231,7 @@ def _process_core(self, image: Image.Image) -> Image.Image: detected_map = self._compute_lineart_hwc(input_image) detected_map = remove_pad(detected_map) - + detected_map_cpu = detected_map.byte().cpu().numpy() return Image.fromarray(detected_map_cpu) diff --git a/src/streamdiffusion/preprocessing/processors/temporal_net_tensorrt.py b/src/streamdiffusion/preprocessing/processors/temporal_net_tensorrt.py index db41ea476..cd136dabe 100644 --- a/src/streamdiffusion/preprocessing/processors/temporal_net_tensorrt.py +++ b/src/streamdiffusion/preprocessing/processors/temporal_net_tensorrt.py @@ -9,7 +9,6 @@ from .base import PipelineAwareProcessor - # Try to import TensorRT dependencies try: from collections import OrderedDict @@ -24,7 +23,10 @@ # Try to import torchvision for RAFT model try: - from torchvision.models.optical_flow import Raft_Small_Weights, raft_small + from torchvision.models.optical_flow import ( # noqa: F401 # TODO: pre-existing, untouched by this refactor + Raft_Small_Weights, + raft_small, + ) from torchvision.utils import flow_to_image TORCHVISION_AVAILABLE = True diff --git a/src/streamdiffusion/preprocessing/processors/trt_base.py b/src/streamdiffusion/preprocessing/processors/trt_base.py index b97778b21..11b376bab 100644 --- a/src/streamdiffusion/preprocessing/processors/trt_base.py +++ b/src/streamdiffusion/preprocessing/processors/trt_base.py @@ -28,7 +28,6 @@ from .base import BasePreprocessor - logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- diff --git a/src/streamdiffusion/stream_parameter_updater.py b/src/streamdiffusion/stream_parameter_updater.py index 42b9c20da..8355dbc46 100644 --- a/src/streamdiffusion/stream_parameter_updater.py +++ b/src/streamdiffusion/stream_parameter_updater.py @@ -13,7 +13,6 @@ ) from .preprocessing.orchestrator_user import OrchestratorUser - logger = logging.getLogger(__name__) @@ -1529,7 +1528,7 @@ def _update_ipadapter_config(self, desired_config: Dict[str, Any]) -> None: else: self.stream.ipadapter.set_scale(desired_scale) # Update our tracking attribute - setattr(self.stream.ipadapter, "scale", desired_scale) + self.stream.ipadapter.scale = desired_scale except Exception: # Do not add fallback mechanisms raise @@ -1539,7 +1538,7 @@ def _update_ipadapter_config(self, desired_config: Dict[str, Any]) -> None: # Tell diffusers_ipadapter to set the scale self.stream.ipadapter.set_scale(desired_scale) # Update our tracking attribute - setattr(self.stream.ipadapter, "scale", desired_scale) + self.stream.ipadapter.scale = desired_scale # Update enabled state if provided if "enabled" in desired_config and desired_config["enabled"] is not None: @@ -1551,14 +1550,14 @@ def _update_ipadapter_config(self, desired_config: Dict[str, Any]) -> None: logger.info( f"_update_ipadapter_config: Updating enabled state: {current_enabled} → {enabled_state}" ) - setattr(self.stream.ipadapter, "enabled", enabled_state) + self.stream.ipadapter.enabled = enabled_state # Update weight type if provided (affects per-layer distribution and/or per-step factor) if "weight_type" in desired_config and desired_config["weight_type"] is not None: weight_type = desired_config["weight_type"] # Update IPAdapter instance if hasattr(self.stream, "ipadapter"): - setattr(self.stream.ipadapter, "weight_type", weight_type) + self.stream.ipadapter.weight_type = weight_type # For PyTorch UNet, immediately apply a per-layer scale vector so layers reflect selection types try: @@ -1581,7 +1580,7 @@ def _update_ipadapter_config(self, desired_config: Dict[str, Any]) -> None: else: self.stream.ipadapter.set_scale(base_weight) # Keep our tracking attribute in sync - setattr(self.stream.ipadapter, "scale", base_weight) + self.stream.ipadapter.scale = base_weight except Exception: # Do not add fallback mechanisms raise @@ -1673,7 +1672,7 @@ def _get_current_hook_config(self, hook_type: str) -> List[Dict[str, Any]]: config = [] for i, processor in enumerate(processors): proc_config = { - "type": getattr(processor, "__class__").__name__, + "type": processor.__class__.__name__, "order": getattr(processor, "order", i), "enabled": getattr(processor, "enabled", True), } @@ -1783,8 +1782,8 @@ def _update_hook_config(self, hook_type: str, desired_config: List[Dict[str, Any ) # Copy attributes from old processor - setattr(new_processor, "order", getattr(existing_processor, "order", i)) - setattr(new_processor, "enabled", enabled) + new_processor.order = getattr(existing_processor, "order", i) + new_processor.enabled = enabled # Set parameters if hasattr(new_processor, "params"): @@ -1797,7 +1796,7 @@ def _update_hook_config(self, hook_type: str, desired_config: List[Dict[str, Any else: # Same type, just update attributes logger.info(f"_update_hook_config: Same type, updating attributes for processor {i}") - setattr(existing_processor, "enabled", enabled) + existing_processor.enabled = enabled # Update parameters if hasattr(existing_processor, "params"): diff --git a/src/streamdiffusion/tools/compile_depth_anything_tensorrt.py b/src/streamdiffusion/tools/compile_depth_anything_tensorrt.py index b842ec101..222b55412 100644 --- a/src/streamdiffusion/tools/compile_depth_anything_tensorrt.py +++ b/src/streamdiffusion/tools/compile_depth_anything_tensorrt.py @@ -16,12 +16,11 @@ import fire import torch - logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") logger = logging.getLogger(__name__) try: - import tensorrt as trt # noqa: E402 + import tensorrt as trt from streamdiffusion.acceleration.tensorrt.utilities import BUILD_TRT_LOGGER diff --git a/src/streamdiffusion/tools/compile_raft_tensorrt.py b/src/streamdiffusion/tools/compile_raft_tensorrt.py index 30ff70b91..5d4de5899 100644 --- a/src/streamdiffusion/tools/compile_raft_tensorrt.py +++ b/src/streamdiffusion/tools/compile_raft_tensorrt.py @@ -1,10 +1,10 @@ -import torch import logging from pathlib import Path -from typing import Optional + import fire +import torch -logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") logger = logging.getLogger(__name__) try: @@ -12,7 +12,6 @@ from streamdiffusion.acceleration.tensorrt.utilities import BUILD_TRT_LOGGER - TENSORRT_AVAILABLE = True except ImportError: TENSORRT_AVAILABLE = False @@ -20,7 +19,8 @@ logger.error("TensorRT not available. Please install it first.") try: - from torchvision.models.optical_flow import raft_small, Raft_Small_Weights + from torchvision.models.optical_flow import Raft_Small_Weights, raft_small + TORCHVISION_AVAILABLE = True except ImportError: TORCHVISION_AVAILABLE = False @@ -33,11 +33,11 @@ def export_raft_to_onnx( min_width: int = 512, max_height: int = 512, max_width: int = 512, - device: str = "cuda" + device: str = "cuda", ) -> bool: """ Export RAFT model to ONNX format - + Args: onnx_path: Path to save the ONNX model min_height: Minimum input height for the model @@ -45,41 +45,41 @@ def export_raft_to_onnx( max_height: Maximum input height for the model max_width: Maximum input width for the model device: Device to use for export - + Returns: True if successful, False otherwise """ if not TORCHVISION_AVAILABLE: logger.error("torchvision is required but not installed") return False - + logger.info(f"Exporting RAFT model to ONNX: {onnx_path}") logger.info(f"Resolution range: {min_height}x{min_width} - {max_height}x{max_width}") - + try: # Load RAFT model logger.info("Loading RAFT Small model...") raft_model = raft_small(weights=Raft_Small_Weights.DEFAULT, progress=True) raft_model = raft_model.to(device=device) raft_model.eval() - + # Create dummy inputs using max resolution for export dummy_frame1 = torch.randn(1, 3, max_height, max_width).to(device) dummy_frame2 = torch.randn(1, 3, max_height, max_width).to(device) - + # Apply RAFT preprocessing if available weights = Raft_Small_Weights.DEFAULT - if hasattr(weights, 'transforms') and weights.transforms is not None: + if hasattr(weights, "transforms") and weights.transforms is not None: transforms = weights.transforms() dummy_frame1, dummy_frame2 = transforms(dummy_frame1, dummy_frame2) - + # Make batch, height, and width dimensions dynamic dynamic_axes = { "frame1": {0: "batch_size", 2: "height", 3: "width"}, "frame2": {0: "batch_size", 2: "height", 3: "width"}, "flow": {0: "batch_size", 2: "height", 3: "width"}, } - + logger.info("Exporting to ONNX...") with torch.no_grad(): torch.onnx.export( @@ -87,22 +87,23 @@ def export_raft_to_onnx( (dummy_frame1, dummy_frame2), str(onnx_path), verbose=False, - input_names=['frame1', 'frame2'], - output_names=['flow'], + input_names=["frame1", "frame2"], + output_names=["flow"], opset_version=17, export_params=True, dynamic_axes=dynamic_axes, ) - + del raft_model torch.cuda.empty_cache() - + logger.info(f"Successfully exported ONNX model to {onnx_path}") return True - + except Exception as e: logger.error(f"Failed to export ONNX model: {e}") import traceback + traceback.print_exc() return False @@ -115,11 +116,11 @@ def build_tensorrt_engine( max_height: int = 512, max_width: int = 512, fp16: bool = True, - workspace_size_gb: int = 4 + workspace_size_gb: int = 4, ) -> bool: """ Build TensorRT engine from ONNX model - + Args: onnx_path: Path to the ONNX model engine_path: Path to save the TensorRT engine @@ -129,75 +130,74 @@ def build_tensorrt_engine( max_width: Maximum input width for optimization fp16: Enable FP16 precision mode workspace_size_gb: Maximum workspace size in GB - + Returns: True if successful, False otherwise """ if not TENSORRT_AVAILABLE: logger.error("TensorRT is required but not installed") return False - + if not onnx_path.exists(): logger.error(f"ONNX model not found: {onnx_path}") return False - + logger.info(f"Building TensorRT engine from ONNX model: {onnx_path}") logger.info(f"Output path: {engine_path}") logger.info(f"Resolution range: {min_height}x{min_width} - {max_height}x{max_width}") logger.info(f"FP16 mode: {fp16}") logger.info("This may take several minutes...") - + try: builder = trt.Builder(BUILD_TRT_LOGGER) network = builder.create_network() # EXPLICIT_BATCH deprecated/ignored in TRT 10.x parser = trt.OnnxParser(network, BUILD_TRT_LOGGER) - logger.info("Parsing ONNX model...") - with open(onnx_path, 'rb') as model: + with open(onnx_path, "rb") as model: if not parser.parse(model.read()): logger.error("Failed to parse ONNX model") for error in range(parser.num_errors): logger.error(f"Parser error: {parser.get_error(error)}") return False - + logger.info("Configuring TensorRT builder...") config = builder.create_builder_config() - + config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, workspace_size_gb * (1 << 30)) - + if fp16: config.set_flag(trt.BuilderFlag.FP16) logger.info("FP16 mode enabled") - + # Calculate optimal resolution (middle point) opt_height = (min_height + max_height) // 2 opt_width = (min_width + max_width) // 2 - + profile = builder.create_optimization_profile() min_shape = (1, 3, min_height, min_width) opt_shape = (1, 3, opt_height, opt_width) max_shape = (1, 3, max_height, max_width) - + profile.set_shape("frame1", min_shape, opt_shape, max_shape) profile.set_shape("frame2", min_shape, opt_shape, max_shape) config.add_optimization_profile(profile) - + logger.info("Building TensorRT engine... (this will take a while)") engine = builder.build_serialized_network(network, config) - + if engine is None: logger.error("Failed to build TensorRT engine") return False - + logger.info(f"Saving engine to {engine_path}") engine_path.parent.mkdir(parents=True, exist_ok=True) - with open(engine_path, 'wb') as f: + with open(engine_path, "wb") as f: f.write(engine) - + logger.info(f"Successfully built and saved TensorRT engine: {engine_path}") - logger.info(f"Engine size: {engine_path.stat().st_size / (1024*1024):.2f} MB") - + logger.info(f"Engine size: {engine_path.stat().st_size / (1024 * 1024):.2f} MB") + # Delete ONNX file after successful engine creation try: if onnx_path.exists(): @@ -205,12 +205,13 @@ def build_tensorrt_engine( logger.info(f"Deleted ONNX file: {onnx_path}") except Exception as e: logger.warning(f"Failed to delete ONNX file: {e}") - + return True - + except Exception as e: logger.error(f"Failed to build TensorRT engine: {e}") import traceback + traceback.print_exc() return False @@ -222,11 +223,11 @@ def compile_raft( device: str = "cuda", fp16: bool = True, workspace_size_gb: int = 4, - force_rebuild: bool = False + force_rebuild: bool = False, ): """ Main function to compile RAFT model to TensorRT engine - + Args: min_resolution: Minimum input resolution as "HxW" (e.g., "512x512") (default: "512x512") max_resolution: Maximum input resolution as "HxW" (e.g., "1024x1024") (default: "512x512") @@ -240,46 +241,46 @@ def compile_raft( logger.error("TensorRT is not available. Please install it first using:") logger.error(" python -m streamdiffusion.tools.install-tensorrt") return - + if not TORCHVISION_AVAILABLE: logger.error("torchvision is not available. Please install it first using:") logger.error(" pip install torchvision") return - + # Parse resolution strings try: - min_height, min_width = map(int, min_resolution.split('x')) + min_height, min_width = map(int, min_resolution.split("x")) except ValueError: logger.error(f"Invalid min_resolution format: {min_resolution}. Expected format: HxW (e.g., 512x512)") return - + try: - max_height, max_width = map(int, max_resolution.split('x')) + max_height, max_width = map(int, max_resolution.split("x")) except ValueError: logger.error(f"Invalid max_resolution format: {max_resolution}. Expected format: HxW (e.g., 1024x1024)") return - + output_path = Path(output_dir) output_path.mkdir(parents=True, exist_ok=True) - + # Add resolution suffix to filenames onnx_path = output_path / f"raft_small_min_{min_resolution}_max_{max_resolution}.onnx" engine_path = output_path / f"raft_small_min_{min_resolution}_max_{max_resolution}.engine" - - logger.info("="*80) + + logger.info("=" * 80) logger.info("RAFT TensorRT Compilation") - logger.info("="*80) + logger.info("=" * 80) logger.info(f"Output directory: {output_path.absolute()}") logger.info(f"Resolution range: {min_resolution} - {max_resolution}") logger.info(f"ONNX path: {onnx_path}") logger.info(f"Engine path: {engine_path}") - logger.info("="*80) - + logger.info("=" * 80) + if engine_path.exists() and not force_rebuild: logger.info(f"TensorRT engine already exists: {engine_path}") logger.info("Use --force_rebuild to rebuild it") return - + if not onnx_path.exists() or force_rebuild: logger.info("\n[Step 1/2] Exporting RAFT to ONNX...") if not export_raft_to_onnx(onnx_path, min_height, min_width, max_height, max_width, device): @@ -287,21 +288,22 @@ def compile_raft( return else: logger.info(f"\n[Step 1/2] ONNX model already exists: {onnx_path}") - + logger.info("\n[Step 2/2] Building TensorRT engine...") - if not build_tensorrt_engine(onnx_path, engine_path, min_height, min_width, max_height, max_width, fp16, workspace_size_gb): + if not build_tensorrt_engine( + onnx_path, engine_path, min_height, min_width, max_height, max_width, fp16, workspace_size_gb + ): logger.error("Failed to build TensorRT engine") return - - logger.info("\n" + "="*80) + + logger.info("\n" + "=" * 80) logger.info("✓ Compilation completed successfully!") - logger.info("="*80) + logger.info("=" * 80) logger.info(f"Engine path: {engine_path.absolute()}") logger.info("\nYou can now use this engine in TemporalNetTensorRTPreprocessor:") logger.info(f' engine_path="{engine_path.absolute()}"') - logger.info("="*80) + logger.info("=" * 80) if __name__ == "__main__": fire.Fire(compile_raft) - diff --git a/src/streamdiffusion/tools/cuda_l2_cache.py b/src/streamdiffusion/tools/cuda_l2_cache.py index 176a158d5..d9858bb40 100644 --- a/src/streamdiffusion/tools/cuda_l2_cache.py +++ b/src/streamdiffusion/tools/cuda_l2_cache.py @@ -24,7 +24,6 @@ import torch - # ============================================================================= # Environment Controls # ============================================================================= diff --git a/src/streamdiffusion/tools/gpu_profiler.py b/src/streamdiffusion/tools/gpu_profiler.py index 3f4efbbd4..8fa33975e 100644 --- a/src/streamdiffusion/tools/gpu_profiler.py +++ b/src/streamdiffusion/tools/gpu_profiler.py @@ -41,7 +41,6 @@ from functools import wraps from typing import Any, Callable, Dict, Generator, List, Optional - # ───────────────────────────────────────────────────────────────────────────── # RegionStats — per-region histogram with percentile support # ───────────────────────────────────────────────────────────────────────────── @@ -50,7 +49,7 @@ class RegionStats: """Histogram-based timing statistics for a named profiling region.""" - __slots__ = ("name", "samples", "count", "total_ms") + __slots__ = ("count", "name", "samples", "total_ms") MAX_SAMPLES = 10_000 # cap to avoid unbounded memory @@ -123,16 +122,16 @@ class _RegionCtx: On exit: optional NVTX range_pop + CUDA event elapsed_time -> RegionStats. """ - __slots__ = ("_profiler", "_name", "_nvtx", "_start_evt", "_end_evt") + __slots__ = ("_end_evt", "_name", "_nvtx", "_profiler", "_start_evt") - def __init__(self, profiler: "GPUProfiler", name: str) -> None: + def __init__(self, profiler: GPUProfiler, name: str) -> None: self._profiler = profiler self._name = name self._nvtx = profiler._nvtx_enabled self._start_evt = None self._end_evt = None - def __enter__(self) -> "_RegionCtx": + def __enter__(self) -> _RegionCtx: p = self._profiler if self._nvtx: p._torch.cuda.nvtx.range_push(self._name) @@ -158,7 +157,7 @@ class _NullCtx: __slots__ = () - def __enter__(self) -> "_NullCtx": + def __enter__(self) -> _NullCtx: return self def __exit__(self, *_: object) -> None: @@ -471,10 +470,10 @@ class _NullProfiler: __slots__ = () - def region(self, name: str) -> _NullCtx: # noqa: ARG002 + def region(self, name: str) -> _NullCtx: return _NULL_CTX - def trace(self, name: str) -> Callable: # noqa: ARG002 + def trace(self, name: str) -> Callable: """Return identity decorator — function is NOT wrapped.""" def decorator(fn: Callable) -> Callable: @@ -483,45 +482,45 @@ def decorator(fn: Callable) -> Callable: return decorator def mark(self, name: str) -> None: - pass # noqa: E704 + pass def begin(self, name: str) -> None: - pass # noqa: E704 + pass def end(self, name: str) -> None: - pass # noqa: E704 + pass def nsys_start(self) -> None: - pass # noqa: E704 + pass def nsys_stop(self) -> None: - pass # noqa: E704 + pass def step(self) -> None: - pass # noqa: E704 + pass def flush(self) -> None: - pass # noqa: E704 + pass def report(self, top_n: int = 30) -> None: - pass # noqa: E704, ARG002 + pass def reset(self) -> None: - pass # noqa: E704 + pass @contextmanager - def torch_trace(self, path: Optional[str] = None, warmup: int = 1, active: int = 5) -> Generator[None, None, None]: # noqa: ARG002 + def torch_trace(self, path: Optional[str] = None, warmup: int = 1, active: int = 5) -> Generator[None, None, None]: yield @contextmanager - def memory_trace(self, path: str = "mem_snapshot.pkl") -> Generator[None, None, None]: # noqa: ARG002 + def memory_trace(self, path: str = "mem_snapshot.pkl") -> Generator[None, None, None]: yield - def export_stats(self, path: str = "gpu_profile_stats.json") -> None: # noqa: ARG002 + def export_stats(self, path: str = "gpu_profile_stats.json") -> None: pass def configure(self, **kwargs: Any) -> None: - pass # noqa: E704, ARG002 + pass # ───────────────────────────────────────────────────────────────────────────── @@ -616,7 +615,7 @@ def configure_from_dict(cfg: Dict[str, Any]) -> None: "nvtx": true, "events": true, "memory": false, - "trace_path": "profiler_logs/trace.json" + "trace_path": "profiler_logs/trace.json", } } """ diff --git a/src/streamdiffusion/utils/__init__.py b/src/streamdiffusion/utils/__init__.py index b40413d24..b383ab919 100644 --- a/src/streamdiffusion/utils/__init__.py +++ b/src/streamdiffusion/utils/__init__.py @@ -1,6 +1,5 @@ from .reporting import report_error - __all__ = [ "report_error", ] diff --git a/src/streamdiffusion/wrapper.py b/src/streamdiffusion/wrapper.py index b09d76e2a..1b6d373ef 100644 --- a/src/streamdiffusion/wrapper.py +++ b/src/streamdiffusion/wrapper.py @@ -15,7 +15,6 @@ from .tools.gpu_profiler import configure as _configure_profiler from .tools.gpu_profiler import profiler - logger = logging.getLogger(__name__) @@ -58,10 +57,7 @@ class StreamDiffusionWrapper: wrapper.prepare([("cat", 0.7), ("dog", 0.3)]) # Prompt + seed blending - wrapper.prepare( - prompt=[("style1", 0.6), ("style2", 0.4)], - seed_list=[(123, 0.8), (456, 0.2)] - ) + wrapper.prepare(prompt=[("style1", 0.6), ("style2", 0.4)], seed_list=[(123, 0.8), (456, 0.2)]) ``` ## Runtime Updates: @@ -73,10 +69,7 @@ class StreamDiffusionWrapper: wrapper.update_prompt([("new1", 0.5), ("new2", 0.5)]) # Update combined parameters - wrapper.update_stream_params( - prompt_list=[("bird", 0.6), ("fish", 0.4)], - seed_list=[(789, 0.3), (101, 0.7)] - ) + wrapper.update_stream_params(prompt_list=[("bird", 0.6), ("fish", 0.4)], seed_list=[(789, 0.3), (101, 0.7)]) ``` ## Weight Management: @@ -2653,7 +2646,7 @@ def _install_cached_proc(attn_module): fp8=fp8 or bool(cfg.get("fp8", False)), ) try: - setattr(engine, "model_id", cfg["model_id"]) + engine.model_id = cfg["model_id"] except Exception: pass compiled_cn_engines.append(engine) @@ -2662,7 +2655,7 @@ def _install_cached_proc(attn_module): f"Failed to compile/load ControlNet engine for {cfg.get('model_id')}: {e}" ) if compiled_cn_engines: - setattr(stream, "controlnet_engines", compiled_cn_engines) + stream.controlnet_engines = compiled_cn_engines try: logger.info( f"Compiled/loaded {len(compiled_cn_engines)} ControlNet TensorRT engine(s)" @@ -2883,7 +2876,7 @@ def get_stream_state(self, include_caches: bool = False) -> Dict[str, Any]: num_inference_steps = None try: if hasattr(stream, "timesteps") and stream.timesteps is not None: - num_inference_steps = int(len(stream.timesteps)) + num_inference_steps = len(stream.timesteps) except Exception as e: logger.debug(f"Failed to derive num_inference_steps from stream.timesteps: {e}", exc_info=True) diff --git a/tests/manual/smoke_self_build_preprocessors.py b/tests/manual/smoke_self_build_preprocessors.py index 96d6f49c6..1ffcb1bc9 100644 --- a/tests/manual/smoke_self_build_preprocessors.py +++ b/tests/manual/smoke_self_build_preprocessors.py @@ -27,7 +27,6 @@ import torch - logging.basicConfig( level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s — %(message)s", @@ -193,7 +192,7 @@ def main() -> None: logger.info(f"GPU: {torch.cuda.get_device_name(0)}") logger.info("TensorRT: ", end="") try: - import tensorrt as trt # noqa: F401 + import tensorrt as trt logger.info(f"{trt.__version__}") except ImportError: diff --git a/tests/quality/regenerate_golden.py b/tests/quality/regenerate_golden.py index 7c86fd449..18c6592cf 100644 --- a/tests/quality/regenerate_golden.py +++ b/tests/quality/regenerate_golden.py @@ -19,12 +19,10 @@ import os import sys - sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) from streamdiffusion import StreamDiffusionWrapper - logger = logging.getLogger("quality.regenerate") TESTS_QUALITY_DIR = os.path.dirname(os.path.abspath(__file__)) diff --git a/tests/quality/run_compare.py b/tests/quality/run_compare.py index cdaba96c8..64f116f4d 100644 --- a/tests/quality/run_compare.py +++ b/tests/quality/run_compare.py @@ -22,7 +22,6 @@ import yaml - sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) logger = logging.getLogger("quality.run_compare") diff --git a/tests/unit/test_cn_preprocessor_residency.py b/tests/unit/test_cn_preprocessor_residency.py index fe142e97f..a2d69c90c 100644 --- a/tests/unit/test_cn_preprocessor_residency.py +++ b/tests/unit/test_cn_preprocessor_residency.py @@ -11,7 +11,6 @@ import pytest - # --------------------------------------------------------------------------- # CN-coupled type → expected preprocessor mappings # (matches the plan table and the updated CN_MODEL_REGISTRY 'preprocessor' fields) diff --git a/tests/unit/test_config_extraction_golden.py b/tests/unit/test_config_extraction_golden.py index b4b8f749a..459ab245e 100644 --- a/tests/unit/test_config_extraction_golden.py +++ b/tests/unit/test_config_extraction_golden.py @@ -15,7 +15,6 @@ from streamdiffusion.config import _extract_prepare_params, _extract_wrapper_params - MINIMAL_CONFIG = {"model_id": "stabilityai/sd-turbo"} EXPECTED_WRAPPER_PARAMS = { diff --git a/tests/unit/test_derived_tensor_sync.py b/tests/unit/test_derived_tensor_sync.py index c89575847..d4a6abc06 100644 --- a/tests/unit/test_derived_tensor_sync.py +++ b/tests/unit/test_derived_tensor_sync.py @@ -13,18 +13,16 @@ """ import torch -import pytest from streamdiffusion.stream_parameter_updater import StreamParameterUpdater - # --------------------------------------------------------------------------- # helpers # --------------------------------------------------------------------------- + def _make_mock_lcm_scheduler(num_steps=50, device="cpu"): """Minimal scheduler shell with alphas_cumprod and get_scalings_for_boundary_condition_discrete.""" - import types from diffusers import LCMScheduler sched = object.__new__(LCMScheduler) @@ -46,12 +44,17 @@ def _scalings(timestep): return sched -def _make_stream_shell(t_index_list, device="cpu", dtype=torch.float32, - frame_bff_size=1, use_denoising_batch=True, - cfg_type="self", do_add_noise=True): +def _make_stream_shell( + t_index_list, + device="cpu", + dtype=torch.float32, + frame_bff_size=1, + use_denoising_batch=True, + cfg_type="self", + do_add_noise=True, +): """Minimal StreamDiffusion pipeline shell for updater testing.""" import types - from diffusers import LCMScheduler stream = types.SimpleNamespace() stream.device = device @@ -106,9 +109,7 @@ def _make_stream_shell(t_index_list, device="cpu", dtype=torch.float32, stream._beta_next = torch.cat( [stream.beta_prod_t_sqrt[1:], torch.ones_like(stream.beta_prod_t_sqrt[0:1])], dim=0 ) - stream._init_noise_rotated = torch.cat( - [stream.init_noise[1:], stream.init_noise[0:1]], dim=0 - ) + stream._init_noise_rotated = torch.cat([stream.init_noise[1:], stream.init_noise[0:1]], dim=0) else: stream._alpha_next = None stream._beta_next = None @@ -129,6 +130,7 @@ def _make_updater(stream): # tests # --------------------------------------------------------------------------- + class TestDerivedTensorSync: """F2: _update_timestep_calculations must keep _alpha_next/_beta_next in sync.""" @@ -144,13 +146,11 @@ def test_alpha_next_updated_after_t_index_change(self): # Change t_index values (same length, different values) updater._update_timestep_values_only([14, 28]) - expected = torch.cat( - [stream.alpha_prod_t_sqrt[1:], torch.ones_like(stream.alpha_prod_t_sqrt[0:1])], dim=0 + expected = torch.cat([stream.alpha_prod_t_sqrt[1:], torch.ones_like(stream.alpha_prod_t_sqrt[0:1])], dim=0) + assert not torch.allclose(old_alpha_next, stream._alpha_next), "_alpha_next was not updated (stale)" + assert torch.allclose(stream._alpha_next, expected, atol=1e-5), ( + f"_alpha_next mismatch: max_diff={(stream._alpha_next - expected).abs().max().item():.6f}" ) - assert not torch.allclose(old_alpha_next, stream._alpha_next), \ - "_alpha_next was not updated (stale)" - assert torch.allclose(stream._alpha_next, expected, atol=1e-5), \ - f"_alpha_next mismatch: max_diff={( stream._alpha_next - expected).abs().max().item():.6f}" def test_beta_next_updated_after_t_index_change(self): """After a same-length value-only t_index change, _beta_next must equal @@ -161,13 +161,11 @@ def test_beta_next_updated_after_t_index_change(self): updater._update_timestep_values_only([14, 28]) - expected = torch.cat( - [stream.beta_prod_t_sqrt[1:], torch.ones_like(stream.beta_prod_t_sqrt[0:1])], dim=0 - ) - assert not torch.allclose(old_beta_next, stream._beta_next), \ - "_beta_next was not updated (stale)" - assert torch.allclose(stream._beta_next, expected, atol=1e-5), \ + expected = torch.cat([stream.beta_prod_t_sqrt[1:], torch.ones_like(stream.beta_prod_t_sqrt[0:1])], dim=0) + assert not torch.allclose(old_beta_next, stream._beta_next), "_beta_next was not updated (stale)" + assert torch.allclose(stream._beta_next, expected, atol=1e-5), ( f"_beta_next mismatch: max_diff={(stream._beta_next - expected).abs().max().item():.6f}" + ) def test_init_noise_rotated_stays_consistent_after_t_index_change(self): """_init_noise_rotated must equal cat([init_noise[1:], init_noise[0:1]]) @@ -179,14 +177,14 @@ def test_init_noise_rotated_stays_consistent_after_t_index_change(self): updater._update_timestep_values_only([14, 28]) # init_noise should be unchanged - assert torch.allclose(stream.init_noise, saved_init_noise), \ + assert torch.allclose(stream.init_noise, saved_init_noise), ( "init_noise was unexpectedly mutated by _update_timestep_values_only" - - expected_rotated = torch.cat( - [stream.init_noise[1:], stream.init_noise[0:1]], dim=0 ) - assert torch.allclose(stream._init_noise_rotated, expected_rotated, atol=1e-6), \ + + expected_rotated = torch.cat([stream.init_noise[1:], stream.init_noise[0:1]], dim=0) + assert torch.allclose(stream._init_noise_rotated, expected_rotated, atol=1e-6), ( "_init_noise_rotated out of sync with init_noise after t_index update" + ) def test_no_update_when_derived_tensors_not_initialized(self): """When _alpha_next is None (non-batched or non-RCFG-self), updater must @@ -201,18 +199,21 @@ def test_no_update_when_derived_tensors_not_initialized(self): def test_warn_on_do_add_noise_false_high_beta(self, caplog): """When do_add_noise=False and inter-step beta_sqrt > 0.75, a warning must be logged.""" import logging + stream = _make_stream_shell([14, 28], do_add_noise=False) updater = _make_updater(stream) with caplog.at_level(logging.WARNING, logger="streamdiffusion.stream_parameter_updater"): updater._update_timestep_values_only([14, 28]) - assert any("do_add_noise=False" in r.message for r in caplog.records), \ + assert any("do_add_noise=False" in r.message for r in caplog.records), ( "Expected do_add_noise bleed-risk warning not emitted" + ) def test_no_warn_when_do_add_noise_true(self, caplog): """When do_add_noise=True, no bleed-risk warning should be logged.""" import logging + stream = _make_stream_shell([14, 28], do_add_noise=True) updater = _make_updater(stream) diff --git a/tests/unit/test_ipc_producer_stream.py b/tests/unit/test_ipc_producer_stream.py index 1b4059cbb..6238ab475 100644 --- a/tests/unit/test_ipc_producer_stream.py +++ b/tests/unit/test_ipc_producer_stream.py @@ -21,7 +21,6 @@ import unittest from unittest.mock import MagicMock - # --------------------------------------------------------------------------- # Helpers: minimal stubs so wrapper.py can be imported without CUDA / diffusers # --------------------------------------------------------------------------- diff --git a/tests/unit/test_l2tc_dynamic_shapes.py b/tests/unit/test_l2tc_dynamic_shapes.py index 1d3d3ca79..c778d9ab7 100644 --- a/tests/unit/test_l2tc_dynamic_shapes.py +++ b/tests/unit/test_l2tc_dynamic_shapes.py @@ -26,7 +26,6 @@ import pytest - try: from streamdiffusion.acceleration.tensorrt import utilities as trt_utilities diff --git a/tests/unit/test_normal_bae_fallback.py b/tests/unit/test_normal_bae_fallback.py index 3478315d2..a5c33adb5 100644 --- a/tests/unit/test_normal_bae_fallback.py +++ b/tests/unit/test_normal_bae_fallback.py @@ -20,7 +20,6 @@ import pytest - # --------------------------------------------------------------------------- # Skip guard — controlnet_aux must be importable # --------------------------------------------------------------------------- diff --git a/tests/unit/test_param_schema.py b/tests/unit/test_param_schema.py index cb5271044..6488baf8f 100644 --- a/tests/unit/test_param_schema.py +++ b/tests/unit/test_param_schema.py @@ -26,7 +26,6 @@ from streamdiffusion.stream_parameter_updater import StreamParameterUpdater from streamdiffusion.wrapper import StreamDiffusionWrapper - WRAPPER_ONLY_PARAMS = {"use_safety_checker", "safety_checker_threshold"} @@ -45,7 +44,7 @@ def test_updater_param_names_is_ordered_subsequence_of_param_names(self): def test_wrapper_only_params_excluded_from_updater(self): assert not (WRAPPER_ONLY_PARAMS & set(UPDATER_PARAM_NAMES)) - assert WRAPPER_ONLY_PARAMS <= set(PARAM_NAMES) + assert set(PARAM_NAMES) >= WRAPPER_ONLY_PARAMS def test_no_duplicate_names(self): assert len(PARAM_NAMES) == len(set(PARAM_NAMES)) diff --git a/tests/unit/test_param_updater_binding.py b/tests/unit/test_param_updater_binding.py index a87630a9c..b08412a31 100644 --- a/tests/unit/test_param_updater_binding.py +++ b/tests/unit/test_param_updater_binding.py @@ -24,7 +24,6 @@ from streamdiffusion.stream_parameter_updater import StreamParameterUpdater - # --------------------------------------------------------------------------- # Fixtures — mirror the minimal-fake-stream pattern from # tests/unit/test_prompt_interpolation.py so __init__ runs without a real pipeline. @@ -51,7 +50,7 @@ def _make_updater(**kwargs) -> StreamParameterUpdater: stream = _fake_stream() orig_attach = StreamParameterUpdater.attach_orchestrator - def _noop_attach(self, s): # noqa: ANN001 + def _noop_attach(self, s): self._preprocessing_orchestrator = None StreamParameterUpdater.attach_orchestrator = _noop_attach @@ -95,7 +94,7 @@ def test_positional_flag_binding_is_rejected(): """The keyword-only barrier makes the original bug a hard TypeError.""" stream = _fake_stream() with pytest.raises(TypeError): - StreamParameterUpdater(stream, False, False) # noqa: F841 + StreamParameterUpdater(stream, False, False) # --------------------------------------------------------------------------- diff --git a/tests/unit/test_phase3_correctness.py b/tests/unit/test_phase3_correctness.py index 49459b63e..a6997254b 100644 --- a/tests/unit/test_phase3_correctness.py +++ b/tests/unit/test_phase3_correctness.py @@ -34,7 +34,6 @@ from streamdiffusion.pipeline import StreamDiffusion from streamdiffusion.wrapper import StreamDiffusionWrapper - # --------------------------------------------------------------------------- # StreamDiffusion.prepare() -- mutable default removed # --------------------------------------------------------------------------- diff --git a/tests/unit/test_prompt_interpolation.py b/tests/unit/test_prompt_interpolation.py index d081efa35..1fe0cbe76 100644 --- a/tests/unit/test_prompt_interpolation.py +++ b/tests/unit/test_prompt_interpolation.py @@ -15,7 +15,6 @@ from streamdiffusion.stream_parameter_updater import StreamParameterUpdater - # --------------------------------------------------------------------------- # Minimal fake stream that satisfies the fields accessed during __init__ and # _apply_prompt_blending without touching the real pipeline. @@ -46,7 +45,7 @@ def _make_updater() -> StreamParameterUpdater: # a real PreprocessingOrchestrator. orig_attach = StreamParameterUpdater.attach_orchestrator - def _noop_attach(self, s): # noqa: ANN001 + def _noop_attach(self, s): self._preprocessing_orchestrator = None StreamParameterUpdater.attach_orchestrator = _noop_attach diff --git a/tests/unit/test_safety_checker.py b/tests/unit/test_safety_checker.py index 1e2b1cbbb..bcb657f92 100644 --- a/tests/unit/test_safety_checker.py +++ b/tests/unit/test_safety_checker.py @@ -29,7 +29,6 @@ from streamdiffusion.wrapper import StreamDiffusionWrapper - # --------------------------------------------------------------------------- # helpers # --------------------------------------------------------------------------- diff --git a/tests/unit/test_sync_free_output_5_2.py b/tests/unit/test_sync_free_output_5_2.py index 76780f1fe..3e3fd9747 100644 --- a/tests/unit/test_sync_free_output_5_2.py +++ b/tests/unit/test_sync_free_output_5_2.py @@ -32,7 +32,6 @@ from streamdiffusion.preprocessing.preprocessing_orchestrator import PreprocessingOrchestrator from streamdiffusion.wrapper import StreamDiffusionWrapper - pytestmark = pytest.mark.skipif( not torch.cuda.is_available(), reason="Sub-phase 5.2 output paths require CUDA (pin_memory / cuda.Event)", diff --git a/tests/unit/test_td_pending_params.py b/tests/unit/test_td_pending_params.py index 23bf4c6ae..8500db425 100644 --- a/tests/unit/test_td_pending_params.py +++ b/tests/unit/test_td_pending_params.py @@ -19,7 +19,6 @@ from streamdiffusion.param_schema import PARAM_NAMES - # --------------------------------------------------------------------------- # Minimal faithful replica of the three methods under test. # Copy-pasted from td_manager.py and frozen here so any future regression in diff --git a/tests/unit/test_trt_atomic_engine_write.py b/tests/unit/test_trt_atomic_engine_write.py index 4b874a2f8..215d6042d 100644 --- a/tests/unit/test_trt_atomic_engine_write.py +++ b/tests/unit/test_trt_atomic_engine_write.py @@ -16,7 +16,6 @@ import pytest - # --------------------------------------------------------------------------- # Import guard — skip all tests if utilities.py's dependencies are unavailable # --------------------------------------------------------------------------- diff --git a/tests/unit/test_trt_engine_guards.py b/tests/unit/test_trt_engine_guards.py index 759bc252a..a4c373497 100644 --- a/tests/unit/test_trt_engine_guards.py +++ b/tests/unit/test_trt_engine_guards.py @@ -14,7 +14,6 @@ import pytest - # --------------------------------------------------------------------------- # Import guard — skip all tests if TRT is unavailable # --------------------------------------------------------------------------- diff --git a/tests/unit/test_wrapper_exception_hygiene.py b/tests/unit/test_wrapper_exception_hygiene.py index af10d1e68..0227c25d0 100644 --- a/tests/unit/test_wrapper_exception_hygiene.py +++ b/tests/unit/test_wrapper_exception_hygiene.py @@ -34,7 +34,6 @@ from streamdiffusion import wrapper as wrapper_module from streamdiffusion.wrapper import StreamDiffusionWrapper, _is_oom_error - # --------------------------------------------------------------------------- # _is_oom_error # --------------------------------------------------------------------------- diff --git a/tests/unit/test_zero_copy_staging_5_6.py b/tests/unit/test_zero_copy_staging_5_6.py index dbba00bbf..310520e89 100644 --- a/tests/unit/test_zero_copy_staging_5_6.py +++ b/tests/unit/test_zero_copy_staging_5_6.py @@ -20,7 +20,6 @@ import pytest - try: from streamdiffusion.acceleration.tensorrt.utilities import _staging_action diff --git a/tools/summarize_audit.py b/tools/summarize_audit.py index f865c1447..da2e58620 100644 --- a/tools/summarize_audit.py +++ b/tools/summarize_audit.py @@ -32,7 +32,6 @@ import tomllib - # ============================================================================ # PACKAGE CLASSIFICATION CONSTANTS # ============================================================================ @@ -1399,7 +1398,7 @@ def generate_markdown_report( if has_safe: lines.append("**Safe to Keep (Development Tools)**:") for category in safe_categories: - if category in orphan_data and orphan_data[category]: + if orphan_data.get(category): tag, description = CATEGORY_LABELS.get(category, ("[?]", category)) for pkg in sorted(orphan_data[category], key=lambda x: x["name"]): lines.append(f"- `{pkg['name']}` ({pkg['version']})") @@ -1410,7 +1409,7 @@ def generate_markdown_report( if has_domain: lines.append("**Domain-Specific Packages (Auto-Detected)**:") for category in domain_categories: - if category in orphan_data and orphan_data[category]: + if orphan_data.get(category): tag, description = CATEGORY_LABELS.get(category, ("[?]", category)) lines.append(f"- {description}:") for pkg in sorted(orphan_data[category], key=lambda x: x["name"]): @@ -1645,7 +1644,7 @@ def main() -> None: print(f"Error: File not found: {json_file}", file=sys.stderr) sys.exit(1) - with open(json_file, "r", encoding="utf-8") as f: + with open(json_file, encoding="utf-8") as f: content = f.read() # Handle pip-audit header line (e.g., "No known vulnerabilities found") diff --git a/utils/viewer.py b/utils/viewer.py index 2bd90984b..a85ca72b7 100644 --- a/utils/viewer.py +++ b/utils/viewer.py @@ -9,7 +9,6 @@ from streamdiffusion.image_utils import postprocess_image - sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..")) From 6bd9df9bd5c54bd0126ab7faf735303f9dcc7653 Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 12 Jul 2026 01:59:17 -0400 Subject: [PATCH 26/37] chore: add pyrefly baseline freezing pre-existing type errors --- pyrefly-baseline.json | 9568 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 9568 insertions(+) create mode 100644 pyrefly-baseline.json diff --git a/pyrefly-baseline.json b/pyrefly-baseline.json new file mode 100644 index 000000000..913287883 --- /dev/null +++ b/pyrefly-baseline.json @@ -0,0 +1,9568 @@ +{ + "errors": [ + { + "line": 254, + "column": 22, + "stop_line": 254, + "stop_column": 35, + "path": "scripts\\profiling\\profile_nsys.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `str` is not assignable to parameter `acceleration` with type `Literal['none', 'tensorrt', 'xformers']` in function `streamdiffusion.wrapper.StreamDiffusionWrapper.__init__`", + "concise_description": "Argument `str` is not assignable to parameter `acceleration` with type `Literal['none', 'tensorrt', 'xformers']` in function `streamdiffusion.wrapper.StreamDiffusionWrapper.__init__`", + "severity": "error" + }, + { + "line": 350, + "column": 1, + "stop_line": 350, + "stop_column": 38, + "path": "scripts\\profiling\\profile_nsys.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `cudaProfilerStart`", + "concise_description": "Object of class `NoneType` has no attribute `cudaProfilerStart`", + "severity": "error" + }, + { + "line": 359, + "column": 1, + "stop_line": 359, + "stop_column": 37, + "path": "scripts\\profiling\\profile_nsys.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `cudaProfilerStop`", + "concise_description": "Object of class `NoneType` has no attribute `cudaProfilerStop`", + "severity": "error" + }, + { + "line": 103, + "column": 12, + "stop_line": 103, + "stop_column": 18, + "path": "scripts\\test_lora_sanity.py", + "code": -2, + "name": "bad-return", + "description": "Returned type `Tensor | list[Tensor]` is not assignable to declared return type `Image`", + "concise_description": "Returned type `Tensor | list[Tensor]` is not assignable to declared return type `Image`", + "severity": "error" + }, + { + "line": 97, + "column": 33, + "stop_line": 97, + "stop_column": 38, + "path": "src\\streamdiffusion\\_patches\\diffusers_kvo_patch.py", + "code": -2, + "name": "bad-assignment", + "description": "`(self: Unknown, attn: Unknown, hidden_states: Unknown, encoder_hidden_states: Unknown | None = None, attention_mask: Unknown | None = None, temb: Unknown | None = None, kvo_cache: Unknown | None = None, *args: Unknown, **kwargs: Unknown) -> tuple[Tensor, Unknown | None]` is not assignable to attribute `__call__` with type `(self: AttnProcessor2_0, attn: Attention, hidden_states: Tensor, encoder_hidden_states: Tensor | None = None, attention_mask: Tensor | None = None, temb: Tensor | None = None, kvo_cache: list[Tensor] | None = None, *args: Unknown, **kwargs: Unknown) -> Tensor`", + "concise_description": "`(self: Unknown, attn: Unknown, hidden_states: Unknown, encoder_hidden_states: Unknown | None = None, attention_mask: Unknown | None = None, temb: Unknown | None = None, kvo_cache: Unknown | None = None, *args: Unknown, **kwargs: Unknown) -> tuple[Tensor, Unknown | None]` is not assignable to attribute `__call__` with type `(self: AttnProcessor2_0, attn: Attention, hidden_states: Tensor, encoder_hidden_states: Tensor | None = None, attention_mask: Tensor | None = None, temb: Tensor | None = None, kvo_cache: list[Tensor] | None = None, *args: Unknown, **kwargs: Unknown) -> Tensor`", + "severity": "error" + }, + { + "line": 174, + "column": 60, + "stop_line": 174, + "stop_column": 96, + "path": "src\\streamdiffusion\\_patches\\diffusers_kvo_patch.py", + "code": -2, + "name": "unsupported-operation", + "description": "`None` is not subscriptable", + "concise_description": "`None` is not subscriptable", + "severity": "error" + }, + { + "line": 177, + "column": 48, + "stop_line": 177, + "stop_column": 64, + "path": "src\\streamdiffusion\\_patches\\diffusers_kvo_patch.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `reshape`", + "concise_description": "Object of class `NoneType` has no attribute `reshape`", + "severity": "error" + }, + { + "line": 199, + "column": 27, + "stop_line": 199, + "stop_column": 35, + "path": "src\\streamdiffusion\\_patches\\diffusers_kvo_patch.py", + "code": -2, + "name": "unbound-name", + "description": "`gate_msa` may be uninitialized", + "concise_description": "`gate_msa` may be uninitialized", + "severity": "error" + }, + { + "line": 201, + "column": 27, + "stop_line": 201, + "stop_column": 35, + "path": "src\\streamdiffusion\\_patches\\diffusers_kvo_patch.py", + "code": -2, + "name": "unbound-name", + "description": "`gate_msa` may be uninitialized", + "concise_description": "`gate_msa` may be uninitialized", + "severity": "error" + }, + { + "line": 218, + "column": 64, + "stop_line": 218, + "stop_column": 100, + "path": "src\\streamdiffusion\\_patches\\diffusers_kvo_patch.py", + "code": -2, + "name": "unsupported-operation", + "description": "`None` is not subscriptable", + "concise_description": "`None` is not subscriptable", + "severity": "error" + }, + { + "line": 236, + "column": 60, + "stop_line": 236, + "stop_column": 96, + "path": "src\\streamdiffusion\\_patches\\diffusers_kvo_patch.py", + "code": -2, + "name": "unsupported-operation", + "description": "`None` is not subscriptable", + "concise_description": "`None` is not subscriptable", + "severity": "error" + }, + { + "line": 241, + "column": 60, + "stop_line": 241, + "stop_column": 69, + "path": "src\\streamdiffusion\\_patches\\diffusers_kvo_patch.py", + "code": -2, + "name": "unbound-name", + "description": "`scale_mlp` may be uninitialized", + "concise_description": "`scale_mlp` may be uninitialized", + "severity": "error" + }, + { + "line": 241, + "column": 82, + "stop_line": 241, + "stop_column": 91, + "path": "src\\streamdiffusion\\_patches\\diffusers_kvo_patch.py", + "code": -2, + "name": "unbound-name", + "description": "`shift_mlp` may be uninitialized", + "concise_description": "`shift_mlp` may be uninitialized", + "severity": "error" + }, + { + "line": 255, + "column": 25, + "stop_line": 255, + "stop_column": 33, + "path": "src\\streamdiffusion\\_patches\\diffusers_kvo_patch.py", + "code": -2, + "name": "unbound-name", + "description": "`gate_mlp` may be uninitialized", + "concise_description": "`gate_mlp` may be uninitialized", + "severity": "error" + }, + { + "line": 257, + "column": 25, + "stop_line": 257, + "stop_column": 33, + "path": "src\\streamdiffusion\\_patches\\diffusers_kvo_patch.py", + "code": -2, + "name": "unbound-name", + "description": "`gate_mlp` may be uninitialized", + "concise_description": "`gate_mlp` may be uninitialized", + "severity": "error" + }, + { + "line": 341, + "column": 26, + "stop_line": 341, + "stop_column": 34, + "path": "src\\streamdiffusion\\_patches\\diffusers_kvo_patch.py", + "code": -2, + "name": "unbound-name", + "description": "`residual` may be uninitialized", + "concise_description": "`residual` may be uninitialized", + "severity": "error" + }, + { + "line": 342, + "column": 28, + "stop_line": 342, + "stop_column": 38, + "path": "src\\streamdiffusion\\_patches\\diffusers_kvo_patch.py", + "code": -2, + "name": "unbound-name", + "description": "`batch_size` may be uninitialized", + "concise_description": "`batch_size` may be uninitialized", + "severity": "error" + }, + { + "line": 343, + "column": 24, + "stop_line": 343, + "stop_column": 30, + "path": "src\\streamdiffusion\\_patches\\diffusers_kvo_patch.py", + "code": -2, + "name": "unbound-name", + "description": "`height` may be uninitialized", + "concise_description": "`height` may be uninitialized", + "severity": "error" + }, + { + "line": 344, + "column": 23, + "stop_line": 344, + "stop_column": 28, + "path": "src\\streamdiffusion\\_patches\\diffusers_kvo_patch.py", + "code": -2, + "name": "unbound-name", + "description": "`width` may be uninitialized", + "concise_description": "`width` may be uninitialized", + "severity": "error" + }, + { + "line": 345, + "column": 27, + "stop_line": 345, + "stop_column": 36, + "path": "src\\streamdiffusion\\_patches\\diffusers_kvo_patch.py", + "code": -2, + "name": "unbound-name", + "description": "`inner_dim` may be uninitialized", + "concise_description": "`inner_dim` may be uninitialized", + "severity": "error" + }, + { + "line": 354, + "column": 35, + "stop_line": 354, + "stop_column": 52, + "path": "src\\streamdiffusion\\_patches\\diffusers_kvo_patch.py", + "code": -2, + "name": "unbound-name", + "description": "`embedded_timestep` may be uninitialized", + "concise_description": "`embedded_timestep` may be uninitialized", + "severity": "error" + }, + { + "line": 355, + "column": 24, + "stop_line": 355, + "stop_column": 30, + "path": "src\\streamdiffusion\\_patches\\diffusers_kvo_patch.py", + "code": -2, + "name": "unbound-name", + "description": "`height` may be uninitialized", + "concise_description": "`height` may be uninitialized", + "severity": "error" + }, + { + "line": 356, + "column": 23, + "stop_line": 356, + "stop_column": 28, + "path": "src\\streamdiffusion\\_patches\\diffusers_kvo_patch.py", + "code": -2, + "name": "unbound-name", + "description": "`width` may be uninitialized", + "concise_description": "`width` may be uninitialized", + "severity": "error" + }, + { + "line": 360, + "column": 21, + "stop_line": 360, + "stop_column": 27, + "path": "src\\streamdiffusion\\_patches\\diffusers_kvo_patch.py", + "code": -2, + "name": "unbound-name", + "description": "`output` may be uninitialized", + "concise_description": "`output` may be uninitialized", + "severity": "error" + }, + { + "line": 362, + "column": 48, + "stop_line": 362, + "stop_column": 54, + "path": "src\\streamdiffusion\\_patches\\diffusers_kvo_patch.py", + "code": -2, + "name": "unbound-name", + "description": "`output` may be uninitialized", + "concise_description": "`output` may be uninitialized", + "severity": "error" + }, + { + "line": 408, + "column": 34, + "stop_line": 408, + "stop_column": 48, + "path": "src\\streamdiffusion\\_patches\\diffusers_kvo_patch.py", + "code": -2, + "name": "bad-index", + "description": "Cannot index into `object`\n Object of class `object` has no attribute `__getitem__`", + "concise_description": "Cannot index into `object`", + "severity": "error" + }, + { + "line": 426, + "column": 39, + "stop_line": 426, + "stop_column": 47, + "path": "src\\streamdiffusion\\_patches\\diffusers_kvo_patch.py", + "code": -2, + "name": "bad-assignment", + "description": "`(self: Unknown, hidden_states: Unknown, temb: Unknown | None = None, encoder_hidden_states: Unknown | None = None, attention_mask: Unknown | None = None, cross_attention_kwargs: Unknown | None = None, encoder_attention_mask: Unknown | None = None, kvo_cache: object | Unknown = ...) -> tuple[Unknown, list[Unknown]] | Unknown` is not assignable to attribute `forward` with type `(self: UNetMidBlock2DCrossAttn, hidden_states: Tensor, temb: Tensor | None = None, encoder_hidden_states: Tensor | None = None, attention_mask: Tensor | None = None, cross_attention_kwargs: dict[str, Any] | None = None, encoder_attention_mask: Tensor | None = None, kvo_cache: list[Tensor] | None = None) -> Tensor`", + "concise_description": "`(self: Unknown, hidden_states: Unknown, temb: Unknown | None = None, encoder_hidden_states: Unknown | None = None, attention_mask: Unknown | None = None, cross_attention_kwargs: Unknown | None = None, encoder_attention_mask: Unknown | None = None, kvo_cache: object | Unknown = ...) -> tuple[Unknown, list[Unknown]] | Unknown` is not assignable to attribute `forward` with type `(self: UNetMidBlock2DCrossAttn, hidden_states: Tensor, temb: Tensor | None = None, encoder_hidden_states: Tensor | None = None, attention_mask: Tensor | None = None, cross_attention_kwargs: dict[str, Any] | None = None, encoder_attention_mask: Tensor | None = None, kvo_cache: list[Tensor] | None = None) -> Tensor`", + "severity": "error" + }, + { + "line": 474, + "column": 34, + "stop_line": 474, + "stop_column": 46, + "path": "src\\streamdiffusion\\_patches\\diffusers_kvo_patch.py", + "code": -2, + "name": "bad-index", + "description": "Cannot index into `object`\n Object of class `object` has no attribute `__getitem__`", + "concise_description": "Cannot index into `object`", + "severity": "error" + }, + { + "line": 501, + "column": 36, + "stop_line": 501, + "stop_column": 44, + "path": "src\\streamdiffusion\\_patches\\diffusers_kvo_patch.py", + "code": -2, + "name": "bad-assignment", + "description": "`(self: Unknown, hidden_states: Unknown, temb: Unknown | None = None, encoder_hidden_states: Unknown | None = None, attention_mask: Unknown | None = None, cross_attention_kwargs: Unknown | None = None, encoder_attention_mask: Unknown | None = None, additional_residuals: Unknown | None = None, kvo_cache: object | Unknown = ...) -> tuple[Unknown, Unknown] | tuple[Unknown, Unknown, list[Unknown]]` is not assignable to attribute `forward` with type `(self: CrossAttnDownBlock2D, hidden_states: Tensor, temb: Tensor | None = None, encoder_hidden_states: Tensor | None = None, attention_mask: Tensor | None = None, cross_attention_kwargs: dict[str, Any] | None = None, encoder_attention_mask: Tensor | None = None, additional_residuals: Tensor | None = None, kvo_cache: list[Tensor] | None = None) -> tuple[Tensor, tuple[Tensor, ...]]`", + "concise_description": "`(self: Unknown, hidden_states: Unknown, temb: Unknown | None = None, encoder_hidden_states: Unknown | None = None, attention_mask: Unknown | None = None, cross_attention_kwargs: Unknown | None = None, encoder_attention_mask: Unknown | None = None, additional_residuals: Unknown | None = None, kvo_cache: object | Unknown = ...) -> tuple[Unknown, Unknown] | tuple[Unknown, Unknown, list[Unknown]]` is not assignable to attribute `forward` with type `(self: CrossAttnDownBlock2D, hidden_states: Tensor, temb: Tensor | None = None, encoder_hidden_states: Tensor | None = None, attention_mask: Tensor | None = None, cross_attention_kwargs: dict[str, Any] | None = None, encoder_attention_mask: Tensor | None = None, additional_residuals: Tensor | None = None, kvo_cache: list[Tensor] | None = None) -> tuple[Tensor, tuple[Tensor, ...]]`", + "severity": "error" + }, + { + "line": 544, + "column": 52, + "stop_line": 544, + "stop_column": 63, + "path": "src\\streamdiffusion\\_patches\\diffusers_kvo_patch.py", + "code": -2, + "name": "not-callable", + "description": "Expected a callable, got `None`", + "concise_description": "Expected a callable, got `None`", + "severity": "error" + }, + { + "line": 587, + "column": 34, + "stop_line": 587, + "stop_column": 42, + "path": "src\\streamdiffusion\\_patches\\diffusers_kvo_patch.py", + "code": -2, + "name": "bad-assignment", + "description": "`(self: Unknown, hidden_states: Unknown, res_hidden_states_tuple: Unknown, temb: Unknown | None = None, encoder_hidden_states: Unknown | None = None, cross_attention_kwargs: Unknown | None = None, upsample_size: Unknown | None = None, attention_mask: Unknown | None = None, encoder_attention_mask: Unknown | None = None, kvo_cache: Unknown | None = None) -> tuple[Unknown, list[Unknown]]` is not assignable to attribute `forward` with type `(self: CrossAttnUpBlock2D, hidden_states: Tensor, res_hidden_states_tuple: tuple[Tensor, ...], temb: Tensor | None = None, encoder_hidden_states: Tensor | None = None, cross_attention_kwargs: dict[str, Any] | None = None, upsample_size: int | None = None, attention_mask: Tensor | None = None, encoder_attention_mask: Tensor | None = None, kvo_cache: list[Tensor] | None = None) -> Tensor`", + "concise_description": "`(self: Unknown, hidden_states: Unknown, res_hidden_states_tuple: Unknown, temb: Unknown | None = None, encoder_hidden_states: Unknown | None = None, cross_attention_kwargs: Unknown | None = None, upsample_size: Unknown | None = None, attention_mask: Unknown | None = None, encoder_attention_mask: Unknown | None = None, kvo_cache: Unknown | None = None) -> tuple[Unknown, list[Unknown]]` is not assignable to attribute `forward` with type `(self: CrossAttnUpBlock2D, hidden_states: Tensor, res_hidden_states_tuple: tuple[Tensor, ...], temb: Tensor | None = None, encoder_hidden_states: Tensor | None = None, cross_attention_kwargs: dict[str, Any] | None = None, upsample_size: int | None = None, attention_mask: Tensor | None = None, encoder_attention_mask: Tensor | None = None, kvo_cache: list[Tensor] | None = None) -> Tensor`", + "severity": "error" + }, + { + "line": 687, + "column": 39, + "stop_line": 687, + "stop_column": 75, + "path": "src\\streamdiffusion\\_patches\\diffusers_kvo_patch.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Unknown | None` is not assignable to parameter `obj` with type `Sized` in function `len`\n Protocol `Sized` requires attribute `__len__`", + "concise_description": "Argument `Unknown | None` is not assignable to parameter `obj` with type `Sized` in function `len`", + "severity": "error" + }, + { + "line": 688, + "column": 68, + "stop_line": 688, + "stop_column": 108, + "path": "src\\streamdiffusion\\_patches\\diffusers_kvo_patch.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `pop`", + "concise_description": "Object of class `NoneType` has no attribute `pop`", + "severity": "error" + }, + { + "line": 706, + "column": 39, + "stop_line": 706, + "stop_column": 75, + "path": "src\\streamdiffusion\\_patches\\diffusers_kvo_patch.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Unknown | None` is not assignable to parameter `obj` with type `Sized` in function `len`\n Protocol `Sized` requires attribute `__len__`", + "concise_description": "Argument `Unknown | None` is not assignable to parameter `obj` with type `Sized` in function `len`", + "severity": "error" + }, + { + "line": 707, + "column": 31, + "stop_line": 707, + "stop_column": 71, + "path": "src\\streamdiffusion\\_patches\\diffusers_kvo_patch.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `pop`", + "concise_description": "Object of class `NoneType` has no attribute `pop`", + "severity": "error" + }, + { + "line": 740, + "column": 25, + "stop_line": 740, + "stop_column": 61, + "path": "src\\streamdiffusion\\_patches\\diffusers_kvo_patch.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Unknown | None` is not assignable to parameter `obj` with type `Sized` in function `len`\n Protocol `Sized` requires attribute `__len__`", + "concise_description": "Argument `Unknown | None` is not assignable to parameter `obj` with type `Sized` in function `len`", + "severity": "error" + }, + { + "line": 741, + "column": 37, + "stop_line": 741, + "stop_column": 76, + "path": "src\\streamdiffusion\\_patches\\diffusers_kvo_patch.py", + "code": -2, + "name": "unsupported-operation", + "description": "`None` is not subscriptable", + "concise_description": "`None` is not subscriptable", + "severity": "error" + }, + { + "line": 743, + "column": 27, + "stop_line": 743, + "stop_column": 67, + "path": "src\\streamdiffusion\\_patches\\diffusers_kvo_patch.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `pop`", + "concise_description": "Object of class `NoneType` has no attribute `pop`", + "severity": "error" + }, + { + "line": 3, + "column": 1, + "stop_line": 3, + "stop_column": 90, + "path": "src\\streamdiffusion\\acceleration\\sfast\\__init__.py", + "code": -2, + "name": "missing-import", + "description": "Cannot find module `sfast.compilers.stable_diffusion_pipeline_compiler`\n Looked in these locations (from config in `D:\\dev\\SDTD_032_dev\\StreamDiffusion\\pyproject.toml`):\n Search path (from config file): [\"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\src\"]\n Import root (inferred from project layout): \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\src\"\n Site package path queried from interpreter: [\"C:\\\\Users\\\\Inter\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python311\\\\DLLs\", \"C:\\\\Users\\\\Inter\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python311\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\venv\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\venv\\\\Lib\\\\site-packages\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\src\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\venv\\\\Lib\\\\site-packages\\\\win32\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\venv\\\\Lib\\\\site-packages\\\\win32\\\\lib\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\venv\\\\Lib\\\\site-packages\\\\Pythonwin\"]", + "concise_description": "Cannot find module `sfast.compilers.stable_diffusion_pipeline_compiler`", + "severity": "error" + }, + { + "line": 16, + "column": 20, + "stop_line": 16, + "stop_column": 28, + "path": "src\\streamdiffusion\\acceleration\\sfast\\__init__.py", + "code": -2, + "name": "missing-import", + "description": "Cannot find module `xformers`\n Looked in these locations (from config in `D:\\dev\\SDTD_032_dev\\StreamDiffusion\\pyproject.toml`):\n Search path (from config file): [\"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\src\"]\n Import root (inferred from project layout): \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\src\"\n Site package path queried from interpreter: [\"C:\\\\Users\\\\Inter\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python311\\\\DLLs\", \"C:\\\\Users\\\\Inter\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python311\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\venv\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\venv\\\\Lib\\\\site-packages\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\src\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\venv\\\\Lib\\\\site-packages\\\\win32\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\venv\\\\Lib\\\\site-packages\\\\win32\\\\lib\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\venv\\\\Lib\\\\site-packages\\\\Pythonwin\"]", + "concise_description": "Cannot find module `xformers`", + "severity": "error" + }, + { + "line": 33, + "column": 9, + "stop_line": 33, + "stop_column": 16, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\__init__.py", + "code": -2, + "name": "bad-override", + "description": "Class member `StableDiffusionSafetyCheckerWrapper.forward` overrides parent class `StableDiffusionSafetyChecker` in an inconsistent manner\n `StableDiffusionSafetyCheckerWrapper.forward` has type `(self: StableDiffusionSafetyCheckerWrapper, clip_input: Unknown) -> Tensor`, which is not assignable to `(self: StableDiffusionSafetyCheckerWrapper, clip_input: Unknown, images: Unknown) -> tuple[Unknown, list[bool]]`, the type of `StableDiffusionSafetyChecker.forward`\n Signature mismatch:\n ...CheckerWrapper, clip_input: Unknown, images: Unknown) -> tuple[Unknown, list[bool]]: ...\n ^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^ return type\n |\n parameters\n ...CheckerWrapper, clip_input: Unknown) -> Tensor: ...\n ^ ^^^^^^ return type\n |\n parameters", + "concise_description": "Class member `StableDiffusionSafetyCheckerWrapper.forward` overrides parent class `StableDiffusionSafetyChecker` in an inconsistent manner", + "severity": "error" + }, + { + "line": 59, + "column": 33, + "stop_line": 59, + "stop_column": 48, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\__init__.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `AutoencoderKL` has no attribute `encode`", + "concise_description": "Object of class `AutoencoderKL` has no attribute `encode`", + "severity": "error" + }, + { + "line": 91, + "column": 11, + "stop_line": 91, + "stop_column": 17, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\__init__.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `AutoencoderKL` has no attribute `to`", + "concise_description": "Object of class `AutoencoderKL` has no attribute `to`", + "severity": "error" + }, + { + "line": 153, + "column": 12, + "stop_line": 153, + "stop_column": 19, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\__init__.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `UNet2DConditionModel` has no attribute `to`", + "concise_description": "Object of class `UNet2DConditionModel` has no attribute `to`", + "severity": "error" + }, + { + "line": 187, + "column": 18, + "stop_line": 187, + "stop_column": 31, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\__init__.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `ControlNetModel` has no attribute `to`", + "concise_description": "Object of class `ControlNetModel` has no attribute `to`", + "severity": "error" + }, + { + "line": 256, + "column": 46, + "stop_line": 256, + "stop_column": 97, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\builder.py", + "code": -2, + "name": "unsupported-operation", + "description": "Cannot set item in `dict[str, dict[str, str]]`\n Argument `dict[str, float | str]` is not assignable to parameter `value` with type `dict[str, str]` in function `dict.__setitem__`", + "concise_description": "Cannot set item in `dict[str, dict[str, str]]`", + "severity": "error" + }, + { + "line": 276, + "column": 48, + "stop_line": 276, + "stop_column": 99, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\builder.py", + "code": -2, + "name": "unsupported-operation", + "description": "Cannot set item in `dict[str, dict[str, str]]`\n Argument `dict[str, float | str]` is not assignable to parameter `value` with type `dict[str, str]` in function `dict.__setitem__`", + "concise_description": "Cannot set item in `dict[str, dict[str, str]]`", + "severity": "error" + }, + { + "line": 408, + "column": 48, + "stop_line": 408, + "stop_column": 99, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\builder.py", + "code": -2, + "name": "unsupported-operation", + "description": "Cannot set item in `dict[str, dict[str, str]]`\n Argument `dict[str, float | str]` is not assignable to parameter `value` with type `dict[str, str]` in function `dict.__setitem__`", + "concise_description": "Cannot set item in `dict[str, dict[str, str]]`", + "severity": "error" + }, + { + "line": 419, + "column": 27, + "stop_line": 419, + "stop_column": 38, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\builder.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `Runtime` in module `tensorrt`", + "concise_description": "No attribute `Runtime` in module `tensorrt`", + "severity": "error" + }, + { + "line": 423, + "column": 58, + "stop_line": 423, + "stop_column": 84, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\builder.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `LayerInformationFormat` in module `tensorrt`", + "concise_description": "No attribute `LayerInformationFormat` in module `tensorrt`", + "severity": "error" + }, + { + "line": 462, + "column": 36, + "stop_line": 462, + "stop_column": 59, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\builder.py", + "code": -2, + "name": "bad-typed-dict-key", + "description": "`float` is not assignable to TypedDict key `total_elapsed_s` with type `bool | dict[str, dict[str, str]] | int | str`", + "concise_description": "`float` is not assignable to TypedDict key `total_elapsed_s` with type `bool | dict[str, dict[str, str]] | int | str`", + "severity": "error" + }, + { + "line": 467, + "column": 39, + "stop_line": 467, + "stop_column": 93, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\builder.py", + "code": -2, + "name": "bad-typed-dict-key", + "description": "`float` is not assignable to TypedDict key `engine_size_mb` with type `bool | dict[str, dict[str, str]] | int | str`", + "concise_description": "`float` is not assignable to TypedDict key `engine_size_mb` with type `bool | dict[str, dict[str, str]] | int | str`", + "severity": "error" + }, + { + "line": 142, + "column": 20, + "stop_line": 142, + "stop_column": 86, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "unsupported-operation", + "description": "`/` is not supported between `Path` and `(path: Unknown, cuda_stream: Unknown, **kwargs: Unknown) -> ControlNetModelEngine`\n Argument `(path: Unknown, cuda_stream: Unknown, **kwargs: Unknown) -> ControlNetModelEngine` is not assignable to parameter `key` with type `PathLike[str] | str` in function `pathlib.PurePath.__truediv__`\n Protocol `PathLike` requires attribute `__fspath__`", + "concise_description": "`/` is not supported between `Path` and `(path: Unknown, cuda_stream: Unknown, **kwargs: Unknown) -> ControlNetModelEngine`", + "severity": "error" + }, + { + "line": 142, + "column": 20, + "stop_line": 142, + "stop_column": 86, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "unsupported-operation", + "description": "`/` is not supported between `Path` and `(path: Unknown, cuda_stream: Unknown, **kwargs: Unknown) -> UNet2DConditionModelEngine`\n Argument `(path: Unknown, cuda_stream: Unknown, **kwargs: Unknown) -> UNet2DConditionModelEngine` is not assignable to parameter `key` with type `PathLike[str] | str` in function `pathlib.PurePath.__truediv__`\n Protocol `PathLike` requires attribute `__fspath__`", + "concise_description": "`/` is not supported between `Path` and `(path: Unknown, cuda_stream: Unknown, **kwargs: Unknown) -> UNet2DConditionModelEngine`", + "severity": "error" + }, + { + "line": 142, + "column": 20, + "stop_line": 142, + "stop_column": 86, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "unsupported-operation", + "description": "`/` is not supported between `Path` and `(path: Unknown, cuda_stream: Unknown, **kwargs: Unknown) -> str`\n Argument `(path: Unknown, cuda_stream: Unknown, **kwargs: Unknown) -> str` is not assignable to parameter `key` with type `PathLike[str] | str` in function `pathlib.PurePath.__truediv__`\n Protocol `PathLike` requires attribute `__fspath__`", + "concise_description": "`/` is not supported between `Path` and `(path: Unknown, cuda_stream: Unknown, **kwargs: Unknown) -> str`", + "severity": "error" + }, + { + "line": 142, + "column": 20, + "stop_line": 142, + "stop_column": 86, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "unsupported-operation", + "description": "`/` is not supported between `Path` and `(controlnet: diffusers.models.controlnets.controlnet.ControlNetModel | diffusers.utils.dummy_pt_objects.ControlNetModel, model_data: BaseModel, onnx_path: str, onnx_opt_path: str, engine_path: str, opt_batch_size: int | None = None, engine_build_options: dict[Unknown, Unknown] | None = None) -> None`\n Argument `(controlnet: diffusers.models.controlnets.controlnet.ControlNetModel | diffusers.utils.dummy_pt_objects.ControlNetModel, model_data: BaseModel, onnx_path: str, onnx_opt_path: str, engine_path: str, opt_batch_size: int | None = None, engine_build_options: dict[Unknown, Unknown] | None = None) -> None` is not assignable to parameter `key` with type `PathLike[str] | str` in function `pathlib.PurePath.__truediv__`\n Protocol `PathLike` requires attribute `__fspath__`", + "concise_description": "`/` is not supported between `Path` and `(controlnet: diffusers.models.controlnets.controlnet.ControlNetModel | diffusers.utils.dummy_pt_objects.ControlNetModel, model_data: BaseModel, onnx_path: str, onnx_opt_path: str, engine_path: str, opt_batch_size: int | None = None, engine_build_options: dict[Unknown, Unknown] | None = None) -> None`", + "severity": "error" + }, + { + "line": 142, + "column": 20, + "stop_line": 142, + "stop_column": 86, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "unsupported-operation", + "description": "`/` is not supported between `Path` and `(safety_checker: StableDiffusionSafetyCheckerWrapper, model_data: BaseModel, onnx_path: str, onnx_opt_path: str, engine_path: str, opt_batch_size: int | None = None, engine_build_options: dict[Unknown, Unknown] | None = None) -> None`\n Argument `(safety_checker: StableDiffusionSafetyCheckerWrapper, model_data: BaseModel, onnx_path: str, onnx_opt_path: str, engine_path: str, opt_batch_size: int | None = None, engine_build_options: dict[Unknown, Unknown] | None = None) -> None` is not assignable to parameter `key` with type `PathLike[str] | str` in function `pathlib.PurePath.__truediv__`\n Protocol `PathLike` requires attribute `__fspath__`", + "concise_description": "`/` is not supported between `Path` and `(safety_checker: StableDiffusionSafetyCheckerWrapper, model_data: BaseModel, onnx_path: str, onnx_opt_path: str, engine_path: str, opt_batch_size: int | None = None, engine_build_options: dict[Unknown, Unknown] | None = None) -> None`", + "severity": "error" + }, + { + "line": 142, + "column": 20, + "stop_line": 142, + "stop_column": 86, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "unsupported-operation", + "description": "`/` is not supported between `Path` and `(unet: diffusers.models.unets.unet_2d_condition.UNet2DConditionModel | diffusers.utils.dummy_pt_objects.UNet2DConditionModel, model_data: BaseModel, onnx_path: str, onnx_opt_path: str, engine_path: str, opt_batch_size: int | None = None, engine_build_options: dict[Unknown, Unknown] | None = None) -> None`\n Argument `(unet: diffusers.models.unets.unet_2d_condition.UNet2DConditionModel | diffusers.utils.dummy_pt_objects.UNet2DConditionModel, model_data: BaseModel, onnx_path: str, onnx_opt_path: str, engine_path: str, opt_batch_size: int | None = None, engine_build_options: dict[Unknown, Unknown] | None = None) -> None` is not assignable to parameter `key` with type `PathLike[str] | str` in function `pathlib.PurePath.__truediv__`\n Protocol `PathLike` requires attribute `__fspath__`", + "concise_description": "`/` is not supported between `Path` and `(unet: diffusers.models.unets.unet_2d_condition.UNet2DConditionModel | diffusers.utils.dummy_pt_objects.UNet2DConditionModel, model_data: BaseModel, onnx_path: str, onnx_opt_path: str, engine_path: str, opt_batch_size: int | None = None, engine_build_options: dict[Unknown, Unknown] | None = None) -> None`", + "severity": "error" + }, + { + "line": 142, + "column": 20, + "stop_line": 142, + "stop_column": 86, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "unsupported-operation", + "description": "`/` is not supported between `Path` and `(vae: diffusers.models.autoencoders.autoencoder_kl.AutoencoderKL | diffusers.utils.dummy_pt_objects.AutoencoderKL, model_data: BaseModel, onnx_path: str, onnx_opt_path: str, engine_path: str, opt_batch_size: int | None = None, engine_build_options: dict[Unknown, Unknown] | None = None) -> None`\n Argument `(vae: diffusers.models.autoencoders.autoencoder_kl.AutoencoderKL | diffusers.utils.dummy_pt_objects.AutoencoderKL, model_data: BaseModel, onnx_path: str, onnx_opt_path: str, engine_path: str, opt_batch_size: int | None = None, engine_build_options: dict[Unknown, Unknown] | None = None) -> None` is not assignable to parameter `key` with type `PathLike[str] | str` in function `pathlib.PurePath.__truediv__`\n Protocol `PathLike` requires attribute `__fspath__`", + "concise_description": "`/` is not supported between `Path` and `(vae: diffusers.models.autoencoders.autoencoder_kl.AutoencoderKL | diffusers.utils.dummy_pt_objects.AutoencoderKL, model_data: BaseModel, onnx_path: str, onnx_opt_path: str, engine_path: str, opt_batch_size: int | None = None, engine_build_options: dict[Unknown, Unknown] | None = None) -> None`", + "severity": "error" + }, + { + "line": 142, + "column": 20, + "stop_line": 142, + "stop_column": 86, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "unsupported-operation", + "description": "`/` is not supported between `Path` and `(vae: TorchVAEEncoder, model_data: BaseModel, onnx_path: str, onnx_opt_path: str, engine_path: str, opt_batch_size: int | None = None, engine_build_options: dict[Unknown, Unknown] | None = None) -> None`\n Argument `(vae: TorchVAEEncoder, model_data: BaseModel, onnx_path: str, onnx_opt_path: str, engine_path: str, opt_batch_size: int | None = None, engine_build_options: dict[Unknown, Unknown] | None = None) -> None` is not assignable to parameter `key` with type `PathLike[str] | str` in function `pathlib.PurePath.__truediv__`\n Protocol `PathLike` requires attribute `__fspath__`", + "concise_description": "`/` is not supported between `Path` and `(vae: TorchVAEEncoder, model_data: BaseModel, onnx_path: str, onnx_opt_path: str, engine_path: str, opt_batch_size: int | None = None, engine_build_options: dict[Unknown, Unknown] | None = None) -> None`", + "severity": "error" + }, + { + "line": 211, + "column": 20, + "stop_line": 211, + "stop_column": 55, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "unsupported-operation", + "description": "`/` is not supported between `Path` and `(path: Unknown, cuda_stream: Unknown, **kwargs: Unknown) -> ControlNetModelEngine`\n Argument `(path: Unknown, cuda_stream: Unknown, **kwargs: Unknown) -> ControlNetModelEngine` is not assignable to parameter `key` with type `PathLike[str] | str` in function `pathlib.PurePath.__truediv__`\n Protocol `PathLike` requires attribute `__fspath__`", + "concise_description": "`/` is not supported between `Path` and `(path: Unknown, cuda_stream: Unknown, **kwargs: Unknown) -> ControlNetModelEngine`", + "severity": "error" + }, + { + "line": 211, + "column": 20, + "stop_line": 211, + "stop_column": 55, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "unsupported-operation", + "description": "`/` is not supported between `Path` and `(path: Unknown, cuda_stream: Unknown, **kwargs: Unknown) -> UNet2DConditionModelEngine`\n Argument `(path: Unknown, cuda_stream: Unknown, **kwargs: Unknown) -> UNet2DConditionModelEngine` is not assignable to parameter `key` with type `PathLike[str] | str` in function `pathlib.PurePath.__truediv__`\n Protocol `PathLike` requires attribute `__fspath__`", + "concise_description": "`/` is not supported between `Path` and `(path: Unknown, cuda_stream: Unknown, **kwargs: Unknown) -> UNet2DConditionModelEngine`", + "severity": "error" + }, + { + "line": 211, + "column": 20, + "stop_line": 211, + "stop_column": 55, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "unsupported-operation", + "description": "`/` is not supported between `Path` and `(path: Unknown, cuda_stream: Unknown, **kwargs: Unknown) -> str`\n Argument `(path: Unknown, cuda_stream: Unknown, **kwargs: Unknown) -> str` is not assignable to parameter `key` with type `PathLike[str] | str` in function `pathlib.PurePath.__truediv__`\n Protocol `PathLike` requires attribute `__fspath__`", + "concise_description": "`/` is not supported between `Path` and `(path: Unknown, cuda_stream: Unknown, **kwargs: Unknown) -> str`", + "severity": "error" + }, + { + "line": 211, + "column": 20, + "stop_line": 211, + "stop_column": 55, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "unsupported-operation", + "description": "`/` is not supported between `Path` and `(controlnet: diffusers.models.controlnets.controlnet.ControlNetModel | diffusers.utils.dummy_pt_objects.ControlNetModel, model_data: BaseModel, onnx_path: str, onnx_opt_path: str, engine_path: str, opt_batch_size: int | None = None, engine_build_options: dict[Unknown, Unknown] | None = None) -> None`\n Argument `(controlnet: diffusers.models.controlnets.controlnet.ControlNetModel | diffusers.utils.dummy_pt_objects.ControlNetModel, model_data: BaseModel, onnx_path: str, onnx_opt_path: str, engine_path: str, opt_batch_size: int | None = None, engine_build_options: dict[Unknown, Unknown] | None = None) -> None` is not assignable to parameter `key` with type `PathLike[str] | str` in function `pathlib.PurePath.__truediv__`\n Protocol `PathLike` requires attribute `__fspath__`", + "concise_description": "`/` is not supported between `Path` and `(controlnet: diffusers.models.controlnets.controlnet.ControlNetModel | diffusers.utils.dummy_pt_objects.ControlNetModel, model_data: BaseModel, onnx_path: str, onnx_opt_path: str, engine_path: str, opt_batch_size: int | None = None, engine_build_options: dict[Unknown, Unknown] | None = None) -> None`", + "severity": "error" + }, + { + "line": 211, + "column": 20, + "stop_line": 211, + "stop_column": 55, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "unsupported-operation", + "description": "`/` is not supported between `Path` and `(safety_checker: StableDiffusionSafetyCheckerWrapper, model_data: BaseModel, onnx_path: str, onnx_opt_path: str, engine_path: str, opt_batch_size: int | None = None, engine_build_options: dict[Unknown, Unknown] | None = None) -> None`\n Argument `(safety_checker: StableDiffusionSafetyCheckerWrapper, model_data: BaseModel, onnx_path: str, onnx_opt_path: str, engine_path: str, opt_batch_size: int | None = None, engine_build_options: dict[Unknown, Unknown] | None = None) -> None` is not assignable to parameter `key` with type `PathLike[str] | str` in function `pathlib.PurePath.__truediv__`\n Protocol `PathLike` requires attribute `__fspath__`", + "concise_description": "`/` is not supported between `Path` and `(safety_checker: StableDiffusionSafetyCheckerWrapper, model_data: BaseModel, onnx_path: str, onnx_opt_path: str, engine_path: str, opt_batch_size: int | None = None, engine_build_options: dict[Unknown, Unknown] | None = None) -> None`", + "severity": "error" + }, + { + "line": 211, + "column": 20, + "stop_line": 211, + "stop_column": 55, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "unsupported-operation", + "description": "`/` is not supported between `Path` and `(unet: diffusers.models.unets.unet_2d_condition.UNet2DConditionModel | diffusers.utils.dummy_pt_objects.UNet2DConditionModel, model_data: BaseModel, onnx_path: str, onnx_opt_path: str, engine_path: str, opt_batch_size: int | None = None, engine_build_options: dict[Unknown, Unknown] | None = None) -> None`\n Argument `(unet: diffusers.models.unets.unet_2d_condition.UNet2DConditionModel | diffusers.utils.dummy_pt_objects.UNet2DConditionModel, model_data: BaseModel, onnx_path: str, onnx_opt_path: str, engine_path: str, opt_batch_size: int | None = None, engine_build_options: dict[Unknown, Unknown] | None = None) -> None` is not assignable to parameter `key` with type `PathLike[str] | str` in function `pathlib.PurePath.__truediv__`\n Protocol `PathLike` requires attribute `__fspath__`", + "concise_description": "`/` is not supported between `Path` and `(unet: diffusers.models.unets.unet_2d_condition.UNet2DConditionModel | diffusers.utils.dummy_pt_objects.UNet2DConditionModel, model_data: BaseModel, onnx_path: str, onnx_opt_path: str, engine_path: str, opt_batch_size: int | None = None, engine_build_options: dict[Unknown, Unknown] | None = None) -> None`", + "severity": "error" + }, + { + "line": 211, + "column": 20, + "stop_line": 211, + "stop_column": 55, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "unsupported-operation", + "description": "`/` is not supported between `Path` and `(vae: diffusers.models.autoencoders.autoencoder_kl.AutoencoderKL | diffusers.utils.dummy_pt_objects.AutoencoderKL, model_data: BaseModel, onnx_path: str, onnx_opt_path: str, engine_path: str, opt_batch_size: int | None = None, engine_build_options: dict[Unknown, Unknown] | None = None) -> None`\n Argument `(vae: diffusers.models.autoencoders.autoencoder_kl.AutoencoderKL | diffusers.utils.dummy_pt_objects.AutoencoderKL, model_data: BaseModel, onnx_path: str, onnx_opt_path: str, engine_path: str, opt_batch_size: int | None = None, engine_build_options: dict[Unknown, Unknown] | None = None) -> None` is not assignable to parameter `key` with type `PathLike[str] | str` in function `pathlib.PurePath.__truediv__`\n Protocol `PathLike` requires attribute `__fspath__`", + "concise_description": "`/` is not supported between `Path` and `(vae: diffusers.models.autoencoders.autoencoder_kl.AutoencoderKL | diffusers.utils.dummy_pt_objects.AutoencoderKL, model_data: BaseModel, onnx_path: str, onnx_opt_path: str, engine_path: str, opt_batch_size: int | None = None, engine_build_options: dict[Unknown, Unknown] | None = None) -> None`", + "severity": "error" + }, + { + "line": 211, + "column": 20, + "stop_line": 211, + "stop_column": 55, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "unsupported-operation", + "description": "`/` is not supported between `Path` and `(vae: TorchVAEEncoder, model_data: BaseModel, onnx_path: str, onnx_opt_path: str, engine_path: str, opt_batch_size: int | None = None, engine_build_options: dict[Unknown, Unknown] | None = None) -> None`\n Argument `(vae: TorchVAEEncoder, model_data: BaseModel, onnx_path: str, onnx_opt_path: str, engine_path: str, opt_batch_size: int | None = None, engine_build_options: dict[Unknown, Unknown] | None = None) -> None` is not assignable to parameter `key` with type `PathLike[str] | str` in function `pathlib.PurePath.__truediv__`\n Protocol `PathLike` requires attribute `__fspath__`", + "concise_description": "`/` is not supported between `Path` and `(vae: TorchVAEEncoder, model_data: BaseModel, onnx_path: str, onnx_opt_path: str, engine_path: str, opt_batch_size: int | None = None, engine_build_options: dict[Unknown, Unknown] | None = None) -> None`", + "severity": "error" + }, + { + "line": 346, + "column": 29, + "stop_line": 346, + "stop_column": 35, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "not-callable", + "description": "Expected a callable, got `str`", + "concise_description": "Expected a callable, got `str`", + "severity": "error" + }, + { + "line": 346, + "column": 35, + "stop_line": 346, + "stop_column": 75, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `onnx_path` in function `streamdiffusion.acceleration.tensorrt.compile_controlnet`", + "concise_description": "Missing argument `onnx_path` in function `streamdiffusion.acceleration.tensorrt.compile_controlnet`", + "severity": "error" + }, + { + "line": 346, + "column": 35, + "stop_line": 346, + "stop_column": 75, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `onnx_opt_path` in function `streamdiffusion.acceleration.tensorrt.compile_controlnet`", + "concise_description": "Missing argument `onnx_opt_path` in function `streamdiffusion.acceleration.tensorrt.compile_controlnet`", + "severity": "error" + }, + { + "line": 346, + "column": 35, + "stop_line": 346, + "stop_column": 75, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `engine_path` in function `streamdiffusion.acceleration.tensorrt.compile_controlnet`", + "concise_description": "Missing argument `engine_path` in function `streamdiffusion.acceleration.tensorrt.compile_controlnet`", + "severity": "error" + }, + { + "line": 346, + "column": 35, + "stop_line": 346, + "stop_column": 75, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `onnx_path` in function `streamdiffusion.acceleration.tensorrt.compile_safety_checker`", + "concise_description": "Missing argument `onnx_path` in function `streamdiffusion.acceleration.tensorrt.compile_safety_checker`", + "severity": "error" + }, + { + "line": 346, + "column": 35, + "stop_line": 346, + "stop_column": 75, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `onnx_opt_path` in function `streamdiffusion.acceleration.tensorrt.compile_safety_checker`", + "concise_description": "Missing argument `onnx_opt_path` in function `streamdiffusion.acceleration.tensorrt.compile_safety_checker`", + "severity": "error" + }, + { + "line": 346, + "column": 35, + "stop_line": 346, + "stop_column": 75, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `engine_path` in function `streamdiffusion.acceleration.tensorrt.compile_safety_checker`", + "concise_description": "Missing argument `engine_path` in function `streamdiffusion.acceleration.tensorrt.compile_safety_checker`", + "severity": "error" + }, + { + "line": 346, + "column": 35, + "stop_line": 346, + "stop_column": 75, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `onnx_path` in function `streamdiffusion.acceleration.tensorrt.compile_unet`", + "concise_description": "Missing argument `onnx_path` in function `streamdiffusion.acceleration.tensorrt.compile_unet`", + "severity": "error" + }, + { + "line": 346, + "column": 35, + "stop_line": 346, + "stop_column": 75, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `onnx_opt_path` in function `streamdiffusion.acceleration.tensorrt.compile_unet`", + "concise_description": "Missing argument `onnx_opt_path` in function `streamdiffusion.acceleration.tensorrt.compile_unet`", + "severity": "error" + }, + { + "line": 346, + "column": 35, + "stop_line": 346, + "stop_column": 75, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `engine_path` in function `streamdiffusion.acceleration.tensorrt.compile_unet`", + "concise_description": "Missing argument `engine_path` in function `streamdiffusion.acceleration.tensorrt.compile_unet`", + "severity": "error" + }, + { + "line": 346, + "column": 35, + "stop_line": 346, + "stop_column": 75, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `onnx_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_decoder`", + "concise_description": "Missing argument `onnx_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_decoder`", + "severity": "error" + }, + { + "line": 346, + "column": 35, + "stop_line": 346, + "stop_column": 75, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `onnx_opt_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_decoder`", + "concise_description": "Missing argument `onnx_opt_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_decoder`", + "severity": "error" + }, + { + "line": 346, + "column": 35, + "stop_line": 346, + "stop_column": 75, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `engine_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_decoder`", + "concise_description": "Missing argument `engine_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_decoder`", + "severity": "error" + }, + { + "line": 346, + "column": 35, + "stop_line": 346, + "stop_column": 75, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `onnx_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_encoder`", + "concise_description": "Missing argument `onnx_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_encoder`", + "severity": "error" + }, + { + "line": 346, + "column": 35, + "stop_line": 346, + "stop_column": 75, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `onnx_opt_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_encoder`", + "concise_description": "Missing argument `onnx_opt_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_encoder`", + "severity": "error" + }, + { + "line": 346, + "column": 35, + "stop_line": 346, + "stop_column": 75, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `engine_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_encoder`", + "concise_description": "Missing argument `engine_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_encoder`", + "severity": "error" + }, + { + "line": 346, + "column": 36, + "stop_line": 346, + "stop_column": 47, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Path` is not assignable to parameter `controlnet` with type `diffusers.models.controlnets.controlnet.ControlNetModel | diffusers.utils.dummy_pt_objects.ControlNetModel` in function `streamdiffusion.acceleration.tensorrt.compile_controlnet`", + "concise_description": "Argument `Path` is not assignable to parameter `controlnet` with type `diffusers.models.controlnets.controlnet.ControlNetModel | diffusers.utils.dummy_pt_objects.ControlNetModel` in function `streamdiffusion.acceleration.tensorrt.compile_controlnet`", + "severity": "error" + }, + { + "line": 346, + "column": 36, + "stop_line": 346, + "stop_column": 47, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Path` is not assignable to parameter `safety_checker` with type `StableDiffusionSafetyCheckerWrapper` in function `streamdiffusion.acceleration.tensorrt.compile_safety_checker`", + "concise_description": "Argument `Path` is not assignable to parameter `safety_checker` with type `StableDiffusionSafetyCheckerWrapper` in function `streamdiffusion.acceleration.tensorrt.compile_safety_checker`", + "severity": "error" + }, + { + "line": 346, + "column": 36, + "stop_line": 346, + "stop_column": 47, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Path` is not assignable to parameter `unet` with type `diffusers.models.unets.unet_2d_condition.UNet2DConditionModel | diffusers.utils.dummy_pt_objects.UNet2DConditionModel` in function `streamdiffusion.acceleration.tensorrt.compile_unet`", + "concise_description": "Argument `Path` is not assignable to parameter `unet` with type `diffusers.models.unets.unet_2d_condition.UNet2DConditionModel | diffusers.utils.dummy_pt_objects.UNet2DConditionModel` in function `streamdiffusion.acceleration.tensorrt.compile_unet`", + "severity": "error" + }, + { + "line": 346, + "column": 36, + "stop_line": 346, + "stop_column": 47, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Path` is not assignable to parameter `vae` with type `diffusers.models.autoencoders.autoencoder_kl.AutoencoderKL | diffusers.utils.dummy_pt_objects.AutoencoderKL` in function `streamdiffusion.acceleration.tensorrt.compile_vae_decoder`", + "concise_description": "Argument `Path` is not assignable to parameter `vae` with type `diffusers.models.autoencoders.autoencoder_kl.AutoencoderKL | diffusers.utils.dummy_pt_objects.AutoencoderKL` in function `streamdiffusion.acceleration.tensorrt.compile_vae_decoder`", + "severity": "error" + }, + { + "line": 346, + "column": 36, + "stop_line": 346, + "stop_column": 47, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Path` is not assignable to parameter `vae` with type `TorchVAEEncoder` in function `streamdiffusion.acceleration.tensorrt.compile_vae_encoder`", + "concise_description": "Argument `Path` is not assignable to parameter `vae` with type `TorchVAEEncoder` in function `streamdiffusion.acceleration.tensorrt.compile_vae_encoder`", + "severity": "error" + }, + { + "line": 346, + "column": 49, + "stop_line": 346, + "stop_column": 74, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `dict[Unknown, Unknown] | None` is not assignable to parameter `model_data` with type `BaseModel` in function `streamdiffusion.acceleration.tensorrt.compile_controlnet`", + "concise_description": "Argument `dict[Unknown, Unknown] | None` is not assignable to parameter `model_data` with type `BaseModel` in function `streamdiffusion.acceleration.tensorrt.compile_controlnet`", + "severity": "error" + }, + { + "line": 346, + "column": 49, + "stop_line": 346, + "stop_column": 74, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `dict[Unknown, Unknown] | None` is not assignable to parameter `model_data` with type `BaseModel` in function `streamdiffusion.acceleration.tensorrt.compile_safety_checker`", + "concise_description": "Argument `dict[Unknown, Unknown] | None` is not assignable to parameter `model_data` with type `BaseModel` in function `streamdiffusion.acceleration.tensorrt.compile_safety_checker`", + "severity": "error" + }, + { + "line": 346, + "column": 49, + "stop_line": 346, + "stop_column": 74, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `dict[Unknown, Unknown] | None` is not assignable to parameter `model_data` with type `BaseModel` in function `streamdiffusion.acceleration.tensorrt.compile_unet`", + "concise_description": "Argument `dict[Unknown, Unknown] | None` is not assignable to parameter `model_data` with type `BaseModel` in function `streamdiffusion.acceleration.tensorrt.compile_unet`", + "severity": "error" + }, + { + "line": 346, + "column": 49, + "stop_line": 346, + "stop_column": 74, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `dict[Unknown, Unknown] | None` is not assignable to parameter `model_data` with type `BaseModel` in function `streamdiffusion.acceleration.tensorrt.compile_vae_decoder`", + "concise_description": "Argument `dict[Unknown, Unknown] | None` is not assignable to parameter `model_data` with type `BaseModel` in function `streamdiffusion.acceleration.tensorrt.compile_vae_decoder`", + "severity": "error" + }, + { + "line": 346, + "column": 49, + "stop_line": 346, + "stop_column": 74, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `dict[Unknown, Unknown] | None` is not assignable to parameter `model_data` with type `BaseModel` in function `streamdiffusion.acceleration.tensorrt.compile_vae_encoder`", + "concise_description": "Argument `dict[Unknown, Unknown] | None` is not assignable to parameter `model_data` with type `BaseModel` in function `streamdiffusion.acceleration.tensorrt.compile_vae_encoder`", + "severity": "error" + }, + { + "line": 351, + "column": 20, + "stop_line": 351, + "stop_column": 26, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "not-callable", + "description": "Expected a callable, got `str`", + "concise_description": "Expected a callable, got `str`", + "severity": "error" + }, + { + "line": 351, + "column": 26, + "stop_line": 356, + "stop_column": 14, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `onnx_path` in function `streamdiffusion.acceleration.tensorrt.compile_controlnet`", + "concise_description": "Missing argument `onnx_path` in function `streamdiffusion.acceleration.tensorrt.compile_controlnet`", + "severity": "error" + }, + { + "line": 351, + "column": 26, + "stop_line": 356, + "stop_column": 14, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `onnx_opt_path` in function `streamdiffusion.acceleration.tensorrt.compile_controlnet`", + "concise_description": "Missing argument `onnx_opt_path` in function `streamdiffusion.acceleration.tensorrt.compile_controlnet`", + "severity": "error" + }, + { + "line": 351, + "column": 26, + "stop_line": 356, + "stop_column": 14, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `engine_path` in function `streamdiffusion.acceleration.tensorrt.compile_controlnet`", + "concise_description": "Missing argument `engine_path` in function `streamdiffusion.acceleration.tensorrt.compile_controlnet`", + "severity": "error" + }, + { + "line": 351, + "column": 26, + "stop_line": 356, + "stop_column": 14, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `onnx_path` in function `streamdiffusion.acceleration.tensorrt.compile_safety_checker`", + "concise_description": "Missing argument `onnx_path` in function `streamdiffusion.acceleration.tensorrt.compile_safety_checker`", + "severity": "error" + }, + { + "line": 351, + "column": 26, + "stop_line": 356, + "stop_column": 14, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `onnx_opt_path` in function `streamdiffusion.acceleration.tensorrt.compile_safety_checker`", + "concise_description": "Missing argument `onnx_opt_path` in function `streamdiffusion.acceleration.tensorrt.compile_safety_checker`", + "severity": "error" + }, + { + "line": 351, + "column": 26, + "stop_line": 356, + "stop_column": 14, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `engine_path` in function `streamdiffusion.acceleration.tensorrt.compile_safety_checker`", + "concise_description": "Missing argument `engine_path` in function `streamdiffusion.acceleration.tensorrt.compile_safety_checker`", + "severity": "error" + }, + { + "line": 351, + "column": 26, + "stop_line": 356, + "stop_column": 14, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `onnx_path` in function `streamdiffusion.acceleration.tensorrt.compile_unet`", + "concise_description": "Missing argument `onnx_path` in function `streamdiffusion.acceleration.tensorrt.compile_unet`", + "severity": "error" + }, + { + "line": 351, + "column": 26, + "stop_line": 356, + "stop_column": 14, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `onnx_opt_path` in function `streamdiffusion.acceleration.tensorrt.compile_unet`", + "concise_description": "Missing argument `onnx_opt_path` in function `streamdiffusion.acceleration.tensorrt.compile_unet`", + "severity": "error" + }, + { + "line": 351, + "column": 26, + "stop_line": 356, + "stop_column": 14, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `engine_path` in function `streamdiffusion.acceleration.tensorrt.compile_unet`", + "concise_description": "Missing argument `engine_path` in function `streamdiffusion.acceleration.tensorrt.compile_unet`", + "severity": "error" + }, + { + "line": 351, + "column": 26, + "stop_line": 356, + "stop_column": 14, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `onnx_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_decoder`", + "concise_description": "Missing argument `onnx_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_decoder`", + "severity": "error" + }, + { + "line": 351, + "column": 26, + "stop_line": 356, + "stop_column": 14, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `onnx_opt_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_decoder`", + "concise_description": "Missing argument `onnx_opt_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_decoder`", + "severity": "error" + }, + { + "line": 351, + "column": 26, + "stop_line": 356, + "stop_column": 14, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `engine_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_decoder`", + "concise_description": "Missing argument `engine_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_decoder`", + "severity": "error" + }, + { + "line": 351, + "column": 26, + "stop_line": 356, + "stop_column": 14, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `onnx_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_encoder`", + "concise_description": "Missing argument `onnx_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_encoder`", + "severity": "error" + }, + { + "line": 351, + "column": 26, + "stop_line": 356, + "stop_column": 14, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `onnx_opt_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_encoder`", + "concise_description": "Missing argument `onnx_opt_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_encoder`", + "severity": "error" + }, + { + "line": 351, + "column": 26, + "stop_line": 356, + "stop_column": 14, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `engine_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_encoder`", + "concise_description": "Missing argument `engine_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_encoder`", + "severity": "error" + }, + { + "line": 352, + "column": 17, + "stop_line": 352, + "stop_column": 28, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Path` is not assignable to parameter `controlnet` with type `diffusers.models.controlnets.controlnet.ControlNetModel | diffusers.utils.dummy_pt_objects.ControlNetModel` in function `streamdiffusion.acceleration.tensorrt.compile_controlnet`", + "concise_description": "Argument `Path` is not assignable to parameter `controlnet` with type `diffusers.models.controlnets.controlnet.ControlNetModel | diffusers.utils.dummy_pt_objects.ControlNetModel` in function `streamdiffusion.acceleration.tensorrt.compile_controlnet`", + "severity": "error" + }, + { + "line": 352, + "column": 17, + "stop_line": 352, + "stop_column": 28, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Path` is not assignable to parameter `safety_checker` with type `StableDiffusionSafetyCheckerWrapper` in function `streamdiffusion.acceleration.tensorrt.compile_safety_checker`", + "concise_description": "Argument `Path` is not assignable to parameter `safety_checker` with type `StableDiffusionSafetyCheckerWrapper` in function `streamdiffusion.acceleration.tensorrt.compile_safety_checker`", + "severity": "error" + }, + { + "line": 352, + "column": 17, + "stop_line": 352, + "stop_column": 28, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Path` is not assignable to parameter `unet` with type `diffusers.models.unets.unet_2d_condition.UNet2DConditionModel | diffusers.utils.dummy_pt_objects.UNet2DConditionModel` in function `streamdiffusion.acceleration.tensorrt.compile_unet`", + "concise_description": "Argument `Path` is not assignable to parameter `unet` with type `diffusers.models.unets.unet_2d_condition.UNet2DConditionModel | diffusers.utils.dummy_pt_objects.UNet2DConditionModel` in function `streamdiffusion.acceleration.tensorrt.compile_unet`", + "severity": "error" + }, + { + "line": 352, + "column": 17, + "stop_line": 352, + "stop_column": 28, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Path` is not assignable to parameter `vae` with type `diffusers.models.autoencoders.autoencoder_kl.AutoencoderKL | diffusers.utils.dummy_pt_objects.AutoencoderKL` in function `streamdiffusion.acceleration.tensorrt.compile_vae_decoder`", + "concise_description": "Argument `Path` is not assignable to parameter `vae` with type `diffusers.models.autoencoders.autoencoder_kl.AutoencoderKL | diffusers.utils.dummy_pt_objects.AutoencoderKL` in function `streamdiffusion.acceleration.tensorrt.compile_vae_decoder`", + "severity": "error" + }, + { + "line": 352, + "column": 17, + "stop_line": 352, + "stop_column": 28, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Path` is not assignable to parameter `vae` with type `TorchVAEEncoder` in function `streamdiffusion.acceleration.tensorrt.compile_vae_encoder`", + "concise_description": "Argument `Path` is not assignable to parameter `vae` with type `TorchVAEEncoder` in function `streamdiffusion.acceleration.tensorrt.compile_vae_encoder`", + "severity": "error" + }, + { + "line": 353, + "column": 17, + "stop_line": 353, + "stop_column": 42, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `dict[Unknown, Unknown] | None` is not assignable to parameter `model_data` with type `BaseModel` in function `streamdiffusion.acceleration.tensorrt.compile_controlnet`", + "concise_description": "Argument `dict[Unknown, Unknown] | None` is not assignable to parameter `model_data` with type `BaseModel` in function `streamdiffusion.acceleration.tensorrt.compile_controlnet`", + "severity": "error" + }, + { + "line": 353, + "column": 17, + "stop_line": 353, + "stop_column": 42, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `dict[Unknown, Unknown] | None` is not assignable to parameter `model_data` with type `BaseModel` in function `streamdiffusion.acceleration.tensorrt.compile_safety_checker`", + "concise_description": "Argument `dict[Unknown, Unknown] | None` is not assignable to parameter `model_data` with type `BaseModel` in function `streamdiffusion.acceleration.tensorrt.compile_safety_checker`", + "severity": "error" + }, + { + "line": 353, + "column": 17, + "stop_line": 353, + "stop_column": 42, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `dict[Unknown, Unknown] | None` is not assignable to parameter `model_data` with type `BaseModel` in function `streamdiffusion.acceleration.tensorrt.compile_unet`", + "concise_description": "Argument `dict[Unknown, Unknown] | None` is not assignable to parameter `model_data` with type `BaseModel` in function `streamdiffusion.acceleration.tensorrt.compile_unet`", + "severity": "error" + }, + { + "line": 353, + "column": 17, + "stop_line": 353, + "stop_column": 42, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `dict[Unknown, Unknown] | None` is not assignable to parameter `model_data` with type `BaseModel` in function `streamdiffusion.acceleration.tensorrt.compile_vae_decoder`", + "concise_description": "Argument `dict[Unknown, Unknown] | None` is not assignable to parameter `model_data` with type `BaseModel` in function `streamdiffusion.acceleration.tensorrt.compile_vae_decoder`", + "severity": "error" + }, + { + "line": 353, + "column": 17, + "stop_line": 353, + "stop_column": 42, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `dict[Unknown, Unknown] | None` is not assignable to parameter `model_data` with type `BaseModel` in function `streamdiffusion.acceleration.tensorrt.compile_vae_encoder`", + "concise_description": "Argument `dict[Unknown, Unknown] | None` is not assignable to parameter `model_data` with type `BaseModel` in function `streamdiffusion.acceleration.tensorrt.compile_vae_encoder`", + "severity": "error" + }, + { + "line": 354, + "column": 17, + "stop_line": 354, + "stop_column": 27, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "unexpected-keyword", + "description": "Unexpected keyword argument `model_type` in function `streamdiffusion.acceleration.tensorrt.compile_controlnet`", + "concise_description": "Unexpected keyword argument `model_type` in function `streamdiffusion.acceleration.tensorrt.compile_controlnet`", + "severity": "error" + }, + { + "line": 354, + "column": 17, + "stop_line": 354, + "stop_column": 27, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "unexpected-keyword", + "description": "Unexpected keyword argument `model_type` in function `streamdiffusion.acceleration.tensorrt.compile_safety_checker`", + "concise_description": "Unexpected keyword argument `model_type` in function `streamdiffusion.acceleration.tensorrt.compile_safety_checker`", + "severity": "error" + }, + { + "line": 354, + "column": 17, + "stop_line": 354, + "stop_column": 27, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "unexpected-keyword", + "description": "Unexpected keyword argument `model_type` in function `streamdiffusion.acceleration.tensorrt.compile_unet`", + "concise_description": "Unexpected keyword argument `model_type` in function `streamdiffusion.acceleration.tensorrt.compile_unet`", + "severity": "error" + }, + { + "line": 354, + "column": 17, + "stop_line": 354, + "stop_column": 27, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "unexpected-keyword", + "description": "Unexpected keyword argument `model_type` in function `streamdiffusion.acceleration.tensorrt.compile_vae_decoder`", + "concise_description": "Unexpected keyword argument `model_type` in function `streamdiffusion.acceleration.tensorrt.compile_vae_decoder`", + "severity": "error" + }, + { + "line": 354, + "column": 17, + "stop_line": 354, + "stop_column": 27, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "unexpected-keyword", + "description": "Unexpected keyword argument `model_type` in function `streamdiffusion.acceleration.tensorrt.compile_vae_encoder`", + "concise_description": "Unexpected keyword argument `model_type` in function `streamdiffusion.acceleration.tensorrt.compile_vae_encoder`", + "severity": "error" + }, + { + "line": 355, + "column": 17, + "stop_line": 355, + "stop_column": 31, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "unexpected-keyword", + "description": "Unexpected keyword argument `use_cuda_graph` in function `streamdiffusion.acceleration.tensorrt.compile_controlnet`", + "concise_description": "Unexpected keyword argument `use_cuda_graph` in function `streamdiffusion.acceleration.tensorrt.compile_controlnet`", + "severity": "error" + }, + { + "line": 355, + "column": 17, + "stop_line": 355, + "stop_column": 31, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "unexpected-keyword", + "description": "Unexpected keyword argument `use_cuda_graph` in function `streamdiffusion.acceleration.tensorrt.compile_safety_checker`", + "concise_description": "Unexpected keyword argument `use_cuda_graph` in function `streamdiffusion.acceleration.tensorrt.compile_safety_checker`", + "severity": "error" + }, + { + "line": 355, + "column": 17, + "stop_line": 355, + "stop_column": 31, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "unexpected-keyword", + "description": "Unexpected keyword argument `use_cuda_graph` in function `streamdiffusion.acceleration.tensorrt.compile_unet`", + "concise_description": "Unexpected keyword argument `use_cuda_graph` in function `streamdiffusion.acceleration.tensorrt.compile_unet`", + "severity": "error" + }, + { + "line": 355, + "column": 17, + "stop_line": 355, + "stop_column": 31, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "unexpected-keyword", + "description": "Unexpected keyword argument `use_cuda_graph` in function `streamdiffusion.acceleration.tensorrt.compile_vae_decoder`", + "concise_description": "Unexpected keyword argument `use_cuda_graph` in function `streamdiffusion.acceleration.tensorrt.compile_vae_decoder`", + "severity": "error" + }, + { + "line": 355, + "column": 17, + "stop_line": 355, + "stop_column": 31, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "unexpected-keyword", + "description": "Unexpected keyword argument `use_cuda_graph` in function `streamdiffusion.acceleration.tensorrt.compile_vae_encoder`", + "concise_description": "Unexpected keyword argument `use_cuda_graph` in function `streamdiffusion.acceleration.tensorrt.compile_vae_encoder`", + "severity": "error" + }, + { + "line": 358, + "column": 20, + "stop_line": 358, + "stop_column": 26, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "not-callable", + "description": "Expected a callable, got `str`", + "concise_description": "Expected a callable, got `str`", + "severity": "error" + }, + { + "line": 358, + "column": 26, + "stop_line": 358, + "stop_column": 66, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `onnx_path` in function `streamdiffusion.acceleration.tensorrt.compile_controlnet`", + "concise_description": "Missing argument `onnx_path` in function `streamdiffusion.acceleration.tensorrt.compile_controlnet`", + "severity": "error" + }, + { + "line": 358, + "column": 26, + "stop_line": 358, + "stop_column": 66, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `onnx_opt_path` in function `streamdiffusion.acceleration.tensorrt.compile_controlnet`", + "concise_description": "Missing argument `onnx_opt_path` in function `streamdiffusion.acceleration.tensorrt.compile_controlnet`", + "severity": "error" + }, + { + "line": 358, + "column": 26, + "stop_line": 358, + "stop_column": 66, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `engine_path` in function `streamdiffusion.acceleration.tensorrt.compile_controlnet`", + "concise_description": "Missing argument `engine_path` in function `streamdiffusion.acceleration.tensorrt.compile_controlnet`", + "severity": "error" + }, + { + "line": 358, + "column": 26, + "stop_line": 358, + "stop_column": 66, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `onnx_path` in function `streamdiffusion.acceleration.tensorrt.compile_safety_checker`", + "concise_description": "Missing argument `onnx_path` in function `streamdiffusion.acceleration.tensorrt.compile_safety_checker`", + "severity": "error" + }, + { + "line": 358, + "column": 26, + "stop_line": 358, + "stop_column": 66, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `onnx_opt_path` in function `streamdiffusion.acceleration.tensorrt.compile_safety_checker`", + "concise_description": "Missing argument `onnx_opt_path` in function `streamdiffusion.acceleration.tensorrt.compile_safety_checker`", + "severity": "error" + }, + { + "line": 358, + "column": 26, + "stop_line": 358, + "stop_column": 66, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `engine_path` in function `streamdiffusion.acceleration.tensorrt.compile_safety_checker`", + "concise_description": "Missing argument `engine_path` in function `streamdiffusion.acceleration.tensorrt.compile_safety_checker`", + "severity": "error" + }, + { + "line": 358, + "column": 26, + "stop_line": 358, + "stop_column": 66, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `onnx_path` in function `streamdiffusion.acceleration.tensorrt.compile_unet`", + "concise_description": "Missing argument `onnx_path` in function `streamdiffusion.acceleration.tensorrt.compile_unet`", + "severity": "error" + }, + { + "line": 358, + "column": 26, + "stop_line": 358, + "stop_column": 66, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `onnx_opt_path` in function `streamdiffusion.acceleration.tensorrt.compile_unet`", + "concise_description": "Missing argument `onnx_opt_path` in function `streamdiffusion.acceleration.tensorrt.compile_unet`", + "severity": "error" + }, + { + "line": 358, + "column": 26, + "stop_line": 358, + "stop_column": 66, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `engine_path` in function `streamdiffusion.acceleration.tensorrt.compile_unet`", + "concise_description": "Missing argument `engine_path` in function `streamdiffusion.acceleration.tensorrt.compile_unet`", + "severity": "error" + }, + { + "line": 358, + "column": 26, + "stop_line": 358, + "stop_column": 66, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `onnx_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_decoder`", + "concise_description": "Missing argument `onnx_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_decoder`", + "severity": "error" + }, + { + "line": 358, + "column": 26, + "stop_line": 358, + "stop_column": 66, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `onnx_opt_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_decoder`", + "concise_description": "Missing argument `onnx_opt_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_decoder`", + "severity": "error" + }, + { + "line": 358, + "column": 26, + "stop_line": 358, + "stop_column": 66, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `engine_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_decoder`", + "concise_description": "Missing argument `engine_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_decoder`", + "severity": "error" + }, + { + "line": 358, + "column": 26, + "stop_line": 358, + "stop_column": 66, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `onnx_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_encoder`", + "concise_description": "Missing argument `onnx_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_encoder`", + "severity": "error" + }, + { + "line": 358, + "column": 26, + "stop_line": 358, + "stop_column": 66, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `onnx_opt_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_encoder`", + "concise_description": "Missing argument `onnx_opt_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_encoder`", + "severity": "error" + }, + { + "line": 358, + "column": 26, + "stop_line": 358, + "stop_column": 66, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "missing-argument", + "description": "Missing argument `engine_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_encoder`", + "concise_description": "Missing argument `engine_path` in function `streamdiffusion.acceleration.tensorrt.compile_vae_encoder`", + "severity": "error" + }, + { + "line": 358, + "column": 27, + "stop_line": 358, + "stop_column": 38, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Path` is not assignable to parameter `controlnet` with type `diffusers.models.controlnets.controlnet.ControlNetModel | diffusers.utils.dummy_pt_objects.ControlNetModel` in function `streamdiffusion.acceleration.tensorrt.compile_controlnet`", + "concise_description": "Argument `Path` is not assignable to parameter `controlnet` with type `diffusers.models.controlnets.controlnet.ControlNetModel | diffusers.utils.dummy_pt_objects.ControlNetModel` in function `streamdiffusion.acceleration.tensorrt.compile_controlnet`", + "severity": "error" + }, + { + "line": 358, + "column": 27, + "stop_line": 358, + "stop_column": 38, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Path` is not assignable to parameter `safety_checker` with type `StableDiffusionSafetyCheckerWrapper` in function `streamdiffusion.acceleration.tensorrt.compile_safety_checker`", + "concise_description": "Argument `Path` is not assignable to parameter `safety_checker` with type `StableDiffusionSafetyCheckerWrapper` in function `streamdiffusion.acceleration.tensorrt.compile_safety_checker`", + "severity": "error" + }, + { + "line": 358, + "column": 27, + "stop_line": 358, + "stop_column": 38, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Path` is not assignable to parameter `unet` with type `diffusers.models.unets.unet_2d_condition.UNet2DConditionModel | diffusers.utils.dummy_pt_objects.UNet2DConditionModel` in function `streamdiffusion.acceleration.tensorrt.compile_unet`", + "concise_description": "Argument `Path` is not assignable to parameter `unet` with type `diffusers.models.unets.unet_2d_condition.UNet2DConditionModel | diffusers.utils.dummy_pt_objects.UNet2DConditionModel` in function `streamdiffusion.acceleration.tensorrt.compile_unet`", + "severity": "error" + }, + { + "line": 358, + "column": 27, + "stop_line": 358, + "stop_column": 38, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Path` is not assignable to parameter `vae` with type `diffusers.models.autoencoders.autoencoder_kl.AutoencoderKL | diffusers.utils.dummy_pt_objects.AutoencoderKL` in function `streamdiffusion.acceleration.tensorrt.compile_vae_decoder`", + "concise_description": "Argument `Path` is not assignable to parameter `vae` with type `diffusers.models.autoencoders.autoencoder_kl.AutoencoderKL | diffusers.utils.dummy_pt_objects.AutoencoderKL` in function `streamdiffusion.acceleration.tensorrt.compile_vae_decoder`", + "severity": "error" + }, + { + "line": 358, + "column": 27, + "stop_line": 358, + "stop_column": 38, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Path` is not assignable to parameter `vae` with type `TorchVAEEncoder` in function `streamdiffusion.acceleration.tensorrt.compile_vae_encoder`", + "concise_description": "Argument `Path` is not assignable to parameter `vae` with type `TorchVAEEncoder` in function `streamdiffusion.acceleration.tensorrt.compile_vae_encoder`", + "severity": "error" + }, + { + "line": 358, + "column": 40, + "stop_line": 358, + "stop_column": 65, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `dict[Unknown, Unknown] | None` is not assignable to parameter `model_data` with type `BaseModel` in function `streamdiffusion.acceleration.tensorrt.compile_controlnet`", + "concise_description": "Argument `dict[Unknown, Unknown] | None` is not assignable to parameter `model_data` with type `BaseModel` in function `streamdiffusion.acceleration.tensorrt.compile_controlnet`", + "severity": "error" + }, + { + "line": 358, + "column": 40, + "stop_line": 358, + "stop_column": 65, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `dict[Unknown, Unknown] | None` is not assignable to parameter `model_data` with type `BaseModel` in function `streamdiffusion.acceleration.tensorrt.compile_safety_checker`", + "concise_description": "Argument `dict[Unknown, Unknown] | None` is not assignable to parameter `model_data` with type `BaseModel` in function `streamdiffusion.acceleration.tensorrt.compile_safety_checker`", + "severity": "error" + }, + { + "line": 358, + "column": 40, + "stop_line": 358, + "stop_column": 65, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `dict[Unknown, Unknown] | None` is not assignable to parameter `model_data` with type `BaseModel` in function `streamdiffusion.acceleration.tensorrt.compile_unet`", + "concise_description": "Argument `dict[Unknown, Unknown] | None` is not assignable to parameter `model_data` with type `BaseModel` in function `streamdiffusion.acceleration.tensorrt.compile_unet`", + "severity": "error" + }, + { + "line": 358, + "column": 40, + "stop_line": 358, + "stop_column": 65, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `dict[Unknown, Unknown] | None` is not assignable to parameter `model_data` with type `BaseModel` in function `streamdiffusion.acceleration.tensorrt.compile_vae_decoder`", + "concise_description": "Argument `dict[Unknown, Unknown] | None` is not assignable to parameter `model_data` with type `BaseModel` in function `streamdiffusion.acceleration.tensorrt.compile_vae_decoder`", + "severity": "error" + }, + { + "line": 358, + "column": 40, + "stop_line": 358, + "stop_column": 65, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\engine_manager.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `dict[Unknown, Unknown] | None` is not assignable to parameter `model_data` with type `BaseModel` in function `streamdiffusion.acceleration.tensorrt.compile_vae_encoder`", + "concise_description": "Argument `dict[Unknown, Unknown] | None` is not assignable to parameter `model_data` with type `BaseModel` in function `streamdiffusion.acceleration.tensorrt.compile_vae_encoder`", + "severity": "error" + }, + { + "line": 33, + "column": 21, + "stop_line": 33, + "stop_column": 33, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_ipadapter_export.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `UNet2DConditionModel` has no attribute `to`", + "concise_description": "Object of class `UNet2DConditionModel` has no attribute `to`", + "severity": "error" + }, + { + "line": 53, + "column": 26, + "stop_line": 53, + "stop_column": 51, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_ipadapter_export.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `UNet2DConditionModel` has no attribute `attn_processors`", + "concise_description": "Object of class `UNet2DConditionModel` has no attribute `attn_processors`", + "severity": "error" + }, + { + "line": 75, + "column": 26, + "stop_line": 75, + "stop_column": 51, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_ipadapter_export.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `UNet2DConditionModel` has no attribute `attn_processors`", + "concise_description": "Object of class `UNet2DConditionModel` has no attribute `attn_processors`", + "severity": "error" + }, + { + "line": 102, + "column": 36, + "stop_line": 102, + "stop_column": 52, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_ipadapter_export.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `UNet2DConditionModel` has no attribute `device`", + "concise_description": "Object of class `UNet2DConditionModel` has no attribute `device`", + "severity": "error" + }, + { + "line": 111, + "column": 13, + "stop_line": 111, + "stop_column": 41, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_ipadapter_export.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `UNet2DConditionModel` has no attribute `set_attn_processor`", + "concise_description": "Object of class `UNet2DConditionModel` has no attribute `set_attn_processor`", + "severity": "error" + }, + { + "line": 137, + "column": 36, + "stop_line": 137, + "stop_column": 61, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_ipadapter_export.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `UNet2DConditionModel` has no attribute `attn_processors`", + "concise_description": "Object of class `UNet2DConditionModel` has no attribute `attn_processors`", + "severity": "error" + }, + { + "line": 143, + "column": 67, + "stop_line": 143, + "stop_column": 83, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_ipadapter_export.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `UNet2DConditionModel` has no attribute `config`", + "concise_description": "Object of class `UNet2DConditionModel` has no attribute `config`", + "severity": "error" + }, + { + "line": 149, + "column": 35, + "stop_line": 149, + "stop_column": 51, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_ipadapter_export.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `UNet2DConditionModel` has no attribute `config`", + "concise_description": "Object of class `UNet2DConditionModel` has no attribute `config`", + "severity": "error" + }, + { + "line": 152, + "column": 49, + "stop_line": 152, + "stop_column": 65, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_ipadapter_export.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `UNet2DConditionModel` has no attribute `config`", + "concise_description": "Object of class `UNet2DConditionModel` has no attribute `config`", + "severity": "error" + }, + { + "line": 155, + "column": 35, + "stop_line": 155, + "stop_column": 51, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_ipadapter_export.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `UNet2DConditionModel` has no attribute `config`", + "concise_description": "Object of class `UNet2DConditionModel` has no attribute `config`", + "severity": "error" + }, + { + "line": 158, + "column": 35, + "stop_line": 158, + "stop_column": 51, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_ipadapter_export.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `UNet2DConditionModel` has no attribute `config`", + "concise_description": "Object of class `UNet2DConditionModel` has no attribute `config`", + "severity": "error" + }, + { + "line": 162, + "column": 40, + "stop_line": 162, + "stop_column": 55, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_ipadapter_export.py", + "code": -2, + "name": "unsupported-operation", + "description": "Cannot set item in `dict[Unknown, AttnProcessor]`\n Argument `AttnProcessor | AttnProcessor2_0` is not assignable to parameter `value` with type `AttnProcessor` in function `dict.__setitem__`", + "concise_description": "Cannot set item in `dict[Unknown, AttnProcessor]`", + "severity": "error" + }, + { + "line": 169, + "column": 26, + "stop_line": 169, + "stop_column": 42, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_ipadapter_export.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `UNet2DConditionModel` has no attribute `device`", + "concise_description": "Object of class `UNet2DConditionModel` has no attribute `device`", + "severity": "error" + }, + { + "line": 174, + "column": 40, + "stop_line": 174, + "stop_column": 44, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_ipadapter_export.py", + "code": -2, + "name": "unsupported-operation", + "description": "Cannot set item in `dict[Unknown, AttnProcessor]`\n Argument `TRTIPAttnProcessor | TRTIPAttnProcessor2_0` is not assignable to parameter `value` with type `AttnProcessor` in function `dict.__setitem__`", + "concise_description": "Cannot set item in `dict[Unknown, AttnProcessor]`", + "severity": "error" + }, + { + "line": 176, + "column": 13, + "stop_line": 176, + "stop_column": 41, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_ipadapter_export.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `UNet2DConditionModel` has no attribute `set_attn_processor`", + "concise_description": "Object of class `UNet2DConditionModel` has no attribute `set_attn_processor`", + "severity": "error" + }, + { + "line": 218, + "column": 44, + "stop_line": 218, + "stop_column": 61, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_ipadapter_export.py", + "code": -2, + "name": "bad-index", + "description": "Cannot index into `Tensor`\n Argument `Module | Tensor` is not assignable to parameter `indices` with type `EllipsisType | SupportsIndex | Tensor | _NestedSequence[EllipsisType | Tensor | bool | int | slice[Any, Any, Any] | None] | bool | int | slice[Any, Any, Any] | tuple[_Index, ...] | None` in function `torch._C.TensorBase.__getitem__`", + "concise_description": "Cannot index into `Tensor`", + "severity": "error" + }, + { + "line": 220, + "column": 96, + "stop_line": 220, + "stop_column": 100, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_ipadapter_export.py", + "code": -2, + "name": "bad-function-definition", + "description": "Default `None` is not assignable to parameter `ipadapter_scale` with type `Tensor`", + "concise_description": "Default `None` is not assignable to parameter `ipadapter_scale` with type `Tensor`", + "severity": "error" + }, + { + "line": 254, + "column": 16, + "stop_line": 254, + "stop_column": 25, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_ipadapter_export.py", + "code": -2, + "name": "not-callable", + "description": "Expected a callable, got `UNet2DConditionModel`", + "concise_description": "Expected a callable, got `UNet2DConditionModel`", + "severity": "error" + }, + { + "line": 279, + "column": 55, + "stop_line": 279, + "stop_column": 59, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_ipadapter_export.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `diffusers.models.unets.unet_2d_condition.UNet2DConditionModel | diffusers.utils.dummy_pt_objects.UNet2DConditionModel` is not assignable to parameter `unet` with type `diffusers.models.unets.unet_2d_condition.UNet2DConditionModel` in function `streamdiffusion.model_detection.detect_model_from_diffusers_unet`", + "concise_description": "Argument `diffusers.models.unets.unet_2d_condition.UNet2DConditionModel | diffusers.utils.dummy_pt_objects.UNet2DConditionModel` is not assignable to parameter `unet` with type `diffusers.models.unets.unet_2d_condition.UNet2DConditionModel` in function `streamdiffusion.model_detection.detect_model_from_diffusers_unet`", + "severity": "error" + }, + { + "line": 280, + "column": 31, + "stop_line": 280, + "stop_column": 42, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_ipadapter_export.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `UNet2DConditionModel` has no attribute `config`", + "concise_description": "Object of class `UNet2DConditionModel` has no attribute `config`", + "severity": "error" + }, + { + "line": 283, + "column": 31, + "stop_line": 283, + "stop_column": 51, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_ipadapter_export.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `UNet2DConditionModel` has no attribute `attn_processors`", + "concise_description": "Object of class `UNet2DConditionModel` has no attribute `attn_processors`", + "severity": "error" + }, + { + "line": 21, + "column": 5, + "stop_line": 21, + "stop_column": 76, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_sdxl_export.py", + "code": -2, + "name": "missing-import", + "description": "Cannot find module `diffusers.models.transformers.clip_text_model`\n Looked in these locations (from config in `D:\\dev\\SDTD_032_dev\\StreamDiffusion\\pyproject.toml`):\n Search path (from config file): [\"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\src\"]\n Import root (inferred from project layout): \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\src\"\n Site package path queried from interpreter: [\"C:\\\\Users\\\\Inter\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python311\\\\DLLs\", \"C:\\\\Users\\\\Inter\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python311\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\venv\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\venv\\\\Lib\\\\site-packages\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\src\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\venv\\\\Lib\\\\site-packages\\\\win32\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\venv\\\\Lib\\\\site-packages\\\\win32\\\\lib\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\venv\\\\Lib\\\\site-packages\\\\Pythonwin\"]", + "concise_description": "Cannot find module `diffusers.models.transformers.clip_text_model`", + "severity": "error" + }, + { + "line": 24, + "column": 9, + "stop_line": 24, + "stop_column": 67, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_sdxl_export.py", + "code": -2, + "name": "missing-import", + "description": "Cannot find module `diffusers.models.clip_text_model`\n Looked in these locations (from config in `D:\\dev\\SDTD_032_dev\\StreamDiffusion\\pyproject.toml`):\n Search path (from config file): [\"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\src\"]\n Import root (inferred from project layout): \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\src\"\n Site package path queried from interpreter: [\"C:\\\\Users\\\\Inter\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python311\\\\DLLs\", \"C:\\\\Users\\\\Inter\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python311\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\venv\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\venv\\\\Lib\\\\site-packages\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\src\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\venv\\\\Lib\\\\site-packages\\\\win32\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\venv\\\\Lib\\\\site-packages\\\\win32\\\\lib\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\venv\\\\Lib\\\\site-packages\\\\Pythonwin\"]", + "concise_description": "Cannot find module `diffusers.models.clip_text_model`", + "severity": "error" + }, + { + "line": 95, + "column": 29, + "stop_line": 95, + "stop_column": 50, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_sdxl_export.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `config`", + "concise_description": "Object of class `NoneType` has no attribute `config`", + "severity": "error" + }, + { + "line": 96, + "column": 21, + "stop_line": 96, + "stop_column": 42, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_sdxl_export.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `config`", + "concise_description": "Object of class `NoneType` has no attribute `config`", + "severity": "error" + }, + { + "line": 209, + "column": 25, + "stop_line": 209, + "stop_column": 32, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_sdxl_export.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `UNet2DConditionModel` has no attribute `to`", + "concise_description": "Object of class `UNet2DConditionModel` has no attribute `to`", + "severity": "error" + }, + { + "line": 268, + "column": 16, + "stop_line": 268, + "stop_column": 41, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_sdxl_export.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `SDXLConditioningHandler` has no attribute `conditioning_handler`", + "concise_description": "Object of class `SDXLConditioningHandler` has no attribute `conditioning_handler`", + "severity": "error" + }, + { + "line": 285, + "column": 29, + "stop_line": 285, + "stop_column": 49, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_sdxl_export.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `SDXLConditioningHandler` has no attribute `supported_calls`", + "concise_description": "Object of class `SDXLConditioningHandler` has no attribute `supported_calls`", + "severity": "error" + }, + { + "line": 300, + "column": 37, + "stop_line": 300, + "stop_column": 41, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_sdxl_export.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `diffusers.models.unets.unet_2d_condition.UNet2DConditionModel | diffusers.utils.dummy_pt_objects.UNet2DConditionModel` is not assignable to parameter `model` with type `Module` in function `streamdiffusion.model_detection.detect_model`", + "concise_description": "Argument `diffusers.models.unets.unet_2d_condition.UNet2DConditionModel | diffusers.utils.dummy_pt_objects.UNet2DConditionModel` is not assignable to parameter `model` with type `Module` in function `streamdiffusion.model_detection.detect_model`", + "severity": "error" + }, + { + "line": 29, + "column": 18, + "stop_line": 29, + "stop_column": 34, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_unified_export.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `UNet2DConditionModel` has no attribute `down_blocks`", + "concise_description": "Object of class `UNet2DConditionModel` has no attribute `down_blocks`", + "severity": "error" + }, + { + "line": 37, + "column": 16, + "stop_line": 37, + "stop_column": 30, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_unified_export.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `UNet2DConditionModel` has no attribute `mid_block`", + "concise_description": "Object of class `UNet2DConditionModel` has no attribute `mid_block`", + "severity": "error" + }, + { + "line": 37, + "column": 50, + "stop_line": 37, + "stop_column": 64, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_unified_export.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `UNet2DConditionModel` has no attribute `mid_block`", + "concise_description": "Object of class `UNet2DConditionModel` has no attribute `mid_block`", + "severity": "error" + }, + { + "line": 38, + "column": 27, + "stop_line": 38, + "stop_column": 41, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_unified_export.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `UNet2DConditionModel` has no attribute `mid_block`", + "concise_description": "Object of class `UNet2DConditionModel` has no attribute `mid_block`", + "severity": "error" + }, + { + "line": 44, + "column": 18, + "stop_line": 44, + "stop_column": 32, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_unified_export.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `UNet2DConditionModel` has no attribute `up_blocks`", + "concise_description": "Object of class `UNet2DConditionModel` has no attribute `up_blocks`", + "severity": "error" + }, + { + "line": 95, + "column": 45, + "stop_line": 95, + "stop_column": 52, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_unified_export.py", + "code": -2, + "name": "no-matching-overload", + "description": "No matching overload found for function `sum` called with arguments: (int)\n Possible overloads:\n (iterable: Iterable[Literal[-20, -19, -18, -17, -16, -15, -14, -13, -12, -11, -10, -9, -8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25] | bool], /, start: int = 0) -> int [closest match]\n (iterable: Iterable[_SupportsSumNoDefaultT], /) -> Literal[0] | _SupportsSumNoDefaultT\n (iterable: Iterable[_AddableT1], /, start: _AddableT2) -> _AddableT1 | _AddableT2", + "concise_description": "No matching overload found for function `sum` called with arguments: (int)", + "severity": "error" + }, + { + "line": 111, + "column": 17, + "stop_line": 111, + "stop_column": 26, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_unified_export.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `diffusers.models.unets.unet_2d_condition.UNet2DConditionModel | diffusers.utils.dummy_pt_objects.UNet2DConditionModel` is not assignable to parameter `unet` with type `diffusers.models.unets.unet_2d_condition.UNet2DConditionModel` in function `streamdiffusion.acceleration.tensorrt.export_wrappers.unet_controlnet_export.create_controlnet_wrapper`", + "concise_description": "Argument `diffusers.models.unets.unet_2d_condition.UNet2DConditionModel | diffusers.utils.dummy_pt_objects.UNet2DConditionModel` is not assignable to parameter `unet` with type `diffusers.models.unets.unet_2d_condition.UNet2DConditionModel` in function `streamdiffusion.acceleration.tensorrt.export_wrappers.unet_controlnet_export.create_controlnet_wrapper`", + "severity": "error" + }, + { + "line": 211, + "column": 15, + "stop_line": 211, + "stop_column": 24, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_unified_export.py", + "code": -2, + "name": "not-callable", + "description": "Expected a callable, got `UNet2DConditionModel`", + "concise_description": "Expected a callable, got `UNet2DConditionModel`", + "severity": "error" + }, + { + "line": 279, + "column": 24, + "stop_line": 279, + "stop_column": 56, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_unified_export.py", + "code": -2, + "name": "bad-return", + "description": "Returned type `tuple[Unknown, Unknown, *tuple[Unknown, ...]]` is not assignable to declared return type `Tensor`", + "concise_description": "Returned type `tuple[Unknown, Unknown, *tuple[Unknown, ...]]` is not assignable to declared return type `Tensor`", + "severity": "error" + }, + { + "line": 284, + "column": 20, + "stop_line": 284, + "stop_column": 102, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\export_wrappers\\unet_unified_export.py", + "code": -2, + "name": "bad-return", + "description": "Returned type `tuple[Unknown, Unknown, *tuple[Unknown, ...]] | Unknown` is not assignable to declared return type `Tensor`", + "concise_description": "Returned type `tuple[Unknown, Unknown, *tuple[Unknown, ...]] | Unknown` is not assignable to declared return type `Tensor`", + "severity": "error" + }, + { + "line": 123, + "column": 38, + "stop_line": 123, + "stop_column": 81, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\fp8_quantize.py", + "code": -2, + "name": "bad-context-manager", + "description": "Cannot use `autocast` as a context manager\n Return type `Literal[False] | object | None` of function `autocast.__exit__` is not assignable to expected return type `bool | None`", + "concise_description": "Cannot use `autocast` as a context manager", + "severity": "error" + }, + { + "line": 310, + "column": 38, + "stop_line": 310, + "stop_column": 81, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\fp8_quantize.py", + "code": -2, + "name": "bad-context-manager", + "description": "Cannot use `autocast` as a context manager\n Return type `Literal[False] | object | None` of function `autocast.__exit__` is not assignable to expected return type `bool | None`", + "concise_description": "Cannot use `autocast` as a context manager", + "severity": "error" + }, + { + "line": 117, + "column": 30, + "stop_line": 117, + "stop_column": 102, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\attention_processors.py", + "code": -2, + "name": "bad-assignment", + "description": "`Tensor` is not assignable to variable `attention_mask` with type `FloatTensor | None`", + "concise_description": "`Tensor` is not assignable to variable `attention_mask` with type `FloatTensor | None`", + "severity": "error" + }, + { + "line": 118, + "column": 30, + "stop_line": 118, + "stop_column": 49, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\attention_processors.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `view`", + "concise_description": "Object of class `NoneType` has no attribute `view`", + "severity": "error" + }, + { + "line": 118, + "column": 78, + "stop_line": 118, + "stop_column": 98, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\attention_processors.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `shape`", + "concise_description": "Object of class `NoneType` has no attribute `shape`", + "severity": "error" + }, + { + "line": 131, + "column": 37, + "stop_line": 131, + "stop_column": 91, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\attention_processors.py", + "code": -2, + "name": "bad-assignment", + "description": "`Tensor` is not assignable to variable `encoder_hidden_states` with type `FloatTensor | None`", + "concise_description": "`Tensor` is not assignable to variable `encoder_hidden_states` with type `FloatTensor | None`", + "severity": "error" + }, + { + "line": 152, + "column": 41, + "stop_line": 152, + "stop_column": 63, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\attention_processors.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `transpose`", + "concise_description": "Object of class `NoneType` has no attribute `transpose`", + "severity": "error" + }, + { + "line": 163, + "column": 25, + "stop_line": 165, + "stop_column": 10, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\attention_processors.py", + "code": -2, + "name": "bad-assignment", + "description": "`Tensor` is not assignable to variable `hidden_states` with type `FloatTensor`", + "concise_description": "`Tensor` is not assignable to variable `hidden_states` with type `FloatTensor`", + "severity": "error" + }, + { + "line": 167, + "column": 25, + "stop_line": 167, + "stop_column": 101, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\attention_processors.py", + "code": -2, + "name": "bad-assignment", + "description": "`Tensor` is not assignable to variable `hidden_states` with type `FloatTensor`", + "concise_description": "`Tensor` is not assignable to variable `hidden_states` with type `FloatTensor`", + "severity": "error" + }, + { + "line": 168, + "column": 25, + "stop_line": 168, + "stop_column": 54, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\attention_processors.py", + "code": -2, + "name": "bad-assignment", + "description": "`Tensor` is not assignable to variable `hidden_states` with type `FloatTensor`", + "concise_description": "`Tensor` is not assignable to variable `hidden_states` with type `FloatTensor`", + "severity": "error" + }, + { + "line": 176, + "column": 81, + "stop_line": 176, + "stop_column": 88, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\attention_processors.py", + "code": -2, + "name": "unbound-name", + "description": "`channel` may be uninitialized", + "concise_description": "`channel` may be uninitialized", + "severity": "error" + }, + { + "line": 176, + "column": 90, + "stop_line": 176, + "stop_column": 96, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\attention_processors.py", + "code": -2, + "name": "unbound-name", + "description": "`height` may be uninitialized", + "concise_description": "`height` may be uninitialized", + "severity": "error" + }, + { + "line": 176, + "column": 98, + "stop_line": 176, + "stop_column": 103, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\attention_processors.py", + "code": -2, + "name": "unbound-name", + "description": "`width` may be uninitialized", + "concise_description": "`width` may be uninitialized", + "severity": "error" + }, + { + "line": 186, + "column": 42, + "stop_line": 186, + "stop_column": 50, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\attention_processors.py", + "code": -2, + "name": "unbound-name", + "description": "`curr_key` may be uninitialized", + "concise_description": "`curr_key` may be uninitialized", + "severity": "error" + }, + { + "line": 186, + "column": 65, + "stop_line": 186, + "stop_column": 75, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\attention_processors.py", + "code": -2, + "name": "unbound-name", + "description": "`curr_value` may be uninitialized", + "concise_description": "`curr_value` may be uninitialized", + "severity": "error" + }, + { + "line": 215, + "column": 61, + "stop_line": 215, + "stop_column": 70, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\attention_processors.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Tensor` is not assignable to parameter `threshold` with type `float` in function `get_nn_feats`", + "concise_description": "Argument `Tensor` is not assignable to parameter `threshold` with type `float` in function `get_nn_feats`", + "severity": "error" + }, + { + "line": 38, + "column": 9, + "stop_line": 38, + "stop_column": 24, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\controlnet_models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `ControlNetTRT.get_input_names` overrides parent class `BaseModel` in an inconsistent manner\n `ControlNetTRT.get_input_names` has type `(self: ControlNetTRT) -> list[str]`, which is not assignable to `(self: ControlNetTRT) -> None`, the type of `BaseModel.get_input_names`\n Signature mismatch:\n expected: def get_input_names(self: ControlNetTRT) -> None: ...\n ^^^^ return type\n found: def get_input_names(self: ControlNetTRT) -> list[str]: ...\n ^^^^^^^^^ return type", + "concise_description": "Class member `ControlNetTRT.get_input_names` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 42, + "column": 9, + "stop_line": 42, + "stop_column": 25, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\controlnet_models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `ControlNetTRT.get_output_names` overrides parent class `BaseModel` in an inconsistent manner\n `ControlNetTRT.get_output_names` has type `(self: ControlNetTRT) -> list[str]`, which is not assignable to `(self: ControlNetTRT) -> None`, the type of `BaseModel.get_output_names`\n Signature mismatch:\n expected: def get_output_names(self: ControlNetTRT) -> None: ...\n ^^^^ return type\n found: def get_output_names(self: ControlNetTRT) -> list[str]: ...\n ^^^^^^^^^ return type", + "concise_description": "Class member `ControlNetTRT.get_output_names` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 47, + "column": 9, + "stop_line": 47, + "stop_column": 25, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\controlnet_models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `ControlNetTRT.get_dynamic_axes` overrides parent class `BaseModel` in an inconsistent manner\n `ControlNetTRT.get_dynamic_axes` has type `(self: ControlNetTRT) -> dict[str, dict[int, str]]`, which is not assignable to `(self: ControlNetTRT) -> None`, the type of `BaseModel.get_dynamic_axes`\n Signature mismatch:\n expected: def get_dynamic_axes(self: ControlNetTRT) -> None: ...\n ^^^^ return type\n found: def get_dynamic_axes(self: ControlNetTRT) -> dict[str, dict[int, str]]: ...\n ^^^^^^^^^^^^^^^^^^^^^^^^^ return type", + "concise_description": "Class member `ControlNetTRT.get_dynamic_axes` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 58, + "column": 9, + "stop_line": 58, + "stop_column": 26, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\controlnet_models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `ControlNetTRT.get_input_profile` overrides parent class `BaseModel` in an inconsistent manner\n `ControlNetTRT.get_input_profile` has type `(self: ControlNetTRT, batch_size: Unknown, image_height: Unknown, image_width: Unknown, static_batch: Unknown, static_shape: Unknown) -> dict[str, list[tuple[int | Unknown] | tuple[Unknown]] | list[tuple[int | Unknown, int, int | Unknown] | tuple[Unknown, int, int | Unknown]] | list[tuple[int | Unknown, int, int | Unknown, int | Unknown] | tuple[Unknown, int, int | Unknown, int | Unknown]]]`, which is not assignable to `(self: ControlNetTRT, batch_size: Unknown, image_height: Unknown, image_width: Unknown, static_batch: Unknown, static_shape: Unknown) -> None`, the type of `BaseModel.get_input_profile`\n Signature mismatch:\n ...-> None: ...\n ^^^^ return type\n ...-> dict[str, list[tuple[int | Unknown] | tuple[Unknown]] | list[tuple[int | Unknown, int, int | Unknown] | tuple[Unknown, int, int | Unknown]] | list[tuple[int | Unknown, int, int | Unknown, int | Unknown] | tuple[Unknown, int, int | Unknown, int | Unknown]]]: ...\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ return type", + "concise_description": "Class member `ControlNetTRT.get_input_profile` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 105, + "column": 9, + "stop_line": 105, + "stop_column": 25, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\controlnet_models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `ControlNetTRT.get_sample_input` overrides parent class `BaseModel` in an inconsistent manner\n `ControlNetTRT.get_sample_input` has type `(self: ControlNetTRT, batch_size: Unknown, image_height: Unknown, image_width: Unknown) -> tuple[Tensor, Tensor, Tensor, Tensor]`, which is not assignable to `(self: ControlNetTRT, batch_size: Unknown, image_height: Unknown, image_width: Unknown) -> None`, the type of `BaseModel.get_sample_input`\n Signature mismatch:\n ...ight: Unknown, image_width: Unknown) -> None: ...\n ^^^^ return type\n ...ight: Unknown, image_width: Unknown) -> tuple[Tensor, Tensor, Tensor, Tensor]: ...\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ return type", + "concise_description": "Class member `ControlNetTRT.get_sample_input` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 181, + "column": 9, + "stop_line": 181, + "stop_column": 42, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\controlnet_models.py", + "code": -2, + "name": "unsupported-operation", + "description": "Cannot set item in `None`\n Object of class `NoneType` has no attribute `__setitem__`", + "concise_description": "Cannot set item in `None`", + "severity": "error" + }, + { + "line": 207, + "column": 9, + "stop_line": 207, + "stop_column": 27, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\controlnet_models.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `update`", + "concise_description": "Object of class `NoneType` has no attribute `update`", + "severity": "error" + }, + { + "line": 210, + "column": 9, + "stop_line": 210, + "stop_column": 25, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\controlnet_models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `ControlNetSDXLTRT.get_sample_input` overrides parent class `ControlNetTRT` in an inconsistent manner\n `ControlNetSDXLTRT.get_sample_input` has type `(self: ControlNetSDXLTRT, batch_size: Unknown, image_height: Unknown, image_width: Unknown) -> tuple[Tensor, Tensor, Tensor, Tensor, Tensor, Tensor, Tensor]`, which is not assignable to `(self: ControlNetSDXLTRT, batch_size: Unknown, image_height: Unknown, image_width: Unknown) -> tuple[Tensor, Tensor, Tensor, Tensor]`, the type of `ControlNetTRT.get_sample_input`\n Signature mismatch:\n ...nknown) -> tuple[Tensor, Tensor, Tensor, Tensor]: ...\n ^ return type\n ...nknown) -> tuple[Tensor, Tensor, Tensor, Tensor, Tensor, Tensor, Tensor]: ...\n ^^^^^^^^^^^^^^^^^^^^^^^^ return type", + "concise_description": "Class member `ControlNetSDXLTRT.get_sample_input` overrides parent class `ControlNetTRT` in an inconsistent manner", + "severity": "error" + }, + { + "line": 291, + "column": 34, + "stop_line": 295, + "stop_column": 10, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\controlnet_models.py", + "code": -2, + "name": "bad-typed-dict-key", + "description": "`list[tuple[int | Unknown, int] | tuple[Unknown, int]]` is not assignable to TypedDict key `text_embeds` with type `list[tuple[int | Unknown] | tuple[Unknown]] | list[tuple[int | Unknown, int, int | Unknown] | tuple[Unknown, int, int | Unknown]] | list[tuple[int | Unknown, int, int | Unknown, int | Unknown] | tuple[Unknown, int, int | Unknown, int | Unknown]]`", + "concise_description": "`list[tuple[int | Unknown, int] | tuple[Unknown, int]]` is not assignable to TypedDict key `text_embeds` with type `list[tuple[int | Unknown] | tuple[Unknown]] | list[tuple[int | Unknown, int, int | Unknown] | tuple[Unknown, int, int | Unknown]] | list[tuple[int | Unknown, int, int | Unknown, int | Unknown] | tuple[Unknown, int, int | Unknown, int | Unknown]]`", + "severity": "error" + }, + { + "line": 298, + "column": 31, + "stop_line": 302, + "stop_column": 10, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\controlnet_models.py", + "code": -2, + "name": "bad-typed-dict-key", + "description": "`list[tuple[int | Unknown, int] | tuple[Unknown, int]]` is not assignable to TypedDict key `time_ids` with type `list[tuple[int | Unknown] | tuple[Unknown]] | list[tuple[int | Unknown, int, int | Unknown] | tuple[Unknown, int, int | Unknown]] | list[tuple[int | Unknown, int, int | Unknown, int | Unknown] | tuple[Unknown, int, int | Unknown, int | Unknown]]`", + "concise_description": "`list[tuple[int | Unknown, int] | tuple[Unknown, int]]` is not assignable to TypedDict key `time_ids` with type `list[tuple[int | Unknown] | tuple[Unknown]] | list[tuple[int | Unknown, int, int | Unknown] | tuple[Unknown, int, int | Unknown]] | list[tuple[int | Unknown, int, int | Unknown, int | Unknown] | tuple[Unknown, int, int | Unknown, int | Unknown]]`", + "severity": "error" + }, + { + "line": 29, + "column": 44, + "stop_line": 29, + "stop_column": 58, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "missing-module-attribute", + "description": "Could not import `fold_constants` from `polygraphy.backend.onnx.loader`", + "concise_description": "Could not import `fold_constants` from `polygraphy.backend.onnx.loader`", + "severity": "error" + }, + { + "line": 282, + "column": 9, + "stop_line": 282, + "stop_column": 24, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `CLIP.get_input_names` overrides parent class `BaseModel` in an inconsistent manner\n `CLIP.get_input_names` has type `(self: CLIP) -> list[str]`, which is not assignable to `(self: CLIP) -> None`, the type of `BaseModel.get_input_names`\n Signature mismatch:\n expected: def get_input_names(self: CLIP) -> None: ...\n ^^^^ return type\n found: def get_input_names(self: CLIP) -> list[str]: ...\n ^^^^^^^^^ return type", + "concise_description": "Class member `CLIP.get_input_names` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 285, + "column": 9, + "stop_line": 285, + "stop_column": 25, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `CLIP.get_output_names` overrides parent class `BaseModel` in an inconsistent manner\n `CLIP.get_output_names` has type `(self: CLIP) -> list[str]`, which is not assignable to `(self: CLIP) -> None`, the type of `BaseModel.get_output_names`\n Signature mismatch:\n expected: def get_output_names(self: CLIP) -> None: ...\n ^^^^ return type\n found: def get_output_names(self: CLIP) -> list[str]: ...\n ^^^^^^^^^ return type", + "concise_description": "Class member `CLIP.get_output_names` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 288, + "column": 9, + "stop_line": 288, + "stop_column": 25, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `CLIP.get_dynamic_axes` overrides parent class `BaseModel` in an inconsistent manner\n `CLIP.get_dynamic_axes` has type `(self: CLIP) -> dict[str, dict[int, str]]`, which is not assignable to `(self: CLIP) -> None`, the type of `BaseModel.get_dynamic_axes`\n Signature mismatch:\n expected: def get_dynamic_axes(self: CLIP) -> None: ...\n ^^^^ return type\n found: def get_dynamic_axes(self: CLIP) -> dict[str, dict[int, str]]: ...\n ^^^^^^^^^^^^^^^^^^^^^^^^^ return type", + "concise_description": "Class member `CLIP.get_dynamic_axes` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 291, + "column": 9, + "stop_line": 291, + "stop_column": 26, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `CLIP.get_input_profile` overrides parent class `BaseModel` in an inconsistent manner\n `CLIP.get_input_profile` has type `(self: CLIP, batch_size: Unknown, image_height: Unknown, image_width: Unknown, static_batch: Unknown, static_shape: Unknown) -> dict[str, list[tuple[int | Unknown, int | Unknown] | tuple[Unknown, int | Unknown]]]`, which is not assignable to `(self: CLIP, batch_size: Unknown, image_height: Unknown, image_width: Unknown, static_batch: Unknown, static_shape: Unknown) -> None`, the type of `BaseModel.get_input_profile`\n Signature mismatch:\n ...ape: Unknown) -> None: ...\n ^^^^ return type\n ...ape: Unknown) -> dict[str, list[tuple[int | Unknown, int | Unknown] | tuple[Unknown, int | Unknown]]]: ...\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ return type", + "concise_description": "Class member `CLIP.get_input_profile` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 304, + "column": 9, + "stop_line": 304, + "stop_column": 23, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `CLIP.get_shape_dict` overrides parent class `BaseModel` in an inconsistent manner\n `CLIP.get_shape_dict` has type `(self: CLIP, batch_size: Unknown, image_height: Unknown, image_width: Unknown) -> dict[str, tuple[Unknown, int | Unknown] | tuple[Unknown, int | Unknown, int | Unknown]]`, which is not assignable to `(self: CLIP, batch_size: Unknown, image_height: Unknown, image_width: Unknown) -> None`, the type of `BaseModel.get_shape_dict`\n Signature mismatch:\n ...h: Unknown) -> None: ...\n ^^^^ return type\n ...h: Unknown) -> dict[str, tuple[Unknown, int | Unknown] | tuple[Unknown, int | Unknown, int | Unknown]]: ...\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ return type", + "concise_description": "Class member `CLIP.get_shape_dict` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 311, + "column": 9, + "stop_line": 311, + "stop_column": 25, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `CLIP.get_sample_input` overrides parent class `BaseModel` in an inconsistent manner\n `CLIP.get_sample_input` has type `(self: CLIP, batch_size: Unknown, image_height: Unknown, image_width: Unknown) -> Tensor`, which is not assignable to `(self: CLIP, batch_size: Unknown, image_height: Unknown, image_width: Unknown) -> None`, the type of `BaseModel.get_sample_input`\n Signature mismatch:\n expected: def get_sample_input(self: CLIP, batch_size: Unknown, image_height: Unknown, image_width: Unknown) -> None: ...\n ^^^^ return type\n found: def get_sample_input(self: CLIP, batch_size: Unknown, image_height: Unknown, image_width: Unknown) -> Tensor: ...\n ^^^^^^ return type", + "concise_description": "Class member `CLIP.get_sample_input` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 341, + "column": 9, + "stop_line": 341, + "stop_column": 24, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `SafetyChecker.get_input_names` overrides parent class `BaseModel` in an inconsistent manner\n `SafetyChecker.get_input_names` has type `(self: SafetyChecker) -> list[str]`, which is not assignable to `(self: SafetyChecker) -> None`, the type of `BaseModel.get_input_names`\n Signature mismatch:\n expected: def get_input_names(self: SafetyChecker) -> None: ...\n ^^^^ return type\n found: def get_input_names(self: SafetyChecker) -> list[str]: ...\n ^^^^^^^^^ return type", + "concise_description": "Class member `SafetyChecker.get_input_names` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 344, + "column": 9, + "stop_line": 344, + "stop_column": 25, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `SafetyChecker.get_output_names` overrides parent class `BaseModel` in an inconsistent manner\n `SafetyChecker.get_output_names` has type `(self: SafetyChecker) -> list[str]`, which is not assignable to `(self: SafetyChecker) -> None`, the type of `BaseModel.get_output_names`\n Signature mismatch:\n expected: def get_output_names(self: SafetyChecker) -> None: ...\n ^^^^ return type\n found: def get_output_names(self: SafetyChecker) -> list[str]: ...\n ^^^^^^^^^ return type", + "concise_description": "Class member `SafetyChecker.get_output_names` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 347, + "column": 9, + "stop_line": 347, + "stop_column": 25, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `SafetyChecker.get_dynamic_axes` overrides parent class `BaseModel` in an inconsistent manner\n `SafetyChecker.get_dynamic_axes` has type `(self: SafetyChecker) -> dict[str, dict[int, str]]`, which is not assignable to `(self: SafetyChecker) -> None`, the type of `BaseModel.get_dynamic_axes`\n Signature mismatch:\n expected: def get_dynamic_axes(self: SafetyChecker) -> None: ...\n ^^^^ return type\n found: def get_dynamic_axes(self: SafetyChecker) -> dict[str, dict[int, str]]: ...\n ^^^^^^^^^^^^^^^^^^^^^^^^^ return type", + "concise_description": "Class member `SafetyChecker.get_dynamic_axes` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 350, + "column": 9, + "stop_line": 350, + "stop_column": 26, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `SafetyChecker.get_input_profile` overrides parent class `BaseModel` in an inconsistent manner\n `SafetyChecker.get_input_profile` has type `(self: SafetyChecker, batch_size: Unknown, *args: Unknown, **kwargs: Unknown) -> dict[str, list[tuple[int | Unknown, int, int, int] | tuple[Unknown, int, int, int]]]`, which is not assignable to `(self: SafetyChecker, batch_size: Unknown, image_height: Unknown, image_width: Unknown, static_batch: Unknown, static_shape: Unknown) -> None`, the type of `BaseModel.get_input_profile`\n Signature mismatch:\n ...n, image_height: Unknown, image_width: Unknown, static_batch: Unknown, static_shape: Unknown) -> None: ...\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ parameters ^^^^ return type\n ...n, *args: Unknown, **kwargs: Unknown) -> dict[str, list[tuple[int | Unknown, int, int, int] | tuple[Unknown, int, int, int]]]: ...\n ^^^^^^^^^^^^^^^^^^^^^^^^ parameters ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ return type", + "concise_description": "Class member `SafetyChecker.get_input_profile` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 359, + "column": 9, + "stop_line": 359, + "stop_column": 23, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `SafetyChecker.get_shape_dict` overrides parent class `BaseModel` in an inconsistent manner\n `SafetyChecker.get_shape_dict` has type `(self: SafetyChecker, batch_size: Unknown, *args: Unknown, **kwargs: Unknown) -> dict[str, tuple[Unknown] | tuple[Unknown, int, int, int]]`, which is not assignable to `(self: SafetyChecker, batch_size: Unknown, image_height: Unknown, image_width: Unknown) -> None`, the type of `BaseModel.get_shape_dict`\n Signature mismatch:\n ...: Unknown, image_height: Unknown, image_width: Unknown) -> None: ...\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ parameters ^^^^ return type\n ...: Unknown, *args: Unknown, **kwargs: Unknown) -> dict[str, tuple[Unknown] | tuple[Unknown, int, int, int]]: ...\n ^^^^^^^^^^^^^^^^^^^^^^^^ parameters ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ return type", + "concise_description": "Class member `SafetyChecker.get_shape_dict` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 365, + "column": 9, + "stop_line": 365, + "stop_column": 25, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `SafetyChecker.get_sample_input` overrides parent class `BaseModel` in an inconsistent manner\n `SafetyChecker.get_sample_input` has type `(self: SafetyChecker, batch_size: Unknown, *args: Unknown, **kwargs: Unknown) -> tuple[Tensor]`, which is not assignable to `(self: SafetyChecker, batch_size: Unknown, image_height: Unknown, image_width: Unknown) -> None`, the type of `BaseModel.get_sample_input`\n Signature mismatch:\n expected: def get_sample_input(self: SafetyChecker, batch_size: Unknown, image_height: Unknown, image_width: Unknown) -> None: ...\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ parameters ^^^^ return type\n found: def get_sample_input(self: SafetyChecker, batch_size: Unknown, *args: Unknown, **kwargs: Unknown) -> tuple[Tensor]: ...\n ^^^^^^^^^^^^^^^^^^^^^^^^ parameters ^^^^^^^^^^^^^ return type", + "concise_description": "Class member `SafetyChecker.get_sample_input` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 378, + "column": 9, + "stop_line": 378, + "stop_column": 24, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `NSFWDetector.get_input_names` overrides parent class `BaseModel` in an inconsistent manner\n `NSFWDetector.get_input_names` has type `(self: NSFWDetector) -> list[str]`, which is not assignable to `(self: NSFWDetector) -> None`, the type of `BaseModel.get_input_names`\n Signature mismatch:\n expected: def get_input_names(self: NSFWDetector) -> None: ...\n ^^^^ return type\n found: def get_input_names(self: NSFWDetector) -> list[str]: ...\n ^^^^^^^^^ return type", + "concise_description": "Class member `NSFWDetector.get_input_names` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 381, + "column": 9, + "stop_line": 381, + "stop_column": 25, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `NSFWDetector.get_output_names` overrides parent class `BaseModel` in an inconsistent manner\n `NSFWDetector.get_output_names` has type `(self: NSFWDetector) -> list[str]`, which is not assignable to `(self: NSFWDetector) -> None`, the type of `BaseModel.get_output_names`\n Signature mismatch:\n expected: def get_output_names(self: NSFWDetector) -> None: ...\n ^^^^ return type\n found: def get_output_names(self: NSFWDetector) -> list[str]: ...\n ^^^^^^^^^ return type", + "concise_description": "Class member `NSFWDetector.get_output_names` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 384, + "column": 9, + "stop_line": 384, + "stop_column": 25, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `NSFWDetector.get_dynamic_axes` overrides parent class `BaseModel` in an inconsistent manner\n `NSFWDetector.get_dynamic_axes` has type `(self: NSFWDetector) -> dict[str, dict[int, str]]`, which is not assignable to `(self: NSFWDetector) -> None`, the type of `BaseModel.get_dynamic_axes`\n Signature mismatch:\n expected: def get_dynamic_axes(self: NSFWDetector) -> None: ...\n ^^^^ return type\n found: def get_dynamic_axes(self: NSFWDetector) -> dict[str, dict[int, str]]: ...\n ^^^^^^^^^^^^^^^^^^^^^^^^^ return type", + "concise_description": "Class member `NSFWDetector.get_dynamic_axes` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 387, + "column": 9, + "stop_line": 387, + "stop_column": 26, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `NSFWDetector.get_input_profile` overrides parent class `BaseModel` in an inconsistent manner\n `NSFWDetector.get_input_profile` has type `(self: NSFWDetector, batch_size: Unknown, *args: Unknown, **kwargs: Unknown) -> dict[str, list[tuple[int | Unknown, int, int, int] | tuple[Unknown, int, int, int]]]`, which is not assignable to `(self: NSFWDetector, batch_size: Unknown, image_height: Unknown, image_width: Unknown, static_batch: Unknown, static_shape: Unknown) -> None`, the type of `BaseModel.get_input_profile`\n Signature mismatch:\n ...n, image_height: Unknown, image_width: Unknown, static_batch: Unknown, static_shape: Unknown) -> None: ...\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ parameters ^^^^ return type\n ...n, *args: Unknown, **kwargs: Unknown) -> dict[str, list[tuple[int | Unknown, int, int, int] | tuple[Unknown, int, int, int]]]: ...\n ^^^^^^^^^^^^^^^^^^^^^^^^ parameters ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ return type", + "concise_description": "Class member `NSFWDetector.get_input_profile` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 396, + "column": 9, + "stop_line": 396, + "stop_column": 23, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `NSFWDetector.get_shape_dict` overrides parent class `BaseModel` in an inconsistent manner\n `NSFWDetector.get_shape_dict` has type `(self: NSFWDetector, batch_size: Unknown, *args: Unknown, **kwargs: Unknown) -> dict[str, tuple[Unknown, int] | tuple[Unknown, int, int, int]]`, which is not assignable to `(self: NSFWDetector, batch_size: Unknown, image_height: Unknown, image_width: Unknown) -> None`, the type of `BaseModel.get_shape_dict`\n Signature mismatch:\n ...Unknown, image_height: Unknown, image_width: Unknown) -> None: ...\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ parameters ^^^^ return type\n ...Unknown, *args: Unknown, **kwargs: Unknown) -> dict[str, tuple[Unknown, int] | tuple[Unknown, int, int, int]]: ...\n ^^^^^^^^^^^^^^^^^^^^^^^^ parameters ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ return type", + "concise_description": "Class member `NSFWDetector.get_shape_dict` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 402, + "column": 9, + "stop_line": 402, + "stop_column": 25, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `NSFWDetector.get_sample_input` overrides parent class `BaseModel` in an inconsistent manner\n `NSFWDetector.get_sample_input` has type `(self: NSFWDetector, batch_size: Unknown, *args: Unknown, **kwargs: Unknown) -> tuple[Tensor]`, which is not assignable to `(self: NSFWDetector, batch_size: Unknown, image_height: Unknown, image_width: Unknown) -> None`, the type of `BaseModel.get_sample_input`\n Signature mismatch:\n expected: def get_sample_input(self: NSFWDetector, batch_size: Unknown, image_height: Unknown, image_width: Unknown) -> None: ...\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ parameters ^^^^ return type\n found: def get_sample_input(self: NSFWDetector, batch_size: Unknown, *args: Unknown, **kwargs: Unknown) -> tuple[Tensor]: ...\n ^^^^^^^^^^^^^^^^^^^^^^^^ parameters ^^^^^^^^^^^^^ return type", + "concise_description": "Class member `NSFWDetector.get_sample_input` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 409, + "column": 38, + "stop_line": 409, + "stop_column": 42, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-function-definition", + "description": "Default `None` is not assignable to parameter `unet` with type `UNet2DConditionModel`", + "concise_description": "Default `None` is not assignable to parameter `unet` with type `UNet2DConditionModel`", + "severity": "error" + }, + { + "line": 423, + "column": 30, + "stop_line": 423, + "stop_column": 34, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-function-definition", + "description": "Default `None` is not assignable to parameter `num_ip_layers` with type `int`", + "concise_description": "Default `None` is not assignable to parameter `num_ip_layers` with type `int`", + "severity": "error" + }, + { + "line": 622, + "column": 9, + "stop_line": 622, + "stop_column": 24, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `UNet.get_input_names` overrides parent class `BaseModel` in an inconsistent manner\n `UNet.get_input_names` has type `(self: UNet) -> list[str]`, which is not assignable to `(self: UNet) -> None`, the type of `BaseModel.get_input_names`\n Signature mismatch:\n expected: def get_input_names(self: UNet) -> None: ...\n ^^^^ return type\n found: def get_input_names(self: UNet) -> list[str]: ...\n ^^^^^^^^^ return type", + "concise_description": "Class member `UNet.get_input_names` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 644, + "column": 9, + "stop_line": 644, + "stop_column": 25, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `UNet.get_output_names` overrides parent class `BaseModel` in an inconsistent manner\n `UNet.get_output_names` has type `(self: UNet) -> list[str]`, which is not assignable to `(self: UNet) -> None`, the type of `BaseModel.get_output_names`\n Signature mismatch:\n expected: def get_output_names(self: UNet) -> None: ...\n ^^^^ return type\n found: def get_output_names(self: UNet) -> list[str]: ...\n ^^^^^^^^^ return type", + "concise_description": "Class member `UNet.get_output_names` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 665, + "column": 9, + "stop_line": 665, + "stop_column": 25, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `UNet.get_dynamic_axes` overrides parent class `BaseModel` in an inconsistent manner\n `UNet.get_dynamic_axes` has type `(self: UNet) -> dict[str, dict[int, str]]`, which is not assignable to `(self: UNet) -> None`, the type of `BaseModel.get_dynamic_axes`\n Signature mismatch:\n expected: def get_dynamic_axes(self: UNet) -> None: ...\n ^^^^ return type\n found: def get_dynamic_axes(self: UNet) -> dict[str, dict[int, str]]: ...\n ^^^^^^^^^^^^^^^^^^^^^^^^^ return type", + "concise_description": "Class member `UNet.get_dynamic_axes` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 707, + "column": 9, + "stop_line": 707, + "stop_column": 26, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `UNet.get_input_profile` overrides parent class `BaseModel` in an inconsistent manner\n `UNet.get_input_profile` has type `(self: UNet, batch_size: Unknown, image_height: Unknown, image_width: Unknown, static_batch: Unknown, static_shape: Unknown) -> dict[str, list[tuple[int | Unknown] | tuple[Unknown]] | list[tuple[int | Unknown, int | Unknown, int | Unknown] | tuple[Unknown, int | Unknown, int | Unknown]] | list[tuple[int | Unknown, int | Unknown, int | Unknown, int | Unknown] | tuple[Unknown, int | Unknown, int | Unknown, int | Unknown]]]`, which is not assignable to `(self: UNet, batch_size: Unknown, image_height: Unknown, image_width: Unknown, static_batch: Unknown, static_shape: Unknown) -> None`, the type of `BaseModel.get_input_profile`\n Signature mismatch:\n ...-> None: ...\n ^^^^ return type\n ...-> dict[str, list[tuple[int | Unknown] | tuple[Unknown]] | list[tuple[int | Unknown, int | Unknown, int | Unknown] | tuple[Unknown, int | Unknown, int | Unknown]] | list[tuple[int | Unknown, int | Unknown, int | Unknown, int | Unknown] | tuple[Unknown, int | Unknown, int | Unknown, int | Unknown]]]: ...\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ return type", + "concise_description": "Class member `UNet.get_input_profile` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 814, + "column": 33, + "stop_line": 814, + "stop_column": 41, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "unsupported-operation", + "description": "Cannot set item in `dict[str, list[tuple[int | Unknown] | tuple[Unknown]] | list[tuple[int | Unknown, int | Unknown, int | Unknown] | tuple[Unknown, int | Unknown, int | Unknown]] | list[tuple[int | Unknown, int | Unknown, int | Unknown, int | Unknown] | tuple[Unknown, int | Unknown, int | Unknown, int | Unknown]]]`\n Argument `list[tuple[Unknown, Unknown, Unknown, Unknown, Unknown]]` is not assignable to parameter `value` with type `list[tuple[int | Unknown] | tuple[Unknown]] | list[tuple[int | Unknown, int | Unknown, int | Unknown] | tuple[Unknown, int | Unknown, int | Unknown]] | list[tuple[int | Unknown, int | Unknown, int | Unknown, int | Unknown] | tuple[Unknown, int | Unknown, int | Unknown, int | Unknown]]` in function `dict.__setitem__`", + "concise_description": "Cannot set item in `dict[str, list[tuple[int | Unknown] | tuple[Unknown]] | list[tuple[int | Unknown, int | Unknown, int | Unknown] | tuple[Unknown, int | Unknown, int | Unknown]] | list[tuple[int | Unknown, int | Unknown, int | Unknown, int | Unknown] | tuple[Unknown, int | Unknown, int | Unknown, int | Unknown]]]`", + "severity": "error" + }, + { + "line": 827, + "column": 9, + "stop_line": 827, + "stop_column": 23, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `UNet.get_shape_dict` overrides parent class `BaseModel` in an inconsistent manner\n `UNet.get_shape_dict` has type `(self: UNet, batch_size: Unknown, image_height: Unknown, image_width: Unknown) -> dict[str, tuple[Unknown] | tuple[Unknown, int | Unknown, int | Unknown] | tuple[Unknown, int | Unknown, Unknown, Unknown] | tuple[Unknown, int, Unknown, Unknown]]`, which is not assignable to `(self: UNet, batch_size: Unknown, image_height: Unknown, image_width: Unknown) -> None`, the type of `BaseModel.get_shape_dict`\n Signature mismatch:\n ...-> None: ...\n ^^^^ return type\n ...-> dict[str, tuple[Unknown] | tuple[Unknown, int | Unknown, int | Unknown] | tuple[Unknown, int | Unknown, Unknown, Unknown] | tuple[Unknown, int, Unknown, Unknown]]: ...\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ return type", + "concise_description": "Class member `UNet.get_shape_dict` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 856, + "column": 39, + "stop_line": 856, + "stop_column": 96, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "unsupported-operation", + "description": "Cannot set item in `dict[str, tuple[Unknown] | tuple[Unknown, int | Unknown, int | Unknown] | tuple[Unknown, int | Unknown, Unknown, Unknown] | tuple[Unknown, int, Unknown, Unknown]]`\n Argument `tuple[Literal[2], int, Unknown, Unknown, Unknown]` is not assignable to parameter `value` with type `tuple[Unknown] | tuple[Unknown, int | Unknown, int | Unknown] | tuple[Unknown, int | Unknown, Unknown, Unknown] | tuple[Unknown, int, Unknown, Unknown]` in function `dict.__setitem__`", + "concise_description": "Cannot set item in `dict[str, tuple[Unknown] | tuple[Unknown, int | Unknown, int | Unknown] | tuple[Unknown, int | Unknown, Unknown, Unknown] | tuple[Unknown, int, Unknown, Unknown]]`", + "severity": "error" + }, + { + "line": 857, + "column": 40, + "stop_line": 857, + "stop_column": 78, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "unsupported-operation", + "description": "Cannot set item in `dict[str, tuple[Unknown] | tuple[Unknown, int | Unknown, int | Unknown] | tuple[Unknown, int | Unknown, Unknown, Unknown] | tuple[Unknown, int, Unknown, Unknown]]`\n Argument `tuple[Literal[2], Literal[1], Unknown, Unknown, Unknown]` is not assignable to parameter `value` with type `tuple[Unknown] | tuple[Unknown, int | Unknown, int | Unknown] | tuple[Unknown, int | Unknown, Unknown, Unknown] | tuple[Unknown, int, Unknown, Unknown]` in function `dict.__setitem__`", + "concise_description": "Cannot set item in `dict[str, tuple[Unknown] | tuple[Unknown, int | Unknown, int | Unknown] | tuple[Unknown, int | Unknown, Unknown, Unknown] | tuple[Unknown, int, Unknown, Unknown]]`", + "severity": "error" + }, + { + "line": 871, + "column": 9, + "stop_line": 871, + "stop_column": 25, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `UNet.get_sample_input` overrides parent class `BaseModel` in an inconsistent manner\n `UNet.get_sample_input` has type `(self: UNet, batch_size: Unknown, image_height: Unknown, image_width: Unknown) -> tuple[Tensor, ...]`, which is not assignable to `(self: UNet, batch_size: Unknown, image_height: Unknown, image_width: Unknown) -> None`, the type of `BaseModel.get_sample_input`\n Signature mismatch:\n expected: def get_sample_input(self: UNet, batch_size: Unknown, image_height: Unknown, image_width: Unknown) -> None: ...\n ^^^^ return type\n found: def get_sample_input(self: UNet, batch_size: Unknown, image_height: Unknown, image_width: Unknown) -> tuple[Tensor, ...]: ...\n ^^^^^^^^^^^^^^^^^^ return type", + "concise_description": "Class member `UNet.get_sample_input` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 967, + "column": 9, + "stop_line": 967, + "stop_column": 24, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `VAE.get_input_names` overrides parent class `BaseModel` in an inconsistent manner\n `VAE.get_input_names` has type `(self: VAE) -> list[str]`, which is not assignable to `(self: VAE) -> None`, the type of `BaseModel.get_input_names`\n Signature mismatch:\n expected: def get_input_names(self: VAE) -> None: ...\n ^^^^ return type\n found: def get_input_names(self: VAE) -> list[str]: ...\n ^^^^^^^^^ return type", + "concise_description": "Class member `VAE.get_input_names` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 970, + "column": 9, + "stop_line": 970, + "stop_column": 25, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `VAE.get_output_names` overrides parent class `BaseModel` in an inconsistent manner\n `VAE.get_output_names` has type `(self: VAE) -> list[str]`, which is not assignable to `(self: VAE) -> None`, the type of `BaseModel.get_output_names`\n Signature mismatch:\n expected: def get_output_names(self: VAE) -> None: ...\n ^^^^ return type\n found: def get_output_names(self: VAE) -> list[str]: ...\n ^^^^^^^^^ return type", + "concise_description": "Class member `VAE.get_output_names` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 973, + "column": 9, + "stop_line": 973, + "stop_column": 25, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `VAE.get_dynamic_axes` overrides parent class `BaseModel` in an inconsistent manner\n `VAE.get_dynamic_axes` has type `(self: VAE) -> dict[str, dict[int, str]]`, which is not assignable to `(self: VAE) -> None`, the type of `BaseModel.get_dynamic_axes`\n Signature mismatch:\n expected: def get_dynamic_axes(self: VAE) -> None: ...\n ^^^^ return type\n found: def get_dynamic_axes(self: VAE) -> dict[str, dict[int, str]]: ...\n ^^^^^^^^^^^^^^^^^^^^^^^^^ return type", + "concise_description": "Class member `VAE.get_dynamic_axes` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 979, + "column": 9, + "stop_line": 979, + "stop_column": 26, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `VAE.get_input_profile` overrides parent class `BaseModel` in an inconsistent manner\n `VAE.get_input_profile` has type `(self: VAE, batch_size: Unknown, image_height: Unknown, image_width: Unknown, static_batch: Unknown, static_shape: Unknown) -> dict[str, list[tuple[int | Unknown, int, int | Unknown, int | Unknown] | tuple[Unknown, int, Unknown, Unknown]]]`, which is not assignable to `(self: VAE, batch_size: Unknown, image_height: Unknown, image_width: Unknown, static_batch: Unknown, static_shape: Unknown) -> None`, the type of `BaseModel.get_input_profile`\n Signature mismatch:\n ...-> None: ...\n ^^^^ return type\n ...-> dict[str, list[tuple[int | Unknown, int, int | Unknown, int | Unknown] | tuple[Unknown, int, Unknown, Unknown]]]: ...\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ return type", + "concise_description": "Class member `VAE.get_input_profile` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 1001, + "column": 9, + "stop_line": 1001, + "stop_column": 23, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `VAE.get_shape_dict` overrides parent class `BaseModel` in an inconsistent manner\n `VAE.get_shape_dict` has type `(self: VAE, batch_size: Unknown, image_height: Unknown, image_width: Unknown) -> dict[str, tuple[Unknown, int, Unknown, Unknown]]`, which is not assignable to `(self: VAE, batch_size: Unknown, image_height: Unknown, image_width: Unknown) -> None`, the type of `BaseModel.get_shape_dict`\n Signature mismatch:\n ... Unknown, image_width: Unknown) -> None: ...\n ^^^^ return type\n ... Unknown, image_width: Unknown) -> dict[str, tuple[Unknown, int, Unknown, Unknown]]: ...\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ return type", + "concise_description": "Class member `VAE.get_shape_dict` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 1008, + "column": 9, + "stop_line": 1008, + "stop_column": 25, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `VAE.get_sample_input` overrides parent class `BaseModel` in an inconsistent manner\n `VAE.get_sample_input` has type `(self: VAE, batch_size: Unknown, image_height: Unknown, image_width: Unknown) -> Tensor`, which is not assignable to `(self: VAE, batch_size: Unknown, image_height: Unknown, image_width: Unknown) -> None`, the type of `BaseModel.get_sample_input`\n Signature mismatch:\n expected: def get_sample_input(self: VAE, batch_size: Unknown, image_height: Unknown, image_width: Unknown) -> None: ...\n ^^^^ return type\n found: def get_sample_input(self: VAE, batch_size: Unknown, image_height: Unknown, image_width: Unknown) -> Tensor: ...\n ^^^^^^ return type", + "concise_description": "Class member `VAE.get_sample_input` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 1030, + "column": 9, + "stop_line": 1030, + "stop_column": 24, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `VAEEncoder.get_input_names` overrides parent class `BaseModel` in an inconsistent manner\n `VAEEncoder.get_input_names` has type `(self: VAEEncoder) -> list[str]`, which is not assignable to `(self: VAEEncoder) -> None`, the type of `BaseModel.get_input_names`\n Signature mismatch:\n expected: def get_input_names(self: VAEEncoder) -> None: ...\n ^^^^ return type\n found: def get_input_names(self: VAEEncoder) -> list[str]: ...\n ^^^^^^^^^ return type", + "concise_description": "Class member `VAEEncoder.get_input_names` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 1033, + "column": 9, + "stop_line": 1033, + "stop_column": 25, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `VAEEncoder.get_output_names` overrides parent class `BaseModel` in an inconsistent manner\n `VAEEncoder.get_output_names` has type `(self: VAEEncoder) -> list[str]`, which is not assignable to `(self: VAEEncoder) -> None`, the type of `BaseModel.get_output_names`\n Signature mismatch:\n expected: def get_output_names(self: VAEEncoder) -> None: ...\n ^^^^ return type\n found: def get_output_names(self: VAEEncoder) -> list[str]: ...\n ^^^^^^^^^ return type", + "concise_description": "Class member `VAEEncoder.get_output_names` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 1036, + "column": 9, + "stop_line": 1036, + "stop_column": 25, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `VAEEncoder.get_dynamic_axes` overrides parent class `BaseModel` in an inconsistent manner\n `VAEEncoder.get_dynamic_axes` has type `(self: VAEEncoder) -> dict[str, dict[int, str]]`, which is not assignable to `(self: VAEEncoder) -> None`, the type of `BaseModel.get_dynamic_axes`\n Signature mismatch:\n expected: def get_dynamic_axes(self: VAEEncoder) -> None: ...\n ^^^^ return type\n found: def get_dynamic_axes(self: VAEEncoder) -> dict[str, dict[int, str]]: ...\n ^^^^^^^^^^^^^^^^^^^^^^^^^ return type", + "concise_description": "Class member `VAEEncoder.get_dynamic_axes` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 1042, + "column": 9, + "stop_line": 1042, + "stop_column": 26, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `VAEEncoder.get_input_profile` overrides parent class `BaseModel` in an inconsistent manner\n `VAEEncoder.get_input_profile` has type `(self: VAEEncoder, batch_size: Unknown, image_height: Unknown, image_width: Unknown, static_batch: Unknown, static_shape: Unknown) -> dict[str, list[tuple[int | Unknown, int, int | Unknown, int | Unknown] | tuple[Unknown, int, Unknown, Unknown]]]`, which is not assignable to `(self: VAEEncoder, batch_size: Unknown, image_height: Unknown, image_width: Unknown, static_batch: Unknown, static_shape: Unknown) -> None`, the type of `BaseModel.get_input_profile`\n Signature mismatch:\n ...-> None: ...\n ^^^^ return type\n ...-> dict[str, list[tuple[int | Unknown, int, int | Unknown, int | Unknown] | tuple[Unknown, int, Unknown, Unknown]]]: ...\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ return type", + "concise_description": "Class member `VAEEncoder.get_input_profile` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 1068, + "column": 9, + "stop_line": 1068, + "stop_column": 23, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `VAEEncoder.get_shape_dict` overrides parent class `BaseModel` in an inconsistent manner\n `VAEEncoder.get_shape_dict` has type `(self: VAEEncoder, batch_size: Unknown, image_height: Unknown, image_width: Unknown) -> dict[str, tuple[Unknown, int, Unknown, Unknown]]`, which is not assignable to `(self: VAEEncoder, batch_size: Unknown, image_height: Unknown, image_width: Unknown) -> None`, the type of `BaseModel.get_shape_dict`\n Signature mismatch:\n ... Unknown, image_width: Unknown) -> None: ...\n ^^^^ return type\n ... Unknown, image_width: Unknown) -> dict[str, tuple[Unknown, int, Unknown, Unknown]]: ...\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ return type", + "concise_description": "Class member `VAEEncoder.get_shape_dict` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 1075, + "column": 9, + "stop_line": 1075, + "stop_column": 25, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\models.py", + "code": -2, + "name": "bad-override", + "description": "Class member `VAEEncoder.get_sample_input` overrides parent class `BaseModel` in an inconsistent manner\n `VAEEncoder.get_sample_input` has type `(self: VAEEncoder, batch_size: Unknown, image_height: Unknown, image_width: Unknown) -> Tensor`, which is not assignable to `(self: VAEEncoder, batch_size: Unknown, image_height: Unknown, image_width: Unknown) -> None`, the type of `BaseModel.get_sample_input`\n Signature mismatch:\n expected: def get_sample_input(self: VAEEncoder, batch_size: Unknown, image_height: Unknown, image_width: Unknown) -> None: ...\n ^^^^ return type\n found: def get_sample_input(self: VAEEncoder, batch_size: Unknown, image_height: Unknown, image_width: Unknown) -> Tensor: ...\n ^^^^^^ return type", + "concise_description": "Class member `VAEEncoder.get_sample_input` overrides parent class `BaseModel` in an inconsistent manner", + "severity": "error" + }, + { + "line": 18, + "column": 31, + "stop_line": 18, + "stop_column": 47, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\utils.py", + "code": -2, + "name": "not-iterable", + "description": "Type `Module` is not iterable\n Expected `__getitem__` to be a callable, got `Module | Tensor`", + "concise_description": "Type `Module` is not iterable", + "severity": "error" + }, + { + "line": 20, + "column": 36, + "stop_line": 20, + "stop_column": 65, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\utils.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `Tensor` has no attribute `transformer_blocks`", + "concise_description": "Object of class `Tensor` has no attribute `transformer_blocks`", + "severity": "error" + }, + { + "line": 49, + "column": 31, + "stop_line": 49, + "stop_column": 47, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\utils.py", + "code": -2, + "name": "not-iterable", + "description": "Type `Module` is not iterable\n Expected `__getitem__` to be a callable, got `Module | Tensor`", + "concise_description": "Type `Module` is not iterable", + "severity": "error" + }, + { + "line": 51, + "column": 36, + "stop_line": 51, + "stop_column": 65, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\utils.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `Tensor` has no attribute `transformer_blocks`", + "concise_description": "Object of class `Tensor` has no attribute `transformer_blocks`", + "severity": "error" + }, + { + "line": 148, + "column": 31, + "stop_line": 148, + "stop_column": 47, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\utils.py", + "code": -2, + "name": "not-iterable", + "description": "Type `Module` is not iterable\n Expected `__getitem__` to be a callable, got `Module | Tensor`", + "concise_description": "Type `Module` is not iterable", + "severity": "error" + }, + { + "line": 149, + "column": 26, + "stop_line": 149, + "stop_column": 55, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\utils.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `Tensor` has no attribute `transformer_blocks`", + "concise_description": "Object of class `Tensor` has no attribute `transformer_blocks`", + "severity": "error" + }, + { + "line": 162, + "column": 31, + "stop_line": 162, + "stop_column": 47, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\utils.py", + "code": -2, + "name": "not-iterable", + "description": "Type `Module` is not iterable\n Expected `__getitem__` to be a callable, got `Module | Tensor`", + "concise_description": "Type `Module` is not iterable", + "severity": "error" + }, + { + "line": 163, + "column": 26, + "stop_line": 163, + "stop_column": 55, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\models\\utils.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `Tensor` has no attribute `transformer_blocks`", + "concise_description": "Object of class `Tensor` has no attribute `transformer_blocks`", + "severity": "error" + }, + { + "line": 39, + "column": 31, + "stop_line": 39, + "stop_column": 66, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\runtime_engines\\controlnet_engine.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_shape`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_shape`", + "severity": "error" + }, + { + "line": 110, + "column": 16, + "stop_line": 110, + "stop_column": 29, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\runtime_engines\\controlnet_engine.py", + "code": -2, + "name": "bad-return", + "description": "Returned type `dict[str, tuple[int, int, int, int]]` is not assignable to declared return type `dict[str, tuple[int, ...]]`", + "concise_description": "Returned type `dict[str, tuple[int, int, int, int]]` is not assignable to declared return type `dict[str, tuple[int, ...]]`", + "severity": "error" + }, + { + "line": 151, + "column": 26, + "stop_line": 151, + "stop_column": 41, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\runtime_engines\\controlnet_engine.py", + "code": -2, + "name": "no-matching-overload", + "description": "No matching overload found for function `typing.MutableMapping.update` called with arguments: (dict[str, tuple[int, ...]])\n Possible overloads:\n (m: SupportsKeysAndGetItem[str, Size], /) -> None [closest match]\n (m: SupportsKeysAndGetItem[str, Size], /, **kwargs: Size) -> None\n (m: Iterable[tuple[str, Size]], /) -> None\n (m: Iterable[tuple[str, Size]], /, **kwargs: Size) -> None\n (**kwargs: Size) -> None", + "concise_description": "No matching overload found for function `typing.MutableMapping.update` called with arguments: (dict[str, tuple[int, ...]])", + "severity": "error" + }, + { + "line": 186, + "column": 16, + "stop_line": 186, + "stop_column": 38, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\runtime_engines\\controlnet_engine.py", + "code": -2, + "name": "bad-return", + "description": "Returned type `tuple[list[Tensor], Tensor | None]` is not assignable to declared return type `tuple[list[Tensor], Tensor]`", + "concise_description": "Returned type `tuple[list[Tensor], Tensor | None]` is not assignable to declared return type `tuple[list[Tensor], Tensor]`", + "severity": "error" + }, + { + "line": 43, + "column": 39, + "stop_line": 43, + "stop_column": 54, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-module-attribute", + "description": "Could not import `bytes_from_path` from `polygraphy.backend.common`", + "concise_description": "Could not import `bytes_from_path` from `polygraphy.backend.common`", + "severity": "error" + }, + { + "line": 44, + "column": 36, + "stop_line": 44, + "stop_column": 53, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-module-attribute", + "description": "Could not import `engine_from_bytes` from `polygraphy.backend.trt`", + "concise_description": "Could not import `engine_from_bytes` from `polygraphy.backend.trt`", + "severity": "error" + }, + { + "line": 54, + "column": 23, + "stop_line": 54, + "stop_column": 34, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `ILogger` in module `tensorrt`", + "concise_description": "No attribute `ILogger` in module `tensorrt`", + "severity": "error" + }, + { + "line": 73, + "column": 9, + "stop_line": 73, + "stop_column": 20, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `ILogger` in module `tensorrt`", + "concise_description": "No attribute `ILogger` in module `tensorrt`", + "severity": "error" + }, + { + "line": 114, + "column": 9, + "stop_line": 114, + "stop_column": 20, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `Builder` in module `tensorrt`", + "concise_description": "No attribute `Builder` in module `tensorrt`", + "severity": "error" + }, + { + "line": 276, + "column": 14, + "stop_line": 276, + "stop_column": 32, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `IBuilderConfig` in module `tensorrt`", + "concise_description": "No attribute `IBuilderConfig` in module `tensorrt`", + "severity": "error" + }, + { + "line": 317, + "column": 25, + "stop_line": 317, + "stop_column": 52, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `TilingOptimizationLevel` in module `tensorrt`", + "concise_description": "No attribute `TilingOptimizationLevel` in module `tensorrt`", + "severity": "error" + }, + { + "line": 318, + "column": 25, + "stop_line": 318, + "stop_column": 52, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `TilingOptimizationLevel` in module `tensorrt`", + "concise_description": "No attribute `TilingOptimizationLevel` in module `tensorrt`", + "severity": "error" + }, + { + "line": 319, + "column": 29, + "stop_line": 319, + "stop_column": 56, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `TilingOptimizationLevel` in module `tensorrt`", + "concise_description": "No attribute `TilingOptimizationLevel` in module `tensorrt`", + "severity": "error" + }, + { + "line": 320, + "column": 25, + "stop_line": 320, + "stop_column": 52, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `TilingOptimizationLevel` in module `tensorrt`", + "concise_description": "No attribute `TilingOptimizationLevel` in module `tensorrt`", + "severity": "error" + }, + { + "line": 322, + "column": 82, + "stop_line": 322, + "stop_column": 109, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `TilingOptimizationLevel` in module `tensorrt`", + "concise_description": "No attribute `TilingOptimizationLevel` in module `tensorrt`", + "severity": "error" + }, + { + "line": 351, + "column": 29, + "stop_line": 351, + "stop_column": 44, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `BuilderFlag` in module `tensorrt`", + "concise_description": "No attribute `BuilderFlag` in module `tensorrt`", + "severity": "error" + }, + { + "line": 362, + "column": 40, + "stop_line": 362, + "stop_column": 58, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `PreviewFeature` in module `tensorrt`", + "concise_description": "No attribute `PreviewFeature` in module `tensorrt`", + "severity": "error" + }, + { + "line": 389, + "column": 20, + "stop_line": 389, + "stop_column": 36, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `TacticSource` in module `tensorrt`", + "concise_description": "No attribute `TacticSource` in module `tensorrt`", + "severity": "error" + }, + { + "line": 391, + "column": 27, + "stop_line": 391, + "stop_column": 43, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `TacticSource` in module `tensorrt`", + "concise_description": "No attribute `TacticSource` in module `tensorrt`", + "severity": "error" + }, + { + "line": 392, + "column": 29, + "stop_line": 392, + "stop_column": 45, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `TacticSource` in module `tensorrt`", + "concise_description": "No attribute `TacticSource` in module `tensorrt`", + "severity": "error" + }, + { + "line": 393, + "column": 29, + "stop_line": 393, + "stop_column": 45, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `TacticSource` in module `tensorrt`", + "concise_description": "No attribute `TacticSource` in module `tensorrt`", + "severity": "error" + }, + { + "line": 394, + "column": 29, + "stop_line": 394, + "stop_column": 45, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `TacticSource` in module `tensorrt`", + "concise_description": "No attribute `TacticSource` in module `tensorrt`", + "severity": "error" + }, + { + "line": 430, + "column": 31, + "stop_line": 430, + "stop_column": 39, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "unsupported-operation", + "description": "Cannot set item in `dict[type[complex128] | type[complex64] | type[float16] | type[float32] | type[float64] | type[int16] | type[int32] | type[int64] | type[int8] | type[uint8], dtype]`\n Argument `type[bool_]` is not assignable to parameter `key` with type `type[complex128] | type[complex64] | type[float16] | type[float32] | type[float64] | type[int16] | type[int32] | type[int64] | type[int8] | type[uint8]` in function `dict.__setitem__`", + "concise_description": "Cannot set item in `dict[type[complex128] | type[complex64] | type[float16] | type[float32] | type[float64] | type[int16] | type[int32] | type[int64] | type[int8] | type[uint8], dtype]`", + "severity": "error" + }, + { + "line": 432, + "column": 31, + "stop_line": 432, + "stop_column": 38, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `bool` in module `numpy`", + "concise_description": "No attribute `bool` in module `numpy`", + "severity": "error" + }, + { + "line": 471, + "column": 19, + "stop_line": 471, + "stop_column": 32, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `IProfiler` in module `tensorrt`", + "concise_description": "No attribute `IProfiler` in module `tensorrt`", + "severity": "error" + }, + { + "line": 519, + "column": 27, + "stop_line": 519, + "stop_column": 35, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "bad-index", + "description": "Cannot index into `deque[Unknown]`\n Argument `slice[int, int, int | Unknown]` is not assignable to parameter `key` with type `SupportsIndex` in function `collections.deque.__getitem__`\n Protocol `SupportsIndex` requires attribute `__index__`", + "concise_description": "Cannot index into `deque[Unknown]`", + "severity": "error" + }, + { + "line": 684, + "column": 20, + "stop_line": 684, + "stop_column": 32, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `Refitter` in module `tensorrt`", + "concise_description": "No attribute `Refitter` in module `tensorrt`", + "severity": "error" + }, + { + "line": 688, + "column": 24, + "stop_line": 688, + "stop_column": 39, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `WeightsRole` in module `tensorrt`", + "concise_description": "No attribute `WeightsRole` in module `tensorrt`", + "severity": "error" + }, + { + "line": 690, + "column": 26, + "stop_line": 690, + "stop_column": 41, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `WeightsRole` in module `tensorrt`", + "concise_description": "No attribute `WeightsRole` in module `tensorrt`", + "severity": "error" + }, + { + "line": 722, + "column": 32, + "stop_line": 722, + "stop_column": 47, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `WeightsRole` in module `tensorrt`", + "concise_description": "No attribute `WeightsRole` in module `tensorrt`", + "severity": "error" + }, + { + "line": 724, + "column": 34, + "stop_line": 724, + "stop_column": 49, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `WeightsRole` in module `tensorrt`", + "concise_description": "No attribute `WeightsRole` in module `tensorrt`", + "severity": "error" + }, + { + "line": 782, + "column": 19, + "stop_line": 782, + "stop_column": 30, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `Builder` in module `tensorrt`", + "concise_description": "No attribute `Builder` in module `tensorrt`", + "severity": "error" + }, + { + "line": 787, + "column": 18, + "stop_line": 787, + "stop_column": 32, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `OnnxParser` in module `tensorrt`", + "concise_description": "No attribute `OnnxParser` in module `tensorrt`", + "severity": "error" + }, + { + "line": 788, + "column": 25, + "stop_line": 788, + "stop_column": 43, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `OnnxParserFlag` in module `tensorrt`", + "concise_description": "No attribute `OnnxParserFlag` in module `tensorrt`", + "severity": "error" + }, + { + "line": 801, + "column": 42, + "stop_line": 801, + "stop_column": 64, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `ProfilingVerbosity` in module `tensorrt`", + "concise_description": "No attribute `ProfilingVerbosity` in module `tensorrt`", + "severity": "error" + }, + { + "line": 807, + "column": 29, + "stop_line": 807, + "stop_column": 44, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `BuilderFlag` in module `tensorrt`", + "concise_description": "No attribute `BuilderFlag` in module `tensorrt`", + "severity": "error" + }, + { + "line": 808, + "column": 25, + "stop_line": 808, + "stop_column": 40, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `BuilderFlag` in module `tensorrt`", + "concise_description": "No attribute `BuilderFlag` in module `tensorrt`", + "severity": "error" + }, + { + "line": 811, + "column": 29, + "stop_line": 811, + "stop_column": 44, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `BuilderFlag` in module `tensorrt`", + "concise_description": "No attribute `BuilderFlag` in module `tensorrt`", + "severity": "error" + }, + { + "line": 815, + "column": 42, + "stop_line": 815, + "stop_column": 60, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `MemoryPoolType` in module `tensorrt`", + "concise_description": "No attribute `MemoryPoolType` in module `tensorrt`", + "severity": "error" + }, + { + "line": 902, + "column": 19, + "stop_line": 902, + "stop_column": 30, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `Builder` in module `tensorrt`", + "concise_description": "No attribute `Builder` in module `tensorrt`", + "severity": "error" + }, + { + "line": 906, + "column": 34, + "stop_line": 906, + "stop_column": 67, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `NetworkDefinitionCreationFlag` in module `tensorrt`", + "concise_description": "No attribute `NetworkDefinitionCreationFlag` in module `tensorrt`", + "severity": "error" + }, + { + "line": 909, + "column": 18, + "stop_line": 909, + "stop_column": 32, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `OnnxParser` in module `tensorrt`", + "concise_description": "No attribute `OnnxParser` in module `tensorrt`", + "severity": "error" + }, + { + "line": 912, + "column": 25, + "stop_line": 912, + "stop_column": 43, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `OnnxParserFlag` in module `tensorrt`", + "concise_description": "No attribute `OnnxParserFlag` in module `tensorrt`", + "severity": "error" + }, + { + "line": 924, + "column": 42, + "stop_line": 924, + "stop_column": 64, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `ProfilingVerbosity` in module `tensorrt`", + "concise_description": "No attribute `ProfilingVerbosity` in module `tensorrt`", + "severity": "error" + }, + { + "line": 932, + "column": 20, + "stop_line": 932, + "stop_column": 35, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `BuilderFlag` in module `tensorrt`", + "concise_description": "No attribute `BuilderFlag` in module `tensorrt`", + "severity": "error" + }, + { + "line": 935, + "column": 29, + "stop_line": 935, + "stop_column": 44, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `BuilderFlag` in module `tensorrt`", + "concise_description": "No attribute `BuilderFlag` in module `tensorrt`", + "severity": "error" + }, + { + "line": 936, + "column": 29, + "stop_line": 936, + "stop_column": 44, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `BuilderFlag` in module `tensorrt`", + "concise_description": "No attribute `BuilderFlag` in module `tensorrt`", + "severity": "error" + }, + { + "line": 937, + "column": 29, + "stop_line": 937, + "stop_column": 44, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `BuilderFlag` in module `tensorrt`", + "concise_description": "No attribute `BuilderFlag` in module `tensorrt`", + "severity": "error" + }, + { + "line": 938, + "column": 29, + "stop_line": 938, + "stop_column": 44, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `BuilderFlag` in module `tensorrt`", + "concise_description": "No attribute `BuilderFlag` in module `tensorrt`", + "severity": "error" + }, + { + "line": 943, + "column": 42, + "stop_line": 943, + "stop_column": 60, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `MemoryPoolType` in module `tensorrt`", + "concise_description": "No attribute `MemoryPoolType` in module `tensorrt`", + "severity": "error" + }, + { + "line": 1019, + "column": 24, + "stop_line": 1019, + "stop_column": 60, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `create_execution_context`", + "concise_description": "Object of class `NoneType` has no attribute `create_execution_context`", + "severity": "error" + }, + { + "line": 1049, + "column": 26, + "stop_line": 1049, + "stop_column": 52, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `num_io_tensors`", + "concise_description": "Object of class `NoneType` has no attribute `num_io_tensors`", + "severity": "error" + }, + { + "line": 1050, + "column": 20, + "stop_line": 1050, + "stop_column": 47, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_name`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_name`", + "severity": "error" + }, + { + "line": 1055, + "column": 25, + "stop_line": 1055, + "stop_column": 53, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_shape`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_shape`", + "severity": "error" + }, + { + "line": 1057, + "column": 25, + "stop_line": 1057, + "stop_column": 53, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_dtype`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_dtype`", + "severity": "error" + }, + { + "line": 1063, + "column": 33, + "stop_line": 1063, + "stop_column": 45, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `DataType` in module `tensorrt`", + "concise_description": "No attribute `DataType` in module `tensorrt`", + "severity": "error" + }, + { + "line": 1067, + "column": 20, + "stop_line": 1067, + "stop_column": 47, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_mode`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_mode`", + "severity": "error" + }, + { + "line": 1069, + "column": 24, + "stop_line": 1069, + "stop_column": 40, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `TensorIOMode` in module `tensorrt`", + "concise_description": "No attribute `TensorIOMode` in module `tensorrt`", + "severity": "error" + }, + { + "line": 1070, + "column": 24, + "stop_line": 1070, + "stop_column": 52, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `set_input_shape`", + "concise_description": "Object of class `NoneType` has no attribute `set_input_shape`", + "severity": "error" + }, + { + "line": 1150, + "column": 34, + "stop_line": 1150, + "stop_column": 60, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `num_io_tensors`", + "concise_description": "Object of class `NoneType` has no attribute `num_io_tensors`", + "severity": "error" + }, + { + "line": 1151, + "column": 28, + "stop_line": 1151, + "stop_column": 55, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_name`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_name`", + "severity": "error" + }, + { + "line": 1152, + "column": 24, + "stop_line": 1152, + "stop_column": 51, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_mode`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_mode`", + "severity": "error" + }, + { + "line": 1152, + "column": 61, + "stop_line": 1152, + "stop_column": 77, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `TensorIOMode` in module `tensorrt`", + "concise_description": "No attribute `TensorIOMode` in module `tensorrt`", + "severity": "error" + }, + { + "line": 1230, + "column": 24, + "stop_line": 1230, + "stop_column": 55, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `set_tensor_address`", + "concise_description": "Object of class `NoneType` has no attribute `set_tensor_address`", + "severity": "error" + }, + { + "line": 1245, + "column": 35, + "stop_line": 1245, + "stop_column": 64, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `execute_async_v3`", + "concise_description": "Object of class `NoneType` has no attribute `execute_async_v3`", + "severity": "error" + }, + { + "line": 1258, + "column": 35, + "stop_line": 1258, + "stop_column": 64, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `execute_async_v3`", + "concise_description": "Object of class `NoneType` has no attribute `execute_async_v3`", + "severity": "error" + }, + { + "line": 1276, + "column": 21, + "stop_line": 1276, + "stop_column": 50, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `execute_async_v3`", + "concise_description": "Object of class `NoneType` has no attribute `execute_async_v3`", + "severity": "error" + }, + { + "line": 1280, + "column": 27, + "stop_line": 1280, + "stop_column": 56, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `execute_async_v3`", + "concise_description": "Object of class `NoneType` has no attribute `execute_async_v3`", + "severity": "error" + }, + { + "line": 1303, + "column": 9, + "stop_line": 1303, + "stop_column": 116, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "bad-assignment", + "description": "`ndarray[Unknown, Unknown]` is not assignable to variable `images` with type `Tensor`", + "concise_description": "`ndarray[Unknown, Unknown]` is not assignable to variable `images` with type `Tensor`", + "severity": "error" + }, + { + "line": 1305, + "column": 29, + "stop_line": 1305, + "stop_column": 30, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Tensor` is not assignable to parameter `obj` with type `SupportsArrayInterface` in function `PIL.Image.fromarray`\n Protocol `SupportsArrayInterface` requires attribute `__array_interface__`", + "concise_description": "Argument `Tensor` is not assignable to parameter `obj` with type `SupportsArrayInterface` in function `PIL.Image.fromarray`", + "severity": "error" + }, + { + "line": 1320, + "column": 17, + "stop_line": 1320, + "stop_column": 47, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "bad-assignment", + "description": "`ndarray[Any, dtype[Any]]` is not assignable to variable `image` with type `Image`", + "concise_description": "`ndarray[Any, dtype[Any]]` is not assignable to variable `image` with type `Image`", + "severity": "error" + }, + { + "line": 1321, + "column": 13, + "stop_line": 1321, + "stop_column": 24, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "bad-index", + "description": "Cannot index into `Image`\n Object of class `Image` has no attribute `__getitem__`", + "concise_description": "Cannot index into `Image`", + "severity": "error" + }, + { + "line": 1322, + "column": 13, + "stop_line": 1322, + "stop_column": 87, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "bad-assignment", + "description": "`Tensor` is not assignable to variable `image` with type `Image`", + "concise_description": "`Tensor` is not assignable to variable `image` with type `Image`", + "severity": "error" + }, + { + "line": 1324, + "column": 16, + "stop_line": 1324, + "stop_column": 43, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "bad-assignment", + "description": "`ndarray[Any, dtype[Any]]` is not assignable to variable `mask` with type `Image`", + "concise_description": "`ndarray[Any, dtype[Any]]` is not assignable to variable `mask` with type `Image`", + "severity": "error" + }, + { + "line": 1325, + "column": 16, + "stop_line": 1325, + "stop_column": 27, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `Image` has no attribute `astype`", + "concise_description": "Object of class `Image` has no attribute `astype`", + "severity": "error" + }, + { + "line": 1329, + "column": 12, + "stop_line": 1329, + "stop_column": 71, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "bad-assignment", + "description": "`Tensor` is not assignable to variable `mask` with type `Image`", + "concise_description": "`Tensor` is not assignable to variable `mask` with type `Image`", + "severity": "error" + }, + { + "line": 1331, + "column": 20, + "stop_line": 1331, + "stop_column": 40, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "unsupported-operation", + "description": "`*` is not supported between `Image` and `bool`\n Argument `Image` is not assignable to parameter `value` with type `int` in function `int.__rmul__`", + "concise_description": "`*` is not supported between `Image` and `bool`", + "severity": "error" + }, + { + "line": 1331, + "column": 29, + "stop_line": 1331, + "stop_column": 39, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "unsupported-operation", + "description": "`<` is not supported between `Image` and `float`\n Argument `Image` is not assignable to parameter `value` with type `float` in function `float.__gt__`", + "concise_description": "`<` is not supported between `Image` and `float`", + "severity": "error" + }, + { + "line": 1346, + "column": 13, + "stop_line": 1346, + "stop_column": 21, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "unexpected-keyword", + "description": "Unexpected keyword argument `hf_token` in function `streamdiffusion.acceleration.tensorrt.models.models.CLIP.__init__`", + "concise_description": "Unexpected keyword argument `hf_token` in function `streamdiffusion.acceleration.tensorrt.models.models.CLIP.__init__`", + "severity": "error" + }, + { + "line": 1352, + "column": 13, + "stop_line": 1352, + "stop_column": 21, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "unexpected-keyword", + "description": "Unexpected keyword argument `hf_token` in function `streamdiffusion.acceleration.tensorrt.models.models.UNet.__init__`", + "concise_description": "Unexpected keyword argument `hf_token` in function `streamdiffusion.acceleration.tensorrt.models.models.UNet.__init__`", + "severity": "error" + }, + { + "line": 1360, + "column": 13, + "stop_line": 1360, + "stop_column": 21, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "unexpected-keyword", + "description": "Unexpected keyword argument `hf_token` in function `streamdiffusion.acceleration.tensorrt.models.models.VAE.__init__`", + "concise_description": "Unexpected keyword argument `hf_token` in function `streamdiffusion.acceleration.tensorrt.models.models.VAE.__init__`", + "severity": "error" + }, + { + "line": 1363, + "column": 13, + "stop_line": 1363, + "stop_column": 26, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "unexpected-keyword", + "description": "Unexpected keyword argument `embedding_dim` in function `streamdiffusion.acceleration.tensorrt.models.models.VAE.__init__`", + "concise_description": "Unexpected keyword argument `embedding_dim` in function `streamdiffusion.acceleration.tensorrt.models.models.VAE.__init__`", + "severity": "error" + }, + { + "line": 1366, + "column": 13, + "stop_line": 1366, + "stop_column": 21, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "unexpected-keyword", + "description": "Unexpected keyword argument `hf_token` in function `streamdiffusion.acceleration.tensorrt.models.models.VAEEncoder.__init__`", + "concise_description": "Unexpected keyword argument `hf_token` in function `streamdiffusion.acceleration.tensorrt.models.models.VAEEncoder.__init__`", + "severity": "error" + }, + { + "line": 1369, + "column": 13, + "stop_line": 1369, + "stop_column": 26, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "unexpected-keyword", + "description": "Unexpected keyword argument `embedding_dim` in function `streamdiffusion.acceleration.tensorrt.models.models.VAEEncoder.__init__`", + "concise_description": "Unexpected keyword argument `embedding_dim` in function `streamdiffusion.acceleration.tensorrt.models.models.VAEEncoder.__init__`", + "severity": "error" + }, + { + "line": 1532, + "column": 34, + "stop_line": 1532, + "stop_column": 56, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "bad-context-manager", + "description": "Cannot use `autocast` as a context manager\n Return type `Literal[False] | object | None` of function `autocast.__exit__` is not assignable to expected return type `bool | None`", + "concise_description": "Cannot use `autocast` as a context manager", + "severity": "error" + }, + { + "line": 1542, + "column": 13, + "stop_line": 1542, + "stop_column": 19, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `None` is not assignable to parameter `args` with type `tuple[Any, ...]` in function `torch.onnx.export`", + "concise_description": "Argument `None` is not assignable to parameter `args` with type `tuple[Any, ...]` in function `torch.onnx.export`", + "severity": "error" + }, + { + "line": 1622, + "column": 13, + "stop_line": 1622, + "stop_column": 27, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `ModelProto | None` is not assignable to parameter `proto` with type `ModelProto | bytes` in function `onnx.save_model`", + "concise_description": "Argument `ModelProto | None` is not assignable to parameter `proto` with type `ModelProto | bytes` in function `onnx.save_model`", + "severity": "error" + }, + { + "line": 1635, + "column": 19, + "stop_line": 1635, + "stop_column": 33, + "path": "src\\streamdiffusion\\acceleration\\tensorrt\\utilities.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `ModelProto | None` is not assignable to parameter `proto` with type `ModelProto | bytes` in function `onnx.save_model`", + "concise_description": "Argument `ModelProto | None` is not assignable to parameter `proto` with type `ModelProto | bytes` in function `onnx.save_model`", + "severity": "error" + }, + { + "line": 37, + "column": 21, + "stop_line": 37, + "stop_column": 45, + "path": "src\\streamdiffusion\\image_filter.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `item`", + "concise_description": "Object of class `NoneType` has no attribute `item`", + "severity": "error" + }, + { + "line": 47, + "column": 9, + "stop_line": 47, + "stop_column": 34, + "path": "src\\streamdiffusion\\image_filter.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `copy_`", + "concise_description": "Object of class `NoneType` has no attribute `copy_`", + "severity": "error" + }, + { + "line": 13, + "column": 12, + "stop_line": 13, + "stop_column": 36, + "path": "src\\streamdiffusion\\image_utils.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `ndarray` has no attribute `clamp`", + "concise_description": "Object of class `ndarray` has no attribute `clamp`", + "severity": "error" + }, + { + "line": 20, + "column": 14, + "stop_line": 20, + "stop_column": 62, + "path": "src\\streamdiffusion\\image_utils.py", + "code": -2, + "name": "bad-assignment", + "description": "`ndarray[Unknown, Unknown]` is not assignable to variable `images` with type `Tensor`", + "concise_description": "`ndarray[Unknown, Unknown]` is not assignable to variable `images` with type `Tensor`", + "severity": "error" + }, + { + "line": 21, + "column": 12, + "stop_line": 21, + "stop_column": 18, + "path": "src\\streamdiffusion\\image_utils.py", + "code": -2, + "name": "bad-return", + "description": "Returned type `Tensor` is not assignable to declared return type `ndarray[Unknown, Unknown]`", + "concise_description": "Returned type `Tensor` is not assignable to declared return type `ndarray[Unknown, Unknown]`", + "severity": "error" + }, + { + "line": 37, + "column": 12, + "stop_line": 37, + "stop_column": 22, + "path": "src\\streamdiffusion\\image_utils.py", + "code": -2, + "name": "bad-return", + "description": "Returned type `list[Image]` is not assignable to declared return type `Image`", + "concise_description": "Returned type `list[Image]` is not assignable to declared return type `Image`", + "severity": "error" + }, + { + "line": 62, + "column": 13, + "stop_line": 62, + "stop_column": 31, + "path": "src\\streamdiffusion\\image_utils.py", + "code": -2, + "name": "bad-assignment", + "description": "`ndarray[Unknown, Unknown]` is not assignable to variable `image` with type `Tensor`", + "concise_description": "`ndarray[Unknown, Unknown]` is not assignable to variable `image` with type `Tensor`", + "severity": "error" + }, + { + "line": 68, + "column": 29, + "stop_line": 68, + "stop_column": 34, + "path": "src\\streamdiffusion\\image_utils.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Tensor` is not assignable to parameter `images` with type `ndarray[Unknown, Unknown]` in function `numpy_to_pil`", + "concise_description": "Argument `Tensor` is not assignable to parameter `images` with type `ndarray[Unknown, Unknown]` in function `numpy_to_pil`", + "severity": "error" + }, + { + "line": 10, + "column": 5, + "stop_line": 10, + "stop_column": 77, + "path": "src\\streamdiffusion\\model_detection.py", + "code": -2, + "name": "missing-import", + "description": "Cannot find module `diffusers.models.transformers.mm_dit`\n Looked in these locations (from config in `D:\\dev\\SDTD_032_dev\\StreamDiffusion\\pyproject.toml`):\n Search path (from config file): [\"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\src\"]\n Import root (inferred from project layout): \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\src\"\n Site package path queried from interpreter: [\"C:\\\\Users\\\\Inter\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python311\\\\DLLs\", \"C:\\\\Users\\\\Inter\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python311\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\venv\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\venv\\\\Lib\\\\site-packages\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\src\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\venv\\\\Lib\\\\site-packages\\\\win32\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\venv\\\\Lib\\\\site-packages\\\\win32\\\\lib\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\venv\\\\Lib\\\\site-packages\\\\Pythonwin\"]", + "concise_description": "Cannot find module `diffusers.models.transformers.mm_dit`", + "severity": "error" + }, + { + "line": 100, + "column": 12, + "stop_line": 100, + "stop_column": 22, + "path": "src\\streamdiffusion\\model_detection.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `Tensor` has no attribute `get`", + "concise_description": "Object of class `Tensor` has no attribute `get`", + "severity": "error" + }, + { + "line": 104, + "column": 16, + "stop_line": 104, + "stop_column": 26, + "path": "src\\streamdiffusion\\model_detection.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `Tensor` has no attribute `get`", + "concise_description": "Object of class `Tensor` has no attribute `get`", + "severity": "error" + }, + { + "line": 107, + "column": 35, + "stop_line": 107, + "stop_column": 45, + "path": "src\\streamdiffusion\\model_detection.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `Tensor` has no attribute `get`", + "concise_description": "Object of class `Tensor` has no attribute `get`", + "severity": "error" + }, + { + "line": 145, + "column": 30, + "stop_line": 145, + "stop_column": 40, + "path": "src\\streamdiffusion\\model_detection.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `Tensor` has no attribute `get`", + "concise_description": "Object of class `Tensor` has no attribute `get`", + "severity": "error" + }, + { + "line": 281, + "column": 22, + "stop_line": 281, + "stop_column": 50, + "path": "src\\streamdiffusion\\model_detection.py", + "code": -2, + "name": "bad-index", + "description": "Cannot index into `object`\n Object of class `object` has no attribute `__getitem__`", + "concise_description": "Cannot index into `object`", + "severity": "error" + }, + { + "line": 281, + "column": 54, + "stop_line": 281, + "stop_column": 79, + "path": "src\\streamdiffusion\\model_detection.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `dict` has no attribute `block_out_channels`", + "concise_description": "Object of class `dict` has no attribute `block_out_channels`", + "severity": "error" + }, + { + "line": 282, + "column": 32, + "stop_line": 282, + "stop_column": 57, + "path": "src\\streamdiffusion\\model_detection.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `object` is not assignable to parameter `iterable` with type `Iterable[@_]` in function `tuple.__new__`\n Protocol `Iterable` requires attribute `__iter__`", + "concise_description": "Argument `object` is not assignable to parameter `iterable` with type `Iterable[@_]` in function `tuple.__new__`", + "severity": "error" + }, + { + "line": 295, + "column": 19, + "stop_line": 295, + "stop_column": 45, + "path": "src\\streamdiffusion\\model_detection.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `dict` has no attribute `cross_attention_dim`", + "concise_description": "Object of class `dict` has no attribute `cross_attention_dim`", + "severity": "error" + }, + { + "line": 296, + "column": 19, + "stop_line": 296, + "stop_column": 37, + "path": "src\\streamdiffusion\\model_detection.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `dict` has no attribute `in_channels`", + "concise_description": "Object of class `dict` has no attribute `in_channels`", + "severity": "error" + }, + { + "line": 162, + "column": 57, + "stop_line": 162, + "stop_column": 75, + "path": "src\\streamdiffusion\\modules\\controlnet_module.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `width`", + "concise_description": "Object of class `NoneType` has no attribute `width`", + "severity": "error" + }, + { + "line": 163, + "column": 58, + "stop_line": 163, + "stop_column": 77, + "path": "src\\streamdiffusion\\modules\\controlnet_module.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `height`", + "concise_description": "Object of class `NoneType` has no attribute `height`", + "severity": "error" + }, + { + "line": 165, + "column": 21, + "stop_line": 165, + "stop_column": 40, + "path": "src\\streamdiffusion\\modules\\controlnet_module.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `BasePreprocessor` has no attribute `image_width`", + "concise_description": "Object of class `BasePreprocessor` has no attribute `image_width`", + "severity": "error" + }, + { + "line": 167, + "column": 21, + "stop_line": 167, + "stop_column": 41, + "path": "src\\streamdiffusion\\modules\\controlnet_module.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `BasePreprocessor` has no attribute `image_height`", + "concise_description": "Object of class `BasePreprocessor` has no attribute `image_height`", + "severity": "error" + }, + { + "line": 212, + "column": 60, + "stop_line": 212, + "stop_column": 78, + "path": "src\\streamdiffusion\\modules\\controlnet_module.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `width`", + "concise_description": "Object of class `NoneType` has no attribute `width`", + "severity": "error" + }, + { + "line": 212, + "column": 80, + "stop_line": 212, + "stop_column": 99, + "path": "src\\streamdiffusion\\modules\\controlnet_module.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `height`", + "concise_description": "Object of class `NoneType` has no attribute `height`", + "severity": "error" + }, + { + "line": 217, + "column": 21, + "stop_line": 217, + "stop_column": 50, + "path": "src\\streamdiffusion\\modules\\controlnet_module.py", + "code": -2, + "name": "unsupported-operation", + "description": "Cannot set item in `list[Tensor | None]`\n No matching overload found for function `list.__setitem__` called with arguments: (int, Tensor | tuple[Tensor, Tensor])\n Possible overloads:\n (key: SupportsIndex, value: Tensor | None, /) -> None [closest match]\n (key: slice[SupportsIndex | None, SupportsIndex | None, SupportsIndex | Unknown | None], value: Iterable[Tensor | None], /) -> None", + "concise_description": "Cannot set item in `list[Tensor | None]`", + "severity": "error" + }, + { + "line": 223, + "column": 48, + "stop_line": 223, + "stop_column": 59, + "path": "src\\streamdiffusion\\modules\\controlnet_module.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `str` is not assignable to parameter `device` with type `device` in function `ControlNetModule.prepare_frame_tensors`", + "concise_description": "Argument `str` is not assignable to parameter `device` with type `device` in function `ControlNetModule.prepare_frame_tensors`", + "severity": "error" + }, + { + "line": 228, + "column": 56, + "stop_line": 228, + "stop_column": 74, + "path": "src\\streamdiffusion\\modules\\controlnet_module.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `width`", + "concise_description": "Object of class `NoneType` has no attribute `width`", + "severity": "error" + }, + { + "line": 228, + "column": 76, + "stop_line": 228, + "stop_column": 95, + "path": "src\\streamdiffusion\\modules\\controlnet_module.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `height`", + "concise_description": "Object of class `NoneType` has no attribute `height`", + "severity": "error" + }, + { + "line": 245, + "column": 40, + "stop_line": 245, + "stop_column": 51, + "path": "src\\streamdiffusion\\modules\\controlnet_module.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `str` is not assignable to parameter `device` with type `device` in function `ControlNetModule.prepare_frame_tensors`", + "concise_description": "Argument `str` is not assignable to parameter `device` with type `device` in function `ControlNetModule.prepare_frame_tensors`", + "severity": "error" + }, + { + "line": 512, + "column": 37, + "stop_line": 512, + "stop_column": 63, + "path": "src\\streamdiffusion\\modules\\controlnet_module.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `prompt_embeds`", + "concise_description": "Object of class `NoneType` has no attribute `prompt_embeds`", + "severity": "error" + }, + { + "line": 547, + "column": 49, + "stop_line": 547, + "stop_column": 58, + "path": "src\\streamdiffusion\\modules\\controlnet_module.py", + "code": -2, + "name": "unsupported-operation", + "description": "Cannot set item in `dict[str, bool]`\n Argument `int` is not assignable to parameter `key` with type `str` in function `dict.__setitem__`", + "concise_description": "Cannot set item in `dict[str, bool]`", + "severity": "error" + }, + { + "line": 647, + "column": 17, + "stop_line": 647, + "stop_column": 42, + "path": "src\\streamdiffusion\\modules\\controlnet_module.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `copy_`", + "concise_description": "Object of class `NoneType` has no attribute `copy_`", + "severity": "error" + }, + { + "line": 652, + "column": 21, + "stop_line": 652, + "stop_column": 45, + "path": "src\\streamdiffusion\\modules\\controlnet_module.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `add_`", + "concise_description": "Object of class `NoneType` has no attribute `add_`", + "severity": "error" + }, + { + "line": 676, + "column": 51, + "stop_line": 676, + "stop_column": 69, + "path": "src\\streamdiffusion\\modules\\controlnet_module.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `width`", + "concise_description": "Object of class `NoneType` has no attribute `width`", + "severity": "error" + }, + { + "line": 676, + "column": 71, + "stop_line": 676, + "stop_column": 90, + "path": "src\\streamdiffusion\\modules\\controlnet_module.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `height`", + "concise_description": "Object of class `NoneType` has no attribute `height`", + "severity": "error" + }, + { + "line": 679, + "column": 16, + "stop_line": 679, + "stop_column": 45, + "path": "src\\streamdiffusion\\modules\\controlnet_module.py", + "code": -2, + "name": "bad-return", + "description": "Returned type `Tensor | tuple[Tensor, Tensor] | None` is not assignable to declared return type `Tensor`", + "concise_description": "Returned type `Tensor | tuple[Tensor, Tensor] | None` is not assignable to declared return type `Tensor`", + "severity": "error" + }, + { + "line": 694, + "column": 56, + "stop_line": 694, + "stop_column": 77, + "path": "src\\streamdiffusion\\modules\\controlnet_module.py", + "code": -2, + "name": "bad-typed-dict-key", + "description": "`int` is not assignable to TypedDict key `conditioning_channels` with type `dtype`", + "concise_description": "`int` is not assignable to TypedDict key `conditioning_channels` with type `dtype`", + "severity": "error" + }, + { + "line": 717, + "column": 59, + "stop_line": 717, + "stop_column": 63, + "path": "src\\streamdiffusion\\modules\\controlnet_module.py", + "code": -2, + "name": "bad-typed-dict-key", + "description": "`Literal[True]` is not assignable to TypedDict key `local_files_only` with type `dtype`", + "concise_description": "`Literal[True]` is not assignable to TypedDict key `local_files_only` with type `dtype`", + "severity": "error" + }, + { + "line": 721, + "column": 55, + "stop_line": 721, + "stop_column": 59, + "path": "src\\streamdiffusion\\modules\\controlnet_module.py", + "code": -2, + "name": "bad-typed-dict-key", + "description": "`Literal[True]` is not assignable to TypedDict key `local_files_only` with type `dtype`", + "concise_description": "`Literal[True]` is not assignable to TypedDict key `local_files_only` with type `dtype`", + "severity": "error" + }, + { + "line": 726, + "column": 55, + "stop_line": 726, + "stop_column": 59, + "path": "src\\streamdiffusion\\modules\\controlnet_module.py", + "code": -2, + "name": "bad-typed-dict-key", + "description": "`Literal[True]` is not assignable to TypedDict key `local_files_only` with type `dtype`", + "concise_description": "`Literal[True]` is not assignable to TypedDict key `local_files_only` with type `dtype`", + "severity": "error" + }, + { + "line": 741, + "column": 39, + "stop_line": 741, + "stop_column": 47, + "path": "src\\streamdiffusion\\modules\\controlnet_module.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `str` is not assignable to parameter `value` with type `Module | Tensor` in function `torch.nn.modules.module.Module.__setattr__`", + "concise_description": "Argument `str` is not assignable to parameter `value` with type `Module | Tensor` in function `torch.nn.modules.module.Module.__setattr__`", + "severity": "error" + }, + { + "line": 33, + "column": 16, + "stop_line": 33, + "stop_column": 71, + "path": "src\\streamdiffusion\\modules\\image_processing_module.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `execute_pipeline_chain`", + "concise_description": "Object of class `NoneType` has no attribute `execute_pipeline_chain`", + "severity": "error" + }, + { + "line": 68, + "column": 9, + "stop_line": 68, + "stop_column": 24, + "path": "src\\streamdiffusion\\modules\\image_processing_module.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `BasePreprocessor` has no attribute `order`", + "concise_description": "Object of class `BasePreprocessor` has no attribute `order`", + "severity": "error" + }, + { + "line": 71, + "column": 9, + "stop_line": 71, + "stop_column": 26, + "path": "src\\streamdiffusion\\modules\\image_processing_module.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `BasePreprocessor` has no attribute `enabled`", + "concise_description": "Object of class `BasePreprocessor` has no attribute `enabled`", + "severity": "error" + }, + { + "line": 80, + "column": 21, + "stop_line": 80, + "stop_column": 42, + "path": "src\\streamdiffusion\\modules\\image_processing_module.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `BasePreprocessor` has no attribute `image_width`", + "concise_description": "Object of class `BasePreprocessor` has no attribute `image_width`", + "severity": "error" + }, + { + "line": 82, + "column": 21, + "stop_line": 82, + "stop_column": 43, + "path": "src\\streamdiffusion\\modules\\image_processing_module.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `BasePreprocessor` has no attribute `image_height`", + "concise_description": "Object of class `BasePreprocessor` has no attribute `image_height`", + "severity": "error" + }, + { + "line": 131, + "column": 16, + "stop_line": 131, + "stop_column": 75, + "path": "src\\streamdiffusion\\modules\\image_processing_module.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `process_pipelined`", + "concise_description": "Object of class `NoneType` has no attribute `process_pipelined`", + "severity": "error" + }, + { + "line": 170, + "column": 16, + "stop_line": 170, + "stop_column": 67, + "path": "src\\streamdiffusion\\modules\\image_processing_module.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `process_pipelined`", + "concise_description": "Object of class `NoneType` has no attribute `process_pipelined`", + "severity": "error" + }, + { + "line": 369, + "column": 13, + "stop_line": 369, + "stop_column": 34, + "path": "src\\streamdiffusion\\modules\\ipadapter_module.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `IPAdapter` has no attribute `weight_type`", + "concise_description": "Object of class `IPAdapter` has no attribute `weight_type`", + "severity": "error" + }, + { + "line": 370, + "column": 13, + "stop_line": 370, + "stop_column": 28, + "path": "src\\streamdiffusion\\modules\\ipadapter_module.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `IPAdapter` has no attribute `scale`", + "concise_description": "Object of class `IPAdapter` has no attribute `scale`", + "severity": "error" + }, + { + "line": 371, + "column": 13, + "stop_line": 371, + "stop_column": 30, + "path": "src\\streamdiffusion\\modules\\ipadapter_module.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `IPAdapter` has no attribute `enabled`", + "concise_description": "Object of class `IPAdapter` has no attribute `enabled`", + "severity": "error" + }, + { + "line": 534, + "column": 49, + "stop_line": 534, + "stop_column": 57, + "path": "src\\streamdiffusion\\modules\\ipadapter_module.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Any | None` is not assignable to parameter `x` with type `Buffer | SupportsFloat | SupportsIndex | str` in function `float.__new__`\n Protocol `Buffer` requires attribute `__buffer__`", + "concise_description": "Argument `Any | None` is not assignable to parameter `x` with type `Buffer | SupportsFloat | SupportsIndex | str` in function `float.__new__`", + "severity": "error" + }, + { + "line": 33, + "column": 16, + "stop_line": 33, + "stop_column": 71, + "path": "src\\streamdiffusion\\modules\\latent_processing_module.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `execute_pipeline_chain`", + "concise_description": "Object of class `NoneType` has no attribute `execute_pipeline_chain`", + "severity": "error" + }, + { + "line": 50, + "column": 67, + "stop_line": 50, + "stop_column": 79, + "path": "src\\streamdiffusion\\modules\\latent_processing_module.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `LatentProcessingModule` has no attribute `_stream`", + "concise_description": "Object of class `LatentProcessingModule` has no attribute `_stream`", + "severity": "error" + }, + { + "line": 66, + "column": 9, + "stop_line": 66, + "stop_column": 24, + "path": "src\\streamdiffusion\\modules\\latent_processing_module.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `BasePreprocessor` has no attribute `order`", + "concise_description": "Object of class `BasePreprocessor` has no attribute `order`", + "severity": "error" + }, + { + "line": 69, + "column": 9, + "stop_line": 69, + "stop_column": 26, + "path": "src\\streamdiffusion\\modules\\latent_processing_module.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `BasePreprocessor` has no attribute `enabled`", + "concise_description": "Object of class `BasePreprocessor` has no attribute `enabled`", + "severity": "error" + }, + { + "line": 78, + "column": 44, + "stop_line": 78, + "stop_column": 65, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StableDiffusionPipeline` has no attribute `vae_scale_factor`", + "concise_description": "Object of class `StableDiffusionPipeline` has no attribute `vae_scale_factor`", + "severity": "error" + }, + { + "line": 79, + "column": 42, + "stop_line": 79, + "stop_column": 63, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StableDiffusionPipeline` has no attribute `vae_scale_factor`", + "concise_description": "Object of class `StableDiffusionPipeline` has no attribute `vae_scale_factor`", + "severity": "error" + }, + { + "line": 89, + "column": 41, + "stop_line": 89, + "stop_column": 50, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StableDiffusionPipeline` has no attribute `unet`", + "concise_description": "Object of class `StableDiffusionPipeline` has no attribute `unet`", + "severity": "error" + }, + { + "line": 137, + "column": 50, + "stop_line": 137, + "stop_column": 71, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StableDiffusionPipeline` has no attribute `vae_scale_factor`", + "concise_description": "Object of class `StableDiffusionPipeline` has no attribute `vae_scale_factor`", + "severity": "error" + }, + { + "line": 138, + "column": 73, + "stop_line": 138, + "stop_column": 87, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StableDiffusionPipeline` has no attribute `scheduler`", + "concise_description": "Object of class `StableDiffusionPipeline` has no attribute `scheduler`", + "severity": "error" + }, + { + "line": 140, + "column": 29, + "stop_line": 140, + "stop_column": 46, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StableDiffusionPipeline` has no attribute `text_encoder`", + "concise_description": "Object of class `StableDiffusionPipeline` has no attribute `text_encoder`", + "severity": "error" + }, + { + "line": 141, + "column": 21, + "stop_line": 141, + "stop_column": 30, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StableDiffusionPipeline` has no attribute `unet`", + "concise_description": "Object of class `StableDiffusionPipeline` has no attribute `unet`", + "severity": "error" + }, + { + "line": 142, + "column": 20, + "stop_line": 142, + "stop_column": 28, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StableDiffusionPipeline` has no attribute `vae`", + "concise_description": "Object of class `StableDiffusionPipeline` has no attribute `vae`", + "severity": "error" + }, + { + "line": 230, + "column": 54, + "stop_line": 230, + "stop_column": 57, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "bad-typed-dict-key", + "description": "`Literal[100]` is not assignable to TypedDict key `original_inference_steps` with type `str`", + "concise_description": "`Literal[100]` is not assignable to TypedDict key `original_inference_steps` with type `str`", + "severity": "error" + }, + { + "line": 230, + "column": 54, + "stop_line": 230, + "stop_column": 57, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "unsupported-operation", + "description": "Cannot set item in `dict[str, str]`\n Argument `Literal[100]` is not assignable to parameter `value` with type `str` in function `dict.__setitem__`", + "concise_description": "Cannot set item in `dict[str, str]`", + "severity": "error" + }, + { + "line": 277, + "column": 20, + "stop_line": 280, + "stop_column": 14, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "bad-return", + "description": "Returned type `dict[str, Tensor | None]` is not assignable to declared return type `dict[str, Tensor] | None`", + "concise_description": "Returned type `dict[str, Tensor | None]` is not assignable to declared return type `dict[str, Tensor] | None`", + "severity": "error" + }, + { + "line": 299, + "column": 27, + "stop_line": 299, + "stop_column": 52, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "unsupported-operation", + "description": "`None` is not subscriptable", + "concise_description": "`None` is not subscriptable", + "severity": "error" + }, + { + "line": 300, + "column": 25, + "stop_line": 300, + "stop_column": 50, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "unsupported-operation", + "description": "`None` is not subscriptable", + "concise_description": "`None` is not subscriptable", + "severity": "error" + }, + { + "line": 301, + "column": 27, + "stop_line": 301, + "stop_column": 49, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "unsupported-operation", + "description": "`None` is not subscriptable", + "concise_description": "`None` is not subscriptable", + "severity": "error" + }, + { + "line": 302, + "column": 25, + "stop_line": 302, + "stop_column": 47, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "unsupported-operation", + "description": "`None` is not subscriptable", + "concise_description": "`None` is not subscriptable", + "severity": "error" + }, + { + "line": 316, + "column": 31, + "stop_line": 316, + "stop_column": 58, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `expand`", + "concise_description": "Object of class `NoneType` has no attribute `expand`", + "severity": "error" + }, + { + "line": 317, + "column": 28, + "stop_line": 317, + "stop_column": 52, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `expand`", + "concise_description": "Object of class `NoneType` has no attribute `expand`", + "severity": "error" + }, + { + "line": 320, + "column": 27, + "stop_line": 320, + "stop_column": 52, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "unsupported-operation", + "description": "`None` is not subscriptable", + "concise_description": "`None` is not subscriptable", + "severity": "error" + }, + { + "line": 320, + "column": 56, + "stop_line": 320, + "stop_column": 82, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `shape`", + "concise_description": "Object of class `NoneType` has no attribute `shape`", + "severity": "error" + }, + { + "line": 321, + "column": 27, + "stop_line": 321, + "stop_column": 49, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "unsupported-operation", + "description": "`None` is not subscriptable", + "concise_description": "`None` is not subscriptable", + "severity": "error" + }, + { + "line": 321, + "column": 53, + "stop_line": 321, + "stop_column": 76, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `shape`", + "concise_description": "Object of class `NoneType` has no attribute `shape`", + "severity": "error" + }, + { + "line": 322, + "column": 31, + "stop_line": 322, + "stop_column": 49, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `expand`", + "concise_description": "Object of class `NoneType` has no attribute `expand`", + "severity": "error" + }, + { + "line": 323, + "column": 28, + "stop_line": 323, + "stop_column": 46, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `expand`", + "concise_description": "Object of class `NoneType` has no attribute `expand`", + "severity": "error" + }, + { + "line": 344, + "column": 13, + "stop_line": 344, + "stop_column": 40, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StableDiffusionPipeline` has no attribute `load_lora_weights`", + "concise_description": "Object of class `StableDiffusionPipeline` has no attribute `load_lora_weights`", + "severity": "error" + }, + { + "line": 363, + "column": 17, + "stop_line": 363, + "stop_column": 44, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StableDiffusionPipeline` has no attribute `load_lora_weights`", + "concise_description": "Object of class `StableDiffusionPipeline` has no attribute `load_lora_weights`", + "severity": "error" + }, + { + "line": 384, + "column": 9, + "stop_line": 384, + "stop_column": 28, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StableDiffusionPipeline` has no attribute `fuse_lora`", + "concise_description": "Object of class `StableDiffusionPipeline` has no attribute `fuse_lora`", + "severity": "error" + }, + { + "line": 435, + "column": 38, + "stop_line": 435, + "stop_column": 42, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "bad-assignment", + "description": "`None` is not assignable to attribute `x_t_latent_buffer` with type `Tensor`", + "concise_description": "`None` is not assignable to attribute `x_t_latent_buffer` with type `Tensor`", + "severity": "error" + }, + { + "line": 451, + "column": 30, + "stop_line": 451, + "stop_column": 53, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StableDiffusionPipeline` has no attribute `encode_prompt`", + "concise_description": "Object of class `StableDiffusionPipeline` has no attribute `encode_prompt`", + "severity": "error" + }, + { + "line": 482, + "column": 53, + "stop_line": 482, + "stop_column": 73, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "unbound-name", + "description": "`uncond_prompt_embeds` may be uninitialized", + "concise_description": "`uncond_prompt_embeds` may be uninitialized", + "severity": "error" + }, + { + "line": 515, + "column": 30, + "stop_line": 515, + "stop_column": 53, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StableDiffusionPipeline` has no attribute `encode_prompt`", + "concise_description": "Object of class `StableDiffusionPipeline` has no attribute `encode_prompt`", + "severity": "error" + }, + { + "line": 530, + "column": 49, + "stop_line": 530, + "stop_column": 69, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "unbound-name", + "description": "`uncond_prompt_embeds` may be uninitialized", + "concise_description": "`uncond_prompt_embeds` may be uninitialized", + "severity": "error" + }, + { + "line": 643, + "column": 44, + "stop_line": 643, + "stop_column": 48, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "bad-assignment", + "description": "`None` is not assignable to attribute `_sub_timesteps_expanded` with type `Tensor`", + "concise_description": "`None` is not assignable to attribute `_sub_timesteps_expanded` with type `Tensor`", + "severity": "error" + }, + { + "line": 739, + "column": 29, + "stop_line": 739, + "stop_column": 88, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `LCMScheduler` has no attribute `get_scalings_for_boundary_condition_discrete`", + "concise_description": "Object of class `LCMScheduler` has no attribute `get_scalings_for_boundary_condition_discrete`", + "severity": "error" + }, + { + "line": 763, + "column": 44, + "stop_line": 763, + "stop_column": 48, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "bad-function-definition", + "description": "Default `None` is not assignable to parameter `scheduler` with type `Literal['lcm', 'tcd']`", + "concise_description": "Default `None` is not assignable to parameter `scheduler` with type `Literal['lcm', 'tcd']`", + "severity": "error" + }, + { + "line": 764, + "column": 89, + "stop_line": 764, + "stop_column": 93, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "bad-function-definition", + "description": "Default `None` is not assignable to parameter `sampler` with type `Literal['beta', 'ddim', 'karras', 'normal', 'sgm_uniform', 'simple']`", + "concise_description": "Default `None` is not assignable to parameter `sampler` with type `Literal['beta', 'ddim', 'karras', 'normal', 'sgm_uniform', 'simple']`", + "severity": "error" + }, + { + "line": 781, + "column": 93, + "stop_line": 781, + "stop_column": 112, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StableDiffusionPipeline` has no attribute `scheduler`", + "concise_description": "Object of class `StableDiffusionPipeline` has no attribute `scheduler`", + "severity": "error" + }, + { + "line": 829, + "column": 17, + "stop_line": 829, + "stop_column": 41, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "unsupported-operation", + "description": "`None` is not subscriptable", + "concise_description": "`None` is not subscriptable", + "severity": "error" + }, + { + "line": 830, + "column": 17, + "stop_line": 830, + "stop_column": 41, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "unsupported-operation", + "description": "`None` is not subscriptable", + "concise_description": "`None` is not subscriptable", + "severity": "error" + }, + { + "line": 832, + "column": 17, + "stop_line": 832, + "stop_column": 36, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "unsupported-operation", + "description": "`None` is not subscriptable", + "concise_description": "`None` is not subscriptable", + "severity": "error" + }, + { + "line": 833, + "column": 17, + "stop_line": 833, + "stop_column": 36, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "unsupported-operation", + "description": "`None` is not subscriptable", + "concise_description": "`None` is not subscriptable", + "severity": "error" + }, + { + "line": 834, + "column": 26, + "stop_line": 834, + "stop_column": 41, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "bad-assignment", + "description": "`Unknown | None` is not assignable to variable `t_list` with type `Tensor | list[int]`", + "concise_description": "`Unknown | None` is not assignable to variable `t_list` with type `Tensor | list[int]`", + "severity": "error" + }, + { + "line": 836, + "column": 17, + "stop_line": 836, + "stop_column": 56, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "unsupported-operation", + "description": "`None` is not subscriptable", + "concise_description": "`None` is not subscriptable", + "severity": "error" + }, + { + "line": 837, + "column": 17, + "stop_line": 837, + "stop_column": 56, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "unsupported-operation", + "description": "`None` is not subscriptable", + "concise_description": "`None` is not subscriptable", + "severity": "error" + }, + { + "line": 839, + "column": 17, + "stop_line": 839, + "stop_column": 47, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "unsupported-operation", + "description": "`None` is not subscriptable", + "concise_description": "`None` is not subscriptable", + "severity": "error" + }, + { + "line": 840, + "column": 17, + "stop_line": 840, + "stop_column": 47, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "unsupported-operation", + "description": "`None` is not subscriptable", + "concise_description": "`None` is not subscriptable", + "severity": "error" + }, + { + "line": 841, + "column": 26, + "stop_line": 841, + "stop_column": 41, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "bad-assignment", + "description": "`Unknown | None` is not assignable to variable `t_list` with type `Tensor | list[int]`", + "concise_description": "`Unknown | None` is not assignable to variable `t_list` with type `Tensor | list[int]`", + "severity": "error" + }, + { + "line": 856, + "column": 34, + "stop_line": 856, + "stop_column": 58, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `shape`", + "concise_description": "Object of class `NoneType` has no attribute `shape`", + "severity": "error" + }, + { + "line": 883, + "column": 36, + "stop_line": 883, + "stop_column": 54, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Tensor | Unknown | None` is not assignable to parameter `x_t_latent` with type `Tensor` in function `streamdiffusion.hooks.StepCtx.__init__`", + "concise_description": "Argument `Tensor | Unknown | None` is not assignable to parameter `x_t_latent` with type `Tensor` in function `streamdiffusion.hooks.StepCtx.__init__`", + "severity": "error" + }, + { + "line": 884, + "column": 32, + "stop_line": 884, + "stop_column": 38, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Tensor | list[int]` is not assignable to parameter `t_list` with type `Tensor` in function `streamdiffusion.hooks.StepCtx.__init__`", + "concise_description": "Argument `Tensor | list[int]` is not assignable to parameter `t_list` with type `Tensor` in function `streamdiffusion.hooks.StepCtx.__init__`", + "severity": "error" + }, + { + "line": 1029, + "column": 30, + "stop_line": 1029, + "stop_column": 47, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "unbound-name", + "description": "`noise_pred_uncond` may be uninitialized", + "concise_description": "`noise_pred_uncond` may be uninitialized", + "severity": "error" + }, + { + "line": 1040, + "column": 27, + "stop_line": 1040, + "stop_column": 53, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "unsupported-operation", + "description": "`*` is not supported between `None` and `Tensor`\n Argument `None` is not assignable to parameter `other` with type `Tensor | bool | complex | float | int` in function `torch._C.TensorBase.__rmul__`", + "concise_description": "`*` is not supported between `None` and `Tensor`", + "severity": "error" + }, + { + "line": 1041, + "column": 27, + "stop_line": 1041, + "stop_column": 52, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "unsupported-operation", + "description": "`/` is not supported between `Tensor` and `None`\n Argument `None` is not assignable to parameter `other` with type `Tensor | bool | complex | float | int` in function `torch._C.TensorBase.__truediv__`", + "concise_description": "`/` is not supported between `Tensor` and `None`", + "severity": "error" + }, + { + "line": 1042, + "column": 36, + "stop_line": 1042, + "stop_column": 70, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "unsupported-operation", + "description": "`+` is not supported between `None` and `Tensor`\n Argument `None` is not assignable to parameter `other` with type `Tensor | bool | complex | float | int` in function `torch._C.TensorBase.__radd__`", + "concise_description": "`+` is not supported between `None` and `Tensor`", + "severity": "error" + }, + { + "line": 1076, + "column": 61, + "stop_line": 1076, + "stop_column": 88, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Unknown | None` is not assignable to parameter `iterable` with type `Iterable[Unknown]` in function `enumerate.__new__`\n Protocol `Iterable` requires attribute `__iter__`", + "concise_description": "Argument `Unknown | None` is not assignable to parameter `iterable` with type `Iterable[Unknown]` in function `enumerate.__new__`", + "severity": "error" + }, + { + "line": 1098, + "column": 18, + "stop_line": 1098, + "stop_column": 58, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "bad-context-manager", + "description": "Cannot use `autocast` as a context manager\n Return type `Literal[False] | object | None` of function `autocast.__exit__` is not assignable to expected return type `bool | None`", + "concise_description": "Cannot use `autocast` as a context manager", + "severity": "error" + }, + { + "line": 1109, + "column": 18, + "stop_line": 1109, + "stop_column": 58, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "bad-context-manager", + "description": "Cannot use `autocast` as a context manager\n Return type `Literal[False] | object | None` of function `autocast.__exit__` is not assignable to expected return type `bool | None`", + "concise_description": "Cannot use `autocast` as a context manager", + "severity": "error" + }, + { + "line": 1123, + "column": 21, + "stop_line": 1123, + "stop_column": 69, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "unsupported-operation", + "description": "`None` is not subscriptable", + "concise_description": "`None` is not subscriptable", + "severity": "error" + }, + { + "line": 1124, + "column": 21, + "stop_line": 1124, + "stop_column": 69, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "unsupported-operation", + "description": "`None` is not subscriptable", + "concise_description": "`None` is not subscriptable", + "severity": "error" + }, + { + "line": 1125, + "column": 34, + "stop_line": 1125, + "stop_column": 59, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "bad-assignment", + "description": "`Unknown | None` is not assignable to variable `x_t_latent` with type `Tensor`", + "concise_description": "`Unknown | None` is not assignable to variable `x_t_latent` with type `Tensor`", + "severity": "error" + }, + { + "line": 1149, + "column": 46, + "stop_line": 1149, + "stop_column": 50, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "bad-assignment", + "description": "`None` is not assignable to attribute `x_t_latent_buffer` with type `Tensor`", + "concise_description": "`None` is not assignable to attribute `x_t_latent_buffer` with type `Tensor`", + "severity": "error" + }, + { + "line": 1171, + "column": 67, + "stop_line": 1171, + "stop_column": 73, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Tensor | Unknown | None` is not assignable to parameter `x_t_latent` with type `Tensor` in function `StreamDiffusion.unet_step`", + "concise_description": "Argument `Tensor | Unknown | None` is not assignable to parameter `x_t_latent` with type `Tensor` in function `StreamDiffusion.unet_step`", + "severity": "error" + }, + { + "line": 1175, + "column": 40, + "stop_line": 1175, + "stop_column": 59, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `TCDScheduler` has no attribute `step`", + "concise_description": "Object of class `TCDScheduler` has no attribute `step`", + "severity": "error" + }, + { + "line": 1182, + "column": 67, + "stop_line": 1182, + "stop_column": 73, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Tensor | Unknown | None` is not assignable to parameter `x_t_latent` with type `Tensor` in function `StreamDiffusion.unet_step`", + "concise_description": "Argument `Tensor | Unknown | None` is not assignable to parameter `x_t_latent` with type `Tensor` in function `StreamDiffusion.unet_step`", + "severity": "error" + }, + { + "line": 1198, + "column": 20, + "stop_line": 1198, + "stop_column": 32, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "bad-return", + "description": "Returned type `Tensor | Unknown | None` is not assignable to declared return type `Tensor`", + "concise_description": "Returned type `Tensor | Unknown | None` is not assignable to declared return type `Tensor`", + "severity": "error" + }, + { + "line": 1232, + "column": 28, + "stop_line": 1232, + "stop_column": 50, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "bad-return", + "description": "Returned type `Unknown | None` is not assignable to declared return type `Tensor`", + "concise_description": "Returned type `Unknown | None` is not assignable to declared return type `Tensor`", + "severity": "error" + }, + { + "line": 1306, + "column": 43, + "stop_line": 1306, + "stop_column": 95, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "bad-assignment", + "description": "`float` is not assignable to attribute `inference_time_ema` with type `int`", + "concise_description": "`float` is not assignable to attribute `inference_time_ema` with type `int`", + "severity": "error" + }, + { + "line": 1441, + "column": 52, + "stop_line": 1441, + "stop_column": 110, + "path": "src\\streamdiffusion\\pipeline.py", + "code": -2, + "name": "bad-typed-dict-key", + "description": "`dict[str, Unknown]` is not assignable to TypedDict key `added_cond_kwargs` with type `Tensor | bool`", + "concise_description": "`dict[str, Unknown]` is not assignable to TypedDict key `added_cond_kwargs` with type `Tensor | bool`", + "severity": "error" + }, + { + "line": 133, + "column": 42, + "stop_line": 133, + "stop_column": 57, + "path": "src\\streamdiffusion\\preprocessing\\pipeline_preprocessing_orchestrator.py", + "code": -2, + "name": "unbound-name", + "description": "`original_stream` may be uninitialized", + "concise_description": "`original_stream` may be uninitialized", + "severity": "error" + }, + { + "line": 166, + "column": 42, + "stop_line": 166, + "stop_column": 57, + "path": "src\\streamdiffusion\\preprocessing\\postprocessing_orchestrator.py", + "code": -2, + "name": "unbound-name", + "description": "`original_stream` may be uninitialized", + "concise_description": "`original_stream` may be uninitialized", + "severity": "error" + }, + { + "line": 47, + "column": 9, + "stop_line": 47, + "stop_column": 21, + "path": "src\\streamdiffusion\\preprocessing\\preprocessing_orchestrator.py", + "code": -2, + "name": "bad-override", + "description": "Class member `PreprocessingOrchestrator.process_sync` overrides parent class `BaseOrchestrator` in an inconsistent manner\n `PreprocessingOrchestrator.process_sync` has type `(self: PreprocessingOrchestrator, control_image: ControlImage, preprocessors: list[Any | None], scales: list[float] = ..., stream_width: int = ..., stream_height: int = ..., index: int | None = None, processing_type: str = 'controlnet') -> list[Tensor | None] | list[tuple[Tensor, Tensor]]`, which is not assignable to `(self: PreprocessingOrchestrator, input_data: ControlImage, *args: Unknown, **kwargs: Unknown) -> list[Tensor | None]`, the type of `BaseOrchestrator.process_sync`\n Signature mismatch:\n ...r, input_data: ControlImage, *args: Unknown, **kwargs: Unknown) -> list[Tensor | None]: ...\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ parameters ^ return type\n ...r, control_image: ControlImage, preprocessors: list[Any | None], scales: list[float] = ..., stream_width: int = ..., stream_height: int = ..., index: int | None = None, processing_type: str = 'controlnet') -> list[Tensor | None] | list[tuple[Tensor, Tensor]]: ...\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ parameters ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ return type", + "concise_description": "Class member `PreprocessingOrchestrator.process_sync` overrides parent class `BaseOrchestrator` in an inconsistent manner", + "severity": "error" + }, + { + "line": 51, + "column": 31, + "stop_line": 51, + "stop_column": 35, + "path": "src\\streamdiffusion\\preprocessing\\preprocessing_orchestrator.py", + "code": -2, + "name": "bad-function-definition", + "description": "Default `None` is not assignable to parameter `scales` with type `list[float]`", + "concise_description": "Default `None` is not assignable to parameter `scales` with type `list[float]`", + "severity": "error" + }, + { + "line": 52, + "column": 29, + "stop_line": 52, + "stop_column": 33, + "path": "src\\streamdiffusion\\preprocessing\\preprocessing_orchestrator.py", + "code": -2, + "name": "bad-function-definition", + "description": "Default `None` is not assignable to parameter `stream_width` with type `int`", + "concise_description": "Default `None` is not assignable to parameter `stream_width` with type `int`", + "severity": "error" + }, + { + "line": 53, + "column": 30, + "stop_line": 53, + "stop_column": 34, + "path": "src\\streamdiffusion\\preprocessing\\preprocessing_orchestrator.py", + "code": -2, + "name": "bad-function-definition", + "description": "Default `None` is not assignable to parameter `stream_height` with type `int`", + "concise_description": "Default `None` is not assignable to parameter `stream_height` with type `int`", + "severity": "error" + }, + { + "line": 208, + "column": 42, + "stop_line": 208, + "stop_column": 57, + "path": "src\\streamdiffusion\\preprocessing\\preprocessing_orchestrator.py", + "code": -2, + "name": "unbound-name", + "description": "`original_stream` may be uninitialized", + "concise_description": "`original_stream` may be uninitialized", + "severity": "error" + }, + { + "line": 211, + "column": 52, + "stop_line": 211, + "stop_column": 56, + "path": "src\\streamdiffusion\\preprocessing\\preprocessing_orchestrator.py", + "code": -2, + "name": "bad-function-definition", + "description": "Default `None` is not assignable to parameter `preprocessors` with type `list[Any | None]`", + "concise_description": "Default `None` is not assignable to parameter `preprocessors` with type `list[Any | None]`", + "severity": "error" + }, + { + "line": 211, + "column": 80, + "stop_line": 211, + "stop_column": 84, + "path": "src\\streamdiffusion\\preprocessing\\preprocessing_orchestrator.py", + "code": -2, + "name": "bad-function-definition", + "description": "Default `None` is not assignable to parameter `scales` with type `list[float]`", + "concise_description": "Default `None` is not assignable to parameter `scales` with type `list[float]`", + "severity": "error" + }, + { + "line": 340, + "column": 9, + "stop_line": 340, + "stop_column": 32, + "path": "src\\streamdiffusion\\preprocessing\\preprocessing_orchestrator.py", + "code": -2, + "name": "unsupported-operation", + "description": "Cannot set item in `list[None]`\n No matching overload found for function `list.__setitem__` called with arguments: (int, Tensor)\n Possible overloads:\n (key: SupportsIndex, value: None, /) -> None [closest match]\n (key: slice[SupportsIndex | None, SupportsIndex | None, SupportsIndex | Unknown | None], value: Iterable[None], /) -> None", + "concise_description": "Cannot set item in `list[None]`", + "severity": "error" + }, + { + "line": 342, + "column": 16, + "stop_line": 342, + "stop_column": 32, + "path": "src\\streamdiffusion\\preprocessing\\preprocessing_orchestrator.py", + "code": -2, + "name": "bad-return", + "description": "Returned type `list[None]` is not assignable to declared return type `list[Tensor | None]`", + "concise_description": "Returned type `list[None]` is not assignable to declared return type `list[Tensor | None]`", + "severity": "error" + }, + { + "line": 375, + "column": 16, + "stop_line": 375, + "stop_column": 32, + "path": "src\\streamdiffusion\\preprocessing\\preprocessing_orchestrator.py", + "code": -2, + "name": "bad-return", + "description": "Returned type `list[None]` is not assignable to declared return type `list[Tensor | None]`", + "concise_description": "Returned type `list[None]` is not assignable to declared return type `list[Tensor | None]`", + "severity": "error" + }, + { + "line": 426, + "column": 16, + "stop_line": 426, + "stop_column": 23, + "path": "src\\streamdiffusion\\preprocessing\\preprocessing_orchestrator.py", + "code": -2, + "name": "bad-return", + "description": "Returned type `list[None]` is not assignable to declared return type `list[tuple[Tensor, Tensor]]`", + "concise_description": "Returned type `list[None]` is not assignable to declared return type `list[tuple[Tensor, Tensor]]`", + "severity": "error" + }, + { + "line": 477, + "column": 17, + "stop_line": 477, + "stop_column": 77, + "path": "src\\streamdiffusion\\preprocessing\\preprocessing_orchestrator.py", + "code": -2, + "name": "missing-import", + "description": "Cannot find module `streamdiffusion.preprocessing.processors.temporal_net`\n Looked in these locations (from config in `D:\\dev\\SDTD_032_dev\\StreamDiffusion\\pyproject.toml`):\n Search path (from config file): [\"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\src\"]\n Import root (inferred from project layout): \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\src\"\n Site package path queried from interpreter: [\"C:\\\\Users\\\\Inter\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python311\\\\DLLs\", \"C:\\\\Users\\\\Inter\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python311\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\venv\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\venv\\\\Lib\\\\site-packages\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\src\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\venv\\\\Lib\\\\site-packages\\\\win32\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\venv\\\\Lib\\\\site-packages\\\\win32\\\\lib\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\venv\\\\Lib\\\\site-packages\\\\Pythonwin\"]", + "concise_description": "Cannot find module `streamdiffusion.preprocessing.processors.temporal_net`", + "severity": "error" + }, + { + "line": 564, + "column": 69, + "stop_line": 564, + "stop_column": 88, + "path": "src\\streamdiffusion\\preprocessing\\preprocessing_orchestrator.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `device` is not assignable to parameter `device` with type `str` in function `PreprocessingOrchestrator._pil_to_tensor_safe`", + "concise_description": "Argument `device` is not assignable to parameter `device` with type `str` in function `PreprocessingOrchestrator._pil_to_tensor_safe`", + "severity": "error" + }, + { + "line": 705, + "column": 29, + "stop_line": 705, + "stop_column": 33, + "path": "src\\streamdiffusion\\preprocessing\\preprocessing_orchestrator.py", + "code": -2, + "name": "bad-function-definition", + "description": "Default `None` is not assignable to parameter `stream_width` with type `int`", + "concise_description": "Default `None` is not assignable to parameter `stream_width` with type `int`", + "severity": "error" + }, + { + "line": 706, + "column": 30, + "stop_line": 706, + "stop_column": 34, + "path": "src\\streamdiffusion\\preprocessing\\preprocessing_orchestrator.py", + "code": -2, + "name": "bad-function-definition", + "description": "Default `None` is not assignable to parameter `stream_height` with type `int`", + "concise_description": "Default `None` is not assignable to parameter `stream_height` with type `int`", + "severity": "error" + }, + { + "line": 110, + "column": 48, + "stop_line": 110, + "stop_column": 81, + "path": "src\\streamdiffusion\\preprocessing\\processors\\__init__.py", + "code": -2, + "name": "bad-typed-dict-key", + "description": "`type[DepthAnythingTensorrtPreprocessor] | None` is not assignable to TypedDict key `depth_tensorrt` with type `type[BlurPreprocessor] | type[CannyPreprocessor] | type[DepthPreprocessor] | type[ExternalPreprocessor] | type[FeedbackPreprocessor] | type[HEDPreprocessor] | type[LatentFeedbackPreprocessor] | type[LineartPreprocessor] | type[OpenPosePreprocessor] | type[PassthroughPreprocessor] | type[RealESRGANProcessor] | type[ScribblePreprocessor] | type[SharpenPreprocessor] | type[SoftEdgePreprocessor] | type[StandardLineartPreprocessor] | type[UpscalePreprocessor]`", + "concise_description": "`type[DepthAnythingTensorrtPreprocessor] | None` is not assignable to TypedDict key `depth_tensorrt` with type `type[BlurPreprocessor] | type[CannyPreprocessor] | type[DepthPreprocessor] | type[ExternalPreprocessor] | type[FeedbackPreprocessor] | type[HEDPreprocessor] | type[LatentFeedbackPreprocessor] | type[LineartPreprocessor] | type[OpenPosePreprocessor] | type[PassthroughPreprocessor] | type[RealESRGANProcessor] | type[ScribblePreprocessor] | type[SharpenPreprocessor] | type[SoftEdgePreprocessor] | type[StandardLineartPreprocessor] | type[UpscalePreprocessor]`", + "severity": "error" + }, + { + "line": 113, + "column": 47, + "stop_line": 113, + "stop_column": 78, + "path": "src\\streamdiffusion\\preprocessing\\processors\\__init__.py", + "code": -2, + "name": "bad-typed-dict-key", + "description": "`type[YoloNasPoseTensorrtPreprocessor] | None` is not assignable to TypedDict key `pose_tensorrt` with type `type[BlurPreprocessor] | type[CannyPreprocessor] | type[DepthPreprocessor] | type[ExternalPreprocessor] | type[FeedbackPreprocessor] | type[HEDPreprocessor] | type[LatentFeedbackPreprocessor] | type[LineartPreprocessor] | type[OpenPosePreprocessor] | type[PassthroughPreprocessor] | type[RealESRGANProcessor] | type[ScribblePreprocessor] | type[SharpenPreprocessor] | type[SoftEdgePreprocessor] | type[StandardLineartPreprocessor] | type[UpscalePreprocessor]`", + "concise_description": "`type[YoloNasPoseTensorrtPreprocessor] | None` is not assignable to TypedDict key `pose_tensorrt` with type `type[BlurPreprocessor] | type[CannyPreprocessor] | type[DepthPreprocessor] | type[ExternalPreprocessor] | type[FeedbackPreprocessor] | type[HEDPreprocessor] | type[LatentFeedbackPreprocessor] | type[LineartPreprocessor] | type[OpenPosePreprocessor] | type[PassthroughPreprocessor] | type[RealESRGANProcessor] | type[ScribblePreprocessor] | type[SharpenPreprocessor] | type[SoftEdgePreprocessor] | type[StandardLineartPreprocessor] | type[UpscalePreprocessor]`", + "severity": "error" + }, + { + "line": 116, + "column": 55, + "stop_line": 116, + "stop_column": 86, + "path": "src\\streamdiffusion\\preprocessing\\processors\\__init__.py", + "code": -2, + "name": "bad-typed-dict-key", + "description": "`type[TemporalNetTensorRTPreprocessor] | None` is not assignable to TypedDict key `temporal_net_tensorrt` with type `type[BlurPreprocessor] | type[CannyPreprocessor] | type[DepthPreprocessor] | type[ExternalPreprocessor] | type[FeedbackPreprocessor] | type[HEDPreprocessor] | type[LatentFeedbackPreprocessor] | type[LineartPreprocessor] | type[OpenPosePreprocessor] | type[PassthroughPreprocessor] | type[RealESRGANProcessor] | type[ScribblePreprocessor] | type[SharpenPreprocessor] | type[SoftEdgePreprocessor] | type[StandardLineartPreprocessor] | type[UpscalePreprocessor]`", + "concise_description": "`type[TemporalNetTensorRTPreprocessor] | None` is not assignable to TypedDict key `temporal_net_tensorrt` with type `type[BlurPreprocessor] | type[CannyPreprocessor] | type[DepthPreprocessor] | type[ExternalPreprocessor] | type[FeedbackPreprocessor] | type[HEDPreprocessor] | type[LatentFeedbackPreprocessor] | type[LineartPreprocessor] | type[OpenPosePreprocessor] | type[PassthroughPreprocessor] | type[RealESRGANProcessor] | type[ScribblePreprocessor] | type[SharpenPreprocessor] | type[SoftEdgePreprocessor] | type[StandardLineartPreprocessor] | type[UpscalePreprocessor]`", + "severity": "error" + }, + { + "line": 120, + "column": 48, + "stop_line": 120, + "stop_column": 73, + "path": "src\\streamdiffusion\\preprocessing\\processors\\__init__.py", + "code": -2, + "name": "bad-typed-dict-key", + "description": "`type[MediaPipePosePreprocessor] | None` is not assignable to TypedDict key `mediapipe_pose` with type `type[BlurPreprocessor] | type[CannyPreprocessor] | type[DepthPreprocessor] | type[ExternalPreprocessor] | type[FeedbackPreprocessor] | type[HEDPreprocessor] | type[LatentFeedbackPreprocessor] | type[LineartPreprocessor] | type[OpenPosePreprocessor] | type[PassthroughPreprocessor] | type[RealESRGANProcessor] | type[ScribblePreprocessor] | type[SharpenPreprocessor] | type[SoftEdgePreprocessor] | type[StandardLineartPreprocessor] | type[UpscalePreprocessor]`", + "concise_description": "`type[MediaPipePosePreprocessor] | None` is not assignable to TypedDict key `mediapipe_pose` with type `type[BlurPreprocessor] | type[CannyPreprocessor] | type[DepthPreprocessor] | type[ExternalPreprocessor] | type[FeedbackPreprocessor] | type[HEDPreprocessor] | type[LatentFeedbackPreprocessor] | type[LineartPreprocessor] | type[OpenPosePreprocessor] | type[PassthroughPreprocessor] | type[RealESRGANProcessor] | type[ScribblePreprocessor] | type[SharpenPreprocessor] | type[SoftEdgePreprocessor] | type[StandardLineartPreprocessor] | type[UpscalePreprocessor]`", + "severity": "error" + }, + { + "line": 123, + "column": 56, + "stop_line": 123, + "stop_column": 89, + "path": "src\\streamdiffusion\\preprocessing\\processors\\__init__.py", + "code": -2, + "name": "bad-typed-dict-key", + "description": "`type[MediaPipeSegmentationPreprocessor] | None` is not assignable to TypedDict key `mediapipe_segmentation` with type `type[BlurPreprocessor] | type[CannyPreprocessor] | type[DepthPreprocessor] | type[ExternalPreprocessor] | type[FeedbackPreprocessor] | type[HEDPreprocessor] | type[LatentFeedbackPreprocessor] | type[LineartPreprocessor] | type[OpenPosePreprocessor] | type[PassthroughPreprocessor] | type[RealESRGANProcessor] | type[ScribblePreprocessor] | type[SharpenPreprocessor] | type[SoftEdgePreprocessor] | type[StandardLineartPreprocessor] | type[UpscalePreprocessor]`", + "concise_description": "`type[MediaPipeSegmentationPreprocessor] | None` is not assignable to TypedDict key `mediapipe_segmentation` with type `type[BlurPreprocessor] | type[CannyPreprocessor] | type[DepthPreprocessor] | type[ExternalPreprocessor] | type[FeedbackPreprocessor] | type[HEDPreprocessor] | type[LatentFeedbackPreprocessor] | type[LineartPreprocessor] | type[OpenPosePreprocessor] | type[PassthroughPreprocessor] | type[RealESRGANProcessor] | type[ScribblePreprocessor] | type[SharpenPreprocessor] | type[SoftEdgePreprocessor] | type[StandardLineartPreprocessor] | type[UpscalePreprocessor]`", + "severity": "error" + }, + { + "line": 127, + "column": 46, + "stop_line": 127, + "stop_column": 69, + "path": "src\\streamdiffusion\\preprocessing\\processors\\__init__.py", + "code": -2, + "name": "bad-typed-dict-key", + "description": "`type[HEDTensorrtPreprocessor] | None` is not assignable to TypedDict key `hed_tensorrt` with type `type[BlurPreprocessor] | type[CannyPreprocessor] | type[DepthPreprocessor] | type[ExternalPreprocessor] | type[FeedbackPreprocessor] | type[HEDPreprocessor] | type[LatentFeedbackPreprocessor] | type[LineartPreprocessor] | type[OpenPosePreprocessor] | type[PassthroughPreprocessor] | type[RealESRGANProcessor] | type[ScribblePreprocessor] | type[SharpenPreprocessor] | type[SoftEdgePreprocessor] | type[StandardLineartPreprocessor] | type[UpscalePreprocessor]`", + "concise_description": "`type[HEDTensorrtPreprocessor] | None` is not assignable to TypedDict key `hed_tensorrt` with type `type[BlurPreprocessor] | type[CannyPreprocessor] | type[DepthPreprocessor] | type[ExternalPreprocessor] | type[FeedbackPreprocessor] | type[HEDPreprocessor] | type[LatentFeedbackPreprocessor] | type[LineartPreprocessor] | type[OpenPosePreprocessor] | type[PassthroughPreprocessor] | type[RealESRGANProcessor] | type[ScribblePreprocessor] | type[SharpenPreprocessor] | type[SoftEdgePreprocessor] | type[StandardLineartPreprocessor] | type[UpscalePreprocessor]`", + "severity": "error" + }, + { + "line": 130, + "column": 51, + "stop_line": 130, + "stop_column": 79, + "path": "src\\streamdiffusion\\preprocessing\\processors\\__init__.py", + "code": -2, + "name": "bad-typed-dict-key", + "description": "`type[ScribbleTensorrtPreprocessor] | None` is not assignable to TypedDict key `scribble_tensorrt` with type `type[BlurPreprocessor] | type[CannyPreprocessor] | type[DepthPreprocessor] | type[ExternalPreprocessor] | type[FeedbackPreprocessor] | type[HEDPreprocessor] | type[LatentFeedbackPreprocessor] | type[LineartPreprocessor] | type[OpenPosePreprocessor] | type[PassthroughPreprocessor] | type[RealESRGANProcessor] | type[ScribblePreprocessor] | type[SharpenPreprocessor] | type[SoftEdgePreprocessor] | type[StandardLineartPreprocessor] | type[UpscalePreprocessor]`", + "concise_description": "`type[ScribbleTensorrtPreprocessor] | None` is not assignable to TypedDict key `scribble_tensorrt` with type `type[BlurPreprocessor] | type[CannyPreprocessor] | type[DepthPreprocessor] | type[ExternalPreprocessor] | type[FeedbackPreprocessor] | type[HEDPreprocessor] | type[LatentFeedbackPreprocessor] | type[LineartPreprocessor] | type[OpenPosePreprocessor] | type[PassthroughPreprocessor] | type[RealESRGANProcessor] | type[ScribblePreprocessor] | type[SharpenPreprocessor] | type[SoftEdgePreprocessor] | type[StandardLineartPreprocessor] | type[UpscalePreprocessor]`", + "severity": "error" + }, + { + "line": 133, + "column": 53, + "stop_line": 133, + "stop_column": 82, + "path": "src\\streamdiffusion\\preprocessing\\processors\\__init__.py", + "code": -2, + "name": "bad-typed-dict-key", + "description": "`type[NormalBaeTensorrtPreprocessor] | None` is not assignable to TypedDict key `normal_bae_tensorrt` with type `type[BlurPreprocessor] | type[CannyPreprocessor] | type[DepthPreprocessor] | type[ExternalPreprocessor] | type[FeedbackPreprocessor] | type[HEDPreprocessor] | type[LatentFeedbackPreprocessor] | type[LineartPreprocessor] | type[OpenPosePreprocessor] | type[PassthroughPreprocessor] | type[RealESRGANProcessor] | type[ScribblePreprocessor] | type[SharpenPreprocessor] | type[SoftEdgePreprocessor] | type[StandardLineartPreprocessor] | type[UpscalePreprocessor]`", + "concise_description": "`type[NormalBaeTensorrtPreprocessor] | None` is not assignable to TypedDict key `normal_bae_tensorrt` with type `type[BlurPreprocessor] | type[CannyPreprocessor] | type[DepthPreprocessor] | type[ExternalPreprocessor] | type[FeedbackPreprocessor] | type[HEDPreprocessor] | type[LatentFeedbackPreprocessor] | type[LineartPreprocessor] | type[OpenPosePreprocessor] | type[PassthroughPreprocessor] | type[RealESRGANProcessor] | type[ScribblePreprocessor] | type[SharpenPreprocessor] | type[SoftEdgePreprocessor] | type[StandardLineartPreprocessor] | type[UpscalePreprocessor]`", + "severity": "error" + }, + { + "line": 333, + "column": 50, + "stop_line": 333, + "stop_column": 54, + "path": "src\\streamdiffusion\\preprocessing\\processors\\__init__.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `ModuleSpec | None` is not assignable to parameter `spec` with type `ModuleSpec` in function `_frozen_importlib.module_from_spec`", + "concise_description": "Argument `ModuleSpec | None` is not assignable to parameter `spec` with type `ModuleSpec` in function `_frozen_importlib.module_from_spec`", + "severity": "error" + }, + { + "line": 334, + "column": 9, + "stop_line": 334, + "stop_column": 20, + "path": "src\\streamdiffusion\\preprocessing\\processors\\__init__.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `loader`", + "concise_description": "Object of class `NoneType` has no attribute `loader`", + "severity": "error" + }, + { + "line": 117, + "column": 64, + "stop_line": 117, + "stop_column": 77, + "path": "src\\streamdiffusion\\preprocessing\\processors\\base.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `LANCZOS` in module `PIL.Image`", + "concise_description": "No attribute `LANCZOS` in module `PIL.Image`", + "severity": "error" + }, + { + "line": 113, + "column": 9, + "stop_line": 113, + "stop_column": 29, + "path": "src\\streamdiffusion\\preprocessing\\processors\\blur.py", + "code": -2, + "name": "bad-override-param-name", + "description": "Class member `BlurPreprocessor._process_tensor_core` overrides parent class `BasePreprocessor` in an inconsistent manner\n Got parameter name `image_tensor`, expected `tensor`", + "concise_description": "Class member `BlurPreprocessor._process_tensor_core` overrides parent class `BasePreprocessor` in an inconsistent manner", + "severity": "error" + }, + { + "line": 100, + "column": 9, + "stop_line": 100, + "stop_column": 29, + "path": "src\\streamdiffusion\\preprocessing\\processors\\canny.py", + "code": -2, + "name": "bad-override-param-name", + "description": "Class member `CannyPreprocessor._process_tensor_core` overrides parent class `BasePreprocessor` in an inconsistent manner\n Got parameter name `image_tensor`, expected `tensor`", + "concise_description": "Class member `CannyPreprocessor._process_tensor_core` overrides parent class `BasePreprocessor` in an inconsistent manner", + "severity": "error" + }, + { + "line": 123, + "column": 27, + "stop_line": 123, + "stop_column": 62, + "path": "src\\streamdiffusion\\preprocessing\\processors\\canny.py", + "code": -2, + "name": "no-matching-overload", + "description": "No matching overload found for function `torch._C._VariableFunctions.conv2d` called with arguments: (Tensor, Tensor | None, padding=Literal[2])\n Possible overloads:\n (input: Tensor, weight: Tensor, bias: Tensor | None = None, stride: Sequence[SymInt | int] | SymInt | int = 1, padding: Sequence[SymInt | int] | SymInt | int = 0, dilation: Sequence[SymInt | int] | SymInt | int = 1, groups: SymInt | int = 1) -> Tensor [closest match]\n (input: Tensor, weight: Tensor, bias: Tensor | None = None, stride: Sequence[SymInt | int] | SymInt | int = 1, padding: str = 'valid', dilation: Sequence[SymInt | int] | SymInt | int = 1, groups: SymInt | int = 1) -> Tensor", + "concise_description": "No matching overload found for function `torch._C._VariableFunctions.conv2d` called with arguments: (Tensor, Tensor | None, padding=Literal[2])", + "severity": "error" + }, + { + "line": 126, + "column": 22, + "stop_line": 126, + "stop_column": 57, + "path": "src\\streamdiffusion\\preprocessing\\processors\\canny.py", + "code": -2, + "name": "no-matching-overload", + "description": "No matching overload found for function `torch._C._VariableFunctions.conv2d` called with arguments: (Unknown, Tensor | None, padding=Literal[1])\n Possible overloads:\n (input: Tensor, weight: Tensor, bias: Tensor | None = None, stride: Sequence[SymInt | int] | SymInt | int = 1, padding: Sequence[SymInt | int] | SymInt | int = 0, dilation: Sequence[SymInt | int] | SymInt | int = 1, groups: SymInt | int = 1) -> Tensor [closest match]\n (input: Tensor, weight: Tensor, bias: Tensor | None = None, stride: Sequence[SymInt | int] | SymInt | int = 1, padding: str = 'valid', dilation: Sequence[SymInt | int] | SymInt | int = 1, groups: SymInt | int = 1) -> Tensor", + "concise_description": "No matching overload found for function `torch._C._VariableFunctions.conv2d` called with arguments: (Unknown, Tensor | None, padding=Literal[1])", + "severity": "error" + }, + { + "line": 127, + "column": 22, + "stop_line": 127, + "stop_column": 57, + "path": "src\\streamdiffusion\\preprocessing\\processors\\canny.py", + "code": -2, + "name": "no-matching-overload", + "description": "No matching overload found for function `torch._C._VariableFunctions.conv2d` called with arguments: (Unknown, Tensor | None, padding=Literal[1])\n Possible overloads:\n (input: Tensor, weight: Tensor, bias: Tensor | None = None, stride: Sequence[SymInt | int] | SymInt | int = 1, padding: Sequence[SymInt | int] | SymInt | int = 0, dilation: Sequence[SymInt | int] | SymInt | int = 1, groups: SymInt | int = 1) -> Tensor [closest match]\n (input: Tensor, weight: Tensor, bias: Tensor | None = None, stride: Sequence[SymInt | int] | SymInt | int = 1, padding: str = 'valid', dilation: Sequence[SymInt | int] | SymInt | int = 1, groups: SymInt | int = 1) -> Tensor", + "concise_description": "No matching overload found for function `torch._C._VariableFunctions.conv2d` called with arguments: (Unknown, Tensor | None, padding=Literal[1])", + "severity": "error" + }, + { + "line": 70, + "column": 78, + "stop_line": 70, + "stop_column": 91, + "path": "src\\streamdiffusion\\preprocessing\\processors\\depth.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `LANCZOS` in module `PIL.Image`", + "concise_description": "No attribute `LANCZOS` in module `PIL.Image`", + "severity": "error" + }, + { + "line": 90, + "column": 9, + "stop_line": 90, + "stop_column": 29, + "path": "src\\streamdiffusion\\preprocessing\\processors\\depth.py", + "code": -2, + "name": "bad-override-param-name", + "description": "Class member `DepthPreprocessor._process_tensor_core` overrides parent class `BasePreprocessor` in an inconsistent manner\n Got parameter name `image_tensor`, expected `tensor`", + "concise_description": "Class member `DepthPreprocessor._process_tensor_core` overrides parent class `BasePreprocessor` in an inconsistent manner", + "severity": "error" + }, + { + "line": 35, + "column": 43, + "stop_line": 35, + "stop_column": 47, + "path": "src\\streamdiffusion\\preprocessing\\processors\\depth_tensorrt.py", + "code": -2, + "name": "bad-function-definition", + "description": "Default `None` is not assignable to parameter `engine_path` with type `str`", + "concise_description": "Default `None` is not assignable to parameter `engine_path` with type `str`", + "severity": "error" + }, + { + "line": 122, + "column": 9, + "stop_line": 122, + "stop_column": 29, + "path": "src\\streamdiffusion\\preprocessing\\processors\\depth_tensorrt.py", + "code": -2, + "name": "bad-override-param-name", + "description": "Class member `DepthAnythingTensorrtPreprocessor._process_tensor_core` overrides parent class `BasePreprocessor` in an inconsistent manner\n Got parameter name `image_tensor`, expected `tensor`", + "concise_description": "Class member `DepthAnythingTensorrtPreprocessor._process_tensor_core` overrides parent class `BasePreprocessor` in an inconsistent manner", + "severity": "error" + }, + { + "line": 75, + "column": 18, + "stop_line": 75, + "stop_column": 28, + "path": "src\\streamdiffusion\\preprocessing\\processors\\hed.py", + "code": -2, + "name": "not-callable", + "description": "Expected a callable, got `None`", + "concise_description": "Expected a callable, got `None`", + "severity": "error" + }, + { + "line": 86, + "column": 67, + "stop_line": 86, + "stop_column": 80, + "path": "src\\streamdiffusion\\preprocessing\\processors\\hed.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `LANCZOS` in module `PIL.Image`", + "concise_description": "No attribute `LANCZOS` in module `PIL.Image`", + "severity": "error" + }, + { + "line": 90, + "column": 9, + "stop_line": 90, + "stop_column": 29, + "path": "src\\streamdiffusion\\preprocessing\\processors\\hed.py", + "code": -2, + "name": "bad-override-param-name", + "description": "Class member `HEDPreprocessor._process_tensor_core` overrides parent class `BasePreprocessor` in an inconsistent manner\n Got parameter name `image_tensor`, expected `tensor`", + "concise_description": "Class member `HEDPreprocessor._process_tensor_core` overrides parent class `BasePreprocessor` in an inconsistent manner", + "severity": "error" + }, + { + "line": 118, + "column": 17, + "stop_line": 118, + "stop_column": 22, + "path": "src\\streamdiffusion\\preprocessing\\processors\\hed_tensorrt.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Tensor` is not assignable to parameter `args` with type `tuple[Any, ...]` in function `torch.onnx.export`", + "concise_description": "Argument `Tensor` is not assignable to parameter `args` with type `tuple[Any, ...]` in function `torch.onnx.export`", + "severity": "error" + }, + { + "line": 45, + "column": 9, + "stop_line": 45, + "stop_column": 22, + "path": "src\\streamdiffusion\\preprocessing\\processors\\ipadapter_embedding.py", + "code": -2, + "name": "bad-override", + "description": "Class member `IPAdapterEmbeddingPreprocessor._process_core` overrides parent class `BasePreprocessor` in an inconsistent manner\n `IPAdapterEmbeddingPreprocessor._process_core` has type `(self: IPAdapterEmbeddingPreprocessor, image: Image) -> tuple[Tensor, Tensor]`, which is not assignable to `(self: IPAdapterEmbeddingPreprocessor, image: Image) -> Image`, the type of `BasePreprocessor._process_core`\n Signature mismatch:\n expected: def _process_core(self: IPAdapterEmbeddingPreprocessor, image: Image) -> Image: ...\n ^^^^^ return type\n found: def _process_core(self: IPAdapterEmbeddingPreprocessor, image: Image) -> tuple[Tensor, Tensor]: ...\n ^^^^^^^^^^^^^^^^^^^^^ return type", + "concise_description": "Class member `IPAdapterEmbeddingPreprocessor._process_core` overrides parent class `BasePreprocessor` in an inconsistent manner", + "severity": "error" + }, + { + "line": 77, + "column": 9, + "stop_line": 77, + "stop_column": 29, + "path": "src\\streamdiffusion\\preprocessing\\processors\\ipadapter_embedding.py", + "code": -2, + "name": "bad-override", + "description": "Class member `IPAdapterEmbeddingPreprocessor._process_tensor_core` overrides parent class `BasePreprocessor` in an inconsistent manner\n `IPAdapterEmbeddingPreprocessor._process_tensor_core` has type `(self: IPAdapterEmbeddingPreprocessor, tensor: Tensor) -> tuple[Tensor, Tensor]`, which is not assignable to `(self: IPAdapterEmbeddingPreprocessor, tensor: Tensor) -> Tensor`, the type of `BasePreprocessor._process_tensor_core`\n Signature mismatch:\n expected: def _process_tensor_core(self: IPAdapterEmbeddingPreprocessor, tensor: Tensor) -> Tensor: ...\n ^^^^^^ return type\n found: def _process_tensor_core(self: IPAdapterEmbeddingPreprocessor, tensor: Tensor) -> tuple[Tensor, Tensor]: ...\n ^^^^^^^^^^^^^^^^^^^^^ return type", + "concise_description": "Class member `IPAdapterEmbeddingPreprocessor._process_tensor_core` overrides parent class `BasePreprocessor` in an inconsistent manner", + "severity": "error" + }, + { + "line": 99, + "column": 9, + "stop_line": 99, + "stop_column": 16, + "path": "src\\streamdiffusion\\preprocessing\\processors\\ipadapter_embedding.py", + "code": -2, + "name": "bad-override", + "description": "Class member `IPAdapterEmbeddingPreprocessor.process` overrides parent class `BasePreprocessor` in an inconsistent manner\n `IPAdapterEmbeddingPreprocessor.process` has type `(self: IPAdapterEmbeddingPreprocessor, image: Image | Tensor) -> tuple[Tensor, Tensor]`, which is not assignable to `(self: IPAdapterEmbeddingPreprocessor, image: Image | Tensor | ndarray[Unknown, Unknown]) -> Image`, the type of `BasePreprocessor.process`\n Signature mismatch:\n expected: def process(self: IPAdapterEmbeddingPreprocessor, image: Image | Tensor | ndarray[Unknown, Unknown]) -> Image: ...\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^ return type\n |\n parameters\n found: def process(self: IPAdapterEmbeddingPreprocessor, image: Image | Tensor) -> tuple[Tensor, Tensor]: ...\n ^ ^^^^^^^^^^^^^^^^^^^^^ return type\n |\n parameters", + "concise_description": "Class member `IPAdapterEmbeddingPreprocessor.process` overrides parent class `BasePreprocessor` in an inconsistent manner", + "severity": "error" + }, + { + "line": 109, + "column": 9, + "stop_line": 109, + "stop_column": 23, + "path": "src\\streamdiffusion\\preprocessing\\processors\\ipadapter_embedding.py", + "code": -2, + "name": "bad-override", + "description": "Class member `IPAdapterEmbeddingPreprocessor.process_tensor` overrides parent class `BasePreprocessor` in an inconsistent manner\n `IPAdapterEmbeddingPreprocessor.process_tensor` has type `(self: IPAdapterEmbeddingPreprocessor, image_tensor: Tensor) -> tuple[Tensor, Tensor]`, which is not assignable to `(self: IPAdapterEmbeddingPreprocessor, image_tensor: Tensor) -> Tensor`, the type of `BasePreprocessor.process_tensor`\n Signature mismatch:\n expected: def process_tensor(self: IPAdapterEmbeddingPreprocessor, image_tensor: Tensor) -> Tensor: ...\n ^^^^^^ return type\n found: def process_tensor(self: IPAdapterEmbeddingPreprocessor, image_tensor: Tensor) -> tuple[Tensor, Tensor]: ...\n ^^^^^^^^^^^^^^^^^^^^^ return type", + "concise_description": "Class member `IPAdapterEmbeddingPreprocessor.process_tensor` overrides parent class `BasePreprocessor` in an inconsistent manner", + "severity": "error" + }, + { + "line": 84, + "column": 9, + "stop_line": 84, + "stop_column": 30, + "path": "src\\streamdiffusion\\preprocessing\\processors\\latent_feedback.py", + "code": -2, + "name": "bad-override-param-name", + "description": "Class member `LatentFeedbackPreprocessor.validate_tensor_input` overrides parent class `PipelineAwareProcessor` in an inconsistent manner\n Got parameter name `latent_tensor`, expected `image_tensor`", + "concise_description": "Class member `LatentFeedbackPreprocessor.validate_tensor_input` overrides parent class `PipelineAwareProcessor` in an inconsistent manner", + "severity": "error" + }, + { + "line": 104, + "column": 82, + "stop_line": 104, + "stop_column": 95, + "path": "src\\streamdiffusion\\preprocessing\\processors\\lineart.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `LANCZOS` in module `PIL.Image`", + "concise_description": "No attribute `LANCZOS` in module `PIL.Image`", + "severity": "error" + }, + { + "line": 112, + "column": 16, + "stop_line": 112, + "stop_column": 29, + "path": "src\\streamdiffusion\\preprocessing\\processors\\lineart.py", + "code": -2, + "name": "bad-return", + "description": "Returned type `Image | int | ndarray[Any, dtype[signedinteger[Any]]] | Unknown` is not assignable to declared return type `Image`", + "concise_description": "Returned type `Image | int | ndarray[Any, dtype[signedinteger[Any]]] | Unknown` is not assignable to declared return type `Image`", + "severity": "error" + }, + { + "line": 558, + "column": 78, + "stop_line": 558, + "stop_column": 91, + "path": "src\\streamdiffusion\\preprocessing\\processors\\mediapipe_pose.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `LANCZOS` in module `PIL.Image`", + "concise_description": "No attribute `LANCZOS` in module `PIL.Image`", + "severity": "error" + }, + { + "line": 566, + "column": 12, + "stop_line": 566, + "stop_column": 34, + "path": "src\\streamdiffusion\\preprocessing\\processors\\mediapipe_pose.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NamedTupleFallback` has no attribute `pose_landmarks`", + "concise_description": "Object of class `NamedTupleFallback` has no attribute `pose_landmarks`", + "severity": "error" + }, + { + "line": 568, + "column": 17, + "stop_line": 568, + "stop_column": 48, + "path": "src\\streamdiffusion\\preprocessing\\processors\\mediapipe_pose.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `object` has no attribute `landmark`", + "concise_description": "Object of class `object` has no attribute `landmark`", + "severity": "error" + }, + { + "line": 577, + "column": 16, + "stop_line": 577, + "stop_column": 43, + "path": "src\\streamdiffusion\\preprocessing\\processors\\mediapipe_pose.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NamedTupleFallback` has no attribute `left_hand_landmarks`", + "concise_description": "Object of class `NamedTupleFallback` has no attribute `left_hand_landmarks`", + "severity": "error" + }, + { + "line": 579, + "column": 33, + "stop_line": 579, + "stop_column": 69, + "path": "src\\streamdiffusion\\preprocessing\\processors\\mediapipe_pose.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `object` has no attribute `landmark`", + "concise_description": "Object of class `object` has no attribute `landmark`", + "severity": "error" + }, + { + "line": 582, + "column": 16, + "stop_line": 582, + "stop_column": 44, + "path": "src\\streamdiffusion\\preprocessing\\processors\\mediapipe_pose.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NamedTupleFallback` has no attribute `right_hand_landmarks`", + "concise_description": "Object of class `NamedTupleFallback` has no attribute `right_hand_landmarks`", + "severity": "error" + }, + { + "line": 584, + "column": 33, + "stop_line": 584, + "stop_column": 70, + "path": "src\\streamdiffusion\\preprocessing\\processors\\mediapipe_pose.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `object` has no attribute `landmark`", + "concise_description": "Object of class `object` has no attribute `landmark`", + "severity": "error" + }, + { + "line": 591, + "column": 9, + "stop_line": 591, + "stop_column": 29, + "path": "src\\streamdiffusion\\preprocessing\\processors\\mediapipe_pose.py", + "code": -2, + "name": "bad-override-param-name", + "description": "Class member `MediaPipePosePreprocessor._process_tensor_core` overrides parent class `BasePreprocessor` in an inconsistent manner\n Got parameter name `image_tensor`, expected `tensor`", + "concise_description": "Class member `MediaPipePosePreprocessor._process_tensor_core` overrides parent class `BasePreprocessor` in an inconsistent manner", + "severity": "error" + }, + { + "line": 219, + "column": 78, + "stop_line": 219, + "stop_column": 91, + "path": "src\\streamdiffusion\\preprocessing\\processors\\mediapipe_segmentation.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `LANCZOS` in module `PIL.Image`", + "concise_description": "No attribute `LANCZOS` in module `PIL.Image`", + "severity": "error" + }, + { + "line": 225, + "column": 12, + "stop_line": 225, + "stop_column": 37, + "path": "src\\streamdiffusion\\preprocessing\\processors\\mediapipe_segmentation.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NamedTupleFallback` has no attribute `segmentation_mask`", + "concise_description": "Object of class `NamedTupleFallback` has no attribute `segmentation_mask`", + "severity": "error" + }, + { + "line": 228, + "column": 47, + "stop_line": 228, + "stop_column": 51, + "path": "src\\streamdiffusion\\preprocessing\\processors\\mediapipe_segmentation.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `object | Unknown` is not assignable to parameter `mask` with type `ndarray[Unknown, Unknown]` in function `MediaPipeSegmentationPreprocessor._apply_mask_smoothing`", + "concise_description": "Argument `object | Unknown` is not assignable to parameter `mask` with type `ndarray[Unknown, Unknown]` in function `MediaPipeSegmentationPreprocessor._apply_mask_smoothing`", + "severity": "error" + }, + { + "line": 245, + "column": 9, + "stop_line": 245, + "stop_column": 29, + "path": "src\\streamdiffusion\\preprocessing\\processors\\mediapipe_segmentation.py", + "code": -2, + "name": "bad-override-param-name", + "description": "Class member `MediaPipeSegmentationPreprocessor._process_tensor_core` overrides parent class `BasePreprocessor` in an inconsistent manner\n Got parameter name `image_tensor`, expected `tensor`", + "concise_description": "Class member `MediaPipeSegmentationPreprocessor._process_tensor_core` overrides parent class `BasePreprocessor` in an inconsistent manner", + "severity": "error" + }, + { + "line": 79, + "column": 17, + "stop_line": 79, + "stop_column": 22, + "path": "src\\streamdiffusion\\preprocessing\\processors\\normal_bae_tensorrt.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Tensor` is not assignable to parameter `args` with type `tuple[Any, ...]` in function `torch.onnx.export`", + "concise_description": "Argument `Tensor` is not assignable to parameter `args` with type `tuple[Any, ...]` in function `torch.onnx.export`", + "severity": "error" + }, + { + "line": 185, + "column": 9, + "stop_line": 185, + "stop_column": 29, + "path": "src\\streamdiffusion\\preprocessing\\processors\\normal_bae_tensorrt.py", + "code": -2, + "name": "bad-override-param-name", + "description": "Class member `_NormalBaeTorchGPU._process_tensor_core` overrides parent class `BasePreprocessor` in an inconsistent manner\n Got parameter name `image_tensor`, expected `tensor`", + "concise_description": "Class member `_NormalBaeTorchGPU._process_tensor_core` overrides parent class `BasePreprocessor` in an inconsistent manner", + "severity": "error" + }, + { + "line": 285, + "column": 17, + "stop_line": 285, + "stop_column": 22, + "path": "src\\streamdiffusion\\preprocessing\\processors\\normal_bae_tensorrt.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Tensor` is not assignable to parameter `args` with type `tuple[Any, ...]` in function `torch.onnx.export`", + "concise_description": "Argument `Tensor` is not assignable to parameter `args` with type `tuple[Any, ...]` in function `torch.onnx.export`", + "severity": "error" + }, + { + "line": 145, + "column": 78, + "stop_line": 145, + "stop_column": 91, + "path": "src\\streamdiffusion\\preprocessing\\processors\\openpose.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `LANCZOS` in module `PIL.Image`", + "concise_description": "No attribute `LANCZOS` in module `PIL.Image`", + "severity": "error" + }, + { + "line": 152, + "column": 59, + "stop_line": 152, + "stop_column": 72, + "path": "src\\streamdiffusion\\preprocessing\\processors\\openpose.py", + "code": -2, + "name": "unexpected-keyword", + "description": "Unexpected keyword argument `hand_and_face` in function `FallbackDetector.__call__`", + "concise_description": "Unexpected keyword argument `hand_and_face` in function `FallbackDetector.__call__`", + "severity": "error" + }, + { + "line": 215, + "column": 43, + "stop_line": 215, + "stop_column": 47, + "path": "src\\streamdiffusion\\preprocessing\\processors\\pose_tensorrt.py", + "code": -2, + "name": "bad-function-definition", + "description": "Default `None` is not assignable to parameter `engine_path` with type `str`", + "concise_description": "Default `None` is not assignable to parameter `engine_path` with type `str`", + "severity": "error" + }, + { + "line": 305, + "column": 9, + "stop_line": 305, + "stop_column": 29, + "path": "src\\streamdiffusion\\preprocessing\\processors\\pose_tensorrt.py", + "code": -2, + "name": "bad-override-param-name", + "description": "Class member `YoloNasPoseTensorrtPreprocessor._process_tensor_core` overrides parent class `BasePreprocessor` in an inconsistent manner\n Got parameter name `image_tensor`, expected `tensor`", + "concise_description": "Class member `YoloNasPoseTensorrtPreprocessor._process_tensor_core` overrides parent class `BasePreprocessor` in an inconsistent manner", + "severity": "error" + }, + { + "line": 20, + "column": 5, + "stop_line": 20, + "stop_column": 37, + "path": "src\\streamdiffusion\\preprocessing\\processors\\realesrgan_trt.py", + "code": -2, + "name": "missing-import", + "description": "Cannot find module `spandrel`\n Looked in these locations (from config in `D:\\dev\\SDTD_032_dev\\StreamDiffusion\\pyproject.toml`):\n Search path (from config file): [\"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\src\"]\n Import root (inferred from project layout): \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\src\"\n Site package path queried from interpreter: [\"C:\\\\Users\\\\Inter\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python311\\\\DLLs\", \"C:\\\\Users\\\\Inter\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python311\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\venv\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\venv\\\\Lib\\\\site-packages\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\src\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\venv\\\\Lib\\\\site-packages\\\\win32\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\venv\\\\Lib\\\\site-packages\\\\win32\\\\lib\", \"D:\\\\dev\\\\SDTD_032_dev\\\\StreamDiffusion\\\\venv\\\\Lib\\\\site-packages\\\\Pythonwin\"]", + "concise_description": "Cannot find module `spandrel`", + "severity": "error" + }, + { + "line": 50, + "column": 35, + "stop_line": 50, + "stop_column": 43, + "path": "src\\streamdiffusion\\preprocessing\\processors\\realesrgan_trt.py", + "code": -2, + "name": "unsupported-operation", + "description": "Cannot set item in `dict[type[complex128] | type[complex64] | type[float16] | type[float32] | type[float64] | type[int16] | type[int32] | type[int64] | type[int8] | type[uint8], dtype]`\n Argument `type[bool_]` is not assignable to parameter `key` with type `type[complex128] | type[complex64] | type[float16] | type[float32] | type[float64] | type[int16] | type[int32] | type[int64] | type[int8] | type[uint8]` in function `dict.__setitem__`", + "concise_description": "Cannot set item in `dict[type[complex128] | type[complex64] | type[float16] | type[float32] | type[float64] | type[int16] | type[int32] | type[int64] | type[int8] | type[uint8], dtype]`", + "severity": "error" + }, + { + "line": 52, + "column": 35, + "stop_line": 52, + "stop_column": 42, + "path": "src\\streamdiffusion\\preprocessing\\processors\\realesrgan_trt.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `bool` in module `numpy`", + "concise_description": "No attribute `bool` in module `numpy`", + "severity": "error" + }, + { + "line": 80, + "column": 24, + "stop_line": 80, + "stop_column": 60, + "path": "src\\streamdiffusion\\preprocessing\\processors\\realesrgan_trt.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `create_execution_context`", + "concise_description": "Object of class `NoneType` has no attribute `create_execution_context`", + "severity": "error" + }, + { + "line": 86, + "column": 9, + "stop_line": 86, + "stop_column": 37, + "path": "src\\streamdiffusion\\preprocessing\\processors\\realesrgan_trt.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `set_input_shape`", + "concise_description": "Object of class `NoneType` has no attribute `set_input_shape`", + "severity": "error" + }, + { + "line": 89, + "column": 26, + "stop_line": 89, + "stop_column": 52, + "path": "src\\streamdiffusion\\preprocessing\\processors\\realesrgan_trt.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `num_io_tensors`", + "concise_description": "Object of class `NoneType` has no attribute `num_io_tensors`", + "severity": "error" + }, + { + "line": 90, + "column": 20, + "stop_line": 90, + "stop_column": 47, + "path": "src\\streamdiffusion\\preprocessing\\processors\\realesrgan_trt.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_name`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_name`", + "severity": "error" + }, + { + "line": 91, + "column": 21, + "stop_line": 91, + "stop_column": 50, + "path": "src\\streamdiffusion\\preprocessing\\processors\\realesrgan_trt.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_shape`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_shape`", + "severity": "error" + }, + { + "line": 92, + "column": 32, + "stop_line": 92, + "stop_column": 60, + "path": "src\\streamdiffusion\\preprocessing\\processors\\realesrgan_trt.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_dtype`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_dtype`", + "severity": "error" + }, + { + "line": 123, + "column": 17, + "stop_line": 123, + "stop_column": 48, + "path": "src\\streamdiffusion\\preprocessing\\processors\\realesrgan_trt.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `set_tensor_address`", + "concise_description": "Object of class `NoneType` has no attribute `set_tensor_address`", + "severity": "error" + }, + { + "line": 126, + "column": 27, + "stop_line": 126, + "stop_column": 56, + "path": "src\\streamdiffusion\\preprocessing\\processors\\realesrgan_trt.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `execute_async_v3`", + "concise_description": "Object of class `NoneType` has no attribute `execute_async_v3`", + "severity": "error" + }, + { + "line": 289, + "column": 17, + "stop_line": 289, + "stop_column": 27, + "path": "src\\streamdiffusion\\preprocessing\\processors\\realesrgan_trt.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Tensor` is not assignable to parameter `args` with type `tuple[Any, ...]` in function `torch.onnx.export`", + "concise_description": "Argument `Tensor` is not assignable to parameter `args` with type `tuple[Any, ...]` in function `torch.onnx.export`", + "severity": "error" + }, + { + "line": 331, + "column": 23, + "stop_line": 331, + "stop_column": 34, + "path": "src\\streamdiffusion\\preprocessing\\processors\\realesrgan_trt.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `Builder` in module `tensorrt`", + "concise_description": "No attribute `Builder` in module `tensorrt`", + "severity": "error" + }, + { + "line": 331, + "column": 35, + "stop_line": 331, + "stop_column": 45, + "path": "src\\streamdiffusion\\preprocessing\\processors\\realesrgan_trt.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `Logger` in module `tensorrt`", + "concise_description": "No attribute `Logger` in module `tensorrt`", + "severity": "error" + }, + { + "line": 331, + "column": 46, + "stop_line": 331, + "stop_column": 56, + "path": "src\\streamdiffusion\\preprocessing\\processors\\realesrgan_trt.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `Logger` in module `tensorrt`", + "concise_description": "No attribute `Logger` in module `tensorrt`", + "severity": "error" + }, + { + "line": 333, + "column": 22, + "stop_line": 333, + "stop_column": 36, + "path": "src\\streamdiffusion\\preprocessing\\processors\\realesrgan_trt.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `OnnxParser` in module `tensorrt`", + "concise_description": "No attribute `OnnxParser` in module `tensorrt`", + "severity": "error" + }, + { + "line": 333, + "column": 46, + "stop_line": 333, + "stop_column": 56, + "path": "src\\streamdiffusion\\preprocessing\\processors\\realesrgan_trt.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `Logger` in module `tensorrt`", + "concise_description": "No attribute `Logger` in module `tensorrt`", + "severity": "error" + }, + { + "line": 333, + "column": 57, + "stop_line": 333, + "stop_column": 67, + "path": "src\\streamdiffusion\\preprocessing\\processors\\realesrgan_trt.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `Logger` in module `tensorrt`", + "concise_description": "No attribute `Logger` in module `tensorrt`", + "severity": "error" + }, + { + "line": 344, + "column": 29, + "stop_line": 344, + "stop_column": 44, + "path": "src\\streamdiffusion\\preprocessing\\processors\\realesrgan_trt.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `BuilderFlag` in module `tensorrt`", + "concise_description": "No attribute `BuilderFlag` in module `tensorrt`", + "severity": "error" + }, + { + "line": 442, + "column": 64, + "stop_line": 442, + "stop_column": 77, + "path": "src\\streamdiffusion\\preprocessing\\processors\\realesrgan_trt.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `LANCZOS` in module `PIL.Image`", + "concise_description": "No attribute `LANCZOS` in module `PIL.Image`", + "severity": "error" + }, + { + "line": 34, + "column": 18, + "stop_line": 34, + "stop_column": 28, + "path": "src\\streamdiffusion\\preprocessing\\processors\\scribble.py", + "code": -2, + "name": "not-callable", + "description": "Expected a callable, got `None`", + "concise_description": "Expected a callable, got `None`", + "severity": "error" + }, + { + "line": 45, + "column": 67, + "stop_line": 45, + "stop_column": 80, + "path": "src\\streamdiffusion\\preprocessing\\processors\\scribble.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `LANCZOS` in module `PIL.Image`", + "concise_description": "No attribute `LANCZOS` in module `PIL.Image`", + "severity": "error" + }, + { + "line": 246, + "column": 9, + "stop_line": 246, + "stop_column": 29, + "path": "src\\streamdiffusion\\preprocessing\\processors\\sharpen.py", + "code": -2, + "name": "bad-override-param-name", + "description": "Class member `SharpenPreprocessor._process_tensor_core` overrides parent class `BasePreprocessor` in an inconsistent manner\n Got parameter name `image_tensor`, expected `tensor`", + "concise_description": "Class member `SharpenPreprocessor._process_tensor_core` overrides parent class `BasePreprocessor` in an inconsistent manner", + "severity": "error" + }, + { + "line": 218, + "column": 9, + "stop_line": 218, + "stop_column": 29, + "path": "src\\streamdiffusion\\preprocessing\\processors\\soft_edge.py", + "code": -2, + "name": "bad-override-param-name", + "description": "Class member `SoftEdgePreprocessor._process_tensor_core` overrides parent class `BasePreprocessor` in an inconsistent manner\n Got parameter name `image_tensor`, expected `tensor`", + "concise_description": "Class member `SoftEdgePreprocessor._process_tensor_core` overrides parent class `BasePreprocessor` in an inconsistent manner", + "severity": "error" + }, + { + "line": 231, + "column": 24, + "stop_line": 231, + "stop_column": 34, + "path": "src\\streamdiffusion\\preprocessing\\processors\\soft_edge.py", + "code": -2, + "name": "not-callable", + "description": "Expected a callable, got `None`", + "concise_description": "Expected a callable, got `None`", + "severity": "error" + }, + { + "line": 17, + "column": 43, + "stop_line": 17, + "stop_column": 58, + "path": "src\\streamdiffusion\\preprocessing\\processors\\temporal_net_tensorrt.py", + "code": -2, + "name": "missing-module-attribute", + "description": "Could not import `bytes_from_path` from `polygraphy.backend.common`", + "concise_description": "Could not import `bytes_from_path` from `polygraphy.backend.common`", + "severity": "error" + }, + { + "line": 18, + "column": 40, + "stop_line": 18, + "stop_column": 57, + "path": "src\\streamdiffusion\\preprocessing\\processors\\temporal_net_tensorrt.py", + "code": -2, + "name": "missing-module-attribute", + "description": "Could not import `engine_from_bytes` from `polygraphy.backend.trt`", + "concise_description": "Could not import `engine_from_bytes` from `polygraphy.backend.trt`", + "severity": "error" + }, + { + "line": 52, + "column": 31, + "stop_line": 52, + "stop_column": 39, + "path": "src\\streamdiffusion\\preprocessing\\processors\\temporal_net_tensorrt.py", + "code": -2, + "name": "unsupported-operation", + "description": "Cannot set item in `dict[type[complex128] | type[complex64] | type[float16] | type[float32] | type[float64] | type[int16] | type[int32] | type[int64] | type[int8] | type[uint8], dtype]`\n Argument `type[bool_]` is not assignable to parameter `key` with type `type[complex128] | type[complex64] | type[float16] | type[float32] | type[float64] | type[int16] | type[int32] | type[int64] | type[int8] | type[uint8]` in function `dict.__setitem__`", + "concise_description": "Cannot set item in `dict[type[complex128] | type[complex64] | type[float16] | type[float32] | type[float64] | type[int16] | type[int32] | type[int64] | type[int8] | type[uint8], dtype]`", + "severity": "error" + }, + { + "line": 54, + "column": 31, + "stop_line": 54, + "stop_column": 38, + "path": "src\\streamdiffusion\\preprocessing\\processors\\temporal_net_tensorrt.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `bool` in module `numpy`", + "concise_description": "No attribute `bool` in module `numpy`", + "severity": "error" + }, + { + "line": 74, + "column": 24, + "stop_line": 74, + "stop_column": 60, + "path": "src\\streamdiffusion\\preprocessing\\processors\\temporal_net_tensorrt.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `create_execution_context`", + "concise_description": "Object of class `NoneType` has no attribute `create_execution_context`", + "severity": "error" + }, + { + "line": 85, + "column": 26, + "stop_line": 85, + "stop_column": 52, + "path": "src\\streamdiffusion\\preprocessing\\processors\\temporal_net_tensorrt.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `num_io_tensors`", + "concise_description": "Object of class `NoneType` has no attribute `num_io_tensors`", + "severity": "error" + }, + { + "line": 86, + "column": 20, + "stop_line": 86, + "stop_column": 47, + "path": "src\\streamdiffusion\\preprocessing\\processors\\temporal_net_tensorrt.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_name`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_name`", + "severity": "error" + }, + { + "line": 87, + "column": 21, + "stop_line": 87, + "stop_column": 50, + "path": "src\\streamdiffusion\\preprocessing\\processors\\temporal_net_tensorrt.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_shape`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_shape`", + "severity": "error" + }, + { + "line": 88, + "column": 25, + "stop_line": 88, + "stop_column": 53, + "path": "src\\streamdiffusion\\preprocessing\\processors\\temporal_net_tensorrt.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_dtype`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_dtype`", + "severity": "error" + }, + { + "line": 94, + "column": 33, + "stop_line": 94, + "stop_column": 45, + "path": "src\\streamdiffusion\\preprocessing\\processors\\temporal_net_tensorrt.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `DataType` in module `tensorrt`", + "concise_description": "No attribute `DataType` in module `tensorrt`", + "severity": "error" + }, + { + "line": 99, + "column": 16, + "stop_line": 99, + "stop_column": 43, + "path": "src\\streamdiffusion\\preprocessing\\processors\\temporal_net_tensorrt.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_mode`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_mode`", + "severity": "error" + }, + { + "line": 99, + "column": 53, + "stop_line": 99, + "stop_column": 69, + "path": "src\\streamdiffusion\\preprocessing\\processors\\temporal_net_tensorrt.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `TensorIOMode` in module `tensorrt`", + "concise_description": "No attribute `TensorIOMode` in module `tensorrt`", + "severity": "error" + }, + { + "line": 103, + "column": 17, + "stop_line": 103, + "stop_column": 45, + "path": "src\\streamdiffusion\\preprocessing\\processors\\temporal_net_tensorrt.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `set_input_shape`", + "concise_description": "Object of class `NoneType` has no attribute `set_input_shape`", + "severity": "error" + }, + { + "line": 105, + "column": 25, + "stop_line": 105, + "stop_column": 54, + "path": "src\\streamdiffusion\\preprocessing\\processors\\temporal_net_tensorrt.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_shape`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_shape`", + "severity": "error" + }, + { + "line": 108, + "column": 25, + "stop_line": 108, + "stop_column": 54, + "path": "src\\streamdiffusion\\preprocessing\\processors\\temporal_net_tensorrt.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_shape`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_shape`", + "severity": "error" + }, + { + "line": 139, + "column": 24, + "stop_line": 139, + "stop_column": 51, + "path": "src\\streamdiffusion\\preprocessing\\processors\\temporal_net_tensorrt.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_mode`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_mode`", + "severity": "error" + }, + { + "line": 139, + "column": 61, + "stop_line": 139, + "stop_column": 77, + "path": "src\\streamdiffusion\\preprocessing\\processors\\temporal_net_tensorrt.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `TensorIOMode` in module `tensorrt`", + "concise_description": "No attribute `TensorIOMode` in module `tensorrt`", + "severity": "error" + }, + { + "line": 140, + "column": 25, + "stop_line": 140, + "stop_column": 53, + "path": "src\\streamdiffusion\\preprocessing\\processors\\temporal_net_tensorrt.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `set_input_shape`", + "concise_description": "Object of class `NoneType` has no attribute `set_input_shape`", + "severity": "error" + }, + { + "line": 146, + "column": 30, + "stop_line": 146, + "stop_column": 56, + "path": "src\\streamdiffusion\\preprocessing\\processors\\temporal_net_tensorrt.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `num_io_tensors`", + "concise_description": "Object of class `NoneType` has no attribute `num_io_tensors`", + "severity": "error" + }, + { + "line": 147, + "column": 24, + "stop_line": 147, + "stop_column": 51, + "path": "src\\streamdiffusion\\preprocessing\\processors\\temporal_net_tensorrt.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_name`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_name`", + "severity": "error" + }, + { + "line": 148, + "column": 25, + "stop_line": 148, + "stop_column": 54, + "path": "src\\streamdiffusion\\preprocessing\\processors\\temporal_net_tensorrt.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_shape`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_shape`", + "severity": "error" + }, + { + "line": 149, + "column": 36, + "stop_line": 149, + "stop_column": 64, + "path": "src\\streamdiffusion\\preprocessing\\processors\\temporal_net_tensorrt.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_dtype`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_dtype`", + "severity": "error" + }, + { + "line": 162, + "column": 13, + "stop_line": 162, + "stop_column": 44, + "path": "src\\streamdiffusion\\preprocessing\\processors\\temporal_net_tensorrt.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `set_tensor_address`", + "concise_description": "Object of class `NoneType` has no attribute `set_tensor_address`", + "severity": "error" + }, + { + "line": 165, + "column": 19, + "stop_line": 165, + "stop_column": 48, + "path": "src\\streamdiffusion\\preprocessing\\processors\\temporal_net_tensorrt.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `execute_async_v3`", + "concise_description": "Object of class `NoneType` has no attribute `execute_async_v3`", + "severity": "error" + }, + { + "line": 232, + "column": 28, + "stop_line": 232, + "stop_column": 32, + "path": "src\\streamdiffusion\\preprocessing\\processors\\temporal_net_tensorrt.py", + "code": -2, + "name": "bad-function-definition", + "description": "Default `None` is not assignable to parameter `engine_path` with type `str`", + "concise_description": "Default `None` is not assignable to parameter `engine_path` with type `str`", + "severity": "error" + }, + { + "line": 39, + "column": 43, + "stop_line": 39, + "stop_column": 58, + "path": "src\\streamdiffusion\\preprocessing\\processors\\trt_base.py", + "code": -2, + "name": "missing-module-attribute", + "description": "Could not import `bytes_from_path` from `polygraphy.backend.common`", + "concise_description": "Could not import `bytes_from_path` from `polygraphy.backend.common`", + "severity": "error" + }, + { + "line": 40, + "column": 40, + "stop_line": 40, + "stop_column": 57, + "path": "src\\streamdiffusion\\preprocessing\\processors\\trt_base.py", + "code": -2, + "name": "missing-module-attribute", + "description": "Could not import `engine_from_bytes` from `polygraphy.backend.trt`", + "concise_description": "Could not import `engine_from_bytes` from `polygraphy.backend.trt`", + "severity": "error" + }, + { + "line": 100, + "column": 24, + "stop_line": 100, + "stop_column": 60, + "path": "src\\streamdiffusion\\preprocessing\\processors\\trt_base.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `create_execution_context`", + "concise_description": "Object of class `NoneType` has no attribute `create_execution_context`", + "severity": "error" + }, + { + "line": 113, + "column": 75, + "stop_line": 113, + "stop_column": 79, + "path": "src\\streamdiffusion\\preprocessing\\processors\\trt_base.py", + "code": -2, + "name": "bad-function-definition", + "description": "Default `None` is not assignable to parameter `input_shape` with type `tuple[Unknown, ...]`", + "concise_description": "Default `None` is not assignable to parameter `input_shape` with type `tuple[Unknown, ...]`", + "severity": "error" + }, + { + "line": 129, + "column": 26, + "stop_line": 129, + "stop_column": 52, + "path": "src\\streamdiffusion\\preprocessing\\processors\\trt_base.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `num_io_tensors`", + "concise_description": "Object of class `NoneType` has no attribute `num_io_tensors`", + "severity": "error" + }, + { + "line": 130, + "column": 20, + "stop_line": 130, + "stop_column": 47, + "path": "src\\streamdiffusion\\preprocessing\\processors\\trt_base.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_name`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_name`", + "severity": "error" + }, + { + "line": 131, + "column": 16, + "stop_line": 131, + "stop_column": 43, + "path": "src\\streamdiffusion\\preprocessing\\processors\\trt_base.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_mode`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_mode`", + "severity": "error" + }, + { + "line": 131, + "column": 53, + "stop_line": 131, + "stop_column": 69, + "path": "src\\streamdiffusion\\preprocessing\\processors\\trt_base.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `TensorIOMode` in module `tensorrt`", + "concise_description": "No attribute `TensorIOMode` in module `tensorrt`", + "severity": "error" + }, + { + "line": 134, + "column": 24, + "stop_line": 134, + "stop_column": 52, + "path": "src\\streamdiffusion\\preprocessing\\processors\\trt_base.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `set_input_shape`", + "concise_description": "Object of class `NoneType` has no attribute `set_input_shape`", + "severity": "error" + }, + { + "line": 142, + "column": 38, + "stop_line": 142, + "stop_column": 67, + "path": "src\\streamdiffusion\\preprocessing\\processors\\trt_base.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_shape`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_shape`", + "severity": "error" + }, + { + "line": 149, + "column": 24, + "stop_line": 149, + "stop_column": 52, + "path": "src\\streamdiffusion\\preprocessing\\processors\\trt_base.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `set_input_shape`", + "concise_description": "Object of class `NoneType` has no attribute `set_input_shape`", + "severity": "error" + }, + { + "line": 156, + "column": 26, + "stop_line": 156, + "stop_column": 52, + "path": "src\\streamdiffusion\\preprocessing\\processors\\trt_base.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `num_io_tensors`", + "concise_description": "Object of class `NoneType` has no attribute `num_io_tensors`", + "severity": "error" + }, + { + "line": 157, + "column": 20, + "stop_line": 157, + "stop_column": 47, + "path": "src\\streamdiffusion\\preprocessing\\processors\\trt_base.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_name`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_name`", + "severity": "error" + }, + { + "line": 158, + "column": 32, + "stop_line": 158, + "stop_column": 60, + "path": "src\\streamdiffusion\\preprocessing\\processors\\trt_base.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_dtype`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_dtype`", + "severity": "error" + }, + { + "line": 159, + "column": 24, + "stop_line": 159, + "stop_column": 51, + "path": "src\\streamdiffusion\\preprocessing\\processors\\trt_base.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_mode`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_mode`", + "severity": "error" + }, + { + "line": 159, + "column": 61, + "stop_line": 159, + "stop_column": 77, + "path": "src\\streamdiffusion\\preprocessing\\processors\\trt_base.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `TensorIOMode` in module `tensorrt`", + "concise_description": "No attribute `TensorIOMode` in module `tensorrt`", + "severity": "error" + }, + { + "line": 164, + "column": 31, + "stop_line": 164, + "stop_column": 60, + "path": "src\\streamdiffusion\\preprocessing\\processors\\trt_base.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_shape`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_shape`", + "severity": "error" + }, + { + "line": 204, + "column": 28, + "stop_line": 204, + "stop_column": 56, + "path": "src\\streamdiffusion\\preprocessing\\processors\\trt_base.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `set_input_shape`", + "concise_description": "Object of class `NoneType` has no attribute `set_input_shape`", + "severity": "error" + }, + { + "line": 212, + "column": 32, + "stop_line": 212, + "stop_column": 60, + "path": "src\\streamdiffusion\\preprocessing\\processors\\trt_base.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `set_input_shape`", + "concise_description": "Object of class `NoneType` has no attribute `set_input_shape`", + "severity": "error" + }, + { + "line": 223, + "column": 38, + "stop_line": 223, + "stop_column": 64, + "path": "src\\streamdiffusion\\preprocessing\\processors\\trt_base.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `num_io_tensors`", + "concise_description": "Object of class `NoneType` has no attribute `num_io_tensors`", + "severity": "error" + }, + { + "line": 224, + "column": 32, + "stop_line": 224, + "stop_column": 59, + "path": "src\\streamdiffusion\\preprocessing\\processors\\trt_base.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_name`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_name`", + "severity": "error" + }, + { + "line": 225, + "column": 24, + "stop_line": 225, + "stop_column": 51, + "path": "src\\streamdiffusion\\preprocessing\\processors\\trt_base.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_mode`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_mode`", + "severity": "error" + }, + { + "line": 225, + "column": 65, + "stop_line": 225, + "stop_column": 81, + "path": "src\\streamdiffusion\\preprocessing\\processors\\trt_base.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `TensorIOMode` in module `tensorrt`", + "concise_description": "No attribute `TensorIOMode` in module `tensorrt`", + "severity": "error" + }, + { + "line": 226, + "column": 47, + "stop_line": 226, + "stop_column": 76, + "path": "src\\streamdiffusion\\preprocessing\\processors\\trt_base.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_shape`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_shape`", + "severity": "error" + }, + { + "line": 250, + "column": 20, + "stop_line": 250, + "stop_column": 51, + "path": "src\\streamdiffusion\\preprocessing\\processors\\trt_base.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `set_tensor_address`", + "concise_description": "Object of class `NoneType` has no attribute `set_tensor_address`", + "severity": "error" + }, + { + "line": 266, + "column": 19, + "stop_line": 266, + "stop_column": 48, + "path": "src\\streamdiffusion\\preprocessing\\processors\\trt_base.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `execute_async_v3`", + "concise_description": "Object of class `NoneType` has no attribute `execute_async_v3`", + "severity": "error" + }, + { + "line": 462, + "column": 19, + "stop_line": 462, + "stop_column": 30, + "path": "src\\streamdiffusion\\preprocessing\\processors\\trt_base.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `Builder` in module `tensorrt`", + "concise_description": "No attribute `Builder` in module `tensorrt`", + "severity": "error" + }, + { + "line": 462, + "column": 31, + "stop_line": 462, + "stop_column": 41, + "path": "src\\streamdiffusion\\preprocessing\\processors\\trt_base.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `Logger` in module `tensorrt`", + "concise_description": "No attribute `Logger` in module `tensorrt`", + "severity": "error" + }, + { + "line": 462, + "column": 42, + "stop_line": 462, + "stop_column": 52, + "path": "src\\streamdiffusion\\preprocessing\\processors\\trt_base.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `Logger` in module `tensorrt`", + "concise_description": "No attribute `Logger` in module `tensorrt`", + "severity": "error" + }, + { + "line": 464, + "column": 18, + "stop_line": 464, + "stop_column": 32, + "path": "src\\streamdiffusion\\preprocessing\\processors\\trt_base.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `OnnxParser` in module `tensorrt`", + "concise_description": "No attribute `OnnxParser` in module `tensorrt`", + "severity": "error" + }, + { + "line": 464, + "column": 42, + "stop_line": 464, + "stop_column": 52, + "path": "src\\streamdiffusion\\preprocessing\\processors\\trt_base.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `Logger` in module `tensorrt`", + "concise_description": "No attribute `Logger` in module `tensorrt`", + "severity": "error" + }, + { + "line": 464, + "column": 53, + "stop_line": 464, + "stop_column": 63, + "path": "src\\streamdiffusion\\preprocessing\\processors\\trt_base.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `Logger` in module `tensorrt`", + "concise_description": "No attribute `Logger` in module `tensorrt`", + "severity": "error" + }, + { + "line": 472, + "column": 25, + "stop_line": 472, + "stop_column": 40, + "path": "src\\streamdiffusion\\preprocessing\\processors\\trt_base.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `BuilderFlag` in module `tensorrt`", + "concise_description": "No attribute `BuilderFlag` in module `tensorrt`", + "severity": "error" + }, + { + "line": 555, + "column": 9, + "stop_line": 555, + "stop_column": 29, + "path": "src\\streamdiffusion\\preprocessing\\processors\\trt_base.py", + "code": -2, + "name": "bad-override-param-name", + "description": "Class member `SelfBuildingTRTPreprocessor._process_tensor_core` overrides parent class `BasePreprocessor` in an inconsistent manner\n Got parameter name `image_tensor`, expected `tensor`", + "concise_description": "Class member `SelfBuildingTRTPreprocessor._process_tensor_core` overrides parent class `BasePreprocessor` in an inconsistent manner", + "severity": "error" + }, + { + "line": 50, + "column": 25, + "stop_line": 50, + "stop_column": 39, + "path": "src\\streamdiffusion\\preprocessing\\processors\\upscale.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `BILINEAR` in module `PIL.Image`", + "concise_description": "No attribute `BILINEAR` in module `PIL.Image`", + "severity": "error" + }, + { + "line": 51, + "column": 24, + "stop_line": 51, + "stop_column": 37, + "path": "src\\streamdiffusion\\preprocessing\\processors\\upscale.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `LANCZOS` in module `PIL.Image`", + "concise_description": "No attribute `LANCZOS` in module `PIL.Image`", + "severity": "error" + }, + { + "line": 52, + "column": 24, + "stop_line": 52, + "stop_column": 37, + "path": "src\\streamdiffusion\\preprocessing\\processors\\upscale.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `BICUBIC` in module `PIL.Image`", + "concise_description": "No attribute `BICUBIC` in module `PIL.Image`", + "severity": "error" + }, + { + "line": 53, + "column": 24, + "stop_line": 53, + "stop_column": 37, + "path": "src\\streamdiffusion\\preprocessing\\processors\\upscale.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `NEAREST` in module `PIL.Image`", + "concise_description": "No attribute `NEAREST` in module `PIL.Image`", + "severity": "error" + }, + { + "line": 66, + "column": 69, + "stop_line": 66, + "stop_column": 83, + "path": "src\\streamdiffusion\\preprocessing\\processors\\upscale.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `BILINEAR` in module `PIL.Image`", + "concise_description": "No attribute `BILINEAR` in module `PIL.Image`", + "severity": "error" + }, + { + "line": 197, + "column": 58, + "stop_line": 197, + "stop_column": 62, + "path": "src\\streamdiffusion\\stream_parameter_updater.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `None` is not assignable to parameter `scales` with type `list[float]` in function `streamdiffusion.preprocessing.preprocessing_orchestrator.PreprocessingOrchestrator.process_sync`", + "concise_description": "Argument `None` is not assignable to parameter `scales` with type `list[float]` in function `streamdiffusion.preprocessing.preprocessing_orchestrator.PreprocessingOrchestrator.process_sync`", + "severity": "error" + }, + { + "line": 202, + "column": 58, + "stop_line": 202, + "stop_column": 78, + "path": "src\\streamdiffusion\\stream_parameter_updater.py", + "code": -2, + "name": "unsupported-operation", + "description": "Cannot set item in `dict[str, tuple[Tensor, Tensor]]`\n Argument `Tensor | tuple[Tensor, Tensor]` is not assignable to parameter `value` with type `tuple[Tensor, Tensor]` in function `dict.__setitem__`", + "concise_description": "Cannot set item in `dict[str, tuple[Tensor, Tensor]]`", + "severity": "error" + }, + { + "line": 900, + "column": 29, + "stop_line": 900, + "stop_column": 95, + "path": "src\\streamdiffusion\\stream_parameter_updater.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `LCMScheduler` has no attribute `get_scalings_for_boundary_condition_discrete`", + "concise_description": "Object of class `LCMScheduler` has no attribute `get_scalings_for_boundary_condition_discrete`", + "severity": "error" + }, + { + "line": 1382, + "column": 34, + "stop_line": 1382, + "stop_column": 61, + "path": "src\\streamdiffusion\\stream_parameter_updater.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Any | None` is not assignable to parameter `model_id` with type `str` in function `streamdiffusion.modules.controlnet_module.ControlNetConfig.__init__`", + "concise_description": "Argument `Any | None` is not assignable to parameter `model_id` with type `str` in function `streamdiffusion.modules.controlnet_module.ControlNetConfig.__init__`", + "severity": "error" + }, + { + "line": 1785, + "column": 25, + "stop_line": 1785, + "stop_column": 44, + "path": "src\\streamdiffusion\\stream_parameter_updater.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `BasePreprocessor` has no attribute `order`", + "concise_description": "Object of class `BasePreprocessor` has no attribute `order`", + "severity": "error" + }, + { + "line": 1786, + "column": 25, + "stop_line": 1786, + "stop_column": 46, + "path": "src\\streamdiffusion\\stream_parameter_updater.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `BasePreprocessor` has no attribute `enabled`", + "concise_description": "Object of class `BasePreprocessor` has no attribute `enabled`", + "severity": "error" + }, + { + "line": 84, + "column": 13, + "stop_line": 84, + "stop_column": 24, + "path": "src\\streamdiffusion\\tools\\compile_depth_anything_tensorrt.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Tensor` is not assignable to parameter `args` with type `tuple[Any, ...]` in function `torch.onnx.export`", + "concise_description": "Argument `Tensor` is not assignable to parameter `args` with type `tuple[Any, ...]` in function `torch.onnx.export`", + "severity": "error" + }, + { + "line": 121, + "column": 19, + "stop_line": 121, + "stop_column": 30, + "path": "src\\streamdiffusion\\tools\\compile_depth_anything_tensorrt.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `Builder` in module `tensorrt`", + "concise_description": "No attribute `Builder` in module `tensorrt`", + "severity": "error" + }, + { + "line": 123, + "column": 18, + "stop_line": 123, + "stop_column": 32, + "path": "src\\streamdiffusion\\tools\\compile_depth_anything_tensorrt.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `OnnxParser` in module `tensorrt`", + "concise_description": "No attribute `OnnxParser` in module `tensorrt`", + "severity": "error" + }, + { + "line": 134, + "column": 38, + "stop_line": 134, + "stop_column": 56, + "path": "src\\streamdiffusion\\tools\\compile_depth_anything_tensorrt.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `MemoryPoolType` in module `tensorrt`", + "concise_description": "No attribute `MemoryPoolType` in module `tensorrt`", + "severity": "error" + }, + { + "line": 137, + "column": 29, + "stop_line": 137, + "stop_column": 44, + "path": "src\\streamdiffusion\\tools\\compile_depth_anything_tensorrt.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `BuilderFlag` in module `tensorrt`", + "concise_description": "No attribute `BuilderFlag` in module `tensorrt`", + "severity": "error" + }, + { + "line": 152, + "column": 19, + "stop_line": 152, + "stop_column": 30, + "path": "src\\streamdiffusion\\tools\\compile_raft_tensorrt.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `Builder` in module `tensorrt`", + "concise_description": "No attribute `Builder` in module `tensorrt`", + "severity": "error" + }, + { + "line": 154, + "column": 18, + "stop_line": 154, + "stop_column": 32, + "path": "src\\streamdiffusion\\tools\\compile_raft_tensorrt.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `OnnxParser` in module `tensorrt`", + "concise_description": "No attribute `OnnxParser` in module `tensorrt`", + "severity": "error" + }, + { + "line": 167, + "column": 38, + "stop_line": 167, + "stop_column": 56, + "path": "src\\streamdiffusion\\tools\\compile_raft_tensorrt.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `MemoryPoolType` in module `tensorrt`", + "concise_description": "No attribute `MemoryPoolType` in module `tensorrt`", + "severity": "error" + }, + { + "line": 170, + "column": 29, + "stop_line": 170, + "stop_column": 44, + "path": "src\\streamdiffusion\\tools\\compile_raft_tensorrt.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `BuilderFlag` in module `tensorrt`", + "concise_description": "No attribute `BuilderFlag` in module `tensorrt`", + "severity": "error" + }, + { + "line": 137, + "column": 13, + "stop_line": 137, + "stop_column": 26, + "path": "src\\streamdiffusion\\tools\\gpu_profiler.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `cuda`", + "concise_description": "Object of class `NoneType` has no attribute `cuda`", + "severity": "error" + }, + { + "line": 139, + "column": 31, + "stop_line": 139, + "stop_column": 44, + "path": "src\\streamdiffusion\\tools\\gpu_profiler.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `cuda`", + "concise_description": "Object of class `NoneType` has no attribute `cuda`", + "severity": "error" + }, + { + "line": 140, + "column": 29, + "stop_line": 140, + "stop_column": 42, + "path": "src\\streamdiffusion\\tools\\gpu_profiler.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `cuda`", + "concise_description": "Object of class `NoneType` has no attribute `cuda`", + "severity": "error" + }, + { + "line": 147, + "column": 13, + "stop_line": 147, + "stop_column": 33, + "path": "src\\streamdiffusion\\tools\\gpu_profiler.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `record`", + "concise_description": "Object of class `NoneType` has no attribute `record`", + "severity": "error" + }, + { + "line": 152, + "column": 13, + "stop_line": 152, + "stop_column": 26, + "path": "src\\streamdiffusion\\tools\\gpu_profiler.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `cuda`", + "concise_description": "Object of class `NoneType` has no attribute `cuda`", + "severity": "error" + }, + { + "line": 393, + "column": 26, + "stop_line": 393, + "stop_column": 38, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `bool | None` is not assignable to parameter `use_lcm_lora` with type `bool` in function `StreamDiffusionWrapper._load_model`", + "concise_description": "Argument `bool | None` is not assignable to parameter `use_lcm_lora` with type `bool` in function `StreamDiffusionWrapper._load_model`", + "severity": "error" + }, + { + "line": 579, + "column": 17, + "stop_line": 579, + "stop_column": 34, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StableDiffusionPipeline` has no attribute `text_encoder`", + "concise_description": "Object of class `StableDiffusionPipeline` has no attribute `text_encoder`", + "severity": "error" + }, + { + "line": 582, + "column": 17, + "stop_line": 582, + "stop_column": 36, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StableDiffusionPipeline` has no attribute `text_encoder_2`", + "concise_description": "Object of class `StableDiffusionPipeline` has no attribute `text_encoder_2`", + "severity": "error" + }, + { + "line": 596, + "column": 13, + "stop_line": 596, + "stop_column": 30, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StableDiffusionPipeline` has no attribute `text_encoder`", + "concise_description": "Object of class `StableDiffusionPipeline` has no attribute `text_encoder`", + "severity": "error" + }, + { + "line": 598, + "column": 13, + "stop_line": 598, + "stop_column": 32, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StableDiffusionPipeline` has no attribute `text_encoder_2`", + "concise_description": "Object of class `StableDiffusionPipeline` has no attribute `text_encoder_2`", + "severity": "error" + }, + { + "line": 829, + "column": 20, + "stop_line": 829, + "stop_column": 63, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "bad-return", + "description": "Returned type `Image | Tensor | list[Image] | ndarray[Unknown, Unknown]` is not assignable to declared return type `Tensor | list[Tensor]`", + "concise_description": "Returned type `Image | Tensor | list[Image] | ndarray[Unknown, Unknown]` is not assignable to declared return type `Tensor | list[Tensor]`", + "severity": "error" + }, + { + "line": 832, + "column": 20, + "stop_line": 832, + "stop_column": 47, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "bad-return", + "description": "Returned type `Image | Tensor | list[Image] | ndarray[Unknown, Unknown]` is not assignable to declared return type `Tensor | list[Tensor]`", + "concise_description": "Returned type `Image | Tensor | list[Image] | ndarray[Unknown, Unknown]` is not assignable to declared return type `Tensor | list[Tensor]`", + "severity": "error" + }, + { + "line": 832, + "column": 33, + "stop_line": 832, + "stop_column": 38, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Image | Tensor | str | None` is not assignable to parameter `image` with type `Image | Tensor | str` in function `StreamDiffusionWrapper.img2img`", + "concise_description": "Argument `Image | Tensor | str | None` is not assignable to parameter `image` with type `Image | Tensor | str` in function `StreamDiffusionWrapper.img2img`", + "severity": "error" + }, + { + "line": 834, + "column": 20, + "stop_line": 834, + "stop_column": 40, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "bad-return", + "description": "Returned type `Image | Tensor | list[Image] | ndarray[Unknown, Unknown]` is not assignable to declared return type `Tensor | list[Tensor]`", + "concise_description": "Returned type `Image | Tensor | list[Image] | ndarray[Unknown, Unknown]` is not assignable to declared return type `Tensor | list[Tensor]`", + "severity": "error" + }, + { + "line": 946, + "column": 17, + "stop_line": 946, + "stop_column": 83, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "bad-assignment", + "description": "`Image | Tensor | list[Image] | ndarray[Unknown, Unknown]` is not assignable to variable `image` with type `Image | Tensor | str`", + "concise_description": "`Image | Tensor | list[Image] | ndarray[Unknown, Unknown]` is not assignable to variable `image` with type `Image | Tensor | str`", + "severity": "error" + }, + { + "line": 948, + "column": 16, + "stop_line": 948, + "stop_column": 21, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "bad-return", + "description": "Returned type `Image | Tensor | str` is not assignable to declared return type `Image | Tensor | list[Image] | ndarray[Unknown, Unknown]`", + "concise_description": "Returned type `Image | Tensor | str` is not assignable to declared return type `Image | Tensor | list[Image] | ndarray[Unknown, Unknown]`", + "severity": "error" + }, + { + "line": 1029, + "column": 20, + "stop_line": 1029, + "stop_column": 24, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "bad-return", + "description": "Returned type `None` is not assignable to declared return type `Image | Tensor | list[Image] | ndarray[Unknown, Unknown]`", + "concise_description": "Returned type `None` is not assignable to declared return type `Image | Tensor | list[Image] | ndarray[Unknown, Unknown]`", + "severity": "error" + }, + { + "line": 1056, + "column": 17, + "stop_line": 1056, + "stop_column": 39, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `record`", + "concise_description": "Object of class `NoneType` has no attribute `record`", + "severity": "error" + }, + { + "line": 1057, + "column": 17, + "stop_line": 1057, + "stop_column": 44, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `synchronize`", + "concise_description": "Object of class `NoneType` has no attribute `synchronize`", + "severity": "error" + }, + { + "line": 1075, + "column": 20, + "stop_line": 1075, + "stop_column": 85, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "bad-index", + "description": "Cannot index into `Image`\n Object of class `Image` has no attribute `__getitem__`", + "concise_description": "Cannot index into `Image`", + "severity": "error" + }, + { + "line": 1127, + "column": 26, + "stop_line": 1127, + "stop_column": 49, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `str | None` is not assignable to parameter `shm_name` with type `str` in function `cuda_link._exporter_port.FrameSpec.__init__`", + "concise_description": "Argument `str | None` is not assignable to parameter `shm_name` with type `str` in function `cuda_link._exporter_port.FrameSpec.__init__`", + "severity": "error" + }, + { + "line": 1185, + "column": 26, + "stop_line": 1185, + "stop_column": 62, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `str | None` is not assignable to parameter `shm_name` with type `str` in function `cuda_link._exporter_port.FrameSpec.__init__`", + "concise_description": "Argument `str | None` is not assignable to parameter `shm_name` with type `str` in function `cuda_link._exporter_port.FrameSpec.__init__`", + "severity": "error" + }, + { + "line": 1332, + "column": 13, + "stop_line": 1332, + "stop_column": 35, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `record`", + "concise_description": "Object of class `NoneType` has no attribute `record`", + "severity": "error" + }, + { + "line": 1333, + "column": 13, + "stop_line": 1333, + "stop_column": 40, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `synchronize`", + "concise_description": "Object of class `NoneType` has no attribute `synchronize`", + "severity": "error" + }, + { + "line": 1601, + "column": 24, + "stop_line": 1601, + "stop_column": 30, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "not-callable", + "description": "Expected a callable, got `None`", + "concise_description": "Expected a callable, got `None`", + "severity": "error" + }, + { + "line": 1601, + "column": 24, + "stop_line": 1601, + "stop_column": 51, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `to`", + "concise_description": "Object of class `NoneType` has no attribute `to`", + "severity": "error" + }, + { + "line": 1610, + "column": 32, + "stop_line": 1610, + "stop_column": 74, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "not-callable", + "description": "Expected a callable, got `None`", + "concise_description": "Expected a callable, got `None`", + "severity": "error" + }, + { + "line": 1640, + "column": 17, + "stop_line": 1640, + "stop_column": 34, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StableDiffusionXLPipeline` has no attribute `text_encoder`", + "concise_description": "Object of class `StableDiffusionXLPipeline` has no attribute `text_encoder`", + "severity": "error" + }, + { + "line": 1642, + "column": 17, + "stop_line": 1642, + "stop_column": 36, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StableDiffusionXLPipeline` has no attribute `text_encoder_2`", + "concise_description": "Object of class `StableDiffusionXLPipeline` has no attribute `text_encoder_2`", + "severity": "error" + }, + { + "line": 1645, + "column": 17, + "stop_line": 1645, + "stop_column": 26, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StableDiffusionXLPipeline` has no attribute `unet`", + "concise_description": "Object of class `StableDiffusionXLPipeline` has no attribute `unet`", + "severity": "error" + }, + { + "line": 1647, + "column": 17, + "stop_line": 1647, + "stop_column": 25, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StableDiffusionXLPipeline` has no attribute `vae`", + "concise_description": "Object of class `StableDiffusionXLPipeline` has no attribute `vae`", + "severity": "error" + }, + { + "line": 1653, + "column": 41, + "stop_line": 1653, + "stop_column": 50, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StableDiffusionXLPipeline` has no attribute `unet`", + "concise_description": "Object of class `StableDiffusionXLPipeline` has no attribute `unet`", + "severity": "error" + }, + { + "line": 1708, + "column": 60, + "stop_line": 1708, + "stop_column": 69, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StableDiffusionXLPipeline` has no attribute `unet`", + "concise_description": "Object of class `StableDiffusionXLPipeline` has no attribute `unet`", + "severity": "error" + }, + { + "line": 1713, + "column": 18, + "stop_line": 1713, + "stop_column": 22, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline | diffusers.utils.dummy_torch_and_transformers_objects.StableDiffusionXLPipeline | Unknown` is not assignable to parameter `pipe` with type `diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline | diffusers.utils.dummy_torch_and_transformers_objects.StableDiffusionPipeline` in function `streamdiffusion.pipeline.StreamDiffusion.__init__`", + "concise_description": "Argument `diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline | diffusers.utils.dummy_torch_and_transformers_objects.StableDiffusionXLPipeline | Unknown` is not assignable to parameter `pipe` with type `diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline | diffusers.utils.dummy_torch_and_transformers_objects.StableDiffusionPipeline` in function `streamdiffusion.pipeline.StreamDiffusion.__init__`", + "severity": "error" + }, + { + "line": 1742, + "column": 17, + "stop_line": 1742, + "stop_column": 26, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StableDiffusionXLPipeline` has no attribute `unet`", + "concise_description": "Object of class `StableDiffusionXLPipeline` has no attribute `unet`", + "severity": "error" + }, + { + "line": 1760, + "column": 17, + "stop_line": 1760, + "stop_column": 26, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StableDiffusionXLPipeline` has no attribute `unet`", + "concise_description": "Object of class `StableDiffusionXLPipeline` has no attribute `unet`", + "severity": "error" + }, + { + "line": 1820, + "column": 21, + "stop_line": 1820, + "stop_column": 42, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StableDiffusionPipeline` has no attribute `fuse_lora`", + "concise_description": "Object of class `StableDiffusionPipeline` has no attribute `fuse_lora`", + "severity": "error" + }, + { + "line": 1823, + "column": 17, + "stop_line": 1823, + "stop_column": 48, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StableDiffusionPipeline` has no attribute `unload_lora_weights`", + "concise_description": "Object of class `StableDiffusionPipeline` has no attribute `unload_lora_weights`", + "severity": "error" + }, + { + "line": 1830, + "column": 21, + "stop_line": 1830, + "stop_column": 52, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StableDiffusionPipeline` has no attribute `unload_lora_weights`", + "concise_description": "Object of class `StableDiffusionPipeline` has no attribute `unload_lora_weights`", + "severity": "error" + }, + { + "line": 1849, + "column": 30, + "stop_line": 1849, + "stop_column": 72, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `to`", + "concise_description": "Object of class `NoneType` has no attribute `to`", + "severity": "error" + }, + { + "line": 1853, + "column": 30, + "stop_line": 1853, + "stop_column": 77, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `to`", + "concise_description": "Object of class `NoneType` has no attribute `to`", + "severity": "error" + }, + { + "line": 1857, + "column": 17, + "stop_line": 1857, + "stop_column": 25, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StableDiffusionXLPipeline` has no attribute `vae`", + "concise_description": "Object of class `StableDiffusionXLPipeline` has no attribute `vae`", + "severity": "error" + }, + { + "line": 1861, + "column": 17, + "stop_line": 1861, + "stop_column": 71, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StableDiffusionPipeline` has no attribute `enable_xformers_memory_efficient_attention`", + "concise_description": "Object of class `StableDiffusionPipeline` has no attribute `enable_xformers_memory_efficient_attention`", + "severity": "error" + }, + { + "line": 1883, + "column": 48, + "stop_line": 1883, + "stop_column": 58, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Path | str | None` is not assignable to parameter `engine_dir` with type `str` in function `streamdiffusion.acceleration.tensorrt.engine_manager.EngineManager.__init__`", + "concise_description": "Argument `Path | str | None` is not assignable to parameter `engine_dir` with type `str` in function `streamdiffusion.acceleration.tensorrt.engine_manager.EngineManager.__init__`", + "severity": "error" + }, + { + "line": 1990, + "column": 31, + "stop_line": 1990, + "stop_column": 40, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "unbound-name", + "description": "`is_faceid` may be uninitialized", + "concise_description": "`is_faceid` may be uninitialized", + "severity": "error" + }, + { + "line": 2124, + "column": 29, + "stop_line": 2124, + "stop_column": 53, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StreamDiffusion` has no attribute `_ipadapter_module`", + "concise_description": "Object of class `StreamDiffusion` has no attribute `_ipadapter_module`", + "severity": "error" + }, + { + "line": 2158, + "column": 68, + "stop_line": 2158, + "stop_column": 90, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "unbound-name", + "description": "`_saved_unet_processors` is uninitialized", + "concise_description": "`_saved_unet_processors` is uninitialized", + "severity": "error" + }, + { + "line": 2178, + "column": 64, + "stop_line": 2178, + "stop_column": 86, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "unbound-name", + "description": "`_saved_unet_processors` is uninitialized", + "concise_description": "`_saved_unet_processors` is uninitialized", + "severity": "error" + }, + { + "line": 2222, + "column": 35, + "stop_line": 2222, + "stop_column": 79, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `int | None` is not assignable to parameter `num_ip_layers` with type `int` in function `streamdiffusion.acceleration.tensorrt.models.models.UNet.__init__`", + "concise_description": "Argument `int | None` is not assignable to parameter `num_ip_layers` with type `int` in function `streamdiffusion.acceleration.tensorrt.models.models.UNet.__init__`", + "severity": "error" + }, + { + "line": 2246, + "column": 41, + "stop_line": 2246, + "stop_column": 60, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `list[list[Unknown]] | list[Unknown]` is not assignable to parameter `kvo_cache_structure` with type `list[int] | None` in function `streamdiffusion.acceleration.tensorrt.export_wrappers.unet_unified_export.UnifiedExportWrapper.__init__`", + "concise_description": "Argument `list[list[Unknown]] | list[Unknown]` is not assignable to parameter `kvo_cache_structure` with type `list[int] | None` in function `streamdiffusion.acceleration.tensorrt.export_wrappers.unet_unified_export.UnifiedExportWrapper.__init__`", + "severity": "error" + }, + { + "line": 2393, + "column": 56, + "stop_line": 2393, + "stop_column": 67, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "bad-typed-dict-key", + "description": "`diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline | diffusers.utils.dummy_torch_and_transformers_objects.StableDiffusionPipeline` is not assignable to TypedDict key `pipe_ref` with type `bool | int`", + "concise_description": "`diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline | diffusers.utils.dummy_torch_and_transformers_objects.StableDiffusionPipeline` is not assignable to TypedDict key `pipe_ref` with type `bool | int`", + "severity": "error" + }, + { + "line": 2398, + "column": 66, + "stop_line": 2398, + "stop_column": 91, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "bad-typed-dict-key", + "description": "`float` is not assignable to TypedDict key `fp8_guidance_scale` with type `bool | int`", + "concise_description": "`float` is not assignable to TypedDict key `fp8_guidance_scale` with type `bool | int`", + "severity": "error" + }, + { + "line": 2403, + "column": 65, + "stop_line": 2403, + "stop_column": 106, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "bad-typed-dict-key", + "description": "`int | None` is not assignable to TypedDict key `fp8_num_ip_layers` with type `bool | int`", + "concise_description": "`int | None` is not assignable to TypedDict key `fp8_num_ip_layers` with type `bool | int`", + "severity": "error" + }, + { + "line": 2489, + "column": 29, + "stop_line": 2489, + "stop_column": 57, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StableDiffusionPipeline` has no attribute `vae_scale_factor`", + "concise_description": "Object of class `StableDiffusionPipeline` has no attribute `vae_scale_factor`", + "severity": "error" + }, + { + "line": 2492, + "column": 25, + "stop_line": 2492, + "stop_column": 42, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `AutoencoderKLEngine` has no attribute `config`", + "concise_description": "Object of class `AutoencoderKLEngine` has no attribute `config`", + "severity": "error" + }, + { + "line": 2493, + "column": 25, + "stop_line": 2493, + "stop_column": 41, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `AutoencoderKLEngine` has no attribute `dtype`", + "concise_description": "Object of class `AutoencoderKLEngine` has no attribute `dtype`", + "severity": "error" + }, + { + "line": 2535, + "column": 38, + "stop_line": 2535, + "stop_column": 61, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `str | None` is not assignable to parameter `model_id_or_path` with type `str` in function `streamdiffusion.acceleration.tensorrt.engine_manager.EngineManager.get_engine_path`", + "concise_description": "Argument `str | None` is not assignable to parameter `model_id_or_path` with type `str` in function `streamdiffusion.acceleration.tensorrt.engine_manager.EngineManager.get_engine_path`", + "severity": "error" + }, + { + "line": 2568, + "column": 29, + "stop_line": 2568, + "stop_column": 48, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Path` is not assignable to parameter `filepath` with type `str` in function `streamdiffusion.acceleration.tensorrt.runtime_engines.unet_engine.NSFWDetectorEngine.__init__`", + "concise_description": "Argument `Path` is not assignable to parameter `filepath` with type `str` in function `streamdiffusion.acceleration.tensorrt.runtime_engines.unet_engine.NSFWDetectorEngine.__init__`", + "severity": "error" + }, + { + "line": 2614, + "column": 17, + "stop_line": 2614, + "stop_column": 42, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StreamDiffusion` has no attribute `_controlnet_module`", + "concise_description": "Object of class `StreamDiffusion` has no attribute `_controlnet_module`", + "severity": "error" + }, + { + "line": 2630, + "column": 42, + "stop_line": 2630, + "stop_column": 56, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "unbound-name", + "description": "`engine_manager` may be uninitialized", + "concise_description": "`engine_manager` may be uninitialized", + "severity": "error" + }, + { + "line": 2637, + "column": 49, + "stop_line": 2637, + "stop_column": 60, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "unbound-name", + "description": "`cuda_stream` may be uninitialized", + "concise_description": "`cuda_stream` may be uninitialized", + "severity": "error" + }, + { + "line": 2643, + "column": 49, + "stop_line": 2643, + "stop_column": 60, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "unbound-name", + "description": "`load_engine` may be uninitialized", + "concise_description": "`load_engine` may be uninitialized", + "severity": "error" + }, + { + "line": 2658, + "column": 29, + "stop_line": 2658, + "stop_column": 54, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StreamDiffusion` has no attribute `controlnet_engines`", + "concise_description": "Object of class `StreamDiffusion` has no attribute `controlnet_engines`", + "severity": "error" + }, + { + "line": 2712, + "column": 17, + "stop_line": 2712, + "stop_column": 41, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StreamDiffusion` has no attribute `_ipadapter_module`", + "concise_description": "Object of class `StreamDiffusion` has no attribute `_ipadapter_module`", + "severity": "error" + }, + { + "line": 2724, + "column": 56, + "stop_line": 2724, + "stop_column": 83, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "unbound-name", + "description": "`_saved_unet_processors_post` is uninitialized", + "concise_description": "`_saved_unet_processors_post` is uninitialized", + "severity": "error" + }, + { + "line": 2752, + "column": 17, + "stop_line": 2752, + "stop_column": 51, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StreamDiffusion` has no attribute `_image_preprocessing_module`", + "concise_description": "Object of class `StreamDiffusion` has no attribute `_image_preprocessing_module`", + "severity": "error" + }, + { + "line": 2764, + "column": 17, + "stop_line": 2764, + "stop_column": 52, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StreamDiffusion` has no attribute `_image_postprocessing_module`", + "concise_description": "Object of class `StreamDiffusion` has no attribute `_image_postprocessing_module`", + "severity": "error" + }, + { + "line": 2776, + "column": 17, + "stop_line": 2776, + "stop_column": 52, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StreamDiffusion` has no attribute `_latent_preprocessing_module`", + "concise_description": "Object of class `StreamDiffusion` has no attribute `_latent_preprocessing_module`", + "severity": "error" + }, + { + "line": 2788, + "column": 17, + "stop_line": 2788, + "stop_column": 53, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StreamDiffusion` has no attribute `_latent_postprocessing_module`", + "concise_description": "Object of class `StreamDiffusion` has no attribute `_latent_postprocessing_module`", + "severity": "error" + }, + { + "line": 2811, + "column": 16, + "stop_line": 2811, + "stop_column": 52, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StreamDiffusion` has no attribute `get_last_processed_image`", + "concise_description": "Object of class `StreamDiffusion` has no attribute `get_last_processed_image`", + "severity": "error" + }, + { + "line": 2819, + "column": 13, + "stop_line": 2819, + "stop_column": 44, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StreamDiffusion` has no attribute `cleanup_controlnets`", + "concise_description": "Object of class `StreamDiffusion` has no attribute `cleanup_controlnets`", + "severity": "error" + }, + { + "line": 2828, + "column": 13, + "stop_line": 2828, + "stop_column": 43, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StreamDiffusion` has no attribute `_controlnet_module`", + "concise_description": "Object of class `StreamDiffusion` has no attribute `_controlnet_module`", + "severity": "error" + }, + { + "line": 2889, + "column": 23, + "stop_line": 2889, + "stop_column": 59, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `type`", + "concise_description": "Object of class `NoneType` has no attribute `type`", + "severity": "error" + }, + { + "line": 3044, + "column": 29, + "stop_line": 3044, + "stop_column": 63, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StreamDiffusion` has no attribute `controlnet_engine_pool`", + "concise_description": "Object of class `StreamDiffusion` has no attribute `controlnet_engine_pool`", + "severity": "error" + }, + { + "line": 3059, + "column": 27, + "stop_line": 3059, + "stop_column": 31, + "path": "src\\streamdiffusion\\wrapper.py", + "code": -2, + "name": "bad-assignment", + "description": "`None` is not assignable to attribute `stream` with type `StreamDiffusion`", + "concise_description": "`None` is not assignable to attribute `stream` with type `StreamDiffusion`", + "severity": "error" + }, + { + "line": 51, + "column": 26, + "stop_line": 51, + "stop_column": 38, + "path": "tests\\unit\\test_controlnet_residual_merge.py", + "code": -2, + "name": "bad-assignment", + "description": "`list[_FakeCN]` is not assignable to attribute `controlnets` with type `list[ControlNetModel | None]`", + "concise_description": "`list[_FakeCN]` is not assignable to attribute `controlnets` with type `list[ControlNetModel | None]`", + "severity": "error" + }, + { + "line": 155, + "column": 30, + "stop_line": 155, + "stop_column": 36, + "path": "tests\\unit\\test_controlnet_residual_merge.py", + "code": -2, + "name": "bad-assignment", + "description": "`list[_FakeCN]` is not assignable to attribute `controlnets` with type `list[ControlNetModel | None]`", + "concise_description": "`list[_FakeCN]` is not assignable to attribute `controlnets` with type `list[ControlNetModel | None]`", + "severity": "error" + }, + { + "line": 33, + "column": 5, + "stop_line": 33, + "stop_column": 25, + "path": "tests\\unit\\test_derived_tensor_sync.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `LCMScheduler` has no attribute `alphas_cumprod`", + "concise_description": "Object of class `LCMScheduler` has no attribute `alphas_cumprod`", + "severity": "error" + }, + { + "line": 43, + "column": 5, + "stop_line": 43, + "stop_column": 55, + "path": "tests\\unit\\test_derived_tensor_sync.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `LCMScheduler` has no attribute `get_scalings_for_boundary_condition_discrete`", + "concise_description": "Object of class `LCMScheduler` has no attribute `get_scalings_for_boundary_condition_discrete`", + "severity": "error" + }, + { + "line": 125, + "column": 5, + "stop_line": 125, + "stop_column": 18, + "path": "tests\\unit\\test_derived_tensor_sync.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StreamParameterUpdater` has no attribute `_lock`", + "concise_description": "Object of class `StreamParameterUpdater` has no attribute `_lock`", + "severity": "error" + }, + { + "line": 56, + "column": 9, + "stop_line": 56, + "stop_column": 26, + "path": "tests\\unit\\test_ipc_producer_stream.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `ModuleType` has no attribute `Tensor`", + "concise_description": "Object of class `ModuleType` has no attribute `Tensor`", + "severity": "error" + }, + { + "line": 61, + "column": 9, + "stop_line": 61, + "stop_column": 33, + "path": "tests\\unit\\test_ipc_producer_stream.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `ModuleType` has no attribute `current_stream`", + "concise_description": "Object of class `ModuleType` has no attribute `current_stream`", + "severity": "error" + }, + { + "line": 62, + "column": 9, + "stop_line": 62, + "stop_column": 24, + "path": "tests\\unit\\test_ipc_producer_stream.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `ModuleType` has no attribute `Event`", + "concise_description": "Object of class `ModuleType` has no attribute `Event`", + "severity": "error" + }, + { + "line": 63, + "column": 9, + "stop_line": 63, + "stop_column": 25, + "path": "tests\\unit\\test_ipc_producer_stream.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `ModuleType` has no attribute `Stream`", + "concise_description": "Object of class `ModuleType` has no attribute `Stream`", + "severity": "error" + }, + { + "line": 64, + "column": 9, + "stop_line": 64, + "stop_column": 24, + "path": "tests\\unit\\test_ipc_producer_stream.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `ModuleType` has no attribute `cuda`", + "concise_description": "Object of class `ModuleType` has no attribute `cuda`", + "severity": "error" + }, + { + "line": 77, + "column": 9, + "stop_line": 77, + "stop_column": 32, + "path": "tests\\unit\\test_ipc_producer_stream.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `ModuleType` has no attribute `GpuFrame`", + "concise_description": "Object of class `ModuleType` has no attribute `GpuFrame`", + "severity": "error" + }, + { + "line": 78, + "column": 9, + "stop_line": 78, + "stop_column": 36, + "path": "tests\\unit\\test_ipc_producer_stream.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `ModuleType` has no attribute `FrameOutcome`", + "concise_description": "Object of class `ModuleType` has no attribute `FrameOutcome`", + "severity": "error" + }, + { + "line": 124, + "column": 30, + "stop_line": 124, + "stop_column": 48, + "path": "tests\\unit\\test_ipc_producer_stream.py", + "code": -2, + "name": "missing-attribute", + "description": "Class `FakeGpuFrame` has no class attribute `calls`", + "concise_description": "Class `FakeGpuFrame` has no class attribute `calls`", + "severity": "error" + }, + { + "line": 125, + "column": 16, + "stop_line": 125, + "stop_column": 34, + "path": "tests\\unit\\test_ipc_producer_stream.py", + "code": -2, + "name": "missing-attribute", + "description": "Class `FakeGpuFrame` has no class attribute `calls`", + "concise_description": "Class `FakeGpuFrame` has no class attribute `calls`", + "severity": "error" + }, + { + "line": 63, + "column": 20, + "stop_line": 63, + "stop_column": 36, + "path": "tests\\unit\\test_l2tc_dynamic_shapes.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `_FakeModelData` is not assignable to parameter `model_data` with type `BaseModel` in function `streamdiffusion.acceleration.tensorrt.utilities.build_engine`", + "concise_description": "Argument `_FakeModelData` is not assignable to parameter `model_data` with type `BaseModel` in function `streamdiffusion.acceleration.tensorrt.utilities.build_engine`", + "severity": "error" + }, + { + "line": 56, + "column": 50, + "stop_line": 56, + "stop_column": 62, + "path": "tests\\unit\\test_param_updater_binding.py", + "code": -2, + "name": "bad-assignment", + "description": "`(self: Unknown, s: Unknown) -> None` is not assignable to attribute `attach_orchestrator` with type `(self: StreamParameterUpdater, stream: Unknown) -> None`\n Positional parameter name mismatch: got `s`, want `stream`", + "concise_description": "`(self: Unknown, s: Unknown) -> None` is not assignable to attribute `attach_orchestrator` with type `(self: StreamParameterUpdater, stream: Unknown) -> None`", + "severity": "error" + }, + { + "line": 97, + "column": 40, + "stop_line": 97, + "stop_column": 45, + "path": "tests\\unit\\test_param_updater_binding.py", + "code": -2, + "name": "bad-argument-count", + "description": "Expected 1 positional argument, got 3 in function `streamdiffusion.stream_parameter_updater.StreamParameterUpdater.__init__`", + "concise_description": "Expected 1 positional argument, got 3 in function `streamdiffusion.stream_parameter_updater.StreamParameterUpdater.__init__`", + "severity": "error" + }, + { + "line": 76, + "column": 16, + "stop_line": 76, + "stop_column": 41, + "path": "tests\\unit\\test_phase3_correctness.py", + "code": -2, + "name": "bad-assignment", + "description": "`_FakeStream` is not assignable to attribute `stream` with type `StreamDiffusion`", + "concise_description": "`_FakeStream` is not assignable to attribute `stream` with type `StreamDiffusion`", + "severity": "error" + }, + { + "line": 91, + "column": 16, + "stop_line": 91, + "stop_column": 40, + "path": "tests\\unit\\test_phase3_correctness.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StreamDiffusion` has no attribute `captured_kwargs`", + "concise_description": "Object of class `StreamDiffusion` has no attribute `captured_kwargs`", + "severity": "error" + }, + { + "line": 92, + "column": 16, + "stop_line": 92, + "stop_column": 44, + "path": "tests\\unit\\test_phase3_correctness.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `object` has no attribute `get`", + "concise_description": "Object of class `object` has no attribute `get`", + "severity": "error" + }, + { + "line": 97, + "column": 34, + "stop_line": 97, + "stop_column": 55, + "path": "tests\\unit\\test_phase3_correctness.py", + "code": -2, + "name": "bad-assignment", + "description": "`(**kwargs: Unknown) -> None` is not assignable to attribute `update_stream_params` with type `(self: StreamDiffusionWrapper, num_inference_steps: int | None = None, guidance_scale: float | None = None, delta: float | None = None, t_index_list: list[int] | None = None, seed: int | None = None, prompt_list: list[tuple[str, float]] | None = None, negative_prompt: str | None = None, prompt_interpolation_method: PromptInterpolationMethod = 'slerp', normalize_prompt_weights: bool | None = None, seed_list: list[tuple[int, float]] | None = None, seed_interpolation_method: SeedInterpolationMethod = 'linear', normalize_seed_weights: bool | None = None, controlnet_config: list[dict[str, Any]] | None = None, ipadapter_config: dict[str, Any] | None = None, image_preprocessing_config: list[dict[str, Any]] | None = None, image_postprocessing_config: list[dict[str, Any]] | None = None, latent_preprocessing_config: list[dict[str, Any]] | None = None, latent_postprocessing_config: list[dict[str, Any]] | None = None, use_safety_checker: bool | None = None, safety_checker_threshold: float | None = None, cache_maxframes: int | None = None, cache_interval: int | None = None, cn_cache_interval: int | None = None, fi_strength: float | None = None, fi_threshold: float | None = None) -> None`", + "concise_description": "`(**kwargs: Unknown) -> None` is not assignable to attribute `update_stream_params` with type `(self: StreamDiffusionWrapper, num_inference_steps: int | None = None, guidance_scale: float | None = None, delta: float | None = None, t_index_list: list[int] | None = None, seed: int | None = None, prompt_list: list[tuple[str, float]] | None = None, negative_prompt: str | None = None, prompt_interpolation_method: PromptInterpolationMethod = 'slerp', normalize_prompt_weights: bool | None = None, seed_list: list[tuple[int, float]] | None = None, seed_interpolation_method: SeedInterpolationMethod = 'linear', normalize_seed_weights: bool | None = None, controlnet_config: list[dict[str, Any]] | None = None, ipadapter_config: dict[str, Any] | None = None, image_preprocessing_config: list[dict[str, Any]] | None = None, image_postprocessing_config: list[dict[str, Any]] | None = None, latent_preprocessing_config: list[dict[str, Any]] | None = None, latent_postprocessing_config: list[dict[str, Any]] | None = None, use_safety_checker: bool | None = None, safety_checker_threshold: float | None = None, cache_maxframes: int | None = None, cache_interval: int | None = None, cn_cache_interval: int | None = None, fi_strength: float | None = None, fi_threshold: float | None = None) -> None`", + "severity": "error" + }, + { + "line": 99, + "column": 16, + "stop_line": 99, + "stop_column": 40, + "path": "tests\\unit\\test_phase3_correctness.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StreamDiffusion` has no attribute `captured_kwargs`", + "concise_description": "Object of class `StreamDiffusion` has no attribute `captured_kwargs`", + "severity": "error" + }, + { + "line": 100, + "column": 16, + "stop_line": 100, + "stop_column": 44, + "path": "tests\\unit\\test_phase3_correctness.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `object` has no attribute `get`", + "concise_description": "Object of class `object` has no attribute `get`", + "severity": "error" + }, + { + "line": 107, + "column": 20, + "stop_line": 107, + "stop_column": 31, + "path": "tests\\unit\\test_phase3_correctness.py", + "code": -2, + "name": "bad-assignment", + "description": "`_FakeStream` is not assignable to attribute `stream` with type `StreamDiffusion`", + "concise_description": "`_FakeStream` is not assignable to attribute `stream` with type `StreamDiffusion`", + "severity": "error" + }, + { + "line": 111, + "column": 16, + "stop_line": 111, + "stop_column": 40, + "path": "tests\\unit\\test_phase3_correctness.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `StreamDiffusion` has no attribute `captured_kwargs`", + "concise_description": "Object of class `StreamDiffusion` has no attribute `captured_kwargs`", + "severity": "error" + }, + { + "line": 139, + "column": 16, + "stop_line": 139, + "stop_column": 28, + "path": "tests\\unit\\test_phase3_correctness.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `Image` has no attribute `data_ptr`\nObject of class `list` has no attribute `data_ptr`\nObject of class `ndarray` has no attribute `data_ptr`", + "concise_description": "Object of class `Image` has no attribute `data_ptr`\nObject of class `list` has no attribute `data_ptr`\nObject of class `ndarray` has no attribute `data_ptr`", + "severity": "error" + }, + { + "line": 140, + "column": 28, + "stop_line": 140, + "stop_column": 31, + "path": "tests\\unit\\test_phase3_correctness.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Image | Tensor | list[Image] | ndarray[Unknown, Unknown]` is not assignable to parameter `input` with type `Tensor` in function `torch._C._VariableFunctions.equal`", + "concise_description": "Argument `Image | Tensor | list[Image] | ndarray[Unknown, Unknown]` is not assignable to parameter `input` with type `Tensor` in function `torch._C._VariableFunctions.equal`", + "severity": "error" + }, + { + "line": 145, + "column": 32, + "stop_line": 145, + "stop_column": 35, + "path": "tests\\unit\\test_phase3_correctness.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Image | Tensor | list[Image] | ndarray[Unknown, Unknown]` is not assignable to parameter `input` with type `Tensor` in function `torch._C._VariableFunctions.equal`", + "concise_description": "Argument `Image | Tensor | list[Image] | ndarray[Unknown, Unknown]` is not assignable to parameter `input` with type `Tensor` in function `torch._C._VariableFunctions.equal`", + "severity": "error" + }, + { + "line": 51, + "column": 50, + "stop_line": 51, + "stop_column": 62, + "path": "tests\\unit\\test_prompt_interpolation.py", + "code": -2, + "name": "bad-assignment", + "description": "`(self: Unknown, s: Unknown) -> None` is not assignable to attribute `attach_orchestrator` with type `(self: StreamParameterUpdater, stream: Unknown) -> None`\n Positional parameter name mismatch: got `s`, want `stream`", + "concise_description": "`(self: Unknown, s: Unknown) -> None` is not assignable to attribute `attach_orchestrator` with type `(self: StreamParameterUpdater, stream: Unknown) -> None`", + "severity": "error" + }, + { + "line": 283, + "column": 41, + "stop_line": 283, + "stop_column": 58, + "path": "tests\\unit\\test_prompt_interpolation.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Literal['cosine_weignted']` is not assignable to parameter `prompt_interpolation_method` with type `Literal['cosine_weighted', 'linear', 'slerp']` in function `streamdiffusion.stream_parameter_updater.StreamParameterUpdater._apply_prompt_blending`", + "concise_description": "Argument `Literal['cosine_weignted']` is not assignable to parameter `prompt_interpolation_method` with type `Literal['cosine_weighted', 'linear', 'slerp']` in function `streamdiffusion.stream_parameter_updater.StreamParameterUpdater._apply_prompt_blending`", + "severity": "error" + }, + { + "line": 296, + "column": 45, + "stop_line": 296, + "stop_column": 62, + "path": "tests\\unit\\test_prompt_interpolation.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Literal['cosine_weignted']` is not assignable to parameter `prompt_interpolation_method` with type `Literal['cosine_weighted', 'linear', 'slerp']` in function `streamdiffusion.stream_parameter_updater.StreamParameterUpdater._apply_prompt_blending`", + "concise_description": "Argument `Literal['cosine_weignted']` is not assignable to parameter `prompt_interpolation_method` with type `Literal['cosine_weighted', 'linear', 'slerp']` in function `streamdiffusion.stream_parameter_updater.StreamParameterUpdater._apply_prompt_blending`", + "severity": "error" + }, + { + "line": 297, + "column": 45, + "stop_line": 297, + "stop_column": 62, + "path": "tests\\unit\\test_prompt_interpolation.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Literal['cosine_weignted']` is not assignable to parameter `prompt_interpolation_method` with type `Literal['cosine_weighted', 'linear', 'slerp']` in function `streamdiffusion.stream_parameter_updater.StreamParameterUpdater._apply_prompt_blending`", + "concise_description": "Argument `Literal['cosine_weignted']` is not assignable to parameter `prompt_interpolation_method` with type `Literal['cosine_weighted', 'linear', 'slerp']` in function `streamdiffusion.stream_parameter_updater.StreamParameterUpdater._apply_prompt_blending`", + "severity": "error" + }, + { + "line": 47, + "column": 38, + "stop_line": 47, + "stop_column": 51, + "path": "tests\\unit\\test_safety_checker.py", + "code": -2, + "name": "bad-assignment", + "description": "`str` is not assignable to attribute `safety_checker_fallback_type` with type `Literal['blank', 'previous']`", + "concise_description": "`str` is not assignable to attribute `safety_checker_fallback_type` with type `Literal['blank', 'previous']`", + "severity": "error" + }, + { + "line": 127, + "column": 28, + "stop_line": 127, + "stop_column": 48, + "path": "tests\\unit\\test_safety_checker.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Tensor | None` is not assignable to parameter `input` with type `Tensor` in function `torch._C._VariableFunctions.equal`", + "concise_description": "Argument `Tensor | None` is not assignable to parameter `input` with type `Tensor` in function `torch._C._VariableFunctions.equal`", + "severity": "error" + }, + { + "line": 239, + "column": 20, + "stop_line": 239, + "stop_column": 39, + "path": "tests\\unit\\test_safety_checker.py", + "code": -2, + "name": "bad-assignment", + "description": "`device` is not assignable to attribute `device` with type `Literal['cpu', 'cuda']`", + "concise_description": "`device` is not assignable to attribute `device` with type `Literal['cpu', 'cuda']`", + "severity": "error" + }, + { + "line": 250, + "column": 20, + "stop_line": 250, + "stop_column": 33, + "path": "tests\\unit\\test_safety_checker.py", + "code": -2, + "name": "bad-assignment", + "description": "`TestApplySafetyChecker.test_skip_diffusion_routes_through_safety_checker._FakeStream` is not assignable to attribute `stream` with type `StreamDiffusion`", + "concise_description": "`TestApplySafetyChecker.test_skip_diffusion_routes_through_safety_checker._FakeStream` is not assignable to attribute `stream` with type `StreamDiffusion`", + "severity": "error" + }, + { + "line": 253, + "column": 31, + "stop_line": 253, + "stop_column": 42, + "path": "tests\\unit\\test_safety_checker.py", + "code": -2, + "name": "bad-assignment", + "description": "`(t: Unknown) -> Unknown` is not assignable to attribute `_normalize_on_gpu` with type `(self: StreamDiffusionWrapper, image_tensor: Tensor) -> Tensor`\n Positional parameter name mismatch: got `t`, want `image_tensor`", + "concise_description": "`(t: Unknown) -> Unknown` is not assignable to attribute `_normalize_on_gpu` with type `(self: StreamDiffusionWrapper, image_tensor: Tensor) -> Tensor`", + "severity": "error" + }, + { + "line": 254, + "column": 33, + "stop_line": 254, + "stop_column": 44, + "path": "tests\\unit\\test_safety_checker.py", + "code": -2, + "name": "bad-assignment", + "description": "`(t: Unknown) -> Unknown` is not assignable to attribute `_denormalize_on_gpu` with type `(self: StreamDiffusionWrapper, image_tensor: Tensor) -> Tensor`\n Positional parameter name mismatch: got `t`, want `image_tensor`", + "concise_description": "`(t: Unknown) -> Unknown` is not assignable to attribute `_denormalize_on_gpu` with type `(self: StreamDiffusionWrapper, image_tensor: Tensor) -> Tensor`", + "severity": "error" + }, + { + "line": 257, + "column": 31, + "stop_line": 257, + "stop_column": 60, + "path": "tests\\unit\\test_safety_checker.py", + "code": -2, + "name": "bad-assignment", + "description": "`(t: Unknown, output_type: Unknown = ...) -> Unknown` is not assignable to attribute `postprocess_image` with type `(self: StreamDiffusionWrapper, image_tensor: Tensor, output_type: str = 'pil') -> Image | Tensor | list[Image] | ndarray[Unknown, Unknown]`\n Positional parameter name mismatch: got `t`, want `image_tensor`", + "concise_description": "`(t: Unknown, output_type: Unknown = ...) -> Unknown` is not assignable to attribute `postprocess_image` with type `(self: StreamDiffusionWrapper, image_tensor: Tensor, output_type: str = 'pil') -> Image | Tensor | list[Image] | ndarray[Unknown, Unknown]`", + "severity": "error" + }, + { + "line": 268, + "column": 18, + "stop_line": 268, + "stop_column": 42, + "path": "tests\\unit\\test_safety_checker.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `float` has no attribute `clamp`\nObject of class `ndarray` has no attribute `clamp`", + "concise_description": "Object of class `float` has no attribute `clamp`\nObject of class `ndarray` has no attribute `clamp`", + "severity": "error" + }, + { + "line": 268, + "column": 19, + "stop_line": 268, + "stop_column": 29, + "path": "tests\\unit\\test_safety_checker.py", + "code": -2, + "name": "unsupported-operation", + "description": "`/` is not supported between `Image` and `Literal[2]`\n Argument `Image` is not assignable to parameter `value` with type `int` in function `int.__rtruediv__`", + "concise_description": "`/` is not supported between `Image` and `Literal[2]`", + "severity": "error" + }, + { + "line": 268, + "column": 19, + "stop_line": 268, + "stop_column": 29, + "path": "tests\\unit\\test_safety_checker.py", + "code": -2, + "name": "unsupported-operation", + "description": "`/` is not supported between `list[Image]` and `Literal[2]`\n Argument `list[Image]` is not assignable to parameter `value` with type `int` in function `int.__rtruediv__`", + "concise_description": "`/` is not supported between `list[Image]` and `Literal[2]`", + "severity": "error" + }, + { + "line": 193, + "column": 21, + "stop_line": 193, + "stop_column": 53, + "path": "tests\\unit\\test_sync_free_output_5_2.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `data_ptr`", + "concise_description": "Object of class `NoneType` has no attribute `data_ptr`", + "severity": "error" + }, + { + "line": 195, + "column": 21, + "stop_line": 195, + "stop_column": 53, + "path": "tests\\unit\\test_sync_free_output_5_2.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `data_ptr`", + "concise_description": "Object of class `NoneType` has no attribute `data_ptr`", + "severity": "error" + }, + { + "line": 74, + "column": 14, + "stop_line": 74, + "stop_column": 30, + "path": "tests\\unit\\test_trt_engine_guards.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `TensorIOMode` in module `tensorrt`", + "concise_description": "No attribute `TensorIOMode` in module `tensorrt`", + "severity": "error" + }, + { + "line": 74, + "column": 38, + "stop_line": 74, + "stop_column": 54, + "path": "tests\\unit\\test_trt_engine_guards.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `TensorIOMode` in module `tensorrt`", + "concise_description": "No attribute `TensorIOMode` in module `tensorrt`", + "severity": "error" + }, + { + "line": 162, + "column": 9, + "stop_line": 162, + "stop_column": 37, + "path": "tests\\unit\\test_trt_engine_guards.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_shape`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_shape`", + "severity": "error" + }, + { + "line": 180, + "column": 9, + "stop_line": 180, + "stop_column": 36, + "path": "tests\\unit\\test_trt_engine_guards.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `set_input_shape`", + "concise_description": "Object of class `NoneType` has no attribute `set_input_shape`", + "severity": "error" + }, + { + "line": 205, + "column": 9, + "stop_line": 205, + "stop_column": 34, + "path": "tests\\unit\\test_trt_engine_guards.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `num_io_tensors`", + "concise_description": "Object of class `NoneType` has no attribute `num_io_tensors`", + "severity": "error" + }, + { + "line": 206, + "column": 9, + "stop_line": 206, + "stop_column": 35, + "path": "tests\\unit\\test_trt_engine_guards.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_name`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_name`", + "severity": "error" + }, + { + "line": 207, + "column": 9, + "stop_line": 207, + "stop_column": 35, + "path": "tests\\unit\\test_trt_engine_guards.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_mode`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_mode`", + "severity": "error" + }, + { + "line": 207, + "column": 60, + "stop_line": 207, + "stop_column": 76, + "path": "tests\\unit\\test_trt_engine_guards.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `TensorIOMode` in module `tensorrt`", + "concise_description": "No attribute `TensorIOMode` in module `tensorrt`", + "severity": "error" + }, + { + "line": 209, + "column": 9, + "stop_line": 209, + "stop_column": 37, + "path": "tests\\unit\\test_trt_engine_guards.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_shape`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_shape`", + "severity": "error" + }, + { + "line": 227, + "column": 9, + "stop_line": 227, + "stop_column": 34, + "path": "tests\\unit\\test_trt_engine_guards.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `num_io_tensors`", + "concise_description": "Object of class `NoneType` has no attribute `num_io_tensors`", + "severity": "error" + }, + { + "line": 228, + "column": 9, + "stop_line": 228, + "stop_column": 35, + "path": "tests\\unit\\test_trt_engine_guards.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_name`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_name`", + "severity": "error" + }, + { + "line": 229, + "column": 9, + "stop_line": 229, + "stop_column": 35, + "path": "tests\\unit\\test_trt_engine_guards.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_mode`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_mode`", + "severity": "error" + }, + { + "line": 229, + "column": 60, + "stop_line": 229, + "stop_column": 76, + "path": "tests\\unit\\test_trt_engine_guards.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `TensorIOMode` in module `tensorrt`", + "concise_description": "No attribute `TensorIOMode` in module `tensorrt`", + "severity": "error" + }, + { + "line": 230, + "column": 9, + "stop_line": 230, + "stop_column": 36, + "path": "tests\\unit\\test_trt_engine_guards.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_dtype`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_dtype`", + "severity": "error" + }, + { + "line": 230, + "column": 61, + "stop_line": 230, + "stop_column": 73, + "path": "tests\\unit\\test_trt_engine_guards.py", + "code": -2, + "name": "missing-attribute", + "description": "No attribute `DataType` in module `tensorrt`", + "concise_description": "No attribute `DataType` in module `tensorrt`", + "severity": "error" + }, + { + "line": 233, + "column": 9, + "stop_line": 233, + "stop_column": 37, + "path": "tests\\unit\\test_trt_engine_guards.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `NoneType` has no attribute `get_tensor_shape`", + "concise_description": "Object of class `NoneType` has no attribute `get_tensor_shape`", + "severity": "error" + }, + { + "line": 240, + "column": 39, + "stop_line": 240, + "stop_column": 59, + "path": "tests\\unit\\test_trt_engine_guards.py", + "code": -2, + "name": "bad-assignment", + "description": "`(_: Unknown) -> type[float32]` is not assignable to attribute `nptype` with type `(trt_type: Unknown) -> Unknown`\n Positional parameter name mismatch: got `_`, want `trt_type`", + "concise_description": "`(_: Unknown) -> type[float32]` is not assignable to attribute `nptype` with type `(trt_type: Unknown) -> Unknown`", + "severity": "error" + }, + { + "line": 79, + "column": 16, + "stop_line": 79, + "stop_column": 35, + "path": "tests\\unit\\test_wrapper_exception_hygiene.py", + "code": -2, + "name": "bad-assignment", + "description": "`device` is not assignable to attribute `device` with type `Literal['cpu', 'cuda']`", + "concise_description": "`device` is not assignable to attribute `device` with type `Literal['cpu', 'cuda']`", + "severity": "error" + }, + { + "line": 1464, + "column": 74, + "stop_line": 1464, + "stop_column": 86, + "path": "tools\\summarize_audit.py", + "code": -2, + "name": "unbound-name", + "description": "`project_name` may be uninitialized", + "concise_description": "`project_name` may be uninitialized", + "severity": "error" + }, + { + "line": 1491, + "column": 21, + "stop_line": 1491, + "stop_column": 31, + "path": "tools\\summarize_audit.py", + "code": -2, + "name": "unbound-name", + "description": "`categories` may be uninitialized", + "concise_description": "`categories` may be uninitialized", + "severity": "error" + }, + { + "line": 1690, + "column": 31, + "stop_line": 1690, + "stop_column": 40, + "path": "tools\\summarize_audit.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `list[Unknown] | None` is not assignable to parameter `tree_data` with type `list[Unknown]` in function `print_dependency_analysis`", + "concise_description": "Argument `list[Unknown] | None` is not assignable to parameter `tree_data` with type `list[Unknown]` in function `print_dependency_analysis`", + "severity": "error" + }, + { + "line": 28, + "column": 52, + "stop_line": 28, + "stop_column": 57, + "path": "utils\\viewer.py", + "code": -2, + "name": "bad-argument-type", + "description": "Argument `Literal[512]` is not assignable to parameter `size` with type `tuple[int, int] | None` in function `PIL.ImageTk.PhotoImage.__init__`", + "concise_description": "Argument `Literal[512]` is not assignable to parameter `size` with type `tuple[int, int] | None` in function `PIL.ImageTk.PhotoImage.__init__`", + "severity": "error" + }, + { + "line": 30, + "column": 5, + "stop_line": 30, + "stop_column": 16, + "path": "utils\\viewer.py", + "code": -2, + "name": "missing-attribute", + "description": "Object of class `Label` has no attribute `image`", + "concise_description": "Object of class `Label` has no attribute `image`", + "severity": "error" + }, + { + "line": 51, + "column": 28, + "stop_line": 56, + "stop_column": 18, + "path": "utils\\viewer.py", + "code": -2, + "name": "bad-argument-type", + "description": "Unpacked argument `tuple[Tensor | Unknown, Label]` is not assignable to parameter `*args` with type `tuple[Image, Label]` in function `tkinter.Misc.after`", + "concise_description": "Unpacked argument `tuple[Tensor | Unknown, Label]` is not assignable to parameter `*args` with type `tuple[Image, Label]` in function `tkinter.Misc.after`", + "severity": "error" + }, + { + "line": 54, + "column": 21, + "stop_line": 54, + "stop_column": 84, + "path": "utils\\viewer.py", + "code": -2, + "name": "bad-index", + "description": "Cannot index into `Image`\n Object of class `Image` has no attribute `__getitem__`", + "concise_description": "Cannot index into `Image`", + "severity": "error" + } + ] +} \ No newline at end of file From d23e79d906e68a2cd9d450e0e092c6f6867b4e6f Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 12 Jul 2026 02:28:36 -0400 Subject: [PATCH 27/37] style: whitelist idiomatic call-defaults; re-enable B008/RUF009 - flake8-bugbear.extend-immutable-calls: fastapi.Depends/File + torch.device - drop B008/RUF009 from ignore (FastAPI DI + torch.device dataclass defaults are intentional) --- pyproject.toml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8be14c1a3..7765600b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,11 +21,15 @@ ignore = [ "RUF001", "RUF002", "RUF003", # ambiguous-unicode in math docstrings "RUF012", "RUF013", "RUF005", "RUF015", "RUF059", # annotation/misc deferral "B905", "B007", # zip-strict / unused-loop-var (behaviour-sensitive) - "B004", "B006", "B008", "B018", "B026", "B904", "NPY002", # TODO: real-bug candidates — re-enable in a follow-up pass - "RUF006", "RUF009", # TODO: real-bug candidates — re-enable in a follow-up pass + "B004", "B006", "B018", "B026", "B904", "NPY002", # TODO: real-bug candidates — re-enable in a follow-up pass + "RUF006", # TODO: real-bug candidates — re-enable in a follow-up pass "RUF022", "RUF046", "UP008", # stylistic ] +[tool.ruff.lint.flake8-bugbear] +# Calls safe to use as argument/dataclass defaults — silences B006/B008/RUF009 for these. +extend-immutable-calls = ["fastapi.Depends", "fastapi.File", "torch.device"] + [tool.ruff.lint.per-file-ignores] "__init__.py" = ["E402", "F401", "F811"] # The only two files using `from ... import *`: From d5140d7b688b0368b26ea1841bf7c31ed6f93b62 Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 12 Jul 2026 02:43:17 -0400 Subject: [PATCH 28/37] fix: chain exception causes + resolve real-bug lints; re-enable rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - B904: add `raise … from e`/`from None` across src, scripts, and demo routes (47 sites) - B004 callable(); B006 None-sentinel default; B018 discard via `_ =`; B026 reorder *args - RUF006: retain InputManager background-task references (prevent GC cancellation) - NPY002: wrapper random-seed path uses np.random.default_rng().integers - drop the fixed codes from [tool.ruff.lint].ignore (rules now enforce) --- demo/realtime-img2img/input_control.py | 5 +++- .../routes/common/api_utils.py | 2 +- demo/realtime-img2img/routes/controlnet.py | 28 +++++++++---------- demo/realtime-img2img/routes/debug.py | 8 +++--- demo/realtime-img2img/routes/inference.py | 4 +-- demo/realtime-img2img/routes/input_sources.py | 4 +-- demo/realtime-img2img/routes/ipadapter.py | 8 +++--- demo/realtime-img2img/routes/parameters.py | 12 ++++---- .../realtime-img2img/routes/pipeline_hooks.py | 16 +++++------ demo/realtime-img2img/util.py | 2 +- examples/screen/main.py | 4 ++- pyproject.toml | 2 -- pyrefly-baseline.json | 12 ++++++++ setup.py | 2 +- .../_patches/diffusers_kvo_patch.py | 2 +- .../acceleration/tensorrt/fp8_quantize.py | 2 +- src/streamdiffusion/pip_utils.py | 2 +- .../processors/faceid_embedding.py | 2 +- .../preprocessing/processors/lineart.py | 2 +- .../preprocessing/processors/openpose.py | 2 +- .../processors/temporal_net_tensorrt.py | 2 +- src/streamdiffusion/wrapper.py | 2 +- 22 files changed, 70 insertions(+), 55 deletions(-) diff --git a/demo/realtime-img2img/input_control.py b/demo/realtime-img2img/input_control.py index 3814708fb..ba895019a 100644 --- a/demo/realtime-img2img/input_control.py +++ b/demo/realtime-img2img/input_control.py @@ -161,6 +161,7 @@ class InputManager: def __init__(self): self.inputs: Dict[str, InputControl] = {} self.parameter_update_callback: Optional[Callable[[str, float], None]] = None + self._background_tasks: set = set() def add_input(self, input_id: str, input_control: InputControl) -> None: """Add an input control""" @@ -171,7 +172,9 @@ def add_input(self, input_id: str, input_control: InputControl) -> None: def remove_input(self, input_id: str) -> None: """Remove an input control""" if input_id in self.inputs: - asyncio.create_task(self.inputs[input_id].stop()) + task = asyncio.create_task(self.inputs[input_id].stop()) + self._background_tasks.add(task) + task.add_done_callback(self._background_tasks.discard) del self.inputs[input_id] logging.info(f"InputManager: Removed input control {input_id}") diff --git a/demo/realtime-img2img/routes/common/api_utils.py b/demo/realtime-img2img/routes/common/api_utils.py index 5937deac6..6095ba1b9 100644 --- a/demo/realtime-img2img/routes/common/api_utils.py +++ b/demo/realtime-img2img/routes/common/api_utils.py @@ -42,7 +42,7 @@ async def handle_api_request( except Exception as e: logging.exception(f"{operation_name}: Failed to parse request: {e}") - raise HTTPException(status_code=400, detail=f"Invalid request format: {e!s}") + raise HTTPException(status_code=400, detail=f"Invalid request format: {e!s}") from e def create_success_response(message: str, **extra_data) -> JSONResponse: diff --git a/demo/realtime-img2img/routes/controlnet.py b/demo/realtime-img2img/routes/controlnet.py index de87f4e35..80ff36b6b 100644 --- a/demo/realtime-img2img/routes/controlnet.py +++ b/demo/realtime-img2img/routes/controlnet.py @@ -47,7 +47,7 @@ async def upload_controlnet_config(file: UploadFile = File(...), app_instance=De try: config_data = yaml.safe_load(content.decode("utf-8")) except yaml.YAMLError as e: - raise HTTPException(status_code=400, detail=f"Invalid YAML format: {e!s}") + raise HTTPException(status_code=400, detail=f"Invalid YAML format: {e!s}") from e # YAML is source of truth - completely replace any runtime modifications app_instance.app_state.uploaded_config = config_data @@ -108,7 +108,7 @@ async def upload_controlnet_config(file: UploadFile = File(...), app_instance=De ) except (ValueError, TypeError): - raise HTTPException(status_code=400, detail="Invalid width/height values in config") + raise HTTPException(status_code=400, detail="Invalid width/height values in config") from None # Build current resolution string current_resolution = None @@ -164,7 +164,7 @@ async def upload_controlnet_config(file: UploadFile = File(...), app_instance=De except Exception as e: logging.exception(f"upload_controlnet_config: Failed to upload configuration: {e}") - raise HTTPException(status_code=500, detail=f"Failed to upload configuration: {e!s}") + raise HTTPException(status_code=500, detail=f"Failed to upload configuration: {e!s}") from e @router.get("/controlnet/info") @@ -207,7 +207,7 @@ async def get_current_blending_config(app_instance=Depends(get_app_instance)): ) except Exception as e: - raise handle_api_error(e, "get_current_blending_config") + raise handle_api_error(e, "get_current_blending_config") from e @router.post("/controlnet/update-strength") @@ -242,7 +242,7 @@ async def update_controlnet_strength(request: Request, app_instance=Depends(get_ return create_success_response(f"Updated ControlNet {controlnet_index} strength to {strength}") except Exception as e: - raise handle_api_error(e, "update_controlnet_strength") + raise handle_api_error(e, "update_controlnet_strength") from e @router.get("/controlnet/available") @@ -302,7 +302,7 @@ async def get_available_controlnets_endpoint( ) except Exception as e: - raise handle_api_error(e, "get_available_controlnets_endpoint") + raise handle_api_error(e, "get_available_controlnets_endpoint") from e @router.post("/controlnet/add") @@ -381,7 +381,7 @@ async def add_controlnet( ) except Exception as e: - raise handle_api_error(e, "add_controlnet") + raise handle_api_error(e, "add_controlnet") from e @router.get("/controlnet/status") @@ -409,7 +409,7 @@ async def get_controlnet_status(app_instance=Depends(get_app_instance)): ) except Exception as e: - raise handle_api_error(e, "get_controlnet_status") + raise handle_api_error(e, "get_controlnet_status") from e @router.post("/controlnet/remove") @@ -459,7 +459,7 @@ async def remove_controlnet(request: Request, app_instance=Depends(get_app_insta ) except Exception as e: - raise handle_api_error(e, "remove_controlnet") + raise handle_api_error(e, "remove_controlnet") from e # Preprocessor endpoints (closely related to ControlNet) @@ -502,7 +502,7 @@ async def get_preprocessors_info(app_instance=Depends(get_app_instance)): ) except Exception as e: - raise handle_api_error(e, "get_preprocessors_info") + raise handle_api_error(e, "get_preprocessors_info") from e @router.post("/preprocessors/switch") @@ -563,7 +563,7 @@ async def switch_preprocessor(request: Request, app_instance=Depends(get_app_ins return create_success_response(f"Switched ControlNet {controlnet_index} preprocessor to {preprocessor_name}") except Exception as e: - raise handle_api_error(e, "switch_preprocessor") + raise handle_api_error(e, "switch_preprocessor") from e @router.post("/preprocessors/update-params") @@ -575,7 +575,7 @@ async def update_preprocessor_params(request: Request, app_instance=Depends(get_ data = await request.json() except Exception as json_error: logging.error(f"update_preprocessor_params: JSON parsing failed: {json_error}") - raise HTTPException(status_code=400, detail=f"Invalid JSON: {json_error}") + raise HTTPException(status_code=400, detail=f"Invalid JSON: {json_error}") from json_error controlnet_index = data.get("controlnet_index") params = data.get("params", {}) @@ -640,7 +640,7 @@ async def update_preprocessor_params(request: Request, app_instance=Depends(get_ except Exception as e: logging.exception(f"update_preprocessor_params: Exception occurred: {e!s}") - raise handle_api_error(e, "update_preprocessor_params") + raise handle_api_error(e, "update_preprocessor_params") from e @router.get("/preprocessors/current-params/{controlnet_index}") @@ -700,4 +700,4 @@ async def get_current_preprocessor_params(controlnet_index: int, app_instance=De ) except Exception as e: - raise handle_api_error(e, "get_current_preprocessor_params") + raise handle_api_error(e, "get_current_preprocessor_params") from e diff --git a/demo/realtime-img2img/routes/debug.py b/demo/realtime-img2img/routes/debug.py index 28b8db974..e7bf7db95 100644 --- a/demo/realtime-img2img/routes/debug.py +++ b/demo/realtime-img2img/routes/debug.py @@ -36,7 +36,7 @@ async def enable_debug_mode(app_instance=Depends(get_app_instance)): ) except Exception as e: logging.exception(f"enable_debug_mode: Failed to enable debug mode: {e}") - raise HTTPException(status_code=500, detail=f"Failed to enable debug mode: {e!s}") + raise HTTPException(status_code=500, detail=f"Failed to enable debug mode: {e!s}") from e @router.post("/disable", response_model=DebugResponse) @@ -56,7 +56,7 @@ async def disable_debug_mode(app_instance=Depends(get_app_instance)): ) except Exception as e: logging.exception(f"disable_debug_mode: Failed to disable debug mode: {e}") - raise HTTPException(status_code=500, detail=f"Failed to disable debug mode: {e!s}") + raise HTTPException(status_code=500, detail=f"Failed to disable debug mode: {e!s}") from e @router.post("/step", response_model=DebugResponse) @@ -81,7 +81,7 @@ async def step_frame(app_instance=Depends(get_app_instance)): raise except Exception as e: logging.exception(f"step_frame: Failed to step frame: {e}") - raise HTTPException(status_code=500, detail=f"Failed to step frame: {e!s}") + raise HTTPException(status_code=500, detail=f"Failed to step frame: {e!s}") from e @router.get("/status", response_model=DebugResponse) @@ -96,4 +96,4 @@ async def get_debug_status(app_instance=Depends(get_app_instance)): ) except Exception as e: logging.exception(f"get_debug_status: Failed to get debug status: {e}") - raise HTTPException(status_code=500, detail=f"Failed to get debug status: {e!s}") + raise HTTPException(status_code=500, detail=f"Failed to get debug status: {e!s}") from e diff --git a/demo/realtime-img2img/routes/inference.py b/demo/realtime-img2img/routes/inference.py index 82a14138e..38d6186a0 100644 --- a/demo/realtime-img2img/routes/inference.py +++ b/demo/realtime-img2img/routes/inference.py @@ -187,7 +187,7 @@ async def generate_frames(): except Exception as e: logging.exception(f"stream: Error in streaming endpoint: {e}") - raise HTTPException(status_code=500, detail=f"Streaming failed: {e!s}") + raise HTTPException(status_code=500, detail=f"Streaming failed: {e!s}") from e @router.get("/state") @@ -246,7 +246,7 @@ async def get_app_state( except Exception as e: logging.error(f"get_app_state: Error getting application state: {e}") - raise HTTPException(status_code=500, detail=f"Failed to get application state: {e!s}") + raise HTTPException(status_code=500, detail=f"Failed to get application state: {e!s}") from e @router.get("/settings") diff --git a/demo/realtime-img2img/routes/input_sources.py b/demo/realtime-img2img/routes/input_sources.py index 0b130503d..5e01e9a42 100644 --- a/demo/realtime-img2img/routes/input_sources.py +++ b/demo/realtime-img2img/routes/input_sources.py @@ -58,7 +58,7 @@ async def set_input_source(request: Request, app_instance=Depends(get_app_instan try: source_type = InputSourceType(source_type_str) except ValueError: - raise HTTPException(status_code=400, detail=f"Invalid source type: {source_type_str}") + raise HTTPException(status_code=400, detail=f"Invalid source type: {source_type_str}") from None # Validate index for controlnet if component == "controlnet" and index is None: @@ -138,7 +138,7 @@ async def upload_component_image( except Exception as e: # Clean up file if image processing fails file_path.unlink(missing_ok=True) - raise HTTPException(status_code=400, detail=f"Invalid image file: {e!s}") + raise HTTPException(status_code=400, detail=f"Invalid image file: {e!s}") from e # Get input source manager and set source manager = _get_input_source_manager(app_instance) diff --git a/demo/realtime-img2img/routes/ipadapter.py b/demo/realtime-img2img/routes/ipadapter.py index 3c574ffb7..db924e6d4 100644 --- a/demo/realtime-img2img/routes/ipadapter.py +++ b/demo/realtime-img2img/routes/ipadapter.py @@ -41,7 +41,7 @@ async def get_default_image(): ) except Exception as e: - raise handle_api_error(e, "get_default_image") + raise handle_api_error(e, "get_default_image") from e @router.post("/ipadapter/update-scale") @@ -67,7 +67,7 @@ async def update_ipadapter_scale(request: Request, app_instance=Depends(get_app_ return create_success_response(f"Updated IPAdapter scale to {scale}") except Exception as e: - raise handle_api_error(e, "update_ipadapter_scale") + raise handle_api_error(e, "update_ipadapter_scale") from e @router.post("/ipadapter/update-weight-type") @@ -93,7 +93,7 @@ async def update_ipadapter_weight_type(request: Request, app_instance=Depends(ge return create_success_response(f"Updated IPAdapter weight type to {weight_type}") except Exception as e: - raise handle_api_error(e, "update_ipadapter_weight_type") + raise handle_api_error(e, "update_ipadapter_weight_type") from e @router.post("/ipadapter/update-enabled") @@ -118,4 +118,4 @@ async def update_ipadapter_enabled(request: Request, app_instance=Depends(get_ap return create_success_response(f"IPAdapter {'enabled' if enabled else 'disabled'} successfully") except Exception as e: - raise handle_api_error(e, "update_ipadapter_enabled") + raise handle_api_error(e, "update_ipadapter_enabled") from e diff --git a/demo/realtime-img2img/routes/parameters.py b/demo/realtime-img2img/routes/parameters.py index 60354d240..3526e9322 100644 --- a/demo/realtime-img2img/routes/parameters.py +++ b/demo/realtime-img2img/routes/parameters.py @@ -55,7 +55,7 @@ async def update_params(request: Request, app_instance=Depends(get_app_instance) logging.info(f"update_params: {message}") return JSONResponse({"status": "success", "message": message}) except ValueError: - raise HTTPException(status_code=400, detail="Invalid resolution format") + raise HTTPException(status_code=400, detail="Invalid resolution format") from None else: raise HTTPException( status_code=400, detail="Resolution must be {width: int, height: int} or 'widthxheight' string" @@ -102,7 +102,7 @@ async def update_params(request: Request, app_instance=Depends(get_app_instance) except Exception as e: logging.exception(f"update_params: Failed to update parameters: {e}") - raise HTTPException(status_code=500, detail=f"Failed to update parameters: {e!s}") + raise HTTPException(status_code=500, detail=f"Failed to update parameters: {e!s}") from e async def _update_single_parameter( @@ -125,7 +125,7 @@ async def _update_single_parameter( return create_success_response(f"Updated {parameter_name} to {value}", **{parameter_name: value}) except Exception as e: - raise handle_api_error(e, operation_name) + raise handle_api_error(e, operation_name) from e @router.post("/update-guidance-scale") @@ -230,7 +230,7 @@ async def update_blending(request: Request, app_instance=Depends(get_app_instanc return create_success_response(f"Updated {' and '.join(updated_types)} blending", updated_types=updated_types) except Exception as e: - raise handle_api_error(e, "update_blending") + raise handle_api_error(e, "update_blending") from e @router.post("/blending/update-prompt-weight") @@ -256,7 +256,7 @@ async def update_prompt_weight(request: Request, app_instance=Depends(get_app_in return create_success_response(f"Updated prompt weight {index} to {weight}") except Exception as e: - raise handle_api_error(e, "update_prompt_weight") + raise handle_api_error(e, "update_prompt_weight") from e @router.post("/blending/update-seed-weight") @@ -282,4 +282,4 @@ async def update_seed_weight(request: Request, app_instance=Depends(get_app_inst return create_success_response(f"Updated seed weight {index} to {weight}") except Exception as e: - raise handle_api_error(e, "update_seed_weight") + raise handle_api_error(e, "update_seed_weight") from e diff --git a/demo/realtime-img2img/routes/pipeline_hooks.py b/demo/realtime-img2img/routes/pipeline_hooks.py index 82a82eb42..eece7dcda 100644 --- a/demo/realtime-img2img/routes/pipeline_hooks.py +++ b/demo/realtime-img2img/routes/pipeline_hooks.py @@ -27,7 +27,7 @@ async def get_pipeline_hooks_info_config(app_instance=Depends(get_app_instance)) } return JSONResponse(hooks_info) except Exception as e: - raise handle_api_error(e, "get_pipeline_hooks_info_config") + raise handle_api_error(e, "get_pipeline_hooks_info_config") from e # Individual hook type endpoints that frontend expects @@ -123,7 +123,7 @@ async def get_hook_processors_info(hook_type: str, app_instance=Depends(get_app_ ) except Exception as e: - raise handle_api_error(e, "get_hook_processors_info") + raise handle_api_error(e, "get_hook_processors_info") from e @router.post("/pipeline-hooks/{hook_type}/add") @@ -178,7 +178,7 @@ async def add_hook_processor(hook_type: str, request: Request, app_instance=Depe return create_success_response(f"Added {processor_type} processor to {hook_type}") except Exception as e: - raise handle_api_error(e, "add_hook_processor") + raise handle_api_error(e, "add_hook_processor") from e @router.delete("/pipeline-hooks/{hook_type}/remove/{processor_index}") @@ -215,7 +215,7 @@ async def remove_hook_processor(hook_type: str, processor_index: int, app_instan return create_success_response(f"Removed processor {processor_index} from {hook_type}") except Exception as e: - raise handle_api_error(e, "remove_hook_processor") + raise handle_api_error(e, "remove_hook_processor") from e @router.post("/pipeline-hooks/{hook_type}/toggle") @@ -263,7 +263,7 @@ async def toggle_hook_processor(hook_type: str, request: Request, app_instance=D ) except Exception as e: - raise handle_api_error(e, "toggle_hook_processor") + raise handle_api_error(e, "toggle_hook_processor") from e @router.post("/pipeline-hooks/{hook_type}/switch") @@ -346,7 +346,7 @@ async def switch_hook_processor(hook_type: str, request: Request, app_instance=D return create_success_response(f"Switched processor {processor_index} in {hook_type} to {new_processor_type}") except Exception as e: - raise handle_api_error(e, "switch_hook_processor") + raise handle_api_error(e, "switch_hook_processor") from e @router.post("/pipeline-hooks/{hook_type}/update-params") @@ -439,7 +439,7 @@ async def update_hook_processor_params(hook_type: str, request: Request, app_ins except Exception as e: logging.exception(f"update_hook_processor_params: Exception occurred: {e!s}") logging.error(f"update_hook_processor_params: Exception type: {type(e).__name__}") - raise handle_api_error(e, "update_hook_processor_params") + raise handle_api_error(e, "update_hook_processor_params") from e @router.get("/pipeline-hooks/{hook_type}/current-params/{processor_index}") @@ -500,4 +500,4 @@ async def get_current_hook_processor_params( ) except Exception as e: - raise handle_api_error(e, "get_current_hook_processor_params") + raise handle_api_error(e, "get_current_hook_processor_params") from e diff --git a/demo/realtime-img2img/util.py b/demo/realtime-img2img/util.py index 11d34665d..3d818d254 100644 --- a/demo/realtime-img2img/util.py +++ b/demo/realtime-img2img/util.py @@ -11,7 +11,7 @@ def get_pipeline_class(pipeline_name: str) -> ModuleType: try: module = import_module(f"pipelines.{pipeline_name}") except ModuleNotFoundError: - raise ValueError(f"Pipeline {pipeline_name} module not found") + raise ValueError(f"Pipeline {pipeline_name} module not found") from None pipeline_class = getattr(module, "Pipeline", None) diff --git a/examples/screen/main.py b/examples/screen/main.py index 3bc47de45..918b46c61 100644 --- a/examples/screen/main.py +++ b/examples/screen/main.py @@ -28,9 +28,11 @@ def screen( event: threading.Event, height: int = 512, width: int = 512, - monitor: Dict[str, int] = {"top": 300, "left": 200, "width": 512, "height": 512}, + monitor: Optional[Dict[str, int]] = None, ): global inputs + if monitor is None: + monitor = {"top": 300, "left": 200, "width": 512, "height": 512} with mss.mss() as sct: while True: if event.is_set(): diff --git a/pyproject.toml b/pyproject.toml index 7765600b9..16d082195 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,8 +21,6 @@ ignore = [ "RUF001", "RUF002", "RUF003", # ambiguous-unicode in math docstrings "RUF012", "RUF013", "RUF005", "RUF015", "RUF059", # annotation/misc deferral "B905", "B007", # zip-strict / unused-loop-var (behaviour-sensitive) - "B004", "B006", "B018", "B026", "B904", "NPY002", # TODO: real-bug candidates — re-enable in a follow-up pass - "RUF006", # TODO: real-bug candidates — re-enable in a follow-up pass "RUF022", "RUF046", "UP008", # stylistic ] diff --git a/pyrefly-baseline.json b/pyrefly-baseline.json index 913287883..55e05f59b 100644 --- a/pyrefly-baseline.json +++ b/pyrefly-baseline.json @@ -6564,6 +6564,18 @@ "concise_description": "Unexpected keyword argument `hand_and_face` in function `FallbackDetector.__call__`", "severity": "error" }, + { + "line": 159, + "column": 16, + "stop_line": 159, + "stop_column": 26, + "path": "src\\streamdiffusion\\preprocessing\\processors\\openpose.py", + "code": -2, + "name": "bad-return", + "description": "Returned type `Image | object | Unknown` is not assignable to declared return type `Image`", + "concise_description": "Returned type `Image | object | Unknown` is not assignable to declared return type `Image`", + "severity": "error" + }, { "line": 215, "column": 43, diff --git a/setup.py b/setup.py index d9c674cd5..459a4d65a 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ def _check_torch_installed(): " pip install --index-url https://download.pytorch.org/whl/cu12x torch torchvision\n" "Replace the index URL and versions to match your CUDA runtime." ) - raise RuntimeError(msg) + raise RuntimeError(msg) from None if not torch.version.cuda: raise RuntimeError( diff --git a/src/streamdiffusion/_patches/diffusers_kvo_patch.py b/src/streamdiffusion/_patches/diffusers_kvo_patch.py index 214b66b7b..b813ef059 100644 --- a/src/streamdiffusion/_patches/diffusers_kvo_patch.py +++ b/src/streamdiffusion/_patches/diffusers_kvo_patch.py @@ -86,10 +86,10 @@ def _call( self, attn, hidden_states, + *args, encoder_hidden_states=encoder_hidden_states, attention_mask=attention_mask, temb=temb, - *args, **kwargs, ) return result, kvo_cache diff --git a/src/streamdiffusion/acceleration/tensorrt/fp8_quantize.py b/src/streamdiffusion/acceleration/tensorrt/fp8_quantize.py index f0a5ff638..2593adcfd 100644 --- a/src/streamdiffusion/acceleration/tensorrt/fp8_quantize.py +++ b/src/streamdiffusion/acceleration/tensorrt/fp8_quantize.py @@ -125,7 +125,7 @@ def _hook(module, args, kwargs): for i, batch in enumerate(batches): logger.info(f"[FP8] Capture batch {i + 1}/{len(batches)}: {batch[0][:60]}") try: - pipe( + _ = pipe( prompt=batch if len(batch) > 1 else batch[0], num_inference_steps=num_inference_steps, output_type="latent", diff --git a/src/streamdiffusion/pip_utils.py b/src/streamdiffusion/pip_utils.py index 670c5ca9b..36d58faa5 100644 --- a/src/streamdiffusion/pip_utils.py +++ b/src/streamdiffusion/pip_utils.py @@ -23,7 +23,7 @@ def _check_torch_installed(): " pip install --index-url https://download.pytorch.org/whl/cu12x torch torchvision\n" "Replace the index URL and versions to match your CUDA runtime." ) - raise RuntimeError(msg) + raise RuntimeError(msg) from None if not torch.version.cuda: raise RuntimeError( diff --git a/src/streamdiffusion/preprocessing/processors/faceid_embedding.py b/src/streamdiffusion/preprocessing/processors/faceid_embedding.py index 31bb042ae..1629a2e92 100644 --- a/src/streamdiffusion/preprocessing/processors/faceid_embedding.py +++ b/src/streamdiffusion/preprocessing/processors/faceid_embedding.py @@ -75,7 +75,7 @@ def _process_core(self, image: Image.Image) -> Tuple[torch.Tensor, torch.Tensor] except Exception as e: msg = f"FaceIDEmbeddingPreprocessor: Failed to extract face embeddings: {e}" report_error(msg) - raise RuntimeError(msg) + raise RuntimeError(msg) from e def update_faceid_v2_weight(self, weight: float) -> None: self.faceid_v2_weight = float(weight) diff --git a/src/streamdiffusion/preprocessing/processors/lineart.py b/src/streamdiffusion/preprocessing/processors/lineart.py index 6e9607dee..01ffe3ec0 100644 --- a/src/streamdiffusion/preprocessing/processors/lineart.py +++ b/src/streamdiffusion/preprocessing/processors/lineart.py @@ -15,7 +15,7 @@ CONTROLNET_AUX_AVAILABLE = False raise ImportError( "LineartPreprocessor: controlnet_aux is required for real-time optimization. Install with: pip install controlnet_aux" - ) + ) from None # TODO provide gpu native lineart detection diff --git a/src/streamdiffusion/preprocessing/processors/openpose.py b/src/streamdiffusion/preprocessing/processors/openpose.py index eb591cfd4..d489c2ddb 100644 --- a/src/streamdiffusion/preprocessing/processors/openpose.py +++ b/src/streamdiffusion/preprocessing/processors/openpose.py @@ -147,7 +147,7 @@ def _process_core(self, image: Image.Image) -> Image.Image: include_hands = self.params.get("include_hands", False) include_face = self.params.get("include_face", False) - if CONTROLNET_AUX_AVAILABLE and hasattr(self.detector, "__call__"): + if CONTROLNET_AUX_AVAILABLE and callable(self.detector): try: pose_image = self.detector(image_resized, hand_and_face=include_hands or include_face) except Exception as e: diff --git a/src/streamdiffusion/preprocessing/processors/temporal_net_tensorrt.py b/src/streamdiffusion/preprocessing/processors/temporal_net_tensorrt.py index cd136dabe..9cefbc0e5 100644 --- a/src/streamdiffusion/preprocessing/processors/temporal_net_tensorrt.py +++ b/src/streamdiffusion/preprocessing/processors/temporal_net_tensorrt.py @@ -326,7 +326,7 @@ def _load_tensorrt_engine(self): f"Failed to load TensorRT engine from {self.engine_path}: {e}\n" f"Make sure the engine was built with a resolution range that includes {self.height}x{self.width}.\n" f"For example: python -m streamdiffusion.tools.compile_raft_tensorrt --min_resolution 512x512 --max_resolution 1024x1024" - ) + ) from e def _process_core(self, image: Image.Image) -> Image.Image: """ diff --git a/src/streamdiffusion/wrapper.py b/src/streamdiffusion/wrapper.py index 1b6d373ef..11ce00787 100644 --- a/src/streamdiffusion/wrapper.py +++ b/src/streamdiffusion/wrapper.py @@ -428,7 +428,7 @@ def __init__( return if seed < 0: # Random seed - seed = np.random.randint(0, 1000000) + seed = int(np.random.default_rng().integers(0, 1000000)) self.stream.prepare( "", From d25a291e3183acdf64dfbafc545cb39a125d3415 Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 12 Jul 2026 02:43:57 -0400 Subject: [PATCH 29/37] fix: harden RealESRGAN fallback loader with torch.load(weights_only=True) - only unguarded torch.load repo-wide; prevents arbitrary-code exec on untrusted checkpoints --- src/streamdiffusion/preprocessing/processors/realesrgan_trt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/streamdiffusion/preprocessing/processors/realesrgan_trt.py b/src/streamdiffusion/preprocessing/processors/realesrgan_trt.py index d08697e42..74514abcc 100644 --- a/src/streamdiffusion/preprocessing/processors/realesrgan_trt.py +++ b/src/streamdiffusion/preprocessing/processors/realesrgan_trt.py @@ -254,7 +254,7 @@ def _load_pytorch_model(self): """Load PyTorch model from file""" if not SPANDREL_AVAILABLE: # Fallback loading without spandrel - state_dict = torch.load(self.model_path, map_location=self.device) # noqa: F841 # TODO: pre-existing, untouched by this refactor + state_dict = torch.load(self.model_path, map_location=self.device, weights_only=True) # noqa: F841 # TODO: pre-existing, untouched by this refactor # This is a simplified approach - real implementation would need model architecture return From 658dcfa21b99c4dbf946ae106ed564761f96a3bb Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 13 Jul 2026 19:29:24 -0400 Subject: [PATCH 30/37] chore: add Claude review workflow to PR2 branch for head-ref validation --- .github/workflows/claude-code-review.yml | 47 ++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 .github/workflows/claude-code-review.yml diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml new file mode 100644 index 000000000..bef9532a1 --- /dev/null +++ b/.github/workflows/claude-code-review.yml @@ -0,0 +1,47 @@ +name: Claude Code Review + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + # base-branch filter: PRs *into* SDTD_032_dev, or into any branch in the + # chained remediation-PR stack (2-5), so Claude also fires on chained PRs. + branches: + [ + SDTD_032_dev, + refactor/remediation-phases-1-4, + perf/unet-step-subphase-5x, + perf/trt-zero-copy-audit-hardening, + refactor/param-schema-fp8-preflight, + ] + +jobs: + review: + if: github.event.pull_request.draft == false + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + issues: write + id-token: write + steps: + - uses: actions/checkout@v4 + with: { fetch-depth: 0 } + + - uses: anthropics/claude-code-action@v1 + with: + claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + track_progress: true # forces tag mode -> sticky comment updates in place + use_sticky_comment: true # one review comment across pushes, not a new one each time + show_full_output: true # surfaces full tool output/denials in the run log for debugging + plugin_marketplaces: "https://github.com/anthropics/claude-code.git" + plugins: "code-review@claude-code-plugins" + prompt: "/code-review:code-review ${{ github.repository }}/pull/${{ github.event.pull_request.number }}" + claude_args: >- + --max-turns 40 + --allowedTools "Bash(git diff:*),Bash(git status:*),Bash(git log:*),Bash(git fetch:*),Bash(git merge-base:*),Bash(git rev-parse:*),Bash(git remote:*),Bash(git branch:*),Bash(git show:*),Bash(echo:*),Bash(gh pr diff:*),Bash(gh pr view:*),Bash(gh release view:*)" + --append-system-prompt "Repo-specific focus: real-time diffusion hot-path discipline (flag + .item()/.cpu()/.numpy()/print(tensor) in step/frame loops), no hard-coded .cuda()/device, + best-effort error paths must never raise into the render loop, use raise ... from e, TensorRT + zero-copy buffer safety (dtype/shape/lifetime). Honor intentional style: keep + typing.Dict/Optional/Union (no PEP585/604 rewrites), line-length 119, single-letter math vars + OK, torch.device valid as a default arg. Skip nits ruff/pyrefly already cover." From ed0521befed4fd17c10498462354b152c1862e55 Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 13 Jul 2026 19:41:14 -0400 Subject: [PATCH 31/37] chore: widen Claude review allowedTools (Skill, python/pytest, /tmp Write) to unblock permission-denied max-turns failures --- .github/workflows/claude-code-review.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml index bef9532a1..b83bd4db1 100644 --- a/.github/workflows/claude-code-review.yml +++ b/.github/workflows/claude-code-review.yml @@ -38,7 +38,7 @@ jobs: prompt: "/code-review:code-review ${{ github.repository }}/pull/${{ github.event.pull_request.number }}" claude_args: >- --max-turns 40 - --allowedTools "Bash(git diff:*),Bash(git status:*),Bash(git log:*),Bash(git fetch:*),Bash(git merge-base:*),Bash(git rev-parse:*),Bash(git remote:*),Bash(git branch:*),Bash(git show:*),Bash(echo:*),Bash(gh pr diff:*),Bash(gh pr view:*),Bash(gh release view:*)" + --allowedTools "Bash(git diff:*),Bash(git status:*),Bash(git log:*),Bash(git fetch:*),Bash(git merge-base:*),Bash(git rev-parse:*),Bash(git remote:*),Bash(git branch:*),Bash(git show:*),Bash(echo:*),Bash(gh pr diff:*),Bash(gh pr view:*),Bash(gh release view:*),Bash(python:*),Bash(python3:*),Bash(pytest:*),Skill,Write(/tmp/*)" --append-system-prompt "Repo-specific focus: real-time diffusion hot-path discipline (flag .item()/.cpu()/.numpy()/print(tensor) in step/frame loops), no hard-coded .cuda()/device, best-effort error paths must never raise into the render loop, use raise ... from e, TensorRT From a6e00fc05ec18a58488ad6743bff59e3a4286249 Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 13 Jul 2026 19:51:55 -0400 Subject: [PATCH 32/37] chore: raise Claude review max-turns to 60, discourage filesystem-wide find and output redirection --- .github/workflows/claude-code-review.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml index b83bd4db1..7505cecef 100644 --- a/.github/workflows/claude-code-review.yml +++ b/.github/workflows/claude-code-review.yml @@ -37,11 +37,14 @@ jobs: plugins: "code-review@claude-code-plugins" prompt: "/code-review:code-review ${{ github.repository }}/pull/${{ github.event.pull_request.number }}" claude_args: >- - --max-turns 40 + --max-turns 60 --allowedTools "Bash(git diff:*),Bash(git status:*),Bash(git log:*),Bash(git fetch:*),Bash(git merge-base:*),Bash(git rev-parse:*),Bash(git remote:*),Bash(git branch:*),Bash(git show:*),Bash(echo:*),Bash(gh pr diff:*),Bash(gh pr view:*),Bash(gh release view:*),Bash(python:*),Bash(python3:*),Bash(pytest:*),Skill,Write(/tmp/*)" --append-system-prompt "Repo-specific focus: real-time diffusion hot-path discipline (flag .item()/.cpu()/.numpy()/print(tensor) in step/frame loops), no hard-coded .cuda()/device, best-effort error paths must never raise into the render loop, use raise ... from e, TensorRT zero-copy buffer safety (dtype/shape/lifetime). Honor intentional style: keep typing.Dict/Optional/Union (no PEP585/604 rewrites), line-length 119, single-letter math vars - OK, torch.device valid as a default arg. Skip nits ruff/pyrefly already cover." + OK, torch.device valid as a default arg. Skip nits ruff/pyrefly already cover. + Tool usage: never run filesystem-wide search (find /, grep -r /), stay inside the repo + checkout. Never redirect Bash output to a file with > or >>, it is blocked; pipe to head/tail + or just read the direct stdout instead." From 47a87eb1f35fac747329bb1672bfa4d718b02130 Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 13 Jul 2026 20:32:55 -0400 Subject: [PATCH 33/37] fix: correct NSFW verdict-to-frame attribution and guard pin_memory on CPU-only builds _apply_safety_checker previously read the prior frame's async verdict but applied it to the current frame's pixels, letting an isolated NSFW frame leak through and wrongly blanking the next clean frame. Buffer the raw frame in _pending_frame and emit it once its own verdict lands (+1 frame latency, first call now emits a black startup frame instead of raw pixels). Also guard .pin_memory() behind torch.cuda.is_available() so the CPU-only test contract in test_safety_checker.py holds. --- src/streamdiffusion/wrapper.py | 72 +++++++----- tests/unit/test_safety_checker.py | 182 +++++++++++++++++++++--------- 2 files changed, 170 insertions(+), 84 deletions(-) diff --git a/src/streamdiffusion/wrapper.py b/src/streamdiffusion/wrapper.py index 17055f54c..873d33ef7 100644 --- a/src/streamdiffusion/wrapper.py +++ b/src/streamdiffusion/wrapper.py @@ -380,6 +380,9 @@ def __init__( # _apply_safety_checker). Distinct from _output_pin_buf, which is uint8 # image-shaped and reallocated by the output-postprocessing paths. self._nsfw_prob_pin: Optional[torch.Tensor] = None + # Raw frame awaiting its own verdict; buffered one call for correct-attribution + # delayed emission (see _apply_safety_checker). + self._pending_frame: Optional[torch.Tensor] = None self.fp8 = fp8 self.static_shapes = static_shapes self.fp8_allow_fp16_fallback = fp8_allow_fp16_fallback @@ -1364,18 +1367,24 @@ def _apply_safety_checker(self, image_tensor: torch.Tensor) -> torch.Tensor: CUDA-IPC export path: postprocess_image() exports the frame inside its own body and returns None, making any post-hoc substitution unreachable and unsafe. - 1-frame-delayed async readback (mirrors SimilarImageFilter, image_filter.py): the - classifier result for THIS frame is read on the NEXT call, so the substitution below - acts on the PREVIOUS frame's verdict while this frame's classification + pinned - `non_blocking` copy is merely launched. This avoids forcing a cudaStreamSynchronize - on the hot path. No explicit sync guards the pinned read — the same trade-off - SimilarImageFilter already makes: stream ordering means the async copy is enqueued - before all of this frame's remaining GPU work, and by the time Python reaches this - call again next frame (after a full diffusion step + decode, and — for "np"/"pil" - output — a hard _d2h_event.synchronize() in postprocess_image) the tiny 4-byte D2H - copy has long since landed in practice. Worst case on other output types is a stale - read of an even-older value, i.e. one extra frame of detection delay, not corrupted - data. Accepted trade-off for an opt-in, off-by-default, TensorRT-only feature. + 1-frame-delayed async classification with delayed EMISSION (mirrors the async-launch + idea in SimilarImageFilter, image_filter.py, but — unlike that filter — buffers the raw + frame so each frame is gated on its OWN verdict, never a neighbor's). Each call: + reads the pinned verdict for the frame buffered on the PREVIOUS call (now landed), + emits that buffered frame gated on its own verdict, launches classification for THIS + frame, and buffers this frame for the next call. No explicit sync guards the pinned + read — the same trade-off SimilarImageFilter already makes: stream ordering means the + async copy is enqueued before all of this frame's remaining GPU work, and by the time + Python reaches this call again next frame (after a full diffusion step + decode, and — + for "np"/"pil" output — a hard _d2h_event.synchronize() in postprocess_image) the tiny + 4-byte D2H copy has long since landed in practice. This avoids forcing a + cudaStreamSynchronize on the hot path. Accepted trade-off for an opt-in, off-by-default, + TensorRT-only feature. + + Two consequences are inherent to gating each frame on its own async verdict without a + sync: (1) output is delayed by exactly one frame; (2) the very first call has no buffered + frame yet, so it emits a black startup frame instead of passing raw pixels through + unscreened, and the final buffered frame of a stream is never emitted at shutdown. Parameters ---------- @@ -1385,8 +1394,9 @@ def _apply_safety_checker(self, image_tensor: torch.Tensor) -> torch.Tensor: Returns ------- torch.Tensor - The original tensor when content is clean, or a fallback tensor (previous clean - frame, or all-black encoded as -1.0 in diffusion range) when content is flagged. + The previously-buffered frame, unchanged when clean, or a fallback tensor (previous + clean frame, or all-black encoded as -1.0 in diffusion range) when flagged. On the + very first call, an all-black startup frame (no buffered frame exists yet). """ if not self.use_safety_checker: return image_tensor @@ -1394,35 +1404,40 @@ def _apply_safety_checker(self, image_tensor: torch.Tensor) -> torch.Tensor: # Denormalize to [0, 1] NCHW for the classifier; stays on GPU. denormalized = self._denormalize_on_gpu(image_tensor) - if self._nsfw_prob_pin is None: - # First frame: no prior async result exists yet. Launch this frame's - # classification so the next frame has a real verdict, but pass this one - # through unflagged (documented 1-frame pass-through edge case). - self._nsfw_prob_pin = torch.zeros(1, dtype=torch.float32, device="cpu").pin_memory() + if self._pending_frame is None: + # First call: nothing buffered yet, so no frame can be gated on its own verdict. + # Prime the pipeline (launch this frame's classification, buffer it) and emit a + # black safety frame rather than ungated pixels. + pin = torch.zeros(1, dtype=torch.float32, device="cpu") + if torch.cuda.is_available(): + pin = pin.pin_memory() + self._nsfw_prob_pin = pin self.safety_checker(denormalized, self._nsfw_prob_pin) - if self.safety_checker_fallback_type == "previous": - self._prev_clean_tensor = image_tensor.clone() - return image_tensor + self._pending_frame = image_tensor.clone() + return torch.full_like(image_tensor, -1.0) - # Step 1: read the PREVIOUS frame's async result (pinned CPU read, no GPU sync). + # Step 1: read the PENDING frame's own async result (pinned CPU read, no GPU sync). flagged = self._nsfw_prob_pin.item() >= self.safety_checker_threshold # Step 2: launch THIS frame's classification + async pinned copy (no sync). self.safety_checker(denormalized, self._nsfw_prob_pin) - # Step 3: branch on the PREVIOUS frame's verdict (1-frame-delayed, no stall). + # Step 3: rotate the pending frame out and this frame in. + pending = self._pending_frame + self._pending_frame = image_tensor.clone() + if flagged: logger.info("NSFW content detected, applying safety fallback frame") if self.safety_checker_fallback_type == "previous" and self._prev_clean_tensor is not None: return self._prev_clean_tensor # -1.0 in diffusion range → 0.0 after denormalization → true black on every output # path (pt, np, pil, CUDA-IPC). - return torch.full_like(image_tensor, -1.0) + return torch.full_like(pending, -1.0) - # Content is clean — cache it for the "previous" fallback strategy. + # Pending frame is clean — cache it for the "previous" fallback strategy. if self.safety_checker_fallback_type == "previous": - self._prev_clean_tensor = image_tensor.clone() - return image_tensor + self._prev_clean_tensor = pending + return pending def _load_model( self, @@ -3072,6 +3087,7 @@ def cleanup_gpu_memory(self) -> None: self._ipc_pack_buf = None self._ipc_pack_unit_buf = None self._nsfw_prob_pin = None + self._pending_frame = None # Force multiple garbage collection cycles for thorough cleanup for i in range(3): diff --git a/tests/unit/test_safety_checker.py b/tests/unit/test_safety_checker.py index 1e2b1cbbb..b4c1d91a7 100644 --- a/tests/unit/test_safety_checker.py +++ b/tests/unit/test_safety_checker.py @@ -16,13 +16,23 @@ raw diffusion-range [-1, 1] pipeline tensor so it is always a real tensor regardless of output path. -Sub-phase 5.3: the checker's verdict is now read 1-frame-delayed from a pinned -async buffer (mirrors SimilarImageFilter) instead of a synchronous .item() call. -`safety_checker` is now `(tensor, prob_pin) -> None`: it writes into `prob_pin` -rather than returning a bool, and `_apply_safety_checker` reads the *previous* -call's pinned value before launching (and writing) this frame's result. Tests -below seed `w._nsfw_prob_pin` directly where they need to bypass the -first-frame pass-through and exercise a specific verdict. +Sub-phase 5.3 (delayed-emission model): the checker's verdict is read +1-frame-delayed from a pinned async buffer (mirrors the async-launch idea in +SimilarImageFilter), but unlike a plain delayed *readback*, the raw frame +itself is buffered in `_pending_frame` so each frame is gated on its OWN +verdict rather than a neighbor's. `safety_checker` is `(tensor, prob_pin) -> +None`: it writes into `prob_pin` rather than returning a bool. Each call to +`_apply_safety_checker`: + 1. reads the verdict for the frame buffered on the PREVIOUS call (now + landed in the pin), + 2. launches classification for the CURRENT frame, + 3. emits the PENDING (previously buffered) frame, gated on its own verdict, + 4. buffers the CURRENT frame for the next call. +This delays output by exactly one frame. The very first call has no buffered +frame yet, so it emits a black startup frame and primes the pipeline instead +of passing raw pixels through unscreened. Tests below seed `w._nsfw_prob_pin` +and `w._pending_frame` directly where they need to bypass the first-call +priming and exercise a specific verdict against a specific frame. """ import torch @@ -48,6 +58,7 @@ def _make_wrapper( w.safety_checker_fallback_type = fallback_type w._prev_clean_tensor = None w._nsfw_prob_pin = None + w._pending_frame = None # safety_checker is set per-test w.safety_checker = None return w @@ -92,15 +103,16 @@ def capturing_checker(tensor, prob_pin): # ── Case 2: NSFW → black frame (blank fallback) ────────────────────────── def test_nsfw_blank_fallback_is_black(self): w = _make_wrapper(fallback_type="blank") - w.safety_checker = lambda t, pin: pin.fill_(1.0) # always "flag" + w.safety_checker = lambda t, pin: None # verdict is pre-seeded below - frame1 = torch.randn(1, 3, 64, 64) - _ = w._apply_safety_checker(frame1) # first frame: pass-through, primes the pin + pending = torch.randn(1, 3, 64, 64) + w._pending_frame = pending + w._nsfw_prob_pin = torch.tensor([1.0]) # pending frame's own verdict: flagged - frame2 = torch.randn(1, 3, 64, 64) - result = w._apply_safety_checker(frame2) # reads frame1's flagged verdict + current = torch.randn(1, 3, 64, 64) + result = w._apply_safety_checker(current) - assert result is not frame2, "flagged frame should be replaced" + assert result is not pending, "flagged pending frame should be replaced" # Denormalize: (x/2+0.5).clamp(0,1); -1.0 → 0.0 = black denorm = (result / 2 + 0.5).clamp(0, 1) assert _black_denorm(denorm), "NSFW blank fallback should produce a black frame" @@ -108,37 +120,33 @@ def test_nsfw_blank_fallback_is_black(self): # ── Case 3: NSFW → previous frame ──────────────────────────────────────── def test_nsfw_previous_fallback_returns_cached_clean_frame(self): w = _make_wrapper(fallback_type="previous") - call_count = [0] + w.safety_checker = lambda t, pin: None # verdict is pre-seeded below - def checker(t, pin): - call_count[0] += 1 - # Call #1 (launched during frame1) writes a flagged prob; frame2's - # read consumes that async result (1-frame delay). - pin.fill_(1.0 if call_count[0] == 1 else 0.0) - - w.safety_checker = checker + cached_clean = torch.randn(1, 3, 64, 64) + w._prev_clean_tensor = cached_clean - clean = torch.randn(1, 3, 64, 64) - _ = w._apply_safety_checker(clean) # first frame: pass-through, primes _prev_clean_tensor + pending = torch.randn(1, 3, 64, 64) + w._pending_frame = pending + w._nsfw_prob_pin = torch.tensor([1.0]) # pending frame's own verdict: flagged - flagged_frame = torch.randn(1, 3, 64, 64) - result = w._apply_safety_checker(flagged_frame) + current = torch.randn(1, 3, 64, 64) + result = w._apply_safety_checker(current) - assert result is w._prev_clean_tensor, "previous-fallback should return the cached clean tensor" - assert torch.equal(w._prev_clean_tensor, clean) + assert result is cached_clean, "previous-fallback should return the cached clean tensor" # ── Case 4: Clean frame passthrough ────────────────────────────────────── def test_clean_frame_returned_unchanged(self): w = _make_wrapper() - w.safety_checker = lambda t, pin: pin.fill_(0.0) # never flag + w.safety_checker = lambda t, pin: None # verdict is pre-seeded below - frame1 = torch.randn(1, 3, 64, 64) - _ = w._apply_safety_checker(frame1) # first frame: pass-through, primes the pin + pending = torch.randn(1, 3, 64, 64) + w._pending_frame = pending + w._nsfw_prob_pin = torch.tensor([0.0]) # pending frame's own verdict: clean - frame2 = torch.randn(1, 3, 64, 64) - result = w._apply_safety_checker(frame2) + current = torch.randn(1, 3, 64, 64) + result = w._apply_safety_checker(current) - assert torch.equal(result, frame2), "clean frame should be returned unchanged" + assert torch.equal(result, pending), "clean pending frame should be emitted unchanged" # ── Case 5: use_safety_checker=False bypasses entirely ─────────────────── def test_disabled_bypasses_checker(self): @@ -159,14 +167,14 @@ def should_not_be_called(t, pin): # ── Case 6: flagged verdict with no cached frame → black ───────────────── def test_nsfw_previous_fallback_no_cache_falls_back_to_black(self): """ - If a frame is flagged (per the previous frame's pinned verdict) and + If the pending frame is flagged (per its own landed verdict) and _prev_clean_tensor is None, the fallback should still produce a black - frame rather than raise. Seeds _nsfw_prob_pin directly to bypass the - first-frame pass-through and exercise this state combination. + frame rather than raise. """ w = _make_wrapper(fallback_type="previous") w.safety_checker = lambda t, pin: pin.fill_(0.0) - w._nsfw_prob_pin = torch.tensor([1.0]) # a prior flagged verdict already landed + w._pending_frame = torch.randn(1, 3, 64, 64) + w._nsfw_prob_pin = torch.tensor([1.0]) # pending frame's own verdict: flagged assert w._prev_clean_tensor is None # no cache yet dummy = torch.randn(1, 3, 64, 64) @@ -179,49 +187,57 @@ def test_nsfw_previous_fallback_no_cache_falls_back_to_black(self): def test_clean_frame_cached_for_previous_strategy(self): w = _make_wrapper(fallback_type="previous") w.safety_checker = lambda t, pin: pin.fill_(0.0) - w._nsfw_prob_pin = torch.tensor([0.0]) # a prior clean verdict already landed + pending = torch.randn(1, 3, 64, 64) + w._pending_frame = pending + w._nsfw_prob_pin = torch.tensor([0.0]) # pending frame's own verdict: clean assert w._prev_clean_tensor is None dummy = torch.randn(1, 3, 64, 64) w._apply_safety_checker(dummy) assert w._prev_clean_tensor is not None, ( - "clean frame with previous strategy should populate _prev_clean_tensor" + "clean pending frame with previous strategy should populate _prev_clean_tensor" ) - assert torch.equal(w._prev_clean_tensor, dummy) + assert torch.equal(w._prev_clean_tensor, pending) # ── Case 8: clean frame NOT cached for blank strategy ──────────────────── def test_clean_frame_not_cached_for_blank_strategy(self): w = _make_wrapper(fallback_type="blank") w.safety_checker = lambda t, pin: pin.fill_(0.0) - w._nsfw_prob_pin = torch.tensor([0.0]) # a prior clean verdict already landed + w._pending_frame = torch.randn(1, 3, 64, 64) + w._nsfw_prob_pin = torch.tensor([0.0]) # pending frame's own verdict: clean dummy = torch.randn(1, 3, 64, 64) w._apply_safety_checker(dummy) assert w._prev_clean_tensor is None, "_prev_clean_tensor should stay None when fallback_type='blank'" - # ── Case 9: first frame always passes through ──────────────────────────── - def test_first_frame_always_passes_through_regardless_of_verdict(self): + # ── Case 9: first call emits black and buffers ──────────────────────────── + def test_first_frame_emits_black_and_buffers(self): """ - The very first frame has no prior async result to read, so it must - pass through unscreened even if the classifier would flag it — the - documented 1-frame pass-through edge case of the delayed-readback design. + The very first call has no buffered frame yet, so it cannot gate + anything on its own verdict. It must emit a black startup frame + (never ungated pixels) and buffer the raw frame for the next call. """ w = _make_wrapper(fallback_type="blank") - w.safety_checker = lambda t, pin: pin.fill_(1.0) # would flag if read this frame + w.safety_checker = lambda t, pin: pin.fill_(1.0) # would flag if read this call dummy = torch.randn(1, 3, 64, 64) result = w._apply_safety_checker(dummy) - assert torch.equal(result, dummy), "first frame must pass through (no prior verdict to act on)" + denorm = (result / 2 + 0.5).clamp(0, 1) + assert _black_denorm(denorm), "first call must emit a black startup frame" + assert w._pending_frame is not None and torch.equal(w._pending_frame, dummy), ( + "first call must buffer the raw frame for next call's emission" + ) # ── Case 10: _process_skip_diffusion wiring — checker is called, result is black ── def test_skip_diffusion_routes_through_safety_checker(self): """ Verify that _process_skip_diffusion actually feeds its tensor through - _apply_safety_checker before postprocess_image, and that a flagged frame - produces the black substitution rather than passing through unscreened. + _apply_safety_checker before postprocess_image, and that a flagged + pending frame produces the black substitution rather than passing + through unscreened. This is a wiring test — the behaviour of _apply_safety_checker itself is covered by the earlier cases. We stub all model-dependent calls with @@ -231,11 +247,12 @@ def test_skip_diffusion_routes_through_safety_checker(self): def capturing_checker(tensor, pin): received.append(tensor) - pin.fill_(0.0) # this frame's own async result — irrelevant to this frame's own read + pin.fill_(0.0) # this frame's own async result — irrelevant to this call's read w = _make_wrapper(fallback_type="blank") w.safety_checker = capturing_checker - w._nsfw_prob_pin = torch.tensor([1.0]) # a prior frame's flagged verdict already landed + w._pending_frame = torch.randn(1, 3, 64, 64) # a prior frame awaiting emission + w._nsfw_prob_pin = torch.tensor([1.0]) # that pending frame's own verdict: flagged w.mode = "img2img" w.device = torch.device("cpu") w.dtype = torch.float32 @@ -265,8 +282,61 @@ def _apply_image_postprocessing_hooks(self, t): assert len(received) == 1, "safety checker should be called exactly once" assert isinstance(received[0], torch.Tensor), "checker arg must be a Tensor, not None" - # Flagged frame (per the pre-seeded pinned verdict) must produce the black substitution + # Flagged pending frame (per the pre-seeded pinned verdict) must produce the black substitution denorm = (result / 2 + 0.5).clamp(0, 1) - assert torch.allclose(denorm, torch.zeros_like(denorm)), ( - "flagged skip-diffusion frame should produce a black output" - ) + assert torch.allclose(denorm, torch.zeros_like(denorm)), "flagged pending frame should produce a black output" + + # ── Case 11: pin_memory is guarded on CPU-only builds ──────────────────── + def test_pin_memory_guarded_on_cpu(self): + """ + Regression test: the first call used to unconditionally call + .pin_memory(), which raises on CPU-only / no-CUDA-driver PyTorch + builds. It must be guarded and still produce a usable CPU tensor. + """ + w = _make_wrapper(fallback_type="blank") + w.safety_checker = lambda t, pin: pin.fill_(0.0) + + dummy = torch.randn(1, 3, 64, 64) + w._apply_safety_checker(dummy) # must not raise + + assert isinstance(w._nsfw_prob_pin, torch.Tensor) + assert w._nsfw_prob_pin.device.type == "cpu" + + # ── Case 12: isolated NSFW frame is blanked, not leaked ────────────────── + def test_isolated_nsfw_frame_is_blanked_not_leaked(self): + """ + Drives [clean, NSFW, clean] through the checker and asserts the NSFW + frame is the one substituted (never emitted raw) and the surrounding + clean frames pass through unscreened — proving the old bug (verdict + misattributed to the wrong frame, causing an isolated NSFW frame to + leak and the following clean frame to be wrongly blanked) is fixed. + """ + verdicts = {"clean1": 0.0, "nsfw": 1.0, "clean2": 0.0} + order = ["clean1", "nsfw", "clean2"] + call_index = [0] + + def checker(t, pin): + key = order[call_index[0]] + call_index[0] += 1 + pin.fill_(verdicts[key]) + + w = _make_wrapper(fallback_type="blank", threshold=0.5) + w.safety_checker = checker + + clean1 = torch.randn(1, 3, 64, 64) + nsfw = torch.randn(1, 3, 64, 64) + clean2 = torch.randn(1, 3, 64, 64) + + r0 = w._apply_safety_checker(clean1) # priming call: black startup frame + r1 = w._apply_safety_checker(nsfw) # emits clean1 (clean) + r2 = w._apply_safety_checker(clean2) # emits nsfw (flagged -> black) + # A 4th call would be needed to emit clean2; verify what we can without it. + + denorm0 = (r0 / 2 + 0.5).clamp(0, 1) + assert _black_denorm(denorm0), "priming call must emit black, not raw pixels" + + assert torch.equal(r1, clean1), "clean1 must be emitted unchanged, gated on its own verdict" + + denorm2 = (r2 / 2 + 0.5).clamp(0, 1) + assert _black_denorm(denorm2), "the NSFW frame itself must be the one blanked, not a neighbor" + assert not torch.equal(r2, clean2), "clean2 must never be substituted for the NSFW frame's verdict" From 2d243692a7357161839e6b3a64d2c3cf0bcc8f62 Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 13 Jul 2026 21:10:11 -0400 Subject: [PATCH 34/37] chore: add Claude review workflow to PR3 branch for head-ref validation --- .github/workflows/claude-code-review.yml | 50 ++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .github/workflows/claude-code-review.yml diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml new file mode 100644 index 000000000..7505cecef --- /dev/null +++ b/.github/workflows/claude-code-review.yml @@ -0,0 +1,50 @@ +name: Claude Code Review + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + # base-branch filter: PRs *into* SDTD_032_dev, or into any branch in the + # chained remediation-PR stack (2-5), so Claude also fires on chained PRs. + branches: + [ + SDTD_032_dev, + refactor/remediation-phases-1-4, + perf/unet-step-subphase-5x, + perf/trt-zero-copy-audit-hardening, + refactor/param-schema-fp8-preflight, + ] + +jobs: + review: + if: github.event.pull_request.draft == false + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + issues: write + id-token: write + steps: + - uses: actions/checkout@v4 + with: { fetch-depth: 0 } + + - uses: anthropics/claude-code-action@v1 + with: + claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + track_progress: true # forces tag mode -> sticky comment updates in place + use_sticky_comment: true # one review comment across pushes, not a new one each time + show_full_output: true # surfaces full tool output/denials in the run log for debugging + plugin_marketplaces: "https://github.com/anthropics/claude-code.git" + plugins: "code-review@claude-code-plugins" + prompt: "/code-review:code-review ${{ github.repository }}/pull/${{ github.event.pull_request.number }}" + claude_args: >- + --max-turns 60 + --allowedTools "Bash(git diff:*),Bash(git status:*),Bash(git log:*),Bash(git fetch:*),Bash(git merge-base:*),Bash(git rev-parse:*),Bash(git remote:*),Bash(git branch:*),Bash(git show:*),Bash(echo:*),Bash(gh pr diff:*),Bash(gh pr view:*),Bash(gh release view:*),Bash(python:*),Bash(python3:*),Bash(pytest:*),Skill,Write(/tmp/*)" + --append-system-prompt "Repo-specific focus: real-time diffusion hot-path discipline (flag + .item()/.cpu()/.numpy()/print(tensor) in step/frame loops), no hard-coded .cuda()/device, + best-effort error paths must never raise into the render loop, use raise ... from e, TensorRT + zero-copy buffer safety (dtype/shape/lifetime). Honor intentional style: keep + typing.Dict/Optional/Union (no PEP585/604 rewrites), line-length 119, single-letter math vars + OK, torch.device valid as a default arg. Skip nits ruff/pyrefly already cover. + Tool usage: never run filesystem-wide search (find /, grep -r /), stay inside the repo + checkout. Never redirect Bash output to a file with > or >>, it is blocked; pipe to head/tail + or just read the direct stdout instead." From 4f8b75a3547a177822e9fdc2984e8420bcdd212a Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 13 Jul 2026 21:21:53 -0400 Subject: [PATCH 35/37] chore: add Claude review workflow to PR4 branch for head-ref validation --- .github/workflows/claude-code-review.yml | 50 ++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .github/workflows/claude-code-review.yml diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml new file mode 100644 index 000000000..7505cecef --- /dev/null +++ b/.github/workflows/claude-code-review.yml @@ -0,0 +1,50 @@ +name: Claude Code Review + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + # base-branch filter: PRs *into* SDTD_032_dev, or into any branch in the + # chained remediation-PR stack (2-5), so Claude also fires on chained PRs. + branches: + [ + SDTD_032_dev, + refactor/remediation-phases-1-4, + perf/unet-step-subphase-5x, + perf/trt-zero-copy-audit-hardening, + refactor/param-schema-fp8-preflight, + ] + +jobs: + review: + if: github.event.pull_request.draft == false + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + issues: write + id-token: write + steps: + - uses: actions/checkout@v4 + with: { fetch-depth: 0 } + + - uses: anthropics/claude-code-action@v1 + with: + claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + track_progress: true # forces tag mode -> sticky comment updates in place + use_sticky_comment: true # one review comment across pushes, not a new one each time + show_full_output: true # surfaces full tool output/denials in the run log for debugging + plugin_marketplaces: "https://github.com/anthropics/claude-code.git" + plugins: "code-review@claude-code-plugins" + prompt: "/code-review:code-review ${{ github.repository }}/pull/${{ github.event.pull_request.number }}" + claude_args: >- + --max-turns 60 + --allowedTools "Bash(git diff:*),Bash(git status:*),Bash(git log:*),Bash(git fetch:*),Bash(git merge-base:*),Bash(git rev-parse:*),Bash(git remote:*),Bash(git branch:*),Bash(git show:*),Bash(echo:*),Bash(gh pr diff:*),Bash(gh pr view:*),Bash(gh release view:*),Bash(python:*),Bash(python3:*),Bash(pytest:*),Skill,Write(/tmp/*)" + --append-system-prompt "Repo-specific focus: real-time diffusion hot-path discipline (flag + .item()/.cpu()/.numpy()/print(tensor) in step/frame loops), no hard-coded .cuda()/device, + best-effort error paths must never raise into the render loop, use raise ... from e, TensorRT + zero-copy buffer safety (dtype/shape/lifetime). Honor intentional style: keep + typing.Dict/Optional/Union (no PEP585/604 rewrites), line-length 119, single-letter math vars + OK, torch.device valid as a default arg. Skip nits ruff/pyrefly already cover. + Tool usage: never run filesystem-wide search (find /, grep -r /), stay inside the repo + checkout. Never redirect Bash output to a file with > or >>, it is blocked; pipe to head/tail + or just read the direct stdout instead." From 0761485fd621bb355a4c2fee3a5f01534556a471 Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 13 Jul 2026 21:29:42 -0400 Subject: [PATCH 36/37] chore: add Claude review workflow to PR5 branch for head-ref validation --- .github/workflows/claude-code-review.yml | 50 ++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .github/workflows/claude-code-review.yml diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml new file mode 100644 index 000000000..7505cecef --- /dev/null +++ b/.github/workflows/claude-code-review.yml @@ -0,0 +1,50 @@ +name: Claude Code Review + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + # base-branch filter: PRs *into* SDTD_032_dev, or into any branch in the + # chained remediation-PR stack (2-5), so Claude also fires on chained PRs. + branches: + [ + SDTD_032_dev, + refactor/remediation-phases-1-4, + perf/unet-step-subphase-5x, + perf/trt-zero-copy-audit-hardening, + refactor/param-schema-fp8-preflight, + ] + +jobs: + review: + if: github.event.pull_request.draft == false + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + issues: write + id-token: write + steps: + - uses: actions/checkout@v4 + with: { fetch-depth: 0 } + + - uses: anthropics/claude-code-action@v1 + with: + claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + track_progress: true # forces tag mode -> sticky comment updates in place + use_sticky_comment: true # one review comment across pushes, not a new one each time + show_full_output: true # surfaces full tool output/denials in the run log for debugging + plugin_marketplaces: "https://github.com/anthropics/claude-code.git" + plugins: "code-review@claude-code-plugins" + prompt: "/code-review:code-review ${{ github.repository }}/pull/${{ github.event.pull_request.number }}" + claude_args: >- + --max-turns 60 + --allowedTools "Bash(git diff:*),Bash(git status:*),Bash(git log:*),Bash(git fetch:*),Bash(git merge-base:*),Bash(git rev-parse:*),Bash(git remote:*),Bash(git branch:*),Bash(git show:*),Bash(echo:*),Bash(gh pr diff:*),Bash(gh pr view:*),Bash(gh release view:*),Bash(python:*),Bash(python3:*),Bash(pytest:*),Skill,Write(/tmp/*)" + --append-system-prompt "Repo-specific focus: real-time diffusion hot-path discipline (flag + .item()/.cpu()/.numpy()/print(tensor) in step/frame loops), no hard-coded .cuda()/device, + best-effort error paths must never raise into the render loop, use raise ... from e, TensorRT + zero-copy buffer safety (dtype/shape/lifetime). Honor intentional style: keep + typing.Dict/Optional/Union (no PEP585/604 rewrites), line-length 119, single-letter math vars + OK, torch.device valid as a default arg. Skip nits ruff/pyrefly already cover. + Tool usage: never run filesystem-wide search (find /, grep -r /), stay inside the repo + checkout. Never redirect Bash output to a file with > or >>, it is blocked; pipe to head/tail + or just read the direct stdout instead." From c46b8b9ab6e9b5f24c0f3d282f6b54372438d4ec Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 18 Jul 2026 20:18:12 -0400 Subject: [PATCH 37/37] chore: fix import ordering in merge-carried diagnostics files --- src/streamdiffusion/utils/diagnostics.py | 1 - tests/unit/test_diagnostics.py | 1 - 2 files changed, 2 deletions(-) diff --git a/src/streamdiffusion/utils/diagnostics.py b/src/streamdiffusion/utils/diagnostics.py index 398be09e6..feeb328c4 100644 --- a/src/streamdiffusion/utils/diagnostics.py +++ b/src/streamdiffusion/utils/diagnostics.py @@ -27,7 +27,6 @@ from pathlib import Path from typing import Any, Dict, Optional - SCHEMA_VERSION = "v1" # Only these env-var prefixes are dumped -- never the full os.environ (avoids leaking secrets). ENV_ALLOWLIST_PREFIXES = ("CUDALINK_", "HF_", "SD_", "SDTD_") diff --git a/tests/unit/test_diagnostics.py b/tests/unit/test_diagnostics.py index d7ac5d6be..3720382f3 100644 --- a/tests/unit/test_diagnostics.py +++ b/tests/unit/test_diagnostics.py @@ -15,7 +15,6 @@ from streamdiffusion.utils import diagnostics - # --------------------------------------------------------------------------- # format_report_text # ---------------------------------------------------------------------------