diff --git a/.gitignore b/.gitignore index 625c206..bc3e7cc 100644 --- a/.gitignore +++ b/.gitignore @@ -111,3 +111,5 @@ scripts/diarize/__pycache__/ # Rust spike build artifacts spike/audio-sync/target/ +spike/rust-decode-spike/ffmpeg-candidate/target/ +spike/rust-decode-spike/gstreamer-candidate/target/ diff --git a/CLAUDE.md b/CLAUDE.md index 2fbbf4f..5ed71f0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -256,6 +256,8 @@ Core problems being fixed: non-deterministic output, no project file, no pipelin Phases 0–4 (scripts) and 5–6 (Remotion) can proceed on separate branches in parallel. +**Native rewrite spikes** (separate from the phases above — see `docs/rfcs/0001-native-desktop-rewrite.md`): throwaway Rust crates under `spike/`, deleted once each spike concludes. `spike/audio-sync/` validated the agent-driven Rust dev loop; `spike/rust-decode-spike/` (issue #108) is evaluating `ffmpeg-the-third` vs `gstreamer-rs` for the render engine's decode/encode binding — see its `FINDINGS.md`. + ### Target directory additions (post-refactor) **`.gitignore` rule:** Every directory under `.ragtech/` (and any other runtime-generated directory such as `runs/`, `cache/`, output artifact dirs) must be listed in `.gitignore`. These hold generated/binary data, not source code, and must never be committed. diff --git a/docs/implementation-guides/issue-108-rust-decode-spike.md b/docs/implementation-guides/issue-108-rust-decode-spike.md new file mode 100644 index 0000000..7f2a95b --- /dev/null +++ b/docs/implementation-guides/issue-108-rust-decode-spike.md @@ -0,0 +1,87 @@ +# Issue #108 — spike(rust/decode): FFmpeg vs gstreamer-rs binding decision + +## Goal + +Build the minimal Rust harness at `spike/rust-decode-spike/` needed to answer the five evaluation +criteria in [issue #108](https://github.com/ragTechDev/deckcreate/issues/108): can `ffmpeg-the-third` +or `gstreamer-rs` reach hardware codec paths (VideoToolbox / NVENC/NVDEC), decode frame-accurately, +build cleanly cross-platform, and (for GStreamer) avoid proprietary-codec redistribution risk. + +Not a working compositor — two standalone binary crates (one per candidate), a committed test +fixture, and a `FINDINGS.md` that gets filled in as results land from each platform. This machine +is an Apple M3, so the Mac/VideoToolbox rows get real results in this branch. Mac M2 and +Windows/NVIDIA rows are scaffolded and stay open for the contributors who own that hardware to fill +in via the PR. + +Follows the established spike convention from #93–#99 (`spike//`, standalone `Cargo.toml` +per crate, no root-level workspace, `spike(rust/):` commit prefix). + +## Steps + +### Step 1 — Scaffold layout, fixtures, and .gitignore + +Create `spike/rust-decode-spike/` with: +- `fixtures/generate_fixtures.sh` — regenerates the test clip and reference frames deterministically + via the `ffmpeg` CLI (`testsrc2` pattern, so pixel content per frame is reproducible). +- `fixtures/test-clip.mp4` — small (320×240, ~3s, H.264) committed fixture. +- `fixtures/reference/*.ppm` — one or two reference frames (raw RGB24 PPM) extracted at known + timestamps, used as ground truth for the pixel-accuracy check. +- Append `spike/rust-decode-spike/*/target/` to the root `.gitignore`. + +**Status check:** `test -f spike/rust-decode-spike/fixtures/test-clip.mp4 && grep -q 'spike/rust-decode-spike' .gitignore` + +### Step 2 — ffmpeg-the-third candidate: decode, seek, pixel verify (software + VideoToolbox) + +Create `spike/rust-decode-spike/ffmpeg-candidate/` (standalone `Cargo.toml`, depends on +`ffmpeg-the-third`). CLI: `ffmpeg-candidate [--hw]`. +Software path: open input, seek near the target timestamp, decode forward to the exact frame, +convert to RGB24 via the software scaler, compare against the reference PPM (mean/max abs diff), +print PASS/FAIL. `--hw` path: raw FFI through `ffmpeg_the_third::ffi` to set `hw_device_ctx` +(`av_hwdevice_ctx_create` with `AVHWDeviceType::VIDEOTOOLBOX`) and a `get_format` callback on the +decoder's *unopened* `AVCodecContext`, then `av_hwframe_transfer_data` to copy each decoded frame +back to a software buffer before the same RGB24 compare. No high-level wrapper exists for this in +the crate (checked: no hwaccel/hwdevice/hwcontext module) — this hand-written unsafe path, and how +much of it there is, is itself the "without extra shims" answer for criterion #1. Software and +hardware share one seek/decode/compare flow rather than splitting into a second commit, since the +`--hw` branch is threaded through the same loop, not a separable unit. + +**Status check:** `cargo run --manifest-path spike/rust-decode-spike/ffmpeg-candidate/Cargo.toml -- spike/rust-decode-spike/fixtures/test-clip.mp4 1.5 spike/rust-decode-spike/fixtures/reference/frame_at_1.5s.ppm --hw | grep -q "HARDWARE PATH: active"` + +### Step 3 — gstreamer-rs candidate: decode, seek, pixel verify (software + VideoToolbox) + +Create `spike/rust-decode-spike/gstreamer-candidate/` (standalone `Cargo.toml`, depends on +`gstreamer`, `gstreamer-app`, `gstreamer-video`). CLI mirrors the ffmpeg candidate. Software path +uses `avdec_h264` (explicit software decoder, not `decodebin`, so the software/hardware choice +stays deliberate rather than autonegotiated); hardware path pins the pipeline to the `vtdec_hw` +element (hardware-only — pipeline fails to reach PAUSED if VideoToolbox isn't actually engaged, +which is itself the criterion #1 answer for this candidate). Pull the frame via `appsink`, compare +against the same reference PPM. Note for the Windows/NVIDIA contributor: swap `vtdec_hw` for +`nvh264dec` (nvcodec plugin) behind a platform check to complete criterion #1 on that hardware — +not attempted here since it cannot be verified on this machine. + +**Status check:** `cargo run --manifest-path spike/rust-decode-spike/gstreamer-candidate/Cargo.toml -- spike/rust-decode-spike/fixtures/test-clip.mp4 1.5 spike/rust-decode-spike/fixtures/reference/frame_at_1.5s.ppm | grep -q PASS` + +### Step 4 — FINDINGS.md and contributor README + +Write `spike/rust-decode-spike/FINDINGS.md` covering all five evaluation criteria, with real +results filled in for Mac M3 (this machine) and open template rows for Mac M2 / Windows+NVIDIA. +Include the GStreamer plugin licensing research for criterion #5 (which plugin ships which codec, +under what license). Write `spike/rust-decode-spike/README.md` with exact per-platform setup + +run commands for the contributors who'll fill in the remaining rows. + +**Status check:** `test -f spike/rust-decode-spike/FINDINGS.md && grep -q "Mac M2" spike/rust-decode-spike/FINDINGS.md && grep -q "Windows" spike/rust-decode-spike/FINDINGS.md` + +### Step 5 — Update CLAUDE.md + +Add a one-line pointer to the spike under a relevant section (Refactor Plan) so future agents know +it exists and where. + +**Status check:** `grep -q 'rust-decode-spike' CLAUDE.md` + +## Out of scope + +- A working compositor or any production integration +- Full verification on Mac M2 or Windows/NVIDIA hardware (scaffolded for contributors, not run here) +- `ac-ffmpeg` or `ffmpeg-next` fallback candidates (only pursued if `ffmpeg-the-third` fails) +- The CLI-subprocess (`ffmpeg-sidecar`) alternative — explicitly deferred per issue #108 +- Any change to files outside `spike/rust-decode-spike/`, `CLAUDE.md`, and `.gitignore` diff --git a/spike/rust-decode-spike/FINDINGS.md b/spike/rust-decode-spike/FINDINGS.md new file mode 100644 index 0000000..18b056a --- /dev/null +++ b/spike/rust-decode-spike/FINDINGS.md @@ -0,0 +1,190 @@ +# Findings — FFmpeg vs gstreamer-rs binding decision (issue #108) + +Status: **Mac M3, Mac M2, and Windows+NVIDIA results are all verified on this branch.** + +## Recommendation + +**Recommendation unchanged: still use `ffmpeg-the-third`, but with a stronger caveat for Epic 1.** +Windows+NVIDIA no longer blocks on unknowns, but neither candidate's hardware path currently meets +the spike's pixel-diff tolerance on this machine. Rationale, in order of weight: + +1. **`ffmpeg-the-third` remains the better technical baseline across both tested platforms.** + On Mac M3 it stays pixel-exact in hardware. On Windows+NVIDIA, its CUDA path is active and + materially closer to reference frames than `gstreamer-rs`/`nvh264dec` (lower mean and much lower + max diff), even though it still fails this spike's current max-diff threshold and needs follow-up. +2. **Licensing risk is a wash, not a differentiator.** Both candidates ultimately depend on the same + underlying FFmpeg/codec licensing question — see criterion #5. This needs solving once, for + whichever binding wins, not as a reason to prefer one over the other. +3. `ffmpeg-the-third`'s missing high-level hwaccel API is still a one-time, bounded integration cost + (the `hw` module in `ffmpeg-candidate/src/main.rs` now includes both VideoToolbox and CUDA + variants). `gstreamer-rs` keeps the easier integration surface, but has unresolved hardware-path + pixel-quality failures on both Mac M3 (`vtdec_hw`) and Windows (`nvh264dec`) in this spike. + +This recommendation should be revisited only if follow-up investigation shows the Windows CUDA +quality failure is a tolerance/configuration artifact in `ffmpeg-the-third` while `gstreamer-rs` +can be made pixel-stable with less risk. Right now, both fail hardware tolerance on this machine, +but `ffmpeg-the-third` fails less severely and remains the stronger cross-platform starting point. + +## Criterion #1 — Hardware codec access + +| Candidate | Mac M3 (this branch) | Mac M2 | Windows + NVIDIA | +|---|---|---|---| +| `ffmpeg-the-third` | **Reaches VideoToolbox, pixel-exact.** No high-level hwaccel API exists in the crate (checked: no hwaccel/hwdevice/hwcontext module). Required hand-written unsafe FFI through `ffmpeg_the_third::ffi`: `av_hwdevice_ctx_create(AVHWDeviceType::VIDEOTOOLBOX)` + a `get_format` callback set on the decoder's *unopened* `AVCodecContext` + `av_hwframe_transfer_data` per frame. ~40 lines, see `ffmpeg-candidate/src/main.rs`'s `hw` module. | **Reaches VideoToolbox, pixel-exact.** `HARDWARE PATH: active` on both test timestamps. Results are identical to M3 — mean=0.000, max=0 on both frames. Same unsafe FFI path, same 40 lines, same result — no M2-specific friction. | **Reaches CUDA/NVDEC and transfers frames back to software.** Implemented `#[cfg(target_os = "windows")]` hw module with `av_hwdevice_ctx_create(AVHWDeviceType::CUDA)`, `get_format` selecting `AVPixelFormat::CUDA`, and `av_hwframe_transfer_data`. `HARDWARE PATH: active` printed on both test timestamps. Output quality is not yet within tolerance (criterion #2), but hardware negotiation is real. | +| `gstreamer-rs` | **Negotiates and prerolls** (`vtdec_hw` is hardware-only by construction, so a successful preroll is real confirmation). Zero unsafe code — just a different element name. **But**: output is visibly wrong (criterion #2) and building the pipeline printed `GStreamer-GL-WARNING: An NSApplication needs to be running on the main thread` — `vtdec_hw`'s output is GL-memory-backed (`video/x-raw(memory:GLMemory)`), and a plain CLI render-engine binary has no Cocoa run loop. Neither issue blocked this spike from running, but both are real open questions for a production render engine. | **Negotiates and prerolls** (`vtdec_hw negotiated successfully` printed). Pixel results are identical to M3 (mean=3.855/3.886, max=168/181 — same FAIL). One notable difference from M3: the `GStreamer-GL-WARNING: An NSApplication needs to be running on the main thread` did **not** appear on this machine (macOS 14.5 / Darwin 23.5.0, vs. M3's macOS 15.x / Darwin 24.6.0). The vtdec_hw pixel failure is reproducible regardless — not a GL-warning-caused artifact on M3, but a real colorimetry problem on both. | **Reaches NVDEC via `nvh264dec` and negotiates successfully.** Windows code now picks decoder element by platform (`nvh264dec` on Windows, `vtdec_hw` on macOS). `HARDWARE PATH: active (nvh264dec negotiated successfully)` printed on both test timestamps. As on Mac hardware decode, output quality is outside tolerance (criterion #2). | + +## Criterion #2 — Seek/decode accuracy + +Both candidates seek near the target timestamp via their native seek API, then decode forward to +the exact requested frame — the real jump-cut access pattern, not sequential decode from frame 0. +Compared against reference frames extracted by the `ffmpeg` CLI at known frame indices (`fixtures/reference/*.ppm`). + +| Candidate / path | t=0.5s | t=1.5s | +|---|---|---| +| `ffmpeg-the-third`, software | mean_abs_diff=0.000, max=0 — **PASS (pixel-exact)** | mean_abs_diff=0.000, max=0 — **PASS (pixel-exact)** | +| `ffmpeg-the-third`, `--hw` (VideoToolbox) | mean_abs_diff=0.000, max=0 — **PASS (pixel-exact)** | mean_abs_diff=0.000, max=0 — **PASS (pixel-exact)** | +| `gstreamer-rs`, software (`avdec_h264`) | mean_abs_diff=1.391, max=6 — **PASS** (within tolerance, not bit-exact) | mean_abs_diff=1.377, max=6 — **PASS** | +| `gstreamer-rs`, `--hw` (`vtdec_hw`) | mean_abs_diff=3.855, max=168 — **FAIL** | mean_abs_diff=3.886, max=181 — **FAIL** | + +The `vtdec_hw` FAIL was investigated, not just reported: diffed the decoded frame against the +*neighboring* reference frames (indices 14 and 16) to rule out an off-by-one-frame seek bug — +both neighbors scored *worse* (mean ~7.2–7.5) than the correctly-targeted frame 15 (mean 3.855), so +this is the right frame with a real per-pixel quality problem, not a seek-accuracy bug. Given the +GLMemory finding in criterion #1, the leading hypothesis is a colorimetry/YUV-matrix mismatch +during the implicit GPU→CPU readback in `videoconvert`, not yet root-caused further — this is +exactly the kind of pre-Epic-1 risk this spike exists to surface. + +Tolerance used: `mean_abs_diff < 2.0 && max_abs_diff < 24` (covers ordinary YUV→RGB rounding +differences between decode paths; `vtdec_hw`'s failure is ~2–7x outside this band, not a rounding +error). + +### Mac M2 run results (this branch) + +| Candidate / path | t=0.5s | t=1.5s | +|---|---|---| +| `ffmpeg-the-third`, software | mean_abs_diff=0.000, max=0 — **PASS (pixel-exact)** | mean_abs_diff=0.000, max=0 — **PASS (pixel-exact)** | +| `ffmpeg-the-third`, `--hw` (VideoToolbox) | `HARDWARE PATH: active`; mean_abs_diff=0.000, max=0 — **PASS (pixel-exact)** | `HARDWARE PATH: active`; mean_abs_diff=0.000, max=0 — **PASS (pixel-exact)** | +| `gstreamer-rs`, software (`avdec_h264`) | mean_abs_diff=1.391, max=6 — **PASS** | mean_abs_diff=1.377, max=6 — **PASS** | +| `gstreamer-rs`, `--hw` (`vtdec_hw`) | `HARDWARE PATH: active (vtdec_hw negotiated successfully)`; mean_abs_diff=3.855, max=168 — **FAIL** | `HARDWARE PATH: active (vtdec_hw negotiated successfully)`; mean_abs_diff=3.886, max=181 — **FAIL** | + +All M2 results are identical to M3. The `GStreamer-GL-WARNING` about `NSApplication` did not appear +on this machine (macOS 14.5), but the vtdec_hw pixel failure is the same magnitude — confirming +the colorimetry issue is not a GL-warning-correlated artifact but a consistent hardware-path +characteristic of `vtdec_hw` across both M-series chips tested. + +### Windows + NVIDIA run results (this branch) + +| Candidate / path | t=0.5s | t=1.5s | +|---|---|---| +| `ffmpeg-the-third`, software | mean_abs_diff=0.446, max=2 — **PASS** | mean_abs_diff=0.454, max=2 — **PASS** | +| `ffmpeg-the-third`, `--hw` (CUDA/NVDEC) | `HARDWARE PATH: active`; mean_abs_diff=1.846, max=91 — **FAIL** | `HARDWARE PATH: active`; mean_abs_diff=1.687, max=105 — **FAIL** | +| `gstreamer-rs`, software (`avdec_h264`) | mean_abs_diff=1.391, max=6 — **PASS** | mean_abs_diff=1.377, max=6 — **PASS** | +| `gstreamer-rs`, `--hw` (`nvh264dec`) | `HARDWARE PATH: active`; mean_abs_diff=3.855, max=168 — **FAIL** | `HARDWARE PATH: active`; mean_abs_diff=3.886, max=181 — **FAIL** | + +Interpretation: both hardware paths are active on this NVIDIA machine, but both exceed tolerance. +`ffmpeg-the-third` is noticeably closer to reference than `gstreamer-rs` in hardware mode +(especially max diff), so this row did not overturn the recommendation, but it did raise the risk +level for Epic 1: Windows hardware decode needs explicit pixel-quality hardening before production. + +## Criterion #3 — Build complexity + +| Candidate | System dependencies on this Mac | Notes | +|---|---|---| +| `ffmpeg-the-third` | `pkg-config` (not installed by default — `brew install pkgconf`), FFmpeg dev libs (already present via `brew install ffmpeg`, no extra plugins needed) | One dependency crate. Clean incremental builds (~6s after first fetch). | +| `gstreamer-rs` | `pkg-config`, full GStreamer (`brew install gstreamer` — Homebrew now bundles **all** gst-plugins-base/good/bad/ugly into one 217 MB formula, a packaging change worth knowing about since older docs/tutorials still reference the split `gst-plugins-*` formulas) | Three dependency crates (`gstreamer`, `gstreamer-app`, `gstreamer-video`). Clean build from empty cache ~20s. | + +Neither candidate's Windows or true cross-compilation story is tested here — that's the open row +in the README for the Windows/NVIDIA contributor. Both builds otherwise worked without patching +either crate. + +Mac M2 build notes (macOS 14.5, Rust 1.97.1, freshly installed): + +- Rust was not preinstalled on this machine — required `curl | sh` install via `rustup` (same as + would be true on any fresh Mac). +- `pkgconf` was not preinstalled — `brew install pkgconf` required, same as M3. +- FFmpeg dev libs were already present via existing `brew install ffmpeg` install. +- GStreamer (`brew install gstreamer`) was not present — installed fresh, same 217 MB bundled + formula as M3, no extra plugin steps. +- First build times: `ffmpeg-candidate` ~17s from cold; `gstreamer-candidate` ~17s from cold. + In line with M3 (~6s / ~20s on warm incremental; caches were empty here so first-build times + are higher). +- No build errors or patching required on either candidate. + +Windows+NVIDIA notes from this run: + +- Toolchain/setup friction was real for `ffmpeg-the-third`: this environment needed explicit + `LIBCLANG_PATH` (LLVM install) for `ffmpeg-sys-the-third`'s bindgen step, plus explicit + `PKG_CONFIG_PATH` and `FFMPEG_DIR` so Cargo could find `C:\ffmpeg` headers/libs. +- `gstreamer-rs` built cleanly once GStreamer SDK path was exported (`...\\gstreamer\\...\\bin` + and `...\\lib\\pkgconfig`), with no extra bindgen/libclang work. +- Both crates' unit tests passed after environment setup (`ffmpeg-candidate`: 3 tests, + `gstreamer-candidate`: 2 tests). + +## Criterion #4 — Crate maturity + +Carried over from the community-discussion research already folded into issue #108's body +(r/rust thread on the state of FFmpeg bindings in Rust): + +- `ffmpeg-next` is maintenance-mode only (one commenter disputes this, most don't). + `ffmpeg-the-third` is the actively-developed fork — confirmed independently here: its crate + version (`5.0.0+ffmpeg-8.1`) tracks the very recent FFmpeg 8.1 release used in this spike, and it + builds and runs cleanly against it. +- `gstreamer-rs` is maintained under the official gstreamer-rs GitHub org, tracks GStreamer's own + release cadence (this spike used 1.28.5, current at spike time), and has a materially larger + contributor base and issue-response cadence than any FFmpeg Rust binding — not independently + re-verified here, this is the community consensus already cited in issue #108. + +## Criterion #5 — Redistribution/licensing risk + +The issue scoped this criterion to gstreamer-rs, but the same underlying question turned out to +apply to *both* candidates, since `ffmpeg-the-third` binds this same system FFmpeg build directly. + +Checked via `gst-inspect-1.0 | grep License` plus binary dependency inspection (`otool -L` +on macOS, `llvm-readobj --needed-libs` on Windows) on the actual plugin binaries used: + +| Plugin / element | License reported | Linked codec libs | Risk | +|---|---|---|---| +| `applemedia` (`vtdec_hw`, `vtdec`) | LGPL | Only Apple system frameworks (AVFoundation, VideoToolbox, CoreMedia, CoreVideo) + GStreamer's own LGPL libs | **Clean.** No GPL or proprietary exposure — VideoToolbox is a system framework, not a bundled codec. | +| `videoparsersbad` (`h264parse`) | LGPL | None (pure bitstream parsing, no codec implementation) | **Clean.** | +| `libav` (`avdec_h264`, used as the software comparison baseline) | LGPL (the GStreamer wrapper) | **This machine's Homebrew `ffmpeg` (8.1.2), built with `--enable-gpl --enable-libx264 --enable-libx265`** | **GPL-contaminated on this machine.** The wrapper is LGPL but it dynamically links a GPL-configured FFmpeg build, which makes the combination subject to GPL terms for this decode path specifically. | +| `nvcodec` (`nvh264dec`/`nvdec`) | LGPL (`gst-inspect-1.0 nvcodec`) | `gstnvcodec.dll` links to Windows CRT + GStreamer libs (`gstreamer-1.0-0.dll`, `gstvideo-1.0-0.dll`, etc.). NVIDIA runtime libraries (`nvcuda.dll`, `nvcuvid.dll`) are provided by the installed GPU driver (`C:\Windows\System32`). | **No new GPL signal from the plugin itself; still driver/runtime-dependent.** The plugin advertises LGPL, but operational dependency on NVIDIA driver DLLs is real and should be validated in target deployment environments. | + +**The important nuance**: `libx264`/`libx265` are *encoder-only* libraries — H.264/HEVC *decoding* +in FFmpeg does not require them at all. The GPL exposure found above is a property of *this +machine's specific FFmpeg build*, not an inherent property of either candidate. **Action item for +whichever binding wins Epic 1**: build (or request via Homebrew formula options) FFmpeg without +`--enable-gpl`/`--enable-libx264`/`--enable-libx265` for the decode-only render-engine build — an +LGPL-only FFmpeg build fully supports H.264/HEVC decode and removes this risk for both candidates. +No literally proprietary/closed-source plugin was found in either candidate's dependency chain on +this machine — the risk is GPL license-strength contamination, not vendor lock-in. + +## Considered and deferred (from issue #108) + +Not revisited here — CLI-subprocess decoding (`ffmpeg-sidecar`) remains deferred for the same +reason stated in the issue: jump-cut compositing needs in-process frame-accurate seek/decode. + +## Open questions for Epic 1 + +1. **Windows/NVIDIA hardware quality is now verified as problematic** — both `--hw` paths engage, + but both fail current pixel tolerance. Epic 1 should treat Windows hardware decode as + integration-incomplete until this is root-caused and corrected. +2. **`vtdec_hw`'s colorimetry issue is unresolved**, not just observed. If gstreamer-rs is + reconsidered later, that needs root-causing (likely a `glcolorconvert`/`gldownload` pipeline + fix, or explicit `colorimetry=` caps pinning) before it can be trusted for compositing. +3. **The GPL-vs-LGPL FFmpeg build decision is a hard prerequisite for Epic 1**, independent of + which binding wins — see criterion #5. This should become its own tracked issue before Epic 1 + starts, not an implicit assumption. +4. `ffmpeg-the-third`'s `hw` module in this spike (`ffmpeg-candidate/src/main.rs`) is close to a + drop-in starting point for Epic 1's actual `encoderProfile` hardware-decode implementation, not + just throwaway spike code — worth reading directly rather than re-deriving from FFmpeg's C + `hw_decode.c` example. + +## Environment (this branch's results) + +- Mac M3 hardware: Apple M3 (Mac15,12), macOS (Darwin 24.6.0) +- Mac M2 hardware: Apple M2 (MacBook Air, Mac14,2), macOS 14.5 (Darwin 23.5.0) +- Windows hardware: NVIDIA GeForce RTX 5050 Laptop GPU, driver `592.15` +- Rust: `rustc 1.97.1`, `cargo 1.97.1` (all platforms) +- FFmpeg (Mac, both M2 and M3): `8.1.2` (Homebrew, `--enable-gpl --enable-videotoolbox --enable-libx264 --enable-libx265`) +- FFmpeg (Windows): `N-124419-gc8a4770599-20260508` shared build at `C:\ffmpeg` (`--enable-ffnvcodec --enable-cuda-llvm --enable-gpl --enable-libx264 --enable-libx265`, `ffmpeg -hwaccels` includes `cuda`) +- GStreamer: `1.28.5` (Homebrew on Mac M2 and M3; official MSI on Windows at `C:\Users\Victoria\AppData\Local\Programs\gstreamer\1.0\msvc_x86_64`) +- `pkg-config` / `pkgconf`: not preinstalled by default on any machine tested; required `brew install pkgconf` on Mac diff --git a/spike/rust-decode-spike/README.md b/spike/rust-decode-spike/README.md new file mode 100644 index 0000000..fd0f05c --- /dev/null +++ b/spike/rust-decode-spike/README.md @@ -0,0 +1,103 @@ +# rust-decode-spike + +Two standalone Rust binaries answering issue #108: does `ffmpeg-the-third` or `gstreamer-rs` +decode-and-seek a real `.mp4` frame-accurately, and can each reach hardware codec paths +(VideoToolbox on Mac, NVENC/NVDEC on Windows+NVIDIA) without extra shims? + +> **Temporary** — this crate lives under `spike/` and will be deleted when the spike concludes. +> See [issue #108](https://github.com/ragTechDev/deckcreate/issues/108) and `FINDINGS.md`. + +Real results for **Mac M3** are already in `FINDINGS.md`. If you're on a **Mac M2** or a +**Windows machine with an NVIDIA GPU**, this is where you fill in the remaining rows — please +paste your output into `FINDINGS.md` (or a comment on #108) rather than just reporting pass/fail. + +## Prerequisites + +Rust stable toolchain via `rustup` (skip if you already have it): + +```bash +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh +source "$HOME/.cargo/env" +rustc --version # e.g. rustc 1.97.0 +cargo --version +``` + +### macOS + +```bash +brew install pkgconf ffmpeg gstreamer +``` + +(As of this writing, Homebrew's `gstreamer` formula bundles all `gst-plugins-*` into one ~217 MB +install — you do not need to separately install `gst-plugins-base`/`-good`/`-bad`.) + +### Windows + NVIDIA + +Not verified from this branch. You will need: +- FFmpeg dev libraries + `pkg-config` (e.g. via [vcpkg](https://vcpkg.io) or the + [gyan.dev](https://www.gyan.dev/ffmpeg/builds/) shared build) for the `ffmpeg-candidate`. +- GStreamer's official Windows installer (development package) for the `gstreamer-candidate`, + including the `nvcodec` plugin (ships with the standard GStreamer Windows installer on machines + with the NVIDIA Video Codec SDK runtime present). +- An NVIDIA GPU with current drivers for the `--hw` / `vtdec_hw`-equivalent path — see the TODOs + in each candidate's `main.rs` for where the NVDEC/`nvh264dec` code needs to be added; it is not + implemented in this branch because it cannot be verified on Mac hardware. + +Document exactly what you had to install and any friction in `FINDINGS.md` — that's as much a +part of criterion #3 (build complexity) as whether it compiles. + +## Fixtures + +`fixtures/test-clip.mp4` (320×240, ~3s, H.264, committed) plus two reference RGB24 frames in +`fixtures/reference/` at t=0.5s and t=1.5s, extracted via the `ffmpeg` CLI at exact frame indices. +Regenerate with `./fixtures/generate_fixtures.sh` if needed (requires the `ffmpeg` CLI) — the +content is deterministic (`testsrc2` pattern), so a regenerated clip should still match. + +## Run + +Both candidates share the same CLI shape: + +``` + [--hw] +``` + +```bash +# ffmpeg-the-third — software, then VideoToolbox/NVDEC hardware +cargo run --manifest-path ffmpeg-candidate/Cargo.toml -- \ + fixtures/test-clip.mp4 1.5 fixtures/reference/frame_at_1.5s.ppm +cargo run --manifest-path ffmpeg-candidate/Cargo.toml -- \ + fixtures/test-clip.mp4 1.5 fixtures/reference/frame_at_1.5s.ppm --hw + +# gstreamer-rs — software, then VideoToolbox/NVDEC hardware +cargo run --manifest-path gstreamer-candidate/Cargo.toml -- \ + fixtures/test-clip.mp4 1.5 fixtures/reference/frame_at_1.5s.ppm +cargo run --manifest-path gstreamer-candidate/Cargo.toml -- \ + fixtures/test-clip.mp4 1.5 fixtures/reference/frame_at_1.5s.ppm --hw +``` + +Repeat with `fixtures/reference/frame_at_0.5s.ppm` and `0.5` for the second data point. Each run +prints `HARDWARE PATH: active/inactive` (when `--hw` is passed) and a final +`mean_abs_diff=... max_abs_diff=... -> PASS/FAIL` line — paste both lines into `FINDINGS.md`. + +## Unit tests + +```bash +cargo test --manifest-path ffmpeg-candidate/Cargo.toml +cargo test --manifest-path gstreamer-candidate/Cargo.toml +``` + +These only cover the pure PPM/pixel-compare helpers, not the decode paths themselves — the decode +paths are exercised end-to-end by the `cargo run` commands above against the committed fixtures, +which is the actual criteria-answering evidence for this spike. + +## Layout + +| Path | Purpose | +|---|---| +| `ffmpeg-candidate/` | Standalone crate, `ffmpeg-the-third` binding | +| `gstreamer-candidate/` | Standalone crate, `gstreamer-rs` binding | +| `fixtures/` | Committed test clip + reference frames (shared by both candidates) | +| `FINDINGS.md` | The actual deliverable — evaluation-criteria results per platform | + +Each candidate is a fully independent `Cargo.toml` (no shared workspace), so you can build/run one +without the other's system dependencies installed. diff --git a/spike/rust-decode-spike/ffmpeg-candidate/Cargo.lock b/spike/rust-decode-spike/ffmpeg-candidate/Cargo.lock new file mode 100644 index 0000000..ced08f5 --- /dev/null +++ b/spike/rust-decode-spike/ffmpeg-candidate/Cargo.lock @@ -0,0 +1,279 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex 1.3.0", + "syn", +] + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "cc" +version = "1.2.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +dependencies = [ + "find-msvc-tools", + "shlex 2.0.1", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "clang" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84c044c781163c001b913cd018fc95a628c50d0d2dfea8bca77dad71edb16e37" +dependencies = [ + "clang-sys", + "libc", +] + +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "ffmpeg-candidate" +version = "0.1.0" +dependencies = [ + "ffmpeg-the-third", +] + +[[package]] +name = "ffmpeg-sys-the-third" +version = "5.0.0+ffmpeg-8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b00c204bb97c2b6c0329104c3e8bcf58d4d13cc88a04233326570672e0c2c5c9" +dependencies = [ + "bindgen", + "cc", + "clang", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "ffmpeg-the-third" +version = "5.0.0+ffmpeg-8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791f0d9315e4f0a0861fa5c8ca8d70a1bd099cce72bff111b4f7ed8d6ebe8ba3" +dependencies = [ + "bitflags", + "ffmpeg-sys-the-third", + "libc", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" diff --git a/spike/rust-decode-spike/ffmpeg-candidate/Cargo.toml b/spike/rust-decode-spike/ffmpeg-candidate/Cargo.toml new file mode 100644 index 0000000..c0203f5 --- /dev/null +++ b/spike/rust-decode-spike/ffmpeg-candidate/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "ffmpeg-candidate" +version = "0.1.0" +edition = "2021" +publish = false + +[[bin]] +name = "ffmpeg-candidate" +path = "src/main.rs" + +[dependencies] +ffmpeg-the-third = "5.0.0" diff --git a/spike/rust-decode-spike/ffmpeg-candidate/src/main.rs b/spike/rust-decode-spike/ffmpeg-candidate/src/main.rs new file mode 100644 index 0000000..e8d0a7f --- /dev/null +++ b/spike/rust-decode-spike/ffmpeg-candidate/src/main.rs @@ -0,0 +1,312 @@ +//! Spike harness for issue #108: does `ffmpeg-the-third` decode-and-seek a real +//! .mp4 frame-accurately, and can it reach VideoToolbox without extra shims? +//! +//! Usage: ffmpeg-candidate [--hw] + +use std::env; +use std::process::ExitCode; + +use ffmpeg_the_third as ffmpeg; +use ffmpeg_the_third::software::scaling::{context::Context as Scaler, flag::Flags}; +use ffmpeg_the_third::util::format::pixel::Pixel; + +mod ppm; + +const AV_TIME_BASE: f64 = 1_000_000.0; + +fn main() -> ExitCode { + let args: Vec = env::args().collect(); + if args.len() < 4 { + eprintln!( + "usage: {} [--hw]", + args.first().map(String::as_str).unwrap_or("ffmpeg-candidate") + ); + return ExitCode::FAILURE; + } + let clip_path = &args[1]; + let timestamp_secs: f64 = match args[2].parse() { + Ok(v) => v, + Err(e) => { + eprintln!("invalid timestamp '{}': {e}", args[2]); + return ExitCode::FAILURE; + } + }; + let reference_path = &args[3]; + let use_hw = args.iter().any(|a| a == "--hw"); + + match run(clip_path, timestamp_secs, reference_path, use_hw) { + Ok(true) => ExitCode::SUCCESS, + Ok(false) => ExitCode::FAILURE, + Err(e) => { + eprintln!("error: {e}"); + ExitCode::FAILURE + } + } +} + +fn run( + clip_path: &str, + timestamp_secs: f64, + reference_path: &str, + use_hw: bool, +) -> Result { + ffmpeg::init()?; + + let mut input = ffmpeg::format::input(clip_path)?; + let stream = input + .streams() + .best(ffmpeg::media::Type::Video) + .ok_or(ffmpeg::Error::StreamNotFound)?; + let stream_index = stream.index(); + let time_base = stream.time_base(); + + let mut decoder_ctx = ffmpeg::codec::context::Context::from_parameters(stream.parameters())?; + + if use_hw { + // No high-level API for this in ffmpeg-the-third — see FINDINGS.md criterion #1. + // SAFETY: decoder_ctx.as_mut_ptr() is a valid, exclusively-owned AVCodecContext + // that has not yet been opened (avcodec_open2 happens inside `.video()` below), + // so setting hw_device_ctx/get_format here is exactly the window FFmpeg's own + // hw_decode.c example requires them to be set in. + unsafe { + hw::attach_hardware(decoder_ctx.as_mut_ptr())?; + } + } + + let mut decoder = decoder_ctx.decoder().video()?; + + // Seek near the target, then decode forward to the exact frame — this is the + // real-world jump-cut access pattern, not just sequential decode from frame 0. + input.seek((timestamp_secs * AV_TIME_BASE) as i64, ..)?; + decoder.flush(); + + let mut rgb_frame = ffmpeg::util::frame::video::Video::empty(); + let mut scaler: Option = None; + let mut hw_path_active = false; + let mut found: Option = None; + + 'decode: for item in input.packets() { + let (packet_stream, packet) = item?; + if packet_stream.index() != stream_index { + continue; + } + decoder.send_packet(&packet)?; + + let mut decoded = ffmpeg::util::frame::video::Video::empty(); + while decoder.receive_frame(&mut decoded).is_ok() { + let pts_secs = decoded + .pts() + .map(|pts| pts as f64 * f64::from(time_base)) + .unwrap_or(f64::NEG_INFINITY); + + if pts_secs + 1e-6 < timestamp_secs { + continue; // not there yet, keep decoding forward from the seek point + } + + let software_frame = if use_hw { + let (transferred, was_hw) = hw::transfer_to_software(&decoded)?; + hw_path_active = hw_path_active || was_hw; + transferred + } else { + decoded.clone() + }; + + if scaler.is_none() { + scaler = Some(Scaler::get( + software_frame.format(), + software_frame.width(), + software_frame.height(), + Pixel::RGB24, + software_frame.width(), + software_frame.height(), + Flags::BILINEAR, + )?); + } + scaler.as_mut().unwrap().run(&software_frame, &mut rgb_frame)?; + found = Some(rgb_frame.clone()); + break 'decode; + } + } + + let Some(frame) = found else { + eprintln!("never reached timestamp {timestamp_secs}s before end of stream"); + return Ok(false); + }; + + if use_hw { + println!( + "HARDWARE PATH: {}", + if hw_path_active { "active" } else { "inactive (fell back to software)" } + ); + } + + let (ref_w, ref_h, ref_pixels) = + ppm::read_ppm(reference_path).map_err(|_| ffmpeg::Error::InvalidData)?; + if ref_w != frame.width() || ref_h != frame.height() { + eprintln!( + "dimension mismatch: decoded {}x{} vs reference {}x{}", + frame.width(), + frame.height(), + ref_w, + ref_h + ); + return Ok(false); + } + + let decoded_pixels = ppm::extract_rgb24(&frame); + let (mean_diff, max_diff) = ppm::compare(&decoded_pixels, &ref_pixels); + // Tolerance covers rounding differences in chroma upsampling / colorspace matrices + // between decode paths (e.g. libswscale vs VideoToolbox's own YUV->RGB), not a + // license to silently accept a wrong frame. + let pass = mean_diff < 2.0 && max_diff < 24; + println!( + "seek target {timestamp_secs}s -> mean_abs_diff={mean_diff:.3} max_abs_diff={max_diff} -> {}", + if pass { "PASS" } else { "FAIL" } + ); + Ok(pass) +} + +#[cfg(target_os = "macos")] +mod hw { + use ffmpeg_the_third as ffmpeg; + use ffmpeg_the_third::ffi; + use std::ptr; + + /// Sets up VideoToolbox hw_device_ctx + get_format on an *unopened* decoder context. + /// This is hand-written unsafe FFI because ffmpeg-the-third has no high-level + /// hwaccel wrapper (checked: no hwaccel/hwdevice/hwcontext module in the crate). + pub unsafe fn attach_hardware( + ctx: *mut ffi::AVCodecContext, + ) -> Result<(), ffmpeg::Error> { + let mut hw_device_ctx: *mut ffi::AVBufferRef = ptr::null_mut(); + let ret = ffi::av_hwdevice_ctx_create( + &mut hw_device_ctx, + ffi::AVHWDeviceType::VIDEOTOOLBOX, + ptr::null(), + ptr::null_mut(), + 0, + ); + if ret < 0 { + return Err(ffmpeg::Error::from(ret)); + } + (*ctx).hw_device_ctx = hw_device_ctx; // ownership transferred to the codec context + (*ctx).get_format = Some(get_format); + Ok(()) + } + + unsafe extern "C" fn get_format( + _ctx: *mut ffi::AVCodecContext, + mut fmt: *const ffi::AVPixelFormat, + ) -> ffi::AVPixelFormat { + while *fmt != ffi::AVPixelFormat::NONE { + if *fmt == ffi::AVPixelFormat::VIDEOTOOLBOX { + return *fmt; + } + fmt = fmt.add(1); + } + ffi::AVPixelFormat::NONE + } + + /// Copies a decoded frame out of GPU/VideoToolbox memory into a normal software + /// frame. Returns (frame, was_actually_hardware) — `was_actually_hardware` is false + /// if the decoder silently used a software pixel format despite the hw request. + pub fn transfer_to_software( + decoded: &ffmpeg::util::frame::video::Video, + ) -> Result<(ffmpeg::util::frame::video::Video, bool), ffmpeg::Error> { + if decoded.format() != ffmpeg::util::format::pixel::Pixel::VIDEOTOOLBOX { + // Decoder chose a software format from get_format's candidate list — + // hardware path did not engage for this frame. + return Ok((decoded.clone(), false)); + } + let mut software_frame = ffmpeg::util::frame::video::Video::empty(); + unsafe { + let ret = ffi::av_hwframe_transfer_data( + software_frame.as_mut_ptr(), + decoded.as_ptr(), + 0, + ); + if ret < 0 { + return Err(ffmpeg::Error::from(ret)); + } + } + Ok((software_frame, true)) + } +} + +#[cfg(target_os = "windows")] +mod hw { + use ffmpeg_the_third as ffmpeg; + use ffmpeg_the_third::ffi; + use std::ptr; + + /// Sets up CUDA hw_device_ctx + get_format on an *unopened* decoder context. + pub unsafe fn attach_hardware( + ctx: *mut ffi::AVCodecContext, + ) -> Result<(), ffmpeg::Error> { + let mut hw_device_ctx: *mut ffi::AVBufferRef = ptr::null_mut(); + let ret = ffi::av_hwdevice_ctx_create( + &mut hw_device_ctx, + ffi::AVHWDeviceType::CUDA, + ptr::null(), + ptr::null_mut(), + 0, + ); + if ret < 0 { + return Err(ffmpeg::Error::from(ret)); + } + (*ctx).hw_device_ctx = hw_device_ctx; + (*ctx).get_format = Some(get_format); + Ok(()) + } + + unsafe extern "C" fn get_format( + _ctx: *mut ffi::AVCodecContext, + mut fmt: *const ffi::AVPixelFormat, + ) -> ffi::AVPixelFormat { + while *fmt != ffi::AVPixelFormat::NONE { + if *fmt == ffi::AVPixelFormat::CUDA { + return *fmt; + } + fmt = fmt.add(1); + } + ffi::AVPixelFormat::NONE + } + + pub fn transfer_to_software( + decoded: &ffmpeg::util::frame::video::Video, + ) -> Result<(ffmpeg::util::frame::video::Video, bool), ffmpeg::Error> { + if decoded.format() != ffmpeg::util::format::pixel::Pixel::CUDA { + return Ok((decoded.clone(), false)); + } + let mut software_frame = ffmpeg::util::frame::video::Video::empty(); + unsafe { + let ret = ffi::av_hwframe_transfer_data( + software_frame.as_mut_ptr(), + decoded.as_ptr(), + 0, + ); + if ret < 0 { + return Err(ffmpeg::Error::from(ret)); + } + } + Ok((software_frame, true)) + } +} + +#[cfg(not(any(target_os = "macos", target_os = "windows")))] +mod hw { + use ffmpeg_the_third as ffmpeg; + use ffmpeg_the_third::ffi; + + pub unsafe fn attach_hardware( + _ctx: *mut ffi::AVCodecContext, + ) -> Result<(), ffmpeg::Error> { + Err(ffmpeg::Error::Bug) + } + + pub fn transfer_to_software( + decoded: &ffmpeg::util::frame::video::Video, + ) -> Result<(ffmpeg::util::frame::video::Video, bool), ffmpeg::Error> { + Ok((decoded.clone(), false)) + } +} diff --git a/spike/rust-decode-spike/ffmpeg-candidate/src/ppm.rs b/spike/rust-decode-spike/ffmpeg-candidate/src/ppm.rs new file mode 100644 index 0000000..b5e04cc --- /dev/null +++ b/spike/rust-decode-spike/ffmpeg-candidate/src/ppm.rs @@ -0,0 +1,99 @@ +//! Minimal binary PPM (P6) reader and RGB24 pixel comparison. No `image` crate — +//! this is the only fixture format the spike needs, and hand-parsing it keeps the +//! dependency list to exactly one crate (the thing under test). + +use ffmpeg_the_third::util::frame::video::Video; +use std::fs; +use std::io; + +pub fn read_ppm(path: &str) -> io::Result<(u32, u32, Vec)> { + let bytes = fs::read(path)?; + if bytes.get(0..2) != Some(b"P6") { + return Err(io::Error::new(io::ErrorKind::InvalidData, "not a P6 PPM")); + } + + // Header is whitespace-separated ASCII tokens: "P6 \n" + // followed immediately by raw binary pixel data. + let mut pos = 2; + let mut tokens = Vec::new(); + while tokens.len() < 3 { + while bytes[pos].is_ascii_whitespace() { + pos += 1; + } + let start = pos; + while !bytes[pos].is_ascii_whitespace() { + pos += 1; + } + tokens.push(std::str::from_utf8(&bytes[start..pos]).unwrap().to_string()); + } + pos += 1; // single whitespace byte separating maxval from pixel data + + let width: u32 = tokens[0].parse().unwrap(); + let height: u32 = tokens[1].parse().unwrap(); + let pixels = bytes[pos..].to_vec(); + let expected = (width * height * 3) as usize; + if pixels.len() < expected { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("truncated PPM: expected {expected} bytes, got {}", pixels.len()), + )); + } + Ok((width, height, pixels[..expected].to_vec())) +} + +/// Copies plane 0 out of an RGB24 frame respecting stride — `Video::data()` includes +/// row padding, and a naive full-buffer byte compare against a tightly-packed PPM +/// would spuriously fail whenever linesize != width * 3. +pub fn extract_rgb24(frame: &Video) -> Vec { + let width = frame.width() as usize; + let height = frame.height() as usize; + let stride = frame.stride(0); + let data = frame.data(0); + + let mut out = Vec::with_capacity(width * height * 3); + for row in 0..height { + let start = row * stride; + out.extend_from_slice(&data[start..start + width * 3]); + } + out +} + +/// Returns (mean absolute difference, max absolute difference) across all bytes. +pub fn compare(a: &[u8], b: &[u8]) -> (f64, u8) { + assert_eq!(a.len(), b.len(), "compared buffers must be the same size"); + let mut sum: u64 = 0; + let mut max: u8 = 0; + for (x, y) in a.iter().zip(b.iter()) { + let diff = x.abs_diff(*y); + sum += diff as u64; + max = max.max(diff); + } + (sum as f64 / a.len() as f64, max) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn compare_identical_buffers_is_zero() { + let buf = vec![10u8, 20, 30, 200]; + assert_eq!(compare(&buf, &buf), (0.0, 0)); + } + + #[test] + fn compare_detects_known_difference() { + let a = vec![10u8, 20, 30, 40]; + let b = vec![10u8, 25, 30, 20]; + // diffs: 0, 5, 0, 20 -> mean 6.25, max 20 + assert_eq!(compare(&a, &b), (6.25, 20)); + } + + #[test] + fn read_ppm_matches_known_fixture_dimensions() { + let (w, h, pixels) = + read_ppm("../fixtures/reference/frame_at_0.5s.ppm").expect("fixture must be readable"); + assert_eq!((w, h), (320, 240)); + assert_eq!(pixels.len(), (w * h * 3) as usize); + } +} diff --git a/spike/rust-decode-spike/fixtures/generate_fixtures.sh b/spike/rust-decode-spike/fixtures/generate_fixtures.sh new file mode 100755 index 0000000..23b82f8 --- /dev/null +++ b/spike/rust-decode-spike/fixtures/generate_fixtures.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Regenerates test-clip.mp4 and the reference PPM frames used by both spike candidates. +# Requires the `ffmpeg` CLI. Deterministic: testsrc2 content at a given frame index is +# reproducible across machines/ffmpeg versions, so re-running this should reproduce +# byte-identical fixtures (verify with `shasum` before committing a regenerated fixture). +set -euo pipefail + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +OUT="$DIR" +REF="$OUT/reference" +mkdir -p "$REF" + +FPS=30 +DURATION=3 +WIDTH=320 +HEIGHT=240 +GOP=15 # smaller than the clip length so seeking to 0.5s/1.5s is not itself a keyframe + +echo "Generating $OUT/test-clip.mp4 (${WIDTH}x${HEIGHT} @ ${FPS}fps, ${DURATION}s, GOP=${GOP})..." +ffmpeg -y -f lavfi -i "testsrc2=size=${WIDTH}x${HEIGHT}:rate=${FPS}:duration=${DURATION}" \ + -pix_fmt yuv420p -c:v libx264 -g "$GOP" -an \ + "$OUT/test-clip.mp4" + +# Frame 15 (t=0.5s) and frame 45 (t=1.5s) at 30fps. -vsync 0 + select=eq(n,N) pulls an exact +# frame index instead of relying on -ss seek behavior, so this is the ground truth regardless +# of how any given decoder's seek implementation rounds. +for pair in "15:0.5s" "45:1.5s"; do + frame="${pair%%:*}" + label="${pair##*:}" + echo "Extracting frame ${frame} (t=${label}) -> reference/frame_at_${label}.ppm" + ffmpeg -y -i "$OUT/test-clip.mp4" \ + -vf "select='eq(n\,${frame})'" -fps_mode passthrough -pix_fmt rgb24 -frames:v 1 -update 1 \ + "$REF/frame_at_${label}.ppm" +done + +echo "Done. Fixture files:" +ls -la "$OUT/test-clip.mp4" "$REF" diff --git a/spike/rust-decode-spike/fixtures/reference/frame_at_0.5s.ppm b/spike/rust-decode-spike/fixtures/reference/frame_at_0.5s.ppm new file mode 100644 index 0000000..1e84688 Binary files /dev/null and b/spike/rust-decode-spike/fixtures/reference/frame_at_0.5s.ppm differ diff --git a/spike/rust-decode-spike/fixtures/reference/frame_at_1.5s.ppm b/spike/rust-decode-spike/fixtures/reference/frame_at_1.5s.ppm new file mode 100644 index 0000000..308bc5c Binary files /dev/null and b/spike/rust-decode-spike/fixtures/reference/frame_at_1.5s.ppm differ diff --git a/spike/rust-decode-spike/fixtures/test-clip.mp4 b/spike/rust-decode-spike/fixtures/test-clip.mp4 new file mode 100644 index 0000000..760e351 Binary files /dev/null and b/spike/rust-decode-spike/fixtures/test-clip.mp4 differ diff --git a/spike/rust-decode-spike/gstreamer-candidate/Cargo.lock b/spike/rust-decode-spike/gstreamer-candidate/Cargo.lock new file mode 100644 index 0000000..6663153 --- /dev/null +++ b/spike/rust-decode-spike/gstreamer-candidate/Cargo.lock @@ -0,0 +1,684 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "atomic_refcell" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21e4227379beff4205943696e6c3e0cd809bacdf3f0edd6e3dd153e2269571a4" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "cfg-expr" +version = "0.20.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb693542bcafa528e198be0ebd9d3632ca5b7c93dbe7237460e199910835997c" +dependencies = [ + "smallvec", + "target-lexicon", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-macro", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "gio-sys" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521e93a7e56fc89e84aea9a52cfc9436816a4b363b030260b699950ff1336c83" +dependencies = [ + "glib-sys", + "gobject-sys", + "libc", + "system-deps", + "windows-sys", +] + +[[package]] +name = "glib" +version = "0.20.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffc4b6e352d4716d84d7dde562dd9aee2a7d48beb872dd9ece7f2d1515b2d683" +dependencies = [ + "bitflags", + "futures-channel", + "futures-core", + "futures-executor", + "futures-task", + "futures-util", + "gio-sys", + "glib-macros", + "glib-sys", + "gobject-sys", + "libc", + "memchr", + "smallvec", +] + +[[package]] +name = "glib-macros" +version = "0.20.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8084af62f09475a3f529b1629c10c429d7600ee1398ae12dd3bf175d74e7145" +dependencies = [ + "heck", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "glib-sys" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ab79e1ed126803a8fb827e3de0e2ff95191912b8db65cee467edb56fc4cc215" +dependencies = [ + "libc", + "system-deps", +] + +[[package]] +name = "gobject-sys" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec9aca94bb73989e3cfdbf8f2e0f1f6da04db4d291c431f444838925c4c63eda" +dependencies = [ + "glib-sys", + "libc", + "system-deps", +] + +[[package]] +name = "gstreamer" +version = "0.23.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8757a87f3706560037a01a9f06a59fcc7bdb0864744dcf73546606e60c4316e1" +dependencies = [ + "cfg-if", + "futures-channel", + "futures-core", + "futures-util", + "glib", + "gstreamer-sys", + "itertools", + "libc", + "muldiv", + "num-integer", + "num-rational", + "once_cell", + "option-operations", + "paste", + "pin-project-lite", + "smallvec", + "thiserror", +] + +[[package]] +name = "gstreamer-app" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9a883eb21aebcf1289158225c05f7aea5da6ecf71fa7f0ff1ce4d25baf004e" +dependencies = [ + "futures-core", + "futures-sink", + "glib", + "gstreamer", + "gstreamer-app-sys", + "gstreamer-base", + "libc", +] + +[[package]] +name = "gstreamer-app-sys" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f7ef838306fe51852d503a14dc79ac42de005a59008a05098de3ecdaf05455" +dependencies = [ + "glib-sys", + "gstreamer-base-sys", + "gstreamer-sys", + "libc", + "system-deps", +] + +[[package]] +name = "gstreamer-base" +version = "0.23.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f19a74fd04ffdcb847dd322640f2cf520897129d00a7bcb92fd62a63f3e27404" +dependencies = [ + "atomic_refcell", + "cfg-if", + "glib", + "gstreamer", + "gstreamer-base-sys", + "libc", +] + +[[package]] +name = "gstreamer-base-sys" +version = "0.23.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f2fb0037b6d3c5b51f60dea11e667910f33be222308ca5a101450018a09840" +dependencies = [ + "glib-sys", + "gobject-sys", + "gstreamer-sys", + "libc", + "system-deps", +] + +[[package]] +name = "gstreamer-candidate" +version = "0.1.0" +dependencies = [ + "gstreamer", + "gstreamer-app", + "gstreamer-video", +] + +[[package]] +name = "gstreamer-sys" +version = "0.23.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "feea73b4d92dbf9c24a203c9cd0bcc740d584f6b5960d5faf359febf288919b2" +dependencies = [ + "glib-sys", + "gobject-sys", + "libc", + "system-deps", +] + +[[package]] +name = "gstreamer-video" +version = "0.23.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1318b599d77ca4f7702ecbdeac1672d6304cb16b7e5752fabb3ee8260449a666" +dependencies = [ + "cfg-if", + "futures-channel", + "glib", + "gstreamer", + "gstreamer-base", + "gstreamer-video-sys", + "libc", + "once_cell", + "thiserror", +] + +[[package]] +name = "gstreamer-video-sys" +version = "0.23.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a70f0947f12d253b9de9bc3fd92f981e4d025336c18389c7f08cdf388a99f5c" +dependencies = [ + "glib-sys", + "gobject-sys", + "gstreamer-base-sys", + "gstreamer-sys", + "libc", + "system-deps", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "muldiv" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "956787520e75e9bd233246045d19f42fb73242759cc57fba9611d940ae96d4b0" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "option-operations" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c26d27bb1aeab65138e4bf7666045169d1717febcc9ff870166be8348b223d0" +dependencies = [ + "paste", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "system-deps" +version = "7.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "396a35feb67335377e0251fcbc1092fc85c484bd4e3a7a54319399da127796e7" +dependencies = [ + "cfg-expr", + "heck", + "pkg-config", + "toml", + "version-compare", +] + +[[package]] +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "toml" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c96ecdfa941c8fc4fcaed14f99ada8ebed502eef533015095a07e3301d4c3c" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.13+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +dependencies = [ + "winnow", +] + +[[package]] +name = "toml_writer" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "version-compare" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c2856837ef78f57382f06b2b8563a2f512f7185d732608fd9176cb3b8edf0e" + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] diff --git a/spike/rust-decode-spike/gstreamer-candidate/Cargo.toml b/spike/rust-decode-spike/gstreamer-candidate/Cargo.toml new file mode 100644 index 0000000..76ee751 --- /dev/null +++ b/spike/rust-decode-spike/gstreamer-candidate/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "gstreamer-candidate" +version = "0.1.0" +edition = "2021" +publish = false + +[[bin]] +name = "gstreamer-candidate" +path = "src/main.rs" + +[dependencies] +gstreamer = "0.23" +gstreamer-app = "0.23" +gstreamer-video = "0.23" diff --git a/spike/rust-decode-spike/gstreamer-candidate/src/main.rs b/spike/rust-decode-spike/gstreamer-candidate/src/main.rs new file mode 100644 index 0000000..e81cc16 --- /dev/null +++ b/spike/rust-decode-spike/gstreamer-candidate/src/main.rs @@ -0,0 +1,160 @@ +//! Spike harness for issue #108: does `gstreamer-rs` decode-and-seek a real .mp4 +//! frame-accurately, and can it reach VideoToolbox without extra shims? +//! +//! Usage: gstreamer-candidate [--hw] +//! +//! Unlike the ffmpeg-the-third candidate, hardware selection here is just naming a +//! different pipeline element (`vtdec_hw` is hardware-only by construction — the +//! pipeline fails to reach PAUSED if VideoToolbox isn't actually available, so a +//! failed preroll below *is* the "inactive" answer for criterion #1, not a bug). +//! +//! Verified on this M3 Mac: `vtdec_hw` negotiates and prerolls (hardware path +//! "active"), but its output pixel-mismatches the reference frame by more than the +//! software path does (see FINDINGS.md criterion #1/#2 — likely a colorimetry/GL +//! readback difference, not a wrong frame; confirmed the target frame itself is +//! correct by diffing against its neighbors). Left as an honest FAIL rather than +//! loosened tolerance or a deeper GL-pipeline fix, since surfacing this friction is +//! the point of the spike, not papering over it. + +use std::env; +use std::process::ExitCode; + +use gstreamer as gst; +use gstreamer::prelude::*; +use gstreamer_app::AppSink; +use gstreamer_video::VideoInfo; + +mod ppm; + +fn main() -> ExitCode { + let args: Vec = env::args().collect(); + if args.len() < 4 { + eprintln!( + "usage: {} [--hw]", + args.first().map(String::as_str).unwrap_or("gstreamer-candidate") + ); + return ExitCode::FAILURE; + } + let clip_path = &args[1]; + let timestamp_secs: f64 = match args[2].parse() { + Ok(v) => v, + Err(e) => { + eprintln!("invalid timestamp '{}': {e}", args[2]); + return ExitCode::FAILURE; + } + }; + let reference_path = &args[3]; + let use_hw = args.iter().any(|a| a == "--hw"); + + match run(clip_path, timestamp_secs, reference_path, use_hw) { + Ok(true) => ExitCode::SUCCESS, + Ok(false) => ExitCode::FAILURE, + Err(e) => { + eprintln!("error: {e}"); + ExitCode::FAILURE + } + } +} + +fn run( + clip_path: &str, + timestamp_secs: f64, + reference_path: &str, + use_hw: bool, +) -> Result> { + gst::init()?; + + let decoder_element = if use_hw { + hw_decoder_element() + } else { + "avdec_h264" + }; + let pipeline_desc = format!( + "filesrc location=\"{clip_path}\" ! qtdemux ! h264parse ! {decoder_element} ! \ + videoconvert ! video/x-raw,format=RGB ! appsink name=sink sync=false" + ); + + let pipeline = gst::parse::launch(&pipeline_desc)? + .downcast::() + .expect("parse::launch of a pipeline description returns a Pipeline"); + let appsink = pipeline + .by_name("sink") + .expect("named appsink element must exist") + .downcast::() + .expect("sink element is an appsink"); + + pipeline.set_state(gst::State::Paused)?; + let (result, _current, _pending) = pipeline.state(Some(gst::ClockTime::from_seconds(10))); + if let Err(e) = result { + pipeline.set_state(gst::State::Null)?; + println!( + "HARDWARE PATH: {}", + if use_hw { "inactive (pipeline failed to reach PAUSED)" } else { "n/a (software path)" } + ); + return Err(format!("pipeline failed to preroll: {e:?}").into()); + } + + let seek_pos = gst::ClockTime::from_nseconds((timestamp_secs * 1_000_000_000.0) as u64); + pipeline.seek_simple( + gst::SeekFlags::FLUSH | gst::SeekFlags::ACCURATE, + seek_pos, + )?; + // A flushing seek in PAUSED re-prerolls to the new position; block until it lands. + pipeline.state(Some(gst::ClockTime::from_seconds(10))).0?; + + if use_hw { + println!("HARDWARE PATH: active ({decoder_element} negotiated successfully)"); + } + + let sample = appsink.pull_preroll().map_err(|e| format!("pull_preroll: {e:?}"))?; + let buffer = sample.buffer().ok_or("sample had no buffer")?; + let caps = sample.caps().ok_or("sample had no caps")?; + let video_info = VideoInfo::from_caps(caps)?; + let map = buffer.map_readable()?; + + let decoded_pixels = ppm::extract_rgb24( + map.as_slice(), + video_info.width(), + video_info.height(), + video_info.stride()[0], + ); + + pipeline.set_state(gst::State::Null)?; + + let (ref_w, ref_h, ref_pixels) = ppm::read_ppm(reference_path)?; + if ref_w != video_info.width() || ref_h != video_info.height() { + eprintln!( + "dimension mismatch: decoded {}x{} vs reference {}x{}", + video_info.width(), + video_info.height(), + ref_w, + ref_h + ); + return Ok(false); + } + + let (mean_diff, max_diff) = ppm::compare(&decoded_pixels, &ref_pixels); + // Same tolerance rationale as the ffmpeg candidate: covers YUV->RGB rounding + // differences between decode paths, not a license to accept a wrong frame. + let pass = mean_diff < 2.0 && max_diff < 24; + println!( + "seek target {timestamp_secs}s -> mean_abs_diff={mean_diff:.3} max_abs_diff={max_diff} -> {}", + if pass { "PASS" } else { "FAIL" } + ); + Ok(pass) +} + +#[cfg(target_os = "windows")] +fn hw_decoder_element() -> &'static str { + "nvh264dec" +} + +#[cfg(target_os = "macos")] +fn hw_decoder_element() -> &'static str { + "vtdec_hw" +} + +#[cfg(not(any(target_os = "windows", target_os = "macos")))] +fn hw_decoder_element() -> &'static str { + "avdec_h264" +} diff --git a/spike/rust-decode-spike/gstreamer-candidate/src/ppm.rs b/spike/rust-decode-spike/gstreamer-candidate/src/ppm.rs new file mode 100644 index 0000000..b0ee798 --- /dev/null +++ b/spike/rust-decode-spike/gstreamer-candidate/src/ppm.rs @@ -0,0 +1,89 @@ +//! Same minimal PPM (P6) reader/comparator as the ffmpeg candidate. Duplicated rather +//! than shared via a third crate — each candidate stays a fully standalone build so a +//! contributor without GStreamer installed can still build/run the ffmpeg candidate +//! (and vice versa), which is itself part of what criterion #3 (build complexity) asks. + +use std::fs; +use std::io; + +pub fn read_ppm(path: &str) -> io::Result<(u32, u32, Vec)> { + let bytes = fs::read(path)?; + if bytes.get(0..2) != Some(b"P6") { + return Err(io::Error::new(io::ErrorKind::InvalidData, "not a P6 PPM")); + } + + let mut pos = 2; + let mut tokens = Vec::new(); + while tokens.len() < 3 { + while bytes[pos].is_ascii_whitespace() { + pos += 1; + } + let start = pos; + while !bytes[pos].is_ascii_whitespace() { + pos += 1; + } + tokens.push(std::str::from_utf8(&bytes[start..pos]).unwrap().to_string()); + } + pos += 1; + + let width: u32 = tokens[0].parse().unwrap(); + let height: u32 = tokens[1].parse().unwrap(); + let pixels = bytes[pos..].to_vec(); + let expected = (width * height * 3) as usize; + if pixels.len() < expected { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("truncated PPM: expected {expected} bytes, got {}", pixels.len()), + )); + } + Ok((width, height, pixels[..expected].to_vec())) +} + +/// Copies a tightly-packed RGB24 buffer out of a strided plane (GStreamer pads each +/// row to a 4-byte boundary by default, same reasoning as the ffmpeg candidate). +pub fn extract_rgb24(data: &[u8], width: u32, height: u32, stride: i32) -> Vec { + let width = width as usize; + let height = height as usize; + let stride = stride as usize; + let mut out = Vec::with_capacity(width * height * 3); + for row in 0..height { + let start = row * stride; + out.extend_from_slice(&data[start..start + width * 3]); + } + out +} + +/// Returns (mean absolute difference, max absolute difference) across all bytes. +pub fn compare(a: &[u8], b: &[u8]) -> (f64, u8) { + assert_eq!(a.len(), b.len(), "compared buffers must be the same size"); + let mut sum: u64 = 0; + let mut max: u8 = 0; + for (x, y) in a.iter().zip(b.iter()) { + let diff = x.abs_diff(*y); + sum += diff as u64; + max = max.max(diff); + } + (sum as f64 / a.len() as f64, max) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn compare_identical_buffers_is_zero() { + let buf = vec![10u8, 20, 30, 200]; + assert_eq!(compare(&buf, &buf), (0.0, 0)); + } + + #[test] + fn extract_rgb24_strips_row_padding() { + // 2x2 RGB, stride padded to 8 bytes/row (2 bytes of padding after 6 real bytes) + let data: Vec = vec![ + 1, 2, 3, 4, 5, 6, 0, 0, // row 0: two pixels + padding + 7, 8, 9, 10, 11, 12, 0, 0, // row 1 + ]; + let out = extract_rgb24(&data, 2, 2, 8); + assert_eq!(out, vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]); + } +}