frames

README.md

@@ -32,6 +32,10 @@ Runwave's recording pipeline is **gstreamer-only** as other methods lead to audi
 - **gstreamer 1.x** with `ximagesrc`, `pulsesrc`, `vp8enc`, `opusenc`,
   `webmmux`, and `filesink` available on `PATH` as `gst-launch-1.0` (override
   via the `RUNWAVE_GSTREAMER` env var or the `gstreamerPath` start option).
+- **ffmpeg and ffprobe** available on `PATH` for the default repeated-frame
+  removal pass that produces the shorter final recording. Pass
+  `repeatedFrameRemoval: false` to keep the raw gstreamer recording without
+  post-processing.
 - **An X server or Xvfb.** `DISPLAY` must be set to a display that Chromium
   or a native Linux game can render into and that `ximagesrc` can read.
 - **PulseAudio running.** `pactl info` must succeed. Chromium's audio must be
@@ -61,7 +65,7 @@ From a private GitHub repo in a task Dockerfile:
 RUN apt-get update && apt-get install -y \
     gstreamer1.0-tools gstreamer1.0-plugins-good gstreamer1.0-plugins-bad \
     gstreamer1.0-plugins-ugly gstreamer1.0-x gstreamer1.0-pulseaudio \
-    pulseaudio xvfb xdotool
+    ffmpeg pulseaudio xvfb xdotool
 RUN npm install -g https://github.com/parsewave/runwave.git
 RUN npx playwright install --with-deps chromium
 ```

controller/bin/run-repeated-frame-removal-examples.js

@@ -0,0 +1,115 @@
+const fs = require('fs');
+const path = require('path');
+const { removeRepeatedFrames } = require('../src/repeated-frame-remover');
+
+const SOURCE_DIRS = [
+  '/Users/plato/code/mm-sapphire/cruft/af4809-playtests/videos',
+  '/Users/plato/code/mm-sapphire/cruft/pr501-playtests/videos',
+];
+
+const OUTPUT_ROOT = path.resolve(__dirname, '..', '..', 'cruft', 'repeated-frame-removal-examples');
+const SIMILARITY_THRESHOLD = 0.98;
+const EDGE_FRAME_COUNT = 10;
+const CONCURRENCY = Number(process.env.REPEATED_FRAME_REMOVAL_CONCURRENCY || 4);
+
+function ensureDir(dir) {
+  fs.mkdirSync(dir, { recursive: true });
+  return dir;
+}
+
+function safeName(value) {
+  return String(value)
+    .replace(/[^a-zA-Z0-9._-]+/g, '-')
+    .replace(/^-+|-+$/g, '') || 'video';
+}
+
+function listSourceVideos() {
+  return SOURCE_DIRS.flatMap((dir) => {
+    const group = safeName(path.basename(path.dirname(dir)));
+    return fs.readdirSync(dir)
+      .filter((entry) => /\.(webm|mp4|mov|mkv)$/i.test(entry))
+      .sort()
+      .map((entry) => ({
+        group,
+        name: path.basename(entry, path.extname(entry)),
+        ext: path.extname(entry),
+        sourcePath: path.join(dir, entry),
+      }));
+  });
+}
+
+function elapsedMs(startedAt) {
+  return Number((process.hrtime.bigint() - startedAt) / 1000000n);
+}
+
+async function processVideo(video) {
+  const startedAt = process.hrtime.bigint();
+  const videoDir = ensureDir(path.join(OUTPUT_ROOT, `${video.group}-${safeName(video.name)}`));
+  const inputPath = path.join(videoDir, `input${video.ext}`);
+  const outputPath = path.join(videoDir, 'output.mp4');
+  const reportPath = path.join(videoDir, 'report.json');
+
+  fs.copyFileSync(video.sourcePath, inputPath);
+  console.log(`Processing ${video.group}/${video.name}${video.ext}`);
+  const summary = await removeRepeatedFrames(inputPath, outputPath, {
+    similarityThreshold: SIMILARITY_THRESHOLD,
+    edgeFrameCount: EDGE_FRAME_COUNT,
+  });
+  const durationMs = elapsedMs(startedAt);
+  const report = {
+    sourcePath: video.sourcePath,
+    durationMs,
+    ...summary,
+  };
+
+  fs.writeFileSync(reportPath, JSON.stringify(report, null, 2));
+  console.log(`  removed ${summary.removedFrameCount}/${summary.frameCount} frames in ${durationMs}ms -> ${outputPath}`);
+  return report;
+}
+
+async function mapWithConcurrency(items, concurrency, fn) {
+  const results = new Array(items.length);
+  let nextIndex = 0;
+
+  async function worker() {
+    while (nextIndex < items.length) {
+      const index = nextIndex;
+      nextIndex += 1;
+      results[index] = await fn(items[index], index);
+    }
+  }
+
+  const workerCount = Math.max(1, Math.min(items.length, Math.floor(concurrency) || 1));
+  await Promise.all(Array.from({ length: workerCount }, () => worker()));
+  return results;
+}
+
+async function main() {
+  ensureDir(OUTPUT_ROOT);
+
+  const startedAt = process.hrtime.bigint();
+  const startedAtIso = new Date().toISOString();
+  const videos = listSourceVideos();
+  const summaries = await mapWithConcurrency(videos, CONCURRENCY, (video) => processVideo(video));
+  const durationMs = elapsedMs(startedAt);
+
+  const indexPath = path.join(OUTPUT_ROOT, 'summary.json');
+  fs.writeFileSync(indexPath, JSON.stringify({
+    startedAt: startedAtIso,
+    finishedAt: new Date().toISOString(),
+    durationMs,
+    videoCount: summaries.length,
+    concurrency: Math.max(1, Math.min(videos.length, Math.floor(CONCURRENCY) || 1)),
+    similarityThreshold: SIMILARITY_THRESHOLD,
+    edgeFrameCount: EDGE_FRAME_COUNT,
+    videos: summaries,
+  }, null, 2));
+  console.log(`Wrote ${indexPath} after ${durationMs}ms`);
+}
+
+if (require.main === module) {
+  main().catch((error) => {
+    console.error(error.stack || error.message);
+    process.exitCode = 1;
+  });
+}

runwave/controller/README.md

@@ -6,7 +6,8 @@ executes timed input sequences, records WebM output, and writes per-action
 artifacts.
 
 See the top-level [Requirements](../../README.md#requirements) section for the
-Linux, gstreamer, X server/Xvfb, and PulseAudio setup required by `record: true`.
+Linux, gstreamer, ffmpeg/ffprobe, X server/Xvfb, and PulseAudio setup required
+by `record: true`.
 
 ## Start
 
@@ -86,6 +87,10 @@ Useful `start` options:
   sources with `videoSource` and `audioSource`. (`recordAudio` is accepted as a
   legacy alias for `record`; they mean the same thing - audio is always captured
   when recording.)
+- `repeatedFrameRemoval`: when `record` is enabled, post-process the final WebM
+  by default to remove repeated-frame runs before returning the stop response.
+  Pass `false` to keep only the raw recording. When enabled, the raw capture is
+  renamed with a `_raw` suffix.
 - `headless`: defaults to `true`; set `false` to watch the browser.
 - `channel`: optional Playwright browser channel, such as `chrome` or `msedge`.
 - `executablePath`: optional explicit browser executable path.
@@ -238,7 +243,15 @@ runwave-controller '{"action":"sessions"}'
 When `record: true` is set, `stop` returns `video` and `audioVideo` pointing at
 the same recorded audio/video WebM. All recording goes through gstreamer - see
 the top-level [Requirements](../../README.md#requirements) section for the
-mandatory environment (Linux, gstreamer, X server/Xvfb, PulseAudio).
+mandatory environment (Linux, gstreamer, ffmpeg/ffprobe, X server/Xvfb,
+PulseAudio).
+
+By default, `video` and `audioVideo` point at the shorter WebM at the normal
+recording path after repeated-frame runs are removed. The original uncut capture
+is preserved as `rawVideo` / `rawAudioVideo`, typically
+`video/000-runwave-with-audio_raw.webm`, and `repeatedFrameRemoval` reports the
+removed frame ranges. Pass `repeatedFrameRemoval: false` on `start` to return
+the raw recording path without post-processing.
 
 ## State
 
@@ -277,7 +290,11 @@ Each turn writes:
 - `response.json`: the main CLI response.
 - `*.png`: screenshots captured during that operation.
 - `NNN-<action_name>.json`: detailed sequence log for `step` operations.
-- `video/000-runwave-with-audio.webm`: final gstreamer audio+video recording.
+- `video/000-runwave-with-audio.webm`: final shorter audio/video recording when
+  repeated-frame removal is enabled, or the raw gstreamer recording when it is
+  disabled.
+- `video/000-runwave-with-audio_raw.webm`: original recording when repeated
+  frame removal is enabled.
 
 Active sessions are tracked as JSON files in `.runwave-sessions/` by default.
 The matching session file is removed by `stop`.

runwave/controller/bin/remove-repeated-frames.js

@@ -0,0 +1,9 @@
+module.exports = require('../src/repeated-frame-remover');
+
+if (require.main === module) {
+  const { main } = module.exports;
+  main(process.argv.slice(2)).catch((error) => {
+    console.error(error.stack || error.message);
+    process.exitCode = 1;
+  });
+}

runwave/controller/src/action-handler.js

@@ -143,7 +143,7 @@ async function writeStopResponse(runtime, input) {
       runtime.profiler.mark('action.stop.screenshot_error', { action_name: input.action_name, error: screenshotError });
     }
   }
-  const recording = await runtime.profiler.time('action.stop.browser_close', { action_name: input.action_name }, () => browser.close());
+  const recording = await runtime.profiler.time('action.stop.browser_close', { action_name: input.action_name }, () => browser.close(input));
   const video = typeof recording === 'string' ? recording : recording.video;
 
   runtime.profiler.timeSync('action.stop.remove_session_file', { sessionFile: runtime.sessionFile }, () =>
@@ -160,6 +160,9 @@ async function writeStopResponse(runtime, input) {
   });
   if (recording && typeof recording === 'object') {
     if (recording.audioVideo) payload.audioVideo = recording.audioVideo;
+    if (recording.rawVideo) payload.rawVideo = recording.rawVideo;
+    if (recording.rawAudioVideo) payload.rawAudioVideo = recording.rawAudioVideo;
+    if (recording.repeatedFrameRemoval) payload.repeatedFrameRemoval = recording.repeatedFrameRemoval;
   }
   if (screenshot) payload.screenshot = screenshot;
   if (stateError) payload.stateError = stateError;

runwave/controller/src/browser-session.js

@@ -7,6 +7,11 @@ const { AudioVideoRecorder } = require('./audio-recorder');
 const { ensureDir, safeName, sleep, timestamp } = require('./file-utils');
 const { drawGridOnScreenshot } = require('./grid-overlay');
 const { parseArgList, targetUrl } = require('./protocol');
+const {
+  isRepeatedFrameRemovalEnabled,
+  removeRepeatedFramesInPlace,
+  repeatedFrameRemovalOptions,
+} = require('./repeated-frame-remover');
 const { readPageState } = require('./state-reader');
 
 const DEFAULT_CHROMIUM_ARGS = [
@@ -484,17 +489,43 @@ class BrowserSession {
     });
   }
 
-  async close() {
+  async close(overrides = {}) {
+    const hasRepeatedFrameRemovalOverride = Object.prototype.hasOwnProperty.call(overrides, 'repeatedFrameRemoval');
+    const closeConfig = {
+      ...this.config,
+      ...overrides,
+      repeatedFrameRemoval: hasRepeatedFrameRemovalOverride
+        ? overrides.repeatedFrameRemoval
+        : this.config.repeatedFrameRemoval,
+    };
     let audioVideoPath = null;
+    let rawVideoPath = null;
+    let repeatedFrameRemoval = null;
     let closeError = null;
     try {
       if (this.audioRecorder) {
         audioVideoPath = await this.time('browser.close.audio_video_stop', () => this.audioRecorder.stop());
       }
+      if (audioVideoPath && isRepeatedFrameRemovalEnabled(closeConfig)) {
+        const processed = await this.time('browser.close.remove_repeated_frames', { video: audioVideoPath }, () =>
+          removeRepeatedFramesInPlace(audioVideoPath, repeatedFrameRemovalOptions())
+        );
+        audioVideoPath = processed.video;
+        rawVideoPath = processed.rawVideo;
+        repeatedFrameRemoval = processed.repeatedFrameRemoval;
+      }
+    } catch (error) {
+      closeError = error;
+    }
+    try {
       if (this.context) await this.time('browser.close.context_close', () => this.context.close());
+    } catch (error) {
+      if (!closeError) closeError = error;
+    }
+    try {
       if (this.browser) await this.time('browser.close.browser_close', () => this.browser.close());
     } catch (error) {
-      closeError = error;
+      if (!closeError) closeError = error;
     }
     try {
       await this.stopGameProcess();
@@ -505,6 +536,9 @@ class BrowserSession {
     return {
       video: audioVideoPath,
       audioVideo: audioVideoPath || undefined,
+      rawVideo: rawVideoPath || undefined,
+      rawAudioVideo: rawVideoPath || undefined,
+      repeatedFrameRemoval: repeatedFrameRemoval || undefined,
     };
   }
 }

runwave/controller/src/linux-session.js

@@ -5,6 +5,11 @@ const { AudioVideoRecorder, defaultVideoSource, parseX11VideoSource } = require(
 const { ensureDir, safeName, sleep, timestamp } = require('./file-utils');
 const { drawGridOnScreenshot } = require('./grid-overlay');
 const { parseArgList } = require('./protocol');
+const {
+  isRepeatedFrameRemovalEnabled,
+  removeRepeatedFramesInPlace,
+  repeatedFrameRemovalOptions,
+} = require('./repeated-frame-remover');
 
 const DEFAULT_WINDOW_WAIT_MS = 15000;
 const DEFAULT_LINUX_LAUNCH_SETTLE_MS = 30000;
@@ -597,27 +602,58 @@ class LinuxSession {
     };
   }
 
-  async close() {
+  async close(overrides = {}) {
+    const hasRepeatedFrameRemovalOverride = Object.prototype.hasOwnProperty.call(overrides, 'repeatedFrameRemoval');
+    const closeConfig = {
+      ...this.config,
+      ...overrides,
+      repeatedFrameRemoval: hasRepeatedFrameRemovalOverride
+        ? overrides.repeatedFrameRemoval
+        : this.config.repeatedFrameRemoval,
+    };
     let audioVideoPath = null;
-    if (this.audioRecorder) {
-      audioVideoPath = await this.time('linux.close.audio_video_stop', () => this.audioRecorder.stop());
+    let rawVideoPath = null;
+    let repeatedFrameRemoval = null;
+    let closeError = null;
+    try {
+      if (this.audioRecorder) {
+        audioVideoPath = await this.time('linux.close.audio_video_stop', () => this.audioRecorder.stop());
+      }
+      if (audioVideoPath && isRepeatedFrameRemovalEnabled(closeConfig)) {
+        const processed = await this.time('linux.close.remove_repeated_frames', { video: audioVideoPath }, () =>
+          removeRepeatedFramesInPlace(audioVideoPath, repeatedFrameRemovalOptions())
+        );
+        audioVideoPath = processed.video;
+        rawVideoPath = processed.rawVideo;
+        repeatedFrameRemoval = processed.repeatedFrameRemoval;
+      }
+    } catch (error) {
+      closeError = error;
     }
-    if (this.process && !processHasClosed(this.process)) {
-      await this.time('linux.close.terminate_process', async () => {
-        signalProcessGroup(this.process, 'SIGTERM');
-        if (!(await waitForProcessClose(this.process, 5000))) {
-          signalProcessGroup(this.process, 'SIGKILL');
-          await waitForProcessClose(this.process, 5000);
-        }
-      });
-    } else if (this.windowId) {
-      try {
-        this.timeSync('linux.close.window_close', () => this.xdotool(['windowclose', this.windowId], { timeoutMs: 1000 }));
-      } catch {}
+    try {
+      if (this.process && !processHasClosed(this.process)) {
+        await this.time('linux.close.terminate_process', async () => {
+          signalProcessGroup(this.process, 'SIGTERM');
+          if (!(await waitForProcessClose(this.process, 5000))) {
+            signalProcessGroup(this.process, 'SIGKILL');
+            await waitForProcessClose(this.process, 5000);
+          }
+        });
+      } else if (this.windowId) {
+        try {
+          this.timeSync('linux.close.window_close', () => this.xdotool(['windowclose', this.windowId], { timeoutMs: 1000 }));
+        } catch {}
+      }
+    } catch (error) {
+      if (!closeError) closeError = error;
     }
+    if (closeError) throw closeError;
     return {
       video: audioVideoPath,
       audioVideo: audioVideoPath || undefined,
+      rawVideo: rawVideoPath || undefined,
+      rawAudioVideo: rawVideoPath || undefined,
+      repeatedFrameRemoval: repeatedFrameRemoval || undefined,
     };
   }
 }