diff --git a/README.md b/README.md index 6b415a1..b73a5f7 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,10 @@ Runwave's recording pipeline is **gstreamer-only** as other methods lead to audi - **gstreamer 1.x** with `ximagesrc`, `pulsesrc`, `vp8enc`, `opusenc`, `webmmux`, and `filesink` available on `PATH` as `gst-launch-1.0` (override via the `RUNWAVE_GSTREAMER` env var or the `gstreamerPath` start option). +- **ffmpeg and ffprobe** available on `PATH` for the default repeated-frame + removal pass that produces the shorter final recording. Pass + `repeatedFrameRemoval: false` to keep the raw gstreamer recording without + post-processing. - **An X server or Xvfb.** `DISPLAY` must be set to a display that Chromium or a native Linux game can render into and that `ximagesrc` can read. - **PulseAudio running.** `pactl info` must succeed. Chromium's audio must be @@ -61,7 +65,7 @@ From a private GitHub repo in a task Dockerfile: RUN apt-get update && apt-get install -y \ gstreamer1.0-tools gstreamer1.0-plugins-good gstreamer1.0-plugins-bad \ gstreamer1.0-plugins-ugly gstreamer1.0-x gstreamer1.0-pulseaudio \ - pulseaudio xvfb xdotool + ffmpeg pulseaudio xvfb xdotool RUN npm install -g https://github.com/parsewave/runwave.git RUN npx playwright install --with-deps chromium ``` diff --git a/controller/bin/run-repeated-frame-removal-examples.js b/controller/bin/run-repeated-frame-removal-examples.js new file mode 100644 index 0000000..9dcf33b --- /dev/null +++ b/controller/bin/run-repeated-frame-removal-examples.js @@ -0,0 +1,115 @@ +const fs = require('fs'); +const path = require('path'); +const { removeRepeatedFrames } = require('../src/repeated-frame-remover'); + +const SOURCE_DIRS = [ + '/Users/plato/code/mm-sapphire/cruft/af4809-playtests/videos', + '/Users/plato/code/mm-sapphire/cruft/pr501-playtests/videos', +]; + +const OUTPUT_ROOT = path.resolve(__dirname, '..', '..', 'cruft', 'repeated-frame-removal-examples'); +const SIMILARITY_THRESHOLD = 0.98; +const EDGE_FRAME_COUNT = 10; +const CONCURRENCY = Number(process.env.REPEATED_FRAME_REMOVAL_CONCURRENCY || 4); + +function ensureDir(dir) { + fs.mkdirSync(dir, { recursive: true }); + return dir; +} + +function safeName(value) { + return String(value) + .replace(/[^a-zA-Z0-9._-]+/g, '-') + .replace(/^-+|-+$/g, '') || 'video'; +} + +function listSourceVideos() { + return SOURCE_DIRS.flatMap((dir) => { + const group = safeName(path.basename(path.dirname(dir))); + return fs.readdirSync(dir) + .filter((entry) => /\.(webm|mp4|mov|mkv)$/i.test(entry)) + .sort() + .map((entry) => ({ + group, + name: path.basename(entry, path.extname(entry)), + ext: path.extname(entry), + sourcePath: path.join(dir, entry), + })); + }); +} + +function elapsedMs(startedAt) { + return Number((process.hrtime.bigint() - startedAt) / 1000000n); +} + +async function processVideo(video) { + const startedAt = process.hrtime.bigint(); + const videoDir = ensureDir(path.join(OUTPUT_ROOT, `${video.group}-${safeName(video.name)}`)); + const inputPath = path.join(videoDir, `input${video.ext}`); + const outputPath = path.join(videoDir, 'output.mp4'); + const reportPath = path.join(videoDir, 'report.json'); + + fs.copyFileSync(video.sourcePath, inputPath); + console.log(`Processing ${video.group}/${video.name}${video.ext}`); + const summary = await removeRepeatedFrames(inputPath, outputPath, { + similarityThreshold: SIMILARITY_THRESHOLD, + edgeFrameCount: EDGE_FRAME_COUNT, + }); + const durationMs = elapsedMs(startedAt); + const report = { + sourcePath: video.sourcePath, + durationMs, + ...summary, + }; + + fs.writeFileSync(reportPath, JSON.stringify(report, null, 2)); + console.log(` removed ${summary.removedFrameCount}/${summary.frameCount} frames in ${durationMs}ms -> ${outputPath}`); + return report; +} + +async function mapWithConcurrency(items, concurrency, fn) { + const results = new Array(items.length); + let nextIndex = 0; + + async function worker() { + while (nextIndex < items.length) { + const index = nextIndex; + nextIndex += 1; + results[index] = await fn(items[index], index); + } + } + + const workerCount = Math.max(1, Math.min(items.length, Math.floor(concurrency) || 1)); + await Promise.all(Array.from({ length: workerCount }, () => worker())); + return results; +} + +async function main() { + ensureDir(OUTPUT_ROOT); + + const startedAt = process.hrtime.bigint(); + const startedAtIso = new Date().toISOString(); + const videos = listSourceVideos(); + const summaries = await mapWithConcurrency(videos, CONCURRENCY, (video) => processVideo(video)); + const durationMs = elapsedMs(startedAt); + + const indexPath = path.join(OUTPUT_ROOT, 'summary.json'); + fs.writeFileSync(indexPath, JSON.stringify({ + startedAt: startedAtIso, + finishedAt: new Date().toISOString(), + durationMs, + videoCount: summaries.length, + concurrency: Math.max(1, Math.min(videos.length, Math.floor(CONCURRENCY) || 1)), + similarityThreshold: SIMILARITY_THRESHOLD, + edgeFrameCount: EDGE_FRAME_COUNT, + videos: summaries, + }, null, 2)); + console.log(`Wrote ${indexPath} after ${durationMs}ms`); +} + +if (require.main === module) { + main().catch((error) => { + console.error(error.stack || error.message); + process.exitCode = 1; + }); +} diff --git a/runwave/controller/README.md b/runwave/controller/README.md index 5277922..23c6bef 100644 --- a/runwave/controller/README.md +++ b/runwave/controller/README.md @@ -6,7 +6,8 @@ executes timed input sequences, records WebM output, and writes per-action artifacts. See the top-level [Requirements](../../README.md#requirements) section for the -Linux, gstreamer, X server/Xvfb, and PulseAudio setup required by `record: true`. +Linux, gstreamer, ffmpeg/ffprobe, X server/Xvfb, and PulseAudio setup required +by `record: true`. ## Start @@ -86,6 +87,10 @@ Useful `start` options: sources with `videoSource` and `audioSource`. (`recordAudio` is accepted as a legacy alias for `record`; they mean the same thing - audio is always captured when recording.) +- `repeatedFrameRemoval`: when `record` is enabled, post-process the final WebM + by default to remove repeated-frame runs before returning the stop response. + Pass `false` to keep only the raw recording. When enabled, the raw capture is + renamed with a `_raw` suffix. - `headless`: defaults to `true`; set `false` to watch the browser. - `channel`: optional Playwright browser channel, such as `chrome` or `msedge`. - `executablePath`: optional explicit browser executable path. @@ -238,7 +243,15 @@ runwave-controller '{"action":"sessions"}' When `record: true` is set, `stop` returns `video` and `audioVideo` pointing at the same recorded audio/video WebM. All recording goes through gstreamer - see the top-level [Requirements](../../README.md#requirements) section for the -mandatory environment (Linux, gstreamer, X server/Xvfb, PulseAudio). +mandatory environment (Linux, gstreamer, ffmpeg/ffprobe, X server/Xvfb, +PulseAudio). + +By default, `video` and `audioVideo` point at the shorter WebM at the normal +recording path after repeated-frame runs are removed. The original uncut capture +is preserved as `rawVideo` / `rawAudioVideo`, typically +`video/000-runwave-with-audio_raw.webm`, and `repeatedFrameRemoval` reports the +removed frame ranges. Pass `repeatedFrameRemoval: false` on `start` to return +the raw recording path without post-processing. ## State @@ -277,7 +290,11 @@ Each turn writes: - `response.json`: the main CLI response. - `*.png`: screenshots captured during that operation. - `NNN-.json`: detailed sequence log for `step` operations. -- `video/000-runwave-with-audio.webm`: final gstreamer audio+video recording. +- `video/000-runwave-with-audio.webm`: final shorter audio/video recording when + repeated-frame removal is enabled, or the raw gstreamer recording when it is + disabled. +- `video/000-runwave-with-audio_raw.webm`: original recording when repeated + frame removal is enabled. Active sessions are tracked as JSON files in `.runwave-sessions/` by default. The matching session file is removed by `stop`. diff --git a/runwave/controller/bin/remove-repeated-frames.js b/runwave/controller/bin/remove-repeated-frames.js new file mode 100644 index 0000000..a7260df --- /dev/null +++ b/runwave/controller/bin/remove-repeated-frames.js @@ -0,0 +1,9 @@ +module.exports = require('../src/repeated-frame-remover'); + +if (require.main === module) { + const { main } = module.exports; + main(process.argv.slice(2)).catch((error) => { + console.error(error.stack || error.message); + process.exitCode = 1; + }); +} diff --git a/runwave/controller/src/action-handler.js b/runwave/controller/src/action-handler.js index 534bfa2..71d9279 100644 --- a/runwave/controller/src/action-handler.js +++ b/runwave/controller/src/action-handler.js @@ -143,7 +143,7 @@ async function writeStopResponse(runtime, input) { runtime.profiler.mark('action.stop.screenshot_error', { action_name: input.action_name, error: screenshotError }); } } - const recording = await runtime.profiler.time('action.stop.browser_close', { action_name: input.action_name }, () => browser.close()); + const recording = await runtime.profiler.time('action.stop.browser_close', { action_name: input.action_name }, () => browser.close(input)); const video = typeof recording === 'string' ? recording : recording.video; runtime.profiler.timeSync('action.stop.remove_session_file', { sessionFile: runtime.sessionFile }, () => @@ -160,6 +160,9 @@ async function writeStopResponse(runtime, input) { }); if (recording && typeof recording === 'object') { if (recording.audioVideo) payload.audioVideo = recording.audioVideo; + if (recording.rawVideo) payload.rawVideo = recording.rawVideo; + if (recording.rawAudioVideo) payload.rawAudioVideo = recording.rawAudioVideo; + if (recording.repeatedFrameRemoval) payload.repeatedFrameRemoval = recording.repeatedFrameRemoval; } if (screenshot) payload.screenshot = screenshot; if (stateError) payload.stateError = stateError; diff --git a/runwave/controller/src/browser-session.js b/runwave/controller/src/browser-session.js index cc94c8d..ad8faab 100644 --- a/runwave/controller/src/browser-session.js +++ b/runwave/controller/src/browser-session.js @@ -7,6 +7,11 @@ const { AudioVideoRecorder } = require('./audio-recorder'); const { ensureDir, safeName, sleep, timestamp } = require('./file-utils'); const { drawGridOnScreenshot } = require('./grid-overlay'); const { parseArgList, targetUrl } = require('./protocol'); +const { + isRepeatedFrameRemovalEnabled, + removeRepeatedFramesInPlace, + repeatedFrameRemovalOptions, +} = require('./repeated-frame-remover'); const { readPageState } = require('./state-reader'); const DEFAULT_CHROMIUM_ARGS = [ @@ -484,17 +489,43 @@ class BrowserSession { }); } - async close() { + async close(overrides = {}) { + const hasRepeatedFrameRemovalOverride = Object.prototype.hasOwnProperty.call(overrides, 'repeatedFrameRemoval'); + const closeConfig = { + ...this.config, + ...overrides, + repeatedFrameRemoval: hasRepeatedFrameRemovalOverride + ? overrides.repeatedFrameRemoval + : this.config.repeatedFrameRemoval, + }; let audioVideoPath = null; + let rawVideoPath = null; + let repeatedFrameRemoval = null; let closeError = null; try { if (this.audioRecorder) { audioVideoPath = await this.time('browser.close.audio_video_stop', () => this.audioRecorder.stop()); } + if (audioVideoPath && isRepeatedFrameRemovalEnabled(closeConfig)) { + const processed = await this.time('browser.close.remove_repeated_frames', { video: audioVideoPath }, () => + removeRepeatedFramesInPlace(audioVideoPath, repeatedFrameRemovalOptions()) + ); + audioVideoPath = processed.video; + rawVideoPath = processed.rawVideo; + repeatedFrameRemoval = processed.repeatedFrameRemoval; + } + } catch (error) { + closeError = error; + } + try { if (this.context) await this.time('browser.close.context_close', () => this.context.close()); + } catch (error) { + if (!closeError) closeError = error; + } + try { if (this.browser) await this.time('browser.close.browser_close', () => this.browser.close()); } catch (error) { - closeError = error; + if (!closeError) closeError = error; } try { await this.stopGameProcess(); @@ -505,6 +536,9 @@ class BrowserSession { return { video: audioVideoPath, audioVideo: audioVideoPath || undefined, + rawVideo: rawVideoPath || undefined, + rawAudioVideo: rawVideoPath || undefined, + repeatedFrameRemoval: repeatedFrameRemoval || undefined, }; } } diff --git a/runwave/controller/src/linux-session.js b/runwave/controller/src/linux-session.js index 49bc8e5..c180c0a 100644 --- a/runwave/controller/src/linux-session.js +++ b/runwave/controller/src/linux-session.js @@ -5,6 +5,11 @@ const { AudioVideoRecorder, defaultVideoSource, parseX11VideoSource } = require( const { ensureDir, safeName, sleep, timestamp } = require('./file-utils'); const { drawGridOnScreenshot } = require('./grid-overlay'); const { parseArgList } = require('./protocol'); +const { + isRepeatedFrameRemovalEnabled, + removeRepeatedFramesInPlace, + repeatedFrameRemovalOptions, +} = require('./repeated-frame-remover'); const DEFAULT_WINDOW_WAIT_MS = 15000; const DEFAULT_LINUX_LAUNCH_SETTLE_MS = 30000; @@ -597,27 +602,58 @@ class LinuxSession { }; } - async close() { + async close(overrides = {}) { + const hasRepeatedFrameRemovalOverride = Object.prototype.hasOwnProperty.call(overrides, 'repeatedFrameRemoval'); + const closeConfig = { + ...this.config, + ...overrides, + repeatedFrameRemoval: hasRepeatedFrameRemovalOverride + ? overrides.repeatedFrameRemoval + : this.config.repeatedFrameRemoval, + }; let audioVideoPath = null; - if (this.audioRecorder) { - audioVideoPath = await this.time('linux.close.audio_video_stop', () => this.audioRecorder.stop()); + let rawVideoPath = null; + let repeatedFrameRemoval = null; + let closeError = null; + try { + if (this.audioRecorder) { + audioVideoPath = await this.time('linux.close.audio_video_stop', () => this.audioRecorder.stop()); + } + if (audioVideoPath && isRepeatedFrameRemovalEnabled(closeConfig)) { + const processed = await this.time('linux.close.remove_repeated_frames', { video: audioVideoPath }, () => + removeRepeatedFramesInPlace(audioVideoPath, repeatedFrameRemovalOptions()) + ); + audioVideoPath = processed.video; + rawVideoPath = processed.rawVideo; + repeatedFrameRemoval = processed.repeatedFrameRemoval; + } + } catch (error) { + closeError = error; } - if (this.process && !processHasClosed(this.process)) { - await this.time('linux.close.terminate_process', async () => { - signalProcessGroup(this.process, 'SIGTERM'); - if (!(await waitForProcessClose(this.process, 5000))) { - signalProcessGroup(this.process, 'SIGKILL'); - await waitForProcessClose(this.process, 5000); - } - }); - } else if (this.windowId) { - try { - this.timeSync('linux.close.window_close', () => this.xdotool(['windowclose', this.windowId], { timeoutMs: 1000 })); - } catch {} + try { + if (this.process && !processHasClosed(this.process)) { + await this.time('linux.close.terminate_process', async () => { + signalProcessGroup(this.process, 'SIGTERM'); + if (!(await waitForProcessClose(this.process, 5000))) { + signalProcessGroup(this.process, 'SIGKILL'); + await waitForProcessClose(this.process, 5000); + } + }); + } else if (this.windowId) { + try { + this.timeSync('linux.close.window_close', () => this.xdotool(['windowclose', this.windowId], { timeoutMs: 1000 })); + } catch {} + } + } catch (error) { + if (!closeError) closeError = error; } + if (closeError) throw closeError; return { video: audioVideoPath, audioVideo: audioVideoPath || undefined, + rawVideo: rawVideoPath || undefined, + rawAudioVideo: rawVideoPath || undefined, + repeatedFrameRemoval: repeatedFrameRemoval || undefined, }; } } diff --git a/runwave/controller/src/protocol.js b/runwave/controller/src/protocol.js index 74c3763..621549f 100644 --- a/runwave/controller/src/protocol.js +++ b/runwave/controller/src/protocol.js @@ -24,6 +24,8 @@ function usage() { cwd: '/absolute/path/to/game', windowTitle: 'Native Game', record: true, + repeatedFrameRemoval: true, + keyAliases: { right: 'd', left: 'a', jump: 'w' }, }, { action: 'step', @@ -141,6 +143,16 @@ function optionalPositiveInteger(value) { return Number.isInteger(number) && number > 0 ? number : null; } +function repeatedFrameRemovalEnabled(input) { + return Boolean(input.record || input.recordAudio) && input.repeatedFrameRemoval !== false; +} + +function repeatedFrameRemovalConfig(input) { + return { + enabled: repeatedFrameRemovalEnabled(input), + }; +} + function linuxStartConfig(input = {}) { const launch = input.launch && typeof input.launch === 'object' ? input.launch : {}; const launchEnv = input.env ?? launch.env; @@ -189,6 +201,7 @@ function startSessionConfig(input, options = {}) { deviceScaleFactor: optionalNumber(input.deviceScaleFactor, 1), record, videoSize: record ? normalizeSize(input.videoSize || input.viewport, viewport) : null, + repeatedFrameRemoval: record ? repeatedFrameRemovalConfig(input) : { enabled: false }, }, defaults: { keyAliases: sortedObject(input.keyAliases), @@ -261,6 +274,8 @@ module.exports = { linuxStartConfig, webStartConfig, parseArgList, + repeatedFrameRemovalConfig, + repeatedFrameRemovalEnabled, startSessionConfig, diffStartSessionConfig, isListSessionsAction, diff --git a/runwave/controller/src/repeated-frame-remover.js b/runwave/controller/src/repeated-frame-remover.js new file mode 100644 index 0000000..6ed86ae --- /dev/null +++ b/runwave/controller/src/repeated-frame-remover.js @@ -0,0 +1,627 @@ +const fs = require('fs'); +const path = require('path'); +const { spawn } = require('child_process'); + +const DEFAULT_EDGE_FRAME_COUNT = 10; +const DEFAULT_SIMILARITY_THRESHOLD = 0.98; +const DEFAULT_PIXEL_TOLERANCE = 3; + +function isRepeatedFrameRemovalEnabled(config = {}) { + return Boolean(config.record || config.recordAudio) && config.repeatedFrameRemoval !== false; +} + +function repeatedFrameRemovalOptions() { + return { + edgeFrameCount: DEFAULT_EDGE_FRAME_COUNT, + similarityThreshold: DEFAULT_SIMILARITY_THRESHOLD, + pixelTolerance: DEFAULT_PIXEL_TOLERANCE, + comparisonWidth: 160, + }; +} + +function ensureDir(dir) { + fs.mkdirSync(dir, { recursive: true }); + return dir; +} + +function spawnChecked(command, args, options = {}) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + stdio: ['ignore', 'ignore', 'pipe'], + ...options, + }); + const stderr = []; + + child.stderr.on('data', (chunk) => stderr.push(Buffer.from(chunk))); + child.on('error', reject); + child.on('close', (code, signal) => { + if (code === 0) { + resolve({ code, signal, stderr: Buffer.concat(stderr).toString('utf8') }); + return; + } + + const tail = Buffer.concat(stderr).toString('utf8').trim().slice(-2000); + reject(new Error(`${command} failed with ${signal || `exit code ${code}`}${tail ? `:\n${tail}` : ''}`)); + }); + }); +} + +function parseFrameMd5Line(line) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith('#')) return null; + + const parts = trimmed.split(',').map((part) => part.trim()); + const hash = parts[parts.length - 1]; + if (!/^[a-f0-9]{32}$/i.test(hash)) return null; + + return { + streamIndex: Number(parts[0]), + dts: Number(parts[1]), + pts: Number(parts[2]), + duration: Number(parts[3]), + size: Number(parts[4]), + hash: hash.toLowerCase(), + }; +} + +function collectFrameHashes(inputPath, options = {}) { + const ffmpegPath = options.ffmpegPath || 'ffmpeg'; + const args = [ + '-hide_banner', + '-v', + 'error', + '-i', + inputPath, + '-map', + '0:v:0', + '-an', + '-f', + 'framemd5', + '-', + ]; + + return new Promise((resolve, reject) => { + const child = spawn(ffmpegPath, args, { stdio: ['ignore', 'pipe', 'pipe'] }); + const frames = []; + const stderr = []; + let buffer = ''; + + child.stdout.on('data', (chunk) => { + buffer += chunk.toString('utf8'); + const lines = buffer.split(/\r?\n/); + buffer = lines.pop() || ''; + + for (const line of lines) { + const frame = parseFrameMd5Line(line); + if (frame) frames.push(frame); + } + }); + + child.stderr.on('data', (chunk) => stderr.push(Buffer.from(chunk))); + child.on('error', reject); + child.on('close', (code, signal) => { + if (buffer) { + const frame = parseFrameMd5Line(buffer); + if (frame) frames.push(frame); + } + + if (code === 0) { + resolve(frames); + return; + } + + const tail = Buffer.concat(stderr).toString('utf8').trim().slice(-2000); + reject(new Error(`${ffmpegPath} framemd5 failed with ${signal || `exit code ${code}`}${tail ? `:\n${tail}` : ''}`)); + }); + }); +} + +function findRepeatedFrameRemovalRanges(frames, options = {}) { + const edgeFrameCount = Number(options.edgeFrameCount ?? DEFAULT_EDGE_FRAME_COUNT); + if (!Number.isInteger(edgeFrameCount) || edgeFrameCount < 1) { + throw new Error(`edgeFrameCount must be a positive integer, got ${options.edgeFrameCount}`); + } + + const hashes = frames.map((frame) => (typeof frame === 'string' ? frame : frame.hash)); + const ranges = []; + let runStart = 0; + + for (let index = 1; index <= hashes.length; index += 1) { + if (index < hashes.length && hashes[index] === hashes[runStart]) continue; + + pushRemovalRangeForRun(ranges, runStart, index - 1, edgeFrameCount, { hash: hashes[runStart] }); + + runStart = index; + } + + return ranges; +} + +function pushRemovalRangeForRun(ranges, runStart, runEnd, edgeFrameCount, extra = {}) { + const runLength = runEnd - runStart + 1; + const removeStart = runStart + edgeFrameCount; + const removeEnd = runEnd - edgeFrameCount; + + if (removeStart <= removeEnd) { + ranges.push({ + start: removeStart, + end: removeEnd, + removedFrames: removeEnd - removeStart + 1, + runStart, + runEnd, + runLength, + ...extra, + }); + } +} + +function normalizeSimilarityThreshold(value) { + const threshold = Number(value ?? DEFAULT_SIMILARITY_THRESHOLD); + if (!Number.isFinite(threshold) || threshold < 0 || threshold > 1) { + throw new Error(`similarityThreshold must be between 0 and 1, got ${value}`); + } + return threshold; +} + +function comparisonSize(mediaInfo, options = {}) { + const width = Number(mediaInfo.width); + const height = Number(mediaInfo.height); + if (!Number.isFinite(width) || width < 1 || !Number.isFinite(height) || height < 1) { + throw new Error(`could not determine video dimensions`); + } + + const requestedWidth = Number(options.comparisonWidth ?? 160); + const comparisonWidth = Math.max(1, Math.min(width, Math.round(requestedWidth))); + const comparisonHeight = Math.max(1, Math.round(height * (comparisonWidth / width))); + + return { + width: comparisonWidth, + height: comparisonHeight, + frameByteLength: comparisonWidth * comparisonHeight, + }; +} + +function normalizePixelTolerance(value) { + const tolerance = Number(value ?? DEFAULT_PIXEL_TOLERANCE); + if (!Number.isFinite(tolerance) || tolerance < 0 || tolerance > 255) { + throw new Error(`pixelTolerance must be between 0 and 255, got ${value}`); + } + return tolerance; +} + +function frameSimilarity(a, b, options = {}) { + if (a.length !== b.length) throw new Error(`cannot compare frames with different sizes`); + + const pixelTolerance = normalizePixelTolerance(options.pixelTolerance); + let matchingPixels = 0; + for (let index = 0; index < a.length; index += 1) { + if (Math.abs(a[index] - b[index]) <= pixelTolerance) matchingPixels += 1; + } + + return matchingPixels / a.length; +} + +async function findSimilarFrameRemovalRanges(inputPath, options = {}) { + const ffmpegPath = options.ffmpegPath || 'ffmpeg'; + const mediaInfo = options.mediaInfo || await readMediaInfo(inputPath, options); + const edgeFrameCount = Number(options.edgeFrameCount ?? DEFAULT_EDGE_FRAME_COUNT); + if (!Number.isInteger(edgeFrameCount) || edgeFrameCount < 1) { + throw new Error(`edgeFrameCount must be a positive integer, got ${options.edgeFrameCount}`); + } + + const similarityThreshold = normalizeSimilarityThreshold(options.similarityThreshold); + const pixelTolerance = normalizePixelTolerance(options.pixelTolerance); + const size = comparisonSize(mediaInfo, options); + const args = [ + '-hide_banner', + '-v', + 'error', + '-i', + inputPath, + '-map', + '0:v:0', + '-an', + '-vf', + `scale=${size.width}:${size.height},format=gray`, + '-f', + 'rawvideo', + '-', + ]; + + return new Promise((resolve, reject) => { + const child = spawn(ffmpegPath, args, { stdio: ['ignore', 'pipe', 'pipe'] }); + const stderr = []; + const removalRanges = []; + let buffered = Buffer.alloc(0); + let anchor = null; + let frameIndex = 0; + let runStart = 0; + let weakestSimilarity = 1; + + function finishRun(runEnd) { + pushRemovalRangeForRun(removalRanges, runStart, runEnd, edgeFrameCount, { + weakestSimilarity: Number(weakestSimilarity.toFixed(6)), + }); + } + + function processFrame(frame) { + if (!anchor) { + anchor = Buffer.from(frame); + frameIndex += 1; + return; + } + + const similarity = frameSimilarity(anchor, frame, { pixelTolerance }); + if (similarity >= similarityThreshold) { + weakestSimilarity = Math.min(weakestSimilarity, similarity); + frameIndex += 1; + return; + } + + finishRun(frameIndex - 1); + anchor = Buffer.from(frame); + runStart = frameIndex; + weakestSimilarity = 1; + frameIndex += 1; + } + + child.stdout.on('data', (chunk) => { + buffered = buffered.length ? Buffer.concat([buffered, chunk]) : chunk; + while (buffered.length >= size.frameByteLength) { + processFrame(buffered.subarray(0, size.frameByteLength)); + buffered = buffered.subarray(size.frameByteLength); + } + }); + + child.stderr.on('data', (chunk) => stderr.push(Buffer.from(chunk))); + child.on('error', reject); + child.on('close', (code, signal) => { + if (code !== 0) { + const tail = Buffer.concat(stderr).toString('utf8').trim().slice(-2000); + reject(new Error(`${ffmpegPath} raw frame decode failed with ${signal || `exit code ${code}`}${tail ? `:\n${tail}` : ''}`)); + return; + } + + if (buffered.length) { + reject(new Error(`raw frame decode ended with ${buffered.length} trailing bytes; expected whole ${size.frameByteLength}-byte frames`)); + return; + } + + if (frameIndex > 0) finishRun(frameIndex - 1); + resolve({ + frameCount: frameIndex, + mediaInfo, + removalRanges, + detection: { + mode: 'similarity', + similarityThreshold, + pixelTolerance, + comparisonWidth: size.width, + comparisonHeight: size.height, + pixelFormat: 'gray', + }, + }); + }); + }); +} + +function escapeSelectFunction(name, values) { + return `${name}(${values.join('\\,')})`; +} + +function buildVideoRemovalExpression(removalRanges) { + return removalRanges + .map((range) => escapeSelectFunction('between', ['n', range.start, range.end])) + .join('+'); +} + +function buildVideoFilter(removalRanges, fps) { + const safeFps = String(fps || '').trim(); + if (!safeFps || safeFps === '0/0') throw new Error(`invalid fps for setpts: ${fps}`); + + const setPts = `setpts=N/(${safeFps})/TB`; + if (!removalRanges.length) return setPts; + + return `select=not(${buildVideoRemovalExpression(removalRanges)}),${setPts}`; +} + +function fpsToNumber(fps) { + const text = String(fps || '').trim(); + const match = text.match(/^(\d+)\/(\d+)$/); + if (match) { + const denominator = Number(match[2]); + return denominator ? Number(match[1]) / denominator : NaN; + } + return Number(text); +} + +function fixedSeconds(value) { + return Number(value).toFixed(6).replace(/0+$/, '').replace(/\.$/, '.0'); +} + +function buildAudioRemovalExpression(removalRanges, fps) { + const fpsNumber = fpsToNumber(fps); + if (!Number.isFinite(fpsNumber) || fpsNumber <= 0) { + throw new Error(`invalid fps for audio trimming: ${fps}`); + } + + return removalRanges + .map((range) => { + const start = fixedSeconds(range.start / fpsNumber); + const end = fixedSeconds((range.end + 1) / fpsNumber); + return `${escapeSelectFunction('gte', ['t', start])}*${escapeSelectFunction('lt', ['t', end])}`; + }) + .join('+'); +} + +function buildAudioFilter(removalRanges, fps) { + const setPts = 'asetpts=N/SR/TB'; + if (!removalRanges.length) return setPts; + return `aselect=not(${buildAudioRemovalExpression(removalRanges, fps)}),${setPts}`; +} + +function parseRate(rate) { + const text = String(rate || '').trim(); + const match = text.match(/^(\d+)\/(\d+)$/); + if (!match) return text; + + const numerator = Number(match[1]); + const denominator = Number(match[2]); + if (!denominator) return ''; + return denominator === 1 ? String(numerator) : `${numerator}/${denominator}`; +} + +async function readVideoFps(inputPath, options = {}) { + const mediaInfo = await readMediaInfo(inputPath, options); + return mediaInfo.fps; +} + +async function readMediaInfo(inputPath, options = {}) { + const ffprobePath = options.ffprobePath || 'ffprobe'; + const args = [ + '-v', + 'error', + '-show_entries', + 'stream=codec_type,width,height,avg_frame_rate,r_frame_rate', + '-of', + 'json', + inputPath, + ]; + + return new Promise((resolve, reject) => { + const child = spawn(ffprobePath, args, { stdio: ['ignore', 'pipe', 'pipe'] }); + const stdout = []; + const stderr = []; + + child.stdout.on('data', (chunk) => stdout.push(Buffer.from(chunk))); + child.stderr.on('data', (chunk) => stderr.push(Buffer.from(chunk))); + child.on('error', reject); + child.on('close', (code, signal) => { + if (code !== 0) { + const tail = Buffer.concat(stderr).toString('utf8').trim().slice(-2000); + reject(new Error(`${ffprobePath} failed with ${signal || `exit code ${code}`}${tail ? `:\n${tail}` : ''}`)); + return; + } + + const parsed = JSON.parse(Buffer.concat(stdout).toString('utf8')); + const streams = parsed.streams || []; + const videoStream = streams.find((stream) => stream.codec_type === 'video'); + const fps = parseRate(videoStream?.avg_frame_rate) || parseRate(videoStream?.r_frame_rate); + if (!fps) reject(new Error(`could not determine video frame rate for ${inputPath}`)); + else { + resolve({ + fps, + width: videoStream.width, + height: videoStream.height, + hasAudio: streams.some((stream) => stream.codec_type === 'audio'), + }); + } + }); + }); +} + +async function encodeWithoutRepeatedFrames(inputPath, outputPath, removalRanges, options = {}) { + const ffmpegPath = options.ffmpegPath || 'ffmpeg'; + const mediaInfo = options.mediaInfo || await readMediaInfo(inputPath, options); + const fps = options.fps || mediaInfo.fps; + const videoFilter = buildVideoFilter(removalRanges, fps); + const shouldIncludeAudio = options.includeAudio ?? mediaInfo.hasAudio; + const outputExt = path.extname(outputPath).toLowerCase(); + const isMp4 = outputExt === '.mp4' || outputExt === '.m4v' || outputExt === '.mov'; + const videoCodec = options.videoCodec || (isMp4 ? 'libx264' : 'libvpx'); + ensureDir(path.dirname(outputPath)); + + const args = [ + '-hide_banner', + '-y', + '-v', + options.logLevel || 'error', + '-i', + inputPath, + ]; + + if (shouldIncludeAudio && mediaInfo.hasAudio) { + args.push( + '-filter_complex', + `[0:v:0]${videoFilter}[v];[0:a:0]${buildAudioFilter(removalRanges, fps)}[a]`, + '-map', + '[v]', + '-map', + '[a]' + ); + } else { + args.push( + '-map', + '0:v:0', + '-an', + '-vf', + videoFilter + ); + } + + args.push( + '-fps_mode', + 'cfr', + '-r', + fps, + '-c:v', + videoCodec + ); + + if (videoCodec === 'libvpx') { + args.push( + '-deadline', + options.deadline || 'realtime', + '-cpu-used', + String(options.cpuUsed ?? 8), + '-crf', + String(options.crf ?? 30), + '-b:v', + String(options.videoBitrate ?? '0') + ); + } else if (videoCodec === 'libx264') { + args.push( + '-preset', + options.preset || 'veryfast', + '-crf', + String(options.crf ?? 23), + '-pix_fmt', + 'yuv420p' + ); + } else { + args.push( + '-crf', + String(options.crf ?? 30) + ); + } + + if (shouldIncludeAudio && mediaInfo.hasAudio) { + args.push( + '-c:a', + options.audioCodec || (isMp4 ? 'aac' : 'libopus'), + '-b:a', + String(options.audioBitrate || '64k') + ); + } + + args.push( + outputPath, + ); + + await spawnChecked(ffmpegPath, args); + return { + outputPath, + fps, + videoFilter, + }; +} + +async function removeRepeatedFrames(inputPath, outputPath, options = {}) { + const analysis = await findSimilarFrameRemovalRanges(inputPath, options); + const removalRanges = analysis.removalRanges; + const removedFrameCount = removalRanges.reduce((sum, range) => sum + range.removedFrames, 0); + const encode = await encodeWithoutRepeatedFrames(inputPath, outputPath, removalRanges, { + ...options, + mediaInfo: analysis.mediaInfo, + }); + + return { + inputPath, + outputPath, + frameCount: analysis.frameCount, + keptFrameCount: analysis.frameCount - removedFrameCount, + removedFrameCount, + removalRanges, + detection: analysis.detection, + fps: encode.fps, + videoFilter: encode.videoFilter, + }; +} + +function rawVideoPath(videoPath, options = {}) { + const parsed = path.parse(videoPath); + const suffix = options.rawSuffix || '_raw'; + const preferred = path.join(parsed.dir, `${parsed.name}${suffix}${parsed.ext}`); + if (!fs.existsSync(preferred)) return preferred; + + for (let index = 2; index < 1000; index += 1) { + const candidate = path.join(parsed.dir, `${parsed.name}${suffix}-${index}${parsed.ext}`); + if (!fs.existsSync(candidate)) return candidate; + } + + throw new Error(`could not find available raw video name for ${videoPath}`); +} + +async function removeRepeatedFramesInPlace(videoPath, options = {}) { + if (!videoPath || !fs.existsSync(videoPath)) { + return null; + } + + const rawPath = options.rawPath || rawVideoPath(videoPath, options); + fs.renameSync(videoPath, rawPath); + + try { + const summary = await removeRepeatedFrames(rawPath, videoPath, options); + return { + video: videoPath, + rawVideo: rawPath, + repeatedFrameRemoval: summary, + }; + } catch (error) { + if (!fs.existsSync(videoPath) && fs.existsSync(rawPath)) { + fs.renameSync(rawPath, videoPath); + } + throw error; + } +} + +function printUsage() { + console.error('Usage: node runwave/controller/bin/remove-repeated-frames.js '); +} + +async function main(argv) { + const args = [...argv]; + const inputPath = args.shift(); + const outputPath = args.shift(); + if (!inputPath || !outputPath) { + printUsage(); + process.exitCode = 2; + return; + } + + if (args.length) { + throw new Error(`unknown argument: ${args[0]}`); + } + + const summary = await removeRepeatedFrames(inputPath, outputPath, repeatedFrameRemovalOptions()); + console.log(JSON.stringify(summary, null, 2)); +} + +if (require.main === module) { + main(process.argv.slice(2)).catch((error) => { + console.error(error.stack || error.message); + process.exitCode = 1; + }); +} + +module.exports = { + DEFAULT_EDGE_FRAME_COUNT, + DEFAULT_PIXEL_TOLERANCE, + DEFAULT_SIMILARITY_THRESHOLD, + buildAudioFilter, + buildVideoFilter, + collectFrameHashes, + encodeWithoutRepeatedFrames, + findRepeatedFrameRemovalRanges, + findSimilarFrameRemovalRanges, + frameSimilarity, + isRepeatedFrameRemovalEnabled, + main, + parseFrameMd5Line, + rawVideoPath, + readMediaInfo, + readVideoFps, + repeatedFrameRemovalOptions, + removeRepeatedFrames, + removeRepeatedFramesInPlace, +}; diff --git a/runwave/controller/test/browser-session.test.js b/runwave/controller/test/browser-session.test.js index 9739ade..182c7f6 100644 --- a/runwave/controller/test/browser-session.test.js +++ b/runwave/controller/test/browser-session.test.js @@ -10,6 +10,11 @@ const { pageViewportVideoSource, webLaunchConfig, } = require('../src/browser-session'); +const { + isRepeatedFrameRemovalEnabled, + rawVideoPath, + repeatedFrameRemovalOptions, +} = require('../src/repeated-frame-remover'); test('chromium launch args leave non-recording runs unchanged', () => { const args = chromiumLaunchArgs({ record: false }, {}); @@ -78,6 +83,27 @@ test('browser viewport stabilizer hides overflow and prevents scrolling keys', ( assert.match(source, /preventDefault/); }); +test('repeated frame removal config defaults on for recordings', () => { + assert.equal(isRepeatedFrameRemovalEnabled({ record: true }), true); + assert.equal(isRepeatedFrameRemovalEnabled({ recordAudio: true }), true); + assert.equal(isRepeatedFrameRemovalEnabled({ record: false }), false); + assert.equal(isRepeatedFrameRemovalEnabled({ record: true, repeatedFrameRemoval: true }), true); + assert.equal(isRepeatedFrameRemovalEnabled({ record: true, repeatedFrameRemoval: false }), false); +}); + +test('repeated frame removal options are hard-coded', () => { + assert.deepEqual(repeatedFrameRemovalOptions(), { + edgeFrameCount: 10, + similarityThreshold: 0.98, + pixelTolerance: 3, + comparisonWidth: 160, + }); +}); + +test('rawVideoPath appends raw suffix before extension', () => { + assert.equal(rawVideoPath('/tmp/video/000-runwave-with-audio.webm'), '/tmp/video/000-runwave-with-audio_raw.webm'); +}); + test('browser clicks hold the mouse down for the normalized click interval', async () => { const session = new BrowserSession({ url: 'about:blank' }, { runDir: os.tmpdir() }); const calls = []; diff --git a/runwave/controller/test/repeated-frame-remover.test.js b/runwave/controller/test/repeated-frame-remover.test.js new file mode 100644 index 0000000..273fd33 --- /dev/null +++ b/runwave/controller/test/repeated-frame-remover.test.js @@ -0,0 +1,86 @@ +const assert = require('node:assert/strict'); +const test = require('node:test'); + +const { + DEFAULT_SIMILARITY_THRESHOLD, + DEFAULT_EDGE_FRAME_COUNT, + DEFAULT_PIXEL_TOLERANCE, + buildVideoFilter, + buildAudioFilter, + findRepeatedFrameRemovalRanges, + frameSimilarity, + parseFrameMd5Line, +} = require('../src/repeated-frame-remover'); + +test('parseFrameMd5Line reads frame hash rows', () => { + assert.deepEqual( + parseFrameMd5Line('0, 7, 7, 1, 1382400, 4ac1f0bdba53f0f96e5cb4aaa499dfac'), + { + streamIndex: 0, + dts: 7, + pts: 7, + duration: 1, + size: 1382400, + hash: '4ac1f0bdba53f0f96e5cb4aaa499dfac', + } + ); + assert.equal(parseFrameMd5Line('#stream#, dts, pts, duration, size, hash'), null); +}); + +test('findRepeatedFrameRemovalRanges honors explicit edge frame counts', () => { + const hashes = [ + ...Array(5).fill('a'), + ...Array(6).fill('b'), + ...Array(10).fill('c'), + ...Array(11).fill('d'), + ...Array(14).fill('e'), + 'f', + ]; + + assert.deepEqual(findRepeatedFrameRemovalRanges(hashes, { edgeFrameCount: 5 }), [ + { + start: 26, + end: 26, + removedFrames: 1, + runStart: 21, + runEnd: 31, + runLength: 11, + hash: 'd', + }, + { + start: 37, + end: 40, + removedFrames: 4, + runStart: 32, + runEnd: 45, + runLength: 14, + hash: 'e', + }, + ]); +}); + +test('buildVideoFilter removes frame ranges and closes timestamps at the original fps', () => { + assert.equal( + buildVideoFilter([ + { start: 5, end: 9 }, + { start: 20, end: 22 }, + ], '25'), + 'select=not(between(n\\,5\\,9)+between(n\\,20\\,22)),setpts=N/(25)/TB' + ); + assert.equal(buildVideoFilter([], '25'), 'setpts=N/(25)/TB'); +}); + +test('frameSimilarity scores normalized byte similarity', () => { + assert.equal(DEFAULT_EDGE_FRAME_COUNT, 10); + assert.equal(DEFAULT_SIMILARITY_THRESHOLD, 0.98); + assert.equal(DEFAULT_PIXEL_TOLERANCE, 3); + assert.equal(frameSimilarity(Buffer.from([0, 100]), Buffer.from([0, 110])), 0.5); + assert.equal(frameSimilarity(Buffer.from([0, 100]), Buffer.from([0, 110]), { pixelTolerance: 10 }), 1); +}); + +test('buildAudioFilter removes matching time spans for removed video frames', () => { + assert.equal( + buildAudioFilter([{ start: 5, end: 9 }], '25'), + 'aselect=not(gte(t\\,0.2)*lt(t\\,0.4)),asetpts=N/SR/TB' + ); +}); diff --git a/runwave/controller/test/session-cleanup.test.js b/runwave/controller/test/session-cleanup.test.js index 07586f7..eabf4bf 100644 --- a/runwave/controller/test/session-cleanup.test.js +++ b/runwave/controller/test/session-cleanup.test.js @@ -56,6 +56,41 @@ test('start reuses a live session only when the start configuration matches', as } }); +test('start configuration records repeated frame removal settings', () => { + const config = startSessionConfig({ + action: 'start', + action_name: 'start-cut-video', + url: 'http://127.0.0.1:3000/', + record: true, + repeatedFrameRemoval: true, + }); + + assert.deepEqual(config.context.repeatedFrameRemoval, { enabled: true }); +}); + +test('start configuration enables repeated frame removal by default for recordings', () => { + const config = startSessionConfig({ + action: 'start', + action_name: 'start-short-video', + url: 'http://127.0.0.1:3000/', + record: true, + }); + + assert.deepEqual(config.context.repeatedFrameRemoval, { enabled: true }); +}); + +test('start configuration can disable repeated frame removal', () => { + const config = startSessionConfig({ + action: 'start', + action_name: 'start-raw-video', + url: 'http://127.0.0.1:3000/', + record: true, + repeatedFrameRemoval: false, + }); + + assert.deepEqual(config.context.repeatedFrameRemoval, { enabled: false }); +}); + test('start rejects a live session with a different target instead of silently reusing it', async () => { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'runwave-target-mismatch-')); const originalStart = { diff --git a/stress-test/README.md b/stress-test/README.md index cd6ceb6..6822f48 100644 --- a/stress-test/README.md +++ b/stress-test/README.md @@ -69,8 +69,10 @@ one GStreamer process to record the cropped X display plus `runwave_sink.monitor into `video/000-runwave-with-audio.webm`. This keeps audio from concurrent games on the same worker out of each other's recordings and avoids post-mux sync offsets. -Recorded fleet jobs use this GStreamer path; there is no Playwright video-only -fallback for production playtests. +Recorded fleet jobs use this GStreamer path and post-process it by default to +remove repeated frames, leaving the shorter video at the normal path and the raw +capture with a `_raw` suffix. There is no Playwright video-only fallback for +production playtests. The default game source is `s3://pw-cruft/games`. Override it with: diff --git a/stress-test/remote/playtest-runner.Dockerfile b/stress-test/remote/playtest-runner.Dockerfile index c040fd5..7b93d5f 100644 --- a/stress-test/remote/playtest-runner.Dockerfile +++ b/stress-test/remote/playtest-runner.Dockerfile @@ -9,6 +9,7 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends \ ca-certificates \ curl \ + ffmpeg \ git \ gstreamer1.0-plugins-base \ gstreamer1.0-plugins-good \ diff --git a/stress-test/remote/run-playtest.js b/stress-test/remote/run-playtest.js index dc4f6c1..5a9f533 100755 --- a/stress-test/remote/run-playtest.js +++ b/stress-test/remote/run-playtest.js @@ -213,6 +213,7 @@ function startOverridesFromJob(job = {}, audioCapture = {}) { chromiumArgsMode: job.chromiumArgsMode, keyAliases: job.keyAliases, gridScreenshots: job.gridScreenshots, + repeatedFrameRemoval: job.repeatedFrameRemoval, markGridRows: job.markGridRows, markGridCols: job.markGridCols, launchSettleMs: job.launchSettleMs,